Bump version to 1.6.3

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ryan Ward
2026-06-12 14:17:06 -05:00
co-authored by Claude Opus 4.6
parent 4ba9706e35
commit 59cfd0e2f5
152 changed files with 8276 additions and 1664 deletions
+29
View File
@@ -0,0 +1,29 @@
[package]
name = "local_inference"
authors.workspace = true
edition = "2024"
publish.workspace = true
license.workspace = true
[features]
default = ["cpu"]
cpu = []
metal = ["candle-core/metal", "candle-nn/metal", "candle-transformers/metal"]
cuda = ["candle-core/cuda", "candle-nn/cuda", "candle-transformers/cuda"]
[dependencies]
anyhow.workspace = true
async-trait.workspace = true
candle-core = "0.9.2"
candle-nn = "0.9.2"
candle-transformers = "0.9.2"
directories.workspace = true
hf-hub = { version = "0.4", features = ["tokio"] }
log.workspace = true
serde.workspace = true
serde_json.workspace = true
tokenizers = "0.21.4"
tokio = { workspace = true, features = ["fs", "sync", "rt", "macros", "rt-multi-thread"] }
[dev-dependencies]
env_logger = "0.10"
@@ -0,0 +1,66 @@
use local_inference::{
Device, InferenceEngine, InferenceTask, InputClassificationInput, InputClassificationTask,
TabNamingInput, TabNamingTask,
};
#[tokio::main]
async fn main() -> anyhow::Result<()> {
env_logger::init();
println!("Initializing inference engine (this may download the model on first run)...");
let engine = InferenceEngine::new(Device::best_available()).await?;
println!("Engine ready!\n");
// --- Input Classification ---
let test_inputs = vec![
"ls -la",
"git status",
"what files are in this directory?",
"explain this error to me",
"docker compose up -d",
"how do I fix this segfault?",
"cd /tmp && rm -rf build/",
"refactor the auth module to use JWT",
];
println!("=== Input Classification ===\n");
for input in &test_inputs {
let result = InputClassificationTask
.run(
&engine,
InputClassificationInput {
user_input: input.to_string(),
recent_commands: vec!["git log".into(), "npm test".into()],
is_follow_up: false,
},
)
.await?;
println!(
" {:50} -> {:?} (confidence: {:.2})",
format!("\"{}\"", input),
result.category,
result.confidence
);
}
// --- Tab Naming ---
println!("\n=== Tab Naming ===\n");
let tab_name = TabNamingTask
.run(
&engine,
TabNamingInput {
recent_commands: vec![
"git checkout feature/auth".into(),
"cargo test".into(),
"vim src/auth/mod.rs".into(),
],
working_directory: "/home/user/projects/myapp".into(),
},
)
.await?;
println!(" Suggested tab name: \"{tab_name}\"");
Ok(())
}
@@ -0,0 +1,115 @@
use anyhow::Result;
use candle_core::Tensor;
#[derive(Clone)]
pub struct GenerationConfig {
pub max_tokens: usize,
pub temperature: f64,
pub top_p: f64,
}
impl Default for GenerationConfig {
fn default() -> Self {
Self {
max_tokens: 32,
temperature: 0.7,
top_p: 0.9,
}
}
}
impl GenerationConfig {
pub fn deterministic() -> Self {
Self {
max_tokens: 32,
temperature: 0.0,
top_p: 1.0,
}
}
pub fn creative() -> Self {
Self {
max_tokens: 64,
temperature: 0.9,
top_p: 0.95,
}
}
}
pub fn sample(logits: &Tensor, config: &GenerationConfig) -> Result<u32> {
if config.temperature == 0.0 {
return greedy(logits);
}
let logits = logits.to_dtype(candle_core::DType::F32)?;
let logits_vec = logits.to_vec1::<f32>()?;
// Apply temperature
let scaled: Vec<f64> = logits_vec
.iter()
.map(|&x| (x as f64) / config.temperature)
.collect();
// Softmax
let max_val = scaled.iter().cloned().fold(f64::NEG_INFINITY, f64::max);
let exp: Vec<f64> = scaled.iter().map(|&x| (x - max_val).exp()).collect();
let sum: f64 = exp.iter().sum();
let probs: Vec<f64> = exp.iter().map(|&x| x / sum).collect();
// Top-p (nucleus) sampling
let mut indexed_probs: Vec<(usize, f64)> = probs.iter().copied().enumerate().collect();
indexed_probs.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap_or(std::cmp::Ordering::Equal));
let mut cumulative = 0.0;
let mut candidates = Vec::new();
for (idx, prob) in &indexed_probs {
cumulative += prob;
candidates.push((*idx, *prob));
if cumulative >= config.top_p {
break;
}
}
// Renormalize candidates
let candidate_sum: f64 = candidates.iter().map(|(_, p)| p).sum();
let threshold = simple_rng() * candidate_sum;
let mut acc = 0.0;
for (idx, prob) in &candidates {
acc += prob;
if acc >= threshold {
return Ok(*idx as u32);
}
}
// Fallback to top candidate
Ok(candidates[0].0 as u32)
}
fn greedy(logits: &Tensor) -> Result<u32> {
let logits = logits.to_dtype(candle_core::DType::F32)?;
let logits_vec = logits.to_vec1::<f32>()?;
let max_idx = logits_vec
.iter()
.enumerate()
.max_by(|(_, a), (_, b)| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal))
.map(|(idx, _)| idx)
.unwrap_or(0);
Ok(max_idx as u32)
}
fn simple_rng() -> f64 {
use std::collections::hash_map::DefaultHasher;
use std::hash::{Hash, Hasher};
use std::time::SystemTime;
let mut hasher = DefaultHasher::new();
SystemTime::now()
.duration_since(SystemTime::UNIX_EPOCH)
.unwrap_or_default()
.as_nanos()
.hash(&mut hasher);
std::thread::current().id().hash(&mut hasher);
let hash = hasher.finish();
(hash as f64) / (u64::MAX as f64)
}
+309
View File
@@ -0,0 +1,309 @@
mod generation;
mod model_loader;
use std::path::PathBuf;
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::Arc;
use anyhow::{Context as _, Result};
use candle_core::{DType, Tensor};
use candle_transformers::models::llama::{Cache, Config, Llama, LlamaConfig};
use tokenizers::Tokenizer;
use tokio::sync::Mutex;
pub use generation::GenerationConfig;
/// A token that can be used to cancel an in-progress generation.
/// Clone it and pass to `generate_cancellable`, then call `cancel()` to
/// interrupt the generation loop between tokens.
#[derive(Clone)]
pub struct CancellationToken {
cancelled: Arc<AtomicBool>,
}
impl CancellationToken {
pub fn new() -> Self {
Self {
cancelled: Arc::new(AtomicBool::new(false)),
}
}
pub fn cancel(&self) {
self.cancelled.store(true, Ordering::Relaxed);
}
pub fn is_cancelled(&self) -> bool {
self.cancelled.load(Ordering::Relaxed)
}
}
const HF_REPO: &str = "HuggingFaceTB/SmolLM2-135M-Instruct";
const MODEL_FILENAME: &str = "model.safetensors";
const TOKENIZER_FILENAME: &str = "tokenizer.json";
const CONFIG_FILENAME: &str = "config.json";
#[derive(Debug, Clone, Copy)]
pub enum Device {
Cpu,
#[cfg(feature = "metal")]
Metal,
#[cfg(feature = "cuda")]
Cuda(usize),
}
impl Device {
pub fn best_available() -> Self {
#[cfg(feature = "metal")]
{
return Device::Metal;
}
#[cfg(feature = "cuda")]
{
return Device::Cuda(0);
}
#[cfg(not(any(feature = "metal", feature = "cuda")))]
{
Device::Cpu
}
}
fn to_candle_device(&self) -> Result<candle_core::Device> {
match self {
Device::Cpu => Ok(candle_core::Device::Cpu),
#[cfg(feature = "metal")]
Device::Metal => Ok(candle_core::Device::new_metal(0)?),
#[cfg(feature = "cuda")]
Device::Cuda(ordinal) => Ok(candle_core::Device::new_cuda(*ordinal)?),
}
}
}
pub struct InferenceEngine {
model: Arc<Mutex<Llama>>,
cache: Arc<Mutex<Cache>>,
config: Config,
tokenizer: Tokenizer,
device: candle_core::Device,
}
impl InferenceEngine {
pub async fn new(device: Device) -> Result<Self> {
let candle_device = device.to_candle_device()?;
let model_dir = model_loader::ensure_model_available().await?;
let config_path = model_dir.join(CONFIG_FILENAME);
let config_str = tokio::fs::read_to_string(&config_path)
.await
.with_context(|| format!("reading config from {}", config_path.display()))?;
let llama_config: LlamaConfig =
serde_json::from_str(&config_str).context("parsing model config")?;
let config = llama_config.into_config(false);
let model_path = model_dir.join(MODEL_FILENAME);
let vb = unsafe {
candle_nn::VarBuilder::from_mmaped_safetensors(
&[model_path],
DType::F32,
&candle_device,
)?
};
let model = Llama::load(vb, &config).context("loading SmolLM2 model")?;
let cache = Cache::new(true, DType::F32, &config, &candle_device)?;
let tokenizer_path = model_dir.join(TOKENIZER_FILENAME);
let tokenizer_bytes = tokio::fs::read(&tokenizer_path)
.await
.with_context(|| format!("reading tokenizer from {}", tokenizer_path.display()))?;
let tokenizer =
Tokenizer::from_bytes(&tokenizer_bytes).map_err(|e| anyhow::anyhow!("{e}"))?;
log::info!(
"Local inference engine initialized (device: {:?}, model: {})",
device,
HF_REPO
);
Ok(Self {
model: Arc::new(Mutex::new(model)),
cache: Arc::new(Mutex::new(cache)),
config,
tokenizer,
device: candle_device,
})
}
pub fn tokenizer(&self) -> &Tokenizer {
&self.tokenizer
}
pub fn device(&self) -> &candle_core::Device {
&self.device
}
pub async fn generate_cancellable(
&self,
prompt: &str,
config: &GenerationConfig,
cancel: &CancellationToken,
) -> Result<String> {
let encoding = self
.tokenizer
.encode(prompt, true)
.map_err(|e| anyhow::anyhow!("{e}"))?;
let input_ids = encoding.get_ids().to_vec();
let mut tokens = input_ids.clone();
let eos_token_id = self
.tokenizer
.token_to_id("</s>")
.or_else(|| self.tokenizer.token_to_id("<|endoftext|>"))
.or_else(|| self.tokenizer.token_to_id("<|im_end|>"));
let model = self.model.lock().await;
let mut cache = self.cache.lock().await;
if cancel.is_cancelled() {
anyhow::bail!("cancelled before generation started");
}
// Reset cache for new generation
*cache = Cache::new(true, DType::F32, &self.config, &self.device)?;
// Prefill: process all input tokens at once
let input_tensor = Tensor::new(input_ids.as_slice(), &self.device)?.unsqueeze(0)?;
let logits = model.forward(&input_tensor, 0, &mut cache)?;
if cancel.is_cancelled() {
anyhow::bail!("cancelled during prefill");
}
// Sample first generated token from the last logits position
let first_logits = logits.squeeze(0)?;
let next_token = generation::sample(&first_logits, config)?;
if let Some(eos) = eos_token_id {
if next_token == eos {
let generated_tokens = &tokens[input_ids.len()..];
let output = self
.tokenizer
.decode(generated_tokens, true)
.map_err(|e| anyhow::anyhow!("{e}"))?;
return Ok(output.trim().to_string());
}
}
tokens.push(next_token);
// Decode: generate one token at a time, checking cancellation between tokens
for _i in 1..config.max_tokens {
if cancel.is_cancelled() {
anyhow::bail!("cancelled during generation");
}
let pos = tokens.len() - 1;
let next_input = Tensor::new(&[*tokens.last().unwrap()], &self.device)?.unsqueeze(0)?;
let logits = model.forward(&next_input, pos, &mut cache)?;
let next_logits = logits.squeeze(0)?;
let next_token = generation::sample(&next_logits, config)?;
if let Some(eos) = eos_token_id {
if next_token == eos {
break;
}
}
tokens.push(next_token);
}
let generated_tokens = &tokens[input_ids.len()..];
let output = self
.tokenizer
.decode(generated_tokens, true)
.map_err(|e| anyhow::anyhow!("{e}"))?;
Ok(output.trim().to_string())
}
pub async fn generate(&self, prompt: &str, config: &GenerationConfig) -> Result<String> {
let encoding = self
.tokenizer
.encode(prompt, true)
.map_err(|e| anyhow::anyhow!("{e}"))?;
let input_ids = encoding.get_ids().to_vec();
let mut tokens = input_ids.clone();
let eos_token_id = self
.tokenizer
.token_to_id("</s>")
.or_else(|| self.tokenizer.token_to_id("<|endoftext|>"))
.or_else(|| self.tokenizer.token_to_id("<|im_end|>"));
let model = self.model.lock().await;
let mut cache = self.cache.lock().await;
// Reset cache for new generation
*cache = Cache::new(true, DType::F32, &self.config, &self.device)?;
// Prefill: process all input tokens at once
let input_tensor = Tensor::new(input_ids.as_slice(), &self.device)?.unsqueeze(0)?;
let logits = model.forward(&input_tensor, 0, &mut cache)?;
// Sample first generated token from the last logits position
let first_logits = logits.squeeze(0)?;
let next_token = generation::sample(&first_logits, config)?;
if let Some(eos) = eos_token_id {
if next_token == eos {
let generated_tokens = &tokens[input_ids.len()..];
let output = self
.tokenizer
.decode(generated_tokens, true)
.map_err(|e| anyhow::anyhow!("{e}"))?;
return Ok(output.trim().to_string());
}
}
tokens.push(next_token);
// Decode: generate one token at a time
for _i in 1..config.max_tokens {
let pos = tokens.len() - 1;
let next_input = Tensor::new(&[*tokens.last().unwrap()], &self.device)?.unsqueeze(0)?;
let logits = model.forward(&next_input, pos, &mut cache)?;
let next_logits = logits.squeeze(0)?;
let next_token = generation::sample(&next_logits, config)?;
if let Some(eos) = eos_token_id {
if next_token == eos {
break;
}
}
tokens.push(next_token);
}
let generated_tokens = &tokens[input_ids.len()..];
let output = self
.tokenizer
.decode(generated_tokens, true)
.map_err(|e| anyhow::anyhow!("{e}"))?;
Ok(output.trim().to_string())
}
pub fn model_dir() -> PathBuf {
model_loader::model_cache_dir()
}
pub async fn is_model_downloaded() -> bool {
let dir = Self::model_dir();
tokio::fs::metadata(dir.join(MODEL_FILENAME)).await.is_ok()
&& tokio::fs::metadata(dir.join(TOKENIZER_FILENAME))
.await
.is_ok()
&& tokio::fs::metadata(dir.join(CONFIG_FILENAME)).await.is_ok()
}
}
@@ -0,0 +1,74 @@
use std::path::PathBuf;
use anyhow::{Context, Result};
use super::{CONFIG_FILENAME, HF_REPO, MODEL_FILENAME, TOKENIZER_FILENAME};
pub fn model_cache_dir() -> PathBuf {
directories::ProjectDirs::from("", "", "galaxy")
.map(|dirs| dirs.cache_dir().join("models").join("smollm2-135m"))
.unwrap_or_else(|| PathBuf::from(".galaxy/models/smollm2-135m"))
}
pub async fn ensure_model_available() -> Result<PathBuf> {
let cache_dir = model_cache_dir();
let model_path = cache_dir.join(MODEL_FILENAME);
let tokenizer_path = cache_dir.join(TOKENIZER_FILENAME);
let config_path = cache_dir.join(CONFIG_FILENAME);
if model_path.exists() && tokenizer_path.exists() && config_path.exists() {
log::debug!("Model already cached at {}", cache_dir.display());
return Ok(cache_dir);
}
log::info!("Downloading SmolLM2-135M model from HuggingFace...");
tokio::fs::create_dir_all(&cache_dir)
.await
.with_context(|| format!("creating cache dir {}", cache_dir.display()))?;
let api = hf_hub::api::tokio::Api::new().context("initializing HuggingFace API")?;
let repo = api.model(HF_REPO.to_string());
let downloaded_model = repo
.get(MODEL_FILENAME)
.await
.context("downloading model weights")?;
let downloaded_tokenizer = repo
.get(TOKENIZER_FILENAME)
.await
.context("downloading tokenizer")?;
let downloaded_config = repo
.get(CONFIG_FILENAME)
.await
.context("downloading config")?;
// hf-hub caches files itself, but we symlink/copy to our canonical location
// for predictable access
link_or_copy(&downloaded_model, &model_path).await?;
link_or_copy(&downloaded_tokenizer, &tokenizer_path).await?;
link_or_copy(&downloaded_config, &config_path).await?;
log::info!("Model downloaded and cached at {}", cache_dir.display());
Ok(cache_dir)
}
async fn link_or_copy(src: &std::path::Path, dst: &std::path::Path) -> Result<()> {
if dst.exists() {
return Ok(());
}
// Try symlink first (saves disk space)
#[cfg(unix)]
{
if tokio::fs::symlink(src, dst).await.is_ok() {
return Ok(());
}
}
// Fall back to copy
tokio::fs::copy(src, dst)
.await
.with_context(|| format!("copying {} to {}", src.display(), dst.display()))?;
Ok(())
}
+22
View File
@@ -0,0 +1,22 @@
pub mod engine;
pub mod tasks;
use async_trait::async_trait;
pub use engine::{CancellationToken, Device, GenerationConfig, InferenceEngine};
pub use tasks::{
InputCategory, InputClassificationInput, InputClassificationResult, InputClassificationTask,
PromptSuggestionInput, PromptSuggestionTask, TabNamingInput, TabNamingTask,
};
#[async_trait]
pub trait InferenceTask: Send + Sync {
type Input: Send;
type Output: Send;
async fn run(
&self,
engine: &InferenceEngine,
input: Self::Input,
) -> anyhow::Result<Self::Output>;
}
@@ -0,0 +1,118 @@
use async_trait::async_trait;
use crate::InferenceTask;
use crate::engine::{CancellationToken, GenerationConfig, InferenceEngine};
pub struct InputClassificationTask;
#[derive(Clone)]
pub struct InputClassificationInput {
pub user_input: String,
pub recent_commands: Vec<String>,
pub is_follow_up: bool,
}
#[derive(Debug, Clone, PartialEq)]
pub enum InputCategory {
Shell,
AgentPrompt,
}
pub struct InputClassificationResult {
pub category: InputCategory,
pub confidence: f32,
}
impl InputClassificationTask {
fn build_prompt(input: &InputClassificationInput) -> String {
let history = if input.recent_commands.is_empty() {
String::from("(none)")
} else {
input
.recent_commands
.iter()
.take(3)
.map(|c| format!("- {c}"))
.collect::<Vec<_>>()
.join("\n")
};
let follow_up_hint = if input.is_follow_up {
" The user just received an AI response, so this may be a follow-up."
} else {
""
};
format!(
"<|im_start|>system\n\
You classify terminal input. Respond with ONLY one word: \"shell\" or \"agent\".\n\
\"shell\" = a CLI command the user wants to execute.\n\
\"agent\" = a natural language prompt for an AI assistant.{follow_up_hint}\
<|im_end|>\n\
<|im_start|>user\n\
Recent commands:\n{history}\n\
Classify this input: \"{}\"\
<|im_end|>\n\
<|im_start|>assistant\n",
input.user_input
)
}
fn parse_output(output: &str) -> InputClassificationResult {
let output_lower = output.trim().to_lowercase();
let category = if output_lower.contains("shell") || output_lower.contains("command") {
InputCategory::Shell
} else {
InputCategory::AgentPrompt
};
let confidence = if output_lower == "shell" || output_lower == "agent" {
0.95
} else {
0.7
};
InputClassificationResult {
category,
confidence,
}
}
pub async fn run_cancellable(
&self,
engine: &InferenceEngine,
input: InputClassificationInput,
cancel: &CancellationToken,
) -> anyhow::Result<InputClassificationResult> {
let prompt = Self::build_prompt(&input);
let config = GenerationConfig {
max_tokens: 4,
temperature: 0.0,
top_p: 1.0,
};
let output = engine.generate_cancellable(&prompt, &config, cancel).await?;
Ok(Self::parse_output(&output))
}
}
#[async_trait]
impl InferenceTask for InputClassificationTask {
type Input = InputClassificationInput;
type Output = InputClassificationResult;
async fn run(
&self,
engine: &InferenceEngine,
input: Self::Input,
) -> anyhow::Result<InputClassificationResult> {
let prompt = Self::build_prompt(&input);
let config = GenerationConfig {
max_tokens: 4,
temperature: 0.0,
top_p: 1.0,
};
let output = engine.generate(&prompt, &config).await?;
Ok(Self::parse_output(&output))
}
}
+9
View File
@@ -0,0 +1,9 @@
mod input_classification;
mod prompt_suggestion;
mod tab_naming;
pub use input_classification::{
InputCategory, InputClassificationInput, InputClassificationResult, InputClassificationTask,
};
pub use prompt_suggestion::{PromptSuggestionInput, PromptSuggestionTask};
pub use tab_naming::{TabNamingInput, TabNamingTask};
@@ -0,0 +1,64 @@
use async_trait::async_trait;
use crate::InferenceTask;
use crate::engine::{GenerationConfig, InferenceEngine};
pub struct PromptSuggestionTask;
pub struct PromptSuggestionInput {
pub recent_commands: Vec<String>,
pub current_input: String,
pub working_directory: String,
}
#[async_trait]
impl InferenceTask for PromptSuggestionTask {
type Input = PromptSuggestionInput;
type Output = Vec<String>;
async fn run(
&self,
engine: &InferenceEngine,
input: Self::Input,
) -> anyhow::Result<Vec<String>> {
let history = input
.recent_commands
.iter()
.take(5)
.map(|c| format!("- {c}"))
.collect::<Vec<_>>()
.join("\n");
let prompt = format!(
"<|im_start|>system\n\
You suggest terminal commands. Give exactly 3 suggestions, one per line. \
No numbering, no explanation, just the commands.\
<|im_end|>\n\
<|im_start|>user\n\
Directory: {}\n\
Recent commands:\n{}\n\
Current partial input: \"{}\"\n\
Suggest 3 likely next commands:\
<|im_end|>\n\
<|im_start|>assistant\n",
input.working_directory, history, input.current_input
);
let config = GenerationConfig {
max_tokens: 64,
temperature: 0.6,
top_p: 0.9,
};
let output = engine.generate(&prompt, &config).await?;
let suggestions: Vec<String> = output
.lines()
.map(|l| l.trim().to_string())
.filter(|l| !l.is_empty())
.take(3)
.collect();
Ok(suggestions)
}
}
@@ -0,0 +1,60 @@
use async_trait::async_trait;
use crate::InferenceTask;
use crate::engine::{GenerationConfig, InferenceEngine};
pub struct TabNamingTask;
pub struct TabNamingInput {
pub recent_commands: Vec<String>,
pub working_directory: String,
}
#[async_trait]
impl InferenceTask for TabNamingTask {
type Input = TabNamingInput;
type Output = String;
async fn run(&self, engine: &InferenceEngine, input: Self::Input) -> anyhow::Result<String> {
let commands_str = input
.recent_commands
.iter()
.take(5)
.map(|c| format!("- {c}"))
.collect::<Vec<_>>()
.join("\n");
let prompt = format!(
"<|im_start|>system\n\
You name terminal tabs. Respond with ONLY a short name (2-4 words max). No explanation.\
<|im_end|>\n\
<|im_start|>user\n\
Directory: {}\n\
Recent commands:\n{}\n\
What should this tab be named?\
<|im_end|>\n\
<|im_start|>assistant\n",
input.working_directory, commands_str
);
let config = GenerationConfig {
max_tokens: 12,
temperature: 0.3,
top_p: 0.9,
};
let output = engine.generate(&prompt, &config).await?;
// Clean up: take only the first line, strip quotes
let name = output
.lines()
.next()
.unwrap_or(&output)
.trim()
.trim_matches('"')
.trim_matches('\'')
.to_string();
Ok(name)
}
}
@@ -0,0 +1,335 @@
//! End-to-end scenario tests that validate realistic usage patterns for
//! the local inference engine across all task types (classification,
//! prompt suggestion, tab naming).
use local_inference::{
Device, InferenceEngine, InferenceTask, InputCategory, InputClassificationInput,
InputClassificationTask, PromptSuggestionInput, PromptSuggestionTask, TabNamingInput,
TabNamingTask,
};
use std::time::Instant;
async fn get_engine() -> InferenceEngine {
InferenceEngine::new(Device::Cpu)
.await
.expect("Failed to initialize engine — is the model downloaded?")
}
// --- Full workflow scenarios ---
/// Simulates a user session where they type various inputs and the classifier
/// routes them correctly between shell and agent modes.
#[tokio::test]
async fn scenario_user_session_mode_switching() {
let engine = get_engine().await;
struct TestCase {
input: &'static str,
recent: Vec<&'static str>,
follow_up: bool,
expected: InputCategory,
}
let cases = vec![
TestCase {
input: "cd ~/projects/myapp",
recent: vec!["ls", "pwd"],
follow_up: false,
expected: InputCategory::Shell,
},
TestCase {
input: "npm start",
recent: vec!["cd ~/projects/myapp", "ls"],
follow_up: false,
expected: InputCategory::Shell,
},
TestCase {
input: "why is my server crashing on startup?",
recent: vec!["npm start", "cd ~/projects/myapp"],
follow_up: false,
expected: InputCategory::AgentPrompt,
},
TestCase {
input: "can you also check the environment variables?",
recent: vec!["npm start"],
follow_up: true,
expected: InputCategory::AgentPrompt,
},
TestCase {
input: "export NODE_ENV=production",
recent: vec!["npm start"],
follow_up: false,
expected: InputCategory::Shell,
},
];
println!("\n--- User Session Mode Switching Scenario ---");
for case in &cases {
let result = InputClassificationTask
.run(
&engine,
InputClassificationInput {
user_input: case.input.to_string(),
recent_commands: case.recent.iter().map(|s| s.to_string()).collect(),
is_follow_up: case.follow_up,
},
)
.await
.expect("classification failed");
println!(
" {:?} -> {:?} (expected {:?}, confidence {:.2})",
case.input, result.category, case.expected, result.confidence
);
assert_eq!(
result.category, case.expected,
"Misclassified {:?}",
case.input
);
}
}
/// Simulates prompt suggestions being generated after various shell commands.
#[tokio::test]
async fn scenario_prompt_suggestions_after_commands() {
let engine = get_engine().await;
println!("\n--- Prompt Suggestions After Commands Scenario ---");
// After a failed build
let suggestions = PromptSuggestionTask
.run(
&engine,
PromptSuggestionInput {
recent_commands: vec![
"cargo build".into(),
"cargo test -- --nocapture".into(),
"cargo build".into(), // repeated = likely still failing
],
current_input: String::new(),
working_directory: "/home/user/projects/rust-app".into(),
},
)
.await
.expect("suggestion failed");
println!(" After repeated builds: {suggestions:?}");
assert!(!suggestions.is_empty());
// After git workflow
let suggestions = PromptSuggestionTask
.run(
&engine,
PromptSuggestionInput {
recent_commands: vec![
"git add .".into(),
"git commit -m 'wip'".into(),
"git push".into(),
],
current_input: String::new(),
working_directory: "/home/user/projects/feature-branch".into(),
},
)
.await
.expect("suggestion failed");
println!(" After git workflow: {suggestions:?}");
assert!(!suggestions.is_empty());
// With partial input
let suggestions = PromptSuggestionTask
.run(
&engine,
PromptSuggestionInput {
recent_commands: vec!["docker ps".into(), "docker logs app".into()],
current_input: "docker".to_string(),
working_directory: "/home/user/deployments".into(),
},
)
.await
.expect("suggestion failed");
println!(" With 'docker' partial input: {suggestions:?}");
assert!(!suggestions.is_empty());
}
/// Simulates tab naming for different development contexts.
#[tokio::test]
async fn scenario_tab_naming_development_contexts() {
let engine = get_engine().await;
println!("\n--- Tab Naming for Development Contexts ---");
struct TabCase {
label: &'static str,
commands: Vec<&'static str>,
cwd: &'static str,
}
let cases = vec![
TabCase {
label: "Git operations",
commands: vec!["git log --oneline", "git branch -a", "git fetch origin"],
cwd: "/home/user/projects/galaxy",
},
TabCase {
label: "Docker deployment",
commands: vec!["docker compose up -d", "docker ps", "docker logs web"],
cwd: "/home/user/services/api",
},
TabCase {
label: "Python data science",
commands: vec!["jupyter notebook", "pip install pandas", "python analysis.py"],
cwd: "/home/user/research/data-pipeline",
},
TabCase {
label: "Rust development",
commands: vec!["cargo build", "cargo test", "cargo clippy"],
cwd: "/home/user/projects/my-crate",
},
];
for case in &cases {
let name = TabNamingTask
.run(
&engine,
TabNamingInput {
recent_commands: case.commands.iter().map(|s| s.to_string()).collect(),
working_directory: case.cwd.to_string(),
},
)
.await
.expect("tab naming failed");
println!(" {}: \"{}\"", case.label, name);
assert!(!name.is_empty(), "tab name should not be empty");
assert!(
name.split_whitespace().count() <= 6,
"tab name too long for '{}': \"{}\"",
case.label,
name
);
}
}
/// Tests performance: all tasks should complete quickly enough for interactive use.
#[tokio::test]
async fn scenario_performance_under_load() {
let engine = get_engine().await;
println!("\n--- Performance Under Load ---");
// Classification should be fast (< 500ms on CPU)
let start = Instant::now();
for _ in 0..5 {
InputClassificationTask
.run(
&engine,
InputClassificationInput {
user_input: "git push origin main".to_string(),
recent_commands: vec!["git add .".into(), "git commit -m 'test'".into()],
is_follow_up: false,
},
)
.await
.expect("classification failed");
}
let classification_time = start.elapsed();
println!(
" 5 classifications: {:?} (avg {:?})",
classification_time,
classification_time / 5
);
// Tab naming
let start = Instant::now();
TabNamingTask
.run(
&engine,
TabNamingInput {
recent_commands: vec!["make build".into(), "make test".into()],
working_directory: "/home/user/project".into(),
},
)
.await
.expect("tab naming failed");
let tab_time = start.elapsed();
println!(" 1 tab naming: {:?}", tab_time);
// Prompt suggestion
let start = Instant::now();
PromptSuggestionTask
.run(
&engine,
PromptSuggestionInput {
recent_commands: vec!["ls".into(), "cd src".into()],
current_input: String::new(),
working_directory: "/home/user".into(),
},
)
.await
.expect("prompt suggestion failed");
let suggest_time = start.elapsed();
println!(" 1 prompt suggestion: {:?}", suggest_time);
// Classification should be under 2s per call on CPU (generous limit for CI)
assert!(
classification_time / 5 < std::time::Duration::from_secs(2),
"classification too slow: {:?} per call",
classification_time / 5
);
}
/// Tests that the engine can be reused across many calls without degradation.
#[tokio::test]
async fn scenario_engine_stability_across_calls() {
let engine = get_engine().await;
println!("\n--- Engine Stability Across Calls ---");
// Mix different task types in sequence
for i in 0..3 {
// Classification
let result = InputClassificationTask
.run(
&engine,
InputClassificationInput {
user_input: format!("test command {i}"),
recent_commands: vec![],
is_follow_up: false,
},
)
.await
.expect("classification should not degrade");
assert!(result.confidence > 0.0);
// Tab naming
let name = TabNamingTask
.run(
&engine,
TabNamingInput {
recent_commands: vec![format!("cmd {i}")],
working_directory: format!("/tmp/test-{i}"),
},
)
.await
.expect("tab naming should not degrade");
assert!(!name.is_empty());
// Prompt suggestion
let suggestions = PromptSuggestionTask
.run(
&engine,
PromptSuggestionInput {
recent_commands: vec![format!("action {i}")],
current_input: String::new(),
working_directory: "/tmp".into(),
},
)
.await
.expect("prompt suggestion should not degrade");
assert!(!suggestions.is_empty());
println!(" Round {}: all tasks passed", i + 1);
}
}
+406
View File
@@ -0,0 +1,406 @@
use local_inference::{
Device, GenerationConfig, InferenceEngine, InferenceTask, InputCategory,
InputClassificationInput, InputClassificationTask, PromptSuggestionInput, PromptSuggestionTask,
TabNamingInput, TabNamingTask,
};
async fn get_engine() -> InferenceEngine {
InferenceEngine::new(Device::Cpu)
.await
.expect("Failed to initialize engine — is the model downloaded?")
}
#[tokio::test]
async fn test_engine_initializes() {
let engine = get_engine().await;
assert!(engine.tokenizer().get_vocab_size(true) > 0);
}
#[tokio::test]
async fn test_basic_generation() {
let engine = get_engine().await;
let config = GenerationConfig::deterministic();
let output = engine
.generate("The capital of France is", &config)
.await
.expect("generation failed");
assert!(!output.is_empty(), "generated output should not be empty");
println!("Generated: {output}");
}
#[tokio::test]
async fn test_classify_shell_command() {
let engine = get_engine().await;
let result = InputClassificationTask
.run(
&engine,
InputClassificationInput {
user_input: "ls -la /tmp".to_string(),
recent_commands: vec!["cd /tmp".into(), "mkdir test".into()],
is_follow_up: false,
},
)
.await
.expect("classification failed");
println!(
"\"ls -la /tmp\" -> {:?} (confidence: {:.2})",
result.category, result.confidence
);
assert_eq!(result.category, InputCategory::Shell);
}
#[tokio::test]
async fn test_classify_agent_prompt() {
let engine = get_engine().await;
let result = InputClassificationTask
.run(
&engine,
InputClassificationInput {
user_input: "explain how async works in rust".to_string(),
recent_commands: vec!["cargo build".into()],
is_follow_up: false,
},
)
.await
.expect("classification failed");
println!(
"\"explain how async works in rust\" -> {:?} (confidence: {:.2})",
result.category, result.confidence
);
assert_eq!(result.category, InputCategory::AgentPrompt);
}
#[tokio::test]
async fn test_classify_ambiguous_input() {
let engine = get_engine().await;
// "docker" alone could be either — we just want to make sure it doesn't panic
let result = InputClassificationTask
.run(
&engine,
InputClassificationInput {
user_input: "docker".to_string(),
recent_commands: vec![],
is_follow_up: false,
},
)
.await
.expect("classification failed");
println!(
"\"docker\" -> {:?} (confidence: {:.2})",
result.category, result.confidence
);
// Either classification is fine, just assert it doesn't crash
assert!(result.confidence > 0.0);
}
#[tokio::test]
async fn test_tab_naming() {
let engine = get_engine().await;
let name = TabNamingTask
.run(
&engine,
TabNamingInput {
recent_commands: vec![
"git log --oneline".into(),
"git diff".into(),
"git add .".into(),
],
working_directory: "/home/user/projects/galaxy".into(),
},
)
.await
.expect("tab naming failed");
println!("Tab name: \"{name}\"");
assert!(!name.is_empty());
// Tab names should be short
assert!(
name.split_whitespace().count() <= 6,
"tab name too long: \"{name}\""
);
}
#[tokio::test]
async fn test_prompt_suggestions() {
let engine = get_engine().await;
let suggestions = PromptSuggestionTask
.run(
&engine,
PromptSuggestionInput {
recent_commands: vec!["cargo build".into(), "cargo test".into()],
current_input: "cargo".to_string(),
working_directory: "/home/user/projects/galaxy".into(),
},
)
.await
.expect("prompt suggestion failed");
println!("Suggestions: {suggestions:?}");
assert!(
!suggestions.is_empty(),
"should have at least one suggestion"
);
}
#[tokio::test]
async fn test_generation_respects_max_tokens() {
let engine = get_engine().await;
let config = GenerationConfig {
max_tokens: 5,
temperature: 0.0,
top_p: 1.0,
};
let output = engine
.generate("Once upon a time", &config)
.await
.expect("generation failed");
let token_count = engine
.tokenizer()
.encode(output.as_str(), false)
.map(|e| e.get_ids().len())
.unwrap_or(0);
println!("Generated ({token_count} tokens): \"{output}\"");
// Should be roughly around max_tokens (could be less if EOS hit)
assert!(token_count <= 6, "generated too many tokens: {token_count}");
}
// --- Edge case tests ---
#[tokio::test]
async fn test_classify_empty_input() {
let engine = get_engine().await;
let result = InputClassificationTask
.run(
&engine,
InputClassificationInput {
user_input: String::new(),
recent_commands: vec![],
is_follow_up: false,
},
)
.await
.expect("classification should not panic on empty input");
println!(
"empty input -> {:?} (confidence: {:.2})",
result.category, result.confidence
);
assert!(result.confidence > 0.0);
}
#[tokio::test]
async fn test_classify_very_long_input() {
let engine = get_engine().await;
let long_input = "a".repeat(500);
let result = InputClassificationTask
.run(
&engine,
InputClassificationInput {
user_input: long_input,
recent_commands: vec!["ls".into()],
is_follow_up: false,
},
)
.await
.expect("classification should handle long input");
println!(
"long input -> {:?} (confidence: {:.2})",
result.category, result.confidence
);
assert!(result.confidence > 0.0);
}
#[tokio::test]
async fn test_classify_special_characters() {
let engine = get_engine().await;
let result = InputClassificationTask
.run(
&engine,
InputClassificationInput {
user_input: "find . -name '*.rs' | xargs grep -l 'TODO'".to_string(),
recent_commands: vec!["grep -rn test .".into()],
is_follow_up: false,
},
)
.await
.expect("classification should handle special characters");
println!(
"pipe command -> {:?} (confidence: {:.2})",
result.category, result.confidence
);
assert_eq!(result.category, InputCategory::Shell);
}
#[tokio::test]
async fn test_classify_follow_up_context() {
let engine = get_engine().await;
let result = InputClassificationTask
.run(
&engine,
InputClassificationInput {
user_input: "can you also add error handling?".to_string(),
recent_commands: vec!["cargo build".into()],
is_follow_up: true,
},
)
.await
.expect("classification should handle follow-up");
println!(
"follow-up -> {:?} (confidence: {:.2})",
result.category, result.confidence
);
assert_eq!(result.category, InputCategory::AgentPrompt);
}
#[tokio::test]
async fn test_classify_multiple_sequential_calls() {
let engine = get_engine().await;
let inputs = vec![
("git status", InputCategory::Shell),
("what does this error mean?", InputCategory::AgentPrompt),
("npm install express", InputCategory::Shell),
("refactor this to use async/await", InputCategory::AgentPrompt),
];
for (input, expected_category) in inputs {
let result = InputClassificationTask
.run(
&engine,
InputClassificationInput {
user_input: input.to_string(),
recent_commands: vec!["ls".into()],
is_follow_up: false,
},
)
.await
.expect("classification failed");
println!("\"{input}\" -> {:?} (confidence: {:.2})", result.category, result.confidence);
assert_eq!(
result.category, expected_category,
"expected {expected_category:?} for \"{input}\", got {:?}",
result.category
);
}
}
#[tokio::test]
async fn test_tab_naming_empty_commands() {
let engine = get_engine().await;
let name = TabNamingTask
.run(
&engine,
TabNamingInput {
recent_commands: vec![],
working_directory: "/home/user".into(),
},
)
.await
.expect("tab naming should handle empty commands");
println!("Tab name (no commands): \"{name}\"");
assert!(!name.is_empty());
}
#[tokio::test]
async fn test_tab_naming_deep_directory() {
let engine = get_engine().await;
let name = TabNamingTask
.run(
&engine,
TabNamingInput {
recent_commands: vec!["python train.py".into(), "tensorboard --logdir=runs".into()],
working_directory: "/home/user/projects/ml-research/experiments/transformer-v2"
.into(),
},
)
.await
.expect("tab naming failed");
println!("Tab name (deep dir): \"{name}\"");
assert!(!name.is_empty());
assert!(
name.split_whitespace().count() <= 6,
"tab name too long: \"{name}\""
);
}
#[tokio::test]
async fn test_prompt_suggestions_empty_input() {
let engine = get_engine().await;
let suggestions = PromptSuggestionTask
.run(
&engine,
PromptSuggestionInput {
recent_commands: vec!["git status".into(), "git add .".into()],
current_input: String::new(),
working_directory: "/home/user/project".into(),
},
)
.await
.expect("prompt suggestion should handle empty input");
println!("Suggestions (empty input): {suggestions:?}");
assert!(
!suggestions.is_empty(),
"should suggest something even with empty input"
);
}
#[tokio::test]
async fn test_prompt_suggestions_no_history() {
let engine = get_engine().await;
let suggestions = PromptSuggestionTask
.run(
&engine,
PromptSuggestionInput {
recent_commands: vec![],
current_input: "docker".to_string(),
working_directory: "/tmp".into(),
},
)
.await
.expect("prompt suggestion should handle no history");
println!("Suggestions (no history): {suggestions:?}");
assert!(
!suggestions.is_empty(),
"should suggest something even without history"
);
}
#[tokio::test]
async fn test_deterministic_classification() {
let engine = get_engine().await;
let input = InputClassificationInput {
user_input: "ls -la".to_string(),
recent_commands: vec!["cd /tmp".into()],
is_follow_up: false,
};
let result1 = InputClassificationTask
.run(&engine, input.clone())
.await
.expect("first classification failed");
let result2 = InputClassificationTask
.run(&engine, input)
.await
.expect("second classification failed");
// With temperature 0.0, results should be deterministic
assert_eq!(
result1.category, result2.category,
"deterministic classification should yield consistent results"
);
}