Bump version to 1.6.3
Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.6
parent
4ba9706e35
commit
59cfd0e2f5
@@ -0,0 +1,29 @@
|
||||
[package]
|
||||
name = "local_inference"
|
||||
authors.workspace = true
|
||||
edition = "2024"
|
||||
publish.workspace = true
|
||||
license.workspace = true
|
||||
|
||||
[features]
|
||||
default = ["cpu"]
|
||||
cpu = []
|
||||
metal = ["candle-core/metal", "candle-nn/metal", "candle-transformers/metal"]
|
||||
cuda = ["candle-core/cuda", "candle-nn/cuda", "candle-transformers/cuda"]
|
||||
|
||||
[dependencies]
|
||||
anyhow.workspace = true
|
||||
async-trait.workspace = true
|
||||
candle-core = "0.9.2"
|
||||
candle-nn = "0.9.2"
|
||||
candle-transformers = "0.9.2"
|
||||
directories.workspace = true
|
||||
hf-hub = { version = "0.4", features = ["tokio"] }
|
||||
log.workspace = true
|
||||
serde.workspace = true
|
||||
serde_json.workspace = true
|
||||
tokenizers = "0.21.4"
|
||||
tokio = { workspace = true, features = ["fs", "sync", "rt", "macros", "rt-multi-thread"] }
|
||||
|
||||
[dev-dependencies]
|
||||
env_logger = "0.10"
|
||||
@@ -0,0 +1,66 @@
|
||||
use local_inference::{
|
||||
Device, InferenceEngine, InferenceTask, InputClassificationInput, InputClassificationTask,
|
||||
TabNamingInput, TabNamingTask,
|
||||
};
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() -> anyhow::Result<()> {
|
||||
env_logger::init();
|
||||
|
||||
println!("Initializing inference engine (this may download the model on first run)...");
|
||||
let engine = InferenceEngine::new(Device::best_available()).await?;
|
||||
println!("Engine ready!\n");
|
||||
|
||||
// --- Input Classification ---
|
||||
let test_inputs = vec![
|
||||
"ls -la",
|
||||
"git status",
|
||||
"what files are in this directory?",
|
||||
"explain this error to me",
|
||||
"docker compose up -d",
|
||||
"how do I fix this segfault?",
|
||||
"cd /tmp && rm -rf build/",
|
||||
"refactor the auth module to use JWT",
|
||||
];
|
||||
|
||||
println!("=== Input Classification ===\n");
|
||||
for input in &test_inputs {
|
||||
let result = InputClassificationTask
|
||||
.run(
|
||||
&engine,
|
||||
InputClassificationInput {
|
||||
user_input: input.to_string(),
|
||||
recent_commands: vec!["git log".into(), "npm test".into()],
|
||||
is_follow_up: false,
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
|
||||
println!(
|
||||
" {:50} -> {:?} (confidence: {:.2})",
|
||||
format!("\"{}\"", input),
|
||||
result.category,
|
||||
result.confidence
|
||||
);
|
||||
}
|
||||
|
||||
// --- Tab Naming ---
|
||||
println!("\n=== Tab Naming ===\n");
|
||||
let tab_name = TabNamingTask
|
||||
.run(
|
||||
&engine,
|
||||
TabNamingInput {
|
||||
recent_commands: vec![
|
||||
"git checkout feature/auth".into(),
|
||||
"cargo test".into(),
|
||||
"vim src/auth/mod.rs".into(),
|
||||
],
|
||||
working_directory: "/home/user/projects/myapp".into(),
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
|
||||
println!(" Suggested tab name: \"{tab_name}\"");
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,115 @@
|
||||
use anyhow::Result;
|
||||
use candle_core::Tensor;
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct GenerationConfig {
|
||||
pub max_tokens: usize,
|
||||
pub temperature: f64,
|
||||
pub top_p: f64,
|
||||
}
|
||||
|
||||
impl Default for GenerationConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
max_tokens: 32,
|
||||
temperature: 0.7,
|
||||
top_p: 0.9,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl GenerationConfig {
|
||||
pub fn deterministic() -> Self {
|
||||
Self {
|
||||
max_tokens: 32,
|
||||
temperature: 0.0,
|
||||
top_p: 1.0,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn creative() -> Self {
|
||||
Self {
|
||||
max_tokens: 64,
|
||||
temperature: 0.9,
|
||||
top_p: 0.95,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn sample(logits: &Tensor, config: &GenerationConfig) -> Result<u32> {
|
||||
if config.temperature == 0.0 {
|
||||
return greedy(logits);
|
||||
}
|
||||
|
||||
let logits = logits.to_dtype(candle_core::DType::F32)?;
|
||||
let logits_vec = logits.to_vec1::<f32>()?;
|
||||
|
||||
// Apply temperature
|
||||
let scaled: Vec<f64> = logits_vec
|
||||
.iter()
|
||||
.map(|&x| (x as f64) / config.temperature)
|
||||
.collect();
|
||||
|
||||
// Softmax
|
||||
let max_val = scaled.iter().cloned().fold(f64::NEG_INFINITY, f64::max);
|
||||
let exp: Vec<f64> = scaled.iter().map(|&x| (x - max_val).exp()).collect();
|
||||
let sum: f64 = exp.iter().sum();
|
||||
let probs: Vec<f64> = exp.iter().map(|&x| x / sum).collect();
|
||||
|
||||
// Top-p (nucleus) sampling
|
||||
let mut indexed_probs: Vec<(usize, f64)> = probs.iter().copied().enumerate().collect();
|
||||
indexed_probs.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap_or(std::cmp::Ordering::Equal));
|
||||
|
||||
let mut cumulative = 0.0;
|
||||
let mut candidates = Vec::new();
|
||||
for (idx, prob) in &indexed_probs {
|
||||
cumulative += prob;
|
||||
candidates.push((*idx, *prob));
|
||||
if cumulative >= config.top_p {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// Renormalize candidates
|
||||
let candidate_sum: f64 = candidates.iter().map(|(_, p)| p).sum();
|
||||
let threshold = simple_rng() * candidate_sum;
|
||||
|
||||
let mut acc = 0.0;
|
||||
for (idx, prob) in &candidates {
|
||||
acc += prob;
|
||||
if acc >= threshold {
|
||||
return Ok(*idx as u32);
|
||||
}
|
||||
}
|
||||
|
||||
// Fallback to top candidate
|
||||
Ok(candidates[0].0 as u32)
|
||||
}
|
||||
|
||||
fn greedy(logits: &Tensor) -> Result<u32> {
|
||||
let logits = logits.to_dtype(candle_core::DType::F32)?;
|
||||
let logits_vec = logits.to_vec1::<f32>()?;
|
||||
let max_idx = logits_vec
|
||||
.iter()
|
||||
.enumerate()
|
||||
.max_by(|(_, a), (_, b)| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal))
|
||||
.map(|(idx, _)| idx)
|
||||
.unwrap_or(0);
|
||||
Ok(max_idx as u32)
|
||||
}
|
||||
|
||||
fn simple_rng() -> f64 {
|
||||
use std::collections::hash_map::DefaultHasher;
|
||||
use std::hash::{Hash, Hasher};
|
||||
use std::time::SystemTime;
|
||||
|
||||
let mut hasher = DefaultHasher::new();
|
||||
SystemTime::now()
|
||||
.duration_since(SystemTime::UNIX_EPOCH)
|
||||
.unwrap_or_default()
|
||||
.as_nanos()
|
||||
.hash(&mut hasher);
|
||||
std::thread::current().id().hash(&mut hasher);
|
||||
let hash = hasher.finish();
|
||||
(hash as f64) / (u64::MAX as f64)
|
||||
}
|
||||
@@ -0,0 +1,309 @@
|
||||
mod generation;
|
||||
mod model_loader;
|
||||
|
||||
use std::path::PathBuf;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::sync::Arc;
|
||||
|
||||
use anyhow::{Context as _, Result};
|
||||
use candle_core::{DType, Tensor};
|
||||
use candle_transformers::models::llama::{Cache, Config, Llama, LlamaConfig};
|
||||
use tokenizers::Tokenizer;
|
||||
use tokio::sync::Mutex;
|
||||
|
||||
pub use generation::GenerationConfig;
|
||||
|
||||
/// A token that can be used to cancel an in-progress generation.
|
||||
/// Clone it and pass to `generate_cancellable`, then call `cancel()` to
|
||||
/// interrupt the generation loop between tokens.
|
||||
#[derive(Clone)]
|
||||
pub struct CancellationToken {
|
||||
cancelled: Arc<AtomicBool>,
|
||||
}
|
||||
|
||||
impl CancellationToken {
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
cancelled: Arc::new(AtomicBool::new(false)),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn cancel(&self) {
|
||||
self.cancelled.store(true, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
pub fn is_cancelled(&self) -> bool {
|
||||
self.cancelled.load(Ordering::Relaxed)
|
||||
}
|
||||
}
|
||||
|
||||
const HF_REPO: &str = "HuggingFaceTB/SmolLM2-135M-Instruct";
|
||||
const MODEL_FILENAME: &str = "model.safetensors";
|
||||
const TOKENIZER_FILENAME: &str = "tokenizer.json";
|
||||
const CONFIG_FILENAME: &str = "config.json";
|
||||
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
pub enum Device {
|
||||
Cpu,
|
||||
#[cfg(feature = "metal")]
|
||||
Metal,
|
||||
#[cfg(feature = "cuda")]
|
||||
Cuda(usize),
|
||||
}
|
||||
|
||||
impl Device {
|
||||
pub fn best_available() -> Self {
|
||||
#[cfg(feature = "metal")]
|
||||
{
|
||||
return Device::Metal;
|
||||
}
|
||||
#[cfg(feature = "cuda")]
|
||||
{
|
||||
return Device::Cuda(0);
|
||||
}
|
||||
#[cfg(not(any(feature = "metal", feature = "cuda")))]
|
||||
{
|
||||
Device::Cpu
|
||||
}
|
||||
}
|
||||
|
||||
fn to_candle_device(&self) -> Result<candle_core::Device> {
|
||||
match self {
|
||||
Device::Cpu => Ok(candle_core::Device::Cpu),
|
||||
#[cfg(feature = "metal")]
|
||||
Device::Metal => Ok(candle_core::Device::new_metal(0)?),
|
||||
#[cfg(feature = "cuda")]
|
||||
Device::Cuda(ordinal) => Ok(candle_core::Device::new_cuda(*ordinal)?),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub struct InferenceEngine {
|
||||
model: Arc<Mutex<Llama>>,
|
||||
cache: Arc<Mutex<Cache>>,
|
||||
config: Config,
|
||||
tokenizer: Tokenizer,
|
||||
device: candle_core::Device,
|
||||
}
|
||||
|
||||
impl InferenceEngine {
|
||||
pub async fn new(device: Device) -> Result<Self> {
|
||||
let candle_device = device.to_candle_device()?;
|
||||
let model_dir = model_loader::ensure_model_available().await?;
|
||||
|
||||
let config_path = model_dir.join(CONFIG_FILENAME);
|
||||
let config_str = tokio::fs::read_to_string(&config_path)
|
||||
.await
|
||||
.with_context(|| format!("reading config from {}", config_path.display()))?;
|
||||
let llama_config: LlamaConfig =
|
||||
serde_json::from_str(&config_str).context("parsing model config")?;
|
||||
let config = llama_config.into_config(false);
|
||||
|
||||
let model_path = model_dir.join(MODEL_FILENAME);
|
||||
let vb = unsafe {
|
||||
candle_nn::VarBuilder::from_mmaped_safetensors(
|
||||
&[model_path],
|
||||
DType::F32,
|
||||
&candle_device,
|
||||
)?
|
||||
};
|
||||
|
||||
let model = Llama::load(vb, &config).context("loading SmolLM2 model")?;
|
||||
let cache = Cache::new(true, DType::F32, &config, &candle_device)?;
|
||||
|
||||
let tokenizer_path = model_dir.join(TOKENIZER_FILENAME);
|
||||
let tokenizer_bytes = tokio::fs::read(&tokenizer_path)
|
||||
.await
|
||||
.with_context(|| format!("reading tokenizer from {}", tokenizer_path.display()))?;
|
||||
let tokenizer =
|
||||
Tokenizer::from_bytes(&tokenizer_bytes).map_err(|e| anyhow::anyhow!("{e}"))?;
|
||||
|
||||
log::info!(
|
||||
"Local inference engine initialized (device: {:?}, model: {})",
|
||||
device,
|
||||
HF_REPO
|
||||
);
|
||||
|
||||
Ok(Self {
|
||||
model: Arc::new(Mutex::new(model)),
|
||||
cache: Arc::new(Mutex::new(cache)),
|
||||
config,
|
||||
tokenizer,
|
||||
device: candle_device,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn tokenizer(&self) -> &Tokenizer {
|
||||
&self.tokenizer
|
||||
}
|
||||
|
||||
pub fn device(&self) -> &candle_core::Device {
|
||||
&self.device
|
||||
}
|
||||
|
||||
pub async fn generate_cancellable(
|
||||
&self,
|
||||
prompt: &str,
|
||||
config: &GenerationConfig,
|
||||
cancel: &CancellationToken,
|
||||
) -> Result<String> {
|
||||
let encoding = self
|
||||
.tokenizer
|
||||
.encode(prompt, true)
|
||||
.map_err(|e| anyhow::anyhow!("{e}"))?;
|
||||
|
||||
let input_ids = encoding.get_ids().to_vec();
|
||||
let mut tokens = input_ids.clone();
|
||||
|
||||
let eos_token_id = self
|
||||
.tokenizer
|
||||
.token_to_id("</s>")
|
||||
.or_else(|| self.tokenizer.token_to_id("<|endoftext|>"))
|
||||
.or_else(|| self.tokenizer.token_to_id("<|im_end|>"));
|
||||
|
||||
let model = self.model.lock().await;
|
||||
let mut cache = self.cache.lock().await;
|
||||
|
||||
if cancel.is_cancelled() {
|
||||
anyhow::bail!("cancelled before generation started");
|
||||
}
|
||||
|
||||
// Reset cache for new generation
|
||||
*cache = Cache::new(true, DType::F32, &self.config, &self.device)?;
|
||||
|
||||
// Prefill: process all input tokens at once
|
||||
let input_tensor = Tensor::new(input_ids.as_slice(), &self.device)?.unsqueeze(0)?;
|
||||
let logits = model.forward(&input_tensor, 0, &mut cache)?;
|
||||
|
||||
if cancel.is_cancelled() {
|
||||
anyhow::bail!("cancelled during prefill");
|
||||
}
|
||||
|
||||
// Sample first generated token from the last logits position
|
||||
let first_logits = logits.squeeze(0)?;
|
||||
let next_token = generation::sample(&first_logits, config)?;
|
||||
|
||||
if let Some(eos) = eos_token_id {
|
||||
if next_token == eos {
|
||||
let generated_tokens = &tokens[input_ids.len()..];
|
||||
let output = self
|
||||
.tokenizer
|
||||
.decode(generated_tokens, true)
|
||||
.map_err(|e| anyhow::anyhow!("{e}"))?;
|
||||
return Ok(output.trim().to_string());
|
||||
}
|
||||
}
|
||||
tokens.push(next_token);
|
||||
|
||||
// Decode: generate one token at a time, checking cancellation between tokens
|
||||
for _i in 1..config.max_tokens {
|
||||
if cancel.is_cancelled() {
|
||||
anyhow::bail!("cancelled during generation");
|
||||
}
|
||||
|
||||
let pos = tokens.len() - 1;
|
||||
let next_input = Tensor::new(&[*tokens.last().unwrap()], &self.device)?.unsqueeze(0)?;
|
||||
let logits = model.forward(&next_input, pos, &mut cache)?;
|
||||
|
||||
let next_logits = logits.squeeze(0)?;
|
||||
let next_token = generation::sample(&next_logits, config)?;
|
||||
|
||||
if let Some(eos) = eos_token_id {
|
||||
if next_token == eos {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
tokens.push(next_token);
|
||||
}
|
||||
|
||||
let generated_tokens = &tokens[input_ids.len()..];
|
||||
let output = self
|
||||
.tokenizer
|
||||
.decode(generated_tokens, true)
|
||||
.map_err(|e| anyhow::anyhow!("{e}"))?;
|
||||
|
||||
Ok(output.trim().to_string())
|
||||
}
|
||||
|
||||
pub async fn generate(&self, prompt: &str, config: &GenerationConfig) -> Result<String> {
|
||||
let encoding = self
|
||||
.tokenizer
|
||||
.encode(prompt, true)
|
||||
.map_err(|e| anyhow::anyhow!("{e}"))?;
|
||||
|
||||
let input_ids = encoding.get_ids().to_vec();
|
||||
let mut tokens = input_ids.clone();
|
||||
|
||||
let eos_token_id = self
|
||||
.tokenizer
|
||||
.token_to_id("</s>")
|
||||
.or_else(|| self.tokenizer.token_to_id("<|endoftext|>"))
|
||||
.or_else(|| self.tokenizer.token_to_id("<|im_end|>"));
|
||||
|
||||
let model = self.model.lock().await;
|
||||
let mut cache = self.cache.lock().await;
|
||||
|
||||
// Reset cache for new generation
|
||||
*cache = Cache::new(true, DType::F32, &self.config, &self.device)?;
|
||||
|
||||
// Prefill: process all input tokens at once
|
||||
let input_tensor = Tensor::new(input_ids.as_slice(), &self.device)?.unsqueeze(0)?;
|
||||
let logits = model.forward(&input_tensor, 0, &mut cache)?;
|
||||
|
||||
// Sample first generated token from the last logits position
|
||||
let first_logits = logits.squeeze(0)?;
|
||||
let next_token = generation::sample(&first_logits, config)?;
|
||||
|
||||
if let Some(eos) = eos_token_id {
|
||||
if next_token == eos {
|
||||
let generated_tokens = &tokens[input_ids.len()..];
|
||||
let output = self
|
||||
.tokenizer
|
||||
.decode(generated_tokens, true)
|
||||
.map_err(|e| anyhow::anyhow!("{e}"))?;
|
||||
return Ok(output.trim().to_string());
|
||||
}
|
||||
}
|
||||
tokens.push(next_token);
|
||||
|
||||
// Decode: generate one token at a time
|
||||
for _i in 1..config.max_tokens {
|
||||
let pos = tokens.len() - 1;
|
||||
let next_input = Tensor::new(&[*tokens.last().unwrap()], &self.device)?.unsqueeze(0)?;
|
||||
let logits = model.forward(&next_input, pos, &mut cache)?;
|
||||
|
||||
let next_logits = logits.squeeze(0)?;
|
||||
let next_token = generation::sample(&next_logits, config)?;
|
||||
|
||||
if let Some(eos) = eos_token_id {
|
||||
if next_token == eos {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
tokens.push(next_token);
|
||||
}
|
||||
|
||||
let generated_tokens = &tokens[input_ids.len()..];
|
||||
let output = self
|
||||
.tokenizer
|
||||
.decode(generated_tokens, true)
|
||||
.map_err(|e| anyhow::anyhow!("{e}"))?;
|
||||
|
||||
Ok(output.trim().to_string())
|
||||
}
|
||||
|
||||
pub fn model_dir() -> PathBuf {
|
||||
model_loader::model_cache_dir()
|
||||
}
|
||||
|
||||
pub async fn is_model_downloaded() -> bool {
|
||||
let dir = Self::model_dir();
|
||||
tokio::fs::metadata(dir.join(MODEL_FILENAME)).await.is_ok()
|
||||
&& tokio::fs::metadata(dir.join(TOKENIZER_FILENAME))
|
||||
.await
|
||||
.is_ok()
|
||||
&& tokio::fs::metadata(dir.join(CONFIG_FILENAME)).await.is_ok()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,74 @@
|
||||
use std::path::PathBuf;
|
||||
|
||||
use anyhow::{Context, Result};
|
||||
|
||||
use super::{CONFIG_FILENAME, HF_REPO, MODEL_FILENAME, TOKENIZER_FILENAME};
|
||||
|
||||
pub fn model_cache_dir() -> PathBuf {
|
||||
directories::ProjectDirs::from("", "", "galaxy")
|
||||
.map(|dirs| dirs.cache_dir().join("models").join("smollm2-135m"))
|
||||
.unwrap_or_else(|| PathBuf::from(".galaxy/models/smollm2-135m"))
|
||||
}
|
||||
|
||||
pub async fn ensure_model_available() -> Result<PathBuf> {
|
||||
let cache_dir = model_cache_dir();
|
||||
|
||||
let model_path = cache_dir.join(MODEL_FILENAME);
|
||||
let tokenizer_path = cache_dir.join(TOKENIZER_FILENAME);
|
||||
let config_path = cache_dir.join(CONFIG_FILENAME);
|
||||
|
||||
if model_path.exists() && tokenizer_path.exists() && config_path.exists() {
|
||||
log::debug!("Model already cached at {}", cache_dir.display());
|
||||
return Ok(cache_dir);
|
||||
}
|
||||
|
||||
log::info!("Downloading SmolLM2-135M model from HuggingFace...");
|
||||
tokio::fs::create_dir_all(&cache_dir)
|
||||
.await
|
||||
.with_context(|| format!("creating cache dir {}", cache_dir.display()))?;
|
||||
|
||||
let api = hf_hub::api::tokio::Api::new().context("initializing HuggingFace API")?;
|
||||
let repo = api.model(HF_REPO.to_string());
|
||||
|
||||
let downloaded_model = repo
|
||||
.get(MODEL_FILENAME)
|
||||
.await
|
||||
.context("downloading model weights")?;
|
||||
let downloaded_tokenizer = repo
|
||||
.get(TOKENIZER_FILENAME)
|
||||
.await
|
||||
.context("downloading tokenizer")?;
|
||||
let downloaded_config = repo
|
||||
.get(CONFIG_FILENAME)
|
||||
.await
|
||||
.context("downloading config")?;
|
||||
|
||||
// hf-hub caches files itself, but we symlink/copy to our canonical location
|
||||
// for predictable access
|
||||
link_or_copy(&downloaded_model, &model_path).await?;
|
||||
link_or_copy(&downloaded_tokenizer, &tokenizer_path).await?;
|
||||
link_or_copy(&downloaded_config, &config_path).await?;
|
||||
|
||||
log::info!("Model downloaded and cached at {}", cache_dir.display());
|
||||
Ok(cache_dir)
|
||||
}
|
||||
|
||||
async fn link_or_copy(src: &std::path::Path, dst: &std::path::Path) -> Result<()> {
|
||||
if dst.exists() {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
// Try symlink first (saves disk space)
|
||||
#[cfg(unix)]
|
||||
{
|
||||
if tokio::fs::symlink(src, dst).await.is_ok() {
|
||||
return Ok(());
|
||||
}
|
||||
}
|
||||
|
||||
// Fall back to copy
|
||||
tokio::fs::copy(src, dst)
|
||||
.await
|
||||
.with_context(|| format!("copying {} to {}", src.display(), dst.display()))?;
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
pub mod engine;
|
||||
pub mod tasks;
|
||||
|
||||
use async_trait::async_trait;
|
||||
|
||||
pub use engine::{CancellationToken, Device, GenerationConfig, InferenceEngine};
|
||||
pub use tasks::{
|
||||
InputCategory, InputClassificationInput, InputClassificationResult, InputClassificationTask,
|
||||
PromptSuggestionInput, PromptSuggestionTask, TabNamingInput, TabNamingTask,
|
||||
};
|
||||
|
||||
#[async_trait]
|
||||
pub trait InferenceTask: Send + Sync {
|
||||
type Input: Send;
|
||||
type Output: Send;
|
||||
|
||||
async fn run(
|
||||
&self,
|
||||
engine: &InferenceEngine,
|
||||
input: Self::Input,
|
||||
) -> anyhow::Result<Self::Output>;
|
||||
}
|
||||
@@ -0,0 +1,118 @@
|
||||
use async_trait::async_trait;
|
||||
|
||||
use crate::InferenceTask;
|
||||
use crate::engine::{CancellationToken, GenerationConfig, InferenceEngine};
|
||||
|
||||
pub struct InputClassificationTask;
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct InputClassificationInput {
|
||||
pub user_input: String,
|
||||
pub recent_commands: Vec<String>,
|
||||
pub is_follow_up: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub enum InputCategory {
|
||||
Shell,
|
||||
AgentPrompt,
|
||||
}
|
||||
|
||||
pub struct InputClassificationResult {
|
||||
pub category: InputCategory,
|
||||
pub confidence: f32,
|
||||
}
|
||||
|
||||
impl InputClassificationTask {
|
||||
fn build_prompt(input: &InputClassificationInput) -> String {
|
||||
let history = if input.recent_commands.is_empty() {
|
||||
String::from("(none)")
|
||||
} else {
|
||||
input
|
||||
.recent_commands
|
||||
.iter()
|
||||
.take(3)
|
||||
.map(|c| format!("- {c}"))
|
||||
.collect::<Vec<_>>()
|
||||
.join("\n")
|
||||
};
|
||||
|
||||
let follow_up_hint = if input.is_follow_up {
|
||||
" The user just received an AI response, so this may be a follow-up."
|
||||
} else {
|
||||
""
|
||||
};
|
||||
|
||||
format!(
|
||||
"<|im_start|>system\n\
|
||||
You classify terminal input. Respond with ONLY one word: \"shell\" or \"agent\".\n\
|
||||
\"shell\" = a CLI command the user wants to execute.\n\
|
||||
\"agent\" = a natural language prompt for an AI assistant.{follow_up_hint}\
|
||||
<|im_end|>\n\
|
||||
<|im_start|>user\n\
|
||||
Recent commands:\n{history}\n\
|
||||
Classify this input: \"{}\"\
|
||||
<|im_end|>\n\
|
||||
<|im_start|>assistant\n",
|
||||
input.user_input
|
||||
)
|
||||
}
|
||||
|
||||
fn parse_output(output: &str) -> InputClassificationResult {
|
||||
let output_lower = output.trim().to_lowercase();
|
||||
|
||||
let category = if output_lower.contains("shell") || output_lower.contains("command") {
|
||||
InputCategory::Shell
|
||||
} else {
|
||||
InputCategory::AgentPrompt
|
||||
};
|
||||
|
||||
let confidence = if output_lower == "shell" || output_lower == "agent" {
|
||||
0.95
|
||||
} else {
|
||||
0.7
|
||||
};
|
||||
|
||||
InputClassificationResult {
|
||||
category,
|
||||
confidence,
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn run_cancellable(
|
||||
&self,
|
||||
engine: &InferenceEngine,
|
||||
input: InputClassificationInput,
|
||||
cancel: &CancellationToken,
|
||||
) -> anyhow::Result<InputClassificationResult> {
|
||||
let prompt = Self::build_prompt(&input);
|
||||
let config = GenerationConfig {
|
||||
max_tokens: 4,
|
||||
temperature: 0.0,
|
||||
top_p: 1.0,
|
||||
};
|
||||
let output = engine.generate_cancellable(&prompt, &config, cancel).await?;
|
||||
Ok(Self::parse_output(&output))
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl InferenceTask for InputClassificationTask {
|
||||
type Input = InputClassificationInput;
|
||||
type Output = InputClassificationResult;
|
||||
|
||||
async fn run(
|
||||
&self,
|
||||
engine: &InferenceEngine,
|
||||
input: Self::Input,
|
||||
) -> anyhow::Result<InputClassificationResult> {
|
||||
let prompt = Self::build_prompt(&input);
|
||||
let config = GenerationConfig {
|
||||
max_tokens: 4,
|
||||
temperature: 0.0,
|
||||
top_p: 1.0,
|
||||
};
|
||||
let output = engine.generate(&prompt, &config).await?;
|
||||
Ok(Self::parse_output(&output))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
mod input_classification;
|
||||
mod prompt_suggestion;
|
||||
mod tab_naming;
|
||||
|
||||
pub use input_classification::{
|
||||
InputCategory, InputClassificationInput, InputClassificationResult, InputClassificationTask,
|
||||
};
|
||||
pub use prompt_suggestion::{PromptSuggestionInput, PromptSuggestionTask};
|
||||
pub use tab_naming::{TabNamingInput, TabNamingTask};
|
||||
@@ -0,0 +1,64 @@
|
||||
use async_trait::async_trait;
|
||||
|
||||
use crate::InferenceTask;
|
||||
use crate::engine::{GenerationConfig, InferenceEngine};
|
||||
|
||||
pub struct PromptSuggestionTask;
|
||||
|
||||
pub struct PromptSuggestionInput {
|
||||
pub recent_commands: Vec<String>,
|
||||
pub current_input: String,
|
||||
pub working_directory: String,
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl InferenceTask for PromptSuggestionTask {
|
||||
type Input = PromptSuggestionInput;
|
||||
type Output = Vec<String>;
|
||||
|
||||
async fn run(
|
||||
&self,
|
||||
engine: &InferenceEngine,
|
||||
input: Self::Input,
|
||||
) -> anyhow::Result<Vec<String>> {
|
||||
let history = input
|
||||
.recent_commands
|
||||
.iter()
|
||||
.take(5)
|
||||
.map(|c| format!("- {c}"))
|
||||
.collect::<Vec<_>>()
|
||||
.join("\n");
|
||||
|
||||
let prompt = format!(
|
||||
"<|im_start|>system\n\
|
||||
You suggest terminal commands. Give exactly 3 suggestions, one per line. \
|
||||
No numbering, no explanation, just the commands.\
|
||||
<|im_end|>\n\
|
||||
<|im_start|>user\n\
|
||||
Directory: {}\n\
|
||||
Recent commands:\n{}\n\
|
||||
Current partial input: \"{}\"\n\
|
||||
Suggest 3 likely next commands:\
|
||||
<|im_end|>\n\
|
||||
<|im_start|>assistant\n",
|
||||
input.working_directory, history, input.current_input
|
||||
);
|
||||
|
||||
let config = GenerationConfig {
|
||||
max_tokens: 64,
|
||||
temperature: 0.6,
|
||||
top_p: 0.9,
|
||||
};
|
||||
|
||||
let output = engine.generate(&prompt, &config).await?;
|
||||
|
||||
let suggestions: Vec<String> = output
|
||||
.lines()
|
||||
.map(|l| l.trim().to_string())
|
||||
.filter(|l| !l.is_empty())
|
||||
.take(3)
|
||||
.collect();
|
||||
|
||||
Ok(suggestions)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,60 @@
|
||||
use async_trait::async_trait;
|
||||
|
||||
use crate::InferenceTask;
|
||||
use crate::engine::{GenerationConfig, InferenceEngine};
|
||||
|
||||
pub struct TabNamingTask;
|
||||
|
||||
pub struct TabNamingInput {
|
||||
pub recent_commands: Vec<String>,
|
||||
pub working_directory: String,
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl InferenceTask for TabNamingTask {
|
||||
type Input = TabNamingInput;
|
||||
type Output = String;
|
||||
|
||||
async fn run(&self, engine: &InferenceEngine, input: Self::Input) -> anyhow::Result<String> {
|
||||
let commands_str = input
|
||||
.recent_commands
|
||||
.iter()
|
||||
.take(5)
|
||||
.map(|c| format!("- {c}"))
|
||||
.collect::<Vec<_>>()
|
||||
.join("\n");
|
||||
|
||||
let prompt = format!(
|
||||
"<|im_start|>system\n\
|
||||
You name terminal tabs. Respond with ONLY a short name (2-4 words max). No explanation.\
|
||||
<|im_end|>\n\
|
||||
<|im_start|>user\n\
|
||||
Directory: {}\n\
|
||||
Recent commands:\n{}\n\
|
||||
What should this tab be named?\
|
||||
<|im_end|>\n\
|
||||
<|im_start|>assistant\n",
|
||||
input.working_directory, commands_str
|
||||
);
|
||||
|
||||
let config = GenerationConfig {
|
||||
max_tokens: 12,
|
||||
temperature: 0.3,
|
||||
top_p: 0.9,
|
||||
};
|
||||
|
||||
let output = engine.generate(&prompt, &config).await?;
|
||||
|
||||
// Clean up: take only the first line, strip quotes
|
||||
let name = output
|
||||
.lines()
|
||||
.next()
|
||||
.unwrap_or(&output)
|
||||
.trim()
|
||||
.trim_matches('"')
|
||||
.trim_matches('\'')
|
||||
.to_string();
|
||||
|
||||
Ok(name)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,335 @@
|
||||
//! End-to-end scenario tests that validate realistic usage patterns for
|
||||
//! the local inference engine across all task types (classification,
|
||||
//! prompt suggestion, tab naming).
|
||||
|
||||
use local_inference::{
|
||||
Device, InferenceEngine, InferenceTask, InputCategory, InputClassificationInput,
|
||||
InputClassificationTask, PromptSuggestionInput, PromptSuggestionTask, TabNamingInput,
|
||||
TabNamingTask,
|
||||
};
|
||||
use std::time::Instant;
|
||||
|
||||
async fn get_engine() -> InferenceEngine {
|
||||
InferenceEngine::new(Device::Cpu)
|
||||
.await
|
||||
.expect("Failed to initialize engine — is the model downloaded?")
|
||||
}
|
||||
|
||||
// --- Full workflow scenarios ---
|
||||
|
||||
/// Simulates a user session where they type various inputs and the classifier
|
||||
/// routes them correctly between shell and agent modes.
|
||||
#[tokio::test]
|
||||
async fn scenario_user_session_mode_switching() {
|
||||
let engine = get_engine().await;
|
||||
|
||||
struct TestCase {
|
||||
input: &'static str,
|
||||
recent: Vec<&'static str>,
|
||||
follow_up: bool,
|
||||
expected: InputCategory,
|
||||
}
|
||||
|
||||
let cases = vec![
|
||||
TestCase {
|
||||
input: "cd ~/projects/myapp",
|
||||
recent: vec!["ls", "pwd"],
|
||||
follow_up: false,
|
||||
expected: InputCategory::Shell,
|
||||
},
|
||||
TestCase {
|
||||
input: "npm start",
|
||||
recent: vec!["cd ~/projects/myapp", "ls"],
|
||||
follow_up: false,
|
||||
expected: InputCategory::Shell,
|
||||
},
|
||||
TestCase {
|
||||
input: "why is my server crashing on startup?",
|
||||
recent: vec!["npm start", "cd ~/projects/myapp"],
|
||||
follow_up: false,
|
||||
expected: InputCategory::AgentPrompt,
|
||||
},
|
||||
TestCase {
|
||||
input: "can you also check the environment variables?",
|
||||
recent: vec!["npm start"],
|
||||
follow_up: true,
|
||||
expected: InputCategory::AgentPrompt,
|
||||
},
|
||||
TestCase {
|
||||
input: "export NODE_ENV=production",
|
||||
recent: vec!["npm start"],
|
||||
follow_up: false,
|
||||
expected: InputCategory::Shell,
|
||||
},
|
||||
];
|
||||
|
||||
println!("\n--- User Session Mode Switching Scenario ---");
|
||||
for case in &cases {
|
||||
let result = InputClassificationTask
|
||||
.run(
|
||||
&engine,
|
||||
InputClassificationInput {
|
||||
user_input: case.input.to_string(),
|
||||
recent_commands: case.recent.iter().map(|s| s.to_string()).collect(),
|
||||
is_follow_up: case.follow_up,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("classification failed");
|
||||
|
||||
println!(
|
||||
" {:?} -> {:?} (expected {:?}, confidence {:.2})",
|
||||
case.input, result.category, case.expected, result.confidence
|
||||
);
|
||||
assert_eq!(
|
||||
result.category, case.expected,
|
||||
"Misclassified {:?}",
|
||||
case.input
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// Simulates prompt suggestions being generated after various shell commands.
|
||||
#[tokio::test]
|
||||
async fn scenario_prompt_suggestions_after_commands() {
|
||||
let engine = get_engine().await;
|
||||
|
||||
println!("\n--- Prompt Suggestions After Commands Scenario ---");
|
||||
|
||||
// After a failed build
|
||||
let suggestions = PromptSuggestionTask
|
||||
.run(
|
||||
&engine,
|
||||
PromptSuggestionInput {
|
||||
recent_commands: vec![
|
||||
"cargo build".into(),
|
||||
"cargo test -- --nocapture".into(),
|
||||
"cargo build".into(), // repeated = likely still failing
|
||||
],
|
||||
current_input: String::new(),
|
||||
working_directory: "/home/user/projects/rust-app".into(),
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("suggestion failed");
|
||||
|
||||
println!(" After repeated builds: {suggestions:?}");
|
||||
assert!(!suggestions.is_empty());
|
||||
|
||||
// After git workflow
|
||||
let suggestions = PromptSuggestionTask
|
||||
.run(
|
||||
&engine,
|
||||
PromptSuggestionInput {
|
||||
recent_commands: vec![
|
||||
"git add .".into(),
|
||||
"git commit -m 'wip'".into(),
|
||||
"git push".into(),
|
||||
],
|
||||
current_input: String::new(),
|
||||
working_directory: "/home/user/projects/feature-branch".into(),
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("suggestion failed");
|
||||
|
||||
println!(" After git workflow: {suggestions:?}");
|
||||
assert!(!suggestions.is_empty());
|
||||
|
||||
// With partial input
|
||||
let suggestions = PromptSuggestionTask
|
||||
.run(
|
||||
&engine,
|
||||
PromptSuggestionInput {
|
||||
recent_commands: vec!["docker ps".into(), "docker logs app".into()],
|
||||
current_input: "docker".to_string(),
|
||||
working_directory: "/home/user/deployments".into(),
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("suggestion failed");
|
||||
|
||||
println!(" With 'docker' partial input: {suggestions:?}");
|
||||
assert!(!suggestions.is_empty());
|
||||
}
|
||||
|
||||
/// Simulates tab naming for different development contexts.
|
||||
#[tokio::test]
|
||||
async fn scenario_tab_naming_development_contexts() {
|
||||
let engine = get_engine().await;
|
||||
|
||||
println!("\n--- Tab Naming for Development Contexts ---");
|
||||
|
||||
struct TabCase {
|
||||
label: &'static str,
|
||||
commands: Vec<&'static str>,
|
||||
cwd: &'static str,
|
||||
}
|
||||
|
||||
let cases = vec![
|
||||
TabCase {
|
||||
label: "Git operations",
|
||||
commands: vec!["git log --oneline", "git branch -a", "git fetch origin"],
|
||||
cwd: "/home/user/projects/galaxy",
|
||||
},
|
||||
TabCase {
|
||||
label: "Docker deployment",
|
||||
commands: vec!["docker compose up -d", "docker ps", "docker logs web"],
|
||||
cwd: "/home/user/services/api",
|
||||
},
|
||||
TabCase {
|
||||
label: "Python data science",
|
||||
commands: vec!["jupyter notebook", "pip install pandas", "python analysis.py"],
|
||||
cwd: "/home/user/research/data-pipeline",
|
||||
},
|
||||
TabCase {
|
||||
label: "Rust development",
|
||||
commands: vec!["cargo build", "cargo test", "cargo clippy"],
|
||||
cwd: "/home/user/projects/my-crate",
|
||||
},
|
||||
];
|
||||
|
||||
for case in &cases {
|
||||
let name = TabNamingTask
|
||||
.run(
|
||||
&engine,
|
||||
TabNamingInput {
|
||||
recent_commands: case.commands.iter().map(|s| s.to_string()).collect(),
|
||||
working_directory: case.cwd.to_string(),
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("tab naming failed");
|
||||
|
||||
println!(" {}: \"{}\"", case.label, name);
|
||||
assert!(!name.is_empty(), "tab name should not be empty");
|
||||
assert!(
|
||||
name.split_whitespace().count() <= 6,
|
||||
"tab name too long for '{}': \"{}\"",
|
||||
case.label,
|
||||
name
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// Tests performance: all tasks should complete quickly enough for interactive use.
|
||||
#[tokio::test]
|
||||
async fn scenario_performance_under_load() {
|
||||
let engine = get_engine().await;
|
||||
|
||||
println!("\n--- Performance Under Load ---");
|
||||
|
||||
// Classification should be fast (< 500ms on CPU)
|
||||
let start = Instant::now();
|
||||
for _ in 0..5 {
|
||||
InputClassificationTask
|
||||
.run(
|
||||
&engine,
|
||||
InputClassificationInput {
|
||||
user_input: "git push origin main".to_string(),
|
||||
recent_commands: vec!["git add .".into(), "git commit -m 'test'".into()],
|
||||
is_follow_up: false,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("classification failed");
|
||||
}
|
||||
let classification_time = start.elapsed();
|
||||
println!(
|
||||
" 5 classifications: {:?} (avg {:?})",
|
||||
classification_time,
|
||||
classification_time / 5
|
||||
);
|
||||
|
||||
// Tab naming
|
||||
let start = Instant::now();
|
||||
TabNamingTask
|
||||
.run(
|
||||
&engine,
|
||||
TabNamingInput {
|
||||
recent_commands: vec!["make build".into(), "make test".into()],
|
||||
working_directory: "/home/user/project".into(),
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("tab naming failed");
|
||||
let tab_time = start.elapsed();
|
||||
println!(" 1 tab naming: {:?}", tab_time);
|
||||
|
||||
// Prompt suggestion
|
||||
let start = Instant::now();
|
||||
PromptSuggestionTask
|
||||
.run(
|
||||
&engine,
|
||||
PromptSuggestionInput {
|
||||
recent_commands: vec!["ls".into(), "cd src".into()],
|
||||
current_input: String::new(),
|
||||
working_directory: "/home/user".into(),
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("prompt suggestion failed");
|
||||
let suggest_time = start.elapsed();
|
||||
println!(" 1 prompt suggestion: {:?}", suggest_time);
|
||||
|
||||
// Classification should be under 2s per call on CPU (generous limit for CI)
|
||||
assert!(
|
||||
classification_time / 5 < std::time::Duration::from_secs(2),
|
||||
"classification too slow: {:?} per call",
|
||||
classification_time / 5
|
||||
);
|
||||
}
|
||||
|
||||
/// Tests that the engine can be reused across many calls without degradation.
|
||||
#[tokio::test]
|
||||
async fn scenario_engine_stability_across_calls() {
|
||||
let engine = get_engine().await;
|
||||
|
||||
println!("\n--- Engine Stability Across Calls ---");
|
||||
|
||||
// Mix different task types in sequence
|
||||
for i in 0..3 {
|
||||
// Classification
|
||||
let result = InputClassificationTask
|
||||
.run(
|
||||
&engine,
|
||||
InputClassificationInput {
|
||||
user_input: format!("test command {i}"),
|
||||
recent_commands: vec![],
|
||||
is_follow_up: false,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("classification should not degrade");
|
||||
assert!(result.confidence > 0.0);
|
||||
|
||||
// Tab naming
|
||||
let name = TabNamingTask
|
||||
.run(
|
||||
&engine,
|
||||
TabNamingInput {
|
||||
recent_commands: vec![format!("cmd {i}")],
|
||||
working_directory: format!("/tmp/test-{i}"),
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("tab naming should not degrade");
|
||||
assert!(!name.is_empty());
|
||||
|
||||
// Prompt suggestion
|
||||
let suggestions = PromptSuggestionTask
|
||||
.run(
|
||||
&engine,
|
||||
PromptSuggestionInput {
|
||||
recent_commands: vec![format!("action {i}")],
|
||||
current_input: String::new(),
|
||||
working_directory: "/tmp".into(),
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("prompt suggestion should not degrade");
|
||||
assert!(!suggestions.is_empty());
|
||||
|
||||
println!(" Round {}: all tasks passed", i + 1);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,406 @@
|
||||
use local_inference::{
|
||||
Device, GenerationConfig, InferenceEngine, InferenceTask, InputCategory,
|
||||
InputClassificationInput, InputClassificationTask, PromptSuggestionInput, PromptSuggestionTask,
|
||||
TabNamingInput, TabNamingTask,
|
||||
};
|
||||
|
||||
async fn get_engine() -> InferenceEngine {
|
||||
InferenceEngine::new(Device::Cpu)
|
||||
.await
|
||||
.expect("Failed to initialize engine — is the model downloaded?")
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_engine_initializes() {
|
||||
let engine = get_engine().await;
|
||||
assert!(engine.tokenizer().get_vocab_size(true) > 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_basic_generation() {
|
||||
let engine = get_engine().await;
|
||||
let config = GenerationConfig::deterministic();
|
||||
let output = engine
|
||||
.generate("The capital of France is", &config)
|
||||
.await
|
||||
.expect("generation failed");
|
||||
assert!(!output.is_empty(), "generated output should not be empty");
|
||||
println!("Generated: {output}");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_classify_shell_command() {
|
||||
let engine = get_engine().await;
|
||||
let result = InputClassificationTask
|
||||
.run(
|
||||
&engine,
|
||||
InputClassificationInput {
|
||||
user_input: "ls -la /tmp".to_string(),
|
||||
recent_commands: vec!["cd /tmp".into(), "mkdir test".into()],
|
||||
is_follow_up: false,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("classification failed");
|
||||
|
||||
println!(
|
||||
"\"ls -la /tmp\" -> {:?} (confidence: {:.2})",
|
||||
result.category, result.confidence
|
||||
);
|
||||
assert_eq!(result.category, InputCategory::Shell);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_classify_agent_prompt() {
|
||||
let engine = get_engine().await;
|
||||
let result = InputClassificationTask
|
||||
.run(
|
||||
&engine,
|
||||
InputClassificationInput {
|
||||
user_input: "explain how async works in rust".to_string(),
|
||||
recent_commands: vec!["cargo build".into()],
|
||||
is_follow_up: false,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("classification failed");
|
||||
|
||||
println!(
|
||||
"\"explain how async works in rust\" -> {:?} (confidence: {:.2})",
|
||||
result.category, result.confidence
|
||||
);
|
||||
assert_eq!(result.category, InputCategory::AgentPrompt);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_classify_ambiguous_input() {
|
||||
let engine = get_engine().await;
|
||||
|
||||
// "docker" alone could be either — we just want to make sure it doesn't panic
|
||||
let result = InputClassificationTask
|
||||
.run(
|
||||
&engine,
|
||||
InputClassificationInput {
|
||||
user_input: "docker".to_string(),
|
||||
recent_commands: vec![],
|
||||
is_follow_up: false,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("classification failed");
|
||||
|
||||
println!(
|
||||
"\"docker\" -> {:?} (confidence: {:.2})",
|
||||
result.category, result.confidence
|
||||
);
|
||||
// Either classification is fine, just assert it doesn't crash
|
||||
assert!(result.confidence > 0.0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_tab_naming() {
|
||||
let engine = get_engine().await;
|
||||
let name = TabNamingTask
|
||||
.run(
|
||||
&engine,
|
||||
TabNamingInput {
|
||||
recent_commands: vec![
|
||||
"git log --oneline".into(),
|
||||
"git diff".into(),
|
||||
"git add .".into(),
|
||||
],
|
||||
working_directory: "/home/user/projects/galaxy".into(),
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("tab naming failed");
|
||||
|
||||
println!("Tab name: \"{name}\"");
|
||||
assert!(!name.is_empty());
|
||||
// Tab names should be short
|
||||
assert!(
|
||||
name.split_whitespace().count() <= 6,
|
||||
"tab name too long: \"{name}\""
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_prompt_suggestions() {
|
||||
let engine = get_engine().await;
|
||||
let suggestions = PromptSuggestionTask
|
||||
.run(
|
||||
&engine,
|
||||
PromptSuggestionInput {
|
||||
recent_commands: vec!["cargo build".into(), "cargo test".into()],
|
||||
current_input: "cargo".to_string(),
|
||||
working_directory: "/home/user/projects/galaxy".into(),
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("prompt suggestion failed");
|
||||
|
||||
println!("Suggestions: {suggestions:?}");
|
||||
assert!(
|
||||
!suggestions.is_empty(),
|
||||
"should have at least one suggestion"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_generation_respects_max_tokens() {
|
||||
let engine = get_engine().await;
|
||||
let config = GenerationConfig {
|
||||
max_tokens: 5,
|
||||
temperature: 0.0,
|
||||
top_p: 1.0,
|
||||
};
|
||||
|
||||
let output = engine
|
||||
.generate("Once upon a time", &config)
|
||||
.await
|
||||
.expect("generation failed");
|
||||
|
||||
let token_count = engine
|
||||
.tokenizer()
|
||||
.encode(output.as_str(), false)
|
||||
.map(|e| e.get_ids().len())
|
||||
.unwrap_or(0);
|
||||
|
||||
println!("Generated ({token_count} tokens): \"{output}\"");
|
||||
// Should be roughly around max_tokens (could be less if EOS hit)
|
||||
assert!(token_count <= 6, "generated too many tokens: {token_count}");
|
||||
}
|
||||
|
||||
// --- Edge case tests ---
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_classify_empty_input() {
|
||||
let engine = get_engine().await;
|
||||
let result = InputClassificationTask
|
||||
.run(
|
||||
&engine,
|
||||
InputClassificationInput {
|
||||
user_input: String::new(),
|
||||
recent_commands: vec![],
|
||||
is_follow_up: false,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("classification should not panic on empty input");
|
||||
|
||||
println!(
|
||||
"empty input -> {:?} (confidence: {:.2})",
|
||||
result.category, result.confidence
|
||||
);
|
||||
assert!(result.confidence > 0.0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_classify_very_long_input() {
|
||||
let engine = get_engine().await;
|
||||
let long_input = "a".repeat(500);
|
||||
let result = InputClassificationTask
|
||||
.run(
|
||||
&engine,
|
||||
InputClassificationInput {
|
||||
user_input: long_input,
|
||||
recent_commands: vec!["ls".into()],
|
||||
is_follow_up: false,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("classification should handle long input");
|
||||
|
||||
println!(
|
||||
"long input -> {:?} (confidence: {:.2})",
|
||||
result.category, result.confidence
|
||||
);
|
||||
assert!(result.confidence > 0.0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_classify_special_characters() {
|
||||
let engine = get_engine().await;
|
||||
let result = InputClassificationTask
|
||||
.run(
|
||||
&engine,
|
||||
InputClassificationInput {
|
||||
user_input: "find . -name '*.rs' | xargs grep -l 'TODO'".to_string(),
|
||||
recent_commands: vec!["grep -rn test .".into()],
|
||||
is_follow_up: false,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("classification should handle special characters");
|
||||
|
||||
println!(
|
||||
"pipe command -> {:?} (confidence: {:.2})",
|
||||
result.category, result.confidence
|
||||
);
|
||||
assert_eq!(result.category, InputCategory::Shell);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_classify_follow_up_context() {
|
||||
let engine = get_engine().await;
|
||||
let result = InputClassificationTask
|
||||
.run(
|
||||
&engine,
|
||||
InputClassificationInput {
|
||||
user_input: "can you also add error handling?".to_string(),
|
||||
recent_commands: vec!["cargo build".into()],
|
||||
is_follow_up: true,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("classification should handle follow-up");
|
||||
|
||||
println!(
|
||||
"follow-up -> {:?} (confidence: {:.2})",
|
||||
result.category, result.confidence
|
||||
);
|
||||
assert_eq!(result.category, InputCategory::AgentPrompt);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_classify_multiple_sequential_calls() {
|
||||
let engine = get_engine().await;
|
||||
|
||||
let inputs = vec![
|
||||
("git status", InputCategory::Shell),
|
||||
("what does this error mean?", InputCategory::AgentPrompt),
|
||||
("npm install express", InputCategory::Shell),
|
||||
("refactor this to use async/await", InputCategory::AgentPrompt),
|
||||
];
|
||||
|
||||
for (input, expected_category) in inputs {
|
||||
let result = InputClassificationTask
|
||||
.run(
|
||||
&engine,
|
||||
InputClassificationInput {
|
||||
user_input: input.to_string(),
|
||||
recent_commands: vec!["ls".into()],
|
||||
is_follow_up: false,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("classification failed");
|
||||
|
||||
println!("\"{input}\" -> {:?} (confidence: {:.2})", result.category, result.confidence);
|
||||
assert_eq!(
|
||||
result.category, expected_category,
|
||||
"expected {expected_category:?} for \"{input}\", got {:?}",
|
||||
result.category
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_tab_naming_empty_commands() {
|
||||
let engine = get_engine().await;
|
||||
let name = TabNamingTask
|
||||
.run(
|
||||
&engine,
|
||||
TabNamingInput {
|
||||
recent_commands: vec![],
|
||||
working_directory: "/home/user".into(),
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("tab naming should handle empty commands");
|
||||
|
||||
println!("Tab name (no commands): \"{name}\"");
|
||||
assert!(!name.is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_tab_naming_deep_directory() {
|
||||
let engine = get_engine().await;
|
||||
let name = TabNamingTask
|
||||
.run(
|
||||
&engine,
|
||||
TabNamingInput {
|
||||
recent_commands: vec!["python train.py".into(), "tensorboard --logdir=runs".into()],
|
||||
working_directory: "/home/user/projects/ml-research/experiments/transformer-v2"
|
||||
.into(),
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("tab naming failed");
|
||||
|
||||
println!("Tab name (deep dir): \"{name}\"");
|
||||
assert!(!name.is_empty());
|
||||
assert!(
|
||||
name.split_whitespace().count() <= 6,
|
||||
"tab name too long: \"{name}\""
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_prompt_suggestions_empty_input() {
|
||||
let engine = get_engine().await;
|
||||
let suggestions = PromptSuggestionTask
|
||||
.run(
|
||||
&engine,
|
||||
PromptSuggestionInput {
|
||||
recent_commands: vec!["git status".into(), "git add .".into()],
|
||||
current_input: String::new(),
|
||||
working_directory: "/home/user/project".into(),
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("prompt suggestion should handle empty input");
|
||||
|
||||
println!("Suggestions (empty input): {suggestions:?}");
|
||||
assert!(
|
||||
!suggestions.is_empty(),
|
||||
"should suggest something even with empty input"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_prompt_suggestions_no_history() {
|
||||
let engine = get_engine().await;
|
||||
let suggestions = PromptSuggestionTask
|
||||
.run(
|
||||
&engine,
|
||||
PromptSuggestionInput {
|
||||
recent_commands: vec![],
|
||||
current_input: "docker".to_string(),
|
||||
working_directory: "/tmp".into(),
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("prompt suggestion should handle no history");
|
||||
|
||||
println!("Suggestions (no history): {suggestions:?}");
|
||||
assert!(
|
||||
!suggestions.is_empty(),
|
||||
"should suggest something even without history"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_deterministic_classification() {
|
||||
let engine = get_engine().await;
|
||||
let input = InputClassificationInput {
|
||||
user_input: "ls -la".to_string(),
|
||||
recent_commands: vec!["cd /tmp".into()],
|
||||
is_follow_up: false,
|
||||
};
|
||||
|
||||
let result1 = InputClassificationTask
|
||||
.run(&engine, input.clone())
|
||||
.await
|
||||
.expect("first classification failed");
|
||||
let result2 = InputClassificationTask
|
||||
.run(&engine, input)
|
||||
.await
|
||||
.expect("second classification failed");
|
||||
|
||||
// With temperature 0.0, results should be deterministic
|
||||
assert_eq!(
|
||||
result1.category, result2.category,
|
||||
"deterministic classification should yield consistent results"
|
||||
);
|
||||
}
|
||||
Reference in New Issue
Block a user