Rig setup

This commit is contained in:
Ryan Ward
2026-08-18 01:40:11 -05:00
parent 56e3b51d48
commit a203d359bf
16 changed files with 2495 additions and 89 deletions
@@ -302,6 +302,7 @@ pub enum ModelFailureDisposition {
#[derive(Clone, Debug, Default, PartialEq, Eq)]
pub struct ProviderRunRestoreNormalization {
pub interrupted_model_call: bool,
pub permission_call_ids_reset: Vec<String>,
pub interrupted_call_ids: Vec<String>,
pub recovery_call_ids: Vec<String>,
@@ -350,6 +351,9 @@ pub enum ProviderRunProtocolError {
InvalidDriverObservation {
message: String,
},
InvalidRestoredState {
message: String,
},
EpochExhausted,
Terminal,
}
@@ -409,6 +413,9 @@ impl fmt::Display for ProviderRunProtocolError {
Self::InvalidDriverObservation { message } => {
write!(f, "invalid driver observation: {message}")
}
Self::InvalidRestoredState { message } => {
write!(f, "invalid restored provider run: {message}")
}
Self::EpochExhausted => f.write_str("provider run epoch is exhausted"),
Self::Terminal => f.write_str("provider run is already terminal"),
}
@@ -502,6 +509,226 @@ impl ProviderRun {
)
}
/// Validates persisted state before restore normalization can mutate it or external work can
/// be reconstructed from it.
pub fn validate_restored_state(&self) -> Result<(), ProviderRunProtocolError> {
let invalid = |message: String| ProviderRunProtocolError::InvalidRestoredState { message };
if self.id.as_str().is_empty() {
return Err(invalid("run ID must not be empty".to_string()));
}
if self.profile.as_str().is_empty() {
return Err(invalid("request profile must not be empty".to_string()));
}
if self.limits.max_model_turns == 0 {
return Err(invalid("model-turn limit must be at least one".to_string()));
}
if self.model_turns > self.limits.max_model_turns {
return Err(invalid(format!(
"model-turn counter {} exceeds limit {}",
self.model_turns, self.limits.max_model_turns
)));
}
if self.epoch.get() < u64::from(self.model_turns) {
return Err(invalid(format!(
"epoch {} is behind model-turn counter {}",
self.epoch.get(),
self.model_turns
)));
}
let retry_slots = u64::from(self.model_turns)
.saturating_add(1)
.saturating_mul(u64::from(self.limits.max_model_retries_per_turn));
if u64::from(self.model_retries) > retry_slots {
return Err(invalid(format!(
"model-retry counter {} exceeds maximum possible {}",
self.model_retries, retry_slots
)));
}
if let Some(work_id) = self.active_work_id() {
self.validate_restored_work_id(work_id)?;
}
match &self.state {
ProviderRunState::ReadyToCallModel => {}
ProviderRunState::AwaitingModel { call } => {
if self.model_turns >= self.limits.max_model_turns {
return Err(invalid(
"awaiting a model call after reaching the model-turn limit".to_string(),
));
}
if call.retry_attempt > self.limits.max_model_retries_per_turn {
return Err(invalid(format!(
"pending retry attempt {} exceeds per-turn limit {}",
call.retry_attempt, self.limits.max_model_retries_per_turn
)));
}
if call.retry_attempt > self.model_retries {
return Err(invalid(format!(
"pending retry attempt {} exceeds total retry counter {}",
call.retry_attempt, self.model_retries
)));
}
if (call.retry_attempt == 0) != call.last_error.is_none() {
return Err(invalid(
"pending retry error does not match its retry attempt".to_string(),
));
}
if call
.last_error
.as_ref()
.is_some_and(|error| !error.recoverable)
{
return Err(invalid(
"pending retry retains a non-recoverable model error".to_string(),
));
}
}
ProviderRunState::ResolvingModel { turn } => {
self.validate_post_model_phase()?;
validate_model_turn(turn).map_err(|error| invalid(error.to_string()))?;
if self.transcript.last() != Some(&assistant_message(turn)) {
return Err(invalid(
"resolving model turn does not own the latest transcript message"
.to_string(),
));
}
if !usage_contains(&self.usage, &turn.usage) {
return Err(invalid(
"aggregate usage does not include the resolving model turn".to_string(),
));
}
}
ProviderRunState::AwaitingTools { batch } => {
self.validate_post_model_phase()?;
if batch.calls.is_empty() {
return Err(invalid("pending tool batch is empty".to_string()));
}
let mut call_ids = HashSet::new();
for pending in &batch.calls {
if pending.call.id.is_empty() {
return Err(invalid("pending tool call ID is empty".to_string()));
}
if !call_ids.insert(pending.call.id.as_str()) {
return Err(invalid(format!(
"duplicate pending tool call ID '{}'",
pending.call.id
)));
}
validate_pending_tool_state(pending).map_err(invalid)?;
}
let transcript_calls = self
.transcript
.last()
.map(tool_calls_from_message)
.unwrap_or_default();
if transcript_calls
!= batch
.calls
.iter()
.map(|pending| pending.call.clone())
.collect::<Vec<_>>()
{
return Err(invalid(
"pending tool batch does not match the latest assistant message"
.to_string(),
));
}
}
ProviderRunState::AwaitingDriver { .. } => {
self.validate_post_model_phase()?;
let Some(message) = self.transcript.last() else {
return Err(invalid(
"driver wait is missing its assistant transcript message".to_string(),
));
};
if message.role != MessageRole::Assistant {
return Err(invalid(
"driver wait does not follow an assistant transcript message".to_string(),
));
}
if !tool_calls_from_message(message).is_empty() {
return Err(invalid(
"driver wait follows an uncommitted assistant tool call".to_string(),
));
}
}
ProviderRunState::Done { .. } => {
if self.model_turns == 0 {
return Err(invalid(
"completed run has no completed model turn".to_string(),
));
}
}
ProviderRunState::Failed { failure } => match failure.kind {
ProviderRunFailureKind::ModelCall
if !failure
.source
.as_ref()
.is_some_and(|source| !source.recoverable) =>
{
return Err(invalid(
"model-call failure lacks a non-recoverable source".to_string(),
));
}
ProviderRunFailureKind::RetryLimitExceeded
if !failure
.source
.as_ref()
.is_some_and(|source| source.recoverable) =>
{
return Err(invalid(
"retry-limit failure lacks a recoverable source".to_string(),
));
}
ProviderRunFailureKind::TurnLimitExceeded
if self.model_turns < self.limits.max_model_turns =>
{
return Err(invalid(
"turn-limit failure occurred before reaching the limit".to_string(),
));
}
ProviderRunFailureKind::ModelCall
| ProviderRunFailureKind::RetryLimitExceeded
| ProviderRunFailureKind::TurnLimitExceeded
| ProviderRunFailureKind::Protocol
| ProviderRunFailureKind::Projection
| ProviderRunFailureKind::Restore
| ProviderRunFailureKind::ExternalWork => {}
},
ProviderRunState::Cancelled { .. } => {}
}
Ok(())
}
fn validate_restored_work_id(
&self,
work_id: &ExternalWorkId,
) -> Result<(), ProviderRunProtocolError> {
validate_work_id(&self.current_work_id(), work_id).map_err(|_| {
ProviderRunProtocolError::InvalidRestoredState {
message: format!(
"active work identity {}:{} does not match run {}:{}",
work_id.run_id.as_str(),
work_id.epoch.get(),
self.id.as_str(),
self.epoch.get()
),
}
})
}
fn validate_post_model_phase(&self) -> Result<(), ProviderRunProtocolError> {
if self.model_turns == 0 {
Err(ProviderRunProtocolError::InvalidRestoredState {
message: format!("{:?} phase has no completed model turn", self.state.phase()),
})
} else {
Ok(())
}
}
pub fn normalize_after_restore(
&mut self,
) -> Result<ProviderRunRestoreNormalization, ProviderRunProtocolError> {
@@ -512,6 +739,21 @@ impl ProviderRun {
&mut self,
recoverable_call_ids: &HashSet<String>,
) -> Result<ProviderRunRestoreNormalization, ProviderRunProtocolError> {
if matches!(self.state, ProviderRunState::AwaitingModel { .. }) {
self.state = ProviderRunState::Failed {
failure: ProviderRunFailure {
kind: ProviderRunFailureKind::Restore,
message: "The model call was interrupted by application restart after dispatch may have begun. Its outcome is unknown, so it was not replayed to avoid duplicate billing or output."
.to_string(),
source: None,
},
};
return Ok(ProviderRunRestoreNormalization {
interrupted_model_call: true,
..ProviderRunRestoreNormalization::default()
});
}
let ProviderRunState::AwaitingTools { batch } = &mut self.state else {
return Ok(ProviderRunRestoreNormalization::default());
};
@@ -1220,6 +1462,84 @@ fn validate_work_id(
}
}
fn validate_pending_tool_state(pending: &PendingToolCall) -> Result<(), String> {
match &pending.state {
PendingToolCallState::PermissionPending { request } => {
if request.id.is_empty() {
return Err(format!(
"permission request for '{}' has an empty request ID",
pending.call.id
));
}
if request.call_id != pending.call.id {
return Err(format!(
"permission request for '{}' belongs to call '{}'",
pending.call.id, request.call_id
));
}
}
PendingToolCallState::Approved {
request_id,
decision,
} => {
if request_id.is_empty() {
return Err(format!(
"approved tool call '{}' has an empty request ID",
pending.call.id
));
}
if matches!(decision, PermissionDecision::Denied { .. }) {
return Err(format!(
"approved tool call '{}' contains a denied decision",
pending.call.id
));
}
}
PendingToolCallState::Resolved { result } if result.call_id != pending.call.id => {
return Err(format!(
"resolved result for '{}' belongs to call '{}'",
pending.call.id, result.call_id
));
}
PendingToolCallState::Proposed
| PendingToolCallState::Executing
| PendingToolCallState::RecoveryPending
| PendingToolCallState::Resolved { .. } => {}
}
Ok(())
}
fn tool_calls_from_message(message: &ConversationMessage) -> Vec<ToolCall> {
let MessageContent::MultiPart(parts) = &message.content else {
return Vec::new();
};
parts
.iter()
.filter_map(|part| match part {
ContentPart::ToolUse {
tool_use_id,
name,
input,
} => Some(ToolCall {
id: tool_use_id.clone(),
name: name.clone(),
arguments: input.clone(),
}),
ContentPart::Text(_)
| ContentPart::Reasoning { .. }
| ContentPart::ToolResult { .. }
| ContentPart::Image { .. } => None,
})
.collect()
}
fn usage_contains(total: &Usage, part: &Usage) -> bool {
total.input_tokens >= part.input_tokens
&& total.output_tokens >= part.output_tokens
&& total.cached_input_tokens >= part.cached_input_tokens
&& total.cache_creation_input_tokens >= part.cache_creation_input_tokens
}
fn validate_model_turn(turn: &CompletedModelTurn) -> Result<(), ProviderRunProtocolError> {
for part in &turn.assistant_content {
match part {
@@ -88,6 +88,36 @@ fn assert_serialization_round_trip(run: &ProviderRun) {
assert_eq!(&restored, run);
}
fn mutate_run_json(run: &ProviderRun, mutate: impl FnOnce(&mut serde_json::Value)) -> ProviderRun {
let mut value = serde_json::to_value(run).unwrap();
mutate(&mut value);
serde_json::from_value(value).unwrap()
}
fn restored_state_error(run: &ProviderRun) -> String {
let ProviderRunProtocolError::InvalidRestoredState { message } =
run.validate_restored_state().unwrap_err()
else {
panic!("expected restored-state validation error");
};
message
}
fn awaiting_tool_run() -> ProviderRun {
let mut run = run();
accept_tool_turn(
&mut run,
tool_turn(
vec![
tool_call("first", "read_files"),
tool_call("second", "grep"),
],
&["read_files", "grep"],
),
);
run
}
#[test]
fn next_step_reemits_identical_pending_model_work() {
let mut run = run();
@@ -740,7 +770,228 @@ fn every_nonterminal_phase_round_trips_through_json() {
}
#[test]
fn restore_normalization_preserves_safe_nonterminal_states() {
fn restored_state_validation_accepts_valid_snapshots_in_every_phase() {
let ready = run();
let mut awaiting_model = ready.clone();
let call = next_model_call(&mut awaiting_model);
let mut resolving = awaiting_model.clone();
resolving
.accept_model_turn(&call.work_id, text_turn("done"))
.unwrap();
let mut awaiting_driver = resolving.clone();
assert_eq!(awaiting_driver.next_step().unwrap(), None);
let awaiting_tools = awaiting_tool_run();
let mut done = awaiting_driver.clone();
let work_id = awaiting_driver.active_work_id().unwrap().clone();
done.complete(&work_id).unwrap();
let mut failed = ready.clone();
failed
.fail(ProviderRunFailureKind::ExternalWork, "failed")
.unwrap();
let mut cancelled = ready;
cancelled.cancel("cancelled").unwrap();
for candidate in [
awaiting_model,
resolving,
awaiting_tools,
awaiting_driver,
done,
failed,
cancelled,
] {
candidate.validate_restored_state().unwrap();
}
}
#[test]
fn restored_state_validation_rejects_active_work_run_and_epoch_mismatches() {
let mut awaiting_model = run();
next_model_call(&mut awaiting_model);
let wrong_run = mutate_run_json(&awaiting_model, |value| {
value["state"]["AwaitingModel"]["call"]["work_id"]["run_id"] = json!("other-run");
});
assert!(restored_state_error(&wrong_run).contains("active work identity other-run:0"));
let wrong_epoch = mutate_run_json(&awaiting_model, |value| {
value["state"]["AwaitingModel"]["call"]["work_id"]["epoch"] = json!(9);
});
assert!(restored_state_error(&wrong_epoch).contains("active work identity run-1:9"));
}
#[test]
fn restored_state_validation_rejects_duplicate_tool_call_ids() {
let corrupted = mutate_run_json(&awaiting_tool_run(), |value| {
value["state"]["AwaitingTools"]["batch"]["calls"][1]["call"]["id"] = json!("first");
});
assert_eq!(
restored_state_error(&corrupted),
"duplicate pending tool call ID 'first'"
);
}
#[test]
fn restored_state_validation_rejects_result_and_permission_call_ownership() {
let mut resolved = awaiting_tool_run();
let work_id = resolved.active_work_id().unwrap().clone();
resolved
.complete_tool(&work_id, successful_result("first", "done"))
.unwrap();
let wrong_result = mutate_run_json(&resolved, |value| {
value["state"]["AwaitingTools"]["batch"]["calls"][0]["state"]["Resolved"]["result"]["call_id"] =
json!("second");
});
assert_eq!(
restored_state_error(&wrong_result),
"resolved result for 'first' belongs to call 'second'"
);
let mut permission = awaiting_tool_run();
let work_id = permission.active_work_id().unwrap().clone();
permission
.request_tool_permission(
&work_id,
PermissionRequest {
id: "request-1".to_string(),
call_id: "first".to_string(),
kind: PermissionKind::Read,
reason: None,
},
)
.unwrap();
let wrong_permission = mutate_run_json(&permission, |value| {
value["state"]["AwaitingTools"]["batch"]["calls"][0]["state"]["PermissionPending"]["request"]
["call_id"] = json!("second");
});
assert_eq!(
restored_state_error(&wrong_permission),
"permission request for 'first' belongs to call 'second'"
);
}
#[test]
fn restored_state_validation_rejects_phase_specific_corruption() {
let empty_batch = mutate_run_json(&awaiting_tool_run(), |value| {
value["state"]["AwaitingTools"]["batch"]["calls"] = json!([]);
});
assert_eq!(
restored_state_error(&empty_batch),
"pending tool batch is empty"
);
let mismatched_batch = mutate_run_json(&awaiting_tool_run(), |value| {
value["state"]["AwaitingTools"]["batch"]["calls"][0]["call"]["name"] =
json!("different_tool");
});
assert!(restored_state_error(&mismatched_batch).contains("latest assistant message"));
let resolving_without_turn = mutate_run_json(&awaiting_tool_run(), |value| {
value["model_turns"] = json!(0);
});
assert!(restored_state_error(&resolving_without_turn).contains("has no completed model turn"));
let mut resolving = run();
let call = next_model_call(&mut resolving);
resolving
.accept_model_turn(&call.work_id, text_turn("done"))
.unwrap();
let invalid_turn = mutate_run_json(&resolving, |value| {
value["state"]["ResolvingModel"]["turn"]["assistant_content"] = json!([{
"ToolUse": {
"tool_use_id": "injected",
"name": "read_files",
"input": {}
}
}]);
});
assert!(restored_state_error(&invalid_turn).contains("assistant_content"));
let mut awaiting_driver = resolving;
assert_eq!(awaiting_driver.next_step().unwrap(), None);
let wrong_driver_owner = mutate_run_json(&awaiting_driver, |value| {
let last = value["transcript"]
.as_array_mut()
.unwrap()
.last_mut()
.unwrap();
last["role"] = json!("User");
});
assert!(restored_state_error(&wrong_driver_owner).contains("does not follow an assistant"));
}
#[test]
fn restored_state_validation_rejects_retry_counter_and_terminal_corruption() {
let zero_limit = mutate_run_json(&run(), |value| {
value["limits"]["max_model_turns"] = json!(0);
});
assert_eq!(
restored_state_error(&zero_limit),
"model-turn limit must be at least one"
);
let excessive_retries = mutate_run_json(&run(), |value| {
value["model_retries"] = json!(3);
});
assert!(restored_state_error(&excessive_retries).contains("model-retry counter"));
let mut awaiting_model = run();
next_model_call(&mut awaiting_model);
let inconsistent_retry = mutate_run_json(&awaiting_model, |value| {
value["state"]["AwaitingModel"]["call"]["retry_attempt"] = json!(1);
});
assert!(restored_state_error(&inconsistent_retry).contains("total retry counter"));
let completed_without_turn = mutate_run_json(&run(), |value| {
value["state"] = json!({"Done": {"completion": {"stop_reason": "Completed"}}});
});
assert_eq!(
restored_state_error(&completed_without_turn),
"completed run has no completed model turn"
);
let early_turn_limit = mutate_run_json(&run(), |value| {
value["state"] = json!({
"Failed": {"failure": {
"kind": "TurnLimitExceeded",
"message": "bad",
"source": null
}}
});
});
assert_eq!(
restored_state_error(&early_turn_limit),
"turn-limit failure occurred before reaching the limit"
);
let missing_retry_source = mutate_run_json(&run(), |value| {
value["state"] = json!({
"Failed": {"failure": {
"kind": "RetryLimitExceeded",
"message": "bad",
"source": null
}}
});
});
assert_eq!(
restored_state_error(&missing_retry_source),
"retry-limit failure lacks a recoverable source"
);
}
#[test]
fn restored_state_validation_rejects_empty_run_identity_and_profile() {
let empty_run = mutate_run_json(&run(), |value| value["id"] = json!(""));
assert_eq!(restored_state_error(&empty_run), "run ID must not be empty");
let empty_profile = mutate_run_json(&run(), |value| value["profile"] = json!(""));
assert_eq!(
restored_state_error(&empty_profile),
"request profile must not be empty"
);
}
#[test]
fn restore_normalization_preserves_model_work_outside_the_uncertain_dispatch_boundary() {
let ready = run();
let mut awaiting_model = ready.clone();
let call = next_model_call(&mut awaiting_model);
@@ -752,7 +1003,7 @@ fn restore_normalization_preserves_safe_nonterminal_states() {
let mut awaiting_driver = resolving.clone();
assert_eq!(awaiting_driver.next_step().unwrap(), None);
for mut candidate in [ready, awaiting_model, resolving, awaiting_driver] {
for mut candidate in [ready, resolving, awaiting_driver] {
let before = candidate.clone();
assert_eq!(
candidate.normalize_after_restore().unwrap(),
@@ -762,6 +1013,58 @@ fn restore_normalization_preserves_safe_nonterminal_states() {
}
}
#[test]
fn crash_after_model_acceptance_before_checkpoint_does_not_replay_the_persisted_call() {
let mut live_run = run();
let call = next_model_call(&mut live_run);
let serialized_at_dispatch_boundary = serde_json::to_string(&live_run).unwrap();
// Simulate remote acceptance followed by a crash before the accepted turn is checkpointed.
live_run
.accept_model_turn(&call.work_id, text_turn("accepted but not checkpointed"))
.unwrap();
assert_eq!(live_run.model_turns(), 1);
let mut restored: ProviderRun = serde_json::from_str(&serialized_at_dispatch_boundary).unwrap();
let normalization = restored.normalize_after_restore().unwrap();
assert!(normalization.interrupted_model_call);
let ProviderRunState::Failed { failure } = restored.state() else {
panic!("uncertain model work must become terminal on restore");
};
assert_eq!(failure.kind, ProviderRunFailureKind::Restore);
assert!(failure.message.contains("outcome is unknown"));
assert!(failure.message.contains("not replayed"));
assert_eq!(restored.active_work_id(), None);
assert!(matches!(
restored.next_step().unwrap(),
Some(ProviderRunStep::Done(ProviderRunOutcome::Failed(_)))
));
// The persisted dispatch identity remains useful for diagnostics but can never be called again.
assert_eq!(call.work_id.epoch, RunEpoch::new(0));
}
#[test]
fn restore_after_model_acceptance_keeps_the_committed_turn_without_replaying() {
let mut run = run();
let call = next_model_call(&mut run);
run.accept_model_turn(&call.work_id, text_turn("accepted output"))
.unwrap();
let serialized_after_acceptance = serde_json::to_string(&run).unwrap();
let mut restored: ProviderRun = serde_json::from_str(&serialized_after_acceptance).unwrap();
assert_eq!(
restored.normalize_after_restore().unwrap(),
ProviderRunRestoreNormalization::default()
);
assert_eq!(restored.state().phase(), ProviderRunPhase::ResolvingModel);
assert_eq!(restored.model_turns(), 1);
assert_eq!(restored.next_step().unwrap(), None);
assert_eq!(restored.state().phase(), ProviderRunPhase::AwaitingDriver);
}
#[test]
fn restore_normalization_reproposes_permissions_and_interrupts_unsafe_tools() {
let mut run = run();