Rig setup
This commit is contained in:
@@ -302,6 +302,7 @@ pub enum ModelFailureDisposition {
|
||||
|
||||
#[derive(Clone, Debug, Default, PartialEq, Eq)]
|
||||
pub struct ProviderRunRestoreNormalization {
|
||||
pub interrupted_model_call: bool,
|
||||
pub permission_call_ids_reset: Vec<String>,
|
||||
pub interrupted_call_ids: Vec<String>,
|
||||
pub recovery_call_ids: Vec<String>,
|
||||
@@ -350,6 +351,9 @@ pub enum ProviderRunProtocolError {
|
||||
InvalidDriverObservation {
|
||||
message: String,
|
||||
},
|
||||
InvalidRestoredState {
|
||||
message: String,
|
||||
},
|
||||
EpochExhausted,
|
||||
Terminal,
|
||||
}
|
||||
@@ -409,6 +413,9 @@ impl fmt::Display for ProviderRunProtocolError {
|
||||
Self::InvalidDriverObservation { message } => {
|
||||
write!(f, "invalid driver observation: {message}")
|
||||
}
|
||||
Self::InvalidRestoredState { message } => {
|
||||
write!(f, "invalid restored provider run: {message}")
|
||||
}
|
||||
Self::EpochExhausted => f.write_str("provider run epoch is exhausted"),
|
||||
Self::Terminal => f.write_str("provider run is already terminal"),
|
||||
}
|
||||
@@ -502,6 +509,226 @@ impl ProviderRun {
|
||||
)
|
||||
}
|
||||
|
||||
/// Validates persisted state before restore normalization can mutate it or external work can
|
||||
/// be reconstructed from it.
|
||||
pub fn validate_restored_state(&self) -> Result<(), ProviderRunProtocolError> {
|
||||
let invalid = |message: String| ProviderRunProtocolError::InvalidRestoredState { message };
|
||||
|
||||
if self.id.as_str().is_empty() {
|
||||
return Err(invalid("run ID must not be empty".to_string()));
|
||||
}
|
||||
if self.profile.as_str().is_empty() {
|
||||
return Err(invalid("request profile must not be empty".to_string()));
|
||||
}
|
||||
if self.limits.max_model_turns == 0 {
|
||||
return Err(invalid("model-turn limit must be at least one".to_string()));
|
||||
}
|
||||
if self.model_turns > self.limits.max_model_turns {
|
||||
return Err(invalid(format!(
|
||||
"model-turn counter {} exceeds limit {}",
|
||||
self.model_turns, self.limits.max_model_turns
|
||||
)));
|
||||
}
|
||||
if self.epoch.get() < u64::from(self.model_turns) {
|
||||
return Err(invalid(format!(
|
||||
"epoch {} is behind model-turn counter {}",
|
||||
self.epoch.get(),
|
||||
self.model_turns
|
||||
)));
|
||||
}
|
||||
let retry_slots = u64::from(self.model_turns)
|
||||
.saturating_add(1)
|
||||
.saturating_mul(u64::from(self.limits.max_model_retries_per_turn));
|
||||
if u64::from(self.model_retries) > retry_slots {
|
||||
return Err(invalid(format!(
|
||||
"model-retry counter {} exceeds maximum possible {}",
|
||||
self.model_retries, retry_slots
|
||||
)));
|
||||
}
|
||||
|
||||
if let Some(work_id) = self.active_work_id() {
|
||||
self.validate_restored_work_id(work_id)?;
|
||||
}
|
||||
|
||||
match &self.state {
|
||||
ProviderRunState::ReadyToCallModel => {}
|
||||
ProviderRunState::AwaitingModel { call } => {
|
||||
if self.model_turns >= self.limits.max_model_turns {
|
||||
return Err(invalid(
|
||||
"awaiting a model call after reaching the model-turn limit".to_string(),
|
||||
));
|
||||
}
|
||||
if call.retry_attempt > self.limits.max_model_retries_per_turn {
|
||||
return Err(invalid(format!(
|
||||
"pending retry attempt {} exceeds per-turn limit {}",
|
||||
call.retry_attempt, self.limits.max_model_retries_per_turn
|
||||
)));
|
||||
}
|
||||
if call.retry_attempt > self.model_retries {
|
||||
return Err(invalid(format!(
|
||||
"pending retry attempt {} exceeds total retry counter {}",
|
||||
call.retry_attempt, self.model_retries
|
||||
)));
|
||||
}
|
||||
if (call.retry_attempt == 0) != call.last_error.is_none() {
|
||||
return Err(invalid(
|
||||
"pending retry error does not match its retry attempt".to_string(),
|
||||
));
|
||||
}
|
||||
if call
|
||||
.last_error
|
||||
.as_ref()
|
||||
.is_some_and(|error| !error.recoverable)
|
||||
{
|
||||
return Err(invalid(
|
||||
"pending retry retains a non-recoverable model error".to_string(),
|
||||
));
|
||||
}
|
||||
}
|
||||
ProviderRunState::ResolvingModel { turn } => {
|
||||
self.validate_post_model_phase()?;
|
||||
validate_model_turn(turn).map_err(|error| invalid(error.to_string()))?;
|
||||
if self.transcript.last() != Some(&assistant_message(turn)) {
|
||||
return Err(invalid(
|
||||
"resolving model turn does not own the latest transcript message"
|
||||
.to_string(),
|
||||
));
|
||||
}
|
||||
if !usage_contains(&self.usage, &turn.usage) {
|
||||
return Err(invalid(
|
||||
"aggregate usage does not include the resolving model turn".to_string(),
|
||||
));
|
||||
}
|
||||
}
|
||||
ProviderRunState::AwaitingTools { batch } => {
|
||||
self.validate_post_model_phase()?;
|
||||
if batch.calls.is_empty() {
|
||||
return Err(invalid("pending tool batch is empty".to_string()));
|
||||
}
|
||||
let mut call_ids = HashSet::new();
|
||||
for pending in &batch.calls {
|
||||
if pending.call.id.is_empty() {
|
||||
return Err(invalid("pending tool call ID is empty".to_string()));
|
||||
}
|
||||
if !call_ids.insert(pending.call.id.as_str()) {
|
||||
return Err(invalid(format!(
|
||||
"duplicate pending tool call ID '{}'",
|
||||
pending.call.id
|
||||
)));
|
||||
}
|
||||
validate_pending_tool_state(pending).map_err(invalid)?;
|
||||
}
|
||||
let transcript_calls = self
|
||||
.transcript
|
||||
.last()
|
||||
.map(tool_calls_from_message)
|
||||
.unwrap_or_default();
|
||||
if transcript_calls
|
||||
!= batch
|
||||
.calls
|
||||
.iter()
|
||||
.map(|pending| pending.call.clone())
|
||||
.collect::<Vec<_>>()
|
||||
{
|
||||
return Err(invalid(
|
||||
"pending tool batch does not match the latest assistant message"
|
||||
.to_string(),
|
||||
));
|
||||
}
|
||||
}
|
||||
ProviderRunState::AwaitingDriver { .. } => {
|
||||
self.validate_post_model_phase()?;
|
||||
let Some(message) = self.transcript.last() else {
|
||||
return Err(invalid(
|
||||
"driver wait is missing its assistant transcript message".to_string(),
|
||||
));
|
||||
};
|
||||
if message.role != MessageRole::Assistant {
|
||||
return Err(invalid(
|
||||
"driver wait does not follow an assistant transcript message".to_string(),
|
||||
));
|
||||
}
|
||||
if !tool_calls_from_message(message).is_empty() {
|
||||
return Err(invalid(
|
||||
"driver wait follows an uncommitted assistant tool call".to_string(),
|
||||
));
|
||||
}
|
||||
}
|
||||
ProviderRunState::Done { .. } => {
|
||||
if self.model_turns == 0 {
|
||||
return Err(invalid(
|
||||
"completed run has no completed model turn".to_string(),
|
||||
));
|
||||
}
|
||||
}
|
||||
ProviderRunState::Failed { failure } => match failure.kind {
|
||||
ProviderRunFailureKind::ModelCall
|
||||
if !failure
|
||||
.source
|
||||
.as_ref()
|
||||
.is_some_and(|source| !source.recoverable) =>
|
||||
{
|
||||
return Err(invalid(
|
||||
"model-call failure lacks a non-recoverable source".to_string(),
|
||||
));
|
||||
}
|
||||
ProviderRunFailureKind::RetryLimitExceeded
|
||||
if !failure
|
||||
.source
|
||||
.as_ref()
|
||||
.is_some_and(|source| source.recoverable) =>
|
||||
{
|
||||
return Err(invalid(
|
||||
"retry-limit failure lacks a recoverable source".to_string(),
|
||||
));
|
||||
}
|
||||
ProviderRunFailureKind::TurnLimitExceeded
|
||||
if self.model_turns < self.limits.max_model_turns =>
|
||||
{
|
||||
return Err(invalid(
|
||||
"turn-limit failure occurred before reaching the limit".to_string(),
|
||||
));
|
||||
}
|
||||
ProviderRunFailureKind::ModelCall
|
||||
| ProviderRunFailureKind::RetryLimitExceeded
|
||||
| ProviderRunFailureKind::TurnLimitExceeded
|
||||
| ProviderRunFailureKind::Protocol
|
||||
| ProviderRunFailureKind::Projection
|
||||
| ProviderRunFailureKind::Restore
|
||||
| ProviderRunFailureKind::ExternalWork => {}
|
||||
},
|
||||
ProviderRunState::Cancelled { .. } => {}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn validate_restored_work_id(
|
||||
&self,
|
||||
work_id: &ExternalWorkId,
|
||||
) -> Result<(), ProviderRunProtocolError> {
|
||||
validate_work_id(&self.current_work_id(), work_id).map_err(|_| {
|
||||
ProviderRunProtocolError::InvalidRestoredState {
|
||||
message: format!(
|
||||
"active work identity {}:{} does not match run {}:{}",
|
||||
work_id.run_id.as_str(),
|
||||
work_id.epoch.get(),
|
||||
self.id.as_str(),
|
||||
self.epoch.get()
|
||||
),
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
fn validate_post_model_phase(&self) -> Result<(), ProviderRunProtocolError> {
|
||||
if self.model_turns == 0 {
|
||||
Err(ProviderRunProtocolError::InvalidRestoredState {
|
||||
message: format!("{:?} phase has no completed model turn", self.state.phase()),
|
||||
})
|
||||
} else {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
pub fn normalize_after_restore(
|
||||
&mut self,
|
||||
) -> Result<ProviderRunRestoreNormalization, ProviderRunProtocolError> {
|
||||
@@ -512,6 +739,21 @@ impl ProviderRun {
|
||||
&mut self,
|
||||
recoverable_call_ids: &HashSet<String>,
|
||||
) -> Result<ProviderRunRestoreNormalization, ProviderRunProtocolError> {
|
||||
if matches!(self.state, ProviderRunState::AwaitingModel { .. }) {
|
||||
self.state = ProviderRunState::Failed {
|
||||
failure: ProviderRunFailure {
|
||||
kind: ProviderRunFailureKind::Restore,
|
||||
message: "The model call was interrupted by application restart after dispatch may have begun. Its outcome is unknown, so it was not replayed to avoid duplicate billing or output."
|
||||
.to_string(),
|
||||
source: None,
|
||||
},
|
||||
};
|
||||
return Ok(ProviderRunRestoreNormalization {
|
||||
interrupted_model_call: true,
|
||||
..ProviderRunRestoreNormalization::default()
|
||||
});
|
||||
}
|
||||
|
||||
let ProviderRunState::AwaitingTools { batch } = &mut self.state else {
|
||||
return Ok(ProviderRunRestoreNormalization::default());
|
||||
};
|
||||
@@ -1220,6 +1462,84 @@ fn validate_work_id(
|
||||
}
|
||||
}
|
||||
|
||||
fn validate_pending_tool_state(pending: &PendingToolCall) -> Result<(), String> {
|
||||
match &pending.state {
|
||||
PendingToolCallState::PermissionPending { request } => {
|
||||
if request.id.is_empty() {
|
||||
return Err(format!(
|
||||
"permission request for '{}' has an empty request ID",
|
||||
pending.call.id
|
||||
));
|
||||
}
|
||||
if request.call_id != pending.call.id {
|
||||
return Err(format!(
|
||||
"permission request for '{}' belongs to call '{}'",
|
||||
pending.call.id, request.call_id
|
||||
));
|
||||
}
|
||||
}
|
||||
PendingToolCallState::Approved {
|
||||
request_id,
|
||||
decision,
|
||||
} => {
|
||||
if request_id.is_empty() {
|
||||
return Err(format!(
|
||||
"approved tool call '{}' has an empty request ID",
|
||||
pending.call.id
|
||||
));
|
||||
}
|
||||
if matches!(decision, PermissionDecision::Denied { .. }) {
|
||||
return Err(format!(
|
||||
"approved tool call '{}' contains a denied decision",
|
||||
pending.call.id
|
||||
));
|
||||
}
|
||||
}
|
||||
PendingToolCallState::Resolved { result } if result.call_id != pending.call.id => {
|
||||
return Err(format!(
|
||||
"resolved result for '{}' belongs to call '{}'",
|
||||
pending.call.id, result.call_id
|
||||
));
|
||||
}
|
||||
PendingToolCallState::Proposed
|
||||
| PendingToolCallState::Executing
|
||||
| PendingToolCallState::RecoveryPending
|
||||
| PendingToolCallState::Resolved { .. } => {}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn tool_calls_from_message(message: &ConversationMessage) -> Vec<ToolCall> {
|
||||
let MessageContent::MultiPart(parts) = &message.content else {
|
||||
return Vec::new();
|
||||
};
|
||||
parts
|
||||
.iter()
|
||||
.filter_map(|part| match part {
|
||||
ContentPart::ToolUse {
|
||||
tool_use_id,
|
||||
name,
|
||||
input,
|
||||
} => Some(ToolCall {
|
||||
id: tool_use_id.clone(),
|
||||
name: name.clone(),
|
||||
arguments: input.clone(),
|
||||
}),
|
||||
ContentPart::Text(_)
|
||||
| ContentPart::Reasoning { .. }
|
||||
| ContentPart::ToolResult { .. }
|
||||
| ContentPart::Image { .. } => None,
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn usage_contains(total: &Usage, part: &Usage) -> bool {
|
||||
total.input_tokens >= part.input_tokens
|
||||
&& total.output_tokens >= part.output_tokens
|
||||
&& total.cached_input_tokens >= part.cached_input_tokens
|
||||
&& total.cache_creation_input_tokens >= part.cache_creation_input_tokens
|
||||
}
|
||||
|
||||
fn validate_model_turn(turn: &CompletedModelTurn) -> Result<(), ProviderRunProtocolError> {
|
||||
for part in &turn.assistant_content {
|
||||
match part {
|
||||
|
||||
@@ -88,6 +88,36 @@ fn assert_serialization_round_trip(run: &ProviderRun) {
|
||||
assert_eq!(&restored, run);
|
||||
}
|
||||
|
||||
fn mutate_run_json(run: &ProviderRun, mutate: impl FnOnce(&mut serde_json::Value)) -> ProviderRun {
|
||||
let mut value = serde_json::to_value(run).unwrap();
|
||||
mutate(&mut value);
|
||||
serde_json::from_value(value).unwrap()
|
||||
}
|
||||
|
||||
fn restored_state_error(run: &ProviderRun) -> String {
|
||||
let ProviderRunProtocolError::InvalidRestoredState { message } =
|
||||
run.validate_restored_state().unwrap_err()
|
||||
else {
|
||||
panic!("expected restored-state validation error");
|
||||
};
|
||||
message
|
||||
}
|
||||
|
||||
fn awaiting_tool_run() -> ProviderRun {
|
||||
let mut run = run();
|
||||
accept_tool_turn(
|
||||
&mut run,
|
||||
tool_turn(
|
||||
vec![
|
||||
tool_call("first", "read_files"),
|
||||
tool_call("second", "grep"),
|
||||
],
|
||||
&["read_files", "grep"],
|
||||
),
|
||||
);
|
||||
run
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn next_step_reemits_identical_pending_model_work() {
|
||||
let mut run = run();
|
||||
@@ -740,7 +770,228 @@ fn every_nonterminal_phase_round_trips_through_json() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn restore_normalization_preserves_safe_nonterminal_states() {
|
||||
fn restored_state_validation_accepts_valid_snapshots_in_every_phase() {
|
||||
let ready = run();
|
||||
let mut awaiting_model = ready.clone();
|
||||
let call = next_model_call(&mut awaiting_model);
|
||||
let mut resolving = awaiting_model.clone();
|
||||
resolving
|
||||
.accept_model_turn(&call.work_id, text_turn("done"))
|
||||
.unwrap();
|
||||
let mut awaiting_driver = resolving.clone();
|
||||
assert_eq!(awaiting_driver.next_step().unwrap(), None);
|
||||
let awaiting_tools = awaiting_tool_run();
|
||||
let mut done = awaiting_driver.clone();
|
||||
let work_id = awaiting_driver.active_work_id().unwrap().clone();
|
||||
done.complete(&work_id).unwrap();
|
||||
let mut failed = ready.clone();
|
||||
failed
|
||||
.fail(ProviderRunFailureKind::ExternalWork, "failed")
|
||||
.unwrap();
|
||||
let mut cancelled = ready;
|
||||
cancelled.cancel("cancelled").unwrap();
|
||||
|
||||
for candidate in [
|
||||
awaiting_model,
|
||||
resolving,
|
||||
awaiting_tools,
|
||||
awaiting_driver,
|
||||
done,
|
||||
failed,
|
||||
cancelled,
|
||||
] {
|
||||
candidate.validate_restored_state().unwrap();
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn restored_state_validation_rejects_active_work_run_and_epoch_mismatches() {
|
||||
let mut awaiting_model = run();
|
||||
next_model_call(&mut awaiting_model);
|
||||
let wrong_run = mutate_run_json(&awaiting_model, |value| {
|
||||
value["state"]["AwaitingModel"]["call"]["work_id"]["run_id"] = json!("other-run");
|
||||
});
|
||||
assert!(restored_state_error(&wrong_run).contains("active work identity other-run:0"));
|
||||
|
||||
let wrong_epoch = mutate_run_json(&awaiting_model, |value| {
|
||||
value["state"]["AwaitingModel"]["call"]["work_id"]["epoch"] = json!(9);
|
||||
});
|
||||
assert!(restored_state_error(&wrong_epoch).contains("active work identity run-1:9"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn restored_state_validation_rejects_duplicate_tool_call_ids() {
|
||||
let corrupted = mutate_run_json(&awaiting_tool_run(), |value| {
|
||||
value["state"]["AwaitingTools"]["batch"]["calls"][1]["call"]["id"] = json!("first");
|
||||
});
|
||||
assert_eq!(
|
||||
restored_state_error(&corrupted),
|
||||
"duplicate pending tool call ID 'first'"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn restored_state_validation_rejects_result_and_permission_call_ownership() {
|
||||
let mut resolved = awaiting_tool_run();
|
||||
let work_id = resolved.active_work_id().unwrap().clone();
|
||||
resolved
|
||||
.complete_tool(&work_id, successful_result("first", "done"))
|
||||
.unwrap();
|
||||
let wrong_result = mutate_run_json(&resolved, |value| {
|
||||
value["state"]["AwaitingTools"]["batch"]["calls"][0]["state"]["Resolved"]["result"]["call_id"] =
|
||||
json!("second");
|
||||
});
|
||||
assert_eq!(
|
||||
restored_state_error(&wrong_result),
|
||||
"resolved result for 'first' belongs to call 'second'"
|
||||
);
|
||||
|
||||
let mut permission = awaiting_tool_run();
|
||||
let work_id = permission.active_work_id().unwrap().clone();
|
||||
permission
|
||||
.request_tool_permission(
|
||||
&work_id,
|
||||
PermissionRequest {
|
||||
id: "request-1".to_string(),
|
||||
call_id: "first".to_string(),
|
||||
kind: PermissionKind::Read,
|
||||
reason: None,
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
let wrong_permission = mutate_run_json(&permission, |value| {
|
||||
value["state"]["AwaitingTools"]["batch"]["calls"][0]["state"]["PermissionPending"]["request"]
|
||||
["call_id"] = json!("second");
|
||||
});
|
||||
assert_eq!(
|
||||
restored_state_error(&wrong_permission),
|
||||
"permission request for 'first' belongs to call 'second'"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn restored_state_validation_rejects_phase_specific_corruption() {
|
||||
let empty_batch = mutate_run_json(&awaiting_tool_run(), |value| {
|
||||
value["state"]["AwaitingTools"]["batch"]["calls"] = json!([]);
|
||||
});
|
||||
assert_eq!(
|
||||
restored_state_error(&empty_batch),
|
||||
"pending tool batch is empty"
|
||||
);
|
||||
|
||||
let mismatched_batch = mutate_run_json(&awaiting_tool_run(), |value| {
|
||||
value["state"]["AwaitingTools"]["batch"]["calls"][0]["call"]["name"] =
|
||||
json!("different_tool");
|
||||
});
|
||||
assert!(restored_state_error(&mismatched_batch).contains("latest assistant message"));
|
||||
|
||||
let resolving_without_turn = mutate_run_json(&awaiting_tool_run(), |value| {
|
||||
value["model_turns"] = json!(0);
|
||||
});
|
||||
assert!(restored_state_error(&resolving_without_turn).contains("has no completed model turn"));
|
||||
|
||||
let mut resolving = run();
|
||||
let call = next_model_call(&mut resolving);
|
||||
resolving
|
||||
.accept_model_turn(&call.work_id, text_turn("done"))
|
||||
.unwrap();
|
||||
let invalid_turn = mutate_run_json(&resolving, |value| {
|
||||
value["state"]["ResolvingModel"]["turn"]["assistant_content"] = json!([{
|
||||
"ToolUse": {
|
||||
"tool_use_id": "injected",
|
||||
"name": "read_files",
|
||||
"input": {}
|
||||
}
|
||||
}]);
|
||||
});
|
||||
assert!(restored_state_error(&invalid_turn).contains("assistant_content"));
|
||||
|
||||
let mut awaiting_driver = resolving;
|
||||
assert_eq!(awaiting_driver.next_step().unwrap(), None);
|
||||
let wrong_driver_owner = mutate_run_json(&awaiting_driver, |value| {
|
||||
let last = value["transcript"]
|
||||
.as_array_mut()
|
||||
.unwrap()
|
||||
.last_mut()
|
||||
.unwrap();
|
||||
last["role"] = json!("User");
|
||||
});
|
||||
assert!(restored_state_error(&wrong_driver_owner).contains("does not follow an assistant"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn restored_state_validation_rejects_retry_counter_and_terminal_corruption() {
|
||||
let zero_limit = mutate_run_json(&run(), |value| {
|
||||
value["limits"]["max_model_turns"] = json!(0);
|
||||
});
|
||||
assert_eq!(
|
||||
restored_state_error(&zero_limit),
|
||||
"model-turn limit must be at least one"
|
||||
);
|
||||
|
||||
let excessive_retries = mutate_run_json(&run(), |value| {
|
||||
value["model_retries"] = json!(3);
|
||||
});
|
||||
assert!(restored_state_error(&excessive_retries).contains("model-retry counter"));
|
||||
|
||||
let mut awaiting_model = run();
|
||||
next_model_call(&mut awaiting_model);
|
||||
let inconsistent_retry = mutate_run_json(&awaiting_model, |value| {
|
||||
value["state"]["AwaitingModel"]["call"]["retry_attempt"] = json!(1);
|
||||
});
|
||||
assert!(restored_state_error(&inconsistent_retry).contains("total retry counter"));
|
||||
|
||||
let completed_without_turn = mutate_run_json(&run(), |value| {
|
||||
value["state"] = json!({"Done": {"completion": {"stop_reason": "Completed"}}});
|
||||
});
|
||||
assert_eq!(
|
||||
restored_state_error(&completed_without_turn),
|
||||
"completed run has no completed model turn"
|
||||
);
|
||||
|
||||
let early_turn_limit = mutate_run_json(&run(), |value| {
|
||||
value["state"] = json!({
|
||||
"Failed": {"failure": {
|
||||
"kind": "TurnLimitExceeded",
|
||||
"message": "bad",
|
||||
"source": null
|
||||
}}
|
||||
});
|
||||
});
|
||||
assert_eq!(
|
||||
restored_state_error(&early_turn_limit),
|
||||
"turn-limit failure occurred before reaching the limit"
|
||||
);
|
||||
|
||||
let missing_retry_source = mutate_run_json(&run(), |value| {
|
||||
value["state"] = json!({
|
||||
"Failed": {"failure": {
|
||||
"kind": "RetryLimitExceeded",
|
||||
"message": "bad",
|
||||
"source": null
|
||||
}}
|
||||
});
|
||||
});
|
||||
assert_eq!(
|
||||
restored_state_error(&missing_retry_source),
|
||||
"retry-limit failure lacks a recoverable source"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn restored_state_validation_rejects_empty_run_identity_and_profile() {
|
||||
let empty_run = mutate_run_json(&run(), |value| value["id"] = json!(""));
|
||||
assert_eq!(restored_state_error(&empty_run), "run ID must not be empty");
|
||||
|
||||
let empty_profile = mutate_run_json(&run(), |value| value["profile"] = json!(""));
|
||||
assert_eq!(
|
||||
restored_state_error(&empty_profile),
|
||||
"request profile must not be empty"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn restore_normalization_preserves_model_work_outside_the_uncertain_dispatch_boundary() {
|
||||
let ready = run();
|
||||
let mut awaiting_model = ready.clone();
|
||||
let call = next_model_call(&mut awaiting_model);
|
||||
@@ -752,7 +1003,7 @@ fn restore_normalization_preserves_safe_nonterminal_states() {
|
||||
let mut awaiting_driver = resolving.clone();
|
||||
assert_eq!(awaiting_driver.next_step().unwrap(), None);
|
||||
|
||||
for mut candidate in [ready, awaiting_model, resolving, awaiting_driver] {
|
||||
for mut candidate in [ready, resolving, awaiting_driver] {
|
||||
let before = candidate.clone();
|
||||
assert_eq!(
|
||||
candidate.normalize_after_restore().unwrap(),
|
||||
@@ -762,6 +1013,58 @@ fn restore_normalization_preserves_safe_nonterminal_states() {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn crash_after_model_acceptance_before_checkpoint_does_not_replay_the_persisted_call() {
|
||||
let mut live_run = run();
|
||||
let call = next_model_call(&mut live_run);
|
||||
let serialized_at_dispatch_boundary = serde_json::to_string(&live_run).unwrap();
|
||||
|
||||
// Simulate remote acceptance followed by a crash before the accepted turn is checkpointed.
|
||||
live_run
|
||||
.accept_model_turn(&call.work_id, text_turn("accepted but not checkpointed"))
|
||||
.unwrap();
|
||||
assert_eq!(live_run.model_turns(), 1);
|
||||
|
||||
let mut restored: ProviderRun = serde_json::from_str(&serialized_at_dispatch_boundary).unwrap();
|
||||
|
||||
let normalization = restored.normalize_after_restore().unwrap();
|
||||
|
||||
assert!(normalization.interrupted_model_call);
|
||||
let ProviderRunState::Failed { failure } = restored.state() else {
|
||||
panic!("uncertain model work must become terminal on restore");
|
||||
};
|
||||
assert_eq!(failure.kind, ProviderRunFailureKind::Restore);
|
||||
assert!(failure.message.contains("outcome is unknown"));
|
||||
assert!(failure.message.contains("not replayed"));
|
||||
assert_eq!(restored.active_work_id(), None);
|
||||
assert!(matches!(
|
||||
restored.next_step().unwrap(),
|
||||
Some(ProviderRunStep::Done(ProviderRunOutcome::Failed(_)))
|
||||
));
|
||||
|
||||
// The persisted dispatch identity remains useful for diagnostics but can never be called again.
|
||||
assert_eq!(call.work_id.epoch, RunEpoch::new(0));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn restore_after_model_acceptance_keeps_the_committed_turn_without_replaying() {
|
||||
let mut run = run();
|
||||
let call = next_model_call(&mut run);
|
||||
run.accept_model_turn(&call.work_id, text_turn("accepted output"))
|
||||
.unwrap();
|
||||
let serialized_after_acceptance = serde_json::to_string(&run).unwrap();
|
||||
let mut restored: ProviderRun = serde_json::from_str(&serialized_after_acceptance).unwrap();
|
||||
|
||||
assert_eq!(
|
||||
restored.normalize_after_restore().unwrap(),
|
||||
ProviderRunRestoreNormalization::default()
|
||||
);
|
||||
assert_eq!(restored.state().phase(), ProviderRunPhase::ResolvingModel);
|
||||
assert_eq!(restored.model_turns(), 1);
|
||||
assert_eq!(restored.next_step().unwrap(), None);
|
||||
assert_eq!(restored.state().phase(), ProviderRunPhase::AwaitingDriver);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn restore_normalization_reproposes_permissions_and_interrupts_unsafe_tools() {
|
||||
let mut run = run();
|
||||
|
||||
Reference in New Issue
Block a user