Skip to content

Commit 4105a2f

Browse files
authored
Merge pull request anomalyco#885 from affaan-m/feat/ecc2-crash-resume
feat(ecc2): add crash resume session recovery
2 parents cf439dd + 0166231 commit 4105a2f

5 files changed

Lines changed: 229 additions & 2 deletions

File tree

‎ecc2/src/main.rs‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,11 @@ enum Commands {
4545
/// Session ID or alias
4646
session_id: String,
4747
},
48+
/// Resume a failed or stopped session
49+
Resume {
50+
/// Session ID or alias
51+
session_id: String,
52+
},
4853
/// Run as background daemon
4954
Daemon,
5055
#[command(hide = true)]
@@ -99,6 +104,10 @@ async fn main() -> Result<()> {
99104
session::manager::stop_session(&db, &session_id).await?;
100105
println!("Session stopped: {session_id}");
101106
}
107+
Some(Commands::Resume { session_id }) => {
108+
let resumed_id = session::manager::resume_session(&db, &session_id).await?;
109+
println!("Session resumed: {resumed_id}");
110+
}
102111
Some(Commands::Daemon) => {
103112
println!("Starting ECC daemon...");
104113
session::daemon::run(db, cfg).await?;
@@ -115,3 +124,19 @@ async fn main() -> Result<()> {
115124

116125
Ok(())
117126
}
127+
128+
#[cfg(test)]
129+
mod tests {
130+
use super::*;
131+
132+
#[test]
133+
fn cli_parses_resume_command() {
134+
let cli = Cli::try_parse_from(["ecc", "resume", "deadbeef"])
135+
.expect("resume subcommand should parse");
136+
137+
match cli.command {
138+
Some(Commands::Resume { session_id }) => assert_eq!(session_id, "deadbeef"),
139+
_ => panic!("expected resume subcommand"),
140+
}
141+
}
142+
}

‎ecc2/src/session/daemon.rs‎

Lines changed: 132 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ use crate::config::Config;
1010
/// and cleans up stale resources.
1111
pub async fn run(db: StateStore, cfg: Config) -> Result<()> {
1212
tracing::info!("ECC daemon started");
13+
resume_crashed_sessions(&db)?;
1314

1415
let heartbeat_interval = Duration::from_secs(cfg.heartbeat_interval_secs);
1516
let timeout = Duration::from_secs(cfg.session_timeout_secs);
@@ -23,6 +24,43 @@ pub async fn run(db: StateStore, cfg: Config) -> Result<()> {
2324
}
2425
}
2526

27+
pub fn resume_crashed_sessions(db: &StateStore) -> Result<()> {
28+
let failed_sessions = resume_crashed_sessions_with(db, pid_is_alive)?;
29+
if failed_sessions > 0 {
30+
tracing::warn!("Marked {failed_sessions} crashed sessions as failed during daemon startup");
31+
}
32+
Ok(())
33+
}
34+
35+
fn resume_crashed_sessions_with<F>(db: &StateStore, is_pid_alive: F) -> Result<usize>
36+
where
37+
F: Fn(u32) -> bool,
38+
{
39+
let sessions = db.list_sessions()?;
40+
let mut failed_sessions = 0;
41+
42+
for session in sessions {
43+
if session.state != SessionState::Running {
44+
continue;
45+
}
46+
47+
let is_alive = session.pid.is_some_and(&is_pid_alive);
48+
if is_alive {
49+
continue;
50+
}
51+
52+
tracing::warn!(
53+
"Session {} was left running with stale pid {:?}; marking it failed",
54+
session.id,
55+
session.pid
56+
);
57+
db.update_state_and_pid(&session.id, &SessionState::Failed, None)?;
58+
failed_sessions += 1;
59+
}
60+
61+
Ok(failed_sessions)
62+
}
63+
2664
fn check_sessions(db: &StateStore, timeout: Duration) -> Result<()> {
2765
let sessions = db.list_sessions()?;
2866

@@ -38,9 +76,102 @@ fn check_sessions(db: &StateStore, timeout: Duration) -> Result<()> {
3876

3977
if elapsed > timeout {
4078
tracing::warn!("Session {} timed out after {:?}", session.id, elapsed);
41-
db.update_state(&session.id, &SessionState::Failed)?;
79+
db.update_state_and_pid(&session.id, &SessionState::Failed, None)?;
4280
}
4381
}
4482

4583
Ok(())
4684
}
85+
86+
#[cfg(unix)]
87+
fn pid_is_alive(pid: u32) -> bool {
88+
if pid == 0 {
89+
return false;
90+
}
91+
92+
// SAFETY: kill(pid, 0) probes process existence without delivering a signal.
93+
let result = unsafe { libc::kill(pid as libc::pid_t, 0) };
94+
if result == 0 {
95+
return true;
96+
}
97+
98+
matches!(
99+
std::io::Error::last_os_error().raw_os_error(),
100+
Some(code) if code == libc::EPERM
101+
)
102+
}
103+
104+
#[cfg(not(unix))]
105+
fn pid_is_alive(_pid: u32) -> bool {
106+
false
107+
}
108+
109+
#[cfg(test)]
110+
mod tests {
111+
use super::*;
112+
use crate::session::{Session, SessionMetrics, SessionState};
113+
use std::path::PathBuf;
114+
115+
fn temp_db_path() -> PathBuf {
116+
std::env::temp_dir().join(format!("ecc2-daemon-test-{}.db", uuid::Uuid::new_v4()))
117+
}
118+
119+
fn sample_session(id: &str, state: SessionState, pid: Option<u32>) -> Session {
120+
let now = chrono::Utc::now();
121+
Session {
122+
id: id.to_string(),
123+
task: "Recover crashed worker".to_string(),
124+
agent_type: "claude".to_string(),
125+
state,
126+
pid,
127+
worktree: None,
128+
created_at: now,
129+
updated_at: now,
130+
metrics: SessionMetrics::default(),
131+
}
132+
}
133+
134+
#[test]
135+
fn resume_crashed_sessions_marks_dead_running_sessions_failed() -> Result<()> {
136+
let path = temp_db_path();
137+
let store = StateStore::open(&path)?;
138+
store.insert_session(&sample_session(
139+
"deadbeef",
140+
SessionState::Running,
141+
Some(4242),
142+
))?;
143+
144+
resume_crashed_sessions_with(&store, |_| false)?;
145+
146+
let session = store
147+
.get_session("deadbeef")?
148+
.expect("session should still exist");
149+
assert_eq!(session.state, SessionState::Failed);
150+
assert_eq!(session.pid, None);
151+
152+
let _ = std::fs::remove_file(path);
153+
Ok(())
154+
}
155+
156+
#[test]
157+
fn resume_crashed_sessions_keeps_live_running_sessions_running() -> Result<()> {
158+
let path = temp_db_path();
159+
let store = StateStore::open(&path)?;
160+
store.insert_session(&sample_session(
161+
"alive123",
162+
SessionState::Running,
163+
Some(7777),
164+
))?;
165+
166+
resume_crashed_sessions_with(&store, |_| true)?;
167+
168+
let session = store
169+
.get_session("alive123")?
170+
.expect("session should still exist");
171+
assert_eq!(session.state, SessionState::Running);
172+
assert_eq!(session.pid, Some(7777));
173+
174+
let _ = std::fs::remove_file(path);
175+
Ok(())
176+
}
177+
}

‎ecc2/src/session/manager.rs‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,21 @@ pub async fn stop_session(db: &StateStore, id: &str) -> Result<()> {
3636
stop_session_with_options(db, id, true).await
3737
}
3838

39+
pub async fn resume_session(db: &StateStore, id: &str) -> Result<String> {
40+
let session = resolve_session(db, id)?;
41+
42+
if session.state == SessionState::Completed {
43+
anyhow::bail!("Completed sessions cannot be resumed: {}", session.id);
44+
}
45+
46+
if session.state == SessionState::Running {
47+
anyhow::bail!("Session is already running: {}", session.id);
48+
}
49+
50+
db.update_state_and_pid(&session.id, &SessionState::Pending, None)?;
51+
Ok(session.id)
52+
}
53+
3954
fn agent_program(agent_type: &str) -> Result<PathBuf> {
4055
match agent_type {
4156
"claude" => Ok(PathBuf::from("claude")),
@@ -575,6 +590,36 @@ mod tests {
575590
Ok(())
576591
}
577592

593+
#[tokio::test(flavor = "current_thread")]
594+
async fn resume_session_requeues_failed_session() -> Result<()> {
595+
let tempdir = TestDir::new("manager-resume-session")?;
596+
let cfg = build_config(tempdir.path());
597+
let db = StateStore::open(&cfg.db_path)?;
598+
let now = Utc::now();
599+
600+
db.insert_session(&Session {
601+
id: "deadbeef".to_string(),
602+
task: "resume previous task".to_string(),
603+
agent_type: "claude".to_string(),
604+
state: SessionState::Failed,
605+
pid: Some(31337),
606+
worktree: None,
607+
created_at: now - Duration::minutes(1),
608+
updated_at: now,
609+
metrics: SessionMetrics::default(),
610+
})?;
611+
612+
let resumed_id = resume_session(&db, "deadbeef").await?;
613+
let resumed = db
614+
.get_session(&resumed_id)?
615+
.context("resumed session should exist")?;
616+
617+
assert_eq!(resumed.state, SessionState::Pending);
618+
assert_eq!(resumed.pid, None);
619+
620+
Ok(())
621+
}
622+
578623
#[test]
579624
fn get_status_supports_latest_alias() -> Result<()> {
580625
let tempdir = TestDir::new("manager-latest-status")?;

‎ecc2/src/session/store.rs‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -124,6 +124,29 @@ impl StateStore {
124124
Ok(())
125125
}
126126

127+
pub fn update_state_and_pid(
128+
&self,
129+
session_id: &str,
130+
state: &SessionState,
131+
pid: Option<u32>,
132+
) -> Result<()> {
133+
let updated = self.conn.execute(
134+
"UPDATE sessions SET state = ?1, pid = ?2, updated_at = ?3 WHERE id = ?4",
135+
rusqlite::params![
136+
state.to_string(),
137+
pid.map(i64::from),
138+
chrono::Utc::now().to_rfc3339(),
139+
session_id,
140+
],
141+
)?;
142+
143+
if updated == 0 {
144+
anyhow::bail!("Session not found: {session_id}");
145+
}
146+
147+
Ok(())
148+
}
149+
127150
pub fn update_state(&self, session_id: &str, state: &SessionState) -> Result<()> {
128151
let current_state = self
129152
.conn

‎ecc2/src/tui/dashboard.rs‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -422,7 +422,10 @@ impl Dashboard {
422422

423423
pub fn stop_selected(&mut self) {
424424
if let Some(session) = self.sessions.get(self.selected_session) {
425-
if let Err(error) = self.db.update_state(&session.id, &SessionState::Stopped) {
425+
if let Err(error) =
426+
self.db
427+
.update_state_and_pid(&session.id, &SessionState::Stopped, None)
428+
{
426429
tracing::warn!("Failed to stop session {}: {error}", session.id);
427430
return;
428431
}

0 commit comments

Comments
 (0)