diff --git a/crates/zesdex-backend/src/app/mod.rs b/crates/zesdex-backend/src/app/mod.rs index 47c99a8..ffba1d9 100644 --- a/crates/zesdex-backend/src/app/mod.rs +++ b/crates/zesdex-backend/src/app/mod.rs @@ -10,4 +10,5 @@ pub mod review; pub mod runtime; pub mod state; pub mod subagent; +pub mod util; pub mod workflow; diff --git a/crates/zesdex-backend/src/app/subagent/engine.rs b/crates/zesdex-backend/src/app/subagent/engine.rs index 4a048d1..a2fcf8d 100644 --- a/crates/zesdex-backend/src/app/subagent/engine.rs +++ b/crates/zesdex-backend/src/app/subagent/engine.rs @@ -14,25 +14,13 @@ use super::workspace::generate_workspace_tree; use crate::dto::chat::message::ChatMessage; use crate::dto::provider::request::ToolDef; use crate::tool::tool_is_risky; -use std::time::{Duration, SystemTime, UNIX_EPOCH}; +use crate::app::util::backoff::backoff_seconds; +use std::time::Duration; use tokio::sync::mpsc; -/// Tiny jitter helper so retry backoffs don't arrive in lockstep. -fn retry_jitter_ns(range_ns: u64) -> u64 { - SystemTime::now() - .duration_since(UNIX_EPOCH) - .unwrap_or_default() - .subsec_nanos() as u64 - % range_ns -} - -/// Exponential backoff with ±25% jitter for subagent step retries. +/// Exponential backoff with ±25% jitter for subagent step retries, capped at 16s. fn step_retry_delay(attempt: u32) -> Duration { - let base_secs = (2u64).pow(attempt).min(16); // 2s, 4s, 8s, 16s cap - let quarter = (base_secs * 250_000_000).max(100_000_000); - let offset = retry_jitter_ns(quarter); - let ns = base_secs * 1_000_000_000 + offset - quarter / 2; - Duration::from_nanos(ns) + backoff_seconds(attempt, 16) } /// Heuristic to decide whether the error is worth retrying. diff --git a/crates/zesdex-backend/src/app/util/backoff.rs b/crates/zesdex-backend/src/app/util/backoff.rs new file mode 100644 index 0000000..d6cf554 --- /dev/null +++ b/crates/zesdex-backend/src/app/util/backoff.rs @@ -0,0 +1,30 @@ +//! Exponential backoff with jitter. +//! +//! Three use cases (subagent, provider, workflow) all share the same formula +//! with different caps. This module provides a single implementation. + +use std::time::{Duration, SystemTime, UNIX_EPOCH}; + +/// Compute an exponential backoff with ±25% jitter. +/// +/// `attempt` is 0-based (first retry -> attempt=0 -> base=1s, +/// second retry -> attempt=1 -> base=2s, etc.). +/// `max_secs` sets the cap. +pub fn backoff_seconds(attempt: u32, max_secs: u64) -> Duration { + let base_secs = (2u64).pow(attempt).min(max_secs); + let quarter = (base_secs * 250_000_000).max(100_000_000); // 25% of base, min 100ms + let offset = jitter_ns(quarter); + // ±25%: offset in [0, quarter), so result = base - quarter/2 + offset + // which lies in [base - 25%, base + 25%). + let ns = base_secs * 1_000_000_000 + offset - quarter / 2; + Duration::from_nanos(ns) +} + +/// Return a jitter offset in the range [0, range_ns). +fn jitter_ns(range_ns: u64) -> u64 { + let nanos = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_nanos() as u64; + nanos % range_ns +} diff --git a/crates/zesdex-backend/src/app/util/mod.rs b/crates/zesdex-backend/src/app/util/mod.rs new file mode 100644 index 0000000..c02e7fc --- /dev/null +++ b/crates/zesdex-backend/src/app/util/mod.rs @@ -0,0 +1,3 @@ +//! Utility modules for shared helpers. + +pub mod backoff; diff --git a/crates/zesdex-backend/src/app/workflow/engine/mod.rs b/crates/zesdex-backend/src/app/workflow/engine/mod.rs index 9eaef8f..2f97979 100644 --- a/crates/zesdex-backend/src/app/workflow/engine/mod.rs +++ b/crates/zesdex-backend/src/app/workflow/engine/mod.rs @@ -28,7 +28,7 @@ use std::sync::{ atomic::{AtomicBool, Ordering}, Arc, Mutex, }; -use std::time::{Duration, SystemTime, UNIX_EPOCH}; +use std::time::Duration; /// The lifecycle state of an agent within a workflow run. #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] @@ -399,18 +399,9 @@ fn spawn_single_agent(sp: SpawnCtx<'_>) -> anyhow::Result { let bg_abort_thread = bg_abort.clone(); let bg_name_thread = bg_name.clone(); std::thread::spawn(move || { - // Retry wrapper: jittered backoff 1s → 2s. - let retry_backoff = |attempt: u32| { - let base_secs = (2u64).pow(attempt).min(8); - let quarter = (base_secs * 250_000_000).max(100_000_000); - let nanos = SystemTime::now() - .duration_since(UNIX_EPOCH) - .unwrap_or_default() - .subsec_nanos() as u64; - let offset = nanos % quarter; - let ns = base_secs * 1_000_000_000 + offset - quarter / 2; - Duration::from_nanos(ns) - }; + // Retry wrapper: jittered backoff with 8s cap. + let retry_backoff = + |attempt: u32| crate::app::util::backoff::backoff_seconds(attempt, 8); for attempt in 1..=2 { // Don't retry if aborted. diff --git a/crates/zesdex-backend/src/service/provider.rs b/crates/zesdex-backend/src/service/provider.rs index 05f964e..f5e138a 100644 --- a/crates/zesdex-backend/src/service/provider.rs +++ b/crates/zesdex-backend/src/service/provider.rs @@ -26,9 +26,10 @@ use anyhow::Result; use std::sync::atomic::AtomicBool; -use std::time::{Duration, SystemTime, UNIX_EPOCH}; +use std::time::Duration; use crate::app::runtime::stream::turn::StreamedTurn; +use crate::app::util::backoff::backoff_seconds; use crate::app::runtime::stream::{SseParser, StreamEvent}; use crate::dto::chat::message::ChatMessage; use crate::dto::provider::request::{ChatRequest, StreamOptions, ToolDef}; @@ -43,30 +44,11 @@ const REQUEST_TIMEOUT: Duration = Duration::from_mins(1); // Retry helpers // --------------------------------------------------------------------------- -/// Return a pseudo-random jitter offset in the range [0, range_ns). -/// -/// Uses the full epoch nanoseconds (wrapped to u64) instead of the -/// sub-second component so the jitter range scales with `range_ns` -/// rather than being capped at ~1 s. -fn jitter_ns(range_ns: u64) -> u64 { - let nanos = SystemTime::now() - .duration_since(UNIX_EPOCH) - .unwrap_or_default() - .as_nanos() as u64; - nanos % range_ns -} - /// Exponential backoff with ±25% jitter, capped at 30 seconds. /// /// `attempt` is 1-based (first retry → attempt=1). fn backoff_duration(attempt: u32) -> Duration { - let base_secs = (2u64).pow(attempt).min(30); - let half_range = (base_secs * 250_000_000).max(100_000_000); // 25% of base, min 100ms - let offset = jitter_ns(half_range * 2); // [0, 50% of base) - // ±25% jitter: subtract half_range so the result varies - // between base-25% and base+25%. - let ns = base_secs * 1_000_000_000 + offset - half_range; - Duration::from_nanos(ns) + backoff_seconds(attempt, 30) } /// Is the error an auth / billing failure that retrying won't fix? @@ -95,12 +77,8 @@ fn is_rate_limit(err_str: &str) -> bool { /// Return a rate-appropriate backoff (longer for 429). fn backoff_for_error(attempt: u32, err_str: &str) -> Duration { if is_rate_limit(err_str) { - // Rate limits need more time to drain — start at 5s instead of 2s. - let base_secs = (5u64 * (2u64).pow(attempt.saturating_sub(1))).min(60); - let half_range = (base_secs * 250_000_000).max(100_000_000); - let offset = jitter_ns(half_range * 2); - let ns = base_secs * 1_000_000_000 + offset - half_range; - Duration::from_nanos(ns) + // Rate limits need more time to drain — backoff capped at 60s. + backoff_seconds(attempt, 60) } else { backoff_duration(attempt) }