refactor: unify 3 backoff implementations into shared helper
This commit is contained in:
@@ -10,4 +10,5 @@ pub mod review;
|
||||
pub mod runtime;
|
||||
pub mod state;
|
||||
pub mod subagent;
|
||||
pub mod util;
|
||||
pub mod workflow;
|
||||
|
||||
@@ -14,25 +14,13 @@ use super::workspace::generate_workspace_tree;
|
||||
use crate::dto::chat::message::ChatMessage;
|
||||
use crate::dto::provider::request::ToolDef;
|
||||
use crate::tool::tool_is_risky;
|
||||
use std::time::{Duration, SystemTime, UNIX_EPOCH};
|
||||
use crate::app::util::backoff::backoff_seconds;
|
||||
use std::time::Duration;
|
||||
use tokio::sync::mpsc;
|
||||
|
||||
/// Tiny jitter helper so retry backoffs don't arrive in lockstep.
|
||||
fn retry_jitter_ns(range_ns: u64) -> u64 {
|
||||
SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.unwrap_or_default()
|
||||
.subsec_nanos() as u64
|
||||
% range_ns
|
||||
}
|
||||
|
||||
/// Exponential backoff with ±25% jitter for subagent step retries.
|
||||
/// Exponential backoff with ±25% jitter for subagent step retries, capped at 16s.
|
||||
fn step_retry_delay(attempt: u32) -> Duration {
|
||||
let base_secs = (2u64).pow(attempt).min(16); // 2s, 4s, 8s, 16s cap
|
||||
let quarter = (base_secs * 250_000_000).max(100_000_000);
|
||||
let offset = retry_jitter_ns(quarter);
|
||||
let ns = base_secs * 1_000_000_000 + offset - quarter / 2;
|
||||
Duration::from_nanos(ns)
|
||||
backoff_seconds(attempt, 16)
|
||||
}
|
||||
|
||||
/// Heuristic to decide whether the error is worth retrying.
|
||||
|
||||
@@ -0,0 +1,30 @@
|
||||
//! Exponential backoff with jitter.
|
||||
//!
|
||||
//! Three use cases (subagent, provider, workflow) all share the same formula
|
||||
//! with different caps. This module provides a single implementation.
|
||||
|
||||
use std::time::{Duration, SystemTime, UNIX_EPOCH};
|
||||
|
||||
/// Compute an exponential backoff with ±25% jitter.
|
||||
///
|
||||
/// `attempt` is 0-based (first retry -> attempt=0 -> base=1s,
|
||||
/// second retry -> attempt=1 -> base=2s, etc.).
|
||||
/// `max_secs` sets the cap.
|
||||
pub fn backoff_seconds(attempt: u32, max_secs: u64) -> Duration {
|
||||
let base_secs = (2u64).pow(attempt).min(max_secs);
|
||||
let quarter = (base_secs * 250_000_000).max(100_000_000); // 25% of base, min 100ms
|
||||
let offset = jitter_ns(quarter);
|
||||
// ±25%: offset in [0, quarter), so result = base - quarter/2 + offset
|
||||
// which lies in [base - 25%, base + 25%).
|
||||
let ns = base_secs * 1_000_000_000 + offset - quarter / 2;
|
||||
Duration::from_nanos(ns)
|
||||
}
|
||||
|
||||
/// Return a jitter offset in the range [0, range_ns).
|
||||
fn jitter_ns(range_ns: u64) -> u64 {
|
||||
let nanos = SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.unwrap_or_default()
|
||||
.as_nanos() as u64;
|
||||
nanos % range_ns
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
//! Utility modules for shared helpers.
|
||||
|
||||
pub mod backoff;
|
||||
@@ -28,7 +28,7 @@ use std::sync::{
|
||||
atomic::{AtomicBool, Ordering},
|
||||
Arc, Mutex,
|
||||
};
|
||||
use std::time::{Duration, SystemTime, UNIX_EPOCH};
|
||||
use std::time::Duration;
|
||||
|
||||
/// The lifecycle state of an agent within a workflow run.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||||
@@ -399,18 +399,9 @@ fn spawn_single_agent(sp: SpawnCtx<'_>) -> anyhow::Result<String> {
|
||||
let bg_abort_thread = bg_abort.clone();
|
||||
let bg_name_thread = bg_name.clone();
|
||||
std::thread::spawn(move || {
|
||||
// Retry wrapper: jittered backoff 1s → 2s.
|
||||
let retry_backoff = |attempt: u32| {
|
||||
let base_secs = (2u64).pow(attempt).min(8);
|
||||
let quarter = (base_secs * 250_000_000).max(100_000_000);
|
||||
let nanos = SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.unwrap_or_default()
|
||||
.subsec_nanos() as u64;
|
||||
let offset = nanos % quarter;
|
||||
let ns = base_secs * 1_000_000_000 + offset - quarter / 2;
|
||||
Duration::from_nanos(ns)
|
||||
};
|
||||
// Retry wrapper: jittered backoff with 8s cap.
|
||||
let retry_backoff =
|
||||
|attempt: u32| crate::app::util::backoff::backoff_seconds(attempt, 8);
|
||||
|
||||
for attempt in 1..=2 {
|
||||
// Don't retry if aborted.
|
||||
|
||||
@@ -26,9 +26,10 @@
|
||||
|
||||
use anyhow::Result;
|
||||
use std::sync::atomic::AtomicBool;
|
||||
use std::time::{Duration, SystemTime, UNIX_EPOCH};
|
||||
use std::time::Duration;
|
||||
|
||||
use crate::app::runtime::stream::turn::StreamedTurn;
|
||||
use crate::app::util::backoff::backoff_seconds;
|
||||
use crate::app::runtime::stream::{SseParser, StreamEvent};
|
||||
use crate::dto::chat::message::ChatMessage;
|
||||
use crate::dto::provider::request::{ChatRequest, StreamOptions, ToolDef};
|
||||
@@ -43,30 +44,11 @@ const REQUEST_TIMEOUT: Duration = Duration::from_mins(1);
|
||||
// Retry helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Return a pseudo-random jitter offset in the range [0, range_ns).
|
||||
///
|
||||
/// Uses the full epoch nanoseconds (wrapped to u64) instead of the
|
||||
/// sub-second component so the jitter range scales with `range_ns`
|
||||
/// rather than being capped at ~1 s.
|
||||
fn jitter_ns(range_ns: u64) -> u64 {
|
||||
let nanos = SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.unwrap_or_default()
|
||||
.as_nanos() as u64;
|
||||
nanos % range_ns
|
||||
}
|
||||
|
||||
/// Exponential backoff with ±25% jitter, capped at 30 seconds.
|
||||
///
|
||||
/// `attempt` is 1-based (first retry → attempt=1).
|
||||
fn backoff_duration(attempt: u32) -> Duration {
|
||||
let base_secs = (2u64).pow(attempt).min(30);
|
||||
let half_range = (base_secs * 250_000_000).max(100_000_000); // 25% of base, min 100ms
|
||||
let offset = jitter_ns(half_range * 2); // [0, 50% of base)
|
||||
// ±25% jitter: subtract half_range so the result varies
|
||||
// between base-25% and base+25%.
|
||||
let ns = base_secs * 1_000_000_000 + offset - half_range;
|
||||
Duration::from_nanos(ns)
|
||||
backoff_seconds(attempt, 30)
|
||||
}
|
||||
|
||||
/// Is the error an auth / billing failure that retrying won't fix?
|
||||
@@ -95,12 +77,8 @@ fn is_rate_limit(err_str: &str) -> bool {
|
||||
/// Return a rate-appropriate backoff (longer for 429).
|
||||
fn backoff_for_error(attempt: u32, err_str: &str) -> Duration {
|
||||
if is_rate_limit(err_str) {
|
||||
// Rate limits need more time to drain — start at 5s instead of 2s.
|
||||
let base_secs = (5u64 * (2u64).pow(attempt.saturating_sub(1))).min(60);
|
||||
let half_range = (base_secs * 250_000_000).max(100_000_000);
|
||||
let offset = jitter_ns(half_range * 2);
|
||||
let ns = base_secs * 1_000_000_000 + offset - half_range;
|
||||
Duration::from_nanos(ns)
|
||||
// Rate limits need more time to drain — backoff capped at 60s.
|
||||
backoff_seconds(attempt, 60)
|
||||
} else {
|
||||
backoff_duration(attempt)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user