refactor: unify 3 backoff implementations into shared helper

This commit is contained in:
asepharyana
2026-07-18 03:18:27 +07:00
parent e32501ee59
commit 431c8d3b89
6 changed files with 47 additions and 56 deletions
+1
View File
@@ -10,4 +10,5 @@ pub mod review;
pub mod runtime;
pub mod state;
pub mod subagent;
pub mod util;
pub mod workflow;
@@ -14,25 +14,13 @@ use super::workspace::generate_workspace_tree;
use crate::dto::chat::message::ChatMessage;
use crate::dto::provider::request::ToolDef;
use crate::tool::tool_is_risky;
use std::time::{Duration, SystemTime, UNIX_EPOCH};
use crate::app::util::backoff::backoff_seconds;
use std::time::Duration;
use tokio::sync::mpsc;
/// Tiny jitter helper so retry backoffs don't arrive in lockstep.
fn retry_jitter_ns(range_ns: u64) -> u64 {
SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap_or_default()
.subsec_nanos() as u64
% range_ns
}
/// Exponential backoff with ±25% jitter for subagent step retries.
/// Exponential backoff with ±25% jitter for subagent step retries, capped at 16s.
fn step_retry_delay(attempt: u32) -> Duration {
let base_secs = (2u64).pow(attempt).min(16); // 2s, 4s, 8s, 16s cap
let quarter = (base_secs * 250_000_000).max(100_000_000);
let offset = retry_jitter_ns(quarter);
let ns = base_secs * 1_000_000_000 + offset - quarter / 2;
Duration::from_nanos(ns)
backoff_seconds(attempt, 16)
}
/// Heuristic to decide whether the error is worth retrying.
@@ -0,0 +1,30 @@
//! Exponential backoff with jitter.
//!
//! Three use cases (subagent, provider, workflow) all share the same formula
//! with different caps. This module provides a single implementation.
use std::time::{Duration, SystemTime, UNIX_EPOCH};
/// Compute an exponential backoff with ±25% jitter.
///
/// `attempt` is 0-based (first retry -> attempt=0 -> base=1s,
/// second retry -> attempt=1 -> base=2s, etc.).
/// `max_secs` sets the cap.
pub fn backoff_seconds(attempt: u32, max_secs: u64) -> Duration {
let base_secs = (2u64).pow(attempt).min(max_secs);
let quarter = (base_secs * 250_000_000).max(100_000_000); // 25% of base, min 100ms
let offset = jitter_ns(quarter);
// ±25%: offset in [0, quarter), so result = base - quarter/2 + offset
// which lies in [base - 25%, base + 25%).
let ns = base_secs * 1_000_000_000 + offset - quarter / 2;
Duration::from_nanos(ns)
}
/// Return a jitter offset in the range [0, range_ns).
fn jitter_ns(range_ns: u64) -> u64 {
let nanos = SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap_or_default()
.as_nanos() as u64;
nanos % range_ns
}
@@ -0,0 +1,3 @@
//! Utility modules for shared helpers.
pub mod backoff;
@@ -28,7 +28,7 @@ use std::sync::{
atomic::{AtomicBool, Ordering},
Arc, Mutex,
};
use std::time::{Duration, SystemTime, UNIX_EPOCH};
use std::time::Duration;
/// The lifecycle state of an agent within a workflow run.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
@@ -399,18 +399,9 @@ fn spawn_single_agent(sp: SpawnCtx<'_>) -> anyhow::Result<String> {
let bg_abort_thread = bg_abort.clone();
let bg_name_thread = bg_name.clone();
std::thread::spawn(move || {
// Retry wrapper: jittered backoff 1s → 2s.
let retry_backoff = |attempt: u32| {
let base_secs = (2u64).pow(attempt).min(8);
let quarter = (base_secs * 250_000_000).max(100_000_000);
let nanos = SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap_or_default()
.subsec_nanos() as u64;
let offset = nanos % quarter;
let ns = base_secs * 1_000_000_000 + offset - quarter / 2;
Duration::from_nanos(ns)
};
// Retry wrapper: jittered backoff with 8s cap.
let retry_backoff =
|attempt: u32| crate::app::util::backoff::backoff_seconds(attempt, 8);
for attempt in 1..=2 {
// Don't retry if aborted.
+5 -27
View File
@@ -26,9 +26,10 @@
use anyhow::Result;
use std::sync::atomic::AtomicBool;
use std::time::{Duration, SystemTime, UNIX_EPOCH};
use std::time::Duration;
use crate::app::runtime::stream::turn::StreamedTurn;
use crate::app::util::backoff::backoff_seconds;
use crate::app::runtime::stream::{SseParser, StreamEvent};
use crate::dto::chat::message::ChatMessage;
use crate::dto::provider::request::{ChatRequest, StreamOptions, ToolDef};
@@ -43,30 +44,11 @@ const REQUEST_TIMEOUT: Duration = Duration::from_mins(1);
// Retry helpers
// ---------------------------------------------------------------------------
/// Return a pseudo-random jitter offset in the range [0, range_ns).
///
/// Uses the full epoch nanoseconds (wrapped to u64) instead of the
/// sub-second component so the jitter range scales with `range_ns`
/// rather than being capped at ~1s.
fn jitter_ns(range_ns: u64) -> u64 {
let nanos = SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap_or_default()
.as_nanos() as u64;
nanos % range_ns
}
/// Exponential backoff with ±25% jitter, capped at 30 seconds.
///
/// `attempt` is 1-based (first retry → attempt=1).
fn backoff_duration(attempt: u32) -> Duration {
let base_secs = (2u64).pow(attempt).min(30);
let half_range = (base_secs * 250_000_000).max(100_000_000); // 25% of base, min 100ms
let offset = jitter_ns(half_range * 2); // [0, 50% of base)
// ±25% jitter: subtract half_range so the result varies
// between base-25% and base+25%.
let ns = base_secs * 1_000_000_000 + offset - half_range;
Duration::from_nanos(ns)
backoff_seconds(attempt, 30)
}
/// Is the error an auth / billing failure that retrying won't fix?
@@ -95,12 +77,8 @@ fn is_rate_limit(err_str: &str) -> bool {
/// Return a rate-appropriate backoff (longer for 429).
fn backoff_for_error(attempt: u32, err_str: &str) -> Duration {
if is_rate_limit(err_str) {
// Rate limits need more time to drain — start at 5s instead of 2s.
let base_secs = (5u64 * (2u64).pow(attempt.saturating_sub(1))).min(60);
let half_range = (base_secs * 250_000_000).max(100_000_000);
let offset = jitter_ns(half_range * 2);
let ns = base_secs * 1_000_000_000 + offset - half_range;
Duration::from_nanos(ns)
// Rate limits need more time to drain — backoff capped at 60s.
backoff_seconds(attempt, 60)
} else {
backoff_duration(attempt)
}