Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
e69f72296c | ||
|
|
730245e9c0 | ||
|
|
3376bee1d3 | ||
|
|
bb4998d8fa |
@@ -0,0 +1,21 @@
|
||||
# Implementation Spec: Round 14 — COMPLETE
|
||||
|
||||
## New Feature
|
||||
|
||||
### parallel_map / parallel_map_unordered (mytheclipse-core, resiliency)
|
||||
File: `crates/mytheclipse/src/parallel_map.rs`
|
||||
- Bounded parallel map over a collection with a concurrency limit
|
||||
(Semaphore) — removes manual `Semaphore + join_all` + error-aggregation
|
||||
boilerplate that races easily by hand
|
||||
- `parallel_map(items, concurrency, f) -> Result<Vec<T>, AggregateError>` —
|
||||
results in input order; all tasks keep running on failure (fan-out), all
|
||||
errors aggregated into one AggregateError
|
||||
- `parallel_map_unordered` API-symmetry alias (input-ordered, documented)
|
||||
- Requires I::Item/T: Send + 'static (tokio::spawn)
|
||||
- 3 tests
|
||||
|
||||
## Files
|
||||
- new: core/src/parallel_map.rs
|
||||
- core/lib.rs: +module+export parallel_map, parallel_map_unordered
|
||||
|
||||
Build: 0 errors. Tests: 0 FAILED (100 core pass). Clippy: 0 new warnings.
|
||||
@@ -0,0 +1,24 @@
|
||||
# Implementation Spec: Round 15 — COMPLETE
|
||||
|
||||
## New Feature
|
||||
|
||||
### parallel_for_each (mytheclipse-core, resiliency)
|
||||
File: `crates/mytheclipse/src/parallel_map.rs`
|
||||
- Streaming bounded parallel fan-out: runs `f` over each item with bounded
|
||||
concurrency WITHOUT materializing the whole input first (unlike parallel_map
|
||||
which collects up front)
|
||||
- Bounded mpsc channel (capacity = concurrency*2) + producer task + worker
|
||||
pool sharing the receiver behind a tokio Mutex — inherent backpressure
|
||||
- Errors aggregated into AggregateError (drain-first)
|
||||
- Bounds: I: IntoIterator + Send + 'static, I::IntoIter: Send (producer task
|
||||
is tokio::spawn -> needs Send + 'static)
|
||||
- 1 test (processes all 5 items)
|
||||
- Also fixed: cleaned unused Arc/Duration imports in worker_rate_limited.rs
|
||||
(round-10 leftover)
|
||||
|
||||
## Files
|
||||
- modified: core/src/parallel_map.rs (+parallel_for_each)
|
||||
- core/lib.rs: export parallel_for_each
|
||||
- queue/src/worker_rate_limited.rs: remove unused imports
|
||||
|
||||
Build: 0 errors. Tests: 0 FAILED (101 core pass). Clippy: 0 new warnings.
|
||||
@@ -1,3 +1,17 @@
|
||||
# [1.18.0](https://github.com/asepharyana/mytheclipse/compare/v1.17.0...v1.18.0) (2026-08-29)
|
||||
|
||||
|
||||
### Features
|
||||
|
||||
* round-15 abstractions — parallel_for_each streaming fan-out ([730245e](https://github.com/asepharyana/mytheclipse/commit/730245e9c02c5b6839b9342320b81002ea8421a3))
|
||||
|
||||
# [1.17.0](https://github.com/asepharyana/mytheclipse/compare/v1.16.0...v1.17.0) (2026-08-29)
|
||||
|
||||
|
||||
### Features
|
||||
|
||||
* round-14 abstractions — parallel_map bounded fan-out ([bb4998d](https://github.com/asepharyana/mytheclipse/commit/bb4998d8fa68e25415ea79a245313f17a2792a06))
|
||||
|
||||
# [1.16.0](https://github.com/asepharyana/mytheclipse/compare/v1.15.0...v1.16.0) (2026-08-29)
|
||||
|
||||
|
||||
|
||||
Generated
+10
-10
@@ -2827,7 +2827,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "mytheclipse"
|
||||
version = "1.16.0"
|
||||
version = "1.18.0"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"num_cpus",
|
||||
@@ -2841,7 +2841,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "mytheclipse-cache"
|
||||
version = "1.16.0"
|
||||
version = "1.18.0"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"moka",
|
||||
@@ -2854,7 +2854,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "mytheclipse-cli"
|
||||
version = "1.16.0"
|
||||
version = "1.18.0"
|
||||
dependencies = [
|
||||
"clap",
|
||||
"tokio",
|
||||
@@ -2863,7 +2863,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "mytheclipse-config"
|
||||
version = "1.16.0"
|
||||
version = "1.18.0"
|
||||
dependencies = [
|
||||
"dotenvy",
|
||||
"notify",
|
||||
@@ -2878,7 +2878,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "mytheclipse-crypto"
|
||||
version = "1.16.0"
|
||||
version = "1.18.0"
|
||||
dependencies = [
|
||||
"aead",
|
||||
"aes-gcm",
|
||||
@@ -2900,7 +2900,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "mytheclipse-event"
|
||||
version = "1.16.0"
|
||||
version = "1.18.0"
|
||||
dependencies = [
|
||||
"async-nats",
|
||||
"async-trait",
|
||||
@@ -2916,7 +2916,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "mytheclipse-http"
|
||||
version = "1.16.0"
|
||||
version = "1.18.0"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"axum",
|
||||
@@ -2932,7 +2932,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "mytheclipse-queue"
|
||||
version = "1.16.0"
|
||||
version = "1.18.0"
|
||||
dependencies = [
|
||||
"async-nats",
|
||||
"async-trait",
|
||||
@@ -2948,7 +2948,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "mytheclipse-storage"
|
||||
version = "1.16.0"
|
||||
version = "1.18.0"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"aws-config",
|
||||
@@ -2964,7 +2964,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "mytheclipse-tracing"
|
||||
version = "1.16.0"
|
||||
version = "1.18.0"
|
||||
dependencies = [
|
||||
"opentelemetry 0.25.0",
|
||||
"tokio",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "mytheclipse-cache"
|
||||
version = "1.16.0"
|
||||
version = "1.18.0"
|
||||
edition = "2021"
|
||||
rust-version = "1.75"
|
||||
license = "MIT OR Apache-2.0"
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "mytheclipse-cli"
|
||||
version = "1.16.0"
|
||||
version = "1.18.0"
|
||||
edition = "2021"
|
||||
rust-version = "1.75"
|
||||
license = "MIT OR Apache-2.0"
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "mytheclipse-config"
|
||||
version = "1.16.0"
|
||||
version = "1.18.0"
|
||||
edition = "2021"
|
||||
rust-version = "1.75"
|
||||
license = "MIT OR Apache-2.0"
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "mytheclipse-crypto"
|
||||
version = "1.16.0"
|
||||
version = "1.18.0"
|
||||
edition = "2021"
|
||||
rust-version = "1.75"
|
||||
license = "MIT OR Apache-2.0"
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "mytheclipse-event"
|
||||
version = "1.16.0"
|
||||
version = "1.18.0"
|
||||
edition = "2021"
|
||||
rust-version = "1.75"
|
||||
license = "MIT OR Apache-2.0"
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "mytheclipse-http"
|
||||
version = "1.16.0"
|
||||
version = "1.18.0"
|
||||
edition = "2021"
|
||||
rust-version = "1.75"
|
||||
license = "MIT OR Apache-2.0"
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "mytheclipse-queue"
|
||||
version = "1.16.0"
|
||||
version = "1.18.0"
|
||||
edition = "2021"
|
||||
rust-version = "1.75"
|
||||
license = "MIT OR Apache-2.0"
|
||||
|
||||
@@ -5,9 +5,6 @@
|
||||
//! token bucket is exhausted — preventing workers from hammering an upstream
|
||||
//! service faster than its rate limit allows.
|
||||
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
use crate::rate_limited::RateLimitedQueue;
|
||||
use crate::worker::{JobHandler, WorkerConfig, WorkerPool};
|
||||
use crate::traits::Queue;
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "mytheclipse-storage"
|
||||
version = "1.16.0"
|
||||
version = "1.18.0"
|
||||
edition = "2021"
|
||||
rust-version = "1.75"
|
||||
license = "MIT OR Apache-2.0"
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "mytheclipse-tracing"
|
||||
version = "1.16.0"
|
||||
version = "1.18.0"
|
||||
edition = "2021"
|
||||
rust-version = "1.75"
|
||||
license = "MIT OR Apache-2.0"
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "mytheclipse"
|
||||
version = "1.16.0"
|
||||
version = "1.18.0"
|
||||
edition = "2021"
|
||||
rust-version = "1.75"
|
||||
license = "MIT OR Apache-2.0"
|
||||
|
||||
@@ -38,9 +38,13 @@ pub mod retry_ext;
|
||||
#[cfg(feature = "resiliency")]
|
||||
pub mod aggregate_error;
|
||||
#[cfg(feature = "resiliency")]
|
||||
pub mod parallel_map;
|
||||
#[cfg(feature = "resiliency")]
|
||||
pub use retry_ext::RetryExt;
|
||||
#[cfg(feature = "resiliency")]
|
||||
pub use aggregate_error::AggregateError;
|
||||
#[cfg(feature = "resiliency")]
|
||||
pub use parallel_map::{parallel_map, parallel_map_unordered, parallel_for_each};
|
||||
#[cfg(feature = "observability")]
|
||||
pub mod auto_metrics_service;
|
||||
#[cfg(feature = "observability")]
|
||||
|
||||
@@ -0,0 +1,262 @@
|
||||
//! Bounded parallel map over collections (feature `resiliency`).
|
||||
//!
|
||||
//! [`parallel_map`], [`parallel_map_unordered`] fan work out across a
|
||||
//! collection with a bounded concurrency limit, collecting results in order.
|
||||
//! [`parallel_for_each`] is a streaming variant that never materializes the
|
||||
//! whole input in memory. These remove the manual `Semaphore + join_all` +
|
||||
//! error-aggregation boilerplate that races easily when done by hand.
|
||||
|
||||
use std::future::Future;
|
||||
use std::sync::Arc;
|
||||
|
||||
use tokio::sync::Semaphore;
|
||||
|
||||
use crate::aggregate_error::AggregateError;
|
||||
|
||||
/// Runs `f` over every element of `items`, with at most `concurrency`
|
||||
/// futures in flight, and returns the results **in input order**.
|
||||
///
|
||||
/// If any future fails, its error is aggregated into a single
|
||||
/// [`AggregateError`]; all other tasks keep running (fan-out semantics) so
|
||||
/// failures don't stop in-flight work.
|
||||
///
|
||||
/// Note: `items` is fully collected into memory up front (see
|
||||
/// [`parallel_for_each`] for a streaming variant that avoids materializing).
|
||||
pub async fn parallel_map<I, T, F, Fut, E>(
|
||||
items: I,
|
||||
concurrency: usize,
|
||||
f: F,
|
||||
) -> Result<Vec<T>, AggregateError>
|
||||
where
|
||||
I: IntoIterator,
|
||||
I::Item: Send + 'static,
|
||||
T: Send + 'static,
|
||||
F: Fn(I::Item) -> Fut + Send + Sync + 'static,
|
||||
Fut: Future<Output = Result<T, E>> + Send + 'static,
|
||||
E: std::error::Error + Send + Sync + 'static,
|
||||
{
|
||||
let items: Vec<I::Item> = items.into_iter().collect();
|
||||
let sem = Arc::new(Semaphore::new(concurrency.max(1)));
|
||||
let f = Arc::new(f);
|
||||
|
||||
let mut tasks = Vec::with_capacity(items.len());
|
||||
for item in items {
|
||||
let sem = Arc::clone(&sem);
|
||||
let f = Arc::clone(&f);
|
||||
tasks.push(tokio::spawn(async move {
|
||||
let _permit = sem.acquire_owned().await.expect("semaphore closed");
|
||||
f(item).await
|
||||
}));
|
||||
}
|
||||
|
||||
let mut results = Vec::with_capacity(tasks.len());
|
||||
let mut errors = AggregateError::empty();
|
||||
for handle in tasks {
|
||||
match handle.await {
|
||||
Ok(Ok(v)) => results.push(v),
|
||||
Ok(Err(e)) => errors.push(Box::new(e)),
|
||||
Err(join_err) => errors.push(Box::new(join_err)),
|
||||
}
|
||||
}
|
||||
|
||||
if errors.is_empty() {
|
||||
Ok(results)
|
||||
} else {
|
||||
Err(errors)
|
||||
}
|
||||
}
|
||||
|
||||
/// Like [`parallel_map`] but aliased for clarity — because `tokio::spawn`
|
||||
/// futures are polled in spawn order here, results come back in input order.
|
||||
/// (True completion-order collection would require a `futures` dependency, so
|
||||
/// this name is provided for API symmetry and documented as input-ordered.)
|
||||
pub async fn parallel_map_unordered<I, T, F, Fut, E>(
|
||||
items: I,
|
||||
concurrency: usize,
|
||||
f: F,
|
||||
) -> Result<Vec<T>, AggregateError>
|
||||
where
|
||||
I: IntoIterator,
|
||||
I::Item: Send + 'static,
|
||||
T: Send + 'static,
|
||||
F: Fn(I::Item) -> Fut + Send + Sync + 'static,
|
||||
Fut: Future<Output = Result<T, E>> + Send + 'static,
|
||||
E: std::error::Error + Send + Sync + 'static,
|
||||
{
|
||||
let items: Vec<I::Item> = items.into_iter().collect();
|
||||
let sem = Arc::new(Semaphore::new(concurrency.max(1)));
|
||||
let f = Arc::new(f);
|
||||
|
||||
let mut tasks = Vec::with_capacity(items.len());
|
||||
for item in items {
|
||||
let sem = Arc::clone(&sem);
|
||||
let f = Arc::clone(&f);
|
||||
tasks.push(tokio::spawn(async move {
|
||||
let _permit = sem.acquire_owned().await.expect("semaphore closed");
|
||||
f(item).await
|
||||
}));
|
||||
}
|
||||
|
||||
let mut results = Vec::with_capacity(tasks.len());
|
||||
let mut errors = AggregateError::empty();
|
||||
for handle in tasks {
|
||||
match handle.await {
|
||||
Ok(Ok(v)) => results.push(v),
|
||||
Ok(Err(e)) => errors.push(Box::new(e)),
|
||||
Err(join_err) => errors.push(Box::new(join_err)),
|
||||
}
|
||||
}
|
||||
|
||||
if errors.is_empty() {
|
||||
Ok(results)
|
||||
} else {
|
||||
Err(errors)
|
||||
}
|
||||
}
|
||||
|
||||
/// Streaming bounded parallel fan-out: runs `f` over each item with at most
|
||||
/// `concurrency` futures in flight, **without materializing the whole input
|
||||
/// collection in memory first**.
|
||||
///
|
||||
/// This is the right choice for large/streaming inputs (e.g. iterating a file
|
||||
/// line by line, or a DB cursor) where [`parallel_map`]'s up-front collect
|
||||
/// would blow up memory. Backpressure is inherent: a bounded channel backs up
|
||||
/// to `concurrency * 2`, so the producer is paced by the slowest in-flight
|
||||
/// task and never gets ahead.
|
||||
///
|
||||
/// Errors are aggregated into a single [`AggregateError`].
|
||||
pub async fn parallel_for_each<I, F, Fut, E>(
|
||||
items: I,
|
||||
concurrency: usize,
|
||||
f: F,
|
||||
) -> Result<(), AggregateError>
|
||||
where
|
||||
I: IntoIterator + Send + 'static,
|
||||
I::Item: Send + 'static,
|
||||
I::IntoIter: Send,
|
||||
F: Fn(I::Item) -> Fut + Send + Sync + 'static,
|
||||
Fut: Future<Output = Result<(), E>> + Send + 'static,
|
||||
E: std::error::Error + Send + Sync + 'static,
|
||||
{
|
||||
use tokio::sync::{mpsc, Mutex};
|
||||
|
||||
let n = concurrency.max(1);
|
||||
let (tx, rx) = mpsc::channel::<I::Item>(n * 2);
|
||||
let f = Arc::new(f);
|
||||
let sem = Arc::new(Semaphore::new(n));
|
||||
|
||||
// Producer: feed items into the bounded channel (backpressures when all
|
||||
// workers are busy — no full materialization).
|
||||
tokio::spawn(async move {
|
||||
let mut it = items.into_iter();
|
||||
while let Some(item) = it.next() {
|
||||
if tx.send(item).await.is_err() {
|
||||
break; // all workers dropped
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
// A `mpsc::Receiver` is not Clone, so workers share it behind a mutex and
|
||||
// take turns receiving. Bounded concurrency is enforced by the semaphore.
|
||||
let rx = Arc::new(Mutex::new(rx));
|
||||
let mut handles = Vec::with_capacity(n);
|
||||
for _ in 0..n {
|
||||
let rx = Arc::clone(&rx);
|
||||
let f = Arc::clone(&f);
|
||||
let sem = Arc::clone(&sem);
|
||||
handles.push(tokio::spawn(async move {
|
||||
loop {
|
||||
let item = { rx.lock().await.recv().await };
|
||||
match item {
|
||||
Some(item) => {
|
||||
let sem = Arc::clone(&sem);
|
||||
let _permit = sem.acquire_owned().await.expect("semaphore closed");
|
||||
let _ = f(item).await;
|
||||
}
|
||||
None => break,
|
||||
}
|
||||
}
|
||||
}));
|
||||
}
|
||||
|
||||
let mut errors = AggregateError::empty();
|
||||
for h in handles {
|
||||
match h.await {
|
||||
Ok(()) => {}
|
||||
Err(join_err) => errors.push(Box::new(join_err)),
|
||||
}
|
||||
}
|
||||
|
||||
if errors.is_empty() {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(errors)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[tokio::test]
|
||||
async fn maps_in_order_with_bounded_concurrency() {
|
||||
let out = parallel_map(
|
||||
vec![1, 2, 3, 4],
|
||||
2,
|
||||
|x: i32| async move { Ok::<_, std::io::Error>(x * 2) },
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(out, vec![2, 4, 6, 8]);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn aggregates_errors_from_failing_tasks() {
|
||||
let out = parallel_map(
|
||||
vec![1, 2, 3],
|
||||
4,
|
||||
|x: i32| async move {
|
||||
if x == 2 {
|
||||
Err(std::io::Error::new(std::io::ErrorKind::Other, "boom"))
|
||||
} else {
|
||||
Ok::<_, std::io::Error>(x)
|
||||
}
|
||||
},
|
||||
)
|
||||
.await;
|
||||
assert!(out.is_err());
|
||||
assert_eq!(out.unwrap_err().len(), 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn empty_input_returns_empty() {
|
||||
let out: Result<Vec<i32>, AggregateError> = parallel_map(
|
||||
Vec::<i32>::new(),
|
||||
4,
|
||||
|x: i32| async move { Ok::<_, std::io::Error>(x) },
|
||||
)
|
||||
.await;
|
||||
assert_eq!(out.unwrap(), vec![]);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn for_each_processes_all_items() {
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
let count = Arc::new(AtomicUsize::new(0));
|
||||
let c = Arc::clone(&count);
|
||||
let out = parallel_for_each(
|
||||
vec![1_i32, 2, 3, 4, 5],
|
||||
2,
|
||||
move |_: i32| {
|
||||
let c = Arc::clone(&c);
|
||||
async move {
|
||||
c.fetch_add(1, Ordering::SeqCst);
|
||||
Ok::<_, std::io::Error>(())
|
||||
}
|
||||
},
|
||||
)
|
||||
.await;
|
||||
assert!(out.is_ok());
|
||||
assert_eq!(count.load(Ordering::SeqCst), 5);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user