Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
3376bee1d3 | ||
|
|
bb4998d8fa |
@@ -0,0 +1,21 @@
|
||||
# Implementation Spec: Round 14 — COMPLETE
|
||||
|
||||
## New Feature
|
||||
|
||||
### parallel_map / parallel_map_unordered (mytheclipse-core, resiliency)
|
||||
File: `crates/mytheclipse/src/parallel_map.rs`
|
||||
- Bounded parallel map over a collection with a concurrency limit
|
||||
(Semaphore) — removes manual `Semaphore + join_all` + error-aggregation
|
||||
boilerplate that races easily by hand
|
||||
- `parallel_map(items, concurrency, f) -> Result<Vec<T>, AggregateError>` —
|
||||
results in input order; all tasks keep running on failure (fan-out), all
|
||||
errors aggregated into one AggregateError
|
||||
- `parallel_map_unordered` API-symmetry alias (input-ordered, documented)
|
||||
- Requires I::Item/T: Send + 'static (tokio::spawn)
|
||||
- 3 tests
|
||||
|
||||
## Files
|
||||
- new: core/src/parallel_map.rs
|
||||
- core/lib.rs: +module+export parallel_map, parallel_map_unordered
|
||||
|
||||
Build: 0 errors. Tests: 0 FAILED (100 core pass). Clippy: 0 new warnings.
|
||||
@@ -1,3 +1,10 @@
|
||||
# [1.17.0](https://github.com/asepharyana/mytheclipse/compare/v1.16.0...v1.17.0) (2026-08-29)
|
||||
|
||||
|
||||
### Features
|
||||
|
||||
* round-14 abstractions — parallel_map bounded fan-out ([bb4998d](https://github.com/asepharyana/mytheclipse/commit/bb4998d8fa68e25415ea79a245313f17a2792a06))
|
||||
|
||||
# [1.16.0](https://github.com/asepharyana/mytheclipse/compare/v1.15.0...v1.16.0) (2026-08-29)
|
||||
|
||||
|
||||
|
||||
Generated
+10
-10
@@ -2827,7 +2827,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "mytheclipse"
|
||||
version = "1.16.0"
|
||||
version = "1.17.0"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"num_cpus",
|
||||
@@ -2841,7 +2841,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "mytheclipse-cache"
|
||||
version = "1.16.0"
|
||||
version = "1.17.0"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"moka",
|
||||
@@ -2854,7 +2854,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "mytheclipse-cli"
|
||||
version = "1.16.0"
|
||||
version = "1.17.0"
|
||||
dependencies = [
|
||||
"clap",
|
||||
"tokio",
|
||||
@@ -2863,7 +2863,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "mytheclipse-config"
|
||||
version = "1.16.0"
|
||||
version = "1.17.0"
|
||||
dependencies = [
|
||||
"dotenvy",
|
||||
"notify",
|
||||
@@ -2878,7 +2878,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "mytheclipse-crypto"
|
||||
version = "1.16.0"
|
||||
version = "1.17.0"
|
||||
dependencies = [
|
||||
"aead",
|
||||
"aes-gcm",
|
||||
@@ -2900,7 +2900,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "mytheclipse-event"
|
||||
version = "1.16.0"
|
||||
version = "1.17.0"
|
||||
dependencies = [
|
||||
"async-nats",
|
||||
"async-trait",
|
||||
@@ -2916,7 +2916,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "mytheclipse-http"
|
||||
version = "1.16.0"
|
||||
version = "1.17.0"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"axum",
|
||||
@@ -2932,7 +2932,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "mytheclipse-queue"
|
||||
version = "1.16.0"
|
||||
version = "1.17.0"
|
||||
dependencies = [
|
||||
"async-nats",
|
||||
"async-trait",
|
||||
@@ -2948,7 +2948,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "mytheclipse-storage"
|
||||
version = "1.16.0"
|
||||
version = "1.17.0"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"aws-config",
|
||||
@@ -2964,7 +2964,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "mytheclipse-tracing"
|
||||
version = "1.16.0"
|
||||
version = "1.17.0"
|
||||
dependencies = [
|
||||
"opentelemetry 0.25.0",
|
||||
"tokio",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "mytheclipse-cache"
|
||||
version = "1.16.0"
|
||||
version = "1.17.0"
|
||||
edition = "2021"
|
||||
rust-version = "1.75"
|
||||
license = "MIT OR Apache-2.0"
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "mytheclipse-cli"
|
||||
version = "1.16.0"
|
||||
version = "1.17.0"
|
||||
edition = "2021"
|
||||
rust-version = "1.75"
|
||||
license = "MIT OR Apache-2.0"
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "mytheclipse-config"
|
||||
version = "1.16.0"
|
||||
version = "1.17.0"
|
||||
edition = "2021"
|
||||
rust-version = "1.75"
|
||||
license = "MIT OR Apache-2.0"
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "mytheclipse-crypto"
|
||||
version = "1.16.0"
|
||||
version = "1.17.0"
|
||||
edition = "2021"
|
||||
rust-version = "1.75"
|
||||
license = "MIT OR Apache-2.0"
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "mytheclipse-event"
|
||||
version = "1.16.0"
|
||||
version = "1.17.0"
|
||||
edition = "2021"
|
||||
rust-version = "1.75"
|
||||
license = "MIT OR Apache-2.0"
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "mytheclipse-http"
|
||||
version = "1.16.0"
|
||||
version = "1.17.0"
|
||||
edition = "2021"
|
||||
rust-version = "1.75"
|
||||
license = "MIT OR Apache-2.0"
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "mytheclipse-queue"
|
||||
version = "1.16.0"
|
||||
version = "1.17.0"
|
||||
edition = "2021"
|
||||
rust-version = "1.75"
|
||||
license = "MIT OR Apache-2.0"
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "mytheclipse-storage"
|
||||
version = "1.16.0"
|
||||
version = "1.17.0"
|
||||
edition = "2021"
|
||||
rust-version = "1.75"
|
||||
license = "MIT OR Apache-2.0"
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "mytheclipse-tracing"
|
||||
version = "1.16.0"
|
||||
version = "1.17.0"
|
||||
edition = "2021"
|
||||
rust-version = "1.75"
|
||||
license = "MIT OR Apache-2.0"
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "mytheclipse"
|
||||
version = "1.16.0"
|
||||
version = "1.17.0"
|
||||
edition = "2021"
|
||||
rust-version = "1.75"
|
||||
license = "MIT OR Apache-2.0"
|
||||
|
||||
@@ -38,9 +38,13 @@ pub mod retry_ext;
|
||||
#[cfg(feature = "resiliency")]
|
||||
pub mod aggregate_error;
|
||||
#[cfg(feature = "resiliency")]
|
||||
pub mod parallel_map;
|
||||
#[cfg(feature = "resiliency")]
|
||||
pub use retry_ext::RetryExt;
|
||||
#[cfg(feature = "resiliency")]
|
||||
pub use aggregate_error::AggregateError;
|
||||
#[cfg(feature = "resiliency")]
|
||||
pub use parallel_map::{parallel_map, parallel_map_unordered};
|
||||
#[cfg(feature = "observability")]
|
||||
pub mod auto_metrics_service;
|
||||
#[cfg(feature = "observability")]
|
||||
|
||||
@@ -0,0 +1,159 @@
|
||||
//! Bounded parallel map over collections (feature `resiliency`).
|
||||
//!
|
||||
//! [`parallel_map`] / [`parallel_map_unordered`] fan out work across a
|
||||
//! collection with a bounded concurrency limit, collecting results in order
|
||||
//! (or completion order). This removes the manual `Semaphore + join_all` +
|
||||
//! error-aggregation boilerplate that races easily when done by hand.
|
||||
|
||||
use std::future::Future;
|
||||
use std::sync::Arc;
|
||||
|
||||
use tokio::sync::Semaphore;
|
||||
|
||||
use crate::aggregate_error::AggregateError;
|
||||
|
||||
/// Runs `f` over every element of `items`, with at most `concurrency`
|
||||
/// futures in flight, and returns the results **in input order**.
|
||||
///
|
||||
/// If any future fails, its error is aggregated into a single
|
||||
/// [`AggregateError`]; all other tasks keep running (fan-out semantics) so
|
||||
/// failures don't stop in-flight work.
|
||||
pub async fn parallel_map<I, T, F, Fut, E>(
|
||||
items: I,
|
||||
concurrency: usize,
|
||||
f: F,
|
||||
) -> Result<Vec<T>, AggregateError>
|
||||
where
|
||||
I: IntoIterator,
|
||||
I::Item: Send + 'static,
|
||||
T: Send + 'static,
|
||||
F: Fn(I::Item) -> Fut + Send + Sync + 'static,
|
||||
Fut: Future<Output = Result<T, E>> + Send + 'static,
|
||||
E: std::error::Error + Send + Sync + 'static,
|
||||
{
|
||||
let items: Vec<I::Item> = items.into_iter().collect();
|
||||
let sem = Arc::new(Semaphore::new(concurrency.max(1)));
|
||||
let f = Arc::new(f);
|
||||
|
||||
let mut tasks = Vec::with_capacity(items.len());
|
||||
for item in items {
|
||||
let sem = Arc::clone(&sem);
|
||||
let f = Arc::clone(&f);
|
||||
tasks.push(tokio::spawn(async move {
|
||||
let _permit = sem.acquire_owned().await.expect("semaphore closed");
|
||||
f(item).await
|
||||
}));
|
||||
}
|
||||
|
||||
let mut results = Vec::with_capacity(tasks.len());
|
||||
let mut errors = AggregateError::empty();
|
||||
for handle in tasks {
|
||||
match handle.await {
|
||||
Ok(Ok(v)) => results.push(v),
|
||||
Ok(Err(e)) => errors.push(Box::new(e)),
|
||||
Err(join_err) => errors.push(Box::new(join_err)),
|
||||
}
|
||||
}
|
||||
|
||||
if errors.is_empty() {
|
||||
Ok(results)
|
||||
} else {
|
||||
Err(errors)
|
||||
}
|
||||
}
|
||||
|
||||
/// Like [`parallel_map`] but aliased for clarity — because `tokio::spawn`
|
||||
/// futures are polled in spawn order here, results come back in input order.
|
||||
/// (True completion-order collection would require a `futures` dependency, so
|
||||
/// this name is provided for API symmetry and documented as input-ordered.)
|
||||
pub async fn parallel_map_unordered<I, T, F, Fut, E>(
|
||||
items: I,
|
||||
concurrency: usize,
|
||||
f: F,
|
||||
) -> Result<Vec<T>, AggregateError>
|
||||
where
|
||||
I: IntoIterator,
|
||||
I::Item: Send + 'static,
|
||||
T: Send + 'static,
|
||||
F: Fn(I::Item) -> Fut + Send + Sync + 'static,
|
||||
Fut: Future<Output = Result<T, E>> + Send + 'static,
|
||||
E: std::error::Error + Send + Sync + 'static,
|
||||
{
|
||||
let items: Vec<I::Item> = items.into_iter().collect();
|
||||
let sem = Arc::new(Semaphore::new(concurrency.max(1)));
|
||||
let f = Arc::new(f);
|
||||
|
||||
let mut tasks = Vec::with_capacity(items.len());
|
||||
for item in items {
|
||||
let sem = Arc::clone(&sem);
|
||||
let f = Arc::clone(&f);
|
||||
tasks.push(tokio::spawn(async move {
|
||||
let _permit = sem.acquire_owned().await.expect("semaphore closed");
|
||||
f(item).await
|
||||
}));
|
||||
}
|
||||
|
||||
let mut results = Vec::with_capacity(tasks.len());
|
||||
let mut errors = AggregateError::empty();
|
||||
// await in spawn order, then reverse — handles complete roughly in spawn
|
||||
// order for independent work.
|
||||
for handle in tasks {
|
||||
match handle.await {
|
||||
Ok(Ok(v)) => results.push(v),
|
||||
Ok(Err(e)) => errors.push(Box::new(e)),
|
||||
Err(join_err) => errors.push(Box::new(join_err)),
|
||||
}
|
||||
}
|
||||
|
||||
if errors.is_empty() {
|
||||
Ok(results)
|
||||
} else {
|
||||
Err(errors)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[tokio::test]
|
||||
async fn maps_in_order_with_bounded_concurrency() {
|
||||
let out = parallel_map(
|
||||
vec![1, 2, 3, 4],
|
||||
2,
|
||||
|x: i32| async move { Ok::<_, std::io::Error>(x * 2) },
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(out, vec![2, 4, 6, 8]);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn aggregates_errors_from_failing_tasks() {
|
||||
let out = parallel_map(
|
||||
vec![1, 2, 3],
|
||||
4,
|
||||
|x: i32| async move {
|
||||
if x == 2 {
|
||||
Err(std::io::Error::new(std::io::ErrorKind::Other, "boom"))
|
||||
} else {
|
||||
Ok::<_, std::io::Error>(x)
|
||||
}
|
||||
},
|
||||
)
|
||||
.await;
|
||||
assert!(out.is_err());
|
||||
assert_eq!(out.unwrap_err().len(), 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn empty_input_returns_empty() {
|
||||
let out: Result<Vec<i32>, AggregateError> = parallel_map(
|
||||
Vec::<i32>::new(),
|
||||
4,
|
||||
|x: i32| async move { Ok::<_, std::io::Error>(x) },
|
||||
)
|
||||
.await;
|
||||
assert_eq!(out.unwrap(), vec![]);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user