feat(ml-service): add Prometheus metrics and request telemetry instrumentation

apps/ml-service/.gitignore

@@ -1,3 +1,6 @@
 .venv/
 __pycache__/
-target/
+target/
+.claude/
+
+.codegraph/

apps/ml-service/Cargo.lock

@@ -111,7 +111,7 @@ dependencies = [
  "num-traits",
  "pastey",
  "rayon",
- "thiserror",
+ "thiserror 2.0.18",
  "v_frame",
  "y4m",
 ]
@@ -402,7 +402,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb"
 dependencies = [
  "libc",
- "windows-sys",
+ "windows-sys 0.61.2",
 ]
 
 [[package]]
@@ -457,6 +457,12 @@ dependencies = [
  "miniz_oxide",
 ]
 
+[[package]]
+name = "fnv"
+version = "1.0.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
+
 [[package]]
 name = "foldhash"
 version = "0.1.5"
@@ -587,6 +593,12 @@ version = "0.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
 
+[[package]]
+name = "hex"
+version = "0.4.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70"
+
 [[package]]
 name = "hmac-sha256"
 version = "1.1.14"
@@ -801,6 +813,12 @@ dependencies = [
  "cc",
 ]
 
+[[package]]
+name = "linux-raw-sys"
+version = "0.4.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab"
+
 [[package]]
 name = "linux-raw-sys"
 version = "0.12.1"
@@ -902,7 +920,7 @@ checksum = "50b7e5b27aa02a74bac8c3f23f448f8d87ff11f92d3aac1a6ed369ee08cc56c1"
 dependencies = [
  "libc",
  "wasi",
- "windows-sys",
+ "windows-sys 0.61.2",
 ]
 
 [[package]]
@@ -1000,7 +1018,7 @@ version = "0.50.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5"
 dependencies = [
- "windows-sys",
+ "windows-sys 0.61.2",
 ]
 
 [[package]]
@@ -1253,6 +1271,28 @@ dependencies = [
  "unicode-ident",
 ]
 
+[[package]]
+name = "procfs"
+version = "0.17.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cc5b72d8145275d844d4b5f6d4e1eef00c8cd889edb6035c21675d1bb1f45c9f"
+dependencies = [
+ "bitflags",
+ "hex",
+ "procfs-core",
+ "rustix 0.38.44",
+]
+
+[[package]]
+name = "procfs-core"
+version = "0.17.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "239df02d8349b06fc07398a3a1697b06418223b1c7725085e801e7c0fc6a12ec"
+dependencies = [
+ "bitflags",
+ "hex",
+]
+
 [[package]]
 name = "profiling"
 version = "1.0.18"
@@ -1272,6 +1312,43 @@ dependencies = [
  "syn",
 ]
 
+[[package]]
+name = "prometheus"
+version = "0.14.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3ca5326d8d0b950a9acd87e6a3f94745394f62e4dae1b1ee22b2bc0c394af43a"
+dependencies = [
+ "cfg-if",
+ "fnv",
+ "lazy_static",
+ "libc",
+ "memchr",
+ "parking_lot",
+ "procfs",
+ "protobuf",
+ "thiserror 2.0.18",
+]
+
+[[package]]
+name = "protobuf"
+version = "3.7.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d65a1d4ddae7d8b5de68153b48f6aa3bba8cb002b243dbdbc55a5afbc98f99f4"
+dependencies = [
+ "once_cell",
+ "protobuf-support",
+ "thiserror 1.0.69",
+]
+
+[[package]]
+name = "protobuf-support"
+version = "3.7.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3e36c2f31e0a47f9280fb347ef5e461ffcd2c52dd520d8e216b52f93b0b0d7d6"
+dependencies = [
+ "thiserror 1.0.69",
+]
+
 [[package]]
 name = "pxfm"
 version = "0.1.29"
@@ -1373,7 +1450,7 @@ dependencies = [
  "rand",
  "rand_chacha",
  "simd_helpers",
- "thiserror",
+ "thiserror 2.0.18",
  "v_frame",
  "wasm-bindgen",
 ]
@@ -1451,6 +1528,19 @@ version = "0.8.53"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "47b34b781b31e5d73e9fbc8689c70551fd1ade9a19e3e28cfec8580a79290cc4"
 
+[[package]]
+name = "rustix"
+version = "0.38.44"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154"
+dependencies = [
+ "bitflags",
+ "errno",
+ "libc",
+ "linux-raw-sys 0.4.15",
+ "windows-sys 0.59.0",
+]
+
 [[package]]
 name = "rustix"
 version = "1.1.4"
@@ -1460,8 +1550,8 @@ dependencies = [
  "bitflags",
  "errno",
  "libc",
- "linux-raw-sys",
+ "linux-raw-sys 0.12.1",
- "windows-sys",
+ "windows-sys 0.61.2",
 ]
 
 [[package]]
@@ -1491,7 +1581,7 @@ version = "0.1.29"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "91c1b7e4904c873ef0710c1f407dde2e6287de2bebc1bbbf7d430bb7cbffd939"
 dependencies = [
- "windows-sys",
+ "windows-sys 0.61.2",
 ]
 
 [[package]]
@@ -1644,7 +1734,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e"
 dependencies = [
  "libc",
- "windows-sys",
+ "windows-sys 0.61.2",
 ]
 
 [[package]]
@@ -1705,8 +1795,17 @@ dependencies = [
  "fastrand",
  "getrandom 0.4.2",
  "once_cell",
- "rustix",
+ "rustix 1.1.4",
- "windows-sys",
+ "windows-sys 0.61.2",
+]
+
+[[package]]
+name = "thiserror"
+version = "1.0.69"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52"
+dependencies = [
+ "thiserror-impl 1.0.69",
 ]
 
 [[package]]
@@ -1715,7 +1814,18 @@ version = "2.0.18"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4"
 dependencies = [
- "thiserror-impl",
+ "thiserror-impl 2.0.18",
+]
+
+[[package]]
+name = "thiserror-impl"
+version = "1.0.69"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
 ]
 
 [[package]]
@@ -1763,7 +1873,7 @@ dependencies = [
  "pin-project-lite",
  "socket2",
  "tokio-macros",
- "windows-sys",
+ "windows-sys 0.61.2",
 ]
 
 [[package]]
@@ -2090,6 +2200,15 @@ version = "0.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5"
 
+[[package]]
+name = "windows-sys"
+version = "0.59.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b"
+dependencies = [
+ "windows-targets",
+]
+
 [[package]]
 name = "windows-sys"
 version = "0.61.2"
@@ -2099,6 +2218,70 @@ dependencies = [
  "windows-link",
 ]
 
+[[package]]
+name = "windows-targets"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973"
+dependencies = [
+ "windows_aarch64_gnullvm",
+ "windows_aarch64_msvc",
+ "windows_i686_gnu",
+ "windows_i686_gnullvm",
+ "windows_i686_msvc",
+ "windows_x86_64_gnu",
+ "windows_x86_64_gnullvm",
+ "windows_x86_64_msvc",
+]
+
+[[package]]
+name = "windows_aarch64_gnullvm"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3"
+
+[[package]]
+name = "windows_aarch64_msvc"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469"
+
+[[package]]
+name = "windows_i686_gnu"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b"
+
+[[package]]
+name = "windows_i686_gnullvm"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66"
+
+[[package]]
+name = "windows_i686_msvc"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66"
+
+[[package]]
+name = "windows_x86_64_gnu"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78"
+
+[[package]]
+name = "windows_x86_64_gnullvm"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d"
+
+[[package]]
+name = "windows_x86_64_msvc"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"
+
 [[package]]
 name = "wit-bindgen"
 version = "0.51.0"
@@ -2208,6 +2391,7 @@ dependencies = [
  "image",
  "ndarray",
  "ort",
+ "prometheus",
  "serde",
  "serde_json",
  "temp-env",

apps/ml-service/Cargo.toml

@@ -9,6 +9,7 @@ axum = { version = "0.7", features = ["multipart"] }
 image = "0.25"
 ndarray = "0.17"
 ort = { version = "2.0.0-rc.10", features = ["download-binaries", "ndarray"] }
+prometheus = { version = "0.14.0", features = ["process"] }
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
 tokio = { version = "1.0", features = ["macros", "rt-multi-thread", "net"] }

apps/ml-service/src/main.rs

@@ -3,6 +3,7 @@ mod error;
 mod image;
 mod model;
 mod routes;
+mod telemetry;
 
 use anyhow::Result;
 use config::Config;
@@ -32,6 +33,9 @@ async fn main() -> Result<()> {
         "Model service initialized"
     );
 
+    // Set model load status metric
+    telemetry::model_load_status().set(if model.is_loaded() { 1.0 } else { 0.0 });
+
     // Create AppState
     let state = AppState { model };
 

apps/ml-service/src/routes.rs

@@ -3,6 +3,8 @@ use crate::config::{LABELS, SERVICE_NAME, SERVICE_VERSION};
 use crate::model::{ModelService, Prediction};
 use crate::error::ServiceError;
 use crate::image::preprocess_image;
+use crate::telemetry;
+use crate::telemetry::RequestMetricsGuard;
 use axum::{
     extract::{State, Multipart},
     routing::{get, post},
@@ -64,22 +66,34 @@ pub fn prediction_response(prediction: Prediction) -> PredictionResponse {
     }
 }
 
+pub async fn metrics() -> (axum::http::StatusCode, String) {
+    (axum::http::StatusCode::OK, telemetry::encode_metrics())
+}
+
 pub async fn health(State(state): State<AppState>) -> Json<HealthResponse> {
-    Json(health_response(state.model.is_loaded()))
+    let _guard = RequestMetricsGuard::new();
+    let res = health_response(state.model.is_loaded());
+    _guard.finish();
+    Json(res)
 }
 
 pub async fn metadata(State(state): State<AppState>) -> Json<MetadataResponse> {
-    Json(metadata_response(
+    let _guard = RequestMetricsGuard::new();
+    let res = metadata_response(
         state.model.model_path().to_string_lossy().to_string(),
         state.model.is_loaded(),
         state.model.input_size(),
-    ))
+    );
+    _guard.finish();
+    Json(res)
 }
 
 pub async fn predict(
     State(state): State<AppState>,
     mut multipart: Multipart,
 ) -> Result<Json<PredictionResponse>, ServiceError> {
+    let _guard = RequestMetricsGuard::new();
+
     // Extract the file field from multipart
     let mut file_data = None;
     while let Ok(Some(field)) = multipart.next_field().await {
@@ -121,6 +135,10 @@ pub async fn predict(
     // Run prediction
     let prediction = state.model.predict(input)?;
 
+    // Record business and request telemetry
+    telemetry::predictions_total().inc();
+    _guard.finish();
+
     Ok(Json(prediction_response(prediction)))
 }
 
@@ -129,6 +147,7 @@ pub fn router(state: AppState) -> Router {
         .route("/health", get(health))
         .route("/metadata", get(metadata))
         .route("/predict", post(predict))
+        .route("/metrics", get(metrics))
         .with_state(state)
 }
 

apps/ml-service/src/telemetry.rs

@@ -0,0 +1,118 @@
+use prometheus::{Counter, Gauge, Histogram, HistogramOpts, Registry, TextEncoder};
+use std::sync::OnceLock;
+use std::time::Instant;
+
+fn global_registry() -> &'static Registry {
+    static REGISTRY: OnceLock<Registry> = OnceLock::new();
+    REGISTRY.get_or_init(|| {
+        Registry::new_custom(Some("zeavis_ml".to_string()), None).expect("create registry")
+    })
+}
+
+macro_rules! define_metric {
+    ($name:ident, $ty:ty, $init:expr) => {
+        pub fn $name() -> &'static $ty {
+            static METRIC: OnceLock<$ty> = OnceLock::new();
+            METRIC.get_or_init(|| {
+                let m = $init;
+                global_registry()
+                    .register(Box::new(m.clone()))
+                    .expect(concat!("register ", stringify!($name)));
+                m
+            })
+        }
+    };
+}
+
+// ── HTTP Metrics ────────────────────────────────────────
+
+define_metric!(
+    http_requests_total,
+    Counter,
+    Counter::new("zeavis_ml_http_requests_total", "Total number of HTTP requests")
+        .expect("create counter")
+);
+
+define_metric!(
+    http_request_duration_seconds,
+    Histogram,
+    Histogram::with_opts(
+        HistogramOpts::new(
+            "zeavis_ml_http_request_duration_seconds",
+            "HTTP request duration in seconds",
+        )
+        .buckets(vec![0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0]),
+    )
+    .expect("create histogram")
+);
+
+define_metric!(
+    http_requests_active,
+    Gauge,
+    Gauge::new(
+        "zeavis_ml_http_requests_active",
+        "Number of active HTTP requests",
+    )
+    .expect("create gauge")
+);
+
+// ── Business Metrics ────────────────────────────────────
+
+define_metric!(
+    predictions_total,
+    Counter,
+    Counter::new(
+        "zeavis_ml_predictions_total",
+        "Total number of prediction requests",
+    )
+    .expect("create counter")
+);
+
+define_metric!(
+    model_load_status,
+    Gauge,
+    Gauge::new(
+        "zeavis_ml_model_load_status",
+        "Model load status (1 = loaded, 0 = not loaded)",
+    )
+    .expect("create gauge")
+);
+
+// ── Request Guard (Drop-based cleanup for active gauge) ─
+
+pub struct RequestMetricsGuard {
+    start: Instant,
+}
+
+impl RequestMetricsGuard {
+    pub fn new() -> Self {
+        http_requests_active().inc();
+        Self {
+            start: Instant::now(),
+        }
+    }
+
+    /// Record duration and request count before the guard drops.
+    pub fn finish(&self) {
+        http_request_duration_seconds().observe(self.start.elapsed().as_secs_f64());
+        http_requests_total().inc();
+    }
+}
+
+impl Drop for RequestMetricsGuard {
+    fn drop(&mut self) {
+        http_requests_active().dec();
+    }
+}
+
+// ── Export ──────────────────────────────────────────────
+
+pub fn encode_metrics() -> String {
+    let encoder = TextEncoder::new();
+    let mut buffer = String::new();
+    let metric_families = global_registry().gather();
+    encoder
+        .encode_utf8(&metric_families, &mut buffer)
+        .unwrap();
+    buffer
+}