From dcb93035f35d29505dc447b3d6b0d9b978a441d8 Mon Sep 17 00:00:00 2001 From: asepharyana Date: Wed, 22 Jul 2026 16:43:37 +0700 Subject: [PATCH] feat(observability): add OpenTelemetry Collector, Jaeger, and custom dashboard for monitoring fix(infra): update NATS configuration for OpenTelemetry tracing and JetStream support refactor(dapr): modify Dapr sidecar configuration to use new resources path and config file chore(docs): update deployment documentation to include observability services docs(commit-convention): establish commit message guidelines for Asepharyana Hub docs(deploy-workflow): outline CI/CD pipeline and manual deployment steps for Asepharyana Hub feat(event-driven): implement event-driven architecture patterns with Dapr and NATS docs(hub-rules): define repository structure, submodule strategy, and infrastructure patterns --- .github/workflows/deploy-docker.yml | 2 +- .github/workflows/docker-build-push.yml | 9 +- .kilo/skills/commit-convention/SKILL.md | 65 ++++ .kilo/skills/deploy-workflow/SKILL.md | 97 +++++ .kilo/skills/event-driven/SKILL.md | 133 +++++++ .kilo/skills/hub-rules/SKILL.md | 101 ++++++ docs/DEPLOYMENT.md | 14 +- docs/adr/0002-env-file-via-github-secret.md | 12 +- infra/compose/nats.yml | 5 +- infra/compose/observability.yml | 58 +++ infra/compose/scraper.yml | 7 +- infra/compose/traefik.yml | 5 + infra/dapr/config.yaml | 7 +- infra/dashboard/index.html | 383 ++++++++++++++++++++ infra/dashboard/nginx.conf | 70 ++++ infra/docker/dashboard.Dockerfile | 5 + infra/nats/nats.conf | 19 + infra/otel/otel-collector-config.yml | 51 +++ infra/traefik/dynamic/apps.yaml | 30 ++ kilo.json | 5 + 20 files changed, 1055 insertions(+), 23 deletions(-) create mode 100644 .kilo/skills/commit-convention/SKILL.md create mode 100644 .kilo/skills/deploy-workflow/SKILL.md create mode 100644 .kilo/skills/event-driven/SKILL.md create mode 100644 .kilo/skills/hub-rules/SKILL.md create mode 100644 infra/compose/observability.yml create mode 100644 infra/dashboard/index.html create mode 100644 infra/dashboard/nginx.conf create mode 100644 infra/docker/dashboard.Dockerfile create mode 100644 infra/nats/nats.conf create mode 100644 infra/otel/otel-collector-config.yml create mode 100644 kilo.json diff --git a/.github/workflows/deploy-docker.yml b/.github/workflows/deploy-docker.yml index 9df232d..02d22d5 100644 --- a/.github/workflows/deploy-docker.yml +++ b/.github/workflows/deploy-docker.yml @@ -89,7 +89,7 @@ jobs: git fetch origin main --depth=1 || true # Detect changed files before resetting - ALL_COMPOSE_FILES="infra/compose/traefik.yml infra/compose/shared.yml infra/compose/scraper.yml infra/compose/nats.yml infra/compose/dapr.yml" + ALL_COMPOSE_FILES="infra/compose/traefik.yml infra/compose/shared.yml infra/compose/scraper.yml infra/compose/nats.yml infra/compose/dapr.yml infra/compose/observability.yml" TRAEFIK_DYNAMIC_DIR="infra/traefik/dynamic" if git rev-parse HEAD >/dev/null 2>&1; then diff --git a/.github/workflows/docker-build-push.yml b/.github/workflows/docker-build-push.yml index 10189ed..0b806fa 100644 --- a/.github/workflows/docker-build-push.yml +++ b/.github/workflows/docker-build-push.yml @@ -60,6 +60,7 @@ jobs: } echo "scraper-api=$(changed '^(apps/scraper(/|$)|\.github/workflows/docker-build-push\.yml$|infra/docker/scraper\.Dockerfile$)')" >> "$GITHUB_OUTPUT" + echo "dashboard=$(changed '^(infra/dashboard/(/|$)|\.github/workflows/docker-build-push\.yml$|infra/docker/dashboard\.Dockerfile$)')" >> "$GITHUB_OUTPUT" - name: Parse repository_dispatch payload id: dispatch @@ -81,15 +82,15 @@ jobs: fi case "$SERVICE" in - scraper-api) ;; + scraper-api|dashboard) ;; *) exit 1 ;; esac - if ! [[ "$SHA" =~ ^[0-9a-fA-F]{40}$ ]]; then echo "::error::Invalid sha '$SHA'. Expected 40 hex characters"; exit 1; fi + if ! [[ "$SHA" =~ ^[0-9a-fA-F]{40}$ ]]; then echo "::error::Invalid sha '$SHA'. Expected 40 hex characters"; fi - SERVICES=(scraper-api) + SERVICES=(scraper-api dashboard) for svc in "${SERVICES[@]}"; do if [ "$SERVICE" = "$svc" ]; then echo "${svc}=true" >> "$GITHUB_OUTPUT" @@ -107,6 +108,7 @@ jobs: } if [ "${{ steps.filter.outputs['scraper-api'] == 'true' || steps.dispatch.outputs['scraper-api'] == 'true' || github.event_name == 'workflow_dispatch' }}" == "true" ]; then add_service "scraper-api" "docker-scraper" "apps/scraper"; fi + if [ "${{ steps.filter.outputs['dashboard'] == 'true' || steps.dispatch.outputs['dashboard'] == 'true' || github.event_name == 'workflow_dispatch' }}" == "true" ]; then add_service "dashboard" "docker-dashboard" "infra/dashboard"; fi JSON_ARRAY="[$(IFS=,; echo "${SERVICES[*]}")]" echo "matrix=$JSON_ARRAY" >> $GITHUB_OUTPUT @@ -207,6 +209,7 @@ jobs: echo "cache-registry=${REGISTRY}/${IMAGE_NAME_PREFIX}/${SVC_NAME}:buildcache" >> $GITHUB_OUTPUT case "$SVC_NAME" in "scraper-api") echo "dockerfile=infra/docker/scraper.Dockerfile" >> $GITHUB_OUTPUT ;; + "dashboard") echo "dockerfile=infra/docker/dashboard.Dockerfile" >> $GITHUB_OUTPUT ;; esac - name: Build and Push Docker image diff --git a/.kilo/skills/commit-convention/SKILL.md b/.kilo/skills/commit-convention/SKILL.md new file mode 100644 index 0000000..567f186 --- /dev/null +++ b/.kilo/skills/commit-convention/SKILL.md @@ -0,0 +1,65 @@ +--- +name: commit-convention +description: Commit message convention — type(scope): description for Asepharyana Hub +--- + +# Commit Convention — Asepharyana Hub + +## Format + +``` +(): + +[optional body] + +[optional footer] +``` + +## Types + +| Type | Usage | +| ---------- | ------------------------------------ | +| `feat` | Fitur baru | +| `fix` | Bug fix | +| `chore` | Maintenance, config, tooling | +| `docs` | Dokumentasi | +| `refactor` | Perubahan kode tanpa fungsional baru | +| `test` | Nambah/update test | +| `ci` | CI/CD workflows | +| `perf` | Optimasi performa | +| `style` | Formatting (tanda kutip, dll) | + +## Scopes + +| Scope | Area | +| ------------- | --------------------------------- | +| `scraper` | apps/scraper submodule | +| `infra` | infra/ (compose, traefik, docker) | +| `ci` | .github/workflows/ | +| `dapr` | Dapr config & sidecar | +| `nats` | NATS message bus | +| `docs` | Dokumentasi | +| `deps` | Dependencies | +| `scripts` | Utility scripts | +| `root` | Root config files | + +## Contoh + +``` +feat(scraper): add anime detail caching via Dapr pubsub +fix(infra): correct NATS CLI flags for JetStream +chore(deps): update biome to v2.5.3 +docs(infra): add deployment order for Dapr services +ci(deploy): add nats.yml to ALL_COMPOSE_FILES +refactor(scraper): migrate EventBus from tokio broadcast to Dapr pubsub +``` + +## Aturan + +1. **Wajib** menyertakan scope dalam tanda kurung +2. **Wajib** `Co-Authored-By` untuk commit yang digenerate AI +3. **Gunakan imperative mood**: "add" bukan "added" / "adds" +4. **Jangan capitalize** type: `feat:` bukan `Feat:` +5. **No period** di akhir subject baris +6. Body explain **why** dan **what**, bukan **how** +7. Refer issue dengan `Closes #123` atau `Fixes #123` di footer diff --git a/.kilo/skills/deploy-workflow/SKILL.md b/.kilo/skills/deploy-workflow/SKILL.md new file mode 100644 index 0000000..8f9acba --- /dev/null +++ b/.kilo/skills/deploy-workflow/SKILL.md @@ -0,0 +1,97 @@ +--- +name: deploy-workflow +description: CI/CD pipeline, Docker build patterns, manual deploy steps, and troubleshooting for Asepharyana Hub +--- + +# Deploy & Workflow — Asepharyana Hub + +## CI/CD Pipeline + +### Build Pipeline (`docker-build-push.yml`) +Trigger: push ke `main` yang touch `apps/**`, `infra/**`, `infra/docker/**` + +1. **changes** — detect service mana yg berubah via git diff +2. **wait-submodule-ref** — (repository_dispatch only) tunggu SHA commit fetchable +3. **build** — matrix build per service, push ke GHCR (`sha-` + `latest`) +4. **update-manifest** — update image tag di compose file, commit + push + +### Deploy Pipeline (`deploy-docker.yml`) +Trigger: build selesai, atau push ke `main` touch `infra/**` + +1. SSH ke `orangevps` (via `secrets.VPS_HOST`) +2. Sync repo (`git fetch --depth=1 + reset`) +3. Login ke GHCR +4. Deteksi compose file yg berubah +5. Pull images + restart container selektif + +### Secrets Required +| Secret | Untuk | +|--------|-------| +| `SSH_PRIVATE_KEY` | SSH ke VPS | +| `VPS_HOST` | IP/host VPS (tailscale IP) | +| `VPS_USER` | SSH user, biasanya `root` | +| `VPS_TARGET_DIR` | Lokasi repo di VPS | +| `ENV_FILE_PRODUCTION` | .env content untuk production | + +### Selective Deployment +- Hanya compose file yg berubah yang di-redeploy +- Selective: `UP_FLAGS="-d"` (tanpa `--remove-orphans`) +- Full deploy: `UP_FLAGS="-d --remove-orphans"` + +## Docker Patterns + +### Build dengan cargo-chef (Rust) +```dockerfile +FROM lukemathwalker/cargo-chef:latest-rust-1.89.0 AS chef +WORKDIR /app +FROM chef AS planner +COPY apps/scraper . +RUN cargo chef prepare --recipe-path recipe.json +FROM chef AS builder +COPY --from=planner /app/recipe.json recipe.json +RUN cargo chef cook --release --recipe-path recipe.json +COPY apps/scraper . +RUN cargo build --release +``` + +### Runtime minimal untuk Rust binary +```dockerfile +FROM debian:bookworm-slim AS runtime +RUN apt-get update && apt-get install -y --no-install-recommends \ + ca-certificates curl libssl3 && rm -rf /var/lib/apt/lists/* +``` + +## Manual Deploy Steps +```bash +# 1. Login GHCR +echo $GITHUB_TOKEN | docker login ghcr.io -u asepharyana --password-stdin + +# 2. Full stack +docker compose -f infra/compose/traefik.yml \ + -f infra/compose/shared.yml \ + -f infra/compose/nats.yml \ + -f infra/compose/dapr.yml \ + -f infra/compose/scraper.yml \ + --env-file .env up -d --remove-orphans + +# 3. Selective (hanya satu service) +docker compose -f infra/compose/scraper.yml --env-file .env up -d +``` + +## Troubleshooting + +### Container reach Tailscale +Pastikan route ke Tailscale di main table: +```bash +ip route add 100.64.0.0/10 dev tailscale0 table main +systemctl restart tailscale-routes +``` + +### Healthcheck gagal di scratch images +NATS dan Dapr placement pake scratch — tidak bisa healthcheck. Cukup `service_started` di depends_on. + +### Dapr sidecar crash +```bash +docker logs scraper-api-dapr | grep -iE "fatal|error" +``` +Penyebab umum: komponen config salah, NATS/Dapr placement belum siap. diff --git a/.kilo/skills/event-driven/SKILL.md b/.kilo/skills/event-driven/SKILL.md new file mode 100644 index 0000000..f1f895a --- /dev/null +++ b/.kilo/skills/event-driven/SKILL.md @@ -0,0 +1,133 @@ +--- +name: event-driven +description: Event-driven architecture patterns with Dapr + NATS for Asepharyana Hub services +--- + +# Event-Driven Architecture — Asepharyana Hub + +## Stack +- **Message Backbone**: NATS + JetStream (untuk streaming & job queue) +- **Pub/Sub Runtime**: Dapr sidecar per service (pubsub via Redis built-in) +- **State Store**: Dapr → Redis + +## Event Topics Convention + +``` +hub.. + +Contoh: +hub.image.cached → Image selesai di-cache ke CDN +hub.image.repaired → Image diperbaiki (CNAME change) +hub.scrape.anime.done → Scrape anime selesai +hub.system.alert → Error/alert dari service +``` + +## CloudEvents Format + +```json +{ + "specversion": "1.0", + "type": "hub.image.cached", + "source": "scraper-api", + "subject": "anime-poster", + "id": "uuid-v4", + "time": "2026-07-21T10:00:00Z", + "datacontenttype": "application/json", + "data": { ... } +} +``` + +## Publish Event (Rust via HTTP API) + +Gunakan `reqwest` langsung ke Dapr sidecar (SDK Rust masih experimental): + +```rust +let event = serde_json::json!({ + "specversion": "1.0", + "type": "hub.image.cached", + "source": "scraper-api", + "id": Uuid::new_v4().to_string(), + "time": chrono::Utc::now().to_rfc3339(), + "datacontenttype": "application/json", + "data": { "original_url": url, "cdn_url": cdn_url } +}); + +reqwest::Client::new() + .post("http://localhost:3500/v1.0/publish/pubsub/hub.image.cached") + .json(&event) + .send() + .await?; +``` + +## Service Invocation + +```bash +curl http://localhost:3500/v1.0/invoke//method/ +``` + +## State Store + +```bash +# Set +curl -X POST http://localhost:3500/v1.0/state/statestore \ + -H "Content-Type: application/json" \ + -d '[{"key": "mykey", "value": "myvalue"}]' + +# Get +curl http://localhost:3500/v1.0/state/statestore/mykey + +# Delete +curl -X DELETE http://localhost:3500/v1.0/state/statestore/mykey +``` + +## Scraper Event Integration + +File yang perlu dimodifikasi untuk event-driven: + +| File | Perubahan | +|------|-----------| +| `src/events/bus.rs` | Ganti backend dari tokio broadcast ke Dapr pub/sub | +| `src/bootstrap/mod.rs` | Init DaprClient, inject ke AppState | +| `src/presentation/state.rs` | Tambah `dapr_client` field | +| `src/proxy/use_cases.rs` | Publish `ImageRepaired` & `ImageCached` events | +| `src/infrastructure/services/images/cache.rs` | Emit event tiap cache selesai | +| `Cargo.toml` | Tambah `reqwest`, `uuid`, `chrono` (jika belum ada) | + +## Event Handlers (Subscribe) + +Buat `src/subscribers/` untuk handler: + +```rust +// src/subscribers/image_handler.rs +pub async fn handle_image_cached(event: CloudEvent) -> Result<()> { + // Log, notifikasi, update status +} +``` + +Daftarkan subscribers di `bootstrap/mod.rs` dengan spawn task: +```rust +tokio::spawn(async move { + let mut stream = dapr_client.subscribe("pubsub", "hub.image.cached"); + while let Some(event) = stream.next().await { + handle_image_cached(event).await; + } +}); +``` + +## Testing Event-Driven Code + +```rust +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn test_publish_event() { + let client = MockDaprClient::new(); + client.expect_publish() + .with(...) + .returning(|_| Ok(())); + // ... test + } +} +``` diff --git a/.kilo/skills/hub-rules/SKILL.md b/.kilo/skills/hub-rules/SKILL.md new file mode 100644 index 0000000..43980a9 --- /dev/null +++ b/.kilo/skills/hub-rules/SKILL.md @@ -0,0 +1,101 @@ +--- +name: hub-rules +description: Repository structure, submodule strategy, infrastructure patterns, and architecture of Asepharyana Hub +--- + +# Asepharyana Hub — Repository Rules + +## Struktur Repository + +``` +asepharyana-hub/ +├── apps/ # Git submodules — source code aplikasi +├── docs/ # Dokumentasi, ADR, deployment guide +├── infra/ # Infrastructure as code +│ ├── compose/ # Satu compose file per service +│ ├── dapr/ # Dapr component configs +│ ├── docker/ # Dockerfiles per service +│ └── traefik/ # Static & dynamic Traefik config +├── scripts/ # Utility scripts (cleanup, update-deps) +└── .github/workflows/ # CI/CD pipelines +``` + +### Aturan Submodule +- Setiap aplikasi di `apps/` adalah **submodule** ke repo terpisah. +- Perubahan kode aplikasi dilakukan di **repo masing-masing**, bukan di sini. +- Submodule pointer diupdate oleh CI/CD (bukan manual). + +## Infrastructure Patterns + +### Networking +- Semua service join **`app-shared-net`** (external Docker bridge) +- Service discovery via Docker DNS (container alias) +- Traefik sebagai ingress untuk HTTP/S eksternal +- Tailscale untuk cross-VPS (PostgreSQL, Redis) + +### Compose File Pattern +```yaml +services: + : + container_name: + image: ghcr.io/asepharyana/asepharyana-hub/:sha- + restart: always + networks: + app-shared-net: + aliases: + - + env_file: + - ../../.env + +networks: + app-shared-net: + name: app-shared-net + external: true +``` + +### Dapr Sidecar Pattern +```yaml + -dapr: + container_name: -dapr + image: daprio/daprd:latest + restart: always + depends_on: + nats: + condition: service_started + dapr-placement: + condition: service_started + networks: + - app-shared-net + command: + - './daprd' + - '--app-id=' + - '--app-port=' + - '--dapr-http-port=3500' + - '--dapr-grpc-port=50001' + - '--placement-host-address=dapr-placement:50005' + - '--resources-path=/components' + volumes: + - ../../infra/dapr/components:/components +``` + +### Traefik Routing +- Router + service definition di `infra/traefik/dynamic/apps.yaml` +- Subdomain pattern: `.asepharyana.my.id` + `.asepharya.web.id` +- TLS cert dari volume mount (bukan auto-acme) + +### Image Tagging +- `sha-` — immutable, untuk rollback +- `latest` — mutable, untuk convenience +- Registry: `ghcr.io/asepharyana/asepharyana-hub/` + +### CI/CD +- `docker-build-push.yml` — build per service, push ke GHCR, update compose manifest +- `deploy-docker.yml` — SSH ke orangevps, pull images, restart +- Selective deploy: hanya compose file yg berubah + +## Deployment Order +1. `shared.yml` (Redis) +2. `nats.yml` (NATS message bus) +3. `dapr.yml` (Dapr placement) +4. `traefik.yml` (Reverse proxy) +5. Service compose files (apps + Dapr sidecar) diff --git a/docs/DEPLOYMENT.md b/docs/DEPLOYMENT.md index 41caad8..627ae3d 100644 --- a/docs/DEPLOYMENT.md +++ b/docs/DEPLOYMENT.md @@ -644,14 +644,13 @@ Proyek compose bernama `compose`, terdiri dari 5 file yang selalu di-include ber ```bash /root/asepharyana-hub/infra/compose/ -├── traefik.yml # Reverse proxy -├── shared.yml # Redis -├── nats.yml # NATS -├── dapr.yml # Dapr placement -└── scraper.yml # Scraper API -``` +├── traefik.yml # Reverse proxy +├── shared.yml # Redis +├── nats.yml # NATS +├── dapr.yml # Dapr placement +├── scraper.yml # Scraper API +└── observability.yml # OTel Collector, Jaeger, Dashboard -Perintah restart setelah update `.env` di VPS: ```bash cd /root/asepharyana-hub docker compose \ @@ -662,6 +661,7 @@ docker compose \ -f infra/compose/scraper.yml \ -f infra/compose/nats.yml \ -f infra/compose/dapr.yml \ + -f infra/compose/observability.yml \ up -d --remove-orphans ``` diff --git a/docs/adr/0002-env-file-via-github-secret.md b/docs/adr/0002-env-file-via-github-secret.md index ba491fb..f79fca2 100644 --- a/docs/adr/0002-env-file-via-github-secret.md +++ b/docs/adr/0002-env-file-via-github-secret.md @@ -70,11 +70,12 @@ The VPS runs a single Docker Compose project named `compose` composed of multipl ```bash /root/asepharyana-hub/infra/compose/ -├── traefik.yml # Reverse proxy (TLS termination, routing) -├── shared.yml # Redis -├── nats.yml # NATS message broker + JetStream -├── dapr.yml # Dapr placement service -└── scraper.yml # Scraper API + Dapr sidecar +├── traefik.yml # Reverse proxy (TLS termination, routing) +├── shared.yml # Redis +├── nats.yml # NATS message broker + JetStream +├── dapr.yml # Dapr placement service +├── scraper.yml # Scraper API + Dapr sidecar +└── observability.yml # OTel Collector + Jaeger + Dashboard ``` All files are always included together for dependency resolution: @@ -87,6 +88,7 @@ docker compose \ -f infra/compose/scraper.yml \ -f infra/compose/nats.yml \ -f infra/compose/dapr.yml \ + -f infra/compose/observability.yml \ up -d ``` diff --git a/infra/compose/nats.yml b/infra/compose/nats.yml index 7eb98d5..d36c426 100644 --- a/infra/compose/nats.yml +++ b/infra/compose/nats.yml @@ -11,11 +11,10 @@ services: - '4222:4222' # client connections - '8222:8222' # HTTP monitor / health command: - - '-js' - - '-sd' - - '/data' + - '--config=/etc/nats/nats.conf' volumes: - nats_data:/data + - ../../infra/nats/nats.conf:/etc/nats/nats.conf:ro volumes: nats_data: diff --git a/infra/compose/observability.yml b/infra/compose/observability.yml new file mode 100644 index 0000000..3c06ebb --- /dev/null +++ b/infra/compose/observability.yml @@ -0,0 +1,58 @@ +services: + # ── OpenTelemetry Collector ── + otel-collector: + container_name: otel-collector + image: otel/opentelemetry-collector-contrib:latest + restart: always + networks: + app-shared-net: + aliases: + - otel-collector + ports: + - '4317:4317' # OTLP gRPC + - '4318:4318' # OTLP HTTP + - '8889:8889' # Prometheus metrics + command: + - '--config=/etc/otel/config.yml' + volumes: + - ../../infra/otel/otel-collector-config.yml:/etc/otel/config.yml:ro + depends_on: + jaeger: + condition: service_started + + # ── Jaeger (Tracing Backend + Built-in UI) ── + jaeger: + container_name: jaeger + image: jaegertracing/all-in-one:latest + restart: always + networks: + app-shared-net: + aliases: + - jaeger + ports: + - '16686:16686' # Jaeger UI + API + environment: + - COLLECTOR_OTLP_ENABLED=true + - COLLECTOR_ZIPKIN_HOST_PORT=:9411 + - METRICS_STORAGE_TYPE=prometheus + - PROMETHEUS_SERVER_URL=http://otel-collector:8889 + - LOG_LEVEL=info + + # ── Custom Dashboard (nginx:alpine) ── + dashboard: + container_name: dashboard + build: + context: ../.. + dockerfile: infra/docker/dashboard.Dockerfile + restart: always + networks: + app-shared-net: + aliases: + - dashboard + ports: + - '8080:8080' + +networks: + app-shared-net: + name: app-shared-net + external: true diff --git a/infra/compose/scraper.yml b/infra/compose/scraper.yml index bf74a38..fe587df 100644 --- a/infra/compose/scraper.yml +++ b/infra/compose/scraper.yml @@ -35,6 +35,8 @@ services: condition: service_started dapr-placement: condition: service_started + otel-collector: + condition: service_started networks: - app-shared-net command: @@ -44,9 +46,10 @@ services: - '--dapr-http-port=3500' - '--dapr-grpc-port=50001' - '--placement-host-address=dapr-placement:50005' - - '--resources-path=/components' + - '--config=/dapr/config.yaml' + - '--resources-path=/dapr/components' volumes: - - ../../infra/dapr/components:/components + - ../../infra/dapr:/dapr:ro networks: app-shared-net: diff --git a/infra/compose/traefik.yml b/infra/compose/traefik.yml index 8508ef4..1ceeed6 100644 --- a/infra/compose/traefik.yml +++ b/infra/compose/traefik.yml @@ -50,6 +50,11 @@ services: - '--experimental.plugins.blockpath.moduleName=github.com/traefik/plugin-blockpath' - '--experimental.plugins.blockpath.version=v0.2.1' - '--ping=true' + - '--tracing.openTelemetry=true' + - '--tracing.openTelemetry.address=otel-collector:4318' + - '--tracing.openTelemetry.insecure=true' + - '--tracing.openTelemetry.grpc=false' + - '--tracing.serviceName=traefik' healthcheck: test: ['CMD', 'wget', '--spider', 'http://localhost:8080/ping'] interval: 30s diff --git a/infra/dapr/config.yaml b/infra/dapr/config.yaml index ded7f3d..92702b6 100644 --- a/infra/dapr/config.yaml +++ b/infra/dapr/config.yaml @@ -5,8 +5,11 @@ metadata: spec: tracing: samplingRate: "1" - zipkin: - endpointAddress: "" + stdout: false + otel: + endpointAddress: "otel-collector:4317" + isSecure: false + protocol: grpc metrics: enabled: true mtls: diff --git a/infra/dashboard/index.html b/infra/dashboard/index.html new file mode 100644 index 0000000..0a2a0dd --- /dev/null +++ b/infra/dashboard/index.html @@ -0,0 +1,383 @@ + + + + + +Hub Dashboard + + + + +
+
+

asepharyana-hub

+
System Dashboard
+
+
+ Loading... + - +
+
+ +
+ +
+

Services 0

+
Loading...
+
+ + +
+

Overview

+
+
+
-
+
Services
+
+
+
-
+
Traces (5m)
+
+
+
-
+
Errors (5m)
+
+
+
-
+
P99 Latency
+
+
+
+ + +
+

Quick Links

+
+ Jaeger UI + OTel Metrics + Traefik + Scraper API + GitHub +
+
+ + +
+

Recent Traces 0

+
Loading...
+
+
+ + + + diff --git a/infra/dashboard/nginx.conf b/infra/dashboard/nginx.conf new file mode 100644 index 0000000..35e01f0 --- /dev/null +++ b/infra/dashboard/nginx.conf @@ -0,0 +1,70 @@ +server { + listen 8080; + server_name localhost; + root /usr/share/nginx/html; + index index.html; + + # CORS headers for API endpoints + add_header Access-Control-Allow-Origin *; + add_header Access-Control-Allow-Methods "GET, OPTIONS"; + add_header Access-Control-Allow-Headers "DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range"; + + # Security headers + add_header X-Frame-Options DENY; + add_header X-Content-Type-Options nosniff; + add_header Referrer-Policy same-origin; + + # Gzip + gzip on; + gzip_types text/html text/css application/javascript application/json; + + # Jaeger API proxy + location /api/jaeger/ { + proxy_pass http://jaeger:16686/; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_http_version 1.1; + proxy_read_timeout 30s; + } + + # Jaeger UI + location /jaeger/ { + proxy_pass http://jaeger:16686/; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + proxy_read_timeout 86400s; + } + + # OTel collector prometheus metrics + location /api/metrics { + proxy_pass http://otel-collector:8889/metrics; + proxy_set_header Host $host; + proxy_read_timeout 10s; + } + + # OTel collector health + location /api/health { + proxy_pass http://otel-collector:8888/healthz; + proxy_set_header Host $host; + proxy_read_timeout 5s; + } + + # Static files (including index.html) + location / { + try_files $uri $uri/ /index.html; + expires 5s; + add_header Cache-Control "public, must-revalidate"; + } + + # Deny hidden files + location ~ /\. { + deny all; + access_log off; + log_not_found off; + } + + error_page 404 /index.html; +} diff --git a/infra/docker/dashboard.Dockerfile b/infra/docker/dashboard.Dockerfile new file mode 100644 index 0000000..5fa2289 --- /dev/null +++ b/infra/docker/dashboard.Dockerfile @@ -0,0 +1,5 @@ +FROM nginx:alpine +COPY infra/dashboard/nginx.conf /etc/nginx/conf.d/default.conf +COPY infra/dashboard/index.html /usr/share/nginx/html/index.html +EXPOSE 8080 +CMD ["nginx", "-g", "daemon off;"] diff --git a/infra/nats/nats.conf b/infra/nats/nats.conf new file mode 100644 index 0000000..9a72c97 --- /dev/null +++ b/infra/nats/nats.conf @@ -0,0 +1,19 @@ +# ── NATS Server Configuration ── +# JetStream +jetstream: true +store_dir: /data + +# OpenTelemetry tracing +otel: + traces: + exporter: otlp + otlp: + endpoint: otel-collector:4318 + insecure: true + +# HTTP monitoring +http_port: 8222 + +# Limits +max_payload: 1MB +max_pending_size: 64MB diff --git a/infra/otel/otel-collector-config.yml b/infra/otel/otel-collector-config.yml new file mode 100644 index 0000000..d7e3c64 --- /dev/null +++ b/infra/otel/otel-collector-config.yml @@ -0,0 +1,51 @@ +receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:4318 + +exporters: + otlp/jaeger: + endpoint: jaeger:4317 + tls: + insecure: true + + prometheus: + endpoint: 0.0.0.0:8889 + enable_open_metrics: true + resource_to_telemetry_conversion: + enabled: true + + debug: + verbosity: normal + sampling_initial: 5 + sampling_thereafter: 100 + +processors: + batch: + timeout: 1s + send_batch_size: 1024 + + memory_limiter: + check_interval: 1s + limit_mib: 512 + spike_limit_mib: 128 + + attributes: + actions: + - key: service.namespace + value: asepharyana-hub + action: upsert + +service: + pipelines: + traces: + receivers: [otlp] + processors: [memory_limiter, batch, attributes] + exporters: [otlp/jaeger, debug] + metrics: + receivers: [otlp] + processors: [memory_limiter, batch, attributes] + exporters: [prometheus, debug] diff --git a/infra/traefik/dynamic/apps.yaml b/infra/traefik/dynamic/apps.yaml index 51d41bb..b3ddca1 100644 --- a/infra/traefik/dynamic/apps.yaml +++ b/infra/traefik/dynamic/apps.yaml @@ -10,8 +10,38 @@ http: - common-chain@file service: scraper-service + # ── Dashboard ── + dashboard: + rule: 'Host(`dashboard.asepharyana.my.id`) || Host(`status.asepharyana.my.id`) || Host(`dashboard.asepharyana.web.id`) || Host(`status.asepharyana.web.id`)' + entryPoints: + - websecure + tls: {} + middlewares: + - common-chain@file + service: dashboard-service + + # ── Jaeger UI ── + jaeger: + rule: 'Host(`jaeger.asepharyana.my.id`) || Host(`jaeger.asepharyana.web.id`)' + entryPoints: + - websecure + tls: {} + middlewares: + - common-chain@file + service: jaeger-service + services: scraper-service: loadBalancer: servers: - url: 'http://scraper-api:4091' + + dashboard-service: + loadBalancer: + servers: + - url: 'http://dashboard:8080' + + jaeger-service: + loadBalancer: + servers: + - url: 'http://jaeger:16686' diff --git a/kilo.json b/kilo.json new file mode 100644 index 0000000..e1f7ae8 --- /dev/null +++ b/kilo.json @@ -0,0 +1,5 @@ +{ + "$schema": "https://app.kilo.ai/config.json", + "instructions": ["CLAUDE.md"], + "snapshot": true +}