From ea168b4b20b03cf0348356cd58de30943a8915d3 Mon Sep 17 00:00:00 2001 From: asepharyana Date: Tue, 21 Jul 2026 17:26:02 +0700 Subject: [PATCH] feat: add NATS + Dapr infrastructure for event-driven architecture - infra/compose/nats.yml: NATS server with JetStream, persistence volume, healthcheck - infra/compose/dapr.yml: Dapr placement service for sidecar coordination - infra/dapr/: Dapr global config, pubsub component (NATS), statestore (Redis) - infra/compose/scraper.yml: add Dapr sidecar container + depends_on nats/dapr - docs/add-dapr-service.md: guide for integrating Dapr into new services - docs/add-new-app.md: add Dapr sidecar step - ARCHITECTURE.md: add NATS/Dapr to infra table + service mesh diagram - infra/README.md: update layout and deployment order - deploy-docker.yml: add nats.yml + dapr.yml to ALL_COMPOSE_FILES Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/deploy-docker.yml | 2 +- ARCHITECTURE.md | 36 +++++- CHANGELOG.md | 5 + README.md | 14 ++- docs/add-dapr-service.md | 152 ++++++++++++++++++++++++++ docs/add-new-app.md | 38 +++++++ infra/README.md | 21 +++- infra/compose/dapr.yml | 19 ++++ infra/compose/nats.yml | 32 ++++++ infra/compose/scraper.yml | 28 +++++ infra/dapr/components/pubsub.yaml | 12 ++ infra/dapr/components/statestore.yaml | 14 +++ infra/dapr/config.yaml | 13 +++ 13 files changed, 370 insertions(+), 16 deletions(-) create mode 100644 docs/add-dapr-service.md create mode 100644 infra/compose/dapr.yml create mode 100644 infra/compose/nats.yml create mode 100644 infra/dapr/components/pubsub.yaml create mode 100644 infra/dapr/components/statestore.yaml create mode 100644 infra/dapr/config.yaml diff --git a/.github/workflows/deploy-docker.yml b/.github/workflows/deploy-docker.yml index 55580fb..e8359cb 100644 --- a/.github/workflows/deploy-docker.yml +++ b/.github/workflows/deploy-docker.yml @@ -89,7 +89,7 @@ jobs: git fetch origin main --depth=1 || true # Detect changed files before resetting - ALL_COMPOSE_FILES="infra/compose/traefik.yml infra/compose/shared.yml infra/compose/scraper.yml" + ALL_COMPOSE_FILES="infra/compose/traefik.yml infra/compose/shared.yml infra/compose/scraper.yml infra/compose/nats.yml infra/compose/dapr.yml" TRAEFIK_DYNAMIC_DIR="infra/traefik/dynamic" if git rev-parse HEAD >/dev/null 2>&1; then diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index cd1f4e9..d6b2bd8 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -42,7 +42,9 @@ asepharyana-hub/ | Container Runtime | Docker + Docker Compose | Service isolation and orchestration | | Container Registry | GHCR (ghcr.io) | Docker image storage | | Networking | Tailscale | Secure overlay network between VPS nodes | -| Cache | Redis (Alpine) | Session store, rate limit counters, caching | +| Message Bus | NATS + JetStream | Event-driven pub/sub, job queues, streaming | +| Runtime Sidecar | Dapr | Service invocation, pub/sub abstraction, state management | +| Cache & State | Redis (Alpine) | Session store, rate limit counters, caching, Dapr state store | | CI/CD | GitHub Actions | Build, test, deploy automation | ## Infrastructure @@ -261,24 +263,50 @@ git commit -m "chore(scraper): update submodule to latest" ## Service Mesh & Inter-Service Communication +### HTTP (External + Internal via Traefik) +External traffic and internal HTTP calls route through Traefik. Services on `app-shared-net` can also communicate directly by container name. + +### Event-Driven (NATS + Dapr) +NATS with JetStream provides a persistent message backbone. Each service has a Dapr sidecar that abstracts pub/sub, service invocation, and state management. + ```mermaid -graph LR +graph TB subgraph "External" WWW[Internet] end subgraph "Orange VPS" TRAEFIK[Traefik :443] - + subgraph "app-shared-net" - SCRAPER[scraper-api
:4091] + NATS[NATS + JetStream
:4222] + DAPR_PLACEMENT[Dapr Placement
:50005] + + subgraph "Service: scraper-api" + SCRAPER[scraper-api
:4091] + DAPR_SIDECAR[Dapr Sidecar
:3500] + SCRAPER --- DAPR_SIDECAR + end end + + DAPR_SIDECAR -.->|gRPC pub/sub| NATS + DAPR_SIDECAR -.->|placement| DAPR_PLACEMENT end WWW -->|HTTPS| TRAEFIK TRAEFIK --> SCRAPER ``` +### Communication Patterns + +| Pattern | Mechanism | Use Case | +|---------|-----------|----------| +| External HTTP | Traefik → Service | User requests, API calls | +| Internal HTTP | Service → Service (via Traefik or direct) | Synchronous queries | +| Pub/Sub Event | Dapr sidecar → NATS JetStream | Async notifications, image cache events | +| Service Invocation | Dapr sidecar gRPC | Cross-service RPC with retry & observability | +| State Store | Dapr → Redis | Shared state, job progress | + ## Observability - **Traefik access logs**: JSON format, logged at INFO level diff --git a/CHANGELOG.md b/CHANGELOG.md index da8e0fc..97f7171 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Optimized `scraper.Dockerfile`: removed Node.js and Chromium from runtime image. - Simplified CI/CD workflows: removed orphan container reference, commented code blocks. - Cleaned up scripts: removed stale MySQL config, fixed package references, simplified update-deps. +- Added NATS + JetStream message broker infrastructure (`infra/compose/nats.yml`). +- Added Dapr runtime infrastructure: placement service, sidecar pattern, pub/sub + state store components. +- Integrated Dapr sidecar into scraper service (`infra/compose/scraper.yml`). +- Updated deployment order: shared → NATS → Dapr → Traefik → apps. +- Added `docs/add-dapr-service.md` guide for adding Dapr to new services. ### Removed diff --git a/README.md b/README.md index 6c55d3e..8181ca8 100644 --- a/README.md +++ b/README.md @@ -5,17 +5,21 @@ Aplikasi dipisah sebagai submodule agar frontend, API, dan service pendukung bis ## Services -| Service | Path | Notes | -| :------ | :------------- | :------------------ | -| Scraper | `apps/scraper` | Web scraper service | +| Service | Path | Notes | +| :------ | :------------- | :----------------------------- | +| Scraper | `apps/scraper` | Web scraper service + Dapr SDK | +| NATS | — | Message broker + JetStream | +| Dapr | — | Sidecar runtime (per service) | ## Infrastructure File compose berada di `infra/compose/`: - `traefik.yml`: reverse proxy Traefik untuk semua layanan. -- `shared.yml`: Redis. -- `scraper.yml`: manifest deploy per service (image GHCR bertag SHA). +- `shared.yml`: Redis (cache + Dapr state store). +- `nats.yml`: NATS message broker dengan JetStream persistence. +- `dapr.yml`: Dapr placement service untuk koordinasi sidecar. +- `scraper.yml`: manifest deploy per service (app + Dapr sidecar). Dockerfile per service berada di `infra/docker/`. diff --git a/docs/add-dapr-service.md b/docs/add-dapr-service.md new file mode 100644 index 0000000..ac97ee3 --- /dev/null +++ b/docs/add-dapr-service.md @@ -0,0 +1,152 @@ +# Menambahkan Dapr ke Service Baru + +Panduan integrasi Dapr runtime sidecar untuk service di `asepharyana-hub`. + +## Prasyarat + +- NATS server berjalan (`infra/compose/nats.yml`) +- Dapr placement service berjalan (`infra/compose/dapr.yml`) + +## 1. Compose File + +Setiap service butuh sidecar container Dapr. Contoh: + +```yaml +services: + app: + container_name: app + image: ghcr.io/asepharyana/asepharyana-hub/app:latest + restart: always + depends_on: + dapr-placement: + condition: service_healthy + nats: + condition: service_healthy + networks: + app-shared-net: + aliases: + - app + env_file: + - ../../.env + + app-dapr: + container_name: app-dapr + image: daprio/daprd:latest + restart: always + depends_on: + dapr-placement: + condition: service_healthy + nats: + condition: service_healthy + networks: + - app-shared-net + command: + - './daprd' + - '--app-id=app' + - '--app-port=3000' + - '--dapr-http-port=3500' + - '--dapr-grpc-port=50001' + - '--placement-host-address=dapr-placement:50005' + - '--components-path=/components' + volumes: + - ../../infra/dapr/components:/components + +networks: + app-shared-net: + name: app-shared-net + external: true +``` + +## 2. Mengakses Dapr dari Service + +### Via HTTP API (semua bahasa) + +Sidecar listen di `localhost:3500`: + +```bash +# Publish event +curl -X POST http://localhost:3500/v1.0/publish/pubsub/hub.event.type \ + -H "Content-Type: application/json" \ + -d '{"key": "value"}' + +# Service invocation +curl http://localhost:3500/v1.0/invoke/app/method/endpoint + +# State store +curl -X POST http://localhost:3500/v1.0/state/statestore \ + -H "Content-Type: application/json" \ + -d '[{"key": "mykey", "value": "myvalue"}]' +``` + +### Via Dapr SDK (Rust) + +Tambah ke `Cargo.toml`: + +```toml +dapr-sdk = { version = "0.15", features = ["pubsub", "http"] } +tokio-stream = "0.1" +``` + +Contoh publish event: + +```rust +use dapr_sdk::client::{Client, Event}; +use dapr_sdk::DaprClient; + +let client = DaprClient::new("127.0.0.1", 3500).await?; +client.publish_event("pubsub", "hub.image.cached", serde_json::json!({ + "original_url": url, + "cdn_url": cdn_url, +})).await?; +``` + +Contoh subscribe event: + +```rust +let mut stream = client.subscribe_events("pubsub", "hub.image.cached").await?; +while let Some(event) = stream.next().await { + let data: MyEvent = serde_json::from_slice(&event.data)?; + // handle event +} +``` + +## 3. Event Topics Convention + +Gunakan prefix `hub.` untuk semua event: + +| Topic | Payload | Description | +|-------|---------|-------------| +| `hub.image.cached` | `{original_url, cdn_url, source}` | Image selesai di-cache | +| `hub.image.repaired` | `{old_url, new_url}` | CNAME image diperbaiki | +| `hub.scrape.anime.done` | `{source, slug, duration}` | Scrape anime selesai | +| `hub.system.alert` | `{service, level, message}` | Error/alert dari service | + +## 4. Local Development + +Untuk development tanpa Docker: + +```bash +# 1. Install Dapr CLI +# 2. Init Dapr local +dapr init + +# 3. Run service dengan sidecar +dapr run --app-id app --app-port 3000 --dapr-http-port 3500 \ + --resources-path ./infra/dapr/components \ + -- cargo run +``` + +## 5. Verifikasi + +```bash +# Sidecar health +curl http://localhost:3500/v1.0/healthz + +# Publish test event +curl -X POST http://localhost:3500/v1.0/publish/pubsub/hub.test \ + -H "Content-Type: application/json" \ + -d '{"test": true}' + +# NATS stream stats +curl http://localhost:8222/jszetstream +``` diff --git a/docs/add-new-app.md b/docs/add-new-app.md index 5f8d9a0..6b3be7b 100644 --- a/docs/add-new-app.md +++ b/docs/add-new-app.md @@ -52,6 +52,44 @@ networks: Gunakan `app-shared-net` agar service dapat diakses oleh Traefik dan service lain. +## 3.5. Tambahkan Dapr sidecar (wajib untuk pub/sub) + +Setiap service yang ingin menggunakan Dapr pub/sub atau service invocation harus punya sidecar. +Tambah di `infra/compose/.yml`: + +```yaml + -dapr: + container_name: -dapr + image: daprio/daprd:latest + restart: always + depends_on: + dapr-placement: + condition: service_healthy + nats: + condition: service_healthy + networks: + - app-shared-net + command: + - './daprd' + - '--app-id=' + - '--app-port=' + - '--dapr-http-port=3500' + - '--dapr-grpc-port=50001' + - '--placement-host-address=dapr-placement:50005' + - '--components-path=/components' + volumes: + - ../../infra/dapr/components:/components +``` + +Pastikan juga app container punya `depends_on` ke dapr-placement dan nats: +```yaml + depends_on: + dapr-placement: + condition: service_healthy + nats: + condition: service_healthy +``` + ## 4. Tambahkan route Traefik Update `infra/traefik/dynamic/apps.yaml`: diff --git a/infra/README.md b/infra/README.md index 29e2f05..6813b83 100644 --- a/infra/README.md +++ b/infra/README.md @@ -9,16 +9,19 @@ infra/ ├── compose/ # One compose file per stack/service │ ├── traefik.yml # Public reverse proxy │ ├── shared.yml # Shared Redis -│ └── scraper.yml # Scraper API +│ ├── nats.yml # NATS message broker + JetStream +│ ├── dapr.yml # Dapr placement service +│ └── scraper.yml # Scraper API (app + Dapr sidecar) ├── docker/ # Dockerfiles and image runtime helpers +├── dapr/ # Dapr component configs +│ ├── config.yaml # Global Dapr configuration +│ └── components/ # Pub/sub (NATS), state store (Redis) ├── traefik/ # Static and dynamic Traefik configuration │ ├── dynamic/ # Routers, services, middlewares, TLS certs │ └── TRAEFIK_ENV_CONFIG.md └── config/ # Service bootstrap configuration ``` -Archived configs that are not deployed live under `docs/config/`. - ## First-time setup Create the shared Docker network before starting any service: @@ -34,12 +37,18 @@ Create `.env` from `.env.example` and fill production values. Do not commit `.en The GitHub deploy workflow combines the active compose files automatically. For manual deployment, use this order: ```bash +# 1. Shared services docker compose -f infra/compose/shared.yml up -d + +# 2. Message bus + Dapr placement +docker compose -f infra/compose/nats.yml up -d +docker compose -f infra/compose/dapr.yml up -d + +# 3. Reverse proxy docker compose -f infra/compose/traefik.yml up -d -docker compose \ - -f infra/compose/scraper.yml \ - up -d +# 4. Application services (with Dapr sidecars) +docker compose -f infra/compose/scraper.yml up -d ``` ## Environment variables diff --git a/infra/compose/dapr.yml b/infra/compose/dapr.yml new file mode 100644 index 0000000..b148642 --- /dev/null +++ b/infra/compose/dapr.yml @@ -0,0 +1,19 @@ +services: + dapr-placement: + container_name: dapr-placement + image: daprio/dapr:latest + restart: always + networks: + - app-shared-net + command: ["./placement", "--port", "50005"] + healthcheck: + test: ['CMD-SHELL', 'wget --spider http://localhost:50005/healthz || exit 1'] + interval: 15s + timeout: 5s + retries: 3 + start_period: 10s + +networks: + app-shared-net: + name: app-shared-net + external: true diff --git a/infra/compose/nats.yml b/infra/compose/nats.yml new file mode 100644 index 0000000..8641189 --- /dev/null +++ b/infra/compose/nats.yml @@ -0,0 +1,32 @@ +services: + nats: + container_name: nats + image: nats:latest + restart: always + networks: + app-shared-net: + aliases: + - nats + ports: + - '4222:4222' # client connections + - '8222:8222' # HTTP monitor / health + command: + - '--jetstream' + - '--store_dir=/data' + - '--max_payload=8MB' + volumes: + - nats_data:/data + healthcheck: + test: ['CMD-SHELL', 'wget --spider http://localhost:8222/healthz || exit 1'] + interval: 15s + timeout: 5s + retries: 3 + start_period: 10s + +volumes: + nats_data: + +networks: + app-shared-net: + name: app-shared-net + external: true diff --git a/infra/compose/scraper.yml b/infra/compose/scraper.yml index 2479f20..7adfa1e 100644 --- a/infra/compose/scraper.yml +++ b/infra/compose/scraper.yml @@ -3,6 +3,11 @@ services: container_name: scraper-api image: ghcr.io/asepharyana/asepharyana-hub/scraper-api:sha-c4ecf8f restart: always + depends_on: + dapr-placement: + condition: service_healthy + nats: + condition: service_healthy networks: app-shared-net: aliases: @@ -20,6 +25,29 @@ services: - JWT_SECRET=${JWT_SECRET:?JWT_SECRET is required} - DATABASE_URL=${DATABASE_URL} - EXTERNAL_BROWSERLESS_WS=ws://100.121.180.82:3001 + + scraper-api-dapr: + container_name: scraper-api-dapr + image: daprio/daprd:latest + restart: always + depends_on: + dapr-placement: + condition: service_healthy + nats: + condition: service_healthy + networks: + - app-shared-net + command: + - './daprd' + - '--app-id=scraper-api' + - '--app-port=4091' + - '--dapr-http-port=3500' + - '--dapr-grpc-port=50001' + - '--placement-host-address=dapr-placement:50005' + - '--components-path=/components' + volumes: + - ../../infra/dapr/components:/components + networks: app-shared-net: name: app-shared-net diff --git a/infra/dapr/components/pubsub.yaml b/infra/dapr/components/pubsub.yaml new file mode 100644 index 0000000..b7ddd4f --- /dev/null +++ b/infra/dapr/components/pubsub.yaml @@ -0,0 +1,12 @@ +apiVersion: dapr.io/v1alpha1 +kind: Component +metadata: + name: pubsub +spec: + type: pubsub.natsstreaming + version: v1 + metadata: + - name: natsURL + value: nats://nats:4222 + - name: natsStreamingClusterID + value: dapr-cluster diff --git a/infra/dapr/components/statestore.yaml b/infra/dapr/components/statestore.yaml new file mode 100644 index 0000000..7445ccd --- /dev/null +++ b/infra/dapr/components/statestore.yaml @@ -0,0 +1,14 @@ +apiVersion: dapr.io/v1alpha1 +kind: Component +metadata: + name: statestore +spec: + type: state.redis + version: v1 + metadata: + - name: redisHost + value: redis:6379 + - name: redisPassword + value: "" + - name: keyPrefix + value: "dapr" diff --git a/infra/dapr/config.yaml b/infra/dapr/config.yaml new file mode 100644 index 0000000..ded7f3d --- /dev/null +++ b/infra/dapr/config.yaml @@ -0,0 +1,13 @@ +apiVersion: dapr.io/v1alpha1 +kind: Configuration +metadata: + name: dapr-config +spec: + tracing: + samplingRate: "1" + zipkin: + endpointAddress: "" + metrics: + enabled: true + mtls: + enabled: false