diff --git a/.bun-version b/.bun-version deleted file mode 100644 index 17e63e7..0000000 --- a/.bun-version +++ /dev/null @@ -1 +0,0 @@ -1.3.11 diff --git a/.claude/settings.json b/.claude/settings.json deleted file mode 100644 index 5668f6d..0000000 --- a/.claude/settings.json +++ /dev/null @@ -1,73 +0,0 @@ -{ - "skills": [ - { - "name": "clean-code", - "filePattern": ".claude/skills/clean-code/SKILL.md", - "description": "Clean Code, Clean Architecture, SOLID, TDD — dari kana-best-practice-engineering" - }, - { - "name": "hub-rules", - "filePattern": ".claude/skills/hub-rules.md", - "description": "Aturan repository hub, submodule, infra patterns, dan arsitektur" - }, - { - "name": "commit-convention", - "filePattern": ".claude/skills/commit-convention.md", - "description": "Commit message convention — type(scope): description" - }, - { - "name": "event-driven", - "filePattern": ".claude/skills/event-driven.md", - "description": "Event-driven patterns with Dapr + NATS untuk hub services" - }, - { - "name": "deploy-workflow", - "filePattern": ".claude/skills/deploy-workflow.md", - "description": "CI/CD pipeline, Docker patterns, deployment guide" - } - ], - "hooks": { - "PreToolUse": [ - { - "matcher": "", - "hooks": [ - { - "type": "command", - "command": "serena-hooks remind --client=claude-code" - } - ] - }, - { - "matcher": "mcp__serena__*", - "hooks": [ - { - "type": "command", - "command": "serena-hooks auto-approve --client=claude-code" - } - ] - } - ], - "SessionStart": [ - { - "matcher": "", - "hooks": [ - { - "type": "command", - "command": "serena-hooks activate --client=claude-code" - } - ] - } - ], - "SessionEnd": [ - { - "matcher": "", - "hooks": [ - { - "type": "command", - "command": "serena-hooks cleanup --client=claude-code" - } - ] - } - ] - } -} diff --git a/.claude/skills/clean-code b/.claude/skills/clean-code deleted file mode 120000 index f307457..0000000 --- a/.claude/skills/clean-code +++ /dev/null @@ -1 +0,0 @@ -/home/asephs/kana-best-practice-engineering/skills/clean-code \ No newline at end of file diff --git a/.claude/skills/commit-convention.md b/.claude/skills/commit-convention.md deleted file mode 100644 index e3c6251..0000000 --- a/.claude/skills/commit-convention.md +++ /dev/null @@ -1,65 +0,0 @@ ---- -name: commit-convention -description: Enforce commit message convention untuk Asepharyana Hub ---- - -# Commit Convention — Asepharyana Hub - -## Format - -``` -(): - -[optional body] - -[optional footer] -``` - -## Types - -| Type | Usage | -| ---------- | ------------------------------------ | -| `feat` | Fitur baru | -| `fix` | Bug fix | -| `chore` | Maintenance, config, tooling | -| `docs` | Dokumentasi | -| `refactor` | Perubahan kode tanpa fungsional baru | -| `test` | Nambah/update test | -| `ci` | CI/CD workflows | -| `perf` | Optimasi performa | -| `style` | Formatting (tanda kutip, dll) | - -## Scopes - -| Scope | Area | -| ------------- | --------------------------------- | -| `scraper` | apps/scraper submodule | -| `infra` | infra/ (compose, traefik, docker) | -| `ci` | .github/workflows/ | -| `dapr` | Dapr config & sidecar | -| `nats` | NATS message bus | -| `docs` | Dokumentasi | -| `deps` | Dependencies | -| `scripts` | Utility scripts | -| `root` | Root config files | - -## Contoh - -``` -feat(scraper): add anime detail caching via Dapr pubsub -fix(infra): correct NATS CLI flags for JetStream -chore(deps): update biome to v2.5.3 -docs(infra): add deployment order for Dapr services -ci(deploy): add nats.yml to ALL_COMPOSE_FILES -refactor(scraper): migrate EventBus from tokio broadcast to Dapr pubsub -``` - -## Aturan - -1. **Wajib** menyertakan scope dalam tanda kurung -2. **Wajib** `Co-Authored-By` untuk commit yang digenerate AI -3. **Gunakan imperative mood**: "add" bukan "added" / "adds" -4. **Jangan capitalize** type: `feat:` bukan `Feat:` -5. **No period** di akhir subject baris -6. Body explain **why** dan **what**, bukan **how** -7. Refer issue dengan `Closes #123` atau `Fixes #123` di footer diff --git a/.claude/skills/deploy-workflow.md b/.claude/skills/deploy-workflow.md deleted file mode 100644 index 9797943..0000000 --- a/.claude/skills/deploy-workflow.md +++ /dev/null @@ -1,103 +0,0 @@ ---- -name: deploy-workflow -description: Panduan deploy, CI/CD, dan Nix/systemd patterns untuk Asepharyana Hub ---- - -# Deploy & Workflow — Asepharyana Hub - -## CI/CD Pipeline - -### Build Pipeline (`docker-build-push.yml`) -Trigger: push ke `main` yang touch `apps/**`, `infra/**`, `infra/docker/**` - -1. **changes** — detect service mana yg berubah via git diff -2. **wait-submodule-ref** — (repository_dispatch only) tunggu SHA commit fetchable -3. **build** — matrix build per service, push ke GHCR (`sha-` + `latest`) -4. **update-manifest** — update image tag di compose file, commit + push - -### Deploy Pipeline (`deploy-docker.yml`) -Trigger: build selesai, atau push ke `main` touch `infra/**` - -1. SSH ke `orangevps` (via `secrets.VPS_HOST`) -2. Sync repo (`git fetch --depth=1 + reset`) -3. Login ke GHCR -4. Deteksi compose file yg berubah -5. Pull images + restart container selektif - -### Secrets Required -| Secret | Untuk | -|--------|-------| -| `SSH_PRIVATE_KEY` | SSH ke VPS | -| `VPS_HOST` | IP/host VPS (tailscale IP) | -| `VPS_USER` | SSH user, biasanya `root` | -| `VPS_TARGET_DIR` | Lokasi repo di VPS | -| `ENV_FILE_PRODUCTION` | .env content untuk production | - -### Selective Deployment -- Hanya compose file yg berubah yang di-redeploy -- Selective: `UP_FLAGS="-d"` (tanpa `--remove-orphans`) -- Full deploy: `UP_FLAGS="-d --remove-orphans"` - -## Docker Patterns - -### Build dengan cargo-chef (Rust) -```dockerfile -FROM lukemathwalker/cargo-chef:latest-rust-1.89.0 AS chef -WORKDIR /app -FROM chef AS planner -COPY apps/scraper . -RUN cargo chef prepare --recipe-path recipe.json -FROM chef AS builder -COPY --from=planner /app/recipe.json recipe.json -RUN cargo chef cook --release --recipe-path recipe.json -COPY apps/scraper . -RUN cargo build --release -``` - -### Runtime minimal untuk Rust binary -```dockerfile -FROM debian:bookworm-slim AS runtime -RUN apt-get update && apt-get install -y --no-install-recommends \ - ca-certificates curl libssl3 && rm -rf /var/lib/apt/lists/* -``` - -## Image Tagging -- `sha-` — immutable, untuk rollback -- `latest` — mutable, untuk convenience -- Build cache: `sha--buildcache` -- Registry: `ghcr.io/asepharyana/asepharyana-hub/` - -## Manual Deploy Steps -```bash -# 1. Login GHCR -echo $GITHUB_TOKEN | docker login ghcr.io -u asepharyana --password-stdin - -# 2. Full stack -docker compose -f infra/compose/traefik.yml \ - -f infra/compose/shared.yml \ - -f infra/compose/nats.yml \ - -f infra/compose/dapr.yml \ - -f infra/compose/scraper.yml \ - --env-file .env up -d --remove-orphans - -# 3. Selective (hanya satu service) -docker compose -f infra/compose/scraper.yml --env-file .env up -d -``` - -## Troubleshooting - -### Container reach Tailscale -Pastikan route ke Tailscale di main table: -```bash -ip route add 100.64.0.0/10 dev tailscale0 table main -systemctl restart tailscale-routes -``` - -### Healthcheck gagal di scratch images -NATS dan Dapr placement pake scratch — tidak bisa healthcheck. Cukup `service_started` di depends_on. - -### Dapr sidecar crash -```bash -docker logs scraper-api-dapr | grep -iE "fatal|error" -``` -Penyebab umum: komponen config salah, NATS/Dapr placement belum siap. diff --git a/.claude/skills/event-driven.md b/.claude/skills/event-driven.md deleted file mode 100644 index e170a60..0000000 --- a/.claude/skills/event-driven.md +++ /dev/null @@ -1,133 +0,0 @@ ---- -name: event-driven -description: Event-driven patterns dengan Dapr + NATS untuk Asepharyana Hub ---- - -# Event-Driven Architecture — Asepharyana Hub - -## Stack -- **Message Backbone**: NATS + JetStream (untuk streaming & job queue) -- **Pub/Sub Runtime**: Dapr sidecar per service (pubsub via Redis built-in) -- **State Store**: Dapr → Redis - -## Event Topics Convention - -``` -hub.. - -Contoh: -hub.image.cached → Image selesai di-cache ke CDN -hub.image.repaired → Image diperbaiki (CNAME change) -hub.scrape.anime.done → Scrape anime selesai -hub.system.alert → Error/alert dari service -``` - -## CloudEvents Format - -```json -{ - "specversion": "1.0", - "type": "hub.image.cached", - "source": "scraper-api", - "subject": "anime-poster", - "id": "uuid-v4", - "time": "2026-07-21T10:00:00Z", - "datacontenttype": "application/json", - "data": { ... } -} -``` - -## Publish Event (Rust via HTTP API) - -Gunakan `reqwest` langsung ke Dapr sidecar (SDK Rust masih experimental): - -```rust -let event = serde_json::json!({ - "specversion": "1.0", - "type": "hub.image.cached", - "source": "scraper-api", - "id": Uuid::new_v4().to_string(), - "time": chrono::Utc::now().to_rfc3339(), - "datacontenttype": "application/json", - "data": { "original_url": url, "cdn_url": cdn_url } -}); - -reqwest::Client::new() - .post("http://localhost:3500/v1.0/publish/pubsub/hub.image.cached") - .json(&event) - .send() - .await?; -``` - -## Service Invocation - -```bash -curl http://localhost:3500/v1.0/invoke//method/ -``` - -## State Store - -```bash -# Set -curl -X POST http://localhost:3500/v1.0/state/statestore \ - -H "Content-Type: application/json" \ - -d '[{"key": "mykey", "value": "myvalue"}]' - -# Get -curl http://localhost:3500/v1.0/state/statestore/mykey - -# Delete -curl -X DELETE http://localhost:3500/v1.0/state/statestore/mykey -``` - -## Scraper Event Integration - -File yang perlu dimodifikasi untuk event-driven: - -| File | Perubahan | -|------|-----------| -| `src/events/bus.rs` | Ganti backend dari tokio broadcast ke Dapr pub/sub | -| `src/bootstrap/mod.rs` | Init DaprClient, inject ke AppState | -| `src/presentation/state.rs` | Tambah `dapr_client` field | -| `src/proxy/use_cases.rs` | Publish `ImageRepaired` & `ImageCached` events | -| `src/infrastructure/services/images/cache.rs` | Emit event tiap cache selesai | -| `Cargo.toml` | Tambah `reqwest`, `uuid`, `chrono` (jika belum ada) | - -## Event Handlers (Subscribe) - -Buat `src/subscribers/` untuk handler: - -```rust -// src/subscribers/image_handler.rs -pub async fn handle_image_cached(event: CloudEvent) -> Result<()> { - // Log, notifikasi, update status -} -``` - -Daftarkan subscribers di `bootstrap/mod.rs` dengan spawn task: -```rust -tokio::spawn(async move { - let mut stream = dapr_client.subscribe("pubsub", "hub.image.cached"); - while let Some(event) = stream.next().await { - handle_image_cached(event).await; - } -}); -``` - -## Testing Event-Driven Code - -```rust -#[cfg(test)] -mod tests { - use super::*; - - #[tokio::test] - async fn test_publish_event() { - let client = MockDaprClient::new(); - client.expect_publish() - .with(...) - .returning(|_| Ok(())); - // ... test - } -} -``` diff --git a/.claude/skills/hub-rules.md b/.claude/skills/hub-rules.md deleted file mode 100644 index 270ab5a..0000000 --- a/.claude/skills/hub-rules.md +++ /dev/null @@ -1,96 +0,0 @@ ---- -name: hub-rules -description: Aturan repository, arsitektur hub, submodule, dan workflow Asepharyana Hub ---- - -# Asepharyana Hub — Repository Rules - -## Struktur Repository - -``` -asepharyana-hub/ -├── apps/ # Git submodules — source code aplikasi -├── docs/ # Dokumentasi, ADR, deployment guide -├── infra/ # Infrastructure as code -│ ├── compose/ # Satu compose file per service -│ ├── dapr/ # Dapr component configs -│ ├── docker/ # Dockerfiles (LEGACY — Docker dihapus) -│ ├── traefik/ # Traefik config (LEGACY — diganti Caddy) -│ └── caddy/ # Caddyfile.prod (reverse proxy produksi) -├── scripts/ # Utility scripts (cleanup, update-deps) -└── .github/workflows/ # CI/CD pipelines -``` - -### Aturan Submodule -- Setiap aplikasi di `apps/` adalah **submodule** ke repo terpisah. -- Perubahan kode aplikasi dilakukan di **repo masing-masing**, bukan di sini. -- Submodule pointer diupdate oleh CI/CD (bukan manual). - -## Infrastructure Patterns - -### Networking -- Semua service Nix/systemd, inter-service via 127.0.0.1: -- Caddy sebagai ingress untuk HTTP/S eksternal (auto-TLS LE, HTTP/3) -- Tailscale untuk cross-VPS (PostgreSQL, Redis) - -### Compose File Pattern -```yaml -services: - : - container_name: - image: ghcr.io/asepharyana/asepharyana-hub/:sha- - restart: always - networks: - app-shared-net: - aliases: - - - env_file: - - ../../.env - -networks: - app-shared-net: - name: app-shared-net - external: true -``` - -### Dapr Sidecar Pattern -```yaml - -dapr: - container_name: -dapr - image: daprio/daprd:latest - restart: always - depends_on: - nats: - condition: service_started - dapr-placement: - condition: service_started - networks: - - app-shared-net - command: - - './daprd' - - '--app-id=' - - '--app-port=' - - '--dapr-http-port=3500' - - '--dapr-grpc-port=50001' - - '--placement-host-address=dapr-placement:50005' - - '--resources-path=/components' - volumes: - - ../../infra/dapr/components:/components -``` - -### Caddy Routing -- Site block di `/etc/caddy/Caddyfile` (ref `infra/caddy/Caddyfile.prod`) -- Subdomain pattern: `.asepharyana.my.id` + `.asepharyana.web.id` -- TLS cert dari volume mount (bukan auto-acme) - -### CI/CD -- `docker-build-push.yml` — build per service, push ke GHCR, update compose manifest -- `deploy-docker.yml` — SSH ke orangevps, pull images, restart -- Selective deploy: hanya compose file yg berubah - -## Deployment Order -1. `shared.yml` (Redis) -2. `nats.yml` (NATS message bus) -3. `dapr.yml` (Dapr placement) -4. Caddy (reverse proxy) -5. Service compose files (apps + Dapr sidecar) diff --git a/.dockerignore b/.dockerignore deleted file mode 100644 index bb13b17..0000000 --- a/.dockerignore +++ /dev/null @@ -1,32 +0,0 @@ -**/.git -**/.gitmodules -**/node_modules -**/dist -**/.output -**/target -**/.svelte-kit -**/.next -**/.DS_Store -**/build -!apps/*/scripts/build/ -!apps/*/src/**/build/ -**/*.log -**/*.pem -.env -.env.* -!.env.example - -# IDE and temporary files -**/.vscode -**/.idea -**/tmp -**/temp -**/.cache -**/coverage -**/.npm -**/.bun -**/.pnpm-store -**/.yarn -**/.cargo-ok -**/*.swp -**/*~ diff --git a/.github/workflows/caddy-deploy.yml b/.github/workflows/caddy-deploy.yml new file mode 100644 index 0000000..5b46596 --- /dev/null +++ b/.github/workflows/caddy-deploy.yml @@ -0,0 +1,76 @@ +name: Deploy Caddy Config + +on: + push: + branches: [main] + paths: + - 'infra/caddy/**' + - 'infra/firewall/**' + - 'infra/systemd/**' + workflow_dispatch: + +concurrency: + group: caddy-deploy + cancel-in-progress: false + +permissions: + contents: read + +env: + VPS_HOST: ${{ secrets.VPS_HOST }} + VPS_USER: ${{ secrets.VPS_USER }} + +jobs: + deploy-caddy: + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@v7 + + - name: Setup SSH key + env: + SSH_KEY: ${{ secrets.SSH_PRIVATE_KEY }} + run: | + mkdir -p ~/.ssh + echo "$SSH_KEY" > ~/.ssh/id_ed25519 + chmod 600 ~/.ssh/id_ed25519 + sed -i 's/\r$//' ~/.ssh/id_ed25519 + ssh-keygen -y -f ~/.ssh/id_ed25519 >/dev/null 2>&1 || { echo "SSH key invalid"; exit 1; } + ssh-keyscan -H "$VPS_HOST" >> ~/.ssh/known_hosts 2>/dev/null + + - name: Validate Caddyfile syntax + run: | + # Basic sanity: no obviously empty file, brace count balanced + test -s infra/caddy/Caddyfile.prod || { echo "Caddyfile.prod missing/empty"; exit 1; } + opens=$(grep -c '{' infra/caddy/Caddyfile.prod || true) + closes=$(grep -c '}' infra/caddy/Caddyfile.prod || true) + echo "braces open=$opens close=$closes" + [ "$opens" = "$closes" ] || { echo "unbalanced braces"; exit 1; } + + - name: Sync Caddyfile to VPS + run: | + set -e + # Backup current config, then push the new one + ssh "$VPS_USER@$VPS_HOST" "sudo cp /etc/caddy/Caddyfile /etc/caddy/Caddyfile.bak-previous" + scp -q infra/caddy/Caddyfile.prod "$VPS_USER@$VPS_HOST":/tmp/Caddyfile.new + ssh "$VPS_USER@$VPS_HOST" "sudo cp /tmp/Caddyfile.new /etc/caddy/Caddyfile && sudo rm -f /tmp/Caddyfile.new" + echo "✅ Caddyfile synced" + + - name: Reload Caddy + run: | + ssh "$VPS_USER@$VPS_HOST" "sudo systemctl reload caddy || sudo systemctl restart caddy" + sleep 3 + ssh "$VPS_USER@$VPS_HOST" "systemctl is-active caddy" + + - name: Verify routes + run: | + set -e + for u in hub.asepharyana.my.id scraper.asepharyana.my.id tools.asepharyana.my.id wiki.asepharyana.my.id upload.asepharyana.my.id ai.asepharyana.my.id; do + code=$(curl -s -o /dev/null -w '%{http_code}' --max-time 10 "https://$u/" || true) + echo "$u -> $code" + # 000/000 means route didn't answer; 404 on root is fine for API-first apps + case "$code" in + 000|502|503|504) echo "::error::$u bad status $code"; exit 1 ;; + esac + done + echo "✅ All routes reachable" \ No newline at end of file diff --git a/.github/workflows/flakehub-publish-rolling.yaml b/.github/workflows/flakehub-publish-rolling.yaml deleted file mode 100644 index cb8c607..0000000 --- a/.github/workflows/flakehub-publish-rolling.yaml +++ /dev/null @@ -1,20 +0,0 @@ -name: Publish to FlakeHub - -on: - push: - branches: [main, master] - workflow_dispatch: - -jobs: - flakehub-publish: - runs-on: ubuntu-latest - permissions: - id-token: write - contents: read - steps: - - uses: actions/checkout@v7 - - uses: DeterminateSystems/determinate-nix-action@main - - uses: DeterminateSystems/flakehub-push@main - with: - visibility: public - rolling: true diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml deleted file mode 100644 index 8bae1a5..0000000 --- a/.github/workflows/lint.yml +++ /dev/null @@ -1,28 +0,0 @@ -name: Lint - -on: - workflow_dispatch: - pull_request: - branches: [main] - paths: - - 'biome.json' - - '*.json' - - '*.js' - push: - branches: [main] - paths: - - 'biome.json' - - '*.json' - - '*.js' - -jobs: - biome: - runs-on: ubuntu-latest - timeout-minutes: 10 - steps: - - uses: actions/checkout@v7 - with: - submodules: recursive - - uses: oven-sh/setup-bun@v2 - - run: bun install --frozen-lockfile - - run: bun run ci diff --git a/.github/workflows/nix-build.yml b/.github/workflows/nix-build.yml deleted file mode 100644 index 381b34b..0000000 --- a/.github/workflows/nix-build.yml +++ /dev/null @@ -1,103 +0,0 @@ -name: Nix Build & Deploy — All Services - -on: - # No `paths` filter: GitHub's path filters do not match submodule gitlink - # changes, so a submodule pointer update (e.g. from update-submodule.yml) - # would never trigger this deploy. Run on every push to main instead. - push: - branches: [main] - workflow_dispatch: - -concurrency: - group: nix-deploy - cancel-in-progress: false - -permissions: - contents: read - id-token: write - -env: - VPS_HOST: ${{ secrets.VPS_HOST }} - VPS_USER: ${{ secrets.VPS_USER }} - -jobs: - build-and-deploy: - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - service: [hub, scraper, tools-gateway, tools-workers, tools-frontend, llm-api] - - steps: - - name: Checkout - uses: actions/checkout@v7 - with: - submodules: recursive - fetch-depth: 0 - - - name: Install Nix - uses: DeterminateSystems/nix-installer-action@v22 - with: - determinate: false - extra-conf: | - sandbox = false - accept-flake-config = true - - - name: Cache Nix - uses: DeterminateSystems/magic-nix-cache-action@v14 - with: - use-flakehub: false - - - name: Build ${{ matrix.service }} - id: build - run: | - nix build .#${{ matrix.service }} --impure --option sandbox false --print-build-logs - STORE_PATH=$(readlink result) - echo "store-path=$STORE_PATH" >> "$GITHUB_OUTPUT" - echo "✅ ${{ matrix.service }}: $STORE_PATH" - - - name: Setup SSH key - if: github.ref == 'refs/heads/main' - env: - SSH_KEY: ${{ secrets.SSH_PRIVATE_KEY }} - run: | - mkdir -p ~/.ssh - echo "$SSH_KEY" > ~/.ssh/id_ed25519 - chmod 600 ~/.ssh/id_ed25519 - sed -i 's/\r$//' ~/.ssh/id_ed25519 - ssh-keygen -y -f ~/.ssh/id_ed25519 >/dev/null 2>&1 || { echo "SSH key invalid"; exit 1; } - ssh-keyscan -H "$VPS_HOST" >> ~/.ssh/known_hosts 2>/dev/null - - - name: Deploy ${{ matrix.service }} to VPS - if: github.ref == 'refs/heads/main' - run: | - STORE_PATH="${{ steps.build.outputs.store-path }}" - echo "=== Copying ${{ matrix.service }}: $STORE_PATH ===" - nix copy --to "ssh://$VPS_USER@$VPS_HOST" "$STORE_PATH" - - echo "=== Updating profile ===" - ssh "$VPS_USER@$VPS_HOST" "sudo /nix/var/nix/profiles/default/bin/nix-env --profile /nix/var/nix/profiles/${{ matrix.service }} --set '$STORE_PATH'" - - echo "=== Restarting service ===" - ssh "$VPS_USER@$VPS_HOST" "sudo systemctl restart ${{ matrix.service }}" || echo " ⚠️ restart failed (may not be enabled yet)" - - echo "✅ ${{ matrix.service }} deployed" - - cleanup: - # Bersihkan sampah Nix di VPS SETELAH deploy: hapus generasi profile lama - # + nix store gc. Profil yang sedang dipakai tidak disentuh. - needs: build-and-deploy - if: always() - runs-on: ubuntu-latest - steps: - - name: Nix GC on VPS - env: - VPS_HOST: ${{ secrets.VPS_HOST }} - VPS_USER: ${{ secrets.VPS_USER }} - SSH_KEY: ${{ secrets.SSH_PRIVATE_KEY }} - run: | - mkdir -p ~/.ssh - echo "$SSH_KEY" > ~/.ssh/id_ed25519 - chmod 600 ~/.ssh/id_ed25519 - ssh-keyscan -H "$VPS_HOST" >> ~/.ssh/known_hosts 2>/dev/null - ssh "$VPS_USER@$VPS_HOST" "sudo /usr/local/bin/nix-gc-vps.sh" || echo "⚠️ Nix GC gagal (non-fatal)" diff --git a/.github/workflows/security.yml b/.github/workflows/security.yml deleted file mode 100644 index 4118425..0000000 --- a/.github/workflows/security.yml +++ /dev/null @@ -1,30 +0,0 @@ -name: Security - -on: - pull_request: - branches: [main] - schedule: - - cron: '0 6 * * 1' # Every Monday - -jobs: - codeql: - runs-on: ubuntu-latest - timeout-minutes: 30 - permissions: - security-events: write - steps: - - uses: actions/checkout@v7 - with: - fetch-depth: 2 - submodules: recursive - - - uses: github/codeql-action/init@v4 - with: - languages: rust - - - name: Build Rust projects for CodeQL analysis - run: | - cargo build --manifest-path apps/scraper/Cargo.toml - cargo build --manifest-path apps/llm-api/Cargo.toml - - - uses: github/codeql-action/analyze@v4 diff --git a/.github/workflows/update-submodule.yml b/.github/workflows/update-submodule.yml deleted file mode 100644 index c3ee7cc..0000000 --- a/.github/workflows/update-submodule.yml +++ /dev/null @@ -1,86 +0,0 @@ -name: Update Submodule Pointer -on: - repository_dispatch: - types: [submodule-updated] - -permissions: - contents: write - -jobs: - update: - runs-on: ubuntu-latest - timeout-minutes: 10 - steps: - - uses: actions/checkout@v7 - - - name: Validate payload - env: - SERVICE: ${{ github.event.client_payload.service }} - SHA: ${{ github.event.client_payload.sha }} - run: | - set -euo pipefail - - if [ -z "${SERVICE:-}" ]; then - echo "::error::Missing service in payload" - exit 1 - fi - - if [ -z "${SHA:-}" ]; then - echo "::error::Missing sha in payload" - exit 1 - fi - - if ! [[ "$SHA" =~ ^[0-9a-fA-F]{40}$ ]]; then - echo "::error::Invalid sha '$SHA'. Expected 40 hex characters." - exit 1 - fi - - case "$SERVICE" in - scraper-api|hub|llm-api|tools) ;; - *) - echo "::error::Unsupported service '$SERVICE'" - exit 1 - ;; - esac - - echo "Payload validated: $SERVICE → $SHA" - - - name: Update submodule pointer - env: - SERVICE: ${{ github.event.client_payload.service }} - SHA: ${{ github.event.client_payload.sha }} - run: | - set -euo pipefail - - # Map service name to submodule path - case "$SERVICE" in - scraper-api) SUBMODULE_PATH="apps/scraper" ;; - llm-api) SUBMODULE_PATH="apps/llm-api" ;; - *) SUBMODULE_PATH="apps/${SERVICE}" ;; - esac - - echo "Updating ${SUBMODULE_PATH} to ${SHA}" - git submodule update --init "${SUBMODULE_PATH}" - cd "${SUBMODULE_PATH}" - git fetch --depth=1 origin master 2>/dev/null || git fetch --depth=1 origin main - git checkout "${SHA}" - cd "${GITHUB_WORKSPACE}" - git add "${SUBMODULE_PATH}" - git diff --cached --quiet && exit 0 - - git config user.name "monrepo-bot" - git config user.email "monrepo-bot@users.noreply.github.com" - git commit -m "chore: update ${SERVICE} to ${SHA:0:12}" - - for attempt in {1..3}; do - if git pull --rebase origin main && git push origin main; then - echo "✅ Push succeeded on attempt $attempt" - exit 0 - fi - echo "⚠️ Push attempt $attempt/3 failed; retrying..." - git rebase --abort 2>/dev/null || true - sleep 3 - done - - echo "::error::Failed to push submodule update after 3 attempts" - exit 1 diff --git a/.gitmodules b/.gitmodules deleted file mode 100644 index eced8d0..0000000 --- a/.gitmodules +++ /dev/null @@ -1,15 +0,0 @@ -[submodule "apps/scraper"] - path = apps/scraper - url = https://github.com/asepharyana/asepharyana-hub-scraper.git -[submodule "apps/hub"] - path = apps/hub - url = https://github.com/asepharyana/asepharyana-hub-hub.git -[submodule "apps/tools"] - path = apps/tools - url = https://github.com/asepharyana/asepharyana-hub-tools.git -[submodule "plugins/hub-guide"] - path = plugins/hub-guide - url = https://github.com/asepharyana/asepharyana-hub-guide.git -[submodule "apps/llm-api"] - path = apps/llm-api - url = https://github.com/asepharyana/asepharyana-hub-llm-api.git diff --git a/.kilo/skills/commit-convention/SKILL.md b/.kilo/skills/commit-convention/SKILL.md deleted file mode 100644 index 567f186..0000000 --- a/.kilo/skills/commit-convention/SKILL.md +++ /dev/null @@ -1,65 +0,0 @@ ---- -name: commit-convention -description: Commit message convention — type(scope): description for Asepharyana Hub ---- - -# Commit Convention — Asepharyana Hub - -## Format - -``` -(): - -[optional body] - -[optional footer] -``` - -## Types - -| Type | Usage | -| ---------- | ------------------------------------ | -| `feat` | Fitur baru | -| `fix` | Bug fix | -| `chore` | Maintenance, config, tooling | -| `docs` | Dokumentasi | -| `refactor` | Perubahan kode tanpa fungsional baru | -| `test` | Nambah/update test | -| `ci` | CI/CD workflows | -| `perf` | Optimasi performa | -| `style` | Formatting (tanda kutip, dll) | - -## Scopes - -| Scope | Area | -| ------------- | --------------------------------- | -| `scraper` | apps/scraper submodule | -| `infra` | infra/ (compose, traefik, docker) | -| `ci` | .github/workflows/ | -| `dapr` | Dapr config & sidecar | -| `nats` | NATS message bus | -| `docs` | Dokumentasi | -| `deps` | Dependencies | -| `scripts` | Utility scripts | -| `root` | Root config files | - -## Contoh - -``` -feat(scraper): add anime detail caching via Dapr pubsub -fix(infra): correct NATS CLI flags for JetStream -chore(deps): update biome to v2.5.3 -docs(infra): add deployment order for Dapr services -ci(deploy): add nats.yml to ALL_COMPOSE_FILES -refactor(scraper): migrate EventBus from tokio broadcast to Dapr pubsub -``` - -## Aturan - -1. **Wajib** menyertakan scope dalam tanda kurung -2. **Wajib** `Co-Authored-By` untuk commit yang digenerate AI -3. **Gunakan imperative mood**: "add" bukan "added" / "adds" -4. **Jangan capitalize** type: `feat:` bukan `Feat:` -5. **No period** di akhir subject baris -6. Body explain **why** dan **what**, bukan **how** -7. Refer issue dengan `Closes #123` atau `Fixes #123` di footer diff --git a/.kilo/skills/deploy-workflow/SKILL.md b/.kilo/skills/deploy-workflow/SKILL.md deleted file mode 100644 index 8f9acba..0000000 --- a/.kilo/skills/deploy-workflow/SKILL.md +++ /dev/null @@ -1,97 +0,0 @@ ---- -name: deploy-workflow -description: CI/CD pipeline, Docker build patterns, manual deploy steps, and troubleshooting for Asepharyana Hub ---- - -# Deploy & Workflow — Asepharyana Hub - -## CI/CD Pipeline - -### Build Pipeline (`docker-build-push.yml`) -Trigger: push ke `main` yang touch `apps/**`, `infra/**`, `infra/docker/**` - -1. **changes** — detect service mana yg berubah via git diff -2. **wait-submodule-ref** — (repository_dispatch only) tunggu SHA commit fetchable -3. **build** — matrix build per service, push ke GHCR (`sha-` + `latest`) -4. **update-manifest** — update image tag di compose file, commit + push - -### Deploy Pipeline (`deploy-docker.yml`) -Trigger: build selesai, atau push ke `main` touch `infra/**` - -1. SSH ke `orangevps` (via `secrets.VPS_HOST`) -2. Sync repo (`git fetch --depth=1 + reset`) -3. Login ke GHCR -4. Deteksi compose file yg berubah -5. Pull images + restart container selektif - -### Secrets Required -| Secret | Untuk | -|--------|-------| -| `SSH_PRIVATE_KEY` | SSH ke VPS | -| `VPS_HOST` | IP/host VPS (tailscale IP) | -| `VPS_USER` | SSH user, biasanya `root` | -| `VPS_TARGET_DIR` | Lokasi repo di VPS | -| `ENV_FILE_PRODUCTION` | .env content untuk production | - -### Selective Deployment -- Hanya compose file yg berubah yang di-redeploy -- Selective: `UP_FLAGS="-d"` (tanpa `--remove-orphans`) -- Full deploy: `UP_FLAGS="-d --remove-orphans"` - -## Docker Patterns - -### Build dengan cargo-chef (Rust) -```dockerfile -FROM lukemathwalker/cargo-chef:latest-rust-1.89.0 AS chef -WORKDIR /app -FROM chef AS planner -COPY apps/scraper . -RUN cargo chef prepare --recipe-path recipe.json -FROM chef AS builder -COPY --from=planner /app/recipe.json recipe.json -RUN cargo chef cook --release --recipe-path recipe.json -COPY apps/scraper . -RUN cargo build --release -``` - -### Runtime minimal untuk Rust binary -```dockerfile -FROM debian:bookworm-slim AS runtime -RUN apt-get update && apt-get install -y --no-install-recommends \ - ca-certificates curl libssl3 && rm -rf /var/lib/apt/lists/* -``` - -## Manual Deploy Steps -```bash -# 1. Login GHCR -echo $GITHUB_TOKEN | docker login ghcr.io -u asepharyana --password-stdin - -# 2. Full stack -docker compose -f infra/compose/traefik.yml \ - -f infra/compose/shared.yml \ - -f infra/compose/nats.yml \ - -f infra/compose/dapr.yml \ - -f infra/compose/scraper.yml \ - --env-file .env up -d --remove-orphans - -# 3. Selective (hanya satu service) -docker compose -f infra/compose/scraper.yml --env-file .env up -d -``` - -## Troubleshooting - -### Container reach Tailscale -Pastikan route ke Tailscale di main table: -```bash -ip route add 100.64.0.0/10 dev tailscale0 table main -systemctl restart tailscale-routes -``` - -### Healthcheck gagal di scratch images -NATS dan Dapr placement pake scratch — tidak bisa healthcheck. Cukup `service_started` di depends_on. - -### Dapr sidecar crash -```bash -docker logs scraper-api-dapr | grep -iE "fatal|error" -``` -Penyebab umum: komponen config salah, NATS/Dapr placement belum siap. diff --git a/.kilo/skills/event-driven/SKILL.md b/.kilo/skills/event-driven/SKILL.md deleted file mode 100644 index f1f895a..0000000 --- a/.kilo/skills/event-driven/SKILL.md +++ /dev/null @@ -1,133 +0,0 @@ ---- -name: event-driven -description: Event-driven architecture patterns with Dapr + NATS for Asepharyana Hub services ---- - -# Event-Driven Architecture — Asepharyana Hub - -## Stack -- **Message Backbone**: NATS + JetStream (untuk streaming & job queue) -- **Pub/Sub Runtime**: Dapr sidecar per service (pubsub via Redis built-in) -- **State Store**: Dapr → Redis - -## Event Topics Convention - -``` -hub.. - -Contoh: -hub.image.cached → Image selesai di-cache ke CDN -hub.image.repaired → Image diperbaiki (CNAME change) -hub.scrape.anime.done → Scrape anime selesai -hub.system.alert → Error/alert dari service -``` - -## CloudEvents Format - -```json -{ - "specversion": "1.0", - "type": "hub.image.cached", - "source": "scraper-api", - "subject": "anime-poster", - "id": "uuid-v4", - "time": "2026-07-21T10:00:00Z", - "datacontenttype": "application/json", - "data": { ... } -} -``` - -## Publish Event (Rust via HTTP API) - -Gunakan `reqwest` langsung ke Dapr sidecar (SDK Rust masih experimental): - -```rust -let event = serde_json::json!({ - "specversion": "1.0", - "type": "hub.image.cached", - "source": "scraper-api", - "id": Uuid::new_v4().to_string(), - "time": chrono::Utc::now().to_rfc3339(), - "datacontenttype": "application/json", - "data": { "original_url": url, "cdn_url": cdn_url } -}); - -reqwest::Client::new() - .post("http://localhost:3500/v1.0/publish/pubsub/hub.image.cached") - .json(&event) - .send() - .await?; -``` - -## Service Invocation - -```bash -curl http://localhost:3500/v1.0/invoke//method/ -``` - -## State Store - -```bash -# Set -curl -X POST http://localhost:3500/v1.0/state/statestore \ - -H "Content-Type: application/json" \ - -d '[{"key": "mykey", "value": "myvalue"}]' - -# Get -curl http://localhost:3500/v1.0/state/statestore/mykey - -# Delete -curl -X DELETE http://localhost:3500/v1.0/state/statestore/mykey -``` - -## Scraper Event Integration - -File yang perlu dimodifikasi untuk event-driven: - -| File | Perubahan | -|------|-----------| -| `src/events/bus.rs` | Ganti backend dari tokio broadcast ke Dapr pub/sub | -| `src/bootstrap/mod.rs` | Init DaprClient, inject ke AppState | -| `src/presentation/state.rs` | Tambah `dapr_client` field | -| `src/proxy/use_cases.rs` | Publish `ImageRepaired` & `ImageCached` events | -| `src/infrastructure/services/images/cache.rs` | Emit event tiap cache selesai | -| `Cargo.toml` | Tambah `reqwest`, `uuid`, `chrono` (jika belum ada) | - -## Event Handlers (Subscribe) - -Buat `src/subscribers/` untuk handler: - -```rust -// src/subscribers/image_handler.rs -pub async fn handle_image_cached(event: CloudEvent) -> Result<()> { - // Log, notifikasi, update status -} -``` - -Daftarkan subscribers di `bootstrap/mod.rs` dengan spawn task: -```rust -tokio::spawn(async move { - let mut stream = dapr_client.subscribe("pubsub", "hub.image.cached"); - while let Some(event) = stream.next().await { - handle_image_cached(event).await; - } -}); -``` - -## Testing Event-Driven Code - -```rust -#[cfg(test)] -mod tests { - use super::*; - - #[tokio::test] - async fn test_publish_event() { - let client = MockDaprClient::new(); - client.expect_publish() - .with(...) - .returning(|_| Ok(())); - // ... test - } -} -``` diff --git a/.kilo/skills/hub-rules/SKILL.md b/.kilo/skills/hub-rules/SKILL.md deleted file mode 100644 index 43980a9..0000000 --- a/.kilo/skills/hub-rules/SKILL.md +++ /dev/null @@ -1,101 +0,0 @@ ---- -name: hub-rules -description: Repository structure, submodule strategy, infrastructure patterns, and architecture of Asepharyana Hub ---- - -# Asepharyana Hub — Repository Rules - -## Struktur Repository - -``` -asepharyana-hub/ -├── apps/ # Git submodules — source code aplikasi -├── docs/ # Dokumentasi, ADR, deployment guide -├── infra/ # Infrastructure as code -│ ├── compose/ # Satu compose file per service -│ ├── dapr/ # Dapr component configs -│ ├── docker/ # Dockerfiles per service -│ └── traefik/ # Static & dynamic Traefik config -├── scripts/ # Utility scripts (cleanup, update-deps) -└── .github/workflows/ # CI/CD pipelines -``` - -### Aturan Submodule -- Setiap aplikasi di `apps/` adalah **submodule** ke repo terpisah. -- Perubahan kode aplikasi dilakukan di **repo masing-masing**, bukan di sini. -- Submodule pointer diupdate oleh CI/CD (bukan manual). - -## Infrastructure Patterns - -### Networking -- Semua service join **`app-shared-net`** (external Docker bridge) -- Service discovery via Docker DNS (container alias) -- Traefik sebagai ingress untuk HTTP/S eksternal -- Tailscale untuk cross-VPS (PostgreSQL, Redis) - -### Compose File Pattern -```yaml -services: - : - container_name: - image: ghcr.io/asepharyana/asepharyana-hub/:sha- - restart: always - networks: - app-shared-net: - aliases: - - - env_file: - - ../../.env - -networks: - app-shared-net: - name: app-shared-net - external: true -``` - -### Dapr Sidecar Pattern -```yaml - -dapr: - container_name: -dapr - image: daprio/daprd:latest - restart: always - depends_on: - nats: - condition: service_started - dapr-placement: - condition: service_started - networks: - - app-shared-net - command: - - './daprd' - - '--app-id=' - - '--app-port=' - - '--dapr-http-port=3500' - - '--dapr-grpc-port=50001' - - '--placement-host-address=dapr-placement:50005' - - '--resources-path=/components' - volumes: - - ../../infra/dapr/components:/components -``` - -### Traefik Routing -- Router + service definition di `infra/traefik/dynamic/apps.yaml` -- Subdomain pattern: `.asepharyana.my.id` + `.asepharya.web.id` -- TLS cert dari volume mount (bukan auto-acme) - -### Image Tagging -- `sha-` — immutable, untuk rollback -- `latest` — mutable, untuk convenience -- Registry: `ghcr.io/asepharyana/asepharyana-hub/` - -### CI/CD -- `docker-build-push.yml` — build per service, push ke GHCR, update compose manifest -- `deploy-docker.yml` — SSH ke orangevps, pull images, restart -- Selective deploy: hanya compose file yg berubah - -## Deployment Order -1. `shared.yml` (Redis) -2. `nats.yml` (NATS message bus) -3. `dapr.yml` (Dapr placement) -4. `traefik.yml` (Reverse proxy) -5. Service compose files (apps + Dapr sidecar) diff --git a/.node-version b/.node-version deleted file mode 100644 index 7af24b7..0000000 --- a/.node-version +++ /dev/null @@ -1 +0,0 @@ -22.11.0 diff --git a/.npmrc b/.npmrc deleted file mode 100644 index d9ca886..0000000 --- a/.npmrc +++ /dev/null @@ -1,2 +0,0 @@ -engine-strict=true -save-exact=true diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md deleted file mode 100644 index dbb1855..0000000 --- a/ARCHITECTURE.md +++ /dev/null @@ -1,324 +0,0 @@ -# Architecture - -## Hub Repository Structure Overview - -``` -asepharyana-hub/ -├── apps/ # Application services (Git submodules) -│ └── scraper/ # Web scraper service -├── docs/ # Documentation -│ ├── adr/ # Architecture Decision Records -│ ├── add-new-app.md # Guide for adding new services -│ └── superpowers/ # Project capabilities tracking -├── infra/ # Infrastructure as code (LEGACY Docker layout) -│ ├── compose/ # Docker Compose files (LEGACY — Docker dihapus 2026-08-02) -│ ├── config/ # Infrastructure configuration -│ ├── docker/ # Dockerfiles (LEGACY) -│ ├── traefik/ # Traefik config (LEGACY — diganti Caddy) -│ └── caddy/ # Caddyfile.prod (reverse proxy produksi) -├── scripts/ # Utility scripts -│ ├── git-hooks/ # Git hook scripts -│ ├── cleanup-ghcr.sh # GHCR image cleanup -│ └── update-deps.sh # Dependency update helper -├── .github/workflows/ # CI/CD pipelines -├── eslint.config.mjs # Root ESLint config -├── package.json # Root formatting/lint helper scripts -└── .prettierrc # Prettier formatting rules -``` - -## Technology Stack - -### Services - -| Service | Language/Runtime | Framework | Database | Key Libraries | -| --------- | ---------------- | --------- | -------- | ------------- | -| **scraper** | _(submodule)_ | — | — | — | - -### Infrastructure - -| Component | Technology | Purpose | -| ------------------ | ----------------------- | ---------------------------------------------------------------- | -| Reverse Proxy | Caddy 2.11.4 | TLS termination (auto-LE), routing, HTTP/3, keep-alive tuning | -| Runtime | Nix + systemd | Service isolation and orchestration (Docker dihapus 2026-08-02) | -| Deployment | GitHub Actions | nix build → nix copy ssh:// → systemctl restart | -| Secrets | Bitwarden Secrets Manager (BWS) | Central secret store, bws-exec wrapper | -| Networking | Tailscale | Secure overlay network between VPS nodes | -| Message Bus | NATS + JetStream | Event-driven pub/sub, job queues, streaming | -| Runtime Sidecar | Dapr | Service invocation, pub/sub abstraction, state management | -| Cache & State | Redis (Alpine) | Session store, rate limit counters, caching, Dapr state store | -| CI/CD | GitHub Actions | Build, test, deploy automation | - -## Infrastructure - -### Caddy Reverse Proxy - -Caddy 2.11.4 runs as the entry point for all HTTP/S traffic (systemd `caddy.service`, `/etc/caddy/Caddyfile`). It is configured via: - -- **Auto-TLS**: Let's Encrypt per-domain (email asepharyana@gmail.com) -- **HTTP/3**: h3 enabled on :443 (QUIC) -- **Snippet `(proxy)`**: shared handler — `encode zstd gzip`, security headers, keep-alive upstream (keepalive 120s, max_conns_per_host 100, dial_timeout 3s) -- **Upload domain** (`upload.asepharyana.my.id`): `flush_interval -1` (streaming), `request_body max_size 0` (unlimited) - -Reference: `infra/caddy/Caddyfile.prod`. Legacy Traefik configs stay under `infra/traefik/` for reference only. - -### Port Mapping (Produksi) - -| Service | Port | Domain | -|---------|------|--------| -| TeleUploader | 4000 | upload.asepharyana.my.id | -| GMW backend | 4001 | (internal) | -| pr-agent | 4002 | pr-agent.asepharyana.my.id | -| hub frontend | 4003 | asepharyana.my.id | -| lidm frontend | 4004 | lidm.asepharyana.my.id | -| lidm backend | 4005 | lidm-api.asepharyana.my.id | -| zeavis API | 4006 | api-zeavisedu.asepharyana.my.id | -| tools frontend | 4007 | tools.asepharyana.my.id | -| tools gateway | 4008 | (internal) | -| GMW proxy | 4009 | imphnen.asepharyana.my.id | -| llm-api | 4010 | ai.asepharyana.my.id | -| zeavisedu nginx | 4011 | zeavisedu.asepharyana.my.id | -| zeavis ML | 4012 | ml-zeavisedu.asepharyana.my.id | -| dashboard | 4013 | dashboard.asepharyana.my.id | -| 9router | 4014 | 9router.asepharyana.my.id | -| scraper | 4091 | scraper.asepharyana.my.id | - -### Nix + systemd Deployment - -Docker dihapus dari produksi (2026-08-02). Semua service deploy via Nix flakes + systemd: - -```bash -nix build .#default --impure --option sandbox false -nix copy --to ssh://vps /nix/store/ -systemctl restart -``` - -CI/CD: GitHub Actions (`deploy.yml`) → nix build → nix copy → systemctl restart. Flake dibatasi `x86_64-linux` (nixpkgs 26.11 drop darwin). - -### Tailscale Networking - -```mermaid -graph TB - subgraph "Tailnet (100.64.0.0/10)" - IMRNES["imrnes (100.121.180.82)"] - ORANGEVPS["orangevps (100.79.111.61)"] - ARCH["archlinux (100.84.39.83)"] - end - - subgraph "imrnes Services" - PG[(PostgreSQL)] - REDIS[Redis] - end - - subgraph "orangevps Services (Nix)" - CADDY[Caddy :443] - SCRAPER[scraper-api :4091] - end - - CADDY --> SCRAPER - - style IMRNES fill:#3a7,color:#fff - style ORANGEVPS fill:#37a,color:#fff - style ARCH fill:#773,color:#fff -``` - -Container-to-Tailscale connectivity requires a systemd service that adds a route to the main routing table: - -``` -ip route add 100.64.0.0/10 dev tailscale0 table main -``` - -This is managed by `/etc/systemd/system/tailscale-routes.service` on the `orangevps` VPS. - -## Data Flow - -### Request Flow (Production) - -```mermaid -sequenceDiagram - participant User as Browser/Client - participant DNS as Cloudflare DNS - participant Caddy as Caddy Proxy - participant App as Application Container - participant DB as PostgreSQL (imrnes via Tailscale) - participant Redis as Redis (imrnes via Tailscale) - - User->>DNS: asepharyana.my.id - DNS->>User: A/AAAA record → orangevps VPS IP - User->>Caddy: HTTPS request :443 - Caddy->>Caddy: TLS termination - Caddy->>Caddy: encode + headers - Caddy->>App: HTTP reverse-proxy (127.0.0.1:) - - alt Database query - App->>DB: sqlx/Drizzle query via Tailscale - DB-->>App: Result set - else Cache lookup - App->>Cache: GET/SET via Tailscale - Cache-->>App: Cached value - end - - App-->>Caddy: HTTP response - Caddy-->>User: HTTPS response -``` - -### CI/CD Pipeline - -```mermaid -flowchart LR - A[Push to main] --> B{Changed paths?} - B -->|apps/** or infra/docker/**| C[Build Docker Images] - B -->|infra/compose/**| D[Deploy to VPS] - B -->|apps/*/src/**/*.ts| E[Lint + TypeCheck] - - C --> F[Push to GHCR] - F --> G[Update Compose tags] - G --> D - - D --> H[SSH into VPS] - H --> I[Pull images] - I --> J[docker compose up -d] - - subgraph "Build Phase" - C - F - G - end - - subgraph "Deploy Phase" - D - H - I - J - end -``` - -## Deployment Architecture - -### Image Tags - -- Every push to `main` triggers Docker builds for changed services -- Images are tagged with both `latest` and `sha-` (e.g., `sha-b0ef947`) -- Compose files are auto-updated to pin the new SHA tag -- This enables deterministic rollbacks by reverting the compose file change - -### VPS Deployment - -The `orangevps` VPS (Tailscale `100.79.111.61`) hosts all application containers: - -1. GitHub Actions SSHes into the VPS -2. Production secrets are written as `.env` -3. The repo is synchronized via `git pull` -4. Changed compose files are detected by `git diff` -5. Docker images are pulled (with retry logic for transient failures) -6. Old containers are removed by `container_name` -7. `docker compose up -d` brings up the new containers -8. Traefik automatically detects the new containers via Docker provider - -### Selective Deployment - -The deploy workflow supports selective updates — if only one compose file changed, only the corresponding service is pulled and recreated, avoiding disruption to other services. - -```mermaid -graph TB - subgraph "Orange VPS" - DIR[/root/asepharyana-hub/] - ENV[.env] - COMPOSE[infra/compose/*.yml] - NET[app-shared-net] - - DIR -->|git pull| COMPOSE - ENV -->|docker compose --env-file| COMPOSE - COMPOSE -->|docker compose pull| IMAGES[(GHCR Images)] - COMPOSE -->|docker compose up -d| CONT[Containers] - CONT --> NET - end - - subgraph "GitHub Actions" - BUILD[Build & Push] - DEPLOY[Deploy Workflow] - BUILD -->|trigger| DEPLOY - DEPLOY -->|SSH| DIR - end - - IMAGES -->|registry| GHCR[ghcr.io/asepharyana] -``` - -## Submodule Strategy - -Each application lives in its own Git repository and is imported as a submodule into `apps/`. This approach: - -- **Enables independent development** — each service can be developed, tested, and versioned separately -- **Pins exact commits** — the super-repository tracks exact submodule SHAs, enabling reproducible deployments -- **Supports `repository_dispatch`** — when a submodule receives a push, it can trigger the super-repository to build and deploy only that service - -### Submodule Lifecycle - -1. Developer pushes to a submodule (e.g., `apps/scraper`) -2. Submodule's GitHub Action dispatches `repository_dispatch` to the super-repo with the service name and new SHA -3. Super-repo detects the dispatch, waits for the SHA to be fetchable, then builds only that service -4. The compose manifest is updated and committed with the new SHA tag -5. The deploy workflow runs and updates only the changed containers - -### Updating Submodules - -```bash -# Update a single submodule to latest -cd apps/scraper -git checkout main -git pull -cd ../.. -git add apps/scraper -git commit -m "chore(scraper): update submodule to latest" -``` - -## Service Mesh & Inter-Service Communication - -### HTTP (External + Internal via Traefik) -External traffic and internal HTTP calls route through Traefik. Services on `app-shared-net` can also communicate directly by container name. - -### Event-Driven (NATS + Dapr) -NATS with JetStream provides a persistent message backbone. Each service has a Dapr sidecar that abstracts pub/sub, service invocation, and state management. - -```mermaid -graph TB - subgraph "External" - WWW[Internet] - end - - subgraph "Orange VPS" - TRAEFIK[Traefik :443] - - subgraph "app-shared-net" - NATS[NATS + JetStream
:4222] - DAPR_PLACEMENT[Dapr Placement
:50005] - - subgraph "Service: scraper-api" - SCRAPER[scraper-api
:4091] - DAPR_SIDECAR[Dapr Sidecar
:3500] - SCRAPER --- DAPR_SIDECAR - end - end - - DAPR_SIDECAR -.->|gRPC pub/sub| NATS - DAPR_SIDECAR -.->|placement| DAPR_PLACEMENT - end - - WWW -->|HTTPS| TRAEFIK - TRAEFIK --> SCRAPER -``` - -### Communication Patterns - -| Pattern | Mechanism | Use Case | -|---------|-----------|----------| -| External HTTP | Traefik → Service | User requests, API calls | -| Internal HTTP | Service → Service (via Traefik or direct) | Synchronous queries | -| Pub/Sub Event | Dapr sidecar → NATS JetStream | Async notifications, image cache events | -| Service Invocation | Dapr sidecar gRPC | Cross-service RPC with retry & observability | -| State Store | Dapr → Redis | Shared state, job progress | - -## Observability - -- **Traefik access logs**: JSON format, logged at INFO level -- **Traefik access logs**: JSON format, logged at INFO level -- **Dashboard**: Traefik dashboard at `traefik.asepharyana.my.id` (secured) diff --git a/Makefile b/Makefile deleted file mode 100644 index fdafbd2..0000000 --- a/Makefile +++ /dev/null @@ -1,22 +0,0 @@ -.PHONY: help dev update-submodules deploy init-submodules status - -SHELL := /bin/bash - -help: ## Show this help - @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-20s\033[0m %s\n", $$1, $$2}' - -dev: ## Start development infrastructure (Redis etc.) - docker compose -f infra/compose/shared.yml up -d - -update-submodules: ## Update all git submodules to latest remote - git submodule update --remote --merge --recursive - -deploy: ## Deploy to VPS (triggers GitHub Actions) - @echo "Push to main to trigger deployment, or run:" - @echo " gh workflow run deploy-docker.yml" - -init-submodules: ## Initialize all submodules - git submodule update --init --recursive - -status: ## Show submodule status - git submodule status diff --git a/README.md b/README.md index 4d1d7db..1f076f5 100644 --- a/README.md +++ b/README.md @@ -1,232 +1,57 @@ -# Architecture +# Asepharyana Infra -## Hub Repository Structure Overview +Reverse-proxy & infrastructure config for [orangevps](https://asepharyana.my.id) (45.127.35.244). -```diff -asepharyana-hub/ -├── apps/ # Application services (Git submodules) -│ └── scraper/ # Web scraper service -├── docs/ # Documentation -│ ├── adr/ # Architecture Decision Records -│ ├── add-new-app.md # Guide for adding new services -│ └── superpowers/ # Project capabilities tracking -├── infra/ # Infrastructure as code -│ ├── compose/ # Docker Compose files per service -│ ├── config/ # Infrastructure configuration -│ ├── docker/ # Dockerfiles per service -│ └── traefik/ # Traefik reverse proxy config -│ └── dynamic/ # Dynamic routing rules (YAML) -├── scripts/ # Utility scripts -│ ├── git-hooks/ # Git hook scripts -│ ├── cleanup-ghcr.sh # GHCR image cleanup -│ └── update-deps.sh # Dependency update helper -├── .github/workflows/ # CI/CD pipelines -├── eslint.config.mjs # Root ESLint config -├── package.json # Root formatting/lint helper scripts -└── .prettierrc # Prettier formatting rules -``` +> **Status (2026-08-28):** Repo ini dulunya monorepo `asepharyana-hub` dengan submodule aplikasi. +> Kini **murni repo infra**: Caddy reverse proxy (source of truth), firewall, drop-in systemd, +> dan docs. Build + deploy tiap aplikasi pindah ke repo masing-masing (self-contained CI). -## Technology Stack +## Repositori Aplikasi (self-contained build & deploy) -### Services +| Repo | Deskripsi | Deploy unit | +|------|-----------|-------------| +| [`asepharyana/hub`](https://github.com/asepharyana/hub) | Portfolio SPA (Next.js, port 4003, dashboard) | `hub` | +| [`asepharyana/scraper`](https://github.com/asepharyana/scraper) | Rust/Axum scraper API (port 4091) | `scraper` | +| [`asepharyana/tools`](https://github.com/asepharyana/tools) | Tools: Rust gateway/workers + Next.js frontend (3500/3501) | `tools-gateway`, `tools-workers`, `tools-frontend` | +| [`asepharyana/llm-api`](https://github.com/asepharyana/llm-api) | Rust LLM API (llama.cpp, port 8080) | `llm-api` | -|| Service | Path | Language/Runtime | Framework | Database | Key Libraries | -||---------|----------------|------------------|-----------|----------|---------------| -|| **scraper** | `apps/scraper` | — | — | — | — | +Tiap repo punya `flake.nix` + `.github/workflows/deploy.yml` sendiri: +`nix build .#` → `nix copy ssh://` → `nix-env --profile` → `systemctl restart`. +Push ke `main` (atau `workflow_dispatch`) langsung deploy; tidak ada lagi pointer submodule. -### Infrastructure +## Infra di Repo Ini -|| Component | Technology | Purpose | -||---------------------|-------------------------|------------------------------------------------------------------| -|| Reverse Proxy | Traefik v3.6 | TLS termination, routing, middleware (rate-limit, headers, auth) | -|| Container Runtime | Docker + Docker Compose | Service isolation and orchestration | -|| Container Registry | GHCR (ghcr.io) | Docker image storage | -|| Networking | Tailscale | Secure overlay network between VPS nodes | -|| Message Bus | NATS + JetStream | Event-driven pub/sub, job queues, streaming | -|| Runtime Sidecar | Dapr | Service invocation, pub/sub abstraction, state management | -|| Cache & State | Redis (Alpine) | Session store, rate limit counters, caching, Dapr state store | -|| CI/CD | GitHub Actions | Build, test, deploy automation | +| Path | Isi | +|------|-----| +| `infra/caddy/Caddyfile.prod` | **Source of truth** `/etc/caddy/Caddyfile` (auto-deploy via CI) | +| `infra/firewall/firewall.sh` | deny-by-default iptables (SSH/80/443/4013/Tailscale/TCPShield) | +| `infra/firewall/99-*.conf` | sysctl hardenings | +| `infra/prometheus/targets.yml` | file_sd targets | +| `infra/systemd/scraper-otel.conf` | drop-in OTEL untuk scraper service | +| `docs/` | arsitektur + operasional (VPS) | -### Infrastructure +## CI/CD -### Traefik Reverse Proxy +| Workflow | Trigger | Aksi | +|----------|---------|------| +| `caddy-deploy.yml` | push main menyentuh `infra/**`, atau manual | sync `Caddyfile.prod` → `/etc/caddy/Caddyfile` → reload → verifikasi rute | -Traefik runs as the entry point for all HTTP/S traffic. It is configured via: - -- **Static config**: CLI arguments in `infra/compose/traefik.yml` — entry points, providers, plugins -- **Dynamic config**: `infra/traefik/dynamic/` — routers, services, middlewares, TLS -- **Docker provider**: Auto-discovers containers with `traefik.enable=true` labels -- **File provider**: Loads `apps.yaml` (routers/services), `middlewares.yaml`, `ssl.yaml` - -Key middleware chains (`infra/traefik/dynamic/middlewares.yaml`): - -- `secure-headers` — SSL redirect, HSTS, XSS protection, CSP -- `compress` — Gzip compression for responses over 256 bytes -- `rate-limit` — 100 avg / 50 burst requests -- `buffer` — 10MB request/response body limit -- `block-sensitive-paths` — blocks `.env`, `.git`, `/wp-admin` etc. -- `common-chain` — composes secure-headers + compress + retry + rate-limit + buffer - -All services route through Traefik on port 443 (TLS), with automatic HTTP-to-HTTPS redirect. - -### Docker Compose - -Each service has its own Compose file under `infra/compose/`. All services join the `app-shared-net` external Docker network, enabling inter-service communication by container name. - -Shared services: - -- `infra/compose/shared.yml` — Redis (alias: `redis`) -- `infra/compose/traefik.yml` — Traefik reverse proxy - -Service compose files are combined during deployment: +## Local Setup / Snapshot VPS ```bash -docker compose -f traefik.yml -f shared.yml -f scraper.yml up -d +# Clone infra repo +git clone https://github.com/asepharyana/infra.git +# Diff config live vs repo +diff /etc/caddy/Caddyfile infra/caddy/Caddyfile.prod +# Koneksi VPS (public) +ssh code@45.127.35.244 ``` -### Tailscale Networking +## Menambahkan Service Baru / Subdomain -### Arsitektur +1. Aplikasi punya repo sendiri + `deploy.yml` (lihat template di repo app yang ada). +2. Registrasi unit systemd di VPS (manual/ops) → app jalan di port lokal. +3. Tambah site block di `infra/caddy/Caddyfile.prod` (pola `import proxy `) → push → CI reload Caddy. +4. (Opsional) Tambah unit ke `MONITORED_UNITS` dashboard hub di repo `asepharyana/hub`. -Semua VPS terhubung via **Tailscale**. Setiap VPS punya IP Tailscale dan service berkomunikasi antar VPS melalui Tailscale network (`100.64.0.0/10`). Container-to-Tailscale connectivity requires a systemd service that adds a route to the main routing table: - -```bash -ip route add 100.64.0.0/10 dev tailscale0 table main -``` - -This is managed by `/etc/systemd/system/tailscale-routes.service` on the `orangevps` VPS. - -### Data Flow - -### Request Flow (Production) - -```mermaid -sequenceDiagram - participant User as Browser/Client - participant DNS as Cloudflare DNS - participant Traefik as Traefik Proxy - participant App as Application Container - participant DB as PostgreSQL (imrnes via Tailscale) - participant Redis as Redis (imrnes via Tailscale) - - User->>DNS: asepharyana.my.id - DNS->>User: A/AAAA record → orangevps VPS IP - User->>Traefik: HTTPS request :443 - Traefik->>Traefik: TLS termination - Traefik->>Traefik: Middleware chain (headers, rate-limit, buffer) - Traefik->>App: HTTP reverse-proxy (internal network) - - alt Database query - App->>DB: sqlx/Drizzle query via Tailscale - DB-->>App: Result set - else Cache lookup - App->>Cache: GET/SET via Tailscale - Cache-->>App: Cached value - end - - App-->>Traefik: HTTP response - Traefik-->>User: HTTPS response -``` - -### CI/CD Pipeline - -```mermaid -flowchart LR - A[Push to main] --> B{Changed paths?} - B -->|apps/** or infra/docker/**| C[Build Docker Images] - B -->|infra/compose/**| D[Deploy to VPS] - B -->|apps/*/src/**/*.ts| E[Lint + TypeCheck] - - C --> F[Push to GHCR] - F --> G[Update Compose tags] - G --> D - - D --> H[SSH into VPS] - H --> I[Pull images] - I --> J[docker compose up -d] - - subgraph "Build Phase" - C - F - G - end - - subgraph "Deploy Phase" - D - H - I - J - end -``` - -### Deployment Architecture - -### Image Tags - -- `latest` — mutable, for convenience -- `sha-` — immutable, for deterministic rollbacks -- Build cache: `sha--buildcache` - -Registry: `ghcr.io/asepharyana/asepharyana-hub/` - -## Deployment Notes - -- Pipeline memakai image tag berbasis commit SHA (`sha-`), bukan `latest`. -- Deploy Compose sekarang mencakup `infra/compose/*.yml` dan `deploy-docker.yml` akan berjalan langsung ketika `infra/compose/**` berubah. -- Selective deployment: hanya compose file yg berubah yang di-redeploy. - -## Networking & Tailscale - -### Arsitektur - -Semua VPS terhubung via **Tailscale**. Setiap VPS punya IP Tailscale dan service berkomunikasi antar VPS melalui Tailscale network (`100.64.0.0/10`). Container-to-Tailscale connectivity requires a systemd service that adds a route to the main routing table: - -```bash -ip route add 100.64.0.0/10 dev tailscale0 table main -``` - -This is managed by `/etc/systemd/system/tailscale-routes.service` on the `orangevps` VPS. - -### Environment Variables - -Service yang connect ke Tailscale IP: - -```env -# PostgreSQL di imrnes -DATABASE_URL=postgres://user:***@100.121.180.82:6432/dbname - -# Redis di imrnes -REDIS_URL=redis://100.121.180.82:6379 -``` - -## Submodule Strategy - -Each application lives in its own Git repository and is imported as a submodule into `apps/`. This approach: - -- **Enables independent development** — each service can be developed, tested, and versioned separately -- **Pins exact commits** — the super-repository tracks exact submodule SHAs, enabling reproducible deployments -- **Supports `repository_dispatch`** — when a submodule receives a push, it can trigger the super-repository to build and deploy only that service - -### Submodule Lifecycle - -1. Developer pushes to a submodule (e.g., `apps/scraper`) -2. Submodule's GitHub Action dispatches `repository_dispatch` to the super-repo with the service name and new SHA -3. Super-repo detects the dispatch, waits for the SHA to be fetchable, then builds only that service -4. The compose manifest is updated and committed with the new SHA tag -5. The deploy workflow runs and updates only the changed containers - -### Updating Submodules - -```bash -# Update a single submodule to latest -cd apps/scraper -git checkout main -git pull -cd ../.. -git add apps/scraper -git commit -m "chore(scraper): update submodule to latest" -``` - -## License - -MIT \ No newline at end of file +Lihat `docs/add-new-app.md` untuk detail. \ No newline at end of file diff --git a/apps/hub b/apps/hub deleted file mode 160000 index 5531076..0000000 --- a/apps/hub +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 55310768a5ac00d3d2799f6e2d431253aee92760 diff --git a/apps/llm-api b/apps/llm-api deleted file mode 160000 index 5f7ead5..0000000 --- a/apps/llm-api +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 5f7ead5503082a71d41a36fd1727325c784e4b79 diff --git a/apps/scraper b/apps/scraper deleted file mode 160000 index 62aa5b0..0000000 --- a/apps/scraper +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 62aa5b0e52859afe3ba9de1c7b11cfe2dacf6c2c diff --git a/apps/tools b/apps/tools deleted file mode 160000 index 036f67d..0000000 --- a/apps/tools +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 036f67d05acb11e8815e365f0aabc04034e74175 diff --git a/biome.json b/biome.json deleted file mode 100644 index 2d34922..0000000 --- a/biome.json +++ /dev/null @@ -1,73 +0,0 @@ -{ - "$schema": "https://biomejs.dev/schemas/2.5.3/schema.json", - "assist": { "actions": { "source": { "organizeImports": "on" } } }, - "linter": { - "enabled": true, - "rules": { - "preset": "recommended", - "a11y": { - "useButtonType": "warn", - "useIframeTitle": "warn", - "noSvgWithoutTitle": "warn", - "useAltText": "warn" - }, - "complexity": { - "noForEach": "off", - "noStaticOnlyClass": "off", - "noThisInStatic": "off", - "useOptionalChain": "off" - }, - "suspicious": { - "noArrayIndexKey": "warn", - "noConsole": "off", - "noExplicitAny": "warn" - }, - "correctness": { - "noUnusedVariables": "error", - "useParseIntRadix": "off" - }, - "style": { - "noNonNullAssertion": "off" - } - } - }, - "formatter": { - "enabled": true, - "formatWithErrors": false, - "indentStyle": "space", - "indentWidth": 2, - "lineWidth": 100, - "lineEnding": "lf" - }, - "javascript": { - "jsxRuntime": "transparent", - "formatter": { - "quoteStyle": "single", - "jsxQuoteStyle": "double", - "trailingCommas": "all", - "semicolons": "always", - "arrowParentheses": "always" - } - }, - "css": { - "parser": { - "tailwindDirectives": true - } - }, - "files": { - "ignoreUnknown": false, - "includes": [ - "**", - "!**/.opencode", - "!**/dist", - "!**/out", - "!**/build", - "!**/node_modules", - "!**/target", - "!**/coverage", - "!**/*.env", - "!**/*.env.*", - "!**/apps/react/src/routeTree.gen.ts" - ] - } -} diff --git a/bun.lock b/bun.lock deleted file mode 100644 index 1003829..0000000 --- a/bun.lock +++ /dev/null @@ -1,31 +0,0 @@ -{ - "lockfileVersion": 1, - "configVersion": 1, - "workspaces": { - "": { - "name": "ultimate-asepharyana.tech", - "devDependencies": { - "@biomejs/biome": "2.5.3", - }, - }, - }, - "packages": { - "@biomejs/biome": ["@biomejs/biome@2.5.3", "", { "optionalDependencies": { "@biomejs/cli-darwin-arm64": "2.5.3", "@biomejs/cli-darwin-x64": "2.5.3", "@biomejs/cli-linux-arm64": "2.5.3", "@biomejs/cli-linux-arm64-musl": "2.5.3", "@biomejs/cli-linux-x64": "2.5.3", "@biomejs/cli-linux-x64-musl": "2.5.3", "@biomejs/cli-win32-arm64": "2.5.3", "@biomejs/cli-win32-x64": "2.5.3" }, "bin": { "biome": "bin/biome" } }, "sha512-MrJswFdei9EfDwwUy2tQrPDpK0AO+RmMFvBoaaJ6ayBc3sUbHdCE+XG5N8vp+5So41ZupZJQm0roHFFhMGVD7A=="], - - "@biomejs/cli-darwin-arm64": ["@biomejs/cli-darwin-arm64@2.5.3", "", { "os": "darwin", "cpu": "arm64" }, "sha512-QhYP9muVQ0nUO5zztFuPbEwi4+94sJWVjaZds9aMi1l/KNZBiUjdiSUrGHsTaMGDXrYl+r4AS2sUKfgH3w+V3g=="], - - "@biomejs/cli-darwin-x64": ["@biomejs/cli-darwin-x64@2.5.3", "", { "os": "darwin", "cpu": "x64" }, "sha512-NC1Ss13UaW7QZX+y8j44bF7AP0jSJdBl6iRhe0MAkvaSqZy+mWg3GaXsrb+eSoHoGDBtaXWEbMVV0iVN2cZ7cQ=="], - - "@biomejs/cli-linux-arm64": ["@biomejs/cli-linux-arm64@2.5.3", "", { "os": "linux", "cpu": "arm64" }, "sha512-ksx1KWeyYW18ILL04msF/J4ZBtBDN33znYK8Z/aNv/vlBVxL9/g3mGP+omgHJKy4+KWbK87vcmmpmurfNjSgiA=="], - - "@biomejs/cli-linux-arm64-musl": ["@biomejs/cli-linux-arm64-musl@2.5.3", "", { "os": "linux", "cpu": "arm64" }, "sha512-fccix0w6xp6csCXgxeC0dU/3ecgRQal0y+cv2SP9ajNlhe7Yrk2Ug7UDe2j9AT9ZDYitkXpvUKgZjjuoYeP4Vg=="], - - "@biomejs/cli-linux-x64": ["@biomejs/cli-linux-x64@2.5.3", "", { "os": "linux", "cpu": "x64" }, "sha512-yMkJtilsgvILDcVkh187aVLTb64xYsrxYajx5kym+r1ULkO5HUOfu9AYKLGQbOVLwJtT2utNw7hhFNg+17mUYA=="], - - "@biomejs/cli-linux-x64-musl": ["@biomejs/cli-linux-x64-musl@2.5.3", "", { "os": "linux", "cpu": "x64" }, "sha512-O/yU9YKRUiHhmcjF2f38PSjseVk3G4VLWYc0G2HWpzdBVREV6G8IGWIVEFf7MFPfWIzNUIvPsEjeAZQIOgnLcQ=="], - - "@biomejs/cli-win32-arm64": ["@biomejs/cli-win32-arm64@2.5.3", "", { "os": "win32", "cpu": "arm64" }, "sha512-cX5z+GYwRcqEok0AH3KSfQGgqYd0Nomfp6Fbe1uiTtELE38hdH2k842wQ9wLNaF/JJ7r4rjJQ4VR+ce+fRmQbw=="], - - "@biomejs/cli-win32-x64": ["@biomejs/cli-win32-x64@2.5.3", "", { "os": "win32", "cpu": "x64" }, "sha512-ExSaJWi4/u6+GXCszlSKpWSjKNbDseAYqqkCznsCsZ/4uidZ/BEqsCc5/3ctlq6dfIubdIIRSVLC/PG9xPl70Q=="], - } -} diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md deleted file mode 100644 index 13533b4..0000000 --- a/docs/ARCHITECTURE.md +++ /dev/null @@ -1,238 +0,0 @@ -# Arsitektur asepharyana-hub - -## Topologi Fisik - -Dua node terhubung via **Tailscale** overlay network: - -``` -┌──────────────────────────────┐ ┌──────────────────────────────┐ -│ orangevps (VPS) │ │ imrnes (Bare-metal) │ -│ IP: 45.127.35.244 │ │ Tailscale: 100.121.180.82 │ -│ Tailscale: 100.x.x.x │◄──────┤ │ -│ │ │ Layanan: │ -│ Layanan: │ │ ├─ PostgreSQL (port 6432) │ -│ ├─ Caddy (port 80/443) │ │ └─ Redis (port 6379) │ -│ ├─ NATS + JetStream │ │ │ -│ ├─ Dapr Placement │ └──────────────────────────────┘ -│ ├─ Redis (cache, Dapr) │ -│ ├─ Scraper API + Dapr │ -│ └─ Hub (Next.js SPA) │ -└──────────────────────────────┘ -``` - -### Konektivitas Container ke Tailscale - -Container di `orangevps` tidak bisa langsung mencapai IP Tailscale (`100.x.x.x`). Route Tailscale harus ditambahkan ke tabel routing utama (`main`) via `tailscale-routes.service` agar traffic dari container bisa melewati host ke Tailscale. - -## Alur Request HTTP (External) - -``` -Internet - │ - ▼ Port 443 -Caddy 2.11.4 (auto-TLS LE, HTTP/3) - ├─ TLS termination (sertifikat dari volume mount) - ├─ Middleware chain: secure-headers → compress → retry → rate-limit → buffer - ├─ Plugin: real-ip (Cloudflare), block-sensitive-paths - │ - ▼ Router matching -Host(`asepharyana.my.id`) || Host(`www.asepharyana.my.id`) → hub -host(`hub.asepharyana.my.id`) → hub (SPA + dashboard) -Host(`scraper.asepharyana.my.id`) || Host(`api.asepharyana.my.id`) → scraper-api - │ - ├─ hub (Next.js, port 4003) - │ ├─ / — Portfolio SPA - │ ├─ /dashboard — Ops dashboard (client-side, auto-refresh 15s) - │ ├─ /api/dashboard — JSON: systemd services, Jaeger traces, Prometheus metrics - │ └─ Metrics via node-exporter + app endpoints - │ - ▼ Service load balancer -http://scraper-api:4091 - │ - ▼ -Scraper API (Rust / Axum) - ├─ Health check: GET /, respon 200 - ├─ REST endpoints - ├─ Database via `DATABASE_URL` (Tailscale → PostgreSQL di imrnes) - ├─ Cache via `REDIS_URL` (Redis lokal di container) - └─ Pub/sub via Dapr sidecar (localhost:3500) -``` - -## Infrastruktur Internal - -### Docker Compose Project - -Semua service berjalan dalam satu Docker Compose project bernama `compose` dan bergabung di network `app-shared-net`: - -| File | Service | Peran | -|------|---------|-------| -| `traefik.yml` | `traefik` | Reverse proxy + TLS + metrics Prometheus | -| `shared.yml` | `redis` | Cache, session store, backend Dapr pub/sub & state | -| `nats.yml` | `nats` | Message broker + JetStream persistent streaming | -| `dapr.yml` | `dapr-placement` | Koordinasi actor placement untuk sidecar Dapr | -| `scraper.yml` | `scraper-api` + `scraper-api-dapr` | Aplikasi Rust + sidecar Dapr | -| systemd hub | `hub` | Next.js SPA portfolio + dashboard | -| `observability.yml` | `otel-collector`, `jaeger`, `prometheus`, `node-exporter` | Tracing, metrics, observability | - -### Dapr Sidecar Pattern - -Setiap aplikasi yang menggunakan Dapr mendapat sidecar container `daprd`: - -``` -┌─────────────────────┐ -│ scraper-api │ -│ (app port 4091) │ -└────────┬────────────┘ - │ localhost:3500 (HTTP) - │ localhost:50001 (gRPC) -┌────────▼────────────┐ -│ scraper-api-dapr │ -│ (daprd sidecar) │ -│ │ -│ Dapr components: │ -│ ├─ pubsub.redis │ -│ └─ state.redis │ -└─────────────────────┘ -``` - -Komponen Dapr: - -| Komponen | Tipe | Backend | -|----------|------|---------| -| `pubsub` | `pubsub.redis` | `redis:6379` | -| `statestore` | `state.redis` | `redis:6379` (prefix `dapr`) | - -### Monitoring & Auto-Discovery - -#### Prometheus Docker Auto-Discovery - -Prometheus menggunakan `docker_sd_configs` untuk auto-detect container yang perlu di-scrape. Cukup tambah label pada container: - -```yaml -labels: - - 'prometheus.io/scrape=true' - - 'prometheus.io/port=8080' # port metrics endpoint - - 'prometheus.io/path=/metrics' # optional, default /metrics -``` - -Prometheus akan auto-detect dan mulai scrape container dalam 15 detik. - -#### Traefik Metrics - -Traefik mengekspos metrics Prometheus di port 8080 (`--metrics.prometheus=true`). Metrics yang tersedia: - -| Metric | Query untuk dashboard | -|--------|----------------------| -| Request rate | `sum(rate(traefik_service_requests_total[1m]))` | -| Latency | `avg(traefik_service_request_duration_seconds_sum / traefik_service_request_duration_seconds_count) * 1000` | -| Error rate | `sum(rate(traefik_service_requests_total{code=~"5.."}[1m]))` | - -Dashboard di `/api/dashboard` returns node metrics + Traefik range data untuk 4 sparkline charts (RPS, latency, errors, trace volume). - -#### Docker Socket Access - -Container yang perlu akses Docker socket (`/var/run/docker.sock`) harus punya group docker (GID 988): - -```yaml -volumes: - - /var/run/docker.sock:/var/run/docker.sock:ro -group_add: - - '988' -```di compose atau `--group-add 988` via CLI. Berlaku untuk `hub` (container list) dan `prometheus` (Docker SD). - -### NATS + JetStream - -NATS berjalan dengan flag `-js` untuk mengaktifkan JetStream. Persistent stream disimpan di volume `nats_data`. Dapr pub/sub routing: - -``` -Service → Dapr sidecar (pubsub.redis) → Redis streams -``` - -> **Catatan:** Saat ini Dapr pub/sub menggunakan Redis, bukan NATS. Jika ingin migrasi ke NATS untuk pub/sub, komponen Dapr perlu diganti dengan `pubsub.nats`. - -## Arsitektur CI/CD - -``` -Push ke main (apps/**, infra/**) - │ - ▼ -docker-build-push.yml - ├─ Phase 1: Detect changed services - ├─ Phase 2: Build & Push image ke GHCR - └─ Phase 3: Update compose manifest + submodule pointer - │ - ▼ (workflow_run trigger) -deploy-docker.yml - ├─ SSH ke orangevps - ├─ Git sync, pull images - ├─ Remove stale containers - └─ Selective restart service -``` - -Submodule update dari remote repo via `repository_dispatch`: - -``` -Push ke asepharyana-hub-scraper - │ - ▼ (repository_dispatch) -update-submodule.yml - ├─ Update submodule pointer - └─ Commit & push ke hub repo - │ - ▼ (repository_dispatch trigger) -docker-build-push.yml - └─ Build, push, deploy -``` - -## Image Tagging Strategy - -| Tag | Contoh | Penggunaan | -|-----|--------|------------| -| `sha-` | `sha-a3c5d74` | Immutable, deterministic rollback | -| `latest` | `latest` | Mutable, convenience | -| `buildcache` | `sha-a3c5d74-buildcache` | Registry-based build cache (internal) | - -## Networking - -### Port Map - -| Port | Service | Deskripsi | -|------|---------|-----------| -| 443 | Traefik | HTTPS eksternal | -| 80 | Traefik | Redirect ke HTTPS | -| 4222 | NATS | Client connections | -| 8222 | NATS | HTTP monitor / health | -| 6379 | Redis | Internal container network | -| 3500 | Dapr sidecar | Dapr HTTP API (per service) | -| 50001 | Dapr sidecar | Dapr gRPC API (per service) | -| 50005 | Dapr placement | Actor placement | -| 4091 | Scraper API | Aplikasi HTTP | - -## Event Topics Convention - -Semua event menggunakan prefix `hub.`: - -| Topic | Payload | Deskripsi | -|-------|---------|-----------| -| `hub.image.cached` | `{original_url, cdn_url, source}` | Image selesai di-cache | -| `hub.image.repaired` | `{old_url, new_url}` | CNAME image diperbaiki | -| `hub.scrape.anime.done` | `{source, slug, duration}` | Scrape anime selesai | -| `hub.system.alert` | `{service, level, message}` | Error/alert dari service | - -## Service Registry (Traefik) - -Domain routing: - -| Subdomain | Service | URL Backend | -|-----------|---------|-------------| -| `asepharyana.my.id` (root) | Hub SPA + dashboard | `http://hub:3000` | -| `www.*` | Hub (alias) | `http://hub:3000` | -| `hub.*` | Hub (alias) | `http://hub:3000` | -| `scraper.*` | Scraper API | `http://scraper-api:4091` | -| `api.*` | Scraper API (alias) | `http://scraper-api:4091` | -| `traefik.*` | Traefik Dashboard | `api@internal` | -| `jaeger.*` | Jaeger UI | `http://jaeger:16686` | - -Semua domain tersedia di: -- `.asepharyana.my.id` -- `.asepharyana.web.id` diff --git a/docs/DEPLOYMENT.md b/docs/DEPLOYMENT.md deleted file mode 100644 index 627ae3d..0000000 --- a/docs/DEPLOYMENT.md +++ /dev/null @@ -1,681 +0,0 @@ -# Deployment Guide - -Panduan deploy aplikasi apapun menggunakan **Docker + Docker Compose + GitHub Actions + VPS**. - -## Arsitektur - -``` -GitHub Repo ──► GitHub Actions ──► Registry (GHCR / Docker Hub / ECR / dll.) - │ - ▼ - VPS () - docker compose pull + up -``` - -## Prerequisites - -- Docker Engine >= 24.x -- Docker Compose v2 (plugin) -- Git -- Akun GitHub dengan akses repo -- SSH key di `~/.ssh/` (default: `id_ed25519`) - -## Konfigurasi VPS Target - -Buat berkas `~/orangevps` (atau sesuaikan dengan env Anda): - -```text -ssh @ -``` - -Contoh isi `~/orangevps`: - -```text -ssh root@45.127.35.244 -``` - -| Parameter | Nilai | Contoh | -|-----------|-------|--------| -| User | `` | `root` | -| Host | `` | `45.127.35.244` | -| SSH Key | `~/.ssh/` | `~/.ssh/id_ed25519` | -| Target Dir di VPS | `` | `/opt/app` atau `/root/app` | - -> Tip: Jika SSH key menggunakan nama selain default, sesuaikan path dan `ssh -i` sesuai. - -## Registry - -Pilih registry untuk menyimpan image Docker. Sesuaikan dengan proyek: - -| Registry | URL | Auth | -|----------|-----|------| -| GitHub Container Registry | `ghcr.io` | `GITHUB_TOKEN` | -| Docker Hub | `docker.io` | username / PAT | -| AWS ECR | `.dkr.ecr..amazonaws.com` | `aws ecr get-login-password` | -| Google GCR | `gcr.io` | `gcloud auth print-access-token` | -| Azure ACR | `.azurecr.io` | `az acr login` | - -Contoh namespace untuk GHCR: - -```text -Registry : ghcr.io -Namespace: -Repo : -``` - -Pastikan package/visibility di registry mengizinkan akses pull dari VPS. - ---- - -## Deploy Otomatis (Recommended) - -Gunakan GitHub Actions untuk otomatisasi build, push, dan deploy. - -### Workflow 1: Build dan Push Image - -File: `.github/workflows/docker-build-push.yml` - -```yaml -name: Build and Push Docker Images - -on: - push: - branches: [main] - workflow_dispatch: - -permissions: - contents: read - packages: write - -jobs: - build: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v6 - - - uses: docker/login-action@v4 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - uses: docker/setup-buildx-action@v4 - - - uses: docker/build-push-action@v7 - with: - context: . - file: Dockerfile - push: true - tags: | - ghcr.io/${{ github.repository }}/:latest - ghcr.io/${{ github.repository }}/:sha-${{ github.sha }} - cache-from: type=registry,ref=ghcr.io/${{ github.repository }}/:buildcache - cache-to: type=registry,ref=ghcr.io/${{ github.repository }}/:buildcache,mode=max -``` - -Ubah `` sesuai service (misal: `app`, `web`, `api`). Jika monorepo, gunakan matrix strategy untuk build beberapa service sekaligus. - -### Workflow 2: Deploy ke VPS - -File: `.github/workflows/deploy-docker.yml` - -```yaml -name: Deploy Docker to VPS - -on: - workflow_run: - workflows: ['Build and Push Docker Images'] - types: [completed] - push: - branches: [main] - workflow_dispatch: - -jobs: - deploy: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v6 - - - name: Deploy to VPS - env: - SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }} - VPS_HOST: ${{ secrets.VPS_HOST }} - VPS_USER: ${{ secrets.VPS_USER }} - VPS_TARGET_DIR: ${{ secrets.VPS_TARGET_DIR }} - ENV_FILE_PRODUCTION: ${{ secrets.ENV_FILE_PRODUCTION }} - run: | - set -euo pipefail - - mkdir -p ~/.ssh - echo "$SSH_PRIVATE_KEY" > ~/.ssh/id_rsa - chmod 600 ~/.ssh/id_rsa - ssh-keyscan -H -t ed25519,rsa "$VPS_HOST" >> ~/.ssh/known_hosts - - SSH_OPTS=(-o ControlMaster=auto -o ControlPath=/tmp/ssh-%r@%h:%p -o ControlPersist=600 -o StrictHostKeyChecking=yes) - - ssh "${SSH_OPTS[@]}" "$VPS_USER@$VPS_HOST" "mkdir -p $VPS_TARGET_DIR && mkdir -p $VPS_TARGET_DIR/infra/compose" - echo "$ENV_FILE_PRODUCTION" > .env.prod - scp "${SSH_OPTS[@]}" .env.prod "$VPS_USER@$VPS_HOST:$VPS_TARGET_DIR/.env" - - ssh "${SSH_OPTS[@]}" "$VPS_USER@$VPS_HOST" bash -s <<'EOF' - set -euo pipefail - cd "$VPS_TARGET_DIR" - - docker network inspect app-shared-net >/dev/null 2>&1 || docker network create app-shared-net - - if [ ! -d ".git" ]; then - git init - git remote add origin https://github.com//.git - fi - git fetch origin main --depth=1 || true - git reset --hard FETCH_HEAD - - docker compose --env-file .env pull - docker compose --env-file .env up -d --remove-orphans - EOF -``` - -### Secrets GitHub yang Diperlukan - -Buka **Settings > Secrets and variables > Actions**: - -| Secret | Deskripsi | -|--------|-----------| -| `SSH_PRIVATE_KEY` | Isi dengan `cat ~/.ssh/` | -| `VPS_HOST` | IP atau domain VPS | -| `VPS_USER` | User SSH (misal: `root`, `ubuntu`, `deploy`) | -| `VPS_TARGET_DIR` | Direktori aplikasi di VPS | -| `ENV_FILE_PRODUCTION` | Isi dengan environment production | - -### Trigger Manual - -```bash -gh workflow run deploy-docker.yml -``` - ---- - -## Dockerfile Patterns - -Pilih pattern sesuai jenis aplikasi. - -### Pattern 1: Multi-stage Build (SPA / static assets) - -```dockerfile -FROM oven/bun:1 AS builder -WORKDIR /app -COPY package.json bun.lock ./ -RUN bun install --frozen-lockfile -COPY . . -RUN bun run build - -FROM nginx:alpine -COPY --from=builder /app/dist /usr/share/nginx/html -COPY nginx.conf /etc/nginx/conf.d/default.conf -EXPOSE 80 -CMD ["nginx", "-g", "daemon off;"] -``` - -### Pattern 2: Single-stage (runtime image) - -```dockerfile -FROM oven/bun:1 -WORKDIR /app -COPY package.json bun.lock ./ -RUN bun install --frozen-lockfile -COPY . . -EXPOSE 3000 -CMD ["bun", "run", "start"] -``` - -### Pattern 3: Compiled binary (Rust / Go / Zig) - -```dockerfile -FROM rust:1 AS builder -WORKDIR /app -COPY . . -RUN cargo build --release - -FROM debian:bookworm-slim -COPY --from=builder /app/target/release/app /usr/local/bin/app -EXPOSE 8080 -CMD ["app"] -``` - ---- - -## Docker Compose Patterns - -### Single service - -```yaml -services: - app: - container_name: app - image: registry.example.com/org/app:latest - restart: always - ports: - - "3000:3000" - environment: - - NODE_ENV=production -``` - -### Multi-service dengan shared network - -```yaml -services: - app: - container_name: app - image: registry.example.com/org/app:latest - restart: always - networks: [app-shared-net] - - redis: - container_name: redis - image: redis:7-alpine - restart: always - networks: [app-shared-net] - -networks: - app-shared-net: - name: app-shared-net - external: true -``` - -### Dengan reverse proxy (Traefik / Caddy / Nginx) - -```yaml -services: - app: - container_name: app - image: registry.example.com/org/app:latest - restart: always - networks: [app-shared-net] - labels: - - 'traefik.enable=true' - - 'traefik.http.routers.app.rule=Host(`app.example.com`)' - - 'traefik.http.routers.app.entrypoints=websecure' - - 'traefik.http.routers.app.tls=true' - - 'traefik.http.services.app.loadbalancer.server.port=3000' - -networks: - app-shared-net: - name: app-shared-net - external: true -``` - ---- - -## Deploy Manual (Lokal) - -### 1. Build dan Push ke Registry - -Login ke registry: - -```bash -echo $GITHUB_TOKEN | docker login ghcr.io -u --password-stdin -``` - -Build dan push: - -```bash -docker build -t ghcr.io///:latest -f Dockerfile . - -docker push ghcr.io///:latest -``` - -Tag tambahan dengan SHA commit: - -```bash -SHORT_SHA=$(git rev-parse --short HEAD) -docker tag ghcr.io///:latest \ - ghcr.io///:sha-${SHORT_SHA} -docker push ghcr.io///:sha-${SHORT_SHA} -``` - -### 2. Pull dan Deploy di VPS - -SSH ke VPS: - -```bash -ssh -i ~/.ssh/ @ -``` - -Clone repo (jika belum): - -```bash -git clone https://github.com//.git -cd -``` - -Buat shared network (hanya sekali): - -```bash -docker network create app-shared-net -``` - -Siapkan environment: - -```bash -cp .env.example .env -# Edit .env sesuai nilai production -nano .env -``` - -Login ke registry di VPS: - -```bash -echo $GITHUB_TOKEN | docker login ghcr.io -u --password-stdin -``` - -Pull gambar terbaru: - -```bash -cd -docker compose -f docker-compose.yml --env-file .env pull -``` - -Deploy (up): - -```bash -docker compose -f docker-compose.yml --env-file .env up -d --remove-orphans -``` - -Verifikasi: - -```bash -docker compose -f docker-compose.yml ps -docker compose -f docker-compose.yml logs -f -``` - ---- - -## Deployment Order (Manual) - -Jika deploy bertahap, gunakan urutan ini: - -```bash -# 1. Shared services (Redis, database, dll.) -docker compose -f infra/compose/shared.yml up -d - -# 2. Reverse proxy -docker compose -f infra/compose/traefik.yml up -d - -# 3. Aplikasi -docker compose \ - -f infra/compose/app1.yml \ - -f infra/compose/app2.yml \ - up -d -``` - ---- - -## Perintah Berguna di VPS - -```bash -# Lihat semua container -docker ps -a - -# Log service -docker logs -f - -# Restart satu service -docker compose -f up -d --force-recreate - -# Hapus network lama (hati-hati) -docker network rm app-shared-net -docker network create app-shared-net - -# Bersihkan image unused -docker image prune -a -f -docker system prune -a -f -``` - ---- - -## Troubleshooting - -### Image tidak bisa di-pull - -Pastikan sudah login ke registry di VPS: - -```bash -docker logout ghcr.io -echo $GITHUB_TOKEN | docker login ghcr.io -u --password-stdin -``` - -Periksa visibility package di registry (harus `Public` atau akses diberikan). - -### Port sudah dipakai - -```bash -docker ps | grep :80 -docker ps | grep :443 -``` - -### Reverse proxy tidak routing - -Periksa label di compose file dan pastikan shared network ada: - -```bash -docker network inspect app-shared-net -docker logs traefik -``` - ---- - -## Environment Variable Management - -### Pola 1: `.env` di VPS (recommended untuk production) - -```bash -# Di VPS -cd -cp .env.example .env -# Edit sesuai production -nano .env -``` - -CI/CD upload `.env` via secret, tidak simpan di repo. - -### Pola 2: Docker secrets (Swarm mode) - -```yaml -services: - app: - image: app:latest - secrets: - - db_password - -secrets: - db_password: - file: ./secrets/db_password.txt -``` - -### Pola 3: External secret manager - -- **HashiCorp Vault**: inject via env atau file -- **AWS Secrets Manager**: `aws secretsmanager get-secret-value` -- **Doppler / Infisical**: unified secret management - ---- - -## Tagging dan Versioning - -### Strategy yang umum - -| Strategy | Contoh tag | Kegunaan | -|----------|-----------|----------| -| Latest + SHA | `latest`, `sha-abc1234` | CI/CD cepat, traceable | -| SemVer | `1.2.3`, `1.2`, `1` | Release publik | -| Git tag mirror | `v1.2.3` | Sync dengan git tag | -| Branch mirror | `main`, `develop` | Preview / staging | - -### Contoh git tag driven deploy - -```bash -git tag v1.2.3 -git push origin v1.2.3 -``` - -CI/CD membaca tag, build image dengan tag yang sama, dan deploy. - ---- - -## Rollback - -### Rollback via registry - -```bash -# Lihat tag yang tersedia -docker manifest inspect ghcr.io/org/app:latest -# atau lihat UI registry - -# Di VPS, edit compose file ke tag sebelumnya -# lalu: -docker compose --env-file .env pull -docker compose --env-file .env up -d --remove-orphans -``` - -### Rollback via git - -```bash -git revert HEAD -git push origin main -# CI/CD otomatis build dan deploy versi sebelumnya -``` - ---- - -## Health Checks - -### Di Dockerfile - -```dockerfile -HEALTHCHECK --interval=30s --timeout=3s --start-period=5s --retries=3 \ - CMD curl -f http://localhost:3000/health || exit 1 -``` - -### Di Docker Compose - -```yaml -services: - app: - image: app:latest - healthcheck: - test: ["CMD", "curl", "-f", "http://localhost:3000/health"] - interval: 30s - timeout: 3s - retries: 3 - start_period: 10s -``` - ---- - -## Monitoring & Observability - -```bash -# Log aggregated -docker compose logs -f --tail=100 - -# Resource usage -docker stats - -# Disk usage -docker system df - -# Cleanup -docker system prune -a -f -``` - ---- - -## Catatan Keamanan - -- Jangan commit `.env` atau SSH private key ke repo. -- Gunakan GitHub Secrets (atau secret manager) untuk credential di CI/CD. -- Rotate token dan key secara berkala. -- Batasi akses SSH ke VPS (ubah port default, gunakan fail2ban). -- Set `StrictHostKeyChecking=yes` pada SSH opsional deployment. - ---- - ---- - -## Proyek Ini: asepharyana-hub - -> Dokumentasi spesifik untuk repo ini. Lihat juga [ADR-0002](adr/0002-env-file-via-github-secret.md). - -### Topologi - -| Host | IP | Peran | -|------|----|-------| -| `orangevps` (VPS) | `45.127.35.244` | Docker host: Traefik, scraper-api, Redis, NATS, Dapr | -| `imrnes` (bare-metal) | `100.121.180.82` (Tailscale) | PostgreSQL (port 6432), Redis (port 6379) | - -### Environment Variables - -**Production `.env` tidak pernah di-commit.** File ini disimpan sebagai GitHub secret `ENV_FILE_PRODUCTION` dan di-SCP ke VPS saat deploy via `deploy-docker.yml`. - -Cara update: -```bash -# Baca current .env dari VPS -ssh root@45.127.35.244 "cat /root/asepharyana-hub/.env" - -# Update GitHub secret (dari output di atas) -cat > /tmp/env-updated << 'EOF' - -EOF -cat /tmp/env-updated | gh secret set ENV_FILE_PRODUCTION --repo asepharyana/asepharyana-hub -``` - -**Jangan manual edit `.env` di VPS tanpa update GitHub secret juga** — nanti ke- overwrite pas deploy berikutnya. - -### Database - -| Variable | Value | -|----------|-------| -| `DATABASE_URL` | `postgres://asephs:hunterz@100.121.180.82:6432/hub` | -| `REDIS_URL` | `redis://redis:6379` (Docker network) | - -### Kompose - -Proyek compose bernama `compose`, terdiri dari 5 file yang selalu di-include bersamaan: - -```bash -/root/asepharyana-hub/infra/compose/ -├── traefik.yml # Reverse proxy -├── shared.yml # Redis -├── nats.yml # NATS -├── dapr.yml # Dapr placement -├── scraper.yml # Scraper API -└── observability.yml # OTel Collector, Jaeger, Dashboard - -```bash -cd /root/asepharyana-hub -docker compose \ - -p compose \ - --env-file .env \ - -f infra/compose/traefik.yml \ - -f infra/compose/shared.yml \ - -f infra/compose/scraper.yml \ - -f infra/compose/nats.yml \ - -f infra/compose/dapr.yml \ - -f infra/compose/observability.yml \ - up -d --remove-orphans -``` - ---- - -## Checklist Deploy Proyek Baru - -1. [ ] Dockerfile ditest lokal (`docker build`, `docker run`) -2. [ ] Docker Compose file valid (`docker compose config`) -3. [ ] `.dockerignore` sesuai (node_modules, .git, .env) -4. [ ] Registry dibuat (GHCR package / Docker Hub repo / ECR / dll.) -5. [ ] GitHub Actions workflow dibuat dengan permission `packages: write` -6. [ ] VPS siap: Docker, Docker Compose, SSH key -7. [ ] Shared network dibuat (`docker network create`) -8. [ ] `.env` production di-VPS atau via secret manager -9. [ ] Reverse proxy (Traefik / Caddy / Nginx) routing ke container -10. [ ] Health check endpoint aktif diff --git a/docs/DEVELOPMENT.md b/docs/DEVELOPMENT.md deleted file mode 100644 index 5e15667..0000000 --- a/docs/DEVELOPMENT.md +++ /dev/null @@ -1,206 +0,0 @@ -# Development Guide - -Panduan setup lingkungan development lokal untuk kontributor `asepharyana-hub`. - -## Prasyarat - -| Tool | Versi Minimal | Catatan | -|------|---------------|---------| -| Git | 2.40+ | Submodule support | -| Docker | 24+ | Dengan Docker Compose v2 plugin | -| Rust | 1.85+ | Hanya untuk `apps/scraper` | -| Bun | 1.x | Root tooling (Biome) | -| Dapr CLI | 1.14+ | Opsional, untuk development dengan Dapr | - -## Setup Awal - -```bash -# 1. Clone repo -git clone https://github.com/asepharyana/asepharyana-hub.git -cd asepharyana-hub - -# 2. Init submodules -make init-submodules - -# 3. Setup environment -cp .env.example .env -# Edit .env sesuai kebutuhan lokal - -# 4. Install root dependencies -bun install -``` - -## Menjalankan Infrastruktur Lokal - -Beberapa service membutuhkan Redis. Jalankan dengan: - -```bash -make dev -# atau equivalen: -docker compose -f infra/compose/shared.yml up -d -``` - -Ini akan menjalankan Redis Alpine di `localhost:6379`. - -### (Opsional) NATS Lokal - -Jika service membutuhkan pub/sub: - -```bash -docker compose -f infra/compose/nats.yml up -d -# NATS client: localhost:4222 -# NATS monitor: localhost:8222 -``` - -### (Opsional) Dapr Placement Lokal - -Jika service membutuhkan sidecar Dapr: - -```bash -docker compose -f infra/compose/dapr.yml up -d -# Dapr placement: localhost:50005 -``` - -## Menjalankan Service Lokal - -### Scraper API (Rust) - -```bash -# Pastikan Redis sudah running (make dev) -cd apps/scraper - -# Cargo run -cargo run - -# Dengan Dapr sidecar (jika placement running) -dapr run \ - --app-id scraper-api \ - --app-port 4091 \ - --dapr-http-port 3500 \ - --resources-path ../../infra/dapr/components \ - -- cargo run -``` - -### Dengan Docker Compose (Full Stack) - -Untuk menjalankan semua service sekaligus: - -```bash -docker compose \ - -f infra/compose/shared.yml \ - -f infra/compose/nats.yml \ - -f infra/compose/dapr.yml \ - -f infra/compose/scraper.yml \ - --env-file .env \ - up -d -``` - -Untuk service baru, tambahkan compose file-nya ke daftar. - -## Update Submodules - -### Pull latest dari semua submodule - -```bash -make update-submodules -# atau: -git submodule update --remote --merge --recursive -``` - -### Check status submodule - -```bash -make status -# atau: -git submodule status -``` - -### Sync .env ke submodule - -```bash -bash scripts/2updateenv.sh -# Copy .env root ke apps/*/ -``` - -## Linting & Formatting - -Root repo menggunakan **Biome** untuk linting dan formatting: - -```bash -bun run check # Lint + format + write -bun run ci # CI mode (no write, exit code on issues) -bun run lint # Lint only -bun run format # Format only -``` - -## Build Docker Image Lokal - -```bash -# Scraper API -docker build -f infra/docker/scraper.Dockerfile -t scraper-api:local . - -# Service baru: tambahkan Dockerfile di infra/docker/ -``` - -## Testing - -Saat ini belum ada test runner di root level. Masing-masing submodule mengelola testing sendiri: - -```bash -# Scraper API (Rust) -cd apps/scraper && cargo test -``` - -## Validasi YAML - -Sebelum commit perubahan infra, validasi semua file YAML: - -```bash -python -c " -import pathlib, yaml -for p in pathlib.Path('infra').rglob('*.yml'): - with open(p) as f: yaml.safe_load(f) - print(f'OK {p}') -for p in pathlib.Path('infra').rglob('*.yaml'): - with open(p) as f: yaml.safe_load(f) - print(f'OK {p}') -" - -for f in infra/compose/*.yml; do - docker compose -f "$f" config >/dev/null && echo "OK $f" -done -``` - -## Git Workflow - -### Commit Convention - -``` -(): -``` - -Type: `feat`, `fix`, `chore`, `docs`, `refactor`, `test`, `ci`, `perf`, `style` -Scope: `scraper`, `infra`, `ci`, `dapr`, `nats`, `docs`, `deps`, `scripts`, `root` - -Contoh: -``` -feat(scraper): add image cache endpoint -fix(infra): correct Traefik rate-limit config -chore(deps): bump biome to 2.5.0 -``` - -### Branch Strategy - -- `main` — production branch, push triggers CI/CD -- Fitur baru: branch dari `main`, PR ke `main` -- Submodule development: dilakukan di repo masing-masing, hub hanya update pointer - -## Deployment ke VPS - -Push ke `main` otomatis trigger CI/CD. Untuk trigger manual: - -```bash -gh workflow run deploy-docker.yml -``` - -Lihat `docs/DEPLOYMENT.md` untuk detail. diff --git a/docs/add-dapr-service.md b/docs/add-dapr-service.md deleted file mode 100644 index bd48f6f..0000000 --- a/docs/add-dapr-service.md +++ /dev/null @@ -1,162 +0,0 @@ -# Menambahkan Dapr ke Service Baru - -Panduan integrasi Dapr runtime sidecar untuk service di `asepharyana-hub`. - -## Prasyarat - -- NATS server berjalan (`infra/compose/nats.yml`) -- Dapr placement service berjalan (`infra/compose/dapr.yml`) - -## 1. Compose File - -Setiap service butuh sidecar container Dapr. Contoh: - -```yaml -services: - app: - container_name: app - image: ghcr.io/asepharyana/asepharyana-hub/app:latest - restart: always - depends_on: - dapr-placement: - condition: service_healthy - nats: - condition: service_healthy - networks: - app-shared-net: - aliases: - - app - env_file: - - ../../.env - - app-dapr: - container_name: app-dapr - image: daprio/daprd:latest - restart: always - depends_on: - dapr-placement: - condition: service_healthy - nats: - condition: service_healthy - networks: - - app-shared-net - depends_on: - dapr-placement: - condition: service_healthy - nats: - condition: service_healthy - otel-collector: - condition: service_started - networks: - - app-shared-net - command: - - './daprd' - - '--app-id=app' - - '--app-port=3000' - - '--dapr-http-port=3500' - - '--dapr-grpc-port=50001' - - '--placement-host-address=dapr-placement:50005' - - '--config=/dapr/config.yaml' - - '--resources-path=/dapr/components' - volumes: - - ../../infra/dapr:/dapr:ro - -networks: - app-shared-net: - name: app-shared-net - external: true -``` - -## 2. Mengakses Dapr dari Service - -### Via HTTP API (semua bahasa) - -Sidecar listen di `localhost:3500`: - -```bash -# Publish event -curl -X POST http://localhost:3500/v1.0/publish/pubsub/hub.event.type \ - -H "Content-Type: application/json" \ - -d '{"key": "value"}' - -# Service invocation -curl http://localhost:3500/v1.0/invoke/app/method/endpoint - -# State store -curl -X POST http://localhost:3500/v1.0/state/statestore \ - -H "Content-Type: application/json" \ - -d '[{"key": "mykey", "value": "myvalue"}]' -``` - -### Via Dapr SDK (Rust) - -Tambah ke `Cargo.toml`: - -```toml -dapr-sdk = { version = "0.15", features = ["pubsub", "http"] } -tokio-stream = "0.1" -``` - -Contoh publish event: - -```rust -use dapr_sdk::client::{Client, Event}; -use dapr_sdk::DaprClient; - -let client = DaprClient::new("127.0.0.1", 3500).await?; -client.publish_event("pubsub", "hub.image.cached", serde_json::json!({ - "original_url": url, - "cdn_url": cdn_url, -})).await?; -``` - -Contoh subscribe event: - -```rust -let mut stream = client.subscribe_events("pubsub", "hub.image.cached").await?; -while let Some(event) = stream.next().await { - let data: MyEvent = serde_json::from_slice(&event.data)?; - // handle event -} -``` - -## 3. Event Topics Convention - -Gunakan prefix `hub.` untuk semua event: - -| Topic | Payload | Description | -|-------|---------|-------------| -| `hub.image.cached` | `{original_url, cdn_url, source}` | Image selesai di-cache | -| `hub.image.repaired` | `{old_url, new_url}` | CNAME image diperbaiki | -| `hub.scrape.anime.done` | `{source, slug, duration}` | Scrape anime selesai | -| `hub.system.alert` | `{service, level, message}` | Error/alert dari service | - -## 4. Local Development - -Untuk development tanpa Docker: - -```bash -# 1. Install Dapr CLI -# 2. Init Dapr local -dapr init - -# 3. Run service dengan sidecar -dapr run --app-id app --app-port 3000 --dapr-http-port 3500 \ - --resources-path ./infra/dapr/components \ - -- cargo run -``` - -## 5. Verifikasi - -```bash -# Sidecar health -curl http://localhost:3500/v1.0/healthz - -# Publish test event -curl -X POST http://localhost:3500/v1.0/publish/pubsub/hub.test \ - -H "Content-Type: application/json" \ - -d '{"test": true}' - -# NATS stream stats -curl http://localhost:8222/jszetstream -``` diff --git a/docs/add-new-app.md b/docs/add-new-app.md index 8f2b52a..f202559 100644 --- a/docs/add-new-app.md +++ b/docs/add-new-app.md @@ -1,167 +1,42 @@ -# Menambahkan Aplikasi Baru ke Deployment +# Menambahkan Service Baru / Subdomain -Dokumen ini menjelaskan langkah menambahkan service baru ke `asepharyana-hub`. Root repo berfungsi sebagai hub: source aplikasi berada di `apps/` sebagai submodule, sedangkan Docker Compose, Traefik, dan workflow deploy tetap berada di root repo. +Panduan untuk menambahkan service baru di ekosistem `asepharyana/infra` (2026-08-28+, pasca monorepo). -## 1. Buat repo aplikasi +## Prinsip -Buat repo baru di GitHub dengan pola nama: +- **Aplikasi hidup di repo sendiri** (`hub`, `scraper`, `tools`, `llm-api`) dengan + `flake.nix` + `.github/workflows/deploy.yml` mandiri. Repo infra TIDAK berisi kode app. +- Repo infra (`asepharyana/infra`) hanya mengatur **reverse proxy & config VPS**. -```text -https://github.com/asepharyana/asepharyana-hub-.git +## Langkah + +1. **Buat repo aplikasi** (contoh pola: `asepharyana/scraper`). +2. **Tambahkan `flake.nix`** di repo app — derivasi Nix (lihat template di repo app yang ada: + Next.js/bun atau Rust/cargo). Nama paket = nama unit systemd. +3. **Tambahkan `.github/workflows/deploy.yml`** (pola `nix build .#` → `nix copy ssh://` + → `nix-env --profile /nix/var/nix/profiles/ --set` → `systemctl restart `). + Secrets yang dibutuhkan: `SSH_PRIVATE_KEY`, `VPS_HOST`, `VPS_USER`. +4. **Di VPS**: buat user systemd + unit (mis. `/etc/systemd/system/.service`, + `ExecStart=/usr/local/bin/bws-exec /nix/var/nix/profiles//bin/`), + pastikan app jalan di port lokal. +5. **Tambah site block** di `infra/caddy/Caddyfile.prod` (pola `import proxy `), + push ke `main` → CI `caddy-deploy.yml` sync + reload + verifikasi rute. +6. **(Opsional)** Tambah unit ke `MONITORED_UNITS` di hub dashboard + (repo `asepharyana/hub`, `src/app/api/dashboard/route.ts`). + +## Contoh site block Caddy + +```caddyfile +nama-app.asepharyana.my.id { + import proxy +} ``` -Lalu tambahkan ke root hub sebagai submodule: +## Verifikasi ```bash -git submodule add https://github.com/asepharyana/asepharyana-hub-.git apps/ -git submodule update --init --recursive -``` +# Dari local +curl -s -o /dev/null -w '%{http_code}\n' https://nama-app.asepharyana.my.id/ -## 2. Tambahkan Dockerfile - -Tambahkan Dockerfile runtime di `infra/docker/.Dockerfile`. - -Gunakan root repo sebagai build context agar Dockerfile bisa mengakses submodule path: - -```bash -docker build -f infra/docker/.Dockerfile -t :local . -``` - -## 3. Tambahkan Compose file - -Buat `infra/compose/.yml`: - -```yaml -services: - : - container_name: - image: ghcr.io/asepharyana/asepharyana-hub/:sha- - restart: always - networks: - app-shared-net: - aliases: - - - env_file: - - ../../.env - -networks: - app-shared-net: - name: app-shared-net - external: true -``` - -Gunakan `app-shared-net` agar service dapat diakses oleh Traefik dan service lain. - -## 3.5. Tambahkan Dapr sidecar (wajib untuk pub/sub) - -Setiap service yang ingin menggunakan Dapr pub/sub atau service invocation harus punya sidecar. -Tambah di `infra/compose/.yml`: - -```yaml - -dapr: - container_name: -dapr - image: daprio/daprd:latest - restart: always - depends_on: - dapr-placement: - condition: service_healthy - nats: - condition: service_healthy - otel-collector: - condition: service_started - networks: - - app-shared-net - command: - - './daprd' - - '--app-id=' - - '--app-port=' - - '--dapr-http-port=3500' - - '--dapr-grpc-port=50001' - - '--placement-host-address=dapr-placement:50005' - - '--config=/dapr/config.yaml' - - '--resources-path=/dapr/components' - volumes: - - ../../infra/dapr:/dapr:ro -``` - -Pastikan juga app container punya `depends_on` ke dapr-placement, nats, dan otel-collector: -```yaml - depends_on: - dapr-placement: - condition: service_healthy - nats: - condition: service_healthy - otel-collector: - condition: service_started -``` - -## 4. Tambahkan route Traefik - -Update `infra/traefik/dynamic/apps.yaml`: - -```yaml -http: - routers: - : - rule: 'Host(`.asepharyana.my.id`) || Host(`.asepharyana.web.id`)' - entryPoints: - - websecure - tls: {} - middlewares: - - common-chain@file - service: -service - - services: - -service: - loadBalancer: - servers: - - url: 'http://:' -``` - -## 5. Update workflow build - -Update `.github/workflows/docker-build-push.yml`: - -1. Tambahkan path detection untuk `apps/` dan `infra/docker/.Dockerfile`. -2. Tambahkan service ke matrix build. -3. Tambahkan mapping Dockerfile di step `Docker metadata`. -4. Tambahkan mapping compose file dan submodule path di step `Update tags and submodules`. - -## 6. Update workflow deploy - -Tambahkan compose file baru ke `ALL_COMPOSE_FILES` di `.github/workflows/deploy-docker.yml`: - -```bash -infra/compose/.yml -``` - -## 7. Update dokumentasi - -Update file berikut bila service baru mengubah arsitektur publik: - -- `README.md` -- `ARCHITECTURE.md` -- `infra/README.md` -- `.gitmodules` - -## 8. Validasi - -Jalankan validasi YAML dan compose rendering: - -```bash -python - <<'PY' -import pathlib, yaml -for path in pathlib.Path('infra').rglob('*.yml'): - with path.open() as fh: - yaml.safe_load(fh) - print(f'OK {path}') -for path in pathlib.Path('infra').rglob('*.yaml'): - with path.open() as fh: - yaml.safe_load(fh) - print(f'OK {path}') -PY - -for f in infra/compose/*.yml; do - docker compose -f "$f" config >/dev/null && echo "OK $f" -done -``` +# Dari VPS +systemctl status \ No newline at end of file diff --git a/docs/adr/0001-use-hub-repo-with-submodules.md b/docs/adr/0001-use-hub-repo-with-submodules.md index 061a7dd..d499aa1 100644 --- a/docs/adr/0001-use-hub-repo-with-submodules.md +++ b/docs/adr/0001-use-hub-repo-with-submodules.md @@ -1,49 +1,38 @@ -# ADR 0001: Use a Hub Repository with App Submodules +# ADR 0001: Infra Repo — Reverse Proxy Config Only (Submodules Removed) ## Status -Accepted +**Superseded** (2026-08-28) — lihat ADR ini sebagai arsip keputusan awal. -## Context +## Context (aslinya) -The project contains multiple independent application services that share one deployment surface: Docker Compose, Traefik routing, GitHub Actions workflows, and operational documentation. +Proyek awal memakai `asepharyana-hub` sebagai monorepo: aplikasi di `apps/` sebagai +git submodule, infra (compose/traefik/dokumen/CI) terpusat di root. Keputusan itu masuk akal +saat semua service berbagi satu deployment surface. -The services should be developed and versioned independently, while deployment infrastructure should remain centralized so production routing and compose manifests stay consistent. +## Decision (aslinya) -## Decision +Gunakan `asepharyana-hub` sebagai root hub: app code submodule, infra + CI di root. -Use `asepharyana-hub` as the root hub repository. +## Superseded By -- Application code lives under `apps/` as Git submodules. -- Infrastructure lives in the root repo under `infra/`. -- Documentation lives in the root repo under `docs/`. -- CI/CD workflows live in the root repo under `.github/workflows/`. -- Root tooling stays minimal: `package.json`, Prettier, ESLint, Makefile helpers, and deployment scripts. +Mulai **2026-08-28** repo dirombak: -Current app submodules: - -| Service | Path | Remote | -| ----------- | -------------- | --------------------------------------- | -| Scraper API | `apps/scraper` | `asepharyana/asepharyana-hub-scraper` | +- **Parent `asepharyana-hub` → `asepharyana/infra`** — murni config reverse proxy (Caddy), + firewall, systemd drop-ins, docs. CI hanya untuk deploy Caddy. +- **App repos di-rename & self-contained**: `hub`, `scraper`, `tools`, `llm-api`. + Masing-masing punya `flake.nix` + `.github/workflows/deploy.yml` sendiri + (`nix build → nix copy → nix-env --profile → systemctl restart`). +- **Submodule dihapus** — tidak ada lagi pointer submodule / repository_dispatch chain. +- `update-submodule.yml`, `notify-parent.yml`, matrix `nix-build.yml` dihapus. ## Consequences ### Positive - -- Each app can evolve in its own repository. -- The hub pins exact submodule revisions for reproducible deployments. -- Deployment infrastructure remains centralized and easier to audit. -- Root tooling stays lightweight and does not impose one build system on every service. +- CI tiap app independen: push ke repo app langsung build+deploy, tak perlu 2 hop. +- Parent kecil & fokus: diff Caddyfile mudah di-audit. +- Tanpa submodule = tanpa `dubious ownership` / pointer drift / fetchGit pin. ### Negative - -- Developers must understand Git submodule workflows. -- Updating a service requires updating the submodule pointer in the hub repo. -- Cross-service changes require coordinating commits across multiple repositories. - -### Mitigations - -- Keep `.gitmodules` accurate and minimal. -- Use `scripts/sync-submodules.sh` for local checkout consistency. -- Document service-addition steps in `docs/add-new-app.md`. -- Keep GitHub Actions responsible for Docker image builds, compose tag updates, and deployments. +- Koordinasi cross-repo manual (app + Caddy bila perlu port baru). +- Repo lama `asepharyana-hub-*` redirect ke nama baru (GitHub auto). \ No newline at end of file diff --git a/docs/adr/0002-env-file-via-github-secret.md b/docs/adr/0002-env-file-via-github-secret.md index f79fca2..889aa94 100644 --- a/docs/adr/0002-env-file-via-github-secret.md +++ b/docs/adr/0002-env-file-via-github-secret.md @@ -1,8 +1,12 @@ # ADR 0002: Production `.env` via GitHub Encrypted Secret +> **LEGACY (2026-08-28):** Repo `asepharyana-hub` sudah dirombak → `asepharyana/infra`. +> Workflow lama yang SCP `.env` ke VPS tidak dipakai lagi (deploy app pindah ke repo masing-masing, +> secrets via Bitwarden `bws-exec`). ADR ini dipertahankan sebagai arsip. + ## Status -Accepted +Accepted (archived) ## Context @@ -33,7 +37,7 @@ Container reads $DATABASE_URL, $JWT_SECRET, etc. ```bash # 1. Read current content from the VPS -ssh root@45.127.35.244 "cat /root/asepharyana-hub/.env" +ssh root@45.127.35.244 "cat /.env" # 2. Pipe updated content to the GitHub secret # (requires gh CLI with repo access) @@ -43,7 +47,7 @@ cat /path/to/updated-env | gh secret set ENV_FILE_PRODUCTION --repo asepharyana/ gh workflow run deploy-docker.yml # OR apply immediately on the VPS (for hotfix): -ssh root@45.127.35.244 "sed -i 's|OLD_VALUE|NEW_VALUE|' /root/asepharyana-hub/.env" +ssh root@45.127.35.244 "sed -i 's|OLD_VALUE|NEW_VALUE|' /.env" # Then restart affected containers ``` @@ -69,7 +73,7 @@ ssh root@45.127.35.244 "sed -i 's|OLD_VALUE|NEW_VALUE|' /root/asepharyana-hub/.e The VPS runs a single Docker Compose project named `compose` composed of multiple files: ```bash -/root/asepharyana-hub/infra/compose/ +/infra/compose/ ├── traefik.yml # Reverse proxy (TLS termination, routing) ├── shared.yml # Redis ├── nats.yml # NATS message broker + JetStream @@ -99,7 +103,7 @@ docker compose \ | `SSH_PRIVATE_KEY` | SSH key for VPS access | | `VPS_HOST` | `45.127.35.244` | | `VPS_USER` | `root` | -| `VPS_TARGET_DIR` | `/root/asepharyana-hub` | +| `VPS_TARGET_DIR` | `` | | `ENV_FILE_PRODUCTION` | Full `.env` content for production | ## Consequences diff --git a/docs/adr/0003-rename-repos-and-split-ci.md b/docs/adr/0003-rename-repos-and-split-ci.md new file mode 100644 index 0000000..4bf4e23 --- /dev/null +++ b/docs/adr/0003-rename-repos-and-split-ci.md @@ -0,0 +1,47 @@ +# ADR 0003: Rename Repositori & Pisahkan CI per Aplikasi + +## Status + +Accepted (2026-08-28) + +## Context + +Monorepo `asepharyana-hub` (app submodule + infra + CI terpusat) punya kelemahan: +- CI build+deploy semua app menyatu di parent (`nix-build.yml` matrix) — setiap push app + butuh 2 hop (notify-parent → update-submodule → nix-build), rawan drift pointer. +- Nama `asepharyana-hub` ambigu (parent & app prefix sama), dan submodule menambah kompleksitas. + +## Decision + +Rombak total: + +| Lama | Baru | Peran | +|------|------|-------| +| `asepharyana-hub` | `asepharyana/infra` | Reverse proxy (Caddy) + firewall + systemd + docs. CI: caddy-deploy saja. | +| `asepharyana-hub-hub` | `asepharyana/hub` | Portfolio SPA. CI mandiri (deploy.yml). | +| `asepharyana-hub-scraper` | `asepharyana/scraper` | Rust scraper API. CI mandiri. | +| `asepharyana-hub-tools` | `asepharyana/tools` | Tools stack (gateway/workers/frontend). CI mandiri. | +| `asepharyana-hub-llm-api` | `asepharyana/llm-api` | Rust LLM API. CI mandiri. | +| `asepharyana-hub-guide` | `asepharyana/hub-guide` | (tidak di-root; plugin guide — diarsipkan) | + +Setiap app repo mendapat: +- `flake.nix` (derivasi build sendiri, tanpa fetchGit submodule) +- `.github/workflows/deploy.yml` (nix build → nix copy → nix-env --profile → systemctl restart) +- Secret `SSH_PRIVATE_KEY`, `VPS_HOST`, `VPS_USER` + +Parent `infra` mendapat: +- Hapus semua submodule + `update-submodule.yml` + matrix `nix-build.yml` +- `.github/workflows/caddy-deploy.yml` (sync Caddyfile → reload → verify) +- Docs diarahkan ulang. + +## Consequences + +- **Positif**: CI per-app independen & cepat; parent kecil; tanpa submodule = tanpa fetchGit pin + / dubious-ownership / pointer churn. Rename GitHub auto-redirect URL lama. +- **Negatif**: koordinasi manual bila app butuh port baru di Caddy; workflow lama di + downstream (skill/cron) perlu update referensi. + +## Referensi + +- `docs/add-new-app.md` — proses menambah service baru +- `infra/caddy/Caddyfile.prod` — pola site block \ No newline at end of file diff --git a/docs/backup-recovery.md b/docs/backup-recovery.md deleted file mode 100644 index c08ecb5..0000000 --- a/docs/backup-recovery.md +++ /dev/null @@ -1,234 +0,0 @@ -# Backup & Disaster Recovery - -## Aset yang Perlu di-Backup - -| Aset | Lokasi | Frekuensi | Metode | -|------|--------|-----------|--------| -| Database PostgreSQL | `imrnes` (100.121.180.82:6432) | Harian | `pg_dump` | -| Volume Redis | `orangevps` (Docker volume) | Opsional | Redis RDB / AOF | -| Volume NATS JetStream | `orangevps` (Docker volume) | Opsional | File copy | -| Docker Compose manifests | GitHub (hub repo) | Real-time | Git | -| Environment variables | GitHub secret `ENV_FILE_PRODUCTION` | Manual | `gh secret set` | -| TLS certificates | `orangevps` (`/root/*.pem`, `*.key`) | Saat renew | SCP | -| Tailscale auth | Tailscale admin console | - | Cloud-managed | -| GitHub Actions secrets | GitHub UI | Manual | Backup list | - -## Database PostgreSQL (Prioritas Tertinggi) - -### Backup Manual - -```bash -# Dari orangevps (via Tailscale) -pg_dump -h 100.121.180.82 -p 6432 -U asephs -d hub \ - --no-owner --no-acl \ - -F c -f /root/db-backups/hub-$(date +%Y%m%d-%H%M%S).dump - -# Atau dari imrnes langsung -pg_dump -U asephs -d hub \ - -F c -f /backup/hub/hub-$(date +%Y%m%d-%H%M%S).dump -``` - -### Restore - -```bash -# Drop dan recreate database -dropdb -h 100.121.180.82 -p 6432 -U asephs hub -createdb -h 100.121.180.82 -p 6432 -U asephs hub - -# Restore dari dump -pg_restore -h 100.121.180.82 -p 6432 -U asephs -d hub \ - --no-owner --no-acl \ - /path/to/backup/hub-20260101-120000.dump -``` - -### Backup Otomatis (via Cron di imrnes) - -```bash -# /etc/cron.d/hub-db-backup -0 2 * * * root pg_dump -U asephs -d hub -F c -f /backup/hub/hub-$(date +\%Y\%m\%d).dump && find /backup/hub -name "hub-*.dump" -mtime +30 -delete -``` - -## Volume Docker - -### Redis - -Redis data bisa di-recover dari NATS events (event sourcing). Jika tidak ada persistence requirement, cukup restart: - -```bash -docker volume rm redis_data -docker compose -f infra/compose/shared.yml up -d -``` - -Jika perlu backup: - -```bash -# Save RDB snapshot -docker exec redis redis-cli SAVE - -# Copy dari volume -docker run --rm -v redis_data:/data -v /backup:/backup alpine cp /data/dump.rdb /backup/redis-$(date +%Y%m%d).rdb -``` - -### NATS JetStream - -```bash -# Backup volume -docker run --rm -v nats_data:/data -v /backup:/backup alpine \ - tar czf /backup/nats-$(date +%Y%m%d).tar.gz -C /data . -``` - -## Environment Variables - -### Backup `.env` dari VPS - -```bash -# Simpan current .env dari VPS -ssh root@45.127.35.244 "cat /root/asepharyana-hub/.env" > .env.backup.$(date +%Y%m%d) - -# Update GitHub secret -cat .env.backup.$(date +%Y%m%d) | gh secret set ENV_FILE_PRODUCTION --repo asepharyana/asepharyana-hub -``` - -### Restore `.env` jika hilang - -```bash -# Buat .env baru dari template -cp .env.example .env - -# Edit secrets (manual dari password manager atau GitHub secret) -# Atau download dari GitHub secret -gh secret list --repo asepharyana/asepharyana-hub -``` - -## TLS Certificates - -### Backup - -```bash -# Di orangevps -tar czf /root/cert-backup-$(date +%Y%m%d).tar.gz \ - /root/asepharyana.my.id.pem \ - /root/asepharyana.my.id.key \ - /root/asepharyana.web.id.pem \ - /root/asepharyana.web.id.key \ - /root/asepharyana-hub/infra/traefik/dynamic/ssl.yaml - -# SCP ke local -scp root@45.127.35.244:/root/cert-backup-*.tar.gz . -``` - -### Restore - -```bash -# SCP ke VPS -scp cert-backup-20260101.tar.gz root@45.127.35.244:/root/ - -# Extract -ssh root@45.127.35.244 "tar xzf /root/cert-backup-20260101.tar.gz -C / && docker restart traefik" -``` - -## Disaster Recovery Scenarios - -### Skenario 1: VPS (orangevps) mati total - -**Dampak:** Semua service down. - -**Recovery:** - -```bash -# 1. Provision VPS baru (atau restore dari snapshot) -# 2. Install Docker + Tailscale -# 3. Clone repo -git clone https://github.com/asepharyana/asepharyana-hub.git /root/asepharyana-hub - -# 4. Setup Tailscale, route service -# 5. Restore .env -echo "" > /root/asepharyana-hub/.env - -# 6. Restore TLS certs -# 7. Create network -docker network create app-shared-net - -# 8. Start services sesuai urutan -cd /root/asepharyana-hub -for f in shared.yml nats.yml dapr.yml traefik.yml scraper.yml; do - docker compose -f infra/compose/$f --env-file .env up -d -done - -# 9. Update DNS jika IP baru -``` - -### Skenario 2: Database (imrnes) mati total - -**Dampak:** Semua service yang butuh database error. - -**Recovery:** - -```bash -# 1. Fix imrnes atau provision server baru -# 2. Setup PostgreSQL -# 3. Restore dari backup terakhir -# 4. Update Tailscale IP jika perlu -# 5. Update .env dan GitHub secret -# 6. Redeploy -``` - -### Skenario 3: GitHub repository hilang - -**Dampak:** Kehilangan CI/CD, tapi Docker images masih ada di GHCR. - -**Recovery:** - -```bash -# 1. Create repo baru di GitHub -# 2. Push dari local clone -git remote add origin-new https://github.com/asepharyana/asepharyana-hub-new.git -git push origin-new main - -# 3. Re-create GitHub secrets -# 4. Re-create workflows -# 5. Update VPS remote -ssh root@45.127.35.244 "cd /root/asepharyana-hub && git remote set-url origin https://github.com/asepharyana/asepharyana-hub-new.git" -``` - -### Skenario 4: GHCR registry tidak bisa diakses - -**Dampak:** Tidak bisa pull image. - -**Recovery:** - -```bash -# 1. Build image langsung di VPS -docker build -f infra/docker/scraper.Dockerfile -t ghcr.io/asepharyana/asepharyana-hub/scraper-api:local . - -# 2. Update compose file untuk sementara -sed -i 's|image: ghcr.io/.*|image: ghcr.io/asepharyana/asepharyana-hub/scraper-api:local|' infra/compose/scraper.yml - -# 3. Start -docker compose -f infra/compose/scraper.yml up -d -``` - -### Skenario 5: Semua server mati (total loss) - -**Recovery:** - -```bash -# 1. Provision VPS baru -# 2. Provision server database baru -# 3. Setup Tailscale -# 4. Clone repo, restore .env, certs -# 5. Restore database dari backup (jika ada) -# 6. Jika tidak ada backup database: -# - Build image dari GHCR -# - Start service dengan database kosong -# - Data akan terisi ulang dari scraping -``` - -## Checklist Pencegahan - -- [ ] Cron job backup database berjalan -- [ ] Backup `.env` disimpan di luar VPS (password manager) -- [ ] TLS certificates backup disimpan di luar VPS -- [ ] GitHub secrets terdaftar (tidak hanya diingat) -- [ ] Docker images bisa di-rebuild dari CI (GHCR sebagai source of truth) -- [ ] Tailscale admin access via multiple accounts diff --git a/docs/ci-cd-pipeline.md b/docs/ci-cd-pipeline.md deleted file mode 100644 index 90a8c74..0000000 --- a/docs/ci-cd-pipeline.md +++ /dev/null @@ -1,248 +0,0 @@ -# CI/CD Pipeline - -Dokumentasi pipeline CI/CD untuk `asepharyana-hub`. Terdiri dari 5 GitHub Actions workflow yang saling terhubung. - -## Workflow Overview - -``` - ┌─────────────┐ - │ Lint │ (PR/push → Biome) - └──────┬──────┘ - │ - Push ke main ─────┼────── repository_dispatch - │ - ┌──────▼──────────────────┐ - │ docker-build-push.yml │ - │ │ - │ Phase 1: Detect │ - │ Phase 2: Build & Push │ - │ Phase 3: Update │ - │ manifests │ - └──────┬──────────────────┘ - │ workflow_run - ┌──────▼──────────────┐ - │ deploy-docker.yml │ - │ SSH → VPS │ - │ Pull → Restart │ - └─────────────────────┘ - - repository_dispatch ──► update-submodule.yml - (dari submodule) (update pointer → commit) - │ - ▼ - docker-build-push.yml - (triggered by push) -``` - -## Workflow Detail - -### 1. Lint (`lint.yml`) - -**Trigger:** PR/push ke `main` yang mengubah `*.json`, `*.js`, `biome.json` - -**Aksi:** -- Checkout repo dengan submodules -- Setup Bun -- `bun install --frozen-lockfile` -- `bun run ci` (Biome CI mode) - -**Permissions:** read-only - -### 2. Build and Push Docker Images (`docker-build-push.yml`) - -**Trigger:** -- Push ke `main` yang mengubah `apps/**`, `infra/**`, atau file workflow -- `repository_dispatch` tipe `submodule-updated` -- `workflow_dispatch` (manual) - -**Concurrency:** Satu workflow per branch (cancel-in-progress=false) - -#### Phase 1: Detect Changes - -Job `changes` mendeteksi service mana yang perlu di-build: - -- **Push event:** `git diff --name-only` antara `before` dan `after` SHA -- **repository_dispatch:** Parse payload `{service, sha}` dan validasi -- **workflow_dispatch:** Build semua service - -Output format matrix: -```json -[{"id":"scraper-api","target":"docker-scraper","path":"apps/scraper"}] -``` - -#### Phase 2: Build & Push (Matrix) - -Job `build` berjalan paralel per service (matrix strategy): - -1. Checkout repo + sync submodule -2. Jika `repository_dispatch`, checkout submodule ke SHA tertentu -3. Login ke GHCR -4. Setup Docker Buildx -5. Build & push dengan tag: - - `ghcr.io/asepharyana/asepharyana-hub/:latest` - - `ghcr.io/asepharyana/asepharyana-hub/:sha-` -6. Build cache: registry-based (`::buildcache`) - -#### Phase 3: Update Manifests - -Job `update-manifest`: - -1. Update image tag di compose file (`infra/compose/.yml`) -2. Jika `repository_dispatch`, update submodule pointer -3. Commit dengan message `chore: update manifests and submodules [skip ci]` -4. Push dengan retry (3 attempts, rebase jika conflict) - -### 3. Deploy Docker to VPS (`deploy-docker.yml`) - -**Trigger:** -- `workflow_run` setelah `docker-build-push.yml` selesai -- Push ke `main` yang mengubah `infra/**` -- `workflow_dispatch` (manual) - -**Concurrency:** Satu deployment dalam satu waktu (`group: deploy-vps`) - -**Aksi di VPS (via SSH):** - -``` -1. Setup SSH multiplexing -2. SCP .env dari GitHub secret ke VPS -3. Docker login ke GHCR -4. Git sync (fetch + reset --hard) -5. Detect changed files: - ├─ Compose stack changes → selective container update - ├─ Traefik dynamic config → SIGHUP - └─ Other infra → full deploy -6. Pull images (retry 3x) -7. Remove stale containers -8. Up services -9. SIGHUP Traefik jika perlu -``` - -### 4. Security Scan (`security.yml`) - -**Trigger:** -- PR ke `main` -- Jadwal: Setiap Senin (`0 6 * * 1`) - -**Aksi:** -- Checkout dengan fetch-depth 2 -- CodeQL init untuk Rust -- `cargo build` di `apps/scraper` -- CodeQL analyze - -### 5. Update Submodule Pointer (`update-submodule.yml`) - -**Trigger:** `repository_dispatch` tipe `submodule-updated` - -**Aksi:** -1. Validasi payload (`service`, `sha`) -2. Map service ke submodule path (e.g., `scraper-api` → `apps/scraper`) -3. Update submodule ke SHA yang diberikan -4. Commit sebagai `monrepo-bot` dengan message: - `chore: update to ` -5. Push dengan retry (3 attempts) - -## Flow Submodule Update - -Flow lengkap ketika code berubah di submodule repo: - -``` -1. Developer push ke asepharyana-hub-scraper -2. GitHub Action di scraper repo kirim repository_dispatch - ke asepharyana-hub -3. update-submodule.yml terima dispatch, update pointer -4. Commit masuk ke hub repo main -5. Commit ini trigger docker-build-push.yml - (push ke main dengan path apps/scraper/**) -6. Build image baru, update compose file -7. Deploy ke VPS -``` - -## Secrets yang Diperlukan - -| Secret | Workflow | Deskripsi | -|--------|----------|-----------| -| `SSH_PRIVATE_KEY` | deploy-docker | SSH key untuk akses VPS | -| `VPS_HOST` | deploy-docker | IP VPS (`45.127.35.244`) | -| `VPS_USER` | deploy-docker | User SSH (`root`) | -| `VPS_TARGET_DIR` | deploy-docker | Dir di VPS (`/root/asepharyana-hub`) | -| `ENV_FILE_PRODUCTION` | deploy-docker | Full `.env` production | - -## Menambahkan Service Baru ke Pipeline - -Untuk menambahkan service baru, update: - -### `docker-build-push.yml` - -1. **Phase 1 — `changes` job:** Tambah detection logic untuk service baru: - -```yaml -echo "new-service=$(changed '^(apps/new-service(/|$)|\.github/workflows/docker-build-push\.yml$|infra/docker/new-service\.Dockerfile$)')" >> "$GITHUB_OUTPUT" -``` - -2. **Phase 1 — `repository_dispatch`:** Tambah case: - -```yaml -case "$SERVICE" in - scraper-api|new-service) ;; -``` - -3. **Phase 1 — `set-matrix`:** Tambah service: - -```bash -if [ "${{ ...['new-service'] == 'true' ... }}" == "true" ]; then add_service "new-service" "docker-new-service" "apps/new-service"; fi -``` - -4. **Phase 2 — `meta` step:** Tambah mapping Dockerfile: - -```bash -"new-service") echo "dockerfile=infra/docker/new-service.Dockerfile" >> $GITHUB_OUTPUT ;; -``` - -5. **Phase 3 — `update-manifest`:** Tambah mapping: - -```bash -SERVICES["new-service"]="new-service.yml" -PATHS["new-service"]="apps/new-service" -``` - -### `deploy-docker.yml` - -Tambah compose file ke `ALL_COMPOSE_FILES`: - -```bash -ALL_COMPOSE_FILES="infra/compose/traefik.yml infra/compose/shared.yml infra/compose/scraper.yml infra/compose/nats.yml infra/compose/dapr.yml infra/compose/new-service.yml" -``` - -## Rollback - -### Rollback Image - -```bash -# Cari SHA tag sebelumnya di GHCR packages -# Update compose file ke tag tersebut -sed -i 's|sha-badcommit|sha-goodcommit|g' infra/compose/scraper.yml -git commit -am "fix: rollback scraper-api to sha-goodcommit" -git push -``` - -### Rollback via Git Revert - -```bash -git revert HEAD -git push origin main -# Pipeline otomatis build dan deploy -``` - -## Monitoring Pipeline - -```bash -# Cek status workflow terbaru -gh run list --limit 5 - -# Lihat log workflow tertentu -gh run view --log - -# Trigger workflow manual -gh workflow run deploy-docker.yml -``` diff --git a/docs/nats-guide.md b/docs/nats-guide.md deleted file mode 100644 index 51905a1..0000000 --- a/docs/nats-guide.md +++ /dev/null @@ -1,286 +0,0 @@ -# NATS + JetStream Guide - -Dokumentasi konfigurasi, penggunaan, dan troubleshooting NATS di infrastruktur `asepharyana-hub`. - -## Arsitektur - -NATS berjalan di container `nats` dengan JetStream diaktifkan (`-js`). Data persistent disimpan di volume Docker `nats_data`. - -``` -Service ──► NATS (port 4222) ──► JetStream (disk) - │ - ├─ Monitoring HTTP: port 8222 - └─ Client connections: port 4222 -``` - -### Hubungan dengan Dapr - -Saat ini Dapr pub/sub menggunakan **Redis** (`pubsub.redis`), bukan NATS. NATS berfungsi sebagai message broker independen untuk: - -- Event streaming antar service -- Persistent job queues -- Pub/sub untuk service yang tidak menggunakan Dapr - -Jika ingin Dapr menggunakan NATS sebagai backend pub/sub, ganti komponen `pubsub.yaml`: - -```yaml -apiVersion: dapr.io/v1alpha1 -kind: Component -metadata: - name: pubsub -spec: - type: pubsub.nats - version: v1 - metadata: - - name: natsURL - value: nats://nats:4222 -``` - -## Konfigurasi Compose - -File: `infra/compose/nats.yml` - -```yaml -services: - nats: - container_name: nats - image: nats:latest - restart: always - networks: - app-shared-net: - aliases: - - nats - ports: - - '4222:4222' # client connections - - '8222:8222' # HTTP monitor - command: - - '-js' # enable JetStream - - '-sd' - - '/data' # storage directory - volumes: - - nats_data:/data -``` - -## CLI Tools - -### Install NATS CLI - -```bash -# Linux -curl -sf https://bin.nats.dev/nats | sh -sudo mv nats /usr/local/bin/ - -# Atau via package manager -# brew install nats-io/nats-tools/nats (macOS) -``` - -### Koneksi ke NATS - -```bash -# Dari host (port 4222 ter-expose) -nats context save hub --server nats://localhost:4222 --description "Hub Production" -nats context select hub - -# Test koneksi -nats server check -nats server info -``` - -### Manage Streams (JetStream) - -```bash -# List semua stream -nats stream list - -# Lihat detail stream -nats stream info - -# Buat stream -nats stream add \ - --subjects "hub.>" \ - --storage file \ - --max-msgs 1000000 \ - --max-bytes 1G \ - --retention limits - -# Hapus stream -nats stream rm - -# Purge (hapus semua message, retain stream) -nats stream purge -``` - -### Pub/Sub - -```bash -# Subscribe ke subject -nats sub "hub.>" -nats sub "hub.image.cached" - -# Publish message -nats pub "hub.test" '{"message": "hello"}' -nats pub "hub.image.cached" '{"original_url": "https://example.com/img.jpg", "cdn_url": "https://cdn.example.com/img.jpg"}' - -# Request-reply -nats request "hub.service.do" '{"task": "process"}' -``` - -### Monitoring via HTTP API - -```bash -# Server info -curl http://localhost:8222/ - -# JetStream info -curl http://localhost:8222/jszetstream - -# Stream detail -curl http://localhost:8222/jszetstream?stream= - -# Consumer info -curl http://localhost:8222/jszetstream?stream=&consumer= - -# Server stats -curl http://localhost:8222/varz - -# Connections -curl http://localhost:8222/connz -``` - -## Event Topics Convention - -Semua topik menggunakan prefix `hub.`: - -| Subject | Payload | Deskripsi | -|---------|---------|-----------| -| `hub.image.cached` | `{original_url, cdn_url, source}` | Image selesai di-cache | -| `hub.image.repaired` | `{old_url, new_url}` | CNAME image diperbaiki | -| `hub.scrape.anime.done` | `{source, slug, duration}` | Scrape anime selesai | -| `hub.system.alert` | `{service, level, message}` | Error/alert dari service | -| `hub.test` | Any | Testing | - -### Wildcard Subjects - -NATS mendukung wildcard: - -- `hub.>` — semua event hub (multi-level) -- `hub.image.*` — semua event image (single-level) -- `hub.*.done` — semua event yang selesai (single-level) - -## JetStream Configuration - -### Storage - -Data JetStream disimpan di volume Docker `nats_data`. - -Lokasi di VPS: -```bash -docker volume inspect nats_data -# atau -ls -la /var/lib/docker/volumes/nats_data/_data/ -``` - -### Memory & Limits - -NATS tidak memiliki konfigurasi limit memori default. Untuk production, pertimbangkan: - -```yaml -command: - - '-js' - - '-sd' - - '/data' - - '--max_pending_size=64MB' - - '--max_payload=1MB' -``` - -Atau gunakan NATS configuration file: - -```yaml -# nats-server.conf -jetstream: - max_memory_store: 256MB - max_file_store: 10GB -``` - -## Troubleshooting - -### Stream data tidak muncul - -```bash -# 1. Cek koneksi NATS -nats server check - -# 2. Cek apakah JetStream aktif -curl http://localhost:8222/jszetstream - -# 3. Cek stream dan message count -nats stream list - -# 4. Subscribe langsung untuk test -nats sub ">" -``` - -### NATS tidak bisa start - -```bash -# Cek log -docker logs nats - -# Cek apakah port 4222 sudah dipakai -ss -tlnp | grep 4222 - -# Cek volume data korup -docker run --rm -v nats_data:/data alpine ls -la /data - -# Restart -docker compose -f infra/compose/nats.yml up -d --force-recreate -``` - -### Disk JetStream penuh - -```bash -# Cek ukuran volume -docker system df -v | grep nats_data - -# Purge stream jika perlu -nats stream purge - -# Atau hapus volume (data hilang!) -docker compose -f infra/compose/nats.yml down -docker volume rm nats_data -docker compose -f infra/compose/nats.yml up -d -``` - -### Slow consumer - -```bash -# Cek consumer lag -nats stream info -# Lihat fields: "Pending" dan "Acknowledgment" - -# Lihat stats server -curl http://localhost:8222/varz | jq '.slow_consumers' -``` - -## Migration: Redis Pub/Sub ke NATS - -Jika ingin migrasi dari Dapr pub/sub Redis ke NATS: - -1. Buat stream NATS untuk topik `hub.>` -2. Update `infra/dapr/components/pubsub.yaml` dari `pubsub.redis` ke `pubsub.nats` -3. Deploy ulang semua service (Dapr sidecar akan reconnect) -4. Verifikasi event flow - -```yaml -# infra/dapr/components/pubsub.yaml (setelah migrasi) -apiVersion: dapr.io/v1alpha1 -kind: Component -metadata: - name: pubsub -spec: - type: pubsub.nats - version: v1 - metadata: - - name: natsURL - value: nats://nats:4222 -``` diff --git a/docs/plan/tools/README.md b/docs/plan/tools/README.md deleted file mode 100644 index cc1f3dc..0000000 --- a/docs/plan/tools/README.md +++ /dev/null @@ -1,60 +0,0 @@ -# Tools — Document Scanner & Media Processing Hub - -Self-hosted, no-install document scanner dan media processing tools yang jalan di browser. Alternatif dari CamScanner, ilovepdf, compressjpeg — tanpa upload ke pihak ketiga. - -## Visi - -Satu platform dengan tools manipulasi file yang **beneran dipake orang setiap hari**. Semua proses di backend Rust — cepat, hemat memory, ga perlu install software. - -## Fitur Utama - -### Phase 1 — Document Scanner (Prioritas) -- Foto dokumen pake HP → auto-detect tepi → lurusin (perspective correction) -- Enhance: iluminasi merata, contrast, sharpen, B&W -- OCR → searchable PDF (teks bisa di-copy, dicari) -- Batch: multi-page → satu PDF -- Fallback crop manual (kalau auto-detect gagal) - -### Phase 2 — Image Tools -- Compress JPEG/PNG/WebP (lossy + lossless, atur kualitas %) -- Resize batch (atur dimensi, semua foto disamain) -- Convert format (HEIC→JPEG, PNG→WebP, SVG→PNG) -- Remove background (ONNX model, Rust runtime) - -### Phase 3 — PDF Tools -- Merge PDF (gabung file) -- Split PDF (ekstrak halaman tertentu) -- Images→PDF (kumpulan foto jadi 1 file) -- PDF→Images (tiap halaman jadi gambar) -- PDF compress (turunkin kualitas embedded images) - -### Phase 4 — Video/Audio Tools -- Compress video (bitrate + resolusi) -- Extract audio (MP4→MP3) -- Trim/crop -- GIF maker -- Audio convert + trim - -## Target User - -Orang yang: -- Punya HP/PC, paham teknologi dasar (buka browser, upload file) -- Butuh scan dokumen tanpa install aplikasi -- Butuh kompres file buat kirim WA/email -- Butuh manipulasi PDF sesekali -- Peduli privasi — ga mau upload file ke server pihak ketiga - -## Prinsip Desain - -1. **Satu task selesai dalam <5 detik** — ga ada loading lama -2. **Drag & drop + preview** — liat hasil sebelum download -3. **Progress realtime** via WebSocket — tau lagi di tahap mana -4. **Batch processing** — banyak file, satu klik -5. **Privasi first** — file otomatis dihapus setelah 1 jam -6. **WASM fallback** — tools ringan jalan di client (tanpa upload) - -## Domain & Branding - -- **Domain**: `tools.asepharyana.my.id` | `tools.asepharyana.web.id` -- **Design**: Twilight Terminal theme (sama kaya portfolio), konsisten visual -- **Dashboard**: Link dari hub dashboard → tools stats (total files processed, storage used) diff --git a/docs/plan/tools/architecture.md b/docs/plan/tools/architecture.md deleted file mode 100644 index 4cbc20e..0000000 --- a/docs/plan/tools/architecture.md +++ /dev/null @@ -1,453 +0,0 @@ -# Architecture - -> **LEGACY (2026-08-02):** Dokumen plan ini ditulis saat infra masih Docker/Traefik. Produksi sekarang Caddy + Nix/systemd dengan port 4000-an. Gunakan hanya sebagai referensi historis. - -## System Overview - -``` -┌────────────────────────────────────────────────────────────────┐ -│ BROWSER │ -│ ┌────────────┐ ┌────────────┐ ┌────────────────────────┐ │ -│ │ Upload │ │ Camera │ │ Preview + Download │ │ -│ │ (drag/drop)│ │ (PWA) │ │ (streaming) │ │ -│ └─────┬──────┘ └─────┬──────┘ └───────────┬────────────┘ │ -│ │ │ │ │ -│ ▼ ▼ ▼ │ -│ ┌──────────────────────────────────────────────────────┐ │ -│ │ WebSocket (progress: processing/step/percentage) │ │ -│ └──────────────────────────────────────────────────────┘ │ -└──────────────────────────┬───────────────────────────────────┘ - │ HTTPS / WSS - ▼ -┌────────────────────────────────────────────────────────────────┐ -│ TRAEFIK (tools.asepharyana.my.id) │ -│ Middleware chain: secure-headers → compress → rate-limit │ -└──────────────────────────┬────────────────────────────────────┘ - │ - ▼ -┌────────────────────────────────────────────────────────────────┐ -│ tools-app (Next.js 16 / TypeScript) │ -│ │ -│ ┌──────────────────┐ ┌─────────────────┐ │ -│ │ Pages/Routes │ │ API Routes │ │ -│ │ / → home │ │ POST /api/upload ──▶ file │ -│ │ /scan → scanner │ │ GET /api/job/:id ─▶ status │ -│ │ /image → image │ │ WS /api/job/:id/ws ─▶ progress │ -│ │ /pdf → pdf tools │ │ GET /api/download/:id ─▶ file │ -│ └──────────────────┘ └─────────────────┘ │ -│ │ -│ Upload validation: MIME type, size limit (50MB), virus scan │ -│ Temp storage bridge ke worker via HTTP/NATS │ -└──────────────────┬────────────────────────────────────────────┘ - │ HTTP (internal) - ▼ -┌────────────────────────────────────────────────────────────────┐ -│ API GATEWAY (Rust / Axum) │ -│ │ -│ ┌──────────────┐ ┌──────────────┐ ┌────────────────────┐ │ -│ │ Upload │ │ Job Manager │ │ Download │ │ -│ │ (streaming │ │ (CRUD job │ │ (stream file, │ │ -│ │ chunked) │ │ status) │ │ auto-delete) │ │ -│ └──────┬───────┘ └──────┬───────┘ └────────────────────┘ │ -│ │ │ │ -│ ▼ ▼ │ -│ ┌────────────────────────────────────────────────────┐ │ -│ │ NATS JetStream │ │ -│ │ ┌──────────┐ ┌──────────┐ ┌──────────────┐ │ │ -│ │ │ scan. │ │ image. │ │ pdf. │ │ │ -│ │ │ jobs │ │ jobs │ │ jobs │ │ │ -│ │ └────┬─────┘ └────┬─────┘ └──────┬───────┘ │ │ -│ │ │ │ │ │ │ -│ │ ▼ ▼ ▼ │ │ -│ │ ┌──────────┐ ┌──────────┐ ┌──────────────┐ │ │ -│ │ │ scan. │ │ image. │ │ pdf. │ │ │ -│ │ │ progress │ │ progress │ │ progress │ │ │ -│ │ └──────────┘ └──────────┘ └──────────────┘ │ │ -│ └────────────────────────────────────────────────────┘ │ -│ │ -│ ┌────────────────────────────────────────────────────┐ │ -│ │ Cache (Redis) │ │ -│ │ - Job metadata (status, progress, timestamps) │ │ -│ │ - Rate limiting (sliding window per IP/tool) │ │ -│ │ - Result metadata (file path, size, type) │ │ -│ └────────────────────────────────────────────────────┘ │ -└──────────────────┬────────────────────────────────────────────┘ - │ consume NATS queue - ▼ -┌────────────────────────────────────────────────────────────────┐ -│ WORKER POOL (Rust / Tokio + Rayon) │ -│ │ -│ ┌─────────────────┐ ┌─────────────────┐ ┌──────────────┐ │ -│ │ Scan Worker │ │ Image Worker │ │ PDF Worker │ │ -│ │ ×4 instances │ │ ×2 instances │ │ ×2 instances│ │ -│ │ │ │ │ │ │ │ -│ │ 1. Load image │ │ 1. Load image │ │ 1. Load PDF │ │ -│ │ 2. Edge detect │ │ 2. Compress │ │ 2. Merge/ │ │ -│ │ 3. Warp │ │ /resize/ │ │ split │ │ -│ │ 4. Enhance │ │ convert │ │ 3. Save │ │ -│ │ 5. OCR │ │ 3. Save │ │ 4. Update │ │ -│ │ 6. Gen PDF │ │ 4. Update │ │ job │ │ -│ │ 7. Update job │ │ job status │ │ status │ │ -│ │ └───────────────┘ └─────────────────┘ └──────────────┘ │ -│ │ │ -│ ▼ │ -│ ┌────────────────────────────────────────────────────┐ │ -│ │ Temp Storage (filesystem volume / S3-compatible) │ │ -│ │ Auto-cleanup: job TTL 1 jam, NATS cron tiap 10m │ │ -│ └────────────────────────────────────────────────────┘ │ -└────────────────────────────────────────────────────────────────┘ -``` - -## Component Diagram - -``` -┌────────────────────────────────────────────┐ -│ apps/tools │ -│ │ -│ ├── frontend/ │ -│ │ ├── pages/ ← Next.js pages │ -│ │ ├── components/ ← React components │ -│ │ ├── lib/ ← utilities │ -│ │ └── public/ ← static assets │ -│ │ │ -│ ├── backend/ ← Rust workspace │ -│ │ ├── gateway/ ← Axum API server │ -│ │ ├── workers/ ← Processing workers │ -│ │ │ ├── scanner/ ← Document scanner │ -│ │ │ ├── image/ ← Image tools │ -│ │ │ └── pdf/ ← PDF tools │ -│ │ └── common/ ← Shared libs │ -│ │ │ -│ └── Dockerfile │ -└────────────────────────────────────────────┘ -``` - -## Data Flow (Document Scanner — Flow Lengkap) - -``` -1. User buka tools.asepharyana.my.id/scan -2. Upload foto via drag-drop atau kamera HP (PWA) -3. Next.js route handler menerima file - ├─ Validasi: MIME type (image/*), max 50MB, virus header scan - └─ Upload chunked ke Gateway internal (HTTP POST) - -4. Gateway menerima stream: - ├─ Simpan ke temp storage - ├─ Buat job record di Redis: {id, tool: "scan", status: "queued", progress: 0} - └─ Publish ke NATS: tools.scan.jobs {job_id, file_path, options} - -5. Scan Worker consume dari NATS: - ├─ Update Redis: status = "processing", progress = 10 - ├─ Load image (image-rs) - ├─ Pipeline (detail di pipeline.md): - │ 1. Edge detection ──▶ progress 25 - │ 2. Perspective warp ──▶ progress 40 - │ 3. Shadow removal ──▶ progress 55 - │ 4. Binarization ──▶ progress 70 - │ 5. Contrast/sharpen ──▶ progress 80 - │ 6. OCR ──▶ progress 90 - │ 7. Generate PDF ──▶ progress 95 - ├─ Simpan file hasil ke temp storage - ├─ Update Redis: status = "completed", progress = 100, result_path, ocr_text - └─ Publish ke NATS: tools.scan.progress {job_id, status, progress} - -6. WebSocket handler di Gateway: - ├─ Subscribe NATS topics tools.scan.progress - ├─ Forward ke browser user (per-job-id filter) - └─ Browser update progress bar + preview - -7. User download PDF: - ├─ GET /api/download/:job_id - ├─ Gateway stream file dari temp storage - └─ Browser save file -``` - -## Tech Stack - -### Frontend (Next.js + TypeScript) - -| Library | Fungsi | -|---------|--------| -| Next.js 16 | App router, API routes | -| shadcn/ui + Tailwind v4 | UI components | -| Framer Motion | Animasi progress, transisi | -| Canvas API | Preview crop manual, image manipulation client-side | -| WebSocket API | Real-time progress | - -### Backend (Rust) - -| Crate | Fungsi | -|-------|--------| -| `axum` | HTTP server (Gateway) | -| `tokio` | Async runtime | -| `image` | Image I/O, resize, convert, compress | -| `imageproc` | Edge detection, contour, thresholding | -| `lopdf` | PDF generation, merge, split, compress | -| `leptess` | Tesseract OCR binding | -| `ort` | ONNX Runtime (background removal) | -| `async-nats` | NATS JetStream client | -| `deadpool-redis` | Redis connection pool | -| `redis` | Redis async client | -| `rayon` | Parallel processing (batch, pixel ops) | -| `serde` | Serialization | -| `tracing` + `opentelemetry` | Observability | -| `uuid` | Job ID generation | - -### Infrastructure - -| Komponen | Fungsi | -|----------|--------| -| NATS JetStream | Job queue, progress pub/sub, scheduler | -| Redis | Job metadata, rate limiting, cache | -| PostgreSQL | Opsional — audit log, usage statistics | -| Tesseract | OCR engine (data files di Docker image) | -| Prometheus | Metrics (jobs/min, queue depth, latency per stage) | - -## Job Queue (NATS Streams & Consumers) - -### Streams - -``` -tools-scan-jobs → 1 stream, mirror to all scan workers -tools-image-jobs → 1 stream, mirror to all image workers -tools-pdf-jobs → 1 stream, mirror to all pdf workers -tools-progress → 1 stream, all progress events (key-value by job_id) -tools-scheduler → 1 stream, cron events -``` - -### Subjects - -``` -tools.scan.jobs.{job_id} → job submission -tools.scan.progress.{job_id} → progress update (fan-out ke Gateway) -tools.image.jobs.{job_id} → job submission -tools.image.progress.{job_id} → progress update -tools.pdf.jobs.{job_id} → job submission -tools.pdf.progress.{job_id} → progress update -tools.scheduler.cleanup → cleanup expired files (every 10 min) -``` - -## Redis Schema - -``` -job:{id} → Hash {status, tool, progress, file_path, result_path, ocr_text, created_at, ttl} -rate_limit:{ip}:{tool} → Sorted Set (sliding window) -file_meta:{hash} → String {original_name, size, mime} -``` - -## Metrics (Prometheus) - -| Metric | Type | Labels | Description | -|--------|------|--------|-------------| -| `tools_jobs_total` | Counter | `tool`, `status` | Total jobs processed | -| `tools_jobs_in_flight` | Gauge | `tool` | Currently processing jobs | -| `tools_queue_depth` | Gauge | `tool` | NATS queue depth | -| `tools_processing_duration` | Histogram | `tool`, `stage` | Duration per stage | -| `tools_file_size_bytes` | Histogram | `tool` | Upload file size distribution | -| `tools_rate_limit_hits` | Counter | `tool` | Rate limit violations | - -## Directory Structure - -``` -apps/tools/ -├── frontend/ -│ ├── src/ -│ │ ├── app/ -│ │ │ ├── page.tsx # Landing page -│ │ │ ├── scan/ -│ │ │ │ ├── page.tsx # Scanner page -│ │ │ │ └── result/[id]/ -│ │ │ │ └── page.tsx # Result page -│ │ │ ├── image/ -│ │ │ │ ├── compress/page.tsx -│ │ │ │ ├── resize/page.tsx -│ │ │ │ ├── convert/page.tsx -│ │ │ │ └── remove-bg/page.tsx -│ │ │ ├── pdf/ -│ │ │ │ ├── merge/page.tsx -│ │ │ │ ├── split/page.tsx -│ │ │ │ ├── images-to-pdf/page.tsx -│ │ │ │ └── compress/page.tsx -│ │ │ ├── api/ -│ │ │ │ ├── upload/route.ts -│ │ │ │ ├── job/[id]/route.ts -│ │ │ │ │ └── ws/route.ts -│ │ │ │ └── download/[id]/route.ts -│ │ │ ├── layout.tsx -│ │ │ └── globals.css -│ │ ├── components/ -│ │ │ ├── upload-zone.tsx # Drag & drop area -│ │ │ ├── progress-bar.tsx # WebSocket-connected progress -│ │ │ ├── preview.tsx # Before/after preview -│ │ │ ├── crop-editor.tsx # Manual corner adjustment -│ │ │ ├── tool-layout.tsx # Consistent tool page layout -│ │ │ └── camera-capture.tsx # PWA camera interface -│ │ ├── hooks/ -│ │ │ ├── use-job-status.ts # WebSocket connection -│ │ │ ├── use-upload.ts # Upload with progress -│ │ │ └── use-camera.ts # Camera access -│ │ └── lib/ -│ │ ├── utils.ts -│ │ └── types.ts -│ ├── next.config.ts -│ ├── package.json -│ └── tsconfig.json -│ -├── backend/ -│ ├── Cargo.toml -│ ├── gateway/ -│ │ ├── Cargo.toml -│ │ └── src/ -│ │ ├── main.rs -│ │ ├── routes/ -│ │ │ ├── mod.rs -│ │ │ ├── upload.rs -│ │ │ ├── job.rs -│ │ │ ├── download.rs -│ │ │ └── ws.rs -│ │ ├── nats/ -│ │ │ ├── mod.rs -│ │ │ └── publisher.rs -│ │ ├── redis/ -│ │ │ ├── mod.rs -│ │ │ ├── job.rs -│ │ │ └── ratelimit.rs -│ │ ├── metrics.rs -│ │ └── config.rs -│ │ -│ ├── workers/ -│ │ ├── Cargo.toml -│ │ └── src/ -│ │ ├── main.rs -│ │ ├── scanner/ -│ │ │ ├── mod.rs -│ │ │ ├── pipeline.rs -│ │ │ ├── edge.rs # Edge detection -│ │ │ ├── warp.rs # Perspective correction -│ │ │ ├── enhance.rs # Shadow removal, B&W, contrast -│ │ │ ├── ocr.rs # Tesseract wrapper -│ │ │ └── pdf.rs # Generate searchable PDF -│ │ ├── image/ -│ │ │ ├── mod.rs -│ │ │ ├── compress.rs -│ │ │ ├── resize.rs -│ │ │ ├── convert.rs -│ │ │ └── remove_bg.rs -│ │ ├── pdf/ -│ │ │ ├── mod.rs -│ │ │ ├── merge.rs -│ │ │ ├── split.rs -│ │ │ ├── extract.rs -│ │ │ └── compress.rs -│ │ ├── nats/ -│ │ │ ├── mod.rs -│ │ │ └── consumer.rs -│ │ └── config.rs -│ │ -│ └── common/ -│ ├── Cargo.toml -│ └── src/ -│ ├── lib.rs -│ ├── types.rs # Shared types (JobStatus, Job, etc.) -│ ├── error.rs # Error types -│ └── nats.rs # NATS subject constants -│ -├── Dockerfile -├── compose.yml # Local dev compose -└── README.md -``` - -## API Design - -### Endpoints - -| Method | Path | Description | -|--------|------|-------------| -| `POST` | `/api/upload` | Upload file, create job | -| `GET` | `/api/job/:id` | Get job status + result metadata | -| `WS` | `/api/job/:id/ws` | WebSocket — realtime progress | -| `GET` | `/api/download/:id` | Download result file | -| `DELETE` | `/api/job/:id` | Cancel job, delete files | -| `GET` | `/health` | Health check | - -### Upload Request - -``` -POST /api/upload -Content-Type: multipart/form-data - -{ - file: , - tool: "scan" | "image-compress" | "image-resize" | "image-convert" | "remove-bg" | - "pdf-merge" | "pdf-split" | "images-to-pdf" | "pdf-compress", - options?: { // tool-specific options - quality?: 80, // compress quality - width?: 1920, // resize width - format?: "webp", // convert format - pages?: "1,3-5", // PDF split pages - dpi?: 300, // scan DPI - enhance?: true, // scan auto-enhance - ocr?: true // scan OCR - } -} -``` - -### Response (202 Accepted) - -```json -{ - "job_id": "uuid", - "status": "queued", - "tool": "scan", - "ws_url": "/api/job/uuid/ws", - "created_at": "2026-07-24T10:00:00Z", - "estimated_seconds": 5 -} -``` - -### WebSocket Messages - -```json -// Server → Client -{ - "type": "progress", - "job_id": "uuid", - "status": "processing", - "progress": 45, - "stage": "warp", - "message": "Meluruskan perspektif dokumen..." -} - -{ - "type": "complete", - "job_id": "uuid", - "status": "completed", - "progress": 100, - "result": { - "download_url": "/api/download/uuid", - "file_name": "scan_20260724.pdf", - "file_size": 1245678, - "pages": 1, - "ocr_text": "Nama: Asep...", - "preview_url": "/api/job/uuid/preview" - } -} - -{ - "type": "error", - "job_id": "uuid", - "status": "failed", - "error": "Edge detection failed: cannot find document boundary" -} -``` - -## Integration with Existing Portfolio - -| Area | Detail | -|------|--------| -| **Domain** | `tools.asepharyana.my.id` — tambah entry di `infra/traefik/dynamic/apps.yaml` | -| **Dashboard** | Link ke tools stats di dashboard hub yang sudah ada | -| **Docker Compose** | `infra/compose/tools.yml` — pola sama kaya `hub.yml` | -| **CI/CD** | Tambah service `tools` di `docker-build-push.yml` | -| **Style** | Ulang Twilight Terminal theme dari hub, konsisten visual branding | -| **Monitoring** | Reuse existing Prometheus + Grafana, tambah metrics tools | diff --git a/docs/plan/tools/implementation.md b/docs/plan/tools/implementation.md deleted file mode 100644 index 23a57b8..0000000 --- a/docs/plan/tools/implementation.md +++ /dev/null @@ -1,1693 +0,0 @@ -# Implementation Plan — Granular Task Breakdown - -> **LEGACY (2026-08-02):** Dokumen plan ini ditulis saat infra masih Docker/Traefik. Produksi sekarang Caddy + Nix/systemd dengan port 4000-an. Gunakan hanya sebagai referensi historis. - -Setiap task adalah unit kerja terkecil yang bisa dikerjakan dalam 1-4 jam. Format: - -``` -[ID] Task description - Files: path/to/file - Accept: criteria yang harus terpenuhi -``` - ---- - -## Phase 1: Foundation + Document Scanner (Prioritas) - -### Milestone 1.1 — Rust Backend Skeleton - -**Goal**: Gateway + Worker connected ke NATS + Redis, upload flow end-to-end. - -#### 1.1.1 — Init Rust Workspace - -``` -[1.1.1] Buat Cargo workspace dengan 4 crate: common, gateway, workers, wasm - Files: - apps/tools/backend/Cargo.toml (workspace definition) - apps/tools/backend/common/Cargo.toml (serde, uuid, chrono) - apps/tools/backend/gateway/Cargo.toml (axum, tokio, tower, async-nats, deadpool-redis, redis) - apps/tools/backend/workers/Cargo.toml (tokio, async-nats, redis, image, imageproc, lopdf, leptess, ort, rayon) - apps/tools/backend/wasm/Cargo.toml (wasm-bindgen, image, console-error-panic) - apps/tools/backend/rust-toolchain.toml (channel = "1.85") - Accept: cargo build —release works untuk semua crate (walaupun main.rs masih empty) -``` - -#### 1.1.2 — Common Types - -``` -[1.1.2] Define shared types: JobStatus enum, Job struct, Tool enum, ScanOptions, ImageOptions, - PdfOptions, UploadResponse, JobProgress - Files: - apps/tools/backend/common/src/lib.rs - apps/tools/backend/common/src/types.rs - Types: - JobStatus { Queued, Processing { stage: String, progress: u8 }, Completed, Failed(String) } - Tool { Scan, ImageCompress, ImageResize, ImageConvert, RemoveBg, - PdfMerge, PdfSplit, ImagesToPdf, PdfCompress, PdfToImages, - VideoCompress, AudioExtract, VideoTrim, GifMaker, AudioConvert } - Job { id: Uuid, tool: Tool, status: JobStatus, file_path: PathBuf, - result_path: Option, file_size: u64, options: Value, - created_at: DateTime, ttl_seconds: u64 } - JobProgress { job_id: Uuid, status: JobStatus, stage: String, progress: u8, message: String } - Accept: Semua type implements Serialize + Deserialize + Debug + Clone -``` - -#### 1.1.3 — Common Errors - -``` -[1.1.3] Define error types dengan thiserror - Files: - apps/tools/backend/common/src/error.rs - Errors: - UploadError (InvalidMime, FileTooLarge, Io, VirusDetected) - PipelineError (ImageLoad, EdgeDetection, Warp, Ocr, PdfGeneration, Timeout, Internal) - NatsError (Publish, Subscribe, JetStream, Timeout) - RedisError (Connection, Query, Serialization) - Accept: Setiap error punya Display + Source chain yang jelas -``` - -#### 1.1.4 — NATS Subjects & Streams - -``` -[1.1.4] Define NATS subject constants + stream configuration - Files: - apps/tools/backend/common/src/nats.rs - Subjects: - tools.scan.jobs → Queue for scan workers - tools.scan.progress → Fan-out progress events - tools.image.jobs - tools.image.progress - tools.pdf.jobs - tools.pdf.progress - tools.video.jobs - tools.video.progress - tools.scheduler.cleanup → Cron: cleanup expired files - Streams: - tools-jobs (max_age: 24h, storage: file) - tools-progress (max_age: 1h, storage: memory) - Accept: Unit test verifikasi format subject string -``` - -#### 1.1.5 — Gateway Config - -``` -[1.1.5] Environment-based configuration loader - Files: - apps/tools/backend/gateway/src/config.rs - Env vars: - GATEWAY_PORT (default: 3001) - NATS_URL (default: nats://localhost:4222) - REDIS_URL (default: redis://localhost:6379) - STORAGE_PATH (default: /data/tools) - MAX_FILE_SIZE_MB (default: 50) - JOB_TTL_SECONDS (default: 3600) - RATE_LIMIT_PER_MINUTE (default: 30) - RUST_LOG (default: info) - Accept: AppConfig struct dengan semua field, load dari env + fallback default -``` - -#### 1.1.6 — Gateway: Upload Route (POST /api/upload) - -``` -[1.1.6] Multipart file upload handler - Files: - apps/tools/backend/gateway/src/routes/upload.rs - apps/tools/backend/gateway/src/routes/mod.rs - Logic: - 1. Extract multipart: file + tool + options - 2. Validate MIME type (image/* for scan/image tools, application/pdf for pdf tools, - video/* for video tools, audio/* for audio tools) - 3. Check file size < MAX_FILE_SIZE_MB - 4. Scan magic bytes: verify actual content matches extension - 5. Save file ke {STORAGE_PATH}/upload/{uuid}.{ext} - 6. Generate job_id (Uuid v4) - 7. Save job metadata ke Redis: SET job:{job_id} → JSON - 8. Publish ke NATS: tools.{tool}.jobs → {job_id, file_path, options} - 9. Return 202: { job_id, status: "queued", tool, ws_url } - Accept: curl upload → 202 + job_id. curl job_status → status field populated -``` - -#### 1.1.7 — Gateway: Job Status Route (GET /api/job/{id}) - -``` -[1.1.7] Job status and metadata retrieval - Files: - apps/tools/backend/gateway/src/routes/job.rs - Logic: - 1. Extract job_id from path - 2. GET job:{job_id} from Redis - 3. Return 404 kalau not found - 4. Return JSON: { job_id, status, tool, progress, stage, message, - result: Option<{ download_url, file_size, file_name, preview_url }>, - created_at, error: Option } - Endpoints: - GET /api/job/{id} → Single job status - GET /api/job/{id}/preview → Preview image (thumbnail) - Accept: curl GET → full job status JSON -``` - -#### 1.1.8 — Gateway: Download Route (GET /api/download/{id}) - -``` -[1.1.8] File download with streaming + auto-cleanup awareness - Files: - apps/tools/backend/gateway/src/routes/download.rs - Logic: - 1. Extract job_id, get status from Redis - 2. Check status == completed → get result_path - 3. Stream file via tokio::fs::File → axum body stream - 4. Set Content-Disposition header with original filename - 5. Set Content-Type based on file extension - 6. Handle 404 (not found), 400 (not completed yet), 410 (expired) - Accept: curl GET /api/download/{id} → file download dengan correct headers -``` - -#### 1.1.9 — Gateway: Health & Metrics - -``` -[1.1.9] Health check + Prometheus metrics endpoint - Files: - apps/tools/backend/gateway/src/routes/health.rs - apps/tools/backend/gateway/src/metrics.rs - Endpoints: - GET /health → 200 OK (used by Traefik health check) - GET /metrics → Prometheus text format - Metrics (via custom counters, no external crate): - tools_uploaded_files_total → counter, labels: tool, status - tools_jobs_total → counter, labels: tool, status - tools_processing_duration_ms → histogram, labels: tool - tools_queue_depth → gauge, labels: tool - Accept: /health returns 200, /metrics returns prometheus-format text -``` - -#### 1.1.10 — Gateway: NATS Publisher - -``` -[1.1.10] NATS connection management + publish helpers - Files: - apps/tools/backend/gateway/src/nats/mod.rs - apps/tools/backend/gateway/src/nats/publisher.rs - Functions: - connect_nats(url: &str) → Result - publish_job(nats: &Connection, tool: Tool, payload: &Job) → Result<()> - publish_progress(nats: &Connection, progress: &JobProgress) → Result<()> - Accept: Integration test: publish message → consume back via subscriber -``` - -#### 1.1.11 — Gateway: Redis Client - -``` -[1.1.11] Redis connection pool + job CRUD operations - Files: - apps/tools/backend/gateway/src/redis/mod.rs - apps/tools/backend/gateway/src/redis/job.rs - apps/tools/backend/gateway/src/redis/ratelimit.rs - Functions: - connect_redis(url: &str) → Result - job_create(redis, job) → Result<()> - job_get(redis, job_id) → Result - job_update(redis, job_id, status) → Result<()> - job_delete(redis, job_id) → Result<()> - rate_limit_check(redis, ip, tool) → Result // sliding window - TTL: Set TTL JOB_TTL_SECONDS on job_create - Accept: Redis integration test: create → get → update → delete → not found -``` - -#### 1.1.12 — Gateway: Main Bootstrap - -``` -[1.1.12] Axum app assembly + graceful shutdown - Files: - apps/tools/backend/gateway/src/main.rs - Logic: - 1. Load config - 2. Init Redis connection pool - 3. Init NATS connection - 4. Build Axum router with all routes - 5. Spawn NATS progress consumer (subscribe tools.*.progress, caches in Redis) - 6. Start HTTP server on configured port - 7. Graceful shutdown on SIGINT/SIGTERM - States: SharedState { redis, nats, config } wrapped in Arc - Accept: cargo run —bin gateway → server listening on :3001 -``` - -#### 1.1.13 — Workers: NATS Consumer - -``` -[1.1.13] NATS JetStream consumer for job queues - Files: - apps/tools/backend/workers/src/main.rs - apps/tools/backend/workers/src/nats/mod.rs - apps/tools/backend/workers/src/nats/consumer.rs - Logic: - 1. Connect to NATS + Redis - 2. Subscribe to JetStream streams: tools.*.jobs - 3. For each message: - a. Deserialize job payload - b. Match tool → dispatch to appropriate handler - c. Acknowledge after handler returns - d. NACK with delay on failure (for retry) - 4. Max delivery: 3, then dead-letter - Max concurrency: configurable (TOOLS_WORKER_CONCURRENCY, default: 4) - Accept: Workers start, consume NATS messages, dispatch to tool handlers -``` - -#### 1.1.14 — Workers: Progress Publisher - -``` -[1.1.14] Progress reporting from worker to NATS → Gateway → WebSocket - Files: - apps/tools/backend/workers/src/nats/progress.rs - Functions: - report_progress(redis, nats, job_id, status, stage, progress, message) - 1. Update Redis: job:{job_id} status + progress - 2. Publish to NATS: tools.{tool}.progress → {job_id, status, stage, progress, message} - Helper: ProgressReporter struct yang implements Clone, bisa dipass ke pipeline - Accept: Worker updates progress → Gateway receives → WebSocket forwards -``` - -#### 1.1.15 — Worker: Scanner Stub - -``` -[1.1.15] Scanner worker yang bisa menerima job dan update progress - Files: - apps/tools/backend/workers/src/scanner/mod.rs - Logic: - 1. Receive job: { job_id, file_path, options } - 2. report_progress(queued → processing:10%) - 3. sleep 2s (simulasi pipeline) - 4. report_progress(processing:50%) - 5. sleep 2s - 6. Copy input file to output (simulasi hasil) - 7. report_progress(completed:100%) - 8. Update Redis with result_path - Accept: Upload → NATS queue → worker consume → progress update → completed -``` - ---- - -### Milestone 1.2 — Scanner Pipeline Core - -**Goal**: Foto miring → lurus + bersih + hitam-putih (belum OCR/PDF). - -#### 1.2.1 — Preprocess: Load & Resize - -``` -[1.2.1] Load image from file, resize if too large, convert to grayscale - Files: - apps/tools/backend/workers/src/scanner/preprocess.rs - Functions: - load_image(path: &Path) → Result - safe_resize(img: DynamicImage, max_dim: u32) → DynamicImage - to_grayscale(img: &DynamicImage) → GrayImage - Rules: - - Resize if max(width, height) > 2000px → scale down, preserve aspect ratio - - Lanczos3 filter untuk downscale (sharpest) - - Support input: JPEG, PNG, WebP, HEIC (if feature enabled) - Accept: Unit test: 12MP image → resize to ≤2000px, verify aspect ratio preserved -``` - -#### 1.2.2 — Edge Detection: Canny + Morphological Close - -``` -[1.2.2] Canny edge detection with morphological operations to connect broken edges - Files: - apps/tools/backend/workers/src/scanner/edge.rs - Functions: - detect_edges(img: &GrayImage) → GrayImage - morphological_close(edges: &GrayImage, kernel_size: u8) → GrayImage - Algorithm: - 1. Gaussian blur (sigma=1.0) on grayscale - 2. Canny with low_threshold=50, high_threshold=150 - 3. Morphological close: dilate → erode with 5x5 kernel - 4. If edge_count < 1% of total pixels → retry Canny(20, 80) - Accept: Unit test: known test images → edge image with continuous document borders -``` - -#### 1.2.3 — Corner Detection: Largest Rectangle Contour - -``` -[1.2.3] Find 4 corners of the document from edge image - Files: - apps/tools/backend/workers/src/scanner/corners.rs - Functions: - find_contours(edges: &GrayImage) → Vec - largest_rectangular_contour(contours: &[Contour]) -> Option - approx_polygon(contour: &Contour, num_vertices: u32) → Option> - order_corners(points: Vec<(f64,f64)>) -> [(f64,f64); 4] - detect_corners(img: &GrayImage) -> Result<[(f64,f64); 4], FallbackReason> - Algorithm: - 1. imageproc::contours::find_contours - 2. Filter by area > 20% of total image - 3. Top 5 largest by contour area - 4. For each: approx polygon, find 4-vertex polygon - 5. Order: top-left, top-right, bottom-right, bottom-left - Accept: Unit test: 5 test images (normal, dark, angle, shadow, cluttered bg) - → correct corners or explicit fallback -``` - -#### 1.2.4 — Perspective Warp: DLT Homography - -``` -[1.2.4] Compute homography matrix via DLT + SVD, apply perspective warp - Files: - apps/tools/backend/workers/src/scanner/warp.rs - Functions: - compute_homography(src: &[(f64,f64);4], dst: &[(f64,f64);4]) → [[f64;3];3] - invert_homography(h: &[[f64;3];3]) → [[f64;3];3] - apply_homography(h: &[[f64;3];3], x: f64, y: f64) → (f64, f64) - bilinear_interpolate(img: &GrayImage, x: f64, y: f64) → Luma - warp_perspective(img: &DynamicImage, corners: [(f64,f64);4]) → DynamicImage - Algorithm: - DLT (Direct Linear Transform): - - 4 point correspondences → 8x9 matrix A - - SVD (via ndarray + ndarray-linalg or nalgebra) - - H = last column of V, reshape to 3x3 - Backward mapping: - - For each output pixel (x,y), compute source (sx,sy) via H_inv - - Bilinear interpolate from source - Accept: Unit test: 4 corners of known grid → warped image is perfectly rectangular -``` - -#### 1.2.5 — Shadow Removal: Illumination Correction - -``` -[1.2.5] Remove uneven lighting, shadows, and glare - Files: - apps/tools/backend/workers/src/scanner/shadow.rs - Functions: - gaussian_blur_large(img: &GrayImage, radius: f64) → GrayImage - subtract_background(img: &GrayImage, background: &GrayImage) -> GrayImage - apply_clahe(img: &GrayImage, tile_size: u8, clip_limit: u8) -> GrayImage - remove_shadow(img: &GrayImage) → GrayImage - Algorithm (primary): - 1. Large Gaussian blur (radius = max_dim/50, min 15px) = illumination estimate - 2. Subtract: pixel = max(0, original - background + mean(background)) - 3. CLAHE: 8x8 tiles, clip limit 3 - Algorithm (fallback - Retinex): - 1. log(I) = log(R) + log(L) - 2. log(R) = log(I) - log(Gaussian*I) - 3. exp(R), normalize to [0,255] - Accept: Unit test: image with shadow gradient → uniform illumination -``` - -#### 1.2.6 — Binarization: Sauvola Local Threshold - -``` -[1.2.6] Convert grayscale to clean black-and-white using adaptive threshold - Files: - apps/tools/backend/workers/src/scanner/binarize.rs - Functions: - compute_integral_image(img: &GrayImage) → Vec - compute_integral_image_sq(img: &GrayImage) -> Vec - local_stats(integral: &[u64], integral_sq: &[u64], - x: i32, y: i32, half_win: i32, w: i32, h: i32) -> (f64, f64) - sauvola_threshold(img: &GrayImage, window_size: u32, k: f64) -> GrayImage - otsu_threshold(img: &GrayImage) -> GrayImage // fallback - Algorithm: - Sauvola: T = m * (1 + k * (s/R - 1)) - - m = local mean (from integral image) - - s = local std dev (from integral image squared) - - k = 0.2 (tunable) - - R = 128 (max std dev for 8-bit) - - window_size = max(width, height) / 30, clamped to [15, 100] - Accept: Unit test: 5 test images → binary output, text readable, background clean white -``` - -#### 1.2.7 — Deskew: Hough Transform Line Detection - -``` -[1.2.7] Detect and correct small rotation (<5°) of text lines - Files: - apps/tools/backend/workers/src/scanner/deskew.rs - Functions: - probabilistic_hough_lines(img: &GrayImage, threshold: u32, - min_line_length: f64, max_gap: f64) -> Vec - median_angle(lines: &[Line]) -> f64 - rotate_image(img: &GrayImage, angle_degrees: f64) -> GrayImage - deskew(img: &GrayImage) -> GrayImage - Algorithm: - 1. Probabilistic Hough line transform - 2. Filter: keep lines with angle between -45° and +45° (skip vertical) - 3. Compute median angle - 4. If |angle| > 0.5° → rotate with Lanczos3, crop to fit - Accept: Unit test: rotated text image 3° → deskewed to <0.5° residual rotation -``` - -#### 1.2.8 — Image Enhancement: Sharpening + Contrast - -``` -[1.2.8] Apply final sharpening and contrast optimization - Files: - apps/tools/backend/workers/src/scanner/enhance.rs - Functions: - unsharp_mask(img: &GrayImage, sigma: f64, amount: f64) -> GrayImage - adjust_contrast(img: &GrayImage, factor: f64) -> GrayImage - remove_noise(img: &GrayImage, threshold: u8) -> GrayImage - enhance_final(img: &GrayImage) -> GrayImage - Algorithm (Unsharp mask): - blurred = gaussian_blur(img, sigma=1.0) - mask = img - blurred - result = img + amount * mask // amount = 1.0 (default) - Accept: Unit test: blurry text → sharpened text, verify no ringing artifacts -``` - -#### 1.2.9 — Pipeline Assembly - -``` -[1.2.9] Connect all pipeline stages with progress reporting - Files: - apps/tools/backend/workers/src/scanner/pipeline.rs - apps/tools/backend/workers/src/scanner/mod.rs (update) - Functions: - ScanPipeline::process(input_path, options, progress: ProgressReporter) → Result - Stages with progress: - 0% → load + preprocess - 15% → edge detection - 25% → corner detection - 35% → perspective warp - 50% → shadow removal - 65% → binarization - 75% → deskew - 85% → final enhance - 100% → complete - ScanResult: - { output_image_path, page_count: 1, image_dimensions, processing_time_ms } - Accept: Full pipeline test with 10 diverse test images → consistent quality output -``` - ---- - -### Milestone 1.3 — Next.js Frontend Foundation - -**Goal**: User bisa upload foto, lihat progress, download hasil. - -#### 1.3.1 — Init Next.js App - -``` -[1.3.1] Create Next.js 16 app with Tailwind v4 + shadcn/ui + TypeScript strict - Files: - apps/tools/frontend/package.json - apps/tools/frontend/next.config.ts - apps/tools/frontend/tsconfig.json - apps/tools/frontend/postcss.config.mjs - apps/tools/frontend/components.json - apps/tools/frontend/biome.json - apps/tools/frontend/src/app/globals.css - Setup: - bun create next-app@latest --typescript --tailwind --eslint - bun add @shadcn/react lucide-react class-variance-authority clsx tailwind-merge framer-motion - npx shadcn@latest init - Add custom CSS variables + Twilight Terminal theme - Accept: bun dev → localhost:3002, halaman kosong dengan Tailwind + shadcn working -``` - -#### 1.3.2 — Root Layout + Theme Provider - -``` -[1.3.2] Layout dengan header, footer, theme provider, fonts - Files: - apps/tools/frontend/src/app/layout.tsx - apps/tools/frontend/src/app/providers.tsx - apps/tools/frontend/src/components/tools/header.tsx - apps/tools/frontend/src/components/tools/footer.tsx - apps/tools/frontend/src/lib/utils.ts - Features: - - Root layout with metadata (title: "Tools — Asep Haryana") - - ThemeProvider (next-themes) wrapping children - - Geist sans font (same as hub portfolio) - - Header: logo "Tools", navigation links, theme toggle, GitHub link - - Footer: copyright, powered by Rust + Next.js badge - - cn() utility from tailwind-merge - Accept: All pages render with header + footer, theme toggle works -``` - -#### 1.3.3 — Landing Page (/) with Tool Cards - -``` -[1.3.3] Card grid showing all available tools with icons - Files: - apps/tools/frontend/src/app/page.tsx - apps/tools/frontend/src/components/tools/tool-card.tsx - apps/tools/frontend/src/components/tools/tool-grid.tsx - Data: - tools = [ - { id: "scan", title: "Document Scanner", desc: "...", icon: ScanIcon, href: "/scan", phase: 1 }, - { id: "image-compress", title: "Compress Image", desc: "...", icon: ... }, - ... - ] - Features: - - Grid responsive: 1 col mobile, 2 col tablet, 3 col desktop - - Each card: icon, title, description, link - - Phase badges: "Available", "Coming Soon" - - Framer Motion stagger animation on mount - Accept: / renders grid of tool cards, each card is clickable link -``` - -#### 1.3.4 — Upload Zone Component - -``` -[1.3.4] Drag & drop upload zone with file validation - Files: - apps/tools/frontend/src/components/tools/upload-zone.tsx - apps/tools/frontend/src/hooks/use-upload.ts - Features: - - Drag & drop area with dashed border - - Click to open file picker - - Accept attribute berdasarkan tool (image/*, application/pdf, video/*, audio/*) - - Validate: file type, max size (50MB), max count (50 for batch) - - Show file name, size, type after selection - - Error state: invalid type, too large, too many - - Drag over highlight animation - - Loading spinner during upload - - Upload progress percentage (from XHR or fetch) - - Cancel upload button - Accept: Drag image file → uploads to server → returns job_id -``` - -#### 1.3.5 — Progress Bar Component - -``` -[1.3.5] Animated progress bar with stage label from WebSocket - Files: - apps/tools/frontend/src/components/tools/progress-bar.tsx - apps/tools/frontend/src/hooks/use-job-status.ts - WebSocket hook (useJobStatus): - - Connect to /api/job/{id}/ws - - Auto-reconnect on disconnect (3 retries) - - Parse JobProgress messages - - Update state: status, progress, stage, message - - Cleanup on unmount - ProgressBar: - - Animated bar (Framer Motion width animation) - - Stage label: "Detecting edges...", "Correcting perspective...", etc. - - Percentage number - - Status badge: Processing (amber pulse), Completed (green), Failed (red) - - Error state with retry button - Accept: Upload → progress bar animates from 0-100% with stage labels -``` - -#### 1.3.6 — Preview Before/After Component - -``` -[1.3.6] Image preview with before/after comparison slider - Files: - apps/tools/frontend/src/components/tools/preview-before-after.tsx - Features: - - Two image layers: original (left) vs processed (right) - - Draggable slider divider - - Click on left = show original, click right = show processed - - Zoom: scroll to zoom, drag to pan - - File size comparison badge: "2.4 MB → 340 KB" - - Responsive: fill container width - Accept: Component renders with two image URLs, slider interaction works -``` - -#### 1.3.7 — Result Preview Component - -``` -[1.3.7] Result display: preview, download, info - Files: - apps/tools/frontend/src/components/tools/result-preview.tsx - Features: - - Show processed file preview (image or icon for PDF/video/audio) - - File info: name, size, dimensions, pages (for PDF) - - Download button with file type icon - - Download as ZIP for batch results - - Copy share link button (if applicable) - - "Process another" button → reset to upload state - - Auto-download option checkbox - Accept: Job completes → result card shows with download button → click downloads file -``` - -#### 1.3.8 — Scanner Page (/scan) - -``` -[1.3.8] Full scanner page: upload → progress → result - Files: - apps/tools/frontend/src/app/scan/page.tsx - apps/tools/frontend/src/app/scan/result/[id]/page.tsx - Page states: - 1. UPLOAD: UploadZone + options (OCR toggle, enhance toggle, DPI selector) - 2. PROCESSING: ProgressBar + stage label + cancel button - 3. RESULT: PreviewBeforeAfter + ResultPreview + "Process Another" - 4. ERROR: Error message with retry + feedback button - Options panel: - - Enable OCR (toggle, default: on) - - Auto-enhance (toggle, default: on) - - Output format (PDF, JPEG, PNG — default: PDF) - - DPI (150, 200, 300, 400 — default: 300) - Flow: - Upload → POST /api/upload → get job_id → connect WS → show progress → - complete → show result with preview + download - Accept: Full user flow: upload → progress → download PDF -``` - -#### 1.3.9 — API Route: Upload Proxy - -``` -[1.3.9] Next.js API route that proxies upload to Rust backend - Files: - apps/tools/frontend/src/app/api/upload/route.ts - Logic: - - Accept multipart/form-data from browser - - Forward to http://tools:3001/api/upload (Rust gateway) - - On 202: return { job_id, ws_url } to client - - On 4xx/5xx: return error to client - - Handle timeout, connection refused gracefully - Accept: POST via browser → proxied to Rust → returns job_id -``` - -#### 1.3.10 — API Route: Job Status & Download Proxy - -``` -[1.3.10] Proxy job status + WebSocket + download to Rust backend - Files: - apps/tools/frontend/src/app/api/job/[id]/route.ts - apps/tools/frontend/src/app/api/job/[id]/ws/route.ts - apps/tools/frontend/src/app/api/download/[id]/route.ts - Features: - GET /api/job/{id} → proxy to Rust - WS /api/job/{id}/ws → proxy WebSocket (Next.js can't do WS in app router, - so use upgrade header or direct client WS to Rust port) - GET /api/download/{id} → stream from Rust - Note: WebSocket langsung dari client ke Rust gateway port (3001), - bukan via Next.js. CORS sudah dihandle di Rust. - Accept: WS connection works: client → Rust gateway → NATS progress → browser -``` - ---- - -### Milestone 1.4 — OCR + Searchable PDF + Infrastructure - -**Goal**: Output searchable PDF, realtime WebSocket progress, auto-cleanup, rate limiting. - -#### 1.4.1 — Tesseract OCR Integration - -``` -[1.4.1] OCR text extraction from processed image - Files: - apps/tools/backend/workers/src/scanner/ocr.rs - Functions: - init_tesseract(lang: &str) -> Result - ocr_text(tess: &mut LepTess, img: &GrayImage) -> Result - ocr_words(tess: &mut LepTess, img: &GrayImage) -> Result> - ocr_with_language(img: &GrayImage, lang: &str) -> Result - OcrWord: { text: String, bbox: {x,y,w,h}, confidence: i32 } - OcrResult: { full_text: String, words: Vec, confidence: f32 } - Languages: "eng+ind" (English + Indonesian) - TESSDATA_PREFIX: /usr/share/tesseract-ocr/5/tessdata - PSM mode: 3 (automatic), fallback 6 (single text block) - Accept: Unit test: known text image → OCR returns text with >80% confidence -``` - -#### 1.4.2 — Searchable PDF Generation - -``` -[1.4.2] Generate PDF with visible image + invisible text layer - Files: - apps/tools/backend/workers/src/scanner/pdf.rs - Functions: - compress_image_jpeg(img: &GrayImage, quality: u8) -> Result> - generate_pdf_page(image_data: &[u8], words: &[OcrWord], - page_width_pt: f64, page_height_pt: f64) -> Result - generate_searchable_pdf(image_data: &[u8], ocr_text: &str, - words: &[OcrWord]) -> Result> - PDF structure: - - Page with MediaBox A4 (595.28 x 841.89) or fit to image aspect ratio - - Image XObject (JPEG DCTDecode, 300 DPI equivalent) - - Content stream: - 1. Place image at full page: q {w} 0 0 {h} 0 0 cm /Im0 Do Q - 2. Invisible text: 3 Tr (rendering mode 3 = neither fill nor stroke) - 3. Each word positioned at its bbox, converted pixels → points - - Metadata: Producer, CreationDate - Accept: Generated PDF → open in browser → text is selectable + searchable -``` - -#### 1.4.3 — Scanner Pipeline: OCR + PDF Integration - -``` -[1.4.3] Connect OCR + PDF generation into the main pipeline - Files: - apps/tools/backend/workers/src/scanner/pipeline.rs (update) - Updated stages: - 75% → deskew - 82% → OCR - 90% → PDF generation - 100% → save + complete - Pipeline now returns ScanResult: - { output_path, page_count, file_size, ocr_text, processing_time_ms } - Accept: Full pipeline test: input image → output searchable PDF file -``` - -#### 1.4.4 — WebSocket Progress Forwarding (Gateway) - -``` -[1.4.4] Gateway subscribes to NATS progress and forwards via WebSocket - Files: - apps/tools/backend/gateway/src/routes/ws.rs - Functions: - ws_handler(ws: WebSocketUpgrade, job_id: Path, state: SharedState) - handle_ws(mut ws: WebSocket, job_id: String, nats: Connection, redis: ConnectionManager) - Logic: - 1. Accept WebSocket upgrade - 2. Subscribe to NATS: tools.*.progress.{job_id} - 3. Forward each message as JSON to WebSocket client - 4. On connection close: unsubscribe from NATS - 5. Keepalive ping every 30 seconds - 6. Send initial status from Redis on connect - Accept: Client connects via WS → receives progress messages in realtime -``` - -#### 1.4.5 — Auto-Cleanup Scheduler - -``` -[1.4.5] Scheduled cleanup of expired files and Redis keys - Files: - apps/tools/backend/workers/src/scheduler/mod.rs - apps/tools/backend/workers/src/scheduler/cleanup.rs - Logic: - 1. Subscribe to NATS cron: tools.scheduler.cleanup (every 10 min) - 2. Scan {STORAGE_PATH} for files older than JOB_TTL_SECONDS - 3. Delete expired files - 4. SCAN Redis for job:* keys with TTL expired, delete orphans - 5. Log: deleted N files, freed M bytes, deleted K orphan keys - 6. Prometheus: tools_cleanup_deleted_files counter - Periodic trigger: NATS cron via tools.scheduler.cleanup subject - Accept: Upload file → wait TTL → file deleted automatically -``` - -#### 1.4.6 — Rate Limiting - -``` -[1.4.6] Rate limiting per IP per tool via Redis sliding window - Files: - apps/tools/backend/gateway/src/redis/ratelimit.rs (update) - apps/tools/backend/gateway/src/routes/upload.rs (middleware) - Algorithm (Sliding Window): - Key: ratelimit:{ip}:{tool}:{minute_bucket} - - ZADD current timestamp - - ZREMRANGEBYSCORE older than 60s - - ZCOUNT → if > RATE_LIMIT_PER_MINUTE → reject - Response on reject: 429 Too Many Requests - { error: "rate_limit_exceeded", retry_after_seconds: 60 } - Middleware: Add to upload route as tower Layer - Accept: curl 31x in 60s → 429 on 31st request -``` - -#### 1.4.7 — Error Handling & Validation (Gateway Middleware) - -``` -[1.4.7] Global error handling and input validation middleware - Files: - apps/tools/backend/gateway/src/middleware/mod.rs - apps/tools/backend/gateway/src/middleware/error_handler.rs - apps/tools/backend/gateway/src/middleware/request_id.rs - Features: - - Request ID middleware (X-Request-Id header, uuid v4) - - JSON error response format: { error: string, code: string, request_id: string } - - 400: invalid input (missing file, missing tool, invalid options) - - 404: job not found / file expired - - 413: file too large - - 429: rate limited - - 500: internal error (logged, not exposed to client) - - Panic recovery layer - Accept: curl with missing fields → 400 JSON error. curl bad tool → 400. -``` - ---- - -### Phase 1 Complete: Document Scanner MVP - -**Acceptance criteria**: -1. User buka tools.asepharyana.my.id → landing page with tool cards -2. Click "Document Scanner" → halaman scan -3. Upload foto dokumen HP (miring, bayangan) → preview upload -4. Progress bar animasi: edge → warp → enhance → binarize → OCR → PDF -5. Download searchable PDF → teks bisa di-copy -6. Rate limit: 30 uploads/min/IP -7. File auto-delete after 1 hour - ---- - -## Phase 2: Scanner Complete + Image Tools - -### Milestone 2.1 — Scanner Robustness (Manual Crop + Camera + Batch) - -#### 2.1.1 — Manual Crop Canvas - -``` -[2.1.1] Interactive canvas with 4 draggable corner handles - Files: - apps/tools/frontend/src/components/tools/crop-editor.tsx - apps/tools/frontend/src/hooks/use-crop-editor.ts - Features: - - Render uploaded image on Canvas - - 4 draggable corner handles (circular, Luma color) - - Connect corners with dashed lines - - Zoom: scroll to zoom, drag canvas to pan - - Grid overlay (rule of thirds) for alignment - - Touch support: pinch zoom, drag handles - - Double-click to auto-detect corners again - - Reset button - - Confirm button → sends corner coordinates + image to server - API: POST /api/scan/manual-crop - { job_id, corners: [{x,y},{x,y},{x,y},{x,y}] } - Accept: User adjust corners → confirm → server applies warp with given corners -``` - -#### 2.1.2 — Fallback Pipeline: Auto → Manual → Process - -``` -[2.1.2] When auto edge detection fails, fall back to manual crop - Files: - apps/tools/backend/gateway/src/routes/upload.rs (update) - apps/tools/frontend/src/app/scan/page.tsx (update) - Flow: - 1. Upload → queue job → worker coba auto-detect - 2. Auto gagal → worker set status = "needs_manual_crop" - → return corner confidence < 0.6 - 3. Gateway returns: { job_id, status: "needs_manual_crop", - manual_crop_url: "/scan/crop/{job_id}" } - 4. Frontend redirect to CropEditor - 5. User adjust corners → POST manual-crop → worker resume pipeline - Accept: Upload bad photo → auto redirect to manual crop → complete pipeline -``` - -#### 2.1.3 — PWA Camera Capture - -``` -[2.1.3] Direct camera capture from browser (PWA) - Files: - apps/tools/frontend/src/components/tools/camera-capture.tsx - apps/tools/frontend/src/hooks/use-camera.ts - Features: - - Access rear camera via getUserMedia - - Live preview in viewfinder - - Auto-focus on tap - - Capture button → freeze frame - - Aspect ratio guide overlay (A4: 1:1.414) - - Reject blurry photos (Laplacian variance check via Canvas API) - - Auto-capture when document detected (stabilize → snap) - - Switch front/rear camera - - Torch/flash toggle (if supported) - - Zoom slider (if supported) - Accept: Click "Use Camera" → camera opens → capture → photo uploaded -``` - -#### 2.1.4 — Batch Multi-Page - -``` -[2.1.4] Multiple pages → one PDF, parallel processing - Files: - apps/tools/frontend/src/components/tools/file-list.tsx - apps/tools/backend/workers/src/scanner/pipeline.rs (update) - Frontend: - - Upload multiple files (drag multiple or multi-select) - - Thumbnail list with drag-to-reorder - - Remove individual pages - - Add more pages button - - Upload all → single group job → N individual jobs - - Per-page progress status in thumbnail list - Backend: - - Group job: { group_id, job_ids: [...], total: N, completed: 0 } - - Process each page via Rayon parallel for - - Merge all PDF pages into single document via lopdf - - Report: per-page progress + overall progress % (completed/total) - Accept: Upload 5 photos → all processed in parallel → single 5-page PDF -``` - -#### 2.1.5 — Scan Options Panel - -``` -[2.1.5] Enhanced options panel for scanner - Files: - apps/tools/frontend/src/components/tools/scan-options.tsx - apps/tools/frontend/src/app/scan/page.tsx (update) - Options: - - Output: PDF, JPEG, PNG - - Quality: 1-100 (slider, default 90) - - OCR: on/off (default: on) - - Language: English, Indonesian, Both (default: Both) - - Auto-enhance: on/off (default: on) - - Color mode: Black & White, Grayscale, Color (default: B&W) - - Page size: A4, Letter, Auto-fit (default: A4) - - DPI: 150/200/300/400 (default: 300) - Accept: Change options → upload → output sesuai pilihan -``` - ---- - -### Milestone 2.2 — WASM Image Tools (Compress, Resize, Convert) - -#### 2.2.1 — WASM Rust Crate - -``` -[2.2.1] Compile image processing to WebAssembly via wasm-pack - Files: - apps/tools/backend/wasm/Cargo.toml - apps/tools/backend/wasm/src/lib.rs - apps/tools/backend/wasm/src/compress.rs - apps/tools/backend/wasm/src/resize.rs - apps/tools/backend/wasm/src/convert.rs - Functions: - #[wasm_bindgen] - fn compress_jpeg(bytes: &[u8], quality: u8) -> Result> - fn compress_png(bytes: &[u8], effort: u8) -> Result> - fn compress_webp(bytes: &[u8], quality: u8) -> Result> - fn resize_image(bytes: &[u8], width: u32, height: u32, fit: String) -> Result> - fn convert_format(bytes: &[u8], target_format: String) -> Result> - fn get_image_info(bytes: &[u8]) -> Result // { width, height, format, size } - Build: wasm-pack build --release --target web - Output: pkg/ directory (wasm binary + JS glue) - Accept: wasm-pack build succeeds. Node.js test: compress results smaller than input -``` - -#### 2.2.2 — WASM Integration in Frontend - -``` -[2.2.2] Load and invoke WASM module from Next.js - Files: - apps/tools/frontend/src/lib/wasm/loader.ts - apps/tools/frontend/src/lib/wasm/image-processor.ts - Functions: - async initWasm() → WASM module instance (lazy load on first use) - async compressInBrowser(file: File, quality: number) → Blob - async resizeInBrowser(file: File, width: number, height: number) → Blob - async convertInBrowser(file: File, format: string) → Blob - Strategy: - - Dynamic import: await import('./pkg/image_wasm.js') - - Lazy init: only load when user visits image tool page - - Code splitting: WASM chunk loaded separately (~1.5MB gzipped) - - Cache: once loaded, keep in memory - - Fallback: if WASM fails → upload to server worker - Accept: Browser loads WASM → image processing runs client-side, no upload -``` - -#### 2.2.3 — Image Compress Page (/image/compress) - -``` -[2.2.3] Full compress page with quality slider and live comparison - Files: - apps/tools/frontend/src/app/image/compress/page.tsx - apps/tools/frontend/src/components/tools/quality-slider.tsx - Features: - - UploadZone (accepts image/*) - - Quality slider: 1-100, live preview update - - File size display: original vs compressed (estimated) - - PreviewBeforeAfter (original vs compressed) - - Format selector: JPEG, PNG, WebP - - Download button - - Batch mode: compress all images in folder - WASM flow: upload → load in WASM → compress → preview → download (no server) - Server fallback: upload → Rust worker compress → download - Accept: Upload photo → adjust quality → live preview → download compressed version -``` - -#### 2.2.4 — Image Resize Page (/image/resize) - -``` -[2.2.4] Resize page with dimension input and aspect ratio lock - Files: - apps/tools/frontend/src/app/image/resize/page.tsx - apps/tools/frontend/src/components/tools/dimension-input.tsx - Features: - - UploadZone - - Input: width + height, auto-fill from original - - Aspect ratio lock toggle (🔗/🔓) - - Preset sizes: 800x600, 1024x768, 1920x1080, Instagram (1080x1080), Custom - - Fit modes: exact (stretch), contain (fit within, add bg), cover (crop to fill) - - Preview: resized dimensions overlay - - Download (or ZIP for batch) - - Batch: resize all images to same dimensions - Accept: Upload photo → set 800px width → lock aspect → download resized image -``` - -#### 2.2.5 — Image Convert Page (/image/convert) - -``` -[2.2.5] Format conversion page - Files: - apps/tools/frontend/src/app/image/convert/page.tsx - Features: - - UploadZone - - From: auto-detected from file - - To: JPEG, PNG, WebP, GIF, BMP, TIFF - - Quality slider (for lossy formats) - - PreviewBeforeAfter - - Download - - Batch: convert all files in folder - Accept: Upload HEIC → convert to JPEG → download -``` - -#### 2.2.6 — Server Fallback for WASM - -``` -[2.2.6] Server-side processing when WASM unavailable - Files: - apps/tools/backend/workers/src/image/mod.rs - apps/tools/backend/workers/src/image/compress.rs - apps/tools/backend/workers/src/image/resize.rs - apps/tools/backend/workers/src/image/convert.rs - Logic: - - Same pipeline as WASM but runs natively - - Use image crate with mozjpeg feature for optimal JPEG - - Resize with Lanczos3 filter - - Convert via image crate format support - Frontend detection: - - try/catch WASM init → if fails, enable "Server Process" button - - Upload → NATS → worker → result (same as scanner flow) - Accept: Disable WASM in browser → upload fallback works → same quality output -``` - ---- - -### Milestone 2.3 — Background Removal (ONNX) - -#### 2.3.1 — ONNX Model Download & Setup - -``` -[2.3.1] Download and package background removal model - Files: - Dockerfile (update — or runtime download script) - models/download-models.sh - Models: - - Primary: MODNet (~25MB, good quality, fast) - - Alternative: DIS_seg (~8MB, decent quality, very fast) - Setup: - - Download model file to models/ - - Verify SHA256 checksum - - Load model at worker startup - - One-time init, cache in memory - ONNX session options: - - InterOpNumThreads: 2 - - IntraOpNumThreads: 4 - - GraphOptimizationLevel: ORT_ENABLE_ALL - Accept: Worker starts → loads ONNX model → ready for inference -``` - -#### 2.3.2 — Pre/Post Processing - -``` -[2.3.2] Image preprocessing and postprocessing for ONNX model - Files: - apps/tools/backend/workers/src/image/remove_bg.rs - Functions: - preprocess_for_model(img: &DynamicImage, target_size: u32) -> Result> - - Resize to model input size (1024x1024 for MODNet) - - Normalize: /255.0, mean=[0.5,0.5,0.5], std=[0.5,0.5,0.5] - - Convert to CHW format (ort::Tensor) - postprocess_mask(output: &ort::Tensor, original_size: (u32,u32)) -> GrayImage - - Sigmoid → threshold 0.5 → resize to original dimensions - - Apply optional smoothing (Gaussian blur sigma=1) - apply_alpha(img: &DynamicImage, mask: &GrayImage, bg_color: Option<[u8;3]>) -> DynamicImage - - If bg_color = None → RGBA with transparent background - - If bg_color = Some([r,g,b]) → composite onto solid color - Accept: Unit test: portrait photo → mask correctly separates foreground -``` - -#### 2.3.3 — Remove Background Page (/image/remove-bg) - -``` -[2.3.3] Background removal page - Files: - apps/tools/frontend/src/app/image/remove-bg/page.tsx - Features: - - UploadZone - - Processing via server worker (too heavy for WASM) - - PreviewBeforeAfter: original → transparent bg - - Background selector: transparent, white, color picker - - Download as PNG (transparent) or JPEG (with bg color) - - Download HD (full resolution) or Web (compressed) - - Batch processing - - Progress bar during inference - Accept: Upload photo → background removed → download with transparent PNG -``` - ---- - -### Phase 2 Complete: Document Scanner Robust + Image Tools - ---- - -## Phase 3: PDF Tools - -### Milestone 3.1 — PDF Worker - -#### 3.1.1 — PDF Merge Worker - -``` -[3.1.1] Merge multiple PDF files into one - Files: - apps/tools/backend/workers/src/pdf/mod.rs - apps/tools/backend/workers/src/pdf/merge.rs - apps/tools/frontend/src/app/pdf/merge/page.tsx - Logic: - 1. Upload 2+ PDF files - 2. Load each via lopdf::Document::load - 3. Iterate pages from each document, append to output - 4. Save merged document - 5. Preserve: page size (use largest MediaBox per page), fonts (embedded) - Frontend: - - UploadZone (multiple, accept .pdf) - - Drag-to-reorder uploaded files - - Remove individual files - - Download merged PDF - Accept: Upload 3 PDFs → download 1 PDF with all pages in order -``` - -#### 3.1.2 — PDF Split Worker - -``` -[3.1.2] Extract specific pages from a PDF - Files: - apps/tools/backend/workers/src/pdf/split.rs - apps/tools/frontend/src/app/pdf/split/page.tsx - Functions: - parse_page_range(input: &str, total_pages: u32) -> Result> - split_pdf(input_path: &Path, pages: &[u32]) -> Result> - Page syntax: "1-3,5,7-9,12" → [1,2,3,5,7,8,9,12] - Frontend: - - UploadZone (single PDF) - - PDF preview with page thumbnails - - Click pages to select/deselect - - Or type page range input - - Download extracted pages as single PDF - Accept: Upload 20-page PDF → extract pages 3-7 → download 5-page PDF -``` - -#### 3.1.3 — Images to PDF Worker - -``` -[3.1.3] Convert multiple images into a single PDF - Files: - apps/tools/backend/workers/src/pdf/images_to_pdf.rs - apps/tools/frontend/src/app/pdf/images-to-pdf/page.tsx - Functions: - images_to_pdf(image_paths: &[PathBuf], options: PdfOptions) -> Result> - Logic: - 1. Load each image via image::open - 2. Convert to JPEG (for PDF embedding) - 3. Create PDF page per image - 4. Fit image to A4 / Letter / Original size - 5. Optional: margin, alignment - Frontend: - - UploadZone (multiple, accept image/*) - - Thumbnail grid with drag-to-reorder - - Page size: A4, Letter, Original - - Orientation: Auto, Portrait, Landscape - - Margin: 0-50mm - - Download PDF - Accept: Upload 5 photos → download 1 PDF, each photo = 1 page -``` - -#### 3.1.4 — PDF Compress Worker - -``` -[3.1.4] Reduce PDF file size by recompressing embedded images - Files: - apps/tools/backend/workers/src/pdf/compress.rs - apps/tools/frontend/src/app/pdf/compress/page.tsx - Algorithm: - 1. Open PDF with lopdf - 2. Find all image XObject streams - 3. For each: decode → re-encode JPEG with lower quality - 4. Replace stream in PDF - 5. Remove unused objects - 6. Linearize PDF for fast web viewing - Compression levels: - - Maximum (q=30): smallest size, visible quality loss - - Balanced (q=50): good balance (default) - - Minimal (q=80): slight size reduction, near-lossless - Frontend: - - UploadZone (single PDF) - - Compression level selector - - Preview: original size vs estimated compressed size - - Download compressed PDF - Accept: Upload 10MB PDF with images → compress → <3MB output -``` - -#### 3.1.5 — PDF to Images Worker - -``` -[3.1.5] Convert each PDF page to an image - Files: - apps/tools/backend/workers/src/pdf/to_images.rs - apps/tools/frontend/src/app/pdf/pdf-to-images/page.tsx - Functions: - pdf_page_to_image(input_path: &Path, page_num: u32, dpi: u32) -> Result> - pdf_to_images(input_path: &Path, dpi: u32, format: &str) -> Result>> - Logic: - 1. Render PDF page to image (via lopdf rasterize or poppler) - 2. Output format: JPEG, PNG, WebP - 3. DPI: 72, 150, 200, 300 (default: 200) - Frontend: - - UploadZone (single PDF) - - DPI selector - - Format selector - - Preview: page thumbnails - - Download as ZIP (all images) or per-page - Accept: Upload PDF → download ZIP of page images -``` - ---- - -## Phase 4: Video/Audio Tools - -### Milestone 4.1 — FFmpeg Worker - -#### 4.1.1 — FFmpeg Binding & Worker Setup - -``` -[4.1.1] Integrate FFmpeg via ffmpeg-next crate - Files: - apps/tools/backend/workers/Cargo.toml (add ffmpeg-next) - apps/tools/backend/workers/src/video/mod.rs - apps/tools/backend/workers/src/audio/mod.rs - Dockerfile (update — install ffmpeg package) - Setup: - - apt-get install ffmpeg - - ffmpeg-next version 7.x - - Initialize: ffmpeg::init() at worker startup - - Test: version check - Accept: Worker starts → ffmpeg initialized → transcoding functions available -``` - -#### 4.1.2 — Video Compress - -``` -[4.1.2] Reduce video file size by lowering bitrate and/or resolution - Files: - apps/tools/backend/workers/src/video/compress.rs - apps/tools/frontend/src/app/video/compress/page.tsx - Options: - - Target size (approximate): 10MB, 25MB, 50MB, 100MB, Custom - - Resolution: 480p, 720p, 1080p, Original - - Quality: Low, Medium, High (CRF: 28, 23, 18) - - Codec: H.264, H.265/HEVC, VP9 - FFmpeg command (via ffmpeg-next API): - - Transcode video stream: libx264, crf, preset medium - - Scale if needed: scale=iw/2:ih/2 - - Copy audio stream (or compress with aac) - Frontend: - - UploadZone (accept video/*, max 500MB) - - Options: target size, resolution, quality - - Progress: from FFmpeg stderr parsed progress - - Preview: thumbnail + duration, original vs estimated size - - Download compressed video - Note: Video processing is HEAVY → max 1 concurrent video job, queue via NATS - Accept: Upload 500MB 1080p video → compress → <100MB 720p output -``` - -#### 4.1.3 — Audio Extract - -``` -[4.1.3] Extract audio track from video file - Files: - apps/tools/backend/workers/src/video/audio_extract.rs - apps/tools/frontend/src/app/video/audio-extract/page.tsx - Options: - - Format: MP3, AAC, WAV, FLAC, OGG - - Quality: 128k, 192k, 320k (for lossy) / Lossless (for FLAC/WAV) - FFmpeg: - - Stream copy if format compatible, else transcode - - Extract best audio stream - Frontend: - - UploadZone (accept video/*) - - Format + quality selector - - Progress - - Download extracted audio - Accept: Upload MP4 → download MP3 with correct audio -``` - -#### 4.1.4 — Video Trim - -``` -[4.1.4] Cut a segment from a video - Files: - apps/tools/backend/workers/src/video/trim.rs - apps/tools/frontend/src/app/video/trim/page.tsx - Options: - - Start time (HH:MM:SS or seconds) - - End time / Duration - FFmpeg: - -ffmpeg -i input -ss start -to end -c copy output (fast seek) - Or re-encode for precise seeking: -ss start -i input -t duration -c libx264 - Frontend: - - UploadZone (accept video/*) - - Preview player with timeline - - Set start/end via sliders or input - - Preview trim result - - Download trimmed video - Accept: Upload 10min video → trim 2:30-5:00 → download 2.5min video -``` - -#### 4.1.5 — GIF Maker - -``` -[4.1.5] Convert video segment to animated GIF - Files: - apps/tools/backend/workers/src/video/gif.rs - apps/tools/frontend/src/app/video/gif-maker/page.tsx - Options: - - Start time, Duration (max 10s for GIF) - - FPS: 10, 15, 20, 24, 30 - - Width: 320, 480, 640, 800 - - Dither: on/off - - Colors: 64, 128, 256 - FFmpeg: - ffmpeg -i input -ss start -t duration -vf "fps=15,scale=480:-1:flags=lanczos,palettegen" palette.png - ffmpeg -i input -i palette.png -ss start -t duration -lavfi "fps=15,scale=480:-1:flags=lanczos[x];[x][1:v]paletteuse" output.gif - Frontend: - - UploadZone (accept video/*) - - Start time + duration sliders - - FPS + width + dither options - - Preview: animated GIF in browser - - Download GIF - Accept: Upload 30s video → trim 2s → download animated GIF -``` - -#### 4.1.6 — Audio Convert - -``` -[4.1.6] Convert audio between formats - Files: - apps/tools/backend/workers/src/audio/convert.rs - apps/tools/frontend/src/app/audio/convert/page.tsx - Formats: - Input: MP3, WAV, FLAC, AAC, OGG, M4A, WMA - Output: MP3, WAV, FLAC, AAC, OGG - Options: - - Bitrate: 128k, 192k, 256k, 320k - - Sample rate: 44100, 48000, 96000 - - Channels: Mono, Stereo (downmix if source is 5.1) - Frontend: - - UploadZone (accept audio/*) - - From (auto-detected) → To selector - - Bitrate + sample rate options - - Download converted audio - Accept: Upload FLAC → convert to 320kbps MP3 → download -``` - ---- - -## Phase 5: Deployment & Polish - -### Milestone 5.1 — Infrastructure - -#### 5.1.1 — Docker Multi-Stage Build - -``` -[5.1.1] Production Dockerfile - Files: - infra/docker/tools.Dockerfile - Stages: - 1. chef (cargo-chef install) - 2. planner (recipe.json) - 3. builder (cargo build —release) - 4. wasm-builder (wasm-pack build) - 5. frontend-builder (bun build) - 6. runtime: debian:bookworm-slim + tesseract + ffmpeg + ONNX model - Runtime dependencies: - - tesseract-ocr + tessdata (eng, ind) - - ffmpeg - - ca-certificates - - libfontconfig1 (for lopdf) - Dockerignore: - - node_modules, target, .next, .git - Build: docker build -f infra/docker/tools.Dockerfile -t tools:latest . - Accept: docker build succeeds, image size <400MB -``` - -#### 5.1.2 — Docker Compose File - -``` -[5.1.2] Compose file for production deployment - Files: - infra/compose/tools.yml - Service definition: - container_name: tools - image: ghcr.io/asepharyana/asepharyana-hub/tools:sha-xxxxx - restart: always - networks: app-shared-net (alias: tools) - env_file: ../../.env - volumes: tools_data:/data/tools - ports: 3001:3001 - depends_on: [redis, nats] - labels: prometheus.io/scrape=true, prometheus.io/port=3001 - Volume: tools_data (docker volume) - Accept: docker compose up → tools container running, connected to redis + nats -``` - -#### 5.1.3 — Traefik Routing - -``` -[5.1.3] Add Traefik router and service for tools - Files: - infra/traefik/dynamic/apps.yaml (update) - Router: - tools: - rule: Host(`tools.asepharyana.my.id`) || Host(`tools.asepharyana.web.id`) - entryPoints: websecure - tls: {} - middlewares: common-chain@file - service: tools-service - Service: - tools-service: - loadBalancer: - servers: - - url: http://tools:3001 - Accept: tools.asepharyana.my.id → loads tools frontend -``` - -#### 5.1.4 — CI/CD Workflow Integration - -``` -[5.1.4] Add tools service to existing build + deploy workflows - Files: - .github/workflows/docker-build-push.yml (update) - .github/workflows/deploy-docker.yml (check — auto-detects compose changes) - Changes: - - Detect changed service (apps/tools/**) - - Build matrix: add tools service - - Dockerfile: tools.Dockerfile - - Path: apps/tools - - Compose file: tools.yml - - Update manifest: sed image tag in compose - Accept: Push to main with apps/tools changes → CI builds + deploys tools -``` - -#### 5.1.5 — Environment Variables Setup - -``` -[5.1.5] Add tools env vars to .env.example - Files: - .env.example (update) - Vars: - # Tools - TOOLS_GATEWAY_PORT=3001 - TOOLS_WORKER_CONCURRENCY=4 - TOOLS_STORAGE_PATH=/data/tools - TOOLS_JOB_TTL_SECONDS=3600 - TOOLS_RATE_LIMIT_PER_MINUTE=30 - TOOLS_MAX_FILE_SIZE_MB=50 - TOOLS_OCR_LANG=eng+ind - Accept: .env.example updated with tools section -``` - ---- - -### Milestone 5.2 — Frontend Polish - -#### 5.2.1 — Theme Integration - -``` -[5.2.1] Apply Twilight Terminal theme consistent with portfolio hub - Files: - apps/tools/frontend/src/app/globals.css (update) - Theme vars (from hub): - --background / --foreground - --primary / --primary-foreground - --card / --card-foreground - --muted / --muted-foreground - Glass effect: .glass { backdrop-filter: blur } - Gradient text: .gradient-text - Terminal cursor blink animation - Same dark/light mode switch mechanism - Accept: tools subdomain → visual style consistent with hub portfolio -``` - -#### 5.2.2 — Responsive Mobile Design - -``` -[5.2.2] All pages responsive for mobile devices - Files: All page/component files (review) - Requirements: - - UploadZone: full-width on mobile, tap-friendly - - Tool cards: single column on mobile - - ProgressBar: always visible, top-fixed on scroll - - CropEditor: touch-drag handles, pinch-zoom - - CameraCapture: fullscreen viewfinder - - FileList: compact thumbnail list on mobile - - Buttons: min 44px touch target - - Bottom sheet instead of modal for options - - Safe area insets for notch devices - Accept: Lighthouse mobile audit >80 for all pages -``` - -#### 5.2.3 — Loading States & Skeleton - -``` -[5.2.3] Skeleton loading states for all pages - Files: - apps/tools/frontend/src/components/tools/skeleton.tsx - apps/tools/frontend/src/app/scan/page.tsx (update) - apps/tools/frontend/src/app/image/compress/page.tsx (update) - (all other tool pages) - Components: - SkeletonCard (pulse animation) - SkeletonUploadZone - SkeletonProgressBar - SkeletonPreview - Accept: All pages show skeleton while loading data/WASM -``` - -#### 5.2.4 — Error Boundaries - -``` -[5.2.4] React error boundaries per page + global - Files: - apps/tools/frontend/src/components/tools/error-boundary.tsx - apps/tools/frontend/src/app/layout.tsx (wrap with ErrorBoundary) - Each tool page: wrap with ErrorBoundary - Behavior: - - Catch React render errors - - Show friendly error message with tool name - - "Try Again" button - - "Report Issue" link (GitHub) - - Log error details to console (future: telemetry) - Accept: Force render error → error boundary shows, app doesn't crash -``` - -#### 5.2.5 — PWA Manifest - -``` -[5.2.5] Progressive Web App configuration - Files: - apps/tools/frontend/public/manifest.json - apps/tools/frontend/src/app/layout.tsx (add manifest link + meta tags) - apps/tools/frontend/public/icons/ (app icons: 192x192, 512x512) - Manifest: - name: "Tools — Asep Haryana" - short_name: "Tools" - description: "Document Scanner, Image & PDF Tools" - start_url: / - display: standalone - background_color: #0a0a1a (dark theme) - theme_color: #0a0a1a - icons: 192x192, 512x512 - Accept: Lighthouse PWA audit >80 -``` - ---- - -### Milestone 5.3 — Monitoring & Observability - -#### 5.3.1 — Prometheus Alerts - -``` -[5.3.1] Alert rules for tools service - Files: - infra/otel/prometheus.yml (update — or separate alert file) - Rules: - - High error rate: rate(tools_jobs_total{status="failed"}[5m]) > 0.1 - - Queue buildup: tools_queue_depth > 50 - - Slow processing: tools_processing_duration_ms{quantile="0.95"} > 10000 - - Low disk space: (disk_free_bytes / disk_total_bytes) < 0.1 (if node_exporter) - Accept: Rules loaded in Prometheus, alert firing correctly -``` - -#### 5.3.2 — Dashboard Integration - -``` -[5.3.2] Add tools metrics to hub dashboard - Files: - apps/hub/src/app/dashboard/page.tsx (update) - apps/hub/src/app/api/dashboard/route.ts (update) - Add to dashboard: - - Card: "Tools" service status (running/degraded/down) - - Quick stats: Total jobs today, Active jobs, Storage used - - Link to tools.asepharyana.my.id - Accept: Dashboard shows tools service status and stats -``` - -#### 5.3.3 — Structured Logging - -``` -[5.3.3] Structured JSON logging for production - Files: - apps/tools/backend/common/src/logging.rs (or in each crate) - apps/tools/backend/gateway/src/main.rs (logging init) - apps/tools/backend/workers/src/main.rs (logging init) - Config: - - Default: human-readable (development) - - JSON mode: RUST_LOG_FORMAT=json (production) - Fields per log: - - timestamp (ISO 8601) - - level (INFO, WARN, ERROR) - - service (gateway / workers / pipeline) - - request_id (if within request context) - - job_id (if within job context) - - message - - duration_ms (for completed processing) - Accept: RUST_LOG_FORMAT=json → JSON-structured log output -``` - ---- - -## Effort Summary - -| Phase | Milestone | Tasks | Estimated Hours | Total Days | -|-------|-----------|-------|-----------------|------------| -| 1 | 1.1 Rust Backend Skeleton | 15 | ~45 | 6 | -| 1 | 1.2 Scanner Pipeline Core | 9 | ~50 | 7 | -| 1 | 1.3 Next.js Frontend | 10 | ~30 | 4 | -| 1 | 1.4 OCR + PDF + Infra | 7 | ~25 | 4 | -| **Phase 1 Total** | **41** | **~150** | **~21** | -| 2 | 2.1 Scanner Robustness | 5 | ~25 | 4 | -| 2 | 2.2 WASM Image Tools | 6 | ~30 | 4 | -| 2 | 2.3 Background Removal | 3 | ~15 | 2 | -| **Phase 2 Total** | **14** | **~70** | **~10** | -| 3 | 3.1 PDF Tools | 5 | ~25 | 4 | -| **Phase 3 Total** | **5** | **~25** | **~4** | -| 4 | 4.1 FFmpeg Worker | 6 | ~35 | 5 | -| **Phase 4 Total** | **6** | **~35** | **~5** | -| 5 | 5.1 Infrastructure | 5 | ~15 | 2 | -| 5 | 5.2 Frontend Polish | 5 | ~20 | 3 | -| 5 | 5.3 Monitoring | 3 | ~10 | 2 | -| **Phase 5 Total** | **13** | **~45** | **~7** | -| **Grand Total** | **79 tasks** | **~325 hours** | **~47 days** | - -> **MVP** (Phase 1 only): 41 tasks, ~21 days -> **Full release** (Phase 1-5): 79 tasks, ~47 days - ---- - -## Critical Path (Phase 1) - -``` -Day 1-2: 1.1.1 → 1.1.2 → 1.1.3 → 1.1.4 → 1.1.5 (workspace + common) -Day 3-5: 1.1.6 → 1.1.7 → 1.1.8 → 1.1.10 → 1.1.14 (gateway routes + redis) -Day 5-6: 1.1.9 → 1.1.11 → 1.1.12 → 1.1.13 (gateway + workers connect) -Day 7-10: 1.2.1 → 1.2.2 → 1.2.3 → 1.2.4 (edge → warp) -Day 10-12: 1.2.5 → 1.2.6 → 1.2.7 → 1.2.8 (shadow → binarize → enhance) -Day 12: 1.2.9 (pipeline assembly) -Day 13-16: 1.3.1 → 1.3.2 → 1.3.3 → 1.3.4 (frontend pages) -Day 14-17: 1.3.5 → 1.3.6 → 1.3.7 → 1.3.8 (components) -Day 15-17: 1.3.9 → 1.3.10 (API proxy routes) -Day 18-19: 1.4.1 → 1.4.2 → 1.4.3 (OCR + PDF) -Day 19: 1.4.4 (WebSocket) -Day 20: 1.4.5 → 1.4.6 → 1.4.7 (cleanup + rate limit + errors) -``` - -> **MVP launch**: Day ~21 — Document Scanner live di tools.asepharyana.my.id \ No newline at end of file diff --git a/docs/plan/tools/infrastructure.md b/docs/plan/tools/infrastructure.md deleted file mode 100644 index b228462..0000000 --- a/docs/plan/tools/infrastructure.md +++ /dev/null @@ -1,429 +0,0 @@ -# Infrastructure & Deployment - -> **LEGACY (2026-08-02):** Dokumen plan ini ditulis saat infra masih Docker/Traefik. Produksi sekarang Caddy + Nix/systemd dengan port 4000-an. Gunakan hanya sebagai referensi historis. - -## Docker Image Architecture - -Project ini punya **satu Docker image** dengan multi-stage build. Backend Rust + Tesseract + ONNX model plus frontend Next.js. - -### Dockerfile Structure - -```dockerfile -# ============================================================ -# Stage 1: Build Rust Backend -# ============================================================ -FROM rust:1.85-slim-bookworm AS chef -RUN cargo install cargo-chef -WORKDIR /app - -FROM chef AS planner -COPY backend/ . -RUN cargo chef prepare --recipe-path recipe.json - -FROM chef AS builder -COPY --from=planner /app/recipe.json recipe.json -RUN cargo chef cook --release --recipe-path recipe.json - -COPY backend/ . -RUN cargo build --release --bin gateway --bin workers - -# ============================================================ -# Stage 2: Build Next.js Frontend -# ============================================================ -FROM oven/bun:1.3 AS frontend-builder -WORKDIR /app -COPY frontend/package.json frontend/bun.lock ./ -RUN bun install --frozen-lockfile -COPY frontend/ . -RUN bun run build - -# ============================================================ -# Stage 3: Production Runtime -# ============================================================ -FROM debian:bookworm-slim AS runtime - -# Install runtime dependencies -RUN apt-get update && apt-get install -y --no-install-recommends \ - tesseract-ocr \ - tesseract-ocr-eng \ - tesseract-ocr-ind \ - ca-certificates \ - fonts-dejavu-core \ - && rm -rf /var/lib/apt/lists/* - -WORKDIR /app - -# Copy Rust binaries -COPY --from=builder /app/target/release/gateway /app/gateway -COPY --from=builder /app/target/release/workers /app/workers - -# Copy Next.js build -COPY --from=frontend-builder /app/.next /app/.next -COPY --from=frontend-builder /app/public /app/public -COPY --from=frontend-builder /app/package.json /app/package.json -COPY --from=frontend-builder /app/node_modules /app/node_modules - -# Copy ONNX model (for background removal) -COPY models/ /app/models/ - -# Create temp storage directory -RUN mkdir -p /data/tools && chmod 1777 /data/tools - -# Environment -ENV TESSDATA_PREFIX=/usr/share/tesseract-ocr/5/tessdata -ENV TOOLS_STORAGE_PATH=/data/tools -ENV TOOLS_GATEWAY_PORT=3001 -ENV TOOLS_WORKER_CONCURRENCY=4 -ENV RUST_LOG=info - -# Expose port -EXPOSE 3001 - -# Run both gateway and workers via supervisor script -COPY scripts/entrypoint.sh /app/entrypoint.sh -RUN chmod +x /app/entrypoint.sh - -CMD ["/app/entrypoint.sh"] -``` - -### Entrypoint Script - -```bash -#!/bin/bash -# Start Gateway (Axum HTTP server) -/app/gateway & -GATEWAY_PID=$! - -# Start Worker(s) -/app/workers & -WORKER_PID=$! - -# Handle graceful shutdown -trap "kill $GATEWAY_PID $WORKER_PID; exit 0" SIGINT SIGTERM - -# Wait for either process to exit -wait -n $GATEWAY_PID $WORKER_PID - -# If one exits, kill the other -kill $GATEWAY_PID $WORKER_PID 2>/dev/null -exit 1 -``` - -### Image Size Estimates - -| Component | Size | -|-----------|------| -| Rust binary (gateway) | ~8 MB | -| Rust binary (workers) | ~15 MB | -| Next.js build | ~10 MB | -| Tesseract + data | ~25 MB | -| ONNX model | ~50 MB | -| Base (Debian slim) | ~80 MB | -| **Total** | **~188 MB** | - -> ONNX model opsional — bisa di-download runtime daripada di-include di image. - ---- - -## Docker Compose - -```yaml -# infra/compose/tools.yml -services: - tools: - container_name: tools - image: ghcr.io/asepharyana/asepharyana-hub/tools:sha-xxxxxxx - restart: always - networks: - app-shared-net: - aliases: - - tools - env_file: - - ../../.env - environment: - - REDIS_URL=redis://redis:6379 - - NATS_URL=nats://nats:4222 - - TOOLS_STORAGE_PATH=/data/tools - - TOOLS_GATEWAY_PORT=3001 - - TOOLS_WORKER_CONCURRENCY=4 - - RUST_LOG=info - volumes: - - tools_data:/data/tools - ports: - - "3001:3001" - depends_on: - redis: - condition: service_started - nats: - condition: service_started - -volumes: - tools_data: - -networks: - app-shared-net: - name: app-shared-net - external: true -``` - -### Environment Variables (`../../.env`) - -```bash -# Tools -TOOLS_GATEWAY_PORT=3001 -TOOLS_WORKER_CONCURRENCY=4 -TOOLS_STORAGE_PATH=/data/tools -TOOLS_JOB_TTL_SECONDS=3600 -TOOLS_RATE_LIMIT_PER_MINUTE=30 -TOOLS_MAX_FILE_SIZE_MB=50 -TOOLS_OCR_LANG=eng+ind - -# Infra (reuse existing) -REDIS_URL=redis://redis:6379 -NATS_URL=nats://nats:4222 -``` - ---- - -## CI/CD Integration - -### Docker Build Workflow - -Tambah service `tools` di `.github/workflows/docker-build-push.yml`: - -```yaml -# Di job "changes" step "Detect changed services" -changed() { - printf '%s\n' "$CHANGED_FILES" | grep -Eq "$1" && echo true || echo false -} -echo "tools=$(changed '^(apps/tools(/|$)|\.github/workflows/docker-build-push\.yml$|infra/docker/tools\.Dockerfile$)')" >> "$GITHUB_OUTPUT" - -# Di job "build" step "Set matrix" -if [ "${{ steps.filter.outputs['tools'] == 'true' || steps.dispatch.outputs['tools'] == 'true' || github.event_name == 'workflow_dispatch' }}" == "true" ]; then - add_service "tools" "docker-tools" "apps/tools" -fi - -# Di job "build" step "Docker metadata" -case "$SVC_NAME" in - "tools") echo "dockerfile=infra/docker/tools.Dockerfile" >> $GITHUB_OUTPUT ;; -esac - -# Di job "update-manifest" -SERVICES["tools"]="tools.yml" -PATHS["tools"]="apps/tools" -``` - -### Deploy Workflow - -Tambah di `.github/workflows/deploy-docker.yml`: -```yaml -# Tidak perlu perubahan — deploy-docker.yml auto-detect compose file changes. -# Kalau compose/tools.yml berubah, service tools akan di-restart. -``` - -### Service Registration (update infra/traefik/dynamic/apps.yaml) - -```yaml -tools: - rule: 'Host(`tools.asepharyana.my.id`) || Host(`tools.asepharyana.web.id`)' - entryPoints: - - websecure - tls: {} - middlewares: - - common-chain@file - service: tools-service - -# ...di bagian services: -tools-service: - loadBalancer: - servers: - - url: 'http://tools:3001' -``` - ---- - -## Monitoring - -### Prometheus Metrics - -Tambahkan label Prometheus ke container tools: - -```yaml -# Di compose tools.yml -labels: - - 'prometheus.io/scrape=true' - - 'prometheus.io/port=3001' - - 'prometheus.io/path=/metrics' -``` - -### Dashboard Integration - -Tambah card di dashboard hub yang sudah ada: - -```tsx -// Di dashboard hub — tambah section "Tools Usage" -// Data dari /api/dashboard → Prometheus query: -// rate(tools_jobs_total[24h]) — jobs per tool per hari -// sum(increase(tools_jobs_total[7d])) — total jobs minggu ini -// tools_jobs_in_flight — current processing -``` - ---- - -## Storage Architecture - -### Temp Storage - -``` -/data/tools/ -├── upload/ # Uploaded files -│ └── {job_id}.{ext} -├── processing/ # Intermediate files (stage-by-stage) -│ └── {job_id}/ -│ ├── 00_original.png -│ ├── 01_grayscale.png -│ ├── 02_edges.png -│ ├── 03_warped.png -│ └── ... -└── output/ # Final output - └── {job_id}.pdf -``` - -### Cleanup Strategy - -| Mekanisme | Timing | -|-----------|--------| -| NATS cron job | Setiap 10 menit | -| Scan files >1 jam | `find /data/tools -mmin +60 -delete` | -| Redis job keys >1 jam | `SCAN 0 MATCH job:*` → TTL check → DEL | -| Storage low warning | Alert via Notification Hub (future) | - ---- - -## Resource Estimation (VPS orangevps) - -### Current Usage - -| Service | CPU | RAM | Disk | -|---------|-----|-----|------| -| Traefik | 0.1 | 50 MB | 10 MB | -| NATS | 0.05 | 30 MB | 10 MB | -| Redis | 0.05 | 10 MB | 5 MB | -| Dapr Placement | 0.02 | 20 MB | 5 MB | -| Scraper API | 0.1 | 30 MB | 50 MB | -| Hub | 0.05 | 120 MB | 200 MB | -| Jaeger | 0.1 | 200 MB | 500 MB | -| Prometheus | 0.1 | 150 MB | 1 GB | -| Node Exporter | 0.02 | 10 MB | 5 MB | -| OTel Collector | 0.05 | 50 MB | 10 MB | -| **Total Current** | **~0.64** | **~670 MB** | **~1.8 GB** | - -### Tools Addition - -| Resources | Estimate | Notes | -|-----------|----------|-------| -| CPU | +1.0 core (burst) | Pipeline processing berat di CPU. Scoring, warp, OCR semua CPU-bound. | -| RAM | +300 MB | Rust binary + image processing buffers + Tesseract + ONNX | -| Disk | +5 GB | Temp files, bisa lebih untuk batch processing. Butuh auto-cleanup ketat. | -| **Total After** | **~1.64 cores** | **~970 MB RAM** | **~6.8 GB disk** | - -> **Catatan**: Kalau VPS cuma punya 1-2 cores, processing akan antri. NATS queue handle ini. Untuk production, pastikan CPU ada >2 cores. - -### Scalability - -``` -VPS 1 core: - - Scanner: ~5-8 detik per page - - Concurrent: 1 job at a time - - Antrian: NATS queue buffer unlimited - -VPS 4+ core: - - Scanner: ~2-3 detik per page - - Concurrent: 4 jobs parallel (1 per worker) - - Rayon: parallel per-page dalam batch -``` - ---- - -## Security Considerations - -| Area | Mitigation | -|------|-----------| -| **Upload validation** | MIME type check (whitelist), magic bytes verification, max size 50MB | -| **Path traversal** | Job ID = UUID v4, no user-controlled filenames in storage | -| **Command injection** | No shell commands — semua processing via Rust crates, FFmpeg via crate binding | -| **Temporary files** | Auto-cleanup, random filenames, restricted permissions (0600) | -| **Rate limiting** | Redis sliding window: 30 requests/min/IP per tool, 429 response | -| **CORS** | Origin terbatas ke domain portfolio | -| **Resource exhaustion** | Max image dimension 8000px, max file count per batch 50, worker concurrency limit | -| **OCR data** | Tesseract data dari package manager, no user-trained models | -| **ONNX model** | Model dari source terpercaya, verify checksum | - ---- - -## Rollback Strategy - -1. **Image tag**: `tools:sha-` immutable — tinggal update compose file ke tag sebelumnya -2. **Data**: Files auto-expire dalam 1 jam — no persistent data migration needed -3. **Traefik**: Cukup restart, TLS certs ga berubah -4. **Monitor**: Prometheus metrics akan langsung show error rate spike - ---- - -## Development Setup (Local) - -Untuk development tanpa Docker: - -```bash -# Terminal 1: Redis + NATS -docker compose -f infra/compose/shared.yml -f infra/compose/nats.yml up -d - -# Terminal 2: Rust workers -cd apps/tools/backend -REDIS_URL=redis://localhost:6379 NATS_URL=nats://localhost:4222 \ -cargo run --bin workers - -# Terminal 3: Rust gateway -REDIS_URL=redis://localhost:6379 NATS_URL=nats://localhost:4222 \ -TOOLS_STORAGE_PATH=/tmp/tools \ -cargo run --bin gateway - -# Terminal 4: Next.js -cd apps/tools/frontend -bun dev --port 3002 -``` - -### Test Pipeline Locally (tanpa NATS/Redis) - -Untuk development pipeline image processing doang: - -```rust -// Di workers/src/scanner/pipeline.rs — test function -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_full_pipeline() { - let pipeline = ScanPipeline::default(); - let result = pipeline.process_sync( - "test_images/scan_miring.jpg", - ScanOptions { ocr: false, enhance: true } - ); - assert!(result.is_ok()); - assert!(result.unwrap().output_path.exists()); - } - - #[test] - fn test_edge_detection_variations() { - // Test dengan berbagai kondisi: kertas putih, background ramai, sudut ekstrim - for case in &["normal.jpg", "dark.jpg", "angle45.jpg", "shadow.jpg"] { - let img = image::open(format!("test_images/{}", case)).unwrap(); - let corners = detect_corners_with_fallback(&img.grayscale().into_luma8()); - assert!(corners.is_ok(), "Failed on: {}", case); - } - } -} -``` - -Test images kumpulin dari foto dokumen real di berbagai kondisi — ini penting buat tuning parameter. diff --git a/docs/plan/tools/pipeline.md b/docs/plan/tools/pipeline.md deleted file mode 100644 index 0752803..0000000 --- a/docs/plan/tools/pipeline.md +++ /dev/null @@ -1,800 +0,0 @@ -# Document Scanner — Processing Pipeline - -> **LEGACY (2026-08-02):** Dokumen plan ini ditulis saat infra masih Docker/Traefik. Produksi sekarang Caddy + Nix/systemd dengan port 4000-an. Gunakan hanya sebagai referensi historis. - -Ini adalah inti dari project. Pipeline mengubah foto dokumen HP jadi dokumen scan yang proper. Setiap tahap dibahas detail teknisnya. - -## Pipeline Overview - -``` -Input: Foto HP (JPEG/PNG/HEIC, 2-12MP) - │ - ▼ -┌──────────────────────────────────┐ -│ 1. Preprocess ──▶ resize + │ -│ konversi grayscale │ -└────────────────┬─────────────────┘ - │ - ▼ -┌──────────────────────────────────┐ -│ 2. Edge Detection ──▶ cari │ -│ kontur dokumen │ -└────────────────┬─────────────────┘ - │ - ▼ -┌──────────────────────────────────┐ -│ 3. Corner Detection ──▶ 4 titik │ -│ sudut dokumen │ -└────────────────┬─────────────────┘ - │ - ▼ -┌──────────────────────────────────┐ -│ 4. Perspective Warp ──▶ lurusin│ -│ (homography) │ -└────────────────┬─────────────────┘ - │ - ▼ -┌──────────────────────────────────┐ -│ 5. Shadow Removal ──▶ iluminasi │ -│ merata │ -└────────────────┬─────────────────┘ - │ - ▼ -┌──────────────────────────────────┐ -│ 6. Binarization ──▶ hitam-putih │ -│ bersih │ -└────────────────┬─────────────────┘ - │ - ▼ -┌──────────────────────────────────┐ -│ 7. Deskew ──▶ lurusin teks │ -│ (kalau masih miring) │ -└────────────────┬─────────────────┘ - │ - ▼ -┌──────────────────────────────────┐ -│ 8. OCR ──▶ extract teks │ -└────────────────┬─────────────────┘ - │ - ▼ -┌──────────────────────────────────┐ -│ 9. Generate PDF ──▶ output │ -│ PDF + hidden text layer │ -└────────────────┬─────────────────┘ - │ - ▼ -Output: searchable PDF + teks OCR -``` - ---- - -## Stage 1: Preprocess - -### Input -- Raw image dari HP (bisa 4000×3000 = 12MP, ~3-5MB JPEG) -- Format: JPEG, PNG, HEIC (via `image` crate, HEIC butuh feature) - -### Proses -```rust -use image::{DynamicImage, imageops}; - -fn preprocess(img: &DynamicImage) -> DynamicImage { - // 1. Resize kalau terlalu besar → max 2000px di sisi terpanjang - // Ini penting: edge detection di resolusi tinggi lambat - // dan ga nambah akurasi secara signifikan - let max_dim = 2000.0; - let (w, h) = (img.width() as f64, img.height() as f64); - let img = if w.max(h) > max_dim { - let scale = max_dim / w.max(h); - let new_w = (w * scale) as u32; - let new_h = (h * scale) as u32; - img.resize_exact(new_w, new_h, imageops::FilterType::Lanczos3) - } else { - img.clone() - }; - - // 2. Grayscale → untuk edge detection - img.grayscale() -} -``` - -### Edge Cases -| Kasus | Penanganan | -|-------|-----------| -| Foto resolusi rendah (<800px) | Skip resize, langsung proses | -| HEIC format | Butuh feature `heic` di `image` crate | -| Grayscale input | `img.grayscale()` no-op | -| Foto malam/noise tinggi | Gaussian blur sebelum edge detection | - ---- - -## Stage 2: Edge Detection - -### Tujuan -Cari tepi dokumen dalam foto. Ini hardest part karena background bisa kacau. - -### Algoritma: Canny Edge Detection + Adaptive Threshold - -```rust -use image::GrayImage; -use imageproc::edges::canny; - -fn detect_edges(img: &GrayImage) -> GrayImage { - // Canny dengan dual threshold - // low: 50, high: 150 — parameter ini harus di-tune - // buat kondisi pencahayaan yang berbeda - canny(img, 50.0, 150.0) -} -``` - -### Masalah & Solusi - -| Masalah | Penyebab | Solusi | -|---------|----------|--------| -| **Tepi dokumen putus** | Kontras rendah, bayangan | Morphological close (dilate → erode) untuk sambungin tepi | -| **Tepi palsu** | Background ramai (meja motif, lantai) | Cari contour terbesar + area terluas = dokumen | -| **Tidak ada tepi** | Background putih, dokumen putih (kertas di meja putih) | Adaptive threshold dulu sebelum Canny, atau fallback ke manual crop | -| **Noise garis** | Texture background | Gaussian blur (kernel 5x5) sebelum Canny | - -### Implementation Detail - -```rust -/// Edge detection yang robust terhadap berbagai kondisi -fn robust_edge_detection(img: &GrayImage) -> GrayImage { - // 1. Gaussian blur untuk noise reduction - let blurred = imageproc::filter::gaussian_blur_f32(img, 3.0); - - // 2. Coba Canny standard - let edges = canny(&blurred, 50.0, 150.0); - - // 3. Morphological close untuk sambung tepi yang putus - let kernel = imageproc::morphology::dilate_square(5); - let closed = imageproc::morphology::close(&edges, &kernel); - - // 4. Kalau jumlah tepi terlalu sedikit (<1% pixels), - // ulang dengan threshold lebih rendah - let edge_count = count_non_zero(&closed); - let total_pixels = (closed.width() * closed.height()) as u32; - if edge_count < total_pixels / 100 { - let edges2 = canny(&blurred, 20.0, 80.0); - return imageproc::morphology::close(&edges2, &kernel); - } - - closed -} -``` - ---- - -## Stage 3: Corner Detection - -### Tujuan -Dari edge image, cari 4 sudut dokumen. - -### Algoritma: Contour Detection → Largest Rectangle - -```rust -use imageproc::contours::{find_contours, Contour}; - -fn find_document_corners(edges: &GrayImage) -> Option<[(f64, f64); 4]> { - // 1. Cari semua contours - let contours = find_contours(edges); - - // 2. Filter: cuma contour dengan area > 20% dari total image - // (dokumen biasanya mengisi sebagian besar frame) - let total_area = edges.width() as f64 * edges.height() as f64; - let docs: Vec<&Contour> = contours - .iter() - .filter(|c| area_perimeter_ratio(c) > 0.3) - .collect(); - - // 3. Approximate polygon → cari yang 4 sisi - for contour in docs { - // Approximate contour ke polygon - let polygon = approximate_polygon(&contour.points, 4); - if let Some(vertices) = polygon { - // Urutkan: top-left, top-right, bottom-right, bottom-left - let corners = order_corners(vertices); - return Some(corners); - } - } - - // 4. Fallback: contour terbesar → bounding rect - contours.iter() - .max_by_key(|c| c.points.len()) - .map(|c| { - let rect = bounding_rect(&c.points); - order_corners(vec![ - (rect.left as f64, rect.top as f64), - (rect.right as f64, rect.top as f64), - (rect.right as f64, rect.bottom as f64), - (rect.left as f64, rect.bottom as f64), - ]) - }) -} -``` - -### Corner Ordering Convention - -``` -(0,0) top-left ────────── top-right (w,0) - │ │ - │ DOKUMEN │ - │ │ -(0,h) bottom-left ────── bottom-right (w,h) -``` - -### Fallback Strategy - -Kalau auto-detect gagal total (contour tidak ketemu, confidence rendah): -1. **Fallback 1**: Coba di resolusi lebih rendah (noise berkurang) -2. **Fallback 2**: Coba adaptive threshold + Canny ulang -3. **Fallback 3**: Minta user crop manual — 4 draggable corners di canvas - -```rust -fn detect_corners_with_fallback(img: &GrayImage) -> Result<[(f64, f64); 4], CropMode> { - // Attempt 1: Resolusi penuh - if let Some(corners) = find_document_corners(img) { - return Ok(corners); - } - - // Attempt 2: Half resolution (noise reduction) - let half = image::imageops::resize(img, img.width() / 2, img.height() / 2, - imageops::FilterType::Lanczos3); - if let Some(corners) = find_document_corners(&half) { - return Ok(corners.map(|(x, y)| (x * 2.0, y * 2.0))); - } - - // Fallback: user manual - Err(CropMode::Manual) -} -``` - ---- - -## Stage 4: Perspective Warp - -### Tujuan -Transform 4 titik sudut ke persegi panjang (rectangular). Koreksi perspektif dari foto miring. - -### Algoritma: Homography - -```rust -use image::{DynamicImage, GrayImage}; -use std::f64::consts::PI; - -fn perspective_warp(img: &DynamicImage, corners: [(f64, f64); 4]) -> DynamicImage { - // Target: persegi panjang dengan aspect ratio dokumen - // Hitung lebar dan tinggi target dari 4 corner - let [tl, tr, br, bl] = corners; - - let width_top = distance(tl, tr); - let width_bot = distance(bl, br); - let width = width_top.max(width_bot).ceil() as u32; - - let height_left = distance(tl, bl); - let height_right = distance(tr, br); - let height = height_left.max(height_right).ceil() as u32; - - // Source points (4 corners dari detection) - let src = [ - tl, // top-left - tr, // top-right - br, // bottom-right - bl, // bottom-left - ]; - - // Destination points (rectangle) - let dst = [ - (0.0, 0.0), // top-left - (width as f64, 0.0), // top-right - (width as f64, height as f64), // bottom-right - (0.0, height as f64), // bottom-left - ]; - - // Hitung homography matrix - let h = compute_homography(&src, &dst); - - // Apply warp (backward mapping + bilinear interpolation) - warp_image(img, &h, width, height) -} -``` - -### Homography Matrix - -``` -H = [h11 h12 h13] x' = (h11*x + h12*y + h13) / (h31*x + h32*y + 1) - [h21 h22 h23] y' = (h21*x + h22*y + h23) / (h31*x + h32*y + 1) - [h31 h32 1 ] -``` - -Komputasi manual (tanpa OpenCV): -```rust -/// Compute homography from 4 point correspondences using DLT algorithm -fn compute_homography(src: &[(f64, f64); 4], dst: &[(f64, f64); 4]) -> [[f64; 3]; 3] { - // Direct Linear Transform - // Bangun matrix A (8x9) dari 4 titik - // Solve Ah = 0 via SVD → h = last column of V - // Reshape ke 3x3 - // - // Detail implementasi: - // Setiap titik correspondence (x,y) → (x',y') menghasilkan 2 baris: - // [-x, -y, -1, 0, 0, 0, x*x', y*x', x'] = 0 - // [ 0, 0, 0, -x, -y, -1, x*y', y*y', y'] = 0 - // - // 4 titik → 8 baris → SVD → H matrix - - // Implementasi SVD atau pakai crate `nalgebra` atau `splines` - todo!("Implement DLT + SVD") -} -``` - -### Image Warp (Backward Mapping) - -```rust -fn warp_image(img: &DynamicImage, h: &[[f64; 3]; 3], width: u32, height: u32) -> DynamicImage { - let gray = img.grayscale().into_luma8(); - let mut output = GrayImage::new(width, height); - - // Inverse homography (backward mapping) - // tiap pixel output = sample dari input - let h_inv = invert_homography(h); - - for y in 0..height { - for x in 0..width { - // Map (x,y) → source image coordinates - let (sx, sy) = apply_homography(&h_inv, x as f64, y as f64); - - // Bilinear interpolation - let pixel = bilinear_interpolate(&gray, sx, sy); - output.put_pixel(x, y, pixel); - } - } - - DynamicImage::ImageLuma8(output) -} -``` - -### Edge Cases - -| Masalah | Solusi | -|---------|--------| -| Dokuen sangat miring (>60°) | Warping mungkin hasilnya gepeng. Deteksi dan skip kalau sudut terlalu ekstrim | -| Output sangat besar | Clamp width/height ke max 3000px | -| Pixel jaggy (aliasing) | Bilinear interpolation (bukan nearest neighbor) | -| Koordinat negative | Clamp ke 0 | -| Warp membuat rasio aneh | Lock aspect ratio ke common (A4=1.414, Letter=1.294) | - ---- - -## Stage 5: Shadow Removal - -### Tujuan -Hilangkan bayangan (dari lampu, jari, atau sudut ruangan). - -### Algoritma: Adaptive Illumination Correction - -Shadow adalah low-frequency variation. Teks adalah high-frequency. Pisahkan pake low-pass filter. - -```rust -fn remove_shadow(img: &GrayImage) -> GrayImage { - let (w, h) = (img.width(), img.height()); - - // 1. Large Gaussian blur untuk estimasi iluminasi background - // Kernel besar (≥sx/50) → cuma dapet variasi iluminasi, bukan teks - let blur_radius = (w.min(h) as f64 / 50.0).max(15.0); - let background = imageproc::filter::gaussian_blur_f32(img, blur_radius); - - // 2. Subtract background dari original - // pixel = max(0, original - background + mean(background)) - let bg_mean = mean_pixel(&background); - let mut corrected = GrayImage::new(w, h); - - for y in 0..h { - for x in 0..w { - let orig = img.get_pixel(x, y)[0] as f32; - let bg = background.get_pixel(x, y)[0] as f32; - let corrected_val = (orig - bg + bg_mean) as u8; - corrected.put_pixel(x, y, Luma([corrected_val])); - } - } - - // 3. CLAHE (Contrast Limited Adaptive Histogram Equalization) - // untuk normalisasi kontras lokal - apply_clahe(&corrected, 8, 4) // 8x8 tiles, clip limit 4 -} -``` - -### Alternatif: Retinex Theory - -```rust -/// Retinex-based illumination correction -/// I(x,y) = R(x,y) × L(x,y) -/// I = observed image, R = reflectance (teks), L = illumination (shadow) -fn retinex_shadow_removal(img: &GrayImage) -> GrayImage { - // Single-scale Retinex - // log(R) = log(I) - log(G * I) - // dimana G = Gaussian kernel - - let float_img = convert_to_float(img); - let blurred = gaussian_blur_float(&float_img, 30.0); - let retinex = element_wise(|p| (p.0.ln() - p.1.ln()), &float_img, &blurred); - - // Normalize ke [0, 255] - normalize_to_u8(&retinex) -} -``` - ---- - -## Stage 6: Binarization - -### Tujuan -Ubah ke hitam-putih bersih — teks hitam, background putih. - -### Algoritma: Sauvola Local Threshold - -Global threshold (Otsu) gagal kalau iluminasi ga merata. Sauvola adaptif per region. - -```rust -fn sauvola_threshold(img: &GrayImage, window_size: u32, k: f32) -> GrayImage { - // Sauvola: T(x,y) = m(x,y) * [1 + k * (s(x,y)/R - 1)] - // m = local mean, s = local std dev, R = max std dev (128), k = parameter (~0.2) - - let (w, h) = (img.width(), img.height()); - let half_win = (window_size / 2) as i32; - let mut output = GrayImage::new(w, h); - - // Integral image for O(1) mean and variance computation - let integral = compute_integral_image(img); - let integral_sq = compute_integral_image_sq(img); - - for y in 0..h { - for x in 0..w { - let (mean, variance) = local_stats(&integral, &integral_sq, - x as i32, y as i32, - half_win, w as i32, h as i32); - let std_dev = variance.sqrt(); - let threshold = mean * (1.0 + k * (std_dev / 128.0 - 1.0)); - - let pixel = img.get_pixel(x, y)[0] as f32; - output.put_pixel(x, y, Luma([if pixel > threshold { 255 } else { 0 }])); - } - } - - output -} -``` - -### Parameter Default - -| Parameter | Value | Notes | -|-----------|-------|-------| -| Window size | max(w,h)/30 | Minimum 15, maksimum 100 | -| k | 0.2 | Lower → lebih sensitif, higher → lebih toleran | - -### Edge Cases - -| Masalah | Solusi | -|---------|--------| -| Dokumen berwarna (bukan putih) | Deteksi warna dominan background, invert logic | -| Background gradasi | Sauvola handle ini lebih baik dari Otsu | -| Foto terlalu gelap | CLAHE dulu sebelum binarization | -| Text tipis/kabur | Morphological erode tipis sesudah binarization | - ---- - -## Stage 7: Deskew - -### Tujuan -Koreksi rotasi sisa (kalau dokumen masih miring sedikit — biasanya <5°). - -### Algoritma: Hough Transform - -```rust -fn deskew(img: &GrayImage) -> GrayImage { - // 1. Cari garis teks via Hough transform - // Probabilistic Hough lebih cepat - let lines = probabilistic_hough_lines(img, 10, PI / 180.0, 50, 50.0, 10.0); - - if lines.is_empty() { - return img.clone(); - } - - // 2. Hitung sudut rata-rata semua garis - let angles: Vec = lines.iter() - .map(|line| line.angle().to_degrees()) - .filter(|a| a.abs() < 45.0) // skip garis vertikal - .collect(); - - if angles.is_empty() { - return img.clone(); - } - - let median_angle = median(&angles); - - // Skip kalau sudutnya <0.5 derajat (ga perlu koreksi) - if median_angle.abs() < 0.5 { - return img.clone(); - } - - // 3. Rotate image - rotate(img, median_angle, imageops::FilterType::Lanczos3) -} -``` - ---- - -## Stage 8: OCR - -### Tujuan -Extract teks dari gambar biar PDF-nya searchable dan teks bisa di-copy. - -### Implementation - -```rust -use leptess::LepTess; - -fn ocr(img: &GrayImage, lang: &str) -> Result { - // 1. Init Tesseract - let mut tess = LepTess::new(Some("/usr/share/tesseract/tessdata"), lang)?; - - // 2. Set image - tess.set_image_from_mem(&img.to_bytes())?; - // 3. Set PSM (Page Segmentation Mode) - // PSM 3 = Fully automatic, default - // PSM 6 = Assume single uniform block of text - // PSM 4 = Assume single column of text - tess.set_source_resolution(300); - - // 4. Recognize - let text = tess.get_utf8_text()?; - - Ok(text) -} - -/// Dapatkan word-level bounding boxes untuk positioning di PDF -fn ocr_words(img: &GrayImage, lang: &str) -> Result, OcrError> { - let mut tess = LepTess::new(Some("/usr/share/tesseract/tessdata"), lang)?; - tess.set_image_from_mem(&img.to_bytes())?; - - let words = tess.get_words() - .iter() - .map(|w| Word { - text: w.text.clone(), - bbox: Bbox { - x: w.x, - y: w.y, - width: w.w, - height: w.h, - }, - confidence: w.confidence, - }) - .collect(); - - Ok(words) -} -``` - -### Output Format - -```rust -struct Word { - text: String, - bbox: Bbox, - confidence: i32, // 0-100 -} -``` - ---- - -## Stage 9: PDF Generation - -### Tujuan -Generate PDF yang: -1. Berisi gambar hasil scan (JPEG compressed) -2. Hidden text layer dari OCR (biar searchable, selectable) - -### Implementation - -```rust -use lopdf::{Document, Object, Stream}; -use std::io::Write; - -fn generate_searchable_pdf( - image_data: &[u8], // JPEG-compressed scan image - ocr_text: &str, // Full OCR text - words: &[Word], // Word positions - page_width: f64, // PDF page width in points - page_height: f64, // PDF page height in points -) -> Result, PdfError> { - let mut doc = Document::new(); - - // 1. Create image XObject - let image_stream = Stream::new( - dictionary! { - "Type" => "XObject", - "Subtype" => "Image", - "Width" => page_width as u32, - "Height" => page_height as u32, - "ColorSpace" => "DeviceGray", - "BitsPerComponent" => 8, - "Filter" => "DCTDecode", // JPEG compression - }, - image_data, - ); - let image_id = doc.add_object(image_stream); - - // 2. Create content stream: place image, then invisible text - // Text layer is invisible (rendering mode 3 = neither fill nor stroke) - let mut content = Vec::new(); - writeln!(content, "q")?; // save state - writeln!(content, "{} 0 0 {} 0 0 cm", page_width, page_height)?; // scale to page - writeln!(content, "/Im0 Do")?; // place image - writeln!(content, "Q")?; // restore state - - // 3. Add invisible text layer (searchable) - for word in words { - let x = word.bbox.x as f64 / DPI * 72.0; // convert pixels → points - let y = (page_height - word.bbox.y as f64 / DPI * 72.0); - writeln!(content, "BT")?; - writeln!(content, "3 Tr")?; // rendering mode: invisible - writeln!(content, "1 Tw")?; // word spacing - writeln!(content, "{} {} Td", x, y)?; // position - writeln!(content, "({}) Tj", escape_pdf_string(&word.text))?; - writeln!(content, "ET")?; - } - - let content_stream = Stream::new( - dictionary! {}, - content, - ); - let content_id = doc.add_object(content_stream); - - // 4. Create page - let page_id = doc.new_object_id(); - let pages_id = doc.new_object_id(); - - doc.objects.insert(page_id, Object::Dictionary(dictionary! { - "Type" => "Page", - "Parent" => pages_id, - "MediaBox" => vec![0.0, 0.0, page_width, page_height], - "Contents" => content_id, - "Resources" => dictionary! { - "XObject" => dictionary! { - "Im0" => image_id, - }, - }, - })); - - // 5. Close and return bytes - let bytes = doc.save_to_bytes()?; - Ok(bytes) -} -``` - -### PDF Coordinate System - -``` -PDF origin = bottom-left -Image origin = top-left - -Perlu flip Y coordinate untuk text layer: -y_pdf = page_height - (y_image / dpi * 72) -``` - ---- - -## Complete Pipeline Assembly - -```rust -pub struct ScanPipeline { - config: PipelineConfig, - metrics: MetricsRecorder, -} - -impl ScanPipeline { - pub async fn process(&self, input_path: &Path, options: ScanOptions) - -> Result - { - let timer = self.metrics.start_timer("scan.full"); - - // 1. Load - let img = image::open(input_path) - .map_err(PipelineError::ImageLoad)?; - self.metrics.stage_duration("load", timer.split()); - - // 2. Preprocess - let gray = preprocess(&img); - self.metrics.stage_duration("preprocess", timer.split()); - - // 3. Edge detection + corners (fallback chain) - let corners = detect_corners_with_fallback(&gray) - .map_err(PipelineError::CornerDetection)?; - self.metrics.stage_duration("corner_detection", timer.split()); - - // 4. Perspective warp - let warped = perspective_warp(&img, corners); // warp from COLOR original, not gray - self.metrics.stage_duration("warp", timer.split()); - - let warped_gray = warped.grayscale().into_luma8(); - - // 5. Shadow removal - let clean = remove_shadow(&warped_gray); - self.metrics.stage_duration("shadow_removal", timer.split()); - - // 6. Binarization - let binary = sauvola_threshold(&clean, 50, 0.2); - self.metrics.stage_duration("binarization", timer.split()); - - // 7. Deskew - let final_image = deskew(&binary); - self.metrics.stage_duration("deskew", timer.split()); - - // 8. Enhance final (sharpening) - let final_image = sharpen(&final_image, 1.0); - self.metrics.stage_duration("sharpen", timer.split()); - - // 9. OCR - let ocr_text = if options.ocr { - Some(ocr(&final_image, "eng")?) - } else { - None - }; - self.metrics.stage_duration("ocr", timer.split()); - - // 10. Generate PDF - let pdf_bytes = generate_searchable_pdf( - &compress_jpeg(&final_image, 90)?, - &ocr_text.unwrap_or_default(), - &[], // word positions (simplified) - A4_WIDTH_PT, - A4_HEIGHT_PT, - )?; - self.metrics.stage_duration("pdf_generation", timer.split()); - - // 11. Save - let output_path = PathBuf::from("/tmp/tools").join(format!("{}.pdf", uuid::Uuid::new_v4())); - std::fs::write(&output_path, &pdf_bytes)?; - - timer.finish(); - - Ok(ScanResult { - output_path, - page_count: 1, - file_size: pdf_bytes.len() as u64, - ocr_text, - }) - } -} -``` - -## Performance Budget - -| Stage | Target | Notes | -|-------|--------|-------| -| Load + Preprocess | <200ms | File I/O + resize | -| Edge + Corner Detection | <500ms | Canny + contour | -| Perspective Warp | <800ms | Per-pixel backward mapping | -| Shadow Removal | <300ms | FFT convolution atau integral image | -| Binarization | <200ms | Integral image | -| Deskew | <300ms | Hough transform | -| OCR | <1.5s | Tesseract, 300dpi | -| PDF Generation | <200ms | lopdf | -| **Total** | **<4s** | Per page | - -> **Catatan**: Target di atas untuk image 12MP (4000×3000). Parallel via Rayon untuk batch processing. - -## Edge Cases Matrix - -| Skenario | Pipeline Behavior | -|----------|------------------| -| Kertas putih di meja putih | Edge detection gagal → fallback ke manual crop | -| Foto dari sudut 45° | Warp koreksi perspektif, output presisi | -| Dokumen terlipat | Edge detection dapet bentuk aneh → fallback manual | -| Bayangan jari | Shadow removal hilangkan | -| Teks pudar/pensil | Sauvola threshold adaptif, contrast enhance dulu | -| Tanda tangan & stempel | OCR bisa gagal di handwriting, tetap di-image | -| Multi-page (buku/kontrak) | Batch upload, masing-masing diproses, digabung 1 PDF | -| Foto malam | CLAHE + strong denoise sebelum edge detection | -| Latar belakang gradasi | Sauvola handle lebih baik dari Otsu | diff --git a/docs/security-guide.md b/docs/security-guide.md deleted file mode 100644 index 2adf2af..0000000 --- a/docs/security-guide.md +++ /dev/null @@ -1,211 +0,0 @@ -# Security Guide - -Praktik keamanan untuk infrastruktur `asepharyana-hub`. - -## Ringkasan - -| Area | Status | Prioritas | -|------|--------|-----------| -| Secrets management | GitHub encrypted secrets | Tinggi | -| TLS termination | Traefik + cert volume mounts | Tinggi | -| Container security | Non-root user (scraper-api) | Sedang | -| Network security | Tailscale overlay, app-shared-net | Sedang | -| Access control | SSH key, GitHub permissions | Sedang | -| Monitoring | Belum ada alert system | Rendah | -| Firewall | UFW/iptables (manual) | Sedang | -| Backup | lihat `docs/backup-recovery.md` | Sedang | - -## Secrets Management - -### Yang Tidak Boleh di-Commit - -- [ ] `.env` production (disimpan sebagai GitHub secret `ENV_FILE_PRODUCTION`) -- [ ] SSH private keys -- [ ] API tokens, JWT secret -- [ ] Docker registry tokens -- [ ] Database passwords -- [ ] TLS certificate private keys - -### GitHub Secrets - -Setting di Settings > Secrets and variables > Actions: - -| Secret | Tujuan | Rotasi | -|--------|--------|--------| -| `SSH_PRIVATE_KEY` | Akses SSH ke VPS | 6 bulan | -| `VPS_HOST` | IP VPS | Tidak berubah | -| `VPS_USER` | User SSH | Tidak berubah | -| `VPS_TARGET_DIR` | Directory di VPS | Tidak berubah | -| `ENV_FILE_PRODUCTION` | Full `.env` production | Saat ada perubahan | - -### Update Secrets dengan aman - -```bash -# Baca current .env dari VPS via SSH -ssh root@45.127.35.244 "cat /root/asepharyana-hub/.env" | gh secret set ENV_FILE_PRODUCTION --repo asepharyana/asepharyana-hub --repos -``` - -### Production `.env` tidak boleh di-commit - -`.env` di root repo adalah untuk development lokal. Production `.env` hanya ada di: -1. GitHub secret `ENV_FILE_PRODUCTION` -2. File `/root/asepharyana-hub/.env` di VPS (hasil SCP dari CI/CD) - -## TLS / SSL - -### Konfigurasi - -```yaml -# Traefik TLS certs dari file mount (bukan auto-ACME) -volumes: - - ${TRAEFIK_CERT_MY_ID_PEM:-/root/asepharyana.my.id.pem}:/etc/traefik/certs/asepharyana.my.id.pem:ro - - ${TRAEFIK_CERT_MY_ID_KEY:-/root/asepharyana.my.id.key}:/etc/traefik/certs/asepharyana.my.id.key:ro -``` - -### Best Practices - -- Certificates disimpan di host (`/root/`), bukan di repo -- Volume mount read-only (`:ro`) -- Private key hanya bisa dibaca oleh root (chmod 600) -- Renew certificates sebelum expired (monitor expiry) -- Dua domain: `asepharyana.my.id` + `asepharyana.web.id` - -## Container Security - -### Non-Root User - -Scraper API berjalan sebagai `appuser` (UID 1001): - -```dockerfile -RUN groupadd -g 1001 appgroup && \ - useradd -u 1001 -g appgroup -s /bin/sh appuser -USER appuser -``` - -Service baru harus mengikuti pattern yang sama. - -### Read-Only Filesystem - -Untuk container yang tidak perlu write ke filesystem: - -```yaml -services: - app: - image: app:latest - read_only: true - tmpfs: - - /tmp -``` - -### Docker Socket - -Hanya Traefik yang perlu akses ke Docker socket (read-only): - -```yaml -volumes: - - /var/run/docker.sock:/var/run/docker.sock:ro -``` - -Service lain tidak boleh mount Docker socket. - -### Image Security - -- Build dari base image resmi dan minimal (`debian:bookworm-slim`, `redis:alpine`, `nats:latest`) -- Multi-stage build untuk production image (tidak include build tools) -- Update base image secara berkala - -## Network Security - -### Firewall (UFW/iptables) - -Di VPS (`orangevps`): - -```bash -# Hanya buka port yang diperlukan -sudo ufw default deny incoming -sudo ufw default allow outgoing -sudo ufw allow 22/tcp # SSH -sudo ufw allow 80/tcp # HTTP redirect -sudo ufw allow 443/tcp # HTTPS -sudo ufw allow 4222/tcp # NATS (jika perlu external akses) -sudo ufw enable -``` - -Di `imrnes`: - -```bash -# Hanya dari Tailscale interface -sudo ufw allow in on tailscale0 to any port 6432 proto tcp # PostgreSQL -sudo ufw allow in on tailscale0 to any port 6379 proto tcp # Redis -sudo ufw enable -``` - -### Network Segmentation - -- Semua container di network `app-shared-net` (internal bridge) -- Tidak ada port yang di-expose ke host kecuali Traefik (80,443) -- Redis hanya accessible via Docker DNS (`redis:6379`) — tidak di-expose -- Database hanya via Tailscale — tidak accessible dari public internet - -### SSH Hardening - -Konfigurasi di `/etc/ssh/sshd_config`: - -``` -Port 22 -PermitRootLogin prohibit-password -PasswordAuthentication no -PubkeyAuthentication yes -AllowUsers root -MaxAuthTries 3 -ClientAliveInterval 300 -ClientAliveCountMax 2 -``` - -## Access Control - -### GitHub Repository - -- `contents: write` hanya untuk workflow `update-manifest` dan `update-submodule` -- `packages: write` hanya untuk workflow `build` -- `security-events: write` hanya untuk workflow `security` -- Branch protection di `main`: require PR review, status checks - -### VPS - -- SSH hanya dengan key-based authentication -- Key disimpan di GitHub secret, bukan di repo -- Rotate SSH key secara berkala (minimal 6 bulan) -- Jangan gunakan password login - -## Monitoring Keamanan - -### Saat Ini - -- Traefik access logs (format JSON, buffer size 100) -- Docker logs via `docker logs` -- CodeQL analysis untuk Rust code (setiap PR + weekly) - -### Rekomendasi - -- [ ] Alert untuk SSH failed login (fail2ban) -- [ ] Log monitoring (Loki / Promtail) -- [ ] Container vulnerability scanning (Trivy / Snyk) -- [ ] Certificate expiry monitoring -- [ ] Disk usage alert -- [ ] Unauthorized access detection - -## Checklist Security - -- [ ] SSH password authentication disabled -- [ ] Root login via SSH key only -- [ ] UFW/iptables configured -- [ ] Docker socket only mounted where necessary (read-only) -- [ ] Container berjalan sebagai non-root user -- [ ] `.env` tidak di-commit -- [ ] GitHub secrets ter-encrypt -- [ ] TLS certificates valid dan belum expired -- [ ] CodeQL analysis berjalan -- [ ] Backup database berjalan -- [ ] SSH key di-rotate -- [ ] Docker image di-scan untuk vulnerability diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index 254d675..9861b6e 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -1,6 +1,6 @@ # Troubleshooting -Kumpulan solusi untuk masalah umum yang spesifik di infrastruktur `asepharyana-hub`. +Kumpulan solusi untuk masalah umum di infrastruktur `asepharyana/infra` (orangevps). > **Catatan (2026-08-02):** Produksi sekarang Caddy + Nix/systemd. Section Traefik/Docker di bawah adalah LEGACY — Docker dan Traefik dihapus dari produksi; gunakan hanya sebagai referensi historis. @@ -29,7 +29,7 @@ Kumpulan solusi untuk masalah umum yang spesifik di infrastruktur `asepharyana-h | `SSH_PRIVATE_KEY` | Wajib | | `VPS_HOST` | Wajib (`45.127.35.244`) | | `VPS_USER` | Wajib (`root`) | -| `VPS_TARGET_DIR` | Wajib (`/root/asepharyana-hub`) | +| ~~`VPS_TARGET_DIR`~~ | Tidak dipakai lagi (CI baru tidak checkout VPS) | | `ENV_FILE_PRODUCTION` | Wajib | ### Workflow build gagal: "Submodule commit not fetchable" @@ -40,7 +40,7 @@ Kumpulan solusi untuk masalah umum yang spesifik di infrastruktur `asepharyana-h ```bash # Cek apakah commit ada di remote -git ls-remote https://github.com/asepharyana/asepharyana-hub-scraper.git +git ls-remote https://github.com/asepharyana/scraper.git # Trigger ulang dispatch dari submodule repo, atau push langsung ke hub ``` diff --git a/flake.lock b/flake.lock deleted file mode 100644 index 617b2f3..0000000 --- a/flake.lock +++ /dev/null @@ -1,61 +0,0 @@ -{ - "nodes": { - "flake-utils": { - "inputs": { - "systems": "systems" - }, - "locked": { - "lastModified": 1731533236, - "narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=", - "owner": "numtide", - "repo": "flake-utils", - "rev": "11707dc2f618dd54ca8739b309ec4fc024de578b", - "type": "github" - }, - "original": { - "owner": "numtide", - "repo": "flake-utils", - "type": "github" - } - }, - "nixpkgs": { - "locked": { - "lastModified": 1785301185, - "narHash": "sha256-eoS3KQTO0aPWXZvIaRbRAzSSHW3l5wdMFXtT1ISfoKA=", - "owner": "NixOS", - "repo": "nixpkgs", - "rev": "9bc02893134c733dd85de46ee4fb2fac696b5529", - "type": "github" - }, - "original": { - "owner": "NixOS", - "ref": "nixpkgs-unstable", - "repo": "nixpkgs", - "type": "github" - } - }, - "root": { - "inputs": { - "flake-utils": "flake-utils", - "nixpkgs": "nixpkgs" - } - }, - "systems": { - "locked": { - "lastModified": 1681028828, - "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", - "owner": "nix-systems", - "repo": "default", - "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", - "type": "github" - }, - "original": { - "owner": "nix-systems", - "repo": "default", - "type": "github" - } - } - }, - "root": "root", - "version": 7 -} diff --git a/flake.nix b/flake.nix deleted file mode 100644 index 5d21506..0000000 --- a/flake.nix +++ /dev/null @@ -1,209 +0,0 @@ -{ - description = "Asepharyana Hub — Nix builds for infrastructure and app services"; - - inputs = { - nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-unstable"; - flake-utils.url = "github:numtide/flake-utils"; - }; - - outputs = { self, nixpkgs, flake-utils }: - flake-utils.lib.eachSystem [ "x86_64-linux" ] (system: - let - pkgs = import nixpkgs { - inherit system; - config.allowUnfree = true; - }; - - # ── mkApp generator ── - mkApp = { name, src, buildScript, installScript, nativeBuildInputs ? [], buildInputs ? [] }: - pkgs.stdenv.mkDerivation { - inherit name src; - - nativeBuildInputs = with pkgs; [ - cacert curl gcc gnumake openssl pkg-config python3 libclang - ] ++ nativeBuildInputs; - - buildInputs = with pkgs; [ - nodejs openssl stdenv.cc.cc.lib libffi - ] ++ buildInputs; - - LIBCLANG_PATH = "${pkgs.libclang.lib}/lib"; - LD_LIBRARY_PATH = "${pkgs.libclang.lib}/lib:${pkgs.stdenv.cc.cc.lib}/lib:${pkgs.libffi}/lib"; - NIX_ENFORCE_PURITY = "0"; - - SSL_CERT_FILE = "${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt"; - NODE_EXTRA_CA_CERTS = "${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt"; - NODE_ENV = "production"; - - phases = [ "unpackPhase" "buildPhase" "installPhase" ]; - buildPhase = '' - export HOME="$TMPDIR" CARGO_HOME="$TMPDIR/.cargo-${name}" - SSL_CERT_FILE=${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt - '' + buildScript; - installPhase = installScript; - }; - - # ── Node.js ── - nodejs = pkgs.nodejs-slim_22; - pnpm = pkgs.pnpm.override { inherit nodejs; }; - - # ── Common Rust build deps ── - cargoDeps = with pkgs; [ rustc cargo clang cmake pkg-config openssl.dev zlib ]; - - # ── Submodule repos — URLs from .gitmodules ── - submoduleRepos = { - hub = "https://github.com/asepharyana/asepharyana-hub-hub.git"; - scraper = "https://github.com/asepharyana/asepharyana-hub-scraper.git"; - tools = "https://github.com/asepharyana/asepharyana-hub-tools.git"; - llm-api = "https://github.com/asepharyana/asepharyana-hub-llm-api.git"; - }; - - # ── Fetch submodule source ── - submoduleSrc = name: builtins.fetchGit { - url = submoduleRepos.${name}; - rev = if name == "hub" then "a90d0c43336a5f000b5856003d2293c420e7d595" - else if name == "scraper" then "62aa5b0e52859afe3ba9de1c7b11cfe2dacf6c2c" - else if name == "tools" then "3956b90c3ce39ffa7ffba8084937f20e11364d6b" - else if name == "llm-api" then "5f7ead5503082a71d41a36fd1727325c784e4b79" - else "HEAD"; - submodules = true; - }; - - # ─── App Derivations ─── - hub = mkApp { - name = "hub-0.1.0"; - src = submoduleSrc "hub"; - - nativeBuildInputs = with pkgs; [ bun ]; - - buildScript = '' - echo "=== Installing dependencies ===" - bun install 2>&1 - echo "=== Building Next.js ===" - bun run build 2>&1 - ''; - - installScript = '' - mkdir -p $out/share/hub $out/bin - cp -r .next $out/share/hub/ - cp -r public $out/share/hub/ 2>/dev/null || true - cp package.json $out/share/hub/ - cp next.config.{ts,mjs,js} $out/share/hub/ 2>/dev/null || true - cp -r node_modules $out/share/hub/ - cat > $out/bin/hub << WRAPPER -#!${pkgs.runtimeShell} -exec ${pkgs.bun}/bin/bun run --cwd $out/share/hub start -WRAPPER - chmod +x $out/bin/hub - ''; - }; - - scraper = mkApp { - name = "scraper-0.1.0"; - src = submoduleSrc "scraper"; - nativeBuildInputs = cargoDeps; - - buildScript = '' - echo "=== Building scraper ===" - cargo build --release 2>&1 - ''; - - installScript = '' - mkdir -p $out/bin - cp target/release/scraper $out/bin/scraper - ''; - }; - - tools-gateway = mkApp { - name = "tools-gateway-0.1.0"; - src = submoduleSrc "tools"; - nativeBuildInputs = cargoDeps ++ [ pkgs.tesseract ]; - - buildScript = '' - cd backend - echo "=== Building tools-gateway ===" - cargo build --release --features tesseract --bin tools-gateway 2>&1 - ''; - - installScript = '' - mkdir -p $out/bin - cp target/release/tools-gateway $out/bin/tools-gateway - ''; - }; - - tools-workers = mkApp { - name = "tools-workers-0.1.0"; - src = submoduleSrc "tools"; - nativeBuildInputs = cargoDeps ++ [ pkgs.tesseract pkgs.leptonica ]; - - buildScript = '' - cd backend - echo "=== Building tools-workers ===" - cargo build --release --features tesseract --bin tools-workers 2>&1 - ''; - installScript = '' - mkdir -p $out/bin - cp target/release/tools-workers $out/bin/tools-workers - ''; - }; - - tools-frontend = mkApp { - name = "tools-frontend-0.1.0"; - src = submoduleSrc "tools"; - nativeBuildInputs = with pkgs; [ bun ]; - - buildScript = '' - cd frontend - echo "=== Installing dependencies ===" - bun install 2>&1 - echo "=== Building Next.js ===" - bun run build 2>&1 - ''; - - installScript = '' - mkdir -p $out/share/tools-frontend $out/bin - cp -r .next $out/share/tools-frontend/ - cp -r public $out/share/tools-frontend/ 2>/dev/null || true - cp package.json $out/share/tools-frontend/ - cp -r node_modules $out/share/tools-frontend/ - cat > $out/bin/tools-frontend << WRAPPER -#!${pkgs.runtimeShell} -exec ${pkgs.bun}/bin/bun run --cwd $out/share/tools-frontend start -WRAPPER - chmod +x $out/bin/tools-frontend - ''; - }; - - llm-api = mkApp { - name = "llm-api-0.1.0"; - src = submoduleSrc "llm-api"; - nativeBuildInputs = cargoDeps ++ [ pkgs.cmake pkgs.gcc ]; - - buildScript = '' - echo "=== Building llm-api ===" - cargo build --release 2>&1 - ''; - - installScript = '' - mkdir -p $out/bin - cp target/release/llm-api $out/bin/llm-api - ''; - }; - - in - { - packages = { - inherit hub scraper tools-gateway tools-workers tools-frontend llm-api; - default = hub; - }; - - apps.hub = { - type = "app"; - program = "${hub}/bin/hub"; - }; - - devShells.default = pkgs.mkShell { - buildInputs = with pkgs; [ nodejs-slim_22 bun pnpm rustc cargo ]; - }; - }); -} diff --git a/infra/README.md b/infra/README.md index 021b40f..2afb4e1 100644 --- a/infra/README.md +++ b/infra/README.md @@ -1,97 +1,40 @@ -# Infrastructure +# Infra — Reverse Proxy & VPS Config -Docker Compose and Traefik configuration for `asepharyana-hub`. +Config produksi untuk **orangevps** (Caddy reverse proxy + firewall + systemd drop-ins). -## Layout +## Caddy (`caddy/`) -```text -infra/ -├── compose/ # One compose file per stack/service -│ ├── traefik.yml # Public reverse proxy -│ ├── shared.yml # Shared Redis -│ ├── nats.yml # NATS message broker + JetStream -│ ├── dapr.yml # Dapr placement service -│ └── scraper.yml # Scraper API (app + Dapr sidecar) -├── docker/ # Dockerfiles and image runtime helpers -├── dapr/ # Dapr component configs -│ ├── config.yaml # Global Dapr configuration -│ └── components/ # Pub/sub (Redis), state store (Redis) -├── traefik/ # Dynamic Traefik configuration -│ ├── dynamic/ # Routers, services, middlewares, TLS certs -│ └── TRAEFIK_ENV_CONFIG.md +- `Caddyfile.prod` — **source of truth** untuk `/etc/caddy/Caddyfile`. +- Site pattern: `.asepharyana.my.id` / `.web.id` → `import proxy `. +- Auto-TLS Let's Encrypt; HTTP/3 default. +- Deploy: push `infra/caddy/**` ke `main` → workflow `caddy-deploy.yml` sync + reload + verify. + +### Update site baru +```caddyfile +myservice.asepharyana.my.id { + import proxy 4022 +} ``` +Commit + push → CI reload caddy → cek `curl -sI https://myservice.asepharyana.my.id`. -## First-time setup +## Firewall (`firewall/`) -Create the shared Docker network before starting any service: +- `firewall.sh` — deny-by-default iptables: + - Public: 22 (SSH), 80/443 (Caddy), 4013 (hermes dashboard), 25565 (Minecraft via TCPShield only) + - Tailscale CGNAT `100.64.0.0/10`: semua port + - Localhost: semua; sisanya DROP + logged. +- `99-hardening.conf`, `99-ssh-optimization.conf` — sysctl drop-ins. -```bash -docker network create app-shared-net -``` +## Prometheus (`prometheus/`) -Create `.env` from `.env.example` and fill production values. Do not commit `.env`. +- `targets.yml` — file_sd targets untuk Prometheus (node-exporter, app /metrics). -## Deployment order +## systemd (`systemd/`) -The GitHub deploy workflow combines the active compose files automatically. For manual deployment, use this order: +- `scraper-otel.conf` — drop-in OTEL exporter untuk `scraper.service`. -```bash -# 1. Shared services -docker compose -f infra/compose/shared.yml up -d +## Deployment Model -# 2. Message bus + Dapr placement -docker compose -f infra/compose/nats.yml up -d -docker compose -f infra/compose/dapr.yml up -d - -# 3. Reverse proxy -docker compose -f infra/compose/traefik.yml up -d - -# 4. Application services (with Dapr sidecars) -docker compose -f infra/compose/scraper.yml up -d -``` - -## Environment variables - -Common variables used by infra compose files: - -```env -DATABASE_URL= -GITHUB_TOKEN= -SHARED_REDIS_EXPOSE=127.0.0.1:6379:6379 -``` - -Traefik certificate path variables are optional because `infra/compose/traefik.yml` provides production-compatible defaults. See `infra/traefik/TRAEFIK_ENV_CONFIG.md` for the full list. - -## Traefik - -Traefik reads dynamic config from `infra/traefik/dynamic/`: - -- `apps.yaml` — routers and upstream services -- `middlewares.yaml` — shared middleware chains -- `ssl.yaml` — TLS certificates for `asepharyana.my.id` and `asepharyana.web.id` - -## Validation - -Run syntax checks after editing infra YAML: - -```bash -python - <<'PY' -import pathlib, yaml -for path in pathlib.Path('infra').rglob('*.yml'): - with path.open() as fh: - yaml.safe_load(fh) - print(f'OK {path}') -for path in pathlib.Path('infra').rglob('*.yaml'): - with path.open() as fh: - yaml.safe_load(fh) - print(f'OK {path}') -PY -``` - -Check compose rendering when Docker is available: - -```bash -for f in infra/compose/*.yml; do - docker compose -f "$f" config >/dev/null && echo "OK $f" -done -``` +Repo `asepharyana/infra` TIDAK build aplikasi. Aplikasi deploy via repo masing-masing +(`hub`, `scraper`, `tools`, `llm-api`) dengan `nix build → nix copy → nix-env --profile → systemctl restart`. +Repo ini hanya mengelola config yang di-sync manual/CI ke VPS. \ No newline at end of file diff --git a/infra/caddy/Caddyfile.prod b/infra/caddy/Caddyfile.prod index 501ee40..a849552 100644 --- a/infra/caddy/Caddyfile.prod +++ b/infra/caddy/Caddyfile.prod @@ -1,5 +1,8 @@ # ── Caddyfile PRODUKSI v2 — TUNED (auto-TLS Let's Encrypt) ── # Tuning: HTTP/3 default, keep-alive upstream, zstd+gzip, timeouts, buffer, TLS 1.3 +# NOTE: file ini adalah source of truth untuk /etc/caddy/Caddyfile di orangevps. +# Changes di-deploy otomatis via .github/workflows/caddy-deploy.yml +# (push ke main → ssh → cp → systemctl reload caddy). { email asepharyana@gmail.com @@ -122,3 +125,34 @@ upload.asepharyana.my.id, upload.asepharyana.web.id { flush_interval -1 } } + +wiki.asepharyana.my.id { + import proxy 4016 + + # Phase 7: observability dashboard (public). + handle /dashboard { + reverse_proxy 127.0.0.1:4020 + } + # Phase 7: Prometheus metrics (public, safe to scrape). + handle /metrics { + reverse_proxy 127.0.0.1:4020 + } + # Phase 4: tRPC API (search, revisions, job status, restore). Routed to the + # API app on :4020 so /trpc/* is reachable on the domain, not swallowed by web. + # Forward the full /trpc path (no prefix strip) to match app.all("/trpc/*"). + handle /trpc/* { + reverse_proxy 127.0.0.1:4020 + } + # Phase 3 git-sync webhook: GitHub POSTs push events here -> BullMQ reindex. + # Forward the full /hooks path (no prefix strip) to match the API's /hooks/reindex route. + handle /hooks/* { + reverse_proxy 127.0.0.1:4020 + } + handle /health { + reverse_proxy 127.0.0.1:4020 + } +} + +mcp.asepharyana.my.id { + import proxy 4021 +} \ No newline at end of file diff --git a/infra/compose/dapr.yml b/infra/compose/dapr.yml deleted file mode 100644 index 0720d04..0000000 --- a/infra/compose/dapr.yml +++ /dev/null @@ -1,13 +0,0 @@ -services: - dapr-placement: - container_name: dapr-placement - image: daprio/dapr:latest - restart: always - networks: - - app-shared-net - command: ["./placement", "--port", "50005"] - -networks: - app-shared-net: - name: app-shared-net - external: true diff --git a/infra/compose/hub.yml.bak b/infra/compose/hub.yml.bak deleted file mode 100644 index 93b4d73..0000000 --- a/infra/compose/hub.yml.bak +++ /dev/null @@ -1,20 +0,0 @@ -services: - hub: - container_name: hub - image: ghcr.io/asepharyana/asepharyana-hub/hub:sha-ac25a51 - restart: always - networks: - app-shared-net: - aliases: - - hub - env_file: - - ../../.env - volumes: - - /var/run/docker.sock:/var/run/docker.sock:ro - group_add: - - '988' - -networks: - app-shared-net: - name: app-shared-net - external: true diff --git a/infra/compose/llm-api.yml.bak b/infra/compose/llm-api.yml.bak deleted file mode 100644 index 7cad8a4..0000000 --- a/infra/compose/llm-api.yml.bak +++ /dev/null @@ -1,36 +0,0 @@ -services: - llm-api: - container_name: llm-api - image: ghcr.io/asepharyana/asepharyana-hub/llm-api:sha-43f0df4 - restart: always - networks: - app-shared-net: - aliases: - - llm-api - env_file: - - ../../.env - environment: - - MODEL_PATH=/models/MiniCPM5-1B-Claude-Opus-Fable5-V2-Thinking-Q8_0.gguf - - API_KEY=${LLM_API_KEY:-} - volumes: - - /root/models/gguf:/models:ro - healthcheck: - test: ['CMD-SHELL', 'curl -so /dev/null --connect-timeout 5 http://localhost:8080/health || test $? -eq 22'] - interval: 30s - timeout: 10s - retries: 5 - start_period: 60s - labels: - prometheus.io/scrape: "true" - prometheus.io/port: "8080" - deploy: - resources: - limits: - memory: 2G - reservations: - memory: 1G - -networks: - app-shared-net: - name: app-shared-net - external: true diff --git a/infra/compose/nats.yml b/infra/compose/nats.yml deleted file mode 100644 index d36c426..0000000 --- a/infra/compose/nats.yml +++ /dev/null @@ -1,25 +0,0 @@ -services: - nats: - container_name: nats - image: nats:latest - restart: always - networks: - app-shared-net: - aliases: - - nats - ports: - - '4222:4222' # client connections - - '8222:8222' # HTTP monitor / health - command: - - '--config=/etc/nats/nats.conf' - volumes: - - nats_data:/data - - ../../infra/nats/nats.conf:/etc/nats/nats.conf:ro - -volumes: - nats_data: - -networks: - app-shared-net: - name: app-shared-net - external: true diff --git a/infra/compose/observability.yml b/infra/compose/observability.yml deleted file mode 100644 index ccd00ce..0000000 --- a/infra/compose/observability.yml +++ /dev/null @@ -1,82 +0,0 @@ -services: - # ── OpenTelemetry Collector ── - otel-collector: - container_name: otel-collector - image: otel/opentelemetry-collector-contrib:latest - restart: always - networks: - app-shared-net: - aliases: - - otel-collector - ports: - - '4317:4317' # OTLP gRPC - - '4318:4318' # OTLP HTTP - - '8889:8889' # Prometheus metrics - command: - - '--config=/etc/otel/config.yml' - volumes: - - ../../infra/otel/otel-collector-config.yml:/etc/otel/config.yml:ro - - # ── Jaeger (Tracing Backend + Built-in UI) ── - jaeger: - container_name: jaeger - image: jaegertracing/all-in-one:latest - restart: always - networks: - app-shared-net: - aliases: - - jaeger - ports: - - '16686:16686' # Jaeger UI + API - environment: - - COLLECTOR_OTLP_ENABLED=true - - COLLECTOR_ZIPKIN_HOST_PORT=:9411 - - METRICS_STORAGE_TYPE=prometheus - - PROMETHEUS_SERVER_URL=http://otel-collector:8889 - - LOG_LEVEL=info - - # ── Prometheus (Metrics Backend) ── - prometheus: - container_name: prometheus - image: prom/prometheus:latest - restart: always - networks: - app-shared-net: - aliases: - - prometheus - ports: - - '9090:9090' - command: - - '--config.file=/etc/prometheus/prometheus.yml' - - '--storage.tsdb.path=/prometheus' - - '--web.console.libraries=/etc/prometheus/console_libraries' - - '--web.console.templates=/etc/prometheus/consoles' - - '--web.enable-lifecycle' - volumes: - - ../../infra/otel/prometheus.yml:/etc/prometheus/prometheus.yml:ro - - /prometheus - - /var/run/docker.sock:/var/run/docker.sock:ro - group_add: - - '988' - - # ── Node Exporter (Host Metrics: CPU, RAM, Disk) ── - node-exporter: - container_name: node-exporter - image: prom/node-exporter:latest - restart: always - networks: - app-shared-net: - aliases: - - node-exporter - command: - - '--web.listen-address=0.0.0.0:9100' - - '--path.rootfs=/host' - - '--path.procfs=/host/proc' - - '--path.sysfs=/host/sys' - volumes: - - /:/host:ro,rslave - -networks: - app-shared-net: - name: app-shared-net - external: true diff --git a/infra/compose/scraper.yml.bak b/infra/compose/scraper.yml.bak deleted file mode 100644 index 82bb727..0000000 --- a/infra/compose/scraper.yml.bak +++ /dev/null @@ -1,57 +0,0 @@ -services: - scraper-api: - container_name: scraper-api - image: ghcr.io/asepharyana/asepharyana-hub/scraper-api:sha-2459541 - restart: always - depends_on: - nats: - condition: service_started - dapr-placement: - condition: service_started - networks: - app-shared-net: - aliases: - - scraper-api - env_file: - - ../../.env - healthcheck: - test: ['CMD-SHELL', 'curl -so /dev/null --connect-timeout 5 http://localhost:4091/ || test $? -eq 22'] - interval: 30s - timeout: 10s - retries: 5 - start_period: 30s - environment: - - REDIS_URL=redis://redis:6379 - - JWT_SECRET=${JWT_SECRET:?JWT_SECRET is required} - - DATABASE_URL=${DATABASE_URL} - - - scraper-api-dapr: - container_name: scraper-api-dapr - image: daprio/daprd:latest - restart: always - depends_on: - nats: - condition: service_started - dapr-placement: - condition: service_started - otel-collector: - condition: service_started - networks: - - app-shared-net - command: - - './daprd' - - '--app-id=scraper-api' - - '--app-port=4091' - - '--dapr-http-port=3500' - - '--dapr-grpc-port=50001' - - '--placement-host-address=dapr-placement:50005' - - '--config=/dapr/config.yaml' - - '--resources-path=/dapr/components' - volumes: - - ../../infra/dapr:/dapr:ro - -networks: - app-shared-net: - name: app-shared-net - external: true diff --git a/infra/compose/shared.yml b/infra/compose/shared.yml deleted file mode 100644 index 9a5f708..0000000 --- a/infra/compose/shared.yml +++ /dev/null @@ -1,27 +0,0 @@ -services: - redis: - container_name: redis - image: 'redis:alpine' - restart: always - networks: - app-shared-net: - aliases: - - redis - ports: - - '${SHARED_REDIS_EXPOSE:-127.0.0.1:6379:6379}' - healthcheck: - test: ['CMD', 'redis-cli', 'ping'] - interval: 10s - timeout: 3s - retries: 3 - start_period: 5s - volumes: - - 'redis_data:/data' - -networks: - app-shared-net: - name: app-shared-net - external: true - -volumes: - redis_data: null diff --git a/infra/compose/tools.yml.bak b/infra/compose/tools.yml.bak deleted file mode 100644 index 4738b97..0000000 --- a/infra/compose/tools.yml.bak +++ /dev/null @@ -1,40 +0,0 @@ -services: - tools: - container_name: tools - image: ghcr.io/asepharyana/asepharyana-hub/tools:sha-ac25a51 - restart: always - networks: - app-shared-net: - aliases: - - tools - env_file: - - ../../.env - environment: - - REDIS_URL=redis://redis:6379 - - NATS_URL=nats://nats:4222 - - STORAGE_PATH=/data/tools - - GATEWAY_PORT=3001 - - TOOLS_WORKER_CONCURRENCY=4 - - RUST_LOG=info - volumes: - - tools_data:/data/tools - labels: - - 'prometheus.io/scrape=true' - - 'prometheus.io/port=3001' - - 'prometheus.io/path=/metrics' - ports: - - "3002:3001" - healthcheck: - test: ['CMD-SHELL', 'wget -q -O /dev/null http://localhost:3001/health || exit 1'] - interval: 30s - timeout: 10s - retries: 5 - start_period: 30s - -volumes: - tools_data: - -networks: - app-shared-net: - name: app-shared-net - external: true \ No newline at end of file diff --git a/infra/compose/traefik.yml b/infra/compose/traefik.yml deleted file mode 100644 index 0f80a75..0000000 --- a/infra/compose/traefik.yml +++ /dev/null @@ -1,97 +0,0 @@ -services: - traefik: - container_name: traefik - image: traefik:v3.6 - restart: always - sysctls: - - net.core.somaxconn=65535 - - net.ipv4.ip_local_port_range=1024 65535 - ulimits: - nofile: - soft: 1048576 - hard: 1048576 - ports: - - '80:80' - - '443:443' - - '443:443/udp' - networks: - - app-shared-net - extra_hosts: - - 'host.docker.internal:10.0.1.1' - command: - - '--api.dashboard=true' - - '--api.insecure=false' - - '--providers.docker=true' - - '--providers.docker.endpoint=unix:///var/run/docker.sock' - - '--providers.docker.exposedByDefault=false' - - '--providers.docker.network=app-shared-net' - - '--providers.docker.watch=true' - - '--providers.file.directory=/etc/traefik/dynamic' - - '--providers.file.watch=true' - - '--entryPoints.web.address=:80' - - '--entryPoints.web.http.redirections.entryPoint.to=websecure' - - '--entryPoints.web.http.redirections.entryPoint.scheme=https' - - '--accesslog=true' - - '--accesslog.bufferingsize=100' - - '--log.level=INFO' - - '--log.format=json' - - '--entryPoints.web.transport.respondingTimeouts.readTimeout=0' - - '--entryPoints.web.transport.respondingTimeouts.writeTimeout=0' - - '--entryPoints.web.transport.respondingTimeouts.idleTimeout=0' - - '--entryPoints.web.transport.lifeCycle.requestAcceptGraceTimeout=15s' - - '--entryPoints.web.transport.lifeCycle.graceTimeOut=10s' - - '--entryPoints.websecure.transport.respondingTimeouts.readTimeout=0' - - '--entryPoints.websecure.transport.respondingTimeouts.writeTimeout=0' - - '--entryPoints.websecure.transport.respondingTimeouts.idleTimeout=0' - - '--entryPoints.websecure.transport.lifeCycle.requestAcceptGraceTimeout=15s' - - '--entryPoints.websecure.transport.lifeCycle.graceTimeOut=10s' - - '--entryPoints.websecure.address=:443' - - '--entryPoints.websecure.http3=true' - # ── Response speed tuning ── - - '--serversTransport.maxIdleConnsPerHost=100' - - '--serversTransport.forwardingTimeouts.dialTimeout=3s' - - '--serversTransport.forwardingTimeouts.idleConnTimeout=180s' - - '--global.checkNewVersion=false' - - '--global.sendAnonymousUsage=false' - - '--experimental.plugins.real-ip.moduleName=github.com/soulbalz/traefik-real-ip' - - '--experimental.plugins.real-ip.version=v1.0.3' - - '--experimental.plugins.blockpath.moduleName=github.com/traefik/plugin-blockpath' - - '--experimental.plugins.blockpath.version=v0.2.1' - - '--ping=true' - - '--metrics.prometheus=true' - - '--metrics.prometheus.addEntryPointsLabels=true' - - '--metrics.prometheus.addServicesLabels=true' - - '--tracing.otlp=true' - - '--tracing.otlp.http=true' - - '--tracing.otlp.http.endpoint=http://otel-collector:4318' - healthcheck: - test: ['CMD', 'wget', '--spider', 'http://localhost:8080/ping'] - interval: 30s - timeout: 5s - retries: 3 - start_period: 15s - environment: - - DOCKER_API_VERSION=1.41 - - GOMEMLIMIT=4096MiB - - GOGC=200 - volumes: - - /var/run/docker.sock:/var/run/docker.sock:ro - - ${TRAEFIK_CONFIG_PATH:-/home/code/asepharyana-hub/infra/traefik/dynamic}:/etc/traefik/dynamic:ro - - ${TRAEFIK_CERT_MY_ID_PEM:-/root/asepharyana.my.id.pem}:/etc/traefik/certs/asepharyana.my.id.pem:ro - - ${TRAEFIK_CERT_MY_ID_KEY:-/root/asepharyana.my.id.key}:/etc/traefik/certs/asepharyana.my.id.key:ro - - ${TRAEFIK_CERT_WEB_ID_PEM:-/root/asepharyana.web.id.pem}:/etc/traefik/certs/asepharyana.web.id.pem:ro - - ${TRAEFIK_CERT_WEB_ID_KEY:-/root/asepharyana.web.id.key}:/etc/traefik/certs/asepharyana.web.id.key:ro - labels: - - 'traefik.enable=true' - - 'traefik.http.routers.traefik.rule=Host(`traefik.asepharyana.my.id`) || Host(`traefik.asepharyana.web.id`)' - - 'traefik.http.routers.traefik.service=api@internal' - - 'traefik.http.routers.traefik.entrypoints=websecure' - - 'traefik.http.routers.traefik.tls=true' - - 'traefik.http.routers.traefik.middlewares=admin-chain@file' - - 'prometheus.io/scrape=true' - - 'prometheus.io/port=8080' - -networks: - app-shared-net: - name: app-shared-net - external: true diff --git a/infra/dapr/components/pubsub.yaml b/infra/dapr/components/pubsub.yaml deleted file mode 100644 index 669a614..0000000 --- a/infra/dapr/components/pubsub.yaml +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: dapr.io/v1alpha1 -kind: Component -metadata: - name: pubsub -spec: - type: pubsub.redis - version: v1 - metadata: - - name: redisHost - value: redis:6379 - - name: redisPassword - value: "" diff --git a/infra/dapr/components/statestore.yaml b/infra/dapr/components/statestore.yaml deleted file mode 100644 index 7445ccd..0000000 --- a/infra/dapr/components/statestore.yaml +++ /dev/null @@ -1,14 +0,0 @@ -apiVersion: dapr.io/v1alpha1 -kind: Component -metadata: - name: statestore -spec: - type: state.redis - version: v1 - metadata: - - name: redisHost - value: redis:6379 - - name: redisPassword - value: "" - - name: keyPrefix - value: "dapr" diff --git a/infra/dapr/config.yaml b/infra/dapr/config.yaml deleted file mode 100644 index 92702b6..0000000 --- a/infra/dapr/config.yaml +++ /dev/null @@ -1,16 +0,0 @@ -apiVersion: dapr.io/v1alpha1 -kind: Configuration -metadata: - name: dapr-config -spec: - tracing: - samplingRate: "1" - stdout: false - otel: - endpointAddress: "otel-collector:4317" - isSecure: false - protocol: grpc - metrics: - enabled: true - mtls: - enabled: false diff --git a/infra/docker/hub.Dockerfile b/infra/docker/hub.Dockerfile deleted file mode 100644 index 54c8862..0000000 --- a/infra/docker/hub.Dockerfile +++ /dev/null @@ -1,21 +0,0 @@ -# ── Build stage ── -FROM oven/bun:1.2 AS builder -WORKDIR /app -COPY apps/hub/package.json apps/hub/bun.lock ./ -RUN bun install --frozen-lockfile -COPY apps/hub . -RUN bun run build - -# ── Runtime ── -FROM oven/bun:1.2 AS runtime -RUN addgroup --system appgroup && adduser --system appuser --ingroup appgroup -WORKDIR /app -COPY --from=builder /app/.next ./.next -COPY --from=builder /app/node_modules ./node_modules -COPY --from=builder /app/public ./public -COPY --from=builder /app/package.json ./ -RUN rm -rf .next/cache && chown -R appuser:appgroup .next -USER appuser -EXPOSE 3000 -ENV PORT=3000 NODE_ENV=production -CMD ["bun", "run", "start"] diff --git a/infra/docker/llm-api.Dockerfile b/infra/docker/llm-api.Dockerfile deleted file mode 100644 index d24bf5c..0000000 --- a/infra/docker/llm-api.Dockerfile +++ /dev/null @@ -1,39 +0,0 @@ -# ── Build stage: cargo-chef ── -FROM lukemathwalker/cargo-chef:latest-rust-1.89.0 AS chef -RUN apt-get update && apt-get install -y --no-install-recommends libclang-dev cmake && rm -rf /var/lib/apt/lists/* -WORKDIR /app - -FROM chef AS planner -COPY apps/llm-api . -RUN cargo chef prepare --recipe-path recipe.json - -FROM chef AS builder -COPY --from=planner /app/recipe.json recipe.json -RUN --mount=type=cache,target=/usr/local/cargo/registry \ - --mount=type=cache,target=/app/target \ - cargo chef cook --release --recipe-path recipe.json - -COPY apps/llm-api . -RUN --mount=type=cache,target=/usr/local/cargo/registry \ - --mount=type=cache,target=/app/target \ - cargo build --release && \ - cp target/release/llm-api /app/llm-api - -# ── Runtime ── -FROM debian:bookworm-slim AS runtime -RUN apt-get update && apt-get install -y --no-install-recommends \ - ca-certificates \ - curl \ - libssl3 \ - libgomp1 \ - && rm -rf /var/lib/apt/lists/* - -RUN groupadd -g 1001 appgroup && \ - useradd -u 1001 -g appgroup -s /bin/sh appuser - -WORKDIR /app -COPY --from=builder /app/llm-api /app/llm-api -USER appuser - -EXPOSE 8080 -CMD ["./llm-api"] diff --git a/infra/docker/scraper.Dockerfile b/infra/docker/scraper.Dockerfile deleted file mode 100644 index eddbc3d..0000000 --- a/infra/docker/scraper.Dockerfile +++ /dev/null @@ -1,37 +0,0 @@ -# ── Build stage: cargo-chef for dependency caching ── -FROM lukemathwalker/cargo-chef:latest-rust-1.89.0 AS chef -WORKDIR /app - -FROM chef AS planner -COPY apps/scraper . -RUN cargo chef prepare --recipe-path recipe.json - -FROM chef AS builder -COPY --from=planner /app/recipe.json recipe.json -RUN --mount=type=cache,target=/usr/local/cargo/registry \ - --mount=type=cache,target=/app/target \ - cargo chef cook --release --recipe-path recipe.json - -COPY apps/scraper . -RUN --mount=type=cache,target=/usr/local/cargo/registry \ - --mount=type=cache,target=/app/target \ - cargo build --release && \ - cp target/release/scraper /app/scraper - -# ── Runtime image ── -FROM debian:bookworm-slim AS runtime -RUN apt-get update && apt-get install -y --no-install-recommends \ - ca-certificates \ - curl \ - libssl3 \ - && rm -rf /var/lib/apt/lists/* - -RUN groupadd -g 1001 appgroup && \ - useradd -u 1001 -g appgroup -s /bin/sh appuser - -WORKDIR /app -COPY --from=builder /app/scraper /app/scraper -USER appuser - -EXPOSE 4091 -CMD ["./scraper"] diff --git a/infra/docker/tools.Dockerfile b/infra/docker/tools.Dockerfile deleted file mode 100644 index 7587188..0000000 --- a/infra/docker/tools.Dockerfile +++ /dev/null @@ -1,93 +0,0 @@ -# ============================================================ -# Stage 1: Build Rust Backend (with cargo-chef caching) -# ============================================================ -FROM lukemathwalker/cargo-chef:latest-rust-1.89.0 AS chef -RUN apt-get update && apt-get install -y --no-install-recommends \ - libleptonica-dev libtesseract-dev clang pkg-config \ - && rm -rf /var/lib/apt/lists/* -WORKDIR /app - -FROM chef AS planner -COPY apps/tools/backend/ . -RUN cargo chef prepare --recipe-path recipe.json - -FROM chef AS builder -RUN apt-get update && apt-get install -y --no-install-recommends \ - libleptonica-dev libtesseract-dev clang pkg-config \ - && rm -rf /var/lib/apt/lists/* - -COPY --from=planner /app/recipe.json recipe.json -RUN --mount=type=cache,target=/usr/local/cargo/registry \ - --mount=type=cache,target=/app/target \ - cargo chef cook --release --recipe-path recipe.json - -COPY apps/tools/backend/ . -RUN --mount=type=cache,target=/usr/local/cargo/registry \ - --mount=type=cache,target=/app/target \ - cargo build --release --features tesseract --bin tools-gateway --bin tools-workers && \ - cp /app/target/release/tools-gateway /app/tools-gateway-bin && \ - cp /app/target/release/tools-workers /app/tools-workers-bin - -# ============================================================ -# Stage 2: Build Next.js Frontend -# ============================================================ -FROM oven/bun:1.3 AS frontend-builder -WORKDIR /app - -# Copy package files first for layer caching -COPY apps/tools/frontend/package.json apps/tools/frontend/bun.lock ./ -RUN bun install --frozen-lockfile - -COPY apps/tools/frontend/ . -RUN bun run build - -# ============================================================ -# Stage 3: Production Runtime -# ============================================================ -FROM debian:bookworm-slim AS runtime - -# Install runtime dependencies -RUN apt-get update && apt-get install -y --no-install-recommends \ - tesseract-ocr \ - tesseract-ocr-eng \ - tesseract-ocr-ind \ - tesseract-ocr-osd \ - ca-certificates \ - fonts-dejavu-core \ - wget \ - && rm -rf /var/lib/apt/lists/* - -WORKDIR /app - -# Copy Rust binaries (cp'd from cache mount in builder stage) -COPY --from=builder /app/tools-gateway-bin /app/gateway -COPY --from=builder /app/tools-workers-bin /app/workers - -# Copy bun binary from frontend builder (needed to run Next.js server) -COPY --from=frontend-builder /usr/local/bin/bun /usr/local/bin/bun - -# Copy Next.js build -COPY --from=frontend-builder /app/.next /app/.next -COPY --from=frontend-builder /app/public /app/public -COPY --from=frontend-builder /app/package.json /app/package.json -COPY --from=frontend-builder /app/node_modules /app/node_modules -COPY --from=frontend-builder /app/next.config.ts /app/next.config.ts - -# Create temp storage directory -RUN mkdir -p /data/tools && chmod 1777 /data/tools - -# Copy entrypoint -COPY apps/tools/scripts/entrypoint.sh /app/entrypoint.sh -RUN chmod +x /app/entrypoint.sh - -# Environment -ENV TESSDATA_PREFIX=/usr/share/tesseract-ocr/5/tessdata -ENV STORAGE_PATH=/data/tools -ENV GATEWAY_PORT=3001 -ENV TOOLS_WORKER_CONCURRENCY=4 -ENV RUST_LOG=info - -# Expose port -EXPOSE 3001 - -CMD ["/app/entrypoint.sh"] \ No newline at end of file diff --git a/infra/nats/nats.conf b/infra/nats/nats.conf deleted file mode 100644 index 6e6e6ec..0000000 --- a/infra/nats/nats.conf +++ /dev/null @@ -1,11 +0,0 @@ -# ── NATS Server Configuration ── - -# JetStream -jetstream: true -store_dir: "/data" - -# HTTP monitoring -http_port: 8222 - -# Limits -max_payload: 1MB diff --git a/infra/otel/otel-collector-config.yml b/infra/otel/otel-collector-config.yml deleted file mode 100644 index e5d257c..0000000 --- a/infra/otel/otel-collector-config.yml +++ /dev/null @@ -1,41 +0,0 @@ -receivers: - otlp: - protocols: - grpc: - endpoint: 0.0.0.0:4317 - http: - endpoint: 0.0.0.0:4318 -exporters: - nop: - prometheus: - endpoint: 0.0.0.0:8889 - enable_open_metrics: true - resource_to_telemetry_conversion: - enabled: true -processors: - batch: - timeout: 1s - send_batch_size: 1024 - memory_limiter: - check_interval: 1s - limit_mib: 512 - spike_limit_mib: 128 - attributes: - actions: - - key: service.namespace - value: asepharyana-hub - action: upsert -extensions: - health_check: - endpoint: 0.0.0.0:13133 -service: - extensions: [health_check] - pipelines: - traces: - receivers: [otlp] - processors: [memory_limiter, batch, attributes] - exporters: [nop] - metrics: - receivers: [otlp] - processors: [memory_limiter, batch, attributes] - exporters: [prometheus] diff --git a/infra/otel/prometheus.yml b/infra/otel/prometheus.yml deleted file mode 100644 index 7942b40..0000000 --- a/infra/otel/prometheus.yml +++ /dev/null @@ -1,45 +0,0 @@ -global: - scrape_interval: 15s - evaluation_interval: 15s - -scrape_configs: - # ── Docker SD: containers with prometheus.io/scrape=true are auto-discovered ── - - job_name: 'docker' - docker_sd_configs: - - host: unix:///var/run/docker.sock - refresh_interval: 15s - filters: - - name: label - values: - - prometheus.io/scrape=true - relabel_configs: - # Address: use container name + label port (Docker DNS resolves names) - - source_labels: [__meta_docker_container_name, __meta_docker_container_label_prometheus_io_port] - regex: '/(.*);(\d+)' - replacement: '${1}:${2}' - target_label: __address__ - # Metrics path from label (default /metrics) - - source_labels: [__meta_docker_container_label_prometheus_io_path] - regex: '(.+)' - target_label: __metrics_path__ - replacement: '${1}' - # Container name label - - source_labels: [__meta_docker_container_name] - regex: '/(.*)' - replacement: '${1}' - target_label: container - - # ── OTel Collector ── - - job_name: 'otel-collector' - scrape_interval: 10s - metrics_path: /metrics - static_configs: - - targets: ['otel-collector:8889'] - labels: - service: otel-collector - - # ── Node Exporter ── - - job_name: 'node' - scrape_interval: 15s - static_configs: - - targets: ['node-exporter:9100'] diff --git a/infra/traefik/TRAEFIK_ENV_CONFIG.md b/infra/traefik/TRAEFIK_ENV_CONFIG.md deleted file mode 100644 index 9db09f1..0000000 --- a/infra/traefik/TRAEFIK_ENV_CONFIG.md +++ /dev/null @@ -1,61 +0,0 @@ -# Traefik Environment Configuration - -This document describes environment variables used to configure Traefik certificate and config paths in production deployments. - -## Certificate Path Environment Variables - -All certificate paths support environment variable substitution with safe fallback defaults. This allows flexible certificate management across different deployment environments without modifying compose files. - -### Configuration Variables - -| Variable | Description | Default Path | Purpose | -| --------------------------- | --------------------------------------------------------------- | ----------------------------------------------- | ------------------------------------------------------- | -| `TRAEFIK_CONFIG_PATH` | Directory containing dynamic Traefik configuration files (YAML) | `/root/asepharyana-hub/infra/traefik/dynamic` | Location of middleware, router, and service definitions | -| `TRAEFIK_CERT_MY_ID_PEM` | Certificate file for asepharyana.my.id | `/root/asepharyana.my.id.pem` | SSL/TLS certificate for asepharyana.my.id domain | -| `TRAEFIK_CERT_MY_ID_KEY` | Key file for asepharyana.my.id | `/root/asepharyana.my.id.key` | SSL/TLS private key for asepharyana.my.id domain | -| `TRAEFIK_CERT_WEB_ID_PEM` | Certificate file for asepharyana.web.id | `/root/asepharyana.web.id.pem` | SSL/TLS certificate for asepharyana.web.id domain | -| `TRAEFIK_CERT_WEB_ID_KEY` | Key file for asepharyana.web.id | `/root/asepharyana.web.id.key` | SSL/TLS private key for asepharyana.web.id domain | - -## Usage - -### Default Behavior (Production) - -If no environment variables are set, Traefik will use the default paths shown above. This is suitable for production deployments where certificates are installed at these standard locations. - -```bash -docker compose -f infra/compose/traefik.yml up -d -``` - -### Custom Paths (Custom Deployments) - -To override paths for a custom deployment, set environment variables before starting services: - -```bash -export TRAEFIK_CONFIG_PATH=/etc/traefik/custom-dynamic - -docker compose -f infra/compose/traefik.yml up -d -``` - -### Via .env File - -Create or update your `.env` file in the deployment directory: - -```env -TRAEFIK_CONFIG_PATH=/root/asepharyana-hub/infra/traefik/dynamic -TRAEFIK_CERT_MY_ID_PEM=/root/asepharyana.my.id.pem -TRAEFIK_CERT_MY_ID_KEY=/root/asepharyana.my.id.key -TRAEFIK_CERT_WEB_ID_PEM=/root/asepharyana.web.id.pem -TRAEFIK_CERT_WEB_ID_KEY=/root/asepharyana.web.id.key -``` - -Then deploy: - -```bash -docker compose --env-file .env -f infra/compose/traefik.yml up -d -``` - -## Notes - -- All certificate paths use read-only mounts (`:ro`) for security -- If a certificate file is missing at the specified path, Docker volume mounting will fail—ensure certificates exist before starting Traefik -- The dynamic configuration directory must contain valid YAML files for Traefik to load properly diff --git a/infra/traefik/dynamic/apps.yaml b/infra/traefik/dynamic/apps.yaml deleted file mode 100644 index 9181053..0000000 --- a/infra/traefik/dynamic/apps.yaml +++ /dev/null @@ -1,227 +0,0 @@ -http: - routers: - # ── Hub (Portfolio SPA) ── - hub: - rule: 'Host(`asepharyana.my.id`) || Host(`www.asepharyana.my.id`) || Host(`asepharyana.web.id`) || Host(`www.asepharyana.web.id`)' - entryPoints: - - websecure - tls: {} - middlewares: - - common-chain@file - service: hub-service - - # ── Scraper API ── - scraper: - rule: 'Host(`scraper.asepharyana.my.id`) || Host(`api.asepharyana.my.id`) || Host(`scraper.asepharyana.web.id`) || Host(`api.asepharyana.web.id`)' - entryPoints: - - websecure - tls: {} - middlewares: - - common-chain@file - service: scraper-service - - # ── LLM API (MiniCPM-V) ── - llm-api: - rule: 'Host(`ai.asepharyana.my.id`) || Host(`ai.asepharyana.web.id`)' - entryPoints: - - websecure - tls: {} - middlewares: - - llm-chain@file - service: llm-api-service - - # ── Tools (Document Scanner & Media Processing) ── - tools: - rule: 'Host(`tools.asepharyana.my.id`) || Host(`tools.asepharyana.web.id`)' - entryPoints: - - websecure - tls: {} - middlewares: - - common-chain@file - service: tools-service - - # ── Jaeger UI ── - jaeger: - rule: 'Host(`jaeger.asepharyana.my.id`) || Host(`jaeger.asepharyana.web.id`)' - entryPoints: - - websecure - tls: {} - middlewares: - - common-chain@file - service: jaeger-service - - # ── 9Router (AI routing gateway) ── - 9router: - rule: 'Host(`9router.asepharyana.my.id`)' - entryPoints: - - websecure - tls: {} - middlewares: - - common-chain@file - service: 9router-service - - # ── PR-Agent (GitHub App webhook) ── - pr-agent: - rule: 'Host(`pr-agent.asepharyana.my.id`)' - entryPoints: - - websecure - tls: {} - middlewares: - - common-chain@file - service: pr-agent-service - - # ── LIDM Frontend ── - lidm-frontend: - rule: 'Host(`lidm.asepharyana.my.id`)' - entryPoints: - - websecure - tls: {} - middlewares: - - common-chain@file - service: lidm-frontend-service - - # ── LIDM Backend API ── - lidm-backend: - rule: 'Host(`lidm-api.asepharyana.my.id`)' - entryPoints: - - websecure - tls: {} - middlewares: - - common-chain@file - service: lidm-backend-service - - # ── ZeaVis Edu Frontend ── - zeavisedu: - rule: 'Host(`zeavisedu.asepharyana.my.id`)' - entryPoints: - - websecure - tls: {} - middlewares: - - common-chain@file - service: zeavisedu-service - - # ── ZeaVis Edu API ── - api-zeavisedu: - rule: 'Host(`api-zeavisedu.asepharyana.my.id`)' - entryPoints: - - websecure - tls: {} - middlewares: - - common-chain@file - service: api-zeavisedu-service - - # ── ZeaVis Edu ML Service ── - ml-zeavisedu: - rule: 'Host(`ml-zeavisedu.asepharyana.my.id`)' - entryPoints: - - websecure - tls: {} - middlewares: - - common-chain@file - service: ml-zeavisedu-service - - # ── Hermes Dashboard ── - hermes-dashboard: - rule: 'Host(`dashboard.asepharyana.my.id`)' - entryPoints: - - websecure - tls: {} - middlewares: - - common-chain@file - service: hermes-dashboard-service - - # ── GMW Discord Automod Dashboard (Nix: gmw-proxy on 8080) ── - gmw: - rule: 'Host(`imphnen.asepharyana.my.id`)' - entryPoints: - - websecure - tls: {} - middlewares: - - common-chain@file - service: gmw-service - - # ── TeleUploader (S3 to Telegram Bridge, Nix: bun on 3000) ── - teleuploader: - rule: 'Host(`upload.asepharyana.my.id`) || Host(`upload.asepharyana.web.id`)' - entryPoints: - - websecure - tls: {} - middlewares: - - upload-chain@file - service: teleuploader-service - - services: - hub-service: - loadBalancer: - servers: - - url: 'http://host.docker.internal:3099' - - scraper-service: - loadBalancer: - servers: - - url: 'http://host.docker.internal:4091' - - tools-service: - loadBalancer: - servers: - - url: 'http://host.docker.internal:3500' - - llm-api-service: - loadBalancer: - servers: - - url: 'http://host.docker.internal:8082' - - jaeger-service: - loadBalancer: - servers: - - url: 'http://jaeger:16686' - - 9router-service: - loadBalancer: - servers: - - url: 'http://host.docker.internal:20128' - - pr-agent-service: - loadBalancer: - servers: - - url: 'http://host.docker.internal:3002' - - lidm-frontend-service: - loadBalancer: - servers: - - url: 'http://host.docker.internal:3100' - - lidm-backend-service: - loadBalancer: - servers: - - url: 'http://host.docker.internal:3101' - - zeavisedu-service: - loadBalancer: - servers: - - url: 'http://host.docker.internal:8088' - - api-zeavisedu-service: - loadBalancer: - servers: - - url: 'http://host.docker.internal:3200' - - ml-zeavisedu-service: - loadBalancer: - servers: - - url: 'http://host.docker.internal:8200' - - hermes-dashboard-service: - loadBalancer: - servers: - - url: 'http://host.docker.internal:9119' - - gmw-service: - loadBalancer: - servers: - - url: 'http://host.docker.internal:8080' - - teleuploader-service: - loadBalancer: - servers: - - url: 'http://host.docker.internal:3000' diff --git a/infra/traefik/dynamic/middlewares.yaml b/infra/traefik/dynamic/middlewares.yaml deleted file mode 100644 index 929d697..0000000 --- a/infra/traefik/dynamic/middlewares.yaml +++ /dev/null @@ -1,100 +0,0 @@ -http: - middlewares: - secure-headers: - headers: - sslRedirect: true - forceSTSHeader: true - stsSeconds: 31536000 - stsIncludeSubdomains: true - stsPreload: true - frameDeny: true - contentTypeNosniff: true - browserXSSFilter: true - referrerPolicy: 'same-origin' - customResponseHeaders: - X-Content-Type-Options: 'nosniff' - X-Frame-Options: 'DENY' - X-XSS-Protection: '1; mode=block' - Referrer-Policy: 'same-origin' - Permissions-Policy: 'geolocation=(), microphone=(), camera=()' - compress: - compress: - minResponseBodyBytes: 256 - excludedContentTypes: - - 'image/*' - - 'application/octet-stream' - retry: - retry: - attempts: 3 - rate-limit: - rateLimit: - average: 100 - burst: 50 - buffer: - buffering: - maxRequestBodyBytes: 10485760 - maxResponseBodyBytes: 10485760 - memRequestBodyBytes: 1048576 - memResponseBodyBytes: 1048576 - admin-chain: - chain: - middlewares: - - secure-headers - - compress - - retry - - # ── Useful Plugins ── - real-ip: - plugin: - real-ip: - excludednetworks: - - '127.0.0.1/32' - realipheader: 'CF-Connecting-IP' - - block-sensitive-paths: - plugin: - blockpath: - regex: - - "^/\\.env" - - "^/\\.git" - - '^/wp-admin' - - "^/wp-login\\.php" - - "^/config\\.php" - - # ── LLM Stream Chain (no buffer/compress — SSE needs real-time) ── - llm-chain: - chain: - middlewares: - - secure-headers - - retry - - rate-limit - - # ── Common Chain ── - common-chain: - chain: - middlewares: - - secure-headers - - compress - - retry - - rate-limit - - buffer - - # ── TeleUploader Chain (2GB body buffer — large file uploads) ── - upload-buffer: - buffering: - maxRequestBodyBytes: 2147483648 - maxResponseBodyBytes: 2147483648 - memRequestBodyBytes: 1048576 - memResponseBodyBytes: 1048576 - upload-rate-limit: - rateLimit: - average: 300 - burst: 100 - period: 1m - upload-chain: - chain: - middlewares: - - secure-headers - - retry - - upload-rate-limit - - upload-buffer diff --git a/infra/traefik/dynamic/ssl.yaml b/infra/traefik/dynamic/ssl.yaml deleted file mode 100644 index c91e9e1..0000000 --- a/infra/traefik/dynamic/ssl.yaml +++ /dev/null @@ -1,11 +0,0 @@ -tls: - certificates: - - certFile: /etc/traefik/certs/asepharyana.my.id.pem - keyFile: /etc/traefik/certs/asepharyana.my.id.key - - certFile: /etc/traefik/certs/asepharyana.web.id.pem - keyFile: /etc/traefik/certs/asepharyana.web.id.key - stores: - default: - defaultCertificate: - certFile: /etc/traefik/certs/asepharyana.my.id.pem - keyFile: /etc/traefik/certs/asepharyana.my.id.key diff --git a/kilo.json b/kilo.json deleted file mode 100644 index e1f7ae8..0000000 --- a/kilo.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "$schema": "https://app.kilo.ai/config.json", - "instructions": ["CLAUDE.md"], - "snapshot": true -} diff --git a/package.json b/package.json deleted file mode 100644 index 68046c4..0000000 --- a/package.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "name": "asepharyana-hub", - "private": true, - "scripts": { - "lint": "biome lint .", - "format": "biome format --write .", - "check": "biome check --write .", - "ci": "biome ci .", - "test": "echo \"Error: no test specified\" && exit 1" - }, - "devDependencies": { - "@biomejs/biome": "2.5.3" - } -} diff --git a/plugins/hub-guide b/plugins/hub-guide deleted file mode 160000 index ffcd3ba..0000000 --- a/plugins/hub-guide +++ /dev/null @@ -1 +0,0 @@ -Subproject commit ffcd3ba8e31b7b50c12825ac66433c4268f55916 diff --git a/scripts/2updateenv.sh b/scripts/2updateenv.sh deleted file mode 100644 index a42dff7..0000000 --- a/scripts/2updateenv.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash -# Copy .env from root to all direct subprojects/packages (not nested, not build, not .git, not .github, not node_modules, not .turbo, not .next, not .vscode, not dist, not public, not src, not coverage, not logs, not .devcontainer, not .yarn, not target) -# Do NOT copy to /apps or /packages root, only to their subfolders. - -ROOT_ENV="./.env" - -if [ ! -f "$ROOT_ENV" ]; then - echo "Root .env file not found at $ROOT_ENV" - exit 1 -fi - -for parent in apps; do - for dir in ./$parent/*/; do - # Remove trailing slash - dir="${dir%/}" - base=$(basename "$dir") - if [[ "$base" =~ ^(\.git|\.github|node_modules|\.turbo|\.next|\.vscode|dist|public|src|coverage|logs|\.devcontainer|\.yarn|target)$ ]]; then - continue - fi - cp "$ROOT_ENV" "$dir/.env" - echo "Copied .env to $dir/.env" - done -done diff --git a/scripts/clean-node-modules.ps1 b/scripts/clean-node-modules.ps1 deleted file mode 100644 index 1489473..0000000 --- a/scripts/clean-node-modules.ps1 +++ /dev/null @@ -1,169 +0,0 @@ -<# -.SYNOPSIS - Clean all node_modules directories in the repository with optional cache and lockfile cleanup. - -.DESCRIPTION - This script recursively finds and removes all 'node_modules' directories starting at the repo root. - Optionally, it can also remove build caches and lockfiles, and run 'pnpm store prune'. - -.PARAMETER IncludeCache - Also remove common cache/build output directories (e.g., .next, .turbo, .vite, node_modules/.cache, dist, build, coverage, out, storybook-static). - -.PARAMETER IncludeLock - Also remove lock files (pnpm-lock.yaml, package-lock.json, yarn.lock) in the repo. - -.PARAMETER PruneStore - After deletion, try to run 'pnpm store prune' if pnpm is installed. - -.PARAMETER Yes - Proceed without interactive confirmation (non-interactive mode). - -.PARAMETER DryRun - Show what would be removed without deleting anything. - -.EXAMPLE - # Preview what will be removed - ./scripts/clean-node-modules.ps1 -DryRun - -.EXAMPLE - # Clean node_modules only, no prompt - ./scripts/clean-node-modules.ps1 -Yes - -.EXAMPLE - # Deep clean including caches and lockfiles, and prune pnpm store - ./scripts/clean-node-modules.ps1 -IncludeCache -IncludeLock -PruneStore -Yes -#> - -[CmdletBinding(SupportsShouldProcess=$true)] -param( - [switch] $IncludeCache, - [switch] $IncludeLock, - [switch] $PruneStore, - [switch] $Yes, - [switch] $DryRun -) - -$ErrorActionPreference = 'Stop' - -function Write-Section($text) { - Write-Host "`n==== $text ====\n" -ForegroundColor Cyan -} - -function Safe-RemoveDirectory { - param( - [Parameter(Mandatory=$true)][string] $Path, - [switch] $Preview - ) - if (-not (Test-Path -LiteralPath $Path)) { return } - if ($Preview) { Write-Host "[dir] $Path"; return } - try { - # Use cmd rmdir for better handling of read-only/long paths on Windows PowerShell 5.1 - & cmd.exe /c "rmdir /s /q \"$Path\"" | Out-Null - } catch { - try { Remove-Item -LiteralPath $Path -Recurse -Force -ErrorAction Stop } catch { - Write-Warning "Failed to remove directory: $Path -> $($_.Exception.Message)" - } - } -} - -function Safe-RemoveFile { - param( - [Parameter(Mandatory=$true)][string] $Path, - [switch] $Preview - ) - if (-not (Test-Path -LiteralPath $Path)) { return } - if ($Preview) { Write-Host "[file] $Path"; return } - try { Remove-Item -LiteralPath $Path -Force -ErrorAction Stop } catch { - Write-Warning "Failed to remove file: $Path -> $($_.Exception.Message)" - } -} - -# Resolve repo root (this script lives in ./scripts) -$RepoRoot = (Resolve-Path (Join-Path $PSScriptRoot '..')).Path -Set-Location -LiteralPath $RepoRoot - -Write-Host "Repo root: $RepoRoot" -ForegroundColor DarkGray - -# Accumulators -$DirsToDelete = New-Object System.Collections.Generic.List[string] -$FilesToDelete = New-Object System.Collections.Generic.List[string] - -# 1) node_modules everywhere (including root) -Write-Section 'Scanning node_modules directories' -$nodeModulesDirs = Get-ChildItem -LiteralPath $RepoRoot -Directory -Recurse -Force -ErrorAction SilentlyContinue | - Where-Object { $_.Name -eq 'node_modules' } - -# Ensure root node_modules is included if present -$rootNM = Join-Path $RepoRoot 'node_modules' -if (Test-Path -LiteralPath $rootNM) { - $DirsToDelete.Add($rootNM) -} -foreach ($d in $nodeModulesDirs) { - if (-not $DirsToDelete.Contains($d.FullName)) { $DirsToDelete.Add($d.FullName) } -} -Write-Host ("Found {0} node_modules directory(ies)" -f $DirsToDelete.Count) - -# 2) Optional caches/output folders -if ($IncludeCache) { - Write-Section 'Scanning cache/output directories' - $cacheNames = @( - '.next', '.turbo', '.vite', '.parcel-cache', '.cache', - 'dist', 'build', 'coverage', 'out', 'storybook-static', - '.wrangler' - ) - # node_modules/.cache is common; include it via name match too - $allDirs = Get-ChildItem -LiteralPath $RepoRoot -Directory -Recurse -Force -ErrorAction SilentlyContinue - foreach ($dir in $allDirs) { - if ($cacheNames -contains $dir.Name) { - if (-not $DirsToDelete.Contains($dir.FullName)) { $DirsToDelete.Add($dir.FullName) } - } - } - Write-Host ("Found {0} cache/output directory(ies)" -f ($DirsToDelete | Where-Object { Test-Path $_ }).Count) -} - -# 3) Optional lockfiles -if ($IncludeLock) { - Write-Section 'Scanning lock files' - $lockGlobs = @('pnpm-lock.yaml', 'package-lock.json', 'yarn.lock') - foreach ($glob in $lockGlobs) { - $files = Get-ChildItem -LiteralPath $RepoRoot -Recurse -Force -File -Filter $glob -ErrorAction SilentlyContinue - foreach ($f in $files) { if (-not $FilesToDelete.Contains($f.FullName)) { $FilesToDelete.Add($f.FullName) } } - } - Write-Host ("Found {0} lock file(s)" -f $FilesToDelete.Count) -} - -# 4) Summary -Write-Section 'Summary' -Write-Host ("Directories to delete: {0}" -f $DirsToDelete.Count) -Write-Host ("Files to delete: {0}" -f $FilesToDelete.Count) - -$preview = $DryRun -or (-not $Yes) -if ($preview) { - Write-Host "Preview mode (no deletions). Use -Yes to confirm, or pass -DryRun:$false to hide this list." -ForegroundColor Yellow - foreach ($dir in $DirsToDelete) { Safe-RemoveDirectory -Path $dir -Preview } - foreach ($fil in $FilesToDelete) { Safe-RemoveFile -Path $fil -Preview } - if (-not $Yes) { Write-Host "\nRun again with -Yes to confirm deletion." -ForegroundColor Yellow } - exit 0 -} - -# 5) Deletion -Write-Section 'Deleting directories' -foreach ($dir in $DirsToDelete) { Safe-RemoveDirectory -Path $dir } - -if ($FilesToDelete.Count -gt 0) { - Write-Section 'Deleting files' - foreach ($fil in $FilesToDelete) { Safe-RemoveFile -Path $fil } -} - -# 6) Optional pnpm store prune -if ($PruneStore) { - Write-Section 'Pruning pnpm store' - $pnpm = Get-Command pnpm -ErrorAction SilentlyContinue - if ($null -ne $pnpm) { - try { & pnpm store prune } catch { Write-Warning "pnpm store prune failed: $($_.Exception.Message)" } - } else { - Write-Warning "pnpm not found on PATH; skipping 'pnpm store prune'." - } -} - -Write-Host "\nDone." -ForegroundColor Green diff --git a/scripts/clean-node-modules.sh b/scripts/clean-node-modules.sh deleted file mode 100644 index 129825a..0000000 --- a/scripts/clean-node-modules.sh +++ /dev/null @@ -1,82 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -# Clean all node_modules directories with optional cache & lockfile cleanup. -# Usage: -# ./scripts/clean-node-modules.sh [--include-cache] [--include-lock] [--prune-store] [--yes] [--dry-run] - -INCLUDE_CACHE=false -INCLUDE_LOCK=false -PRUNE_STORE=false -YES=false -DRY_RUN=false - -for arg in "$@"; do - case "$arg" in - --include-cache) INCLUDE_CACHE=true ;; - --include-lock) INCLUDE_LOCK=true ;; - --prune-store) PRUNE_STORE=true ;; - --yes) YES=true ;; - --dry-run) DRY_RUN=true ;; - *) echo "Unknown option: $arg"; exit 2 ;; - esac -done - -SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &> /dev/null && pwd)" -REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" - -preview() { - if [[ "$DRY_RUN" == true || "$YES" == false ]]; then echo 1; else echo 0; fi -} - -section() { echo -e "\n==== $* ====\n"; } - -DIRS_TO_DELETE=() -FILES_TO_DELETE=() - -section "Scanning node_modules directories" -while IFS= read -r -d '' d; do DIRS_TO_DELETE+=("$d"); done < <(find "$REPO_ROOT" -type d -name node_modules -print0) -if [[ -d "$REPO_ROOT/node_modules" ]]; then DIRS_TO_DELETE+=("$REPO_ROOT/node_modules"); fi - -section "Summary so far" -echo "Found ${#DIRS_TO_DELETE[@]} node_modules directories" - -if [[ "$INCLUDE_CACHE" == true ]]; then - section "Scanning cache/output directories" - while IFS= read -r -d '' d; do DIRS_TO_DELETE+=("$d"); done < <(find "$REPO_ROOT" -type d \( \ - -name .next -o -name .turbo -o -name .vite -o -name .parcel-cache -o -name .cache -o \ - -name dist -o -name build -o -name coverage -o -name out -o -name storybook-static -o -name .wrangler \ - \) -print0) -fi - -if [[ "$INCLUDE_LOCK" == true ]]; then - section "Scanning lock files" - while IFS= read -r -d '' f; do FILES_TO_DELETE+=("$f"); done < <(\ - find "$REPO_ROOT" -type f \( -name pnpm-lock.yaml -o -name package-lock.json -o -name yarn.lock \) -print0) -fi - -section "Summary" -echo "Directories to delete: ${#DIRS_TO_DELETE[@]}" -echo "Files to delete: ${#FILES_TO_DELETE[@]}" - -if [[ $(preview) -eq 1 ]]; then - echo "Preview mode (no deletions). Re-run with --yes to confirm." - for d in "${DIRS_TO_DELETE[@]}"; do echo "[dir] $d"; done - for f in "${FILES_TO_DELETE[@]}"; do echo "[file] $f"; done - exit 0 -fi - -section "Deleting directories" -for d in "${DIRS_TO_DELETE[@]}"; do rm -rf -- "$d" || true; done - -if [[ ${#FILES_TO_DELETE[@]} -gt 0 ]]; then - section "Deleting files" - for f in "${FILES_TO_DELETE[@]}"; do rm -f -- "$f" || true; done -fi - -if [[ "$PRUNE_STORE" == true ]]; then - section "Pruning pnpm store" - if command -v pnpm >/dev/null 2>&1; then pnpm store prune || true; else echo "pnpm not found; skipping"; fi -fi - -echo "Done." diff --git a/scripts/cleanup-ghcr.sh b/scripts/cleanup-ghcr.sh deleted file mode 100644 index 5bdbf91..0000000 --- a/scripts/cleanup-ghcr.sh +++ /dev/null @@ -1,54 +0,0 @@ -#!/bin/bash - -# Cleanup script for old GitHub Container Registry (GHCR) images -# This script uses the 'gh' CLI to delete old package versions. -# Requires 'gh' CLI to be installed and authenticated with 'delete:packages' scope. - -set -e - -# Configuration -ORG="asepharyana" -PACKAGE_NAMES=("asepharyana-hub/scraper-api") - -echo "🚀 Starting GHCR cleanup for $ORG..." - -for PACKAGE in "${PACKAGE_NAMES[@]}"; do - echo "------------------------------------------------" - echo "📦 Checking package: $PACKAGE" - - # List versions that are NOT 'latest' and DON'T match the current SHAs - # This is a safe approach: list all versions and let the user decide or - # filter by date/tag patterns. - - # For simplicity and safety, this script will list versions and - # provide the command to delete them. - # To AUTOMATICALLY delete, uncomment the 'gh api' call below. - - echo "🔍 Fetching versions..." - VERSIONS=$(gh api "/orgs/$ORG/packages/container/$PACKAGE/versions" --paginate -q '.[] | "\(.id) \(.metadata.container.tags[0] // "no-tag") \(.updated_at)"') - - if [ -z "$VERSIONS" ]; then - echo "✅ No versions found for $PACKAGE" - continue - fi - - echo "$VERSIONS" | while read -r ID TAG DATE; do - if [[ "$TAG" == "latest" ]]; then - echo "✨ Skipping latest: $ID ($DATE)" - continue - fi - - # Example: only delete if the tag doesn't start with 'sha-' (adjust as needed) - # Or delete very old ones. - - echo "🗑️ Found old version: $ID | Tag: $TAG | Date: $DATE" - - # UNCOMMENT THE LINE BELOW TO ENABLE AUTOMATIC DELETION - # gh api -X DELETE "/orgs/$ORG/packages/container/$PACKAGE/versions/$ID" - # echo "✅ Deleted $ID" - done -done - -echo "------------------------------------------------" -echo "✅ Cleanup script finished." -echo "💡 Note: Deletion is commented out by default for safety. Edit the script to enable it." diff --git a/scripts/git-hooks/setup-hooks.sh b/scripts/git-hooks/setup-hooks.sh deleted file mode 100755 index 6351754..0000000 --- a/scripts/git-hooks/setup-hooks.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -echo "=== Setting up Git hooks ===" -REPO_ROOT=$(git rev-parse --show-toplevel) - -cat < -set -euo pipefail - -SERVICE="$1" -PROFILE="/nix/var/nix/profiles/${SERVICE}" - -# Find the latest store path for this service -LATEST=$(ls -1d /nix/store/*-"${SERVICE}"-* 2>/dev/null | tail -1) -if [ -z "$LATEST" ]; then - echo "ERROR: No store path found for ${SERVICE}" - exit 1 -fi - -# Update profile -/nix/var/nix/profiles/default/bin/nix-env --profile "$PROFILE" --set "$LATEST" - -# Restart service -systemctl daemon-reload -systemctl enable --now "${SERVICE}" 2>/dev/null || systemctl restart "${SERVICE}" - -echo "Deployed ${SERVICE}: ${LATEST}" -systemctl is-active "${SERVICE}" - -# Collect garbage (safe: only removes unreachable paths) -# nix-collect-garbage -d 2>/dev/null || true diff --git a/scripts/sync-submodules.sh b/scripts/sync-submodules.sh deleted file mode 100755 index 6a00ef1..0000000 --- a/scripts/sync-submodules.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -echo "=== Syncing all submodules ===" - -git submodule update --init --recursive - -echo "" -echo "=== Latest submodule status ===" -git submodule status - -echo "" -echo "✅ All submodules synced" diff --git a/scripts/update-deps.sh b/scripts/update-deps.sh deleted file mode 100644 index 9700667..0000000 --- a/scripts/update-deps.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -echo "=== Updating root dependencies ===" -bun update - -echo "" -echo "=== Updating app submodule dependencies ===" -for app in apps/*/; do - if [ -f "${app}package.json" ]; then - echo "→ $app" - (cd "$app" && bun update 2>/dev/null && echo " ✓ $app updated") || echo " - skipping $app (no bun setup)" - fi -done - -echo "" -echo "✅ All dependencies updated"