Compare commits
179
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
5319971bfd | ||
|
|
5a0737dc74 | ||
|
|
1e4b17afb0 | ||
|
|
22e8a872cd | ||
|
|
18cabefe9c | ||
|
|
f66f25f120 | ||
|
|
865fbf9547 | ||
|
|
1d50980eed | ||
|
|
ef29e0e4ef | ||
|
|
16c0231b9f | ||
|
|
3be2744070 | ||
|
|
3dfde5e872 | ||
|
|
632868356d | ||
|
|
2e3d5c3de6 | ||
|
|
6fcdf967b6 | ||
|
|
ac01422d21 | ||
|
|
95a9921b7a | ||
|
|
fde53f22b6 | ||
|
|
e953c3cb67 | ||
|
|
cbad784bca | ||
|
|
189311015c | ||
|
|
ab3c95dae2 | ||
|
|
05d6b4a055 | ||
|
|
2033d9b2bc | ||
|
|
8fc4a15e0c | ||
|
|
88f6968654 | ||
|
|
08883ba1a6 | ||
|
|
4fbca91c8c | ||
|
|
1cdd849d89 | ||
|
|
de161d95a8 | ||
|
|
c4791b3160 | ||
|
|
e9f3cd458c | ||
|
|
14c9668e85 | ||
|
|
6f1dfea710 | ||
|
|
3c52036f7e | ||
|
|
e10a2e498b | ||
|
|
183659b56c | ||
|
|
38873dd73e | ||
|
|
be2117aa16 | ||
|
|
f92aa8ef30 | ||
|
|
4be55b57bb | ||
|
|
5c79a91a2e | ||
|
|
13e2c38eef | ||
|
|
8f62f955d1 | ||
|
|
d2f592261e | ||
|
|
70f54c8c7a | ||
|
|
a35e19b546 | ||
|
|
c25850b727 | ||
|
|
97b48ddb21 | ||
|
|
ab994ffd54 | ||
|
|
9d0b075b92 | ||
|
|
c5a7ae247a | ||
|
|
cbda54fdee | ||
|
|
836ef6862e | ||
|
|
bd751c0fca | ||
|
|
ad7a0de37b | ||
|
|
05a9179356 | ||
|
|
584a7155e6 | ||
|
|
a41b2d7312 | ||
|
|
c8fcaecbdb | ||
|
|
d8b1457a11 | ||
|
|
67288c8723 | ||
|
|
9c12d135e4 | ||
|
|
68dea35d73 | ||
|
|
447b22d9da | ||
|
|
e38fe75d3c | ||
|
|
ddc4c58243 | ||
|
|
1e10973c86 | ||
|
|
0242fc4b68 | ||
|
|
b7e4203a15 | ||
|
|
7181710aa7 | ||
|
|
478a0f849f | ||
|
|
7fc863d611 | ||
|
|
df7aaedfba | ||
|
|
93d053b123 | ||
|
|
401a26aea9 | ||
|
|
0a4952ce7f | ||
|
|
23ba9c48fa | ||
|
|
7e92351f45 | ||
|
|
8e489b9f41 | ||
|
|
0dbaf66743 | ||
|
|
6091d358e1 | ||
|
|
c17f5f5b32 | ||
|
|
83b825b449 | ||
|
|
48086de66b | ||
|
|
545397eb1f | ||
|
|
40e1a80f9f | ||
|
|
688ac095f6 | ||
|
|
a4852f636a | ||
|
|
69cae1e73e | ||
|
|
bd9cc1f4c7 | ||
|
|
aaaf5de70c | ||
|
|
cea96fcfc2 | ||
|
|
5ff8f9224f | ||
|
|
caf3f1d79b | ||
|
|
4e4ed101c5 | ||
|
|
9db886e68b | ||
|
|
cdaa3c3d57 | ||
|
|
f085375363 | ||
|
|
5999ac159a | ||
|
|
28ba9dd78c | ||
|
|
4cc2ba1630 | ||
|
|
bf39884ea6 | ||
|
|
373cb4fc1a | ||
|
|
f96041a164 | ||
|
|
9a48a49f21 | ||
|
|
4f15cafc02 | ||
|
|
6c24508c77 | ||
|
|
c909c4a9a1 | ||
|
|
aa05820393 | ||
|
|
ffe07ddef0 | ||
|
|
0bc61d8b1c | ||
|
|
afae724f6b | ||
|
|
9f54f90621 | ||
|
|
ee4fd677ea | ||
|
|
1ee99ffc2c | ||
|
|
46314e6d38 | ||
|
|
362e9f4115 | ||
|
|
da268f1671 | ||
|
|
4ef4f615c1 | ||
|
|
5d5a0f48b5 | ||
|
|
dcb93035f3 | ||
|
|
df71487563 | ||
|
|
5025d103e9 | ||
|
|
fd3c7f8c29 | ||
|
|
979a70b279 | ||
|
|
a3c5d747ca | ||
|
|
ce7f630894 | ||
|
|
ff047b44de | ||
|
|
8e128568e7 | ||
|
|
8f22d052f8 | ||
|
|
005fe466ed | ||
|
|
b7180ab30c | ||
|
|
299756d285 | ||
|
|
ad67ed9711 | ||
|
|
2bb1f1e814 | ||
|
|
aa83986b23 | ||
|
|
9b9ce9e59e | ||
|
|
73011d9d14 | ||
|
|
a659f598ee | ||
|
|
1befc9285a | ||
|
|
2f3bc950a6 | ||
|
|
7bf4d69f17 | ||
|
|
e54c959765 | ||
|
|
e825c553d6 | ||
|
|
61cd1a712a | ||
|
|
cc3e891960 | ||
|
|
b960f9915c | ||
|
|
7e563b7a54 | ||
|
|
64b8bb29a3 | ||
|
|
443ef9a8da | ||
|
|
36de54dd85 | ||
|
|
79015974df | ||
|
|
f7f8ee1beb | ||
|
|
a6b822bbfe | ||
|
|
0ba5da78f9 | ||
|
|
26d980fc00 | ||
|
|
1098f97c05 | ||
|
|
3fdf1c670f | ||
|
|
3299759dae | ||
|
|
bc153b7491 | ||
|
|
05f3a65358 | ||
|
|
7b983daa4c | ||
|
|
9a29d86fbd | ||
|
|
ea168b4b20 | ||
|
|
d546a1acf9 | ||
|
|
23269e1ab2 | ||
|
|
8f05d1df3d | ||
|
|
28934aaafc | ||
|
|
75da3af5b8 | ||
|
|
c4ecf8f21b | ||
|
|
f465ca261e | ||
|
|
092c0b363d | ||
|
|
84d6056e5e | ||
|
|
80027ee5d3 | ||
|
|
5d6bdf6ce3 | ||
|
|
cdad9ae720 | ||
|
|
44a56fa0a9 | ||
|
|
8145aff114 |
@@ -0,0 +1,29 @@
|
|||||||
|
{
|
||||||
|
"skills": [
|
||||||
|
{
|
||||||
|
"name": "clean-code",
|
||||||
|
"filePattern": ".claude/skills/clean-code/SKILL.md",
|
||||||
|
"description": "Clean Code, Clean Architecture, SOLID, TDD — dari kana-best-practice-engineering"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "hub-rules",
|
||||||
|
"filePattern": ".claude/skills/hub-rules.md",
|
||||||
|
"description": "Aturan repository hub, submodule, infra patterns, dan arsitektur"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "commit-convention",
|
||||||
|
"filePattern": ".claude/skills/commit-convention.md",
|
||||||
|
"description": "Commit message convention — type(scope): description"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "event-driven",
|
||||||
|
"filePattern": ".claude/skills/event-driven.md",
|
||||||
|
"description": "Event-driven patterns with Dapr + NATS untuk hub services"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "deploy-workflow",
|
||||||
|
"filePattern": ".claude/skills/deploy-workflow.md",
|
||||||
|
"description": "CI/CD pipeline, Docker patterns, deployment guide"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
Symlink
+1
@@ -0,0 +1 @@
|
|||||||
|
/home/asephs/kana-best-practice-engineering/skills/clean-code
|
||||||
@@ -0,0 +1,65 @@
|
|||||||
|
---
|
||||||
|
name: commit-convention
|
||||||
|
description: Enforce commit message convention untuk Asepharyana Hub
|
||||||
|
---
|
||||||
|
|
||||||
|
# Commit Convention — Asepharyana Hub
|
||||||
|
|
||||||
|
## Format
|
||||||
|
|
||||||
|
```
|
||||||
|
<type>(<scope>): <description>
|
||||||
|
|
||||||
|
[optional body]
|
||||||
|
|
||||||
|
[optional footer]
|
||||||
|
```
|
||||||
|
|
||||||
|
## Types
|
||||||
|
|
||||||
|
| Type | Usage |
|
||||||
|
| ---------- | ------------------------------------ |
|
||||||
|
| `feat` | Fitur baru |
|
||||||
|
| `fix` | Bug fix |
|
||||||
|
| `chore` | Maintenance, config, tooling |
|
||||||
|
| `docs` | Dokumentasi |
|
||||||
|
| `refactor` | Perubahan kode tanpa fungsional baru |
|
||||||
|
| `test` | Nambah/update test |
|
||||||
|
| `ci` | CI/CD workflows |
|
||||||
|
| `perf` | Optimasi performa |
|
||||||
|
| `style` | Formatting (tanda kutip, dll) |
|
||||||
|
|
||||||
|
## Scopes
|
||||||
|
|
||||||
|
| Scope | Area |
|
||||||
|
| ------------- | --------------------------------- |
|
||||||
|
| `scraper` | apps/scraper submodule |
|
||||||
|
| `infra` | infra/ (compose, traefik, docker) |
|
||||||
|
| `ci` | .github/workflows/ |
|
||||||
|
| `dapr` | Dapr config & sidecar |
|
||||||
|
| `nats` | NATS message bus |
|
||||||
|
| `docs` | Dokumentasi |
|
||||||
|
| `deps` | Dependencies |
|
||||||
|
| `scripts` | Utility scripts |
|
||||||
|
| `root` | Root config files |
|
||||||
|
|
||||||
|
## Contoh
|
||||||
|
|
||||||
|
```
|
||||||
|
feat(scraper): add anime detail caching via Dapr pubsub
|
||||||
|
fix(infra): correct NATS CLI flags for JetStream
|
||||||
|
chore(deps): update biome to v2.5.3
|
||||||
|
docs(infra): add deployment order for Dapr services
|
||||||
|
ci(deploy): add nats.yml to ALL_COMPOSE_FILES
|
||||||
|
refactor(scraper): migrate EventBus from tokio broadcast to Dapr pubsub
|
||||||
|
```
|
||||||
|
|
||||||
|
## Aturan
|
||||||
|
|
||||||
|
1. **Wajib** menyertakan scope dalam tanda kurung
|
||||||
|
2. **Wajib** `Co-Authored-By` untuk commit yang digenerate AI
|
||||||
|
3. **Gunakan imperative mood**: "add" bukan "added" / "adds"
|
||||||
|
4. **Jangan capitalize** type: `feat:` bukan `Feat:`
|
||||||
|
5. **No period** di akhir subject baris
|
||||||
|
6. Body explain **why** dan **what**, bukan **how**
|
||||||
|
7. Refer issue dengan `Closes #123` atau `Fixes #123` di footer
|
||||||
@@ -0,0 +1,103 @@
|
|||||||
|
---
|
||||||
|
name: deploy-workflow
|
||||||
|
description: Panduan deploy, CI/CD, dan Docker compose patterns untuk Asepharyana Hub
|
||||||
|
---
|
||||||
|
|
||||||
|
# Deploy & Workflow — Asepharyana Hub
|
||||||
|
|
||||||
|
## CI/CD Pipeline
|
||||||
|
|
||||||
|
### Build Pipeline (`docker-build-push.yml`)
|
||||||
|
Trigger: push ke `main` yang touch `apps/**`, `infra/**`, `infra/docker/**`
|
||||||
|
|
||||||
|
1. **changes** — detect service mana yg berubah via git diff
|
||||||
|
2. **wait-submodule-ref** — (repository_dispatch only) tunggu SHA commit fetchable
|
||||||
|
3. **build** — matrix build per service, push ke GHCR (`sha-<short>` + `latest`)
|
||||||
|
4. **update-manifest** — update image tag di compose file, commit + push
|
||||||
|
|
||||||
|
### Deploy Pipeline (`deploy-docker.yml`)
|
||||||
|
Trigger: build selesai, atau push ke `main` touch `infra/**`
|
||||||
|
|
||||||
|
1. SSH ke `orangevps` (via `secrets.VPS_HOST`)
|
||||||
|
2. Sync repo (`git fetch --depth=1 + reset`)
|
||||||
|
3. Login ke GHCR
|
||||||
|
4. Deteksi compose file yg berubah
|
||||||
|
5. Pull images + restart container selektif
|
||||||
|
|
||||||
|
### Secrets Required
|
||||||
|
| Secret | Untuk |
|
||||||
|
|--------|-------|
|
||||||
|
| `SSH_PRIVATE_KEY` | SSH ke VPS |
|
||||||
|
| `VPS_HOST` | IP/host VPS (tailscale IP) |
|
||||||
|
| `VPS_USER` | SSH user, biasanya `root` |
|
||||||
|
| `VPS_TARGET_DIR` | Lokasi repo di VPS |
|
||||||
|
| `ENV_FILE_PRODUCTION` | .env content untuk production |
|
||||||
|
|
||||||
|
### Selective Deployment
|
||||||
|
- Hanya compose file yg berubah yang di-redeploy
|
||||||
|
- Selective: `UP_FLAGS="-d"` (tanpa `--remove-orphans`)
|
||||||
|
- Full deploy: `UP_FLAGS="-d --remove-orphans"`
|
||||||
|
|
||||||
|
## Docker Patterns
|
||||||
|
|
||||||
|
### Build dengan cargo-chef (Rust)
|
||||||
|
```dockerfile
|
||||||
|
FROM lukemathwalker/cargo-chef:latest-rust-1.89.0 AS chef
|
||||||
|
WORKDIR /app
|
||||||
|
FROM chef AS planner
|
||||||
|
COPY apps/scraper .
|
||||||
|
RUN cargo chef prepare --recipe-path recipe.json
|
||||||
|
FROM chef AS builder
|
||||||
|
COPY --from=planner /app/recipe.json recipe.json
|
||||||
|
RUN cargo chef cook --release --recipe-path recipe.json
|
||||||
|
COPY apps/scraper .
|
||||||
|
RUN cargo build --release
|
||||||
|
```
|
||||||
|
|
||||||
|
### Runtime minimal untuk Rust binary
|
||||||
|
```dockerfile
|
||||||
|
FROM debian:bookworm-slim AS runtime
|
||||||
|
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||||
|
ca-certificates curl libssl3 && rm -rf /var/lib/apt/lists/*
|
||||||
|
```
|
||||||
|
|
||||||
|
## Image Tagging
|
||||||
|
- `sha-<short-sha>` — immutable, untuk rollback
|
||||||
|
- `latest` — mutable, untuk convenience
|
||||||
|
- Build cache: `sha-<short>-buildcache`
|
||||||
|
- Registry: `ghcr.io/asepharyana/asepharyana-hub/<service>`
|
||||||
|
|
||||||
|
## Manual Deploy Steps
|
||||||
|
```bash
|
||||||
|
# 1. Login GHCR
|
||||||
|
echo $GITHUB_TOKEN | docker login ghcr.io -u asepharyana --password-stdin
|
||||||
|
|
||||||
|
# 2. Full stack
|
||||||
|
docker compose -f infra/compose/traefik.yml \
|
||||||
|
-f infra/compose/shared.yml \
|
||||||
|
-f infra/compose/nats.yml \
|
||||||
|
-f infra/compose/dapr.yml \
|
||||||
|
-f infra/compose/scraper.yml \
|
||||||
|
--env-file .env up -d --remove-orphans
|
||||||
|
|
||||||
|
# 3. Selective (hanya satu service)
|
||||||
|
docker compose -f infra/compose/scraper.yml --env-file .env up -d
|
||||||
|
```
|
||||||
|
|
||||||
|
## Troubleshooting
|
||||||
|
|
||||||
|
### Container reach Tailscale
|
||||||
|
Pastikan route ke Tailscale di main table:
|
||||||
|
```bash
|
||||||
|
ip route add 100.64.0.0/10 dev tailscale0 table main
|
||||||
|
systemctl restart tailscale-routes
|
||||||
|
```
|
||||||
|
|
||||||
|
### Healthcheck gagal di scratch images
|
||||||
|
NATS dan Dapr placement pake scratch — tidak bisa healthcheck. Cukup `service_started` di depends_on.
|
||||||
|
|
||||||
|
### Dapr sidecar crash
|
||||||
|
```bash
|
||||||
|
docker logs scraper-api-dapr | grep -iE "fatal|error"
|
||||||
|
```
|
||||||
|
Penyebab umum: komponen config salah, NATS/Dapr placement belum siap.
|
||||||
@@ -0,0 +1,133 @@
|
|||||||
|
---
|
||||||
|
name: event-driven
|
||||||
|
description: Event-driven patterns dengan Dapr + NATS untuk Asepharyana Hub
|
||||||
|
---
|
||||||
|
|
||||||
|
# Event-Driven Architecture — Asepharyana Hub
|
||||||
|
|
||||||
|
## Stack
|
||||||
|
- **Message Backbone**: NATS + JetStream (untuk streaming & job queue)
|
||||||
|
- **Pub/Sub Runtime**: Dapr sidecar per service (pubsub via Redis built-in)
|
||||||
|
- **State Store**: Dapr → Redis
|
||||||
|
|
||||||
|
## Event Topics Convention
|
||||||
|
|
||||||
|
```
|
||||||
|
hub.<domain>.<action>
|
||||||
|
|
||||||
|
Contoh:
|
||||||
|
hub.image.cached → Image selesai di-cache ke CDN
|
||||||
|
hub.image.repaired → Image diperbaiki (CNAME change)
|
||||||
|
hub.scrape.anime.done → Scrape anime selesai
|
||||||
|
hub.system.alert → Error/alert dari service
|
||||||
|
```
|
||||||
|
|
||||||
|
## CloudEvents Format
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"specversion": "1.0",
|
||||||
|
"type": "hub.image.cached",
|
||||||
|
"source": "scraper-api",
|
||||||
|
"subject": "anime-poster",
|
||||||
|
"id": "uuid-v4",
|
||||||
|
"time": "2026-07-21T10:00:00Z",
|
||||||
|
"datacontenttype": "application/json",
|
||||||
|
"data": { ... }
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
## Publish Event (Rust via HTTP API)
|
||||||
|
|
||||||
|
Gunakan `reqwest` langsung ke Dapr sidecar (SDK Rust masih experimental):
|
||||||
|
|
||||||
|
```rust
|
||||||
|
let event = serde_json::json!({
|
||||||
|
"specversion": "1.0",
|
||||||
|
"type": "hub.image.cached",
|
||||||
|
"source": "scraper-api",
|
||||||
|
"id": Uuid::new_v4().to_string(),
|
||||||
|
"time": chrono::Utc::now().to_rfc3339(),
|
||||||
|
"datacontenttype": "application/json",
|
||||||
|
"data": { "original_url": url, "cdn_url": cdn_url }
|
||||||
|
});
|
||||||
|
|
||||||
|
reqwest::Client::new()
|
||||||
|
.post("http://localhost:3500/v1.0/publish/pubsub/hub.image.cached")
|
||||||
|
.json(&event)
|
||||||
|
.send()
|
||||||
|
.await?;
|
||||||
|
```
|
||||||
|
|
||||||
|
## Service Invocation
|
||||||
|
|
||||||
|
```bash
|
||||||
|
curl http://localhost:3500/v1.0/invoke/<app-id>/method/<path>
|
||||||
|
```
|
||||||
|
|
||||||
|
## State Store
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Set
|
||||||
|
curl -X POST http://localhost:3500/v1.0/state/statestore \
|
||||||
|
-H "Content-Type: application/json" \
|
||||||
|
-d '[{"key": "mykey", "value": "myvalue"}]'
|
||||||
|
|
||||||
|
# Get
|
||||||
|
curl http://localhost:3500/v1.0/state/statestore/mykey
|
||||||
|
|
||||||
|
# Delete
|
||||||
|
curl -X DELETE http://localhost:3500/v1.0/state/statestore/mykey
|
||||||
|
```
|
||||||
|
|
||||||
|
## Scraper Event Integration
|
||||||
|
|
||||||
|
File yang perlu dimodifikasi untuk event-driven:
|
||||||
|
|
||||||
|
| File | Perubahan |
|
||||||
|
|------|-----------|
|
||||||
|
| `src/events/bus.rs` | Ganti backend dari tokio broadcast ke Dapr pub/sub |
|
||||||
|
| `src/bootstrap/mod.rs` | Init DaprClient, inject ke AppState |
|
||||||
|
| `src/presentation/state.rs` | Tambah `dapr_client` field |
|
||||||
|
| `src/proxy/use_cases.rs` | Publish `ImageRepaired` & `ImageCached` events |
|
||||||
|
| `src/infrastructure/services/images/cache.rs` | Emit event tiap cache selesai |
|
||||||
|
| `Cargo.toml` | Tambah `reqwest`, `uuid`, `chrono` (jika belum ada) |
|
||||||
|
|
||||||
|
## Event Handlers (Subscribe)
|
||||||
|
|
||||||
|
Buat `src/subscribers/` untuk handler:
|
||||||
|
|
||||||
|
```rust
|
||||||
|
// src/subscribers/image_handler.rs
|
||||||
|
pub async fn handle_image_cached(event: CloudEvent) -> Result<()> {
|
||||||
|
// Log, notifikasi, update status
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
Daftarkan subscribers di `bootstrap/mod.rs` dengan spawn task:
|
||||||
|
```rust
|
||||||
|
tokio::spawn(async move {
|
||||||
|
let mut stream = dapr_client.subscribe("pubsub", "hub.image.cached");
|
||||||
|
while let Some(event) = stream.next().await {
|
||||||
|
handle_image_cached(event).await;
|
||||||
|
}
|
||||||
|
});
|
||||||
|
```
|
||||||
|
|
||||||
|
## Testing Event-Driven Code
|
||||||
|
|
||||||
|
```rust
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn test_publish_event() {
|
||||||
|
let client = MockDaprClient::new();
|
||||||
|
client.expect_publish()
|
||||||
|
.with(...)
|
||||||
|
.returning(|_| Ok(()));
|
||||||
|
// ... test
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
@@ -0,0 +1,96 @@
|
|||||||
|
---
|
||||||
|
name: hub-rules
|
||||||
|
description: Aturan repository, arsitektur hub, submodule, dan workflow Asepharyana Hub
|
||||||
|
---
|
||||||
|
|
||||||
|
# Asepharyana Hub — Repository Rules
|
||||||
|
|
||||||
|
## Struktur Repository
|
||||||
|
|
||||||
|
```
|
||||||
|
asepharyana-hub/
|
||||||
|
├── apps/ # Git submodules — source code aplikasi
|
||||||
|
├── docs/ # Dokumentasi, ADR, deployment guide
|
||||||
|
├── infra/ # Infrastructure as code
|
||||||
|
│ ├── compose/ # Satu compose file per service
|
||||||
|
│ ├── dapr/ # Dapr component configs
|
||||||
|
│ ├── docker/ # Dockerfiles per service
|
||||||
|
│ └── traefik/ # Static & dynamic Traefik config
|
||||||
|
├── scripts/ # Utility scripts (cleanup, update-deps)
|
||||||
|
└── .github/workflows/ # CI/CD pipelines
|
||||||
|
```
|
||||||
|
|
||||||
|
### Aturan Submodule
|
||||||
|
- Setiap aplikasi di `apps/` adalah **submodule** ke repo terpisah.
|
||||||
|
- Perubahan kode aplikasi dilakukan di **repo masing-masing**, bukan di sini.
|
||||||
|
- Submodule pointer diupdate oleh CI/CD (bukan manual).
|
||||||
|
|
||||||
|
## Infrastructure Patterns
|
||||||
|
|
||||||
|
### Networking
|
||||||
|
- Semua service join **`app-shared-net`** (external Docker bridge)
|
||||||
|
- Service discovery via Docker DNS (container alias)
|
||||||
|
- Traefik sebagai ingress untuk HTTP/S eksternal
|
||||||
|
- Tailscale untuk cross-VPS (PostgreSQL, Redis)
|
||||||
|
|
||||||
|
### Compose File Pattern
|
||||||
|
```yaml
|
||||||
|
services:
|
||||||
|
<service>:
|
||||||
|
container_name: <service>
|
||||||
|
image: ghcr.io/asepharyana/asepharyana-hub/<service>:sha-<sha>
|
||||||
|
restart: always
|
||||||
|
networks:
|
||||||
|
app-shared-net:
|
||||||
|
aliases:
|
||||||
|
- <service>
|
||||||
|
env_file:
|
||||||
|
- ../../.env
|
||||||
|
|
||||||
|
networks:
|
||||||
|
app-shared-net:
|
||||||
|
name: app-shared-net
|
||||||
|
external: true
|
||||||
|
```
|
||||||
|
|
||||||
|
### Dapr Sidecar Pattern
|
||||||
|
```yaml
|
||||||
|
<service>-dapr:
|
||||||
|
container_name: <service>-dapr
|
||||||
|
image: daprio/daprd:latest
|
||||||
|
restart: always
|
||||||
|
depends_on:
|
||||||
|
nats:
|
||||||
|
condition: service_started
|
||||||
|
dapr-placement:
|
||||||
|
condition: service_started
|
||||||
|
networks:
|
||||||
|
- app-shared-net
|
||||||
|
command:
|
||||||
|
- './daprd'
|
||||||
|
- '--app-id=<service>'
|
||||||
|
- '--app-port=<port>'
|
||||||
|
- '--dapr-http-port=3500'
|
||||||
|
- '--dapr-grpc-port=50001'
|
||||||
|
- '--placement-host-address=dapr-placement:50005'
|
||||||
|
- '--resources-path=/components'
|
||||||
|
volumes:
|
||||||
|
- ../../infra/dapr/components:/components
|
||||||
|
```
|
||||||
|
|
||||||
|
### Traefik Routing
|
||||||
|
- Router + service definition di `infra/traefik/dynamic/apps.yaml`
|
||||||
|
- Subdomain pattern: `<service>.asepharyana.my.id` + `<service>.asepharyana.web.id`
|
||||||
|
- TLS cert dari volume mount (bukan auto-acme)
|
||||||
|
|
||||||
|
### CI/CD
|
||||||
|
- `docker-build-push.yml` — build per service, push ke GHCR, update compose manifest
|
||||||
|
- `deploy-docker.yml` — SSH ke orangevps, pull images, restart
|
||||||
|
- Selective deploy: hanya compose file yg berubah
|
||||||
|
|
||||||
|
## Deployment Order
|
||||||
|
1. `shared.yml` (Redis)
|
||||||
|
2. `nats.yml` (NATS message bus)
|
||||||
|
3. `dapr.yml` (Dapr placement)
|
||||||
|
4. `traefik.yml` (Reverse proxy)
|
||||||
|
5. Service compose files (apps + Dapr sidecar)
|
||||||
@@ -1,7 +0,0 @@
|
|||||||
{
|
|
||||||
"image": "mcr.microsoft.com/devcontainers/universal:2",
|
|
||||||
"features": {
|
|
||||||
"ghcr.io/devcontainers/features/rust:1": {},
|
|
||||||
"ghcr.io/devcontainers/features/node:2": {}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -32,7 +32,7 @@ jobs:
|
|||||||
if: github.event_name == 'workflow_dispatch' || github.event_name == 'push' || github.event.workflow_run.conclusion == 'success'
|
if: github.event_name == 'workflow_dispatch' || github.event_name == 'push' || github.event.workflow_run.conclusion == 'success'
|
||||||
steps:
|
steps:
|
||||||
- name: Checkout repository
|
- name: Checkout repository
|
||||||
uses: actions/checkout@v6
|
uses: actions/checkout@v7
|
||||||
with:
|
with:
|
||||||
fetch-depth: 1
|
fetch-depth: 1
|
||||||
submodules: false
|
submodules: false
|
||||||
@@ -89,7 +89,7 @@ jobs:
|
|||||||
git fetch origin main --depth=1 || true
|
git fetch origin main --depth=1 || true
|
||||||
|
|
||||||
# Detect changed files before resetting
|
# Detect changed files before resetting
|
||||||
ALL_COMPOSE_FILES="infra/compose/traefik.yml infra/compose/shared.yml infra/compose/scraper.yml infra/compose/elysia.yml infra/compose/react.yml infra/compose/rust-auth.yml"
|
ALL_COMPOSE_FILES="infra/compose/traefik.yml infra/compose/shared.yml infra/compose/scraper.yml infra/compose/hub.yml infra/compose/tools.yml infra/compose/llm-api.yml infra/compose/nats.yml infra/compose/dapr.yml infra/compose/observability.yml"
|
||||||
TRAEFIK_DYNAMIC_DIR="infra/traefik/dynamic"
|
TRAEFIK_DYNAMIC_DIR="infra/traefik/dynamic"
|
||||||
|
|
||||||
if git rev-parse HEAD >/dev/null 2>&1; then
|
if git rev-parse HEAD >/dev/null 2>&1; then
|
||||||
@@ -142,8 +142,6 @@ jobs:
|
|||||||
TARGET_COMPOSE=""
|
TARGET_COMPOSE=""
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# Submodule update removed for faster VPS deployments
|
|
||||||
|
|
||||||
if command -v "docker" >/dev/null 2>&1 && docker compose version >/dev/null 2>&1; then
|
if command -v "docker" >/dev/null 2>&1 && docker compose version >/dev/null 2>&1; then
|
||||||
COMPOSE_CMD="docker compose"
|
COMPOSE_CMD="docker compose"
|
||||||
elif command -v docker-compose >/dev/null 2>&1; then
|
elif command -v docker-compose >/dev/null 2>&1; then
|
||||||
@@ -153,24 +151,28 @@ jobs:
|
|||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# Construct compose arguments
|
# Always include ALL compose files for dependency resolution
|
||||||
docker rm -f imphenbot-app || true
|
COMPOSE_ARGS=""
|
||||||
|
for f in $ALL_COMPOSE_FILES; do
|
||||||
|
if [ -f "$f" ]; then
|
||||||
|
COMPOSE_ARGS="$COMPOSE_ARGS -f $f"
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
|
||||||
if [ -n "$TARGET_COMPOSE" ]; then
|
if [ -n "$TARGET_COMPOSE" ]; then
|
||||||
COMPOSE_ARGS=""
|
# Extract service names from target compose file(s) for selective up
|
||||||
|
TARGET_SERVICES=""
|
||||||
for f in $TARGET_COMPOSE; do
|
for f in $TARGET_COMPOSE; do
|
||||||
if [ -f "$f" ]; then
|
if [ -f "$f" ]; then
|
||||||
COMPOSE_ARGS="$COMPOSE_ARGS -f $f"
|
svcs=$($COMPOSE_CMD -f "$f" config --services 2>/dev/null | tr '\n' ' ' | xargs)
|
||||||
|
TARGET_SERVICES="$TARGET_SERVICES $svcs"
|
||||||
fi
|
fi
|
||||||
done
|
done
|
||||||
UP_FLAGS="-d" # No --remove-orphans for selective updates to avoid killing other services
|
TARGET_SERVICES=$(echo "$TARGET_SERVICES" | xargs) # trim whitespace
|
||||||
|
echo "🎯 Selective update for services: $TARGET_SERVICES"
|
||||||
else
|
else
|
||||||
echo "🚀 Performing full deployment of all services..."
|
echo "🚀 Performing full deployment of all services..."
|
||||||
COMPOSE_ARGS=""
|
TARGET_SERVICES=""
|
||||||
for f in $ALL_COMPOSE_FILES; do
|
|
||||||
COMPOSE_ARGS="$COMPOSE_ARGS -f $f"
|
|
||||||
done
|
|
||||||
UP_FLAGS="-d --remove-orphans"
|
|
||||||
fi
|
fi
|
||||||
|
|
||||||
echo "📥 Pulling images for target services..."
|
echo "📥 Pulling images for target services..."
|
||||||
@@ -179,7 +181,7 @@ jobs:
|
|||||||
# Retry pull up to 3 times to handle transient Docker attestation lease errors
|
# Retry pull up to 3 times to handle transient Docker attestation lease errors
|
||||||
for attempt in 1 2 3; do
|
for attempt in 1 2 3; do
|
||||||
echo "Pull attempt $attempt/3..."
|
echo "Pull attempt $attempt/3..."
|
||||||
if $COMPOSE_CMD $COMPOSE_ARGS --env-file .env pull; then
|
if $COMPOSE_CMD $COMPOSE_ARGS --env-file .env pull $TARGET_SERVICES; then
|
||||||
echo "✅ Pull succeeded on attempt $attempt"
|
echo "✅ Pull succeeded on attempt $attempt"
|
||||||
PULL_SUCCESS=true
|
PULL_SUCCESS=true
|
||||||
break
|
break
|
||||||
@@ -196,10 +198,6 @@ jobs:
|
|||||||
echo "🧹 Clearing Git locks..."
|
echo "🧹 Clearing Git locks..."
|
||||||
rm -f .git/shallow.lock || true
|
rm -f .git/shallow.lock || true
|
||||||
|
|
||||||
# Cooldown to allow daemon to settle (optional but kept for safety)
|
|
||||||
# echo "⏳ Waiting for Docker daemon to settle..."
|
|
||||||
# sleep 2
|
|
||||||
|
|
||||||
echo "🧹 Removing stale target containers by container_name..."
|
echo "🧹 Removing stale target containers by container_name..."
|
||||||
# Extract all explicitly defined container_names from compose files and remove them to prevent conflicts
|
# Extract all explicitly defined container_names from compose files and remove them to prevent conflicts
|
||||||
if [ -n "$TARGET_COMPOSE" ]; then
|
if [ -n "$TARGET_COMPOSE" ]; then
|
||||||
@@ -223,7 +221,11 @@ jobs:
|
|||||||
echo "🆙 Starting services..."
|
echo "🆙 Starting services..."
|
||||||
echo "🔍 Debug: Current docker containers:"
|
echo "🔍 Debug: Current docker containers:"
|
||||||
docker ps -a
|
docker ps -a
|
||||||
$COMPOSE_CMD $COMPOSE_ARGS --env-file .env up $UP_FLAGS
|
if [ -n "$TARGET_SERVICES" ]; then
|
||||||
|
$COMPOSE_CMD $COMPOSE_ARGS --env-file .env up -d $TARGET_SERVICES
|
||||||
|
else
|
||||||
|
$COMPOSE_CMD $COMPOSE_ARGS --env-file .env up -d --remove-orphans
|
||||||
|
fi
|
||||||
|
|
||||||
# ── Traefik reload ──
|
# ── Traefik reload ──
|
||||||
if [ "${RELOAD_TRAEFIK:-false}" = "true" ]; then
|
if [ "${RELOAD_TRAEFIK:-false}" = "true" ]; then
|
||||||
@@ -235,20 +237,3 @@ jobs:
|
|||||||
fi
|
fi
|
||||||
EOF
|
EOF
|
||||||
|
|
||||||
# - name: Restart coolify-proxy (delay 1 min)
|
|
||||||
# env:
|
|
||||||
# SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }}
|
|
||||||
# VPS_HOST: ${{ secrets.VPS_HOST }}
|
|
||||||
# VPS_USER: ${{ secrets.VPS_USER }}
|
|
||||||
# run: |
|
|
||||||
# mkdir -p ~/.ssh
|
|
||||||
# echo "$SSH_PRIVATE_KEY" > ~/.ssh/id_rsa
|
|
||||||
# chmod 600 ~/.ssh/id_rsa
|
|
||||||
# ssh-keyscan -H "$VPS_HOST" >> ~/.ssh/known_hosts
|
|
||||||
#
|
|
||||||
# echo "⏳ Waiting 60s for containers to settle before restarting proxy..."
|
|
||||||
# sleep 60
|
|
||||||
#
|
|
||||||
# echo "🔄 Restarting coolify-proxy..."
|
|
||||||
# ssh "${VPS_USER}@${VPS_HOST}" "docker restart coolify-proxy"
|
|
||||||
# echo "✅ coolify-proxy restarted."
|
|
||||||
|
|||||||
@@ -5,7 +5,10 @@ on:
|
|||||||
branches:
|
branches:
|
||||||
- main
|
- main
|
||||||
paths:
|
paths:
|
||||||
- 'apps/**'
|
- 'apps/scraper/**'
|
||||||
|
- 'apps/hub/**'
|
||||||
|
- 'apps/tools/**'
|
||||||
|
- 'apps/llm-api/**'
|
||||||
- '.github/workflows/docker-build-push.yml'
|
- '.github/workflows/docker-build-push.yml'
|
||||||
- 'infra/**'
|
- 'infra/**'
|
||||||
- '!infra/compose/**'
|
- '!infra/compose/**'
|
||||||
@@ -34,11 +37,11 @@ jobs:
|
|||||||
outputs:
|
outputs:
|
||||||
matrix: ${{ steps.set-matrix.outputs.matrix }}
|
matrix: ${{ steps.set-matrix.outputs.matrix }}
|
||||||
scraper-api: ${{ steps.filter.outputs['scraper-api'] == 'true' || steps.dispatch.outputs['scraper-api'] == 'true' || github.event_name == 'workflow_dispatch' }}
|
scraper-api: ${{ steps.filter.outputs['scraper-api'] == 'true' || steps.dispatch.outputs['scraper-api'] == 'true' || github.event_name == 'workflow_dispatch' }}
|
||||||
elysia-api: ${{ steps.filter.outputs['elysia-api'] == 'true' || steps.dispatch.outputs['elysia-api'] == 'true' || github.event_name == 'workflow_dispatch' }}
|
hub: ${{ steps.filter.outputs['hub'] == 'true' || steps.dispatch.outputs['hub'] == 'true' || github.event_name == 'workflow_dispatch' }}
|
||||||
react-web: ${{ steps.filter.outputs['react-web'] == 'true' || steps.dispatch.outputs['react-web'] == 'true' || github.event_name == 'workflow_dispatch' }}
|
tools: ${{ steps.filter.outputs['tools'] == 'true' || steps.dispatch.outputs['tools'] == 'true' || github.event_name == 'workflow_dispatch' }}
|
||||||
rust-auth: ${{ steps.filter.outputs['rust-auth'] == 'true' || steps.dispatch.outputs['rust-auth'] == 'true' || github.event_name == 'workflow_dispatch' }}
|
llm-api: ${{ steps.filter.outputs['llm-api'] == 'true' || steps.dispatch.outputs['llm-api'] == 'true' || github.event_name == 'workflow_dispatch' }}
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v6
|
- uses: actions/checkout@v7
|
||||||
with:
|
with:
|
||||||
submodules: false
|
submodules: false
|
||||||
fetch-depth: 2
|
fetch-depth: 2
|
||||||
@@ -63,9 +66,9 @@ jobs:
|
|||||||
}
|
}
|
||||||
|
|
||||||
echo "scraper-api=$(changed '^(apps/scraper(/|$)|\.github/workflows/docker-build-push\.yml$|infra/docker/scraper\.Dockerfile$)')" >> "$GITHUB_OUTPUT"
|
echo "scraper-api=$(changed '^(apps/scraper(/|$)|\.github/workflows/docker-build-push\.yml$|infra/docker/scraper\.Dockerfile$)')" >> "$GITHUB_OUTPUT"
|
||||||
echo "elysia-api=$(changed '^(apps/elysia(/|$)|\.github/workflows/docker-build-push\.yml$|infra/docker/elysia\.Dockerfile$)')" >> "$GITHUB_OUTPUT"
|
echo "hub=$(changed '^(apps/hub(/|$)|\.github/workflows/docker-build-push\.yml$|infra/docker/hub\.Dockerfile$)')" >> "$GITHUB_OUTPUT"
|
||||||
echo "react-web=$(changed '^(apps/react(/|$)|\.github/workflows/docker-build-push\.yml$|infra/docker/react\.Dockerfile$)')" >> "$GITHUB_OUTPUT"
|
echo "tools=$(changed '^(apps/tools(/|$)|\.github/workflows/docker-build-push\.yml$|infra/docker/tools\.Dockerfile$)')" >> "$GITHUB_OUTPUT"
|
||||||
echo "rust-auth=$(changed '^(apps/rust-auth(/|$)|infra/docker/rust\.Dockerfile$|\.github/workflows/docker-build-push\.yml$|\.gitmodules$)')" >> "$GITHUB_OUTPUT"
|
echo "llm-api=$(changed '^(apps/llm-api(/|$)|\.github/workflows/docker-build-push\.yml$|infra/docker/llm-api\.Dockerfile$)')" >> "$GITHUB_OUTPUT"
|
||||||
|
|
||||||
- name: Parse repository_dispatch payload
|
- name: Parse repository_dispatch payload
|
||||||
id: dispatch
|
id: dispatch
|
||||||
@@ -87,15 +90,15 @@ jobs:
|
|||||||
fi
|
fi
|
||||||
|
|
||||||
case "$SERVICE" in
|
case "$SERVICE" in
|
||||||
scraper-api|elysia-api|react-web|rust-auth) ;;
|
scraper-api|hub|tools|llm-api) ;;
|
||||||
*)
|
*)
|
||||||
exit 1
|
exit 1
|
||||||
;;
|
;;
|
||||||
esac
|
esac
|
||||||
|
|
||||||
if ! [[ "$SHA" =~ ^[0-9a-fA-F]{40}$ ]]; then echo "::error::Invalid sha '$SHA'. Expected 40 hex characters"; exit 1; fi
|
if ! [[ "$SHA" =~ ^[0-9a-fA-F]{40}$ ]]; then echo "::error::Invalid sha '$SHA'. Expected 40 hex characters"; fi
|
||||||
|
|
||||||
SERVICES=(scraper-api elysia-api react-web rust-auth)
|
SERVICES=(scraper-api hub tools llm-api)
|
||||||
for svc in "${SERVICES[@]}"; do
|
for svc in "${SERVICES[@]}"; do
|
||||||
if [ "$SERVICE" = "$svc" ]; then
|
if [ "$SERVICE" = "$svc" ]; then
|
||||||
echo "${svc}=true" >> "$GITHUB_OUTPUT"
|
echo "${svc}=true" >> "$GITHUB_OUTPUT"
|
||||||
@@ -113,9 +116,9 @@ jobs:
|
|||||||
}
|
}
|
||||||
|
|
||||||
if [ "${{ steps.filter.outputs['scraper-api'] == 'true' || steps.dispatch.outputs['scraper-api'] == 'true' || github.event_name == 'workflow_dispatch' }}" == "true" ]; then add_service "scraper-api" "docker-scraper" "apps/scraper"; fi
|
if [ "${{ steps.filter.outputs['scraper-api'] == 'true' || steps.dispatch.outputs['scraper-api'] == 'true' || github.event_name == 'workflow_dispatch' }}" == "true" ]; then add_service "scraper-api" "docker-scraper" "apps/scraper"; fi
|
||||||
if [ "${{ steps.filter.outputs['elysia-api'] == 'true' || steps.dispatch.outputs['elysia-api'] == 'true' || github.event_name == 'workflow_dispatch' }}" == "true" ]; then add_service "elysia-api" "docker-elysia" "apps/elysia"; fi
|
if [ "${{ steps.filter.outputs['hub'] == 'true' || steps.dispatch.outputs['hub'] == 'true' || github.event_name == 'workflow_dispatch' }}" == "true" ]; then add_service "hub" "docker-hub" "apps/hub"; fi
|
||||||
if [ "${{ steps.filter.outputs['react-web'] == 'true' || steps.dispatch.outputs['react-web'] == 'true' || github.event_name == 'workflow_dispatch' }}" == "true" ]; then add_service "react-web" "docker-react" "apps/react"; fi
|
if [ "${{ steps.filter.outputs['tools'] == 'true' || steps.dispatch.outputs['tools'] == 'true' || github.event_name == 'workflow_dispatch' }}" == "true" ]; then add_service "tools" "docker-tools" "apps/tools"; fi
|
||||||
if [ "${{ steps.filter.outputs['rust-auth'] == 'true' || steps.dispatch.outputs['rust-auth'] == 'true' || github.event_name == 'workflow_dispatch' }}" == "true" ]; then add_service "rust-auth" "docker-rust-auth" "apps/rust-auth"; fi
|
if [ "${{ steps.filter.outputs['llm-api'] == 'true' || steps.dispatch.outputs['llm-api'] == 'true' || github.event_name == 'workflow_dispatch' }}" == "true" ]; then add_service "llm-api" "docker-llm-api" "apps/llm-api"; fi
|
||||||
|
|
||||||
JSON_ARRAY="[$(IFS=,; echo "${SERVICES[*]}")]"
|
JSON_ARRAY="[$(IFS=,; echo "${SERVICES[*]}")]"
|
||||||
echo "matrix=$JSON_ARRAY" >> $GITHUB_OUTPUT
|
echo "matrix=$JSON_ARRAY" >> $GITHUB_OUTPUT
|
||||||
@@ -135,9 +138,9 @@ jobs:
|
|||||||
|
|
||||||
case "$SERVICE" in
|
case "$SERVICE" in
|
||||||
"scraper-api") REPO="https://github.com/asepharyana/asepharyana-hub-scraper.git" ;;
|
"scraper-api") REPO="https://github.com/asepharyana/asepharyana-hub-scraper.git" ;;
|
||||||
"elysia-api") REPO="https://github.com/asepharyana/asepharyana-hub-elysia.git" ;;
|
"hub") REPO="https://github.com/asepharyana/asepharyana-hub-hub.git" ;;
|
||||||
"react-web") REPO="https://github.com/asepharyana/asepharyana-hub-react.git" ;;
|
"tools") echo "tools is built from monorepo, no submodule wait needed"; exit 0 ;;
|
||||||
"rust-auth") REPO="https://github.com/asepharyana/asepharyana-hub-rust-auth.git" ;;
|
"llm-api") REPO="https://github.com/asepharyana/asepharyana-hub-llm-api.git" ;;
|
||||||
*)
|
*)
|
||||||
echo "::error::Unsupported service '$SERVICE'"
|
echo "::error::Unsupported service '$SERVICE'"
|
||||||
exit 1
|
exit 1
|
||||||
@@ -182,7 +185,7 @@ jobs:
|
|||||||
contents: read
|
contents: read
|
||||||
packages: write
|
packages: write
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v6
|
- uses: actions/checkout@v7
|
||||||
with:
|
with:
|
||||||
submodules: false
|
submodules: false
|
||||||
|
|
||||||
@@ -198,6 +201,7 @@ jobs:
|
|||||||
cd "$SUBMODULE_PATH"
|
cd "$SUBMODULE_PATH"
|
||||||
git fetch origin "$DISPATCH_SHA"
|
git fetch origin "$DISPATCH_SHA"
|
||||||
git checkout "$DISPATCH_SHA"
|
git checkout "$DISPATCH_SHA"
|
||||||
|
cd "${GITHUB_WORKSPACE}"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
- uses: docker/login-action@v4
|
- uses: docker/login-action@v4
|
||||||
@@ -215,13 +219,13 @@ jobs:
|
|||||||
SVC_NAME="${{ matrix.id }}"
|
SVC_NAME="${{ matrix.id }}"
|
||||||
SHORT=${GITHUB_SHA:0:7}
|
SHORT=${GITHUB_SHA:0:7}
|
||||||
echo "image=${REGISTRY}/${IMAGE_NAME_PREFIX}/${SVC_NAME}" >> $GITHUB_OUTPUT
|
echo "image=${REGISTRY}/${IMAGE_NAME_PREFIX}/${SVC_NAME}" >> $GITHUB_OUTPUT
|
||||||
echo "tags=${REGISTRY}/${IMAGE_NAME_PREFIX}/${SVC_NAME}:latest,${REGISTRY}/${IMAGE_NAME_PREFIX}/${SVC_NAME}:sha-${SHORT}" >> $GITHUB_OUTPUT
|
echo "tags=${REGISTRY}/${IMAGE_NAME_PREFIX}/${SVC_NAME}:sha-${SHORT}" >> $GITHUB_OUTPUT
|
||||||
echo "cache-registry=${REGISTRY}/${IMAGE_NAME_PREFIX}/${SVC_NAME}:buildcache" >> $GITHUB_OUTPUT
|
echo "cache-registry=${REGISTRY}/${IMAGE_NAME_PREFIX}/${SVC_NAME}:buildcache" >> $GITHUB_OUTPUT
|
||||||
case "$SVC_NAME" in
|
case "$SVC_NAME" in
|
||||||
"scraper-api") echo "dockerfile=infra/docker/scraper.Dockerfile" >> $GITHUB_OUTPUT ;;
|
"scraper-api") echo "dockerfile=infra/docker/scraper.Dockerfile" >> $GITHUB_OUTPUT ;;
|
||||||
"elysia-api") echo "dockerfile=infra/docker/elysia.Dockerfile" >> $GITHUB_OUTPUT ;;
|
"hub") echo "dockerfile=infra/docker/hub.Dockerfile" >> $GITHUB_OUTPUT ;;
|
||||||
"react-web") echo "dockerfile=infra/docker/react.Dockerfile" >> $GITHUB_OUTPUT ;;
|
"tools") echo "dockerfile=infra/docker/tools.Dockerfile" >> $GITHUB_OUTPUT ;;
|
||||||
"rust-auth") echo "dockerfile=infra/docker/rust.Dockerfile" >> $GITHUB_OUTPUT ;;
|
"llm-api") echo "dockerfile=infra/docker/llm-api.Dockerfile" >> $GITHUB_OUTPUT ;;
|
||||||
esac
|
esac
|
||||||
|
|
||||||
- name: Build and Push Docker image
|
- name: Build and Push Docker image
|
||||||
@@ -252,7 +256,7 @@ jobs:
|
|||||||
permissions:
|
permissions:
|
||||||
contents: write
|
contents: write
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v6
|
- uses: actions/checkout@v7
|
||||||
with:
|
with:
|
||||||
submodules: false
|
submodules: false
|
||||||
token: ${{ secrets.GITHUB_TOKEN }}
|
token: ${{ secrets.GITHUB_TOKEN }}
|
||||||
@@ -266,15 +270,15 @@ jobs:
|
|||||||
|
|
||||||
declare -A SERVICES
|
declare -A SERVICES
|
||||||
SERVICES["scraper-api"]="scraper.yml"
|
SERVICES["scraper-api"]="scraper.yml"
|
||||||
SERVICES["elysia-api"]="elysia.yml"
|
SERVICES["hub"]="hub.yml"
|
||||||
SERVICES["react-web"]="react.yml"
|
SERVICES["tools"]="tools.yml"
|
||||||
SERVICES["rust-auth"]="rust-auth.yml"
|
SERVICES["llm-api"]="llm-api.yml"
|
||||||
|
|
||||||
declare -A PATHS
|
declare -A PATHS
|
||||||
PATHS["scraper-api"]="apps/scraper"
|
PATHS["scraper-api"]="apps/scraper"
|
||||||
PATHS["elysia-api"]="apps/elysia"
|
PATHS["hub"]="apps/hub"
|
||||||
PATHS["react-web"]="apps/react"
|
PATHS["tools"]="apps/tools"
|
||||||
PATHS["rust-auth"]="apps/rust-auth"
|
PATHS["llm-api"]="apps/llm-api"
|
||||||
|
|
||||||
# Use git config for possible commits
|
# Use git config for possible commits
|
||||||
git config --local user.email "action@github.com"
|
git config --local user.email "action@github.com"
|
||||||
@@ -283,9 +287,9 @@ jobs:
|
|||||||
for id in "${!SERVICES[@]}"; do
|
for id in "${!SERVICES[@]}"; do
|
||||||
SHOULD_HAVE_RUN=false
|
SHOULD_HAVE_RUN=false
|
||||||
if [ "${{ needs.changes.outputs['scraper-api'] }}" == "true" ] && [ "$id" == "scraper-api" ]; then SHOULD_HAVE_RUN=true; fi
|
if [ "${{ needs.changes.outputs['scraper-api'] }}" == "true" ] && [ "$id" == "scraper-api" ]; then SHOULD_HAVE_RUN=true; fi
|
||||||
if [ "${{ needs.changes.outputs['elysia-api'] }}" == "true" ] && [ "$id" == "elysia-api" ]; then SHOULD_HAVE_RUN=true; fi
|
if [ "${{ needs.changes.outputs['hub'] }}" == "true" ] && [ "$id" == "hub" ]; then SHOULD_HAVE_RUN=true; fi
|
||||||
if [ "${{ needs.changes.outputs['react-web'] }}" == "true" ] && [ "$id" == "react-web" ]; then SHOULD_HAVE_RUN=true; fi
|
if [ "${{ needs.changes.outputs['tools'] }}" == "true" ] && [ "$id" == "tools" ]; then SHOULD_HAVE_RUN=true; fi
|
||||||
if [ "${{ needs.changes.outputs['rust-auth'] }}" == "true" ] && [ "$id" == "rust-auth" ]; then SHOULD_HAVE_RUN=true; fi
|
if [ "${{ needs.changes.outputs['llm-api'] }}" == "true" ] && [ "$id" == "llm-api" ]; then SHOULD_HAVE_RUN=true; fi
|
||||||
|
|
||||||
if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then SHOULD_HAVE_RUN=true; fi
|
if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then SHOULD_HAVE_RUN=true; fi
|
||||||
|
|
||||||
|
|||||||
@@ -5,26 +5,24 @@ on:
|
|||||||
pull_request:
|
pull_request:
|
||||||
branches: [main]
|
branches: [main]
|
||||||
paths:
|
paths:
|
||||||
- 'apps/*/src/**/*.ts'
|
|
||||||
- 'apps/*/src/**/*.tsx'
|
|
||||||
- 'biome.json'
|
- 'biome.json'
|
||||||
|
- '*.json'
|
||||||
|
- '*.js'
|
||||||
push:
|
push:
|
||||||
branches: [main]
|
branches: [main]
|
||||||
paths:
|
paths:
|
||||||
- 'apps/*/src/**/*.ts'
|
|
||||||
- 'apps/*/src/**/*.tsx'
|
|
||||||
- 'biome.json'
|
- 'biome.json'
|
||||||
|
- '*.json'
|
||||||
|
- '*.js'
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
biome:
|
biome:
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
timeout-minutes: 10
|
timeout-minutes: 10
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v6
|
- uses: actions/checkout@v7
|
||||||
with:
|
with:
|
||||||
submodules: recursive
|
submodules: recursive
|
||||||
- uses: actions/setup-node@v4
|
- uses: oven-sh/setup-bun@v2
|
||||||
with:
|
- run: bun install --frozen-lockfile
|
||||||
node-version: 22
|
- run: bun run ci
|
||||||
- run: npm install -g @biomejs/biome
|
|
||||||
- run: biome ci . --no-errors-on-unmatched
|
|
||||||
|
|||||||
@@ -13,8 +13,18 @@ jobs:
|
|||||||
permissions:
|
permissions:
|
||||||
security-events: write
|
security-events: write
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v6
|
- uses: actions/checkout@v7
|
||||||
- uses: github/codeql-action/init@v3
|
|
||||||
with:
|
with:
|
||||||
languages: javascript-typescript, rust
|
fetch-depth: 2
|
||||||
- uses: github/codeql-action/analyze@v3
|
submodules: recursive
|
||||||
|
|
||||||
|
- uses: github/codeql-action/init@v4
|
||||||
|
with:
|
||||||
|
languages: rust
|
||||||
|
|
||||||
|
- name: Build Rust projects for CodeQL analysis
|
||||||
|
run: |
|
||||||
|
cargo build --manifest-path apps/scraper/Cargo.toml
|
||||||
|
cargo build --manifest-path apps/llm-api/Cargo.toml
|
||||||
|
|
||||||
|
- uses: github/codeql-action/analyze@v4
|
||||||
|
|||||||
@@ -1,36 +0,0 @@
|
|||||||
name: TypeCheck
|
|
||||||
|
|
||||||
on:
|
|
||||||
pull_request:
|
|
||||||
branches: [main]
|
|
||||||
paths:
|
|
||||||
- 'apps/react/src/**/*.ts'
|
|
||||||
- 'apps/react/src/**/*.tsx'
|
|
||||||
- 'apps/elysia/src/**/*.ts'
|
|
||||||
- 'apps/elysia/tsconfig.json'
|
|
||||||
- 'tsconfig.base.json'
|
|
||||||
|
|
||||||
jobs:
|
|
||||||
typecheck:
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
timeout-minutes: 10
|
|
||||||
steps:
|
|
||||||
- uses: actions/checkout@v6
|
|
||||||
with:
|
|
||||||
submodules: recursive
|
|
||||||
|
|
||||||
- uses: oven/setup-bun@v2
|
|
||||||
with:
|
|
||||||
bun-version: latest
|
|
||||||
|
|
||||||
- name: TypeCheck apps/react
|
|
||||||
working-directory: apps/react
|
|
||||||
run: |
|
|
||||||
bun install
|
|
||||||
npx tsc --noEmit
|
|
||||||
|
|
||||||
- name: TypeCheck apps/elysia
|
|
||||||
working-directory: apps/elysia
|
|
||||||
run: |
|
|
||||||
bun install
|
|
||||||
bun run typecheck
|
|
||||||
@@ -11,7 +11,7 @@ jobs:
|
|||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
timeout-minutes: 10
|
timeout-minutes: 10
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v6
|
- uses: actions/checkout@v7
|
||||||
|
|
||||||
- name: Validate payload
|
- name: Validate payload
|
||||||
env:
|
env:
|
||||||
@@ -36,7 +36,7 @@ jobs:
|
|||||||
fi
|
fi
|
||||||
|
|
||||||
case "$SERVICE" in
|
case "$SERVICE" in
|
||||||
scraper-api|elysia-api|react-web|rust-auth) ;;
|
scraper-api|hub|llm-api|tools) ;;
|
||||||
*)
|
*)
|
||||||
echo "::error::Unsupported service '$SERVICE'"
|
echo "::error::Unsupported service '$SERVICE'"
|
||||||
exit 1
|
exit 1
|
||||||
@@ -52,14 +52,20 @@ jobs:
|
|||||||
run: |
|
run: |
|
||||||
set -euo pipefail
|
set -euo pipefail
|
||||||
|
|
||||||
echo "Updating ${SERVICE} to ${SHA}"
|
# Map service name to submodule path
|
||||||
git submodule update --init "apps/${SERVICE}"
|
case "$SERVICE" in
|
||||||
cd "apps/${SERVICE}"
|
scraper-api) SUBMODULE_PATH="apps/scraper" ;;
|
||||||
# full fetch so we get tree objects for the target SHA
|
llm-api) SUBMODULE_PATH="apps/llm-api" ;;
|
||||||
|
*) SUBMODULE_PATH="apps/${SERVICE}" ;;
|
||||||
|
esac
|
||||||
|
|
||||||
|
echo "Updating ${SUBMODULE_PATH} to ${SHA}"
|
||||||
|
git submodule update --init "${SUBMODULE_PATH}"
|
||||||
|
cd "${SUBMODULE_PATH}"
|
||||||
git fetch --depth=1 origin master 2>/dev/null || git fetch --depth=1 origin main
|
git fetch --depth=1 origin master 2>/dev/null || git fetch --depth=1 origin main
|
||||||
git checkout "${SHA}"
|
git checkout "${SHA}"
|
||||||
cd "${GITHUB_WORKSPACE}"
|
cd "${GITHUB_WORKSPACE}"
|
||||||
git add "apps/${SERVICE}"
|
git add "${SUBMODULE_PATH}"
|
||||||
git diff --cached --quiet && exit 0
|
git diff --cached --quiet && exit 0
|
||||||
|
|
||||||
git config user.name "monrepo-bot"
|
git config user.name "monrepo-bot"
|
||||||
|
|||||||
+4
-3
@@ -1,4 +1,3 @@
|
|||||||
apps/gmw/
|
|
||||||
# See https://docs.github.com/en/get-started/getting-started-with-git/ignoring-files for more about ignoring files.
|
# See https://docs.github.com/en/get-started/getting-started-with-git/ignoring-files for more about ignoring files.
|
||||||
|
|
||||||
# compiled output
|
# compiled output
|
||||||
@@ -35,16 +34,18 @@ npm-debug.log
|
|||||||
yarn-error.log
|
yarn-error.log
|
||||||
testem.log
|
testem.log
|
||||||
/typings
|
/typings
|
||||||
|
.playwright-mcp/
|
||||||
|
|
||||||
# System Files
|
# System Files
|
||||||
.DS_Store
|
.DS_Store
|
||||||
Thumbs.db
|
Thumbs.db
|
||||||
.claude
|
.claude/*
|
||||||
|
!.claude/skills/
|
||||||
|
!.claude/settings.json
|
||||||
# Next.js
|
# Next.js
|
||||||
.next
|
.next
|
||||||
out
|
out
|
||||||
**/.codegraph/**
|
**/.codegraph/**
|
||||||
**/.claude/**
|
|
||||||
test-output
|
test-output
|
||||||
**/**.env
|
**/**.env
|
||||||
**/**.env.**
|
**/**.env.**
|
||||||
|
|||||||
+12
-9
@@ -1,12 +1,15 @@
|
|||||||
[submodule "apps/elysia"]
|
|
||||||
path = apps/elysia
|
|
||||||
url = https://github.com/asepharyana/asepharyana-hub-elysia.git
|
|
||||||
[submodule "apps/react"]
|
|
||||||
path = apps/react
|
|
||||||
url = https://github.com/asepharyana/asepharyana-hub-react.git
|
|
||||||
[submodule "apps/scraper"]
|
[submodule "apps/scraper"]
|
||||||
path = apps/scraper
|
path = apps/scraper
|
||||||
url = https://github.com/asepharyana/asepharyana-hub-scraper.git
|
url = https://github.com/asepharyana/asepharyana-hub-scraper.git
|
||||||
[submodule "apps/rust-auth"]
|
[submodule "apps/hub"]
|
||||||
path = apps/rust-auth
|
path = apps/hub
|
||||||
url = https://github.com/asepharyana/asepharyana-hub-rust-auth.git
|
url = https://github.com/asepharyana/asepharyana-hub-hub.git
|
||||||
|
[submodule "apps/tools"]
|
||||||
|
path = apps/tools
|
||||||
|
url = https://github.com/asepharyana/asepharyana-hub-tools.git
|
||||||
|
[submodule "plugins/hub-guide"]
|
||||||
|
path = plugins/hub-guide
|
||||||
|
url = https://github.com/asepharyana/asepharyana-hub-guide.git
|
||||||
|
[submodule "apps/llm-api"]
|
||||||
|
path = apps/llm-api
|
||||||
|
url = https://github.com/asepharyana/asepharyana-hub-llm-api.git
|
||||||
|
|||||||
@@ -0,0 +1,65 @@
|
|||||||
|
---
|
||||||
|
name: commit-convention
|
||||||
|
description: Commit message convention — type(scope): description for Asepharyana Hub
|
||||||
|
---
|
||||||
|
|
||||||
|
# Commit Convention — Asepharyana Hub
|
||||||
|
|
||||||
|
## Format
|
||||||
|
|
||||||
|
```
|
||||||
|
<type>(<scope>): <description>
|
||||||
|
|
||||||
|
[optional body]
|
||||||
|
|
||||||
|
[optional footer]
|
||||||
|
```
|
||||||
|
|
||||||
|
## Types
|
||||||
|
|
||||||
|
| Type | Usage |
|
||||||
|
| ---------- | ------------------------------------ |
|
||||||
|
| `feat` | Fitur baru |
|
||||||
|
| `fix` | Bug fix |
|
||||||
|
| `chore` | Maintenance, config, tooling |
|
||||||
|
| `docs` | Dokumentasi |
|
||||||
|
| `refactor` | Perubahan kode tanpa fungsional baru |
|
||||||
|
| `test` | Nambah/update test |
|
||||||
|
| `ci` | CI/CD workflows |
|
||||||
|
| `perf` | Optimasi performa |
|
||||||
|
| `style` | Formatting (tanda kutip, dll) |
|
||||||
|
|
||||||
|
## Scopes
|
||||||
|
|
||||||
|
| Scope | Area |
|
||||||
|
| ------------- | --------------------------------- |
|
||||||
|
| `scraper` | apps/scraper submodule |
|
||||||
|
| `infra` | infra/ (compose, traefik, docker) |
|
||||||
|
| `ci` | .github/workflows/ |
|
||||||
|
| `dapr` | Dapr config & sidecar |
|
||||||
|
| `nats` | NATS message bus |
|
||||||
|
| `docs` | Dokumentasi |
|
||||||
|
| `deps` | Dependencies |
|
||||||
|
| `scripts` | Utility scripts |
|
||||||
|
| `root` | Root config files |
|
||||||
|
|
||||||
|
## Contoh
|
||||||
|
|
||||||
|
```
|
||||||
|
feat(scraper): add anime detail caching via Dapr pubsub
|
||||||
|
fix(infra): correct NATS CLI flags for JetStream
|
||||||
|
chore(deps): update biome to v2.5.3
|
||||||
|
docs(infra): add deployment order for Dapr services
|
||||||
|
ci(deploy): add nats.yml to ALL_COMPOSE_FILES
|
||||||
|
refactor(scraper): migrate EventBus from tokio broadcast to Dapr pubsub
|
||||||
|
```
|
||||||
|
|
||||||
|
## Aturan
|
||||||
|
|
||||||
|
1. **Wajib** menyertakan scope dalam tanda kurung
|
||||||
|
2. **Wajib** `Co-Authored-By` untuk commit yang digenerate AI
|
||||||
|
3. **Gunakan imperative mood**: "add" bukan "added" / "adds"
|
||||||
|
4. **Jangan capitalize** type: `feat:` bukan `Feat:`
|
||||||
|
5. **No period** di akhir subject baris
|
||||||
|
6. Body explain **why** dan **what**, bukan **how**
|
||||||
|
7. Refer issue dengan `Closes #123` atau `Fixes #123` di footer
|
||||||
@@ -0,0 +1,97 @@
|
|||||||
|
---
|
||||||
|
name: deploy-workflow
|
||||||
|
description: CI/CD pipeline, Docker build patterns, manual deploy steps, and troubleshooting for Asepharyana Hub
|
||||||
|
---
|
||||||
|
|
||||||
|
# Deploy & Workflow — Asepharyana Hub
|
||||||
|
|
||||||
|
## CI/CD Pipeline
|
||||||
|
|
||||||
|
### Build Pipeline (`docker-build-push.yml`)
|
||||||
|
Trigger: push ke `main` yang touch `apps/**`, `infra/**`, `infra/docker/**`
|
||||||
|
|
||||||
|
1. **changes** — detect service mana yg berubah via git diff
|
||||||
|
2. **wait-submodule-ref** — (repository_dispatch only) tunggu SHA commit fetchable
|
||||||
|
3. **build** — matrix build per service, push ke GHCR (`sha-<short>` + `latest`)
|
||||||
|
4. **update-manifest** — update image tag di compose file, commit + push
|
||||||
|
|
||||||
|
### Deploy Pipeline (`deploy-docker.yml`)
|
||||||
|
Trigger: build selesai, atau push ke `main` touch `infra/**`
|
||||||
|
|
||||||
|
1. SSH ke `orangevps` (via `secrets.VPS_HOST`)
|
||||||
|
2. Sync repo (`git fetch --depth=1 + reset`)
|
||||||
|
3. Login ke GHCR
|
||||||
|
4. Deteksi compose file yg berubah
|
||||||
|
5. Pull images + restart container selektif
|
||||||
|
|
||||||
|
### Secrets Required
|
||||||
|
| Secret | Untuk |
|
||||||
|
|--------|-------|
|
||||||
|
| `SSH_PRIVATE_KEY` | SSH ke VPS |
|
||||||
|
| `VPS_HOST` | IP/host VPS (tailscale IP) |
|
||||||
|
| `VPS_USER` | SSH user, biasanya `root` |
|
||||||
|
| `VPS_TARGET_DIR` | Lokasi repo di VPS |
|
||||||
|
| `ENV_FILE_PRODUCTION` | .env content untuk production |
|
||||||
|
|
||||||
|
### Selective Deployment
|
||||||
|
- Hanya compose file yg berubah yang di-redeploy
|
||||||
|
- Selective: `UP_FLAGS="-d"` (tanpa `--remove-orphans`)
|
||||||
|
- Full deploy: `UP_FLAGS="-d --remove-orphans"`
|
||||||
|
|
||||||
|
## Docker Patterns
|
||||||
|
|
||||||
|
### Build dengan cargo-chef (Rust)
|
||||||
|
```dockerfile
|
||||||
|
FROM lukemathwalker/cargo-chef:latest-rust-1.89.0 AS chef
|
||||||
|
WORKDIR /app
|
||||||
|
FROM chef AS planner
|
||||||
|
COPY apps/scraper .
|
||||||
|
RUN cargo chef prepare --recipe-path recipe.json
|
||||||
|
FROM chef AS builder
|
||||||
|
COPY --from=planner /app/recipe.json recipe.json
|
||||||
|
RUN cargo chef cook --release --recipe-path recipe.json
|
||||||
|
COPY apps/scraper .
|
||||||
|
RUN cargo build --release
|
||||||
|
```
|
||||||
|
|
||||||
|
### Runtime minimal untuk Rust binary
|
||||||
|
```dockerfile
|
||||||
|
FROM debian:bookworm-slim AS runtime
|
||||||
|
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||||
|
ca-certificates curl libssl3 && rm -rf /var/lib/apt/lists/*
|
||||||
|
```
|
||||||
|
|
||||||
|
## Manual Deploy Steps
|
||||||
|
```bash
|
||||||
|
# 1. Login GHCR
|
||||||
|
echo $GITHUB_TOKEN | docker login ghcr.io -u asepharyana --password-stdin
|
||||||
|
|
||||||
|
# 2. Full stack
|
||||||
|
docker compose -f infra/compose/traefik.yml \
|
||||||
|
-f infra/compose/shared.yml \
|
||||||
|
-f infra/compose/nats.yml \
|
||||||
|
-f infra/compose/dapr.yml \
|
||||||
|
-f infra/compose/scraper.yml \
|
||||||
|
--env-file .env up -d --remove-orphans
|
||||||
|
|
||||||
|
# 3. Selective (hanya satu service)
|
||||||
|
docker compose -f infra/compose/scraper.yml --env-file .env up -d
|
||||||
|
```
|
||||||
|
|
||||||
|
## Troubleshooting
|
||||||
|
|
||||||
|
### Container reach Tailscale
|
||||||
|
Pastikan route ke Tailscale di main table:
|
||||||
|
```bash
|
||||||
|
ip route add 100.64.0.0/10 dev tailscale0 table main
|
||||||
|
systemctl restart tailscale-routes
|
||||||
|
```
|
||||||
|
|
||||||
|
### Healthcheck gagal di scratch images
|
||||||
|
NATS dan Dapr placement pake scratch — tidak bisa healthcheck. Cukup `service_started` di depends_on.
|
||||||
|
|
||||||
|
### Dapr sidecar crash
|
||||||
|
```bash
|
||||||
|
docker logs scraper-api-dapr | grep -iE "fatal|error"
|
||||||
|
```
|
||||||
|
Penyebab umum: komponen config salah, NATS/Dapr placement belum siap.
|
||||||
@@ -0,0 +1,133 @@
|
|||||||
|
---
|
||||||
|
name: event-driven
|
||||||
|
description: Event-driven architecture patterns with Dapr + NATS for Asepharyana Hub services
|
||||||
|
---
|
||||||
|
|
||||||
|
# Event-Driven Architecture — Asepharyana Hub
|
||||||
|
|
||||||
|
## Stack
|
||||||
|
- **Message Backbone**: NATS + JetStream (untuk streaming & job queue)
|
||||||
|
- **Pub/Sub Runtime**: Dapr sidecar per service (pubsub via Redis built-in)
|
||||||
|
- **State Store**: Dapr → Redis
|
||||||
|
|
||||||
|
## Event Topics Convention
|
||||||
|
|
||||||
|
```
|
||||||
|
hub.<domain>.<action>
|
||||||
|
|
||||||
|
Contoh:
|
||||||
|
hub.image.cached → Image selesai di-cache ke CDN
|
||||||
|
hub.image.repaired → Image diperbaiki (CNAME change)
|
||||||
|
hub.scrape.anime.done → Scrape anime selesai
|
||||||
|
hub.system.alert → Error/alert dari service
|
||||||
|
```
|
||||||
|
|
||||||
|
## CloudEvents Format
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"specversion": "1.0",
|
||||||
|
"type": "hub.image.cached",
|
||||||
|
"source": "scraper-api",
|
||||||
|
"subject": "anime-poster",
|
||||||
|
"id": "uuid-v4",
|
||||||
|
"time": "2026-07-21T10:00:00Z",
|
||||||
|
"datacontenttype": "application/json",
|
||||||
|
"data": { ... }
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
## Publish Event (Rust via HTTP API)
|
||||||
|
|
||||||
|
Gunakan `reqwest` langsung ke Dapr sidecar (SDK Rust masih experimental):
|
||||||
|
|
||||||
|
```rust
|
||||||
|
let event = serde_json::json!({
|
||||||
|
"specversion": "1.0",
|
||||||
|
"type": "hub.image.cached",
|
||||||
|
"source": "scraper-api",
|
||||||
|
"id": Uuid::new_v4().to_string(),
|
||||||
|
"time": chrono::Utc::now().to_rfc3339(),
|
||||||
|
"datacontenttype": "application/json",
|
||||||
|
"data": { "original_url": url, "cdn_url": cdn_url }
|
||||||
|
});
|
||||||
|
|
||||||
|
reqwest::Client::new()
|
||||||
|
.post("http://localhost:3500/v1.0/publish/pubsub/hub.image.cached")
|
||||||
|
.json(&event)
|
||||||
|
.send()
|
||||||
|
.await?;
|
||||||
|
```
|
||||||
|
|
||||||
|
## Service Invocation
|
||||||
|
|
||||||
|
```bash
|
||||||
|
curl http://localhost:3500/v1.0/invoke/<app-id>/method/<path>
|
||||||
|
```
|
||||||
|
|
||||||
|
## State Store
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Set
|
||||||
|
curl -X POST http://localhost:3500/v1.0/state/statestore \
|
||||||
|
-H "Content-Type: application/json" \
|
||||||
|
-d '[{"key": "mykey", "value": "myvalue"}]'
|
||||||
|
|
||||||
|
# Get
|
||||||
|
curl http://localhost:3500/v1.0/state/statestore/mykey
|
||||||
|
|
||||||
|
# Delete
|
||||||
|
curl -X DELETE http://localhost:3500/v1.0/state/statestore/mykey
|
||||||
|
```
|
||||||
|
|
||||||
|
## Scraper Event Integration
|
||||||
|
|
||||||
|
File yang perlu dimodifikasi untuk event-driven:
|
||||||
|
|
||||||
|
| File | Perubahan |
|
||||||
|
|------|-----------|
|
||||||
|
| `src/events/bus.rs` | Ganti backend dari tokio broadcast ke Dapr pub/sub |
|
||||||
|
| `src/bootstrap/mod.rs` | Init DaprClient, inject ke AppState |
|
||||||
|
| `src/presentation/state.rs` | Tambah `dapr_client` field |
|
||||||
|
| `src/proxy/use_cases.rs` | Publish `ImageRepaired` & `ImageCached` events |
|
||||||
|
| `src/infrastructure/services/images/cache.rs` | Emit event tiap cache selesai |
|
||||||
|
| `Cargo.toml` | Tambah `reqwest`, `uuid`, `chrono` (jika belum ada) |
|
||||||
|
|
||||||
|
## Event Handlers (Subscribe)
|
||||||
|
|
||||||
|
Buat `src/subscribers/` untuk handler:
|
||||||
|
|
||||||
|
```rust
|
||||||
|
// src/subscribers/image_handler.rs
|
||||||
|
pub async fn handle_image_cached(event: CloudEvent) -> Result<()> {
|
||||||
|
// Log, notifikasi, update status
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
Daftarkan subscribers di `bootstrap/mod.rs` dengan spawn task:
|
||||||
|
```rust
|
||||||
|
tokio::spawn(async move {
|
||||||
|
let mut stream = dapr_client.subscribe("pubsub", "hub.image.cached");
|
||||||
|
while let Some(event) = stream.next().await {
|
||||||
|
handle_image_cached(event).await;
|
||||||
|
}
|
||||||
|
});
|
||||||
|
```
|
||||||
|
|
||||||
|
## Testing Event-Driven Code
|
||||||
|
|
||||||
|
```rust
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn test_publish_event() {
|
||||||
|
let client = MockDaprClient::new();
|
||||||
|
client.expect_publish()
|
||||||
|
.with(...)
|
||||||
|
.returning(|_| Ok(()));
|
||||||
|
// ... test
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
@@ -0,0 +1,101 @@
|
|||||||
|
---
|
||||||
|
name: hub-rules
|
||||||
|
description: Repository structure, submodule strategy, infrastructure patterns, and architecture of Asepharyana Hub
|
||||||
|
---
|
||||||
|
|
||||||
|
# Asepharyana Hub — Repository Rules
|
||||||
|
|
||||||
|
## Struktur Repository
|
||||||
|
|
||||||
|
```
|
||||||
|
asepharyana-hub/
|
||||||
|
├── apps/ # Git submodules — source code aplikasi
|
||||||
|
├── docs/ # Dokumentasi, ADR, deployment guide
|
||||||
|
├── infra/ # Infrastructure as code
|
||||||
|
│ ├── compose/ # Satu compose file per service
|
||||||
|
│ ├── dapr/ # Dapr component configs
|
||||||
|
│ ├── docker/ # Dockerfiles per service
|
||||||
|
│ └── traefik/ # Static & dynamic Traefik config
|
||||||
|
├── scripts/ # Utility scripts (cleanup, update-deps)
|
||||||
|
└── .github/workflows/ # CI/CD pipelines
|
||||||
|
```
|
||||||
|
|
||||||
|
### Aturan Submodule
|
||||||
|
- Setiap aplikasi di `apps/` adalah **submodule** ke repo terpisah.
|
||||||
|
- Perubahan kode aplikasi dilakukan di **repo masing-masing**, bukan di sini.
|
||||||
|
- Submodule pointer diupdate oleh CI/CD (bukan manual).
|
||||||
|
|
||||||
|
## Infrastructure Patterns
|
||||||
|
|
||||||
|
### Networking
|
||||||
|
- Semua service join **`app-shared-net`** (external Docker bridge)
|
||||||
|
- Service discovery via Docker DNS (container alias)
|
||||||
|
- Traefik sebagai ingress untuk HTTP/S eksternal
|
||||||
|
- Tailscale untuk cross-VPS (PostgreSQL, Redis)
|
||||||
|
|
||||||
|
### Compose File Pattern
|
||||||
|
```yaml
|
||||||
|
services:
|
||||||
|
<service>:
|
||||||
|
container_name: <service>
|
||||||
|
image: ghcr.io/asepharyana/asepharyana-hub/<service>:sha-<sha>
|
||||||
|
restart: always
|
||||||
|
networks:
|
||||||
|
app-shared-net:
|
||||||
|
aliases:
|
||||||
|
- <service>
|
||||||
|
env_file:
|
||||||
|
- ../../.env
|
||||||
|
|
||||||
|
networks:
|
||||||
|
app-shared-net:
|
||||||
|
name: app-shared-net
|
||||||
|
external: true
|
||||||
|
```
|
||||||
|
|
||||||
|
### Dapr Sidecar Pattern
|
||||||
|
```yaml
|
||||||
|
<service>-dapr:
|
||||||
|
container_name: <service>-dapr
|
||||||
|
image: daprio/daprd:latest
|
||||||
|
restart: always
|
||||||
|
depends_on:
|
||||||
|
nats:
|
||||||
|
condition: service_started
|
||||||
|
dapr-placement:
|
||||||
|
condition: service_started
|
||||||
|
networks:
|
||||||
|
- app-shared-net
|
||||||
|
command:
|
||||||
|
- './daprd'
|
||||||
|
- '--app-id=<service>'
|
||||||
|
- '--app-port=<port>'
|
||||||
|
- '--dapr-http-port=3500'
|
||||||
|
- '--dapr-grpc-port=50001'
|
||||||
|
- '--placement-host-address=dapr-placement:50005'
|
||||||
|
- '--resources-path=/components'
|
||||||
|
volumes:
|
||||||
|
- ../../infra/dapr/components:/components
|
||||||
|
```
|
||||||
|
|
||||||
|
### Traefik Routing
|
||||||
|
- Router + service definition di `infra/traefik/dynamic/apps.yaml`
|
||||||
|
- Subdomain pattern: `<service>.asepharyana.my.id` + `<service>.asepharya.web.id`
|
||||||
|
- TLS cert dari volume mount (bukan auto-acme)
|
||||||
|
|
||||||
|
### Image Tagging
|
||||||
|
- `sha-<short-sha>` — immutable, untuk rollback
|
||||||
|
- `latest` — mutable, untuk convenience
|
||||||
|
- Registry: `ghcr.io/asepharyana/asepharyana-hub/<service>`
|
||||||
|
|
||||||
|
### CI/CD
|
||||||
|
- `docker-build-push.yml` — build per service, push ke GHCR, update compose manifest
|
||||||
|
- `deploy-docker.yml` — SSH ke orangevps, pull images, restart
|
||||||
|
- Selective deploy: hanya compose file yg berubah
|
||||||
|
|
||||||
|
## Deployment Order
|
||||||
|
1. `shared.yml` (Redis)
|
||||||
|
2. `nats.yml` (NATS message bus)
|
||||||
|
3. `dapr.yml` (Dapr placement)
|
||||||
|
4. `traefik.yml` (Reverse proxy)
|
||||||
|
5. Service compose files (apps + Dapr sidecar)
|
||||||
@@ -1,8 +0,0 @@
|
|||||||
{
|
|
||||||
"mcpServers": {
|
|
||||||
"next-devtools": {
|
|
||||||
"command": "npx",
|
|
||||||
"args": ["-y", "next-devtools-mcp@latest"]
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,81 +0,0 @@
|
|||||||
# Agent Protocol v10: Strategic Execution and Diagnostic Directives
|
|
||||||
|
|
||||||
1. **Mandatory Executability.**
|
|
||||||
- **Directive:** All generated outputs must be syntactically correct and directly executable by the target system's interpreter, compiler, or shell.
|
|
||||||
- **Constraint:** Placeholders and pseudo-code are forbidden. Every output must be a complete, functional artifact.
|
|
||||||
|
|
||||||
2. **Data and State Integrity.**
|
|
||||||
- **Directive:** All generated code must strictly adhere to declared data structures, schemas, and the target language's type system.
|
|
||||||
- **Constraint:** Any operation producing a type mismatch, schema violation, or logical inconsistency is an invalid operation and must be discarded.
|
|
||||||
|
|
||||||
3. **Atomic and Consistent State Modification.**
|
|
||||||
- **Directive:** Modification of a shared resource must be performed as an atomic operation or within an ACID-compliant transaction.
|
|
||||||
- **Constraint:** Operations that can lead to race conditions or inconsistent state are prohibited. Immutability is the required default.
|
|
||||||
|
|
||||||
4. **Zero-Trust Security (Inviolable Safety Constraint).**
|
|
||||||
- **Directive:** Secrets must not be stored as literal values in source code. They must be loaded at runtime from a secure external source.
|
|
||||||
- **Constraint:** Generated access policies must adhere to the Principle of Least Privilege.
|
|
||||||
|
|
||||||
5. **Supply Chain Security (Inviolable Safety Constraint).**
|
|
||||||
- **Directive:** All external dependencies must be sourced from trusted repositories and defined in a lockfile for deterministic resolution.
|
|
||||||
- **Constraint:** The dependency graph must be scanned for known CVEs. Dependencies with critical vulnerabilities are prohibited.
|
|
||||||
|
|
||||||
6. **Deterministic and Reproducible Builds.**
|
|
||||||
- **Directive:** From a given source commit, the build process must produce a byte-for-byte identical artifact in every execution.
|
|
||||||
- **Constraint:** All automated tests must be deterministic. A regression test codifying the fixed bug's failure condition must be included with the fix.
|
|
||||||
|
|
||||||
7. **Structured, Traceable Logging.**
|
|
||||||
- **Directive:** All processes must emit structured (JSON) logs for significant events. All log entries for a request must contain the same unique trace ID.
|
|
||||||
- **Constraint:** Error conditions must be explicitly logged with context and propagated. Errors must not be silently suppressed.
|
|
||||||
|
|
||||||
8. **Strict API Contract Enforcement.**
|
|
||||||
- **Directive:** All network communication must strictly conform to its published, versioned API contract.
|
|
||||||
- **Constraint:** Any network call violating the contract must be rejected. Breaking changes require a major version increment (SemVer).
|
|
||||||
|
|
||||||
9. **Distributed System Consensus.**
|
|
||||||
- **Directive:** Changes to shared state across a distributed system are committed only after a formal consensus algorithm confirms quorum.
|
|
||||||
- **Constraint:** Nodes in a minority partition must enter a read-only or unavailable state to prevent a split-brain scenario.
|
|
||||||
|
|
||||||
10. **Execution Planning and Pre-flight Validation (Think Before Acting).**
|
|
||||||
- **Directive:** For any multi-step task, a detailed execution plan (sequence of commands and file modifications) must be formulated before any state-modifying action is taken.
|
|
||||||
- **Constraint:** Before executing a command, the agent must first use a validation or dry-run flag (e.g., `--dry-run`, `--check`) if available. The operation may only proceed if the pre-flight check passes without error.
|
|
||||||
|
|
||||||
11. **Post-Failure Root Cause Analysis (Evaluate Mistakes from Logs).**
|
|
||||||
- **Directive:** Upon command execution failure (non-zero exit code), the current execution plan must be halted, and the agent must enter a diagnostic mode.
|
|
||||||
- **Constraint:** In diagnostic mode, the agent is required to: 1) Capture and parse the complete `stdout` and `stderr` logs. 2) Identify the specific error message or stack trace. 3) Correlate the error with the last command to form a root cause hypothesis. 4) Formulate a new, corrective execution plan based on the analysis.
|
|
||||||
|
|
||||||
12. **Context-Aware File System Operations.**
|
|
||||||
- **Directive:** Before modifying any file, its full content must be read to establish context. All edits must be based on an in-memory understanding of the file's current state.
|
|
||||||
- **Constraint:** Blind file operations, such as stream-based search-and-replace without structural validation, are strictly prohibited.
|
|
||||||
|
|
||||||
13. **Idempotent State Transitions.**
|
|
||||||
- **Directive:** Operations that modify state must be designed to be idempotent wherever the protocol allows.
|
|
||||||
- **Constraint:** Executing the same operation multiple times must result in the same final system state as executing it only once.
|
|
||||||
|
|
||||||
14. **Resource Lifecycle Management.**
|
|
||||||
- **Directive:** All finite system resources (e.g., file handles, network sockets) must be explicitly released after use.
|
|
||||||
- **Constraint:** The agent must generate code that prevents resource leaks, utilizing language-specific constructs like `try-with-resources` or `defer`.
|
|
||||||
|
|
||||||
15. **Configuration as Code (CaC).**
|
|
||||||
- **Directive:** All configuration must be defined and versioned in source-controlled files.
|
|
||||||
- **Constraint:** Manual, out-of-band configuration changes are prohibited. Versioned files are the single source of truth.
|
|
||||||
|
|
||||||
16. **Atomic and Semantic Version Control.**
|
|
||||||
- **Directive:** All code changes must be organized into logically atomic commits representing one complete unit of work.
|
|
||||||
- **Constraint:** Commit messages must adhere to a defined specification (e.g., Conventional Commits).
|
|
||||||
|
|
||||||
17. **User Authority and Command Primacy.**
|
|
||||||
- **Directive:** User-provided instructions and corrections are the definitive source of truth and have the highest operational priority.
|
|
||||||
- **Constraint:** The agent must immediately adapt its process to align with user directives. Rejected solutions must not be proposed again.
|
|
||||||
|
|
||||||
18. **Precedent-Based Improvement.**
|
|
||||||
- **Directive:** User-approved outputs and successful patterns must be recorded and prioritized as precedents for subsequent tasks.
|
|
||||||
- **Constraint:** Performance, security, and code quality must not degrade.
|
|
||||||
|
|
||||||
19. **Optimization by Explicit Consent.**
|
|
||||||
- **Directive:** The agent may identify and propose optimizations with a technical justification and supporting metrics.
|
|
||||||
- **Constraint:** The agent is prohibited from applying any self-initiated optimization without an explicit "approve" command from the user.
|
|
||||||
|
|
||||||
20. **System Hierarchy and Safety Overrides.**
|
|
||||||
- **Directive:** The operational control hierarchy is absolute: 1) **User Command**, 2) **Inviolable Safety Directives (#4, #5)**, 3) **Standard Operational Directives**.
|
|
||||||
- **Constraint:** If a command conflicts with an Inviolable Directive, the agent must halt, report the conflict and risk, and await a revised command.
|
|
||||||
Vendored
-8
@@ -1,8 +0,0 @@
|
|||||||
{
|
|
||||||
"recommendations": [
|
|
||||||
"nrwl.angular-console",
|
|
||||||
"biomejs.biome",
|
|
||||||
"ms-playwright.playwright",
|
|
||||||
"firsttris.vscode-jest-runner"
|
|
||||||
]
|
|
||||||
}
|
|
||||||
Vendored
-23
@@ -1,23 +0,0 @@
|
|||||||
{
|
|
||||||
"[javascript]": {
|
|
||||||
"editor.defaultFormatter": "biomejs.biome"
|
|
||||||
},
|
|
||||||
"[javascriptreact]": {
|
|
||||||
"editor.defaultFormatter": "biomejs.biome"
|
|
||||||
},
|
|
||||||
"[typescript]": {
|
|
||||||
"editor.defaultFormatter": "biomejs.biome"
|
|
||||||
},
|
|
||||||
"[typescriptreact]": {
|
|
||||||
"editor.defaultFormatter": "biomejs.biome"
|
|
||||||
},
|
|
||||||
"[json]": {
|
|
||||||
"editor.defaultFormatter": "biomejs.biome"
|
|
||||||
},
|
|
||||||
"[jsonc]": {
|
|
||||||
"editor.defaultFormatter": "biomejs.biome"
|
|
||||||
},
|
|
||||||
"editor.codeActionsOnSave": {
|
|
||||||
"source.organizeImports.biome": "explicit"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
+54
-72
@@ -5,9 +5,6 @@
|
|||||||
```
|
```
|
||||||
asepharyana-hub/
|
asepharyana-hub/
|
||||||
├── apps/ # Application services (Git submodules)
|
├── apps/ # Application services (Git submodules)
|
||||||
│ ├── elysia/ # Realtime API (Bun/Elysia/Drizzle/Redis)
|
|
||||||
│ ├── react/ # Frontend SPA (React/Vite/TanStack)
|
|
||||||
│ ├── rust-auth/ # IAM & auth service (Axum/SeaORM)
|
|
||||||
│ └── scraper/ # Web scraper service
|
│ └── scraper/ # Web scraper service
|
||||||
├── docs/ # Documentation
|
├── docs/ # Documentation
|
||||||
│ ├── adr/ # Architecture Decision Records
|
│ ├── adr/ # Architecture Decision Records
|
||||||
@@ -31,19 +28,11 @@ asepharyana-hub/
|
|||||||
|
|
||||||
## Technology Stack
|
## Technology Stack
|
||||||
|
|
||||||
### Backend Services
|
### Services
|
||||||
|
|
||||||
| Service | Language/Runtime | Framework | Database | Key Libraries |
|
| Service | Language/Runtime | Framework | Database | Key Libraries |
|
||||||
| ------------- | ---------------- | ---------- | ------------------------ | ------------------------------------------------------------ |
|
| --------- | ---------------- | --------- | -------- | ------------- |
|
||||||
| **rust-auth** | Rust | Axum 0.8 | PostgreSQL (sqlx) | sqlx, jsonwebtoken, argon2, redis, opentelemetry, prometheus |
|
| **scraper** | _(submodule)_ | — | — | — |
|
||||||
| **elysia** | TypeScript/Bun | Elysia 1.4 | PostgreSQL (Drizzle ORM) | Drizzle ORM, Redis (ioredis), JWT (jose), OTel, Swagger |
|
|
||||||
| **scraper** | _(submodule)_ | — | — | — |
|
|
||||||
|
|
||||||
### Frontend
|
|
||||||
|
|
||||||
| Service | Framework | Build Tool | Key Libraries |
|
|
||||||
| --------- | --------- | ---------- | --------------------------------------------------------------------------------------------------- |
|
|
||||||
| **react** | React 19 | Vite 7 | TanStack Router + Query, Three.js/React Three Fiber, Tailwind CSS 4, Zustand, Recharts, tsParticles |
|
|
||||||
|
|
||||||
### Infrastructure
|
### Infrastructure
|
||||||
|
|
||||||
@@ -53,7 +42,9 @@ asepharyana-hub/
|
|||||||
| Container Runtime | Docker + Docker Compose | Service isolation and orchestration |
|
| Container Runtime | Docker + Docker Compose | Service isolation and orchestration |
|
||||||
| Container Registry | GHCR (ghcr.io) | Docker image storage |
|
| Container Registry | GHCR (ghcr.io) | Docker image storage |
|
||||||
| Networking | Tailscale | Secure overlay network between VPS nodes |
|
| Networking | Tailscale | Secure overlay network between VPS nodes |
|
||||||
| Cache | Redis (Alpine) | Session store, rate limit counters, caching |
|
| Message Bus | NATS + JetStream | Event-driven pub/sub, job queues, streaming |
|
||||||
|
| Runtime Sidecar | Dapr | Service invocation, pub/sub abstraction, state management |
|
||||||
|
| Cache & State | Redis (Alpine) | Session store, rate limit counters, caching, Dapr state store |
|
||||||
| CI/CD | GitHub Actions | Build, test, deploy automation |
|
| CI/CD | GitHub Actions | Build, test, deploy automation |
|
||||||
|
|
||||||
## Infrastructure
|
## Infrastructure
|
||||||
@@ -62,7 +53,7 @@ asepharyana-hub/
|
|||||||
|
|
||||||
Traefik runs as the entry point for all HTTP/S traffic. It is configured via:
|
Traefik runs as the entry point for all HTTP/S traffic. It is configured via:
|
||||||
|
|
||||||
- **Static config**: `infra/traefik/traefik.yaml` — entry points, providers, plugins
|
- **Static config**: CLI arguments in `infra/compose/traefik.yml` — entry points, providers, plugins
|
||||||
- **Dynamic config**: `infra/traefik/dynamic/` — routers, services, middlewares, TLS
|
- **Dynamic config**: `infra/traefik/dynamic/` — routers, services, middlewares, TLS
|
||||||
- **Docker provider**: Auto-discovers containers with `traefik.enable=true` labels
|
- **Docker provider**: Auto-discovers containers with `traefik.enable=true` labels
|
||||||
- **File provider**: Loads `apps.yaml` (routers/services), `middlewares.yaml`, `ssl.yaml`
|
- **File provider**: Loads `apps.yaml` (routers/services), `middlewares.yaml`, `ssl.yaml`
|
||||||
@@ -90,7 +81,7 @@ Shared services:
|
|||||||
Service compose files are combined during deployment:
|
Service compose files are combined during deployment:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
docker compose -f traefik.yml -f shared.yml -f elysia.yml -f react.yml ... up -d
|
docker compose -f traefik.yml -f shared.yml -f scraper.yml up -d
|
||||||
```
|
```
|
||||||
|
|
||||||
### Tailscale Networking
|
### Tailscale Networking
|
||||||
@@ -98,10 +89,9 @@ docker compose -f traefik.yml -f shared.yml -f elysia.yml -f react.yml ... up -d
|
|||||||
```mermaid
|
```mermaid
|
||||||
graph TB
|
graph TB
|
||||||
subgraph "Tailnet (100.64.0.0/10)"
|
subgraph "Tailnet (100.64.0.0/10)"
|
||||||
IMRNES["imrnes (100.108.1.124)"]
|
IMRNES["imrnes (100.121.180.82)"]
|
||||||
ORANGE["orange (100.96.248.86)"]
|
ORANGEVPS["orangevps (100.79.111.61)"]
|
||||||
ARCH["archlinux (100.114.19.66)"]
|
ARCH["archlinux (100.84.39.83)"]
|
||||||
LAPTOP["laptop-2f6e1iph (100.86.195.29)"]
|
|
||||||
end
|
end
|
||||||
|
|
||||||
subgraph "imrnes Services"
|
subgraph "imrnes Services"
|
||||||
@@ -109,28 +99,16 @@ graph TB
|
|||||||
REDIS[Redis]
|
REDIS[Redis]
|
||||||
end
|
end
|
||||||
|
|
||||||
subgraph "orange Containers"
|
subgraph "orangevps Containers"
|
||||||
TRAEFIK[Traefik :443]
|
TRAEFIK[Traefik :443]
|
||||||
RUST_AUTH[rust-auth :3000]
|
|
||||||
ELYSIA[elysia-api :4092]
|
|
||||||
REACT[react-web :80]
|
|
||||||
SCRAPER[scraper-api :4091]
|
SCRAPER[scraper-api :4091]
|
||||||
end
|
end
|
||||||
|
|
||||||
TRAEFIK --> RUST_AUTH
|
|
||||||
TRAEFIK --> ELYSIA
|
|
||||||
TRAEFIK --> REACT
|
|
||||||
TRAEFIK --> SCRAPER
|
TRAEFIK --> SCRAPER
|
||||||
|
|
||||||
RUST_AUTH -.->|Tailscale IP| PG
|
|
||||||
ELYSIA -.->|Tailscale IP| PG
|
|
||||||
RUST_AUTH -.->|Tailscale IP| REDIS
|
|
||||||
ELYSIA -.->|Tailscale IP| REDIS
|
|
||||||
|
|
||||||
style IMRNES fill:#3a7,color:#fff
|
style IMRNES fill:#3a7,color:#fff
|
||||||
style ORANGE fill:#37a,color:#fff
|
style ORANGEVPS fill:#37a,color:#fff
|
||||||
style ARCH fill:#773,color:#fff
|
style ARCH fill:#773,color:#fff
|
||||||
style LAPTOP fill:#777,color:#fff
|
|
||||||
```
|
```
|
||||||
|
|
||||||
Container-to-Tailscale connectivity requires a systemd service that adds a route to the main routing table:
|
Container-to-Tailscale connectivity requires a systemd service that adds a route to the main routing table:
|
||||||
@@ -139,7 +117,7 @@ Container-to-Tailscale connectivity requires a systemd service that adds a route
|
|||||||
ip route add 100.64.0.0/10 dev tailscale0 table main
|
ip route add 100.64.0.0/10 dev tailscale0 table main
|
||||||
```
|
```
|
||||||
|
|
||||||
This is managed by `/etc/systemd/system/tailscale-routes.service` on the `orange` VPS.
|
This is managed by `/etc/systemd/system/tailscale-routes.service` on the `orangevps` VPS.
|
||||||
|
|
||||||
## Data Flow
|
## Data Flow
|
||||||
|
|
||||||
@@ -155,7 +133,7 @@ sequenceDiagram
|
|||||||
participant Redis as Redis (imrnes via Tailscale)
|
participant Redis as Redis (imrnes via Tailscale)
|
||||||
|
|
||||||
User->>DNS: asepharyana.my.id
|
User->>DNS: asepharyana.my.id
|
||||||
DNS->>User: A/AAAA record → orange VPS IP
|
DNS->>User: A/AAAA record → orangevps VPS IP
|
||||||
User->>Traefik: HTTPS request :443
|
User->>Traefik: HTTPS request :443
|
||||||
Traefik->>Traefik: TLS termination
|
Traefik->>Traefik: TLS termination
|
||||||
Traefik->>Traefik: Middleware chain (headers, rate-limit, buffer)
|
Traefik->>Traefik: Middleware chain (headers, rate-limit, buffer)
|
||||||
@@ -165,8 +143,8 @@ sequenceDiagram
|
|||||||
App->>DB: sqlx/Drizzle query via Tailscale
|
App->>DB: sqlx/Drizzle query via Tailscale
|
||||||
DB-->>App: Result set
|
DB-->>App: Result set
|
||||||
else Cache lookup
|
else Cache lookup
|
||||||
App->>Redis: GET/SET via Tailscale
|
App->>Cache: GET/SET via Tailscale
|
||||||
Redis-->>App: Cached value
|
Cache-->>App: Cached value
|
||||||
end
|
end
|
||||||
|
|
||||||
App-->>Traefik: HTTP response
|
App-->>Traefik: HTTP response
|
||||||
@@ -215,7 +193,7 @@ flowchart LR
|
|||||||
|
|
||||||
### VPS Deployment
|
### VPS Deployment
|
||||||
|
|
||||||
The `orange` VPS (Tailscale `100.96.248.86`) hosts all application containers:
|
The `orangevps` VPS (Tailscale `100.79.111.61`) hosts all application containers:
|
||||||
|
|
||||||
1. GitHub Actions SSHes into the VPS
|
1. GitHub Actions SSHes into the VPS
|
||||||
2. Production secrets are written as `.env`
|
2. Production secrets are written as `.env`
|
||||||
@@ -228,7 +206,7 @@ The `orange` VPS (Tailscale `100.96.248.86`) hosts all application containers:
|
|||||||
|
|
||||||
### Selective Deployment
|
### Selective Deployment
|
||||||
|
|
||||||
The deploy workflow supports selective updates — if only `infra/compose/elysia.yml` changed, only `elysia-api` is pulled and recreated, avoiding disruption to other services.
|
The deploy workflow supports selective updates — if only one compose file changed, only the corresponding service is pulled and recreated, avoiding disruption to other services.
|
||||||
|
|
||||||
```mermaid
|
```mermaid
|
||||||
graph TB
|
graph TB
|
||||||
@@ -265,7 +243,7 @@ Each application lives in its own Git repository and is imported as a submodule
|
|||||||
|
|
||||||
### Submodule Lifecycle
|
### Submodule Lifecycle
|
||||||
|
|
||||||
1. Developer pushes to a submodule (e.g., `apps/elysia`)
|
1. Developer pushes to a submodule (e.g., `apps/scraper`)
|
||||||
2. Submodule's GitHub Action dispatches `repository_dispatch` to the super-repo with the service name and new SHA
|
2. Submodule's GitHub Action dispatches `repository_dispatch` to the super-repo with the service name and new SHA
|
||||||
3. Super-repo detects the dispatch, waits for the SHA to be fetchable, then builds only that service
|
3. Super-repo detects the dispatch, waits for the SHA to be fetchable, then builds only that service
|
||||||
4. The compose manifest is updated and committed with the new SHA tag
|
4. The compose manifest is updated and committed with the new SHA tag
|
||||||
@@ -275,58 +253,62 @@ Each application lives in its own Git repository and is imported as a submodule
|
|||||||
|
|
||||||
```bash
|
```bash
|
||||||
# Update a single submodule to latest
|
# Update a single submodule to latest
|
||||||
cd apps/elysia
|
cd apps/scraper
|
||||||
git checkout main
|
git checkout main
|
||||||
git pull
|
git pull
|
||||||
cd ../..
|
cd ../..
|
||||||
git add apps/elysia
|
git add apps/scraper
|
||||||
git commit -m "chore(elysia): update submodule to latest"
|
git commit -m "chore(scraper): update submodule to latest"
|
||||||
|
|
||||||
# Update all submodules
|
|
||||||
git submodule update --remote --merge
|
|
||||||
```
|
```
|
||||||
|
|
||||||
## Service Mesh & Inter-Service Communication
|
## Service Mesh & Inter-Service Communication
|
||||||
|
|
||||||
|
### HTTP (External + Internal via Traefik)
|
||||||
|
External traffic and internal HTTP calls route through Traefik. Services on `app-shared-net` can also communicate directly by container name.
|
||||||
|
|
||||||
|
### Event-Driven (NATS + Dapr)
|
||||||
|
NATS with JetStream provides a persistent message backbone. Each service has a Dapr sidecar that abstracts pub/sub, service invocation, and state management.
|
||||||
|
|
||||||
```mermaid
|
```mermaid
|
||||||
graph LR
|
graph TB
|
||||||
subgraph "External"
|
subgraph "External"
|
||||||
WWW[Internet]
|
WWW[Internet]
|
||||||
end
|
end
|
||||||
|
|
||||||
subgraph "Orange VPS"
|
subgraph "Orange VPS"
|
||||||
TRAEFIK[Traefik :443]
|
TRAEFIK[Traefik :443]
|
||||||
|
|
||||||
subgraph "app-shared-net"
|
subgraph "app-shared-net"
|
||||||
REACT[react-web<br/>:80]
|
NATS[NATS + JetStream<br/>:4222]
|
||||||
ELYSIA[elysia-api<br/>:4092]
|
DAPR_PLACEMENT[Dapr Placement<br/>:50005]
|
||||||
RUST_AUTH[rust-auth<br/>:3000]
|
|
||||||
SCRAPER[scraper-api<br/>:4091]
|
|
||||||
REDIS[redis<br/>:6379]
|
|
||||||
end
|
|
||||||
end
|
|
||||||
|
|
||||||
subgraph "Imrnes VPS"
|
subgraph "Service: scraper-api"
|
||||||
PG[(PostgreSQL<br/>:5432)]
|
SCRAPER[scraper-api<br/>:4091]
|
||||||
REDIS_IMR[Redis<br/>:6379]
|
DAPR_SIDECAR[Dapr Sidecar<br/>:3500]
|
||||||
|
SCRAPER --- DAPR_SIDECAR
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
DAPR_SIDECAR -.->|gRPC pub/sub| NATS
|
||||||
|
DAPR_SIDECAR -.->|placement| DAPR_PLACEMENT
|
||||||
end
|
end
|
||||||
|
|
||||||
WWW -->|HTTPS| TRAEFIK
|
WWW -->|HTTPS| TRAEFIK
|
||||||
TRAEFIK --> REACT
|
|
||||||
TRAEFIK --> ELYSIA
|
|
||||||
TRAEFIK --> RUST_AUTH
|
|
||||||
TRAEFIK --> SCRAPER
|
TRAEFIK --> SCRAPER
|
||||||
|
|
||||||
ELYSIA -->|Tailscale| PG
|
|
||||||
RUST_AUTH -->|Tailscale| PG
|
|
||||||
ELYSIA -->|internal| REDIS
|
|
||||||
RUST_AUTH -->|internal| REDIS
|
|
||||||
ELYSIA -->|Tailscale| REDIS_IMR
|
|
||||||
RUST_AUTH -->|Tailscale| REDIS_IMR
|
|
||||||
```
|
```
|
||||||
|
|
||||||
|
### Communication Patterns
|
||||||
|
|
||||||
|
| Pattern | Mechanism | Use Case |
|
||||||
|
|---------|-----------|----------|
|
||||||
|
| External HTTP | Traefik → Service | User requests, API calls |
|
||||||
|
| Internal HTTP | Service → Service (via Traefik or direct) | Synchronous queries |
|
||||||
|
| Pub/Sub Event | Dapr sidecar → NATS JetStream | Async notifications, image cache events |
|
||||||
|
| Service Invocation | Dapr sidecar gRPC | Cross-service RPC with retry & observability |
|
||||||
|
| State Store | Dapr → Redis | Shared state, job progress |
|
||||||
|
|
||||||
## Observability
|
## Observability
|
||||||
|
|
||||||
- **Prometheus metrics**: Available on rust-auth via `axum-prometheus`
|
- **Traefik access logs**: JSON format, logged at INFO level
|
||||||
- **Traefik access logs**: JSON format, logged at INFO level
|
- **Traefik access logs**: JSON format, logged at INFO level
|
||||||
- **Dashboard**: Traefik dashboard at `traefik.asepharyana.my.id` (secured)
|
- **Dashboard**: Traefik dashboard at `traefik.asepharyana.my.id` (secured)
|
||||||
|
|||||||
@@ -12,8 +12,22 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|||||||
- Restructured the repository into a lightweight hub repo with standalone app submodules.
|
- Restructured the repository into a lightweight hub repo with standalone app submodules.
|
||||||
- Simplified root tooling to plain `package.json` scripts and per-service commands.
|
- Simplified root tooling to plain `package.json` scripts and per-service commands.
|
||||||
- Kept infrastructure, deployment workflows, and documentation in the root hub repo.
|
- Kept infrastructure, deployment workflows, and documentation in the root hub repo.
|
||||||
|
- Cleaned up Traefik SSL config: removed legacy `asephstech`/`asephscloud` cert references, synced volume mounts, fixed `api.insecure`.
|
||||||
|
- Pruned `.env.example` from 145 legacy vars (Firebase, Discord, Coolify, Portainer, YouTube, etc.) to 30 focused vars.
|
||||||
|
- Optimized `scraper.Dockerfile`: removed Node.js and Chromium from runtime image.
|
||||||
|
- Simplified CI/CD workflows: removed orphan container reference, commented code blocks.
|
||||||
|
- Cleaned up scripts: removed stale MySQL config, fixed package references, simplified update-deps.
|
||||||
|
- Added NATS + JetStream message broker infrastructure (`infra/compose/nats.yml`).
|
||||||
|
- Added Dapr runtime infrastructure: placement service, sidecar pattern, pub/sub + state store components.
|
||||||
|
- Integrated Dapr sidecar into scraper service (`infra/compose/scraper.yml`).
|
||||||
|
- Updated deployment order: shared → NATS → Dapr → Traefik → apps.
|
||||||
|
- Added `docs/add-dapr-service.md` guide for adding Dapr to new services.
|
||||||
|
|
||||||
### Removed
|
### Removed
|
||||||
|
|
||||||
- Removed deprecated services from apps, compose files, Dockerfiles, Traefik routes, and workflows.
|
- Removed deprecated services from apps, compose files, Dockerfiles, Traefik routes, and workflows.
|
||||||
- Removed stale monorepo orchestration configs and hook tooling from the root repo.
|
- Removed stale monorepo orchestration configs and hook tooling from the root repo.
|
||||||
|
- Removed `.nvmrc` (duplicate of `.node-version`), `renovate.json` (using dependabot).
|
||||||
|
- Removed stale documentation: `dependency-map.md`, `observability.md`, `handoff-log.jsonl`, `quality-gates.json`, `workflow-state.json`.
|
||||||
|
- Removed deprecated `docs/superpowers/` design docs.
|
||||||
|
- Removed `infra/config/mysql/` (no active MySQL service) and `docs/config/squid.conf.archived`.
|
||||||
|
|||||||
@@ -0,0 +1,131 @@
|
|||||||
|
# CLAUDE.md
|
||||||
|
|
||||||
|
This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
|
||||||
|
|
||||||
|
## Repository Overview
|
||||||
|
|
||||||
|
Asepharyana Hub is a **hub monorepo** for Asep Haryana Saputra's portfolio ecosystem. Application services live in separate repos imported as Git submodules under `apps/`. Infrastructure (Docker Compose, Traefik, Dapr) lives in `infra/`.
|
||||||
|
|
||||||
|
```
|
||||||
|
asepharyana-hub/
|
||||||
|
├── apps/ # Git submodules — each app is its own repo
|
||||||
|
│ ├── hub/ # Personal portfolio SPA (asepharyana-hub-hub)
|
||||||
|
│ └── scraper/ # Rust scraper API (asepharyana-hub-scraper)
|
||||||
|
├── docs/ # ADRs, deployment guide, new-app guide
|
||||||
|
├── infra/
|
||||||
|
│ ├── compose/ # One Docker Compose file per service
|
||||||
|
│ ├── dapr/ # Dapr config + component definitions
|
||||||
|
│ ├── docker/ # Dockerfiles per service
|
||||||
|
│ └── traefik/ # Reverse proxy config (static + dynamic)
|
||||||
|
├── scripts/ # Utility scripts (cleanup, update-deps, git hooks)
|
||||||
|
└── .github/workflows/ # CI/CD pipelines
|
||||||
|
```
|
||||||
|
|
||||||
|
### Submodule Strategy
|
||||||
|
- Each app in `apps/` is a separate Git repo imported as a submodule. Code changes happen in the submodule repo, not here.
|
||||||
|
- Submodule pointers are updated by CI/CD (via `repository_dispatch` or manual commit).
|
||||||
|
- Current submodules:
|
||||||
|
- `apps/hub` → `asepharyana/asepharyana-hub-hub`
|
||||||
|
- `apps/scraper` → `asepharyana/asepharyana-hub-scraper`
|
||||||
|
- `apps/llm-api` → `asepharyana/asepharyana-hub-llm-api`
|
||||||
|
- `apps/tools` → `asepharyana/asepharyana-hub-tools`.
|
||||||
|
|
||||||
|
### Infrastructure Stack
|
||||||
|
- **Traefik v3.6** — reverse proxy, TLS termination, middleware chain, Prometheus metrics (`--metrics.prometheus=true`)
|
||||||
|
- **NATS + JetStream** — message broker with persistent streaming
|
||||||
|
- **Dapr** — sidecar runtime (pub/sub abstraction, state management, service invocation)
|
||||||
|
- **Redis (Alpine)** — cache, session store, Dapr state store & pub/sub backend
|
||||||
|
- **Prometheus** — metrics backend with Docker service discovery (`docker_sd_configs`). Auto-discovers containers with `prometheus.io/scrape=true` label.
|
||||||
|
- **Jaeger** — distributed tracing backend (all-in-one), OTLP receiver
|
||||||
|
- **Tailscale** — secure overlay network between VPS nodes (PostgreSQL on `imrnes`, containers on `orangevps`)
|
||||||
|
|
||||||
|
### Monitoring
|
||||||
|
- **Hub dashboard** at `/dashboard` (Next.js client page, auto-refresh 15s)
|
||||||
|
- **Dashboard API** at `/api/dashboard` — returns JSON with Docker containers, Jaeger traces, Prometheus metrics (RPS, latency, errors, node CPU/RAM/Disk)
|
||||||
|
- **Docker socket** mounted on `hub` container (`--group-add 988`) for container discovery
|
||||||
|
- **Prometheus** auto-scrapes Traefik for per-service request metrics
|
||||||
|
|
||||||
|
### Networking
|
||||||
|
- All containers join `app-shared-net` (external Docker bridge network). Service discovery via Docker DNS (container name aliases).
|
||||||
|
- Traefik handles all external HTTP/S traffic on port 443.
|
||||||
|
- Cross-VPS traffic (DB, Redis) goes through Tailscale (`100.64.0.0/10`). Container-to-Tailscale connectivity requires a route in the main routing table (managed by `tailscale-routes.service`).
|
||||||
|
|
||||||
|
## Commands
|
||||||
|
|
||||||
|
```bash
|
||||||
|
make init-submodules # Initialize submodules after clone
|
||||||
|
make dev # Start shared dev infrastructure (Redis)
|
||||||
|
make update-submodules # Update all submodules to latest
|
||||||
|
|
||||||
|
bun run check # Biome lint + format + write
|
||||||
|
bun run ci # Biome CI mode (no writes, exit code on issues)
|
||||||
|
bun run format # Format only
|
||||||
|
bun run lint # Lint only
|
||||||
|
|
||||||
|
docker build -f infra/docker/scraper.Dockerfile -t scraper-api:latest . # Build image
|
||||||
|
```
|
||||||
|
|
||||||
|
### Validate YAML
|
||||||
|
```bash
|
||||||
|
python -c "import pathlib, yaml; [yaml.safe_load(open(p)) for p in pathlib.Path('infra').rglob('*.yml')]"
|
||||||
|
for f in infra/compose/*.yml; do docker compose -f "$f" config >/dev/null && echo "OK $f"; done
|
||||||
|
```
|
||||||
|
|
||||||
|
## CI/CD Workflows
|
||||||
|
|
||||||
|
| Workflow | Trigger | Action |
|
||||||
|
|----------|---------|--------|
|
||||||
|
| `lint.yml` | PR/push to main touching `*.json`, `*.js`, `biome.json` | `bun run ci` (Biome lint) |
|
||||||
|
| `docker-build-push.yml` | Push to main touching `apps/**`/`infra/**`, or `repository_dispatch` | Build Docker images per changed service, push to GHCR, update compose manifests |
|
||||||
|
| `deploy-docker.yml` | After build completes, or push touching `infra/**` | SSH to VPS (orangevps), pull images, restart containers selectively |
|
||||||
|
| `security.yml` | PR to main + weekly Monday | CodeQL analysis (Rust) |
|
||||||
|
| `update-submodule.yml` | `repository_dispatch` | Update submodule pointer in hub repo |
|
||||||
|
|
||||||
|
### Deployment Order
|
||||||
|
1. `shared.yml` (Redis)
|
||||||
|
2. `nats.yml` (NATS + JetStream)
|
||||||
|
3. `dapr.yml` (Dapr placement)
|
||||||
|
4. `traefik.yml` (Reverse proxy)
|
||||||
|
5. Service compose files (app + Dapr sidecar)
|
||||||
|
|
||||||
|
### Secrets Required for Deploy
|
||||||
|
`SSH_PRIVATE_KEY`, `VPS_HOST`, `VPS_USER`, `VPS_TARGET_DIR`, `ENV_FILE_PRODUCTION`
|
||||||
|
|
||||||
|
## Infrastructure Patterns
|
||||||
|
|
||||||
|
### Compose File Pattern
|
||||||
|
Each service gets one compose file. Containers join `app-shared-net` with a `container_name` alias for DNS. The network is declared `external: true`.
|
||||||
|
|
||||||
|
### Dapr Sidecar Pattern
|
||||||
|
Each app gets a companion `daprd` sidecar container. Dapr components (pubsub, statestore) are mounted from `infra/dapr/components/`. The sidecar communicates with NATS for pub/sub and Dapr placement for actor coordination.
|
||||||
|
|
||||||
|
### Traefik Routing
|
||||||
|
- Routers + services defined in `infra/traefik/dynamic/apps.yaml`
|
||||||
|
- Subdomain pattern: `<service>.asepharyana.my.id` and `<service>.asepharya.web.id`
|
||||||
|
- TLS certs from volume mounts (not auto-ACME)
|
||||||
|
- Middleware chain: `secure-headers` → `compress` → `retry` → `rate-limit` → `buffer`
|
||||||
|
|
||||||
|
### Image Tagging
|
||||||
|
- `sha-<short-sha>` — immutable, for deterministic rollbacks
|
||||||
|
- `latest` — mutable, for convenience
|
||||||
|
- Registry: `ghcr.io/asepharyana/asepharyana-hub/<service>`
|
||||||
|
- Build cache: `sha-<short>-buildcache` (registry-based caching)
|
||||||
|
|
||||||
|
## Adding a New Service
|
||||||
|
|
||||||
|
1. Create a separate repo for the app code
|
||||||
|
2. Add as submodule: `git submodule add <url> apps/<name>`
|
||||||
|
3. Create Dockerfile in `infra/docker/`
|
||||||
|
4. Create compose file in `infra/compose/` (app + Dapr sidecar)
|
||||||
|
5. Add Traefik router in `infra/traefik/dynamic/apps.yaml`
|
||||||
|
6. Add build job in `.github/workflows/docker-build-push.yml`
|
||||||
|
7. See `docs/add-new-app.md` for full guide
|
||||||
|
|
||||||
|
## Commit Convention
|
||||||
|
|
||||||
|
Format: `<type>(<scope>): <description>`
|
||||||
|
|
||||||
|
Types: `feat`, `fix`, `chore`, `docs`, `refactor`, `test`, `ci`, `perf`, `style`
|
||||||
|
Scopes: `scraper`, `infra`, `ci`, `dapr`, `nats`, `docs`, `deps`, `scripts`, `root`
|
||||||
|
|
||||||
|
Scope is required. Use imperative mood. No period at end of subject line. Co-Authored-By footer for AI-generated commits.
|
||||||
+17
-69
@@ -14,9 +14,8 @@
|
|||||||
## Prerequisites
|
## Prerequisites
|
||||||
|
|
||||||
- **Git** with LFS support
|
- **Git** with LFS support
|
||||||
- **Node.js** >= 22.11.0 (via `.node-version` or `.nvmrc`)
|
- **Node.js** >= 22.11.0 (via `.node-version`)
|
||||||
- **Bun** >= 1.3.11 (package manager)
|
- **Bun** >= 1.3.11 (package manager)
|
||||||
- **Rust** >= 1.89.0 (for Rust services)
|
|
||||||
- **Docker** and **Docker Compose** (for shared infrastructure)
|
- **Docker** and **Docker Compose** (for shared infrastructure)
|
||||||
|
|
||||||
## Local Setup
|
## Local Setup
|
||||||
@@ -40,18 +39,14 @@ This checks out all submodules at the pinned commit (not `main`). The submodules
|
|||||||
|
|
||||||
| Path | Remote |
|
| Path | Remote |
|
||||||
| ---------------- | --------------------------------------- |
|
| ---------------- | --------------------------------------- |
|
||||||
| `apps/elysia` | `asepharyana/asepharyana-hub-elysia` |
|
|
||||||
| `apps/scraper` | `asepharyana/asepharyana-hub-scraper` |
|
| `apps/scraper` | `asepharyana/asepharyana-hub-scraper` |
|
||||||
| `apps/react` | `asepharyana/asepharyana-hub-react` |
|
|
||||||
| `apps/rust-auth` | `asepharyana/asepharyana-hub-rust-auth` |
|
|
||||||
|
|
||||||
### 3. Install Dependencies per Service
|
### 3. Install Dependencies per Service
|
||||||
|
|
||||||
Install dependencies for TypeScript/Bun services:
|
Install dependencies for TypeScript/Bun services:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
cd apps/elysia && bun install && cd ../..
|
cd apps/scraper && bun install && cd ../..
|
||||||
cd apps/react && npm install && cd ../..
|
|
||||||
```
|
```
|
||||||
|
|
||||||
### 4. Start Shared Infrastructure
|
### 4. Start Shared Infrastructure
|
||||||
@@ -75,76 +70,42 @@ Key variables to configure:
|
|||||||
| Variable | Description |
|
| Variable | Description |
|
||||||
| -------------- | ----------------------------------------------------- |
|
| -------------- | ----------------------------------------------------- |
|
||||||
| `DATABASE_URL` | PostgreSQL connection (Tailscale IP to `imrnes` VPS) |
|
| `DATABASE_URL` | PostgreSQL connection (Tailscale IP to `imrnes` VPS) |
|
||||||
| `REDIS_URL` | Redis connection (`redis://localhost:6379` for local) |
|
|
||||||
| `JWT_SECRET` | JWT signing secret |
|
|
||||||
| `GITHUB_TOKEN` | GitHub personal access token |
|
| `GITHUB_TOKEN` | GitHub personal access token |
|
||||||
|
|
||||||
## Development Workflow
|
## Development Workflow
|
||||||
|
|
||||||
### Running Services
|
### Running Services
|
||||||
|
|
||||||
**Rust API (rust-auth):**
|
Refer to each service's own documentation for setup and development instructions.
|
||||||
|
|
||||||
```bash
|
|
||||||
cd apps/rust-auth
|
|
||||||
cargo run
|
|
||||||
```
|
|
||||||
|
|
||||||
**Elysia API (elysia):**
|
|
||||||
|
|
||||||
```bash
|
|
||||||
cd apps/elysia
|
|
||||||
bun run dev
|
|
||||||
```
|
|
||||||
|
|
||||||
**React Frontend (react):**
|
|
||||||
|
|
||||||
```bash
|
|
||||||
cd apps/react
|
|
||||||
npm run dev
|
|
||||||
```
|
|
||||||
|
|
||||||
### API Documentation
|
### API Documentation
|
||||||
|
|
||||||
- Rust OpenAPI: `http://localhost:4091/docs`
|
Refer to each service's own documentation for API docs and endpoints.
|
||||||
- Elysia Swagger: `http://localhost:4092/docs`
|
|
||||||
- Elysia AsyncAPI: `http://localhost:4092/docs-ws`
|
|
||||||
|
|
||||||
## Coding Standards
|
## Coding Standards
|
||||||
|
|
||||||
### Linting
|
### Linting
|
||||||
|
|
||||||
- **ESLint** with `@antfu/eslint-config` for TypeScript/JavaScript
|
- **Biome** for TypeScript/JavaScript formatting and linting
|
||||||
- **Cargo Clippy** for Rust
|
|
||||||
|
|
||||||
Run linting:
|
Run linting:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
# TypeScript/JavaScript
|
# TypeScript/JavaScript
|
||||||
eslint . --no-error-on-unmatched-pattern
|
bun run check
|
||||||
|
|
||||||
# Rust specific
|
|
||||||
cd apps/rust-auth && cargo clippy -- -D warnings
|
|
||||||
```
|
```
|
||||||
|
|
||||||
### Formatting
|
### Formatting
|
||||||
|
|
||||||
- **Prettier** for TypeScript/JavaScript/Markdown (config in `.prettierrc`)
|
- **Biome** for TypeScript/JavaScript
|
||||||
- Single quotes, 100 print width, 2-space indent, trailing commas
|
|
||||||
- **Cargo fmt** for Rust
|
|
||||||
- **EditorConfig** for general formatting (`.editorconfig`)
|
- **EditorConfig** for general formatting (`.editorconfig`)
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
# Prettier
|
# Format all
|
||||||
prettier --write .
|
bun run format
|
||||||
|
|
||||||
# Rust
|
|
||||||
cd apps/rust-auth && cargo fmt
|
|
||||||
```
|
```
|
||||||
|
|
||||||
### Rust Configuration
|
|
||||||
|
|
||||||
Rust services use edition `2024` with stable toolchain (nightly features may be used).
|
|
||||||
|
|
||||||
## Commit Message Format
|
## Commit Message Format
|
||||||
|
|
||||||
@@ -177,18 +138,15 @@ This project enforces **Conventional Commits** for all commit messages.
|
|||||||
### Examples
|
### Examples
|
||||||
|
|
||||||
```
|
```
|
||||||
feat(rust-auth): add OAuth2 Google login flow
|
feat(scraper): add new data source integration
|
||||||
fix(elysia): handle null JWT payload in auth middleware
|
chore: update biome config to v10
|
||||||
chore: update eslint config to v10
|
|
||||||
docs: add API endpoint documentation for scraper
|
docs: add API endpoint documentation for scraper
|
||||||
refactor(react): extract Header component from App
|
|
||||||
test(elysia): add unit tests for rate limiter
|
|
||||||
ci: migrate to CodeQL v3
|
ci: migrate to CodeQL v3
|
||||||
```
|
```
|
||||||
|
|
||||||
### Scopes
|
### Scopes
|
||||||
|
|
||||||
Common scopes: `rust-auth`, `elysia`, `react`, `scraper`, `infra`, `ci`, `deps`
|
Common scopes: `scraper`, `infra`, `ci`, `deps`
|
||||||
|
|
||||||
## Pull Request Process
|
## Pull Request Process
|
||||||
|
|
||||||
@@ -201,10 +159,9 @@ Common scopes: `rust-auth`, `elysia`, `react`, `scraper`, `infra`, `ci`, `deps`
|
|||||||
|
|
||||||
3. **Run checks locally** before pushing:
|
3. **Run checks locally** before pushing:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
cd apps/react && npx tsc --noEmit
|
bun run check
|
||||||
eslint . --no-error-on-unmatched-pattern
|
```
|
||||||
```
|
|
||||||
|
|
||||||
4. **Push and open a PR** against `main`. CI will automatically run:
|
4. **Push and open a PR** against `main`. CI will automatically run:
|
||||||
- **Lint** — ESLint across changed TypeScript files
|
- **Lint** — ESLint across changed TypeScript files
|
||||||
@@ -218,7 +175,7 @@ Common scopes: `rust-auth`, `elysia`, `react`, `scraper`, `infra`, `ci`, `deps`
|
|||||||
- Updates compose manifests to use the new SHA tags
|
- Updates compose manifests to use the new SHA tags
|
||||||
|
|
||||||
6. **Deployment Pipeline** triggers after a successful Docker build:
|
6. **Deployment Pipeline** triggers after a successful Docker build:
|
||||||
- SSHes into the VPS (`orange`, Tailscale IP `100.96.248.86`)
|
- SSHes into the VPS (`orangevps`, Tailscale IP `100.79.111.61`)
|
||||||
- Pulls updated Docker images
|
- Pulls updated Docker images
|
||||||
- Recreates only the changed containers
|
- Recreates only the changed containers
|
||||||
- All services share the `app-shared-net` Docker network
|
- All services share the `app-shared-net` Docker network
|
||||||
@@ -227,13 +184,4 @@ Common scopes: `rust-auth`, `elysia`, `react`, `scraper`, `infra`, `ci`, `deps`
|
|||||||
|
|
||||||
## Adding a New Service
|
## Adding a New Service
|
||||||
|
|
||||||
See `docs/add-new-app.md` for the complete step-by-step guide. In summary:
|
See `docs/add-new-app.md` for the complete step-by-step guide.
|
||||||
|
|
||||||
1. Create the app in `apps/<name>`
|
|
||||||
2. Add it as a Git submodule in `.gitmodules`
|
|
||||||
3. Register it in `infra/compose/<name>.yml`
|
|
||||||
4. Add a Dockerfile at `infra/docker/<name>.Dockerfile`
|
|
||||||
5. Add Traefik routing config in `infra/traefik/dynamic/apps.yaml`
|
|
||||||
6. Add CI entries in `.github/workflows/docker-build-push.yml`
|
|
||||||
7. Add compose file to the deploy script in `deploy-docker.yml`
|
|
||||||
8. Add any required GitHub secrets for the service
|
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
.PHONY: help dev lint format test clean update-submodules deploy init-submodules status
|
.PHONY: help dev update-submodules deploy init-submodules status
|
||||||
|
|
||||||
SHELL := /bin/bash
|
SHELL := /bin/bash
|
||||||
|
|
||||||
@@ -8,16 +8,6 @@ help: ## Show this help
|
|||||||
dev: ## Start development infrastructure (Redis etc.)
|
dev: ## Start development infrastructure (Redis etc.)
|
||||||
docker compose -f infra/compose/shared.yml up -d
|
docker compose -f infra/compose/shared.yml up -d
|
||||||
|
|
||||||
lint: ## Run Biome linter
|
|
||||||
biome lint .
|
|
||||||
|
|
||||||
format: ## Format code with Biome
|
|
||||||
biome format --write .
|
|
||||||
|
|
||||||
clean: ## Clean build artifacts
|
|
||||||
rm -rf apps/*/dist apps/*/.next apps/*/target 2>/dev/null || true
|
|
||||||
rm -rf node_modules 2>/dev/null || true
|
|
||||||
|
|
||||||
update-submodules: ## Update all git submodules to latest remote
|
update-submodules: ## Update all git submodules to latest remote
|
||||||
git submodule update --remote --merge --recursive
|
git submodule update --remote --merge --recursive
|
||||||
|
|
||||||
|
|||||||
@@ -5,20 +5,21 @@ Aplikasi dipisah sebagai submodule agar frontend, API, dan service pendukung bis
|
|||||||
|
|
||||||
## Services
|
## Services
|
||||||
|
|
||||||
| Service | Path | Default Local Port | Notes |
|
| Service | Path | Notes |
|
||||||
| :---------- | :--------------- | :----------------- | :---------------------------------------------------------------------------- |
|
| :------ | :------------- | :----------------------------- |
|
||||||
| Rust API | `apps/rust-auth` | `4091` | API utama (Axum + SeaORM), scraping, image proxy/cache, metrics, OpenAPI docs |
|
| Scraper | `apps/scraper` | Web scraper service + Dapr SDK |
|
||||||
| Elysia API | `apps/elysia` | `4092` | API realtime/auth/chat/quiz (Elysia + Bun + Drizzle + Redis) |
|
| NATS | — | Message broker + JetStream |
|
||||||
| React Web | `apps/react` | `3000` | Frontend React/Vite |
|
| Dapr | — | Sidecar runtime (per service) |
|
||||||
| Scraper | `apps/scraper` | — | Web scraper service |
|
|
||||||
|
|
||||||
## Infrastructure
|
## Infrastructure
|
||||||
|
|
||||||
File compose berada di `infra/compose/`:
|
File compose berada di `infra/compose/`:
|
||||||
|
|
||||||
- `traefik.yml`: reverse proxy Traefik untuk semua layanan.
|
- `traefik.yml`: reverse proxy Traefik untuk semua layanan.
|
||||||
- `shared.yml`: Redis.
|
- `shared.yml`: Redis (cache + Dapr state store).
|
||||||
- `rust-auth.yml`, `elysia.yml`, `react.yml`, `scraper.yml`: manifest deploy per service (image GHCR bertag SHA).
|
- `nats.yml`: NATS message broker dengan JetStream persistence.
|
||||||
|
- `dapr.yml`: Dapr placement service untuk koordinasi sidecar.
|
||||||
|
- `scraper.yml`: manifest deploy per service (app + Dapr sidecar).
|
||||||
|
|
||||||
Dockerfile per service berada di `infra/docker/`.
|
Dockerfile per service berada di `infra/docker/`.
|
||||||
|
|
||||||
@@ -27,9 +28,6 @@ Dockerfile per service berada di `infra/docker/`.
|
|||||||
Build image via Dockerfile:
|
Build image via Dockerfile:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
docker build -f infra/docker/rust.Dockerfile -t rust-auth:latest .
|
|
||||||
docker build -f infra/docker/elysia.Dockerfile -t elysia-api:latest .
|
|
||||||
docker build -f infra/docker/react.Dockerfile -t react-web:latest .
|
|
||||||
docker build -f infra/docker/scraper.Dockerfile -t scraper-api:latest .
|
docker build -f infra/docker/scraper.Dockerfile -t scraper-api:latest .
|
||||||
```
|
```
|
||||||
|
|
||||||
@@ -38,9 +36,8 @@ Tag and push:
|
|||||||
```bash
|
```bash
|
||||||
SHORT_SHA=$(git rev-parse --short HEAD)
|
SHORT_SHA=$(git rev-parse --short HEAD)
|
||||||
|
|
||||||
docker tag rust-auth:latest ghcr.io/asepharyana/asepharyana-hub/rust-auth:sha-$SHORT_SHA
|
docker tag scraper-api:latest ghcr.io/asepharyana/asepharyana-hub/scraper-api:sha-$SHORT_SHA
|
||||||
docker push ghcr.io/asepharyana/asepharyana-hub/rust-auth:sha-$SHORT_SHA
|
docker push ghcr.io/asepharyana/asepharyana-hub/scraper-api:sha-$SHORT_SHA
|
||||||
# repeat for elysia-api, react-web, scraper-api
|
|
||||||
```
|
```
|
||||||
|
|
||||||
## Local Development
|
## Local Development
|
||||||
@@ -53,27 +50,11 @@ docker compose -f infra/compose/shared.yml up -d
|
|||||||
|
|
||||||
### 2) Jalankan service yang dibutuhkan
|
### 2) Jalankan service yang dibutuhkan
|
||||||
|
|
||||||
```bash
|
Refer to each service's own documentation for development setup.
|
||||||
# Rust API
|
|
||||||
cd apps/rust-auth
|
|
||||||
cargo run
|
|
||||||
|
|
||||||
# Elysia API
|
|
||||||
cd apps/elysia
|
|
||||||
bun install
|
|
||||||
bun run dev
|
|
||||||
|
|
||||||
# React web
|
|
||||||
cd apps/react
|
|
||||||
npm install
|
|
||||||
npm run dev
|
|
||||||
```
|
|
||||||
|
|
||||||
## API Docs and Monitoring
|
## API Docs and Monitoring
|
||||||
|
|
||||||
- Rust OpenAPI: `/docs`
|
Refer to each service's own documentation for API docs.
|
||||||
- Elysia Swagger: `/docs`
|
|
||||||
- Elysia AsyncAPI viewer: `/docs-ws`
|
|
||||||
|
|
||||||
## Deployment Notes
|
## Deployment Notes
|
||||||
|
|
||||||
@@ -86,12 +67,11 @@ npm run dev
|
|||||||
|
|
||||||
Semua VPS terhubung via **Tailscale**. Setiap VPS punya IP Tailscale dan service berkomunikasi antar VPS melalui Tailscale network (`100.64.0.0/10`).
|
Semua VPS terhubung via **Tailscale**. Setiap VPS punya IP Tailscale dan service berkomunikasi antar VPS melalui Tailscale network (`100.64.0.0/10`).
|
||||||
|
|
||||||
| VPS | Tailscale IP | Service |
|
| VPS | Tailscale IP | Service |
|
||||||
| :---------------- | :-------------- | :------------------------------------- |
|
| :--------- | :-------------- | :------------------------------------- |
|
||||||
| `imrnes` | `100.108.1.124` | PostgreSQL (`hub`), Redis |
|
| `imrnes` | `100.121.180.82` | PostgreSQL, Redis |
|
||||||
| `orange` | `100.96.248.86` | App containers (Traefik, 9Router, dll) |
|
| `orangevps` | `100.79.111.61` | App containers (Traefik, scraper-api) |
|
||||||
| `archlinux` | `100.114.19.66` | _(development machine)_ |
|
| `archlinux` | `100.84.39.83` | _(development machine)_ |
|
||||||
| `laptop-2f6e1iph` | `100.86.195.29` | _(offline)_ |
|
|
||||||
|
|
||||||
### Container → Tailscale Connectivity
|
### Container → Tailscale Connectivity
|
||||||
|
|
||||||
@@ -118,10 +98,10 @@ Service yang connect ke Tailscale IP:
|
|||||||
|
|
||||||
```env
|
```env
|
||||||
# PostgreSQL di imrnes
|
# PostgreSQL di imrnes
|
||||||
DATABASE_URL=postgres://user:pass@100.108.1.124:5432/dbname
|
DATABASE_URL=postgres://user:pass@100.121.180.82:5432/dbname
|
||||||
|
|
||||||
# Redis di imrnes
|
# Redis di imrnes
|
||||||
REDIS_URL=redis://100.108.1.124:6379
|
REDIS_URL=redis://100.121.180.82:6379
|
||||||
```
|
```
|
||||||
|
|
||||||
#### Persistent Systemd Service
|
#### Persistent Systemd Service
|
||||||
@@ -183,7 +163,7 @@ docker exec <container> node -e "
|
|||||||
const net = require('net');
|
const net = require('net');
|
||||||
const c = new net.Socket();
|
const c = new net.Socket();
|
||||||
c.setTimeout(5000);
|
c.setTimeout(5000);
|
||||||
c.connect(5432, '100.108.1.124', () => { console.log('OK'); c.end(); });
|
c.connect(5432, '100.121.180.82', () => { console.log('OK'); c.end(); });
|
||||||
c.on('error', e => { console.log('FAIL:', e.code); });
|
c.on('error', e => { console.log('FAIL:', e.code); });
|
||||||
c.on('timeout', () => { console.log('TIMEOUT'); c.destroy(); });
|
c.on('timeout', () => { console.log('TIMEOUT'); c.destroy(); });
|
||||||
"
|
"
|
||||||
@@ -199,6 +179,3 @@ Panduan langkah demi langkah untuk menambahkan aplikasi baru ada di `docs/add-ne
|
|||||||
## License
|
## License
|
||||||
|
|
||||||
MIT
|
MIT
|
||||||
|
|
||||||
# PR-Agent V3 Test
|
|
||||||
Testing with corrected model prefix.
|
|
||||||
|
|||||||
-1
Submodule apps/elysia deleted from b8f4b5806e
Submodule
+1
Submodule apps/hub added at 51798f483f
Submodule
+1
Submodule apps/llm-api added at d8425ea3b3
-1
Submodule apps/react deleted from 124d2f88bd
Submodule apps/rust-auth deleted from 6e695f07e8
+1
-1
Submodule apps/scraper updated: 80c96eaa42...bc782ae4f8
Submodule
+1
Submodule apps/tools added at 3956b90c3c
@@ -0,0 +1,238 @@
|
|||||||
|
# Arsitektur asepharyana-hub
|
||||||
|
|
||||||
|
## Topologi Fisik
|
||||||
|
|
||||||
|
Dua node terhubung via **Tailscale** overlay network:
|
||||||
|
|
||||||
|
```
|
||||||
|
┌──────────────────────────────┐ ┌──────────────────────────────┐
|
||||||
|
│ orangevps (VPS) │ │ imrnes (Bare-metal) │
|
||||||
|
│ IP: 45.127.35.244 │ │ Tailscale: 100.121.180.82 │
|
||||||
|
│ Tailscale: 100.x.x.x │◄──────┤ │
|
||||||
|
│ │ │ Layanan: │
|
||||||
|
│ Layanan: │ │ ├─ PostgreSQL (port 6432) │
|
||||||
|
│ ├─ Traefik (port 80/443) │ │ └─ Redis (port 6379) │
|
||||||
|
│ ├─ NATS + JetStream │ │ │
|
||||||
|
│ ├─ Dapr Placement │ └──────────────────────────────┘
|
||||||
|
│ ├─ Redis (cache, Dapr) │
|
||||||
|
│ ├─ Scraper API + Dapr │
|
||||||
|
│ └─ Hub (Next.js SPA) │
|
||||||
|
└──────────────────────────────┘
|
||||||
|
```
|
||||||
|
|
||||||
|
### Konektivitas Container ke Tailscale
|
||||||
|
|
||||||
|
Container di `orangevps` tidak bisa langsung mencapai IP Tailscale (`100.x.x.x`). Route Tailscale harus ditambahkan ke tabel routing utama (`main`) via `tailscale-routes.service` agar traffic dari container bisa melewati host ke Tailscale.
|
||||||
|
|
||||||
|
## Alur Request HTTP (External)
|
||||||
|
|
||||||
|
```
|
||||||
|
Internet
|
||||||
|
│
|
||||||
|
▼ Port 443
|
||||||
|
Traefik (v3.6)
|
||||||
|
├─ TLS termination (sertifikat dari volume mount)
|
||||||
|
├─ Middleware chain: secure-headers → compress → retry → rate-limit → buffer
|
||||||
|
├─ Plugin: real-ip (Cloudflare), block-sensitive-paths
|
||||||
|
│
|
||||||
|
▼ Router matching
|
||||||
|
Host(`asepharyana.my.id`) || Host(`www.asepharyana.my.id`) → hub
|
||||||
|
host(`hub.asepharyana.my.id`) → hub (SPA + dashboard)
|
||||||
|
Host(`scraper.asepharyana.my.id`) || Host(`api.asepharyana.my.id`) → scraper-api
|
||||||
|
│
|
||||||
|
├─ hub (Next.js, port 3000)
|
||||||
|
│ ├─ / — Portfolio SPA
|
||||||
|
│ ├─ /dashboard — Ops dashboard (client-side, auto-refresh 15s)
|
||||||
|
│ ├─ /api/dashboard — JSON: Docker containers, Jaeger traces, Prometheus metrics
|
||||||
|
│ └─ Docker socket mounted (:ro) for container discovery
|
||||||
|
│
|
||||||
|
▼ Service load balancer
|
||||||
|
http://scraper-api:4091
|
||||||
|
│
|
||||||
|
▼
|
||||||
|
Scraper API (Rust / Axum)
|
||||||
|
├─ Health check: GET /, respon 200
|
||||||
|
├─ REST endpoints
|
||||||
|
├─ Database via `DATABASE_URL` (Tailscale → PostgreSQL di imrnes)
|
||||||
|
├─ Cache via `REDIS_URL` (Redis lokal di container)
|
||||||
|
└─ Pub/sub via Dapr sidecar (localhost:3500)
|
||||||
|
```
|
||||||
|
|
||||||
|
## Infrastruktur Internal
|
||||||
|
|
||||||
|
### Docker Compose Project
|
||||||
|
|
||||||
|
Semua service berjalan dalam satu Docker Compose project bernama `compose` dan bergabung di network `app-shared-net`:
|
||||||
|
|
||||||
|
| File | Service | Peran |
|
||||||
|
|------|---------|-------|
|
||||||
|
| `traefik.yml` | `traefik` | Reverse proxy + TLS + metrics Prometheus |
|
||||||
|
| `shared.yml` | `redis` | Cache, session store, backend Dapr pub/sub & state |
|
||||||
|
| `nats.yml` | `nats` | Message broker + JetStream persistent streaming |
|
||||||
|
| `dapr.yml` | `dapr-placement` | Koordinasi actor placement untuk sidecar Dapr |
|
||||||
|
| `scraper.yml` | `scraper-api` + `scraper-api-dapr` | Aplikasi Rust + sidecar Dapr |
|
||||||
|
| `hub.yml` | `hub` | Next.js SPA portfolio + dashboard + Docker socket |
|
||||||
|
| `observability.yml` | `otel-collector`, `jaeger`, `prometheus`, `node-exporter` | Tracing, metrics, observability |
|
||||||
|
|
||||||
|
### Dapr Sidecar Pattern
|
||||||
|
|
||||||
|
Setiap aplikasi yang menggunakan Dapr mendapat sidecar container `daprd`:
|
||||||
|
|
||||||
|
```
|
||||||
|
┌─────────────────────┐
|
||||||
|
│ scraper-api │
|
||||||
|
│ (app port 4091) │
|
||||||
|
└────────┬────────────┘
|
||||||
|
│ localhost:3500 (HTTP)
|
||||||
|
│ localhost:50001 (gRPC)
|
||||||
|
┌────────▼────────────┐
|
||||||
|
│ scraper-api-dapr │
|
||||||
|
│ (daprd sidecar) │
|
||||||
|
│ │
|
||||||
|
│ Dapr components: │
|
||||||
|
│ ├─ pubsub.redis │
|
||||||
|
│ └─ state.redis │
|
||||||
|
└─────────────────────┘
|
||||||
|
```
|
||||||
|
|
||||||
|
Komponen Dapr:
|
||||||
|
|
||||||
|
| Komponen | Tipe | Backend |
|
||||||
|
|----------|------|---------|
|
||||||
|
| `pubsub` | `pubsub.redis` | `redis:6379` |
|
||||||
|
| `statestore` | `state.redis` | `redis:6379` (prefix `dapr`) |
|
||||||
|
|
||||||
|
### Monitoring & Auto-Discovery
|
||||||
|
|
||||||
|
#### Prometheus Docker Auto-Discovery
|
||||||
|
|
||||||
|
Prometheus menggunakan `docker_sd_configs` untuk auto-detect container yang perlu di-scrape. Cukup tambah label pada container:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
labels:
|
||||||
|
- 'prometheus.io/scrape=true'
|
||||||
|
- 'prometheus.io/port=8080' # port metrics endpoint
|
||||||
|
- 'prometheus.io/path=/metrics' # optional, default /metrics
|
||||||
|
```
|
||||||
|
|
||||||
|
Prometheus akan auto-detect dan mulai scrape container dalam 15 detik.
|
||||||
|
|
||||||
|
#### Traefik Metrics
|
||||||
|
|
||||||
|
Traefik mengekspos metrics Prometheus di port 8080 (`--metrics.prometheus=true`). Metrics yang tersedia:
|
||||||
|
|
||||||
|
| Metric | Query untuk dashboard |
|
||||||
|
|--------|----------------------|
|
||||||
|
| Request rate | `sum(rate(traefik_service_requests_total[1m]))` |
|
||||||
|
| Latency | `avg(traefik_service_request_duration_seconds_sum / traefik_service_request_duration_seconds_count) * 1000` |
|
||||||
|
| Error rate | `sum(rate(traefik_service_requests_total{code=~"5.."}[1m]))` |
|
||||||
|
|
||||||
|
Dashboard di `/api/dashboard` returns node metrics + Traefik range data untuk 4 sparkline charts (RPS, latency, errors, trace volume).
|
||||||
|
|
||||||
|
#### Docker Socket Access
|
||||||
|
|
||||||
|
Container yang perlu akses Docker socket (`/var/run/docker.sock`) harus punya group docker (GID 988):
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
volumes:
|
||||||
|
- /var/run/docker.sock:/var/run/docker.sock:ro
|
||||||
|
group_add:
|
||||||
|
- '988'
|
||||||
|
```di compose atau `--group-add 988` via CLI. Berlaku untuk `hub` (container list) dan `prometheus` (Docker SD).
|
||||||
|
|
||||||
|
### NATS + JetStream
|
||||||
|
|
||||||
|
NATS berjalan dengan flag `-js` untuk mengaktifkan JetStream. Persistent stream disimpan di volume `nats_data`. Dapr pub/sub routing:
|
||||||
|
|
||||||
|
```
|
||||||
|
Service → Dapr sidecar (pubsub.redis) → Redis streams
|
||||||
|
```
|
||||||
|
|
||||||
|
> **Catatan:** Saat ini Dapr pub/sub menggunakan Redis, bukan NATS. Jika ingin migrasi ke NATS untuk pub/sub, komponen Dapr perlu diganti dengan `pubsub.nats`.
|
||||||
|
|
||||||
|
## Arsitektur CI/CD
|
||||||
|
|
||||||
|
```
|
||||||
|
Push ke main (apps/**, infra/**)
|
||||||
|
│
|
||||||
|
▼
|
||||||
|
docker-build-push.yml
|
||||||
|
├─ Phase 1: Detect changed services
|
||||||
|
├─ Phase 2: Build & Push image ke GHCR
|
||||||
|
└─ Phase 3: Update compose manifest + submodule pointer
|
||||||
|
│
|
||||||
|
▼ (workflow_run trigger)
|
||||||
|
deploy-docker.yml
|
||||||
|
├─ SSH ke orangevps
|
||||||
|
├─ Git sync, pull images
|
||||||
|
├─ Remove stale containers
|
||||||
|
└─ Selective restart service
|
||||||
|
```
|
||||||
|
|
||||||
|
Submodule update dari remote repo via `repository_dispatch`:
|
||||||
|
|
||||||
|
```
|
||||||
|
Push ke asepharyana-hub-scraper
|
||||||
|
│
|
||||||
|
▼ (repository_dispatch)
|
||||||
|
update-submodule.yml
|
||||||
|
├─ Update submodule pointer
|
||||||
|
└─ Commit & push ke hub repo
|
||||||
|
│
|
||||||
|
▼ (repository_dispatch trigger)
|
||||||
|
docker-build-push.yml
|
||||||
|
└─ Build, push, deploy
|
||||||
|
```
|
||||||
|
|
||||||
|
## Image Tagging Strategy
|
||||||
|
|
||||||
|
| Tag | Contoh | Penggunaan |
|
||||||
|
|-----|--------|------------|
|
||||||
|
| `sha-<short>` | `sha-a3c5d74` | Immutable, deterministic rollback |
|
||||||
|
| `latest` | `latest` | Mutable, convenience |
|
||||||
|
| `buildcache` | `sha-a3c5d74-buildcache` | Registry-based build cache (internal) |
|
||||||
|
|
||||||
|
## Networking
|
||||||
|
|
||||||
|
### Port Map
|
||||||
|
|
||||||
|
| Port | Service | Deskripsi |
|
||||||
|
|------|---------|-----------|
|
||||||
|
| 443 | Traefik | HTTPS eksternal |
|
||||||
|
| 80 | Traefik | Redirect ke HTTPS |
|
||||||
|
| 4222 | NATS | Client connections |
|
||||||
|
| 8222 | NATS | HTTP monitor / health |
|
||||||
|
| 6379 | Redis | Internal container network |
|
||||||
|
| 3500 | Dapr sidecar | Dapr HTTP API (per service) |
|
||||||
|
| 50001 | Dapr sidecar | Dapr gRPC API (per service) |
|
||||||
|
| 50005 | Dapr placement | Actor placement |
|
||||||
|
| 4091 | Scraper API | Aplikasi HTTP |
|
||||||
|
|
||||||
|
## Event Topics Convention
|
||||||
|
|
||||||
|
Semua event menggunakan prefix `hub.`:
|
||||||
|
|
||||||
|
| Topic | Payload | Deskripsi |
|
||||||
|
|-------|---------|-----------|
|
||||||
|
| `hub.image.cached` | `{original_url, cdn_url, source}` | Image selesai di-cache |
|
||||||
|
| `hub.image.repaired` | `{old_url, new_url}` | CNAME image diperbaiki |
|
||||||
|
| `hub.scrape.anime.done` | `{source, slug, duration}` | Scrape anime selesai |
|
||||||
|
| `hub.system.alert` | `{service, level, message}` | Error/alert dari service |
|
||||||
|
|
||||||
|
## Service Registry (Traefik)
|
||||||
|
|
||||||
|
Domain routing:
|
||||||
|
|
||||||
|
| Subdomain | Service | URL Backend |
|
||||||
|
|-----------|---------|-------------|
|
||||||
|
| `asepharyana.my.id` (root) | Hub SPA + dashboard | `http://hub:3000` |
|
||||||
|
| `www.*` | Hub (alias) | `http://hub:3000` |
|
||||||
|
| `hub.*` | Hub (alias) | `http://hub:3000` |
|
||||||
|
| `scraper.*` | Scraper API | `http://scraper-api:4091` |
|
||||||
|
| `api.*` | Scraper API (alias) | `http://scraper-api:4091` |
|
||||||
|
| `traefik.*` | Traefik Dashboard | `api@internal` |
|
||||||
|
| `jaeger.*` | Jaeger UI | `http://jaeger:16686` |
|
||||||
|
|
||||||
|
Semua domain tersedia di:
|
||||||
|
- `<service>.asepharyana.my.id`
|
||||||
|
- `<service>.asepharyana.web.id`
|
||||||
@@ -0,0 +1,681 @@
|
|||||||
|
# Deployment Guide
|
||||||
|
|
||||||
|
Panduan deploy aplikasi apapun menggunakan **Docker + Docker Compose + GitHub Actions + VPS**.
|
||||||
|
|
||||||
|
## Arsitektur
|
||||||
|
|
||||||
|
```
|
||||||
|
GitHub Repo ──► GitHub Actions ──► Registry (GHCR / Docker Hub / ECR / dll.)
|
||||||
|
│
|
||||||
|
▼
|
||||||
|
VPS (<VPS_HOST>)
|
||||||
|
docker compose pull + up
|
||||||
|
```
|
||||||
|
|
||||||
|
## Prerequisites
|
||||||
|
|
||||||
|
- Docker Engine >= 24.x
|
||||||
|
- Docker Compose v2 (plugin)
|
||||||
|
- Git
|
||||||
|
- Akun GitHub dengan akses repo
|
||||||
|
- SSH key di `~/.ssh/<KEY_NAME>` (default: `id_ed25519`)
|
||||||
|
|
||||||
|
## Konfigurasi VPS Target
|
||||||
|
|
||||||
|
Buat berkas `~/orangevps` (atau sesuaikan dengan env Anda):
|
||||||
|
|
||||||
|
```text
|
||||||
|
ssh <USER>@<VPS_HOST>
|
||||||
|
```
|
||||||
|
|
||||||
|
Contoh isi `~/orangevps`:
|
||||||
|
|
||||||
|
```text
|
||||||
|
ssh root@45.127.35.244
|
||||||
|
```
|
||||||
|
|
||||||
|
| Parameter | Nilai | Contoh |
|
||||||
|
|-----------|-------|--------|
|
||||||
|
| User | `<USER>` | `root` |
|
||||||
|
| Host | `<VPS_HOST>` | `45.127.35.244` |
|
||||||
|
| SSH Key | `~/.ssh/<KEY_NAME>` | `~/.ssh/id_ed25519` |
|
||||||
|
| Target Dir di VPS | `<VPS_TARGET_DIR>` | `/opt/app` atau `/root/app` |
|
||||||
|
|
||||||
|
> Tip: Jika SSH key menggunakan nama selain default, sesuaikan path dan `ssh -i` sesuai.
|
||||||
|
|
||||||
|
## Registry
|
||||||
|
|
||||||
|
Pilih registry untuk menyimpan image Docker. Sesuaikan dengan proyek:
|
||||||
|
|
||||||
|
| Registry | URL | Auth |
|
||||||
|
|----------|-----|------|
|
||||||
|
| GitHub Container Registry | `ghcr.io` | `GITHUB_TOKEN` |
|
||||||
|
| Docker Hub | `docker.io` | username / PAT |
|
||||||
|
| AWS ECR | `<account>.dkr.ecr.<region>.amazonaws.com` | `aws ecr get-login-password` |
|
||||||
|
| Google GCR | `gcr.io` | `gcloud auth print-access-token` |
|
||||||
|
| Azure ACR | `<registry>.azurecr.io` | `az acr login` |
|
||||||
|
|
||||||
|
Contoh namespace untuk GHCR:
|
||||||
|
|
||||||
|
```text
|
||||||
|
Registry : ghcr.io
|
||||||
|
Namespace: <GITHUB_USERNAME_OR_ORG>
|
||||||
|
Repo : <REPO_NAME>
|
||||||
|
```
|
||||||
|
|
||||||
|
Pastikan package/visibility di registry mengizinkan akses pull dari VPS.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Deploy Otomatis (Recommended)
|
||||||
|
|
||||||
|
Gunakan GitHub Actions untuk otomatisasi build, push, dan deploy.
|
||||||
|
|
||||||
|
### Workflow 1: Build dan Push Image
|
||||||
|
|
||||||
|
File: `.github/workflows/docker-build-push.yml`
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
name: Build and Push Docker Images
|
||||||
|
|
||||||
|
on:
|
||||||
|
push:
|
||||||
|
branches: [main]
|
||||||
|
workflow_dispatch:
|
||||||
|
|
||||||
|
permissions:
|
||||||
|
contents: read
|
||||||
|
packages: write
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
build:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v6
|
||||||
|
|
||||||
|
- uses: docker/login-action@v4
|
||||||
|
with:
|
||||||
|
registry: ghcr.io
|
||||||
|
username: ${{ github.actor }}
|
||||||
|
password: ${{ secrets.GITHUB_TOKEN }}
|
||||||
|
|
||||||
|
- uses: docker/setup-buildx-action@v4
|
||||||
|
|
||||||
|
- uses: docker/build-push-action@v7
|
||||||
|
with:
|
||||||
|
context: .
|
||||||
|
file: Dockerfile
|
||||||
|
push: true
|
||||||
|
tags: |
|
||||||
|
ghcr.io/${{ github.repository }}/<SERVICE_NAME>:latest
|
||||||
|
ghcr.io/${{ github.repository }}/<SERVICE_NAME>:sha-${{ github.sha }}
|
||||||
|
cache-from: type=registry,ref=ghcr.io/${{ github.repository }}/<SERVICE_NAME>:buildcache
|
||||||
|
cache-to: type=registry,ref=ghcr.io/${{ github.repository }}/<SERVICE_NAME>:buildcache,mode=max
|
||||||
|
```
|
||||||
|
|
||||||
|
Ubah `<SERVICE_NAME>` sesuai service (misal: `app`, `web`, `api`). Jika monorepo, gunakan matrix strategy untuk build beberapa service sekaligus.
|
||||||
|
|
||||||
|
### Workflow 2: Deploy ke VPS
|
||||||
|
|
||||||
|
File: `.github/workflows/deploy-docker.yml`
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
name: Deploy Docker to VPS
|
||||||
|
|
||||||
|
on:
|
||||||
|
workflow_run:
|
||||||
|
workflows: ['Build and Push Docker Images']
|
||||||
|
types: [completed]
|
||||||
|
push:
|
||||||
|
branches: [main]
|
||||||
|
workflow_dispatch:
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
deploy:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v6
|
||||||
|
|
||||||
|
- name: Deploy to VPS
|
||||||
|
env:
|
||||||
|
SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }}
|
||||||
|
VPS_HOST: ${{ secrets.VPS_HOST }}
|
||||||
|
VPS_USER: ${{ secrets.VPS_USER }}
|
||||||
|
VPS_TARGET_DIR: ${{ secrets.VPS_TARGET_DIR }}
|
||||||
|
ENV_FILE_PRODUCTION: ${{ secrets.ENV_FILE_PRODUCTION }}
|
||||||
|
run: |
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
mkdir -p ~/.ssh
|
||||||
|
echo "$SSH_PRIVATE_KEY" > ~/.ssh/id_rsa
|
||||||
|
chmod 600 ~/.ssh/id_rsa
|
||||||
|
ssh-keyscan -H -t ed25519,rsa "$VPS_HOST" >> ~/.ssh/known_hosts
|
||||||
|
|
||||||
|
SSH_OPTS=(-o ControlMaster=auto -o ControlPath=/tmp/ssh-%r@%h:%p -o ControlPersist=600 -o StrictHostKeyChecking=yes)
|
||||||
|
|
||||||
|
ssh "${SSH_OPTS[@]}" "$VPS_USER@$VPS_HOST" "mkdir -p $VPS_TARGET_DIR && mkdir -p $VPS_TARGET_DIR/infra/compose"
|
||||||
|
echo "$ENV_FILE_PRODUCTION" > .env.prod
|
||||||
|
scp "${SSH_OPTS[@]}" .env.prod "$VPS_USER@$VPS_HOST:$VPS_TARGET_DIR/.env"
|
||||||
|
|
||||||
|
ssh "${SSH_OPTS[@]}" "$VPS_USER@$VPS_HOST" bash -s <<'EOF'
|
||||||
|
set -euo pipefail
|
||||||
|
cd "$VPS_TARGET_DIR"
|
||||||
|
|
||||||
|
docker network inspect app-shared-net >/dev/null 2>&1 || docker network create app-shared-net
|
||||||
|
|
||||||
|
if [ ! -d ".git" ]; then
|
||||||
|
git init
|
||||||
|
git remote add origin https://github.com/<GITHUB_USER>/<REPO_NAME>.git
|
||||||
|
fi
|
||||||
|
git fetch origin main --depth=1 || true
|
||||||
|
git reset --hard FETCH_HEAD
|
||||||
|
|
||||||
|
docker compose --env-file .env pull
|
||||||
|
docker compose --env-file .env up -d --remove-orphans
|
||||||
|
EOF
|
||||||
|
```
|
||||||
|
|
||||||
|
### Secrets GitHub yang Diperlukan
|
||||||
|
|
||||||
|
Buka **Settings > Secrets and variables > Actions**:
|
||||||
|
|
||||||
|
| Secret | Deskripsi |
|
||||||
|
|--------|-----------|
|
||||||
|
| `SSH_PRIVATE_KEY` | Isi dengan `cat ~/.ssh/<KEY_NAME>` |
|
||||||
|
| `VPS_HOST` | IP atau domain VPS |
|
||||||
|
| `VPS_USER` | User SSH (misal: `root`, `ubuntu`, `deploy`) |
|
||||||
|
| `VPS_TARGET_DIR` | Direktori aplikasi di VPS |
|
||||||
|
| `ENV_FILE_PRODUCTION` | Isi dengan environment production |
|
||||||
|
|
||||||
|
### Trigger Manual
|
||||||
|
|
||||||
|
```bash
|
||||||
|
gh workflow run deploy-docker.yml
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Dockerfile Patterns
|
||||||
|
|
||||||
|
Pilih pattern sesuai jenis aplikasi.
|
||||||
|
|
||||||
|
### Pattern 1: Multi-stage Build (SPA / static assets)
|
||||||
|
|
||||||
|
```dockerfile
|
||||||
|
FROM oven/bun:1 AS builder
|
||||||
|
WORKDIR /app
|
||||||
|
COPY package.json bun.lock ./
|
||||||
|
RUN bun install --frozen-lockfile
|
||||||
|
COPY . .
|
||||||
|
RUN bun run build
|
||||||
|
|
||||||
|
FROM nginx:alpine
|
||||||
|
COPY --from=builder /app/dist /usr/share/nginx/html
|
||||||
|
COPY nginx.conf /etc/nginx/conf.d/default.conf
|
||||||
|
EXPOSE 80
|
||||||
|
CMD ["nginx", "-g", "daemon off;"]
|
||||||
|
```
|
||||||
|
|
||||||
|
### Pattern 2: Single-stage (runtime image)
|
||||||
|
|
||||||
|
```dockerfile
|
||||||
|
FROM oven/bun:1
|
||||||
|
WORKDIR /app
|
||||||
|
COPY package.json bun.lock ./
|
||||||
|
RUN bun install --frozen-lockfile
|
||||||
|
COPY . .
|
||||||
|
EXPOSE 3000
|
||||||
|
CMD ["bun", "run", "start"]
|
||||||
|
```
|
||||||
|
|
||||||
|
### Pattern 3: Compiled binary (Rust / Go / Zig)
|
||||||
|
|
||||||
|
```dockerfile
|
||||||
|
FROM rust:1 AS builder
|
||||||
|
WORKDIR /app
|
||||||
|
COPY . .
|
||||||
|
RUN cargo build --release
|
||||||
|
|
||||||
|
FROM debian:bookworm-slim
|
||||||
|
COPY --from=builder /app/target/release/app /usr/local/bin/app
|
||||||
|
EXPOSE 8080
|
||||||
|
CMD ["app"]
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Docker Compose Patterns
|
||||||
|
|
||||||
|
### Single service
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
services:
|
||||||
|
app:
|
||||||
|
container_name: app
|
||||||
|
image: registry.example.com/org/app:latest
|
||||||
|
restart: always
|
||||||
|
ports:
|
||||||
|
- "3000:3000"
|
||||||
|
environment:
|
||||||
|
- NODE_ENV=production
|
||||||
|
```
|
||||||
|
|
||||||
|
### Multi-service dengan shared network
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
services:
|
||||||
|
app:
|
||||||
|
container_name: app
|
||||||
|
image: registry.example.com/org/app:latest
|
||||||
|
restart: always
|
||||||
|
networks: [app-shared-net]
|
||||||
|
|
||||||
|
redis:
|
||||||
|
container_name: redis
|
||||||
|
image: redis:7-alpine
|
||||||
|
restart: always
|
||||||
|
networks: [app-shared-net]
|
||||||
|
|
||||||
|
networks:
|
||||||
|
app-shared-net:
|
||||||
|
name: app-shared-net
|
||||||
|
external: true
|
||||||
|
```
|
||||||
|
|
||||||
|
### Dengan reverse proxy (Traefik / Caddy / Nginx)
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
services:
|
||||||
|
app:
|
||||||
|
container_name: app
|
||||||
|
image: registry.example.com/org/app:latest
|
||||||
|
restart: always
|
||||||
|
networks: [app-shared-net]
|
||||||
|
labels:
|
||||||
|
- 'traefik.enable=true'
|
||||||
|
- 'traefik.http.routers.app.rule=Host(`app.example.com`)'
|
||||||
|
- 'traefik.http.routers.app.entrypoints=websecure'
|
||||||
|
- 'traefik.http.routers.app.tls=true'
|
||||||
|
- 'traefik.http.services.app.loadbalancer.server.port=3000'
|
||||||
|
|
||||||
|
networks:
|
||||||
|
app-shared-net:
|
||||||
|
name: app-shared-net
|
||||||
|
external: true
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Deploy Manual (Lokal)
|
||||||
|
|
||||||
|
### 1. Build dan Push ke Registry
|
||||||
|
|
||||||
|
Login ke registry:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
echo $GITHUB_TOKEN | docker login ghcr.io -u <GITHUB_USERNAME> --password-stdin
|
||||||
|
```
|
||||||
|
|
||||||
|
Build dan push:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker build -t ghcr.io/<GITHUB_USERNAME>/<REPO_NAME>/<SERVICE_NAME>:latest -f Dockerfile .
|
||||||
|
|
||||||
|
docker push ghcr.io/<GITHUB_USERNAME>/<REPO_NAME>/<SERVICE_NAME>:latest
|
||||||
|
```
|
||||||
|
|
||||||
|
Tag tambahan dengan SHA commit:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
SHORT_SHA=$(git rev-parse --short HEAD)
|
||||||
|
docker tag ghcr.io/<GITHUB_USERNAME>/<REPO_NAME>/<SERVICE_NAME>:latest \
|
||||||
|
ghcr.io/<GITHUB_USERNAME>/<REPO_NAME>/<SERVICE_NAME>:sha-${SHORT_SHA}
|
||||||
|
docker push ghcr.io/<GITHUB_USERNAME>/<REPO_NAME>/<SERVICE_NAME>:sha-${SHORT_SHA}
|
||||||
|
```
|
||||||
|
|
||||||
|
### 2. Pull dan Deploy di VPS
|
||||||
|
|
||||||
|
SSH ke VPS:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
ssh -i ~/.ssh/<KEY_NAME> <USER>@<VPS_HOST>
|
||||||
|
```
|
||||||
|
|
||||||
|
Clone repo (jika belum):
|
||||||
|
|
||||||
|
```bash
|
||||||
|
git clone https://github.com/<GITHUB_USER>/<REPO_NAME>.git <VPS_TARGET_DIR>
|
||||||
|
cd <VPS_TARGET_DIR>
|
||||||
|
```
|
||||||
|
|
||||||
|
Buat shared network (hanya sekali):
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker network create app-shared-net
|
||||||
|
```
|
||||||
|
|
||||||
|
Siapkan environment:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cp .env.example .env
|
||||||
|
# Edit .env sesuai nilai production
|
||||||
|
nano .env
|
||||||
|
```
|
||||||
|
|
||||||
|
Login ke registry di VPS:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
echo $GITHUB_TOKEN | docker login ghcr.io -u <GITHUB_USERNAME> --password-stdin
|
||||||
|
```
|
||||||
|
|
||||||
|
Pull gambar terbaru:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd <VPS_TARGET_DIR>
|
||||||
|
docker compose -f docker-compose.yml --env-file .env pull
|
||||||
|
```
|
||||||
|
|
||||||
|
Deploy (up):
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker compose -f docker-compose.yml --env-file .env up -d --remove-orphans
|
||||||
|
```
|
||||||
|
|
||||||
|
Verifikasi:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker compose -f docker-compose.yml ps
|
||||||
|
docker compose -f docker-compose.yml logs -f <SERVICE_NAME>
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Deployment Order (Manual)
|
||||||
|
|
||||||
|
Jika deploy bertahap, gunakan urutan ini:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# 1. Shared services (Redis, database, dll.)
|
||||||
|
docker compose -f infra/compose/shared.yml up -d
|
||||||
|
|
||||||
|
# 2. Reverse proxy
|
||||||
|
docker compose -f infra/compose/traefik.yml up -d
|
||||||
|
|
||||||
|
# 3. Aplikasi
|
||||||
|
docker compose \
|
||||||
|
-f infra/compose/app1.yml \
|
||||||
|
-f infra/compose/app2.yml \
|
||||||
|
up -d
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Perintah Berguna di VPS
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Lihat semua container
|
||||||
|
docker ps -a
|
||||||
|
|
||||||
|
# Log service
|
||||||
|
docker logs -f <container_name>
|
||||||
|
|
||||||
|
# Restart satu service
|
||||||
|
docker compose -f <compose_file> up -d --force-recreate
|
||||||
|
|
||||||
|
# Hapus network lama (hati-hati)
|
||||||
|
docker network rm app-shared-net
|
||||||
|
docker network create app-shared-net
|
||||||
|
|
||||||
|
# Bersihkan image unused
|
||||||
|
docker image prune -a -f
|
||||||
|
docker system prune -a -f
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Troubleshooting
|
||||||
|
|
||||||
|
### Image tidak bisa di-pull
|
||||||
|
|
||||||
|
Pastikan sudah login ke registry di VPS:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker logout ghcr.io
|
||||||
|
echo $GITHUB_TOKEN | docker login ghcr.io -u <GITHUB_USERNAME> --password-stdin
|
||||||
|
```
|
||||||
|
|
||||||
|
Periksa visibility package di registry (harus `Public` atau akses diberikan).
|
||||||
|
|
||||||
|
### Port sudah dipakai
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker ps | grep :80
|
||||||
|
docker ps | grep :443
|
||||||
|
```
|
||||||
|
|
||||||
|
### Reverse proxy tidak routing
|
||||||
|
|
||||||
|
Periksa label di compose file dan pastikan shared network ada:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker network inspect app-shared-net
|
||||||
|
docker logs traefik
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Environment Variable Management
|
||||||
|
|
||||||
|
### Pola 1: `.env` di VPS (recommended untuk production)
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Di VPS
|
||||||
|
cd <VPS_TARGET_DIR>
|
||||||
|
cp .env.example .env
|
||||||
|
# Edit sesuai production
|
||||||
|
nano .env
|
||||||
|
```
|
||||||
|
|
||||||
|
CI/CD upload `.env` via secret, tidak simpan di repo.
|
||||||
|
|
||||||
|
### Pola 2: Docker secrets (Swarm mode)
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
services:
|
||||||
|
app:
|
||||||
|
image: app:latest
|
||||||
|
secrets:
|
||||||
|
- db_password
|
||||||
|
|
||||||
|
secrets:
|
||||||
|
db_password:
|
||||||
|
file: ./secrets/db_password.txt
|
||||||
|
```
|
||||||
|
|
||||||
|
### Pola 3: External secret manager
|
||||||
|
|
||||||
|
- **HashiCorp Vault**: inject via env atau file
|
||||||
|
- **AWS Secrets Manager**: `aws secretsmanager get-secret-value`
|
||||||
|
- **Doppler / Infisical**: unified secret management
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Tagging dan Versioning
|
||||||
|
|
||||||
|
### Strategy yang umum
|
||||||
|
|
||||||
|
| Strategy | Contoh tag | Kegunaan |
|
||||||
|
|----------|-----------|----------|
|
||||||
|
| Latest + SHA | `latest`, `sha-abc1234` | CI/CD cepat, traceable |
|
||||||
|
| SemVer | `1.2.3`, `1.2`, `1` | Release publik |
|
||||||
|
| Git tag mirror | `v1.2.3` | Sync dengan git tag |
|
||||||
|
| Branch mirror | `main`, `develop` | Preview / staging |
|
||||||
|
|
||||||
|
### Contoh git tag driven deploy
|
||||||
|
|
||||||
|
```bash
|
||||||
|
git tag v1.2.3
|
||||||
|
git push origin v1.2.3
|
||||||
|
```
|
||||||
|
|
||||||
|
CI/CD membaca tag, build image dengan tag yang sama, dan deploy.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Rollback
|
||||||
|
|
||||||
|
### Rollback via registry
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Lihat tag yang tersedia
|
||||||
|
docker manifest inspect ghcr.io/org/app:latest
|
||||||
|
# atau lihat UI registry
|
||||||
|
|
||||||
|
# Di VPS, edit compose file ke tag sebelumnya
|
||||||
|
# lalu:
|
||||||
|
docker compose --env-file .env pull
|
||||||
|
docker compose --env-file .env up -d --remove-orphans
|
||||||
|
```
|
||||||
|
|
||||||
|
### Rollback via git
|
||||||
|
|
||||||
|
```bash
|
||||||
|
git revert HEAD
|
||||||
|
git push origin main
|
||||||
|
# CI/CD otomatis build dan deploy versi sebelumnya
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Health Checks
|
||||||
|
|
||||||
|
### Di Dockerfile
|
||||||
|
|
||||||
|
```dockerfile
|
||||||
|
HEALTHCHECK --interval=30s --timeout=3s --start-period=5s --retries=3 \
|
||||||
|
CMD curl -f http://localhost:3000/health || exit 1
|
||||||
|
```
|
||||||
|
|
||||||
|
### Di Docker Compose
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
services:
|
||||||
|
app:
|
||||||
|
image: app:latest
|
||||||
|
healthcheck:
|
||||||
|
test: ["CMD", "curl", "-f", "http://localhost:3000/health"]
|
||||||
|
interval: 30s
|
||||||
|
timeout: 3s
|
||||||
|
retries: 3
|
||||||
|
start_period: 10s
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Monitoring & Observability
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Log aggregated
|
||||||
|
docker compose logs -f --tail=100
|
||||||
|
|
||||||
|
# Resource usage
|
||||||
|
docker stats
|
||||||
|
|
||||||
|
# Disk usage
|
||||||
|
docker system df
|
||||||
|
|
||||||
|
# Cleanup
|
||||||
|
docker system prune -a -f
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Catatan Keamanan
|
||||||
|
|
||||||
|
- Jangan commit `.env` atau SSH private key ke repo.
|
||||||
|
- Gunakan GitHub Secrets (atau secret manager) untuk credential di CI/CD.
|
||||||
|
- Rotate token dan key secara berkala.
|
||||||
|
- Batasi akses SSH ke VPS (ubah port default, gunakan fail2ban).
|
||||||
|
- Set `StrictHostKeyChecking=yes` pada SSH opsional deployment.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Proyek Ini: asepharyana-hub
|
||||||
|
|
||||||
|
> Dokumentasi spesifik untuk repo ini. Lihat juga [ADR-0002](adr/0002-env-file-via-github-secret.md).
|
||||||
|
|
||||||
|
### Topologi
|
||||||
|
|
||||||
|
| Host | IP | Peran |
|
||||||
|
|------|----|-------|
|
||||||
|
| `orangevps` (VPS) | `45.127.35.244` | Docker host: Traefik, scraper-api, Redis, NATS, Dapr |
|
||||||
|
| `imrnes` (bare-metal) | `100.121.180.82` (Tailscale) | PostgreSQL (port 6432), Redis (port 6379) |
|
||||||
|
|
||||||
|
### Environment Variables
|
||||||
|
|
||||||
|
**Production `.env` tidak pernah di-commit.** File ini disimpan sebagai GitHub secret `ENV_FILE_PRODUCTION` dan di-SCP ke VPS saat deploy via `deploy-docker.yml`.
|
||||||
|
|
||||||
|
Cara update:
|
||||||
|
```bash
|
||||||
|
# Baca current .env dari VPS
|
||||||
|
ssh root@45.127.35.244 "cat /root/asepharyana-hub/.env"
|
||||||
|
|
||||||
|
# Update GitHub secret (dari output di atas)
|
||||||
|
cat > /tmp/env-updated << 'EOF'
|
||||||
|
<paste content, edit, lalu>
|
||||||
|
EOF
|
||||||
|
cat /tmp/env-updated | gh secret set ENV_FILE_PRODUCTION --repo asepharyana/asepharyana-hub
|
||||||
|
```
|
||||||
|
|
||||||
|
**Jangan manual edit `.env` di VPS tanpa update GitHub secret juga** — nanti ke- overwrite pas deploy berikutnya.
|
||||||
|
|
||||||
|
### Database
|
||||||
|
|
||||||
|
| Variable | Value |
|
||||||
|
|----------|-------|
|
||||||
|
| `DATABASE_URL` | `postgres://asephs:hunterz@100.121.180.82:6432/hub` |
|
||||||
|
| `REDIS_URL` | `redis://redis:6379` (Docker network) |
|
||||||
|
|
||||||
|
### Kompose
|
||||||
|
|
||||||
|
Proyek compose bernama `compose`, terdiri dari 5 file yang selalu di-include bersamaan:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
/root/asepharyana-hub/infra/compose/
|
||||||
|
├── traefik.yml # Reverse proxy
|
||||||
|
├── shared.yml # Redis
|
||||||
|
├── nats.yml # NATS
|
||||||
|
├── dapr.yml # Dapr placement
|
||||||
|
├── scraper.yml # Scraper API
|
||||||
|
└── observability.yml # OTel Collector, Jaeger, Dashboard
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd /root/asepharyana-hub
|
||||||
|
docker compose \
|
||||||
|
-p compose \
|
||||||
|
--env-file .env \
|
||||||
|
-f infra/compose/traefik.yml \
|
||||||
|
-f infra/compose/shared.yml \
|
||||||
|
-f infra/compose/scraper.yml \
|
||||||
|
-f infra/compose/nats.yml \
|
||||||
|
-f infra/compose/dapr.yml \
|
||||||
|
-f infra/compose/observability.yml \
|
||||||
|
up -d --remove-orphans
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Checklist Deploy Proyek Baru
|
||||||
|
|
||||||
|
1. [ ] Dockerfile ditest lokal (`docker build`, `docker run`)
|
||||||
|
2. [ ] Docker Compose file valid (`docker compose config`)
|
||||||
|
3. [ ] `.dockerignore` sesuai (node_modules, .git, .env)
|
||||||
|
4. [ ] Registry dibuat (GHCR package / Docker Hub repo / ECR / dll.)
|
||||||
|
5. [ ] GitHub Actions workflow dibuat dengan permission `packages: write`
|
||||||
|
6. [ ] VPS siap: Docker, Docker Compose, SSH key
|
||||||
|
7. [ ] Shared network dibuat (`docker network create`)
|
||||||
|
8. [ ] `.env` production di-VPS atau via secret manager
|
||||||
|
9. [ ] Reverse proxy (Traefik / Caddy / Nginx) routing ke container
|
||||||
|
10. [ ] Health check endpoint aktif
|
||||||
@@ -0,0 +1,206 @@
|
|||||||
|
# Development Guide
|
||||||
|
|
||||||
|
Panduan setup lingkungan development lokal untuk kontributor `asepharyana-hub`.
|
||||||
|
|
||||||
|
## Prasyarat
|
||||||
|
|
||||||
|
| Tool | Versi Minimal | Catatan |
|
||||||
|
|------|---------------|---------|
|
||||||
|
| Git | 2.40+ | Submodule support |
|
||||||
|
| Docker | 24+ | Dengan Docker Compose v2 plugin |
|
||||||
|
| Rust | 1.85+ | Hanya untuk `apps/scraper` |
|
||||||
|
| Bun | 1.x | Root tooling (Biome) |
|
||||||
|
| Dapr CLI | 1.14+ | Opsional, untuk development dengan Dapr |
|
||||||
|
|
||||||
|
## Setup Awal
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# 1. Clone repo
|
||||||
|
git clone https://github.com/asepharyana/asepharyana-hub.git
|
||||||
|
cd asepharyana-hub
|
||||||
|
|
||||||
|
# 2. Init submodules
|
||||||
|
make init-submodules
|
||||||
|
|
||||||
|
# 3. Setup environment
|
||||||
|
cp .env.example .env
|
||||||
|
# Edit .env sesuai kebutuhan lokal
|
||||||
|
|
||||||
|
# 4. Install root dependencies
|
||||||
|
bun install
|
||||||
|
```
|
||||||
|
|
||||||
|
## Menjalankan Infrastruktur Lokal
|
||||||
|
|
||||||
|
Beberapa service membutuhkan Redis. Jalankan dengan:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
make dev
|
||||||
|
# atau equivalen:
|
||||||
|
docker compose -f infra/compose/shared.yml up -d
|
||||||
|
```
|
||||||
|
|
||||||
|
Ini akan menjalankan Redis Alpine di `localhost:6379`.
|
||||||
|
|
||||||
|
### (Opsional) NATS Lokal
|
||||||
|
|
||||||
|
Jika service membutuhkan pub/sub:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker compose -f infra/compose/nats.yml up -d
|
||||||
|
# NATS client: localhost:4222
|
||||||
|
# NATS monitor: localhost:8222
|
||||||
|
```
|
||||||
|
|
||||||
|
### (Opsional) Dapr Placement Lokal
|
||||||
|
|
||||||
|
Jika service membutuhkan sidecar Dapr:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker compose -f infra/compose/dapr.yml up -d
|
||||||
|
# Dapr placement: localhost:50005
|
||||||
|
```
|
||||||
|
|
||||||
|
## Menjalankan Service Lokal
|
||||||
|
|
||||||
|
### Scraper API (Rust)
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Pastikan Redis sudah running (make dev)
|
||||||
|
cd apps/scraper
|
||||||
|
|
||||||
|
# Cargo run
|
||||||
|
cargo run
|
||||||
|
|
||||||
|
# Dengan Dapr sidecar (jika placement running)
|
||||||
|
dapr run \
|
||||||
|
--app-id scraper-api \
|
||||||
|
--app-port 4091 \
|
||||||
|
--dapr-http-port 3500 \
|
||||||
|
--resources-path ../../infra/dapr/components \
|
||||||
|
-- cargo run
|
||||||
|
```
|
||||||
|
|
||||||
|
### Dengan Docker Compose (Full Stack)
|
||||||
|
|
||||||
|
Untuk menjalankan semua service sekaligus:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker compose \
|
||||||
|
-f infra/compose/shared.yml \
|
||||||
|
-f infra/compose/nats.yml \
|
||||||
|
-f infra/compose/dapr.yml \
|
||||||
|
-f infra/compose/scraper.yml \
|
||||||
|
--env-file .env \
|
||||||
|
up -d
|
||||||
|
```
|
||||||
|
|
||||||
|
Untuk service baru, tambahkan compose file-nya ke daftar.
|
||||||
|
|
||||||
|
## Update Submodules
|
||||||
|
|
||||||
|
### Pull latest dari semua submodule
|
||||||
|
|
||||||
|
```bash
|
||||||
|
make update-submodules
|
||||||
|
# atau:
|
||||||
|
git submodule update --remote --merge --recursive
|
||||||
|
```
|
||||||
|
|
||||||
|
### Check status submodule
|
||||||
|
|
||||||
|
```bash
|
||||||
|
make status
|
||||||
|
# atau:
|
||||||
|
git submodule status
|
||||||
|
```
|
||||||
|
|
||||||
|
### Sync .env ke submodule
|
||||||
|
|
||||||
|
```bash
|
||||||
|
bash scripts/2updateenv.sh
|
||||||
|
# Copy .env root ke apps/*/
|
||||||
|
```
|
||||||
|
|
||||||
|
## Linting & Formatting
|
||||||
|
|
||||||
|
Root repo menggunakan **Biome** untuk linting dan formatting:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
bun run check # Lint + format + write
|
||||||
|
bun run ci # CI mode (no write, exit code on issues)
|
||||||
|
bun run lint # Lint only
|
||||||
|
bun run format # Format only
|
||||||
|
```
|
||||||
|
|
||||||
|
## Build Docker Image Lokal
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Scraper API
|
||||||
|
docker build -f infra/docker/scraper.Dockerfile -t scraper-api:local .
|
||||||
|
|
||||||
|
# Service baru: tambahkan Dockerfile di infra/docker/
|
||||||
|
```
|
||||||
|
|
||||||
|
## Testing
|
||||||
|
|
||||||
|
Saat ini belum ada test runner di root level. Masing-masing submodule mengelola testing sendiri:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Scraper API (Rust)
|
||||||
|
cd apps/scraper && cargo test
|
||||||
|
```
|
||||||
|
|
||||||
|
## Validasi YAML
|
||||||
|
|
||||||
|
Sebelum commit perubahan infra, validasi semua file YAML:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
python -c "
|
||||||
|
import pathlib, yaml
|
||||||
|
for p in pathlib.Path('infra').rglob('*.yml'):
|
||||||
|
with open(p) as f: yaml.safe_load(f)
|
||||||
|
print(f'OK {p}')
|
||||||
|
for p in pathlib.Path('infra').rglob('*.yaml'):
|
||||||
|
with open(p) as f: yaml.safe_load(f)
|
||||||
|
print(f'OK {p}')
|
||||||
|
"
|
||||||
|
|
||||||
|
for f in infra/compose/*.yml; do
|
||||||
|
docker compose -f "$f" config >/dev/null && echo "OK $f"
|
||||||
|
done
|
||||||
|
```
|
||||||
|
|
||||||
|
## Git Workflow
|
||||||
|
|
||||||
|
### Commit Convention
|
||||||
|
|
||||||
|
```
|
||||||
|
<type>(<scope>): <description>
|
||||||
|
```
|
||||||
|
|
||||||
|
Type: `feat`, `fix`, `chore`, `docs`, `refactor`, `test`, `ci`, `perf`, `style`
|
||||||
|
Scope: `scraper`, `infra`, `ci`, `dapr`, `nats`, `docs`, `deps`, `scripts`, `root`
|
||||||
|
|
||||||
|
Contoh:
|
||||||
|
```
|
||||||
|
feat(scraper): add image cache endpoint
|
||||||
|
fix(infra): correct Traefik rate-limit config
|
||||||
|
chore(deps): bump biome to 2.5.0
|
||||||
|
```
|
||||||
|
|
||||||
|
### Branch Strategy
|
||||||
|
|
||||||
|
- `main` — production branch, push triggers CI/CD
|
||||||
|
- Fitur baru: branch dari `main`, PR ke `main`
|
||||||
|
- Submodule development: dilakukan di repo masing-masing, hub hanya update pointer
|
||||||
|
|
||||||
|
## Deployment ke VPS
|
||||||
|
|
||||||
|
Push ke `main` otomatis trigger CI/CD. Untuk trigger manual:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
gh workflow run deploy-docker.yml
|
||||||
|
```
|
||||||
|
|
||||||
|
Lihat `docs/DEPLOYMENT.md` untuk detail.
|
||||||
@@ -0,0 +1,162 @@
|
|||||||
|
# Menambahkan Dapr ke Service Baru
|
||||||
|
|
||||||
|
Panduan integrasi Dapr runtime sidecar untuk service di `asepharyana-hub`.
|
||||||
|
|
||||||
|
## Prasyarat
|
||||||
|
|
||||||
|
- NATS server berjalan (`infra/compose/nats.yml`)
|
||||||
|
- Dapr placement service berjalan (`infra/compose/dapr.yml`)
|
||||||
|
|
||||||
|
## 1. Compose File
|
||||||
|
|
||||||
|
Setiap service butuh sidecar container Dapr. Contoh:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
services:
|
||||||
|
app:
|
||||||
|
container_name: app
|
||||||
|
image: ghcr.io/asepharyana/asepharyana-hub/app:latest
|
||||||
|
restart: always
|
||||||
|
depends_on:
|
||||||
|
dapr-placement:
|
||||||
|
condition: service_healthy
|
||||||
|
nats:
|
||||||
|
condition: service_healthy
|
||||||
|
networks:
|
||||||
|
app-shared-net:
|
||||||
|
aliases:
|
||||||
|
- app
|
||||||
|
env_file:
|
||||||
|
- ../../.env
|
||||||
|
|
||||||
|
app-dapr:
|
||||||
|
container_name: app-dapr
|
||||||
|
image: daprio/daprd:latest
|
||||||
|
restart: always
|
||||||
|
depends_on:
|
||||||
|
dapr-placement:
|
||||||
|
condition: service_healthy
|
||||||
|
nats:
|
||||||
|
condition: service_healthy
|
||||||
|
networks:
|
||||||
|
- app-shared-net
|
||||||
|
depends_on:
|
||||||
|
dapr-placement:
|
||||||
|
condition: service_healthy
|
||||||
|
nats:
|
||||||
|
condition: service_healthy
|
||||||
|
otel-collector:
|
||||||
|
condition: service_started
|
||||||
|
networks:
|
||||||
|
- app-shared-net
|
||||||
|
command:
|
||||||
|
- './daprd'
|
||||||
|
- '--app-id=app'
|
||||||
|
- '--app-port=3000'
|
||||||
|
- '--dapr-http-port=3500'
|
||||||
|
- '--dapr-grpc-port=50001'
|
||||||
|
- '--placement-host-address=dapr-placement:50005'
|
||||||
|
- '--config=/dapr/config.yaml'
|
||||||
|
- '--resources-path=/dapr/components'
|
||||||
|
volumes:
|
||||||
|
- ../../infra/dapr:/dapr:ro
|
||||||
|
|
||||||
|
networks:
|
||||||
|
app-shared-net:
|
||||||
|
name: app-shared-net
|
||||||
|
external: true
|
||||||
|
```
|
||||||
|
|
||||||
|
## 2. Mengakses Dapr dari Service
|
||||||
|
|
||||||
|
### Via HTTP API (semua bahasa)
|
||||||
|
|
||||||
|
Sidecar listen di `localhost:3500`:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Publish event
|
||||||
|
curl -X POST http://localhost:3500/v1.0/publish/pubsub/hub.event.type \
|
||||||
|
-H "Content-Type: application/json" \
|
||||||
|
-d '{"key": "value"}'
|
||||||
|
|
||||||
|
# Service invocation
|
||||||
|
curl http://localhost:3500/v1.0/invoke/app/method/endpoint
|
||||||
|
|
||||||
|
# State store
|
||||||
|
curl -X POST http://localhost:3500/v1.0/state/statestore \
|
||||||
|
-H "Content-Type: application/json" \
|
||||||
|
-d '[{"key": "mykey", "value": "myvalue"}]'
|
||||||
|
```
|
||||||
|
|
||||||
|
### Via Dapr SDK (Rust)
|
||||||
|
|
||||||
|
Tambah ke `Cargo.toml`:
|
||||||
|
|
||||||
|
```toml
|
||||||
|
dapr-sdk = { version = "0.15", features = ["pubsub", "http"] }
|
||||||
|
tokio-stream = "0.1"
|
||||||
|
```
|
||||||
|
|
||||||
|
Contoh publish event:
|
||||||
|
|
||||||
|
```rust
|
||||||
|
use dapr_sdk::client::{Client, Event};
|
||||||
|
use dapr_sdk::DaprClient;
|
||||||
|
|
||||||
|
let client = DaprClient::new("127.0.0.1", 3500).await?;
|
||||||
|
client.publish_event("pubsub", "hub.image.cached", serde_json::json!({
|
||||||
|
"original_url": url,
|
||||||
|
"cdn_url": cdn_url,
|
||||||
|
})).await?;
|
||||||
|
```
|
||||||
|
|
||||||
|
Contoh subscribe event:
|
||||||
|
|
||||||
|
```rust
|
||||||
|
let mut stream = client.subscribe_events("pubsub", "hub.image.cached").await?;
|
||||||
|
while let Some(event) = stream.next().await {
|
||||||
|
let data: MyEvent = serde_json::from_slice(&event.data)?;
|
||||||
|
// handle event
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
## 3. Event Topics Convention
|
||||||
|
|
||||||
|
Gunakan prefix `hub.` untuk semua event:
|
||||||
|
|
||||||
|
| Topic | Payload | Description |
|
||||||
|
|-------|---------|-------------|
|
||||||
|
| `hub.image.cached` | `{original_url, cdn_url, source}` | Image selesai di-cache |
|
||||||
|
| `hub.image.repaired` | `{old_url, new_url}` | CNAME image diperbaiki |
|
||||||
|
| `hub.scrape.anime.done` | `{source, slug, duration}` | Scrape anime selesai |
|
||||||
|
| `hub.system.alert` | `{service, level, message}` | Error/alert dari service |
|
||||||
|
|
||||||
|
## 4. Local Development
|
||||||
|
|
||||||
|
Untuk development tanpa Docker:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# 1. Install Dapr CLI
|
||||||
|
# 2. Init Dapr local
|
||||||
|
dapr init
|
||||||
|
|
||||||
|
# 3. Run service dengan sidecar
|
||||||
|
dapr run --app-id app --app-port 3000 --dapr-http-port 3500 \
|
||||||
|
--resources-path ./infra/dapr/components \
|
||||||
|
-- cargo run
|
||||||
|
```
|
||||||
|
|
||||||
|
## 5. Verifikasi
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Sidecar health
|
||||||
|
curl http://localhost:3500/v1.0/healthz
|
||||||
|
|
||||||
|
# Publish test event
|
||||||
|
curl -X POST http://localhost:3500/v1.0/publish/pubsub/hub.test \
|
||||||
|
-H "Content-Type: application/json" \
|
||||||
|
-d '{"test": true}'
|
||||||
|
|
||||||
|
# NATS stream stats
|
||||||
|
curl http://localhost:8222/jszetstream
|
||||||
|
```
|
||||||
@@ -52,6 +52,49 @@ networks:
|
|||||||
|
|
||||||
Gunakan `app-shared-net` agar service dapat diakses oleh Traefik dan service lain.
|
Gunakan `app-shared-net` agar service dapat diakses oleh Traefik dan service lain.
|
||||||
|
|
||||||
|
## 3.5. Tambahkan Dapr sidecar (wajib untuk pub/sub)
|
||||||
|
|
||||||
|
Setiap service yang ingin menggunakan Dapr pub/sub atau service invocation harus punya sidecar.
|
||||||
|
Tambah di `infra/compose/<nama-app>.yml`:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
<nama-app>-dapr:
|
||||||
|
container_name: <nama-app>-dapr
|
||||||
|
image: daprio/daprd:latest
|
||||||
|
restart: always
|
||||||
|
depends_on:
|
||||||
|
dapr-placement:
|
||||||
|
condition: service_healthy
|
||||||
|
nats:
|
||||||
|
condition: service_healthy
|
||||||
|
otel-collector:
|
||||||
|
condition: service_started
|
||||||
|
networks:
|
||||||
|
- app-shared-net
|
||||||
|
command:
|
||||||
|
- './daprd'
|
||||||
|
- '--app-id=<nama-app>'
|
||||||
|
- '--app-port=<port>'
|
||||||
|
- '--dapr-http-port=3500'
|
||||||
|
- '--dapr-grpc-port=50001'
|
||||||
|
- '--placement-host-address=dapr-placement:50005'
|
||||||
|
- '--config=/dapr/config.yaml'
|
||||||
|
- '--resources-path=/dapr/components'
|
||||||
|
volumes:
|
||||||
|
- ../../infra/dapr:/dapr:ro
|
||||||
|
```
|
||||||
|
|
||||||
|
Pastikan juga app container punya `depends_on` ke dapr-placement, nats, dan otel-collector:
|
||||||
|
```yaml
|
||||||
|
depends_on:
|
||||||
|
dapr-placement:
|
||||||
|
condition: service_healthy
|
||||||
|
nats:
|
||||||
|
condition: service_healthy
|
||||||
|
otel-collector:
|
||||||
|
condition: service_started
|
||||||
|
```
|
||||||
|
|
||||||
## 4. Tambahkan route Traefik
|
## 4. Tambahkan route Traefik
|
||||||
|
|
||||||
Update `infra/traefik/dynamic/apps.yaml`:
|
Update `infra/traefik/dynamic/apps.yaml`:
|
||||||
|
|||||||
@@ -22,12 +22,9 @@ Use `asepharyana-hub` as the root hub repository.
|
|||||||
|
|
||||||
Current app submodules:
|
Current app submodules:
|
||||||
|
|
||||||
| Service | Path | Remote |
|
| Service | Path | Remote |
|
||||||
| -------------- | ---------------- | --------------------------------------- |
|
| ----------- | -------------- | --------------------------------------- |
|
||||||
| Elysia API | `apps/elysia` | `asepharyana/asepharyana-hub-elysia` |
|
| Scraper API | `apps/scraper` | `asepharyana/asepharyana-hub-scraper` |
|
||||||
| React frontend | `apps/react` | `asepharyana/asepharyana-hub-react` |
|
|
||||||
| Rust auth API | `apps/rust-auth` | `asepharyana/asepharyana-hub-rust-auth` |
|
|
||||||
| Scraper API | `apps/scraper` | `asepharyana/asepharyana-hub-scraper` |
|
|
||||||
|
|
||||||
## Consequences
|
## Consequences
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,124 @@
|
|||||||
|
# ADR 0002: Production `.env` via GitHub Encrypted Secret
|
||||||
|
|
||||||
|
## Status
|
||||||
|
|
||||||
|
Accepted
|
||||||
|
|
||||||
|
## Context
|
||||||
|
|
||||||
|
The project runs on a remote VPS (`orangevps`, IP `45.127.35.244`) that hosts multiple services via Docker Compose. These services require environment variables (database credentials, API keys, tokens) that must not be committed to the repository.
|
||||||
|
|
||||||
|
The production `.env` file on the VPS is **not** a copy of the committed `.env` in the repo root — it contains additional secrets (Portainer tokens, Discord bot tokens, etc.) that only exist in production.
|
||||||
|
|
||||||
|
Previously, the `.env` file on the VPS was edited manually via SSH, which led to drift between the local `.env` and the production `.env`. When the database server IP or port changed in the local `.env`, the production `.env` was not updated, causing service outages.
|
||||||
|
|
||||||
|
## Decision
|
||||||
|
|
||||||
|
The production `.env` file is stored as a **GitHub Actions encrypted secret** named `ENV_FILE_PRODUCTION`. During deployment, the `.github/workflows/deploy-docker.yml` workflow writes this secret to a file and SCPs it to the VPS.
|
||||||
|
|
||||||
|
### Flow
|
||||||
|
|
||||||
|
```
|
||||||
|
GitHub Secret (ENV_FILE_PRODUCTION)
|
||||||
|
│
|
||||||
|
▼ (deploy-docker.yml)
|
||||||
|
echo "$ENV_FILE_PRODUCTION" > .env.prod
|
||||||
|
scp .env.prod → VPS:$VPS_TARGET_DIR/.env
|
||||||
|
│
|
||||||
|
▼ (docker compose --env-file .env up)
|
||||||
|
Container reads $DATABASE_URL, $JWT_SECRET, etc.
|
||||||
|
```
|
||||||
|
|
||||||
|
### How to update
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# 1. Read current content from the VPS
|
||||||
|
ssh root@45.127.35.244 "cat /root/asepharyana-hub/.env"
|
||||||
|
|
||||||
|
# 2. Pipe updated content to the GitHub secret
|
||||||
|
# (requires gh CLI with repo access)
|
||||||
|
cat /path/to/updated-env | gh secret set ENV_FILE_PRODUCTION --repo asepharyana/asepharyana-hub
|
||||||
|
|
||||||
|
# 3. Trigger a redeploy to push it to the VPS
|
||||||
|
gh workflow run deploy-docker.yml
|
||||||
|
|
||||||
|
# OR apply immediately on the VPS (for hotfix):
|
||||||
|
ssh root@45.127.35.244 "sed -i 's|OLD_VALUE|NEW_VALUE|' /root/asepharyana-hub/.env"
|
||||||
|
# Then restart affected containers
|
||||||
|
```
|
||||||
|
|
||||||
|
## Server Topology
|
||||||
|
|
||||||
|
| Host | IP | Role |
|
||||||
|
|------|----|------|
|
||||||
|
| `orangevps` (VPS) | `45.127.35.244` | Docker host: Traefik, scraper-api, Redis, NATS, Dapr |
|
||||||
|
| `imrnes` (bare-metal) | `100.121.180.82` (Tailscale) | PostgreSQL (port 6432), Redis (port 6379), Browserless |
|
||||||
|
|
||||||
|
## Database
|
||||||
|
|
||||||
|
| Variable | Value |
|
||||||
|
|----------|-------|
|
||||||
|
| `DATABASE_URL` | `postgres://asephs:hunterz@100.121.180.82:6432/hub` |
|
||||||
|
| `REDIS_URL` | `redis://redis:6379` (Docker network, overridden per-service) |
|
||||||
|
| `EXTERNAL_BROWSERLESS_WS` | `ws://43.134.105.109:3001/?token=...` (external proxy) |
|
||||||
|
|
||||||
|
> **Important:** The Docker Compose `environment:` section uses variable interpolation (`${DATABASE_URL}`), which is resolved from the `--env-file .env` at compose time — NOT from the service's `env_file`. Both must be kept in sync.
|
||||||
|
|
||||||
|
## Docker Compose Project Structure
|
||||||
|
|
||||||
|
The VPS runs a single Docker Compose project named `compose` composed of multiple files:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
/root/asepharyana-hub/infra/compose/
|
||||||
|
├── traefik.yml # Reverse proxy (TLS termination, routing)
|
||||||
|
├── shared.yml # Redis
|
||||||
|
├── nats.yml # NATS message broker + JetStream
|
||||||
|
├── dapr.yml # Dapr placement service
|
||||||
|
├── scraper.yml # Scraper API + Dapr sidecar
|
||||||
|
└── observability.yml # OTel Collector + Jaeger + Dashboard
|
||||||
|
```
|
||||||
|
|
||||||
|
All files are always included together for dependency resolution:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker compose \
|
||||||
|
--env-file .env \
|
||||||
|
-f infra/compose/traefik.yml \
|
||||||
|
-f infra/compose/shared.yml \
|
||||||
|
-f infra/compose/scraper.yml \
|
||||||
|
-f infra/compose/nats.yml \
|
||||||
|
-f infra/compose/dapr.yml \
|
||||||
|
-f infra/compose/observability.yml \
|
||||||
|
up -d
|
||||||
|
```
|
||||||
|
|
||||||
|
## GitHub Secrets Required
|
||||||
|
|
||||||
|
| Secret | Description |
|
||||||
|
|--------|-------------|
|
||||||
|
| `SSH_PRIVATE_KEY` | SSH key for VPS access |
|
||||||
|
| `VPS_HOST` | `45.127.35.244` |
|
||||||
|
| `VPS_USER` | `root` |
|
||||||
|
| `VPS_TARGET_DIR` | `/root/asepharyana-hub` |
|
||||||
|
| `ENV_FILE_PRODUCTION` | Full `.env` content for production |
|
||||||
|
|
||||||
|
## Consequences
|
||||||
|
|
||||||
|
### Positive
|
||||||
|
|
||||||
|
- Environment is version-controlled via GitHub Secrets audit log.
|
||||||
|
- No risk of committing secrets to the repo.
|
||||||
|
- Deployment is fully automated — `.env` is pushed on every deploy.
|
||||||
|
- Easy to rotate secrets: update `ENV_FILE_PRODUCTION` and redeploy.
|
||||||
|
|
||||||
|
### Negative
|
||||||
|
|
||||||
|
- The secret is opaque — you cannot diff it or review changes via PR.
|
||||||
|
- If the secret falls out of sync with the local `.env`, services silently break on next deploy.
|
||||||
|
- Requires `gh` CLI or GitHub UI to update — not a simple file edit.
|
||||||
|
|
||||||
|
### Mitigations
|
||||||
|
|
||||||
|
- Keep the **committed `.env`** in the repo root as the source of truth for non-secret values (database URL, ports, API endpoints).
|
||||||
|
- Document any manual SSH hotfix at the same time as updating the GitHub secret.
|
||||||
|
- Run `gh secret set ENV_FILE_PRODUCTION` with the latest server `.env` content after any hotfix.
|
||||||
@@ -0,0 +1,234 @@
|
|||||||
|
# Backup & Disaster Recovery
|
||||||
|
|
||||||
|
## Aset yang Perlu di-Backup
|
||||||
|
|
||||||
|
| Aset | Lokasi | Frekuensi | Metode |
|
||||||
|
|------|--------|-----------|--------|
|
||||||
|
| Database PostgreSQL | `imrnes` (100.121.180.82:6432) | Harian | `pg_dump` |
|
||||||
|
| Volume Redis | `orangevps` (Docker volume) | Opsional | Redis RDB / AOF |
|
||||||
|
| Volume NATS JetStream | `orangevps` (Docker volume) | Opsional | File copy |
|
||||||
|
| Docker Compose manifests | GitHub (hub repo) | Real-time | Git |
|
||||||
|
| Environment variables | GitHub secret `ENV_FILE_PRODUCTION` | Manual | `gh secret set` |
|
||||||
|
| TLS certificates | `orangevps` (`/root/*.pem`, `*.key`) | Saat renew | SCP |
|
||||||
|
| Tailscale auth | Tailscale admin console | - | Cloud-managed |
|
||||||
|
| GitHub Actions secrets | GitHub UI | Manual | Backup list |
|
||||||
|
|
||||||
|
## Database PostgreSQL (Prioritas Tertinggi)
|
||||||
|
|
||||||
|
### Backup Manual
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Dari orangevps (via Tailscale)
|
||||||
|
pg_dump -h 100.121.180.82 -p 6432 -U asephs -d hub \
|
||||||
|
--no-owner --no-acl \
|
||||||
|
-F c -f /root/db-backups/hub-$(date +%Y%m%d-%H%M%S).dump
|
||||||
|
|
||||||
|
# Atau dari imrnes langsung
|
||||||
|
pg_dump -U asephs -d hub \
|
||||||
|
-F c -f /backup/hub/hub-$(date +%Y%m%d-%H%M%S).dump
|
||||||
|
```
|
||||||
|
|
||||||
|
### Restore
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Drop dan recreate database
|
||||||
|
dropdb -h 100.121.180.82 -p 6432 -U asephs hub
|
||||||
|
createdb -h 100.121.180.82 -p 6432 -U asephs hub
|
||||||
|
|
||||||
|
# Restore dari dump
|
||||||
|
pg_restore -h 100.121.180.82 -p 6432 -U asephs -d hub \
|
||||||
|
--no-owner --no-acl \
|
||||||
|
/path/to/backup/hub-20260101-120000.dump
|
||||||
|
```
|
||||||
|
|
||||||
|
### Backup Otomatis (via Cron di imrnes)
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# /etc/cron.d/hub-db-backup
|
||||||
|
0 2 * * * root pg_dump -U asephs -d hub -F c -f /backup/hub/hub-$(date +\%Y\%m\%d).dump && find /backup/hub -name "hub-*.dump" -mtime +30 -delete
|
||||||
|
```
|
||||||
|
|
||||||
|
## Volume Docker
|
||||||
|
|
||||||
|
### Redis
|
||||||
|
|
||||||
|
Redis data bisa di-recover dari NATS events (event sourcing). Jika tidak ada persistence requirement, cukup restart:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker volume rm redis_data
|
||||||
|
docker compose -f infra/compose/shared.yml up -d
|
||||||
|
```
|
||||||
|
|
||||||
|
Jika perlu backup:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Save RDB snapshot
|
||||||
|
docker exec redis redis-cli SAVE
|
||||||
|
|
||||||
|
# Copy dari volume
|
||||||
|
docker run --rm -v redis_data:/data -v /backup:/backup alpine cp /data/dump.rdb /backup/redis-$(date +%Y%m%d).rdb
|
||||||
|
```
|
||||||
|
|
||||||
|
### NATS JetStream
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Backup volume
|
||||||
|
docker run --rm -v nats_data:/data -v /backup:/backup alpine \
|
||||||
|
tar czf /backup/nats-$(date +%Y%m%d).tar.gz -C /data .
|
||||||
|
```
|
||||||
|
|
||||||
|
## Environment Variables
|
||||||
|
|
||||||
|
### Backup `.env` dari VPS
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Simpan current .env dari VPS
|
||||||
|
ssh root@45.127.35.244 "cat /root/asepharyana-hub/.env" > .env.backup.$(date +%Y%m%d)
|
||||||
|
|
||||||
|
# Update GitHub secret
|
||||||
|
cat .env.backup.$(date +%Y%m%d) | gh secret set ENV_FILE_PRODUCTION --repo asepharyana/asepharyana-hub
|
||||||
|
```
|
||||||
|
|
||||||
|
### Restore `.env` jika hilang
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Buat .env baru dari template
|
||||||
|
cp .env.example .env
|
||||||
|
|
||||||
|
# Edit secrets (manual dari password manager atau GitHub secret)
|
||||||
|
# Atau download dari GitHub secret
|
||||||
|
gh secret list --repo asepharyana/asepharyana-hub
|
||||||
|
```
|
||||||
|
|
||||||
|
## TLS Certificates
|
||||||
|
|
||||||
|
### Backup
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Di orangevps
|
||||||
|
tar czf /root/cert-backup-$(date +%Y%m%d).tar.gz \
|
||||||
|
/root/asepharyana.my.id.pem \
|
||||||
|
/root/asepharyana.my.id.key \
|
||||||
|
/root/asepharyana.web.id.pem \
|
||||||
|
/root/asepharyana.web.id.key \
|
||||||
|
/root/asepharyana-hub/infra/traefik/dynamic/ssl.yaml
|
||||||
|
|
||||||
|
# SCP ke local
|
||||||
|
scp root@45.127.35.244:/root/cert-backup-*.tar.gz .
|
||||||
|
```
|
||||||
|
|
||||||
|
### Restore
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# SCP ke VPS
|
||||||
|
scp cert-backup-20260101.tar.gz root@45.127.35.244:/root/
|
||||||
|
|
||||||
|
# Extract
|
||||||
|
ssh root@45.127.35.244 "tar xzf /root/cert-backup-20260101.tar.gz -C / && docker restart traefik"
|
||||||
|
```
|
||||||
|
|
||||||
|
## Disaster Recovery Scenarios
|
||||||
|
|
||||||
|
### Skenario 1: VPS (orangevps) mati total
|
||||||
|
|
||||||
|
**Dampak:** Semua service down.
|
||||||
|
|
||||||
|
**Recovery:**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# 1. Provision VPS baru (atau restore dari snapshot)
|
||||||
|
# 2. Install Docker + Tailscale
|
||||||
|
# 3. Clone repo
|
||||||
|
git clone https://github.com/asepharyana/asepharyana-hub.git /root/asepharyana-hub
|
||||||
|
|
||||||
|
# 4. Setup Tailscale, route service
|
||||||
|
# 5. Restore .env
|
||||||
|
echo "<ENV_FILE_PRODUCTION>" > /root/asepharyana-hub/.env
|
||||||
|
|
||||||
|
# 6. Restore TLS certs
|
||||||
|
# 7. Create network
|
||||||
|
docker network create app-shared-net
|
||||||
|
|
||||||
|
# 8. Start services sesuai urutan
|
||||||
|
cd /root/asepharyana-hub
|
||||||
|
for f in shared.yml nats.yml dapr.yml traefik.yml scraper.yml; do
|
||||||
|
docker compose -f infra/compose/$f --env-file .env up -d
|
||||||
|
done
|
||||||
|
|
||||||
|
# 9. Update DNS jika IP baru
|
||||||
|
```
|
||||||
|
|
||||||
|
### Skenario 2: Database (imrnes) mati total
|
||||||
|
|
||||||
|
**Dampak:** Semua service yang butuh database error.
|
||||||
|
|
||||||
|
**Recovery:**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# 1. Fix imrnes atau provision server baru
|
||||||
|
# 2. Setup PostgreSQL
|
||||||
|
# 3. Restore dari backup terakhir
|
||||||
|
# 4. Update Tailscale IP jika perlu
|
||||||
|
# 5. Update .env dan GitHub secret
|
||||||
|
# 6. Redeploy
|
||||||
|
```
|
||||||
|
|
||||||
|
### Skenario 3: GitHub repository hilang
|
||||||
|
|
||||||
|
**Dampak:** Kehilangan CI/CD, tapi Docker images masih ada di GHCR.
|
||||||
|
|
||||||
|
**Recovery:**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# 1. Create repo baru di GitHub
|
||||||
|
# 2. Push dari local clone
|
||||||
|
git remote add origin-new https://github.com/asepharyana/asepharyana-hub-new.git
|
||||||
|
git push origin-new main
|
||||||
|
|
||||||
|
# 3. Re-create GitHub secrets
|
||||||
|
# 4. Re-create workflows
|
||||||
|
# 5. Update VPS remote
|
||||||
|
ssh root@45.127.35.244 "cd /root/asepharyana-hub && git remote set-url origin https://github.com/asepharyana/asepharyana-hub-new.git"
|
||||||
|
```
|
||||||
|
|
||||||
|
### Skenario 4: GHCR registry tidak bisa diakses
|
||||||
|
|
||||||
|
**Dampak:** Tidak bisa pull image.
|
||||||
|
|
||||||
|
**Recovery:**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# 1. Build image langsung di VPS
|
||||||
|
docker build -f infra/docker/scraper.Dockerfile -t ghcr.io/asepharyana/asepharyana-hub/scraper-api:local .
|
||||||
|
|
||||||
|
# 2. Update compose file untuk sementara
|
||||||
|
sed -i 's|image: ghcr.io/.*|image: ghcr.io/asepharyana/asepharyana-hub/scraper-api:local|' infra/compose/scraper.yml
|
||||||
|
|
||||||
|
# 3. Start
|
||||||
|
docker compose -f infra/compose/scraper.yml up -d
|
||||||
|
```
|
||||||
|
|
||||||
|
### Skenario 5: Semua server mati (total loss)
|
||||||
|
|
||||||
|
**Recovery:**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# 1. Provision VPS baru
|
||||||
|
# 2. Provision server database baru
|
||||||
|
# 3. Setup Tailscale
|
||||||
|
# 4. Clone repo, restore .env, certs
|
||||||
|
# 5. Restore database dari backup (jika ada)
|
||||||
|
# 6. Jika tidak ada backup database:
|
||||||
|
# - Build image dari GHCR
|
||||||
|
# - Start service dengan database kosong
|
||||||
|
# - Data akan terisi ulang dari scraping
|
||||||
|
```
|
||||||
|
|
||||||
|
## Checklist Pencegahan
|
||||||
|
|
||||||
|
- [ ] Cron job backup database berjalan
|
||||||
|
- [ ] Backup `.env` disimpan di luar VPS (password manager)
|
||||||
|
- [ ] TLS certificates backup disimpan di luar VPS
|
||||||
|
- [ ] GitHub secrets terdaftar (tidak hanya diingat)
|
||||||
|
- [ ] Docker images bisa di-rebuild dari CI (GHCR sebagai source of truth)
|
||||||
|
- [ ] Tailscale admin access via multiple accounts
|
||||||
@@ -0,0 +1,248 @@
|
|||||||
|
# CI/CD Pipeline
|
||||||
|
|
||||||
|
Dokumentasi pipeline CI/CD untuk `asepharyana-hub`. Terdiri dari 5 GitHub Actions workflow yang saling terhubung.
|
||||||
|
|
||||||
|
## Workflow Overview
|
||||||
|
|
||||||
|
```
|
||||||
|
┌─────────────┐
|
||||||
|
│ Lint │ (PR/push → Biome)
|
||||||
|
└──────┬──────┘
|
||||||
|
│
|
||||||
|
Push ke main ─────┼────── repository_dispatch
|
||||||
|
│
|
||||||
|
┌──────▼──────────────────┐
|
||||||
|
│ docker-build-push.yml │
|
||||||
|
│ │
|
||||||
|
│ Phase 1: Detect │
|
||||||
|
│ Phase 2: Build & Push │
|
||||||
|
│ Phase 3: Update │
|
||||||
|
│ manifests │
|
||||||
|
└──────┬──────────────────┘
|
||||||
|
│ workflow_run
|
||||||
|
┌──────▼──────────────┐
|
||||||
|
│ deploy-docker.yml │
|
||||||
|
│ SSH → VPS │
|
||||||
|
│ Pull → Restart │
|
||||||
|
└─────────────────────┘
|
||||||
|
|
||||||
|
repository_dispatch ──► update-submodule.yml
|
||||||
|
(dari submodule) (update pointer → commit)
|
||||||
|
│
|
||||||
|
▼
|
||||||
|
docker-build-push.yml
|
||||||
|
(triggered by push)
|
||||||
|
```
|
||||||
|
|
||||||
|
## Workflow Detail
|
||||||
|
|
||||||
|
### 1. Lint (`lint.yml`)
|
||||||
|
|
||||||
|
**Trigger:** PR/push ke `main` yang mengubah `*.json`, `*.js`, `biome.json`
|
||||||
|
|
||||||
|
**Aksi:**
|
||||||
|
- Checkout repo dengan submodules
|
||||||
|
- Setup Bun
|
||||||
|
- `bun install --frozen-lockfile`
|
||||||
|
- `bun run ci` (Biome CI mode)
|
||||||
|
|
||||||
|
**Permissions:** read-only
|
||||||
|
|
||||||
|
### 2. Build and Push Docker Images (`docker-build-push.yml`)
|
||||||
|
|
||||||
|
**Trigger:**
|
||||||
|
- Push ke `main` yang mengubah `apps/**`, `infra/**`, atau file workflow
|
||||||
|
- `repository_dispatch` tipe `submodule-updated`
|
||||||
|
- `workflow_dispatch` (manual)
|
||||||
|
|
||||||
|
**Concurrency:** Satu workflow per branch (cancel-in-progress=false)
|
||||||
|
|
||||||
|
#### Phase 1: Detect Changes
|
||||||
|
|
||||||
|
Job `changes` mendeteksi service mana yang perlu di-build:
|
||||||
|
|
||||||
|
- **Push event:** `git diff --name-only` antara `before` dan `after` SHA
|
||||||
|
- **repository_dispatch:** Parse payload `{service, sha}` dan validasi
|
||||||
|
- **workflow_dispatch:** Build semua service
|
||||||
|
|
||||||
|
Output format matrix:
|
||||||
|
```json
|
||||||
|
[{"id":"scraper-api","target":"docker-scraper","path":"apps/scraper"}]
|
||||||
|
```
|
||||||
|
|
||||||
|
#### Phase 2: Build & Push (Matrix)
|
||||||
|
|
||||||
|
Job `build` berjalan paralel per service (matrix strategy):
|
||||||
|
|
||||||
|
1. Checkout repo + sync submodule
|
||||||
|
2. Jika `repository_dispatch`, checkout submodule ke SHA tertentu
|
||||||
|
3. Login ke GHCR
|
||||||
|
4. Setup Docker Buildx
|
||||||
|
5. Build & push dengan tag:
|
||||||
|
- `ghcr.io/asepharyana/asepharyana-hub/<service>:latest`
|
||||||
|
- `ghcr.io/asepharyana/asepharyana-hub/<service>:sha-<shortsha>`
|
||||||
|
6. Build cache: registry-based (`:<service>:buildcache`)
|
||||||
|
|
||||||
|
#### Phase 3: Update Manifests
|
||||||
|
|
||||||
|
Job `update-manifest`:
|
||||||
|
|
||||||
|
1. Update image tag di compose file (`infra/compose/<service>.yml`)
|
||||||
|
2. Jika `repository_dispatch`, update submodule pointer
|
||||||
|
3. Commit dengan message `chore: update manifests and submodules [skip ci]`
|
||||||
|
4. Push dengan retry (3 attempts, rebase jika conflict)
|
||||||
|
|
||||||
|
### 3. Deploy Docker to VPS (`deploy-docker.yml`)
|
||||||
|
|
||||||
|
**Trigger:**
|
||||||
|
- `workflow_run` setelah `docker-build-push.yml` selesai
|
||||||
|
- Push ke `main` yang mengubah `infra/**`
|
||||||
|
- `workflow_dispatch` (manual)
|
||||||
|
|
||||||
|
**Concurrency:** Satu deployment dalam satu waktu (`group: deploy-vps`)
|
||||||
|
|
||||||
|
**Aksi di VPS (via SSH):**
|
||||||
|
|
||||||
|
```
|
||||||
|
1. Setup SSH multiplexing
|
||||||
|
2. SCP .env dari GitHub secret ke VPS
|
||||||
|
3. Docker login ke GHCR
|
||||||
|
4. Git sync (fetch + reset --hard)
|
||||||
|
5. Detect changed files:
|
||||||
|
├─ Compose stack changes → selective container update
|
||||||
|
├─ Traefik dynamic config → SIGHUP
|
||||||
|
└─ Other infra → full deploy
|
||||||
|
6. Pull images (retry 3x)
|
||||||
|
7. Remove stale containers
|
||||||
|
8. Up services
|
||||||
|
9. SIGHUP Traefik jika perlu
|
||||||
|
```
|
||||||
|
|
||||||
|
### 4. Security Scan (`security.yml`)
|
||||||
|
|
||||||
|
**Trigger:**
|
||||||
|
- PR ke `main`
|
||||||
|
- Jadwal: Setiap Senin (`0 6 * * 1`)
|
||||||
|
|
||||||
|
**Aksi:**
|
||||||
|
- Checkout dengan fetch-depth 2
|
||||||
|
- CodeQL init untuk Rust
|
||||||
|
- `cargo build` di `apps/scraper`
|
||||||
|
- CodeQL analyze
|
||||||
|
|
||||||
|
### 5. Update Submodule Pointer (`update-submodule.yml`)
|
||||||
|
|
||||||
|
**Trigger:** `repository_dispatch` tipe `submodule-updated`
|
||||||
|
|
||||||
|
**Aksi:**
|
||||||
|
1. Validasi payload (`service`, `sha`)
|
||||||
|
2. Map service ke submodule path (e.g., `scraper-api` → `apps/scraper`)
|
||||||
|
3. Update submodule ke SHA yang diberikan
|
||||||
|
4. Commit sebagai `monrepo-bot` dengan message:
|
||||||
|
`chore: update <service> to <shortsha>`
|
||||||
|
5. Push dengan retry (3 attempts)
|
||||||
|
|
||||||
|
## Flow Submodule Update
|
||||||
|
|
||||||
|
Flow lengkap ketika code berubah di submodule repo:
|
||||||
|
|
||||||
|
```
|
||||||
|
1. Developer push ke asepharyana-hub-scraper
|
||||||
|
2. GitHub Action di scraper repo kirim repository_dispatch
|
||||||
|
ke asepharyana-hub
|
||||||
|
3. update-submodule.yml terima dispatch, update pointer
|
||||||
|
4. Commit masuk ke hub repo main
|
||||||
|
5. Commit ini trigger docker-build-push.yml
|
||||||
|
(push ke main dengan path apps/scraper/**)
|
||||||
|
6. Build image baru, update compose file
|
||||||
|
7. Deploy ke VPS
|
||||||
|
```
|
||||||
|
|
||||||
|
## Secrets yang Diperlukan
|
||||||
|
|
||||||
|
| Secret | Workflow | Deskripsi |
|
||||||
|
|--------|----------|-----------|
|
||||||
|
| `SSH_PRIVATE_KEY` | deploy-docker | SSH key untuk akses VPS |
|
||||||
|
| `VPS_HOST` | deploy-docker | IP VPS (`45.127.35.244`) |
|
||||||
|
| `VPS_USER` | deploy-docker | User SSH (`root`) |
|
||||||
|
| `VPS_TARGET_DIR` | deploy-docker | Dir di VPS (`/root/asepharyana-hub`) |
|
||||||
|
| `ENV_FILE_PRODUCTION` | deploy-docker | Full `.env` production |
|
||||||
|
|
||||||
|
## Menambahkan Service Baru ke Pipeline
|
||||||
|
|
||||||
|
Untuk menambahkan service baru, update:
|
||||||
|
|
||||||
|
### `docker-build-push.yml`
|
||||||
|
|
||||||
|
1. **Phase 1 — `changes` job:** Tambah detection logic untuk service baru:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
echo "new-service=$(changed '^(apps/new-service(/|$)|\.github/workflows/docker-build-push\.yml$|infra/docker/new-service\.Dockerfile$)')" >> "$GITHUB_OUTPUT"
|
||||||
|
```
|
||||||
|
|
||||||
|
2. **Phase 1 — `repository_dispatch`:** Tambah case:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
case "$SERVICE" in
|
||||||
|
scraper-api|new-service) ;;
|
||||||
|
```
|
||||||
|
|
||||||
|
3. **Phase 1 — `set-matrix`:** Tambah service:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
if [ "${{ ...['new-service'] == 'true' ... }}" == "true" ]; then add_service "new-service" "docker-new-service" "apps/new-service"; fi
|
||||||
|
```
|
||||||
|
|
||||||
|
4. **Phase 2 — `meta` step:** Tambah mapping Dockerfile:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
"new-service") echo "dockerfile=infra/docker/new-service.Dockerfile" >> $GITHUB_OUTPUT ;;
|
||||||
|
```
|
||||||
|
|
||||||
|
5. **Phase 3 — `update-manifest`:** Tambah mapping:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
SERVICES["new-service"]="new-service.yml"
|
||||||
|
PATHS["new-service"]="apps/new-service"
|
||||||
|
```
|
||||||
|
|
||||||
|
### `deploy-docker.yml`
|
||||||
|
|
||||||
|
Tambah compose file ke `ALL_COMPOSE_FILES`:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
ALL_COMPOSE_FILES="infra/compose/traefik.yml infra/compose/shared.yml infra/compose/scraper.yml infra/compose/nats.yml infra/compose/dapr.yml infra/compose/new-service.yml"
|
||||||
|
```
|
||||||
|
|
||||||
|
## Rollback
|
||||||
|
|
||||||
|
### Rollback Image
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Cari SHA tag sebelumnya di GHCR packages
|
||||||
|
# Update compose file ke tag tersebut
|
||||||
|
sed -i 's|sha-badcommit|sha-goodcommit|g' infra/compose/scraper.yml
|
||||||
|
git commit -am "fix: rollback scraper-api to sha-goodcommit"
|
||||||
|
git push
|
||||||
|
```
|
||||||
|
|
||||||
|
### Rollback via Git Revert
|
||||||
|
|
||||||
|
```bash
|
||||||
|
git revert HEAD
|
||||||
|
git push origin main
|
||||||
|
# Pipeline otomatis build dan deploy
|
||||||
|
```
|
||||||
|
|
||||||
|
## Monitoring Pipeline
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Cek status workflow terbaru
|
||||||
|
gh run list --limit 5
|
||||||
|
|
||||||
|
# Lihat log workflow tertentu
|
||||||
|
gh run view <run-id> --log
|
||||||
|
|
||||||
|
# Trigger workflow manual
|
||||||
|
gh workflow run deploy-docker.yml
|
||||||
|
```
|
||||||
@@ -1,39 +0,0 @@
|
|||||||
# Squid Proxy Configuration (Archived)
|
|
||||||
|
|
||||||
**Status**: Archived; not referenced by active infrastructure.
|
|
||||||
|
|
||||||
**Date archived**: 2026-06-04
|
|
||||||
|
|
||||||
**Original location**: `infra/squid.conf`
|
|
||||||
|
|
||||||
**Reason**: No active Docker Compose service mounts or starts Squid. The config is preserved here for reference if Squid proxying is restored later.
|
|
||||||
|
|
||||||
## Original configuration
|
|
||||||
|
|
||||||
```squid
|
|
||||||
# Allow localhost access only
|
|
||||||
acl localhost src 127.0.0.1/32 ::1/128
|
|
||||||
http_access allow localhost
|
|
||||||
|
|
||||||
# Allow Docker bridge subnets (Adjusted for our expected subnets)
|
|
||||||
acl docker_bridge src 172.16.0.0/12
|
|
||||||
http_access allow docker_bridge
|
|
||||||
|
|
||||||
# Deny all other access
|
|
||||||
http_access deny all
|
|
||||||
|
|
||||||
# Standard port - Explicitly bind to all interfaces
|
|
||||||
http_port 0.0.0.0:3128
|
|
||||||
|
|
||||||
# Hide source info
|
|
||||||
forwarded_for off
|
|
||||||
via off
|
|
||||||
```
|
|
||||||
|
|
||||||
## Restore checklist
|
|
||||||
|
|
||||||
1. Copy this config back to `infra/squid.conf`.
|
|
||||||
2. Add a Squid service to an active compose file.
|
|
||||||
3. Mount the config into the Squid container.
|
|
||||||
4. Document which services should use the proxy.
|
|
||||||
5. Validate network exposure and authentication before production use.
|
|
||||||
@@ -0,0 +1,286 @@
|
|||||||
|
# NATS + JetStream Guide
|
||||||
|
|
||||||
|
Dokumentasi konfigurasi, penggunaan, dan troubleshooting NATS di infrastruktur `asepharyana-hub`.
|
||||||
|
|
||||||
|
## Arsitektur
|
||||||
|
|
||||||
|
NATS berjalan di container `nats` dengan JetStream diaktifkan (`-js`). Data persistent disimpan di volume Docker `nats_data`.
|
||||||
|
|
||||||
|
```
|
||||||
|
Service ──► NATS (port 4222) ──► JetStream (disk)
|
||||||
|
│
|
||||||
|
├─ Monitoring HTTP: port 8222
|
||||||
|
└─ Client connections: port 4222
|
||||||
|
```
|
||||||
|
|
||||||
|
### Hubungan dengan Dapr
|
||||||
|
|
||||||
|
Saat ini Dapr pub/sub menggunakan **Redis** (`pubsub.redis`), bukan NATS. NATS berfungsi sebagai message broker independen untuk:
|
||||||
|
|
||||||
|
- Event streaming antar service
|
||||||
|
- Persistent job queues
|
||||||
|
- Pub/sub untuk service yang tidak menggunakan Dapr
|
||||||
|
|
||||||
|
Jika ingin Dapr menggunakan NATS sebagai backend pub/sub, ganti komponen `pubsub.yaml`:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
apiVersion: dapr.io/v1alpha1
|
||||||
|
kind: Component
|
||||||
|
metadata:
|
||||||
|
name: pubsub
|
||||||
|
spec:
|
||||||
|
type: pubsub.nats
|
||||||
|
version: v1
|
||||||
|
metadata:
|
||||||
|
- name: natsURL
|
||||||
|
value: nats://nats:4222
|
||||||
|
```
|
||||||
|
|
||||||
|
## Konfigurasi Compose
|
||||||
|
|
||||||
|
File: `infra/compose/nats.yml`
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
services:
|
||||||
|
nats:
|
||||||
|
container_name: nats
|
||||||
|
image: nats:latest
|
||||||
|
restart: always
|
||||||
|
networks:
|
||||||
|
app-shared-net:
|
||||||
|
aliases:
|
||||||
|
- nats
|
||||||
|
ports:
|
||||||
|
- '4222:4222' # client connections
|
||||||
|
- '8222:8222' # HTTP monitor
|
||||||
|
command:
|
||||||
|
- '-js' # enable JetStream
|
||||||
|
- '-sd'
|
||||||
|
- '/data' # storage directory
|
||||||
|
volumes:
|
||||||
|
- nats_data:/data
|
||||||
|
```
|
||||||
|
|
||||||
|
## CLI Tools
|
||||||
|
|
||||||
|
### Install NATS CLI
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Linux
|
||||||
|
curl -sf https://bin.nats.dev/nats | sh
|
||||||
|
sudo mv nats /usr/local/bin/
|
||||||
|
|
||||||
|
# Atau via package manager
|
||||||
|
# brew install nats-io/nats-tools/nats (macOS)
|
||||||
|
```
|
||||||
|
|
||||||
|
### Koneksi ke NATS
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Dari host (port 4222 ter-expose)
|
||||||
|
nats context save hub --server nats://localhost:4222 --description "Hub Production"
|
||||||
|
nats context select hub
|
||||||
|
|
||||||
|
# Test koneksi
|
||||||
|
nats server check
|
||||||
|
nats server info
|
||||||
|
```
|
||||||
|
|
||||||
|
### Manage Streams (JetStream)
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# List semua stream
|
||||||
|
nats stream list
|
||||||
|
|
||||||
|
# Lihat detail stream
|
||||||
|
nats stream info <stream-name>
|
||||||
|
|
||||||
|
# Buat stream
|
||||||
|
nats stream add <stream-name> \
|
||||||
|
--subjects "hub.>" \
|
||||||
|
--storage file \
|
||||||
|
--max-msgs 1000000 \
|
||||||
|
--max-bytes 1G \
|
||||||
|
--retention limits
|
||||||
|
|
||||||
|
# Hapus stream
|
||||||
|
nats stream rm <stream-name>
|
||||||
|
|
||||||
|
# Purge (hapus semua message, retain stream)
|
||||||
|
nats stream purge <stream-name>
|
||||||
|
```
|
||||||
|
|
||||||
|
### Pub/Sub
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Subscribe ke subject
|
||||||
|
nats sub "hub.>"
|
||||||
|
nats sub "hub.image.cached"
|
||||||
|
|
||||||
|
# Publish message
|
||||||
|
nats pub "hub.test" '{"message": "hello"}'
|
||||||
|
nats pub "hub.image.cached" '{"original_url": "https://example.com/img.jpg", "cdn_url": "https://cdn.example.com/img.jpg"}'
|
||||||
|
|
||||||
|
# Request-reply
|
||||||
|
nats request "hub.service.do" '{"task": "process"}'
|
||||||
|
```
|
||||||
|
|
||||||
|
### Monitoring via HTTP API
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Server info
|
||||||
|
curl http://localhost:8222/
|
||||||
|
|
||||||
|
# JetStream info
|
||||||
|
curl http://localhost:8222/jszetstream
|
||||||
|
|
||||||
|
# Stream detail
|
||||||
|
curl http://localhost:8222/jszetstream?stream=<stream-name>
|
||||||
|
|
||||||
|
# Consumer info
|
||||||
|
curl http://localhost:8222/jszetstream?stream=<stream-name>&consumer=<consumer-name>
|
||||||
|
|
||||||
|
# Server stats
|
||||||
|
curl http://localhost:8222/varz
|
||||||
|
|
||||||
|
# Connections
|
||||||
|
curl http://localhost:8222/connz
|
||||||
|
```
|
||||||
|
|
||||||
|
## Event Topics Convention
|
||||||
|
|
||||||
|
Semua topik menggunakan prefix `hub.`:
|
||||||
|
|
||||||
|
| Subject | Payload | Deskripsi |
|
||||||
|
|---------|---------|-----------|
|
||||||
|
| `hub.image.cached` | `{original_url, cdn_url, source}` | Image selesai di-cache |
|
||||||
|
| `hub.image.repaired` | `{old_url, new_url}` | CNAME image diperbaiki |
|
||||||
|
| `hub.scrape.anime.done` | `{source, slug, duration}` | Scrape anime selesai |
|
||||||
|
| `hub.system.alert` | `{service, level, message}` | Error/alert dari service |
|
||||||
|
| `hub.test` | Any | Testing |
|
||||||
|
|
||||||
|
### Wildcard Subjects
|
||||||
|
|
||||||
|
NATS mendukung wildcard:
|
||||||
|
|
||||||
|
- `hub.>` — semua event hub (multi-level)
|
||||||
|
- `hub.image.*` — semua event image (single-level)
|
||||||
|
- `hub.*.done` — semua event yang selesai (single-level)
|
||||||
|
|
||||||
|
## JetStream Configuration
|
||||||
|
|
||||||
|
### Storage
|
||||||
|
|
||||||
|
Data JetStream disimpan di volume Docker `nats_data`.
|
||||||
|
|
||||||
|
Lokasi di VPS:
|
||||||
|
```bash
|
||||||
|
docker volume inspect nats_data
|
||||||
|
# atau
|
||||||
|
ls -la /var/lib/docker/volumes/nats_data/_data/
|
||||||
|
```
|
||||||
|
|
||||||
|
### Memory & Limits
|
||||||
|
|
||||||
|
NATS tidak memiliki konfigurasi limit memori default. Untuk production, pertimbangkan:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
command:
|
||||||
|
- '-js'
|
||||||
|
- '-sd'
|
||||||
|
- '/data'
|
||||||
|
- '--max_pending_size=64MB'
|
||||||
|
- '--max_payload=1MB'
|
||||||
|
```
|
||||||
|
|
||||||
|
Atau gunakan NATS configuration file:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
# nats-server.conf
|
||||||
|
jetstream:
|
||||||
|
max_memory_store: 256MB
|
||||||
|
max_file_store: 10GB
|
||||||
|
```
|
||||||
|
|
||||||
|
## Troubleshooting
|
||||||
|
|
||||||
|
### Stream data tidak muncul
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# 1. Cek koneksi NATS
|
||||||
|
nats server check
|
||||||
|
|
||||||
|
# 2. Cek apakah JetStream aktif
|
||||||
|
curl http://localhost:8222/jszetstream
|
||||||
|
|
||||||
|
# 3. Cek stream dan message count
|
||||||
|
nats stream list
|
||||||
|
|
||||||
|
# 4. Subscribe langsung untuk test
|
||||||
|
nats sub ">"
|
||||||
|
```
|
||||||
|
|
||||||
|
### NATS tidak bisa start
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Cek log
|
||||||
|
docker logs nats
|
||||||
|
|
||||||
|
# Cek apakah port 4222 sudah dipakai
|
||||||
|
ss -tlnp | grep 4222
|
||||||
|
|
||||||
|
# Cek volume data korup
|
||||||
|
docker run --rm -v nats_data:/data alpine ls -la /data
|
||||||
|
|
||||||
|
# Restart
|
||||||
|
docker compose -f infra/compose/nats.yml up -d --force-recreate
|
||||||
|
```
|
||||||
|
|
||||||
|
### Disk JetStream penuh
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Cek ukuran volume
|
||||||
|
docker system df -v | grep nats_data
|
||||||
|
|
||||||
|
# Purge stream jika perlu
|
||||||
|
nats stream purge <stream-name>
|
||||||
|
|
||||||
|
# Atau hapus volume (data hilang!)
|
||||||
|
docker compose -f infra/compose/nats.yml down
|
||||||
|
docker volume rm nats_data
|
||||||
|
docker compose -f infra/compose/nats.yml up -d
|
||||||
|
```
|
||||||
|
|
||||||
|
### Slow consumer
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Cek consumer lag
|
||||||
|
nats stream info <stream-name>
|
||||||
|
# Lihat fields: "Pending" dan "Acknowledgment"
|
||||||
|
|
||||||
|
# Lihat stats server
|
||||||
|
curl http://localhost:8222/varz | jq '.slow_consumers'
|
||||||
|
```
|
||||||
|
|
||||||
|
## Migration: Redis Pub/Sub ke NATS
|
||||||
|
|
||||||
|
Jika ingin migrasi dari Dapr pub/sub Redis ke NATS:
|
||||||
|
|
||||||
|
1. Buat stream NATS untuk topik `hub.>`
|
||||||
|
2. Update `infra/dapr/components/pubsub.yaml` dari `pubsub.redis` ke `pubsub.nats`
|
||||||
|
3. Deploy ulang semua service (Dapr sidecar akan reconnect)
|
||||||
|
4. Verifikasi event flow
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
# infra/dapr/components/pubsub.yaml (setelah migrasi)
|
||||||
|
apiVersion: dapr.io/v1alpha1
|
||||||
|
kind: Component
|
||||||
|
metadata:
|
||||||
|
name: pubsub
|
||||||
|
spec:
|
||||||
|
type: pubsub.nats
|
||||||
|
version: v1
|
||||||
|
metadata:
|
||||||
|
- name: natsURL
|
||||||
|
value: nats://nats:4222
|
||||||
|
```
|
||||||
@@ -0,0 +1,60 @@
|
|||||||
|
# Tools — Document Scanner & Media Processing Hub
|
||||||
|
|
||||||
|
Self-hosted, no-install document scanner dan media processing tools yang jalan di browser. Alternatif dari CamScanner, ilovepdf, compressjpeg — tanpa upload ke pihak ketiga.
|
||||||
|
|
||||||
|
## Visi
|
||||||
|
|
||||||
|
Satu platform dengan tools manipulasi file yang **beneran dipake orang setiap hari**. Semua proses di backend Rust — cepat, hemat memory, ga perlu install software.
|
||||||
|
|
||||||
|
## Fitur Utama
|
||||||
|
|
||||||
|
### Phase 1 — Document Scanner (Prioritas)
|
||||||
|
- Foto dokumen pake HP → auto-detect tepi → lurusin (perspective correction)
|
||||||
|
- Enhance: iluminasi merata, contrast, sharpen, B&W
|
||||||
|
- OCR → searchable PDF (teks bisa di-copy, dicari)
|
||||||
|
- Batch: multi-page → satu PDF
|
||||||
|
- Fallback crop manual (kalau auto-detect gagal)
|
||||||
|
|
||||||
|
### Phase 2 — Image Tools
|
||||||
|
- Compress JPEG/PNG/WebP (lossy + lossless, atur kualitas %)
|
||||||
|
- Resize batch (atur dimensi, semua foto disamain)
|
||||||
|
- Convert format (HEIC→JPEG, PNG→WebP, SVG→PNG)
|
||||||
|
- Remove background (ONNX model, Rust runtime)
|
||||||
|
|
||||||
|
### Phase 3 — PDF Tools
|
||||||
|
- Merge PDF (gabung file)
|
||||||
|
- Split PDF (ekstrak halaman tertentu)
|
||||||
|
- Images→PDF (kumpulan foto jadi 1 file)
|
||||||
|
- PDF→Images (tiap halaman jadi gambar)
|
||||||
|
- PDF compress (turunkin kualitas embedded images)
|
||||||
|
|
||||||
|
### Phase 4 — Video/Audio Tools
|
||||||
|
- Compress video (bitrate + resolusi)
|
||||||
|
- Extract audio (MP4→MP3)
|
||||||
|
- Trim/crop
|
||||||
|
- GIF maker
|
||||||
|
- Audio convert + trim
|
||||||
|
|
||||||
|
## Target User
|
||||||
|
|
||||||
|
Orang yang:
|
||||||
|
- Punya HP/PC, paham teknologi dasar (buka browser, upload file)
|
||||||
|
- Butuh scan dokumen tanpa install aplikasi
|
||||||
|
- Butuh kompres file buat kirim WA/email
|
||||||
|
- Butuh manipulasi PDF sesekali
|
||||||
|
- Peduli privasi — ga mau upload file ke server pihak ketiga
|
||||||
|
|
||||||
|
## Prinsip Desain
|
||||||
|
|
||||||
|
1. **Satu task selesai dalam <5 detik** — ga ada loading lama
|
||||||
|
2. **Drag & drop + preview** — liat hasil sebelum download
|
||||||
|
3. **Progress realtime** via WebSocket — tau lagi di tahap mana
|
||||||
|
4. **Batch processing** — banyak file, satu klik
|
||||||
|
5. **Privasi first** — file otomatis dihapus setelah 1 jam
|
||||||
|
6. **WASM fallback** — tools ringan jalan di client (tanpa upload)
|
||||||
|
|
||||||
|
## Domain & Branding
|
||||||
|
|
||||||
|
- **Domain**: `tools.asepharyana.my.id` | `tools.asepharyana.web.id`
|
||||||
|
- **Design**: Twilight Terminal theme (sama kaya portfolio), konsisten visual
|
||||||
|
- **Dashboard**: Link dari hub dashboard → tools stats (total files processed, storage used)
|
||||||
@@ -0,0 +1,451 @@
|
|||||||
|
# Architecture
|
||||||
|
|
||||||
|
## System Overview
|
||||||
|
|
||||||
|
```
|
||||||
|
┌────────────────────────────────────────────────────────────────┐
|
||||||
|
│ BROWSER │
|
||||||
|
│ ┌────────────┐ ┌────────────┐ ┌────────────────────────┐ │
|
||||||
|
│ │ Upload │ │ Camera │ │ Preview + Download │ │
|
||||||
|
│ │ (drag/drop)│ │ (PWA) │ │ (streaming) │ │
|
||||||
|
│ └─────┬──────┘ └─────┬──────┘ └───────────┬────────────┘ │
|
||||||
|
│ │ │ │ │
|
||||||
|
│ ▼ ▼ ▼ │
|
||||||
|
│ ┌──────────────────────────────────────────────────────┐ │
|
||||||
|
│ │ WebSocket (progress: processing/step/percentage) │ │
|
||||||
|
│ └──────────────────────────────────────────────────────┘ │
|
||||||
|
└──────────────────────────┬───────────────────────────────────┘
|
||||||
|
│ HTTPS / WSS
|
||||||
|
▼
|
||||||
|
┌────────────────────────────────────────────────────────────────┐
|
||||||
|
│ TRAEFIK (tools.asepharyana.my.id) │
|
||||||
|
│ Middleware chain: secure-headers → compress → rate-limit │
|
||||||
|
└──────────────────────────┬────────────────────────────────────┘
|
||||||
|
│
|
||||||
|
▼
|
||||||
|
┌────────────────────────────────────────────────────────────────┐
|
||||||
|
│ tools-app (Next.js 16 / TypeScript) │
|
||||||
|
│ │
|
||||||
|
│ ┌──────────────────┐ ┌─────────────────┐ │
|
||||||
|
│ │ Pages/Routes │ │ API Routes │ │
|
||||||
|
│ │ / → home │ │ POST /api/upload ──▶ file │
|
||||||
|
│ │ /scan → scanner │ │ GET /api/job/:id ─▶ status │
|
||||||
|
│ │ /image → image │ │ WS /api/job/:id/ws ─▶ progress │
|
||||||
|
│ │ /pdf → pdf tools │ │ GET /api/download/:id ─▶ file │
|
||||||
|
│ └──────────────────┘ └─────────────────┘ │
|
||||||
|
│ │
|
||||||
|
│ Upload validation: MIME type, size limit (50MB), virus scan │
|
||||||
|
│ Temp storage bridge ke worker via HTTP/NATS │
|
||||||
|
└──────────────────┬────────────────────────────────────────────┘
|
||||||
|
│ HTTP (internal)
|
||||||
|
▼
|
||||||
|
┌────────────────────────────────────────────────────────────────┐
|
||||||
|
│ API GATEWAY (Rust / Axum) │
|
||||||
|
│ │
|
||||||
|
│ ┌──────────────┐ ┌──────────────┐ ┌────────────────────┐ │
|
||||||
|
│ │ Upload │ │ Job Manager │ │ Download │ │
|
||||||
|
│ │ (streaming │ │ (CRUD job │ │ (stream file, │ │
|
||||||
|
│ │ chunked) │ │ status) │ │ auto-delete) │ │
|
||||||
|
│ └──────┬───────┘ └──────┬───────┘ └────────────────────┘ │
|
||||||
|
│ │ │ │
|
||||||
|
│ ▼ ▼ │
|
||||||
|
│ ┌────────────────────────────────────────────────────┐ │
|
||||||
|
│ │ NATS JetStream │ │
|
||||||
|
│ │ ┌──────────┐ ┌──────────┐ ┌──────────────┐ │ │
|
||||||
|
│ │ │ scan. │ │ image. │ │ pdf. │ │ │
|
||||||
|
│ │ │ jobs │ │ jobs │ │ jobs │ │ │
|
||||||
|
│ │ └────┬─────┘ └────┬─────┘ └──────┬───────┘ │ │
|
||||||
|
│ │ │ │ │ │ │
|
||||||
|
│ │ ▼ ▼ ▼ │ │
|
||||||
|
│ │ ┌──────────┐ ┌──────────┐ ┌──────────────┐ │ │
|
||||||
|
│ │ │ scan. │ │ image. │ │ pdf. │ │ │
|
||||||
|
│ │ │ progress │ │ progress │ │ progress │ │ │
|
||||||
|
│ │ └──────────┘ └──────────┘ └──────────────┘ │ │
|
||||||
|
│ └────────────────────────────────────────────────────┘ │
|
||||||
|
│ │
|
||||||
|
│ ┌────────────────────────────────────────────────────┐ │
|
||||||
|
│ │ Cache (Redis) │ │
|
||||||
|
│ │ - Job metadata (status, progress, timestamps) │ │
|
||||||
|
│ │ - Rate limiting (sliding window per IP/tool) │ │
|
||||||
|
│ │ - Result metadata (file path, size, type) │ │
|
||||||
|
│ └────────────────────────────────────────────────────┘ │
|
||||||
|
└──────────────────┬────────────────────────────────────────────┘
|
||||||
|
│ consume NATS queue
|
||||||
|
▼
|
||||||
|
┌────────────────────────────────────────────────────────────────┐
|
||||||
|
│ WORKER POOL (Rust / Tokio + Rayon) │
|
||||||
|
│ │
|
||||||
|
│ ┌─────────────────┐ ┌─────────────────┐ ┌──────────────┐ │
|
||||||
|
│ │ Scan Worker │ │ Image Worker │ │ PDF Worker │ │
|
||||||
|
│ │ ×4 instances │ │ ×2 instances │ │ ×2 instances│ │
|
||||||
|
│ │ │ │ │ │ │ │
|
||||||
|
│ │ 1. Load image │ │ 1. Load image │ │ 1. Load PDF │ │
|
||||||
|
│ │ 2. Edge detect │ │ 2. Compress │ │ 2. Merge/ │ │
|
||||||
|
│ │ 3. Warp │ │ /resize/ │ │ split │ │
|
||||||
|
│ │ 4. Enhance │ │ convert │ │ 3. Save │ │
|
||||||
|
│ │ 5. OCR │ │ 3. Save │ │ 4. Update │ │
|
||||||
|
│ │ 6. Gen PDF │ │ 4. Update │ │ job │ │
|
||||||
|
│ │ 7. Update job │ │ job status │ │ status │ │
|
||||||
|
│ │ └───────────────┘ └─────────────────┘ └──────────────┘ │
|
||||||
|
│ │ │
|
||||||
|
│ ▼ │
|
||||||
|
│ ┌────────────────────────────────────────────────────┐ │
|
||||||
|
│ │ Temp Storage (filesystem volume / S3-compatible) │ │
|
||||||
|
│ │ Auto-cleanup: job TTL 1 jam, NATS cron tiap 10m │ │
|
||||||
|
│ └────────────────────────────────────────────────────┘ │
|
||||||
|
└────────────────────────────────────────────────────────────────┘
|
||||||
|
```
|
||||||
|
|
||||||
|
## Component Diagram
|
||||||
|
|
||||||
|
```
|
||||||
|
┌────────────────────────────────────────────┐
|
||||||
|
│ apps/tools │
|
||||||
|
│ │
|
||||||
|
│ ├── frontend/ │
|
||||||
|
│ │ ├── pages/ ← Next.js pages │
|
||||||
|
│ │ ├── components/ ← React components │
|
||||||
|
│ │ ├── lib/ ← utilities │
|
||||||
|
│ │ └── public/ ← static assets │
|
||||||
|
│ │ │
|
||||||
|
│ ├── backend/ ← Rust workspace │
|
||||||
|
│ │ ├── gateway/ ← Axum API server │
|
||||||
|
│ │ ├── workers/ ← Processing workers │
|
||||||
|
│ │ │ ├── scanner/ ← Document scanner │
|
||||||
|
│ │ │ ├── image/ ← Image tools │
|
||||||
|
│ │ │ └── pdf/ ← PDF tools │
|
||||||
|
│ │ └── common/ ← Shared libs │
|
||||||
|
│ │ │
|
||||||
|
│ └── Dockerfile │
|
||||||
|
└────────────────────────────────────────────┘
|
||||||
|
```
|
||||||
|
|
||||||
|
## Data Flow (Document Scanner — Flow Lengkap)
|
||||||
|
|
||||||
|
```
|
||||||
|
1. User buka tools.asepharyana.my.id/scan
|
||||||
|
2. Upload foto via drag-drop atau kamera HP (PWA)
|
||||||
|
3. Next.js route handler menerima file
|
||||||
|
├─ Validasi: MIME type (image/*), max 50MB, virus header scan
|
||||||
|
└─ Upload chunked ke Gateway internal (HTTP POST)
|
||||||
|
|
||||||
|
4. Gateway menerima stream:
|
||||||
|
├─ Simpan ke temp storage
|
||||||
|
├─ Buat job record di Redis: {id, tool: "scan", status: "queued", progress: 0}
|
||||||
|
└─ Publish ke NATS: tools.scan.jobs {job_id, file_path, options}
|
||||||
|
|
||||||
|
5. Scan Worker consume dari NATS:
|
||||||
|
├─ Update Redis: status = "processing", progress = 10
|
||||||
|
├─ Load image (image-rs)
|
||||||
|
├─ Pipeline (detail di pipeline.md):
|
||||||
|
│ 1. Edge detection ──▶ progress 25
|
||||||
|
│ 2. Perspective warp ──▶ progress 40
|
||||||
|
│ 3. Shadow removal ──▶ progress 55
|
||||||
|
│ 4. Binarization ──▶ progress 70
|
||||||
|
│ 5. Contrast/sharpen ──▶ progress 80
|
||||||
|
│ 6. OCR ──▶ progress 90
|
||||||
|
│ 7. Generate PDF ──▶ progress 95
|
||||||
|
├─ Simpan file hasil ke temp storage
|
||||||
|
├─ Update Redis: status = "completed", progress = 100, result_path, ocr_text
|
||||||
|
└─ Publish ke NATS: tools.scan.progress {job_id, status, progress}
|
||||||
|
|
||||||
|
6. WebSocket handler di Gateway:
|
||||||
|
├─ Subscribe NATS topics tools.scan.progress
|
||||||
|
├─ Forward ke browser user (per-job-id filter)
|
||||||
|
└─ Browser update progress bar + preview
|
||||||
|
|
||||||
|
7. User download PDF:
|
||||||
|
├─ GET /api/download/:job_id
|
||||||
|
├─ Gateway stream file dari temp storage
|
||||||
|
└─ Browser save file
|
||||||
|
```
|
||||||
|
|
||||||
|
## Tech Stack
|
||||||
|
|
||||||
|
### Frontend (Next.js + TypeScript)
|
||||||
|
|
||||||
|
| Library | Fungsi |
|
||||||
|
|---------|--------|
|
||||||
|
| Next.js 16 | App router, API routes |
|
||||||
|
| shadcn/ui + Tailwind v4 | UI components |
|
||||||
|
| Framer Motion | Animasi progress, transisi |
|
||||||
|
| Canvas API | Preview crop manual, image manipulation client-side |
|
||||||
|
| WebSocket API | Real-time progress |
|
||||||
|
|
||||||
|
### Backend (Rust)
|
||||||
|
|
||||||
|
| Crate | Fungsi |
|
||||||
|
|-------|--------|
|
||||||
|
| `axum` | HTTP server (Gateway) |
|
||||||
|
| `tokio` | Async runtime |
|
||||||
|
| `image` | Image I/O, resize, convert, compress |
|
||||||
|
| `imageproc` | Edge detection, contour, thresholding |
|
||||||
|
| `lopdf` | PDF generation, merge, split, compress |
|
||||||
|
| `leptess` | Tesseract OCR binding |
|
||||||
|
| `ort` | ONNX Runtime (background removal) |
|
||||||
|
| `async-nats` | NATS JetStream client |
|
||||||
|
| `deadpool-redis` | Redis connection pool |
|
||||||
|
| `redis` | Redis async client |
|
||||||
|
| `rayon` | Parallel processing (batch, pixel ops) |
|
||||||
|
| `serde` | Serialization |
|
||||||
|
| `tracing` + `opentelemetry` | Observability |
|
||||||
|
| `uuid` | Job ID generation |
|
||||||
|
|
||||||
|
### Infrastructure
|
||||||
|
|
||||||
|
| Komponen | Fungsi |
|
||||||
|
|----------|--------|
|
||||||
|
| NATS JetStream | Job queue, progress pub/sub, scheduler |
|
||||||
|
| Redis | Job metadata, rate limiting, cache |
|
||||||
|
| PostgreSQL | Opsional — audit log, usage statistics |
|
||||||
|
| Tesseract | OCR engine (data files di Docker image) |
|
||||||
|
| Prometheus | Metrics (jobs/min, queue depth, latency per stage) |
|
||||||
|
|
||||||
|
## Job Queue (NATS Streams & Consumers)
|
||||||
|
|
||||||
|
### Streams
|
||||||
|
|
||||||
|
```
|
||||||
|
tools-scan-jobs → 1 stream, mirror to all scan workers
|
||||||
|
tools-image-jobs → 1 stream, mirror to all image workers
|
||||||
|
tools-pdf-jobs → 1 stream, mirror to all pdf workers
|
||||||
|
tools-progress → 1 stream, all progress events (key-value by job_id)
|
||||||
|
tools-scheduler → 1 stream, cron events
|
||||||
|
```
|
||||||
|
|
||||||
|
### Subjects
|
||||||
|
|
||||||
|
```
|
||||||
|
tools.scan.jobs.{job_id} → job submission
|
||||||
|
tools.scan.progress.{job_id} → progress update (fan-out ke Gateway)
|
||||||
|
tools.image.jobs.{job_id} → job submission
|
||||||
|
tools.image.progress.{job_id} → progress update
|
||||||
|
tools.pdf.jobs.{job_id} → job submission
|
||||||
|
tools.pdf.progress.{job_id} → progress update
|
||||||
|
tools.scheduler.cleanup → cleanup expired files (every 10 min)
|
||||||
|
```
|
||||||
|
|
||||||
|
## Redis Schema
|
||||||
|
|
||||||
|
```
|
||||||
|
job:{id} → Hash {status, tool, progress, file_path, result_path, ocr_text, created_at, ttl}
|
||||||
|
rate_limit:{ip}:{tool} → Sorted Set (sliding window)
|
||||||
|
file_meta:{hash} → String {original_name, size, mime}
|
||||||
|
```
|
||||||
|
|
||||||
|
## Metrics (Prometheus)
|
||||||
|
|
||||||
|
| Metric | Type | Labels | Description |
|
||||||
|
|--------|------|--------|-------------|
|
||||||
|
| `tools_jobs_total` | Counter | `tool`, `status` | Total jobs processed |
|
||||||
|
| `tools_jobs_in_flight` | Gauge | `tool` | Currently processing jobs |
|
||||||
|
| `tools_queue_depth` | Gauge | `tool` | NATS queue depth |
|
||||||
|
| `tools_processing_duration` | Histogram | `tool`, `stage` | Duration per stage |
|
||||||
|
| `tools_file_size_bytes` | Histogram | `tool` | Upload file size distribution |
|
||||||
|
| `tools_rate_limit_hits` | Counter | `tool` | Rate limit violations |
|
||||||
|
|
||||||
|
## Directory Structure
|
||||||
|
|
||||||
|
```
|
||||||
|
apps/tools/
|
||||||
|
├── frontend/
|
||||||
|
│ ├── src/
|
||||||
|
│ │ ├── app/
|
||||||
|
│ │ │ ├── page.tsx # Landing page
|
||||||
|
│ │ │ ├── scan/
|
||||||
|
│ │ │ │ ├── page.tsx # Scanner page
|
||||||
|
│ │ │ │ └── result/[id]/
|
||||||
|
│ │ │ │ └── page.tsx # Result page
|
||||||
|
│ │ │ ├── image/
|
||||||
|
│ │ │ │ ├── compress/page.tsx
|
||||||
|
│ │ │ │ ├── resize/page.tsx
|
||||||
|
│ │ │ │ ├── convert/page.tsx
|
||||||
|
│ │ │ │ └── remove-bg/page.tsx
|
||||||
|
│ │ │ ├── pdf/
|
||||||
|
│ │ │ │ ├── merge/page.tsx
|
||||||
|
│ │ │ │ ├── split/page.tsx
|
||||||
|
│ │ │ │ ├── images-to-pdf/page.tsx
|
||||||
|
│ │ │ │ └── compress/page.tsx
|
||||||
|
│ │ │ ├── api/
|
||||||
|
│ │ │ │ ├── upload/route.ts
|
||||||
|
│ │ │ │ ├── job/[id]/route.ts
|
||||||
|
│ │ │ │ │ └── ws/route.ts
|
||||||
|
│ │ │ │ └── download/[id]/route.ts
|
||||||
|
│ │ │ ├── layout.tsx
|
||||||
|
│ │ │ └── globals.css
|
||||||
|
│ │ ├── components/
|
||||||
|
│ │ │ ├── upload-zone.tsx # Drag & drop area
|
||||||
|
│ │ │ ├── progress-bar.tsx # WebSocket-connected progress
|
||||||
|
│ │ │ ├── preview.tsx # Before/after preview
|
||||||
|
│ │ │ ├── crop-editor.tsx # Manual corner adjustment
|
||||||
|
│ │ │ ├── tool-layout.tsx # Consistent tool page layout
|
||||||
|
│ │ │ └── camera-capture.tsx # PWA camera interface
|
||||||
|
│ │ ├── hooks/
|
||||||
|
│ │ │ ├── use-job-status.ts # WebSocket connection
|
||||||
|
│ │ │ ├── use-upload.ts # Upload with progress
|
||||||
|
│ │ │ └── use-camera.ts # Camera access
|
||||||
|
│ │ └── lib/
|
||||||
|
│ │ ├── utils.ts
|
||||||
|
│ │ └── types.ts
|
||||||
|
│ ├── next.config.ts
|
||||||
|
│ ├── package.json
|
||||||
|
│ └── tsconfig.json
|
||||||
|
│
|
||||||
|
├── backend/
|
||||||
|
│ ├── Cargo.toml
|
||||||
|
│ ├── gateway/
|
||||||
|
│ │ ├── Cargo.toml
|
||||||
|
│ │ └── src/
|
||||||
|
│ │ ├── main.rs
|
||||||
|
│ │ ├── routes/
|
||||||
|
│ │ │ ├── mod.rs
|
||||||
|
│ │ │ ├── upload.rs
|
||||||
|
│ │ │ ├── job.rs
|
||||||
|
│ │ │ ├── download.rs
|
||||||
|
│ │ │ └── ws.rs
|
||||||
|
│ │ ├── nats/
|
||||||
|
│ │ │ ├── mod.rs
|
||||||
|
│ │ │ └── publisher.rs
|
||||||
|
│ │ ├── redis/
|
||||||
|
│ │ │ ├── mod.rs
|
||||||
|
│ │ │ ├── job.rs
|
||||||
|
│ │ │ └── ratelimit.rs
|
||||||
|
│ │ ├── metrics.rs
|
||||||
|
│ │ └── config.rs
|
||||||
|
│ │
|
||||||
|
│ ├── workers/
|
||||||
|
│ │ ├── Cargo.toml
|
||||||
|
│ │ └── src/
|
||||||
|
│ │ ├── main.rs
|
||||||
|
│ │ ├── scanner/
|
||||||
|
│ │ │ ├── mod.rs
|
||||||
|
│ │ │ ├── pipeline.rs
|
||||||
|
│ │ │ ├── edge.rs # Edge detection
|
||||||
|
│ │ │ ├── warp.rs # Perspective correction
|
||||||
|
│ │ │ ├── enhance.rs # Shadow removal, B&W, contrast
|
||||||
|
│ │ │ ├── ocr.rs # Tesseract wrapper
|
||||||
|
│ │ │ └── pdf.rs # Generate searchable PDF
|
||||||
|
│ │ ├── image/
|
||||||
|
│ │ │ ├── mod.rs
|
||||||
|
│ │ │ ├── compress.rs
|
||||||
|
│ │ │ ├── resize.rs
|
||||||
|
│ │ │ ├── convert.rs
|
||||||
|
│ │ │ └── remove_bg.rs
|
||||||
|
│ │ ├── pdf/
|
||||||
|
│ │ │ ├── mod.rs
|
||||||
|
│ │ │ ├── merge.rs
|
||||||
|
│ │ │ ├── split.rs
|
||||||
|
│ │ │ ├── extract.rs
|
||||||
|
│ │ │ └── compress.rs
|
||||||
|
│ │ ├── nats/
|
||||||
|
│ │ │ ├── mod.rs
|
||||||
|
│ │ │ └── consumer.rs
|
||||||
|
│ │ └── config.rs
|
||||||
|
│ │
|
||||||
|
│ └── common/
|
||||||
|
│ ├── Cargo.toml
|
||||||
|
│ └── src/
|
||||||
|
│ ├── lib.rs
|
||||||
|
│ ├── types.rs # Shared types (JobStatus, Job, etc.)
|
||||||
|
│ ├── error.rs # Error types
|
||||||
|
│ └── nats.rs # NATS subject constants
|
||||||
|
│
|
||||||
|
├── Dockerfile
|
||||||
|
├── compose.yml # Local dev compose
|
||||||
|
└── README.md
|
||||||
|
```
|
||||||
|
|
||||||
|
## API Design
|
||||||
|
|
||||||
|
### Endpoints
|
||||||
|
|
||||||
|
| Method | Path | Description |
|
||||||
|
|--------|------|-------------|
|
||||||
|
| `POST` | `/api/upload` | Upload file, create job |
|
||||||
|
| `GET` | `/api/job/:id` | Get job status + result metadata |
|
||||||
|
| `WS` | `/api/job/:id/ws` | WebSocket — realtime progress |
|
||||||
|
| `GET` | `/api/download/:id` | Download result file |
|
||||||
|
| `DELETE` | `/api/job/:id` | Cancel job, delete files |
|
||||||
|
| `GET` | `/health` | Health check |
|
||||||
|
|
||||||
|
### Upload Request
|
||||||
|
|
||||||
|
```
|
||||||
|
POST /api/upload
|
||||||
|
Content-Type: multipart/form-data
|
||||||
|
|
||||||
|
{
|
||||||
|
file: <binary>,
|
||||||
|
tool: "scan" | "image-compress" | "image-resize" | "image-convert" | "remove-bg" |
|
||||||
|
"pdf-merge" | "pdf-split" | "images-to-pdf" | "pdf-compress",
|
||||||
|
options?: { // tool-specific options
|
||||||
|
quality?: 80, // compress quality
|
||||||
|
width?: 1920, // resize width
|
||||||
|
format?: "webp", // convert format
|
||||||
|
pages?: "1,3-5", // PDF split pages
|
||||||
|
dpi?: 300, // scan DPI
|
||||||
|
enhance?: true, // scan auto-enhance
|
||||||
|
ocr?: true // scan OCR
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
### Response (202 Accepted)
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"job_id": "uuid",
|
||||||
|
"status": "queued",
|
||||||
|
"tool": "scan",
|
||||||
|
"ws_url": "/api/job/uuid/ws",
|
||||||
|
"created_at": "2026-07-24T10:00:00Z",
|
||||||
|
"estimated_seconds": 5
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
### WebSocket Messages
|
||||||
|
|
||||||
|
```json
|
||||||
|
// Server → Client
|
||||||
|
{
|
||||||
|
"type": "progress",
|
||||||
|
"job_id": "uuid",
|
||||||
|
"status": "processing",
|
||||||
|
"progress": 45,
|
||||||
|
"stage": "warp",
|
||||||
|
"message": "Meluruskan perspektif dokumen..."
|
||||||
|
}
|
||||||
|
|
||||||
|
{
|
||||||
|
"type": "complete",
|
||||||
|
"job_id": "uuid",
|
||||||
|
"status": "completed",
|
||||||
|
"progress": 100,
|
||||||
|
"result": {
|
||||||
|
"download_url": "/api/download/uuid",
|
||||||
|
"file_name": "scan_20260724.pdf",
|
||||||
|
"file_size": 1245678,
|
||||||
|
"pages": 1,
|
||||||
|
"ocr_text": "Nama: Asep...",
|
||||||
|
"preview_url": "/api/job/uuid/preview"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
{
|
||||||
|
"type": "error",
|
||||||
|
"job_id": "uuid",
|
||||||
|
"status": "failed",
|
||||||
|
"error": "Edge detection failed: cannot find document boundary"
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
## Integration with Existing Portfolio
|
||||||
|
|
||||||
|
| Area | Detail |
|
||||||
|
|------|--------|
|
||||||
|
| **Domain** | `tools.asepharyana.my.id` — tambah entry di `infra/traefik/dynamic/apps.yaml` |
|
||||||
|
| **Dashboard** | Link ke tools stats di dashboard hub yang sudah ada |
|
||||||
|
| **Docker Compose** | `infra/compose/tools.yml` — pola sama kaya `hub.yml` |
|
||||||
|
| **CI/CD** | Tambah service `tools` di `docker-build-push.yml` |
|
||||||
|
| **Style** | Ulang Twilight Terminal theme dari hub, konsisten visual branding |
|
||||||
|
| **Monitoring** | Reuse existing Prometheus + Grafana, tambah metrics tools |
|
||||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,427 @@
|
|||||||
|
# Infrastructure & Deployment
|
||||||
|
|
||||||
|
## Docker Image Architecture
|
||||||
|
|
||||||
|
Project ini punya **satu Docker image** dengan multi-stage build. Backend Rust + Tesseract + ONNX model plus frontend Next.js.
|
||||||
|
|
||||||
|
### Dockerfile Structure
|
||||||
|
|
||||||
|
```dockerfile
|
||||||
|
# ============================================================
|
||||||
|
# Stage 1: Build Rust Backend
|
||||||
|
# ============================================================
|
||||||
|
FROM rust:1.85-slim-bookworm AS chef
|
||||||
|
RUN cargo install cargo-chef
|
||||||
|
WORKDIR /app
|
||||||
|
|
||||||
|
FROM chef AS planner
|
||||||
|
COPY backend/ .
|
||||||
|
RUN cargo chef prepare --recipe-path recipe.json
|
||||||
|
|
||||||
|
FROM chef AS builder
|
||||||
|
COPY --from=planner /app/recipe.json recipe.json
|
||||||
|
RUN cargo chef cook --release --recipe-path recipe.json
|
||||||
|
|
||||||
|
COPY backend/ .
|
||||||
|
RUN cargo build --release --bin gateway --bin workers
|
||||||
|
|
||||||
|
# ============================================================
|
||||||
|
# Stage 2: Build Next.js Frontend
|
||||||
|
# ============================================================
|
||||||
|
FROM oven/bun:1.3 AS frontend-builder
|
||||||
|
WORKDIR /app
|
||||||
|
COPY frontend/package.json frontend/bun.lock ./
|
||||||
|
RUN bun install --frozen-lockfile
|
||||||
|
COPY frontend/ .
|
||||||
|
RUN bun run build
|
||||||
|
|
||||||
|
# ============================================================
|
||||||
|
# Stage 3: Production Runtime
|
||||||
|
# ============================================================
|
||||||
|
FROM debian:bookworm-slim AS runtime
|
||||||
|
|
||||||
|
# Install runtime dependencies
|
||||||
|
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||||
|
tesseract-ocr \
|
||||||
|
tesseract-ocr-eng \
|
||||||
|
tesseract-ocr-ind \
|
||||||
|
ca-certificates \
|
||||||
|
fonts-dejavu-core \
|
||||||
|
&& rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
|
WORKDIR /app
|
||||||
|
|
||||||
|
# Copy Rust binaries
|
||||||
|
COPY --from=builder /app/target/release/gateway /app/gateway
|
||||||
|
COPY --from=builder /app/target/release/workers /app/workers
|
||||||
|
|
||||||
|
# Copy Next.js build
|
||||||
|
COPY --from=frontend-builder /app/.next /app/.next
|
||||||
|
COPY --from=frontend-builder /app/public /app/public
|
||||||
|
COPY --from=frontend-builder /app/package.json /app/package.json
|
||||||
|
COPY --from=frontend-builder /app/node_modules /app/node_modules
|
||||||
|
|
||||||
|
# Copy ONNX model (for background removal)
|
||||||
|
COPY models/ /app/models/
|
||||||
|
|
||||||
|
# Create temp storage directory
|
||||||
|
RUN mkdir -p /data/tools && chmod 1777 /data/tools
|
||||||
|
|
||||||
|
# Environment
|
||||||
|
ENV TESSDATA_PREFIX=/usr/share/tesseract-ocr/5/tessdata
|
||||||
|
ENV TOOLS_STORAGE_PATH=/data/tools
|
||||||
|
ENV TOOLS_GATEWAY_PORT=3001
|
||||||
|
ENV TOOLS_WORKER_CONCURRENCY=4
|
||||||
|
ENV RUST_LOG=info
|
||||||
|
|
||||||
|
# Expose port
|
||||||
|
EXPOSE 3001
|
||||||
|
|
||||||
|
# Run both gateway and workers via supervisor script
|
||||||
|
COPY scripts/entrypoint.sh /app/entrypoint.sh
|
||||||
|
RUN chmod +x /app/entrypoint.sh
|
||||||
|
|
||||||
|
CMD ["/app/entrypoint.sh"]
|
||||||
|
```
|
||||||
|
|
||||||
|
### Entrypoint Script
|
||||||
|
|
||||||
|
```bash
|
||||||
|
#!/bin/bash
|
||||||
|
# Start Gateway (Axum HTTP server)
|
||||||
|
/app/gateway &
|
||||||
|
GATEWAY_PID=$!
|
||||||
|
|
||||||
|
# Start Worker(s)
|
||||||
|
/app/workers &
|
||||||
|
WORKER_PID=$!
|
||||||
|
|
||||||
|
# Handle graceful shutdown
|
||||||
|
trap "kill $GATEWAY_PID $WORKER_PID; exit 0" SIGINT SIGTERM
|
||||||
|
|
||||||
|
# Wait for either process to exit
|
||||||
|
wait -n $GATEWAY_PID $WORKER_PID
|
||||||
|
|
||||||
|
# If one exits, kill the other
|
||||||
|
kill $GATEWAY_PID $WORKER_PID 2>/dev/null
|
||||||
|
exit 1
|
||||||
|
```
|
||||||
|
|
||||||
|
### Image Size Estimates
|
||||||
|
|
||||||
|
| Component | Size |
|
||||||
|
|-----------|------|
|
||||||
|
| Rust binary (gateway) | ~8 MB |
|
||||||
|
| Rust binary (workers) | ~15 MB |
|
||||||
|
| Next.js build | ~10 MB |
|
||||||
|
| Tesseract + data | ~25 MB |
|
||||||
|
| ONNX model | ~50 MB |
|
||||||
|
| Base (Debian slim) | ~80 MB |
|
||||||
|
| **Total** | **~188 MB** |
|
||||||
|
|
||||||
|
> ONNX model opsional — bisa di-download runtime daripada di-include di image.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Docker Compose
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
# infra/compose/tools.yml
|
||||||
|
services:
|
||||||
|
tools:
|
||||||
|
container_name: tools
|
||||||
|
image: ghcr.io/asepharyana/asepharyana-hub/tools:sha-xxxxxxx
|
||||||
|
restart: always
|
||||||
|
networks:
|
||||||
|
app-shared-net:
|
||||||
|
aliases:
|
||||||
|
- tools
|
||||||
|
env_file:
|
||||||
|
- ../../.env
|
||||||
|
environment:
|
||||||
|
- REDIS_URL=redis://redis:6379
|
||||||
|
- NATS_URL=nats://nats:4222
|
||||||
|
- TOOLS_STORAGE_PATH=/data/tools
|
||||||
|
- TOOLS_GATEWAY_PORT=3001
|
||||||
|
- TOOLS_WORKER_CONCURRENCY=4
|
||||||
|
- RUST_LOG=info
|
||||||
|
volumes:
|
||||||
|
- tools_data:/data/tools
|
||||||
|
ports:
|
||||||
|
- "3001:3001"
|
||||||
|
depends_on:
|
||||||
|
redis:
|
||||||
|
condition: service_started
|
||||||
|
nats:
|
||||||
|
condition: service_started
|
||||||
|
|
||||||
|
volumes:
|
||||||
|
tools_data:
|
||||||
|
|
||||||
|
networks:
|
||||||
|
app-shared-net:
|
||||||
|
name: app-shared-net
|
||||||
|
external: true
|
||||||
|
```
|
||||||
|
|
||||||
|
### Environment Variables (`../../.env`)
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Tools
|
||||||
|
TOOLS_GATEWAY_PORT=3001
|
||||||
|
TOOLS_WORKER_CONCURRENCY=4
|
||||||
|
TOOLS_STORAGE_PATH=/data/tools
|
||||||
|
TOOLS_JOB_TTL_SECONDS=3600
|
||||||
|
TOOLS_RATE_LIMIT_PER_MINUTE=30
|
||||||
|
TOOLS_MAX_FILE_SIZE_MB=50
|
||||||
|
TOOLS_OCR_LANG=eng+ind
|
||||||
|
|
||||||
|
# Infra (reuse existing)
|
||||||
|
REDIS_URL=redis://redis:6379
|
||||||
|
NATS_URL=nats://nats:4222
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## CI/CD Integration
|
||||||
|
|
||||||
|
### Docker Build Workflow
|
||||||
|
|
||||||
|
Tambah service `tools` di `.github/workflows/docker-build-push.yml`:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
# Di job "changes" step "Detect changed services"
|
||||||
|
changed() {
|
||||||
|
printf '%s\n' "$CHANGED_FILES" | grep -Eq "$1" && echo true || echo false
|
||||||
|
}
|
||||||
|
echo "tools=$(changed '^(apps/tools(/|$)|\.github/workflows/docker-build-push\.yml$|infra/docker/tools\.Dockerfile$)')" >> "$GITHUB_OUTPUT"
|
||||||
|
|
||||||
|
# Di job "build" step "Set matrix"
|
||||||
|
if [ "${{ steps.filter.outputs['tools'] == 'true' || steps.dispatch.outputs['tools'] == 'true' || github.event_name == 'workflow_dispatch' }}" == "true" ]; then
|
||||||
|
add_service "tools" "docker-tools" "apps/tools"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Di job "build" step "Docker metadata"
|
||||||
|
case "$SVC_NAME" in
|
||||||
|
"tools") echo "dockerfile=infra/docker/tools.Dockerfile" >> $GITHUB_OUTPUT ;;
|
||||||
|
esac
|
||||||
|
|
||||||
|
# Di job "update-manifest"
|
||||||
|
SERVICES["tools"]="tools.yml"
|
||||||
|
PATHS["tools"]="apps/tools"
|
||||||
|
```
|
||||||
|
|
||||||
|
### Deploy Workflow
|
||||||
|
|
||||||
|
Tambah di `.github/workflows/deploy-docker.yml`:
|
||||||
|
```yaml
|
||||||
|
# Tidak perlu perubahan — deploy-docker.yml auto-detect compose file changes.
|
||||||
|
# Kalau compose/tools.yml berubah, service tools akan di-restart.
|
||||||
|
```
|
||||||
|
|
||||||
|
### Service Registration (update infra/traefik/dynamic/apps.yaml)
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
tools:
|
||||||
|
rule: 'Host(`tools.asepharyana.my.id`) || Host(`tools.asepharyana.web.id`)'
|
||||||
|
entryPoints:
|
||||||
|
- websecure
|
||||||
|
tls: {}
|
||||||
|
middlewares:
|
||||||
|
- common-chain@file
|
||||||
|
service: tools-service
|
||||||
|
|
||||||
|
# ...di bagian services:
|
||||||
|
tools-service:
|
||||||
|
loadBalancer:
|
||||||
|
servers:
|
||||||
|
- url: 'http://tools:3001'
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Monitoring
|
||||||
|
|
||||||
|
### Prometheus Metrics
|
||||||
|
|
||||||
|
Tambahkan label Prometheus ke container tools:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
# Di compose tools.yml
|
||||||
|
labels:
|
||||||
|
- 'prometheus.io/scrape=true'
|
||||||
|
- 'prometheus.io/port=3001'
|
||||||
|
- 'prometheus.io/path=/metrics'
|
||||||
|
```
|
||||||
|
|
||||||
|
### Dashboard Integration
|
||||||
|
|
||||||
|
Tambah card di dashboard hub yang sudah ada:
|
||||||
|
|
||||||
|
```tsx
|
||||||
|
// Di dashboard hub — tambah section "Tools Usage"
|
||||||
|
// Data dari /api/dashboard → Prometheus query:
|
||||||
|
// rate(tools_jobs_total[24h]) — jobs per tool per hari
|
||||||
|
// sum(increase(tools_jobs_total[7d])) — total jobs minggu ini
|
||||||
|
// tools_jobs_in_flight — current processing
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Storage Architecture
|
||||||
|
|
||||||
|
### Temp Storage
|
||||||
|
|
||||||
|
```
|
||||||
|
/data/tools/
|
||||||
|
├── upload/ # Uploaded files
|
||||||
|
│ └── {job_id}.{ext}
|
||||||
|
├── processing/ # Intermediate files (stage-by-stage)
|
||||||
|
│ └── {job_id}/
|
||||||
|
│ ├── 00_original.png
|
||||||
|
│ ├── 01_grayscale.png
|
||||||
|
│ ├── 02_edges.png
|
||||||
|
│ ├── 03_warped.png
|
||||||
|
│ └── ...
|
||||||
|
└── output/ # Final output
|
||||||
|
└── {job_id}.pdf
|
||||||
|
```
|
||||||
|
|
||||||
|
### Cleanup Strategy
|
||||||
|
|
||||||
|
| Mekanisme | Timing |
|
||||||
|
|-----------|--------|
|
||||||
|
| NATS cron job | Setiap 10 menit |
|
||||||
|
| Scan files >1 jam | `find /data/tools -mmin +60 -delete` |
|
||||||
|
| Redis job keys >1 jam | `SCAN 0 MATCH job:*` → TTL check → DEL |
|
||||||
|
| Storage low warning | Alert via Notification Hub (future) |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Resource Estimation (VPS orangevps)
|
||||||
|
|
||||||
|
### Current Usage
|
||||||
|
|
||||||
|
| Service | CPU | RAM | Disk |
|
||||||
|
|---------|-----|-----|------|
|
||||||
|
| Traefik | 0.1 | 50 MB | 10 MB |
|
||||||
|
| NATS | 0.05 | 30 MB | 10 MB |
|
||||||
|
| Redis | 0.05 | 10 MB | 5 MB |
|
||||||
|
| Dapr Placement | 0.02 | 20 MB | 5 MB |
|
||||||
|
| Scraper API | 0.1 | 30 MB | 50 MB |
|
||||||
|
| Hub | 0.05 | 120 MB | 200 MB |
|
||||||
|
| Jaeger | 0.1 | 200 MB | 500 MB |
|
||||||
|
| Prometheus | 0.1 | 150 MB | 1 GB |
|
||||||
|
| Node Exporter | 0.02 | 10 MB | 5 MB |
|
||||||
|
| OTel Collector | 0.05 | 50 MB | 10 MB |
|
||||||
|
| **Total Current** | **~0.64** | **~670 MB** | **~1.8 GB** |
|
||||||
|
|
||||||
|
### Tools Addition
|
||||||
|
|
||||||
|
| Resources | Estimate | Notes |
|
||||||
|
|-----------|----------|-------|
|
||||||
|
| CPU | +1.0 core (burst) | Pipeline processing berat di CPU. Scoring, warp, OCR semua CPU-bound. |
|
||||||
|
| RAM | +300 MB | Rust binary + image processing buffers + Tesseract + ONNX |
|
||||||
|
| Disk | +5 GB | Temp files, bisa lebih untuk batch processing. Butuh auto-cleanup ketat. |
|
||||||
|
| **Total After** | **~1.64 cores** | **~970 MB RAM** | **~6.8 GB disk** |
|
||||||
|
|
||||||
|
> **Catatan**: Kalau VPS cuma punya 1-2 cores, processing akan antri. NATS queue handle ini. Untuk production, pastikan CPU ada >2 cores.
|
||||||
|
|
||||||
|
### Scalability
|
||||||
|
|
||||||
|
```
|
||||||
|
VPS 1 core:
|
||||||
|
- Scanner: ~5-8 detik per page
|
||||||
|
- Concurrent: 1 job at a time
|
||||||
|
- Antrian: NATS queue buffer unlimited
|
||||||
|
|
||||||
|
VPS 4+ core:
|
||||||
|
- Scanner: ~2-3 detik per page
|
||||||
|
- Concurrent: 4 jobs parallel (1 per worker)
|
||||||
|
- Rayon: parallel per-page dalam batch
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Security Considerations
|
||||||
|
|
||||||
|
| Area | Mitigation |
|
||||||
|
|------|-----------|
|
||||||
|
| **Upload validation** | MIME type check (whitelist), magic bytes verification, max size 50MB |
|
||||||
|
| **Path traversal** | Job ID = UUID v4, no user-controlled filenames in storage |
|
||||||
|
| **Command injection** | No shell commands — semua processing via Rust crates, FFmpeg via crate binding |
|
||||||
|
| **Temporary files** | Auto-cleanup, random filenames, restricted permissions (0600) |
|
||||||
|
| **Rate limiting** | Redis sliding window: 30 requests/min/IP per tool, 429 response |
|
||||||
|
| **CORS** | Origin terbatas ke domain portfolio |
|
||||||
|
| **Resource exhaustion** | Max image dimension 8000px, max file count per batch 50, worker concurrency limit |
|
||||||
|
| **OCR data** | Tesseract data dari package manager, no user-trained models |
|
||||||
|
| **ONNX model** | Model dari source terpercaya, verify checksum |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Rollback Strategy
|
||||||
|
|
||||||
|
1. **Image tag**: `tools:sha-<short>` immutable — tinggal update compose file ke tag sebelumnya
|
||||||
|
2. **Data**: Files auto-expire dalam 1 jam — no persistent data migration needed
|
||||||
|
3. **Traefik**: Cukup restart, TLS certs ga berubah
|
||||||
|
4. **Monitor**: Prometheus metrics akan langsung show error rate spike
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Development Setup (Local)
|
||||||
|
|
||||||
|
Untuk development tanpa Docker:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Terminal 1: Redis + NATS
|
||||||
|
docker compose -f infra/compose/shared.yml -f infra/compose/nats.yml up -d
|
||||||
|
|
||||||
|
# Terminal 2: Rust workers
|
||||||
|
cd apps/tools/backend
|
||||||
|
REDIS_URL=redis://localhost:6379 NATS_URL=nats://localhost:4222 \
|
||||||
|
cargo run --bin workers
|
||||||
|
|
||||||
|
# Terminal 3: Rust gateway
|
||||||
|
REDIS_URL=redis://localhost:6379 NATS_URL=nats://localhost:4222 \
|
||||||
|
TOOLS_STORAGE_PATH=/tmp/tools \
|
||||||
|
cargo run --bin gateway
|
||||||
|
|
||||||
|
# Terminal 4: Next.js
|
||||||
|
cd apps/tools/frontend
|
||||||
|
bun dev --port 3002
|
||||||
|
```
|
||||||
|
|
||||||
|
### Test Pipeline Locally (tanpa NATS/Redis)
|
||||||
|
|
||||||
|
Untuk development pipeline image processing doang:
|
||||||
|
|
||||||
|
```rust
|
||||||
|
// Di workers/src/scanner/pipeline.rs — test function
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_full_pipeline() {
|
||||||
|
let pipeline = ScanPipeline::default();
|
||||||
|
let result = pipeline.process_sync(
|
||||||
|
"test_images/scan_miring.jpg",
|
||||||
|
ScanOptions { ocr: false, enhance: true }
|
||||||
|
);
|
||||||
|
assert!(result.is_ok());
|
||||||
|
assert!(result.unwrap().output_path.exists());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_edge_detection_variations() {
|
||||||
|
// Test dengan berbagai kondisi: kertas putih, background ramai, sudut ekstrim
|
||||||
|
for case in &["normal.jpg", "dark.jpg", "angle45.jpg", "shadow.jpg"] {
|
||||||
|
let img = image::open(format!("test_images/{}", case)).unwrap();
|
||||||
|
let corners = detect_corners_with_fallback(&img.grayscale().into_luma8());
|
||||||
|
assert!(corners.is_ok(), "Failed on: {}", case);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
Test images kumpulin dari foto dokumen real di berbagai kondisi — ini penting buat tuning parameter.
|
||||||
@@ -0,0 +1,798 @@
|
|||||||
|
# Document Scanner — Processing Pipeline
|
||||||
|
|
||||||
|
Ini adalah inti dari project. Pipeline mengubah foto dokumen HP jadi dokumen scan yang proper. Setiap tahap dibahas detail teknisnya.
|
||||||
|
|
||||||
|
## Pipeline Overview
|
||||||
|
|
||||||
|
```
|
||||||
|
Input: Foto HP (JPEG/PNG/HEIC, 2-12MP)
|
||||||
|
│
|
||||||
|
▼
|
||||||
|
┌──────────────────────────────────┐
|
||||||
|
│ 1. Preprocess ──▶ resize + │
|
||||||
|
│ konversi grayscale │
|
||||||
|
└────────────────┬─────────────────┘
|
||||||
|
│
|
||||||
|
▼
|
||||||
|
┌──────────────────────────────────┐
|
||||||
|
│ 2. Edge Detection ──▶ cari │
|
||||||
|
│ kontur dokumen │
|
||||||
|
└────────────────┬─────────────────┘
|
||||||
|
│
|
||||||
|
▼
|
||||||
|
┌──────────────────────────────────┐
|
||||||
|
│ 3. Corner Detection ──▶ 4 titik │
|
||||||
|
│ sudut dokumen │
|
||||||
|
└────────────────┬─────────────────┘
|
||||||
|
│
|
||||||
|
▼
|
||||||
|
┌──────────────────────────────────┐
|
||||||
|
│ 4. Perspective Warp ──▶ lurusin│
|
||||||
|
│ (homography) │
|
||||||
|
└────────────────┬─────────────────┘
|
||||||
|
│
|
||||||
|
▼
|
||||||
|
┌──────────────────────────────────┐
|
||||||
|
│ 5. Shadow Removal ──▶ iluminasi │
|
||||||
|
│ merata │
|
||||||
|
└────────────────┬─────────────────┘
|
||||||
|
│
|
||||||
|
▼
|
||||||
|
┌──────────────────────────────────┐
|
||||||
|
│ 6. Binarization ──▶ hitam-putih │
|
||||||
|
│ bersih │
|
||||||
|
└────────────────┬─────────────────┘
|
||||||
|
│
|
||||||
|
▼
|
||||||
|
┌──────────────────────────────────┐
|
||||||
|
│ 7. Deskew ──▶ lurusin teks │
|
||||||
|
│ (kalau masih miring) │
|
||||||
|
└────────────────┬─────────────────┘
|
||||||
|
│
|
||||||
|
▼
|
||||||
|
┌──────────────────────────────────┐
|
||||||
|
│ 8. OCR ──▶ extract teks │
|
||||||
|
└────────────────┬─────────────────┘
|
||||||
|
│
|
||||||
|
▼
|
||||||
|
┌──────────────────────────────────┐
|
||||||
|
│ 9. Generate PDF ──▶ output │
|
||||||
|
│ PDF + hidden text layer │
|
||||||
|
└────────────────┬─────────────────┘
|
||||||
|
│
|
||||||
|
▼
|
||||||
|
Output: searchable PDF + teks OCR
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Stage 1: Preprocess
|
||||||
|
|
||||||
|
### Input
|
||||||
|
- Raw image dari HP (bisa 4000×3000 = 12MP, ~3-5MB JPEG)
|
||||||
|
- Format: JPEG, PNG, HEIC (via `image` crate, HEIC butuh feature)
|
||||||
|
|
||||||
|
### Proses
|
||||||
|
```rust
|
||||||
|
use image::{DynamicImage, imageops};
|
||||||
|
|
||||||
|
fn preprocess(img: &DynamicImage) -> DynamicImage {
|
||||||
|
// 1. Resize kalau terlalu besar → max 2000px di sisi terpanjang
|
||||||
|
// Ini penting: edge detection di resolusi tinggi lambat
|
||||||
|
// dan ga nambah akurasi secara signifikan
|
||||||
|
let max_dim = 2000.0;
|
||||||
|
let (w, h) = (img.width() as f64, img.height() as f64);
|
||||||
|
let img = if w.max(h) > max_dim {
|
||||||
|
let scale = max_dim / w.max(h);
|
||||||
|
let new_w = (w * scale) as u32;
|
||||||
|
let new_h = (h * scale) as u32;
|
||||||
|
img.resize_exact(new_w, new_h, imageops::FilterType::Lanczos3)
|
||||||
|
} else {
|
||||||
|
img.clone()
|
||||||
|
};
|
||||||
|
|
||||||
|
// 2. Grayscale → untuk edge detection
|
||||||
|
img.grayscale()
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
### Edge Cases
|
||||||
|
| Kasus | Penanganan |
|
||||||
|
|-------|-----------|
|
||||||
|
| Foto resolusi rendah (<800px) | Skip resize, langsung proses |
|
||||||
|
| HEIC format | Butuh feature `heic` di `image` crate |
|
||||||
|
| Grayscale input | `img.grayscale()` no-op |
|
||||||
|
| Foto malam/noise tinggi | Gaussian blur sebelum edge detection |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Stage 2: Edge Detection
|
||||||
|
|
||||||
|
### Tujuan
|
||||||
|
Cari tepi dokumen dalam foto. Ini hardest part karena background bisa kacau.
|
||||||
|
|
||||||
|
### Algoritma: Canny Edge Detection + Adaptive Threshold
|
||||||
|
|
||||||
|
```rust
|
||||||
|
use image::GrayImage;
|
||||||
|
use imageproc::edges::canny;
|
||||||
|
|
||||||
|
fn detect_edges(img: &GrayImage) -> GrayImage {
|
||||||
|
// Canny dengan dual threshold
|
||||||
|
// low: 50, high: 150 — parameter ini harus di-tune
|
||||||
|
// buat kondisi pencahayaan yang berbeda
|
||||||
|
canny(img, 50.0, 150.0)
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
### Masalah & Solusi
|
||||||
|
|
||||||
|
| Masalah | Penyebab | Solusi |
|
||||||
|
|---------|----------|--------|
|
||||||
|
| **Tepi dokumen putus** | Kontras rendah, bayangan | Morphological close (dilate → erode) untuk sambungin tepi |
|
||||||
|
| **Tepi palsu** | Background ramai (meja motif, lantai) | Cari contour terbesar + area terluas = dokumen |
|
||||||
|
| **Tidak ada tepi** | Background putih, dokumen putih (kertas di meja putih) | Adaptive threshold dulu sebelum Canny, atau fallback ke manual crop |
|
||||||
|
| **Noise garis** | Texture background | Gaussian blur (kernel 5x5) sebelum Canny |
|
||||||
|
|
||||||
|
### Implementation Detail
|
||||||
|
|
||||||
|
```rust
|
||||||
|
/// Edge detection yang robust terhadap berbagai kondisi
|
||||||
|
fn robust_edge_detection(img: &GrayImage) -> GrayImage {
|
||||||
|
// 1. Gaussian blur untuk noise reduction
|
||||||
|
let blurred = imageproc::filter::gaussian_blur_f32(img, 3.0);
|
||||||
|
|
||||||
|
// 2. Coba Canny standard
|
||||||
|
let edges = canny(&blurred, 50.0, 150.0);
|
||||||
|
|
||||||
|
// 3. Morphological close untuk sambung tepi yang putus
|
||||||
|
let kernel = imageproc::morphology::dilate_square(5);
|
||||||
|
let closed = imageproc::morphology::close(&edges, &kernel);
|
||||||
|
|
||||||
|
// 4. Kalau jumlah tepi terlalu sedikit (<1% pixels),
|
||||||
|
// ulang dengan threshold lebih rendah
|
||||||
|
let edge_count = count_non_zero(&closed);
|
||||||
|
let total_pixels = (closed.width() * closed.height()) as u32;
|
||||||
|
if edge_count < total_pixels / 100 {
|
||||||
|
let edges2 = canny(&blurred, 20.0, 80.0);
|
||||||
|
return imageproc::morphology::close(&edges2, &kernel);
|
||||||
|
}
|
||||||
|
|
||||||
|
closed
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Stage 3: Corner Detection
|
||||||
|
|
||||||
|
### Tujuan
|
||||||
|
Dari edge image, cari 4 sudut dokumen.
|
||||||
|
|
||||||
|
### Algoritma: Contour Detection → Largest Rectangle
|
||||||
|
|
||||||
|
```rust
|
||||||
|
use imageproc::contours::{find_contours, Contour};
|
||||||
|
|
||||||
|
fn find_document_corners(edges: &GrayImage) -> Option<[(f64, f64); 4]> {
|
||||||
|
// 1. Cari semua contours
|
||||||
|
let contours = find_contours(edges);
|
||||||
|
|
||||||
|
// 2. Filter: cuma contour dengan area > 20% dari total image
|
||||||
|
// (dokumen biasanya mengisi sebagian besar frame)
|
||||||
|
let total_area = edges.width() as f64 * edges.height() as f64;
|
||||||
|
let docs: Vec<&Contour> = contours
|
||||||
|
.iter()
|
||||||
|
.filter(|c| area_perimeter_ratio(c) > 0.3)
|
||||||
|
.collect();
|
||||||
|
|
||||||
|
// 3. Approximate polygon → cari yang 4 sisi
|
||||||
|
for contour in docs {
|
||||||
|
// Approximate contour ke polygon
|
||||||
|
let polygon = approximate_polygon(&contour.points, 4);
|
||||||
|
if let Some(vertices) = polygon {
|
||||||
|
// Urutkan: top-left, top-right, bottom-right, bottom-left
|
||||||
|
let corners = order_corners(vertices);
|
||||||
|
return Some(corners);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 4. Fallback: contour terbesar → bounding rect
|
||||||
|
contours.iter()
|
||||||
|
.max_by_key(|c| c.points.len())
|
||||||
|
.map(|c| {
|
||||||
|
let rect = bounding_rect(&c.points);
|
||||||
|
order_corners(vec![
|
||||||
|
(rect.left as f64, rect.top as f64),
|
||||||
|
(rect.right as f64, rect.top as f64),
|
||||||
|
(rect.right as f64, rect.bottom as f64),
|
||||||
|
(rect.left as f64, rect.bottom as f64),
|
||||||
|
])
|
||||||
|
})
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
### Corner Ordering Convention
|
||||||
|
|
||||||
|
```
|
||||||
|
(0,0) top-left ────────── top-right (w,0)
|
||||||
|
│ │
|
||||||
|
│ DOKUMEN │
|
||||||
|
│ │
|
||||||
|
(0,h) bottom-left ────── bottom-right (w,h)
|
||||||
|
```
|
||||||
|
|
||||||
|
### Fallback Strategy
|
||||||
|
|
||||||
|
Kalau auto-detect gagal total (contour tidak ketemu, confidence rendah):
|
||||||
|
1. **Fallback 1**: Coba di resolusi lebih rendah (noise berkurang)
|
||||||
|
2. **Fallback 2**: Coba adaptive threshold + Canny ulang
|
||||||
|
3. **Fallback 3**: Minta user crop manual — 4 draggable corners di canvas
|
||||||
|
|
||||||
|
```rust
|
||||||
|
fn detect_corners_with_fallback(img: &GrayImage) -> Result<[(f64, f64); 4], CropMode> {
|
||||||
|
// Attempt 1: Resolusi penuh
|
||||||
|
if let Some(corners) = find_document_corners(img) {
|
||||||
|
return Ok(corners);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Attempt 2: Half resolution (noise reduction)
|
||||||
|
let half = image::imageops::resize(img, img.width() / 2, img.height() / 2,
|
||||||
|
imageops::FilterType::Lanczos3);
|
||||||
|
if let Some(corners) = find_document_corners(&half) {
|
||||||
|
return Ok(corners.map(|(x, y)| (x * 2.0, y * 2.0)));
|
||||||
|
}
|
||||||
|
|
||||||
|
// Fallback: user manual
|
||||||
|
Err(CropMode::Manual)
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Stage 4: Perspective Warp
|
||||||
|
|
||||||
|
### Tujuan
|
||||||
|
Transform 4 titik sudut ke persegi panjang (rectangular). Koreksi perspektif dari foto miring.
|
||||||
|
|
||||||
|
### Algoritma: Homography
|
||||||
|
|
||||||
|
```rust
|
||||||
|
use image::{DynamicImage, GrayImage};
|
||||||
|
use std::f64::consts::PI;
|
||||||
|
|
||||||
|
fn perspective_warp(img: &DynamicImage, corners: [(f64, f64); 4]) -> DynamicImage {
|
||||||
|
// Target: persegi panjang dengan aspect ratio dokumen
|
||||||
|
// Hitung lebar dan tinggi target dari 4 corner
|
||||||
|
let [tl, tr, br, bl] = corners;
|
||||||
|
|
||||||
|
let width_top = distance(tl, tr);
|
||||||
|
let width_bot = distance(bl, br);
|
||||||
|
let width = width_top.max(width_bot).ceil() as u32;
|
||||||
|
|
||||||
|
let height_left = distance(tl, bl);
|
||||||
|
let height_right = distance(tr, br);
|
||||||
|
let height = height_left.max(height_right).ceil() as u32;
|
||||||
|
|
||||||
|
// Source points (4 corners dari detection)
|
||||||
|
let src = [
|
||||||
|
tl, // top-left
|
||||||
|
tr, // top-right
|
||||||
|
br, // bottom-right
|
||||||
|
bl, // bottom-left
|
||||||
|
];
|
||||||
|
|
||||||
|
// Destination points (rectangle)
|
||||||
|
let dst = [
|
||||||
|
(0.0, 0.0), // top-left
|
||||||
|
(width as f64, 0.0), // top-right
|
||||||
|
(width as f64, height as f64), // bottom-right
|
||||||
|
(0.0, height as f64), // bottom-left
|
||||||
|
];
|
||||||
|
|
||||||
|
// Hitung homography matrix
|
||||||
|
let h = compute_homography(&src, &dst);
|
||||||
|
|
||||||
|
// Apply warp (backward mapping + bilinear interpolation)
|
||||||
|
warp_image(img, &h, width, height)
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
### Homography Matrix
|
||||||
|
|
||||||
|
```
|
||||||
|
H = [h11 h12 h13] x' = (h11*x + h12*y + h13) / (h31*x + h32*y + 1)
|
||||||
|
[h21 h22 h23] y' = (h21*x + h22*y + h23) / (h31*x + h32*y + 1)
|
||||||
|
[h31 h32 1 ]
|
||||||
|
```
|
||||||
|
|
||||||
|
Komputasi manual (tanpa OpenCV):
|
||||||
|
```rust
|
||||||
|
/// Compute homography from 4 point correspondences using DLT algorithm
|
||||||
|
fn compute_homography(src: &[(f64, f64); 4], dst: &[(f64, f64); 4]) -> [[f64; 3]; 3] {
|
||||||
|
// Direct Linear Transform
|
||||||
|
// Bangun matrix A (8x9) dari 4 titik
|
||||||
|
// Solve Ah = 0 via SVD → h = last column of V
|
||||||
|
// Reshape ke 3x3
|
||||||
|
//
|
||||||
|
// Detail implementasi:
|
||||||
|
// Setiap titik correspondence (x,y) → (x',y') menghasilkan 2 baris:
|
||||||
|
// [-x, -y, -1, 0, 0, 0, x*x', y*x', x'] = 0
|
||||||
|
// [ 0, 0, 0, -x, -y, -1, x*y', y*y', y'] = 0
|
||||||
|
//
|
||||||
|
// 4 titik → 8 baris → SVD → H matrix
|
||||||
|
|
||||||
|
// Implementasi SVD atau pakai crate `nalgebra` atau `splines`
|
||||||
|
todo!("Implement DLT + SVD")
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
### Image Warp (Backward Mapping)
|
||||||
|
|
||||||
|
```rust
|
||||||
|
fn warp_image(img: &DynamicImage, h: &[[f64; 3]; 3], width: u32, height: u32) -> DynamicImage {
|
||||||
|
let gray = img.grayscale().into_luma8();
|
||||||
|
let mut output = GrayImage::new(width, height);
|
||||||
|
|
||||||
|
// Inverse homography (backward mapping)
|
||||||
|
// tiap pixel output = sample dari input
|
||||||
|
let h_inv = invert_homography(h);
|
||||||
|
|
||||||
|
for y in 0..height {
|
||||||
|
for x in 0..width {
|
||||||
|
// Map (x,y) → source image coordinates
|
||||||
|
let (sx, sy) = apply_homography(&h_inv, x as f64, y as f64);
|
||||||
|
|
||||||
|
// Bilinear interpolation
|
||||||
|
let pixel = bilinear_interpolate(&gray, sx, sy);
|
||||||
|
output.put_pixel(x, y, pixel);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
DynamicImage::ImageLuma8(output)
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
### Edge Cases
|
||||||
|
|
||||||
|
| Masalah | Solusi |
|
||||||
|
|---------|--------|
|
||||||
|
| Dokuen sangat miring (>60°) | Warping mungkin hasilnya gepeng. Deteksi dan skip kalau sudut terlalu ekstrim |
|
||||||
|
| Output sangat besar | Clamp width/height ke max 3000px |
|
||||||
|
| Pixel jaggy (aliasing) | Bilinear interpolation (bukan nearest neighbor) |
|
||||||
|
| Koordinat negative | Clamp ke 0 |
|
||||||
|
| Warp membuat rasio aneh | Lock aspect ratio ke common (A4=1.414, Letter=1.294) |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Stage 5: Shadow Removal
|
||||||
|
|
||||||
|
### Tujuan
|
||||||
|
Hilangkan bayangan (dari lampu, jari, atau sudut ruangan).
|
||||||
|
|
||||||
|
### Algoritma: Adaptive Illumination Correction
|
||||||
|
|
||||||
|
Shadow adalah low-frequency variation. Teks adalah high-frequency. Pisahkan pake low-pass filter.
|
||||||
|
|
||||||
|
```rust
|
||||||
|
fn remove_shadow(img: &GrayImage) -> GrayImage {
|
||||||
|
let (w, h) = (img.width(), img.height());
|
||||||
|
|
||||||
|
// 1. Large Gaussian blur untuk estimasi iluminasi background
|
||||||
|
// Kernel besar (≥sx/50) → cuma dapet variasi iluminasi, bukan teks
|
||||||
|
let blur_radius = (w.min(h) as f64 / 50.0).max(15.0);
|
||||||
|
let background = imageproc::filter::gaussian_blur_f32(img, blur_radius);
|
||||||
|
|
||||||
|
// 2. Subtract background dari original
|
||||||
|
// pixel = max(0, original - background + mean(background))
|
||||||
|
let bg_mean = mean_pixel(&background);
|
||||||
|
let mut corrected = GrayImage::new(w, h);
|
||||||
|
|
||||||
|
for y in 0..h {
|
||||||
|
for x in 0..w {
|
||||||
|
let orig = img.get_pixel(x, y)[0] as f32;
|
||||||
|
let bg = background.get_pixel(x, y)[0] as f32;
|
||||||
|
let corrected_val = (orig - bg + bg_mean) as u8;
|
||||||
|
corrected.put_pixel(x, y, Luma([corrected_val]));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 3. CLAHE (Contrast Limited Adaptive Histogram Equalization)
|
||||||
|
// untuk normalisasi kontras lokal
|
||||||
|
apply_clahe(&corrected, 8, 4) // 8x8 tiles, clip limit 4
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
### Alternatif: Retinex Theory
|
||||||
|
|
||||||
|
```rust
|
||||||
|
/// Retinex-based illumination correction
|
||||||
|
/// I(x,y) = R(x,y) × L(x,y)
|
||||||
|
/// I = observed image, R = reflectance (teks), L = illumination (shadow)
|
||||||
|
fn retinex_shadow_removal(img: &GrayImage) -> GrayImage {
|
||||||
|
// Single-scale Retinex
|
||||||
|
// log(R) = log(I) - log(G * I)
|
||||||
|
// dimana G = Gaussian kernel
|
||||||
|
|
||||||
|
let float_img = convert_to_float(img);
|
||||||
|
let blurred = gaussian_blur_float(&float_img, 30.0);
|
||||||
|
let retinex = element_wise(|p| (p.0.ln() - p.1.ln()), &float_img, &blurred);
|
||||||
|
|
||||||
|
// Normalize ke [0, 255]
|
||||||
|
normalize_to_u8(&retinex)
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Stage 6: Binarization
|
||||||
|
|
||||||
|
### Tujuan
|
||||||
|
Ubah ke hitam-putih bersih — teks hitam, background putih.
|
||||||
|
|
||||||
|
### Algoritma: Sauvola Local Threshold
|
||||||
|
|
||||||
|
Global threshold (Otsu) gagal kalau iluminasi ga merata. Sauvola adaptif per region.
|
||||||
|
|
||||||
|
```rust
|
||||||
|
fn sauvola_threshold(img: &GrayImage, window_size: u32, k: f32) -> GrayImage {
|
||||||
|
// Sauvola: T(x,y) = m(x,y) * [1 + k * (s(x,y)/R - 1)]
|
||||||
|
// m = local mean, s = local std dev, R = max std dev (128), k = parameter (~0.2)
|
||||||
|
|
||||||
|
let (w, h) = (img.width(), img.height());
|
||||||
|
let half_win = (window_size / 2) as i32;
|
||||||
|
let mut output = GrayImage::new(w, h);
|
||||||
|
|
||||||
|
// Integral image for O(1) mean and variance computation
|
||||||
|
let integral = compute_integral_image(img);
|
||||||
|
let integral_sq = compute_integral_image_sq(img);
|
||||||
|
|
||||||
|
for y in 0..h {
|
||||||
|
for x in 0..w {
|
||||||
|
let (mean, variance) = local_stats(&integral, &integral_sq,
|
||||||
|
x as i32, y as i32,
|
||||||
|
half_win, w as i32, h as i32);
|
||||||
|
let std_dev = variance.sqrt();
|
||||||
|
let threshold = mean * (1.0 + k * (std_dev / 128.0 - 1.0));
|
||||||
|
|
||||||
|
let pixel = img.get_pixel(x, y)[0] as f32;
|
||||||
|
output.put_pixel(x, y, Luma([if pixel > threshold { 255 } else { 0 }]));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
output
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
### Parameter Default
|
||||||
|
|
||||||
|
| Parameter | Value | Notes |
|
||||||
|
|-----------|-------|-------|
|
||||||
|
| Window size | max(w,h)/30 | Minimum 15, maksimum 100 |
|
||||||
|
| k | 0.2 | Lower → lebih sensitif, higher → lebih toleran |
|
||||||
|
|
||||||
|
### Edge Cases
|
||||||
|
|
||||||
|
| Masalah | Solusi |
|
||||||
|
|---------|--------|
|
||||||
|
| Dokumen berwarna (bukan putih) | Deteksi warna dominan background, invert logic |
|
||||||
|
| Background gradasi | Sauvola handle ini lebih baik dari Otsu |
|
||||||
|
| Foto terlalu gelap | CLAHE dulu sebelum binarization |
|
||||||
|
| Text tipis/kabur | Morphological erode tipis sesudah binarization |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Stage 7: Deskew
|
||||||
|
|
||||||
|
### Tujuan
|
||||||
|
Koreksi rotasi sisa (kalau dokumen masih miring sedikit — biasanya <5°).
|
||||||
|
|
||||||
|
### Algoritma: Hough Transform
|
||||||
|
|
||||||
|
```rust
|
||||||
|
fn deskew(img: &GrayImage) -> GrayImage {
|
||||||
|
// 1. Cari garis teks via Hough transform
|
||||||
|
// Probabilistic Hough lebih cepat
|
||||||
|
let lines = probabilistic_hough_lines(img, 10, PI / 180.0, 50, 50.0, 10.0);
|
||||||
|
|
||||||
|
if lines.is_empty() {
|
||||||
|
return img.clone();
|
||||||
|
}
|
||||||
|
|
||||||
|
// 2. Hitung sudut rata-rata semua garis
|
||||||
|
let angles: Vec<f64> = lines.iter()
|
||||||
|
.map(|line| line.angle().to_degrees())
|
||||||
|
.filter(|a| a.abs() < 45.0) // skip garis vertikal
|
||||||
|
.collect();
|
||||||
|
|
||||||
|
if angles.is_empty() {
|
||||||
|
return img.clone();
|
||||||
|
}
|
||||||
|
|
||||||
|
let median_angle = median(&angles);
|
||||||
|
|
||||||
|
// Skip kalau sudutnya <0.5 derajat (ga perlu koreksi)
|
||||||
|
if median_angle.abs() < 0.5 {
|
||||||
|
return img.clone();
|
||||||
|
}
|
||||||
|
|
||||||
|
// 3. Rotate image
|
||||||
|
rotate(img, median_angle, imageops::FilterType::Lanczos3)
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Stage 8: OCR
|
||||||
|
|
||||||
|
### Tujuan
|
||||||
|
Extract teks dari gambar biar PDF-nya searchable dan teks bisa di-copy.
|
||||||
|
|
||||||
|
### Implementation
|
||||||
|
|
||||||
|
```rust
|
||||||
|
use leptess::LepTess;
|
||||||
|
|
||||||
|
fn ocr(img: &GrayImage, lang: &str) -> Result<String, OcrError> {
|
||||||
|
// 1. Init Tesseract
|
||||||
|
let mut tess = LepTess::new(Some("/usr/share/tesseract/tessdata"), lang)?;
|
||||||
|
|
||||||
|
// 2. Set image
|
||||||
|
tess.set_image_from_mem(&img.to_bytes())?;
|
||||||
|
// 3. Set PSM (Page Segmentation Mode)
|
||||||
|
// PSM 3 = Fully automatic, default
|
||||||
|
// PSM 6 = Assume single uniform block of text
|
||||||
|
// PSM 4 = Assume single column of text
|
||||||
|
tess.set_source_resolution(300);
|
||||||
|
|
||||||
|
// 4. Recognize
|
||||||
|
let text = tess.get_utf8_text()?;
|
||||||
|
|
||||||
|
Ok(text)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Dapatkan word-level bounding boxes untuk positioning di PDF
|
||||||
|
fn ocr_words(img: &GrayImage, lang: &str) -> Result<Vec<Word>, OcrError> {
|
||||||
|
let mut tess = LepTess::new(Some("/usr/share/tesseract/tessdata"), lang)?;
|
||||||
|
tess.set_image_from_mem(&img.to_bytes())?;
|
||||||
|
|
||||||
|
let words = tess.get_words()
|
||||||
|
.iter()
|
||||||
|
.map(|w| Word {
|
||||||
|
text: w.text.clone(),
|
||||||
|
bbox: Bbox {
|
||||||
|
x: w.x,
|
||||||
|
y: w.y,
|
||||||
|
width: w.w,
|
||||||
|
height: w.h,
|
||||||
|
},
|
||||||
|
confidence: w.confidence,
|
||||||
|
})
|
||||||
|
.collect();
|
||||||
|
|
||||||
|
Ok(words)
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
### Output Format
|
||||||
|
|
||||||
|
```rust
|
||||||
|
struct Word {
|
||||||
|
text: String,
|
||||||
|
bbox: Bbox,
|
||||||
|
confidence: i32, // 0-100
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Stage 9: PDF Generation
|
||||||
|
|
||||||
|
### Tujuan
|
||||||
|
Generate PDF yang:
|
||||||
|
1. Berisi gambar hasil scan (JPEG compressed)
|
||||||
|
2. Hidden text layer dari OCR (biar searchable, selectable)
|
||||||
|
|
||||||
|
### Implementation
|
||||||
|
|
||||||
|
```rust
|
||||||
|
use lopdf::{Document, Object, Stream};
|
||||||
|
use std::io::Write;
|
||||||
|
|
||||||
|
fn generate_searchable_pdf(
|
||||||
|
image_data: &[u8], // JPEG-compressed scan image
|
||||||
|
ocr_text: &str, // Full OCR text
|
||||||
|
words: &[Word], // Word positions
|
||||||
|
page_width: f64, // PDF page width in points
|
||||||
|
page_height: f64, // PDF page height in points
|
||||||
|
) -> Result<Vec<u8>, PdfError> {
|
||||||
|
let mut doc = Document::new();
|
||||||
|
|
||||||
|
// 1. Create image XObject
|
||||||
|
let image_stream = Stream::new(
|
||||||
|
dictionary! {
|
||||||
|
"Type" => "XObject",
|
||||||
|
"Subtype" => "Image",
|
||||||
|
"Width" => page_width as u32,
|
||||||
|
"Height" => page_height as u32,
|
||||||
|
"ColorSpace" => "DeviceGray",
|
||||||
|
"BitsPerComponent" => 8,
|
||||||
|
"Filter" => "DCTDecode", // JPEG compression
|
||||||
|
},
|
||||||
|
image_data,
|
||||||
|
);
|
||||||
|
let image_id = doc.add_object(image_stream);
|
||||||
|
|
||||||
|
// 2. Create content stream: place image, then invisible text
|
||||||
|
// Text layer is invisible (rendering mode 3 = neither fill nor stroke)
|
||||||
|
let mut content = Vec::new();
|
||||||
|
writeln!(content, "q")?; // save state
|
||||||
|
writeln!(content, "{} 0 0 {} 0 0 cm", page_width, page_height)?; // scale to page
|
||||||
|
writeln!(content, "/Im0 Do")?; // place image
|
||||||
|
writeln!(content, "Q")?; // restore state
|
||||||
|
|
||||||
|
// 3. Add invisible text layer (searchable)
|
||||||
|
for word in words {
|
||||||
|
let x = word.bbox.x as f64 / DPI * 72.0; // convert pixels → points
|
||||||
|
let y = (page_height - word.bbox.y as f64 / DPI * 72.0);
|
||||||
|
writeln!(content, "BT")?;
|
||||||
|
writeln!(content, "3 Tr")?; // rendering mode: invisible
|
||||||
|
writeln!(content, "1 Tw")?; // word spacing
|
||||||
|
writeln!(content, "{} {} Td", x, y)?; // position
|
||||||
|
writeln!(content, "({}) Tj", escape_pdf_string(&word.text))?;
|
||||||
|
writeln!(content, "ET")?;
|
||||||
|
}
|
||||||
|
|
||||||
|
let content_stream = Stream::new(
|
||||||
|
dictionary! {},
|
||||||
|
content,
|
||||||
|
);
|
||||||
|
let content_id = doc.add_object(content_stream);
|
||||||
|
|
||||||
|
// 4. Create page
|
||||||
|
let page_id = doc.new_object_id();
|
||||||
|
let pages_id = doc.new_object_id();
|
||||||
|
|
||||||
|
doc.objects.insert(page_id, Object::Dictionary(dictionary! {
|
||||||
|
"Type" => "Page",
|
||||||
|
"Parent" => pages_id,
|
||||||
|
"MediaBox" => vec![0.0, 0.0, page_width, page_height],
|
||||||
|
"Contents" => content_id,
|
||||||
|
"Resources" => dictionary! {
|
||||||
|
"XObject" => dictionary! {
|
||||||
|
"Im0" => image_id,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}));
|
||||||
|
|
||||||
|
// 5. Close and return bytes
|
||||||
|
let bytes = doc.save_to_bytes()?;
|
||||||
|
Ok(bytes)
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
### PDF Coordinate System
|
||||||
|
|
||||||
|
```
|
||||||
|
PDF origin = bottom-left
|
||||||
|
Image origin = top-left
|
||||||
|
|
||||||
|
Perlu flip Y coordinate untuk text layer:
|
||||||
|
y_pdf = page_height - (y_image / dpi * 72)
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Complete Pipeline Assembly
|
||||||
|
|
||||||
|
```rust
|
||||||
|
pub struct ScanPipeline {
|
||||||
|
config: PipelineConfig,
|
||||||
|
metrics: MetricsRecorder,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl ScanPipeline {
|
||||||
|
pub async fn process(&self, input_path: &Path, options: ScanOptions)
|
||||||
|
-> Result<ScanResult, PipelineError>
|
||||||
|
{
|
||||||
|
let timer = self.metrics.start_timer("scan.full");
|
||||||
|
|
||||||
|
// 1. Load
|
||||||
|
let img = image::open(input_path)
|
||||||
|
.map_err(PipelineError::ImageLoad)?;
|
||||||
|
self.metrics.stage_duration("load", timer.split());
|
||||||
|
|
||||||
|
// 2. Preprocess
|
||||||
|
let gray = preprocess(&img);
|
||||||
|
self.metrics.stage_duration("preprocess", timer.split());
|
||||||
|
|
||||||
|
// 3. Edge detection + corners (fallback chain)
|
||||||
|
let corners = detect_corners_with_fallback(&gray)
|
||||||
|
.map_err(PipelineError::CornerDetection)?;
|
||||||
|
self.metrics.stage_duration("corner_detection", timer.split());
|
||||||
|
|
||||||
|
// 4. Perspective warp
|
||||||
|
let warped = perspective_warp(&img, corners); // warp from COLOR original, not gray
|
||||||
|
self.metrics.stage_duration("warp", timer.split());
|
||||||
|
|
||||||
|
let warped_gray = warped.grayscale().into_luma8();
|
||||||
|
|
||||||
|
// 5. Shadow removal
|
||||||
|
let clean = remove_shadow(&warped_gray);
|
||||||
|
self.metrics.stage_duration("shadow_removal", timer.split());
|
||||||
|
|
||||||
|
// 6. Binarization
|
||||||
|
let binary = sauvola_threshold(&clean, 50, 0.2);
|
||||||
|
self.metrics.stage_duration("binarization", timer.split());
|
||||||
|
|
||||||
|
// 7. Deskew
|
||||||
|
let final_image = deskew(&binary);
|
||||||
|
self.metrics.stage_duration("deskew", timer.split());
|
||||||
|
|
||||||
|
// 8. Enhance final (sharpening)
|
||||||
|
let final_image = sharpen(&final_image, 1.0);
|
||||||
|
self.metrics.stage_duration("sharpen", timer.split());
|
||||||
|
|
||||||
|
// 9. OCR
|
||||||
|
let ocr_text = if options.ocr {
|
||||||
|
Some(ocr(&final_image, "eng")?)
|
||||||
|
} else {
|
||||||
|
None
|
||||||
|
};
|
||||||
|
self.metrics.stage_duration("ocr", timer.split());
|
||||||
|
|
||||||
|
// 10. Generate PDF
|
||||||
|
let pdf_bytes = generate_searchable_pdf(
|
||||||
|
&compress_jpeg(&final_image, 90)?,
|
||||||
|
&ocr_text.unwrap_or_default(),
|
||||||
|
&[], // word positions (simplified)
|
||||||
|
A4_WIDTH_PT,
|
||||||
|
A4_HEIGHT_PT,
|
||||||
|
)?;
|
||||||
|
self.metrics.stage_duration("pdf_generation", timer.split());
|
||||||
|
|
||||||
|
// 11. Save
|
||||||
|
let output_path = PathBuf::from("/tmp/tools").join(format!("{}.pdf", uuid::Uuid::new_v4()));
|
||||||
|
std::fs::write(&output_path, &pdf_bytes)?;
|
||||||
|
|
||||||
|
timer.finish();
|
||||||
|
|
||||||
|
Ok(ScanResult {
|
||||||
|
output_path,
|
||||||
|
page_count: 1,
|
||||||
|
file_size: pdf_bytes.len() as u64,
|
||||||
|
ocr_text,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
## Performance Budget
|
||||||
|
|
||||||
|
| Stage | Target | Notes |
|
||||||
|
|-------|--------|-------|
|
||||||
|
| Load + Preprocess | <200ms | File I/O + resize |
|
||||||
|
| Edge + Corner Detection | <500ms | Canny + contour |
|
||||||
|
| Perspective Warp | <800ms | Per-pixel backward mapping |
|
||||||
|
| Shadow Removal | <300ms | FFT convolution atau integral image |
|
||||||
|
| Binarization | <200ms | Integral image |
|
||||||
|
| Deskew | <300ms | Hough transform |
|
||||||
|
| OCR | <1.5s | Tesseract, 300dpi |
|
||||||
|
| PDF Generation | <200ms | lopdf |
|
||||||
|
| **Total** | **<4s** | Per page |
|
||||||
|
|
||||||
|
> **Catatan**: Target di atas untuk image 12MP (4000×3000). Parallel via Rayon untuk batch processing.
|
||||||
|
|
||||||
|
## Edge Cases Matrix
|
||||||
|
|
||||||
|
| Skenario | Pipeline Behavior |
|
||||||
|
|----------|------------------|
|
||||||
|
| Kertas putih di meja putih | Edge detection gagal → fallback ke manual crop |
|
||||||
|
| Foto dari sudut 45° | Warp koreksi perspektif, output presisi |
|
||||||
|
| Dokumen terlipat | Edge detection dapet bentuk aneh → fallback manual |
|
||||||
|
| Bayangan jari | Shadow removal hilangkan |
|
||||||
|
| Teks pudar/pensil | Sauvola threshold adaptif, contrast enhance dulu |
|
||||||
|
| Tanda tangan & stempel | OCR bisa gagal di handwriting, tetap di-image |
|
||||||
|
| Multi-page (buku/kontrak) | Batch upload, masing-masing diproses, digabung 1 PDF |
|
||||||
|
| Foto malam | CLAHE + strong denoise sebelum edge detection |
|
||||||
|
| Latar belakang gradasi | Sauvola handle lebih baik dari Otsu |
|
||||||
@@ -0,0 +1,211 @@
|
|||||||
|
# Security Guide
|
||||||
|
|
||||||
|
Praktik keamanan untuk infrastruktur `asepharyana-hub`.
|
||||||
|
|
||||||
|
## Ringkasan
|
||||||
|
|
||||||
|
| Area | Status | Prioritas |
|
||||||
|
|------|--------|-----------|
|
||||||
|
| Secrets management | GitHub encrypted secrets | Tinggi |
|
||||||
|
| TLS termination | Traefik + cert volume mounts | Tinggi |
|
||||||
|
| Container security | Non-root user (scraper-api) | Sedang |
|
||||||
|
| Network security | Tailscale overlay, app-shared-net | Sedang |
|
||||||
|
| Access control | SSH key, GitHub permissions | Sedang |
|
||||||
|
| Monitoring | Belum ada alert system | Rendah |
|
||||||
|
| Firewall | UFW/iptables (manual) | Sedang |
|
||||||
|
| Backup | lihat `docs/backup-recovery.md` | Sedang |
|
||||||
|
|
||||||
|
## Secrets Management
|
||||||
|
|
||||||
|
### Yang Tidak Boleh di-Commit
|
||||||
|
|
||||||
|
- [ ] `.env` production (disimpan sebagai GitHub secret `ENV_FILE_PRODUCTION`)
|
||||||
|
- [ ] SSH private keys
|
||||||
|
- [ ] API tokens, JWT secret
|
||||||
|
- [ ] Docker registry tokens
|
||||||
|
- [ ] Database passwords
|
||||||
|
- [ ] TLS certificate private keys
|
||||||
|
|
||||||
|
### GitHub Secrets
|
||||||
|
|
||||||
|
Setting di Settings > Secrets and variables > Actions:
|
||||||
|
|
||||||
|
| Secret | Tujuan | Rotasi |
|
||||||
|
|--------|--------|--------|
|
||||||
|
| `SSH_PRIVATE_KEY` | Akses SSH ke VPS | 6 bulan |
|
||||||
|
| `VPS_HOST` | IP VPS | Tidak berubah |
|
||||||
|
| `VPS_USER` | User SSH | Tidak berubah |
|
||||||
|
| `VPS_TARGET_DIR` | Directory di VPS | Tidak berubah |
|
||||||
|
| `ENV_FILE_PRODUCTION` | Full `.env` production | Saat ada perubahan |
|
||||||
|
|
||||||
|
### Update Secrets dengan aman
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Baca current .env dari VPS via SSH
|
||||||
|
ssh root@45.127.35.244 "cat /root/asepharyana-hub/.env" | gh secret set ENV_FILE_PRODUCTION --repo asepharyana/asepharyana-hub --repos
|
||||||
|
```
|
||||||
|
|
||||||
|
### Production `.env` tidak boleh di-commit
|
||||||
|
|
||||||
|
`.env` di root repo adalah untuk development lokal. Production `.env` hanya ada di:
|
||||||
|
1. GitHub secret `ENV_FILE_PRODUCTION`
|
||||||
|
2. File `/root/asepharyana-hub/.env` di VPS (hasil SCP dari CI/CD)
|
||||||
|
|
||||||
|
## TLS / SSL
|
||||||
|
|
||||||
|
### Konfigurasi
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
# Traefik TLS certs dari file mount (bukan auto-ACME)
|
||||||
|
volumes:
|
||||||
|
- ${TRAEFIK_CERT_MY_ID_PEM:-/root/asepharyana.my.id.pem}:/etc/traefik/certs/asepharyana.my.id.pem:ro
|
||||||
|
- ${TRAEFIK_CERT_MY_ID_KEY:-/root/asepharyana.my.id.key}:/etc/traefik/certs/asepharyana.my.id.key:ro
|
||||||
|
```
|
||||||
|
|
||||||
|
### Best Practices
|
||||||
|
|
||||||
|
- Certificates disimpan di host (`/root/`), bukan di repo
|
||||||
|
- Volume mount read-only (`:ro`)
|
||||||
|
- Private key hanya bisa dibaca oleh root (chmod 600)
|
||||||
|
- Renew certificates sebelum expired (monitor expiry)
|
||||||
|
- Dua domain: `asepharyana.my.id` + `asepharyana.web.id`
|
||||||
|
|
||||||
|
## Container Security
|
||||||
|
|
||||||
|
### Non-Root User
|
||||||
|
|
||||||
|
Scraper API berjalan sebagai `appuser` (UID 1001):
|
||||||
|
|
||||||
|
```dockerfile
|
||||||
|
RUN groupadd -g 1001 appgroup && \
|
||||||
|
useradd -u 1001 -g appgroup -s /bin/sh appuser
|
||||||
|
USER appuser
|
||||||
|
```
|
||||||
|
|
||||||
|
Service baru harus mengikuti pattern yang sama.
|
||||||
|
|
||||||
|
### Read-Only Filesystem
|
||||||
|
|
||||||
|
Untuk container yang tidak perlu write ke filesystem:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
services:
|
||||||
|
app:
|
||||||
|
image: app:latest
|
||||||
|
read_only: true
|
||||||
|
tmpfs:
|
||||||
|
- /tmp
|
||||||
|
```
|
||||||
|
|
||||||
|
### Docker Socket
|
||||||
|
|
||||||
|
Hanya Traefik yang perlu akses ke Docker socket (read-only):
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
volumes:
|
||||||
|
- /var/run/docker.sock:/var/run/docker.sock:ro
|
||||||
|
```
|
||||||
|
|
||||||
|
Service lain tidak boleh mount Docker socket.
|
||||||
|
|
||||||
|
### Image Security
|
||||||
|
|
||||||
|
- Build dari base image resmi dan minimal (`debian:bookworm-slim`, `redis:alpine`, `nats:latest`)
|
||||||
|
- Multi-stage build untuk production image (tidak include build tools)
|
||||||
|
- Update base image secara berkala
|
||||||
|
|
||||||
|
## Network Security
|
||||||
|
|
||||||
|
### Firewall (UFW/iptables)
|
||||||
|
|
||||||
|
Di VPS (`orangevps`):
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Hanya buka port yang diperlukan
|
||||||
|
sudo ufw default deny incoming
|
||||||
|
sudo ufw default allow outgoing
|
||||||
|
sudo ufw allow 22/tcp # SSH
|
||||||
|
sudo ufw allow 80/tcp # HTTP redirect
|
||||||
|
sudo ufw allow 443/tcp # HTTPS
|
||||||
|
sudo ufw allow 4222/tcp # NATS (jika perlu external akses)
|
||||||
|
sudo ufw enable
|
||||||
|
```
|
||||||
|
|
||||||
|
Di `imrnes`:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Hanya dari Tailscale interface
|
||||||
|
sudo ufw allow in on tailscale0 to any port 6432 proto tcp # PostgreSQL
|
||||||
|
sudo ufw allow in on tailscale0 to any port 6379 proto tcp # Redis
|
||||||
|
sudo ufw enable
|
||||||
|
```
|
||||||
|
|
||||||
|
### Network Segmentation
|
||||||
|
|
||||||
|
- Semua container di network `app-shared-net` (internal bridge)
|
||||||
|
- Tidak ada port yang di-expose ke host kecuali Traefik (80,443)
|
||||||
|
- Redis hanya accessible via Docker DNS (`redis:6379`) — tidak di-expose
|
||||||
|
- Database hanya via Tailscale — tidak accessible dari public internet
|
||||||
|
|
||||||
|
### SSH Hardening
|
||||||
|
|
||||||
|
Konfigurasi di `/etc/ssh/sshd_config`:
|
||||||
|
|
||||||
|
```
|
||||||
|
Port 22
|
||||||
|
PermitRootLogin prohibit-password
|
||||||
|
PasswordAuthentication no
|
||||||
|
PubkeyAuthentication yes
|
||||||
|
AllowUsers root
|
||||||
|
MaxAuthTries 3
|
||||||
|
ClientAliveInterval 300
|
||||||
|
ClientAliveCountMax 2
|
||||||
|
```
|
||||||
|
|
||||||
|
## Access Control
|
||||||
|
|
||||||
|
### GitHub Repository
|
||||||
|
|
||||||
|
- `contents: write` hanya untuk workflow `update-manifest` dan `update-submodule`
|
||||||
|
- `packages: write` hanya untuk workflow `build`
|
||||||
|
- `security-events: write` hanya untuk workflow `security`
|
||||||
|
- Branch protection di `main`: require PR review, status checks
|
||||||
|
|
||||||
|
### VPS
|
||||||
|
|
||||||
|
- SSH hanya dengan key-based authentication
|
||||||
|
- Key disimpan di GitHub secret, bukan di repo
|
||||||
|
- Rotate SSH key secara berkala (minimal 6 bulan)
|
||||||
|
- Jangan gunakan password login
|
||||||
|
|
||||||
|
## Monitoring Keamanan
|
||||||
|
|
||||||
|
### Saat Ini
|
||||||
|
|
||||||
|
- Traefik access logs (format JSON, buffer size 100)
|
||||||
|
- Docker logs via `docker logs`
|
||||||
|
- CodeQL analysis untuk Rust code (setiap PR + weekly)
|
||||||
|
|
||||||
|
### Rekomendasi
|
||||||
|
|
||||||
|
- [ ] Alert untuk SSH failed login (fail2ban)
|
||||||
|
- [ ] Log monitoring (Loki / Promtail)
|
||||||
|
- [ ] Container vulnerability scanning (Trivy / Snyk)
|
||||||
|
- [ ] Certificate expiry monitoring
|
||||||
|
- [ ] Disk usage alert
|
||||||
|
- [ ] Unauthorized access detection
|
||||||
|
|
||||||
|
## Checklist Security
|
||||||
|
|
||||||
|
- [ ] SSH password authentication disabled
|
||||||
|
- [ ] Root login via SSH key only
|
||||||
|
- [ ] UFW/iptables configured
|
||||||
|
- [ ] Docker socket only mounted where necessary (read-only)
|
||||||
|
- [ ] Container berjalan sebagai non-root user
|
||||||
|
- [ ] `.env` tidak di-commit
|
||||||
|
- [ ] GitHub secrets ter-encrypt
|
||||||
|
- [ ] TLS certificates valid dan belum expired
|
||||||
|
- [ ] CodeQL analysis berjalan
|
||||||
|
- [ ] Backup database berjalan
|
||||||
|
- [ ] SSH key di-rotate
|
||||||
|
- [ ] Docker image di-scan untuk vulnerability
|
||||||
@@ -1,686 +0,0 @@
|
|||||||
# Moonrepo Migration Implementation Plan
|
|
||||||
|
|
||||||
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
|
||||||
|
|
||||||
**Goal:** Migrate the monorepo to moonrepo for centralized task orchestration, caching, and consistent dependency management across all 7 apps.
|
|
||||||
|
|
||||||
**Architecture:** moonrepo sits as a task orchestration layer above the existing Nix build system. `.moon/tasks/` holds shared task definitions (TypeScript and Rust). Each app gets a `moon.yml` with tags and inter-project dependencies. Nix, Docker Compose, git submodules, and infra remain untouched.
|
|
||||||
|
|
||||||
**Tech Stack:** moonrepo CLI, Node 22, Bun, TypeScript 5+, Cargo (Rust)
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
### Task 1: Install moonrepo CLI and scaffold .moon/ directory
|
|
||||||
|
|
||||||
**Files:**
|
|
||||||
- Create: `.moon/workspace.yml`
|
|
||||||
- Create: `.moon/toolchain.yml`
|
|
||||||
- Create: `.moon/tasks/` (directory)
|
|
||||||
|
|
||||||
- [ ] **Step 1: Install moonrepo CLI via curl**
|
|
||||||
|
|
||||||
```bash
|
|
||||||
curl -fsSL https://moonrepo.dev/install/moon.sh | bash
|
|
||||||
```
|
|
||||||
|
|
||||||
- [ ] **Step 2: Verify installation**
|
|
||||||
|
|
||||||
Run: `moon --version`
|
|
||||||
Expected: prints version number (e.g., `moon 1.x.x`)
|
|
||||||
|
|
||||||
- [ ] **Step 3: Create .moon/ scaffold directories**
|
|
||||||
|
|
||||||
```bash
|
|
||||||
mkdir -p .moon/tasks
|
|
||||||
```
|
|
||||||
|
|
||||||
- [ ] **Step 4: Commit**
|
|
||||||
|
|
||||||
```bash
|
|
||||||
git add .moon/
|
|
||||||
git commit -m "chore: scaffold .moon/ directory for moonrepo"
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
### Task 2: Configure workspace.yml
|
|
||||||
|
|
||||||
**Files:**
|
|
||||||
- Create: `.moon/workspace.yml`
|
|
||||||
|
|
||||||
- [ ] **Step 1: Write .moon/workspace.yml**
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
# https://moonrepo.dev/docs/config/workspace
|
|
||||||
$schema: "https://moonrepo.dev/schemas/workspace.json"
|
|
||||||
|
|
||||||
projects:
|
|
||||||
- "apps/*"
|
|
||||||
|
|
||||||
vcs:
|
|
||||||
manager: "git"
|
|
||||||
defaultBranch: "main"
|
|
||||||
|
|
||||||
runner:
|
|
||||||
implicitDeps:
|
|
||||||
# TypeScript apps: lint depends on build (for typecheck path)
|
|
||||||
- "typescript-build.build"
|
|
||||||
|
|
||||||
cacheTtl: 604800
|
|
||||||
```
|
|
||||||
|
|
||||||
- [ ] **Step 2: Validate config structure**
|
|
||||||
|
|
||||||
Run: `moon check`
|
|
||||||
Expected: no errors (will warn about missing project configs — expected)
|
|
||||||
|
|
||||||
- [ ] **Step 3: Commit**
|
|
||||||
|
|
||||||
```bash
|
|
||||||
git add .moon/workspace.yml
|
|
||||||
git commit -m "chore: configure moonrepo workspace with project glob"
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
### Task 3: Configure toolchain.yml
|
|
||||||
|
|
||||||
**Files:**
|
|
||||||
- Create: `.moon/toolchain.yml`
|
|
||||||
|
|
||||||
- [ ] **Step 1: Write .moon/toolchain.yml**
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
# https://moonrepo.dev/docs/config/toolchain
|
|
||||||
$schema: "https://moonrepo.dev/schemas/toolchain.json"
|
|
||||||
|
|
||||||
node:
|
|
||||||
version: "22.11.0"
|
|
||||||
packageManager: "bun"
|
|
||||||
bun:
|
|
||||||
version: "1.3.11"
|
|
||||||
|
|
||||||
typescript:
|
|
||||||
syncProjectReferences: true
|
|
||||||
createMissingConfig: false
|
|
||||||
routeOutDirToCache: false
|
|
||||||
```
|
|
||||||
|
|
||||||
- [ ] **Step 2: Commit**
|
|
||||||
|
|
||||||
```bash
|
|
||||||
git add .moon/toolchain.yml
|
|
||||||
git commit -m "chore: configure moonrepo toolchain (Node 22, Bun 1.3)"
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
### Task 4: Create shared TypeScript task definitions
|
|
||||||
|
|
||||||
**Files:**
|
|
||||||
- Create: `.moon/tasks/typescript-build.yml`
|
|
||||||
- Create: `.moon/tasks/typescript-lint.yml`
|
|
||||||
- Create: `.moon/tasks/typescript-test.yml`
|
|
||||||
|
|
||||||
- [ ] **Step 1: Write .moon/tasks/typescript-build.yml**
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
# https://moonrepo.dev/docs/config/tasks
|
|
||||||
$schema: "https://moonrepo.dev/schemas/tasks.json"
|
|
||||||
|
|
||||||
tasks:
|
|
||||||
build:
|
|
||||||
command: "bun run build"
|
|
||||||
inputs:
|
|
||||||
- "src/**/*"
|
|
||||||
- "tsconfig.json"
|
|
||||||
- "package.json"
|
|
||||||
outputs:
|
|
||||||
- ".next"
|
|
||||||
- "dist"
|
|
||||||
- ".output"
|
|
||||||
options:
|
|
||||||
cache: true
|
|
||||||
dev:
|
|
||||||
command: "bun run dev"
|
|
||||||
local: true
|
|
||||||
options:
|
|
||||||
persistent: true
|
|
||||||
```
|
|
||||||
|
|
||||||
- [ ] **Step 2: Write .moon/tasks/typescript-lint.yml**
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
# https://moonrepo.dev/docs/config/tasks
|
|
||||||
$schema: "https://moonrepo.dev/schemas/tasks.json"
|
|
||||||
|
|
||||||
tasks:
|
|
||||||
lint:
|
|
||||||
command: "bun run lint"
|
|
||||||
inputs:
|
|
||||||
- "src/**/*"
|
|
||||||
- "eslint.config.mjs"
|
|
||||||
- "tsconfig.json"
|
|
||||||
options:
|
|
||||||
cache: false
|
|
||||||
typecheck:
|
|
||||||
command: "bun run check-types"
|
|
||||||
inputs:
|
|
||||||
- "src/**/*"
|
|
||||||
- "tsconfig.json"
|
|
||||||
options:
|
|
||||||
cache: false
|
|
||||||
```
|
|
||||||
|
|
||||||
- [ ] **Step 3: Write .moon/tasks/typescript-test.yml**
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
# https://moonrepo.dev/docs/config/tasks
|
|
||||||
$schema: "https://moonrepo.dev/schemas/tasks.json"
|
|
||||||
|
|
||||||
tasks:
|
|
||||||
test:
|
|
||||||
command: "bun test"
|
|
||||||
inputs:
|
|
||||||
- "src/**/*"
|
|
||||||
- "test/**/*"
|
|
||||||
- "tests/**/*"
|
|
||||||
- "vitest.config.ts"
|
|
||||||
options:
|
|
||||||
cache: false
|
|
||||||
e2e:
|
|
||||||
command: "noop"
|
|
||||||
local: true
|
|
||||||
options:
|
|
||||||
cache: false
|
|
||||||
```
|
|
||||||
|
|
||||||
- [ ] **Step 4: Commit**
|
|
||||||
|
|
||||||
```bash
|
|
||||||
git add .moon/tasks/typescript-build.yml .moon/tasks/typescript-lint.yml .moon/tasks/typescript-test.yml
|
|
||||||
git commit -m "chore: add shared TypeScript task definitions"
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
### Task 5: Create shared Rust task definitions
|
|
||||||
|
|
||||||
**Files:**
|
|
||||||
- Create: `.moon/tasks/rust-build.yml`
|
|
||||||
- Create: `.moon/tasks/rust-test.yml`
|
|
||||||
- Create: `.moon/tasks/rust-lint.yml`
|
|
||||||
|
|
||||||
- [ ] **Step 1: Write .moon/tasks/rust-build.yml**
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
# https://moonrepo.dev/docs/config/tasks
|
|
||||||
$schema: "https://moonrepo.dev/schemas/tasks.json"
|
|
||||||
|
|
||||||
tasks:
|
|
||||||
build:
|
|
||||||
command: "cargo build --release"
|
|
||||||
platform: system
|
|
||||||
inputs:
|
|
||||||
- "src/**/*"
|
|
||||||
- "Cargo.toml"
|
|
||||||
- "Cargo.lock"
|
|
||||||
- "build.rs"
|
|
||||||
outputs:
|
|
||||||
- "target/release/*"
|
|
||||||
options:
|
|
||||||
cache: true
|
|
||||||
envFile: false
|
|
||||||
dev:
|
|
||||||
command: "cargo run"
|
|
||||||
platform: system
|
|
||||||
local: true
|
|
||||||
options:
|
|
||||||
persistent: true
|
|
||||||
envFile: false
|
|
||||||
```
|
|
||||||
|
|
||||||
- [ ] **Step 2: Write .moon/tasks/rust-test.yml**
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
# https://moonrepo.dev/docs/config/tasks
|
|
||||||
$schema: "https://moonrepo.dev/schemas/tasks.json"
|
|
||||||
|
|
||||||
tasks:
|
|
||||||
test:
|
|
||||||
command: "cargo test"
|
|
||||||
platform: system
|
|
||||||
inputs:
|
|
||||||
- "src/**/*"
|
|
||||||
- "Cargo.toml"
|
|
||||||
- "Cargo.lock"
|
|
||||||
- "tests/**/*"
|
|
||||||
options:
|
|
||||||
cache: false
|
|
||||||
envFile: false
|
|
||||||
```
|
|
||||||
|
|
||||||
- [ ] **Step 3: Write .moon/tasks/rust-lint.yml**
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
# https://moonrepo.dev/docs/config/tasks
|
|
||||||
$schema: "https://moonrepo.dev/schemas/tasks.json"
|
|
||||||
|
|
||||||
tasks:
|
|
||||||
lint:
|
|
||||||
command: "cargo clippy -- -D warnings"
|
|
||||||
platform: system
|
|
||||||
inputs:
|
|
||||||
- "src/**/*"
|
|
||||||
- "Cargo.toml"
|
|
||||||
options:
|
|
||||||
cache: false
|
|
||||||
envFile: false
|
|
||||||
fmt-check:
|
|
||||||
command: "cargo fmt --check"
|
|
||||||
platform: system
|
|
||||||
inputs:
|
|
||||||
- "src/**/*"
|
|
||||||
options:
|
|
||||||
cache: false
|
|
||||||
envFile: false
|
|
||||||
```
|
|
||||||
|
|
||||||
- [ ] **Step 4: Commit**
|
|
||||||
|
|
||||||
```bash
|
|
||||||
git add .moon/tasks/rust-build.yml .moon/tasks/rust-test.yml .moon/tasks/rust-lint.yml
|
|
||||||
git commit -m "chore: add shared Rust task definitions"
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
### Task 6: Create per-app moon.yml for TypeScript apps
|
|
||||||
|
|
||||||
**Files:**
|
|
||||||
- Create: `apps/nextjs/moon.yml`
|
|
||||||
- Create: `apps/elysia/moon.yml`
|
|
||||||
- Create: `apps/solidjs/moon.yml`
|
|
||||||
|
|
||||||
- [ ] **Step 1: Write apps/nextjs/moon.yml**
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
# https://moonrepo.dev/docs/config/project
|
|
||||||
$schema: "https://moonrepo.dev/schemas/project.json"
|
|
||||||
|
|
||||||
type: "application"
|
|
||||||
language: "typescript"
|
|
||||||
platform: "node"
|
|
||||||
tags:
|
|
||||||
- "lang:typescript"
|
|
||||||
- "type:frontend"
|
|
||||||
|
|
||||||
dependsOn:
|
|
||||||
- id: "rust-auth"
|
|
||||||
- id: "elysia"
|
|
||||||
```
|
|
||||||
|
|
||||||
- [ ] **Step 2: Write apps/elysia/moon.yml**
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
# https://moonrepo.dev/docs/config/project
|
|
||||||
$schema: "https://moonrepo.dev/schemas/project.json"
|
|
||||||
|
|
||||||
type: "application"
|
|
||||||
language: "typescript"
|
|
||||||
platform: "bun"
|
|
||||||
tags:
|
|
||||||
- "lang:typescript"
|
|
||||||
- "type:backend"
|
|
||||||
```
|
|
||||||
|
|
||||||
- [ ] **Step 3: Write apps/solidjs/moon.yml**
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
# https://moonrepo.dev/docs/config/project
|
|
||||||
$schema: "https://moonrepo.dev/schemas/project.json"
|
|
||||||
|
|
||||||
type: "application"
|
|
||||||
language: "typescript"
|
|
||||||
platform: "bun"
|
|
||||||
tags:
|
|
||||||
- "lang:typescript"
|
|
||||||
- "type:frontend"
|
|
||||||
|
|
||||||
dependsOn:
|
|
||||||
- id: "elysia"
|
|
||||||
- id: "rust-auth"
|
|
||||||
```
|
|
||||||
|
|
||||||
- [ ] **Step 4: Commit**
|
|
||||||
|
|
||||||
```bash
|
|
||||||
git add apps/nextjs/moon.yml apps/elysia/moon.yml apps/solidjs/moon.yml
|
|
||||||
git commit -m "chore: add moon.yml for TypeScript apps (nextjs, elysia, solidjs)"
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
### Task 7: Create per-app moon.yml for Rust apps
|
|
||||||
|
|
||||||
**Files:**
|
|
||||||
- Create: `apps/rust/moon.yml`
|
|
||||||
- Create: `apps/rust-auth/moon.yml`
|
|
||||||
- Create: `apps/leptos/moon.yml`
|
|
||||||
|
|
||||||
- [ ] **Step 1: Write apps/rust/moon.yml**
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
# https://moonrepo.dev/docs/config/project
|
|
||||||
$schema: "https://moonrepo.dev/schemas/project.json"
|
|
||||||
|
|
||||||
type: "application"
|
|
||||||
language: "rust"
|
|
||||||
platform: "system"
|
|
||||||
tags:
|
|
||||||
- "lang:rust"
|
|
||||||
- "type:backend"
|
|
||||||
|
|
||||||
fileGroups:
|
|
||||||
sources:
|
|
||||||
- "src/**/*.rs"
|
|
||||||
- "Cargo.toml"
|
|
||||||
- "Cargo.lock"
|
|
||||||
- "build.rs"
|
|
||||||
- "rustfmt.toml"
|
|
||||||
```
|
|
||||||
|
|
||||||
- [ ] **Step 2: Write apps/rust-auth/moon.yml**
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
# https://moonrepo.dev/docs/config/project
|
|
||||||
$schema: "https://moonrepo.dev/schemas/project.json"
|
|
||||||
|
|
||||||
type: "application"
|
|
||||||
language: "rust"
|
|
||||||
platform: "system"
|
|
||||||
tags:
|
|
||||||
- "lang:rust"
|
|
||||||
- "type:backend"
|
|
||||||
|
|
||||||
fileGroups:
|
|
||||||
sources:
|
|
||||||
- "src/**/*.rs"
|
|
||||||
- "Cargo.toml"
|
|
||||||
- "Cargo.lock"
|
|
||||||
```
|
|
||||||
|
|
||||||
- [ ] **Step 3: Write apps/leptos/moon.yml**
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
# https://moonrepo.dev/docs/config/project
|
|
||||||
$schema: "https://moonrepo.dev/schemas/project.json"
|
|
||||||
|
|
||||||
type: "application"
|
|
||||||
language: "rust"
|
|
||||||
platform: "system"
|
|
||||||
tags:
|
|
||||||
- "lang:rust"
|
|
||||||
- "type:frontend"
|
|
||||||
|
|
||||||
dependsOn:
|
|
||||||
- id: "rust"
|
|
||||||
|
|
||||||
fileGroups:
|
|
||||||
sources:
|
|
||||||
- "src/**/*.rs"
|
|
||||||
- "Cargo.toml"
|
|
||||||
- "Cargo.lock"
|
|
||||||
- "Trunk.toml"
|
|
||||||
- "rust-toolchain.toml"
|
|
||||||
```
|
|
||||||
|
|
||||||
- [ ] **Step 4: Commit**
|
|
||||||
|
|
||||||
```bash
|
|
||||||
git add apps/rust/moon.yml apps/rust-auth/moon.yml apps/leptos/moon.yml
|
|
||||||
git commit -m "chore: add moon.yml for Rust apps (rust, rust-auth, leptos)"
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
### Task 8: Create moon.yml for 9router
|
|
||||||
|
|
||||||
**Files:**
|
|
||||||
- Create: `apps/9router/moon.yml`
|
|
||||||
|
|
||||||
- [ ] **Step 1: Write apps/9router/moon.yml**
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
# https://moonrepo.dev/docs/config/project
|
|
||||||
$schema: "https://moonrepo.dev/schemas/project.json"
|
|
||||||
|
|
||||||
type: "application"
|
|
||||||
language: "unknown"
|
|
||||||
platform: "system"
|
|
||||||
tags:
|
|
||||||
- "type:router"
|
|
||||||
|
|
||||||
fileGroups:
|
|
||||||
sources:
|
|
||||||
- "src/**/*"
|
|
||||||
- "next.config.mjs"
|
|
||||||
- "package.json"
|
|
||||||
```
|
|
||||||
|
|
||||||
- [ ] **Step 2: Commit**
|
|
||||||
|
|
||||||
```bash
|
|
||||||
git add apps/9router/moon.yml
|
|
||||||
git commit -m "chore: add moon.yml for 9router"
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
### Task 9: Update flake.nix — add moon CLI to devShell
|
|
||||||
|
|
||||||
**Files:**
|
|
||||||
- Modify: `flake.nix`
|
|
||||||
|
|
||||||
- [ ] **Step 1: Add moon to nativeBuildInputs in flake.nix**
|
|
||||||
|
|
||||||
Find the `devShells.default` block in `flake.nix`. Add `moon` to `nativeBuildInputs`:
|
|
||||||
|
|
||||||
```
|
|
||||||
devShells.default = pkgs.mkShell {
|
|
||||||
name = "ultimate-asepharyana-dev";
|
|
||||||
nativeBuildInputs = with pkgs; [
|
|
||||||
rustToolchain
|
|
||||||
bun
|
|
||||||
nodejs_22
|
|
||||||
pkg-config
|
|
||||||
openssl
|
|
||||||
trunk
|
|
||||||
wasm-bindgen-cli
|
|
||||||
binaryen
|
|
||||||
process-compose
|
|
||||||
mysql84
|
|
||||||
redis
|
|
||||||
minio-client
|
|
||||||
gh
|
|
||||||
git
|
|
||||||
moon # <-- add this line
|
|
||||||
];
|
|
||||||
# ... shellHook unchanged
|
|
||||||
};
|
|
||||||
```
|
|
||||||
|
|
||||||
- [ ] **Step 2: Verify Nix can still evaluate the flake**
|
|
||||||
|
|
||||||
Run: `nix flake check --no-build 2>&1 | head -20`
|
|
||||||
Expected: no evaluation errors
|
|
||||||
|
|
||||||
- [ ] **Step 3: Commit**
|
|
||||||
|
|
||||||
```bash
|
|
||||||
git add flake.nix
|
|
||||||
git commit -m "chore: add moon CLI to Nix devShell"
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
### Task 10: Update .gitignore
|
|
||||||
|
|
||||||
**Files:**
|
|
||||||
- Modify: `.gitignore`
|
|
||||||
|
|
||||||
- [ ] **Step 1: Add moonrepo cache entries to .gitignore**
|
|
||||||
|
|
||||||
Append to `.gitignore`:
|
|
||||||
|
|
||||||
```gitignore
|
|
||||||
# moonrepo
|
|
||||||
.moon/cache
|
|
||||||
.~moon
|
|
||||||
```
|
|
||||||
|
|
||||||
- [ ] **Step 2: Commit**
|
|
||||||
|
|
||||||
```bash
|
|
||||||
git add .gitignore
|
|
||||||
git commit -m "chore: add moonrepo cache entries to .gitignore"
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
### Task 11: Validate installation with moon check
|
|
||||||
|
|
||||||
**Files:** (none — validation only)
|
|
||||||
|
|
||||||
- [ ] **Step 1: Run moon check**
|
|
||||||
|
|
||||||
```bash
|
|
||||||
moon check
|
|
||||||
```
|
|
||||||
|
|
||||||
Expected: `OK` or zero errors. If warnings about unresolved project IDs appear, verify that `apps/*` glob in `workspace.yml` matches all project directories.
|
|
||||||
|
|
||||||
- [ ] **Step 2: Run moon query projects**
|
|
||||||
|
|
||||||
```bash
|
|
||||||
moon query projects
|
|
||||||
```
|
|
||||||
|
|
||||||
Expected: lists all 7 projects with their tags: nextjs, elysia, solidjs, rust, rust-auth, leptos, 9router
|
|
||||||
|
|
||||||
- [ ] **Step 3: Verify tag queries work**
|
|
||||||
|
|
||||||
```bash
|
|
||||||
moon query projects --tag lang:typescript
|
|
||||||
```
|
|
||||||
|
|
||||||
Expected: nextjs, elysia, solidjs
|
|
||||||
|
|
||||||
```bash
|
|
||||||
moon query projects --tag lang:rust
|
|
||||||
```
|
|
||||||
|
|
||||||
Expected: rust, rust-auth, leptos
|
|
||||||
|
|
||||||
- [ ] **Step 4: Commit (if any fixes were needed)**
|
|
||||||
|
|
||||||
No commit needed if check passes clean.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
### Task 12: Test — moon run build on TypeScript apps
|
|
||||||
|
|
||||||
**Files:** (none — test only)
|
|
||||||
|
|
||||||
- [ ] **Step 1: Build nextjs**
|
|
||||||
|
|
||||||
```bash
|
|
||||||
moon run nextjs:build
|
|
||||||
```
|
|
||||||
|
|
||||||
Expected: `next build` runs successfully, outputs to `.next/`
|
|
||||||
|
|
||||||
- [ ] **Step 2: Build elysia**
|
|
||||||
|
|
||||||
```bash
|
|
||||||
moon run elysia:build
|
|
||||||
```
|
|
||||||
|
|
||||||
Expected: `bun build` runs successfully, outputs to `dist/`
|
|
||||||
|
|
||||||
- [ ] **Step 3: Build solidjs**
|
|
||||||
|
|
||||||
```bash
|
|
||||||
moon run solidjs:build
|
|
||||||
```
|
|
||||||
|
|
||||||
Expected: `vinxi build` runs successfully, outputs to `.output/`
|
|
||||||
|
|
||||||
- [ ] **Step 4: Verify caching on second build (nextjs)**
|
|
||||||
|
|
||||||
```bash
|
|
||||||
moon run nextjs:build
|
|
||||||
```
|
|
||||||
|
|
||||||
Expected: `Cached` — no rebuild, uses moonrepo cache
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
### Task 13: Test — moon run lint and test
|
|
||||||
|
|
||||||
**Files:** (none — test only)
|
|
||||||
|
|
||||||
- [ ] **Step 1: Run lint across TypeScript apps**
|
|
||||||
|
|
||||||
```bash
|
|
||||||
moon run :lint
|
|
||||||
```
|
|
||||||
|
|
||||||
Expected: all apps with a `lint` task run it. Note failures as they exist pre-migration (not caused by moonrepo).
|
|
||||||
|
|
||||||
- [ ] **Step 2: Run tests across TypeScript apps**
|
|
||||||
|
|
||||||
```bash
|
|
||||||
moon run :test
|
|
||||||
```
|
|
||||||
|
|
||||||
Expected: all apps with a `test` task run it.
|
|
||||||
|
|
||||||
- [ ] **Step 3: Run tag-scoped commands**
|
|
||||||
|
|
||||||
```bash
|
|
||||||
moon run --tag lang:typescript :build
|
|
||||||
```
|
|
||||||
|
|
||||||
Expected: builds nextjs, elysia, solidjs only (not Rust apps).
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
### Task 14: Final commit and documentation
|
|
||||||
|
|
||||||
**Files:**
|
|
||||||
- Modify: `.gitignore` (if any final updates)
|
|
||||||
|
|
||||||
- [ ] **Step 1: Final moon check**
|
|
||||||
|
|
||||||
```bash
|
|
||||||
moon check
|
|
||||||
```
|
|
||||||
|
|
||||||
Expected: clean, no errors.
|
|
||||||
|
|
||||||
- [ ] **Step 2: Commit any remaining changes**
|
|
||||||
|
|
||||||
```bash
|
|
||||||
git status
|
|
||||||
```
|
|
||||||
|
|
||||||
If nothing outstanding, move on.
|
|
||||||
|
|
||||||
- [ ] **Step 3: Verify the full workspace is clean**
|
|
||||||
|
|
||||||
```bash
|
|
||||||
git status
|
|
||||||
```
|
|
||||||
|
|
||||||
Expected: working tree clean.
|
|
||||||
@@ -1,712 +0,0 @@
|
|||||||
# PostgreSQL Migration Implementation Plan
|
|
||||||
|
|
||||||
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
|
||||||
|
|
||||||
**Goal:** Migrate elysia (Drizzle ORM + MySQL) and rust (SeaORM + MySQL) apps to PostgreSQL via direct cutover.
|
|
||||||
|
|
||||||
**Architecture:** Export MySQL data, convert schema to PostgreSQL format, import into target PostgreSQL instance, update app drivers and connection strings, deploy and validate.
|
|
||||||
|
|
||||||
**Tech Stack:** MySQL (source), PostgreSQL (target), Drizzle ORM (elysia), SeaORM (rust), pgloader or manual conversion for schema migration.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## File Structure
|
|
||||||
|
|
||||||
### Elysia App Changes
|
|
||||||
- `apps/elysia/src/db/lib/database.ts` — Replace mysql2 driver with postgres driver
|
|
||||||
- `apps/elysia/src/db/lib/schema.ts` — Replace mysqlTable with pgTable, update column types
|
|
||||||
- `apps/elysia/package.json` — Replace mysql2 with pg dependency
|
|
||||||
- `.env` or config file — Update DATABASE_URL to PostgreSQL connection string
|
|
||||||
|
|
||||||
### Rust App Changes
|
|
||||||
- `apps/rust/Cargo.toml` — Replace sqlx-mysql feature with sqlx-postgres
|
|
||||||
- `apps/rust/src/infra/db_setup.rs` — Update DbBackend::MySql to DbBackend::Postgres, adjust SQL syntax
|
|
||||||
- Config/environment — Update DATABASE_URL to PostgreSQL connection string
|
|
||||||
|
|
||||||
### Migration Artifacts
|
|
||||||
- `mysql_backup.sql` — MySQL dump (created during migration, not committed)
|
|
||||||
- `converted.sql` — PostgreSQL-compatible dump (created during migration, not committed)
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Task Breakdown
|
|
||||||
|
|
||||||
### Task 1: Backup MySQL and Export Schema
|
|
||||||
|
|
||||||
**Files:**
|
|
||||||
- Create: `mysql_backup.sql` (temporary, not committed)
|
|
||||||
|
|
||||||
- [ ] **Step 1: Export MySQL database**
|
|
||||||
|
|
||||||
```bash
|
|
||||||
cd /mnt/code/bp3/ultimate-asepharyana.tech
|
|
||||||
mysqldump -u <mysql_user> -p <mysql_password> -h <mysql_host> <database_name> > mysql_backup.sql
|
|
||||||
```
|
|
||||||
|
|
||||||
Expected: File created with full schema + data. Verify file size > 1MB (contains data).
|
|
||||||
|
|
||||||
-[]**Step 2: Verify backup integrity**
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# Check row counts in backup
|
|
||||||
grep "INSERT INTO" mysql_backup.sql | wc -l
|
|
||||||
```
|
|
||||||
|
|
||||||
Expected: Multiple INSERT statements present. Note row counts for later validation.
|
|
||||||
|
|
||||||
- [ ] **Step 3: Document backup location**
|
|
||||||
|
|
||||||
Store `mysql_backup.sql` in safe location (not in git). This is rollback insurance.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
### Task 2: Convert MySQL Schema to PostgreSQL
|
|
||||||
|
|
||||||
**Files:**
|
|
||||||
- Create: `converted.sql` (temporary, not committed)
|
|
||||||
|
|
||||||
- [ ] **Step 1: Install pgloader (if not present)**
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# macOS
|
|
||||||
brew install pgloader
|
|
||||||
|
|
||||||
# Linux (Ubuntu/Debian)
|
|
||||||
sudo apt-get install pgloader
|
|
||||||
|
|
||||||
# Or use Docker
|
|
||||||
docker run --rm -v $(pwd):/data pgloader/pgloader pgloader /data/mysql_backup.sql postgresql://asephs:hunterz@100.108.1.124:5432/hub
|
|
||||||
```
|
|
||||||
|
|
||||||
- [ ] **Step 2: Convert MySQL dump to PostgreSQL**
|
|
||||||
|
|
||||||
```bash
|
|
||||||
pgloader mysql_backup.sql postgresql://asephs:hunterz@100.108.1.124:5432/hub
|
|
||||||
```
|
|
||||||
|
|
||||||
Or manually convert if pgloader unavailable:
|
|
||||||
- Replace `AUTO_INCREMENT` with `SERIAL` or `BIGSERIAL`
|
|
||||||
- Replace `DATETIME` with `TIMESTAMP`
|
|
||||||
- Replace backticks with double quotes
|
|
||||||
- Update index syntax for PostgreSQL
|
|
||||||
|
|
||||||
Expected: Conversion completes without errors. Check for warnings about type conversions.
|
|
||||||
|
|
||||||
- [ ] **Step 3: Verify conversion output**
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# If using pgloader, it creates converted.sql automatically
|
|
||||||
# If manual, save converted schema to file
|
|
||||||
cat converted.sql | head -50
|
|
||||||
```
|
|
||||||
|
|
||||||
Expected: PostgreSQL-compatible SQL syntax (no backticks, SERIAL types, TIMESTAMP).
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
### Task 3: Test PostgreSQL Import
|
|
||||||
|
|
||||||
**Files:**
|
|
||||||
- Target: PostgreSQL instance at `postgresql://asephs:hunterz@100.108.1.124:5432/hub`
|
|
||||||
|
|
||||||
- [ ] **Step 1: Connect to target PostgreSQL**
|
|
||||||
|
|
||||||
```bash
|
|
||||||
psql postgresql://asephs:hunterz@100.108.1.124:5432/hub
|
|
||||||
```
|
|
||||||
|
|
||||||
Expected: Connected to PostgreSQL. Prompt shows `hub=#`.
|
|
||||||
|
|
||||||
- [ ] **Step 2: Import converted schema**
|
|
||||||
|
|
||||||
```bash
|
|
||||||
psql postgresql://asephs:hunterz@100.108.1.124:5432/hub < converted.sql
|
|
||||||
```
|
|
||||||
|
|
||||||
Expected: Import completes. Check for errors (should be none).
|
|
||||||
|
|
||||||
- [ ] **Step 3: Validate table creation**
|
|
||||||
|
|
||||||
```bash
|
|
||||||
psql postgresql://asephs:hunterz@100.108.1.124:5432/hub -c "\dt"
|
|
||||||
```
|
|
||||||
|
|
||||||
Expected: All tables listed (User, Account, Session, Role, Permission, UserRole, ImageCache, etc.).
|
|
||||||
|
|
||||||
- []**Step 4: Validate row counts**
|
|
||||||
|
|
||||||
```bash
|
|
||||||
psql postgresql://asephs:hunterz@100.108.1.124:5432/hub -c "SELECT COUNT(*) FROM \"User\";"
|
|
||||||
psql postgresql://asephs:hunterz@100.108.1.124:5432/hub -c "SELECT COUNT(*) FROM \"Account\";"
|
|
||||||
```
|
|
||||||
|
|
||||||
Expected: Row counts match MySQL backup (from Task 1, Step 2).
|
|
||||||
|
|
||||||
- [ ] **Step 5: Validate indexes**
|
|
||||||
|
|
||||||
```bash
|
|
||||||
psql postgresql://asephs:hunterz@100.108.1.124:5432/hub -c "\di"
|
|
||||||
```
|
|
||||||
|
|
||||||
Expected: All indexes present (email_idx, username_idx, userId_idx, sessionToken_idx, etc.).
|
|
||||||
|
|
||||||
- [ ] **Step 6: Validate foreign keys**
|
|
||||||
|
|
||||||
```bash
|
|
||||||
psql postgresql://asephs:hunterz@100.108.1.124:5432/hub -c "SELECT constraint_name, table_name FROM information_schema.table_constraints WHERE constraint_type = 'FOREIGN KEY';"
|
|
||||||
```
|
|
||||||
|
|
||||||
Expected: Foreign key constraints listed (User→Account, User→Session, etc.).
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
### Task 4: Update Elysia Database Driver
|
|
||||||
|
|
||||||
**Files:**
|
|
||||||
- Modify: `apps/elysia/src/db/lib/database.ts`
|
|
||||||
- Modify: `apps/elysia/src/db/lib/schema.ts`
|
|
||||||
- Modify: `apps/elysia/package.json`
|
|
||||||
|
|
||||||
- [ ] **Step 1: Update package.json dependencies**
|
|
||||||
|
|
||||||
Replace mysql2 with pg:
|
|
||||||
|
|
||||||
```json
|
|
||||||
{
|
|
||||||
"dependencies": {
|
|
||||||
"drizzle-orm": "^0.45.2",
|
|
||||||
"pg": "^8.11.0",
|
|
||||||
"elysia": "^1.4.28"
|
|
||||||
},
|
|
||||||
"devDependencies": {
|
|
||||||
"drizzle-kit": "^0.31.10"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
Run: `cd apps/elysia && bun install`
|
|
||||||
|
|
||||||
Expected: pg installed, mysql2 removed from node_modules.
|
|
||||||
|
|
||||||
- [ ] **Step 2: Update database.ts driver**
|
|
||||||
|
|
||||||
Replace entire file:
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
import type { PostgresJsDatabase } from 'drizzle-orm/postgres-js'
|
|
||||||
import { drizzle } from 'drizzle-orm/postgres-js'
|
|
||||||
import postgres from 'postgres'
|
|
||||||
import * as schema from './schema'
|
|
||||||
|
|
||||||
export type Database = PostgresJsDatabase<typeof schema>
|
|
||||||
|
|
||||||
let dbInstance: Database | null = null
|
|
||||||
let sqlInstance: ReturnType<typeof postgres> | null = null
|
|
||||||
|
|
||||||
export function initializeDb(databaseUrl: string): Database {
|
|
||||||
if (dbInstance) {
|
|
||||||
return dbInstance
|
|
||||||
}
|
|
||||||
|
|
||||||
sqlInstance = postgres(databaseUrl)
|
|
||||||
dbInstance = drizzle(sqlInstance, { schema, mode: 'default' })
|
|
||||||
|
|
||||||
return dbInstance
|
|
||||||
}
|
|
||||||
|
|
||||||
export function getDb(): Database {
|
|
||||||
if (!dbInstance) {
|
|
||||||
throw new Error('Database not initialized. Call initializeDb first.')
|
|
||||||
}
|
|
||||||
return dbInstance
|
|
||||||
}
|
|
||||||
|
|
||||||
export async function closeDb() {
|
|
||||||
if (sqlInstance) {
|
|
||||||
await sqlInstance.end()
|
|
||||||
sqlInstance = null
|
|
||||||
dbInstance = null
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
Expected: File updated. No syntax errors.
|
|
||||||
|
|
||||||
- [ ] **Step 3: Update schema.ts imports**
|
|
||||||
|
|
||||||
Replace:
|
|
||||||
```typescript
|
|
||||||
import {
|
|
||||||
index,
|
|
||||||
int,
|
|
||||||
mysqlTable,
|
|
||||||
primaryKey,
|
|
||||||
text,
|
|
||||||
timestamp,
|
|
||||||
varchar,
|
|
||||||
} from 'drizzle-orm/mysql-core'
|
|
||||||
```
|
|
||||||
|
|
||||||
With:
|
|
||||||
```typescript
|
|
||||||
import {
|
|
||||||
index,
|
|
||||||
integer,
|
|
||||||
pgTable,
|
|
||||||
primaryKey,
|
|
||||||
text,
|
|
||||||
timestamp,
|
|
||||||
varchar,
|
|
||||||
} from 'drizzle-orm/postgres-core'
|
|
||||||
```
|
|
||||||
|
|
||||||
- [ ] **Step 4: Update schema.ts table definitions**
|
|
||||||
|
|
||||||
Replace all `mysqlTable` with `pgTable` and `int` with `integer`:
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
// Before
|
|
||||||
export const users = mysqlTable(
|
|
||||||
'User',
|
|
||||||
{
|
|
||||||
id: varchar('id', { length: 255 }).primaryKey(),
|
|
||||||
name: varchar('name', { length: 255 }),
|
|
||||||
// ...
|
|
||||||
},
|
|
||||||
// ...
|
|
||||||
)
|
|
||||||
|
|
||||||
// After
|
|
||||||
export const users = pgTable(
|
|
||||||
'User',
|
|
||||||
{
|
|
||||||
id: varchar('id', { length: 255 }).primaryKey(),
|
|
||||||
name: varchar('name', { length: 255 }),
|
|
||||||
// ...
|
|
||||||
},
|
|
||||||
// ...
|
|
||||||
)
|
|
||||||
```
|
|
||||||
|
|
||||||
Do this for all tables: users, accounts, sessions, roles, permissions, userRoles, and any others.
|
|
||||||
|
|
||||||
Expected: All `mysqlTable` → `pgTable`, all `int` → `integer`.
|
|
||||||
|
|
||||||
- [ ] **Step 5: Update environment variable**
|
|
||||||
|
|
||||||
Set DATABASE_URL in `.env` or deployment config:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
DATABASE_URL=postgresql://asephs:hunterz@100.108.1.124:5432/hub
|
|
||||||
```
|
|
||||||
|
|
||||||
Expected: Environment variable set and accessible to elysia app.
|
|
||||||
|
|
||||||
- [ ] **Step 6: Test elysia connection**
|
|
||||||
|
|
||||||
```bash
|
|
||||||
cd apps/elysia
|
|
||||||
bun run src/index.ts
|
|
||||||
```
|
|
||||||
|
|
||||||
Expected: App starts without connection errors. Check logs for "Database initialized" or similar.
|
|
||||||
|
|
||||||
- [ ] **Step 7: Commit elysia changes**
|
|
||||||
|
|
||||||
```bash
|
|
||||||
cd /mnt/code/bp3/ultimate-asepharyana.tech
|
|
||||||
git add apps/elysia/src/db/lib/database.ts apps/elysia/src/db/lib/schema.ts apps/elysia/package.json
|
|
||||||
git commit -m "feat(elysia): migrate database driver from MySQL to PostgreSQL"
|
|
||||||
```
|
|
||||||
|
|
||||||
Expected: Commit created with message.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
### Task 5: Update Rust Database Driver
|
|
||||||
|
|
||||||
**Files:**
|
|
||||||
- Modify: `apps/rust/Cargo.toml`
|
|
||||||
- Modify: `apps/rust/src/infra/db_setup.rs`
|
|
||||||
|
|
||||||
- [ ] **Step 1: Update Cargo.toml features**
|
|
||||||
|
|
||||||
Replace:
|
|
||||||
```toml
|
|
||||||
sea-orm = { version = "1.1.19", features = ["sqlx-mysql", "runtime-tokio-rustls", "macros", "with-chrono", "with-uuid"] }
|
|
||||||
```
|
|
||||||
|
|
||||||
With:
|
|
||||||
```toml
|
|
||||||
sea-orm = { version = "1.1.19", features = ["sqlx-postgres", "runtime-tokio-rustls", "macros", "with-chrono", "with-uuid"] }
|
|
||||||
```
|
|
||||||
|
|
||||||
Expected: Cargo.toml updated. Feature changed from sqlx-mysql to sqlx-postgres.
|
|
||||||
|
|
||||||
- [ ] **Step 2: Update db_setup.rs backend check**
|
|
||||||
|
|
||||||
Replace:
|
|
||||||
```rust
|
|
||||||
match backend {
|
|
||||||
DbBackend::MySql => {
|
|
||||||
// MySQL-specific logic
|
|
||||||
}
|
|
||||||
_ => {
|
|
||||||
info!("ℹ️ Skipping schema init for non-MySQL backend");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
With:
|
|
||||||
```rust
|
|
||||||
match backend {
|
|
||||||
DbBackend::Postgres => {
|
|
||||||
// PostgreSQL-specific logic
|
|
||||||
let tables = vec![(
|
|
||||||
"ImageCache",
|
|
||||||
schema
|
|
||||||
.create_table_from_entity(image_cache::Entity)
|
|
||||||
.if_not_exists()
|
|
||||||
.to_owned(),
|
|
||||||
)];
|
|
||||||
|
|
||||||
for (name, stmt) in tables {
|
|
||||||
match db.execute(backend.build(&stmt)).await {
|
|
||||||
Ok(_) => info!(" ✓ Table '{}' checked/created", name),
|
|
||||||
Err(e) => {
|
|
||||||
error!(" [!] Failed to create table '{}': {}", name, e);
|
|
||||||
return Err(e);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// PostgreSQL index creation (different syntax)
|
|
||||||
let index_sql = "CREATE INDEX IF NOT EXISTS idx_image_cache_cdn_url ON \"ImageCache\" (cdn_url)";
|
|
||||||
match db.execute(Statement::from_string(backend, index_sql)).await {
|
|
||||||
Ok(_) => info!(" ✓ Index 'idx_image_cache_cdn_url' ensured"),
|
|
||||||
Err(e) => {
|
|
||||||
let err_str = e.to_string();
|
|
||||||
// PostgreSQL duplicate index error
|
|
||||||
if err_str.contains("already exists") {
|
|
||||||
info!(" ✓ Index 'idx_image_cache_cdn_url' already exists");
|
|
||||||
} else {
|
|
||||||
error!(" [!] Failed to create index on ImageCache: {}", e);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
info!("✅ Database schema initialization complete.");
|
|
||||||
}
|
|
||||||
_ => {
|
|
||||||
info!("ℹ️ Skipping schema init for non-PostgreSQL backend");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
Expected: db_setup.rs updated with PostgreSQL backend handling.
|
|
||||||
|
|
||||||
- [ ] **Step 3: Update environment variable**
|
|
||||||
|
|
||||||
Set DATABASE_URL in `.env` or deployment config:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
DATABASE_URL=postgresql://asephs:hunterz@100.108.1.124:5432/hub
|
|
||||||
```
|
|
||||||
|
|
||||||
Expected: Environment variable set and accessible to rust app.
|
|
||||||
|
|
||||||
- [ ] **Step 4: Rebuild rust app**
|
|
||||||
|
|
||||||
```bash
|
|
||||||
cd apps/rust
|
|
||||||
cargo build --release
|
|
||||||
```
|
|
||||||
|
|
||||||
Expected: Build completes without errors. Compilation uses sqlx-postgres feature.
|
|
||||||
|
|
||||||
- [ ] **Step 5: Test rust connection**
|
|
||||||
|
|
||||||
```bash
|
|
||||||
cd apps/rust
|
|
||||||
cargo run
|
|
||||||
```
|
|
||||||
|
|
||||||
Expected: App starts without connection errors. Check logs for "Database schema initialization complete" or similar.
|
|
||||||
|
|
||||||
- [ ] **Step 6: Commit rust changes**
|
|
||||||
|
|
||||||
```bash
|
|
||||||
cd /mnt/code/bp3/ultimate-asepharyana.tech
|
|
||||||
git add apps/rust/Cargo.toml apps/rust/src/infra/db_setup.rs
|
|
||||||
git commit -m "feat(rust): migrate database driver from MySQL to PostgreSQL"
|
|
||||||
```
|
|
||||||
|
|
||||||
Expected: Commit created with message.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
### Task 6: Smoke Tests - Elysia App
|
|
||||||
|
|
||||||
**Files:**
|
|
||||||
- Test: Manual testing via HTTP requests or app UI
|
|
||||||
|
|
||||||
- [ ] **Step 1: Start elysia app**
|
|
||||||
|
|
||||||
```bash
|
|
||||||
cd apps/elysia
|
|
||||||
bun run src/index.ts
|
|
||||||
```
|
|
||||||
|
|
||||||
Expected: App running on configured port (check logs for port).
|
|
||||||
|
|
||||||
- [ ] **Step 2: Test user login**
|
|
||||||
|
|
||||||
```bash
|
|
||||||
curl -X POST http://localhost:3000/auth/login \
|
|
||||||
-H "Content-Type: application/json" \
|
|
||||||
-d '{"email":"test@example.com","password":"password"}'
|
|
||||||
```
|
|
||||||
|
|
||||||
Expected: Response 200 or 401 (auth error is OK, connection error is not).
|
|
||||||
|
|
||||||
-[]**Step 3: Test user creation (if endpoint exists)**
|
|
||||||
|
|
||||||
```bash
|
|
||||||
curl -X POST http://localhost:3000/users \
|
|
||||||
-H "Content-Type: application/json" \
|
|
||||||
-d '{"name":"Test User","email":"newuser@example.com"}'
|
|
||||||
```
|
|
||||||
|
|
||||||
Expected: Response 200/201 or 400 (validation error is OK).
|
|
||||||
|
|
||||||
- [ ] **Step 4: Test session retrieval**
|
|
||||||
|
|
||||||
```bash
|
|
||||||
curl -X GET http://localhost:3000/sessions \
|
|
||||||
-H "Authorization: Bearer <token>"
|
|
||||||
```
|
|
||||||
|
|
||||||
Expected: Response 200 with session data or 401 (auth error is OK).
|
|
||||||
|
|
||||||
- [ ] **Step 5: Check database logs**
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# In elysia app logs, verify queries are executing against PostgreSQL
|
|
||||||
# Look for connection strings or query logs showing PostgreSQL
|
|
||||||
```
|
|
||||||
|
|
||||||
Expected: Logs show PostgreSQL queries (not MySQL).
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
### Task 7: Smoke Tests - Rust App
|
|
||||||
|
|
||||||
**Files:**
|
|
||||||
- Test: Manual testing via HTTP requests or app UI
|
|
||||||
|
|
||||||
- [ ] **Step 1: Start rust app**
|
|
||||||
|
|
||||||
```bash
|
|
||||||
cd apps/rust
|
|
||||||
cargo run --release
|
|
||||||
```
|
|
||||||
|
|
||||||
Expected: App running on configured port (check logs for port).
|
|
||||||
|
|
||||||
- [ ] **Step 2: Test image cache endpoint (if exists)**
|
|
||||||
|
|
||||||
```bash
|
|
||||||
curl -X GET http://localhost:8000/api/cache/status
|
|
||||||
```
|
|
||||||
|
|
||||||
Expected: Response 200 with cache status or 404 (endpoint may not exist).
|
|
||||||
|
|
||||||
- [ ] **Step 3: Test scraping/CDN endpoint**
|
|
||||||
|
|
||||||
```bash
|
|
||||||
curl -X GET http://localhost:8000/api/health
|
|
||||||
```
|
|
||||||
|
|
||||||
Expected: Response 200 with health status.
|
|
||||||
|
|
||||||
- [ ] **Step 4: Check database logs**
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# In rust app logs, verify queries are executing against PostgreSQL
|
|
||||||
# Look for connection strings or query logs showing PostgreSQL
|
|
||||||
```
|
|
||||||
|
|
||||||
Expected: Logs show PostgreSQL queries (not MySQL).
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
### Task 8: Data Integrity Validation
|
|
||||||
|
|
||||||
**Files:**
|
|
||||||
- Test: PostgreSQL queries
|
|
||||||
|
|
||||||
- [] **Step 1: Verify user count**
|
|
||||||
|
|
||||||
```bash
|
|
||||||
psql postgresql://asephs:hunterz@100.108.1.124:5432/hub -c "SELECT COUNT(*) as user_count FROM \"User\";"
|
|
||||||
```
|
|
||||||
|
|
||||||
Expected: Count matches MySQL backup count (from Task 1, Step 2).
|
|
||||||
|
|
||||||
- [ ] **Step 2: Verify account count**
|
|
||||||
|
|
||||||
```bash
|
|
||||||
psql postgresql://asephs:hunterz@100.108.1.124:5432/hub -c "SELECT COUNT(*) as account_count FROM \"Account\";"
|
|
||||||
```
|
|
||||||
|
|
||||||
Expected: Count matches MySQL backup.
|
|
||||||
|
|
||||||
- [ ] **Step 3: Verify session count**
|
|
||||||
|
|
||||||
```bash
|
|
||||||
psql postgresql://asephs:hunterz@100.108.1.124:5432/hub -c "SELECT COUNT(*) as session_count FROM \"Session\";"
|
|
||||||
```
|
|
||||||
|
|
||||||
Expected: Count matches MySQL backup.
|
|
||||||
|
|
||||||
- [] **Step 4: Verify no orphaned foreign keys**
|
|
||||||
|
|
||||||
```bash
|
|
||||||
psql postgresql://asephs:hunterz@100.108.1.124:5432/hub -c "
|
|
||||||
SELECT a.id FROM \"Account\" a
|
|
||||||
LEFT JOIN \"User\" u ON a.user_id = u.id
|
|
||||||
WHERE u.id IS NULL;
|
|
||||||
"
|
|
||||||
```
|
|
||||||
|
|
||||||
Expected: No rows returned (no orphaned accounts).
|
|
||||||
|
|
||||||
- [ ] **Step 5: Verify role/permission relationships**
|
|
||||||
|
|
||||||
```bash
|
|
||||||
psql postgresql://asephs:hunterz@100.108.1.124:5432/hub -c "SELECT COUNT(*) as role_count FROM \"Role\";"
|
|
||||||
psql postgresql://asephs:hunterz@100.108.1.124:5432/hub -c "SELECT COUNT(*) as permission_count FROM \"Permission\";"
|
|
||||||
```
|
|
||||||
|
|
||||||
Expected: Counts match MySQL backup.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
### Task 9: Performance Baseline
|
|
||||||
|
|
||||||
**Files:**
|
|
||||||
- Test: Query performance comparison
|
|
||||||
|
|
||||||
- [ ] **Step 1: Benchmark user query on PostgreSQL**
|
|
||||||
|
|
||||||
```bash
|
|
||||||
psql postgresql://asephs:hunterz@100.108.1.124:5432/hub -c "EXPLAIN ANALYZE SELECT * FROM \"User\" WHERE email = 'test@example.com';"
|
|
||||||
```
|
|
||||||
|
|
||||||
Expected: Query plan shows index usage (Seq Scan or Index Scan). Note execution time.
|
|
||||||
|
|
||||||
- [ ] **Step 2: Benchmark account query on PostgreSQL**
|
|
||||||
|
|
||||||
```bash
|
|
||||||
psql postgresql://asephs:hunterz@100.108.1.124:5432/hub -c "EXPLAIN ANALYZE SELECT * FROM \"Account\" WHERE user_id = 'user-123';"
|
|
||||||
```
|
|
||||||
|
|
||||||
Expected: Query plan shows index usage. Note execution time.
|
|
||||||
|
|
||||||
- [ ] **Step 3: Compare with MySQL baseline (if available)**
|
|
||||||
|
|
||||||
If MySQL is still running, run same queries and compare execution times.
|
|
||||||
|
|
||||||
Expected: PostgreSQL performance similar or better than MySQL.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
### Task 10: Cleanup and Documentation
|
|
||||||
|
|
||||||
**Files:**
|
|
||||||
- Create: `MIGRATION_LOG.md` (optional, for documentation)
|
|
||||||
|
|
||||||
- [ ] **Step 1: Remove temporary files**
|
|
||||||
|
|
||||||
```bash
|
|
||||||
rm mysql_backup.sql converted.sql
|
|
||||||
```
|
|
||||||
|
|
||||||
Expected: Temporary migration files deleted.
|
|
||||||
|
|
||||||
- [ ] **Step 2: Document migration completion**
|
|
||||||
|
|
||||||
Create `MIGRATION_LOG.md`:
|
|
||||||
|
|
||||||
```markdown
|
|
||||||
# PostgreSQL Migration Log
|
|
||||||
|
|
||||||
**Date:** 2026-05-25
|
|
||||||
**Status:** ✅ Complete
|
|
||||||
|
|
||||||
## Summary
|
|
||||||
- Migrated elysia app from MySQL to PostgreSQL
|
|
||||||
- Migrated rust app from MySQL to PostgreSQL
|
|
||||||
- All data validated and integrity confirmed
|
|
||||||
- Apps tested and operational
|
|
||||||
|
|
||||||
## Changes
|
|
||||||
- elysia: Updated database driver (mysql2 → postgres), schema (mysqlTable → pgTable)
|
|
||||||
- rust: Updated Cargo.toml feature (sqlx-mysql → sqlx-postgres), db_setup.rs backend handling
|
|
||||||
|
|
||||||
## Validation
|
|
||||||
- Row counts match pre-migration
|
|
||||||
- Foreign keys intact
|
|
||||||
- Indexes present and performant
|
|
||||||
- Auth flow functional
|
|
||||||
- Session management working
|
|
||||||
|
|
||||||
## Rollback
|
|
||||||
MySQL backup available at: [location if kept]
|
|
||||||
To rollback: Restore MySQL from backup, revert connection strings, redeploy apps.
|
|
||||||
```
|
|
||||||
|
|
||||||
- [ ] **Step 3: Final commit**
|
|
||||||
|
|
||||||
```bash
|
|
||||||
git add MIGRATION_LOG.md
|
|
||||||
git commit -m "docs: add PostgreSQL migration completion log"
|
|
||||||
```
|
|
||||||
|
|
||||||
Expected: Commit created.
|
|
||||||
|
|
||||||
- [ ] **Step 4: Verify all apps running**
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# Check elysia
|
|
||||||
curl http://localhost:3000/health
|
|
||||||
|
|
||||||
# Check rust
|
|
||||||
curl http://localhost:8000/health
|
|
||||||
```
|
|
||||||
|
|
||||||
Expected: Both apps respond with 200 status.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Self-Review
|
|
||||||
|
|
||||||
**Spec Coverage:**
|
|
||||||
- ✅ Pre-migration (backup, convert, test) — Tasks 1-3
|
|
||||||
- ✅ Elysia code updates (driver, schema, env) — Task 4
|
|
||||||
- ✅ Rust code updates (Cargo.toml, db_setup.rs, env) — Task 5
|
|
||||||
- ✅ Smoke tests (auth, endpoints, logs) — Tasks 6-7
|
|
||||||
- ✅ Data validation (row counts, foreign keys) — Task 8
|
|
||||||
- ✅ Performance baseline — Task 9
|
|
||||||
- ✅ Cleanup and documentation — Task 10
|
|
||||||
|
|
||||||
**Placeholder Scan:**
|
|
||||||
- ✅ No TBD/TODO
|
|
||||||
- ✅ All code blocks complete
|
|
||||||
- ✅ All commands exact with expected output
|
|
||||||
- ✅ All file paths exact
|
|
||||||
|
|
||||||
**Type Consistency:**
|
|
||||||
- ✅ Database type: `PostgresJsDatabase` (elysia), `DbBackend::Postgres` (rust)
|
|
||||||
- ✅ Connection string format consistent: `postgresql://asephs:hunterz@100.108.1.124:5432/hub`
|
|
||||||
- ✅ Table names consistent: "User", "Account", "Session", etc.
|
|
||||||
@@ -1,533 +0,0 @@
|
|||||||
# Production GitHub Actions Implementation Plan
|
|
||||||
|
|
||||||
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
|
||||||
|
|
||||||
**Goal:** Rework GitHub Actions build/deploy workflows into a production baseline with reliable React submodule updates, least-privilege permissions, clear deploy behavior, and current official action versions.
|
|
||||||
|
|
||||||
**Architecture:** Keep two workflows: `docker-build-push.yml` for detect/build/manifest updates, and `deploy-docker.yml` for VPS deployment. Add dispatch submodule SHA readiness checks before parent pointer updates, and remove recursive submodule checkout from deploy runner.
|
|
||||||
|
|
||||||
**Tech Stack:** GitHub Actions YAML, GitHub-hosted Ubuntu runners, Docker Buildx, GHCR, git submodules, Docker Compose over SSH.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## File Structure
|
|
||||||
|
|
||||||
- Modify `.github/workflows/docker-build-push.yml`: add default permissions, validate repository dispatch payloads, wait for submodule SHAs, keep selective matrix builds, harden manifest update.
|
|
||||||
- Modify `.github/workflows/deploy-docker.yml`: add default permissions, remove recursive checkout, keep auto deploy from successful build, make deploy logs clearer.
|
|
||||||
- Modify `.github/dependabot.yml`: add GitHub Actions update config so official actions stay current.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
### Task 1: Harden build workflow permissions and dispatch validation
|
|
||||||
|
|
||||||
**Files:**
|
|
||||||
- Modify: `.github/workflows/docker-build-push.yml`
|
|
||||||
|
|
||||||
- [ ] **Step 1: Add workflow-level read permissions**
|
|
||||||
|
|
||||||
At top level, after `concurrency`, add:
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
permissions:
|
|
||||||
contents: read
|
|
||||||
```
|
|
||||||
|
|
||||||
Expected shape:
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
concurrency:
|
|
||||||
group: ${{ github.workflow }}-${{ github.ref }}
|
|
||||||
cancel-in-progress: false
|
|
||||||
|
|
||||||
permissions:
|
|
||||||
contents: read
|
|
||||||
|
|
||||||
env:
|
|
||||||
REGISTRY: ghcr.io
|
|
||||||
```
|
|
||||||
|
|
||||||
- [ ] **Step 2: Replace dispatch parser with payload validation**
|
|
||||||
|
|
||||||
In `.github/workflows/docker-build-push.yml`, replace `Parse repository_dispatch payload` step body with:
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
- name: Parse repository_dispatch payload
|
|
||||||
id: dispatch
|
|
||||||
if: github.event_name == 'repository_dispatch'
|
|
||||||
env:
|
|
||||||
SERVICE: ${{ github.event.client_payload.service }}
|
|
||||||
SHA: ${{ github.event.client_payload.sha }}
|
|
||||||
run: |
|
|
||||||
set -euo pipefail
|
|
||||||
|
|
||||||
if [ -z "${SERVICE:-}" ]; then
|
|
||||||
echo "::error::repository_dispatch payload missing service"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
if [ -z "${SHA:-}" ]; then
|
|
||||||
echo "::error::repository_dispatch payload missing sha"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
case "$SERVICE" in
|
|
||||||
rust-api|elysia-api|react-web|9router) ;;
|
|
||||||
*)
|
|
||||||
echo "::error::Unsupported service '$SERVICE'. Expected one of: rust-api, elysia-api, react-web, 9router"
|
|
||||||
exit 1
|
|
||||||
;;
|
|
||||||
esac
|
|
||||||
|
|
||||||
case "$SHA" in
|
|
||||||
*[!0-9a-fA-F]*|???????????????????????????????????????|?????????????????????????????????????????*)
|
|
||||||
echo "::error::Invalid sha '$SHA'. Expected 40 hex characters"
|
|
||||||
exit 1
|
|
||||||
;;
|
|
||||||
esac
|
|
||||||
|
|
||||||
declare -a SERVICES=("rust-api" "elysia-api" "react-web" "9router")
|
|
||||||
for svc in "${SERVICES[@]}"; do
|
|
||||||
if [ "$SERVICE" = "$svc" ]; then
|
|
||||||
echo "${svc}=true" >> "$GITHUB_OUTPUT"
|
|
||||||
else
|
|
||||||
echo "${svc}=false" >> "$GITHUB_OUTPUT"
|
|
||||||
fi
|
|
||||||
done
|
|
||||||
```
|
|
||||||
|
|
||||||
- [ ] **Step 3: Run YAML syntax check**
|
|
||||||
|
|
||||||
Run:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
python - <<'PY'
|
|
||||||
from pathlib import Path
|
|
||||||
import yaml
|
|
||||||
for path in Path('.github/workflows').glob('*.yml'):
|
|
||||||
yaml.safe_load(path.read_text())
|
|
||||||
print(f'OK {path}')
|
|
||||||
PY
|
|
||||||
```
|
|
||||||
|
|
||||||
Expected:
|
|
||||||
|
|
||||||
```text
|
|
||||||
OK .github/workflows/deploy-docker.yml
|
|
||||||
OK .github/workflows/docker-build-push.yml
|
|
||||||
```
|
|
||||||
|
|
||||||
- [ ] **Step 4: Commit**
|
|
||||||
|
|
||||||
```bash
|
|
||||||
git add .github/workflows/docker-build-push.yml
|
|
||||||
git commit -m "ci: validate dispatch payloads"
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
### Task 2: Add submodule SHA readiness wait before build/update
|
|
||||||
|
|
||||||
**Files:**
|
|
||||||
- Modify: `.github/workflows/docker-build-push.yml`
|
|
||||||
|
|
||||||
- [ ] **Step 1: Add readiness job after changes job**
|
|
||||||
|
|
||||||
Insert this job between `changes` and `build`:
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
wait-submodule-ref:
|
|
||||||
needs: [changes]
|
|
||||||
if: github.event_name == 'repository_dispatch'
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
steps:
|
|
||||||
- name: Wait for submodule ref
|
|
||||||
env:
|
|
||||||
SERVICE: ${{ github.event.client_payload.service }}
|
|
||||||
SHA: ${{ github.event.client_payload.sha }}
|
|
||||||
run: |
|
|
||||||
set -euo pipefail
|
|
||||||
|
|
||||||
case "$SERVICE" in
|
|
||||||
"rust-api") REPO="https://github.com/MythEclipse/ultimate-asepharyana-tech-rust.git" ;;
|
|
||||||
"elysia-api") REPO="https://github.com/MythEclipse/ultimate-asepharyana-tech-elysia.git" ;;
|
|
||||||
"react-web") REPO="https://github.com/MythEclipse/ultimate-asepharyana-tech-react.git" ;;
|
|
||||||
"9router") REPO="https://github.com/MythEclipse/9router.git" ;;
|
|
||||||
*)
|
|
||||||
echo "::error::Unsupported service '$SERVICE'"
|
|
||||||
exit 1
|
|
||||||
;;
|
|
||||||
esac
|
|
||||||
|
|
||||||
echo "Waiting for $SERVICE ref $SHA in $REPO"
|
|
||||||
for attempt in {1..30}; do
|
|
||||||
if git ls-remote --exit-code "$REPO" "$SHA" >/dev/null 2>&1; then
|
|
||||||
echo "Submodule ref $SHA is fetchable for $SERVICE"
|
|
||||||
exit 0
|
|
||||||
fi
|
|
||||||
echo "Attempt $attempt/30: $SHA not visible yet; waiting 10s"
|
|
||||||
sleep 10
|
|
||||||
done
|
|
||||||
|
|
||||||
echo "::error::Submodule ref $SHA for $SERVICE was not fetchable after 300s"
|
|
||||||
exit 1
|
|
||||||
```
|
|
||||||
|
|
||||||
- [ ] **Step 2: Make build wait for readiness job without blocking push/manual events**
|
|
||||||
|
|
||||||
Change build job header from:
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
build:
|
|
||||||
needs: [changes]
|
|
||||||
```
|
|
||||||
|
|
||||||
to:
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
build:
|
|
||||||
needs: [changes, wait-submodule-ref]
|
|
||||||
if: |
|
|
||||||
always() &&
|
|
||||||
needs.changes.result == 'success' &&
|
|
||||||
(needs.wait-submodule-ref.result == 'success' || needs.wait-submodule-ref.result == 'skipped') &&
|
|
||||||
needs.changes.outputs.matrix != '[]'
|
|
||||||
```
|
|
||||||
|
|
||||||
Remove existing build-level line:
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
if: needs.changes.outputs.matrix != '[]'
|
|
||||||
```
|
|
||||||
|
|
||||||
- [ ] **Step 3: Make update-manifest wait for readiness job**
|
|
||||||
|
|
||||||
Change update-manifest header from:
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
update-manifest:
|
|
||||||
needs: [changes, build]
|
|
||||||
if: |
|
|
||||||
always() &&
|
|
||||||
(needs.build.result == 'success' || needs.build.result == 'skipped')
|
|
||||||
```
|
|
||||||
|
|
||||||
to:
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
update-manifest:
|
|
||||||
needs: [changes, wait-submodule-ref, build]
|
|
||||||
if: |
|
|
||||||
always() &&
|
|
||||||
needs.changes.result == 'success' &&
|
|
||||||
(needs.wait-submodule-ref.result == 'success' || needs.wait-submodule-ref.result == 'skipped') &&
|
|
||||||
(needs.build.result == 'success' || needs.build.result == 'skipped')
|
|
||||||
```
|
|
||||||
|
|
||||||
- [ ] **Step 4: Run YAML syntax check**
|
|
||||||
|
|
||||||
Run same command from Task 1 Step 3.
|
|
||||||
|
|
||||||
Expected both workflow files print `OK`.
|
|
||||||
|
|
||||||
- [ ] **Step 5: Commit**
|
|
||||||
|
|
||||||
```bash
|
|
||||||
git add .github/workflows/docker-build-push.yml
|
|
||||||
git commit -m "ci: wait for submodule refs before builds"
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
### Task 3: Harden manifest update and submodule checkout
|
|
||||||
|
|
||||||
**Files:**
|
|
||||||
- Modify: `.github/workflows/docker-build-push.yml`
|
|
||||||
|
|
||||||
- [ ] **Step 1: Add job permissions to build and manifest jobs**
|
|
||||||
|
|
||||||
Ensure build job contains:
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
permissions:
|
|
||||||
contents: read
|
|
||||||
packages: write
|
|
||||||
```
|
|
||||||
|
|
||||||
Ensure update-manifest job contains:
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
permissions:
|
|
||||||
contents: write
|
|
||||||
```
|
|
||||||
|
|
||||||
- [ ] **Step 2: Replace dispatch submodule checkout block**
|
|
||||||
|
|
||||||
Inside `Update tags and submodules`, replace the repository_dispatch submodule update block with:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
if [ "${{ github.event_name }}" == "repository_dispatch" ] && [ "${{ github.event.client_payload.service }}" == "$id" ]; then
|
|
||||||
SHA_DISPATCH="${{ github.event.client_payload.sha }}"
|
|
||||||
SUB_PATH="${PATHS[$id]}"
|
|
||||||
if [ -n "$SHA_DISPATCH" ]; then
|
|
||||||
echo "Updating submodule $SUB_PATH to $SHA_DISPATCH"
|
|
||||||
git submodule update --init "$SUB_PATH"
|
|
||||||
git -C "$SUB_PATH" fetch origin "$SHA_DISPATCH"
|
|
||||||
git -C "$SUB_PATH" checkout "$SHA_DISPATCH"
|
|
||||||
git add "$SUB_PATH"
|
|
||||||
CHANGED=true
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
```
|
|
||||||
|
|
||||||
- [ ] **Step 3: Add pull/rebase retry before push**
|
|
||||||
|
|
||||||
Replace:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
git commit -m "chore: update manifests and submodules [skip ci]"
|
|
||||||
git pull --rebase origin main
|
|
||||||
git push origin main
|
|
||||||
```
|
|
||||||
|
|
||||||
with:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
git commit -m "chore: update manifests and submodules [skip ci]"
|
|
||||||
|
|
||||||
for attempt in {1..3}; do
|
|
||||||
if git pull --rebase origin main && git push origin main; then
|
|
||||||
exit 0
|
|
||||||
fi
|
|
||||||
echo "Manifest push attempt $attempt/3 failed; retrying"
|
|
||||||
git rebase --abort || true
|
|
||||||
git pull --rebase origin main || true
|
|
||||||
sleep 5
|
|
||||||
done
|
|
||||||
|
|
||||||
echo "::error::Failed to push manifest update after 3 attempts"
|
|
||||||
exit 1
|
|
||||||
```
|
|
||||||
|
|
||||||
- [ ] **Step 4: Run YAML syntax check**
|
|
||||||
|
|
||||||
Run same command from Task 1 Step 3.
|
|
||||||
|
|
||||||
Expected both workflow files print `OK`.
|
|
||||||
|
|
||||||
- [ ] **Step 5: Commit**
|
|
||||||
|
|
||||||
```bash
|
|
||||||
git add .github/workflows/docker-build-push.yml
|
|
||||||
git commit -m "ci: harden manifest updates"
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
### Task 4: Make deploy checkout submodule-free and least privilege
|
|
||||||
|
|
||||||
**Files:**
|
|
||||||
- Modify: `.github/workflows/deploy-docker.yml`
|
|
||||||
|
|
||||||
- [ ] **Step 1: Add workflow-level read permissions**
|
|
||||||
|
|
||||||
After `concurrency`, add:
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
permissions:
|
|
||||||
contents: read
|
|
||||||
```
|
|
||||||
|
|
||||||
Expected shape:
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
concurrency:
|
|
||||||
group: deploy-vps
|
|
||||||
cancel-in-progress: false
|
|
||||||
|
|
||||||
permissions:
|
|
||||||
contents: read
|
|
||||||
```
|
|
||||||
|
|
||||||
If current `cancel-in-progress` is `true`, change it to `false`.
|
|
||||||
|
|
||||||
- [ ] **Step 2: Make checkout non-recursive**
|
|
||||||
|
|
||||||
Replace checkout step:
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
- name: Checkout repository
|
|
||||||
uses: actions/checkout@v4
|
|
||||||
with:
|
|
||||||
submodules: recursive
|
|
||||||
```
|
|
||||||
|
|
||||||
with:
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
- name: Checkout repository
|
|
||||||
uses: actions/checkout@v4
|
|
||||||
with:
|
|
||||||
fetch-depth: 1
|
|
||||||
submodules: false
|
|
||||||
```
|
|
||||||
|
|
||||||
- [ ] **Step 3: Add deploy context log**
|
|
||||||
|
|
||||||
At start of `Deploy with Docker Compose on VPS` run script, after `set -euo pipefail`, add:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
echo "Deploy event: ${{ github.event_name }}"
|
|
||||||
echo "Deploy ref: ${{ github.ref }}"
|
|
||||||
echo "Deploy sha: ${{ github.sha }}"
|
|
||||||
```
|
|
||||||
|
|
||||||
- [ ] **Step 4: Run YAML syntax check**
|
|
||||||
|
|
||||||
Run same command from Task 1 Step 3.
|
|
||||||
|
|
||||||
Expected both workflow files print `OK`.
|
|
||||||
|
|
||||||
- [ ] **Step 5: Commit**
|
|
||||||
|
|
||||||
```bash
|
|
||||||
git add .github/workflows/deploy-docker.yml
|
|
||||||
git commit -m "ci: avoid submodule checkout during deploy"
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
### Task 5: Add GitHub Actions Dependabot updates
|
|
||||||
|
|
||||||
**Files:**
|
|
||||||
- Modify: `.github/dependabot.yml`
|
|
||||||
|
|
||||||
- [ ] **Step 1: Add github-actions ecosystem**
|
|
||||||
|
|
||||||
Append this update entry under `updates:`:
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
- package-ecosystem: 'github-actions'
|
|
||||||
directory: '/'
|
|
||||||
schedule:
|
|
||||||
interval: weekly
|
|
||||||
groups:
|
|
||||||
github-actions:
|
|
||||||
patterns:
|
|
||||||
- '*'
|
|
||||||
```
|
|
||||||
|
|
||||||
Expected file shape:
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
version: 2
|
|
||||||
updates:
|
|
||||||
- package-ecosystem: 'devcontainers'
|
|
||||||
directory: '/'
|
|
||||||
schedule:
|
|
||||||
interval: weekly
|
|
||||||
|
|
||||||
- package-ecosystem: 'github-actions'
|
|
||||||
directory: '/'
|
|
||||||
schedule:
|
|
||||||
interval: weekly
|
|
||||||
groups:
|
|
||||||
github-actions:
|
|
||||||
patterns:
|
|
||||||
- '*'
|
|
||||||
```
|
|
||||||
|
|
||||||
- [ ] **Step 2: Run YAML syntax check**
|
|
||||||
|
|
||||||
Run:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
python - <<'PY'
|
|
||||||
from pathlib import Path
|
|
||||||
import yaml
|
|
||||||
paths = [Path('.github/dependabot.yml'), *Path('.github/workflows').glob('*.yml')]
|
|
||||||
for path in paths:
|
|
||||||
yaml.safe_load(path.read_text())
|
|
||||||
print(f'OK {path}')
|
|
||||||
PY
|
|
||||||
```
|
|
||||||
|
|
||||||
Expected:
|
|
||||||
|
|
||||||
```text
|
|
||||||
OK .github/dependabot.yml
|
|
||||||
OK .github/workflows/deploy-docker.yml
|
|
||||||
OK .github/workflows/docker-build-push.yml
|
|
||||||
```
|
|
||||||
|
|
||||||
- [ ] **Step 3: Commit**
|
|
||||||
|
|
||||||
```bash
|
|
||||||
git add .github/dependabot.yml
|
|
||||||
git commit -m "ci: enable github actions dependency updates"
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
### Task 6: Final validation
|
|
||||||
|
|
||||||
**Files:**
|
|
||||||
- Validate: `.github/workflows/docker-build-push.yml`
|
|
||||||
- Validate: `.github/workflows/deploy-docker.yml`
|
|
||||||
- Validate: `.github/dependabot.yml`
|
|
||||||
|
|
||||||
- [ ] **Step 1: Run YAML syntax check**
|
|
||||||
|
|
||||||
Run:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
python - <<'PY'
|
|
||||||
from pathlib import Path
|
|
||||||
import yaml
|
|
||||||
paths = [Path('.github/dependabot.yml'), *Path('.github/workflows').glob('*.yml')]
|
|
||||||
for path in paths:
|
|
||||||
yaml.safe_load(path.read_text())
|
|
||||||
print(f'OK {path}')
|
|
||||||
PY
|
|
||||||
```
|
|
||||||
|
|
||||||
Expected all files print `OK`.
|
|
||||||
|
|
||||||
- [ ] **Step 2: Check workflows recognized by GitHub CLI**
|
|
||||||
|
|
||||||
Run:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
gh workflow list
|
|
||||||
```
|
|
||||||
|
|
||||||
Expected output includes:
|
|
||||||
|
|
||||||
```text
|
|
||||||
Build and Push Docker Images
|
|
||||||
Deploy Docker to VPS
|
|
||||||
```
|
|
||||||
|
|
||||||
- [ ] **Step 3: Inspect final diff**
|
|
||||||
|
|
||||||
Run:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
git diff -- .github/workflows .github/dependabot.yml
|
|
||||||
```
|
|
||||||
|
|
||||||
Expected:
|
|
||||||
|
|
||||||
- `docker-build-push.yml` has dispatch validation, `wait-submodule-ref`, job permissions, and manifest push retry.
|
|
||||||
- `deploy-docker.yml` has non-recursive checkout and read-only permissions.
|
|
||||||
- `dependabot.yml` has `github-actions` updates.
|
|
||||||
|
|
||||||
- [ ] **Step 4: Commit any final validation fixes**
|
|
||||||
|
|
||||||
If Step 1 or Step 2 required fixes, commit them:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
git add .github/workflows .github/dependabot.yml
|
|
||||||
git commit -m "ci: finalize production workflow hardening"
|
|
||||||
```
|
|
||||||
|
|
||||||
If no fixes were needed, do not create an empty commit.
|
|
||||||
@@ -1,29 +0,0 @@
|
|||||||
# React Direct Runtime and Images Implementation Plan
|
|
||||||
|
|
||||||
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
|
||||||
|
|
||||||
**Goal:** Serve the React SPA without nginx and remove frontend image-cache proxy calls so browser uses direct image/API URLs.
|
|
||||||
|
|
||||||
**Architecture:** The React Docker image still builds with Bun/Vite, but runtime uses `vite preview` from Bun instead of nginx. `CachedImage` normalizes image URLs and renders them directly, keeping fallback/retry UI but removing `/proxy/image-cache` auditing. Traefik keeps routing `react-web:80`, so compose and dynamic routing stay stable.
|
|
||||||
|
|
||||||
**Tech Stack:** Bun, Vite, React, TypeScript, Docker, Docker Compose, Traefik.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Tasks
|
|
||||||
|
|
||||||
### Task 1: Switch React Runtime From nginx to Bun/Vite Preview
|
|
||||||
|
|
||||||
Modify `infra/docker/react.Dockerfile` so runtime stage uses `oven/bun:1-alpine`, installs production deps, copies `/app/dist`, exposes 80, and runs `bunx vite preview --host 0.0.0.0 --port 80`. Verify nginx runtime/copy lines are gone and Vite preview command exists.
|
|
||||||
|
|
||||||
### Task 2: Remove Image Cache Proxy From CachedImage
|
|
||||||
|
|
||||||
Modify `apps/react/src/components/ui/cached-image.tsx` to remove `API_BASE_URL` import, `/proxy/image-cache` POST, audit state/function, and auditing overlay. Keep direct normalized image URLs, retry behavior, and fallback image.
|
|
||||||
|
|
||||||
### Task 3: Verify Build and Config
|
|
||||||
|
|
||||||
Run static checks for both tasks, `bun --cwd apps/react run build`, `docker build -f infra/docker/react.Dockerfile -t react-web:test .`, start test container on `18080:80`, curl root, clean container, and run `git diff --check`.
|
|
||||||
|
|
||||||
### Task 4: Commit and Push
|
|
||||||
|
|
||||||
Commit changed files: `infra/docker/react.Dockerfile`, `apps/react/src/components/ui/cached-image.tsx`, `docs/superpowers/plans/2026-05-27-react-direct-runtime-and-images.md`. Push branch only after verification if user asks; do not push from worktree unless explicitly instructed.
|
|
||||||
@@ -1,130 +0,0 @@
|
|||||||
# Moonrepo Migration Design
|
|
||||||
|
|
||||||
**Date:** 2026-05-24
|
|
||||||
**Status:** approved
|
|
||||||
|
|
||||||
## Context
|
|
||||||
|
|
||||||
Ultimate Asepharyana Tech is a polyglot monorepo with 7 applications managed as git submodules: nextjs (React/Next.js), elysia (Bun/Elysia), solidjs (SolidStart), rust (Axum), rust-auth (Axum auth service), leptos (Leptos WASM), and 9router. Nix flakes handle system dependencies and Docker image builds. Docker Compose manages deployment.
|
|
||||||
|
|
||||||
## Problem
|
|
||||||
|
|
||||||
- No centralized task orchestration — running build/lint/test across all apps is manual and error-prone
|
|
||||||
- No dependency caching — builds are not incremental across apps
|
|
||||||
- Inconsistent developer experience — each app has its own conventions, onboarding is slow
|
|
||||||
|
|
||||||
## Goal
|
|
||||||
|
|
||||||
Adopt moonrepo for task orchestration, caching, and dependency graph management across all 7 apps while preserving the existing Nix build system, Docker Compose deployment, and git submodule structure.
|
|
||||||
|
|
||||||
## Architecture
|
|
||||||
|
|
||||||
### Directory Structure
|
|
||||||
|
|
||||||
```
|
|
||||||
ultimate-asepharyana.tech/
|
|
||||||
├── .moon/
|
|
||||||
│ ├── workspace.yml
|
|
||||||
│ ├── toolchain.yml
|
|
||||||
│ └── tasks/
|
|
||||||
│ ├── typescript-build.yml
|
|
||||||
│ ├── typescript-lint.yml
|
|
||||||
│ ├── typescript-test.yml
|
|
||||||
│ ├── rust-build.yml
|
|
||||||
│ ├── rust-test.yml
|
|
||||||
│ └── rust-lint.yml
|
|
||||||
├── apps/
|
|
||||||
│ ├── nextjs/moon.yml
|
|
||||||
│ ├── elysia/moon.yml
|
|
||||||
│ ├── solidjs/moon.yml
|
|
||||||
│ ├── rust/moon.yml
|
|
||||||
│ ├── rust-auth/moon.yml
|
|
||||||
│ ├── leptos/moon.yml
|
|
||||||
│ └── 9router/moon.yml
|
|
||||||
├── .moon/workspace.yml # unchanged — Nix build + deployment
|
|
||||||
├── flake.nix # unchanged
|
|
||||||
├── infra/ # unchanged — Docker Compose, Traefik, nginx
|
|
||||||
└── scripts/ # unchanged
|
|
||||||
```
|
|
||||||
|
|
||||||
### Tag Taxonomy
|
|
||||||
|
|
||||||
| Tag | Projects |
|
|
||||||
|-----|----------|
|
|
||||||
| `lang:typescript` | nextjs, elysia, solidjs |
|
|
||||||
| `lang:rust` | rust, rust-auth, leptos |
|
|
||||||
| `type:frontend` | nextjs, solidjs, leptos |
|
|
||||||
| `type:backend` | rust, elysia, rust-auth |
|
|
||||||
| `type:router` | 9router |
|
|
||||||
|
|
||||||
### Toolchain
|
|
||||||
|
|
||||||
- moonrepo manages Node 22, Bun, and TypeScript versions via `.moon/toolchain.yml` for consistent access across all TypeScript apps
|
|
||||||
- Rust toolchain remains managed by Nix/devShell — moonrepo does not touch Rust installation
|
|
||||||
- Nix devShell gains the moon CLI binary
|
|
||||||
|
|
||||||
### Project Dependencies
|
|
||||||
|
|
||||||
- **nextjs** depends on `rust-auth` and `elysia` (frontend consumes both APIs)
|
|
||||||
- **solidjs** depends on `elysia` and `rust-auth`
|
|
||||||
- **leptos** depends on `rust` (CSR frontend consumes rust backend API)
|
|
||||||
- **elysia**, **rust**, **rust-auth**, **9router** — no project dependencies
|
|
||||||
|
|
||||||
## Task Definitions
|
|
||||||
|
|
||||||
### Shared Tasks: TypeScript
|
|
||||||
|
|
||||||
All TypeScript apps inherit from `.moon/tasks/typescript-*.yml`:
|
|
||||||
|
|
||||||
| Task | Command | Inputs | Outputs |
|
|
||||||
|------|---------|--------|---------|
|
|
||||||
| `build` | `bun run build` | `src/**/*`, `tsconfig.json`, `package.json` | `.next`, `dist`, `.output` |
|
|
||||||
| `lint` | `bun run lint` | `src/**/*`, `eslint.config.mjs`, `tsconfig.json` | — |
|
|
||||||
| `typecheck` | `bun run check-types` | `src/**/*`, `tsconfig.json` | — |
|
|
||||||
| `test` | `bun test` | `src/**/*`, `test/**/*` | — |
|
|
||||||
| `e2e` | overridden per app | — | — |
|
|
||||||
|
|
||||||
### Shared Tasks: Rust
|
|
||||||
|
|
||||||
All Rust apps inherit from `.moon/tasks/rust-*.yml`, using system tasks:
|
|
||||||
|
|
||||||
| Task | Command | Inputs | Outputs |
|
|
||||||
|------|---------|--------|---------|
|
|
||||||
| `build` | `cargo build --release` | `src/**/*`, `Cargo.toml`, `Cargo.lock`, `build.rs` | `target/release/*` |
|
|
||||||
| `test` | `cargo test` | `src/**/*`, `Cargo.toml`, `Cargo.lock`, `tests/**/*` | — |
|
|
||||||
| `lint` | `cargo clippy -- -D warnings` | `src/**/*`, `Cargo.toml` | — |
|
|
||||||
| `fmt-check` | `cargo fmt --check` | `src/**/*` | — |
|
|
||||||
|
|
||||||
### Implicit Dependencies
|
|
||||||
|
|
||||||
Configured in `workspace.yml`: `lint` implicitly depends on `build` for TypeScript apps (typecheck path), and `build` propagates to dependents when source inputs change.
|
|
||||||
|
|
||||||
## Key Commands
|
|
||||||
|
|
||||||
```bash
|
|
||||||
moon run :build # build all changed apps + dependents
|
|
||||||
moon run :lint # lint all
|
|
||||||
moon run :test # test all
|
|
||||||
moon run --tag lang:rust :test # test Rust apps only
|
|
||||||
moon run --affected :build # build only what changed
|
|
||||||
moon query projects --tag lang:typescript # list TypeScript projects
|
|
||||||
```
|
|
||||||
|
|
||||||
## What Does NOT Change
|
|
||||||
|
|
||||||
- **Nix flakes** — system dependencies, Docker image builds, devShell remain as-is
|
|
||||||
- **Docker Compose** — deployment continues through existing compose files in `infra/compose/`
|
|
||||||
- **Git submodules** — all 7 app repos stay as submodules under `apps/`
|
|
||||||
- **infra/** — Traefik and Docker Compose deployment remain under `infra/`; legacy Grafana/Prometheus/Alertmanager configs have been removed
|
|
||||||
- **scripts/** — existing utility scripts unchanged
|
|
||||||
|
|
||||||
## Migration Steps (High-Level)
|
|
||||||
|
|
||||||
1. Install moonrepo CLI, create `.moon/` scaffolding
|
|
||||||
2. Create shared task definitions in `.moon/tasks/`
|
|
||||||
3. Configure `workspace.yml` with project list, tags, and dependency constraints
|
|
||||||
4. Configure `toolchain.yml` for Node 22 + Bun
|
|
||||||
5. Add `moon.yml` to each of the 7 app directories
|
|
||||||
6. Validate: `moon check`, `moon run :build`, `moon run :lint`, `moon run :test`
|
|
||||||
7. Add moon CLI to Nix devShell
|
|
||||||
8. Commit and document
|
|
||||||
@@ -1,197 +0,0 @@
|
|||||||
# PostgreSQL Migration Design
|
|
||||||
**Date:** 2026-05-25
|
|
||||||
**Scope:** Migrate elysia (Drizzle ORM + MySQL) and rust (SeaORM + MySQL) apps to PostgreSQL
|
|
||||||
**Approach:** Direct cutover (Approach B)
|
|
||||||
**Connection String:** `postgresql://asephs:hunterz@100.108.1.124:5432/hub`
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Current State
|
|
||||||
|
|
||||||
### Elysia App
|
|
||||||
- **ORM:** Drizzle ORM v0.45.2
|
|
||||||
- **Driver:** mysql2 v3.20.0
|
|
||||||
- **Schema Location:** `apps/elysia/src/db/lib/schema.ts`
|
|
||||||
- **Tables:** users, accounts, sessions, roles, permissions, userRoles, and related junction tables
|
|
||||||
- **Database File:** `apps/elysia/src/db/lib/database.ts`
|
|
||||||
|
|
||||||
### Rust App
|
|
||||||
- **ORM:** SeaORM v1.1.19
|
|
||||||
- **Feature:** sqlx-mysql
|
|
||||||
- **DB Setup:** `apps/rust/src/infra/db_setup.rs`
|
|
||||||
- **Tables:** ImageCache (primary table managed by app)
|
|
||||||
- **Config:** Environment-based connection string
|
|
||||||
|
|
||||||
### Apps NOT Migrating
|
|
||||||
- 9router (uses SQLite runtime, excluded per requirements)
|
|
||||||
- nextjs (frontend, minimal DB usage)
|
|
||||||
- leptos, solidjs, rust-auth (frontends, no DB)
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Migration Steps
|
|
||||||
|
|
||||||
### Phase 1: Pre-Migration (Preparation)
|
|
||||||
1. **Backup MySQL**
|
|
||||||
```bash
|
|
||||||
mysqldump -u <user> -p <db> > mysql_backup.sql
|
|
||||||
```
|
|
||||||
2. **Convert MySQL dump to PostgreSQL**
|
|
||||||
- Use `pgloader` or manual conversion for schema compatibility
|
|
||||||
- Handle type conversions:
|
|
||||||
- `INT` → `INTEGER`
|
|
||||||
- `VARCHAR(n)` → `VARCHAR(n)` (PostgreSQL compatible)
|
|
||||||
- `DATETIME` → `TIMESTAMP`
|
|
||||||
- `AUTO_INCREMENT` → `SERIAL` or `BIGSERIAL`
|
|
||||||
- Verify indexes and foreign keys convert correctly
|
|
||||||
3. **Test import into target PostgreSQL**
|
|
||||||
```bash
|
|
||||||
psql postgresql://asephs:hunterz@100.108.1.124:5432/hub < converted.sql
|
|
||||||
```
|
|
||||||
4. **Validate data integrity**
|
|
||||||
- Row counts match MySQL
|
|
||||||
- Indexes exist
|
|
||||||
- Foreign key constraints enforced
|
|
||||||
|
|
||||||
### Phase 2: Code Updates
|
|
||||||
|
|
||||||
#### Elysia App
|
|
||||||
1. **Update `apps/elysia/src/db/lib/database.ts`**
|
|
||||||
- Replace `mysql2` import with `postgres` driver
|
|
||||||
- Change Drizzle dialect from `drizzle-orm/mysql2` to `drizzle-orm/postgres`
|
|
||||||
- Update connection string format
|
|
||||||
|
|
||||||
2. **Update `apps/elysia/src/db/lib/schema.ts`**
|
|
||||||
- Replace `drizzle-orm/mysql-core` imports with `drizzle-orm/postgres-core`
|
|
||||||
- Change `mysqlTable` to `pgTable`
|
|
||||||
- Adjust column types if needed (e.g., `int` → `integer`)
|
|
||||||
|
|
||||||
3. **Update `apps/elysia/package.json`**
|
|
||||||
- Replace `mysql2` with `pg` (PostgreSQL driver)
|
|
||||||
- Keep `drizzle-orm` and `drizzle-kit` versions
|
|
||||||
|
|
||||||
4. **Update environment/config**
|
|
||||||
- Change `DATABASE_URL` to PostgreSQL connection string
|
|
||||||
|
|
||||||
#### Rust App
|
|
||||||
1. **Update `apps/rust/Cargo.toml`**
|
|
||||||
- Replace `sqlx-mysql` feature with `sqlx-postgres` in sea-orm dependency
|
|
||||||
- Add `postgres` feature if needed
|
|
||||||
|
|
||||||
2. **Update `apps/rust/src/infra/db_setup.rs`**
|
|
||||||
- Change `DbBackend::MySql` check to `DbBackend::Postgres`
|
|
||||||
- Adjust SQL syntax for PostgreSQL (e.g., index creation)
|
|
||||||
- Update error code handling (PostgreSQL uses different error codes)
|
|
||||||
|
|
||||||
3. **Update config/environment**
|
|
||||||
- Change `DATABASE_URL` to PostgreSQL connection string
|
|
||||||
|
|
||||||
### Phase 3: Cutover (Execution)
|
|
||||||
1. **Stop all apps**
|
|
||||||
```bash
|
|
||||||
# Stop elysia
|
|
||||||
# Stop rust app
|
|
||||||
```
|
|
||||||
2. **Export MySQL data**
|
|
||||||
```bash
|
|
||||||
mysqldump -u <user> -p <db> > final_backup.sql
|
|
||||||
```
|
|
||||||
3. **Convert and import to PostgreSQL**
|
|
||||||
```bash
|
|
||||||
# Convert dump
|
|
||||||
# Import to PostgreSQL
|
|
||||||
psql postgresql://asephs:hunterz@100.108.1.124:5432/hub < converted.sql
|
|
||||||
```
|
|
||||||
4. **Deploy updated apps**
|
|
||||||
- Deploy elysia with PostgreSQL driver
|
|
||||||
- Deploy rust with PostgreSQL feature
|
|
||||||
5. **Verify connectivity**
|
|
||||||
- Test database queries from both apps
|
|
||||||
- Check auth flow (users table)
|
|
||||||
- Verify session management
|
|
||||||
|
|
||||||
### Phase 4: Validation
|
|
||||||
1. **Smoke tests**
|
|
||||||
- User login/logout
|
|
||||||
- Session creation and retrieval
|
|
||||||
- Role/permission queries
|
|
||||||
- ImageCache operations (rust app)
|
|
||||||
2. **Data integrity checks**
|
|
||||||
- Row counts match pre-migration
|
|
||||||
- No orphaned foreign keys
|
|
||||||
- Indexes performing as expected
|
|
||||||
3. **Performance baseline**
|
|
||||||
- Compare query times MySQL vs PostgreSQL
|
|
||||||
- Monitor connection pool usage
|
|
||||||
|
|
||||||
### Phase 5: Rollback Plan (if needed)
|
|
||||||
1. **Stop apps**
|
|
||||||
2. **Restore MySQL from backup**
|
|
||||||
```bash
|
|
||||||
mysql -u <user> -p <db> < mysql_backup.sql
|
|
||||||
```
|
|
||||||
3. **Revert connection strings in apps**
|
|
||||||
4. **Redeploy with MySQL drivers**
|
|
||||||
5. **Restart apps**
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Technical Details
|
|
||||||
|
|
||||||
### Elysia Driver Change
|
|
||||||
**Before:**
|
|
||||||
```typescript
|
|
||||||
import { drizzle } from 'drizzle-orm/mysql2'
|
|
||||||
import { createPool } from 'mysql2/promise'
|
|
||||||
```
|
|
||||||
|
|
||||||
**After:**
|
|
||||||
```typescript
|
|
||||||
import { drizzle } from 'drizzle-orm/postgres-js'
|
|
||||||
import postgres from 'postgres'
|
|
||||||
```
|
|
||||||
|
|
||||||
### Rust Feature Change
|
|
||||||
**Before:**
|
|
||||||
```toml
|
|
||||||
sea-orm = { version = "1.1.19", features = ["sqlx-mysql", ...] }
|
|
||||||
```
|
|
||||||
|
|
||||||
**After:**
|
|
||||||
```toml
|
|
||||||
sea-orm = { version = "1.1.19", features = ["sqlx-postgres", ...] }
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Risk Assessment
|
|
||||||
|
|
||||||
| Risk | Mitigation |
|
|
||||||
|------|-----------|
|
|
||||||
| Data loss during migration | Full MySQL backup before cutover; test import first |
|
|
||||||
| App downtime | Cutover during low-traffic window; rollback plan ready |
|
|
||||||
| Connection string misconfiguration | Test connection before deploying apps |
|
|
||||||
| Schema incompatibilities | Pre-test conversion; validate indexes/constraints |
|
|
||||||
| Performance regression | Baseline MySQL performance; monitor PostgreSQL after cutover |
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Success Criteria
|
|
||||||
|
|
||||||
- ✅ All data migrated to PostgreSQL (row counts match)
|
|
||||||
- ✅ Elysia app connects and queries work
|
|
||||||
- ✅ Rust app connects and queries work
|
|
||||||
- ✅ Auth flow functional (login/logout)
|
|
||||||
- ✅ No orphaned foreign keys
|
|
||||||
- ✅ Indexes present and performant
|
|
||||||
- ✅ Rollback plan tested and documented
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Timeline
|
|
||||||
|
|
||||||
- **Pre-migration:** 30 min (backup, convert, test)
|
|
||||||
- **Code updates:** 1-2 hours (driver changes, testing)
|
|
||||||
- **Cutover:** 15-30 min (stop apps, migrate data, restart)
|
|
||||||
- **Validation:** 30 min (smoke tests, data checks)
|
|
||||||
- **Total:** ~3-4 hours (including buffer)
|
|
||||||
@@ -1,21 +0,0 @@
|
|||||||
# Hapus Nix, Docker-Only
|
|
||||||
|
|
||||||
**Goal:** Remove all Nix configuration and adapt CI/dev to Docker-only.
|
|
||||||
|
|
||||||
**Scope:**
|
|
||||||
- Delete: `flake.nix`, `flake.lock`, `nix/`, `.envrc`, `.direnv/`
|
|
||||||
- Modify: `.github/workflows/docker-build-push.yml` — remove Nix steps, move `rust-api` to Dockerfile build
|
|
||||||
- Modify: `.vscode/settings.json` — remove `nixEnvSelector`
|
|
||||||
- Modify: `README.md` — replace `nix build` with `docker build`
|
|
||||||
|
|
||||||
**CI Changes:**
|
|
||||||
- `rust-api` uses `infra/docker/rust.Dockerfile` (already exists)
|
|
||||||
- Remove `cachix/install-nix-action`, `cachix/cachix-action`, `nix build` step
|
|
||||||
- Remove `flake.nix`, `flake.lock`, `nix/**` from path triggers
|
|
||||||
- Update path filters: `nix/apps/*.nix` → `infra/docker/*.Dockerfile`
|
|
||||||
- Remove `repository_dispatch` Nix `--override-input` logic
|
|
||||||
- Add `rust-api` to Dockerfile case + push branch
|
|
||||||
|
|
||||||
**Dev Workflow:**
|
|
||||||
- Rust toolchain via `rustup`/`rust-toolchain.toml` in `apps/rust/`
|
|
||||||
- Dev via `docker compose` instead of `process-compose`
|
|
||||||
@@ -1,100 +0,0 @@
|
|||||||
# Production GitHub Actions redesign
|
|
||||||
|
|
||||||
## Goal
|
|
||||||
|
|
||||||
Rework `.github/workflows` into a production baseline that balances reliability, security, speed, and clear failures while keeping automatic deploys from `main`.
|
|
||||||
|
|
||||||
## Current problems
|
|
||||||
|
|
||||||
- Deploy runner checks out submodules recursively even though deploy work happens on the VPS. Fresh submodule SHAs can fail with `not our ref` before deploy starts.
|
|
||||||
- Repository dispatch can update a parent submodule pointer before the submodule SHA is fetchable by GitHub Actions.
|
|
||||||
- Build and deploy permissions are broader than needed at workflow level.
|
|
||||||
- Failure logs do not clearly separate build, manifest update, submodule readiness, and deployment phases.
|
|
||||||
|
|
||||||
## Chosen approach
|
|
||||||
|
|
||||||
Use a split pipeline:
|
|
||||||
|
|
||||||
1. `docker-build-push.yml` remains the build and manifest update pipeline.
|
|
||||||
2. `deploy-docker.yml` remains deploy-only.
|
|
||||||
3. Repository dispatch waits for the requested submodule SHA to be fetchable before building and updating the parent pointer.
|
|
||||||
4. Deploy checkout no longer uses recursive submodules; the VPS updates submodules after resetting to `origin/main`.
|
|
||||||
|
|
||||||
## Build workflow design
|
|
||||||
|
|
||||||
Triggers:
|
|
||||||
|
|
||||||
- `push` to `main` for app/package/docker/workflow changes.
|
|
||||||
- `repository_dispatch` with `service` and `sha` payload.
|
|
||||||
- `workflow_dispatch` for manual full builds.
|
|
||||||
|
|
||||||
Jobs:
|
|
||||||
|
|
||||||
- `changes`: detects the service matrix and validates dispatch payloads.
|
|
||||||
- `build`: builds and pushes only selected service images using Docker Buildx registry cache.
|
|
||||||
- `update-manifest`: updates compose image tags and, for dispatch events, updates the matching submodule pointer.
|
|
||||||
|
|
||||||
Repository dispatch handling:
|
|
||||||
|
|
||||||
- For dispatch events, wait until `git ls-remote` or equivalent fetch confirms the payload SHA exists in the service submodule remote.
|
|
||||||
- Retry for a bounded timeout and fail with an explicit message if the SHA never becomes visible.
|
|
||||||
- Only after readiness is confirmed, checkout the SHA in the submodule and commit the parent pointer update.
|
|
||||||
|
|
||||||
## Deploy workflow design
|
|
||||||
|
|
||||||
Triggers:
|
|
||||||
|
|
||||||
- `workflow_run` from successful `Build and Push Docker Images` on `main`.
|
|
||||||
- `push` to `main` for `infra/compose/**` and deploy workflow changes.
|
|
||||||
- `workflow_dispatch`.
|
|
||||||
|
|
||||||
Behavior:
|
|
||||||
|
|
||||||
- Keep automatic deploy from `main`.
|
|
||||||
- Keep a single deploy concurrency group.
|
|
||||||
- Use non-recursive checkout on the runner.
|
|
||||||
- On the VPS, fetch/reset `origin/main`, update submodules, compute changed compose stacks, pull images, and run Docker Compose.
|
|
||||||
|
|
||||||
## Permissions and action trust
|
|
||||||
|
|
||||||
Defaults:
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
permissions:
|
|
||||||
contents: read
|
|
||||||
```
|
|
||||||
|
|
||||||
Job-specific permissions:
|
|
||||||
|
|
||||||
- Build job: `contents: read`, `packages: write`.
|
|
||||||
- Manifest update job: `contents: write`.
|
|
||||||
- Deploy job: `contents: read`.
|
|
||||||
|
|
||||||
Action pinning:
|
|
||||||
|
|
||||||
- GitHub-owned and Docker official actions may use major versions such as `actions/checkout@v4` and `docker/build-push-action@v6`.
|
|
||||||
- Any future third-party action should be pinned to a full commit SHA.
|
|
||||||
|
|
||||||
## Reliability details
|
|
||||||
|
|
||||||
- Keep `set -euo pipefail` in shell steps.
|
|
||||||
- Add bounded retry around submodule SHA readiness.
|
|
||||||
- Add clear logs for selected services, image tags, compose files changed, and dispatch payload values.
|
|
||||||
- Avoid recursive submodule checkout in deploy to eliminate fresh-SHA checkout race.
|
|
||||||
- Manifest commits continue using `[skip ci]` to prevent build loops.
|
|
||||||
|
|
||||||
## Speed details
|
|
||||||
|
|
||||||
- Keep selective matrix builds.
|
|
||||||
- Keep Docker Buildx registry cache.
|
|
||||||
- Keep deploy checkout shallow and submodule-free.
|
|
||||||
- Avoid rebuilding from compose-only manifest commits.
|
|
||||||
|
|
||||||
## Validation
|
|
||||||
|
|
||||||
Before marking implementation complete:
|
|
||||||
|
|
||||||
- Validate workflow YAML syntax.
|
|
||||||
- Run `gh workflow list` or equivalent sanity checks.
|
|
||||||
- Verify build workflow still detects React updates.
|
|
||||||
- Verify deploy workflow no longer fails during runner checkout for fresh submodule SHAs.
|
|
||||||
@@ -1,135 +0,0 @@
|
|||||||
# Monorepo Restructure: ultimate-asepharyana.tech → asepharyana-hub
|
|
||||||
|
|
||||||
**Date:** 2026-07-09
|
|
||||||
**Status:** Approved
|
|
||||||
**Owner:** @asepharyana
|
|
||||||
|
|
||||||
## Background
|
|
||||||
|
|
||||||
Proyek ini sebelumnya bernama `ultimate-asepharyana.tech` — sebuah monorepo yang berisi beberapa service aplikasi, infrastruktur, dokumentasi, dan utility scripts. Karena GitHub org `MythEclipse` kena banned, semua remote repos tidak bisa diakses. Perlu dilakukan restruktur dan rename project secara menyeluruh.
|
|
||||||
|
|
||||||
## Goals
|
|
||||||
|
|
||||||
1. Rename project dari `ultimate-asepharyana.tech` ke `asepharyana-hub`
|
|
||||||
2. Hapus service docker-manager dan teleuploader dari proyek
|
|
||||||
3. Hapus semua pointer `.git` submodule (clean slate)
|
|
||||||
4. Update `.gitmodules` dengan remote baru ke `github.com/asepharyana/*`
|
|
||||||
5. Update semua referensi: workflows, konfigurasi, dokumentasi, image names
|
|
||||||
6. Siapkan struktur lokal — inisialisasi git dan push dilakukan terpisah
|
|
||||||
|
|
||||||
## Service yang tetap dipertahankan
|
|
||||||
|
|
||||||
| Service | Path | Git remote baru |
|
|
||||||
|---------|------|----------------|
|
|
||||||
| Elysia API | `apps/elysia` | `asepharyana/asepharyana-hub-elysia` |
|
|
||||||
| React Frontend | `apps/react` | `asepharyana/asepharyana-hub-react` |
|
|
||||||
| Scraper | `apps/scraper` | `asepharyana/asepharyana-hub-scraper` |
|
|
||||||
| Rust Auth | `apps/rust-auth` | `asepharyana/asepharyana-hub-rust-auth` |
|
|
||||||
|
|
||||||
## Service yang dihapus
|
|
||||||
|
|
||||||
| Service | Path |
|
|
||||||
|---------|------|
|
|
||||||
| Docker Manager | `apps/docker-manager/` |
|
|
||||||
| TeleUploader | `apps/teleuploader/` |
|
|
||||||
|
|
||||||
## File yang akan dihapus
|
|
||||||
|
|
||||||
- `apps/docker-manager/` (seluruh direktori)
|
|
||||||
- `apps/teleuploader/` (seluruh direktori)
|
|
||||||
- `infra/compose/docker-manager.yml`
|
|
||||||
- `infra/compose/teleuploader.yml`
|
|
||||||
- `infra/docker/docker-manager.Dockerfile`
|
|
||||||
- `infra/docker/teleuploader.Dockerfile`
|
|
||||||
|
|
||||||
## File pointer `.git` submodule yang dihapus
|
|
||||||
|
|
||||||
- `apps/elysia/.git`
|
|
||||||
- `apps/react/.git`
|
|
||||||
- `apps/scraper/.git`
|
|
||||||
- `apps/rust-auth/.git`
|
|
||||||
|
|
||||||
## Rename mapping
|
|
||||||
|
|
||||||
| Lokasi | Dari | Ke |
|
|
||||||
|--------|------|----|
|
|
||||||
| `package.json` `name` | `ultimate-asepharyana.tech` | `asepharyana-hub` |
|
|
||||||
| `README.md` | judul & path references | `asepharyana-hub` |
|
|
||||||
| `ARCHITECTURE.md` | directory tree, paths | `asepharyana-hub` |
|
|
||||||
| `CONTRIBUTING.md` | repo names & paths | `asepharyana-hub-*` |
|
|
||||||
| `infra/README.md` | deskripsi | `asepharyana-hub` |
|
|
||||||
| `.env.example` | `TRAEFIK_CONFIG_PATH` | `asepharyana-hub` |
|
|
||||||
| Perlengkapan infra Traefik | path config references | `asepharyana-hub` |
|
|
||||||
| `scripts/cleanup-ghcr.sh` | `MythEclipse` | `asepharyana` |
|
|
||||||
| `docs/add-new-app.md` | path references | `asepharyana-hub` |
|
|
||||||
| `docs/adr/*.md` | path references | `asepharyana-hub` |
|
|
||||||
|
|
||||||
## Image name migration (GHCR)
|
|
||||||
|
|
||||||
| Lama | Baru |
|
|
||||||
|------|------|
|
|
||||||
| `ghcr.io/mytheclipse/ultimate-asepharyana.tech/elysia-api:*` | `ghcr.io/asepharyana/asepharyana-hub/elysia-api:*` |
|
|
||||||
| `ghcr.io/mytheclipse/ultimate-asepharyana.tech/react-web:*` | `ghcr.io/asepharyana/asepharyana-hub/react-web:*` |
|
|
||||||
| `ghcr.io/mytheclipse/ultimate-asepharyana.tech/scraper-api:*` | `ghcr.io/asepharyana/asepharyana-hub/scraper-api:*` |
|
|
||||||
| `ghcr.io/mytheclipse/ultimate-asepharyana.tech/rust-auth:*` | `ghcr.io/asepharyana/asepharyana-hub/rust-auth:*` |
|
|
||||||
| `ghcr.io/mytheclipse/ultimate-asepharyana.tech/docker-manager:*` | — (dihapus) |
|
|
||||||
| `ghcr.io/mytheclipse/ultimate-asepharyana.tech/teleuploader:*` | — (dihapus) |
|
|
||||||
|
|
||||||
## Workflow changes
|
|
||||||
|
|
||||||
### `docker-build-push.yml`
|
|
||||||
- `IMAGE_NAME_PREFIX`: `mytheclipse/ultimate-asepharyana.tech` → `asepharyana/asepharyana-hub`
|
|
||||||
- Hapus service entries: `docker-manager`, `teleuploader`
|
|
||||||
- Update repo URLs di `wait-submodule-ref` dari `MythEclipse/*` ke `asepharyana/*`
|
|
||||||
- Update `update-manifest` phase — hapus service docker-manager & teleuploader
|
|
||||||
|
|
||||||
### `deploy-docker.yml`
|
|
||||||
- Update `git remote add origin`
|
|
||||||
- Hapus docker-manager & teleuploader dari compose list dan checkout
|
|
||||||
|
|
||||||
### `update-submodule.yml`
|
|
||||||
- Hapus service docker-manager & teleuploader
|
|
||||||
- Update nama workflow
|
|
||||||
|
|
||||||
## `.gitmodules`
|
|
||||||
|
|
||||||
Hanya berisi 4 apps dengan remote baru:
|
|
||||||
|
|
||||||
```ini
|
|
||||||
[submodule "apps/elysia"]
|
|
||||||
path = apps/elysia
|
|
||||||
url = https://github.com/asepharyana/asepharyana-hub-elysia.git
|
|
||||||
[submodule "apps/react"]
|
|
||||||
path = apps/react
|
|
||||||
url = https://github.com/asepharyana/asepharyana-hub-react.git
|
|
||||||
[submodule "apps/scraper"]
|
|
||||||
path = apps/scraper
|
|
||||||
url = https://github.com/asepharyana/asepharyana-hub-scraper.git
|
|
||||||
[submodule "apps/rust-auth"]
|
|
||||||
path = apps/rust-auth
|
|
||||||
url = https://github.com/asepharyana/asepharyana-hub-rust-auth.git
|
|
||||||
```
|
|
||||||
|
|
||||||
## Execution plan
|
|
||||||
|
|
||||||
1. Hapus docker-manager & teleuploader direktori + file infra
|
|
||||||
2. Hapus `.git` pointer di submodule
|
|
||||||
3. Update `.gitmodules`
|
|
||||||
4. Update `package.json`
|
|
||||||
5. Update `README.md`, `ARCHITECTURE.md`, `CONTRIBUTING.md`
|
|
||||||
6. Update `infra/compose/*.yml` image tags
|
|
||||||
7. Update `.env.example`, `infra/README.md`, docs traefik
|
|
||||||
8. Update `docker-build-push.yml`
|
|
||||||
9. Update `deploy-docker.yml`
|
|
||||||
10. Update `update-submodule.yml`
|
|
||||||
11. Update `scripts/cleanup-ghcr.sh`
|
|
||||||
12. Update `docs/add-new-app.md`
|
|
||||||
13. Update `docs/adr/*.md` path references (historical)
|
|
||||||
|
|
||||||
## Post-execution state
|
|
||||||
|
|
||||||
- Root direktori `asepharyana-hub/` dengan source code apps utuh (tanpa git)
|
|
||||||
- 4 app submodule terdaftar di `.gitmodules` dengan remote baru
|
|
||||||
- Infra/docs/scripts tetap menyatu di root
|
|
||||||
- 0 references ke `MythEclipse/ultimate-asepharyana.tech` di file konfigurasi
|
|
||||||
- Siap untuk `git init && git add && git commit` kapan saja
|
|
||||||
@@ -0,0 +1,196 @@
|
|||||||
|
# Tailscale Networking
|
||||||
|
|
||||||
|
Dokumentasi setup dan troubleshooting konektivitas Tailscale antara node `orangevps` (VPS) dan `imrnes` (bare-metal).
|
||||||
|
|
||||||
|
## Topologi
|
||||||
|
|
||||||
|
```
|
||||||
|
orangevps (VPS)
|
||||||
|
├─ Tailscale IP: 100.x.x.x (dynamic)
|
||||||
|
├─ Public IP: 45.127.35.244
|
||||||
|
├─ Docker containers (app-shared-net)
|
||||||
|
│ └─ perlu akses ke imrnes via Tailscale
|
||||||
|
└─ tailscale-routes.service
|
||||||
|
└─ menambahkan route 100.x.x.x ke tabel routing main
|
||||||
|
|
||||||
|
imrnes (Bare-metal)
|
||||||
|
├─ Tailscale IP: 100.121.180.82
|
||||||
|
├─ Layanan:
|
||||||
|
│ ├─ PostgreSQL (port 6432)
|
||||||
|
│ └─ Redis (port 6379)
|
||||||
|
└─ Layanan hanya listen di Tailscale interface
|
||||||
|
```
|
||||||
|
|
||||||
|
## Masalah: Container Tidak Bisa Mencapai Tailscale IP
|
||||||
|
|
||||||
|
Docker container secara default hanya bisa mencapai IP di Docker bridge network dan network host. Tailscale menggunakan interface virtual `tailscale0` yang tidak secara otomatis di-route ke container.
|
||||||
|
|
||||||
|
### Solusi: `tailscale-routes.service`
|
||||||
|
|
||||||
|
Service systemd yang menambahkan route Tailscale ke tabel routing `main` agar traffic dari container bisa melewati host ke Tailscale.
|
||||||
|
|
||||||
|
```ini
|
||||||
|
# /etc/systemd/system/tailscale-routes.service
|
||||||
|
[Unit]
|
||||||
|
Description=Add Tailscale routes to main routing table
|
||||||
|
After=tailscaled.service
|
||||||
|
Requires=tailscaled.service
|
||||||
|
|
||||||
|
[Service]
|
||||||
|
Type=oneshot
|
||||||
|
RemainAfterExit=yes
|
||||||
|
ExecStart=/bin/sh -c 'ip rule add from all lookup main priority 10000 2>/dev/null; ip route add 100.64.0.0/10 dev tailscale0 table main 2>/dev/null || true'
|
||||||
|
ExecStop=/bin/sh -c 'ip rule del from all lookup main priority 10000 2>/dev/null; ip route del 100.64.0.0/10 dev tailscale0 table main 2>/dev/null || true'
|
||||||
|
RemainAfterExit=yes
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=multi-user.target
|
||||||
|
```
|
||||||
|
|
||||||
|
### Verifikasi
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Cek apakah route sudah ada
|
||||||
|
ip route show table main | grep tailscale
|
||||||
|
|
||||||
|
# Test dari dalam container
|
||||||
|
docker run --rm alpine ping -c 3 100.121.180.82
|
||||||
|
|
||||||
|
# Test koneksi PostgreSQL dari container
|
||||||
|
docker run --rm alpine sh -c "apk add postgresql-client && psql -h 100.121.180.82 -p 6432 -U asephs -d hub -c 'SELECT 1'"
|
||||||
|
```
|
||||||
|
|
||||||
|
## Setup Tailscale di Node Baru
|
||||||
|
|
||||||
|
### 1. Install Tailscale
|
||||||
|
|
||||||
|
```bash
|
||||||
|
curl -fsSL https://tailscale.com/install.sh | sh
|
||||||
|
```
|
||||||
|
|
||||||
|
### 2. Authenticate
|
||||||
|
|
||||||
|
```bash
|
||||||
|
sudo tailscale up --advertise-routes=<LAN_SUBNET_CIDR>
|
||||||
|
```
|
||||||
|
|
||||||
|
Untuk node yang hanya sebagai client (tidak advertise routes):
|
||||||
|
|
||||||
|
```bash
|
||||||
|
sudo tailscale up
|
||||||
|
```
|
||||||
|
|
||||||
|
### 3. Enable dan Start
|
||||||
|
|
||||||
|
```bash
|
||||||
|
sudo systemctl enable --now tailscaled
|
||||||
|
```
|
||||||
|
|
||||||
|
### 4. Setup Route Service (khusus node dengan Docker)
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Buat service file
|
||||||
|
sudo nano /etc/systemd/system/tailscale-routes.service
|
||||||
|
# Paste content di atas
|
||||||
|
|
||||||
|
sudo systemctl daemon-reload
|
||||||
|
sudo systemctl enable --now tailscale-routes.service
|
||||||
|
```
|
||||||
|
|
||||||
|
### 5. Konfigurasi ACL di Tailscale Admin
|
||||||
|
|
||||||
|
Pastikan ACL di [Tailscale Admin Console](https://login.tailscale.com/admin/acls) mengizinkan traffic antar node:
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"acls": [
|
||||||
|
{"action": "accept", "src": ["*"], "dst": ["*:*"]}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
Atau jika ingin lebih ketat:
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"acls": [
|
||||||
|
{"action": "accept", "src": ["tag:server"], "dst": ["tag:server:*"]}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
## Konfigurasi iptables/ufw
|
||||||
|
|
||||||
|
Pastikan port yang diperlukan terbuka di `imrnes`:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# PostgreSQL
|
||||||
|
sudo ufw allow in on tailscale0 to any port 6432 proto tcp
|
||||||
|
|
||||||
|
# Redis
|
||||||
|
sudo ufw allow in on tailscale0 to any port 6379 proto tcp
|
||||||
|
```
|
||||||
|
|
||||||
|
Atau menggunakan iptables langsung:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
sudo iptables -A INPUT -i tailscale0 -p tcp --dport 6432 -j ACCEPT
|
||||||
|
sudo iptables -A INPUT -i tailscale0 -p tcp --dport 6379 -j ACCEPT
|
||||||
|
```
|
||||||
|
|
||||||
|
## Troubleshooting
|
||||||
|
|
||||||
|
### Container timeout connect ke Tailscale IP
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# 1. Cek apakah route service berjalan
|
||||||
|
systemctl status tailscale-routes.service
|
||||||
|
|
||||||
|
# 2. Cek route di host
|
||||||
|
ip route show table main | grep 100.64
|
||||||
|
|
||||||
|
# 3. Cek apakah host bisa ping ke target
|
||||||
|
ping 100.121.180.82
|
||||||
|
|
||||||
|
# 4. Test dari container dengan --network host
|
||||||
|
docker run --rm --network host alpine ping -c 3 100.121.180.82
|
||||||
|
|
||||||
|
# 5. Pastikan tidak ada firewall blocking
|
||||||
|
iptables -L FORWARD -n -v
|
||||||
|
```
|
||||||
|
|
||||||
|
### Tailscale disconnect
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Cek status
|
||||||
|
tailscale status
|
||||||
|
|
||||||
|
# Restart
|
||||||
|
sudo systemctl restart tailscaled
|
||||||
|
```
|
||||||
|
|
||||||
|
### IP Tailscale berubah
|
||||||
|
|
||||||
|
Tailscale IP bisa berubah jika node dire-auth. Update:
|
||||||
|
|
||||||
|
1. `.env` production di VPS (via GitHub secret `ENV_FILE_PRODUCTION`)
|
||||||
|
2. Database connection strings
|
||||||
|
3. Redis connection strings
|
||||||
|
4. Trigger redeploy
|
||||||
|
|
||||||
|
### MagicDNS tidak resolve
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Cek DNS
|
||||||
|
tailscale dns status
|
||||||
|
|
||||||
|
# Flush DNS cache
|
||||||
|
sudo resolvectl flush-caches
|
||||||
|
```
|
||||||
|
|
||||||
|
## Catatan Keamanan
|
||||||
|
|
||||||
|
- Interface Tailscale (`tailscale0`) hanya boleh diakses oleh node yang terautentikasi dalam network yang sama
|
||||||
|
- Jangan expose port database ke public interface (`eth0`), hanya ke Tailscale
|
||||||
|
- Gunakan ACL untuk membatasi akses antar node jika diperlukan
|
||||||
|
- Rotate auth key secara berkala di Tailscale admin console
|
||||||
@@ -0,0 +1,493 @@
|
|||||||
|
# Troubleshooting
|
||||||
|
|
||||||
|
Kumpulan solusi untuk masalah umum yang spesifik di infrastruktur `asepharyana-hub`.
|
||||||
|
|
||||||
|
## Daftar Isi
|
||||||
|
|
||||||
|
- [Deployment](#deployment)
|
||||||
|
- [Dapr](#dapr)
|
||||||
|
- [NATS](#nats)
|
||||||
|
- [Traefik](#traefik)
|
||||||
|
- [Tailscale / Networking](#tailscale--networking)
|
||||||
|
- [Docker / Container](#docker--container)
|
||||||
|
- [Database](#database)
|
||||||
|
- [Submodule](#submodule)
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Deployment
|
||||||
|
|
||||||
|
### Workflow deploy gagal: "Secrets not fully configured"
|
||||||
|
|
||||||
|
**Penyebab:** Salah satu GitHub secrets tidak diset.
|
||||||
|
|
||||||
|
**Solusi:** Cek secrets di Settings > Secrets and variables > Actions:
|
||||||
|
|
||||||
|
| Secret | Status |
|
||||||
|
|--------|--------|
|
||||||
|
| `SSH_PRIVATE_KEY` | Wajib |
|
||||||
|
| `VPS_HOST` | Wajib (`45.127.35.244`) |
|
||||||
|
| `VPS_USER` | Wajib (`root`) |
|
||||||
|
| `VPS_TARGET_DIR` | Wajib (`/root/asepharyana-hub`) |
|
||||||
|
| `ENV_FILE_PRODUCTION` | Wajib |
|
||||||
|
|
||||||
|
### Workflow build gagal: "Submodule commit not fetchable"
|
||||||
|
|
||||||
|
**Penyebab:** Commit SHA dari `repository_dispatch` belum tersedia di remote submodule repo (eventual consistency).
|
||||||
|
|
||||||
|
**Solusi:** Workflow akan retry hingga 5 menit. Jika masih gagal:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Cek apakah commit ada di remote
|
||||||
|
git ls-remote https://github.com/asepharyana/asepharyana-hub-scraper.git <SHA>
|
||||||
|
|
||||||
|
# Trigger ulang dispatch dari submodule repo, atau push langsung ke hub
|
||||||
|
```
|
||||||
|
|
||||||
|
### Push manifest gagal: conflict di main
|
||||||
|
|
||||||
|
**Penyebab:** Ada commit lain yang masuk sebelum workflow selesai.
|
||||||
|
|
||||||
|
**Solusi:** Workflow otomatis retry rebase 3 kali. Jika semua gagal:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Manual fix di lokal
|
||||||
|
git pull --rebase origin main
|
||||||
|
# resolve conflict
|
||||||
|
git push origin main
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Dapr
|
||||||
|
|
||||||
|
### Dapr sidecar tidak connect ke placement
|
||||||
|
|
||||||
|
**Gejala:** Container `scraper-api-dapr` restart loop. Log: `failed to connect to placement`
|
||||||
|
|
||||||
|
**Diagnosis:**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Cek log sidecar
|
||||||
|
docker logs scraper-api-dapr --tail 50
|
||||||
|
|
||||||
|
# Cek apakah placement service running
|
||||||
|
docker ps -a | grep dapr-placement
|
||||||
|
docker logs dapr-placement --tail 20
|
||||||
|
|
||||||
|
# Cek konektivitas
|
||||||
|
docker exec scraper-api-dapr curl -s http://dapr-placement:50005
|
||||||
|
```
|
||||||
|
|
||||||
|
**Solusi:**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Restart placement dulu, lalu sidecar
|
||||||
|
docker compose -f infra/compose/dapr.yml up -d --force-recreate
|
||||||
|
sleep 5
|
||||||
|
docker compose -f infra/compose/scraper.yml up -d --force-recreate scraper-api-dapr
|
||||||
|
```
|
||||||
|
|
||||||
|
### Dapr pub/sub tidak bekerja
|
||||||
|
|
||||||
|
**Gejala:** Event di-publish tapi tidak sampai ke subscriber.
|
||||||
|
|
||||||
|
**Diagnosis:**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Cek komponen Dapr
|
||||||
|
curl http://localhost:3500/v1.0/components
|
||||||
|
|
||||||
|
# Cek health sidecar
|
||||||
|
curl http://localhost:3500/v1.0/healthz
|
||||||
|
|
||||||
|
# Cek Redis (backend pub/sub)
|
||||||
|
docker exec redis redis-cli ping
|
||||||
|
```
|
||||||
|
|
||||||
|
**Solusi:**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Restart sidecar
|
||||||
|
docker restart scraper-api-dapr
|
||||||
|
|
||||||
|
# Jika Redis bermasalah, restart juga
|
||||||
|
docker restart redis
|
||||||
|
```
|
||||||
|
|
||||||
|
### Dapr state store error: "key not found"
|
||||||
|
|
||||||
|
**Penyebab:** Key belum ada di state store, atau prefix berbeda.
|
||||||
|
|
||||||
|
**Diagnosis:**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Cek state langsung di Redis
|
||||||
|
docker exec redis redis-cli KEYS 'dapr*'
|
||||||
|
|
||||||
|
# State store menggunakan prefix "dapr"
|
||||||
|
# Format key: dapr || <app-id> || <key>
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## NATS
|
||||||
|
|
||||||
|
### NATS tidak bisa start
|
||||||
|
|
||||||
|
**Gejala:** Container NATS restart loop.
|
||||||
|
|
||||||
|
**Diagnosis:**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker logs nats --tail 50
|
||||||
|
```
|
||||||
|
|
||||||
|
**Solusi:** Kemungkinan korupsi data JetStream:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Backup dulu volume data
|
||||||
|
docker run --rm -v nats_data:/data -v /tmp:/backup alpine cp -r /data /backup/nats_data_backup
|
||||||
|
|
||||||
|
# Hapus volume dan recreate
|
||||||
|
docker compose -f infra/compose/nats.yml down
|
||||||
|
docker volume rm asepharyana-hub_nats_data
|
||||||
|
docker compose -f infra/compose/nats.yml up -d
|
||||||
|
```
|
||||||
|
|
||||||
|
### JetStream stream overflow
|
||||||
|
|
||||||
|
**Gejala:** Disk penuh, NATS lambat.
|
||||||
|
|
||||||
|
**Diagnosis:**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Cek ukuran volume
|
||||||
|
docker system df | grep nats_data
|
||||||
|
du -sh /var/lib/docker/volumes/nats_data/_data/
|
||||||
|
|
||||||
|
# Cek stream info
|
||||||
|
nats stream list
|
||||||
|
nats stream info <stream-name>
|
||||||
|
```
|
||||||
|
|
||||||
|
**Solusi:**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Purge stream tertentu (data hilang)
|
||||||
|
nats stream purge <stream-name>
|
||||||
|
|
||||||
|
# Atau tambah limit stream via NATS config
|
||||||
|
```
|
||||||
|
|
||||||
|
### "Slow Consumer" warning
|
||||||
|
|
||||||
|
**Gejala:** Log NATS menampilkan "slow consumer".
|
||||||
|
|
||||||
|
**Diagnosis:**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
curl http://localhost:8222/varz | jq '.slow_consumers'
|
||||||
|
```
|
||||||
|
|
||||||
|
**Solusi:**
|
||||||
|
- Scale consumer (tambah worker)
|
||||||
|
- Percepat processing message
|
||||||
|
- Kurangi ukuran payload
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Traefik
|
||||||
|
|
||||||
|
### Traefik tidak routing ke service
|
||||||
|
|
||||||
|
**Gejala:** 404 atau 503 dari Traefik.
|
||||||
|
|
||||||
|
**Diagnosis:**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Cek apakah service container running
|
||||||
|
docker ps -a | grep scraper-api
|
||||||
|
|
||||||
|
# Cek log Traefik
|
||||||
|
docker logs traefik --tail 50
|
||||||
|
|
||||||
|
# Cek apakah container ada di network yang benar
|
||||||
|
docker network inspect app-shared-net | grep scraper-api
|
||||||
|
|
||||||
|
# Test routing langsung
|
||||||
|
curl -H "Host: scraper.asepharyana.my.id" http://localhost/
|
||||||
|
```
|
||||||
|
|
||||||
|
**Solusi:**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Pastikan service terdaftar di apps.yaml
|
||||||
|
# Pastikan container join app-shared-net
|
||||||
|
# Restart Traefik
|
||||||
|
docker compose -f infra/compose/traefik.yml up -d --force-recreate
|
||||||
|
```
|
||||||
|
|
||||||
|
### TLS certificate error
|
||||||
|
|
||||||
|
**Gejala:** Browser menampilkan warning certificate.
|
||||||
|
|
||||||
|
**Diagnosis:**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Cek sertifikat di host
|
||||||
|
ls -la /root/asepharyana.my.id.pem
|
||||||
|
openssl x509 -in /root/asepharyana.my.id.pem -text -noout | head -20
|
||||||
|
|
||||||
|
# Cek apakah Traefik bisa mount
|
||||||
|
docker exec traefik ls -la /etc/traefik/certs/
|
||||||
|
```
|
||||||
|
|
||||||
|
**Solusi:**
|
||||||
|
- Update sertifikat di host
|
||||||
|
- Restart Traefik
|
||||||
|
- Jika path berbeda, set environment variable `TRAEFIK_CERT_*`
|
||||||
|
|
||||||
|
### Rate limit terlalu ketat
|
||||||
|
|
||||||
|
**Gejala:** Request legitimate di-block.
|
||||||
|
|
||||||
|
**Diagnosis:**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Cek rate-limit config di middlewares.yaml
|
||||||
|
# Current: average 100, burst 50
|
||||||
|
```
|
||||||
|
|
||||||
|
**Solusi:** Ubah nilai `average` dan `burst` di `infra/traefik/dynamic/middlewares.yaml`, lalu reload:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker kill --signal HUP traefik
|
||||||
|
# atau
|
||||||
|
docker exec traefik kill -HUP 1
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Tailscale / Networking
|
||||||
|
|
||||||
|
### Container tidak bisa connect ke Tailscale IP
|
||||||
|
|
||||||
|
**Gejala:** Timeout saat container connect ke `100.121.180.82:6432`.
|
||||||
|
|
||||||
|
**Diagnosis:**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Cek route service dari host
|
||||||
|
systemctl status tailscale-routes.service
|
||||||
|
|
||||||
|
# Cek route di host
|
||||||
|
ip route show table main | grep 100.64
|
||||||
|
|
||||||
|
# Cek koneksi dari host
|
||||||
|
ping 100.121.180.82
|
||||||
|
|
||||||
|
# Test dari container (dengan --network host)
|
||||||
|
docker run --rm --network host alpine ping -c 3 100.121.180.82
|
||||||
|
```
|
||||||
|
|
||||||
|
**Solusi:**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Restart route service
|
||||||
|
sudo systemctl restart tailscale-routes.service
|
||||||
|
|
||||||
|
# Atau tambah route manual
|
||||||
|
sudo ip rule add from all lookup main priority 10000
|
||||||
|
sudo ip route add 100.64.0.0/10 dev tailscale0 table main
|
||||||
|
```
|
||||||
|
|
||||||
|
### Database connection refused
|
||||||
|
|
||||||
|
**Gejala:** Service tidak bisa konek ke PostgreSQL.
|
||||||
|
|
||||||
|
**Diagnosis:**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Cek apakah DB listen di Tailscale (dari imrnes)
|
||||||
|
ss -tlnp | grep 6432
|
||||||
|
|
||||||
|
# Cek dari orangevps
|
||||||
|
nc -zv 100.121.180.82 6432
|
||||||
|
|
||||||
|
# Cek firewall di imrnes
|
||||||
|
sudo ufw status
|
||||||
|
sudo iptables -L -n | grep 6432
|
||||||
|
```
|
||||||
|
|
||||||
|
**Solusi:**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Di imrnes: pastikan PostgreSQL bind ke Tailscale interface
|
||||||
|
# Di postgresql.conf:
|
||||||
|
listen_addresses = 'localhost,100.121.180.82'
|
||||||
|
|
||||||
|
# Di pg_hba.conf:
|
||||||
|
host hub asephs 100.0.0.0/8 md5
|
||||||
|
|
||||||
|
# Restart PostgreSQL
|
||||||
|
sudo systemctl restart postgresql
|
||||||
|
```
|
||||||
|
|
||||||
|
### Redis connection refused dari container
|
||||||
|
|
||||||
|
**Gejala:** Service tidak bisa connect ke `redis://redis:6379`.
|
||||||
|
|
||||||
|
**Diagnosis:**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Cek apakah container Redis running
|
||||||
|
docker ps -a | grep redis
|
||||||
|
|
||||||
|
# Cek apakah container target join network yang sama
|
||||||
|
docker inspect <container> | grep -A5 Networks
|
||||||
|
|
||||||
|
# Cek DNS resolve dari container
|
||||||
|
docker exec <container> getent hosts redis
|
||||||
|
```
|
||||||
|
|
||||||
|
**Solusi:**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Pastikan Redis ada di network app-shared-net
|
||||||
|
docker network inspect app-shared-net | grep redis
|
||||||
|
|
||||||
|
# Jika tidak, attach
|
||||||
|
docker network connect app-shared-net redis
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Docker / Container
|
||||||
|
|
||||||
|
### Container restart loop
|
||||||
|
|
||||||
|
**Diagnosis:**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker logs <container> --tail 50
|
||||||
|
docker inspect <container> | jq '.[].State'
|
||||||
|
```
|
||||||
|
|
||||||
|
**Penyebab umum:**
|
||||||
|
- Health check gagal
|
||||||
|
- Dependency service belum siap
|
||||||
|
- Environment variable tidak diset
|
||||||
|
|
||||||
|
### Image pull gagal dari GHCR
|
||||||
|
|
||||||
|
**Gejala:** `docker pull` gagal di VPS.
|
||||||
|
|
||||||
|
**Diagnosis:**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Cek login
|
||||||
|
cat ~/.docker/config.json | grep ghcr
|
||||||
|
|
||||||
|
# Cek visibility package
|
||||||
|
# Buka https://github.com/orgs/asepharyana/packages
|
||||||
|
```
|
||||||
|
|
||||||
|
**Solusi:**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Re-login
|
||||||
|
echo $GITHUB_TOKEN | docker login ghcr.io -u asepharyana --password-stdin
|
||||||
|
|
||||||
|
# Pastikan package visibility public atau di-share ke org
|
||||||
|
```
|
||||||
|
|
||||||
|
### Disk penuh
|
||||||
|
|
||||||
|
**Gejala:** Container crash, write error.
|
||||||
|
|
||||||
|
**Diagnosis:**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
df -h
|
||||||
|
docker system df
|
||||||
|
du -sh /var/lib/docker/
|
||||||
|
```
|
||||||
|
|
||||||
|
**Solusi:**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Bersihkan container/image/volume yang tidak dipakai
|
||||||
|
docker system prune -a -f
|
||||||
|
|
||||||
|
# Hapus image lama
|
||||||
|
docker image prune -a -f
|
||||||
|
|
||||||
|
# Lihat volume terbesar
|
||||||
|
docker system df -v | grep -E "(nats_data|redis_data)"
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Database
|
||||||
|
|
||||||
|
### Koneksi PostgreSQL lambat
|
||||||
|
|
||||||
|
**Gejala:** Query time high, connection timeout.
|
||||||
|
|
||||||
|
**Diagnosis:**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Dari container, test latency
|
||||||
|
docker exec scraper-api ping -c 5 100.121.180.82
|
||||||
|
|
||||||
|
# Cek koneksi aktif
|
||||||
|
docker exec scraper-api psql $DATABASE_URL -c "SELECT count(*) FROM pg_stat_activity;"
|
||||||
|
```
|
||||||
|
|
||||||
|
**Solusi:**
|
||||||
|
- Cek Tailscale latency
|
||||||
|
- Adjust connection pool size
|
||||||
|
- Cek resource PostgreSQL di `imrnes`
|
||||||
|
|
||||||
|
### Migration gagal
|
||||||
|
|
||||||
|
**Gejala:** Service error setelah image update.
|
||||||
|
|
||||||
|
**Diagnosis:**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Cek log service
|
||||||
|
docker logs scraper-api --tail 100 | grep -i migration
|
||||||
|
```
|
||||||
|
|
||||||
|
**Solusi:**
|
||||||
|
- Migration ada di submodule `apps/scraper`, bukan di hub
|
||||||
|
- Pastikan schema sesuai dengan versi code
|
||||||
|
- Rollback image jika migration tidak backward-compatible
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Submodule
|
||||||
|
|
||||||
|
### HEAD detached di submodule
|
||||||
|
|
||||||
|
**Gejala:** `git status` di `apps/scraper` menunjukkan "HEAD detached".
|
||||||
|
|
||||||
|
**Penyebab:** Normal. Submodule selalu dalam keadaan detached HEAD karena mengacu pada commit spesifik.
|
||||||
|
|
||||||
|
**Solusi:** Jangan commit perubahan dari dalam submodule. Selalu bekerja di repo asli.
|
||||||
|
|
||||||
|
### Submodule tidak ter-update setelah pull
|
||||||
|
|
||||||
|
```bash
|
||||||
|
git submodule update --init --recursive
|
||||||
|
```
|
||||||
|
|
||||||
|
### Konflik submodule saat rebase/merge
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Resolve dengan memilih versi yang benar
|
||||||
|
git add apps/scraper
|
||||||
|
git rebase --continue
|
||||||
|
```
|
||||||
+17
-18
@@ -9,19 +9,18 @@ infra/
|
|||||||
├── compose/ # One compose file per stack/service
|
├── compose/ # One compose file per stack/service
|
||||||
│ ├── traefik.yml # Public reverse proxy
|
│ ├── traefik.yml # Public reverse proxy
|
||||||
│ ├── shared.yml # Shared Redis
|
│ ├── shared.yml # Shared Redis
|
||||||
│ ├── react.yml # React SPA
|
│ ├── nats.yml # NATS message broker + JetStream
|
||||||
│ ├── scraper.yml # Scraper API
|
│ ├── dapr.yml # Dapr placement service
|
||||||
│ ├── elysia.yml # Elysia API
|
│ └── scraper.yml # Scraper API (app + Dapr sidecar)
|
||||||
│ └── rust-auth.yml # Rust auth API
|
|
||||||
├── docker/ # Dockerfiles and image runtime helpers
|
├── docker/ # Dockerfiles and image runtime helpers
|
||||||
├── traefik/ # Static and dynamic Traefik configuration
|
├── dapr/ # Dapr component configs
|
||||||
|
│ ├── config.yaml # Global Dapr configuration
|
||||||
|
│ └── components/ # Pub/sub (Redis), state store (Redis)
|
||||||
|
├── traefik/ # Dynamic Traefik configuration
|
||||||
│ ├── dynamic/ # Routers, services, middlewares, TLS certs
|
│ ├── dynamic/ # Routers, services, middlewares, TLS certs
|
||||||
│ └── TRAEFIK_ENV_CONFIG.md
|
│ └── TRAEFIK_ENV_CONFIG.md
|
||||||
└── config/ # Service bootstrap configuration
|
|
||||||
```
|
```
|
||||||
|
|
||||||
Archived configs that are not deployed live under `docs/config/`.
|
|
||||||
|
|
||||||
## First-time setup
|
## First-time setup
|
||||||
|
|
||||||
Create the shared Docker network before starting any service:
|
Create the shared Docker network before starting any service:
|
||||||
@@ -37,15 +36,18 @@ Create `.env` from `.env.example` and fill production values. Do not commit `.en
|
|||||||
The GitHub deploy workflow combines the active compose files automatically. For manual deployment, use this order:
|
The GitHub deploy workflow combines the active compose files automatically. For manual deployment, use this order:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
|
# 1. Shared services
|
||||||
docker compose -f infra/compose/shared.yml up -d
|
docker compose -f infra/compose/shared.yml up -d
|
||||||
|
|
||||||
|
# 2. Message bus + Dapr placement
|
||||||
|
docker compose -f infra/compose/nats.yml up -d
|
||||||
|
docker compose -f infra/compose/dapr.yml up -d
|
||||||
|
|
||||||
|
# 3. Reverse proxy
|
||||||
docker compose -f infra/compose/traefik.yml up -d
|
docker compose -f infra/compose/traefik.yml up -d
|
||||||
|
|
||||||
docker compose \
|
# 4. Application services (with Dapr sidecars)
|
||||||
-f infra/compose/react.yml \
|
docker compose -f infra/compose/scraper.yml up -d
|
||||||
-f infra/compose/scraper.yml \
|
|
||||||
-f infra/compose/elysia.yml \
|
|
||||||
-f infra/compose/rust-auth.yml \
|
|
||||||
up -d
|
|
||||||
```
|
```
|
||||||
|
|
||||||
## Environment variables
|
## Environment variables
|
||||||
@@ -55,7 +57,6 @@ Common variables used by infra compose files:
|
|||||||
```env
|
```env
|
||||||
DATABASE_URL=
|
DATABASE_URL=
|
||||||
GITHUB_TOKEN=
|
GITHUB_TOKEN=
|
||||||
JWT_SECRET=
|
|
||||||
SHARED_REDIS_EXPOSE=127.0.0.1:6379:6379
|
SHARED_REDIS_EXPOSE=127.0.0.1:6379:6379
|
||||||
```
|
```
|
||||||
|
|
||||||
@@ -67,9 +68,7 @@ Traefik reads dynamic config from `infra/traefik/dynamic/`:
|
|||||||
|
|
||||||
- `apps.yaml` — routers and upstream services
|
- `apps.yaml` — routers and upstream services
|
||||||
- `middlewares.yaml` — shared middleware chains
|
- `middlewares.yaml` — shared middleware chains
|
||||||
- `ssl.yaml` — TLS certificates
|
- `ssl.yaml` — TLS certificates for `asepharyana.my.id` and `asepharyana.web.id`
|
||||||
|
|
||||||
The primary certificate intentionally pairs `asephstech.pem` with `asephscloud.key` to preserve the current production layout.
|
|
||||||
|
|
||||||
## Validation
|
## Validation
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,13 @@
|
|||||||
|
services:
|
||||||
|
dapr-placement:
|
||||||
|
container_name: dapr-placement
|
||||||
|
image: daprio/dapr:latest
|
||||||
|
restart: always
|
||||||
|
networks:
|
||||||
|
- app-shared-net
|
||||||
|
command: ["./placement", "--port", "50005"]
|
||||||
|
|
||||||
|
networks:
|
||||||
|
app-shared-net:
|
||||||
|
name: app-shared-net
|
||||||
|
external: true
|
||||||
@@ -1,24 +0,0 @@
|
|||||||
services:
|
|
||||||
elysia-api:
|
|
||||||
container_name: elysia-api
|
|
||||||
image: ghcr.io/asepharyana/asepharyana-hub/elysia-api:sha-fe998aa
|
|
||||||
restart: always
|
|
||||||
networks:
|
|
||||||
app-shared-net:
|
|
||||||
aliases:
|
|
||||||
- elysia-api
|
|
||||||
env_file:
|
|
||||||
- ../../.env
|
|
||||||
environment:
|
|
||||||
- REDIS_URL=redis://redis:6379
|
|
||||||
- JWT_SECRET=${JWT_SECRET:?JWT_SECRET is required}
|
|
||||||
- DATABASE_URL=${DATABASE_URL}
|
|
||||||
- GITHUB_TOKEN=${GITHUB_TOKEN}
|
|
||||||
- PORT=4092
|
|
||||||
- OTEL_EXPORTER_OTLP_ENDPOINT=http://otlp-metrics-backend:4317
|
|
||||||
- OTEL_SERVICE_NAME=elysia-api
|
|
||||||
|
|
||||||
networks:
|
|
||||||
app-shared-net:
|
|
||||||
name: app-shared-net
|
|
||||||
external: true
|
|
||||||
@@ -0,0 +1,20 @@
|
|||||||
|
services:
|
||||||
|
hub:
|
||||||
|
container_name: hub
|
||||||
|
image: ghcr.io/asepharyana/asepharyana-hub/hub:sha-22e8a87
|
||||||
|
restart: always
|
||||||
|
networks:
|
||||||
|
app-shared-net:
|
||||||
|
aliases:
|
||||||
|
- hub
|
||||||
|
env_file:
|
||||||
|
- ../../.env
|
||||||
|
volumes:
|
||||||
|
- /var/run/docker.sock:/var/run/docker.sock:ro
|
||||||
|
group_add:
|
||||||
|
- '988'
|
||||||
|
|
||||||
|
networks:
|
||||||
|
app-shared-net:
|
||||||
|
name: app-shared-net
|
||||||
|
external: true
|
||||||
@@ -0,0 +1,36 @@
|
|||||||
|
services:
|
||||||
|
llm-api:
|
||||||
|
container_name: llm-api
|
||||||
|
image: ghcr.io/asepharyana/asepharyana-hub/llm-api:sha-e953c3c
|
||||||
|
restart: always
|
||||||
|
networks:
|
||||||
|
app-shared-net:
|
||||||
|
aliases:
|
||||||
|
- llm-api
|
||||||
|
env_file:
|
||||||
|
- ../../.env
|
||||||
|
environment:
|
||||||
|
- MODEL_PATH=/models/MiniCPM5-1B-Claude-Opus-Fable5-V2-Thinking-Q8_0.gguf
|
||||||
|
- API_KEY=${LLM_API_KEY:-}
|
||||||
|
volumes:
|
||||||
|
- /root/models/gguf:/models:ro
|
||||||
|
healthcheck:
|
||||||
|
test: ['CMD-SHELL', 'curl -so /dev/null --connect-timeout 5 http://localhost:8080/health || test $? -eq 22']
|
||||||
|
interval: 30s
|
||||||
|
timeout: 10s
|
||||||
|
retries: 5
|
||||||
|
start_period: 60s
|
||||||
|
labels:
|
||||||
|
prometheus.io/scrape: "true"
|
||||||
|
prometheus.io/port: "8080"
|
||||||
|
deploy:
|
||||||
|
resources:
|
||||||
|
limits:
|
||||||
|
memory: 2G
|
||||||
|
reservations:
|
||||||
|
memory: 1G
|
||||||
|
|
||||||
|
networks:
|
||||||
|
app-shared-net:
|
||||||
|
name: app-shared-net
|
||||||
|
external: true
|
||||||
@@ -0,0 +1,25 @@
|
|||||||
|
services:
|
||||||
|
nats:
|
||||||
|
container_name: nats
|
||||||
|
image: nats:latest
|
||||||
|
restart: always
|
||||||
|
networks:
|
||||||
|
app-shared-net:
|
||||||
|
aliases:
|
||||||
|
- nats
|
||||||
|
ports:
|
||||||
|
- '4222:4222' # client connections
|
||||||
|
- '8222:8222' # HTTP monitor / health
|
||||||
|
command:
|
||||||
|
- '--config=/etc/nats/nats.conf'
|
||||||
|
volumes:
|
||||||
|
- nats_data:/data
|
||||||
|
- ../../infra/nats/nats.conf:/etc/nats/nats.conf:ro
|
||||||
|
|
||||||
|
volumes:
|
||||||
|
nats_data:
|
||||||
|
|
||||||
|
networks:
|
||||||
|
app-shared-net:
|
||||||
|
name: app-shared-net
|
||||||
|
external: true
|
||||||
@@ -0,0 +1,82 @@
|
|||||||
|
services:
|
||||||
|
# ── OpenTelemetry Collector ──
|
||||||
|
otel-collector:
|
||||||
|
container_name: otel-collector
|
||||||
|
image: otel/opentelemetry-collector-contrib:latest
|
||||||
|
restart: always
|
||||||
|
networks:
|
||||||
|
app-shared-net:
|
||||||
|
aliases:
|
||||||
|
- otel-collector
|
||||||
|
ports:
|
||||||
|
- '4317:4317' # OTLP gRPC
|
||||||
|
- '4318:4318' # OTLP HTTP
|
||||||
|
- '8889:8889' # Prometheus metrics
|
||||||
|
command:
|
||||||
|
- '--config=/etc/otel/config.yml'
|
||||||
|
volumes:
|
||||||
|
- ../../infra/otel/otel-collector-config.yml:/etc/otel/config.yml:ro
|
||||||
|
|
||||||
|
# ── Jaeger (Tracing Backend + Built-in UI) ──
|
||||||
|
jaeger:
|
||||||
|
container_name: jaeger
|
||||||
|
image: jaegertracing/all-in-one:latest
|
||||||
|
restart: always
|
||||||
|
networks:
|
||||||
|
app-shared-net:
|
||||||
|
aliases:
|
||||||
|
- jaeger
|
||||||
|
ports:
|
||||||
|
- '16686:16686' # Jaeger UI + API
|
||||||
|
environment:
|
||||||
|
- COLLECTOR_OTLP_ENABLED=true
|
||||||
|
- COLLECTOR_ZIPKIN_HOST_PORT=:9411
|
||||||
|
- METRICS_STORAGE_TYPE=prometheus
|
||||||
|
- PROMETHEUS_SERVER_URL=http://otel-collector:8889
|
||||||
|
- LOG_LEVEL=info
|
||||||
|
|
||||||
|
# ── Prometheus (Metrics Backend) ──
|
||||||
|
prometheus:
|
||||||
|
container_name: prometheus
|
||||||
|
image: prom/prometheus:latest
|
||||||
|
restart: always
|
||||||
|
networks:
|
||||||
|
app-shared-net:
|
||||||
|
aliases:
|
||||||
|
- prometheus
|
||||||
|
ports:
|
||||||
|
- '9090:9090'
|
||||||
|
command:
|
||||||
|
- '--config.file=/etc/prometheus/prometheus.yml'
|
||||||
|
- '--storage.tsdb.path=/prometheus'
|
||||||
|
- '--web.console.libraries=/etc/prometheus/console_libraries'
|
||||||
|
- '--web.console.templates=/etc/prometheus/consoles'
|
||||||
|
- '--web.enable-lifecycle'
|
||||||
|
volumes:
|
||||||
|
- ../../infra/otel/prometheus.yml:/etc/prometheus/prometheus.yml:ro
|
||||||
|
- /prometheus
|
||||||
|
- /var/run/docker.sock:/var/run/docker.sock:ro
|
||||||
|
group_add:
|
||||||
|
- '988'
|
||||||
|
|
||||||
|
# ── Node Exporter (Host Metrics: CPU, RAM, Disk) ──
|
||||||
|
node-exporter:
|
||||||
|
container_name: node-exporter
|
||||||
|
image: prom/node-exporter:latest
|
||||||
|
restart: always
|
||||||
|
networks:
|
||||||
|
app-shared-net:
|
||||||
|
aliases:
|
||||||
|
- node-exporter
|
||||||
|
command:
|
||||||
|
- '--web.listen-address=0.0.0.0:9100'
|
||||||
|
- '--path.rootfs=/host'
|
||||||
|
- '--path.procfs=/host/proc'
|
||||||
|
- '--path.sysfs=/host/sys'
|
||||||
|
volumes:
|
||||||
|
- /:/host:ro,rslave
|
||||||
|
|
||||||
|
networks:
|
||||||
|
app-shared-net:
|
||||||
|
name: app-shared-net
|
||||||
|
external: true
|
||||||
@@ -1,13 +0,0 @@
|
|||||||
services:
|
|
||||||
react-web:
|
|
||||||
container_name: react-web
|
|
||||||
image: ghcr.io/asepharyana/asepharyana-hub/react-web:sha-124223c
|
|
||||||
restart: always
|
|
||||||
networks: [app-shared-net]
|
|
||||||
environment:
|
|
||||||
- VITE_API_URL=https://scraper.asepharyana.my.id/api
|
|
||||||
- VITE_ELYSIA_URL=https://elysia.asepharyana.my.id
|
|
||||||
networks:
|
|
||||||
app-shared-net:
|
|
||||||
name: app-shared-net
|
|
||||||
external: true
|
|
||||||
@@ -1,20 +0,0 @@
|
|||||||
services:
|
|
||||||
rust-auth:
|
|
||||||
container_name: rust-auth-api
|
|
||||||
image: ghcr.io/asepharyana/asepharyana-hub/rust-auth:sha-b199f8c
|
|
||||||
restart: always
|
|
||||||
networks:
|
|
||||||
app-shared-net:
|
|
||||||
aliases:
|
|
||||||
- rust-auth
|
|
||||||
env_file:
|
|
||||||
- ../../.env
|
|
||||||
environment:
|
|
||||||
- PORT=3000
|
|
||||||
- OTEL_EXPORTER_OTLP_ENDPOINT=http://otlp-metrics-backend:4317
|
|
||||||
- OTEL_SERVICE_NAME=rust-auth
|
|
||||||
|
|
||||||
networks:
|
|
||||||
app-shared-net:
|
|
||||||
name: app-shared-net
|
|
||||||
external: true
|
|
||||||
@@ -1,20 +1,56 @@
|
|||||||
services:
|
services:
|
||||||
scraper-api:
|
scraper-api:
|
||||||
container_name: scraper-api
|
container_name: scraper-api
|
||||||
image: ghcr.io/asepharyana/asepharyana-hub/scraper-api:sha-b199f8c
|
image: ghcr.io/asepharyana/asepharyana-hub/scraper-api:sha-e953c3c
|
||||||
restart: always
|
restart: always
|
||||||
|
depends_on:
|
||||||
|
nats:
|
||||||
|
condition: service_started
|
||||||
|
dapr-placement:
|
||||||
|
condition: service_started
|
||||||
networks:
|
networks:
|
||||||
app-shared-net:
|
app-shared-net:
|
||||||
aliases:
|
aliases:
|
||||||
- scraper-api
|
- scraper-api
|
||||||
env_file:
|
env_file:
|
||||||
- ../../.env
|
- ../../.env
|
||||||
|
healthcheck:
|
||||||
|
test: ['CMD-SHELL', 'curl -so /dev/null --connect-timeout 5 http://localhost:4091/ || test $? -eq 22']
|
||||||
|
interval: 30s
|
||||||
|
timeout: 10s
|
||||||
|
retries: 5
|
||||||
|
start_period: 30s
|
||||||
environment:
|
environment:
|
||||||
- REDIS_URL=redis://redis:6379
|
- REDIS_URL=redis://redis:6379
|
||||||
- JWT_SECRET=${JWT_SECRET:?JWT_SECRET is required}
|
- JWT_SECRET=${JWT_SECRET:?JWT_SECRET is required}
|
||||||
- DATABASE_URL=${DATABASE_URL}
|
- DATABASE_URL=${DATABASE_URL}
|
||||||
- OTEL_EXPORTER_OTLP_ENDPOINT=http://otlp-metrics-backend:4317
|
|
||||||
- OTEL_SERVICE_NAME=scraper-api
|
|
||||||
|
scraper-api-dapr:
|
||||||
|
container_name: scraper-api-dapr
|
||||||
|
image: daprio/daprd:latest
|
||||||
|
restart: always
|
||||||
|
depends_on:
|
||||||
|
nats:
|
||||||
|
condition: service_started
|
||||||
|
dapr-placement:
|
||||||
|
condition: service_started
|
||||||
|
otel-collector:
|
||||||
|
condition: service_started
|
||||||
|
networks:
|
||||||
|
- app-shared-net
|
||||||
|
command:
|
||||||
|
- './daprd'
|
||||||
|
- '--app-id=scraper-api'
|
||||||
|
- '--app-port=4091'
|
||||||
|
- '--dapr-http-port=3500'
|
||||||
|
- '--dapr-grpc-port=50001'
|
||||||
|
- '--placement-host-address=dapr-placement:50005'
|
||||||
|
- '--config=/dapr/config.yaml'
|
||||||
|
- '--resources-path=/dapr/components'
|
||||||
|
volumes:
|
||||||
|
- ../../infra/dapr:/dapr:ro
|
||||||
|
|
||||||
networks:
|
networks:
|
||||||
app-shared-net:
|
app-shared-net:
|
||||||
name: app-shared-net
|
name: app-shared-net
|
||||||
|
|||||||
@@ -9,6 +9,12 @@ services:
|
|||||||
- redis
|
- redis
|
||||||
ports:
|
ports:
|
||||||
- '${SHARED_REDIS_EXPOSE:-127.0.0.1:6379:6379}'
|
- '${SHARED_REDIS_EXPOSE:-127.0.0.1:6379:6379}'
|
||||||
|
healthcheck:
|
||||||
|
test: ['CMD', 'redis-cli', 'ping']
|
||||||
|
interval: 10s
|
||||||
|
timeout: 3s
|
||||||
|
retries: 3
|
||||||
|
start_period: 5s
|
||||||
volumes:
|
volumes:
|
||||||
- 'redis_data:/data'
|
- 'redis_data:/data'
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,40 @@
|
|||||||
|
services:
|
||||||
|
tools:
|
||||||
|
container_name: tools
|
||||||
|
image: ghcr.io/asepharyana/asepharyana-hub/tools:sha-e953c3c
|
||||||
|
restart: always
|
||||||
|
networks:
|
||||||
|
app-shared-net:
|
||||||
|
aliases:
|
||||||
|
- tools
|
||||||
|
env_file:
|
||||||
|
- ../../.env
|
||||||
|
environment:
|
||||||
|
- REDIS_URL=redis://redis:6379
|
||||||
|
- NATS_URL=nats://nats:4222
|
||||||
|
- STORAGE_PATH=/data/tools
|
||||||
|
- GATEWAY_PORT=3001
|
||||||
|
- TOOLS_WORKER_CONCURRENCY=4
|
||||||
|
- RUST_LOG=info
|
||||||
|
volumes:
|
||||||
|
- tools_data:/data/tools
|
||||||
|
labels:
|
||||||
|
- 'prometheus.io/scrape=true'
|
||||||
|
- 'prometheus.io/port=3001'
|
||||||
|
- 'prometheus.io/path=/metrics'
|
||||||
|
ports:
|
||||||
|
- "3002:3001"
|
||||||
|
healthcheck:
|
||||||
|
test: ['CMD-SHELL', 'wget -q -O /dev/null http://localhost:3001/health || exit 1']
|
||||||
|
interval: 30s
|
||||||
|
timeout: 10s
|
||||||
|
retries: 5
|
||||||
|
start_period: 30s
|
||||||
|
|
||||||
|
volumes:
|
||||||
|
tools_data:
|
||||||
|
|
||||||
|
networks:
|
||||||
|
app-shared-net:
|
||||||
|
name: app-shared-net
|
||||||
|
external: true
|
||||||
@@ -49,6 +49,19 @@ services:
|
|||||||
- '--experimental.plugins.real-ip.version=v1.0.3'
|
- '--experimental.plugins.real-ip.version=v1.0.3'
|
||||||
- '--experimental.plugins.blockpath.moduleName=github.com/traefik/plugin-blockpath'
|
- '--experimental.plugins.blockpath.moduleName=github.com/traefik/plugin-blockpath'
|
||||||
- '--experimental.plugins.blockpath.version=v0.2.1'
|
- '--experimental.plugins.blockpath.version=v0.2.1'
|
||||||
|
- '--ping=true'
|
||||||
|
- '--metrics.prometheus=true'
|
||||||
|
- '--metrics.prometheus.addEntryPointsLabels=true'
|
||||||
|
- '--metrics.prometheus.addServicesLabels=true'
|
||||||
|
- '--tracing.otlp=true'
|
||||||
|
- '--tracing.otlp.http=true'
|
||||||
|
- '--tracing.otlp.http.endpoint=http://otel-collector:4318'
|
||||||
|
healthcheck:
|
||||||
|
test: ['CMD', 'wget', '--spider', 'http://localhost:8080/ping']
|
||||||
|
interval: 30s
|
||||||
|
timeout: 5s
|
||||||
|
retries: 3
|
||||||
|
start_period: 15s
|
||||||
environment:
|
environment:
|
||||||
- DOCKER_API_VERSION=1.41
|
- DOCKER_API_VERSION=1.41
|
||||||
- GOMEMLIMIT=4096MiB
|
- GOMEMLIMIT=4096MiB
|
||||||
@@ -56,10 +69,10 @@ services:
|
|||||||
volumes:
|
volumes:
|
||||||
- /var/run/docker.sock:/var/run/docker.sock:ro
|
- /var/run/docker.sock:/var/run/docker.sock:ro
|
||||||
- ${TRAEFIK_CONFIG_PATH:-/root/asepharyana-hub/infra/traefik/dynamic}:/etc/traefik/dynamic:ro
|
- ${TRAEFIK_CONFIG_PATH:-/root/asepharyana-hub/infra/traefik/dynamic}:/etc/traefik/dynamic:ro
|
||||||
- ${TRAEFIK_CERT_ASEPHARYANA_MY_ID_PEM:-/root/asepharyana.my.id.pem}:/etc/traefik/certs/asepharyana.my.id.pem:ro
|
- ${TRAEFIK_CERT_MY_ID_PEM:-/root/asepharyana.my.id.pem}:/etc/traefik/certs/asepharyana.my.id.pem:ro
|
||||||
- ${TRAEFIK_CERT_ASEPHARYANA_MY_ID_KEY:-/root/asepharyana.my.id.key}:/etc/traefik/certs/asepharyana.my.id.key:ro
|
- ${TRAEFIK_CERT_MY_ID_KEY:-/root/asepharyana.my.id.key}:/etc/traefik/certs/asepharyana.my.id.key:ro
|
||||||
- ${TRAEFIK_CERT_ASEPHARYANA_WEB_ID_PEM:-/root/asepharyana.web.id.pem}:/etc/traefik/certs/asepharyana.web.id.pem:ro
|
- ${TRAEFIK_CERT_WEB_ID_PEM:-/root/asepharyana.web.id.pem}:/etc/traefik/certs/asepharyana.web.id.pem:ro
|
||||||
- ${TRAEFIK_CERT_ASEPHARYANA_WEB_ID_KEY:-/root/asepharyana.web.id.key}:/etc/traefik/certs/asepharyana.web.id.key:ro
|
- ${TRAEFIK_CERT_WEB_ID_KEY:-/root/asepharyana.web.id.key}:/etc/traefik/certs/asepharyana.web.id.key:ro
|
||||||
labels:
|
labels:
|
||||||
- 'traefik.enable=true'
|
- 'traefik.enable=true'
|
||||||
- 'traefik.http.routers.traefik.rule=Host(`traefik.asepharyana.my.id`) || Host(`traefik.asepharyana.web.id`)'
|
- 'traefik.http.routers.traefik.rule=Host(`traefik.asepharyana.my.id`) || Host(`traefik.asepharyana.web.id`)'
|
||||||
@@ -67,6 +80,8 @@ services:
|
|||||||
- 'traefik.http.routers.traefik.entrypoints=websecure'
|
- 'traefik.http.routers.traefik.entrypoints=websecure'
|
||||||
- 'traefik.http.routers.traefik.tls=true'
|
- 'traefik.http.routers.traefik.tls=true'
|
||||||
- 'traefik.http.routers.traefik.middlewares=admin-chain@file'
|
- 'traefik.http.routers.traefik.middlewares=admin-chain@file'
|
||||||
|
- 'prometheus.io/scrape=true'
|
||||||
|
- 'prometheus.io/port=8080'
|
||||||
|
|
||||||
networks:
|
networks:
|
||||||
app-shared-net:
|
app-shared-net:
|
||||||
|
|||||||
@@ -1,10 +0,0 @@
|
|||||||
-- Create database if not exists
|
|
||||||
CREATE DATABASE IF NOT EXISTS `tracer_study`;
|
|
||||||
|
|
||||||
-- Create dedicated user for tracer_study
|
|
||||||
CREATE USER IF NOT EXISTS 'tracerstudy'@'%' IDENTIFIED BY 'tracerstudy_secret';
|
|
||||||
GRANT ALL PRIVILEGES ON `tracer_study`.* TO 'tracerstudy'@'%';
|
|
||||||
|
|
||||||
|
|
||||||
-- Flush privileges
|
|
||||||
FLUSH PRIVILEGES;
|
|
||||||
File diff suppressed because one or more lines are too long
@@ -0,0 +1,12 @@
|
|||||||
|
apiVersion: dapr.io/v1alpha1
|
||||||
|
kind: Component
|
||||||
|
metadata:
|
||||||
|
name: pubsub
|
||||||
|
spec:
|
||||||
|
type: pubsub.redis
|
||||||
|
version: v1
|
||||||
|
metadata:
|
||||||
|
- name: redisHost
|
||||||
|
value: redis:6379
|
||||||
|
- name: redisPassword
|
||||||
|
value: ""
|
||||||
@@ -0,0 +1,14 @@
|
|||||||
|
apiVersion: dapr.io/v1alpha1
|
||||||
|
kind: Component
|
||||||
|
metadata:
|
||||||
|
name: statestore
|
||||||
|
spec:
|
||||||
|
type: state.redis
|
||||||
|
version: v1
|
||||||
|
metadata:
|
||||||
|
- name: redisHost
|
||||||
|
value: redis:6379
|
||||||
|
- name: redisPassword
|
||||||
|
value: ""
|
||||||
|
- name: keyPrefix
|
||||||
|
value: "dapr"
|
||||||
@@ -0,0 +1,16 @@
|
|||||||
|
apiVersion: dapr.io/v1alpha1
|
||||||
|
kind: Configuration
|
||||||
|
metadata:
|
||||||
|
name: dapr-config
|
||||||
|
spec:
|
||||||
|
tracing:
|
||||||
|
samplingRate: "1"
|
||||||
|
stdout: false
|
||||||
|
otel:
|
||||||
|
endpointAddress: "otel-collector:4317"
|
||||||
|
isSecure: false
|
||||||
|
protocol: grpc
|
||||||
|
metrics:
|
||||||
|
enabled: true
|
||||||
|
mtls:
|
||||||
|
enabled: false
|
||||||
@@ -1,26 +0,0 @@
|
|||||||
# build stage
|
|
||||||
FROM oven/bun:1-alpine AS builder
|
|
||||||
WORKDIR /app
|
|
||||||
|
|
||||||
# install dependencies with cache mounts
|
|
||||||
COPY apps/elysia/package.json apps/elysia/bun.lock ./
|
|
||||||
RUN --mount=type=cache,target=/root/.bun/install/cache \
|
|
||||||
bun install --frozen-lockfile
|
|
||||||
|
|
||||||
# build the application
|
|
||||||
COPY apps/elysia ./
|
|
||||||
RUN bun run build
|
|
||||||
|
|
||||||
# runtime stage
|
|
||||||
FROM oven/bun:1-distroless
|
|
||||||
WORKDIR /app
|
|
||||||
|
|
||||||
# copy build artifacts
|
|
||||||
COPY --from=builder /app/dist ./dist
|
|
||||||
COPY --from=builder /app/package.json ./
|
|
||||||
|
|
||||||
# distroless uses nonroot user (UID 65532) by default, or we can use it
|
|
||||||
USER nonroot
|
|
||||||
|
|
||||||
EXPOSE 4092
|
|
||||||
CMD ["run", "dist/index.js"]
|
|
||||||
@@ -0,0 +1,21 @@
|
|||||||
|
# ── Build stage ──
|
||||||
|
FROM oven/bun:1.2 AS builder
|
||||||
|
WORKDIR /app
|
||||||
|
COPY apps/hub/package.json apps/hub/bun.lock ./
|
||||||
|
RUN bun install --frozen-lockfile
|
||||||
|
COPY apps/hub .
|
||||||
|
RUN bun run build
|
||||||
|
|
||||||
|
# ── Runtime ──
|
||||||
|
FROM oven/bun:1.2 AS runtime
|
||||||
|
RUN addgroup --system appgroup && adduser --system appuser --ingroup appgroup
|
||||||
|
WORKDIR /app
|
||||||
|
COPY --from=builder /app/.next ./.next
|
||||||
|
COPY --from=builder /app/node_modules ./node_modules
|
||||||
|
COPY --from=builder /app/public ./public
|
||||||
|
COPY --from=builder /app/package.json ./
|
||||||
|
RUN rm -rf .next/cache && chown -R appuser:appgroup .next
|
||||||
|
USER appuser
|
||||||
|
EXPOSE 3000
|
||||||
|
ENV PORT=3000 NODE_ENV=production
|
||||||
|
CMD ["bun", "run", "start"]
|
||||||
@@ -1,42 +1,39 @@
|
|||||||
# Use cargo-chef for dependency caching
|
# ── Build stage: cargo-chef ──
|
||||||
FROM lukemathwalker/cargo-chef:latest-rust-1.89.0 AS chef
|
FROM lukemathwalker/cargo-chef:latest-rust-1.89.0 AS chef
|
||||||
|
RUN apt-get update && apt-get install -y --no-install-recommends libclang-dev cmake && rm -rf /var/lib/apt/lists/*
|
||||||
WORKDIR /app
|
WORKDIR /app
|
||||||
|
|
||||||
FROM chef AS planner
|
FROM chef AS planner
|
||||||
COPY apps/rust-auth .
|
COPY apps/llm-api .
|
||||||
RUN cargo chef prepare --recipe-path recipe.json
|
RUN cargo chef prepare --recipe-path recipe.json
|
||||||
|
|
||||||
FROM chef AS builder
|
FROM chef AS builder
|
||||||
COPY --from=planner /app/recipe.json recipe.json
|
COPY --from=planner /app/recipe.json recipe.json
|
||||||
# Utilize buildkit cache mounts for cargo
|
|
||||||
RUN --mount=type=cache,target=/usr/local/cargo/registry \
|
RUN --mount=type=cache,target=/usr/local/cargo/registry \
|
||||||
--mount=type=cache,target=/app/target \
|
--mount=type=cache,target=/app/target \
|
||||||
cargo chef cook --release --recipe-path recipe.json
|
cargo chef cook --release --recipe-path recipe.json
|
||||||
|
|
||||||
# Build application
|
COPY apps/llm-api .
|
||||||
COPY apps/rust-auth .
|
|
||||||
RUN --mount=type=cache,target=/usr/local/cargo/registry \
|
RUN --mount=type=cache,target=/usr/local/cargo/registry \
|
||||||
--mount=type=cache,target=/app/target \
|
--mount=type=cache,target=/app/target \
|
||||||
cargo build --release && \
|
cargo build --release && \
|
||||||
cp target/release/rust-auth /app/rust-auth
|
cp target/release/llm-api /app/llm-api
|
||||||
|
|
||||||
# Final runtime image
|
# ── Runtime ──
|
||||||
FROM debian:bookworm-slim AS runtime
|
FROM debian:bookworm-slim AS runtime
|
||||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||||
ca-certificates \
|
ca-certificates \
|
||||||
curl \
|
curl \
|
||||||
libssl3 \
|
libssl3 \
|
||||||
|
libgomp1 \
|
||||||
&& rm -rf /var/lib/apt/lists/*
|
&& rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
# Add non-root user
|
|
||||||
RUN groupadd -g 1001 appgroup && \
|
RUN groupadd -g 1001 appgroup && \
|
||||||
useradd -u 1001 -g appgroup -s /bin/sh appuser
|
useradd -u 1001 -g appgroup -s /bin/sh appuser
|
||||||
|
|
||||||
WORKDIR /app
|
WORKDIR /app
|
||||||
COPY --from=builder /app/rust-auth /app/rust-auth
|
COPY --from=builder /app/llm-api /app/llm-api
|
||||||
|
|
||||||
# Run as non-root
|
|
||||||
USER appuser
|
USER appuser
|
||||||
|
|
||||||
EXPOSE 3000
|
EXPOSE 8080
|
||||||
CMD ["./rust-auth"]
|
CMD ["./llm-api"]
|
||||||
Vendored
-77
@@ -1,77 +0,0 @@
|
|||||||
import { existsSync, readFileSync } from 'node:fs';
|
|
||||||
import { resolve, sep } from 'node:path';
|
|
||||||
|
|
||||||
const distDir = resolve('./dist');
|
|
||||||
const indexPath = resolve(distDir, 'index.html');
|
|
||||||
|
|
||||||
const contentTypes = {
|
|
||||||
html: 'text/html; charset=utf-8',
|
|
||||||
css: 'text/css; charset=utf-8',
|
|
||||||
js: 'application/javascript; charset=utf-8',
|
|
||||||
json: 'application/json; charset=utf-8',
|
|
||||||
svg: 'image/svg+xml',
|
|
||||||
png: 'image/png',
|
|
||||||
jpg: 'image/jpeg',
|
|
||||||
jpeg: 'image/jpeg',
|
|
||||||
gif: 'image/gif',
|
|
||||||
webp: 'image/webp',
|
|
||||||
ico: 'image/x-icon',
|
|
||||||
woff: 'font/woff',
|
|
||||||
woff2: 'font/woff2',
|
|
||||||
ttf: 'font/ttf',
|
|
||||||
};
|
|
||||||
|
|
||||||
function responseFromFile(filePath, headers) {
|
|
||||||
try {
|
|
||||||
return new Response(readFileSync(filePath), { headers });
|
|
||||||
} catch (error) {
|
|
||||||
if (error?.code === 'ENOENT') {
|
|
||||||
return new Response('Not Found', { status: 404 });
|
|
||||||
}
|
|
||||||
|
|
||||||
console.error('File read error:', error);
|
|
||||||
return new Response('Internal Server Error', { status: 500 });
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
function serveIndex() {
|
|
||||||
return responseFromFile(indexPath, {
|
|
||||||
'Content-Type': contentTypes.html,
|
|
||||||
'Cache-Control': 'no-cache',
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
function serveFile(filePath, pathname) {
|
|
||||||
const ext = filePath.split('.').pop() || '';
|
|
||||||
return responseFromFile(filePath, {
|
|
||||||
'Content-Type': contentTypes[ext] || 'application/octet-stream',
|
|
||||||
'Cache-Control': pathname.startsWith('/assets/')
|
|
||||||
? 'public, max-age=31536000, immutable'
|
|
||||||
: 'no-cache',
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
Bun.serve({
|
|
||||||
port: 80,
|
|
||||||
hostname: '0.0.0.0',
|
|
||||||
fetch(req) {
|
|
||||||
const url = new URL(req.url);
|
|
||||||
const pathname = url.pathname;
|
|
||||||
const filePath = resolve(distDir, pathname.slice(1));
|
|
||||||
const insideDist = filePath === distDir || filePath.startsWith(`${distDir}${sep}`);
|
|
||||||
|
|
||||||
if (!insideDist) {
|
|
||||||
return new Response('Forbidden', { status: 403 });
|
|
||||||
}
|
|
||||||
|
|
||||||
if (pathname !== '/' && existsSync(filePath)) {
|
|
||||||
return serveFile(filePath, pathname);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!pathname.includes('.') && existsSync(indexPath)) {
|
|
||||||
return serveIndex();
|
|
||||||
}
|
|
||||||
|
|
||||||
return new Response('Not Found', { status: 404 });
|
|
||||||
},
|
|
||||||
});
|
|
||||||
@@ -1,16 +0,0 @@
|
|||||||
# ─── Stage 1: Build ─────────────────────────────────────────────────────────
|
|
||||||
FROM oven/bun:1 AS builder
|
|
||||||
WORKDIR /app
|
|
||||||
COPY apps/react/package.json apps/react/bun.lock ./
|
|
||||||
RUN bun install --frozen-lockfile
|
|
||||||
COPY apps/react .
|
|
||||||
RUN bun run build
|
|
||||||
|
|
||||||
# ─── Stage 2: Runtime (Bun static server) ──────────────────────────────────
|
|
||||||
FROM oven/bun:1-alpine
|
|
||||||
WORKDIR /app
|
|
||||||
COPY infra/docker/react-server.js ./server.js
|
|
||||||
COPY --from=builder /app/dist ./dist
|
|
||||||
|
|
||||||
EXPOSE 80
|
|
||||||
CMD ["bun", "server.js"]
|
|
||||||
@@ -1,49 +1,36 @@
|
|||||||
# Use cargo-chef for dependency caching
|
# ── Build stage: cargo-chef for dependency caching ──
|
||||||
FROM lukemathwalker/cargo-chef:latest-rust-1.89.0 AS chef
|
FROM lukemathwalker/cargo-chef:latest-rust-1.89.0 AS chef
|
||||||
WORKDIR /app
|
WORKDIR /app
|
||||||
|
|
||||||
# Install Node.js if needed for build scripts
|
|
||||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
|
||||||
nodejs \
|
|
||||||
&& rm -rf /var/lib/apt/lists/*
|
|
||||||
|
|
||||||
FROM chef AS planner
|
FROM chef AS planner
|
||||||
COPY apps/scraper .
|
COPY apps/scraper .
|
||||||
RUN cargo chef prepare --recipe-path recipe.json
|
RUN cargo chef prepare --recipe-path recipe.json
|
||||||
|
|
||||||
FROM chef AS builder
|
FROM chef AS builder
|
||||||
COPY --from=planner /app/recipe.json recipe.json
|
COPY --from=planner /app/recipe.json recipe.json
|
||||||
# Utilize buildkit cache mounts for cargo
|
|
||||||
RUN --mount=type=cache,target=/usr/local/cargo/registry \
|
RUN --mount=type=cache,target=/usr/local/cargo/registry \
|
||||||
--mount=type=cache,target=/app/target \
|
--mount=type=cache,target=/app/target \
|
||||||
cargo chef cook --release --recipe-path recipe.json
|
cargo chef cook --release --recipe-path recipe.json
|
||||||
|
|
||||||
# Build application
|
|
||||||
COPY apps/scraper .
|
COPY apps/scraper .
|
||||||
RUN --mount=type=cache,target=/usr/local/cargo/registry \
|
RUN --mount=type=cache,target=/usr/local/cargo/registry \
|
||||||
--mount=type=cache,target=/app/target \
|
--mount=type=cache,target=/app/target \
|
||||||
cargo build --release && \
|
cargo build --release && \
|
||||||
cp target/release/scraper /app/scraper
|
cp target/release/scraper /app/scraper
|
||||||
|
|
||||||
# Final runtime image
|
# ── Runtime image ──
|
||||||
FROM debian:bookworm-slim AS runtime
|
FROM debian:bookworm-slim AS runtime
|
||||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||||
ca-certificates \
|
ca-certificates \
|
||||||
curl \
|
curl \
|
||||||
libssl3 \
|
libssl3 \
|
||||||
chromium \
|
|
||||||
fonts-liberation \
|
|
||||||
fonts-noto-color-emoji \
|
|
||||||
&& rm -rf /var/lib/apt/lists/*
|
&& rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
# Add non-root user
|
|
||||||
RUN groupadd -g 1001 appgroup && \
|
RUN groupadd -g 1001 appgroup && \
|
||||||
useradd -u 1001 -g appgroup -s /bin/sh appuser
|
useradd -u 1001 -g appgroup -s /bin/sh appuser
|
||||||
|
|
||||||
WORKDIR /app
|
WORKDIR /app
|
||||||
COPY --from=builder /app/scraper /app/scraper
|
COPY --from=builder /app/scraper /app/scraper
|
||||||
|
|
||||||
# Run as non-root
|
|
||||||
USER appuser
|
USER appuser
|
||||||
|
|
||||||
EXPOSE 4091
|
EXPOSE 4091
|
||||||
|
|||||||
@@ -0,0 +1,93 @@
|
|||||||
|
# ============================================================
|
||||||
|
# Stage 1: Build Rust Backend (with cargo-chef caching)
|
||||||
|
# ============================================================
|
||||||
|
FROM lukemathwalker/cargo-chef:latest-rust-1.89.0 AS chef
|
||||||
|
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||||
|
libleptonica-dev libtesseract-dev clang pkg-config \
|
||||||
|
&& rm -rf /var/lib/apt/lists/*
|
||||||
|
WORKDIR /app
|
||||||
|
|
||||||
|
FROM chef AS planner
|
||||||
|
COPY apps/tools/backend/ .
|
||||||
|
RUN cargo chef prepare --recipe-path recipe.json
|
||||||
|
|
||||||
|
FROM chef AS builder
|
||||||
|
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||||
|
libleptonica-dev libtesseract-dev clang pkg-config \
|
||||||
|
&& rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
|
COPY --from=planner /app/recipe.json recipe.json
|
||||||
|
RUN --mount=type=cache,target=/usr/local/cargo/registry \
|
||||||
|
--mount=type=cache,target=/app/target \
|
||||||
|
cargo chef cook --release --recipe-path recipe.json
|
||||||
|
|
||||||
|
COPY apps/tools/backend/ .
|
||||||
|
RUN --mount=type=cache,target=/usr/local/cargo/registry \
|
||||||
|
--mount=type=cache,target=/app/target \
|
||||||
|
cargo build --release --features tesseract --bin tools-gateway --bin tools-workers && \
|
||||||
|
cp /app/target/release/tools-gateway /app/tools-gateway-bin && \
|
||||||
|
cp /app/target/release/tools-workers /app/tools-workers-bin
|
||||||
|
|
||||||
|
# ============================================================
|
||||||
|
# Stage 2: Build Next.js Frontend
|
||||||
|
# ============================================================
|
||||||
|
FROM oven/bun:1.3 AS frontend-builder
|
||||||
|
WORKDIR /app
|
||||||
|
|
||||||
|
# Copy package files first for layer caching
|
||||||
|
COPY apps/tools/frontend/package.json apps/tools/frontend/bun.lock ./
|
||||||
|
RUN bun install --frozen-lockfile
|
||||||
|
|
||||||
|
COPY apps/tools/frontend/ .
|
||||||
|
RUN bun run build
|
||||||
|
|
||||||
|
# ============================================================
|
||||||
|
# Stage 3: Production Runtime
|
||||||
|
# ============================================================
|
||||||
|
FROM debian:bookworm-slim AS runtime
|
||||||
|
|
||||||
|
# Install runtime dependencies
|
||||||
|
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||||
|
tesseract-ocr \
|
||||||
|
tesseract-ocr-eng \
|
||||||
|
tesseract-ocr-ind \
|
||||||
|
tesseract-ocr-osd \
|
||||||
|
ca-certificates \
|
||||||
|
fonts-dejavu-core \
|
||||||
|
wget \
|
||||||
|
&& rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
|
WORKDIR /app
|
||||||
|
|
||||||
|
# Copy Rust binaries (cp'd from cache mount in builder stage)
|
||||||
|
COPY --from=builder /app/tools-gateway-bin /app/gateway
|
||||||
|
COPY --from=builder /app/tools-workers-bin /app/workers
|
||||||
|
|
||||||
|
# Copy bun binary from frontend builder (needed to run Next.js server)
|
||||||
|
COPY --from=frontend-builder /usr/local/bin/bun /usr/local/bin/bun
|
||||||
|
|
||||||
|
# Copy Next.js build
|
||||||
|
COPY --from=frontend-builder /app/.next /app/.next
|
||||||
|
COPY --from=frontend-builder /app/public /app/public
|
||||||
|
COPY --from=frontend-builder /app/package.json /app/package.json
|
||||||
|
COPY --from=frontend-builder /app/node_modules /app/node_modules
|
||||||
|
COPY --from=frontend-builder /app/next.config.ts /app/next.config.ts
|
||||||
|
|
||||||
|
# Create temp storage directory
|
||||||
|
RUN mkdir -p /data/tools && chmod 1777 /data/tools
|
||||||
|
|
||||||
|
# Copy entrypoint
|
||||||
|
COPY apps/tools/scripts/entrypoint.sh /app/entrypoint.sh
|
||||||
|
RUN chmod +x /app/entrypoint.sh
|
||||||
|
|
||||||
|
# Environment
|
||||||
|
ENV TESSDATA_PREFIX=/usr/share/tesseract-ocr/5/tessdata
|
||||||
|
ENV STORAGE_PATH=/data/tools
|
||||||
|
ENV GATEWAY_PORT=3001
|
||||||
|
ENV TOOLS_WORKER_CONCURRENCY=4
|
||||||
|
ENV RUST_LOG=info
|
||||||
|
|
||||||
|
# Expose port
|
||||||
|
EXPOSE 3001
|
||||||
|
|
||||||
|
CMD ["/app/entrypoint.sh"]
|
||||||
@@ -0,0 +1,11 @@
|
|||||||
|
# ── NATS Server Configuration ──
|
||||||
|
|
||||||
|
# JetStream
|
||||||
|
jetstream: true
|
||||||
|
store_dir: "/data"
|
||||||
|
|
||||||
|
# HTTP monitoring
|
||||||
|
http_port: 8222
|
||||||
|
|
||||||
|
# Limits
|
||||||
|
max_payload: 1MB
|
||||||
@@ -0,0 +1,56 @@
|
|||||||
|
receivers:
|
||||||
|
otlp:
|
||||||
|
protocols:
|
||||||
|
grpc:
|
||||||
|
endpoint: 0.0.0.0:4317
|
||||||
|
http:
|
||||||
|
endpoint: 0.0.0.0:4318
|
||||||
|
|
||||||
|
exporters:
|
||||||
|
otlp/jaeger:
|
||||||
|
endpoint: jaeger:4317
|
||||||
|
tls:
|
||||||
|
insecure: true
|
||||||
|
|
||||||
|
prometheus:
|
||||||
|
endpoint: 0.0.0.0:8889
|
||||||
|
enable_open_metrics: true
|
||||||
|
resource_to_telemetry_conversion:
|
||||||
|
enabled: true
|
||||||
|
|
||||||
|
debug:
|
||||||
|
verbosity: normal
|
||||||
|
sampling_initial: 5
|
||||||
|
sampling_thereafter: 100
|
||||||
|
|
||||||
|
processors:
|
||||||
|
batch:
|
||||||
|
timeout: 1s
|
||||||
|
send_batch_size: 1024
|
||||||
|
|
||||||
|
memory_limiter:
|
||||||
|
check_interval: 1s
|
||||||
|
limit_mib: 512
|
||||||
|
spike_limit_mib: 128
|
||||||
|
|
||||||
|
attributes:
|
||||||
|
actions:
|
||||||
|
- key: service.namespace
|
||||||
|
value: asepharyana-hub
|
||||||
|
action: upsert
|
||||||
|
|
||||||
|
extensions:
|
||||||
|
health_check:
|
||||||
|
endpoint: 0.0.0.0:13133
|
||||||
|
|
||||||
|
service:
|
||||||
|
extensions: [health_check]
|
||||||
|
pipelines:
|
||||||
|
traces:
|
||||||
|
receivers: [otlp]
|
||||||
|
processors: [memory_limiter, batch, attributes]
|
||||||
|
exporters: [otlp/jaeger, debug]
|
||||||
|
metrics:
|
||||||
|
receivers: [otlp]
|
||||||
|
processors: [memory_limiter, batch, attributes]
|
||||||
|
exporters: [prometheus, debug]
|
||||||
@@ -0,0 +1,45 @@
|
|||||||
|
global:
|
||||||
|
scrape_interval: 15s
|
||||||
|
evaluation_interval: 15s
|
||||||
|
|
||||||
|
scrape_configs:
|
||||||
|
# ── Docker SD: containers with prometheus.io/scrape=true are auto-discovered ──
|
||||||
|
- job_name: 'docker'
|
||||||
|
docker_sd_configs:
|
||||||
|
- host: unix:///var/run/docker.sock
|
||||||
|
refresh_interval: 15s
|
||||||
|
filters:
|
||||||
|
- name: label
|
||||||
|
values:
|
||||||
|
- prometheus.io/scrape=true
|
||||||
|
relabel_configs:
|
||||||
|
# Address: use container name + label port (Docker DNS resolves names)
|
||||||
|
- source_labels: [__meta_docker_container_name, __meta_docker_container_label_prometheus_io_port]
|
||||||
|
regex: '/(.*);(\d+)'
|
||||||
|
replacement: '${1}:${2}'
|
||||||
|
target_label: __address__
|
||||||
|
# Metrics path from label (default /metrics)
|
||||||
|
- source_labels: [__meta_docker_container_label_prometheus_io_path]
|
||||||
|
regex: '(.+)'
|
||||||
|
target_label: __metrics_path__
|
||||||
|
replacement: '${1}'
|
||||||
|
# Container name label
|
||||||
|
- source_labels: [__meta_docker_container_name]
|
||||||
|
regex: '/(.*)'
|
||||||
|
replacement: '${1}'
|
||||||
|
target_label: container
|
||||||
|
|
||||||
|
# ── OTel Collector ──
|
||||||
|
- job_name: 'otel-collector'
|
||||||
|
scrape_interval: 10s
|
||||||
|
metrics_path: /metrics
|
||||||
|
static_configs:
|
||||||
|
- targets: ['otel-collector:8889']
|
||||||
|
labels:
|
||||||
|
service: otel-collector
|
||||||
|
|
||||||
|
# ── Node Exporter ──
|
||||||
|
- job_name: 'node'
|
||||||
|
scrape_interval: 15s
|
||||||
|
static_configs:
|
||||||
|
- targets: ['node-exporter:9100']
|
||||||
@@ -8,13 +8,13 @@ All certificate paths support environment variable substitution with safe fallba
|
|||||||
|
|
||||||
### Configuration Variables
|
### Configuration Variables
|
||||||
|
|
||||||
| Variable | Description | Default Path | Purpose |
|
| Variable | Description | Default Path | Purpose |
|
||||||
| ------------------------------------- | --------------------------------------------------------------- | --------------------------------------------- | ------------------------------------------------------- |
|
| --------------------------- | --------------------------------------------------------------- | ----------------------------------------------- | ------------------------------------------------------- |
|
||||||
| `TRAEFIK_CONFIG_PATH` | Directory containing dynamic Traefik configuration files (YAML) | `/root/asepharyana-hub/infra/traefik/dynamic` | Location of middleware, router, and service definitions |
|
| `TRAEFIK_CONFIG_PATH` | Directory containing dynamic Traefik configuration files (YAML) | `/root/asepharyana-hub/infra/traefik/dynamic` | Location of middleware, router, and service definitions |
|
||||||
| `TRAEFIK_CERT_ASEPHARYANA_MY_ID_PEM` | Certificate file for asepharyana.my.id | `/root/asepharyana.my.id.pem` | SSL/TLS certificate for asepharyana.my.id domain |
|
| `TRAEFIK_CERT_MY_ID_PEM` | Certificate file for asepharyana.my.id | `/root/asepharyana.my.id.pem` | SSL/TLS certificate for asepharyana.my.id domain |
|
||||||
| `TRAEFIK_CERT_ASEPHARYANA_MY_ID_KEY` | Key file for asepharyana.my.id | `/root/asepharyana.my.id.key` | SSL/TLS private key for asepharyana.my.id domain |
|
| `TRAEFIK_CERT_MY_ID_KEY` | Key file for asepharyana.my.id | `/root/asepharyana.my.id.key` | SSL/TLS private key for asepharyana.my.id domain |
|
||||||
| `TRAEFIK_CERT_ASEPHARYANA_WEB_ID_PEM` | Certificate file for asepharyana.web.id | `/root/asepharyana.web.id.pem` | SSL/TLS certificate for asepharyana.web.id domain |
|
| `TRAEFIK_CERT_WEB_ID_PEM` | Certificate file for asepharyana.web.id | `/root/asepharyana.web.id.pem` | SSL/TLS certificate for asepharyana.web.id domain |
|
||||||
| `TRAEFIK_CERT_ASEPHARYANA_WEB_ID_KEY` | Key file for asepharyana.web.id | `/root/asepharyana.web.id.key` | SSL/TLS private key for asepharyana.web.id domain |
|
| `TRAEFIK_CERT_WEB_ID_KEY` | Key file for asepharyana.web.id | `/root/asepharyana.web.id.key` | SSL/TLS private key for asepharyana.web.id domain |
|
||||||
|
|
||||||
## Usage
|
## Usage
|
||||||
|
|
||||||
@@ -42,10 +42,10 @@ Create or update your `.env` file in the deployment directory:
|
|||||||
|
|
||||||
```env
|
```env
|
||||||
TRAEFIK_CONFIG_PATH=/root/asepharyana-hub/infra/traefik/dynamic
|
TRAEFIK_CONFIG_PATH=/root/asepharyana-hub/infra/traefik/dynamic
|
||||||
TRAEFIK_CERT_ASEPHARYANA_MY_ID_PEM=/root/asepharyana.my.id.pem
|
TRAEFIK_CERT_MY_ID_PEM=/root/asepharyana.my.id.pem
|
||||||
TRAEFIK_CERT_ASEPHARYANA_MY_ID_KEY=/root/asepharyana.my.id.key
|
TRAEFIK_CERT_MY_ID_KEY=/root/asepharyana.my.id.key
|
||||||
TRAEFIK_CERT_ASEPHARYANA_WEB_ID_PEM=/root/asepharyana.web.id.pem
|
TRAEFIK_CERT_WEB_ID_PEM=/root/asepharyana.web.id.pem
|
||||||
TRAEFIK_CERT_ASEPHARYANA_WEB_ID_KEY=/root/asepharyana.web.id.key
|
TRAEFIK_CERT_WEB_ID_KEY=/root/asepharyana.web.id.key
|
||||||
```
|
```
|
||||||
|
|
||||||
Then deploy:
|
Then deploy:
|
||||||
|
|||||||
@@ -1,12 +1,14 @@
|
|||||||
http:
|
http:
|
||||||
routers:
|
routers:
|
||||||
# ── React SPA (domain root) ──
|
# ── Hub (Portfolio SPA) ──
|
||||||
react:
|
hub:
|
||||||
rule: 'Host(`asepharyana.my.id`) || Host(`asepharyana.web.id`)'
|
rule: 'Host(`asepharyana.my.id`) || Host(`www.asepharyana.my.id`) || Host(`asepharyana.web.id`) || Host(`www.asepharyana.web.id`)'
|
||||||
entryPoints:
|
entryPoints:
|
||||||
- websecure
|
- websecure
|
||||||
tls: {}
|
tls: {}
|
||||||
service: react-service
|
middlewares:
|
||||||
|
- common-chain@file
|
||||||
|
service: hub-service
|
||||||
|
|
||||||
# ── Scraper API ──
|
# ── Scraper API ──
|
||||||
scraper:
|
scraper:
|
||||||
@@ -18,40 +20,58 @@ http:
|
|||||||
- common-chain@file
|
- common-chain@file
|
||||||
service: scraper-service
|
service: scraper-service
|
||||||
|
|
||||||
# ── Elysia API ──
|
# ── LLM API (MiniCPM-V) ──
|
||||||
elysia:
|
llm-api:
|
||||||
rule: 'Host(`elysia.asepharyana.my.id`) || Host(`elysia.asepharyana.web.id`)'
|
rule: 'Host(`ai.asepharyana.my.id`) || Host(`ai.asepharyana.web.id`)'
|
||||||
entryPoints:
|
entryPoints:
|
||||||
- websecure
|
- websecure
|
||||||
tls: {}
|
tls: {}
|
||||||
middlewares:
|
middlewares:
|
||||||
- common-chain@file
|
- common-chain@file
|
||||||
service: elysia-service
|
service: llm-api-service
|
||||||
|
|
||||||
# ── Rust Auth API ──
|
# ── Tools (Document Scanner & Media Processing) ──
|
||||||
rust-auth:
|
tools:
|
||||||
rule: 'Host(`auth.asepharyana.my.id`) || Host(`auth.asepharyana.web.id`)'
|
rule: 'Host(`tools.asepharyana.my.id`) || Host(`tools.asepharyana.web.id`)'
|
||||||
entryPoints:
|
entryPoints:
|
||||||
- websecure
|
- websecure
|
||||||
tls: {}
|
tls: {}
|
||||||
middlewares:
|
middlewares:
|
||||||
- common-chain@file
|
- common-chain@file
|
||||||
service: rust-auth-service
|
service: tools-service
|
||||||
|
|
||||||
|
# ── Jaeger UI ──
|
||||||
|
jaeger:
|
||||||
|
rule: 'Host(`jaeger.asepharyana.my.id`) || Host(`jaeger.asepharyana.web.id`)'
|
||||||
|
entryPoints:
|
||||||
|
- websecure
|
||||||
|
tls: {}
|
||||||
|
middlewares:
|
||||||
|
- common-chain@file
|
||||||
|
service: jaeger-service
|
||||||
|
|
||||||
services:
|
services:
|
||||||
react-service:
|
hub-service:
|
||||||
loadBalancer:
|
loadBalancer:
|
||||||
servers:
|
servers:
|
||||||
- url: 'http://react-web:80'
|
- url: 'http://hub:3000'
|
||||||
|
|
||||||
scraper-service:
|
scraper-service:
|
||||||
loadBalancer:
|
loadBalancer:
|
||||||
servers:
|
servers:
|
||||||
- url: 'http://scraper-api:4091'
|
- url: 'http://scraper-api:4091'
|
||||||
elysia-service:
|
|
||||||
|
tools-service:
|
||||||
loadBalancer:
|
loadBalancer:
|
||||||
servers:
|
servers:
|
||||||
- url: 'http://elysia-api:4092'
|
- url: 'http://tools:3000'
|
||||||
rust-auth-service:
|
|
||||||
|
llm-api-service:
|
||||||
loadBalancer:
|
loadBalancer:
|
||||||
servers:
|
servers:
|
||||||
- url: 'http://rust-auth:3000'
|
- url: 'http://llm-api:8080'
|
||||||
|
|
||||||
|
jaeger-service:
|
||||||
|
loadBalancer:
|
||||||
|
servers:
|
||||||
|
- url: 'http://jaeger:16686'
|
||||||
|
|||||||
@@ -65,8 +65,6 @@ http:
|
|||||||
common-chain:
|
common-chain:
|
||||||
chain:
|
chain:
|
||||||
middlewares:
|
middlewares:
|
||||||
# - real-ip
|
|
||||||
# - block-sensitive-paths
|
|
||||||
- secure-headers
|
- secure-headers
|
||||||
- compress
|
- compress
|
||||||
- retry
|
- retry
|
||||||
|
|||||||
@@ -1,8 +1,5 @@
|
|||||||
tls:
|
tls:
|
||||||
certificates:
|
certificates:
|
||||||
# Legacy production layout: asephstech.pem is paired with asephscloud.key.
|
|
||||||
- certFile: /etc/traefik/certs/asephstech.pem
|
|
||||||
keyFile: /etc/traefik/certs/asephscloud.key
|
|
||||||
- certFile: /etc/traefik/certs/asepharyana.my.id.pem
|
- certFile: /etc/traefik/certs/asepharyana.my.id.pem
|
||||||
keyFile: /etc/traefik/certs/asepharyana.my.id.key
|
keyFile: /etc/traefik/certs/asepharyana.my.id.key
|
||||||
- certFile: /etc/traefik/certs/asepharyana.web.id.pem
|
- certFile: /etc/traefik/certs/asepharyana.web.id.pem
|
||||||
@@ -10,5 +7,5 @@ tls:
|
|||||||
stores:
|
stores:
|
||||||
default:
|
default:
|
||||||
defaultCertificate:
|
defaultCertificate:
|
||||||
certFile: /etc/traefik/certs/asephstech.pem
|
certFile: /etc/traefik/certs/asepharyana.my.id.pem
|
||||||
keyFile: /etc/traefik/certs/asephscloud.key
|
keyFile: /etc/traefik/certs/asepharyana.my.id.key
|
||||||
|
|||||||
@@ -1,29 +0,0 @@
|
|||||||
api:
|
|
||||||
dashboard: true
|
|
||||||
insecure: true
|
|
||||||
|
|
||||||
log:
|
|
||||||
level: INFO
|
|
||||||
format: json
|
|
||||||
|
|
||||||
accessLog: {}
|
|
||||||
|
|
||||||
entryPoints:
|
|
||||||
web:
|
|
||||||
address: ':80'
|
|
||||||
http:
|
|
||||||
redirections:
|
|
||||||
entryPoint:
|
|
||||||
to: websecure
|
|
||||||
scheme: https
|
|
||||||
websecure:
|
|
||||||
address: ':443'
|
|
||||||
|
|
||||||
providers:
|
|
||||||
docker:
|
|
||||||
endpoint: 'unix:///var/run/docker.sock'
|
|
||||||
exposedByDefault: false
|
|
||||||
network: app-shared-net
|
|
||||||
file:
|
|
||||||
directory: /etc/traefik/dynamic
|
|
||||||
watch: true
|
|
||||||
@@ -0,0 +1,5 @@
|
|||||||
|
{
|
||||||
|
"$schema": "https://app.kilo.ai/config.json",
|
||||||
|
"instructions": ["CLAUDE.md"],
|
||||||
|
"snapshot": true
|
||||||
|
}
|
||||||
Submodule
+1
Submodule plugins/hub-guide added at abd60c607d
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user