diff --git a/.gitmodules b/.gitmodules index f7c1206..369a887 100644 --- a/.gitmodules +++ b/.gitmodules @@ -10,3 +10,6 @@ [submodule "plugins/hub-guide"] path = plugins/hub-guide url = https://github.com/asepharyana/asepharyana-hub-hub-guide.git +[submodule "apps/llm-api"] + path = apps/llm-api + url = https://github.com/asepharyana/llm-api.git diff --git a/apps/hub b/apps/hub index 62d1865..d793ebf 160000 --- a/apps/hub +++ b/apps/hub @@ -1 +1 @@ -Subproject commit 62d1865c0f29017e421b336a269ec8c5f106555f +Subproject commit d793ebf9b41fa1c9f9453438ccdae76b09eeb4f5 diff --git a/apps/llm-api b/apps/llm-api new file mode 160000 index 0000000..6edf6ce --- /dev/null +++ b/apps/llm-api @@ -0,0 +1 @@ +Subproject commit 6edf6cee5d778a7cf693e1344cdc7b870f36bb13 diff --git a/infra/compose/llm-api.yml b/infra/compose/llm-api.yml new file mode 100644 index 0000000..e64ce93 --- /dev/null +++ b/infra/compose/llm-api.yml @@ -0,0 +1,36 @@ +services: + llm-api: + container_name: llm-api + image: ghcr.io/asepharyana/asepharyana-hub/llm-api:latest + restart: always + networks: + app-shared-net: + aliases: + - llm-api + env_file: + - ../../.env + environment: + - MODEL_PATH=/models/MiniCPM-V-4.6-Q4_K_M.gguf + - API_KEY=${LLM_API_KEY:-} + volumes: + - /root/models/gguf:/models:ro + healthcheck: + test: ['CMD-SHELL', 'curl -so /dev/null --connect-timeout 5 http://localhost:8080/health || test $? -eq 22'] + interval: 30s + timeout: 10s + retries: 5 + start_period: 60s + labels: + prometheus.io/scrape: "true" + prometheus.io/port: "8080" + deploy: + resources: + limits: + memory: 2G + reservations: + memory: 1G + +networks: + app-shared-net: + name: app-shared-net + external: true diff --git a/infra/docker/llm-api.Dockerfile b/infra/docker/llm-api.Dockerfile new file mode 100644 index 0000000..d24bf5c --- /dev/null +++ b/infra/docker/llm-api.Dockerfile @@ -0,0 +1,39 @@ +# ── Build stage: cargo-chef ── +FROM lukemathwalker/cargo-chef:latest-rust-1.89.0 AS chef +RUN apt-get update && apt-get install -y --no-install-recommends libclang-dev cmake && rm -rf /var/lib/apt/lists/* +WORKDIR /app + +FROM chef AS planner +COPY apps/llm-api . +RUN cargo chef prepare --recipe-path recipe.json + +FROM chef AS builder +COPY --from=planner /app/recipe.json recipe.json +RUN --mount=type=cache,target=/usr/local/cargo/registry \ + --mount=type=cache,target=/app/target \ + cargo chef cook --release --recipe-path recipe.json + +COPY apps/llm-api . +RUN --mount=type=cache,target=/usr/local/cargo/registry \ + --mount=type=cache,target=/app/target \ + cargo build --release && \ + cp target/release/llm-api /app/llm-api + +# ── Runtime ── +FROM debian:bookworm-slim AS runtime +RUN apt-get update && apt-get install -y --no-install-recommends \ + ca-certificates \ + curl \ + libssl3 \ + libgomp1 \ + && rm -rf /var/lib/apt/lists/* + +RUN groupadd -g 1001 appgroup && \ + useradd -u 1001 -g appgroup -s /bin/sh appuser + +WORKDIR /app +COPY --from=builder /app/llm-api /app/llm-api +USER appuser + +EXPOSE 8080 +CMD ["./llm-api"] diff --git a/infra/traefik/dynamic/apps.yaml b/infra/traefik/dynamic/apps.yaml index 0cfdc08..34eec08 100644 --- a/infra/traefik/dynamic/apps.yaml +++ b/infra/traefik/dynamic/apps.yaml @@ -20,6 +20,16 @@ http: - common-chain@file service: scraper-service + # ── LLM API (MiniCPM-V) ── + llm-api: + rule: 'Host(`ai.asepharyana.my.id`) || Host(`ai.asepharya.web.id`)' + entryPoints: + - websecure + tls: {} + middlewares: + - common-chain@file + service: llm-api-service + # ── Tools (Document Scanner & Media Processing) ── tools: rule: 'Host(`tools.asepharyana.my.id`) || Host(`tools.asepharyana.web.id`)' @@ -56,6 +66,11 @@ http: servers: - url: 'http://tools:3000' + llm-api-service: + loadBalancer: + servers: + - url: 'http://llm-api:8080' + jaeger-service: loadBalancer: servers: