diff --git a/.hermes/plans/phase4-plan.md b/.hermes/plans/phase4-plan.md new file mode 100644 index 0000000..f3f02c8 --- /dev/null +++ b/.hermes/plans/phase4-plan.md @@ -0,0 +1,105 @@ +# MCPedia Phase 4 — Operability & Correctness Hardening + +> Reinterpretation: the PHASES.md "Scale-out" items (OpenSearch, object storage, +> multi-tenant, distributed workers) are YAGNI at KB scale (4 docs). Phase 4 = +> make the Phase 3 async + revision machinery **correct, secure, observable, and +> deployable** — not speculative infra. Each task below fixes a real gap found +> by reading the code, not a hypothetical need. + +## Tasks + +### T1 — `restoreRevision` must rebuild semantic chunks (CORRECTNESS BUG) +**Root cause:** `packages/core/src/revision.service.ts` `restoreRevision` writes +the old body back into `documents` but never calls `indexChunks(slug, body)`. +So after a restore, keyword search (FTS on `documents.body`) is correct but +`document_chunks`/embeddings stay on the *new* body → semantic + hybrid search +return stale/ghost chunks. + +**Fix:** +- Add `reindexChunks(slug)` to `@mcpedia/core` that re-runs `indexChunks(slug, body)` + using the live `documents.body` (the new body after the update). +- Call it inside `restoreRevision` after the `documents` update (wrap in try/catch + like `indexContentFile` so embed failure doesn't abort the restore). +- Add a unit-style assertion to the MCP smoke test or a small script: restore → + `document_chunks` count matches re-chunked body. + +**Files:** `packages/core/src/revision.service.ts`, `packages/core/src/index.ts`, +`packages/core/src/document.service.ts` (export existing `indexChunks` if needed). + +### T2 — Secure the git-sync webhook (SECURITY) +**Root cause:** `apps/api/src/index.ts` `/hooks/reindex` and `/hooks/index` accept +any request with no `WEBHOOK_SECRET` check — `.env.example` defines `WEBHOOK_SECRET` +but the router never reads it. + +**Fix:** +- In `apps/api/src/index.ts`, compare `c.req.header("x-webhook-secret")` (or + `?secret=`) against `WEBHOOK_SECRET` (from `@mcpedia/config`). If unset/mismatch → + `401`. If `WEBHOOK_SECRET` env is empty, reject at startup with a clear log + (fail-fast, don't run an open endpoint). +- Add `WEBHOOK_SECRET` to `packages/config/src/index.ts` export. +- Document the header in README + verify with curl (401 without secret, 200 with). + +**Files:** `apps/api/src/index.ts`, `packages/config/src/index.ts`, README. + +### T3 — Web UI revisions view (UX) +**Root cause:** Web UI (server components) calls `@mcpedia/core` directly; there is +no revisions surface even though `revisions`/`restoreRevision` tRPC + MCP resource +exist. + +**Fix (server-component only, no client JS):** +- On the doc page (`apps/web/app/[section]/[...slug]/page.tsx`), fetch + `listRevisions(fullSlug, 10)` and render a "History" panel: revision number, + reason, createdAt, body length, and a `/api/revisions/restore` link/POST that + calls the tRPC `restoreRevision` mutation via a server action or a form POST to + a small route handler. Simplest: a `
` + with hidden `id` + a route handler in `apps/web` calling `restoreRevision`. + Keep it read-mostly; restore is a deliberate action. +- Add `apps/web/app/api/revisions/restore/route.ts` (POST) → `restoreRevision(id)` + → `revalidatePath` the doc. + +**Files:** `apps/web/app/[section]/[...slug]/page.tsx`, +`apps/web/app/api/revisions/restore/route.ts`. + +### T4 — Paginate `listRevisions` / `revisions` API (PERF) +**Root cause:** `revision.service.ts` `listRevisions` does `select length(body)` + (fine) but the tRPC `revisions` and MCP resource return *full* revision rows + including the body in some callers; list endpoints should never carry bodies. + +**Fix:** +- Ensure `listRevisions` summary excludes `body` (it already does — `bodyLength` + only). Add `offset` param for paging. Confirm MCP resource uses the summary. +- No behavior change for the doc page (uses summary). + +**Files:** `packages/core/src/revision.service.ts` (add `offset`), router unchanged. + +### T5 — Deployable as supervised services (OPS) +**Root cause:** `apps/api` and `apps/worker` run only ad-hoc; the host already runs +`zeavis` via Nix/systemd. Phase-4 operability = provide a systemd unit (or Nix +service) so `mcpedia-api` + `mcpedia-worker` start on boot and restart on failure. + +**Fix (Nix-first, per MEMORY):** +- Write `mcpedia-api.service` + `mcpedia-worker.service` systemd unit files under + `deploy/` (bun run api / bun run worker, `WorkingDirectory`, `Restart=on-failure`, + `EnvironmentFile` pointing at `.env`, `After=network-online.target`). +- README section "Run as a service" with `cp deploy/*.service /etc/systemd/system && systemctl daemon-reload && systemctl enable --now mcpedia-api mcpedia-worker`. +- **Do NOT** `systemctl` on the host without user confirmation (changing live + services). Provide the files + instructions only; user runs enable. + +**Files:** `deploy/mcpedia-api.service`, `deploy/mcpedia-worker.service`, README. + +## Verification (all real, against imrnes Redis + Postgres) +1. `turbo run typecheck` + `turbo run build` green. +2. T1: script — edit a doc, reindex (new revision + new chunks), restore rev #1, + assert `document_chunks` count for that slug now matches re-chunk of rev #1 body + and `semanticSearch` on a term unique to rev #1 returns it. +3. T2: `curl -XPOST localhost:4020/hooks/reindex` → 401; with + `-H "x-webhook-secret: $WEBHOOK_SECRET"` → 200 + jobId. +4. T3: `next build` includes the History panel; restore form rebuilds chunks + (verified via T1 path through the route handler). +5. T4: `revisions` API returns summaries without body; offset paging works. +6. T5: `systemd-analyze verify deploy/*.service` passes (off-host safe check); + README documents enable steps. + +## Out of scope (YAGNI, keep deferred per PHASES.md) +OpenSearch/Elasticsearch, object storage, multi-tenant, distributed workers, +pgvector migration. Revisit only when corpus > ~10k docs or query latency bites. diff --git a/PHASES.md b/PHASES.md index d1905e1..da60978 100644 --- a/PHASES.md +++ b/PHASES.md @@ -72,9 +72,53 @@ bun run api # Hono+tRPC API on :4020 (added /hooks/* webhooks) - API webhook `POST /hooks/reindex` enqueues → worker drains queue → `queueStatus` reflects counts. -## Phase 4 — Scale-out (only if needed) +## Phase 4 — Operability & Correctness Hardening ✅ DONE + +> Reinterpreted from the original "Scale-out" plan: OpenSearch/object-storage/ +> multi-tenant were flagged YAGNI at KB scale (4 docs), so Phase 4 = make the +> Phase 3 async + revision machinery **correct, secure, observable, deployable**. + +- [x] **T1 — `restoreRevision` rebuilds semantic chunks (CORRECTNESS BUG)** — + previously restore wrote the old body into `documents` but left `document_chunks` + on the *new* body, so semantic/hybrid search went stale after a restore. + `@mcpedia/core` `reindexChunks(slug)` now re-chunks + re-embeds from the live + body; `restoreRevision` calls it after the update (embed failure is logged, not + thrown). Verified: restore → `document_chunks` count matches re-chunk of the + restored body. +- [x] **T2 — Secure git-sync webhook (SECURITY)** — `/hooks/*` now require an + `x-webhook-secret` header matching `WEBHOOK_SECRET` (401 otherwise). API + fails fast at startup if `WEBHOOK_SECRET` is unset (no open endpoint). Added + `WEBHOOK_SECRET` to `@mcpedia/config` + `.env.example`; generated a real secret + in the local `.env` (gitignored). +- [x] **T3 — Web UI revisions view (UX)** — doc page now shows a "History" panel + (revision no, reason, date, body length) with a per-revision Restore button. + Restore POSTs to `apps/web/app/api/revisions/restore/route.ts` → `restoreRevision` + → `revalidatePath` (server-component only, no client JS). +- [x] **T4 — Paginate `listRevisions`** — added `offset` param (summary never + includes body). API `revisions` + MCP resource use the summary. +- [x] **T5 — Deploy as supervised services (OPS)** — `deploy/mcpedia-api.service` + + `deploy/mcpedia-worker.service` systemd units (`Restart=on-failure`, + `EnvironmentFile=.env`, `WorkingDirectory=/home/code/mcpedia`). Enable with: + `cp deploy/*.service /etc/systemd/system && systemctl daemon-reload && + systemctl enable --now mcpedia-api mcpedia-worker`. (Not auto-enabled on host + without explicit user go-ahead.) + +### Verification done (real, against imrnes Redis + Postgres) +- `turbo run typecheck` + `turbo run build` green (incl. `next build` with the + History panel). +- T1: edit → reindex (new revision + chunks) → restore rev #1 → `document_chunks` + count for that slug matches re-chunk of rev #1; `semanticSearch` on a term + unique to rev #1 returns it. +- T2: `curl -XPOST /hooks/reindex` → 401; with `-H "x-webhook-secret: $WEBHOOK_SECRET"` + → 200 + jobId; job drains via worker. +- T3: History panel renders; restore route rebuilds chunks (T1 path). +- T4: `revisions` returns summaries (no body); `offset` paging works. +- T5: `systemd-analyze verify deploy/*.service` passes (off-host safe check). + +## Phase 5 — Deferred scale-out (only when needed) - [ ] Dedicated search engine (OpenSearch/Elasticsearch) — YAGNI until FTS is insufficient +- [ ] pgvector migration (install on imrnes Postgres) — when `real[]` cosine stalls - [ ] Object storage for assets - [ ] Advanced ranking, distributed workers, observability, multi-tenant diff --git a/README.md b/README.md index e728cf6..aced8b7 100644 --- a/README.md +++ b/README.md @@ -140,9 +140,30 @@ Git-sync webhooks (enqueue BullMQ jobs; the worker processes them): push webhook here to auto-reindex on push). - `POST /hooks/index?slug=` — reindex a single document. +> **Security:** both webhooks require an `x-webhook-secret` header that matches +> `WEBHOOK_SECRET` (set in `.env`). The API refuses to start if `WEBHOOK_SECRET` +> is unset, so the hooks are never left open. + `bun run index` now also chunks + embeds (Phase 2 indexer) and snapshots a revision whenever the body changes (Phase 3). See `.env.example` for -`EMBED_*` / `REDIS_*` / `QUEUE_PREFIX` vars. +`EMBED_*` / `REDIS_*` / `QUEUE_PREFIX` / `WEBHOOK_SECRET` vars. + +### Run as a supervised service (Phase 4) + +`deploy/mcpedia-api.service` + `deploy/mcpedia-worker.service` are systemd units +(`Restart=on-failure`, `EnvironmentFile=.env`, `WorkingDirectory=/home/code/mcpedia`). +Enable them with: + +```bash +sudo cp deploy/*.service /etc/systemd/system/ +sudo systemctl daemon-reload +sudo systemctl enable --now mcpedia-api mcpedia-worker +# tail logs +journalctl -u mcpedia-api -u mcpedia-worker -f +``` + +The API should sit behind Caddy (or your reverse proxy) for TLS; expose only +`:4020` internally and the web app publicly. ## Status diff --git a/apps/api/src/index.ts b/apps/api/src/index.ts index 5ad635a..d59001a 100644 --- a/apps/api/src/index.ts +++ b/apps/api/src/index.ts @@ -5,22 +5,40 @@ import { db } from "@mcpedia/db"; import { appRouter } from "./router"; import type { Context } from "./trpc"; import { enqueueIndexDoc, enqueueFullIndex } from "@mcpedia/queue"; +import { WEBHOOK_SECRET } from "@mcpedia/config"; + +// Fail fast: never expose an open git-sync endpoint. If the operator hasn't +// set WEBHOOK_SECRET, refuse to start rather than run an unauthenticated hook. +if (!WEBHOOK_SECRET) { + throw new Error( + "WEBHOOK_SECRET is not set — /hooks/* would be open. Set it (see .env.example) before starting the API.", + ); +} const app = new Hono(); -// Health check. +// Health check (no auth — safe to expose). app.get("/health", (c) => c.json({ ok: true })); +// Shared guard for the git-sync webhooks: require `x-webhook-secret` header to +// match the configured secret. Reject anything else with 401. +function assertWebhookAuth(c: { req: { header: (k: string) => string | undefined } }): boolean { + const provided = c.req.header("x-webhook-secret"); + return provided != null && provided === WEBHOOK_SECRET; +} + // --- Phase 3: Git synchronization hook --- // POST /hooks/reindex -> enqueue a full-corpus reindex (git push webhook) // POST /hooks/index?slug=... -> enqueue a single document reindex // Returns the created job id(s). The worker processes them asynchronously. app.post("/hooks/reindex", async (c) => { + if (!assertWebhookAuth(c)) return c.json({ ok: false, error: "unauthorized" }, 401); const job = await enqueueFullIndex("git-push"); return c.json({ ok: true, jobId: job.id, kind: "full" }); }); app.post("/hooks/index", async (c) => { + if (!assertWebhookAuth(c)) return c.json({ ok: false, error: "unauthorized" }, 401); const slug = c.req.query("slug"); if (!slug) return c.json({ ok: false, error: "slug query param required" }, 400); // slug is the relative path without extension, e.g. docs/websocket/contract diff --git a/apps/web/app/[section]/[...slug]/page.tsx b/apps/web/app/[section]/[...slug]/page.tsx index 3a0bdf2..c335a76 100644 --- a/apps/web/app/[section]/[...slug]/page.tsx +++ b/apps/web/app/[section]/[...slug]/page.tsx @@ -1,6 +1,6 @@ import Link from "next/link"; import { notFound } from "next/navigation"; -import { getDocument, listDocuments, getRelated } from "@mcpedia/core"; +import { getDocument, listDocuments, getRelated, listRevisions } from "@mcpedia/core"; import Markdown from "@/components/Markdown"; export default async function DocPage({ @@ -14,6 +14,7 @@ export default async function DocPage({ if (!doc) notFound(); const related = await getRelated(fullSlug, 5); + const revisions = await listRevisions(fullSlug, 10); return (
@@ -48,6 +49,34 @@ export default async function DocPage({ )} + + {revisions.length > 0 && ( + + )}
); } diff --git a/apps/web/app/api/revisions/restore/route.ts b/apps/web/app/api/revisions/restore/route.ts new file mode 100644 index 0000000..581ded2 --- /dev/null +++ b/apps/web/app/api/revisions/restore/route.ts @@ -0,0 +1,31 @@ +import { NextRequest, NextResponse } from "next/server"; +import { restoreRevision, getRevision } from "@mcpedia/core"; +import { revalidatePath } from "next/cache"; + +// POST /api/revisions/restore — restore a document to a past revision. +// Body (form-urlencoded): id= +// After restoring, we rebuild semantic chunks (handled inside restoreRevision) +// and revalidate the doc page so the Web UI reflects the restored body. +export async function POST(req: NextRequest) { + const form = await req.formData().catch(() => null); + const id = form?.get("id"); + if (typeof id !== "string" || id.length === 0) { + return NextResponse.json({ ok: false, error: "missing id" }, { status: 400 }); + } + + const rev = await getRevision(id); + if (!rev) { + return NextResponse.json({ ok: false, error: "revision not found" }, { status: 404 }); + } + + const result = await restoreRevision(id); + if (!result) { + return NextResponse.json({ ok: false, error: "restore failed" }, { status: 500 }); + } + + // Revalidate the doc route + home so the change is visible immediately. + revalidatePath(`/${result.slug}`); + revalidatePath("/"); + + return NextResponse.redirect(new URL(`/${result.slug}`, req.url)); +} diff --git a/deploy/mcpedia-api.service b/deploy/mcpedia-api.service new file mode 100644 index 0000000..17fc02f --- /dev/null +++ b/deploy/mcpedia-api.service @@ -0,0 +1,24 @@ +[Unit] +Description=MCPedia API (tRPC/Hono + git-sync webhooks) +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +WorkingDirectory=/home/code/mcpedia +# Loads DATABASE_URL, REDIS_*, EMBED_*, WEBHOOK_SECRET from the repo .env +# (.env is gitignored; for prod, point this at a deployed secret file). +EnvironmentFile=/home/code/mcpedia/.env +ExecStart=/usr/bin/env bun run api +Restart=on-failure +RestartSec=5 +User=code +Group=code +# The API needs WEBHOOK_SECRET; fail-fast is built into the app if it's missing. +NoNewPrivileges=true +PrivateTmp=true +MemoryMax=512M +TasksMax=256 + +[Install] +WantedBy=multi-user.target diff --git a/deploy/mcpedia-worker.service b/deploy/mcpedia-worker.service new file mode 100644 index 0000000..ed08f8d --- /dev/null +++ b/deploy/mcpedia-worker.service @@ -0,0 +1,23 @@ +[Unit] +Description=MCPedia indexing/embedding worker (BullMQ) +After=network-online.target redis.service +Wants=network-online.target + +[Service] +Type=simple +WorkingDirectory=/home/code/mcpedia +# Loads DATABASE_URL, REDIS_*, EMBED_* from the repo .env +# (.env is gitignored; for prod, point this at a deployed secret file). +EnvironmentFile=/home/code/mcpedia/.env +ExecStart=/usr/bin/env bun run worker +Restart=on-failure +RestartSec=5 +User=code +Group=code +NoNewPrivileges=true +PrivateTmp=true +MemoryMax=1G +TasksMax=256 + +[Install] +WantedBy=multi-user.target diff --git a/packages/config/src/index.ts b/packages/config/src/index.ts index 2582a16..e4fccea 100644 --- a/packages/config/src/index.ts +++ b/packages/config/src/index.ts @@ -46,6 +46,10 @@ export const REDIS_PASSWORD = process.env.REDIS_PASSWORD ?? ""; // BullMQ key prefix to namespace jobs on the shared Redis instance. export const QUEUE_PREFIX = process.env.QUEUE_PREFIX ?? "mcpedia"; +// Phase 4: git-sync webhook shared secret. The API /hooks/* endpoints require +// this header (x-webhook-secret) to match, so an open port can't trigger reindex. +export const WEBHOOK_SECRET = process.env.WEBHOOK_SECRET ?? ""; + if (!DATABASE_URL) { // Fail fast with an explicit message instead of a cryptic driver error. throw new Error( diff --git a/packages/core/src/index.service.ts b/packages/core/src/index.service.ts index 15ccb0d..a28b8a9 100644 --- a/packages/core/src/index.service.ts +++ b/packages/core/src/index.service.ts @@ -131,6 +131,28 @@ async function snapshotRevision( return true; } +/** + * Rebuild the semantic chunks + embeddings for a slug from its CURRENT live + * `documents.body`. Used after `restoreRevision` so semantic/hybrid search + * stay consistent with the restored body (otherwise chunks would be stale). + * Embed failures are logged, not thrown — FTS remains the source of truth. + */ +export async function reindexChunks(slug: string): Promise { + const [doc] = await db + .select({ body: documents.body }) + .from(documents) + .where(eq(documents.slug, slug)); + if (!doc) return 0; + try { + return await indexChunks(slug, doc.body); + } catch (err) { + console.error( + ` reindexChunks embed FAILED for ${slug}: ${err instanceof Error ? err.message : err}`, + ); + return 0; + } +} + /** * Walk the entire content tree and index every file. Returns aggregate counts. */ diff --git a/packages/core/src/revision.service.ts b/packages/core/src/revision.service.ts index 5916a17..f07288f 100644 --- a/packages/core/src/revision.service.ts +++ b/packages/core/src/revision.service.ts @@ -1,5 +1,6 @@ import { db } from "@mcpedia/db"; import { documents, documentRevisions, documentChunks } from "@mcpedia/db/schema"; +import { reindexChunks } from "./index.service"; import { eq, desc, and, sql } from "drizzle-orm"; import { toMeta } from "./row-map"; import type { DocumentMeta } from "@mcpedia/types"; @@ -14,10 +15,11 @@ export interface RevisionSummary { bodyLength: number; } -/** List revisions for a slug, newest first. */ +/** List revisions for a slug, newest first. `offset` enables paging. */ export async function listRevisions( slug: string, limit = 20, + offset = 0, ): Promise { const [doc] = await db .select({ id: documents.id }) @@ -38,7 +40,8 @@ export async function listRevisions( .from(documentRevisions) .where(eq(documentRevisions.documentId, doc.id)) .orderBy(desc(documentRevisions.revisionNo)) - .limit(limit); + .limit(limit) + .offset(offset); return rows.map((r) => ({ id: r.id, @@ -112,5 +115,10 @@ export async function restoreRevision( }) .where(eq(documents.id, rev.documentId)); + // Rebuild semantic chunks + embeddings from the restored body so semantic + // and hybrid search stay consistent (otherwise document_chunks would hold + // the NEW body's chunks while documents.body holds the OLD/restore body). + await reindexChunks(rev.slug); + return { slug: rev.slug, documentId: rev.documentId }; }