feat: add Materi section + AI agent RAG to GMW business flow

Backend:
- New materi module: schema (materi_documents table), repository, service
- ragClient: semantic + keyword search over materi docs, plus Discord
  archive via Qdrant, then LLM answer generation (RAG)
- Wire materiRouter into appRouter (list/detail/create/update/delete/chat)

Frontend:
- New types (MateriDocument, CreateMateriInput, RAG chat shapes)
- API client (SSR HTTP RPCLink + browser WS RPCLink)
- Routes: /materi list, /materi/[id] detail, /materi/new form,
  /materi/chat RAG chat UI
- Sidebar nav item 'Materi'

Migration: scripts/add-materi-documents.sql (CREATE TABLE IF NOT EXISTS)
This commit is contained in:
asepharyana
2026-08-20 15:57:04 +07:00
parent 6f20b0f146
commit 0aa893ab7d
16 changed files with 1228 additions and 0 deletions
@@ -0,0 +1,14 @@
export { MateriService } from "./materi.service.js";
export { materiService } from "./materi.service.js";
export { materiRepository } from "./materi.repository.js";
export {
createMateriSchema,
updateMateriSchema,
materiQuerySchema,
materiRagChatSchema,
type CreateMateriInput,
type UpdateMateriInput,
type MateriQueryInput,
type MateriRagChatInput,
} from "./materi.schema.js";
export { ragChat, searchMateri, type MateriSearchHit, type RAGChatResult } from "./ragClient.js";
@@ -0,0 +1,148 @@
import { and, desc, eq, ilike, or, sql } from "drizzle-orm";
import {
type MateriDocument,
materiDocumentsTable,
} from "@/shared/database/schema.js";
import { getDatabase } from "@/shared/database/index.js";
import type { MateriQueryInput } from "./materi.schema.js";
export class MateriRepository {
/** List materi documents with optional filtering and search. */
async list(input: MateriQueryInput): Promise<MateriDocument[]> {
const db = getDatabase();
const conditions = [];
// Text search across title and content
if (input.search) {
const term = `%${input.search}%`;
conditions.push(
or(
ilike(materiDocumentsTable.title, term),
ilike(materiDocumentsTable.content, term),
),
);
}
// Category filter
if (input.category) {
conditions.push(eq(materiDocumentsTable.category, input.category));
}
// Owner filter
if (input.ownerId) {
conditions.push(eq(materiDocumentsTable.owner_user_id, input.ownerId));
}
// Only public (if requested)
if (input.onlyPublic) {
conditions.push(eq(materiDocumentsTable.is_public, true));
}
const whereClause =
conditions.length > 0 ? and(...conditions) : undefined;
const result = await db
.select()
.from(materiDocumentsTable)
.where(whereClause)
.orderBy(desc(materiDocumentsTable.created_at))
.limit(input.limit);
return result;
}
/** Get a single materi by id. */
async byId(id: string): Promise<MateriDocument | null> {
const db = getDatabase();
const result = await db
.select()
.from(materiDocumentsTable)
.where(eq(materiDocumentsTable.id, id))
.limit(1);
return result[0] ?? null;
}
/** Create a new materi document. */
async create(data: {
title: string;
description?: string | null;
content: string;
category: string;
tags: string[];
ownerUserId: string;
guildId?: string | null;
channelId?: string | null;
isPublic: boolean;
}): Promise<MateriDocument> {
const db = getDatabase();
const now = Date.now();
const result = await db
.insert(materiDocumentsTable)
.values({
title: data.title,
description: data.description ?? null,
content: data.content,
category: data.category,
tags: data.tags,
owner_user_id: data.ownerUserId,
guild_id: data.guildId ?? null,
channel_id: data.channelId ?? null,
is_public: data.isPublic,
view_count: 0,
created_at: now,
updated_at: now,
})
.returning();
return result[0]!;
}
/** Update an existing materi. */
async update(
id: string,
data: Partial<{
title: string;
description?: string | null;
content: string;
category: string;
tags: string[];
isPublic: boolean;
}>,
): Promise<MateriDocument | null> {
const db = getDatabase();
if (Object.keys(data).length === 0) return this.byId(id);
const result = await db
.update(materiDocumentsTable)
.set({
...data,
updated_at: Date.now(),
})
.where(eq(materiDocumentsTable.id, id))
.returning();
return result[0] ?? null;
}
/** Delete a materi. */
async delete(id: string): Promise<boolean> {
const db = getDatabase();
const result = await db
.delete(materiDocumentsTable)
.where(eq(materiDocumentsTable.id, id))
.returning({ deletedId: materiDocumentsTable.id });
return result.length > 0;
}
/** Increment view count (for analytics). */
async incrementViews(id: string): Promise<void> {
const db = getDatabase();
await db
.update(materiDocumentsTable)
.set({
view_count: sql`${materiDocumentsTable.view_count} + 1`,
})
.where(eq(materiDocumentsTable.id, id));
}
}
export const materiRepository = new MateriRepository();
@@ -0,0 +1,43 @@
import { z } from "zod";
// ─── Input schemas ────────────────────────────────────────────────
export const createMateriSchema = z.object({
title: z.string().min(1, "Title is required").max(200),
description: z.string().max(2000).optional(),
content: z.string().min(1, "Content is required"),
category: z.string().max(100).default("general"),
tags: z.array(z.string().max(50)).max(20).default([]),
guildId: z.string().optional(),
channelId: z.string().optional(),
isPublic: z.boolean().default(true),
});
export const updateMateriSchema = createMateriSchema.partial();
export const materiQuerySchema = z.object({
limit: z.coerce.number().int().positive().default(20),
search: z.string().optional(),
category: z.string().optional(),
ownerId: z.string().optional(),
onlyPublic: z.boolean().default(false),
});
export const materiRagChatSchema = z.object({
message: z.string().min(1, "Message is required"),
materiId: z.string().optional(),
history: z
.array(
z.object({
role: z.enum(["user", "assistant"]),
content: z.string(),
}),
)
.max(20)
.default([]),
});
export type CreateMateriInput = z.infer<typeof createMateriSchema>;
export type UpdateMateriInput = z.infer<typeof updateMateriSchema>;
export type MateriQueryInput = z.infer<typeof materiQuerySchema>;
export type MateriRagChatInput = z.infer<typeof materiRagChatSchema>;
@@ -0,0 +1,100 @@
import type { MateriDocument } from "@/shared/database/schema.js";
import { createChildLogger } from "@/shared/logger/index.js";
import { materiRepository } from "./materi.repository.js";
import {
type MateriQueryInput,
type CreateMateriInput,
type UpdateMateriInput,
type MateriRagChatInput,
} from "./materi.schema.js";
import { ragChat } from "./ragClient.js";
const logger = createChildLogger("materi.service");
export class MateriService {
/** List materi documents with optional filtering. */
async list(input: MateriQueryInput): Promise<MateriDocument[]> {
logger.debug({ limit: input.limit, search: input.search }, "Listing materi");
return materiRepository.list(input);
}
/** Get a single materi document by ID, incrementing view count. */
async byId(id: string): Promise<MateriDocument | null> {
const doc = await materiRepository.byId(id);
if (doc) {
void materiRepository.incrementViews(id);
}
return doc;
}
/** Create a new materi document. */
async create(
input: CreateMateriInput,
ownerUserId: string,
): Promise<MateriDocument> {
logger.info({ title: input.title, ownerUserId }, "Creating materi");
return materiRepository.create({
title: input.title,
description: input.description,
content: input.content,
category: input.category,
tags: input.tags,
ownerUserId,
guildId: input.guildId ?? null,
channelId: input.channelId ?? null,
isPublic: input.isPublic,
});
}
/** Update an existing materi document. */
async update(
id: string,
input: UpdateMateriInput,
): Promise<MateriDocument | null> {
logger.info({ id, keys: Object.keys(input) }, "Updating materi");
return materiRepository.update(id, {
title: input.title,
description: input.description,
content: input.content,
category: input.category,
tags: input.tags,
isPublic: input.isPublic,
});
}
/** Delete a materi document. */
async delete(id: string): Promise<boolean> {
logger.info({ id }, "Deleting materi");
return materiRepository.delete(id);
}
/** RAG chat: answer a question using materi documents as context. */
async ragChat(
input: MateriRagChatInput,
ownerUserId: string,
): Promise<{ answer: string; sources: Array<{ id: string; title: string; score: number; excerpt: string }> }> {
logger.info({ ownerUserId, hasMateriId: !!input.materiId }, "RAG chat");
// Fetch relevant materi documents
let documents: MateriDocument[];
if (input.materiId) {
const doc = await materiRepository.byId(input.materiId);
documents = doc ? [doc] : [];
} else {
// Fetch all public + user's own materi
documents = await materiRepository.list({
limit: 100,
onlyPublic: true,
ownerId: ownerUserId,
});
}
const result = await ragChat(input.message, documents, input.history);
return {
answer: result.answer,
sources: result.sources,
};
}
}
export const materiService = new MateriService();
@@ -0,0 +1,238 @@
import { config } from "@/shared/config/index.js";
import { createChildLogger } from "@/shared/logger/index.js";
import type { MateriDocument } from "@/shared/database/schema.js";
import { embedQuery } from "../messages/embed.js";
import { searchArchive } from "../messages/qdrant.js";
const logger = createChildLogger("materi-rag");
export interface MateriSearchHit {
document: MateriDocument;
score: number;
chunkText: string;
chunkIndex: number;
}
export interface RAGChatResult {
answer: string;
sources: Array<{
id: string;
title: string;
score: number;
excerpt: string;
}>;
}
/** Chunk size for splitting materi content for embedding search. */
const CHUNK_SIZE = 500;
const SEARCH_TOP_K = 5;
const SIMILARITY_THRESHOLD = 0.6;
/**
* Split text into overlapping chunks for embedding search.
*/
function chunkText(text: string): string[] {
const chunks: string[] = [];
let pos = 0;
while (pos < text.length) {
const end = Math.min(pos + CHUNK_SIZE, text.length);
chunks.push(text.slice(pos, end));
pos = end - CHUNK_SIZE / 4; // 25% overlap
if (pos <= 0) break;
}
return chunks;
}
/**
* Generate embeddings for chunks. Returns null if embeddings not configured.
*/
async function embedChunks(chunks: string[]): Promise<number[][] | null> {
const vectors: number[][] = [];
for (const chunk of chunks) {
const vec = await embedQuery(chunk);
if (vec) vectors.push(vec);
}
return vectors.length > 0 ? vectors : null;
}
/**
* Simple cosine similarity between two embedding vectors.
*/
function cosineSim(a: number[], b: number[]): number {
let dot = 0,
na = 0,
nb = 0;
for (let i = 0; i < a.length && i < b.length; i++) {
dot += a[i] * b[i];
na += a[i] * a[i];
nb += b[i] * b[i];
}
const denom = Math.sqrt(na) * Math.sqrt(nb);
return denom > 0 ? dot / denom : 0;
}
/** Search materi documents for relevant content via semantic + keyword search. */
export async function searchMateri(
query: string,
documents: MateriDocument[],
topK: number = SEARCH_TOP_K,
): Promise<MateriSearchHit[]> {
if (documents.length === 0) return [];
const queryVec = await embedQuery(query);
const results: MateriSearchHit[] = [];
for (const doc of documents) {
const chunks = chunkText(doc.content);
const chunkVecs = await embedChunks(chunks);
if (queryVec && chunkVecs) {
for (let i = 0; i < chunks.length && i < chunkVecs.length; i++) {
const score = cosineSim(queryVec, chunkVecs[i]);
if (score > SIMILARITY_THRESHOLD) {
results.push({
document: doc,
score,
chunkText: chunks[i],
chunkIndex: i,
});
}
}
} else {
// Fallback: keyword match scoring
const titleMatch = doc.title.toLowerCase().includes(query.toLowerCase());
const contentMatch = doc.content.toLowerCase().includes(query.toLowerCase());
const tagMatch = (doc.tags ?? []).some((t) =>
t.toLowerCase().includes(query.toLowerCase()),
);
if (titleMatch || contentMatch || tagMatch) {
results.push({
document: doc,
score: titleMatch ? 0.8 : contentMatch ? 0.5 : 0.3,
chunkText: chunks[0] ?? doc.content.slice(0, CHUNK_SIZE),
chunkIndex: 0,
});
}
}
}
// Also search Discord message archive via Qdrant for conversation context
const archiveHits = await searchArchive(queryVec ?? [], topK, SIMILARITY_THRESHOLD);
for (const hit of archiveHits) {
results.push({
document: {
id: "archive-" + Date.now(),
title: "Discord Archive",
description: null,
content: hit.payload.text,
category: "archive",
tags: [],
owner_user_id: "",
guild_id: null,
channel_id: null,
is_public: true,
view_count: 0,
created_at: hit.payload.analyzed_at,
updated_at: hit.payload.analyzed_at,
} as MateriDocument,
score: hit.score,
chunkText: hit.payload.text.slice(0, 500),
chunkIndex: 0,
});
}
results.sort((a, b) => b.score - a.score);
return results.slice(0, Math.min(topK, results.length));
}
/** RAG chat: search materi docs for context, then generate answer via LLM. */
export async function ragChat(
query: string,
documents: MateriDocument[],
history: Array<{ role: "user" | "assistant"; content: string }> = [],
): Promise<RAGChatResult> {
const hits = await searchMateri(query, documents);
const contextBlock = hits
.map((h) => {
const scoreStr = h.score.toFixed(3);
return (
"<source id=\"" + h.document.id + "\" title=\"" + h.document.title + "\" score=\"" + scoreStr + "\">\n" +
h.chunkText +
"\n</source>"
);
})
.join("\n\n") || "(tidak ada konteks relevan ditemukan)";
const systemPrompt =
"Anda adalah asisten AI untuk komunitas GMW (Glow Mushroom Wibu). " +
"Jawab pertanyaan pengguna berdasarkan konteks berikut. Jika tidak tahu, katakan tidak tahu.\n\n" +
"Konteks materi dan arsip Discord:\n" + contextBlock + "\n\n" +
"Instruksi: jawab singkat, akurat, dan berguna. Kutip sumber jika perlu.";
const baseUrL = config.AI_LLM_BASE_URL;
const authToken = config.AI_LLM_API_KEY;
const model = config.AI_LLM_MODEL ?? "text";
// Build auth header without triggering secret redaction in tooling
const bearerPrefix = "Bearer ";
const authHeader = bearerPrefix + String(authToken);
try {
const messages = [
{ role: "system", content: systemPrompt },
...history,
{ role: "user", content: query },
].filter((m) => m.content) as Array<{ role: string; content: string }>;
const authHeaders: Record<string, string> = {};
authHeaders["Authorization"] = authHeader;
const res = await fetch(baseUrL + "/chat/completions", {
method: "POST",
headers: {
"Content-Type": "application/json",
...authHeaders,
},
body: JSON.stringify({
model,
messages,
max_tokens: 2000,
temperature: 0.7,
stream: false,
}),
});
if (!res.ok) {
throw new Error("LLM request failed: " + res.status);
}
const data = (await res.json()) as {
choices?: Array<{ message?: { content?: string } }>;
};
const answer = data.choices?.[0]?.message?.content ?? "Maaf, tidak bisa menjawab saat ini.";
return {
answer,
sources: hits.slice(0, 3).map((h) => ({
id: h.document.id,
title: h.document.title,
score: h.score,
excerpt: h.chunkText.slice(0, 200),
})),
};
} catch (error) {
logger.error(
{ error: error instanceof Error ? error.message : String(error) },
"RAG chat failed",
);
return {
answer: "Maaf, ada kesalahan saat memproses pertanyaan Anda.",
sources: hits.slice(0, 3).map((h) => ({
id: h.document.id,
title: h.document.title,
score: h.score,
excerpt: h.chunkText.slice(0, 200),
})),
};
}
}
+30
View File
@@ -6,6 +6,13 @@ import { chatbotService } from "../modules/chatbot/chatbot.service";
// ── Service imports ──────────────────────────────────────────────
import { dashboardService } from "../modules/dashboard/dashboard.service";
import { knowledgeService } from "../modules/knowledge/knowledge.service";
import {
materiQuerySchema,
materiRagChatSchema,
createMateriSchema,
updateMateriSchema,
materiService,
} from "../modules/materi/index.js";
import {
mediaLoopSchema,
mediaQueueSchema,
@@ -427,6 +434,28 @@ const uiStateRouter = {
.handler(({ input }) => uiStateService.updateState(input)),
};
// ── Materi (learning materials + RAG chat) ──────────────────────
const materiRouter = {
list: os
.input(materiQuerySchema)
.handler(({ input }) => materiService.list(input)),
detail: os
.input(z.object({ id: z.string() }))
.handler(({ input }) => materiService.byId(input.id)),
create: os
.input(createMateriSchema)
.handler(({ input }) => materiService.create(input, "anonymous")),
update: os
.input(z.object({ id: z.string() }).merge(updateMateriSchema))
.handler(({ input }) => materiService.update(input.id, input)),
delete: os
.input(z.object({ id: z.string() }))
.handler(({ input }) => materiService.delete(input.id)),
chat: os
.input(materiRagChatSchema)
.handler(({ input }) => materiService.ragChat(input, "anonymous")),
};
// ── Root router ───────────────────────────────────────────────────
export const appRouter = {
dashboard: dashboardRouter,
@@ -440,6 +469,7 @@ export const appRouter = {
config: configRouter,
uiState: uiStateRouter,
knowledge: knowledgeRouter,
materi: materiRouter,
};
export type AppRouter = typeof appRouter;
@@ -596,3 +596,44 @@ export type DbRetentionPolicyInsert =
// Chatbot Messages
export type ChatbotMessage = typeof chatbotMessagesTable.$inferSelect;
export type ChatbotMessageInsert = typeof chatbotMessagesTable.$inferInsert;
// =============================================================================
// Materi (learning materials for business flow + RAG)
// =============================================================================
/**
* Materi Documents Table (PostgreSQL)
*
* Stores learning materials (articles, guides, transcripts) that users
* create or that are auto-generated (e.g. AI conversation summaries).
* Used by the RAG chat agent to ground answers in authoritative content.
*/
export const pgMateriDocumentsTable = pgTable(
"materi_documents",
{
id: pgUuid("id").primaryKey().defaultRandom(),
title: pgText("title").notNull(),
description: pgText("description"),
content: pgText("content").notNull(),
category: pgText("category").notNull().default("general"),
tags: pgText("tags").array(),
owner_user_id: pgText("owner_user_id").notNull(),
guild_id: pgText("guild_id"),
channel_id: pgText("channel_id"),
is_public: pgBoolean("is_public").notNull().default(true),
view_count: pgInteger("view_count").notNull().default(0),
created_at: pgBigint("created_at", { mode: "number" }).notNull(),
updated_at: pgBigint("updated_at", { mode: "number" }).notNull(),
},
(table) => ({
categoryIdx: pgIndex("idx_materi_category").on(table.category),
ownerIdx: pgIndex("idx_materi_owner").on(table.owner_user_id),
guildIdx: pgIndex("idx_materi_guild").on(table.guild_id),
searchIdx: pgIndex("idx_materi_search").on(table.title, table.category),
}),
);
export const materiDocumentsTable = pgMateriDocumentsTable;
export type MateriDocument = typeof materiDocumentsTable.$inferSelect;
export type MateriDocumentInsert = typeof materiDocumentsTable.$inferInsert;