feat: implement backend image classification with TensorFlow.js model

- Extend shared types for image classification, including PredictionProbability, UploaderMetadata, and ImageClassificationRecord.
- Create image_classifications table in the database with necessary fields and foreign key constraints.
- Implement disease mappers to convert database rows to shared disease records.
- Develop uploader client to handle image uploads to external service.
- Create image model service to load and classify images using TensorFlow.js.
- Add API routes for image classification, including GET for history and POST for new classifications.
- Implement frontend components for image classification form and display results.
- Update dashboard to integrate image classification functionality and display results.
- Document implementation plan for backend image classification.

apps/api/src/db/schema.ts

@@ -1,4 +1,4 @@
-import { pgTable, timestamp, uuid, varchar, text, integer } from 'drizzle-orm/pg-core';
+import { pgTable, timestamp, uuid, varchar, text, integer, jsonb, real } from 'drizzle-orm/pg-core';
 
 export const appEvents = pgTable('app_events', {
   id: uuid('id').primaryKey().defaultRandom(),
@@ -30,3 +30,17 @@ export const manualClassifications = pgTable('manual_classifications', {
   location: varchar('location', { length: 160 }).notNull(),
   createdAt: timestamp('created_at', { withTimezone: true }).notNull().defaultNow(),
 });
+
+export const imageClassifications = pgTable('image_classifications', {
+  id: uuid('id').primaryKey().defaultRandom(),
+  predictedDiseaseSlug: varchar('predicted_disease_slug', { length: 80 })
+    .notNull()
+    .references(() => diseaseCatalog.slug),
+  confidence: real('confidence').notNull(),
+  probabilities: jsonb('probabilities').notNull(),
+  imageUrl: text('image_url').notNull(),
+  originalFileName: varchar('original_file_name', { length: 240 }).notNull(),
+  uploaderPublicId: varchar('uploader_public_id', { length: 160 }).notNull(),
+  uploaderPayload: jsonb('uploader_payload').notNull(),
+  createdAt: timestamp('created_at', { withTimezone: true }).notNull().defaultNow(),
+});

apps/api/src/lib/disease-mappers.ts

@@ -0,0 +1,17 @@
+import type { DiseaseCatalogItem, DiseaseSlug, DiseaseLabel, RiskLevel } from '@zeavis/shared';
+import type { diseaseCatalog } from '../db/schema';
+
+export function toDisease(row: typeof diseaseCatalog.$inferSelect): DiseaseCatalogItem {
+  return {
+    slug: row.slug as DiseaseSlug,
+    label: row.label as DiseaseLabel,
+    commonName: row.commonName,
+    summary: row.summary,
+    description: row.description,
+    symptoms: row.symptoms,
+    recommendations: row.recommendations,
+    riskLevel: row.riskLevel as RiskLevel,
+    accentColor: row.accentColor,
+    displayOrder: row.displayOrder,
+  };
+}

apps/api/src/lib/http-errors.ts

@@ -12,6 +12,13 @@ export function notFound(message: string): Response {
   });
 }
 
+export function badGateway(message: string): Response {
+  return new Response(JSON.stringify({ error: message }), {
+    status: 502,
+    headers: { 'Content-Type': 'application/json' },
+  });
+}
+
 export function serviceUnavailable(message: string): Response {
   return new Response(JSON.stringify({ error: message }), {
     status: 503,

apps/api/src/lib/image-model.ts

@@ -0,0 +1,134 @@
+import * as tf from '@tensorflow/tfjs';
+import * as jpeg from 'jpeg-js';
+import { PNG } from 'pngjs';
+import { existsSync } from 'fs';
+import { resolve } from 'path';
+import type { DiseaseSlug, DiseaseLabel, PredictionProbability } from '@zeavis/shared';
+
+const DISEASE_CLASSES: Array<{ slug: DiseaseSlug; label: DiseaseLabel }> = [
+  { slug: 'bercak-daun', label: 'Bercak Daun' },
+  { slug: 'daun-sehat', label: 'Daun Sehat' },
+  { slug: 'karat-daun', label: 'Karat Daun' },
+  { slug: 'hawar-daun', label: 'Hawar Daun' },
+];
+
+function resolveModelPath() {
+  const candidates = [
+    resolve(process.cwd(), 'Machine_Learning/model/tfjs_model/model.json'),
+    resolve(process.cwd(), '../../Machine_Learning/model/tfjs_model/model.json'),
+  ];
+
+  const modelPath = candidates.find((candidate) => existsSync(candidate));
+  if (!modelPath) {
+    throw new Error('TFJS model file was not found');
+  }
+
+  return modelPath;
+}
+
+let modelPromise: Promise<tf.GraphModel> | null = null;
+
+async function loadModel(): Promise<tf.GraphModel> {
+  if (modelPromise) {
+    return modelPromise;
+  }
+
+  modelPromise = (async () => {
+    try {
+      const fileUrl = `file://${resolveModelPath()}`;
+      return await tf.loadGraphModel(fileUrl);
+    } catch (error) {
+      throw new Error(`Failed to load TFJS model: ${error instanceof Error ? error.message : String(error)}`);
+    }
+  })();
+
+  return modelPromise;
+}
+
+export type ClassificationResult = {
+  predictedDiseaseSlug: DiseaseSlug;
+  confidence: number;
+  probabilities: PredictionProbability[];
+};
+
+export async function classifyImage(file: File): Promise<ClassificationResult> {
+  if (file.type !== 'image/jpeg' && file.type !== 'image/png') {
+    throw new Error('File must be JPEG or PNG');
+  }
+
+  const buffer = await file.arrayBuffer();
+  const uint8Array = new Uint8Array(buffer);
+
+  let imageData: { data: Uint8Array; width: number; height: number };
+
+  if (file.type === 'image/jpeg') {
+    const decoded = jpeg.decode(uint8Array, { useTArray: true });
+    imageData = {
+      data: decoded.data,
+      width: decoded.width,
+      height: decoded.height,
+    };
+  } else {
+    const png = new PNG();
+    await new Promise<void>((resolve, reject) => {
+      png.parse(Buffer.from(uint8Array), (err: Error | null) => {
+        if (err) reject(err);
+        else resolve();
+      });
+    });
+    imageData = {
+      data: png.data,
+      width: png.width,
+      height: png.height,
+    };
+  }
+
+  const imageTensor = tf.tidy(() => {
+    const rgb = new Uint8Array(imageData.width * imageData.height * 3);
+    for (let source = 0, target = 0; source < imageData.data.length; source += 4, target += 3) {
+      rgb[target] = imageData.data[source];
+      rgb[target + 1] = imageData.data[source + 1];
+      rgb[target + 2] = imageData.data[source + 2];
+    }
+
+    return tf
+      .tensor3d(rgb, [imageData.height, imageData.width, 3], 'int32')
+      .resizeBilinear([224, 224])
+      .toFloat()
+      .expandDims(0);
+  });
+
+  try {
+    const model = await loadModel();
+    const predictions = model.predict(imageTensor) as tf.Tensor;
+
+    try {
+      const scoresArray = await predictions.data();
+
+      let maxScore = -Infinity;
+      let maxIndex = 0;
+      for (let i = 0; i < scoresArray.length; i++) {
+        if (scoresArray[i] > maxScore) {
+          maxScore = scoresArray[i];
+          maxIndex = i;
+        }
+      }
+
+      const probabilities: PredictionProbability[] = DISEASE_CLASSES.map((disease, index) => ({
+        diseaseSlug: disease.slug,
+        label: disease.label,
+        confidence: Math.max(0, Math.min(1, scoresArray[index])),
+      })).sort((a, b) => b.confidence - a.confidence);
+
+      return {
+        predictedDiseaseSlug: DISEASE_CLASSES[maxIndex].slug,
+        confidence: Math.max(0, Math.min(1, maxScore)),
+        probabilities,
+      };
+    } finally {
+      predictions.dispose();
+    }
+  } finally {
+    imageTensor.dispose();
+  }
+}

apps/api/src/lib/uploader-client.ts

@@ -0,0 +1,28 @@
+import type { UploaderMetadata } from '@zeavis/shared';
+
+export async function uploadImageToStorage(file: File): Promise<UploaderMetadata> {
+  const formData = new FormData();
+  formData.append('file', file);
+  formData.append('fileName', file.name);
+
+  const response = await fetch('https://upload.asepharyana.tech/api/upload', {
+    method: 'POST',
+    body: formData,
+  });
+
+  if (!response.ok) {
+    throw new Error(`Upload failed with status ${response.status}`);
+  }
+
+  const data = (await response.json()) as Record<string, unknown>;
+
+  if (!data.download_url) {
+    throw new Error('Upload response missing download_url');
+  }
+
+  if (!data.public_id) {
+    throw new Error('Upload response missing public_id');
+  }
+
+  return data as UploaderMetadata;
+}

apps/api/src/routes/classifications.ts

@@ -1,12 +1,215 @@
 import { Elysia } from 'elysia';
-import type { ManualClassificationRequest, ManualClassificationRecord } from '@zeavis/shared';
+import type {
+  DiseaseCatalogItem,
+  DiseaseSlug,
+  ImageClassificationRecord,
+  ManualClassificationRequest,
+  ManualClassificationRecord,
+  PredictionProbability,
+  UploaderMetadata,
+} from '@zeavis/shared';
 import { isDiseaseSlug } from '@zeavis/shared';
 import { createDbClient } from '../db/client';
-import { diseaseCatalog, manualClassifications } from '../db/schema';
-import { badRequest, serviceUnavailable } from '../lib/http-errors';
+import { diseaseCatalog, manualClassifications, imageClassifications } from '../db/schema';
+import { badRequest, badGateway, serviceUnavailable } from '../lib/http-errors';
 import { desc, eq } from 'drizzle-orm';
+import { classifyImage } from '../lib/image-model';
+import { uploadImageToStorage } from '../lib/uploader-client';
+import { toDisease } from '../lib/disease-mappers';
+
+function toImageClassificationRecord(row: {
+  id: string;
+  predictedDiseaseSlug: string;
+  confidence: number;
+  probabilities: unknown;
+  imageUrl: string;
+  originalFileName: string;
+  uploaderPublicId: string;
+  uploaderPayload: unknown;
+  createdAt: Date;
+  disease: {
+    slug: string;
+    label: string;
+    commonName: string;
+    summary: string;
+    description: string;
+    symptoms: string[];
+    recommendations: string[];
+    riskLevel: string;
+    accentColor: string;
+    displayOrder: number;
+  };
+}): ImageClassificationRecord {
+  return {
+    id: row.id,
+    predictedDiseaseSlug: row.predictedDiseaseSlug as DiseaseSlug,
+    confidence: row.confidence,
+    probabilities: row.probabilities as PredictionProbability[],
+    imageUrl: row.imageUrl,
+    originalFileName: row.originalFileName,
+    uploaderPublicId: row.uploaderPublicId,
+    uploader: row.uploaderPayload as UploaderMetadata,
+    createdAt: row.createdAt.toISOString(),
+    disease: toDisease(row.disease as typeof diseaseCatalog.$inferSelect),
+  };
+}
 
 export const classificationRoutes = new Elysia({ prefix: '/api/v1' })
+  .get('/classifications/image', async () => {
+    try {
+      const db = createDbClient();
+      const rows = await db
+        .select({
+          id: imageClassifications.id,
+          predictedDiseaseSlug: imageClassifications.predictedDiseaseSlug,
+          confidence: imageClassifications.confidence,
+          probabilities: imageClassifications.probabilities,
+          imageUrl: imageClassifications.imageUrl,
+          originalFileName: imageClassifications.originalFileName,
+          uploaderPublicId: imageClassifications.uploaderPublicId,
+          uploaderPayload: imageClassifications.uploaderPayload,
+          createdAt: imageClassifications.createdAt,
+          disease: {
+            slug: diseaseCatalog.slug,
+            label: diseaseCatalog.label,
+            commonName: diseaseCatalog.commonName,
+            summary: diseaseCatalog.summary,
+            description: diseaseCatalog.description,
+            symptoms: diseaseCatalog.symptoms,
+            recommendations: diseaseCatalog.recommendations,
+            riskLevel: diseaseCatalog.riskLevel,
+            accentColor: diseaseCatalog.accentColor,
+            displayOrder: diseaseCatalog.displayOrder,
+          },
+        })
+        .from(imageClassifications)
+        .innerJoin(diseaseCatalog, eq(imageClassifications.predictedDiseaseSlug, diseaseCatalog.slug))
+        .orderBy(desc(imageClassifications.createdAt))
+        .limit(20);
+
+      const records: ImageClassificationRecord[] = rows.map(toImageClassificationRecord);
+      return records;
+    } catch (error) {
+      return serviceUnavailable('Database unavailable');
+    }
+  })
+  .post('/classifications/image', async ({ body }) => {
+    let file: File | undefined;
+
+    if (body instanceof FormData) {
+      const formFile = body.get('file');
+      if (formFile instanceof File) {
+        file = formFile;
+      }
+    } else if (typeof body === 'object' && body !== null) {
+      const bodyObj = body as Record<string, unknown>;
+      if (bodyObj.file instanceof File) {
+        file = bodyObj.file;
+      } else if (Array.isArray(bodyObj.file) && bodyObj.file.length > 0 && bodyObj.file[0] instanceof File) {
+        file = bodyObj.file[0];
+      }
+    }
+
+    if (!file) {
+      return badRequest('Missing required field: file');
+    }
+
+    if (file.size === 0) {
+      return badRequest('File is empty');
+    }
+
+    if (file.type !== 'image/jpeg' && file.type !== 'image/png') {
+      return badRequest('File must be JPEG or PNG image');
+    }
+
+    try {
+      let uploaderMetadata: UploaderMetadata;
+      try {
+        uploaderMetadata = await uploadImageToStorage(file);
+      } catch (error) {
+        return badGateway(
+          `Upload service error: ${error instanceof Error ? error.message : 'Unknown error'}`
+        );
+      }
+
+      let classificationResult;
+      try {
+        classificationResult = await classifyImage(file);
+      } catch (error) {
+        return serviceUnavailable(
+          `Model service error: ${error instanceof Error ? error.message : 'Unknown error'}`
+        );
+      }
+
+      const db = createDbClient();
+      let diseaseRow;
+      try {
+        const diseaseRows = await db
+          .select()
+          .from(diseaseCatalog)
+          .where(eq(diseaseCatalog.slug, classificationResult.predictedDiseaseSlug))
+          .limit(1);
+
+        if (diseaseRows.length === 0) {
+          return serviceUnavailable(
+            `Predicted disease "${classificationResult.predictedDiseaseSlug}" not found in catalog`
+          );
+        }
+
+        diseaseRow = diseaseRows[0];
+      } catch (error) {
+        return serviceUnavailable('Database unavailable');
+      }
+
+      let inserted;
+      try {
+        const result = await db
+          .insert(imageClassifications)
+          .values({
+            predictedDiseaseSlug: classificationResult.predictedDiseaseSlug,
+            confidence: classificationResult.confidence,
+            probabilities: classificationResult.probabilities,
+            imageUrl: uploaderMetadata.download_url,
+            originalFileName: file.name,
+            uploaderPublicId: uploaderMetadata.public_id,
+            uploaderPayload: uploaderMetadata,
+          })
+          .returning();
+
+        inserted = result[0];
+      } catch (error) {
+        return serviceUnavailable('Database unavailable');
+      }
+
+      const record: ImageClassificationRecord = toImageClassificationRecord({
+        id: inserted.id,
+        predictedDiseaseSlug: inserted.predictedDiseaseSlug,
+        confidence: inserted.confidence,
+        probabilities: inserted.probabilities,
+        imageUrl: inserted.imageUrl,
+        originalFileName: inserted.originalFileName,
+        uploaderPublicId: inserted.uploaderPublicId,
+        uploaderPayload: inserted.uploaderPayload,
+        createdAt: inserted.createdAt,
+        disease: {
+          slug: diseaseRow.slug,
+          label: diseaseRow.label,
+          commonName: diseaseRow.commonName,
+          summary: diseaseRow.summary,
+          description: diseaseRow.description,
+          symptoms: diseaseRow.symptoms,
+          recommendations: diseaseRow.recommendations,
+          riskLevel: diseaseRow.riskLevel,
+          accentColor: diseaseRow.accentColor,
+          displayOrder: diseaseRow.displayOrder,
+        },
+      });
+
+      return record;
+    } catch (error) {
+      return serviceUnavailable('Internal server error');
+    }
+  })
   .get('/classifications/manual', async () => {
     try {
       const db = createDbClient();

apps/api/src/routes/dashboard.ts

@@ -1,7 +1,7 @@
 import { Elysia } from 'elysia';
 import type { DashboardSummary, RiskLevel } from '@zeavis/shared';
 import { createDbClient } from '../db/client';
-import { diseaseCatalog, manualClassifications } from '../db/schema';
+import { diseaseCatalog, imageClassifications, manualClassifications } from '../db/schema';
 import { serviceUnavailable } from '../lib/http-errors';
 import { desc, eq } from 'drizzle-orm';
 
@@ -10,7 +10,8 @@ export const dashboardRoutes = new Elysia({ prefix: '/api/v1' }).get('/dashboard
     const db = createDbClient();
 
     const diseaseCount = await db.select().from(diseaseCatalog);
-    const classificationCount = await db.select().from(manualClassifications);
+    const manualClassificationCount = await db.select().from(manualClassifications);
+    const imageClassificationCount = await db.select().from(imageClassifications);
 
     const latestClassificationRow = await db
       .select({
@@ -75,7 +76,7 @@ export const dashboardRoutes = new Elysia({ prefix: '/api/v1' }).get('/dashboard
 
     const summary: DashboardSummary = {
       diseaseCount: diseaseCount.length,
-      classificationCount: classificationCount.length,
+      classificationCount: manualClassificationCount.length + imageClassificationCount.length,
       latestClassification,
       riskDistribution,
     };

apps/api/src/types/image-decoders.d.ts

@@ -0,0 +1,8 @@
+declare module 'pngjs' {
+  export class PNG {
+    width: number;
+    height: number;
+    data: Buffer;
+    parse(data: Buffer, callback: (err: Error | null) => void): void;
+  }
+}