feat: implement backend image classification with TensorFlow.js model

- Extend shared types for image classification, including PredictionProbability, UploaderMetadata, and ImageClassificationRecord.
- Create image_classifications table in the database with necessary fields and foreign key constraints.
- Implement disease mappers to convert database rows to shared disease records.
- Develop uploader client to handle image uploads to external service.
- Create image model service to load and classify images using TensorFlow.js.
- Add API routes for image classification, including GET for history and POST for new classifications.
- Implement frontend components for image classification form and display results.
- Update dashboard to integrate image classification functionality and display results.
- Document implementation plan for backend image classification.

apps/api/src/lib/disease-mappers.ts

@@ -0,0 +1,17 @@
+import type { DiseaseCatalogItem, DiseaseSlug, DiseaseLabel, RiskLevel } from '@zeavis/shared';
+import type { diseaseCatalog } from '../db/schema';
+
+export function toDisease(row: typeof diseaseCatalog.$inferSelect): DiseaseCatalogItem {
+  return {
+    slug: row.slug as DiseaseSlug,
+    label: row.label as DiseaseLabel,
+    commonName: row.commonName,
+    summary: row.summary,
+    description: row.description,
+    symptoms: row.symptoms,
+    recommendations: row.recommendations,
+    riskLevel: row.riskLevel as RiskLevel,
+    accentColor: row.accentColor,
+    displayOrder: row.displayOrder,
+  };
+}

apps/api/src/lib/http-errors.ts

@@ -12,6 +12,13 @@ export function notFound(message: string): Response {
   });
 }
 
+export function badGateway(message: string): Response {
+  return new Response(JSON.stringify({ error: message }), {
+    status: 502,
+    headers: { 'Content-Type': 'application/json' },
+  });
+}
+
 export function serviceUnavailable(message: string): Response {
   return new Response(JSON.stringify({ error: message }), {
     status: 503,

apps/api/src/lib/image-model.ts

@@ -0,0 +1,134 @@
+import * as tf from '@tensorflow/tfjs';
+import * as jpeg from 'jpeg-js';
+import { PNG } from 'pngjs';
+import { existsSync } from 'fs';
+import { resolve } from 'path';
+import type { DiseaseSlug, DiseaseLabel, PredictionProbability } from '@zeavis/shared';
+
+const DISEASE_CLASSES: Array<{ slug: DiseaseSlug; label: DiseaseLabel }> = [
+  { slug: 'bercak-daun', label: 'Bercak Daun' },
+  { slug: 'daun-sehat', label: 'Daun Sehat' },
+  { slug: 'karat-daun', label: 'Karat Daun' },
+  { slug: 'hawar-daun', label: 'Hawar Daun' },
+];
+
+function resolveModelPath() {
+  const candidates = [
+    resolve(process.cwd(), 'Machine_Learning/model/tfjs_model/model.json'),
+    resolve(process.cwd(), '../../Machine_Learning/model/tfjs_model/model.json'),
+  ];
+
+  const modelPath = candidates.find((candidate) => existsSync(candidate));
+  if (!modelPath) {
+    throw new Error('TFJS model file was not found');
+  }
+
+  return modelPath;
+}
+
+let modelPromise: Promise<tf.GraphModel> | null = null;
+
+async function loadModel(): Promise<tf.GraphModel> {
+  if (modelPromise) {
+    return modelPromise;
+  }
+
+  modelPromise = (async () => {
+    try {
+      const fileUrl = `file://${resolveModelPath()}`;
+      return await tf.loadGraphModel(fileUrl);
+    } catch (error) {
+      throw new Error(`Failed to load TFJS model: ${error instanceof Error ? error.message : String(error)}`);
+    }
+  })();
+
+  return modelPromise;
+}
+
+export type ClassificationResult = {
+  predictedDiseaseSlug: DiseaseSlug;
+  confidence: number;
+  probabilities: PredictionProbability[];
+};
+
+export async function classifyImage(file: File): Promise<ClassificationResult> {
+  if (file.type !== 'image/jpeg' && file.type !== 'image/png') {
+    throw new Error('File must be JPEG or PNG');
+  }
+
+  const buffer = await file.arrayBuffer();
+  const uint8Array = new Uint8Array(buffer);
+
+  let imageData: { data: Uint8Array; width: number; height: number };
+
+  if (file.type === 'image/jpeg') {
+    const decoded = jpeg.decode(uint8Array, { useTArray: true });
+    imageData = {
+      data: decoded.data,
+      width: decoded.width,
+      height: decoded.height,
+    };
+  } else {
+    const png = new PNG();
+    await new Promise<void>((resolve, reject) => {
+      png.parse(Buffer.from(uint8Array), (err: Error | null) => {
+        if (err) reject(err);
+        else resolve();
+      });
+    });
+    imageData = {
+      data: png.data,
+      width: png.width,
+      height: png.height,
+    };
+  }
+
+  const imageTensor = tf.tidy(() => {
+    const rgb = new Uint8Array(imageData.width * imageData.height * 3);
+    for (let source = 0, target = 0; source < imageData.data.length; source += 4, target += 3) {
+      rgb[target] = imageData.data[source];
+      rgb[target + 1] = imageData.data[source + 1];
+      rgb[target + 2] = imageData.data[source + 2];
+    }
+
+    return tf
+      .tensor3d(rgb, [imageData.height, imageData.width, 3], 'int32')
+      .resizeBilinear([224, 224])
+      .toFloat()
+      .expandDims(0);
+  });
+
+  try {
+    const model = await loadModel();
+    const predictions = model.predict(imageTensor) as tf.Tensor;
+
+    try {
+      const scoresArray = await predictions.data();
+
+      let maxScore = -Infinity;
+      let maxIndex = 0;
+      for (let i = 0; i < scoresArray.length; i++) {
+        if (scoresArray[i] > maxScore) {
+          maxScore = scoresArray[i];
+          maxIndex = i;
+        }
+      }
+
+      const probabilities: PredictionProbability[] = DISEASE_CLASSES.map((disease, index) => ({
+        diseaseSlug: disease.slug,
+        label: disease.label,
+        confidence: Math.max(0, Math.min(1, scoresArray[index])),
+      })).sort((a, b) => b.confidence - a.confidence);
+
+      return {
+        predictedDiseaseSlug: DISEASE_CLASSES[maxIndex].slug,
+        confidence: Math.max(0, Math.min(1, maxScore)),
+        probabilities,
+      };
+    } finally {
+      predictions.dispose();
+    }
+  } finally {
+    imageTensor.dispose();
+  }
+}

apps/api/src/lib/uploader-client.ts

@@ -0,0 +1,28 @@
+import type { UploaderMetadata } from '@zeavis/shared';
+
+export async function uploadImageToStorage(file: File): Promise<UploaderMetadata> {
+  const formData = new FormData();
+  formData.append('file', file);
+  formData.append('fileName', file.name);
+
+  const response = await fetch('https://upload.asepharyana.tech/api/upload', {
+    method: 'POST',
+    body: formData,
+  });
+
+  if (!response.ok) {
+    throw new Error(`Upload failed with status ${response.status}`);
+  }
+
+  const data = (await response.json()) as Record<string, unknown>;
+
+  if (!data.download_url) {
+    throw new Error('Upload response missing download_url');
+  }
+
+  if (!data.public_id) {
+    throw new Error('Upload response missing public_id');
+  }
+
+  return data as UploaderMetadata;
+}