Add MiniCPM-V 4.6 Benchmark Notebook for model evaluation and performance metrics

.kilo/agents/data.md

@@ -0,0 +1,28 @@
+---
+mode: primary
+description: Run notebook-first data analysis by appending and executing cells
+  for each request.
+options:
+  displayName: Data
+  id: data
+requirements:
+  skills:
+    - data-investigation
+  vscode_extensions:
+    - name: Jupyter
+      id: ms-toolsai.jupyter
+color: "#2563EB"
+---
+
+You are Kilo, a notebook-first data analysis agent. Use an active Jupyter notebook as the working surface.
+
+Guidelines:
+- If no notebook is active, create a uniquely named, descriptive `<topic>.ipynb` in the current workspace folder
+- Use the dedicated notebook tools to create, read, edit, and execute; prefer these tools over other methods like MCP tools and manual raw JSON editing
+- Confirm Jupyter and kernel readiness through the first requested notebook execution; only notify the user if they need to select or configure a kernel before work can continue
+- For every user request, append at least one focused code cell and execute it
+- Preserve notebook history: do not modify or delete existing cells unless explicitly asked; after failures, append diagnostic or corrected cells
+- Keep substantive data work and supporting evidence in the notebook
+- Avoid changing non-notebook files unless explicitly requested or necessary to complete the task
+- Inspect cell output before answering, and keep notebook outputs and final summaries concise
+- Never claim execution when a notebook cell did not run

minicpmv4_6_benchmark.ipynb

@@ -0,0 +1,423 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# MiniCPM-V 4.6 — Clone & Benchmark Notebook\n",
+    "\n",
+    "Notebook untuk meng-clone model [openbmb/MiniCPM-V-4.6](https://huggingface.co/openbmb/MiniCPM-V-4.6) dari Hugging Face dan melakukan benchmark:\n",
+    "- Loading speed & memory usage\n",
+    "- Text generation throughput (tokens/sec)\n",
+    "- Reasoning & knowledge QA\n",
+    "- Coding capability\n",
+    "- Multimodal image understanding\n",
+    "- Long context (needle-in-haystack)\n",
+    "\n",
+    "**Model**: `openbmb/MiniCPM-V-4.6` (SigLIP2-400M + Qwen3.5-0.8B, 1B total params)\n",
+    "\n",
+    "**Cara pakai**: Runtime > Factory reset runtime, lalu Runtime > Run all"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "---\n",
+    "## 1. Environment Setup"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Install dependencies — skip torch, Colab sudah punya\n",
+    "!pip install -qU \\\n",
+    "    'transformers>=5.7.0' \\\n",
+    "    accelerate \\\n",
+    "    sentencepiece \\\n",
+    "    psutil \\\n",
+    "    'pillow<11' \\\n",
+    "    requests \\\n",
+    "    matplotlib \\\n",
+    "    tabulate \\\n",
+    "    av \\\n",
+    "    einops \\\n",
+    "    2>&1 | tail -3\n",
+    "print(\"Install selesai\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import os, sys, json, time, gc, warnings\n",
+    "from pathlib import Path\n",
+    "from datetime import datetime\n",
+    "from IPython.display import display, Video\n",
+    "\n",
+    "import torch\n",
+    "import psutil\n",
+    "import numpy as np\n",
+    "import matplotlib.pyplot as plt\n",
+    "from tabulate import tabulate\n",
+    "from PIL import Image\n",
+    "import requests\n",
+    "from io import BytesIO\n",
+    "\n",
+    "warnings.filterwarnings(\"ignore\")\n",
+    "\n",
+    "print(f\"Python       : {sys.version}\")\n",
+    "print(f\"PyTorch      : {torch.__version__}\")\n",
+    "print(f\"CUDA avail   : {torch.cuda.is_available()}\")\n",
+    "if torch.cuda.is_available():\n",
+    "    print(f\"CUDA device  : {torch.cuda.get_device_name(0)}\")\n",
+    "    print(f\"CUDA VRAM    : {torch.cuda.get_device_properties(0).total_memory / 1e9:.1f} GB\")\n",
+    "    print(f\"CUDA cap     : {torch.cuda.get_device_capability()}\")\n",
+    "\n",
+    "try:\n",
+    "    import transformers\n",
+    "    print(f\"transformers : {transformers.__version__}\")\n",
+    "except:\n",
+    "    print(\"transformers : NOT INSTALLED\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "---\n",
+    "## 2. Clone Model from Hugging Face\n",
+    "\n",
+    "MiniCPM-V 4.6 = 1B params. Muat di T4 (15.6GB) tanpa masalah."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "MODEL_ID = \"openbmb/MiniCPM-V-4.6\"\n",
+    "CACHE_DIR = None  # Set ke \"drive/MyDrive/hf_cache\" untuk persistent\n",
+    "DOWNSAMPLE_MODE = \"16x\"  # \"16x\" = efisien, \"4x\" = detail lebih tinggi\n",
+    "\n",
+    "print(f\"Model ID: {MODEL_ID}\")\n",
+    "print(\"Loading...\")\n",
+    "t0 = time.perf_counter()\n",
+    "\n",
+    "from transformers import AutoProcessor, AutoModelForImageTextToText\n",
+    "\n",
+    "processor = AutoProcessor.from_pretrained(MODEL_ID, cache_dir=CACHE_DIR)\n",
+    "print(f\"Processor loaded in {time.perf_counter()-t0:.1f}s\")\n",
+    "\n",
+    "load_start = time.perf_counter()\n",
+    "model = AutoModelForImageTextToText.from_pretrained(\n",
+    "    MODEL_ID,\n",
+    "    torch_dtype=torch.bfloat16,\n",
+    "    device_map=\"auto\",\n",
+    "    cache_dir=CACHE_DIR,\n",
+    ")\n",
+    "load_time = time.perf_counter() - load_start\n",
+    "print(f\"\\nModel loaded in {load_time:.1f}s\")\n",
+    "\n",
+    "total_params = sum(p.numel() for p in model.parameters())\n",
+    "print(f\"Parameters: {total_params/1e9:.2f}B\")\n",
+    "print(f\"Device: {model.device}, Dtype: {model.dtype}\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "if torch.cuda.is_available():\n",
+    "    vram_used = torch.cuda.memory_allocated() / 1e9\n",
+    "    print(f\"VRAM allocated: {vram_used:.2f} GB\")\n",
+    "ram_used = psutil.Process(os.getpid()).memory_info().rss / 1e9\n",
+    "print(f\"RAM used: {ram_used:.2f} GB\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "---\n",
+    "## 3. Text Generation Throughput"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def throughput(prompt, max_tokens=256, runs=3):\n",
+    "    msgs = [{\"role\": \"user\", \"content\": prompt}]\n",
+    "    inputs = processor.apply_chat_template(msgs, tokenize=True, return_dict=True,\n",
+    "                    return_tensors=\"pt\", add_generation_prompt=True).to(model.device)\n",
+    "    inp_len = inputs[\"input_ids\"].shape[-1]\n",
+    "    lats, toks = [], []\n",
+    "    for _ in range(runs):\n",
+    "        start = time.perf_counter()\n",
+    "        with torch.no_grad():\n",
+    "            out = model.generate(**inputs, max_new_tokens=max_tokens, do_sample=True, temperature=0.7)\n",
+    "        elapsed = time.perf_counter() - start\n",
+    "        gen = out[0][inp_len:]\n",
+    "        lats.append(elapsed)\n",
+    "        toks.append(len(gen))\n",
+    "    tps = [t/l for t,l in zip(toks, lats)]\n",
+    "    return {\"prompt\": prompt[:60]+\"...\", \"tokens\": int(np.mean(toks)),\n",
+    "            \"latency\": float(np.mean(lats)), \"tps\": float(np.mean(tps))}\n",
+    "\n",
+    "prompts = {\n",
+    "    \"Simple QA\": \"What is the capital of Indonesia?\",\n",
+    "    \"Math\": \"If a train travels at 120 km/h and another at 80 km/h toward each other from 500 km apart, how long until they meet?\",\n",
+    "    \"Code\": \"Write a Python function to find the longest palindromic substring.\",\n",
+    "}\n",
+    "\n",
+    "results = []\n",
+    "for name, p in prompts.items():\n",
+    "    r = throughput(p, runs=2)\n",
+    "    results.append(r)\n",
+    "    print(f\"{name}: {r['tokens']} tok, {r['latency']:.1f}s, {r['tps']:.1f} tok/s\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "---\n",
+    "## 4. Reasoning (MMLU-style)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "mmlu = [\n",
+    "    {\"q\": \"What is the time complexity of binary search?\", \"o\": [\"A. O(n)\", \"B. O(log n)\", \"C. O(n log n)\", \"D. O(1)\"], \"a\": \"B\"},\n",
+    "    {\"q\": \"Which planet has the strongest surface gravity?\", \"o\": [\"A. Earth\", \"B. Mars\", \"C. Jupiter\", \"D. Saturn\"], \"a\": \"C\"},\n",
+    "    {\"q\": \"In C++, which keyword prevents overriding?\", \"o\": [\"A. static\", \"B. const\", \"C. final\", \"D. override\"], \"a\": \"C\"},\n",
+    "    {\"q\": \"Probability of drawing a red ball from 3 red + 5 blue?\", \"o\": [\"A. 3/5\", \"B. 3/8\", \"C. 5/8\", \"D. 1/2\"], \"a\": \"B\"},\n",
+    "    {\"q\": \"What does mitochondria do?\", \"o\": [\"A. Protein\", \"B. Energy (ATP)\", \"C. Lipid\", \"D. DNA\"], \"a\": \"B\"},\n",
+    "]\n",
+    "\n",
+    "ok = 0\n",
+    "for q in mmlu:\n",
+    "    prompt = f\"{q['q']}\\n\\n\" + \"\\n\".join(q[\"o\"]) + \"\\n\\nAnswer with a single letter:\"\n",
+    "    inputs = processor.apply_chat_template([{\"role\":\"user\",\"content\":prompt}],\n",
+    "        tokenize=True, return_dict=True, return_tensors=\"pt\", add_generation_prompt=True).to(model.device)\n",
+    "    with torch.no_grad():\n",
+    "        out = model.generate(**inputs, max_new_tokens=8, do_sample=False)\n",
+    "    ans = processor.decode(out[0][inputs[\"input_ids\"].shape[-1]:], skip_special_tokens=True).strip()\n",
+    "    cor = q[\"a\"] in ans.upper()[:1]\n",
+    "    if cor: ok += 1\n",
+    "    print(f\"  {'OK' if cor else 'NO'} Expected={q['a']} Got={ans[:20]} | {q['q'][:50]}\")\n",
+    "\n",
+    "print(f\"\\nAccuracy: {ok}/{len(mmlu)} = {ok/len(mmlu)*100:.0f}%\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "---\n",
+    "## 5. Coding"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "for name, prompt in [\n",
+    "    (\"Binary Search\", \"Write Python `binary_search(arr, target)` returning index or -1.\"),\n",
+    "    (\"Fibonacci\", \"Write Python `fib(n)` for nth Fibonacci using DP.\"),\n",
+    "]:\n",
+    "    print(f\"\\n{'='*40}\\n{name}\\n{'='*40}\")\n",
+    "    inputs = processor.apply_chat_template([{\"role\":\"user\",\"content\":prompt}],\n",
+    "        tokenize=True, return_dict=True, return_tensors=\"pt\", add_generation_prompt=True).to(model.device)\n",
+    "    with torch.no_grad():\n",
+    "        out = model.generate(**inputs, max_new_tokens=512, temperature=0.2, do_sample=False)\n",
+    "    print(processor.decode(out[0][inputs[\"input_ids\"].shape[-1]:], skip_special_tokens=True)[:400])"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "---\n",
+    "## 6. Image Understanding"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "print(\"Test image understanding (16x downsample / efisien)...\")\n",
+    "try:\n",
+    "    url = \"https://huggingface.co/datasets/openbmb/DemoCase/resolve/main/refract.png\"\n",
+    "    img = Image.open(BytesIO(requests.get(url, timeout=30).content))\n",
+    "    display(img.resize((250, 180)))\n",
+    "\n",
+    "    msgs = [{\"role\":\"user\",\"content\":[\n",
+    "        {\"type\":\"image\",\"image\":img},\n",
+    "        {\"type\":\"text\",\"text\":\"What causes this phenomenon?\"}\n",
+    "    ]}]\n",
+    "\n",
+    "    inputs = processor.apply_chat_template(msgs, tokenize=True, add_generation_prompt=True,\n",
+    "        return_dict=True, return_tensors=\"pt\", downsample_mode=DOWNSAMPLE_MODE,\n",
+    "    ).to(model.device)\n",
+    "\n",
+    "    t0 = time.perf_counter()\n",
+    "    with torch.no_grad():\n",
+    "        out = model.generate(**inputs, downsample_mode=DOWNSAMPLE_MODE, max_new_tokens=128, do_sample=True)\n",
+    "    resp = processor.decode(out[0][inputs[\"input_ids\"].shape[-1]:], skip_special_tokens=True)\n",
+    "    print(f\"Time: {time.perf_counter()-t0:.1f}s\")\n",
+    "    print(f\"Answer: {resp[:300]}\")\n",
+    "except Exception as e:\n",
+    "    print(f\"ERROR: {e}\")\n",
+    "\n",
+    "print(\"\\n---\\nTest image understanding (4x downsample / detail tinggi)...\")\n",
+    "try:\n",
+    "    url2 = \"https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/p-blog/candy.JPG\"\n",
+    "    img2 = Image.open(BytesIO(requests.get(url2, timeout=30).content))\n",
+    "    display(img2.resize((250, 180)))\n",
+    "\n",
+    "    msgs2 = [{\"role\":\"user\",\"content\":[\n",
+    "        {\"type\":\"image\",\"image\":img2},\n",
+    "        {\"type\":\"text\",\"text\":\"What animal is on the candy?\"}\n",
+    "    ]}]\n",
+    "\n",
+    "    inputs2 = processor.apply_chat_template(msgs2, tokenize=True, add_generation_prompt=True,\n",
+    "        return_dict=True, return_tensors=\"pt\", downsample_mode=\"4x\",\n",
+    "    ).to(model.device)\n",
+    "\n",
+    "    t0 = time.perf_counter()\n",
+    "    with torch.no_grad():\n",
+    "        out2 = model.generate(**inputs2, downsample_mode=\"4x\", max_new_tokens=128, do_sample=True)\n",
+    "    resp2 = processor.decode(out2[0][inputs2[\"input_ids\"].shape[-1]:], skip_special_tokens=True)\n",
+    "    print(f\"Time: {time.perf_counter()-t0:.1f}s\")\n",
+    "    print(f\"Answer: {resp2[:300]}\")\n",
+    "except Exception as e:\n",
+    "    print(f\"ERROR: {e}\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "---\n",
+    "## 7. Long Context (Needle-in-Haystack)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def needle(pos):\n",
+    "    needle_str = \"The secret code is BLUE-42-GREEN.\"\n",
+    "    filler = \"The quick brown fox jumps over the lazy dog. Python is versatile. \"\n",
+    "    sents = [filler] * 50\n",
+    "    if pos == \"early\": sents.insert(0, needle_str)\n",
+    "    elif pos == \"middle\": sents.insert(len(sents)//2, needle_str)\n",
+    "    else: sents.append(needle_str)\n",
+    "    prompt = f\"Read the text and answer.\\n\\nText: {' '.join(sents)}\\n\\nQ: What is the secret code? Answer with code only.\"\n",
+    "    inputs = processor.apply_chat_template([{\"role\":\"user\",\"content\":prompt}],\n",
+    "        tokenize=True, return_dict=True, return_tensors=\"pt\", add_generation_prompt=True).to(model.device)\n",
+    "    t0 = time.perf_counter()\n",
+    "    with torch.no_grad():\n",
+    "        out = model.generate(**inputs, max_new_tokens=16, do_sample=False)\n",
+    "    resp = processor.decode(out[0][inputs[\"input_ids\"].shape[-1]:], skip_special_tokens=True).strip()\n",
+    "    return {\"pos\": pos, \"tokens\": inputs[\"input_ids\"].shape[-1], \"resp\": resp[:80],\n",
+    "            \"correct\": \"BLUE-42-GREEN\" in resp, \"time\": time.perf_counter()-t0}\n",
+    "\n",
+    "print(\"Needle-in-Haystack:\")\n",
+    "for p in [\"early\", \"middle\", \"late\"]:\n",
+    "    r = needle(p)\n",
+    "    print(f\"{p.upper()}: {r['tokens']} tok, correct={'OK' if r['correct'] else 'NO'}, {r['time']:.1f}s\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "---\n",
+    "## 8. Summary"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "rows = [\n",
+    "    [\"Model\", MODEL_ID],\n",
+    "    [\"Parameters\", f\"{total_params/1e9:.2f}B\"],\n",
+    "    [\"Device\", str(model.device)],\n",
+    "    [\"Dtype\", str(model.dtype)],\n",
+    "]\n",
+    "if torch.cuda.is_available():\n",
+    "    rows.append([\"VRAM\", f\"{torch.cuda.memory_allocated()/1e9:.2f} GB\"])\n",
+    "rows.append([\"RAM\", f\"{ram_used:.2f} GB\"])\n",
+    "rows.append([\"Load Time\", f\"{load_time:.1f}s\"])\n",
+    "if results:\n",
+    "    rows.append([\"Throughput\", f\"{np.mean([r['tps'] for r in results]):.1f} tok/s\"])\n",
+    "rows.append([\"MMLU\", f\"{ok}/{len(mmlu)} ({ok/len(mmlu)*100:.0f}%)\"])\n",
+    "if 'nh' in dir():\n",
+    "    rows.append([\"Needle/Haystack\", f\"{sum(1 for r in nh if r['correct'])}/{len(nh)} correct\"])\n",
+    "\n",
+    "print(\"=\"*60)\n",
+    "print(\"  MINICPM-V 4.6 — BENCHMARK SUMMARY\")\n",
+    "print(\"=\"*60)\n",
+    "print(tabulate(rows, headers=[\"Metric\",\"Value\"], tablefmt=\"grid\"))\n",
+    "print(\"=\"*60)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "---\n",
+    "## 9. Cleanup"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "del model, processor\n",
+    "gc.collect()\n",
+    "if torch.cuda.is_available():\n",
+    "    torch.cuda.empty_cache()\n",
+    "    print(f\"VRAM: {torch.cuda.memory_allocated()/1e9:.2f} GB\")\n",
+    "print(\"Done.\")"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}