llm/minicpmv4_6_benchmark.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# MiniCPM-V 4.6 — Clone & Benchmark Notebook\n",
    "\n",
    "Notebook untuk meng-clone model [openbmb/MiniCPM-V-4.6](https://huggingface.co/openbmb/MiniCPM-V-4.6) dari Hugging Face dan melakukan benchmark:\n",
    "- Loading speed & memory usage\n",
    "- Text generation throughput (tokens/sec)\n",
    "- Reasoning & knowledge QA\n",
    "- Coding capability\n",
    "- Multimodal image understanding\n",
    "- Long context (needle-in-haystack)\n",
    "\n",
    "**Model**: `openbmb/MiniCPM-V-4.6` (SigLIP2-400M + Qwen3.5-0.8B, 1B total params)\n",
    "\n",
    "**Cara pakai**: Runtime > Factory reset runtime, lalu Runtime > Run all"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "## 1. Environment Setup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Install dependencies — skip torch, Colab sudah punya\n",
    "!pip install -qU \\\n",
    "    'transformers>=5.7.0' \\\n",
    "    accelerate \\\n",
    "    sentencepiece \\\n",
    "    psutil \\\n",
    "    'pillow<11' \\\n",
    "    requests \\\n",
    "    matplotlib \\\n",
    "    tabulate \\\n",
    "    av \\\n",
    "    einops \\\n",
    "    2>&1 | tail -3\n",
    "print(\"Install selesai\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os, sys, json, time, gc, warnings\n",
    "from pathlib import Path\n",
    "from datetime import datetime\n",
    "from IPython.display import display, Video\n",
    "\n",
    "import torch\n",
    "import psutil\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from tabulate import tabulate\n",
    "from PIL import Image\n",
    "import requests\n",
    "from io import BytesIO\n",
    "\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "\n",
    "print(f\"Python       : {sys.version}\")\n",
    "print(f\"PyTorch      : {torch.__version__}\")\n",
    "print(f\"CUDA avail   : {torch.cuda.is_available()}\")\n",
    "if torch.cuda.is_available():\n",
    "    print(f\"CUDA device  : {torch.cuda.get_device_name(0)}\")\n",
    "    print(f\"CUDA VRAM    : {torch.cuda.get_device_properties(0).total_memory / 1e9:.1f} GB\")\n",
    "    print(f\"CUDA cap     : {torch.cuda.get_device_capability()}\")\n",
    "\n",
    "try:\n",
    "    import transformers\n",
    "    print(f\"transformers : {transformers.__version__}\")\n",
    "except:\n",
    "    print(\"transformers : NOT INSTALLED\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "## 2. Clone Model from Hugging Face\n",
    "\n",
    "MiniCPM-V 4.6 = 1B params. Muat di T4 (15.6GB) tanpa masalah."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "MODEL_ID = \"openbmb/MiniCPM-V-4.6\"\n",
    "CACHE_DIR = None  # Set ke \"drive/MyDrive/hf_cache\" untuk persistent\n",
    "DOWNSAMPLE_MODE = \"16x\"  # \"16x\" = efisien, \"4x\" = detail lebih tinggi\n",
    "\n",
    "print(f\"Model ID: {MODEL_ID}\")\n",
    "print(\"Loading...\")\n",
    "t0 = time.perf_counter()\n",
    "\n",
    "from transformers import AutoProcessor, AutoModelForImageTextToText\n",
    "\n",
    "processor = AutoProcessor.from_pretrained(MODEL_ID, cache_dir=CACHE_DIR)\n",
    "print(f\"Processor loaded in {time.perf_counter()-t0:.1f}s\")\n",
    "\n",
    "load_start = time.perf_counter()\n",
    "model = AutoModelForImageTextToText.from_pretrained(\n",
    "    MODEL_ID,\n",
    "    torch_dtype=torch.bfloat16,\n",
    "    device_map=\"auto\",\n",
    "    cache_dir=CACHE_DIR,\n",
    ")\n",
    "load_time = time.perf_counter() - load_start\n",
    "print(f\"\\nModel loaded in {load_time:.1f}s\")\n",
    "\n",
    "total_params = sum(p.numel() for p in model.parameters())\n",
    "print(f\"Parameters: {total_params/1e9:.2f}B\")\n",
    "print(f\"Device: {model.device}, Dtype: {model.dtype}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if torch.cuda.is_available():\n",
    "    vram_used = torch.cuda.memory_allocated() / 1e9\n",
    "    print(f\"VRAM allocated: {vram_used:.2f} GB\")\n",
    "ram_used = psutil.Process(os.getpid()).memory_info().rss / 1e9\n",
    "print(f\"RAM used: {ram_used:.2f} GB\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "## 3. Text Generation Throughput"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def throughput(prompt, max_tokens=256, runs=3):\n",
    "    msgs = [{\"role\": \"user\", \"content\": prompt}]\n",
    "    inputs = processor.apply_chat_template(msgs, tokenize=True, return_dict=True,\n",
    "                    return_tensors=\"pt\", add_generation_prompt=True).to(model.device)\n",
    "    inp_len = inputs[\"input_ids\"].shape[-1]\n",
    "    lats, toks = [], []\n",
    "    for _ in range(runs):\n",
    "        start = time.perf_counter()\n",
    "        with torch.no_grad():\n",
    "            out = model.generate(**inputs, max_new_tokens=max_tokens, do_sample=True, temperature=0.7)\n",
    "        elapsed = time.perf_counter() - start\n",
    "        gen = out[0][inp_len:]\n",
    "        lats.append(elapsed)\n",
    "        toks.append(len(gen))\n",
    "    tps = [t/l for t,l in zip(toks, lats)]\n",
    "    return {\"prompt\": prompt[:60]+\"...\", \"tokens\": int(np.mean(toks)),\n",
    "            \"latency\": float(np.mean(lats)), \"tps\": float(np.mean(tps))}\n",
    "\n",
    "prompts = {\n",
    "    \"Simple QA\": \"What is the capital of Indonesia?\",\n",
    "    \"Math\": \"If a train travels at 120 km/h and another at 80 km/h toward each other from 500 km apart, how long until they meet?\",\n",
    "    \"Code\": \"Write a Python function to find the longest palindromic substring.\",\n",
    "}\n",
    "\n",
    "results = []\n",
    "for name, p in prompts.items():\n",
    "    r = throughput(p, runs=2)\n",
    "    results.append(r)\n",
    "    print(f\"{name}: {r['tokens']} tok, {r['latency']:.1f}s, {r['tps']:.1f} tok/s\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "## 4. Reasoning (MMLU-style)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mmlu = [\n",
    "    {\"q\": \"What is the time complexity of binary search?\", \"o\": [\"A. O(n)\", \"B. O(log n)\", \"C. O(n log n)\", \"D. O(1)\"], \"a\": \"B\"},\n",
    "    {\"q\": \"Which planet has the strongest surface gravity?\", \"o\": [\"A. Earth\", \"B. Mars\", \"C. Jupiter\", \"D. Saturn\"], \"a\": \"C\"},\n",
    "    {\"q\": \"In C++, which keyword prevents overriding?\", \"o\": [\"A. static\", \"B. const\", \"C. final\", \"D. override\"], \"a\": \"C\"},\n",
    "    {\"q\": \"Probability of drawing a red ball from 3 red + 5 blue?\", \"o\": [\"A. 3/5\", \"B. 3/8\", \"C. 5/8\", \"D. 1/2\"], \"a\": \"B\"},\n",
    "    {\"q\": \"What does mitochondria do?\", \"o\": [\"A. Protein\", \"B. Energy (ATP)\", \"C. Lipid\", \"D. DNA\"], \"a\": \"B\"},\n",
    "]\n",
    "\n",
    "ok = 0\n",
    "for q in mmlu:\n",
    "    prompt = f\"{q['q']}\\n\\n\" + \"\\n\".join(q[\"o\"]) + \"\\n\\nAnswer with a single letter:\"\n",
    "    inputs = processor.apply_chat_template([{\"role\":\"user\",\"content\":prompt}],\n",
    "        tokenize=True, return_dict=True, return_tensors=\"pt\", add_generation_prompt=True).to(model.device)\n",
    "    with torch.no_grad():\n",
    "        out = model.generate(**inputs, max_new_tokens=8, do_sample=False)\n",
    "    ans = processor.decode(out[0][inputs[\"input_ids\"].shape[-1]:], skip_special_tokens=True).strip()\n",
    "    cor = q[\"a\"] in ans.upper()[:1]\n",
    "    if cor: ok += 1\n",
    "    print(f\"  {'OK' if cor else 'NO'} Expected={q['a']} Got={ans[:20]} | {q['q'][:50]}\")\n",
    "\n",
    "print(f\"\\nAccuracy: {ok}/{len(mmlu)} = {ok/len(mmlu)*100:.0f}%\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "## 5. Coding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for name, prompt in [\n",
    "    (\"Binary Search\", \"Write Python `binary_search(arr, target)` returning index or -1.\"),\n",
    "    (\"Fibonacci\", \"Write Python `fib(n)` for nth Fibonacci using DP.\"),\n",
    "]:\n",
    "    print(f\"\\n{'='*40}\\n{name}\\n{'='*40}\")\n",
    "    inputs = processor.apply_chat_template([{\"role\":\"user\",\"content\":prompt}],\n",
    "        tokenize=True, return_dict=True, return_tensors=\"pt\", add_generation_prompt=True).to(model.device)\n",
    "    with torch.no_grad():\n",
    "        out = model.generate(**inputs, max_new_tokens=512, temperature=0.2, do_sample=False)\n",
    "    print(processor.decode(out[0][inputs[\"input_ids\"].shape[-1]:], skip_special_tokens=True)[:400])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "## 6. Image Understanding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"Test image understanding (16x downsample / efisien)...\")\n",
    "try:\n",
    "    url = \"https://huggingface.co/datasets/openbmb/DemoCase/resolve/main/refract.png\"\n",
    "    img = Image.open(BytesIO(requests.get(url, timeout=30).content))\n",
    "    display(img.resize((250, 180)))\n",
    "\n",
    "    msgs = [{\"role\":\"user\",\"content\":[\n",
    "        {\"type\":\"image\",\"image\":img},\n",
    "        {\"type\":\"text\",\"text\":\"What causes this phenomenon?\"}\n",
    "    ]}]\n",
    "\n",
    "    inputs = processor.apply_chat_template(msgs, tokenize=True, add_generation_prompt=True,\n",
    "        return_dict=True, return_tensors=\"pt\", downsample_mode=DOWNSAMPLE_MODE,\n",
    "    ).to(model.device)\n",
    "\n",
    "    t0 = time.perf_counter()\n",
    "    with torch.no_grad():\n",
    "        out = model.generate(**inputs, downsample_mode=DOWNSAMPLE_MODE, max_new_tokens=128, do_sample=True)\n",
    "    resp = processor.decode(out[0][inputs[\"input_ids\"].shape[-1]:], skip_special_tokens=True)\n",
    "    print(f\"Time: {time.perf_counter()-t0:.1f}s\")\n",
    "    print(f\"Answer: {resp[:300]}\")\n",
    "except Exception as e:\n",
    "    print(f\"ERROR: {e}\")\n",
    "\n",
    "print(\"\\n---\\nTest image understanding (4x downsample / detail tinggi)...\")\n",
    "try:\n",
    "    url2 = \"https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/p-blog/candy.JPG\"\n",
    "    img2 = Image.open(BytesIO(requests.get(url2, timeout=30).content))\n",
    "    display(img2.resize((250, 180)))\n",
    "\n",
    "    msgs2 = [{\"role\":\"user\",\"content\":[\n",
    "        {\"type\":\"image\",\"image\":img2},\n",
    "        {\"type\":\"text\",\"text\":\"What animal is on the candy?\"}\n",
    "    ]}]\n",
    "\n",
    "    inputs2 = processor.apply_chat_template(msgs2, tokenize=True, add_generation_prompt=True,\n",
    "        return_dict=True, return_tensors=\"pt\", downsample_mode=\"4x\",\n",
    "    ).to(model.device)\n",
    "\n",
    "    t0 = time.perf_counter()\n",
    "    with torch.no_grad():\n",
    "        out2 = model.generate(**inputs2, downsample_mode=\"4x\", max_new_tokens=128, do_sample=True)\n",
    "    resp2 = processor.decode(out2[0][inputs2[\"input_ids\"].shape[-1]:], skip_special_tokens=True)\n",
    "    print(f\"Time: {time.perf_counter()-t0:.1f}s\")\n",
    "    print(f\"Answer: {resp2[:300]}\")\n",
    "except Exception as e:\n",
    "    print(f\"ERROR: {e}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "## 7. Long Context (Needle-in-Haystack)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def needle(pos):\n",
    "    needle_str = \"The secret code is BLUE-42-GREEN.\"\n",
    "    filler = \"The quick brown fox jumps over the lazy dog. Python is versatile. \"\n",
    "    sents = [filler] * 50\n",
    "    if pos == \"early\": sents.insert(0, needle_str)\n",
    "    elif pos == \"middle\": sents.insert(len(sents)//2, needle_str)\n",
    "    else: sents.append(needle_str)\n",
    "    prompt = f\"Read the text and answer.\\n\\nText: {' '.join(sents)}\\n\\nQ: What is the secret code? Answer with code only.\"\n",
    "    inputs = processor.apply_chat_template([{\"role\":\"user\",\"content\":prompt}],\n",
    "        tokenize=True, return_dict=True, return_tensors=\"pt\", add_generation_prompt=True).to(model.device)\n",
    "    t0 = time.perf_counter()\n",
    "    with torch.no_grad():\n",
    "        out = model.generate(**inputs, max_new_tokens=16, do_sample=False)\n",
    "    resp = processor.decode(out[0][inputs[\"input_ids\"].shape[-1]:], skip_special_tokens=True).strip()\n",
    "    return {\"pos\": pos, \"tokens\": inputs[\"input_ids\"].shape[-1], \"resp\": resp[:80],\n",
    "            \"correct\": \"BLUE-42-GREEN\" in resp, \"time\": time.perf_counter()-t0}\n",
    "\n",
    "print(\"Needle-in-Haystack:\")\n",
    "for p in [\"early\", \"middle\", \"late\"]:\n",
    "    r = needle(p)\n",
    "    print(f\"{p.upper()}: {r['tokens']} tok, correct={'OK' if r['correct'] else 'NO'}, {r['time']:.1f}s\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "## 8. Summary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "rows = [\n",
    "    [\"Model\", MODEL_ID],\n",
    "    [\"Parameters\", f\"{total_params/1e9:.2f}B\"],\n",
    "    [\"Device\", str(model.device)],\n",
    "    [\"Dtype\", str(model.dtype)],\n",
    "]\n",
    "if torch.cuda.is_available():\n",
    "    rows.append([\"VRAM\", f\"{torch.cuda.memory_allocated()/1e9:.2f} GB\"])\n",
    "rows.append([\"RAM\", f\"{ram_used:.2f} GB\"])\n",
    "rows.append([\"Load Time\", f\"{load_time:.1f}s\"])\n",
    "if results:\n",
    "    rows.append([\"Throughput\", f\"{np.mean([r['tps'] for r in results]):.1f} tok/s\"])\n",
    "rows.append([\"MMLU\", f\"{ok}/{len(mmlu)} ({ok/len(mmlu)*100:.0f}%)\"])\n",
    "if 'nh' in dir():\n",
    "    rows.append([\"Needle/Haystack\", f\"{sum(1 for r in nh if r['correct'])}/{len(nh)} correct\"])\n",
    "\n",
    "print(\"=\"*60)\n",
    "print(\"  MINICPM-V 4.6 — BENCHMARK SUMMARY\")\n",
    "print(\"=\"*60)\n",
    "print(tabulate(rows, headers=[\"Metric\",\"Value\"], tablefmt=\"grid\"))\n",
    "print(\"=\"*60)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "## 9. Cleanup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "del model, processor\n",
    "gc.collect()\n",
    "if torch.cuda.is_available():\n",
    "    torch.cuda.empty_cache()\n",
    "    print(f\"VRAM: {torch.cuda.memory_allocated()/1e9:.2f} GB\")\n",
    "print(\"Done.\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}