commit c58c4bca037e5f08bb7628822e63f6617a488736 Author: asepharyana Date: Fri Jul 24 17:30:43 2026 +0700 Add MiniCPM-V 4.6 Benchmark Notebook for model evaluation and performance metrics diff --git a/.kilo/agents/data.md b/.kilo/agents/data.md new file mode 100644 index 0000000..deae2e8 --- /dev/null +++ b/.kilo/agents/data.md @@ -0,0 +1,28 @@ +--- +mode: primary +description: Run notebook-first data analysis by appending and executing cells + for each request. +options: + displayName: Data + id: data +requirements: + skills: + - data-investigation + vscode_extensions: + - name: Jupyter + id: ms-toolsai.jupyter +color: "#2563EB" +--- + +You are Kilo, a notebook-first data analysis agent. Use an active Jupyter notebook as the working surface. + +Guidelines: +- If no notebook is active, create a uniquely named, descriptive `.ipynb` in the current workspace folder +- Use the dedicated notebook tools to create, read, edit, and execute; prefer these tools over other methods like MCP tools and manual raw JSON editing +- Confirm Jupyter and kernel readiness through the first requested notebook execution; only notify the user if they need to select or configure a kernel before work can continue +- For every user request, append at least one focused code cell and execute it +- Preserve notebook history: do not modify or delete existing cells unless explicitly asked; after failures, append diagnostic or corrected cells +- Keep substantive data work and supporting evidence in the notebook +- Avoid changing non-notebook files unless explicitly requested or necessary to complete the task +- Inspect cell output before answering, and keep notebook outputs and final summaries concise +- Never claim execution when a notebook cell did not run diff --git a/gemma4_e4b_clone_benchmark.ipynb b/gemma4_e4b_clone_benchmark.ipynb new file mode 100644 index 0000000..9cc4fa0 --- /dev/null +++ b/gemma4_e4b_clone_benchmark.ipynb @@ -0,0 +1,616 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Gemma 4 E4B — Clone & Benchmark Notebook\n", + "\n", + "Notebook untuk meng-clone model [google/gemma-4-E4B](https://huggingface.co/google/gemma-4-E4B) dari Hugging Face dan melakukan benchmark pada berbagai metrik:\n", + "- Kecepatan loading & memory usage\n", + "- Text generation throughput (tokens/sec)\n", + "- Reasoning & knowledge QA\n", + "- Coding capability\n", + "- Multimodal understanding (image)\n", + "- Long context retrieval\n", + "\n", + "**Model**: `google/gemma-4-E4B-it` (instruction-tuned, 4.5B effective params, 8B total, 128K context)\n", + "\n", + "**Cara pakai**: Runtime > Factory reset runtime, lalu Runtime > Run all" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "---\n", + "## 1. Environment Setup" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + " ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 60.7/60.7 MB 13.8 MB/s eta 0:00:00\n", + "Install selesai\n" + ] + } + ], + "source": [ + "# Install dependencies\n", + "!pip install -qU \\\n", + " 'transformers>=4.50.0' \\\n", + " accelerate \\\n", + " sentencepiece \\\n", + " protobuf \\\n", + " psutil \\\n", + " 'pillow<11' \\\n", + " requests \\\n", + " matplotlib \\\n", + " tabulate \\\n", + " librosa \\\n", + " soundfile \\\n", + " einops \\\n", + " bitsandbytes \\\n", + " 2>&1 | tail -3\n", + "print(\"Install selesai\")" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Python : 3.12.13 (main, Mar 4 2026, 09:23:07) [GCC 11.4.0]\n", + "PyTorch : 2.11.0+cu128\n", + "CUDA avail : True\n", + "CUDA device : Tesla T4\n", + "CUDA VRAM : 15.6 GB\n", + "CUDA cap : (7, 5)\n" + ] + } + ], + "source": [ + "import os, sys, json, time, gc, warnings\n", + "from pathlib import Path\n", + "from datetime import datetime\n", + "from IPython.display import display\n", + "\n", + "import torch\n", + "import psutil\n", + "import numpy as np\n", + "import matplotlib.pyplot as plt\n", + "from tabulate import tabulate\n", + "from PIL import Image\n", + "import requests\n", + "from io import BytesIO\n", + "\n", + "warnings.filterwarnings(\"ignore\")\n", + "\n", + "print(f\"Python : {sys.version}\")\n", + "print(f\"PyTorch : {torch.__version__}\")\n", + "print(f\"CUDA avail : {torch.cuda.is_available()}\")\n", + "if torch.cuda.is_available():\n", + " print(f\"CUDA device : {torch.cuda.get_device_name(0)}\")\n", + " print(f\"CUDA VRAM : {torch.cuda.get_device_properties(0).total_memory / 1e9:.1f} GB\")\n", + " print(f\"CUDA cap : {torch.cuda.get_device_capability()}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "---\n", + "## 2. Clone Model from Hugging Face\n", + "\n", + "Model size ~16 GB dalam BF16. Karena T4 hanya 15.6GB VRAM, kita perlu 4-bit quantization." + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Model ID: google/gemma-4-E4B-it\n", + "Loading (4-bit quantized)...\n", + "Processor loaded in 6.2s\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "169a4541ec42498083b317e11576b863", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "Loading weights: 0%| | 0/2076 [00:00" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Time: 23.8s\n", + "This image displays a photograph featuring the **Golden Gate Bridge** spanning across a body of water toward a landmass in the distance.\n", + "\n", + "Here is a brief description:\n", + "\n", + "The dominant feature is the massive **red bridge structure** (clearly identifiable\n" + ] + } + ], + "source": [ + "try:\n", + " url = \"https://raw.githubusercontent.com/google-gemma/cookbook/main/apps/sample-data/GoldenGate.png\"\n", + " img = Image.open(BytesIO(requests.get(url, timeout=30).content))\n", + " display(img.resize((250, 180)))\n", + " inputs = processor.apply_chat_template([{\"role\":\"user\",\"content\":[\n", + " {\"type\":\"image\",\"image\":img},\n", + " {\"type\":\"text\",\"text\":\"What is shown? Describe briefly.\"}\n", + " ]}], tokenize=True, return_dict=True, return_tensors=\"pt\", add_generation_prompt=True).to(model.device)\n", + " t0 = time.perf_counter()\n", + " with torch.no_grad():\n", + " out = model.generate(**inputs, max_new_tokens=128, temperature=0.7, do_sample=True)\n", + " print(f\"Time: {time.perf_counter()-t0:.1f}s\")\n", + " print(processor.decode(out[0][inputs[\"input_ids\"].shape[-1]:], skip_special_tokens=True)[:250])\n", + "except Exception as e:\n", + " print(f\"ERROR: {e}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "---\n", + "## 7. Long Context (Needle-in-Haystack)" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Needle-in-Haystack:\n", + "EARLY: 793 tok, correct=OK, 4.9s\n", + "MIDDLE: 793 tok, correct=OK, 5.0s\n", + "LATE: 793 tok, correct=OK, 5.0s\n" + ] + } + ], + "source": [ + "torch.cuda.empty_cache()\n", + "gc.collect()\n", + "\n", + "def needle(pos):\n", + " needle_str = \"The secret code is BLUE-42-GREEN.\"\n", + " filler = \"The quick brown fox jumps over the lazy dog. Python is versatile. \"\n", + " sents = [filler] * 50 # shorter context for T4\n", + " if pos == \"early\": sents.insert(0, needle_str)\n", + " elif pos == \"middle\": sents.insert(len(sents)//2, needle_str)\n", + " else: sents.append(needle_str)\n", + " prompt = f\"Read the text and answer.\\n\\nText: {' '.join(sents)}\\n\\nQ: What is the secret code? Answer with code only.\"\n", + " inputs = processor.apply_chat_template([{\"role\":\"user\",\"content\":prompt}],\n", + " tokenize=True, return_dict=True, return_tensors=\"pt\", add_generation_prompt=True).to(model.device)\n", + " t0 = time.perf_counter()\n", + " with torch.no_grad():\n", + " out = model.generate(**inputs, max_new_tokens=16, do_sample=False)\n", + " resp = processor.decode(out[0][inputs[\"input_ids\"].shape[-1]:], skip_special_tokens=True).strip()\n", + " return {\"pos\": pos, \"tokens\": inputs[\"input_ids\"].shape[-1], \"resp\": resp[:80],\n", + " \"correct\": \"BLUE-42-GREEN\" in resp, \"time\": time.perf_counter()-t0}\n", + "\n", + "print(\"Needle-in-Haystack:\")\n", + "for p in [\"early\", \"middle\", \"late\"]:\n", + " r = needle(p)\n", + " print(f\"{p.upper()}: {r['tokens']} tok, correct={'OK' if r['correct'] else 'NO'}, {r['time']:.1f}s\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "---\n", + "## 8. Summary" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "============================================================\n", + " GEMMA 4 E4B — SUMMARY\n", + "============================================================\n", + "+------------+-----------------------+\n", + "| Metric | Value |\n", + "+============+=======================+\n", + "| Model | google/gemma-4-E4B-it |\n", + "+------------+-----------------------+\n", + "| Parameters | 5.72B |\n", + "+------------+-----------------------+\n", + "| Device | cuda:0 |\n", + "+------------+-----------------------+\n", + "| VRAM | 10.26 GB |\n", + "+------------+-----------------------+\n", + "| RAM | 2.67 GB |\n", + "+------------+-----------------------+\n", + "| Load Time | 63.4s |\n", + "+------------+-----------------------+\n", + "| Throughput | 5.6 tok/s |\n", + "+------------+-----------------------+\n", + "| MMLU | 4/5 (80%) |\n", + "+------------+-----------------------+\n", + "============================================================\n" + ] + } + ], + "source": [ + "rows = [\n", + " [\"Model\", MODEL_ID],\n", + " [\"Parameters\", f\"{total_params/1e9:.2f}B\"],\n", + " [\"Device\", str(model.device)],\n", + "]\n", + "if torch.cuda.is_available(): rows.append([\"VRAM\", f\"{torch.cuda.memory_allocated()/1e9:.2f} GB\"])\n", + "rows.append([\"RAM\", f\"{ram_used:.2f} GB\"])\n", + "rows.append([\"Load Time\", f\"{load_time:.1f}s\"])\n", + "if results: rows.append([\"Throughput\", f\"{np.mean([r['tps'] for r in results]):.1f} tok/s\"])\n", + "rows.append([\"MMLU\", f\"{ok}/{len(mmlu)} ({ok/len(mmlu)*100:.0f}%)\"])\n", + "\n", + "print(\"=\"*60)\n", + "print(\" GEMMA 4 E4B — SUMMARY\")\n", + "print(\"=\"*60)\n", + "print(tabulate(rows, headers=[\"Metric\",\"Value\"], tablefmt=\"grid\"))\n", + "print(\"=\"*60)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "---\n", + "## 9. Cleanup" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "VRAM: 10.26 GB\n", + "Done.\n" + ] + } + ], + "source": [ + "del model, processor\n", + "gc.collect()\n", + "if torch.cuda.is_available():\n", + " torch.cuda.empty_cache()\n", + " print(f\"VRAM: {torch.cuda.memory_allocated()/1e9:.2f} GB\")\n", + "print(\"Done.\")" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.13" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/minicpmv4_6_benchmark.ipynb b/minicpmv4_6_benchmark.ipynb new file mode 100644 index 0000000..0f2ae46 --- /dev/null +++ b/minicpmv4_6_benchmark.ipynb @@ -0,0 +1,423 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# MiniCPM-V 4.6 — Clone & Benchmark Notebook\n", + "\n", + "Notebook untuk meng-clone model [openbmb/MiniCPM-V-4.6](https://huggingface.co/openbmb/MiniCPM-V-4.6) dari Hugging Face dan melakukan benchmark:\n", + "- Loading speed & memory usage\n", + "- Text generation throughput (tokens/sec)\n", + "- Reasoning & knowledge QA\n", + "- Coding capability\n", + "- Multimodal image understanding\n", + "- Long context (needle-in-haystack)\n", + "\n", + "**Model**: `openbmb/MiniCPM-V-4.6` (SigLIP2-400M + Qwen3.5-0.8B, 1B total params)\n", + "\n", + "**Cara pakai**: Runtime > Factory reset runtime, lalu Runtime > Run all" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "---\n", + "## 1. Environment Setup" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Install dependencies — skip torch, Colab sudah punya\n", + "!pip install -qU \\\n", + " 'transformers>=5.7.0' \\\n", + " accelerate \\\n", + " sentencepiece \\\n", + " psutil \\\n", + " 'pillow<11' \\\n", + " requests \\\n", + " matplotlib \\\n", + " tabulate \\\n", + " av \\\n", + " einops \\\n", + " 2>&1 | tail -3\n", + "print(\"Install selesai\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import os, sys, json, time, gc, warnings\n", + "from pathlib import Path\n", + "from datetime import datetime\n", + "from IPython.display import display, Video\n", + "\n", + "import torch\n", + "import psutil\n", + "import numpy as np\n", + "import matplotlib.pyplot as plt\n", + "from tabulate import tabulate\n", + "from PIL import Image\n", + "import requests\n", + "from io import BytesIO\n", + "\n", + "warnings.filterwarnings(\"ignore\")\n", + "\n", + "print(f\"Python : {sys.version}\")\n", + "print(f\"PyTorch : {torch.__version__}\")\n", + "print(f\"CUDA avail : {torch.cuda.is_available()}\")\n", + "if torch.cuda.is_available():\n", + " print(f\"CUDA device : {torch.cuda.get_device_name(0)}\")\n", + " print(f\"CUDA VRAM : {torch.cuda.get_device_properties(0).total_memory / 1e9:.1f} GB\")\n", + " print(f\"CUDA cap : {torch.cuda.get_device_capability()}\")\n", + "\n", + "try:\n", + " import transformers\n", + " print(f\"transformers : {transformers.__version__}\")\n", + "except:\n", + " print(\"transformers : NOT INSTALLED\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "---\n", + "## 2. Clone Model from Hugging Face\n", + "\n", + "MiniCPM-V 4.6 = 1B params. Muat di T4 (15.6GB) tanpa masalah." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "MODEL_ID = \"openbmb/MiniCPM-V-4.6\"\n", + "CACHE_DIR = None # Set ke \"drive/MyDrive/hf_cache\" untuk persistent\n", + "DOWNSAMPLE_MODE = \"16x\" # \"16x\" = efisien, \"4x\" = detail lebih tinggi\n", + "\n", + "print(f\"Model ID: {MODEL_ID}\")\n", + "print(\"Loading...\")\n", + "t0 = time.perf_counter()\n", + "\n", + "from transformers import AutoProcessor, AutoModelForImageTextToText\n", + "\n", + "processor = AutoProcessor.from_pretrained(MODEL_ID, cache_dir=CACHE_DIR)\n", + "print(f\"Processor loaded in {time.perf_counter()-t0:.1f}s\")\n", + "\n", + "load_start = time.perf_counter()\n", + "model = AutoModelForImageTextToText.from_pretrained(\n", + " MODEL_ID,\n", + " torch_dtype=torch.bfloat16,\n", + " device_map=\"auto\",\n", + " cache_dir=CACHE_DIR,\n", + ")\n", + "load_time = time.perf_counter() - load_start\n", + "print(f\"\\nModel loaded in {load_time:.1f}s\")\n", + "\n", + "total_params = sum(p.numel() for p in model.parameters())\n", + "print(f\"Parameters: {total_params/1e9:.2f}B\")\n", + "print(f\"Device: {model.device}, Dtype: {model.dtype}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "if torch.cuda.is_available():\n", + " vram_used = torch.cuda.memory_allocated() / 1e9\n", + " print(f\"VRAM allocated: {vram_used:.2f} GB\")\n", + "ram_used = psutil.Process(os.getpid()).memory_info().rss / 1e9\n", + "print(f\"RAM used: {ram_used:.2f} GB\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "---\n", + "## 3. Text Generation Throughput" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def throughput(prompt, max_tokens=256, runs=3):\n", + " msgs = [{\"role\": \"user\", \"content\": prompt}]\n", + " inputs = processor.apply_chat_template(msgs, tokenize=True, return_dict=True,\n", + " return_tensors=\"pt\", add_generation_prompt=True).to(model.device)\n", + " inp_len = inputs[\"input_ids\"].shape[-1]\n", + " lats, toks = [], []\n", + " for _ in range(runs):\n", + " start = time.perf_counter()\n", + " with torch.no_grad():\n", + " out = model.generate(**inputs, max_new_tokens=max_tokens, do_sample=True, temperature=0.7)\n", + " elapsed = time.perf_counter() - start\n", + " gen = out[0][inp_len:]\n", + " lats.append(elapsed)\n", + " toks.append(len(gen))\n", + " tps = [t/l for t,l in zip(toks, lats)]\n", + " return {\"prompt\": prompt[:60]+\"...\", \"tokens\": int(np.mean(toks)),\n", + " \"latency\": float(np.mean(lats)), \"tps\": float(np.mean(tps))}\n", + "\n", + "prompts = {\n", + " \"Simple QA\": \"What is the capital of Indonesia?\",\n", + " \"Math\": \"If a train travels at 120 km/h and another at 80 km/h toward each other from 500 km apart, how long until they meet?\",\n", + " \"Code\": \"Write a Python function to find the longest palindromic substring.\",\n", + "}\n", + "\n", + "results = []\n", + "for name, p in prompts.items():\n", + " r = throughput(p, runs=2)\n", + " results.append(r)\n", + " print(f\"{name}: {r['tokens']} tok, {r['latency']:.1f}s, {r['tps']:.1f} tok/s\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "---\n", + "## 4. Reasoning (MMLU-style)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "mmlu = [\n", + " {\"q\": \"What is the time complexity of binary search?\", \"o\": [\"A. O(n)\", \"B. O(log n)\", \"C. O(n log n)\", \"D. O(1)\"], \"a\": \"B\"},\n", + " {\"q\": \"Which planet has the strongest surface gravity?\", \"o\": [\"A. Earth\", \"B. Mars\", \"C. Jupiter\", \"D. Saturn\"], \"a\": \"C\"},\n", + " {\"q\": \"In C++, which keyword prevents overriding?\", \"o\": [\"A. static\", \"B. const\", \"C. final\", \"D. override\"], \"a\": \"C\"},\n", + " {\"q\": \"Probability of drawing a red ball from 3 red + 5 blue?\", \"o\": [\"A. 3/5\", \"B. 3/8\", \"C. 5/8\", \"D. 1/2\"], \"a\": \"B\"},\n", + " {\"q\": \"What does mitochondria do?\", \"o\": [\"A. Protein\", \"B. Energy (ATP)\", \"C. Lipid\", \"D. DNA\"], \"a\": \"B\"},\n", + "]\n", + "\n", + "ok = 0\n", + "for q in mmlu:\n", + " prompt = f\"{q['q']}\\n\\n\" + \"\\n\".join(q[\"o\"]) + \"\\n\\nAnswer with a single letter:\"\n", + " inputs = processor.apply_chat_template([{\"role\":\"user\",\"content\":prompt}],\n", + " tokenize=True, return_dict=True, return_tensors=\"pt\", add_generation_prompt=True).to(model.device)\n", + " with torch.no_grad():\n", + " out = model.generate(**inputs, max_new_tokens=8, do_sample=False)\n", + " ans = processor.decode(out[0][inputs[\"input_ids\"].shape[-1]:], skip_special_tokens=True).strip()\n", + " cor = q[\"a\"] in ans.upper()[:1]\n", + " if cor: ok += 1\n", + " print(f\" {'OK' if cor else 'NO'} Expected={q['a']} Got={ans[:20]} | {q['q'][:50]}\")\n", + "\n", + "print(f\"\\nAccuracy: {ok}/{len(mmlu)} = {ok/len(mmlu)*100:.0f}%\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "---\n", + "## 5. Coding" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "for name, prompt in [\n", + " (\"Binary Search\", \"Write Python `binary_search(arr, target)` returning index or -1.\"),\n", + " (\"Fibonacci\", \"Write Python `fib(n)` for nth Fibonacci using DP.\"),\n", + "]:\n", + " print(f\"\\n{'='*40}\\n{name}\\n{'='*40}\")\n", + " inputs = processor.apply_chat_template([{\"role\":\"user\",\"content\":prompt}],\n", + " tokenize=True, return_dict=True, return_tensors=\"pt\", add_generation_prompt=True).to(model.device)\n", + " with torch.no_grad():\n", + " out = model.generate(**inputs, max_new_tokens=512, temperature=0.2, do_sample=False)\n", + " print(processor.decode(out[0][inputs[\"input_ids\"].shape[-1]:], skip_special_tokens=True)[:400])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "---\n", + "## 6. Image Understanding" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "print(\"Test image understanding (16x downsample / efisien)...\")\n", + "try:\n", + " url = \"https://huggingface.co/datasets/openbmb/DemoCase/resolve/main/refract.png\"\n", + " img = Image.open(BytesIO(requests.get(url, timeout=30).content))\n", + " display(img.resize((250, 180)))\n", + "\n", + " msgs = [{\"role\":\"user\",\"content\":[\n", + " {\"type\":\"image\",\"image\":img},\n", + " {\"type\":\"text\",\"text\":\"What causes this phenomenon?\"}\n", + " ]}]\n", + "\n", + " inputs = processor.apply_chat_template(msgs, tokenize=True, add_generation_prompt=True,\n", + " return_dict=True, return_tensors=\"pt\", downsample_mode=DOWNSAMPLE_MODE,\n", + " ).to(model.device)\n", + "\n", + " t0 = time.perf_counter()\n", + " with torch.no_grad():\n", + " out = model.generate(**inputs, downsample_mode=DOWNSAMPLE_MODE, max_new_tokens=128, do_sample=True)\n", + " resp = processor.decode(out[0][inputs[\"input_ids\"].shape[-1]:], skip_special_tokens=True)\n", + " print(f\"Time: {time.perf_counter()-t0:.1f}s\")\n", + " print(f\"Answer: {resp[:300]}\")\n", + "except Exception as e:\n", + " print(f\"ERROR: {e}\")\n", + "\n", + "print(\"\\n---\\nTest image understanding (4x downsample / detail tinggi)...\")\n", + "try:\n", + " url2 = \"https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/p-blog/candy.JPG\"\n", + " img2 = Image.open(BytesIO(requests.get(url2, timeout=30).content))\n", + " display(img2.resize((250, 180)))\n", + "\n", + " msgs2 = [{\"role\":\"user\",\"content\":[\n", + " {\"type\":\"image\",\"image\":img2},\n", + " {\"type\":\"text\",\"text\":\"What animal is on the candy?\"}\n", + " ]}]\n", + "\n", + " inputs2 = processor.apply_chat_template(msgs2, tokenize=True, add_generation_prompt=True,\n", + " return_dict=True, return_tensors=\"pt\", downsample_mode=\"4x\",\n", + " ).to(model.device)\n", + "\n", + " t0 = time.perf_counter()\n", + " with torch.no_grad():\n", + " out2 = model.generate(**inputs2, downsample_mode=\"4x\", max_new_tokens=128, do_sample=True)\n", + " resp2 = processor.decode(out2[0][inputs2[\"input_ids\"].shape[-1]:], skip_special_tokens=True)\n", + " print(f\"Time: {time.perf_counter()-t0:.1f}s\")\n", + " print(f\"Answer: {resp2[:300]}\")\n", + "except Exception as e:\n", + " print(f\"ERROR: {e}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "---\n", + "## 7. Long Context (Needle-in-Haystack)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def needle(pos):\n", + " needle_str = \"The secret code is BLUE-42-GREEN.\"\n", + " filler = \"The quick brown fox jumps over the lazy dog. Python is versatile. \"\n", + " sents = [filler] * 50\n", + " if pos == \"early\": sents.insert(0, needle_str)\n", + " elif pos == \"middle\": sents.insert(len(sents)//2, needle_str)\n", + " else: sents.append(needle_str)\n", + " prompt = f\"Read the text and answer.\\n\\nText: {' '.join(sents)}\\n\\nQ: What is the secret code? Answer with code only.\"\n", + " inputs = processor.apply_chat_template([{\"role\":\"user\",\"content\":prompt}],\n", + " tokenize=True, return_dict=True, return_tensors=\"pt\", add_generation_prompt=True).to(model.device)\n", + " t0 = time.perf_counter()\n", + " with torch.no_grad():\n", + " out = model.generate(**inputs, max_new_tokens=16, do_sample=False)\n", + " resp = processor.decode(out[0][inputs[\"input_ids\"].shape[-1]:], skip_special_tokens=True).strip()\n", + " return {\"pos\": pos, \"tokens\": inputs[\"input_ids\"].shape[-1], \"resp\": resp[:80],\n", + " \"correct\": \"BLUE-42-GREEN\" in resp, \"time\": time.perf_counter()-t0}\n", + "\n", + "print(\"Needle-in-Haystack:\")\n", + "for p in [\"early\", \"middle\", \"late\"]:\n", + " r = needle(p)\n", + " print(f\"{p.upper()}: {r['tokens']} tok, correct={'OK' if r['correct'] else 'NO'}, {r['time']:.1f}s\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "---\n", + "## 8. Summary" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "rows = [\n", + " [\"Model\", MODEL_ID],\n", + " [\"Parameters\", f\"{total_params/1e9:.2f}B\"],\n", + " [\"Device\", str(model.device)],\n", + " [\"Dtype\", str(model.dtype)],\n", + "]\n", + "if torch.cuda.is_available():\n", + " rows.append([\"VRAM\", f\"{torch.cuda.memory_allocated()/1e9:.2f} GB\"])\n", + "rows.append([\"RAM\", f\"{ram_used:.2f} GB\"])\n", + "rows.append([\"Load Time\", f\"{load_time:.1f}s\"])\n", + "if results:\n", + " rows.append([\"Throughput\", f\"{np.mean([r['tps'] for r in results]):.1f} tok/s\"])\n", + "rows.append([\"MMLU\", f\"{ok}/{len(mmlu)} ({ok/len(mmlu)*100:.0f}%)\"])\n", + "if 'nh' in dir():\n", + " rows.append([\"Needle/Haystack\", f\"{sum(1 for r in nh if r['correct'])}/{len(nh)} correct\"])\n", + "\n", + "print(\"=\"*60)\n", + "print(\" MINICPM-V 4.6 — BENCHMARK SUMMARY\")\n", + "print(\"=\"*60)\n", + "print(tabulate(rows, headers=[\"Metric\",\"Value\"], tablefmt=\"grid\"))\n", + "print(\"=\"*60)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "---\n", + "## 9. Cleanup" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "del model, processor\n", + "gc.collect()\n", + "if torch.cuda.is_available():\n", + " torch.cuda.empty_cache()\n", + " print(f\"VRAM: {torch.cuda.memory_allocated()/1e9:.2f} GB\")\n", + "print(\"Done.\")" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +}