MythEclipse/llm
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity
Files
main
llm/gemma4_e4b_clone_benchmark.ipynb
T

617 lines
129 KiB
Plaintext
Raw Permalink Normal View History

Add MiniCPM-V 4.6 Benchmark Notebook for model evaluation and performance metrics
2026-07-24 17:30:43 +07:00
{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Gemma 4 E4B — Clone & Benchmark Notebook\n",
"\n",
"Notebook untuk meng-clone model [google/gemma-4-E4B](https://huggingface.co/google/gemma-4-E4B) dari Hugging Face dan melakukan benchmark pada berbagai metrik:\n",
"- Kecepatan loading & memory usage\n",
"- Text generation throughput (tokens/sec)\n",
"- Reasoning & knowledge QA\n",
"- Coding capability\n",
"- Multimodal understanding (image)\n",
"- Long context retrieval\n",
"\n",
"**Model**: `google/gemma-4-E4B-it` (instruction-tuned, 4.5B effective params, 8B total, 128K context)\n",
"\n",
"**Cara pakai**: Runtime > Factory reset runtime, lalu Runtime > Run all"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"---\n",
"## 1. Environment Setup"
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
" ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 60.7/60.7 MB 13.8 MB/s eta 0:00:00\n",
"Install selesai\n"
]
}
],
"source": [
"# Install dependencies\n",
"!pip install -qU \\\n",
" 'transformers>=4.50.0' \\\n",
" accelerate \\\n",
" sentencepiece \\\n",
" protobuf \\\n",
" psutil \\\n",
" 'pillow<11' \\\n",
" requests \\\n",
" matplotlib \\\n",
" tabulate \\\n",
" librosa \\\n",
" soundfile \\\n",
" einops \\\n",
" bitsandbytes \\\n",
" 2>&1 | tail -3\n",
"print(\"Install selesai\")"
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Python : 3.12.13 (main, Mar 4 2026, 09:23:07) [GCC 11.4.0]\n",
"PyTorch : 2.11.0+cu128\n",
"CUDA avail : True\n",
"CUDA device : Tesla T4\n",
"CUDA VRAM : 15.6 GB\n",
"CUDA cap : (7, 5)\n"
]
}
],
"source": [
"import os, sys, json, time, gc, warnings\n",
"from pathlib import Path\n",
"from datetime import datetime\n",
"from IPython.display import display\n",
"\n",
"import torch\n",
"import psutil\n",
"import numpy as np\n",
"import matplotlib.pyplot as plt\n",
"from tabulate import tabulate\n",
"from PIL import Image\n",
"import requests\n",
"from io import BytesIO\n",
"\n",
"warnings.filterwarnings(\"ignore\")\n",
"\n",
"print(f\"Python : {sys.version}\")\n",
"print(f\"PyTorch : {torch.__version__}\")\n",
"print(f\"CUDA avail : {torch.cuda.is_available()}\")\n",
"if torch.cuda.is_available():\n",
" print(f\"CUDA device : {torch.cuda.get_device_name(0)}\")\n",
" print(f\"CUDA VRAM : {torch.cuda.get_device_properties(0).total_memory / 1e9:.1f} GB\")\n",
" print(f\"CUDA cap : {torch.cuda.get_device_capability()}\")"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"---\n",
"## 2. Clone Model from Hugging Face\n",
"\n",
"Model size ~16 GB dalam BF16. Karena T4 hanya 15.6GB VRAM, kita perlu 4-bit quantization."
]
},
{
"cell_type": "code",
"execution_count": 10,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Model ID: google/gemma-4-E4B-it\n",
"Loading (4-bit quantized)...\n",
"Processor loaded in 6.2s\n"
]
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "169a4541ec42498083b317e11576b863",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"Loading weights: 0%| | 0/2076 [00:00<?, ?it/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"\n",
"Model loaded in 63.4s\n",
"Parameters: 5.72B\n",
"Device: cuda:0, Dtype: torch.bfloat16\n"
]
}
],
"source": [
"MODEL_ID = \"google/gemma-4-E4B-it\"\n",
"CACHE_DIR = None\n",
"\n",
"print(f\"Model ID: {MODEL_ID}\")\n",
"print(\"Loading (4-bit quantized)...\")\n",
"t0 = time.perf_counter()\n",
"\n",
"from transformers import AutoProcessor, AutoModelForMultimodalLM, BitsAndBytesConfig\n",
"import accelerate\n",
"\n",
"bnb_config = BitsAndBytesConfig(\n",
" load_in_4bit=True,\n",
" bnb_4bit_compute_dtype=torch.bfloat16,\n",
" bnb_4bit_use_double_quant=True,\n",
")\n",
"\n",
"processor = AutoProcessor.from_pretrained(MODEL_ID, cache_dir=CACHE_DIR)\n",
"print(f\"Processor loaded in {time.perf_counter()-t0:.1f}s\")\n",
"\n",
"load_start = time.perf_counter()\n",
"model = AutoModelForMultimodalLM.from_pretrained(\n",
" MODEL_ID,\n",
" torch_dtype=torch.bfloat16,\n",
" device_map={\"\": \"cuda:0\"},\n",
" max_memory={0: \"14GiB\", \"cpu\": \"48GiB\"},\n",
" cache_dir=CACHE_DIR,\n",
" quantization_config=bnb_config,\n",
")\n",
"load_time = time.perf_counter() - load_start\n",
"print(f\"\\nModel loaded in {load_time:.1f}s\")\n",
"\n",
"total_params = sum(p.numel() for p in model.parameters())\n",
"print(f\"Parameters: {total_params/1e9:.2f}B\")\n",
"print(f\"Device: {model.device}, Dtype: {model.dtype}\")"
]
},
{
"cell_type": "code",
"execution_count": 11,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"VRAM allocated: 9.32 GB\n",
"RAM used: 2.67 GB\n"
]
}
],
"source": [
"if torch.cuda.is_available():\n",
" vram_used = torch.cuda.memory_allocated() / 1e9\n",
" print(f\"VRAM allocated: {vram_used:.2f} GB\")\n",
"ram_used = psutil.Process(os.getpid()).memory_info().rss / 1e9\n",
"print(f\"RAM used: {ram_used:.2f} GB\")"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"---\n",
"## 3. Text Generation Throughput"
]
},
{
"cell_type": "code",
"execution_count": 12,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Simple QA: 42 tok, 7.5s, 5.2 tok/s\n",
"Math: 217 tok, 37.3s, 5.8 tok/s\n",
"Code: 256 tok, 43.7s, 5.9 tok/s\n"
]
}
],
"source": [
"def throughput(prompt, max_tokens=256, runs=3):\n",
" msgs = [{\"role\": \"user\", \"content\": prompt}]\n",
" inputs = processor.apply_chat_template(msgs, tokenize=True, return_dict=True,\n",
" return_tensors=\"pt\", add_generation_prompt=True).to(model.device)\n",
" inp_len = inputs[\"input_ids\"].shape[-1]\n",
" lats, toks = [], []\n",
" for _ in range(runs):\n",
" start = time.perf_counter()\n",
" with torch.no_grad():\n",
" out = model.generate(**inputs, max_new_tokens=max_tokens, do_sample=True, temperature=0.7)\n",
" elapsed = time.perf_counter() - start\n",
" gen = out[0][inp_len:]\n",
" lats.append(elapsed)\n",
" toks.append(len(gen))\n",
" tps = [t/l for t,l in zip(toks, lats)]\n",
" return {\"prompt\": prompt[:60]+\"...\", \"tokens\": int(np.mean(toks)),\n",
" \"latency\": float(np.mean(lats)), \"tps\": float(np.mean(tps))}\n",
"\n",
"prompts = {\n",
" \"Simple QA\": \"What is the capital of Indonesia?\",\n",
" \"Math\": \"If a train travels at 120 km/h and another at 80 km/h toward each other from 500 km apart, how long until they meet?\",\n",
" \"Code\": \"Write a Python function to find the longest palindromic substring.\",\n",
"}\n",
"\n",
"results = []\n",
"for name, p in prompts.items():\n",
" r = throughput(p, runs=2)\n",
" results.append(r)\n",
" print(f\"{name}: {r['tokens']} tok, {r['latency']:.1f}s, {r['tps']:.1f} tok/s\")"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"---\n",
"## 4. Reasoning (MMLU-style)"
]
},
{
"cell_type": "code",
"execution_count": 13,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
" OK Expected=B Got=B | What is the time complexity of binary search?\n",
" OK Expected=C Got=C | Which planet has the strongest surface gravity?\n",
" OK Expected=C Got=C | In C++, which keyword prevents overriding?\n",
" NO Expected=B Got=A | Probability of drawing a red ball from 3 red + 5 b\n",
" OK Expected=B Got=B | What does mitochondria do?\n",
"\n",
"Accuracy: 4/5 = 80%\n"
]
}
],
"source": [
"mmlu = [\n",
" {\"q\": \"What is the time complexity of binary search?\", \"o\": [\"A. O(n)\", \"B. O(log n)\", \"C. O(n log n)\", \"D. O(1)\"], \"a\": \"B\"},\n",
" {\"q\": \"Which planet has the strongest surface gravity?\", \"o\": [\"A. Earth\", \"B. Mars\", \"C. Jupiter\", \"D. Saturn\"], \"a\": \"C\"},\n",
" {\"q\": \"In C++, which keyword prevents overriding?\", \"o\": [\"A. static\", \"B. const\", \"C. final\", \"D. override\"], \"a\": \"C\"},\n",
" {\"q\": \"Probability of drawing a red ball from 3 red + 5 blue?\", \"o\": [\"A. 3/5\", \"B. 3/8\", \"C. 5/8\", \"D. 1/2\"], \"a\": \"B\"},\n",
" {\"q\": \"What does mitochondria do?\", \"o\": [\"A. Protein\", \"B. Energy (ATP)\", \"C. Lipid\", \"D. DNA\"], \"a\": \"B\"},\n",
"]\n",
"\n",
"ok = 0\n",
"for q in mmlu:\n",
" prompt = f\"{q['q']}\\n\\n\" + \"\\n\".join(q[\"o\"]) + \"\\n\\nAnswer with a single letter:\"\n",
" inputs = processor.apply_chat_template([{\"role\":\"user\",\"content\":prompt}],\n",
" tokenize=True, return_dict=True, return_tensors=\"pt\", add_generation_prompt=True).to(model.device)\n",
" with torch.no_grad():\n",
" out = model.generate(**inputs, max_new_tokens=8, do_sample=False)\n",
" ans = processor.decode(out[0][inputs[\"input_ids\"].shape[-1]:], skip_special_tokens=True).strip()\n",
" cor = q[\"a\"] in ans.upper()[:1]\n",
" if cor: ok += 1\n",
" print(f\" {'OK' if cor else 'NO'} Expected={q['a']} Got={ans[:20]} | {q['q'][:50]}\")\n",
"\n",
"print(f\"\\nAccuracy: {ok}/{len(mmlu)} = {ok/len(mmlu)*100:.0f}%\")"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"---\n",
"## 5. Coding"
]
},
{
"cell_type": "code",
"execution_count": 14,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"[transformers] The following generation flags are not valid and may be ignored: ['temperature']. Set `TRANSFORMERS_VERBOSITY=info` for more details.\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"\n",
"========================================\n",
"Binary Search\n",
"========================================\n",
"```python\n",
"def binary_search(arr, target):\n",
" \"\"\"\n",
" Performs a binary search on a sorted array to find the target element.\n",
"\n",
" Args:\n",
" arr: A sorted list of elements (the array to search).\n",
" target: The element whose index is to be found.\n",
"\n",
" Returns:\n",
" The index of the target if found, otherwise -1.\n",
" \"\"\"\n",
" left = 0\n",
" right = len(arr) - 1\n",
"\n",
" while left <= right:\n",
" # \n",
"\n",
"========================================\n",
"Fibonacci\n",
"========================================\n",
"Here are several ways to implement the Fibonacci sequence function `fib(n)` using Dynamic Programming (DP) in Python, depending on whether you want to optimize for space or time complexity.\n",
"\n",
"The standard Fibonacci sequence starts with $F_0 = 0$ and $F_1 = 1$.\n",
"\n",
"---\n",
"\n",
"## 1. Top-Down DP with Memoization (Recursive with Caching)\n",
"\n",
"This is the most direct translation of applying DP to the recursive defin\n"
]
}
],
"source": [
"for name, prompt in [\n",
" (\"Binary Search\", \"Write Python `binary_search(arr, target)` returning index or -1.\"),\n",
" (\"Fibonacci\", \"Write Python `fib(n)` for nth Fibonacci using DP.\"),\n",
"]:\n",
" print(f\"\\n{'='*40}\\n{name}\\n{'='*40}\")\n",
" inputs = processor.apply_chat_template([{\"role\":\"user\",\"content\":prompt}],\n",
" tokenize=True, return_dict=True, return_tensors=\"pt\", add_generation_prompt=True).to(model.device)\n",
" with torch.no_grad():\n",
" out = model.generate(**inputs, max_new_tokens=512, temperature=0.2, do_sample=False)\n",
" print(processor.decode(out[0][inputs[\"input_ids\"].shape[-1]:], skip_special_tokens=True)[:400])"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"---\n",
"## 6. Image Understanding"
]
},
{
"cell_type": "code",
"execution_count": 15,
"metadata": {},
"outputs": [
{
"data": {
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAPoAAAC0CAYAAAC9vjEhAAEAAElEQVR4AXT9V5Bu23bfh83OvXd375zPPjnegHMRLoCLi0AQJAFSNASCIinJskRZKrv8IMnlsvXgB7v8YJdt2VV+U8mUZEklybRFCyQYRBskCBAZuPncfHLaOe/dvXfn9u/3H3N+X59LanV/a80w0hxzzDzXXDP//n/4Tw8OZlo7aHXhxD3DHyFEzMxUiDDtYJbw/cQKJY6QXuJMrsPOIIIzI/QsPymEdDBmi0hoSTFsuJcE8uqXsoiNPAclWvziSEJP4vHMwGdfWgQkjmglFy9y8JTTfmQiTiDcI42SO8AvRyUOkdBXcrkY1CnDZB/ESlmDZulnhrCCFg7uoiGAPOP0TlRSVQElr3KEtPIXTEki7EHbI8jnbNeH6SyKRgh/0OYMwbmnoPLpECP3DoCZlU+PUVb+40t6g1PpUJYDiMlTEH+mK/qND08u01uSznbKEg1W0pxEBbfgkyNxVoz5MJWjuKk5eZUdBC5pjLBwLw0rv/maFBlVgHGUt+BlFj80lNVLKWZM5ADBGY3iD1dphw8g8hYPeLFnZkpb0zBhjSiwPLpsBhfLUMcDdQCUXYSiKEgoD+DESU8ZS7vknbGAaUsTi5QO/7NG4B52BijXTJtXNFmJ4FUGroNoI3J1KDNMwfkz8wVSrNmoSx9UCI6BCddhhO9kImBhBjQ89kw0fypdKiYbRvFZuQzcUnAlokI12IpObYUzuJ1eyPREKGnJrFwFV/xAoqQXGdNV9A5lYeKS7llN2BQrHhDAyteCpbxFpQzYtBM04RVdAT/S2Bni14yJFT4USj6JR5TwMB7twstKqQxb6Ep/1xjhukoDJWcowKMuJfdXhOuhjNLuvsgyxSIcz7RyxF1BhAllJPg9rPhoA8Z1XuGoVCVXQoOGrLIdMUXKADAKX5f5Ia60Ax4IAymqySzoagRdBi35gAJYmpBOUbNyG3IZOgduiVlUJ7Ql3fkLZ9qrMIZxUUtelX+kobCqFA19JAw5x1UVg1TL3ke48F0Z4T0sSR0nrgOK6dVTFNh5A4iIDUGn7MgUFJ+QxjkfP6gxQAIkMkQbihBp6Mn4hEdREtMACdXpjyumTkDV7BVWyicucLQ60Sx0VaR/o0ALXtrTEWSgoD9Da4YfXsNwk78mTkijeJr5ZYRWO2UgUtEdcgGWZ7HR261JV4X3p7V+pAttIOGtbU1kDXKhm+bKGFMjHnBJhz5gDtGOsPjrEm/wVaqCHHlgWg3dN9XRVZfdcPzJWPUfv9jRVnQU2aUHg1DGGd3hKdGsZsQJdSmEuxKIUJVYACbh+4QnrWKGiKBFPzjD7VMa8VeaJKKxpwUtFkUcGAviXqArTcWF6A5RKZROTyHE9Q09FS+hK/31PIQlQL+COfET3tNR0SVrpUlZquIwTrnVX2ypgJFBeC8xjC2ePuuXqDgHJCGBFjflovcCDZ9eKq+kkGM1UgSFCDxgNXp1sjJ4ogMcplFJhq57QSdYokDbpZNBGQfAFMCDWZLW9RQldWkk5zVDMyPRqi4qocYklpvCRZAepj9CyVOgBAjhNagY3OMJK3z9IwmlcEUtwyu+1hdVe3f+nexo+TsLHj1ToKcI9kJ8ljyG9ZZa3kRU+gyVYJdGBDKhCovBhkuXaxgPQeInqB65B6qnJbADqON/spIsfmZsCSkJw+pStkjWecnOdAwanX2lbeCFd+mO9q3LCyV4VFohRhrCmTBpyF5/5XaF7XWZCmsqjy5lUA1VAUIVv3kl/UQUOHeAOh3j/d8/UKtJFTYpDsYuc68aaxW9hMmk0+6ySlN+4hR+uAiWS4peMT0AK7zCEpFIxTSngLZlDr2CFd9GKjqZ8AzXwFroZ/YLRlLVKJWc6SFAK5QmiRKqLnvIUqqeVlUuQzLL2iSPQkMcYysFJsh05yLYGH8p6KUMo4RAxGGkYiSzTRZi9XALYKj5AGPUqnERkAIaSsbqMEldUXqjnTjSG+hZFFrCip/0J0MVtJRrLyC8Ot9wNzHIJafxG2Nv/aNLjHMCkGSYLjXAVQrVoWwG6khWGt0vqXEpO9fQhZKlQAGvuBNa0CidyVaadZXRF5nwAs8rj96rkbbpVA9zYOsPDUAHpaGTpDyyVmWcyiAU//tvMRRohbMtVkBLxxl2VlbBrCqtmQ5TlZN4/HVB5rqs0jJtXt7t5SlbDBx/wIm3AAumPzg+CdDmOjdCeuGEhoVUuD3v/DsGNaSzDy3hTUViCsygXCVTtDTFA0b4tIgMxzQE02NLbY/FS7m7c5LWRCRc7oVj7gupfY80DW6aijFDWtPtNZ7lHilJFLBeRSnuOIuXMfIbGInn5pMk5KpKrSQwILrlSRd/QAbOqB8YyBtSf0KEnmmLo5JgrNrIU6CkRMULR82mJFEIIbhTEahZfxVEcE2gCZAMt+YCK2DBIZSnCi0+Pb5rTdhPXIQHHwZxqQn+x2UhG0ZghtCmEV0AwieTuEnD0CrAnSaBho04XclueB6mMZQucPHqxDq+ipFGUSoe5TekaJWxBOgQZPG3MqisKK4lg+6RL5KW2TRdoW9QZ2ReeKXHY3oFTQU7ZIOecvYE21tK9x00MTsouVIyF7UqoASFUSrF6CZAyDykLhoSOhglrAikgJfUodLmko6S2/AhfzIiZJGx81BGycSPe1LZ406PAWS7zMpllTryRhwrVqjz81IH2kklXmjjytcFDRwhysdV90G3KIgQ8UXk6o+4Qtv4QzoRZkhRmjJtYvkzxPzqFXTSwmQbYWkcAiMcFKOPcjMZV5eGbF2hAtMqVHyUJG4MnchSjnBR2aGESVpq/ICPkhUuAnYmiUdxsY6CL/3IQFSE88lVCdTR6RHhZJRXcJQlvlJ8ZW0Ccqu48gtROMWzIxJauPJW3pHhXYRipNFATHrJ9DwNwKcRySLMjMWpjElzAru/khdQoycoh2gYyTXiQg0ljIIEq/CfQPT8Gq1z0g9MVCSs8hXJiXvi747i1QUC2nROEiRM948sTBBAg3LNu5QOBq5yJD4BCjDh2vO0uFbhq+rB1l/mpqV6R4UlTC5JFAgOqReXEVlQ03uhdVw8NZTT8IcsYg746j2NwmyoYEnV4N/DTJlyFhXvU3rRe6W8Uw4RwAsm3PrNB736wJnilKlOqnQnZaAkEXwxjAmBIMZWyygih+hVBdj76a0+uNqPaaHrroLF7URCDjcFrLLQOEgUUJgLO+2qACuNiAKOtHppHRSJ+udehVVcqqsksliDcxX3UiJRUkm8z4hiCA5jCq+jqz6o1DWoDVTBjRtY0SX+AZ9CQkISD1LIx4/E4hrBJVx3Tp46urlWKoQRqDMc8INbMozoGDuYSj0F128IYQhZbjnLg78uyGQIMIj3pzCB5aFLg6/qWaMYAkE9kV2ioBCXJ0i9ojX/K6juyWdp4i0Dw6EiQ1brqMtWxTCxYrSC4a4hUodLmIhG1GNCM8iTHEy0aehgwFc+h+1AjxzhAhwTv3HiQtiBZ7g/8XqxmPhrFr90bjqnl/iDAqFJW9cf3sgcxYy0CE3LC8qQLw6phIwSmGR5cU9GFH3vo5IQ27jSCRiFLGIR9tHdU7zK6/iJnxdZZgHkGSNNpo6MJczWlDivw8ZdCSOwp2LUnBKvTK+E6E3B69YhuFfguNdz+E00JAGaylTkitHALSMJB+CnnHCrtE7UR7WyOA7RlErFjWwuvoEx8tCl3WiwkfSTOV80uNuRMl1hy63c4ZCYhCdUwgIAUYHxp0B0fHFrgHLYIIqWY1QLjLlTxlGcol9BytsdPiYBga94woaxSAew3Ibi9YrGlccEYIhMzsdG9CeygHF2tHS1Yw/EhKxPfvYWhSk87+rdu6H8QqOHB3bAFG2TE5uVWAiBS6A0DAiduNFgho3GQ1aYwgSuX8CJ1dkmULtJWGjgFgbcopt7RwAKYMGEHzEdOTTV0bjkFDqGRW5SEZkKZlIJ4i16YpQ7/Z5Oa/Dymdg4ymfIkMMhiHjOV+mmay9lFV3GrH+KpnD6rB0OXwMG8+rAdqtrkX4ovTJPumWQYRMDlkFR5QmALDT
"text/plain": [
"<PIL.Image.Image image mode=RGBA size=250x180>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Time: 23.8s\n",
"This image displays a photograph featuring the **Golden Gate Bridge** spanning across a body of water toward a landmass in the distance.\n",
"\n",
"Here is a brief description:\n",
"\n",
"The dominant feature is the massive **red bridge structure** (clearly identifiable\n"
]
}
],
"source": [
"try:\n",
" url = \"https://raw.githubusercontent.com/google-gemma/cookbook/main/apps/sample-data/GoldenGate.png\"\n",
" img = Image.open(BytesIO(requests.get(url, timeout=30).content))\n",
" display(img.resize((250, 180)))\n",
" inputs = processor.apply_chat_template([{\"role\":\"user\",\"content\":[\n",
" {\"type\":\"image\",\"image\":img},\n",
" {\"type\":\"text\",\"text\":\"What is shown? Describe briefly.\"}\n",
" ]}], tokenize=True, return_dict=True, return_tensors=\"pt\", add_generation_prompt=True).to(model.device)\n",
" t0 = time.perf_counter()\n",
" with torch.no_grad():\n",
" out = model.generate(**inputs, max_new_tokens=128, temperature=0.7, do_sample=True)\n",
" print(f\"Time: {time.perf_counter()-t0:.1f}s\")\n",
" print(processor.decode(out[0][inputs[\"input_ids\"].shape[-1]:], skip_special_tokens=True)[:250])\n",
"except Exception as e:\n",
" print(f\"ERROR: {e}\")"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"---\n",
"## 7. Long Context (Needle-in-Haystack)"
]
},
{
"cell_type": "code",
"execution_count": 17,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Needle-in-Haystack:\n",
"EARLY: 793 tok, correct=OK, 4.9s\n",
"MIDDLE: 793 tok, correct=OK, 5.0s\n",
"LATE: 793 tok, correct=OK, 5.0s\n"
]
}
],
"source": [
"torch.cuda.empty_cache()\n",
"gc.collect()\n",
"\n",
"def needle(pos):\n",
" needle_str = \"The secret code is BLUE-42-GREEN.\"\n",
" filler = \"The quick brown fox jumps over the lazy dog. Python is versatile. \"\n",
" sents = [filler] * 50 # shorter context for T4\n",
" if pos == \"early\": sents.insert(0, needle_str)\n",
" elif pos == \"middle\": sents.insert(len(sents)//2, needle_str)\n",
" else: sents.append(needle_str)\n",
" prompt = f\"Read the text and answer.\\n\\nText: {' '.join(sents)}\\n\\nQ: What is the secret code? Answer with code only.\"\n",
" inputs = processor.apply_chat_template([{\"role\":\"user\",\"content\":prompt}],\n",
" tokenize=True, return_dict=True, return_tensors=\"pt\", add_generation_prompt=True).to(model.device)\n",
" t0 = time.perf_counter()\n",
" with torch.no_grad():\n",
" out = model.generate(**inputs, max_new_tokens=16, do_sample=False)\n",
" resp = processor.decode(out[0][inputs[\"input_ids\"].shape[-1]:], skip_special_tokens=True).strip()\n",
" return {\"pos\": pos, \"tokens\": inputs[\"input_ids\"].shape[-1], \"resp\": resp[:80],\n",
" \"correct\": \"BLUE-42-GREEN\" in resp, \"time\": time.perf_counter()-t0}\n",
"\n",
"print(\"Needle-in-Haystack:\")\n",
"for p in [\"early\", \"middle\", \"late\"]:\n",
" r = needle(p)\n",
" print(f\"{p.upper()}: {r['tokens']} tok, correct={'OK' if r['correct'] else 'NO'}, {r['time']:.1f}s\")"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"---\n",
"## 8. Summary"
]
},
{
"cell_type": "code",
"execution_count": 18,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"============================================================\n",
" GEMMA 4 E4B — SUMMARY\n",
"============================================================\n",
"+------------+-----------------------+\n",
"| Metric | Value |\n",
"+============+=======================+\n",
"| Model | google/gemma-4-E4B-it |\n",
"+------------+-----------------------+\n",
"| Parameters | 5.72B |\n",
"+------------+-----------------------+\n",
"| Device | cuda:0 |\n",
"+------------+-----------------------+\n",
"| VRAM | 10.26 GB |\n",
"+------------+-----------------------+\n",
"| RAM | 2.67 GB |\n",
"+------------+-----------------------+\n",
"| Load Time | 63.4s |\n",
"+------------+-----------------------+\n",
"| Throughput | 5.6 tok/s |\n",
"+------------+-----------------------+\n",
"| MMLU | 4/5 (80%) |\n",
"+------------+-----------------------+\n",
"============================================================\n"
]
}
],
"source": [
"rows = [\n",
" [\"Model\", MODEL_ID],\n",
" [\"Parameters\", f\"{total_params/1e9:.2f}B\"],\n",
" [\"Device\", str(model.device)],\n",
"]\n",
"if torch.cuda.is_available(): rows.append([\"VRAM\", f\"{torch.cuda.memory_allocated()/1e9:.2f} GB\"])\n",
"rows.append([\"RAM\", f\"{ram_used:.2f} GB\"])\n",
"rows.append([\"Load Time\", f\"{load_time:.1f}s\"])\n",
"if results: rows.append([\"Throughput\", f\"{np.mean([r['tps'] for r in results]):.1f} tok/s\"])\n",
"rows.append([\"MMLU\", f\"{ok}/{len(mmlu)} ({ok/len(mmlu)*100:.0f}%)\"])\n",
"\n",
"print(\"=\"*60)\n",
"print(\" GEMMA 4 E4B — SUMMARY\")\n",
"print(\"=\"*60)\n",
"print(tabulate(rows, headers=[\"Metric\",\"Value\"], tablefmt=\"grid\"))\n",
"print(\"=\"*60)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"---\n",
"## 9. Cleanup"
]
},
{
"cell_type": "code",
"execution_count": 19,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"VRAM: 10.26 GB\n",
"Done.\n"
]
}
],
"source": [
"del model, processor\n",
"gc.collect()\n",
"if torch.cuda.is_available():\n",
" torch.cuda.empty_cache()\n",
" print(f\"VRAM: {torch.cuda.memory_allocated()/1e9:.2f} GB\")\n",
"print(\"Done.\")"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.12.13"
}
},
"nbformat": 4,
"nbformat_minor": 4
}
Reference in New Issue Copy Permalink