52 KiB
52 KiB
In [6]:
!pip install -r requirements.txtRequirement already satisfied: tensorflow==2.19.0 in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from -r requirements.txt (line 1)) (2.19.0) Requirement already satisfied: tensorflowjs==4.22.0 in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from -r requirements.txt (line 2)) (4.22.0) Requirement already satisfied: gdown in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from -r requirements.txt (line 3)) (6.1.0) Requirement already satisfied: numpy in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from -r requirements.txt (line 4)) (2.1.3) Requirement already satisfied: matplotlib in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from -r requirements.txt (line 5)) (3.10.9) Requirement already satisfied: seaborn in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from -r requirements.txt (line 6)) (0.13.2) Requirement already satisfied: pillow in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from -r requirements.txt (line 7)) (12.2.0) Requirement already satisfied: split-folders in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from -r requirements.txt (line 8)) (0.6.1) Requirement already satisfied: scikit-learn in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from -r requirements.txt (line 9)) (1.9.0) Requirement already satisfied: tf2onnx in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from -r requirements.txt (line 10)) (1.17.0) Requirement already satisfied: onnxruntime in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from -r requirements.txt (line 11)) (1.26.0) Requirement already satisfied: absl-py>=1.0.0 in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from tensorflow==2.19.0->-r requirements.txt (line 1)) (2.4.0) Requirement already satisfied: astunparse>=1.6.0 in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from tensorflow==2.19.0->-r requirements.txt (line 1)) (1.6.3) Requirement already satisfied: flatbuffers>=24.3.25 in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from tensorflow==2.19.0->-r requirements.txt (line 1)) (25.12.19) Requirement already satisfied: gast!=0.5.0,!=0.5.1,!=0.5.2,>=0.2.1 in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from tensorflow==2.19.0->-r requirements.txt (line 1)) (0.7.0) Requirement already satisfied: google-pasta>=0.1.1 in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from tensorflow==2.19.0->-r requirements.txt (line 1)) (0.2.0) Requirement already satisfied: libclang>=13.0.0 in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from tensorflow==2.19.0->-r requirements.txt (line 1)) (18.1.1) Requirement already satisfied: opt-einsum>=2.3.2 in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from tensorflow==2.19.0->-r requirements.txt (line 1)) (3.4.0) Requirement already satisfied: packaging in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from tensorflow==2.19.0->-r requirements.txt (line 1)) (23.2) Requirement already satisfied: protobuf!=4.21.0,!=4.21.1,!=4.21.2,!=4.21.3,!=4.21.4,!=4.21.5,<6.0.0dev,>=3.20.3 in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from tensorflow==2.19.0->-r requirements.txt (line 1)) (5.29.6) Requirement already satisfied: requests<3,>=2.21.0 in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from tensorflow==2.19.0->-r requirements.txt (line 1)) (2.34.2) Requirement already satisfied: setuptools in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from tensorflow==2.19.0->-r requirements.txt (line 1)) (82.0.1) Requirement already satisfied: six>=1.12.0 in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from tensorflow==2.19.0->-r requirements.txt (line 1)) (1.17.0) Requirement already satisfied: termcolor>=1.1.0 in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from tensorflow==2.19.0->-r requirements.txt (line 1)) (3.3.0) Requirement already satisfied: typing-extensions>=3.6.6 in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from tensorflow==2.19.0->-r requirements.txt (line 1)) (4.15.0) Requirement already satisfied: wrapt>=1.11.0 in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from tensorflow==2.19.0->-r requirements.txt (line 1)) (2.2.1) Requirement already satisfied: grpcio<2.0,>=1.24.3 in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from tensorflow==2.19.0->-r requirements.txt (line 1)) (1.81.1) Requirement already satisfied: tensorboard~=2.19.0 in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from tensorflow==2.19.0->-r requirements.txt (line 1)) (2.19.0) Requirement already satisfied: keras>=3.5.0 in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from tensorflow==2.19.0->-r requirements.txt (line 1)) (3.14.1) Requirement already satisfied: h5py>=3.11.0 in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from tensorflow==2.19.0->-r requirements.txt (line 1)) (3.14.0) Requirement already satisfied: ml-dtypes<1.0.0,>=0.5.1 in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from tensorflow==2.19.0->-r requirements.txt (line 1)) (0.5.4) Requirement already satisfied: flax>=0.7.2 in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from tensorflowjs==4.22.0->-r requirements.txt (line 2)) (0.12.7) Requirement already satisfied: importlib_resources>=5.9.0 in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from tensorflowjs==4.22.0->-r requirements.txt (line 2)) (7.1.0) Requirement already satisfied: jax>=0.4.13 in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from tensorflowjs==4.22.0->-r requirements.txt (line 2)) (0.10.1) Requirement already satisfied: jaxlib>=0.4.13 in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from tensorflowjs==4.22.0->-r requirements.txt (line 2)) (0.10.1) Requirement already satisfied: tf-keras>=2.13.0 in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from tensorflowjs==4.22.0->-r requirements.txt (line 2)) (2.19.0) Requirement already satisfied: tensorflow-decision-forests>=1.5.0 in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from tensorflowjs==4.22.0->-r requirements.txt (line 2)) (1.12.0) Requirement already satisfied: tensorflow-hub>=0.16.1 in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from tensorflowjs==4.22.0->-r requirements.txt (line 2)) (0.16.1) Requirement already satisfied: charset_normalizer<4,>=2 in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from requests<3,>=2.21.0->tensorflow==2.19.0->-r requirements.txt (line 1)) (3.4.7) Requirement already satisfied: idna<4,>=2.5 in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from requests<3,>=2.21.0->tensorflow==2.19.0->-r requirements.txt (line 1)) (3.18) Requirement already satisfied: urllib3<3,>=1.26 in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from requests<3,>=2.21.0->tensorflow==2.19.0->-r requirements.txt (line 1)) (2.7.0) Requirement already satisfied: certifi>=2023.5.7 in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from requests<3,>=2.21.0->tensorflow==2.19.0->-r requirements.txt (line 1)) (2026.5.20) Requirement already satisfied: markdown>=2.6.8 in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from tensorboard~=2.19.0->tensorflow==2.19.0->-r requirements.txt (line 1)) (3.10.2) Requirement already satisfied: tensorboard-data-server<0.8.0,>=0.7.0 in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from tensorboard~=2.19.0->tensorflow==2.19.0->-r requirements.txt (line 1)) (0.7.2) Requirement already satisfied: werkzeug>=1.0.1 in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from tensorboard~=2.19.0->tensorflow==2.19.0->-r requirements.txt (line 1)) (3.1.8) Requirement already satisfied: beautifulsoup4 in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from gdown->-r requirements.txt (line 3)) (4.15.0) Requirement already satisfied: filelock in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from gdown->-r requirements.txt (line 3)) (3.29.3) Requirement already satisfied: tqdm in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from gdown->-r requirements.txt (line 3)) (4.68.2) Requirement already satisfied: contourpy>=1.0.1 in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from matplotlib->-r requirements.txt (line 5)) (1.3.3) Requirement already satisfied: cycler>=0.10 in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from matplotlib->-r requirements.txt (line 5)) (0.12.1) Requirement already satisfied: fonttools>=4.22.0 in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from matplotlib->-r requirements.txt (line 5)) (4.63.0) Requirement already satisfied: kiwisolver>=1.3.1 in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from matplotlib->-r requirements.txt (line 5)) (1.5.0) Requirement already satisfied: pyparsing>=3 in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from matplotlib->-r requirements.txt (line 5)) (3.3.2) Requirement already satisfied: python-dateutil>=2.7 in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from matplotlib->-r requirements.txt (line 5)) (2.9.0.post0) Requirement already satisfied: pandas>=1.2 in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from seaborn->-r requirements.txt (line 6)) (3.0.3) Requirement already satisfied: scipy>=1.10.0 in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from scikit-learn->-r requirements.txt (line 9)) (1.17.1) Requirement already satisfied: joblib>=1.4.0 in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from scikit-learn->-r requirements.txt (line 9)) (1.5.3) Requirement already satisfied: narwhals>=2.0.1 in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from scikit-learn->-r requirements.txt (line 9)) (2.22.1) Requirement already satisfied: threadpoolctl>=3.5.0 in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from scikit-learn->-r requirements.txt (line 9)) (3.6.0) Requirement already satisfied: onnx>=1.14.0 in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from tf2onnx->-r requirements.txt (line 10)) (1.21.0) Requirement already satisfied: wheel<1.0,>=0.23.0 in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from astunparse>=1.6.0->tensorflow==2.19.0->-r requirements.txt (line 1)) (0.45.1) Requirement already satisfied: msgpack in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from flax>=0.7.2->tensorflowjs==4.22.0->-r requirements.txt (line 2)) (1.2.0) Requirement already satisfied: optax in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from flax>=0.7.2->tensorflowjs==4.22.0->-r requirements.txt (line 2)) (0.2.8) Requirement already satisfied: orbax-checkpoint in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from flax>=0.7.2->tensorflowjs==4.22.0->-r requirements.txt (line 2)) (0.12.0) Requirement already satisfied: tensorstore in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from flax>=0.7.2->tensorflowjs==4.22.0->-r requirements.txt (line 2)) (0.1.84) Requirement already satisfied: rich>=11.1 in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from flax>=0.7.2->tensorflowjs==4.22.0->-r requirements.txt (line 2)) (15.0.0) Requirement already satisfied: PyYAML>=5.4.1 in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from flax>=0.7.2->tensorflowjs==4.22.0->-r requirements.txt (line 2)) (6.0.3) Requirement already satisfied: treescope>=0.1.7 in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from flax>=0.7.2->tensorflowjs==4.22.0->-r requirements.txt (line 2)) (0.1.10) Requirement already satisfied: namex in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from keras>=3.5.0->tensorflow==2.19.0->-r requirements.txt (line 1)) (0.1.0) Requirement already satisfied: optree in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from keras>=3.5.0->tensorflow==2.19.0->-r requirements.txt (line 1)) (0.19.1) Requirement already satisfied: markdown-it-py>=2.2.0 in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from rich>=11.1->flax>=0.7.2->tensorflowjs==4.22.0->-r requirements.txt (line 2)) (4.2.0) Requirement already satisfied: pygments<3.0.0,>=2.13.0 in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from rich>=11.1->flax>=0.7.2->tensorflowjs==4.22.0->-r requirements.txt (line 2)) (2.20.0) Requirement already satisfied: mdurl~=0.1 in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from markdown-it-py>=2.2.0->rich>=11.1->flax>=0.7.2->tensorflowjs==4.22.0->-r requirements.txt (line 2)) (0.1.2) Requirement already satisfied: wurlitzer in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from tensorflow-decision-forests>=1.5.0->tensorflowjs==4.22.0->-r requirements.txt (line 2)) (3.1.1) Requirement already satisfied: ydf>=0.11.0 in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from tensorflow-decision-forests>=1.5.0->tensorflowjs==4.22.0->-r requirements.txt (line 2)) (0.16.0) Requirement already satisfied: markupsafe>=2.1.1 in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from werkzeug>=1.0.1->tensorboard~=2.19.0->tensorflow==2.19.0->-r requirements.txt (line 1)) (3.0.3) Requirement already satisfied: soupsieve>=1.6.1 in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from beautifulsoup4->gdown->-r requirements.txt (line 3)) (2.8.4) Requirement already satisfied: etils[epath,epy] in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from orbax-checkpoint->flax>=0.7.2->tensorflowjs==4.22.0->-r requirements.txt (line 2)) (1.14.0) Requirement already satisfied: prometheus-client>=0.20.0 in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from orbax-checkpoint->flax>=0.7.2->tensorflowjs==4.22.0->-r requirements.txt (line 2)) (0.25.0) Requirement already satisfied: aiofiles in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from orbax-checkpoint->flax>=0.7.2->tensorflowjs==4.22.0->-r requirements.txt (line 2)) (25.1.0) Requirement already satisfied: humanize in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from orbax-checkpoint->flax>=0.7.2->tensorflowjs==4.22.0->-r requirements.txt (line 2)) (4.15.0) Requirement already satisfied: simplejson>=3.16.0 in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from orbax-checkpoint->flax>=0.7.2->tensorflowjs==4.22.0->-r requirements.txt (line 2)) (4.1.1) Requirement already satisfied: psutil in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from orbax-checkpoint->flax>=0.7.2->tensorflowjs==4.22.0->-r requirements.txt (line 2)) (7.2.2) Requirement already satisfied: uvloop in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from orbax-checkpoint->flax>=0.7.2->tensorflowjs==4.22.0->-r requirements.txt (line 2)) (0.22.1) Requirement already satisfied: fsspec in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from etils[epath,epy]->orbax-checkpoint->flax>=0.7.2->tensorflowjs==4.22.0->-r requirements.txt (line 2)) (2026.4.0) Requirement already satisfied: zipp in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from etils[epath,epy]->orbax-checkpoint->flax>=0.7.2->tensorflowjs==4.22.0->-r requirements.txt (line 2)) (4.1.0) Requirement already satisfied: PySocks!=1.5.7,>=1.5.6 in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from requests[socks]->gdown->-r requirements.txt (line 3)) (1.7.1)
In [7]:
import os
import shutil
import zipfile
import random
from collections import Counter
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
import splitfolders
from PIL import Image
import tensorflow as tf
from tensorflow.keras import layers, models, callbacks
from tensorflow.keras.applications import EfficientNetV2B0
from tensorflow.keras.applications.efficientnet_v2 import preprocess_input
from tensorflow.keras.optimizers import AdamW
from sklearn.metrics import classification_report, confusion_matrix
from sklearn.utils.class_weight import compute_class_weight
# Detect environment
try:
from google.colab import drive
IS_COLAB = True
print("Running on Google Colab")
except ModuleNotFoundError:
IS_COLAB = False
print(f"Running locally (TF {tf.__version__}, GPU: {tf.config.list_physical_devices('GPU')})")
tf.keras.mixed_precision.set_global_policy('float32')
# Hyperparams
IMG_SIZE = (224, 224)
BATCH_SIZE = 32
SEED = 42
random.seed(SEED)
np.random.seed(SEED)
tf.random.set_seed(SEED)
AUTOTUNE = tf.data.AUTOTUNE
print(f"Setup OK. IMG={IMG_SIZE}, BATCH={BATCH_SIZE}")Running locally (TF 2.19.0, GPU: [PhysicalDevice(name='/physical_device:GPU:0', device_type='GPU')]) Setup OK. IMG=(224, 224), BATCH=32
In [8]:
if IS_COLAB:
drive.mount('/content/drive')
archive_path = '/content/drive/MyDrive/jagung/dataset_jagung.zip'
destination_path = '/content/dataset_jagung.zip'
extract_path = '/content/dataset'
else:
import gdown
base = os.getcwd()
archive_path = os.path.join(base, 'dataset_jagung.zip')
destination_path = archive_path
extract_path = os.path.join(base, 'dataset')
DRIVE_FILE_ID = "1s0H2lDOQVCixywk5eZXJz2i9jj4JihxJ"
if not os.path.exists(archive_path):
print("Downloading from Google Drive...")
try:
gdown.download(f"https://drive.google.com/uc?id={DRIVE_FILE_ID}", archive_path, quiet=False)
except Exception as e:
print(f"Download failed: {e}")
if os.path.exists(destination_path):
if not os.path.exists(extract_path) or len(os.listdir(extract_path)) == 0:
os.makedirs(extract_path, exist_ok=True)
print("Extracting dataset...")
try:
with zipfile.ZipFile(destination_path, 'r') as zip_ref:
zip_ref.extractall(path=extract_path)
print("Extraction completed!")
except Exception as e:
print(f"Extraction failed: {e}")
else:
print("Dataset ready.")Dataset ready.
In [9]:
# --- Determine dataset path ---
# Colab: zip extracts to dataset/ with dataset_jagung_v1/ subfolder
# Local: zip extracts classes directly to dataset/
if IS_COLAB:
dataset_path = "/content/dataset/dataset_jagung_v1"
else:
candidate = os.path.join(extract_path, "dataset_jagung_v1")
if os.path.isdir(candidate):
dataset_path = candidate
else:
dataset_path = extract_path # classes are directly in dataset/
print(f"Dataset path: {dataset_path}")
def clean_image_data(directory):
removed_count = 0
for root, dirs, files in os.walk(directory):
for file in files:
file_path = os.path.join(root, file)
try:
img = Image.open(file_path)
img.verify()
img = Image.open(file_path)
if img.mode != 'RGB':
img = img.convert('RGB')
img.save(file_path)
except Exception:
print(f"Removing: {file_path}")
os.remove(file_path)
removed_count += 1
return removed_count
print("Cleaning data...")
removed = clean_image_data(dataset_path)
print(f"Done. {removed} problematic files removed.")Dataset path: /home/asephs/ZeaVis-Edu/Machine_Learning/dataset Cleaning data... Done. 0 problematic files removed.
In [10]:
if IS_COLAB:
output_dir = "/content/dataset_split"
else:
output_dir = os.path.join(os.getcwd(), "dataset_split")
if os.path.exists(output_dir):
shutil.rmtree(output_dir)
print("Splitting dataset 70:15:15...")
splitfolders.ratio(dataset_path, output=output_dir,
seed=SEED, ratio=(0.7, 0.15, 0.15),
group_prefix=None, move=False)
print("Done.")
train_dir = os.path.join(output_dir, 'train')
val_dir = os.path.join(output_dir, 'val')
test_dir = os.path.join(output_dir, 'test')
# Count
def count_images(path):
return sum(len(files) for _, _, files in os.walk(path))
print(f'Train: {count_images(train_dir)} | Val: {count_images(val_dir)} | Test: {count_images(test_dir)}')Splitting dataset 70:15:15...
Copying files: 6988 files [00:08, 826.32 files/s]
Done. Train: 4890 | Val: 1046 | Test: 1052
In [ ]:
# --- Load datasets ---
train_ds = tf.keras.utils.image_dataset_from_directory(
train_dir, shuffle=True, batch_size=BATCH_SIZE, image_size=IMG_SIZE)
val_ds = tf.keras.utils.image_dataset_from_directory(
val_dir, shuffle=True, batch_size=BATCH_SIZE, image_size=IMG_SIZE)
test_ds = tf.keras.utils.image_dataset_from_directory(
test_dir, shuffle=False, batch_size=BATCH_SIZE, image_size=IMG_SIZE)
class_names = train_ds.class_names
NUM_CLASSES = len(class_names)
print(f"Classes ({NUM_CLASSES}): {class_names}")
# --- Helper: one-hot ---
def to_one_hot(image, label):
return image, tf.one_hot(label, NUM_CLASSES)
# --- Beta distribution for MixUp/CutMix ---
def sample_beta_distribution(size, concentration_0=0.2, concentration_1=0.2):
gamma_1 = tf.random.gamma(shape=[size], alpha=concentration_1, dtype=tf.float32)
gamma_2 = tf.random.gamma(shape=[size], alpha=concentration_0, dtype=tf.float32)
return gamma_2 / (gamma_1 + gamma_2 + 1e-8)
# --- MixUp ---
def mix_up(images, labels, alpha=0.2):
batch_size = tf.shape(images)[0]
lambda_param = sample_beta_distribution(batch_size, alpha, alpha)
lambda_param_img = tf.reshape(lambda_param, [batch_size, 1, 1, 1])
random_indices = tf.random.shuffle(tf.range(batch_size))
mixed_images = lambda_param_img * images + (1 - lambda_param_img) * tf.gather(images, random_indices)
labels = tf.cast(labels, tf.float32)
lambda_param_lbl = tf.reshape(lambda_param, [-1, 1])
mixed_labels = lambda_param_lbl * labels + (1 - lambda_param_lbl) * tf.gather(labels, random_indices)
return mixed_images, mixed_labels
# --- CutMix (batch-level via coordinate-grid broadcasting) ---
def cut_mix(images, labels, alpha=0.2):
batch_size = tf.shape(images)[0]
h = tf.shape(images)[1]
w = tf.shape(images)[2]
lambda_param = sample_beta_distribution(batch_size, alpha, alpha) # [B]
random_indices = tf.random.shuffle(tf.range(batch_size))
# Per-sample cut sizes (in pixels)
cut_ratio = tf.sqrt(1.0 - lambda_param) # [B]
r_h = tf.cast(cut_ratio * tf.cast(h, tf.float32), tf.int32) # [B]
r_w = tf.cast(cut_ratio * tf.cast(w, tf.float32), tf.int32) # [B]
# Per-sample random cut centres
cx = tf.random.uniform([batch_size], 0, w, tf.int32) # [B]
cy = tf.random.uniform([batch_size], 0, h, tf.int32) # [B]
half_h = r_h // 2
half_w = r_w // 2
x1 = tf.clip_by_value(cx - half_w, 0, w) # [B]
x2 = tf.clip_by_value(cx + half_w, 0, w) # [B]
y1 = tf.clip_by_value(cy - half_h, 0, h) # [B]
y2 = tf.clip_by_value(cy + half_h, 0, h) # [B]
# Build boolean cut mask via coordinate broadcasting
col_idx = tf.range(w, dtype=tf.int32) # [W]
row_idx = tf.range(h, dtype=tf.int32) # [H]
in_x = tf.logical_and(
tf.reshape(col_idx, [1, 1, w]) >= tf.reshape(x1, [batch_size, 1, 1]),
tf.reshape(col_idx, [1, 1, w]) < tf.reshape(x2, [batch_size, 1, 1])
) # [B, 1, W]
in_y = tf.logical_and(
tf.reshape(row_idx, [1, h, 1]) >= tf.reshape(y1, [batch_size, 1, 1]),
tf.reshape(row_idx, [1, h, 1]) < tf.reshape(y2, [batch_size, 1, 1])
) # [B, H, 1]
cut_mask = tf.cast(tf.logical_and(in_y, in_x), tf.float32) # [B, H, W]
cut_mask = tf.expand_dims(cut_mask, -1) # [B, H, W, 1]
# Mix images
shuffled = tf.gather(images, random_indices)
mixed_images = (1.0 - cut_mask) * images + cut_mask * shuffled
# Mix labels
labels = tf.cast(labels, tf.float32)
lambda_reshaped = tf.reshape(lambda_param, [-1, 1])
mixed_labels = lambda_reshaped * labels + (1.0 - lambda_reshaped) * tf.gather(labels, random_indices)
return mixed_images, mixed_labels
# --- RandomErasing ---
def random_erasing(images, probability=0.5, scale=(0.02, 0.33), ratio=(0.3, 3.3)):
batch_size = tf.shape(images)[0]
h = tf.shape(images)[1]
w = tf.shape(images)[2]
target_area = tf.random.uniform([], scale[0], scale[1]) * tf.cast(h * w, tf.float32)
aspect_ratio = tf.random.uniform([], ratio[0], ratio[1])
erasing_h = tf.cast(tf.math.sqrt(target_area / aspect_ratio), tf.int32)
erasing_w = tf.cast(tf.math.sqrt(target_area * aspect_ratio), tf.int32)
erasing_h = tf.clip_by_value(erasing_h, 1, h - 1)
erasing_w = tf.clip_by_value(erasing_w, 1, w - 1)
cx = tf.random.uniform([], 0, w - erasing_w, tf.int32)
cy = tf.random.uniform([], 0, h - erasing_h, tf.int32)
# Build erase mask via coordinate broadcasting (same pattern as CutMix)
col_idx = tf.range(w, dtype=tf.int32)
row_idx = tf.range(h, dtype=tf.int32)
in_x = tf.logical_and(col_idx >= cx, col_idx < cx + erasing_w) # [W]
in_y = tf.logical_and(row_idx >= cy, row_idx < cy + erasing_h) # [H]
erase_mask = tf.cast(tf.expand_dims(in_y, 1) & tf.expand_dims(in_x, 0), tf.float32) # [H, W]
erase_mask = tf.expand_dims(erase_mask, 0) # [1, H, W]
erase_mask = tf.expand_dims(erase_mask, -1) # [1, H, W, 1]
noise = tf.random.uniform([batch_size, erasing_h, erasing_w, 3], 0.0, 255.0, dtype=tf.float32)
# Pad the noise patch to full image size
paddings = [[0, 0], [cy, h - (cy + erasing_h)], [cx, w - (cx + erasing_w)], [0, 0]]
noise_padded = tf.pad(noise, paddings, constant_values=0.0)
erased = images * (1.0 - erase_mask) + noise_padded * erase_mask
return tf.cond(
tf.random.uniform([]) < probability,
lambda: erased,
lambda: images
)
# --- Geometric augmentation (kept for TTA compatibility) ---
data_augmentation = tf.keras.Sequential([
layers.RandomFlip("horizontal_and_vertical"),
layers.RandomRotation(0.2),
layers.RandomZoom(0.2),
layers.RandomTranslation(0.1, 0.1),
layers.RandomContrast(0.2),
layers.RandomBrightness(0.2)
], name="data_augmentation")
# --- Augment + MixUp/CutMix + RandomErasing pipeline ---
def augment_and_mix(images, labels):
# images are uint8 [0,255], labels are int
# 1. Geometric augmentation
images = data_augmentation(images, training=True)
# 2. Convert to float32 for MixUp/CutMix
images = tf.cast(images, tf.float32)
# 3. Apply CutMix or MixUp with 50% probability
choice = tf.random.uniform([])
labels_onehot = tf.one_hot(labels, NUM_CLASSES)
images, labels_onehot = tf.cond(
choice < 0.3, # 30% MixUp
lambda: mix_up(images, labels_onehot),
lambda: tf.cond(
choice < 0.6, # 30% CutMix
lambda: cut_mix(images, labels_onehot),
lambda: (images, labels_onehot) # 40% no mix
)
)
# 4. Random Erasing
images = random_erasing(images, probability=0.25)
return images, labels_onehot
# --- Preprocessing (EfficientNetV2: [-1,1]) ---
def preprocess_fn(image, label):
return preprocess_input(image), label
# --- Class Weights ---
train_class_counts = Counter()
for cn in class_names:
p = os.path.join(train_dir, cn)
if os.path.exists(p):
train_class_counts[cn] = len(os.listdir(p))
y_integer = []
for i, cn in enumerate(class_names):
y_integer.extend([i] * train_class_counts[cn])
class_weights_array = compute_class_weight('balanced', classes=np.unique(y_integer), y=y_integer)
class_weights_capped = [min(w, 2.5) for w in class_weights_array]
class_weights_tensor = tf.constant(class_weights_capped, dtype=tf.float32)
def add_sample_weight(image, label):
class_indices = tf.argmax(label, axis=-1)
sample_weights = tf.gather(class_weights_tensor, class_indices)
return image, label, sample_weights
print("Class weights (capped at 2.5):")
for i, cn in enumerate(class_names):
print(f" {cn}: {class_weights_capped[i]:.4f}")
# --- Build pipelines ---
# Train: augment → mix → erase → preprocess → onehot → sample_weight → prefetch
train_ds = (train_ds
.map(augment_and_mix, num_parallel_calls=AUTOTUNE)
.map(preprocess_fn, num_parallel_calls=AUTOTUNE)
.map(add_sample_weight, num_parallel_calls=AUTOTUNE)
.prefetch(AUTOTUNE))
# Val/Test: preprocess → onehot → prefetch
val_ds = (val_ds
.map(lambda img, lbl: (preprocess_input(tf.cast(img, tf.float32)), lbl), num_parallel_calls=AUTOTUNE)
.map(to_one_hot, num_parallel_calls=AUTOTUNE)
.prefetch(AUTOTUNE))
test_ds = (test_ds
.map(lambda img, lbl: (preprocess_input(tf.cast(img, tf.float32)), lbl), num_parallel_calls=AUTOTUNE)
.map(to_one_hot, num_parallel_calls=AUTOTUNE)
.prefetch(AUTOTUNE))
print("Data pipelines ready.")In [ ]:
# Visualisasi augmentasi (non-preprocessed)
vis_ds = tf.keras.utils.image_dataset_from_directory(
train_dir, shuffle=True, batch_size=BATCH_SIZE, image_size=IMG_SIZE)
plt.figure(figsize=(16, 10))
for images, labels in vis_ds.take(1):
# Original (4)
for i in range(4):
plt.subplot(3, 4, i + 1)
plt.imshow(images[i].numpy().astype("uint8"))
plt.title(f"Asli: {class_names[labels[i].numpy()]}", fontsize=11)
plt.axis("off")
# Augmented (4) — geometric only
aug = data_augmentation(images, training=True)
for i in range(4):
plt.subplot(3, 4, i + 5)
plt.imshow(aug[i].numpy().astype("uint8"))
plt.title(f"Aug: {class_names[labels[i].numpy()]}", fontsize=11)
plt.axis("off")
# MixUp/CutMix result (4)
aug_f = tf.cast(images, tf.float32)
mixed, _ = mix_up(aug_f, tf.one_hot(labels, NUM_CLASSES))
for i in range(4):
plt.subplot(3, 4, i + 9)
plt.imshow(tf.clip_by_value(mixed[i], 0, 255).numpy().astype("uint8"))
plt.title("MixUp/CutMix", fontsize=11)
plt.axis("off")
plt.suptitle("Contoh Augmentasi Berlapis", fontsize=16)
plt.tight_layout()
plt.show()In [ ]:
def build_model(num_classes):
base_model = EfficientNetV2B0(
input_shape=IMG_SIZE + (3,),
include_top=False,
weights='imagenet',
)
base_model.trainable = False
inputs = tf.keras.Input(shape=IMG_SIZE + (3,))
x = layers.GaussianNoise(0.1)(inputs)
x = base_model(x, training=False)
x = layers.Conv2D(512, (3, 3), padding='same', activation='swish')(x)
x = layers.BatchNormalization()(x)
x = layers.MaxPooling2D((2, 2))(x)
x = layers.Dropout(0.2)(x)
x = layers.Conv2D(256, (3, 3), padding='same', activation='swish')(x)
x = layers.BatchNormalization()(x)
x = layers.GlobalAveragePooling2D()(x)
x = layers.Dropout(0.3)(x)
x = layers.Dense(1024, activation='swish')(x)
x = layers.BatchNormalization()(x)
x = layers.Dropout(0.4)(x)
outputs = layers.Dense(num_classes, activation='softmax', dtype='float32')(x)
return models.Model(inputs, outputs), base_model
if IS_COLAB:
ckpt_dir = '/content/best_model'
else:
ckpt_dir = os.path.join(os.getcwd(), 'best_model')
checkpoint_path = os.path.join(ckpt_dir, 'best_model.keras')
if os.path.exists(checkpoint_path):
print(f"Loading checkpoint: {checkpoint_path}")
try:
model = models.load_model(checkpoint_path, compile=False)
except Exception as e:
print(f"Load failed: {e}. Building fresh.")
model, base_model = build_model(NUM_CLASSES)
else:
print("No checkpoint. Building fresh model.")
model, base_model = build_model(NUM_CLASSES)
os.makedirs(ckpt_dir, exist_ok=True)
model.summary()In [ ]:
log_dir = "logs/fit/" + time.strftime("%Y%m%d-%H%M%S")
import time
checkpoint_cb = callbacks.ModelCheckpoint(
checkpoint_path,
save_best_only=True,
monitor="val_accuracy",
mode="max",
verbose=1
)
early_stopping_cb = callbacks.EarlyStopping(
monitor="val_accuracy",
patience=15,
restore_best_weights=True,
mode="max",
verbose=1
)
reduce_lr_cb = callbacks.ReduceLROnPlateau(
monitor='val_loss',
factor=0.2,
patience=5,
min_lr=1e-8,
verbose=1,
mode="min"
)
csv_logger = callbacks.CSVLogger(os.path.join(ckpt_dir, 'training_log.csv'))
callbacks_list = [checkpoint_cb, early_stopping_cb, reduce_lr_cb, csv_logger]In [ ]:
model.compile(
optimizer=AdamW(use_ema=True, ema_momentum=0.999,
learning_rate=1e-3, weight_decay=1e-4),
loss=tf.keras.losses.CategoricalCrossentropy(label_smoothing=0.2),
metrics=['accuracy']
)
print("Fase 1: Head training (base frozen)...")
history_1 = model.fit(
train_ds,
validation_data=val_ds,
epochs=30,
callbacks=callbacks_list
)In [ ]:
base_model = model.layers[1] # layer[0]=GaussianNoise, layer[1]=EfficientNetV2B0
base_model.trainable = True
for layer in base_model.layers[:-100]:
layer.trainable = False
model.compile(
optimizer=AdamW(use_ema=True, ema_momentum=0.999,
learning_rate=1e-4, weight_decay=1e-4),
loss=tf.keras.losses.CategoricalCrossentropy(label_smoothing=0.2),
metrics=['accuracy']
)
print("Fase 2: Fine-tuning top 100 layers...")
history_2 = model.fit(
train_ds,
validation_data=val_ds,
epochs=60,
initial_epoch=history_1.epoch[-1] + 1,
callbacks=callbacks_list
)In [ ]:
base_model.trainable = True
model.compile(
optimizer=AdamW(use_ema=True, ema_momentum=0.999,
learning_rate=5e-5, weight_decay=1e-4),
loss=tf.keras.losses.CategoricalCrossentropy(label_smoothing=0.2),
metrics=['accuracy']
)
print("Fase 3: Full fine-tuning...")
history_3 = model.fit(
train_ds,
validation_data=val_ds,
epochs=90,
initial_epoch=history_2.epoch[-1] + 1,
callbacks=callbacks_list
)In [ ]:
# Gabungkan history
acc = history_1.history['accuracy'] + history_2.history['accuracy'] + history_3.history['accuracy']
val_acc = history_1.history['val_accuracy'] + history_2.history['val_accuracy'] + history_3.history['val_accuracy']
loss = history_1.history['loss'] + history_2.history['loss'] + history_3.history['loss']
val_loss = history_1.history['val_loss'] + history_2.history['val_loss'] + history_3.history['val_loss']
# Batas antar fase
boundary_1 = len(history_1.history['accuracy']) - 1
boundary_2 = boundary_1 + len(history_2.history['accuracy'])
plt.figure(figsize=(16, 6))
plt.subplot(1, 2, 1)
plt.plot(acc, label='Training Accuracy', linewidth=2)
plt.plot(val_acc, label='Validation Accuracy', linewidth=2)
plt.axvline(x=boundary_1, color='gray', linestyle='--', alpha=0.7, label='Fase 2 start')
plt.axvline(x=boundary_2, color='black', linestyle='--', alpha=0.7, label='Fase 3 start')
plt.legend(fontsize=12)
plt.title('Training & Validation Accuracy', fontsize=14)
plt.xlabel('Epoch')
plt.ylabel('Accuracy')
plt.grid(alpha=0.3)
plt.subplot(1, 2, 2)
plt.plot(loss, label='Training Loss', linewidth=2)
plt.plot(val_loss, label='Validation Loss', linewidth=2)
plt.axvline(x=boundary_1, color='gray', linestyle='--', alpha=0.7, label='Fase 2 start')
plt.axvline(x=boundary_2, color='black', linestyle='--', alpha=0.7, label='Fase 3 start')
plt.legend(fontsize=12)
plt.title('Training & Validation Loss', fontsize=14)
plt.xlabel('Epoch')
plt.ylabel('Loss')
plt.grid(alpha=0.3)
plt.tight_layout()
plt.show()In [ ]:
# Load model terbaik
print(f"Loading best model from {checkpoint_path}...")
best_model = tf.keras.models.load_model(checkpoint_path, compile=False)
# Kumpulkan gambar test asli
raw_test_ds = tf.keras.utils.image_dataset_from_directory(
test_dir, shuffle=False, batch_size=BATCH_SIZE, image_size=IMG_SIZE)
test_images = []
test_labels_true_raw = []
for images, labels in raw_test_ds.unbatch():
test_images.append(images.numpy())
test_labels_true_raw.append(labels.numpy())
test_images = np.array(test_images)
test_labels_true = tf.one_hot(np.array(test_labels_true_raw), NUM_CLASSES).numpy()
TTA_STEPS = 5
tta_predictions = []
for i in range(TTA_STEPS):
aug_images = data_augmentation(test_images, training=True)
aug_images = preprocess_input(aug_images)
preds = best_model.predict(aug_images, batch_size=BATCH_SIZE, verbose=0)
tta_predictions.append(preds)
print(f" TTA step {i+1}/{TTA_STEPS}")
mean_tta = np.mean(tta_predictions, axis=0)
test_preds = np.argmax(mean_tta, axis=1)
test_true = np.argmax(test_labels_true, axis=1)
tta_acc = np.mean(test_preds == test_true)
print(f"\nTest Accuracy (TTA {TTA_STEPS}x): {tta_acc*100:.2f}%")In [ ]:
print("\nClassification Report:\n")
print(classification_report(test_true, test_preds, target_names=class_names))
cm = confusion_matrix(test_true, test_preds)
plt.figure(figsize=(10, 8))
sns.heatmap(cm, annot=True, fmt='d', cmap='Blues',
xticklabels=class_names, yticklabels=class_names)
plt.title('Confusion Matrix', fontsize=14)
plt.xlabel('Predicted', fontsize=12)
plt.ylabel('True', fontsize=12)
plt.tight_layout()
plt.show()In [ ]:
# Simpan model training final (dengan EMA weights)
final_path = os.path.join(ckpt_dir, 'best_model.keras')
model.save(final_path)
print(f"Model saved to {final_path}")
# Simpan juga versi tanpa EMA untuk fallback
model.save(os.path.join(ckpt_dir, 'final_model.keras'))
print("Final model saved.")