feat(ml): implement v3.0 architecture with CBAM and calibrated inference
- Integrate Convolutional Block Attention Module (CBAM) for improved feature focus - Implement temperature scaling and confidence-based status reporting - Automate dataset acquisition using kagglehub - Update ONNX opset to 18 and refine preprocessing validation
This commit is contained in:
MythEclipse
Binary file not shown.
@@ -30,7 +30,7 @@ def main():
|
||||
parser.add_argument(
|
||||
"--opset",
|
||||
type=int,
|
||||
default=13,
|
||||
default=18,
|
||||
help="ONNX opset version to target",
|
||||
)
|
||||
|
||||
|
||||
+1136
-3297
File diff suppressed because one or more lines are too long
@@ -2,6 +2,13 @@ import os
|
||||
import shutil
|
||||
import zipfile
|
||||
import json
|
||||
import logging
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from PIL import Image, UnidentifiedImageError
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s")
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
# ==========================================
|
||||
# KONFIGURASI DAN MAPPING
|
||||
@@ -9,17 +16,25 @@ import json
|
||||
DAFTAR_ZIP = ['dataset_1.zip', 'dataset_2.zip', 'dataset_3.zip']
|
||||
TARGET_DIR = "dataset"
|
||||
|
||||
# Kaggle dataset paths (untuk auto-download)
|
||||
KAGGLE_DS1 = "ndisan/corn-leaf-disease"
|
||||
KAGGLE_DS2 = "smaranjitghose/corn-or-maize-leaf-disease-dataset"
|
||||
|
||||
# Google Drive file ID for dataset_3 fallback (SciDB via alternative host)
|
||||
DS3_GDRIVE_ID = None # Replace with known ID if available
|
||||
|
||||
PEMETAAN_KATEGORI = {
|
||||
"大斑病": "Hawar Daun",
|
||||
"小斑病": "Hawar Daun",
|
||||
"褐斑病": "Bercak Daun",
|
||||
"弯孢霉叶斑病": "Bercak Daun",
|
||||
"圆斑病": "Bercak Daun",
|
||||
"灰斑病": "Bercak Daun",
|
||||
"南方锈病": "Karat Daun",
|
||||
"普通锈病": "Karat Daun",
|
||||
"大斑病": "Hawar Daun",
|
||||
"小斑病": "Hawar Daun",
|
||||
"褐斑病": "Bercak Daun",
|
||||
"弯孢霉叶斑病": "Bercak Daun",
|
||||
"圆斑病": "Bercak Daun",
|
||||
"灰斑病": "Bercak Daun",
|
||||
"南方锈病": "Karat Daun",
|
||||
"普通锈病": "Karat Daun",
|
||||
}
|
||||
|
||||
# Known-corrupt images (empty/broken JPEG headers)
|
||||
DAFTAR_FILE_HAPUS = [
|
||||
"CBS28.jpg",
|
||||
"Corn_Common_Rust (1275).jpg",
|
||||
@@ -28,11 +43,102 @@ DAFTAR_FILE_HAPUS = [
|
||||
"Corn_Gray_Spot (1).jpg"
|
||||
]
|
||||
|
||||
MIN_WIDTH, MIN_HEIGHT = 32, 32
|
||||
MAX_ASPECT_RATIO = 5.0
|
||||
MIN_FILE_SIZE = 512
|
||||
|
||||
|
||||
# ==========================================
|
||||
# TAHAP 0: DOWNLOAD DATASET OTOMATIS
|
||||
# ==========================================
|
||||
def download_dataset_1():
|
||||
"""Download dataset_1 (ndisan/corn-leaf-disease) from Kaggle."""
|
||||
try:
|
||||
import kagglehub
|
||||
log.info(f" Downloading dataset_1 from Kaggle: {KAGGLE_DS1}...")
|
||||
path = kagglehub.dataset_download(KAGGLE_DS1)
|
||||
log.info(f" [OK] dataset_1 downloaded to {path}")
|
||||
return path
|
||||
except ImportError:
|
||||
log.warning(" [SKIP] kagglehub not installed. Install with: pip install kagglehub")
|
||||
return None
|
||||
except Exception as e:
|
||||
log.warning(f" [FAIL] dataset_1 download failed: {e}")
|
||||
log.warning(" Please download manually from https://www.kaggle.com/datasets/ndisan/corn-leaf-disease")
|
||||
return None
|
||||
|
||||
|
||||
def download_dataset_2():
|
||||
"""Download dataset_2 (smaranjitghose/corn-or-maize-leaf-disease-dataset) from Kaggle."""
|
||||
try:
|
||||
import kagglehub
|
||||
log.info(f" Downloading dataset_2 from Kaggle: {KAGGLE_DS2}...")
|
||||
path = kagglehub.dataset_download(KAGGLE_DS2)
|
||||
log.info(f" [OK] dataset_2 downloaded to {path}")
|
||||
return path
|
||||
except ImportError:
|
||||
log.warning(" [SKIP] kagglehub not installed. Install with: pip install kagglehub")
|
||||
return None
|
||||
except Exception as e:
|
||||
log.warning(f" [FAIL] dataset_2 download failed: {e}")
|
||||
log.warning(" Please download manually from https://www.kaggle.com/datasets/smaranjitghose/corn-or-maize-leaf-disease-dataset")
|
||||
return None
|
||||
|
||||
|
||||
def download_dataset_3():
|
||||
"""Download dataset_3 (SciDB China Agricultural Dataset).
|
||||
|
||||
Attempt: Kaggle alternative host, then Google Drive, then warn.
|
||||
If all fail, user must download manually from SciDB.
|
||||
"""
|
||||
# Try Kaggle alternative (if available)
|
||||
KAGGLE_DS3 = "disease-identification/corn-leaf-disease-chinese" # not guaranteed
|
||||
try:
|
||||
import kagglehub
|
||||
path = kagglehub.dataset_download(KAGGLE_DS3)
|
||||
log.info(f" [OK] dataset_3 downloaded from Kaggle mirror to {path}")
|
||||
return path
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
log.warning(" [SKIP] dataset_3 could not be downloaded automatically.")
|
||||
log.warning(" Please download manually from:")
|
||||
log.warning(" https://www.scidb.cn/en/detail?dataSetId=19536c73f6d74946a212719a94f53ab3")
|
||||
return None
|
||||
|
||||
|
||||
def copy_kaggle_download(src_path, target_zip_name):
|
||||
"""Copy kagglehub download into a local ZIP so extraction step works unchanged."""
|
||||
if src_path is None or not os.path.exists(src_path):
|
||||
return False
|
||||
# src_path is a directory; zip it as target_zip_name
|
||||
try:
|
||||
shutil.make_archive(target_zip_name.replace('.zip', ''), 'zip', src_path)
|
||||
log.info(f" [OK] Packed {src_path} -> {target_zip_name}")
|
||||
return True
|
||||
except Exception as e:
|
||||
log.warning(f" [FAIL] Could not pack {src_path}: {e}")
|
||||
return False
|
||||
|
||||
|
||||
def download_dataset(zip_name, download_fn):
|
||||
"""Attempt auto-download if ZIP doesn't exist locally."""
|
||||
if os.path.exists(zip_name):
|
||||
log.info(f" [CACHE] {zip_name} already exists, skipping download.")
|
||||
return True
|
||||
log.info(f"--- Downloading {zip_name} ---")
|
||||
path = download_fn()
|
||||
if path is None:
|
||||
return False
|
||||
# Pack into ZIP for downstream extraction
|
||||
return copy_kaggle_download(path, zip_name)
|
||||
|
||||
|
||||
# ==========================================
|
||||
# TAHAP 1: EKSTRAKSI DATASET
|
||||
# ==========================================
|
||||
def ekstrak_semua_zip():
|
||||
print("--- TAHAP 1: Mengekstrak File ZIP ---")
|
||||
log.info("--- TAHAP 1: Mengekstrak File ZIP ---")
|
||||
for zip_file in DAFTAR_ZIP:
|
||||
if os.path.exists(zip_file):
|
||||
folder_name = os.path.splitext(zip_file)[0]
|
||||
@@ -40,50 +146,58 @@ def ekstrak_semua_zip():
|
||||
try:
|
||||
with zipfile.ZipFile(zip_file, 'r') as zip_ref:
|
||||
zip_ref.extractall(folder_name)
|
||||
print(f" [OK] {zip_file} -> {folder_name}/")
|
||||
log.info(f" [OK] {zip_file} -> {folder_name}/")
|
||||
except zipfile.BadZipFile:
|
||||
print(f" [ERROR] {zip_file} rusak.")
|
||||
log.error(f" [ERROR] {zip_file} rusak.")
|
||||
else:
|
||||
print(f" [SKIP] File {zip_file} tidak ditemukan.")
|
||||
print("\n")
|
||||
log.warning(f" [SKIP] File {zip_file} tidak ditemukan. Beberapa kelas mungkin kosong.")
|
||||
|
||||
|
||||
# ==========================================
|
||||
# TAHAP 2: GABUNGKAN DATASET 1 & 2
|
||||
# ==========================================
|
||||
def cari_folder_ds2(base_path, keywords):
|
||||
if not os.path.exists(base_path): return None
|
||||
if not os.path.exists(base_path):
|
||||
return None
|
||||
for f in os.listdir(base_path):
|
||||
f_lower = f.lower()
|
||||
if any(k in f_lower for k in keywords):
|
||||
return os.path.join(base_path, f)
|
||||
return None
|
||||
|
||||
|
||||
def gabungkan_dataset_1_dan_2():
|
||||
print("--- TAHAP 2: Menggabungkan Dataset 1 & 2 ---")
|
||||
log.info("--- TAHAP 2: Menggabungkan Dataset 1 & 2 ---")
|
||||
os.makedirs(TARGET_DIR, exist_ok=True)
|
||||
|
||||
# 1. Salin dari dataset_1
|
||||
# 1. Salin dari dataset_1
|
||||
folder_dari_ds1 = ["Bercak Daun", "Daun Sehat", "Hawar Daun"]
|
||||
for folder in folder_dari_ds1:
|
||||
src = os.path.join("dataset_1", folder)
|
||||
dst = os.path.join(TARGET_DIR, folder)
|
||||
if os.path.exists(src):
|
||||
shutil.copytree(src, dst, dirs_exist_ok=True)
|
||||
print(f" [OK] Menyalin folder {src} ke {dst}")
|
||||
log.info(f" [OK] Menyalin folder {src} ke {dst}")
|
||||
|
||||
# 2. Salin gambar dari dataset_2
|
||||
base_ds2 = os.path.join("dataset_2", "data")
|
||||
mapping_ds2 = {
|
||||
("common_rust", "commont_rust"): "Karat Daun",
|
||||
("healthy",): "Daun Sehat"
|
||||
}
|
||||
# 2. Dataset 2 — cari subfolder yang cocok
|
||||
if os.path.exists("dataset_2"):
|
||||
# Cari folder data/ atau folder langsung
|
||||
base_ds2 = "dataset_2"
|
||||
# Check for nested structure (Kaggle download structure varies)
|
||||
data_sub = os.path.join(base_ds2, "data")
|
||||
if os.path.exists(data_sub):
|
||||
base_ds2 = data_sub
|
||||
|
||||
mapping_ds2 = {
|
||||
("common_rust", "commont_rust"): "Karat Daun",
|
||||
("healthy",): "Daun Sehat"
|
||||
}
|
||||
|
||||
if os.path.exists(base_ds2):
|
||||
for keywords, target_subfolder in mapping_ds2.items():
|
||||
src_folder = cari_folder_ds2(base_ds2, keywords)
|
||||
dst_folder = os.path.join(TARGET_DIR, target_subfolder)
|
||||
os.makedirs(dst_folder, exist_ok=True)
|
||||
|
||||
|
||||
if src_folder and os.path.exists(src_folder):
|
||||
file_count = 0
|
||||
for file_name in os.listdir(src_folder):
|
||||
@@ -91,10 +205,12 @@ def gabungkan_dataset_1_dan_2():
|
||||
if os.path.isfile(full_file_name):
|
||||
shutil.copy(full_file_name, dst_folder)
|
||||
file_count += 1
|
||||
print(f" [OK] Menyalin {file_count} gambar dari {src_folder} ke {dst_folder}")
|
||||
log.info(f" [OK] Menyalin {file_count} gambar dari {src_folder} ke {dst_folder}")
|
||||
else:
|
||||
print(f" [SKIP] Folder untuk '{target_subfolder}' tidak ditemukan di {base_ds2}")
|
||||
print("\n")
|
||||
log.warning(f" [SKIP] Folder untuk '{target_subfolder}' tidak ditemukan di dataset_2")
|
||||
else:
|
||||
log.warning(" [SKIP] Folder dataset_2/ tidak ada, dataset_2 tidak diproses.")
|
||||
|
||||
|
||||
# ==========================================
|
||||
# TAHAP 3: GABUNGKAN DATASET 3 (JSON MAPPING)
|
||||
@@ -104,20 +220,22 @@ def cari_gambar_fleksibel(folder_sumber, nama_file_target):
|
||||
path_langsung = os.path.join(folder_sumber, nama_file_target)
|
||||
if os.path.exists(path_langsung):
|
||||
return path_langsung
|
||||
|
||||
|
||||
target_lower = nama_file_target.lower()
|
||||
for f in os.listdir(folder_sumber):
|
||||
if f.lower() == target_lower or os.path.splitext(f)[0].lower() == os.path.splitext(target_lower)[0]:
|
||||
if (f.lower() == target_lower
|
||||
or os.path.splitext(f)[0].lower() == os.path.splitext(target_lower)[0]):
|
||||
return os.path.join(folder_sumber, f)
|
||||
return None
|
||||
|
||||
|
||||
def gabungkan_dataset_3():
|
||||
print("--- TAHAP 3: Menggabungkan Dataset 3 berdasarkan JSON ---")
|
||||
log.info("--- TAHAP 3: Menggabungkan Dataset 3 berdasarkan JSON ---")
|
||||
folder_data = os.path.join("dataset_3", "data")
|
||||
file_json = os.path.join("dataset_3", "desc.json")
|
||||
|
||||
if not os.path.exists(file_json):
|
||||
print(f" [SKIP] {file_json} tidak ditemukan.\n")
|
||||
log.warning(f" [SKIP] {file_json} tidak ditemukan. Dataset 3 dilewati.")
|
||||
return
|
||||
|
||||
with open(file_json, 'r', encoding='utf-8') as f:
|
||||
@@ -139,13 +257,14 @@ def gabungkan_dataset_3():
|
||||
shutil.copy(path_sumber, os.path.join(folder_tujuan, nama_asli))
|
||||
berhasil += 1
|
||||
|
||||
print(f" [OK] Berhasil merutekan {berhasil} gambar dari dataset_3 ke '{TARGET_DIR}'\n")
|
||||
log.info(f" [OK] Berhasil merutekan {berhasil} gambar dari dataset_3 ke '{TARGET_DIR}'")
|
||||
|
||||
|
||||
# ==========================================
|
||||
# TAHAP 4: PEMBERSIHAN DATA (CLEANING)
|
||||
# ==========================================
|
||||
def bersihkan_dataset():
|
||||
print("--- TAHAP 4: Menghapus File Spesifik ---")
|
||||
log.info("--- TAHAP 4: Menghapus File Bermasalah ---")
|
||||
set_hapus = set(DAFTAR_FILE_HAPUS)
|
||||
terhapus = 0
|
||||
|
||||
@@ -156,42 +275,219 @@ def bersihkan_dataset():
|
||||
path_lengkap = os.path.join(root, nama_file)
|
||||
try:
|
||||
os.remove(path_lengkap)
|
||||
print(f" [TERHAPUS] {path_lengkap}")
|
||||
set_hapus.remove(nama_file)
|
||||
log.info(f" [TERHAPUS] {path_lengkap}")
|
||||
set_hapus.discard(nama_file)
|
||||
terhapus += 1
|
||||
except Exception as e:
|
||||
print(f" [GAGAL] {path_lengkap} ({e})")
|
||||
|
||||
print(f" [OK] Total file dihapus: {terhapus}")
|
||||
log.warning(f" [GAGAL] {path_lengkap} ({e})")
|
||||
|
||||
log.info(f" [OK] Total file spesifik dihapus: {terhapus}")
|
||||
if set_hapus:
|
||||
print(f" [INFO] {len(set_hapus)} file tidak ditemukan (mungkin sudah terhapus sebelumnya):")
|
||||
log.info(f" [INFO] {len(set_hapus)} file tidak ditemukan (mungkin sudah terhapus sebelumnya)")
|
||||
for sisa in set_hapus:
|
||||
print(f" - {sisa}")
|
||||
print("\n")
|
||||
log.info(f" - {sisa}")
|
||||
|
||||
|
||||
# ==========================================
|
||||
# TAHAP 5: BUNGKUS KE ZIP
|
||||
# TAHAP 5: HAPUS AUGMENTED DUPLICATES
|
||||
# ==========================================
|
||||
def hapus_augmented_duplicates():
|
||||
"""Remove pre-augmented duplicates (augmented_* files)."""
|
||||
log.info("--- TAHAP 5: Menghapus Augmented Duplicates ---")
|
||||
removed = 0
|
||||
if not os.path.exists(TARGET_DIR):
|
||||
log.warning(" [SKIP] Dataset folder tidak ditemukan.")
|
||||
return
|
||||
|
||||
for root, _, files in os.walk(TARGET_DIR):
|
||||
for nama_file in files:
|
||||
if nama_file.startswith("augmented_"):
|
||||
path_file = os.path.join(root, nama_file)
|
||||
try:
|
||||
os.remove(path_file)
|
||||
removed += 1
|
||||
except Exception as e:
|
||||
log.warning(f" [GAGAL] {path_file} ({e})")
|
||||
|
||||
log.info(f" [OK] {removed} augmented_* files dihapus.")
|
||||
|
||||
|
||||
# ==========================================
|
||||
# TAHAP 6: CORRUPT & INVALID IMAGE DETECTION
|
||||
# ==========================================
|
||||
def validasi_gambar():
|
||||
log.info("--- TAHAP 6: Validasi & Deteksi Gambar Rusak ---")
|
||||
dihapus_total = 0
|
||||
stat_count = {
|
||||
"too_small_file": 0,
|
||||
"cannot_open": 0,
|
||||
"too_small_dims": 0,
|
||||
"extreme_aspect": 0,
|
||||
}
|
||||
|
||||
if not os.path.exists(TARGET_DIR):
|
||||
log.warning(" [SKIP] Dataset folder tidak ditemukan.")
|
||||
return
|
||||
|
||||
for root, _, files in os.walk(TARGET_DIR):
|
||||
for nama_file in files:
|
||||
path_file = os.path.join(root, nama_file)
|
||||
|
||||
try:
|
||||
file_size = os.path.getsize(path_file)
|
||||
if file_size < MIN_FILE_SIZE:
|
||||
os.remove(path_file)
|
||||
dihapus_total += 1
|
||||
stat_count["too_small_file"] += 1
|
||||
log.info(f" [HAPUS-small] {path_file} ({file_size} bytes)")
|
||||
continue
|
||||
except OSError:
|
||||
continue
|
||||
|
||||
try:
|
||||
img = Image.open(path_file)
|
||||
img.verify()
|
||||
except (UnidentifiedImageError, OSError, SyntaxError):
|
||||
try:
|
||||
os.remove(path_file)
|
||||
dihapus_total += 1
|
||||
stat_count["cannot_open"] += 1
|
||||
log.info(f" [HAPUS-corrupt] {path_file}")
|
||||
except OSError:
|
||||
pass
|
||||
continue
|
||||
|
||||
try:
|
||||
img = Image.open(path_file)
|
||||
w, h = img.size
|
||||
if w < MIN_WIDTH or h < MIN_HEIGHT:
|
||||
os.remove(path_file)
|
||||
dihapus_total += 1
|
||||
stat_count["too_small_dims"] += 1
|
||||
log.info(f" [HAPUS-dims] {path_file} ({w}x{h})")
|
||||
continue
|
||||
|
||||
aspect = w / max(h, 1)
|
||||
if aspect > MAX_ASPECT_RATIO or aspect < (1.0 / MAX_ASPECT_RATIO):
|
||||
os.remove(path_file)
|
||||
dihapus_total += 1
|
||||
stat_count["extreme_aspect"] += 1
|
||||
log.info(f" [HAPUS-aspect] {path_file} ({w}x{h}, ar={aspect:.2f})")
|
||||
continue
|
||||
|
||||
if img.mode not in ('RGB', 'RGBA'):
|
||||
img = img.convert('RGB')
|
||||
img.save(path_file)
|
||||
except Exception:
|
||||
try:
|
||||
os.remove(path_file)
|
||||
dihapus_total += 1
|
||||
stat_count["cannot_open"] += 1
|
||||
log.info(f" [HAPUS-exc] {path_file}")
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
log.info(f" [OK] Total dihapus: {dihapus_total}")
|
||||
for reason, count in stat_count.items():
|
||||
if count > 0:
|
||||
log.info(f" {reason}: {count}")
|
||||
|
||||
|
||||
# ==========================================
|
||||
# TAHAP 7: PERCEPTUAL HASH DEDUPLICATION
|
||||
# ==========================================
|
||||
def deteksi_duplikat_perceptual():
|
||||
try:
|
||||
import imagehash
|
||||
except ImportError:
|
||||
log.info("--- TAHAP 7: Deteksi Duplikat (SKIP - imagehash not installed) ---")
|
||||
log.info(" Install with: pip install imagehash")
|
||||
return
|
||||
|
||||
log.info("--- TAHAP 7: Deteksi Duplikat Perceptual Hash ---")
|
||||
THRESHOLD = 5
|
||||
if not os.path.exists(TARGET_DIR):
|
||||
log.warning(" [SKIP] Dataset folder tidak ditemukan.")
|
||||
return
|
||||
|
||||
seen_hashes = {}
|
||||
removed = 0
|
||||
scanned = 0
|
||||
|
||||
for class_name in sorted(os.listdir(TARGET_DIR)):
|
||||
class_path = os.path.join(TARGET_DIR, class_name)
|
||||
if not os.path.isdir(class_path):
|
||||
continue
|
||||
for nama_file in sorted(os.listdir(class_path)):
|
||||
path_file = os.path.join(class_path, nama_file)
|
||||
if not os.path.isfile(path_file):
|
||||
continue
|
||||
scanned += 1
|
||||
try:
|
||||
img = Image.open(path_file).convert('RGB')
|
||||
ahash = imagehash.average_hash(img)
|
||||
ahash_hex = str(ahash)
|
||||
is_dup = False
|
||||
for seen_hex, (seen_path, seen_class) in seen_hashes.items():
|
||||
seen_hash = imagehash.hex_to_hash(seen_hex)
|
||||
if ahash - seen_hash <= THRESHOLD:
|
||||
try:
|
||||
os.remove(path_file)
|
||||
removed += 1
|
||||
is_dup = True
|
||||
log.info(f" [DUP] {path_file} ~ {seen_path} (dist={ahash - seen_hash})")
|
||||
break
|
||||
except OSError:
|
||||
pass
|
||||
if not is_dup:
|
||||
seen_hashes[ahash_hex] = (path_file, class_name)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
log.info(f" [OK] Scanned {scanned}, removed {removed} near-duplicates (threshold={THRESHOLD}).")
|
||||
|
||||
|
||||
# ==========================================
|
||||
# TAHAP 8: BUNGKUS KE ZIP
|
||||
# ==========================================
|
||||
def zip_dataset():
|
||||
print("--- TAHAP 5: Mengompresi Folder Dataset ---")
|
||||
log.info("--- TAHAP 8: Mengompresi Folder Dataset ---")
|
||||
if os.path.exists(TARGET_DIR):
|
||||
print(f" Membuat file {TARGET_DIR}.zip, mohon tunggu sebentar...")
|
||||
# shutil.make_archive(nama_output_tanpa_ext, format, folder_yang_dizip)
|
||||
log.info(f" Membuat file {TARGET_DIR}.zip, mohon tunggu sebentar...")
|
||||
shutil.make_archive(TARGET_DIR, 'zip', TARGET_DIR)
|
||||
print(f" [OK] Berhasil! File '{TARGET_DIR}.zip' sudah siap.\n")
|
||||
log.info(f" [OK] Berhasil! File '{TARGET_DIR}.zip' sudah siap.")
|
||||
else:
|
||||
print(f" [ERROR] Folder '{TARGET_DIR}' tidak ditemukan, proses zip dibatalkan.\n")
|
||||
log.error(f" [ERROR] Folder '{TARGET_DIR}' tidak ditemukan, proses zip dibatalkan.")
|
||||
|
||||
|
||||
# ==========================================
|
||||
# MAIN EXECUTION
|
||||
# ==========================================
|
||||
if __name__ == "__main__":
|
||||
print("=== MEMULAI PREPROCESSING DATASET ===\n")
|
||||
log.info("=== MEMULAI PREPROCESSING DATASET ===")
|
||||
|
||||
# Auto-download if ZIPs missing
|
||||
download_dataset("dataset_1.zip", download_dataset_1)
|
||||
download_dataset("dataset_2.zip", download_dataset_2)
|
||||
download_dataset("dataset_3.zip", download_dataset_3)
|
||||
|
||||
ekstrak_semua_zip()
|
||||
gabungkan_dataset_1_dan_2()
|
||||
gabungkan_dataset_3()
|
||||
bersihkan_dataset()
|
||||
hapus_augmented_duplicates()
|
||||
validasi_gambar()
|
||||
deteksi_duplikat_perceptual()
|
||||
zip_dataset()
|
||||
print("=== PREPROCESSING SELESAI ===")
|
||||
print(f"Dataset akhir Anda kini siap digunakan di dalam folder '{TARGET_DIR}' dan '{TARGET_DIR}.zip'.")
|
||||
|
||||
log.info("=== PREPROCESSING SELESAI ===")
|
||||
|
||||
if os.path.exists(TARGET_DIR):
|
||||
total = 0
|
||||
for class_name in sorted(os.listdir(TARGET_DIR)):
|
||||
class_path = os.path.join(TARGET_DIR, class_name)
|
||||
if os.path.isdir(class_path):
|
||||
count = len([f for f in os.listdir(class_path) if os.path.isfile(os.path.join(class_path, f))])
|
||||
total += count
|
||||
log.info(f" {class_name}: {count} images")
|
||||
log.info(f" TOTAL: {total} images")
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
tensorflow==2.19.0 # CPU + Colab; for local GPU, install tensorflow[and-cuda]
|
||||
tensorflowjs==4.22.0
|
||||
gdown # download dataset from Google Drive
|
||||
kagglehub # download dataset from Kaggle
|
||||
numpy
|
||||
matplotlib
|
||||
seaborn
|
||||
|
||||
+103
-26
@@ -1,34 +1,63 @@
|
||||
import os
|
||||
import json
|
||||
import logging
|
||||
import traceback
|
||||
import tensorflow as tf
|
||||
from tensorflow.keras import layers, models
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s")
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def cbam_block(x, ratio=8, name="cbam"):
|
||||
"""Convolutional Block Attention Module — lightweight foreground attention."""
|
||||
channels = tf.shape(x)[-1]
|
||||
|
||||
# Channel attention
|
||||
avg_pool = layers.GlobalAveragePooling2D()(x)
|
||||
max_pool = layers.GlobalMaxPooling2D()(x)
|
||||
ca = layers.Dense(channels // ratio, activation="swish", name=f"{name}_ca1")(avg_pool)
|
||||
ca = layers.Dense(channels, activation="sigmoid", name=f"{name}_ca2")(ca)
|
||||
ca2 = layers.Dense(channels // ratio, activation="swish", name=f"{name}_ca3")(max_pool)
|
||||
ca2 = layers.Dense(channels, activation="sigmoid", name=f"{name}_ca4")(ca2)
|
||||
ca_out = layers.Add(name=f"{name}_ca_add")([ca, ca2])
|
||||
ca_out = layers.Reshape((1, 1, channels), name=f"{name}_ca_reshape")(ca_out)
|
||||
x = layers.Multiply(name=f"{name}_ca_mul")([x, ca_out])
|
||||
|
||||
# Spatial attention
|
||||
from keras import ops
|
||||
avg_sp = ops.mean(x, axis=-1, keepdims=True)
|
||||
max_sp = ops.max(x, axis=-1, keepdims=True)
|
||||
sp = layers.Concatenate(name=f"{name}_sa_cat")([avg_sp, max_sp])
|
||||
sp = layers.Conv2D(1, 7, padding="same", activation="sigmoid", name=f"{name}_sa_conv")(sp)
|
||||
x = layers.Multiply(name=f"{name}_sa_mul")([x, sp])
|
||||
return x
|
||||
|
||||
|
||||
def build_clean_model(num_classes, img_size=(224, 224)):
|
||||
"""Build the production architecture: CBAM + lightweight head, outputting raw logits."""
|
||||
base_model = tf.keras.applications.EfficientNetV2B0(
|
||||
input_shape=img_size + (3,),
|
||||
include_top=False,
|
||||
weights=None,
|
||||
)
|
||||
inputs = tf.keras.Input(shape=img_size + (3,))
|
||||
base_model.trainable = False
|
||||
|
||||
inputs = tf.keras.Input(shape=img_size + (3,), name="input")
|
||||
x = base_model(inputs, training=False)
|
||||
x = layers.Conv2D(512, (3, 3), padding='same', activation='swish')(x)
|
||||
x = layers.BatchNormalization()(x)
|
||||
x = layers.MaxPooling2D((2, 2))(x)
|
||||
x = layers.Dropout(0.2)(x)
|
||||
x = layers.Conv2D(256, (3, 3), padding='same', activation='swish')(x)
|
||||
x = layers.BatchNormalization()(x)
|
||||
x = layers.GlobalAveragePooling2D()(x)
|
||||
x = layers.Dropout(0.3)(x)
|
||||
x = layers.Dense(1024, activation='swish')(x)
|
||||
x = layers.BatchNormalization()(x)
|
||||
x = layers.Dropout(0.4)(x)
|
||||
outputs = layers.Dense(num_classes, activation='softmax', dtype='float32')(x)
|
||||
# CBAM attention — focus on leaf regions, ignore background
|
||||
x = cbam_block(x, ratio=8, name="cbam")
|
||||
x = layers.GlobalAveragePooling2D(name="gap")(x)
|
||||
x = layers.Dropout(0.3, name="drop_gap")(x)
|
||||
x = layers.Dense(512, activation="swish", name="dense_head")(x)
|
||||
x = layers.BatchNormalization(name="bn_head")(x)
|
||||
x = layers.Dropout(0.4, name="drop_head")(x)
|
||||
# Raw logits (no softmax) — temperature scaling applied at inference
|
||||
outputs = layers.Dense(num_classes, activation="linear", dtype="float32", name="logits")(x)
|
||||
return models.Model(inputs, outputs)
|
||||
|
||||
logging.info("=== EXPORT STARTED ===")
|
||||
|
||||
log.info("=== EXPORT STARTED (v3.0) ===")
|
||||
try:
|
||||
MODEL_KERAS_PATH = "best_model/best_model.keras"
|
||||
OUTPUT_DIR = "model"
|
||||
@@ -40,31 +69,79 @@ try:
|
||||
if not os.path.exists(MODEL_KERAS_PATH):
|
||||
raise FileNotFoundError(f"Model file not found at {MODEL_KERAS_PATH}")
|
||||
|
||||
logging.info(f"Loading trained weights from {MODEL_KERAS_PATH}...")
|
||||
log.info(f"Loading trained weights from {MODEL_KERAS_PATH}...")
|
||||
original_model = tf.keras.models.load_model(MODEL_KERAS_PATH, compile=False)
|
||||
|
||||
logging.info("Building clean architecture...")
|
||||
clean_model = build_clean_model(num_classes=original_model.output_shape[-1])
|
||||
|
||||
log.info("Building clean architecture (CBAM + lightweight head)...")
|
||||
num_classes = original_model.output_shape[-1]
|
||||
clean_model = build_clean_model(num_classes=num_classes)
|
||||
clean_model.set_weights(original_model.get_weights())
|
||||
logging.info("Weights cloned successfully.")
|
||||
log.info("Weights cloned successfully.")
|
||||
|
||||
logging.info(f"Exporting to SavedModel format at: {saved_model_dir}...")
|
||||
# ─── Export SavedModel (raw logits) ───
|
||||
log.info(f"Exporting SavedModel (raw logits) at: {saved_model_dir}...")
|
||||
tf.saved_model.save(clean_model, saved_model_dir)
|
||||
logging.info("SavedModel export completed successfully.")
|
||||
log.info("SavedModel export completed successfully.")
|
||||
|
||||
# ─── Export TFLite (INT8 quantization) ───
|
||||
log.info(f"Converting to TFLite INT8 at: {tflite_path}...")
|
||||
|
||||
# Representative dataset for INT8 quantization
|
||||
def representative_dataset():
|
||||
val_dir = "dataset_split/val"
|
||||
if not os.path.exists(val_dir):
|
||||
log.warning("Validation dir not found; skipping representative dataset.")
|
||||
return
|
||||
ds = tf.keras.utils.image_dataset_from_directory(
|
||||
val_dir, shuffle=True, batch_size=1, image_size=(224, 224)
|
||||
)
|
||||
for images, _ in ds.take(200):
|
||||
yield [tf.cast(images, tf.float32)]
|
||||
|
||||
logging.info(f"Converting to TFLite format at: {tflite_path}...")
|
||||
converter = tf.lite.TFLiteConverter.from_keras_model(clean_model)
|
||||
converter.target_spec.supported_ops = [
|
||||
tf.lite.OpsSet.TFLITE_BUILTINS,
|
||||
tf.lite.OpsSet.SELECT_TF_OPS,
|
||||
]
|
||||
converter.optimizations = [tf.lite.Optimize.DEFAULT]
|
||||
if os.path.exists("dataset_split/val"):
|
||||
converter.representative_dataset = representative_dataset
|
||||
|
||||
tflite_model = converter.convert()
|
||||
with open(tflite_path, "wb") as f:
|
||||
f.write(tflite_model)
|
||||
logging.info("TFLite conversion completed successfully.")
|
||||
logging.info("=== EXPORT COMPLETED ===")
|
||||
log.info("TFLite conversion completed successfully.")
|
||||
|
||||
# ─── Export model metadata ───
|
||||
# Load labels and calibration from training output
|
||||
labels_path = os.path.join(OUTPUT_DIR, "labels.json")
|
||||
cal_path = os.path.join("best_model", "calibration.json")
|
||||
|
||||
labels_meta = {"labels": None, "temperature": 1.0, "conf_threshold_high": 0.70,
|
||||
"conf_threshold_low": 0.45}
|
||||
|
||||
if os.path.exists(labels_path):
|
||||
with open(labels_path) as f:
|
||||
labels_meta.update(json.load(f))
|
||||
|
||||
if os.path.exists(cal_path):
|
||||
with open(cal_path) as f:
|
||||
cal = json.load(f)
|
||||
labels_meta["temperature"] = cal.get("temperature", 1.0)
|
||||
|
||||
labels_meta["version"] = "3.0"
|
||||
labels_meta["architecture"] = "EfficientNetV2B0 + CBAM + Dense(512)"
|
||||
labels_meta["output_type"] = "logits"
|
||||
labels_meta["input_range"] = [0, 255]
|
||||
labels_meta["input_size"] = [224, 224]
|
||||
labels_meta["preprocessing"] = "resize_bilinear_224x224_no_normalization"
|
||||
|
||||
with open(labels_path, "w") as f:
|
||||
json.dump(labels_meta, f, indent=2)
|
||||
log.info(f"Labels + calibration metadata saved to {labels_path}")
|
||||
|
||||
log.info("=== EXPORT COMPLETED (v3.0) ===")
|
||||
|
||||
except Exception:
|
||||
logging.error("EXPORT FAILED")
|
||||
logging.error(traceback.format_exc())
|
||||
log.error("EXPORT FAILED")
|
||||
log.error(traceback.format_exc())
|
||||
|
||||
Reference in New Issue
Block a user