bebas
This commit is contained in:
elytra7
@@ -0,0 +1,197 @@
|
||||
import os
|
||||
import shutil
|
||||
import zipfile
|
||||
import json
|
||||
|
||||
# ==========================================
|
||||
# KONFIGURASI DAN MAPPING
|
||||
# ==========================================
|
||||
DAFTAR_ZIP = ['dataset_1.zip', 'dataset_2.zip', 'dataset_3.zip']
|
||||
TARGET_DIR = "dataset"
|
||||
|
||||
PEMETAAN_KATEGORI = {
|
||||
"大斑病": "Hawar Daun",
|
||||
"小斑病": "Hawar Daun",
|
||||
"褐斑病": "Bercak Daun",
|
||||
"弯孢霉叶斑病": "Bercak Daun",
|
||||
"圆斑病": "Bercak Daun",
|
||||
"灰斑病": "Bercak Daun",
|
||||
"南方锈病": "Karat Daun",
|
||||
"普通锈病": "Karat Daun",
|
||||
}
|
||||
|
||||
DAFTAR_FILE_HAPUS = [
|
||||
"CBS28.jpg",
|
||||
"Corn_Common_Rust (1275).jpg",
|
||||
"Corn_Common_Rust (1289).jpg",
|
||||
"Corn_Common_Rust (1295).jpg",
|
||||
"Corn_Gray_Spot (1).jpg"
|
||||
]
|
||||
|
||||
# ==========================================
|
||||
# TAHAP 1: EKSTRAKSI DATASET
|
||||
# ==========================================
|
||||
def ekstrak_semua_zip():
|
||||
print("--- TAHAP 1: Mengekstrak File ZIP ---")
|
||||
for zip_file in DAFTAR_ZIP:
|
||||
if os.path.exists(zip_file):
|
||||
folder_name = os.path.splitext(zip_file)[0]
|
||||
os.makedirs(folder_name, exist_ok=True)
|
||||
try:
|
||||
with zipfile.ZipFile(zip_file, 'r') as zip_ref:
|
||||
zip_ref.extractall(folder_name)
|
||||
print(f" [OK] {zip_file} -> {folder_name}/")
|
||||
except zipfile.BadZipFile:
|
||||
print(f" [ERROR] {zip_file} rusak.")
|
||||
else:
|
||||
print(f" [SKIP] File {zip_file} tidak ditemukan.")
|
||||
print("\n")
|
||||
|
||||
# ==========================================
|
||||
# TAHAP 2: GABUNGKAN DATASET 1 & 2
|
||||
# ==========================================
|
||||
def cari_folder_ds2(base_path, keywords):
|
||||
if not os.path.exists(base_path): return None
|
||||
for f in os.listdir(base_path):
|
||||
f_lower = f.lower()
|
||||
if any(k in f_lower for k in keywords):
|
||||
return os.path.join(base_path, f)
|
||||
return None
|
||||
|
||||
def gabungkan_dataset_1_dan_2():
|
||||
print("--- TAHAP 2: Menggabungkan Dataset 1 & 2 ---")
|
||||
os.makedirs(TARGET_DIR, exist_ok=True)
|
||||
|
||||
# 1. Salin dari dataset_1
|
||||
folder_dari_ds1 = ["Bercak Daun", "Daun Sehat", "Hawar Daun"]
|
||||
for folder in folder_dari_ds1:
|
||||
src = os.path.join("dataset_1", folder)
|
||||
dst = os.path.join(TARGET_DIR, folder)
|
||||
if os.path.exists(src):
|
||||
shutil.copytree(src, dst, dirs_exist_ok=True)
|
||||
print(f" [OK] Menyalin folder {src} ke {dst}")
|
||||
|
||||
# 2. Salin gambar dari dataset_2
|
||||
base_ds2 = os.path.join("dataset_2", "data")
|
||||
mapping_ds2 = {
|
||||
("common_rust", "commont_rust"): "Karat Daun",
|
||||
("healthy",): "Daun Sehat"
|
||||
}
|
||||
|
||||
if os.path.exists(base_ds2):
|
||||
for keywords, target_subfolder in mapping_ds2.items():
|
||||
src_folder = cari_folder_ds2(base_ds2, keywords)
|
||||
dst_folder = os.path.join(TARGET_DIR, target_subfolder)
|
||||
os.makedirs(dst_folder, exist_ok=True)
|
||||
|
||||
if src_folder and os.path.exists(src_folder):
|
||||
file_count = 0
|
||||
for file_name in os.listdir(src_folder):
|
||||
full_file_name = os.path.join(src_folder, file_name)
|
||||
if os.path.isfile(full_file_name):
|
||||
shutil.copy(full_file_name, dst_folder)
|
||||
file_count += 1
|
||||
print(f" [OK] Menyalin {file_count} gambar dari {src_folder} ke {dst_folder}")
|
||||
else:
|
||||
print(f" [SKIP] Folder untuk '{target_subfolder}' tidak ditemukan di {base_ds2}")
|
||||
print("\n")
|
||||
|
||||
# ==========================================
|
||||
# TAHAP 3: GABUNGKAN DATASET 3 (JSON MAPPING)
|
||||
# ==========================================
|
||||
def cari_gambar_fleksibel(folder_sumber, nama_file_target):
|
||||
nama_file_target = nama_file_target.strip()
|
||||
path_langsung = os.path.join(folder_sumber, nama_file_target)
|
||||
if os.path.exists(path_langsung):
|
||||
return path_langsung
|
||||
|
||||
target_lower = nama_file_target.lower()
|
||||
for f in os.listdir(folder_sumber):
|
||||
if f.lower() == target_lower or os.path.splitext(f)[0].lower() == os.path.splitext(target_lower)[0]:
|
||||
return os.path.join(folder_sumber, f)
|
||||
return None
|
||||
|
||||
def gabungkan_dataset_3():
|
||||
print("--- TAHAP 3: Menggabungkan Dataset 3 berdasarkan JSON ---")
|
||||
folder_data = os.path.join("dataset_3", "data")
|
||||
file_json = os.path.join("dataset_3", "desc.json")
|
||||
|
||||
if not os.path.exists(file_json):
|
||||
print(f" [SKIP] {file_json} tidak ditemukan.\n")
|
||||
return
|
||||
|
||||
with open(file_json, 'r', encoding='utf-8') as f:
|
||||
data_label = json.load(f)
|
||||
|
||||
berhasil = 0
|
||||
for item in data_label:
|
||||
image_name = item.get("image_name")
|
||||
label = item.get("label", "").strip()
|
||||
|
||||
if image_name and label in PEMETAAN_KATEGORI:
|
||||
nama_folder_target = PEMETAAN_KATEGORI[label]
|
||||
folder_tujuan = os.path.join(TARGET_DIR, nama_folder_target)
|
||||
os.makedirs(folder_tujuan, exist_ok=True)
|
||||
|
||||
path_sumber = cari_gambar_fleksibel(folder_data, image_name)
|
||||
if path_sumber:
|
||||
nama_asli = os.path.basename(path_sumber)
|
||||
shutil.copy(path_sumber, os.path.join(folder_tujuan, nama_asli))
|
||||
berhasil += 1
|
||||
|
||||
print(f" [OK] Berhasil merutekan {berhasil} gambar dari dataset_3 ke '{TARGET_DIR}'\n")
|
||||
|
||||
# ==========================================
|
||||
# TAHAP 4: PEMBERSIHAN DATA (CLEANING)
|
||||
# ==========================================
|
||||
def bersihkan_dataset():
|
||||
print("--- TAHAP 4: Menghapus File Spesifik ---")
|
||||
set_hapus = set(DAFTAR_FILE_HAPUS)
|
||||
terhapus = 0
|
||||
|
||||
if os.path.exists(TARGET_DIR):
|
||||
for root, _, files in os.walk(TARGET_DIR):
|
||||
for nama_file in files:
|
||||
if nama_file in set_hapus:
|
||||
path_lengkap = os.path.join(root, nama_file)
|
||||
try:
|
||||
os.remove(path_lengkap)
|
||||
print(f" [TERHAPUS] {path_lengkap}")
|
||||
set_hapus.remove(nama_file)
|
||||
terhapus += 1
|
||||
except Exception as e:
|
||||
print(f" [GAGAL] {path_lengkap} ({e})")
|
||||
|
||||
print(f" [OK] Total file dihapus: {terhapus}")
|
||||
if set_hapus:
|
||||
print(f" [INFO] {len(set_hapus)} file tidak ditemukan (mungkin sudah terhapus sebelumnya):")
|
||||
for sisa in set_hapus:
|
||||
print(f" - {sisa}")
|
||||
print("\n")
|
||||
|
||||
# ==========================================
|
||||
# TAHAP 5: BUNGKUS KE ZIP
|
||||
# ==========================================
|
||||
def zip_dataset():
|
||||
print("--- TAHAP 5: Mengompresi Folder Dataset ---")
|
||||
if os.path.exists(TARGET_DIR):
|
||||
print(f" Membuat file {TARGET_DIR}.zip, mohon tunggu sebentar...")
|
||||
# shutil.make_archive(nama_output_tanpa_ext, format, folder_yang_dizip)
|
||||
shutil.make_archive(TARGET_DIR, 'zip', TARGET_DIR)
|
||||
print(f" [OK] Berhasil! File '{TARGET_DIR}.zip' sudah siap.\n")
|
||||
else:
|
||||
print(f" [ERROR] Folder '{TARGET_DIR}' tidak ditemukan, proses zip dibatalkan.\n")
|
||||
|
||||
|
||||
# ==========================================
|
||||
# MAIN EXECUTION
|
||||
# ==========================================
|
||||
if __name__ == "__main__":
|
||||
print("=== MEMULAI PREPROCESSING DATASET ===\n")
|
||||
ekstrak_semua_zip()
|
||||
gabungkan_dataset_1_dan_2()
|
||||
gabungkan_dataset_3()
|
||||
bersihkan_dataset()
|
||||
zip_dataset()
|
||||
print("=== PREPROCESSING SELESAI ===")
|
||||
print(f"Dataset akhir Anda kini siap digunakan di dalam folder '{TARGET_DIR}' dan '{TARGET_DIR}.zip'.")
|
||||
Reference in New Issue
Block a user