chore(Machine_Learning): update notebook execution count and outputs
This commit is contained in:
MythEclipse
+241
-56
@@ -23,20 +23,123 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"execution_count": 6,
|
||||
"id": "94627f52",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Requirement already satisfied: tensorflow==2.19.0 in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from -r requirements.txt (line 1)) (2.19.0)\n",
|
||||
"Requirement already satisfied: tensorflowjs==4.22.0 in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from -r requirements.txt (line 2)) (4.22.0)\n",
|
||||
"Requirement already satisfied: gdown in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from -r requirements.txt (line 3)) (6.1.0)\n",
|
||||
"Requirement already satisfied: numpy in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from -r requirements.txt (line 4)) (2.1.3)\n",
|
||||
"Requirement already satisfied: matplotlib in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from -r requirements.txt (line 5)) (3.10.9)\n",
|
||||
"Requirement already satisfied: seaborn in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from -r requirements.txt (line 6)) (0.13.2)\n",
|
||||
"Requirement already satisfied: pillow in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from -r requirements.txt (line 7)) (12.2.0)\n",
|
||||
"Requirement already satisfied: split-folders in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from -r requirements.txt (line 8)) (0.6.1)\n",
|
||||
"Requirement already satisfied: scikit-learn in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from -r requirements.txt (line 9)) (1.9.0)\n",
|
||||
"Requirement already satisfied: tf2onnx in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from -r requirements.txt (line 10)) (1.17.0)\n",
|
||||
"Requirement already satisfied: onnxruntime in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from -r requirements.txt (line 11)) (1.26.0)\n",
|
||||
"Requirement already satisfied: absl-py>=1.0.0 in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from tensorflow==2.19.0->-r requirements.txt (line 1)) (2.4.0)\n",
|
||||
"Requirement already satisfied: astunparse>=1.6.0 in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from tensorflow==2.19.0->-r requirements.txt (line 1)) (1.6.3)\n",
|
||||
"Requirement already satisfied: flatbuffers>=24.3.25 in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from tensorflow==2.19.0->-r requirements.txt (line 1)) (25.12.19)\n",
|
||||
"Requirement already satisfied: gast!=0.5.0,!=0.5.1,!=0.5.2,>=0.2.1 in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from tensorflow==2.19.0->-r requirements.txt (line 1)) (0.7.0)\n",
|
||||
"Requirement already satisfied: google-pasta>=0.1.1 in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from tensorflow==2.19.0->-r requirements.txt (line 1)) (0.2.0)\n",
|
||||
"Requirement already satisfied: libclang>=13.0.0 in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from tensorflow==2.19.0->-r requirements.txt (line 1)) (18.1.1)\n",
|
||||
"Requirement already satisfied: opt-einsum>=2.3.2 in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from tensorflow==2.19.0->-r requirements.txt (line 1)) (3.4.0)\n",
|
||||
"Requirement already satisfied: packaging in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from tensorflow==2.19.0->-r requirements.txt (line 1)) (23.2)\n",
|
||||
"Requirement already satisfied: protobuf!=4.21.0,!=4.21.1,!=4.21.2,!=4.21.3,!=4.21.4,!=4.21.5,<6.0.0dev,>=3.20.3 in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from tensorflow==2.19.0->-r requirements.txt (line 1)) (5.29.6)\n",
|
||||
"Requirement already satisfied: requests<3,>=2.21.0 in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from tensorflow==2.19.0->-r requirements.txt (line 1)) (2.34.2)\n",
|
||||
"Requirement already satisfied: setuptools in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from tensorflow==2.19.0->-r requirements.txt (line 1)) (82.0.1)\n",
|
||||
"Requirement already satisfied: six>=1.12.0 in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from tensorflow==2.19.0->-r requirements.txt (line 1)) (1.17.0)\n",
|
||||
"Requirement already satisfied: termcolor>=1.1.0 in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from tensorflow==2.19.0->-r requirements.txt (line 1)) (3.3.0)\n",
|
||||
"Requirement already satisfied: typing-extensions>=3.6.6 in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from tensorflow==2.19.0->-r requirements.txt (line 1)) (4.15.0)\n",
|
||||
"Requirement already satisfied: wrapt>=1.11.0 in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from tensorflow==2.19.0->-r requirements.txt (line 1)) (2.2.1)\n",
|
||||
"Requirement already satisfied: grpcio<2.0,>=1.24.3 in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from tensorflow==2.19.0->-r requirements.txt (line 1)) (1.81.1)\n",
|
||||
"Requirement already satisfied: tensorboard~=2.19.0 in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from tensorflow==2.19.0->-r requirements.txt (line 1)) (2.19.0)\n",
|
||||
"Requirement already satisfied: keras>=3.5.0 in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from tensorflow==2.19.0->-r requirements.txt (line 1)) (3.14.1)\n",
|
||||
"Requirement already satisfied: h5py>=3.11.0 in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from tensorflow==2.19.0->-r requirements.txt (line 1)) (3.14.0)\n",
|
||||
"Requirement already satisfied: ml-dtypes<1.0.0,>=0.5.1 in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from tensorflow==2.19.0->-r requirements.txt (line 1)) (0.5.4)\n",
|
||||
"Requirement already satisfied: flax>=0.7.2 in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from tensorflowjs==4.22.0->-r requirements.txt (line 2)) (0.12.7)\n",
|
||||
"Requirement already satisfied: importlib_resources>=5.9.0 in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from tensorflowjs==4.22.0->-r requirements.txt (line 2)) (7.1.0)\n",
|
||||
"Requirement already satisfied: jax>=0.4.13 in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from tensorflowjs==4.22.0->-r requirements.txt (line 2)) (0.10.1)\n",
|
||||
"Requirement already satisfied: jaxlib>=0.4.13 in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from tensorflowjs==4.22.0->-r requirements.txt (line 2)) (0.10.1)\n",
|
||||
"Requirement already satisfied: tf-keras>=2.13.0 in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from tensorflowjs==4.22.0->-r requirements.txt (line 2)) (2.19.0)\n",
|
||||
"Requirement already satisfied: tensorflow-decision-forests>=1.5.0 in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from tensorflowjs==4.22.0->-r requirements.txt (line 2)) (1.12.0)\n",
|
||||
"Requirement already satisfied: tensorflow-hub>=0.16.1 in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from tensorflowjs==4.22.0->-r requirements.txt (line 2)) (0.16.1)\n",
|
||||
"Requirement already satisfied: charset_normalizer<4,>=2 in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from requests<3,>=2.21.0->tensorflow==2.19.0->-r requirements.txt (line 1)) (3.4.7)\n",
|
||||
"Requirement already satisfied: idna<4,>=2.5 in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from requests<3,>=2.21.0->tensorflow==2.19.0->-r requirements.txt (line 1)) (3.18)\n",
|
||||
"Requirement already satisfied: urllib3<3,>=1.26 in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from requests<3,>=2.21.0->tensorflow==2.19.0->-r requirements.txt (line 1)) (2.7.0)\n",
|
||||
"Requirement already satisfied: certifi>=2023.5.7 in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from requests<3,>=2.21.0->tensorflow==2.19.0->-r requirements.txt (line 1)) (2026.5.20)\n",
|
||||
"Requirement already satisfied: markdown>=2.6.8 in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from tensorboard~=2.19.0->tensorflow==2.19.0->-r requirements.txt (line 1)) (3.10.2)\n",
|
||||
"Requirement already satisfied: tensorboard-data-server<0.8.0,>=0.7.0 in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from tensorboard~=2.19.0->tensorflow==2.19.0->-r requirements.txt (line 1)) (0.7.2)\n",
|
||||
"Requirement already satisfied: werkzeug>=1.0.1 in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from tensorboard~=2.19.0->tensorflow==2.19.0->-r requirements.txt (line 1)) (3.1.8)\n",
|
||||
"Requirement already satisfied: beautifulsoup4 in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from gdown->-r requirements.txt (line 3)) (4.15.0)\n",
|
||||
"Requirement already satisfied: filelock in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from gdown->-r requirements.txt (line 3)) (3.29.3)\n",
|
||||
"Requirement already satisfied: tqdm in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from gdown->-r requirements.txt (line 3)) (4.68.2)\n",
|
||||
"Requirement already satisfied: contourpy>=1.0.1 in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from matplotlib->-r requirements.txt (line 5)) (1.3.3)\n",
|
||||
"Requirement already satisfied: cycler>=0.10 in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from matplotlib->-r requirements.txt (line 5)) (0.12.1)\n",
|
||||
"Requirement already satisfied: fonttools>=4.22.0 in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from matplotlib->-r requirements.txt (line 5)) (4.63.0)\n",
|
||||
"Requirement already satisfied: kiwisolver>=1.3.1 in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from matplotlib->-r requirements.txt (line 5)) (1.5.0)\n",
|
||||
"Requirement already satisfied: pyparsing>=3 in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from matplotlib->-r requirements.txt (line 5)) (3.3.2)\n",
|
||||
"Requirement already satisfied: python-dateutil>=2.7 in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from matplotlib->-r requirements.txt (line 5)) (2.9.0.post0)\n",
|
||||
"Requirement already satisfied: pandas>=1.2 in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from seaborn->-r requirements.txt (line 6)) (3.0.3)\n",
|
||||
"Requirement already satisfied: scipy>=1.10.0 in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from scikit-learn->-r requirements.txt (line 9)) (1.17.1)\n",
|
||||
"Requirement already satisfied: joblib>=1.4.0 in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from scikit-learn->-r requirements.txt (line 9)) (1.5.3)\n",
|
||||
"Requirement already satisfied: narwhals>=2.0.1 in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from scikit-learn->-r requirements.txt (line 9)) (2.22.1)\n",
|
||||
"Requirement already satisfied: threadpoolctl>=3.5.0 in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from scikit-learn->-r requirements.txt (line 9)) (3.6.0)\n",
|
||||
"Requirement already satisfied: onnx>=1.14.0 in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from tf2onnx->-r requirements.txt (line 10)) (1.21.0)\n",
|
||||
"Requirement already satisfied: wheel<1.0,>=0.23.0 in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from astunparse>=1.6.0->tensorflow==2.19.0->-r requirements.txt (line 1)) (0.45.1)\n",
|
||||
"Requirement already satisfied: msgpack in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from flax>=0.7.2->tensorflowjs==4.22.0->-r requirements.txt (line 2)) (1.2.0)\n",
|
||||
"Requirement already satisfied: optax in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from flax>=0.7.2->tensorflowjs==4.22.0->-r requirements.txt (line 2)) (0.2.8)\n",
|
||||
"Requirement already satisfied: orbax-checkpoint in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from flax>=0.7.2->tensorflowjs==4.22.0->-r requirements.txt (line 2)) (0.12.0)\n",
|
||||
"Requirement already satisfied: tensorstore in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from flax>=0.7.2->tensorflowjs==4.22.0->-r requirements.txt (line 2)) (0.1.84)\n",
|
||||
"Requirement already satisfied: rich>=11.1 in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from flax>=0.7.2->tensorflowjs==4.22.0->-r requirements.txt (line 2)) (15.0.0)\n",
|
||||
"Requirement already satisfied: PyYAML>=5.4.1 in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from flax>=0.7.2->tensorflowjs==4.22.0->-r requirements.txt (line 2)) (6.0.3)\n",
|
||||
"Requirement already satisfied: treescope>=0.1.7 in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from flax>=0.7.2->tensorflowjs==4.22.0->-r requirements.txt (line 2)) (0.1.10)\n",
|
||||
"Requirement already satisfied: namex in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from keras>=3.5.0->tensorflow==2.19.0->-r requirements.txt (line 1)) (0.1.0)\n",
|
||||
"Requirement already satisfied: optree in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from keras>=3.5.0->tensorflow==2.19.0->-r requirements.txt (line 1)) (0.19.1)\n",
|
||||
"Requirement already satisfied: markdown-it-py>=2.2.0 in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from rich>=11.1->flax>=0.7.2->tensorflowjs==4.22.0->-r requirements.txt (line 2)) (4.2.0)\n",
|
||||
"Requirement already satisfied: pygments<3.0.0,>=2.13.0 in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from rich>=11.1->flax>=0.7.2->tensorflowjs==4.22.0->-r requirements.txt (line 2)) (2.20.0)\n",
|
||||
"Requirement already satisfied: mdurl~=0.1 in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from markdown-it-py>=2.2.0->rich>=11.1->flax>=0.7.2->tensorflowjs==4.22.0->-r requirements.txt (line 2)) (0.1.2)\n",
|
||||
"Requirement already satisfied: wurlitzer in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from tensorflow-decision-forests>=1.5.0->tensorflowjs==4.22.0->-r requirements.txt (line 2)) (3.1.1)\n",
|
||||
"Requirement already satisfied: ydf>=0.11.0 in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from tensorflow-decision-forests>=1.5.0->tensorflowjs==4.22.0->-r requirements.txt (line 2)) (0.16.0)\n",
|
||||
"Requirement already satisfied: markupsafe>=2.1.1 in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from werkzeug>=1.0.1->tensorboard~=2.19.0->tensorflow==2.19.0->-r requirements.txt (line 1)) (3.0.3)\n",
|
||||
"Requirement already satisfied: soupsieve>=1.6.1 in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from beautifulsoup4->gdown->-r requirements.txt (line 3)) (2.8.4)\n",
|
||||
"Requirement already satisfied: etils[epath,epy] in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from orbax-checkpoint->flax>=0.7.2->tensorflowjs==4.22.0->-r requirements.txt (line 2)) (1.14.0)\n",
|
||||
"Requirement already satisfied: prometheus-client>=0.20.0 in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from orbax-checkpoint->flax>=0.7.2->tensorflowjs==4.22.0->-r requirements.txt (line 2)) (0.25.0)\n",
|
||||
"Requirement already satisfied: aiofiles in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from orbax-checkpoint->flax>=0.7.2->tensorflowjs==4.22.0->-r requirements.txt (line 2)) (25.1.0)\n",
|
||||
"Requirement already satisfied: humanize in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from orbax-checkpoint->flax>=0.7.2->tensorflowjs==4.22.0->-r requirements.txt (line 2)) (4.15.0)\n",
|
||||
"Requirement already satisfied: simplejson>=3.16.0 in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from orbax-checkpoint->flax>=0.7.2->tensorflowjs==4.22.0->-r requirements.txt (line 2)) (4.1.1)\n",
|
||||
"Requirement already satisfied: psutil in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from orbax-checkpoint->flax>=0.7.2->tensorflowjs==4.22.0->-r requirements.txt (line 2)) (7.2.2)\n",
|
||||
"Requirement already satisfied: uvloop in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from orbax-checkpoint->flax>=0.7.2->tensorflowjs==4.22.0->-r requirements.txt (line 2)) (0.22.1)\n",
|
||||
"Requirement already satisfied: fsspec in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from etils[epath,epy]->orbax-checkpoint->flax>=0.7.2->tensorflowjs==4.22.0->-r requirements.txt (line 2)) (2026.4.0)\n",
|
||||
"Requirement already satisfied: zipp in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from etils[epath,epy]->orbax-checkpoint->flax>=0.7.2->tensorflowjs==4.22.0->-r requirements.txt (line 2)) (4.1.0)\n",
|
||||
"Requirement already satisfied: PySocks!=1.5.7,>=1.5.6 in /home/asephs/ZeaVis-Edu/.venv/lib/python3.12/site-packages (from requests[socks]->gdown->-r requirements.txt (line 3)) (1.7.1)\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"!pip install -r requirements.txt"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"execution_count": 7,
|
||||
"id": "bad56dcc",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Running locally (TF 2.19.0, GPU: [PhysicalDevice(name='/physical_device:GPU:0', device_type='GPU')])\n",
|
||||
"Setup OK. IMG=(224, 224), BATCH=32\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"import os\n",
|
||||
"import shutil\n",
|
||||
@@ -90,10 +193,18 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"execution_count": 8,
|
||||
"id": "c18354b4",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Dataset ready.\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"if IS_COLAB:\n",
|
||||
" drive.mount('/content/drive')\n",
|
||||
@@ -138,10 +249,20 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"execution_count": 9,
|
||||
"id": "63ee82cc",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Dataset path: /home/asephs/ZeaVis-Edu/Machine_Learning/dataset\n",
|
||||
"Cleaning data...\n",
|
||||
"Done. 0 problematic files removed.\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"# --- Determine dataset path ---\n",
|
||||
"# Colab: zip extracts to dataset/ with dataset_jagung_v1/ subfolder\n",
|
||||
@@ -190,10 +311,40 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"execution_count": 10,
|
||||
"id": "35d34ff6",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Splitting dataset 70:15:15...\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Copying files: 6988 files [00:08, 826.32 files/s] "
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Done.\n",
|
||||
"Train: 4890 | Val: 1046 | Test: 1052\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"if IS_COLAB:\n",
|
||||
" output_dir = \"/content/dataset_split\"\n",
|
||||
@@ -222,6 +373,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "a0ab36ff",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## 5. Data Loader dengan Augmentasi Lanjutan\n",
|
||||
@@ -237,6 +389,7 @@
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "36109306",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
@@ -266,72 +419,104 @@
|
||||
"def mix_up(images, labels, alpha=0.2):\n",
|
||||
" batch_size = tf.shape(images)[0]\n",
|
||||
" lambda_param = sample_beta_distribution(batch_size, alpha, alpha)\n",
|
||||
" lambda_param = tf.reshape(lambda_param, [batch_size, 1, 1, 1])\n",
|
||||
" lambda_param_img = tf.reshape(lambda_param, [batch_size, 1, 1, 1])\n",
|
||||
" random_indices = tf.random.shuffle(tf.range(batch_size))\n",
|
||||
" mixed_images = lambda_param * images + (1 - lambda_param) * tf.gather(images, random_indices)\n",
|
||||
" mixed_images = lambda_param_img * images + (1 - lambda_param_img) * tf.gather(images, random_indices)\n",
|
||||
" labels = tf.cast(labels, tf.float32)\n",
|
||||
" mixed_labels = lambda_param[...,0,0] * labels + (1 - lambda_param[...,0,0]) * tf.gather(labels, random_indices)\n",
|
||||
" lambda_param_lbl = tf.reshape(lambda_param, [-1, 1])\n",
|
||||
" mixed_labels = lambda_param_lbl * labels + (1 - lambda_param_lbl) * tf.gather(labels, random_indices)\n",
|
||||
" return mixed_images, mixed_labels\n",
|
||||
"\n",
|
||||
"# --- CutMix ---\n",
|
||||
"# --- CutMix (batch-level via coordinate-grid broadcasting) ---\n",
|
||||
"def cut_mix(images, labels, alpha=0.2):\n",
|
||||
" batch_size = tf.shape(images)[0]\n",
|
||||
" lambda_param = sample_beta_distribution(batch_size, alpha, alpha)\n",
|
||||
" h = tf.shape(images)[1]\n",
|
||||
" w = tf.shape(images)[2]\n",
|
||||
" \n",
|
||||
" lambda_param = sample_beta_distribution(batch_size, alpha, alpha) # [B]\n",
|
||||
" random_indices = tf.random.shuffle(tf.range(batch_size))\n",
|
||||
" h, w = tf.shape(images)[1], tf.shape(images)[2]\n",
|
||||
" cut_ratio = tf.sqrt(1.0 - lambda_param)\n",
|
||||
" r_h = tf.cast(cut_ratio * tf.cast(h, tf.float32), tf.int32)\n",
|
||||
" r_w = tf.cast(cut_ratio * tf.cast(w, tf.float32), tf.int32)\n",
|
||||
" cx = tf.random.uniform([], 0, w, tf.int32)\n",
|
||||
" cy = tf.random.uniform([], 0, h, tf.int32)\n",
|
||||
" x1 = tf.clip_by_value(cx - r_w // 2, 0, w)\n",
|
||||
" x2 = tf.clip_by_value(cx + r_w // 2, 0, w)\n",
|
||||
" y1 = tf.clip_by_value(cy - r_h // 2, 0, h)\n",
|
||||
" y2 = tf.clip_by_value(cy + r_h // 2, 0, h)\n",
|
||||
" # Create binary mask for the cut region\n",
|
||||
" mask = tf.ones((h, w, 3), dtype=tf.float32)\n",
|
||||
" ones = tf.ones((y2 - y1, x2 - x1, 3), dtype=tf.float32)\n",
|
||||
" mask = tf.tensor_scatter_nd_update(mask, tf.constant([[y1, x1, 0]]), tf.expand_dims(ones[0,0], 0))\n",
|
||||
" # Simpler approach: pad the patch\n",
|
||||
" patch = tf.image.crop_to_bounding_box(tf.gather(images, random_indices), y1, x1, y2 - y1, x2 - x1)\n",
|
||||
" paddings = [[0, 0], [y1, h - y2], [x1, w - x2], [0, 0]]\n",
|
||||
" patch = tf.pad(patch, paddings, constant_values=0)\n",
|
||||
" mixed_images = images * (1 - tf.cast(tf.cast(patch, tf.bool), tf.float32)) + patch\n",
|
||||
" mixed_images = tf.where(tf.cast(patch, tf.bool), patch, images)\n",
|
||||
" \n",
|
||||
" # Per-sample cut sizes (in pixels)\n",
|
||||
" cut_ratio = tf.sqrt(1.0 - lambda_param) # [B]\n",
|
||||
" r_h = tf.cast(cut_ratio * tf.cast(h, tf.float32), tf.int32) # [B]\n",
|
||||
" r_w = tf.cast(cut_ratio * tf.cast(w, tf.float32), tf.int32) # [B]\n",
|
||||
" \n",
|
||||
" # Per-sample random cut centres\n",
|
||||
" cx = tf.random.uniform([batch_size], 0, w, tf.int32) # [B]\n",
|
||||
" cy = tf.random.uniform([batch_size], 0, h, tf.int32) # [B]\n",
|
||||
" \n",
|
||||
" half_h = r_h // 2\n",
|
||||
" half_w = r_w // 2\n",
|
||||
" x1 = tf.clip_by_value(cx - half_w, 0, w) # [B]\n",
|
||||
" x2 = tf.clip_by_value(cx + half_w, 0, w) # [B]\n",
|
||||
" y1 = tf.clip_by_value(cy - half_h, 0, h) # [B]\n",
|
||||
" y2 = tf.clip_by_value(cy + half_h, 0, h) # [B]\n",
|
||||
" \n",
|
||||
" # Build boolean cut mask via coordinate broadcasting\n",
|
||||
" col_idx = tf.range(w, dtype=tf.int32) # [W]\n",
|
||||
" row_idx = tf.range(h, dtype=tf.int32) # [H]\n",
|
||||
" \n",
|
||||
" in_x = tf.logical_and(\n",
|
||||
" tf.reshape(col_idx, [1, 1, w]) >= tf.reshape(x1, [batch_size, 1, 1]),\n",
|
||||
" tf.reshape(col_idx, [1, 1, w]) < tf.reshape(x2, [batch_size, 1, 1])\n",
|
||||
" ) # [B, 1, W]\n",
|
||||
" in_y = tf.logical_and(\n",
|
||||
" tf.reshape(row_idx, [1, h, 1]) >= tf.reshape(y1, [batch_size, 1, 1]),\n",
|
||||
" tf.reshape(row_idx, [1, h, 1]) < tf.reshape(y2, [batch_size, 1, 1])\n",
|
||||
" ) # [B, H, 1]\n",
|
||||
" \n",
|
||||
" cut_mask = tf.cast(tf.logical_and(in_y, in_x), tf.float32) # [B, H, W]\n",
|
||||
" cut_mask = tf.expand_dims(cut_mask, -1) # [B, H, W, 1]\n",
|
||||
" \n",
|
||||
" # Mix images\n",
|
||||
" shuffled = tf.gather(images, random_indices)\n",
|
||||
" mixed_images = (1.0 - cut_mask) * images + cut_mask * shuffled\n",
|
||||
" \n",
|
||||
" # Mix labels\n",
|
||||
" labels = tf.cast(labels, tf.float32)\n",
|
||||
" lambda_reshaped = tf.reshape(lambda_param, [-1, 1])\n",
|
||||
" mixed_labels = lambda_reshaped * labels + (1 - lambda_reshaped) * tf.gather(labels, random_indices)\n",
|
||||
" mixed_labels = lambda_reshaped * labels + (1.0 - lambda_reshaped) * tf.gather(labels, random_indices)\n",
|
||||
" \n",
|
||||
" return mixed_images, mixed_labels\n",
|
||||
"\n",
|
||||
"# --- RandomErasing ---\n",
|
||||
"def random_erasing(images, probability=0.5, scale=(0.02, 0.33), ratio=(0.3, 3.3)):\n",
|
||||
" batch_size = tf.shape(images)[0]\n",
|
||||
" h, w = tf.shape(images)[1], tf.shape(images)[2]\n",
|
||||
" h = tf.shape(images)[1]\n",
|
||||
" w = tf.shape(images)[2]\n",
|
||||
" \n",
|
||||
" target_area = tf.random.uniform([], scale[0], scale[1]) * tf.cast(h * w, tf.float32)\n",
|
||||
" aspect_ratio = tf.random.uniform([], ratio[0], ratio[1])\n",
|
||||
" erasing_h = tf.cast(tf.sqrt(target_area / aspect_ratio), tf.int32)\n",
|
||||
" erasing_w = tf.cast(tf.sqrt(target_area * aspect_ratio), tf.int32)\n",
|
||||
" erasing_h = tf.cast(tf.math.sqrt(target_area / aspect_ratio), tf.int32)\n",
|
||||
" erasing_w = tf.cast(tf.math.sqrt(target_area * aspect_ratio), tf.int32)\n",
|
||||
" erasing_h = tf.clip_by_value(erasing_h, 1, h - 1)\n",
|
||||
" erasing_w = tf.clip_by_value(erasing_w, 1, w - 1)\n",
|
||||
" \n",
|
||||
" cx = tf.random.uniform([], 0, w - erasing_w, tf.int32)\n",
|
||||
" cy = tf.random.uniform([], 0, h - erasing_h, tf.int32)\n",
|
||||
" noise = tf.random.uniform([batch_size, erasing_h, erasing_w, 3], 0, 255, dtype=tf.float32)\n",
|
||||
" mask = tf.ones([batch_size, h, w, 3], dtype=tf.float32)\n",
|
||||
" ones_patch = tf.zeros([batch_size, erasing_h, erasing_w, 3], dtype=tf.float32)\n",
|
||||
" # Scatter the erase region\n",
|
||||
" updates = tf.ones([batch_size, erasing_h * erasing_w * 3], dtype=tf.float32)\n",
|
||||
" # Use simple approach with slicing\n",
|
||||
" result = tf.identity(images)\n",
|
||||
" # Apply noise patch via scatter_nd\n",
|
||||
" batch_indices = tf.reshape(tf.repeat(tf.range(batch_size), erasing_h * erasing_w * 3), [-1, 1])\n",
|
||||
" y_indices = tf.reshape(tf.repeat(tf.range(cy, cy + erasing_h), erasing_w * 3), [-1, 1])\n",
|
||||
" x_indices = tf.reshape(tf.tile(tf.repeat(tf.range(cx, cx + erasing_w), 3), [erasing_h]), [-1, 1])\n",
|
||||
" c_indices = tf.reshape(tf.tile(tf.range(3), [erasing_h * erasing_w]), [-1, 1])\n",
|
||||
" indices = tf.concat([batch_indices, y_indices, x_indices, c_indices], axis=1)\n",
|
||||
" noise_flat = tf.reshape(noise, [-1])\n",
|
||||
" noise_flat = noise_flat[:tf.shape(indices)[0]]\n",
|
||||
" result = tf.tensor_scatter_nd_update(result, indices, noise_flat)\n",
|
||||
" return tf.cond(tf.random.uniform([]) < probability, lambda: result, lambda: images)\n",
|
||||
" \n",
|
||||
" # Build erase mask via coordinate broadcasting (same pattern as CutMix)\n",
|
||||
" col_idx = tf.range(w, dtype=tf.int32)\n",
|
||||
" row_idx = tf.range(h, dtype=tf.int32)\n",
|
||||
" \n",
|
||||
" in_x = tf.logical_and(col_idx >= cx, col_idx < cx + erasing_w) # [W]\n",
|
||||
" in_y = tf.logical_and(row_idx >= cy, row_idx < cy + erasing_h) # [H]\n",
|
||||
" erase_mask = tf.cast(tf.expand_dims(in_y, 1) & tf.expand_dims(in_x, 0), tf.float32) # [H, W]\n",
|
||||
" erase_mask = tf.expand_dims(erase_mask, 0) # [1, H, W]\n",
|
||||
" erase_mask = tf.expand_dims(erase_mask, -1) # [1, H, W, 1]\n",
|
||||
" \n",
|
||||
" noise = tf.random.uniform([batch_size, erasing_h, erasing_w, 3], 0.0, 255.0, dtype=tf.float32)\n",
|
||||
" \n",
|
||||
" # Pad the noise patch to full image size\n",
|
||||
" paddings = [[0, 0], [cy, h - (cy + erasing_h)], [cx, w - (cx + erasing_w)], [0, 0]]\n",
|
||||
" noise_padded = tf.pad(noise, paddings, constant_values=0.0)\n",
|
||||
" \n",
|
||||
" erased = images * (1.0 - erase_mask) + noise_padded * erase_mask\n",
|
||||
" return tf.cond(\n",
|
||||
" tf.random.uniform([]) < probability,\n",
|
||||
" lambda: erased,\n",
|
||||
" lambda: images\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
"# --- Geometric augmentation (kept for TTA compatibility) ---\n",
|
||||
"data_augmentation = tf.keras.Sequential([\n",
|
||||
|
||||
Reference in New Issue
Block a user