In [17]:
%load_ext autoreload
%autoreload 2
The autoreload extension is already loaded. To reload it, use: %reload_ext autoreload
In [18]:
from tensorflow.keras.applications.efficientnet import preprocess_input as efn_preprocess
# Import common modules and set global configs
from project_utils.common_imports import *
# Specific imports for data, augmentation, model, and evaluation
from project_utils.loading_utils import load_and_prepare_data, preprocess_metadata_and_scale, compute_class_weights, compute_oversampling_factors
from project_utils.data_augmentor_and_generator import data_generator_with_metadata, get_steps, SpecTimeMask, SpecFreqMask
from project_utils.model_architect import (
get_model_dispatch, # <-- Use the new unified model dispatcher
macro_f1_fn,
categorical_focal_loss,
AddClsToken,
SafeCheckpointTuner,
)
from project_utils.tuner_utils import run_multimodal_tuner
from project_utils.cv_utils import train_or_cv # <-- Use the new streamlined training/CV function
from project_utils.evaluation_utils import evaluate_on_test_multimodal, plot_training_diagnostics, plot_test_evaluation, CLASS_MAPPING_DICT, evaluate_on_test, get_suffix
# System info
print(f"TensorFlow version: {tf.__version__}")
print(f"GPUs available: {len(tf.config.list_physical_devices('GPU'))}")
os.environ["TF_GPU_ALLOCATOR"] = "cuda_malloc_async"
TensorFlow version: 2.19.0 GPUs available: 1
In [19]:
class CFG:
# ─── Static defaults ────────────────────────────────────────────────────────
seed = 42
debug = False
print_freq = 10000
num_workers = 1
# Parametric settings
model_type = "rescnn" # vit or rescnn
fusion_strategy = "film" # New: 'concat', 'meta_attention', 'film', 'cross_attention', 'bidirectional_attention'
input_dim = 256
train_model = True
use_augmentation = True
use_oversampling = False
use_class_weights = True
use_metadata = True # Set to False for pure audio model
use_transfer_learning = True
use_tuner = False # Set to True to run tuner, False to train with CV or single fold
use_cv = True # Set to True for cross-validation, False for single train/val split
# Training hyperparams
epochs = 30
batch_size = 8
lr = 1.31e-4
weight_decay = 8.12e-6
min_lr = 1e-6
dropout_rate = 0.5
dense_width = 384 # For your dense layers
# Cross-validation
n_fold = 5
# Model / data specifics
in_channels = 1
num_classes = 4
# Augmentation params (adjusted dynamically below)
noise_std = 0.1
max_time_shift = 0.2
max_freq_shift = 0.2
max_mask_time = 0.15
max_mask_freq = 0.15
# Metadata features
metadata_dim = 5 # Will be updated dynamically in loading_utils
# ─── Paths ───────────────────────────────────────────────────────────────────
PROJECT_ROOT_DIR = Path.cwd().parents[2]
OUTPUT_DIR = PROJECT_ROOT_DIR / "data/working"
SPECTROGRAMS_DIR = OUTPUT_DIR / "birdclef25-mel-spectrograms"
working_df_path = PROJECT_ROOT_DIR / "data/raw/downsampled_metadata.csv"
MODELS_DIR = OUTPUT_DIR / "models"
PLOTS_DIR = OUTPUT_DIR / "plots"
RESULTS_CSV_DIR= OUTPUT_DIR / "result_csvs"
# ─── Initialization ──────────────────────────────────────────────────────────
def __init__(self):
# Minimal seed setting
os.environ['PYTHONHASHSEED'] = str(self.seed)
random.seed(self.seed)
np.random.seed(self.seed)
tf.random.set_seed(self.seed)
# 1) Create dirs
for d in (self.MODELS_DIR, self.PLOTS_DIR, self.RESULTS_CSV_DIR):
os.makedirs(d, exist_ok=True)
# 2) Fix augmentation-dependent params
if self.use_augmentation:
self.aug_prob = 1
self.mixup_alpha = 0.2
if self.input_dim == 32:
self.noise_std = 0.065
self.max_time_shift = 0.11
self.max_freq_shift = 0.065
self.max_mask_time = 0.125
self.max_mask_freq = 0.1
elif self.input_dim == 64:
self.noise_std = 0.09
self.max_time_shift = 0.135
self.max_freq_shift = 0.09
self.max_mask_time = 0.175
self.max_mask_freq = 0.135
else: # Fallback for 256
self.noise_std = 0.09
self.max_time_shift = 0.135
self.max_freq_shift = 0.09
self.max_mask_time = 0.175
self.max_mask_freq = 0.135
# 3) Set target shape
self.TARGET_SHAPE = (self.input_dim, self.input_dim)
# ─── Derived properties ─────────────────────────────────────────────────────
@property
def spectrogram_npy(self):
return self.SPECTROGRAMS_DIR / f"birdclef2025_melspec_5sec_{self.input_dim}_{self.input_dim}.npy"
@property
def model_save_name(self):
suffix = ""
if self.use_augmentation: suffix += "_aug"
if self.use_oversampling: suffix += "_os"
if self.use_class_weights: suffix += "_cw"
if self.use_transfer_learning: suffix += "_tl"
if self.use_metadata:
suffix += f"_meta_{self.fusion_strategy}"
if self.model_type.lower() == "vit":
return f"vit_model_{self.input_dim}{suffix}.keras"
elif self.model_type.lower() == "rescnn":
return f"rescnn_model_{self.input_dim}{suffix}.keras"
else:
raise ValueError(f"Unsupported model type: {self.model_type}. Use 'vit' or 'rescnn'.")
cfg = CFG()
In [20]:
f'best_{cfg.model_save_name}'
Out[20]:
'best_rescnn_model_256_aug_cw_tl_meta_film.keras'
In [21]:
# Load pre-computed mel spectrograms
spectrograms = np.load(cfg.spectrogram_npy, allow_pickle=True).item()
print(f"Loaded {len(spectrograms)} pre-computed mel spectrograms")
print(f"Example spectrogram shape: {spectrograms['1139490-CSA36389'].shape}")
Loaded 28564 pre-computed mel spectrograms Example spectrogram shape: (256, 256)
In [22]:
# 1. Load and prepare initial data splits
combined_train_val_df, test_df = load_and_prepare_data(cfg)
# 2. Filter spectrograms to only needed data
needed_keys = set(combined_train_val_df['samplename'].tolist() + test_df['samplename'].tolist())
filtered_spectrograms = {k: spectrograms[k] for k in needed_keys if k in spectrograms}
print(f"Filtered spectrograms to {len(filtered_spectrograms)} entries")
# Manage memory
del spectrograms
gc.collect()
# 3. Create a stable train/validation split for the tuner and final training
# This is crucial for consistent preprocessing
train_df, val_df = train_test_split(
combined_train_val_df,
test_size=0.20,
stratify=combined_train_val_df["y_species_encoded"],
random_state=cfg.seed,
)
Loading full dataset... Train/Val samples: 332 Test samples: 68 Filtered spectrograms to 400 entries
In [23]:
# 4. preprocess and scale metadata
# The scaler is now fit ONLY on train_df and applied to all splits
train_meta, val_meta, test_meta = preprocess_metadata_and_scale(
train_df, val_df, test_df, cfg
)
# 5. Compute class weights and oversampling factors ONLY from the training data
class_weights_dict = compute_class_weights(train_df, use_class_weights=cfg.use_class_weights)
cfg.oversampling_factors = compute_oversampling_factors(train_df)
class_weights = class_weights_dict if cfg.use_class_weights else None
Preprocessing and scaling metadata correctly...
Scaler fit on training data and applied to all splits.
Metadata feature dimension: 5
Computing balanced class weights...
Final class weights: {0: 0.9601449275362319, 1: 0.9888059701492538, 2: 1.0192307692307692, 3: 1.03515625}
Class counts in train/val: {0: 69, 1: 67, 2: 65, 3: 64}
Computed oversampling factors: {0: 1, 1: 1, 2: 1, 3: 1}
In [24]:
if cfg.use_tuner:
# We use the pre-defined train_df and val_df
best_model, tuner = run_multimodal_tuner(
cfg, train_df, val_df,
filtered_spectrograms,
{'train': train_meta, 'val': val_meta}, # Pass metadata as a dict
class_weights,
)
best_model.summary()
plot_model(best_model, show_shapes=True, expand_nested=True, dpi=65)
hp_json_path = cfg.OUTPUT_DIR / f'{cfg.model_save_name.replace(".keras", "")}_tuner' / f"best_hps_{cfg.model_save_name.replace('.keras', '')}.json"
# ② Pretty-print them
print("Best hyper-parameters:")
best_hp = tuner.get_best_hyperparameters(num_trials=1)[0] # <- real H-Ps
for name, val in best_hp.values.items():
print(f" {name}: {val}")
# Now access the best metric as above
best_trial = tuner.oracle.get_best_trials(num_trials=1)[0]
best_val_metric = best_trial.score
print(f"\nBest val_macro_f1_fn: {best_val_metric}")
In [25]:
# --- Train Model (Single Fold or Cross-Validation) ---
# This block now robustly handles hyperparameter loading.
# It attempts to load the best HPs from a tuner file if cfg.use_tuner is True.
# If the file is not found, it falls back to the HPs defined in the CFG object.
# Construct the path to the best hyperparameters file
hp_json_path = None
if cfg.use_tuner:
hp_json_path = cfg.OUTPUT_DIR / f'{cfg.model_save_name.replace(".keras", "")}_tuner' / f"best_hps_{cfg.model_save_name.replace('.keras', '')}.json"
# Check if the file actually exists before setting the path for the function
if not hp_json_path.exists():
print(f"⚠️ Tuner file not found at {hp_json_path}")
print("Falling back to hyperparameters defined in CFG.")
hp_json_path = None
if cfg.train_model:
# The train_or_cv function will use the logic in cv_utils.py
# to either run a single fold or a full cross-validation.
history_dict, metrics_list = train_or_cv(
cfg=cfg,
full_trainval_df=combined_train_val_df,
spectrograms=filtered_spectrograms,
class_weights=class_weights_dict,
best_hp_path=None # Pass the path, which will be None if not found
)
[INFO] No best_hp_path provided for CV, using hyperparameters from config [INFO] No hyperparameter file provided, using config defaults [INFO] Selected grouping strategy: hierarchical [INFO] Using parsed samplename for grouping [INFO] Group-class distribution shape: (332, 4) [SUCCESS] Using StratifiedGroupKFold with 332 groups [INFO] Using StratifiedGroupKFold (hierarchical) for 5-fold CV ================================================== FOLD 1/5 ================================================== [FOLD 1 DIAGNOSTICS] Train size: 265, Val size: 67 Val class distribution: y_species_encoded 0 0.284 1 0.254 2 0.254 3 0.209 Name: proportion, dtype: float64 Epoch 1/30 34/34 ━━━━━━━━━━━━━━━━━━━━ 14s 132ms/step - accuracy: 0.2451 - loss: 0.2226 - macro_f1_fn: 0.1548 - val_accuracy: 0.2500 - val_loss: 0.1946 - val_macro_f1_fn: 0.0929 Epoch 2/30 34/34 ━━━━━━━━━━━━━━━━━━━━ 2s 58ms/step - accuracy: 0.3586 - loss: 0.1684 - macro_f1_fn: 0.2988 - val_accuracy: 0.2917 - val_loss: 0.1959 - val_macro_f1_fn: 0.1103 Epoch 3/30 34/34 ━━━━━━━━━━━━━━━━━━━━ 2s 50ms/step - accuracy: 0.3722 - loss: 0.1595 - macro_f1_fn: 0.3061 - val_accuracy: 0.2500 - val_loss: 0.2144 - val_macro_f1_fn: 0.0982 Epoch 4/30 34/34 ━━━━━━━━━━━━━━━━━━━━ 2s 62ms/step - accuracy: 0.4294 - loss: 0.1444 - macro_f1_fn: 0.3582 - val_accuracy: 0.3056 - val_loss: 0.1764 - val_macro_f1_fn: 0.1435 Epoch 5/30 34/34 ━━━━━━━━━━━━━━━━━━━━ 2s 53ms/step - accuracy: 0.5628 - loss: 0.1257 - macro_f1_fn: 0.4835 - val_accuracy: 0.5000 - val_loss: 0.1625 - val_macro_f1_fn: 0.3509 Epoch 6/30 34/34 ━━━━━━━━━━━━━━━━━━━━ 1s 30ms/step - accuracy: 0.5467 - loss: 0.1255 - macro_f1_fn: 0.4895 - val_accuracy: 0.3056 - val_loss: 0.1673 - val_macro_f1_fn: 0.1265 Epoch 7/30 34/34 ━━━━━━━━━━━━━━━━━━━━ 2s 52ms/step - accuracy: 0.5150 - loss: 0.1232 - macro_f1_fn: 0.4409 - val_accuracy: 0.5139 - val_loss: 0.1391 - val_macro_f1_fn: 0.3709 Epoch 8/30 34/34 ━━━━━━━━━━━━━━━━━━━━ 2s 52ms/step - accuracy: 0.6175 - loss: 0.1147 - macro_f1_fn: 0.5181 - val_accuracy: 0.5972 - val_loss: 0.1489 - val_macro_f1_fn: 0.5127 Epoch 9/30 34/34 ━━━━━━━━━━━━━━━━━━━━ 2s 54ms/step - accuracy: 0.5745 - loss: 0.1218 - macro_f1_fn: 0.4838 - val_accuracy: 0.5139 - val_loss: 0.1570 - val_macro_f1_fn: 0.3672 Epoch 10/30 34/34 ━━━━━━━━━━━━━━━━━━━━ 1s 30ms/step - accuracy: 0.5642 - loss: 0.1310 - macro_f1_fn: 0.4843 - val_accuracy: 0.4167 - val_loss: 0.1532 - val_macro_f1_fn: 0.3293 Epoch 11/30 34/34 ━━━━━━━━━━━━━━━━━━━━ 1s 30ms/step - accuracy: 0.5522 - loss: 0.1186 - macro_f1_fn: 0.4659 - val_accuracy: 0.5556 - val_loss: 0.1409 - val_macro_f1_fn: 0.4757 Epoch 12/30 34/34 ━━━━━━━━━━━━━━━━━━━━ 1s 30ms/step - accuracy: 0.6402 - loss: 0.1060 - macro_f1_fn: 0.5531 - val_accuracy: 0.3889 - val_loss: 0.1417 - val_macro_f1_fn: 0.3052 Epoch 13/30 34/34 ━━━━━━━━━━━━━━━━━━━━ 2s 60ms/step - accuracy: 0.6030 - loss: 0.0984 - macro_f1_fn: 0.5467 - val_accuracy: 0.7500 - val_loss: 0.1156 - val_macro_f1_fn: 0.6637 Epoch 14/30 34/34 ━━━━━━━━━━━━━━━━━━━━ 1s 28ms/step - accuracy: 0.5495 - loss: 0.1129 - macro_f1_fn: 0.4898 - val_accuracy: 0.7500 - val_loss: 0.1243 - val_macro_f1_fn: 0.6078 Epoch 15/30 34/34 ━━━━━━━━━━━━━━━━━━━━ 2s 54ms/step - accuracy: 0.6669 - loss: 0.0922 - macro_f1_fn: 0.5819 - val_accuracy: 0.7500 - val_loss: 0.1272 - val_macro_f1_fn: 0.7009 Epoch 16/30 34/34 ━━━━━━━━━━━━━━━━━━━━ 1s 29ms/step - accuracy: 0.6272 - loss: 0.1004 - macro_f1_fn: 0.5558 - val_accuracy: 0.7222 - val_loss: 0.1060 - val_macro_f1_fn: 0.6440 Epoch 17/30 34/34 ━━━━━━━━━━━━━━━━━━━━ 1s 29ms/step - accuracy: 0.6667 - loss: 0.1105 - macro_f1_fn: 0.6183 - val_accuracy: 0.6806 - val_loss: 0.1231 - val_macro_f1_fn: 0.5817 Epoch 18/30 34/34 ━━━━━━━━━━━━━━━━━━━━ 1s 29ms/step - accuracy: 0.6820 - loss: 0.0947 - macro_f1_fn: 0.5914 - val_accuracy: 0.6944 - val_loss: 0.1162 - val_macro_f1_fn: 0.5749 Epoch 19/30 34/34 ━━━━━━━━━━━━━━━━━━━━ 2s 56ms/step - accuracy: 0.6091 - loss: 0.1045 - macro_f1_fn: 0.5416 - val_accuracy: 0.8333 - val_loss: 0.0962 - val_macro_f1_fn: 0.7341 Epoch 20/30 34/34 ━━━━━━━━━━━━━━━━━━━━ 1s 29ms/step - accuracy: 0.6026 - loss: 0.1110 - macro_f1_fn: 0.5812 - val_accuracy: 0.6806 - val_loss: 0.1291 - val_macro_f1_fn: 0.5923 Epoch 21/30 34/34 ━━━━━━━━━━━━━━━━━━━━ 1s 29ms/step - accuracy: 0.6254 - loss: 0.1098 - macro_f1_fn: 0.5479 - val_accuracy: 0.6528 - val_loss: 0.1144 - val_macro_f1_fn: 0.5913 Epoch 22/30 34/34 ━━━━━━━━━━━━━━━━━━━━ 1s 29ms/step - accuracy: 0.6208 - loss: 0.1153 - macro_f1_fn: 0.5366 - val_accuracy: 0.7917 - val_loss: 0.1100 - val_macro_f1_fn: 0.7126 Epoch 23/30 34/34 ━━━━━━━━━━━━━━━━━━━━ 1s 29ms/step - accuracy: 0.6438 - loss: 0.0895 - macro_f1_fn: 0.6030 - val_accuracy: 0.7778 - val_loss: 0.1087 - val_macro_f1_fn: 0.6950 Epoch 24/30 34/34 ━━━━━━━━━━━━━━━━━━━━ 1s 29ms/step - accuracy: 0.5999 - loss: 0.1219 - macro_f1_fn: 0.5159 - val_accuracy: 0.7778 - val_loss: 0.1156 - val_macro_f1_fn: 0.6843 Epoch 25/30 34/34 ━━━━━━━━━━━━━━━━━━━━ 1s 29ms/step - accuracy: 0.6361 - loss: 0.1045 - macro_f1_fn: 0.5963 - val_accuracy: 0.7639 - val_loss: 0.1080 - val_macro_f1_fn: 0.6680 Epoch 26/30 34/34 ━━━━━━━━━━━━━━━━━━━━ 1s 30ms/step - accuracy: 0.6769 - loss: 0.0907 - macro_f1_fn: 0.6534 - val_accuracy: 0.7500 - val_loss: 0.1195 - val_macro_f1_fn: 0.6733 Epoch 27/30 34/34 ━━━━━━━━━━━━━━━━━━━━ 1s 30ms/step - accuracy: 0.6857 - loss: 0.0987 - macro_f1_fn: 0.6294 - val_accuracy: 0.6667 - val_loss: 0.1256 - val_macro_f1_fn: 0.5599 ================================================== FOLD 2/5 ================================================== [FOLD 2 DIAGNOSTICS] Train size: 266, Val size: 66 Val class distribution: y_species_encoded 0 0.258 3 0.258 1 0.242 2 0.242 Name: proportion, dtype: float64 Epoch 1/30 34/34 ━━━━━━━━━━━━━━━━━━━━ 14s 127ms/step - accuracy: 0.2495 - loss: 0.2191 - macro_f1_fn: 0.1590 - val_accuracy: 0.2361 - val_loss: 0.2012 - val_macro_f1_fn: 0.0908 Epoch 2/30 34/34 ━━━━━━━━━━━━━━━━━━━━ 2s 54ms/step - accuracy: 0.3415 - loss: 0.1690 - macro_f1_fn: 0.3110 - val_accuracy: 0.2500 - val_loss: 0.2082 - val_macro_f1_fn: 0.0952 Epoch 3/30 34/34 ━━━━━━━━━━━━━━━━━━━━ 1s 31ms/step - accuracy: 0.3333 - loss: 0.1754 - macro_f1_fn: 0.2659 - val_accuracy: 0.2222 - val_loss: 0.2015 - val_macro_f1_fn: 0.0862 Epoch 4/30 34/34 ━━━━━━━━━━━━━━━━━━━━ 2s 55ms/step - accuracy: 0.4256 - loss: 0.1482 - macro_f1_fn: 0.3332 - val_accuracy: 0.3333 - val_loss: 0.2034 - val_macro_f1_fn: 0.2216 Epoch 5/30 34/34 ━━━━━━━━━━━━━━━━━━━━ 2s 54ms/step - accuracy: 0.4912 - loss: 0.1404 - macro_f1_fn: 0.4263 - val_accuracy: 0.5694 - val_loss: 0.1725 - val_macro_f1_fn: 0.4366 Epoch 6/30 34/34 ━━━━━━━━━━━━━━━━━━━━ 2s 51ms/step - accuracy: 0.5168 - loss: 0.1349 - macro_f1_fn: 0.4494 - val_accuracy: 0.5556 - val_loss: 0.1674 - val_macro_f1_fn: 0.4670 Epoch 7/30 34/34 ━━━━━━━━━━━━━━━━━━━━ 1s 28ms/step - accuracy: 0.5237 - loss: 0.1310 - macro_f1_fn: 0.4658 - val_accuracy: 0.4167 - val_loss: 0.1690 - val_macro_f1_fn: 0.2654 Epoch 8/30 34/34 ━━━━━━━━━━━━━━━━━━━━ 1s 38ms/step - accuracy: 0.5931 - loss: 0.1218 - macro_f1_fn: 0.5011 - val_accuracy: 0.4583 - val_loss: 0.1673 - val_macro_f1_fn: 0.2669 Epoch 9/30 34/34 ━━━━━━━━━━━━━━━━━━━━ 1s 29ms/step - accuracy: 0.6161 - loss: 0.1172 - macro_f1_fn: 0.5566 - val_accuracy: 0.5000 - val_loss: 0.1621 - val_macro_f1_fn: 0.3825 Epoch 10/30 34/34 ━━━━━━━━━━━━━━━━━━━━ 2s 61ms/step - accuracy: 0.6635 - loss: 0.0965 - macro_f1_fn: 0.5533 - val_accuracy: 0.6389 - val_loss: 0.1457 - val_macro_f1_fn: 0.5272 Epoch 11/30 34/34 ━━━━━━━━━━━━━━━━━━━━ 1s 29ms/step - accuracy: 0.6041 - loss: 0.1110 - macro_f1_fn: 0.5412 - val_accuracy: 0.4722 - val_loss: 0.1613 - val_macro_f1_fn: 0.3687 Epoch 12/30 34/34 ━━━━━━━━━━━━━━━━━━━━ 2s 51ms/step - accuracy: 0.6680 - loss: 0.0948 - macro_f1_fn: 0.5828 - val_accuracy: 0.7222 - val_loss: 0.1158 - val_macro_f1_fn: 0.5676 Epoch 13/30 34/34 ━━━━━━━━━━━━━━━━━━━━ 2s 51ms/step - accuracy: 0.6699 - loss: 0.1158 - macro_f1_fn: 0.6090 - val_accuracy: 0.7083 - val_loss: 0.1204 - val_macro_f1_fn: 0.6319 Epoch 14/30 34/34 ━━━━━━━━━━━━━━━━━━━━ 1s 30ms/step - accuracy: 0.6653 - loss: 0.1108 - macro_f1_fn: 0.6192 - val_accuracy: 0.6111 - val_loss: 0.1397 - val_macro_f1_fn: 0.5428 Epoch 15/30 34/34 ━━━━━━━━━━━━━━━━━━━━ 1s 30ms/step - accuracy: 0.7565 - loss: 0.0933 - macro_f1_fn: 0.6540 - val_accuracy: 0.7500 - val_loss: 0.1071 - val_macro_f1_fn: 0.6279 Epoch 16/30 34/34 ━━━━━━━━━━━━━━━━━━━━ 1s 30ms/step - accuracy: 0.6946 - loss: 0.1082 - macro_f1_fn: 0.6325 - val_accuracy: 0.6250 - val_loss: 0.1299 - val_macro_f1_fn: 0.5404 Epoch 17/30 34/34 ━━━━━━━━━━━━━━━━━━━━ 2s 53ms/step - accuracy: 0.6788 - loss: 0.1072 - macro_f1_fn: 0.6290 - val_accuracy: 0.7361 - val_loss: 0.1002 - val_macro_f1_fn: 0.6400 Epoch 18/30 34/34 ━━━━━━━━━━━━━━━━━━━━ 1s 30ms/step - accuracy: 0.6537 - loss: 0.1084 - macro_f1_fn: 0.5793 - val_accuracy: 0.7083 - val_loss: 0.1168 - val_macro_f1_fn: 0.5843 Epoch 19/30 34/34 ━━━━━━━━━━━━━━━━━━━━ 1s 29ms/step - accuracy: 0.7143 - loss: 0.0997 - macro_f1_fn: 0.6421 - val_accuracy: 0.6667 - val_loss: 0.1126 - val_macro_f1_fn: 0.5688 Epoch 20/30 34/34 ━━━━━━━━━━━━━━━━━━━━ 1s 37ms/step - accuracy: 0.6714 - loss: 0.1061 - macro_f1_fn: 0.5925 - val_accuracy: 0.7222 - val_loss: 0.1127 - val_macro_f1_fn: 0.6363 Epoch 21/30 34/34 ━━━━━━━━━━━━━━━━━━━━ 2s 51ms/step - accuracy: 0.6702 - loss: 0.0949 - macro_f1_fn: 0.6314 - val_accuracy: 0.7500 - val_loss: 0.0980 - val_macro_f1_fn: 0.6622 Epoch 22/30 34/34 ━━━━━━━━━━━━━━━━━━━━ 1s 28ms/step - accuracy: 0.7500 - loss: 0.0947 - macro_f1_fn: 0.6787 - val_accuracy: 0.6806 - val_loss: 0.1099 - val_macro_f1_fn: 0.5930 Epoch 23/30 34/34 ━━━━━━━━━━━━━━━━━━━━ 1s 30ms/step - accuracy: 0.6847 - loss: 0.0993 - macro_f1_fn: 0.5864 - val_accuracy: 0.6667 - val_loss: 0.1178 - val_macro_f1_fn: 0.5267 Epoch 24/30 34/34 ━━━━━━━━━━━━━━━━━━━━ 1s 38ms/step - accuracy: 0.6470 - loss: 0.0998 - macro_f1_fn: 0.5902 - val_accuracy: 0.7083 - val_loss: 0.1052 - val_macro_f1_fn: 0.6058 Epoch 25/30 34/34 ━━━━━━━━━━━━━━━━━━━━ 1s 38ms/step - accuracy: 0.6827 - loss: 0.1011 - macro_f1_fn: 0.6051 - val_accuracy: 0.6944 - val_loss: 0.1059 - val_macro_f1_fn: 0.6259 Epoch 26/30 34/34 ━━━━━━━━━━━━━━━━━━━━ 1s 38ms/step - accuracy: 0.7356 - loss: 0.0976 - macro_f1_fn: 0.6647 - val_accuracy: 0.6806 - val_loss: 0.1036 - val_macro_f1_fn: 0.6320 Epoch 27/30 34/34 ━━━━━━━━━━━━━━━━━━━━ 1s 29ms/step - accuracy: 0.6504 - loss: 0.0949 - macro_f1_fn: 0.5768 - val_accuracy: 0.6667 - val_loss: 0.1132 - val_macro_f1_fn: 0.5679 Epoch 28/30 34/34 ━━━━━━━━━━━━━━━━━━━━ 1s 38ms/step - accuracy: 0.7244 - loss: 0.0947 - macro_f1_fn: 0.6659 - val_accuracy: 0.6806 - val_loss: 0.1140 - val_macro_f1_fn: 0.5869 Epoch 29/30 34/34 ━━━━━━━━━━━━━━━━━━━━ 1s 32ms/step - accuracy: 0.7244 - loss: 0.0990 - macro_f1_fn: 0.6507 - val_accuracy: 0.6806 - val_loss: 0.1084 - val_macro_f1_fn: 0.5473 ================================================== FOLD 3/5 ================================================== [FOLD 3 DIAGNOSTICS] Train size: 265, Val size: 67 Val class distribution: y_species_encoded 0 0.284 1 0.254 3 0.254 2 0.209 Name: proportion, dtype: float64 Epoch 1/30 34/34 ━━━━━━━━━━━━━━━━━━━━ 14s 132ms/step - accuracy: 0.2820 - loss: 0.2036 - macro_f1_fn: 0.1992 - val_accuracy: 0.2361 - val_loss: 0.2086 - val_macro_f1_fn: 0.0911 Epoch 2/30 34/34 ━━━━━━━━━━━━━━━━━━━━ 2s 55ms/step - accuracy: 0.3253 - loss: 0.1706 - macro_f1_fn: 0.2708 - val_accuracy: 0.1806 - val_loss: 0.2174 - val_macro_f1_fn: 0.1207 Epoch 3/30 34/34 ━━━━━━━━━━━━━━━━━━━━ 2s 52ms/step - accuracy: 0.3739 - loss: 0.1699 - macro_f1_fn: 0.3091 - val_accuracy: 0.2083 - val_loss: 0.1999 - val_macro_f1_fn: 0.0752 Epoch 4/30 34/34 ━━━━━━━━━━━━━━━━━━━━ 2s 53ms/step - accuracy: 0.5279 - loss: 0.1346 - macro_f1_fn: 0.4241 - val_accuracy: 0.4583 - val_loss: 0.1807 - val_macro_f1_fn: 0.2715 Epoch 5/30 34/34 ━━━━━━━━━━━━━━━━━━━━ 1s 30ms/step - accuracy: 0.4890 - loss: 0.1364 - macro_f1_fn: 0.4158 - val_accuracy: 0.2917 - val_loss: 0.1823 - val_macro_f1_fn: 0.1866 Epoch 6/30 34/34 ━━━━━━━━━━━━━━━━━━━━ 1s 29ms/step - accuracy: 0.5253 - loss: 0.1378 - macro_f1_fn: 0.4146 - val_accuracy: 0.2500 - val_loss: 0.1937 - val_macro_f1_fn: 0.1598 Epoch 7/30 34/34 ━━━━━━━━━━━━━━━━━━━━ 2s 53ms/step - accuracy: 0.5957 - loss: 0.1136 - macro_f1_fn: 0.5459 - val_accuracy: 0.5833 - val_loss: 0.1529 - val_macro_f1_fn: 0.4360 Epoch 8/30 34/34 ━━━━━━━━━━━━━━━━━━━━ 2s 54ms/step - accuracy: 0.5985 - loss: 0.0998 - macro_f1_fn: 0.5039 - val_accuracy: 0.4444 - val_loss: 0.1453 - val_macro_f1_fn: 0.3277 Epoch 9/30 34/34 ━━━━━━━━━━━━━━━━━━━━ 1s 30ms/step - accuracy: 0.6590 - loss: 0.1003 - macro_f1_fn: 0.5873 - val_accuracy: 0.5139 - val_loss: 0.1516 - val_macro_f1_fn: 0.3864 Epoch 10/30 34/34 ━━━━━━━━━━━━━━━━━━━━ 2s 61ms/step - accuracy: 0.6464 - loss: 0.1029 - macro_f1_fn: 0.5581 - val_accuracy: 0.6389 - val_loss: 0.1324 - val_macro_f1_fn: 0.5831 Epoch 11/30 34/34 ━━━━━━━━━━━━━━━━━━━━ 2s 51ms/step - accuracy: 0.6392 - loss: 0.1090 - macro_f1_fn: 0.5620 - val_accuracy: 0.7083 - val_loss: 0.1098 - val_macro_f1_fn: 0.6035 Epoch 12/30 34/34 ━━━━━━━━━━━━━━━━━━━━ 1s 30ms/step - accuracy: 0.6187 - loss: 0.1086 - macro_f1_fn: 0.5228 - val_accuracy: 0.6667 - val_loss: 0.1097 - val_macro_f1_fn: 0.5671 Epoch 13/30 34/34 ━━━━━━━━━━━━━━━━━━━━ 1s 38ms/step - accuracy: 0.6543 - loss: 0.1044 - macro_f1_fn: 0.5694 - val_accuracy: 0.6667 - val_loss: 0.1134 - val_macro_f1_fn: 0.5431 Epoch 14/30 34/34 ━━━━━━━━━━━━━━━━━━━━ 2s 51ms/step - accuracy: 0.6728 - loss: 0.1018 - macro_f1_fn: 0.5922 - val_accuracy: 0.7222 - val_loss: 0.0981 - val_macro_f1_fn: 0.6124 Epoch 15/30 34/34 ━━━━━━━━━━━━━━━━━━━━ 2s 50ms/step - accuracy: 0.6327 - loss: 0.1108 - macro_f1_fn: 0.5901 - val_accuracy: 0.7361 - val_loss: 0.1030 - val_macro_f1_fn: 0.6353 Epoch 16/30 34/34 ━━━━━━━━━━━━━━━━━━━━ 2s 52ms/step - accuracy: 0.6330 - loss: 0.0968 - macro_f1_fn: 0.5589 - val_accuracy: 0.7083 - val_loss: 0.0967 - val_macro_f1_fn: 0.6573 Epoch 17/30 34/34 ━━━━━━━━━━━━━━━━━━━━ 3s 77ms/step - accuracy: 0.5368 - loss: 0.1235 - macro_f1_fn: 0.4634 - val_accuracy: 0.7917 - val_loss: 0.0908 - val_macro_f1_fn: 0.7226 Epoch 18/30 34/34 ━━━━━━━━━━━━━━━━━━━━ 1s 29ms/step - accuracy: 0.6757 - loss: 0.0967 - macro_f1_fn: 0.5871 - val_accuracy: 0.7083 - val_loss: 0.1058 - val_macro_f1_fn: 0.6483 Epoch 19/30 34/34 ━━━━━━━━━━━━━━━━━━━━ 1s 29ms/step - accuracy: 0.6845 - loss: 0.1038 - macro_f1_fn: 0.5933 - val_accuracy: 0.7639 - val_loss: 0.0934 - val_macro_f1_fn: 0.6673 Epoch 20/30 34/34 ━━━━━━━━━━━━━━━━━━━━ 1s 28ms/step - accuracy: 0.6480 - loss: 0.0970 - macro_f1_fn: 0.5638 - val_accuracy: 0.8194 - val_loss: 0.0833 - val_macro_f1_fn: 0.6862 Epoch 21/30 34/34 ━━━━━━━━━━━━━━━━━━━━ 2s 51ms/step - accuracy: 0.6443 - loss: 0.1027 - macro_f1_fn: 0.5861 - val_accuracy: 0.7917 - val_loss: 0.1008 - val_macro_f1_fn: 0.7302 Epoch 22/30 34/34 ━━━━━━━━━━━━━━━━━━━━ 1s 28ms/step - accuracy: 0.6883 - loss: 0.1047 - macro_f1_fn: 0.5864 - val_accuracy: 0.7639 - val_loss: 0.0998 - val_macro_f1_fn: 0.6471 Epoch 23/30 34/34 ━━━━━━━━━━━━━━━━━━━━ 2s 50ms/step - accuracy: 0.5862 - loss: 0.1153 - macro_f1_fn: 0.5067 - val_accuracy: 0.8333 - val_loss: 0.0762 - val_macro_f1_fn: 0.7719 Epoch 24/30 34/34 ━━━━━━━━━━━━━━━━━━━━ 1s 30ms/step - accuracy: 0.7077 - loss: 0.1009 - macro_f1_fn: 0.6164 - val_accuracy: 0.8056 - val_loss: 0.0901 - val_macro_f1_fn: 0.7455 Epoch 25/30 34/34 ━━━━━━━━━━━━━━━━━━━━ 1s 30ms/step - accuracy: 0.6803 - loss: 0.0925 - macro_f1_fn: 0.6215 - val_accuracy: 0.7361 - val_loss: 0.1042 - val_macro_f1_fn: 0.6319 Epoch 26/30 34/34 ━━━━━━━━━━━━━━━━━━━━ 1s 31ms/step - accuracy: 0.6161 - loss: 0.1107 - macro_f1_fn: 0.5494 - val_accuracy: 0.8611 - val_loss: 0.0701 - val_macro_f1_fn: 0.7341 Epoch 27/30 34/34 ━━━━━━━━━━━━━━━━━━━━ 1s 31ms/step - accuracy: 0.6462 - loss: 0.1012 - macro_f1_fn: 0.5427 - val_accuracy: 0.7639 - val_loss: 0.0957 - val_macro_f1_fn: 0.6548 Epoch 28/30 34/34 ━━━━━━━━━━━━━━━━━━━━ 1s 30ms/step - accuracy: 0.6954 - loss: 0.1044 - macro_f1_fn: 0.6301 - val_accuracy: 0.7917 - val_loss: 0.0877 - val_macro_f1_fn: 0.6973 Epoch 29/30 34/34 ━━━━━━━━━━━━━━━━━━━━ 1s 29ms/step - accuracy: 0.6822 - loss: 0.0971 - macro_f1_fn: 0.5977 - val_accuracy: 0.8611 - val_loss: 0.0769 - val_macro_f1_fn: 0.7656 Epoch 30/30 34/34 ━━━━━━━━━━━━━━━━━━━━ 1s 28ms/step - accuracy: 0.6711 - loss: 0.1025 - macro_f1_fn: 0.6089 - val_accuracy: 0.7639 - val_loss: 0.0900 - val_macro_f1_fn: 0.6193 ================================================== FOLD 4/5 ================================================== [FOLD 4 DIAGNOSTICS] Train size: 266, Val size: 66 Val class distribution: y_species_encoded 1 0.258 2 0.258 0 0.242 3 0.242 Name: proportion, dtype: float64 Epoch 1/30 34/34 ━━━━━━━━━━━━━━━━━━━━ 14s 131ms/step - accuracy: 0.2329 - loss: 0.2171 - macro_f1_fn: 0.1792 - val_accuracy: 0.2500 - val_loss: 0.2076 - val_macro_f1_fn: 0.0961 Epoch 2/30 34/34 ━━━━━━━━━━━━━━━━━━━━ 2s 54ms/step - accuracy: 0.3127 - loss: 0.2053 - macro_f1_fn: 0.2509 - val_accuracy: 0.3056 - val_loss: 0.1970 - val_macro_f1_fn: 0.1878 Epoch 3/30 34/34 ━━━━━━━━━━━━━━━━━━━━ 1s 31ms/step - accuracy: 0.4161 - loss: 0.1507 - macro_f1_fn: 0.3323 - val_accuracy: 0.2361 - val_loss: 0.1852 - val_macro_f1_fn: 0.1123 Epoch 4/30 34/34 ━━━━━━━━━━━━━━━━━━━━ 2s 52ms/step - accuracy: 0.4589 - loss: 0.1461 - macro_f1_fn: 0.4186 - val_accuracy: 0.4861 - val_loss: 0.1740 - val_macro_f1_fn: 0.3056 Epoch 5/30 34/34 ━━━━━━━━━━━━━━━━━━━━ 1s 29ms/step - accuracy: 0.4358 - loss: 0.1504 - macro_f1_fn: 0.3705 - val_accuracy: 0.3750 - val_loss: 0.1775 - val_macro_f1_fn: 0.2491 Epoch 6/30 34/34 ━━━━━━━━━━━━━━━━━━━━ 1s 30ms/step - accuracy: 0.4927 - loss: 0.1402 - macro_f1_fn: 0.4529 - val_accuracy: 0.3750 - val_loss: 0.1564 - val_macro_f1_fn: 0.2027 Epoch 7/30 34/34 ━━━━━━━━━━━━━━━━━━━━ 2s 54ms/step - accuracy: 0.5829 - loss: 0.1225 - macro_f1_fn: 0.4593 - val_accuracy: 0.4722 - val_loss: 0.1457 - val_macro_f1_fn: 0.3372 Epoch 8/30 34/34 ━━━━━━━━━━━━━━━━━━━━ 2s 51ms/step - accuracy: 0.5705 - loss: 0.1154 - macro_f1_fn: 0.5205 - val_accuracy: 0.5833 - val_loss: 0.1409 - val_macro_f1_fn: 0.4551 Epoch 9/30 34/34 ━━━━━━━━━━━━━━━━━━━━ 2s 51ms/step - accuracy: 0.5912 - loss: 0.1279 - macro_f1_fn: 0.5644 - val_accuracy: 0.7778 - val_loss: 0.1268 - val_macro_f1_fn: 0.5518 Epoch 10/30 34/34 ━━━━━━━━━━━━━━━━━━━━ 2s 54ms/step - accuracy: 0.5730 - loss: 0.1180 - macro_f1_fn: 0.5136 - val_accuracy: 0.5278 - val_loss: 0.1241 - val_macro_f1_fn: 0.4018 Epoch 11/30 34/34 ━━━━━━━━━━━━━━━━━━━━ 1s 30ms/step - accuracy: 0.5670 - loss: 0.1195 - macro_f1_fn: 0.5041 - val_accuracy: 0.6528 - val_loss: 0.1212 - val_macro_f1_fn: 0.5298 Epoch 12/30 34/34 ━━━━━━━━━━━━━━━━━━━━ 2s 52ms/step - accuracy: 0.5924 - loss: 0.1054 - macro_f1_fn: 0.5246 - val_accuracy: 0.7500 - val_loss: 0.1096 - val_macro_f1_fn: 0.5781 Epoch 13/30 34/34 ━━━━━━━━━━━━━━━━━━━━ 2s 53ms/step - accuracy: 0.6825 - loss: 0.1005 - macro_f1_fn: 0.6136 - val_accuracy: 0.6250 - val_loss: 0.1159 - val_macro_f1_fn: 0.4863 Epoch 14/30 34/34 ━━━━━━━━━━━━━━━━━━━━ 1s 37ms/step - accuracy: 0.6675 - loss: 0.0965 - macro_f1_fn: 0.6102 - val_accuracy: 0.6250 - val_loss: 0.1247 - val_macro_f1_fn: 0.5076 Epoch 15/30 34/34 ━━━━━━━━━━━━━━━━━━━━ 1s 30ms/step - accuracy: 0.7112 - loss: 0.0933 - macro_f1_fn: 0.5926 - val_accuracy: 0.6944 - val_loss: 0.1157 - val_macro_f1_fn: 0.5622 Epoch 16/30 34/34 ━━━━━━━━━━━━━━━━━━━━ 2s 52ms/step - accuracy: 0.6833 - loss: 0.0949 - macro_f1_fn: 0.6326 - val_accuracy: 0.7361 - val_loss: 0.1080 - val_macro_f1_fn: 0.5967 Epoch 17/30 34/34 ━━━━━━━━━━━━━━━━━━━━ 2s 52ms/step - accuracy: 0.6185 - loss: 0.1136 - macro_f1_fn: 0.5491 - val_accuracy: 0.7639 - val_loss: 0.1021 - val_macro_f1_fn: 0.5987 Epoch 18/30 34/34 ━━━━━━━━━━━━━━━━━━━━ 2s 51ms/step - accuracy: 0.5913 - loss: 0.1139 - macro_f1_fn: 0.5358 - val_accuracy: 0.7639 - val_loss: 0.1024 - val_macro_f1_fn: 0.6376 Epoch 19/30 34/34 ━━━━━━━━━━━━━━━━━━━━ 1s 30ms/step - accuracy: 0.7512 - loss: 0.0931 - macro_f1_fn: 0.6844 - val_accuracy: 0.6806 - val_loss: 0.1277 - val_macro_f1_fn: 0.5550 Epoch 20/30 34/34 ━━━━━━━━━━━━━━━━━━━━ 1s 38ms/step - accuracy: 0.5850 - loss: 0.1249 - macro_f1_fn: 0.5422 - val_accuracy: 0.7361 - val_loss: 0.1024 - val_macro_f1_fn: 0.6084 Epoch 21/30 34/34 ━━━━━━━━━━━━━━━━━━━━ 1s 37ms/step - accuracy: 0.6571 - loss: 0.0979 - macro_f1_fn: 0.5797 - val_accuracy: 0.6806 - val_loss: 0.1170 - val_macro_f1_fn: 0.5406 Epoch 22/30 34/34 ━━━━━━━━━━━━━━━━━━━━ 1s 29ms/step - accuracy: 0.6755 - loss: 0.0954 - macro_f1_fn: 0.5777 - val_accuracy: 0.7639 - val_loss: 0.0964 - val_macro_f1_fn: 0.6014 Epoch 23/30 34/34 ━━━━━━━━━━━━━━━━━━━━ 1s 31ms/step - accuracy: 0.6234 - loss: 0.1111 - macro_f1_fn: 0.5487 - val_accuracy: 0.7361 - val_loss: 0.1027 - val_macro_f1_fn: 0.5810 Epoch 24/30 34/34 ━━━━━━━━━━━━━━━━━━━━ 1s 28ms/step - accuracy: 0.6336 - loss: 0.0960 - macro_f1_fn: 0.5783 - val_accuracy: 0.7222 - val_loss: 0.1048 - val_macro_f1_fn: 0.5665 Epoch 25/30 34/34 ━━━━━━━━━━━━━━━━━━━━ 1s 38ms/step - accuracy: 0.6161 - loss: 0.0968 - macro_f1_fn: 0.5552 - val_accuracy: 0.7222 - val_loss: 0.1110 - val_macro_f1_fn: 0.6025 Epoch 26/30 34/34 ━━━━━━━━━━━━━━━━━━━━ 1s 38ms/step - accuracy: 0.6651 - loss: 0.1083 - macro_f1_fn: 0.5981 - val_accuracy: 0.7500 - val_loss: 0.1142 - val_macro_f1_fn: 0.5765 ================================================== FOLD 5/5 ================================================== [FOLD 5 DIAGNOSTICS] Train size: 266, Val size: 66 Val class distribution: y_species_encoded 1 0.258 2 0.258 0 0.242 3 0.242 Name: proportion, dtype: float64 Epoch 1/30 34/34 ━━━━━━━━━━━━━━━━━━━━ 14s 128ms/step - accuracy: 0.2148 - loss: 0.2286 - macro_f1_fn: 0.1485 - val_accuracy: 0.2639 - val_loss: 0.1997 - val_macro_f1_fn: 0.1102 Epoch 2/30 34/34 ━━━━━━━━━━━━━━━━━━━━ 1s 30ms/step - accuracy: 0.3107 - loss: 0.1821 - macro_f1_fn: 0.2507 - val_accuracy: 0.2083 - val_loss: 0.2014 - val_macro_f1_fn: 0.0815 Epoch 3/30 34/34 ━━━━━━━━━━━━━━━━━━━━ 2s 56ms/step - accuracy: 0.4180 - loss: 0.1593 - macro_f1_fn: 0.2999 - val_accuracy: 0.4028 - val_loss: 0.1858 - val_macro_f1_fn: 0.2682 Epoch 4/30 34/34 ━━━━━━━━━━━━━━━━━━━━ 2s 52ms/step - accuracy: 0.3789 - loss: 0.1515 - macro_f1_fn: 0.3193 - val_accuracy: 0.5278 - val_loss: 0.1721 - val_macro_f1_fn: 0.3954 Epoch 5/30 34/34 ━━━━━━━━━━━━━━━━━━━━ 1s 28ms/step - accuracy: 0.4866 - loss: 0.1412 - macro_f1_fn: 0.3759 - val_accuracy: 0.3889 - val_loss: 0.1739 - val_macro_f1_fn: 0.2508 Epoch 6/30 34/34 ━━━━━━━━━━━━━━━━━━━━ 1s 38ms/step - accuracy: 0.5085 - loss: 0.1324 - macro_f1_fn: 0.4866 - val_accuracy: 0.5417 - val_loss: 0.1453 - val_macro_f1_fn: 0.3862 Epoch 7/30 34/34 ━━━━━━━━━━━━━━━━━━━━ 2s 62ms/step - accuracy: 0.5518 - loss: 0.1236 - macro_f1_fn: 0.4801 - val_accuracy: 0.5694 - val_loss: 0.1582 - val_macro_f1_fn: 0.4471 Epoch 8/30 34/34 ━━━━━━━━━━━━━━━━━━━━ 3s 76ms/step - accuracy: 0.5557 - loss: 0.1282 - macro_f1_fn: 0.4521 - val_accuracy: 0.6389 - val_loss: 0.1530 - val_macro_f1_fn: 0.4730 Epoch 9/30 34/34 ━━━━━━━━━━━━━━━━━━━━ 3s 76ms/step - accuracy: 0.5011 - loss: 0.1249 - macro_f1_fn: 0.4469 - val_accuracy: 0.5833 - val_loss: 0.1491 - val_macro_f1_fn: 0.5333 Epoch 10/30 34/34 ━━━━━━━━━━━━━━━━━━━━ 1s 30ms/step - accuracy: 0.5484 - loss: 0.1158 - macro_f1_fn: 0.4905 - val_accuracy: 0.5694 - val_loss: 0.1225 - val_macro_f1_fn: 0.4400 Epoch 11/30 34/34 ━━━━━━━━━━━━━━━━━━━━ 2s 51ms/step - accuracy: 0.5319 - loss: 0.1209 - macro_f1_fn: 0.4408 - val_accuracy: 0.7222 - val_loss: 0.1329 - val_macro_f1_fn: 0.6079 Epoch 12/30 34/34 ━━━━━━━━━━━━━━━━━━━━ 2s 50ms/step - accuracy: 0.5881 - loss: 0.1133 - macro_f1_fn: 0.5615 - val_accuracy: 0.7500 - val_loss: 0.1312 - val_macro_f1_fn: 0.6528 Epoch 13/30 34/34 ━━━━━━━━━━━━━━━━━━━━ 1s 31ms/step - accuracy: 0.6434 - loss: 0.1057 - macro_f1_fn: 0.5749 - val_accuracy: 0.7361 - val_loss: 0.1101 - val_macro_f1_fn: 0.6428 Epoch 14/30 34/34 ━━━━━━━━━━━━━━━━━━━━ 2s 50ms/step - accuracy: 0.6193 - loss: 0.1082 - macro_f1_fn: 0.5357 - val_accuracy: 0.7778 - val_loss: 0.1212 - val_macro_f1_fn: 0.6704 Epoch 15/30 34/34 ━━━━━━━━━━━━━━━━━━━━ 1s 29ms/step - accuracy: 0.6334 - loss: 0.1004 - macro_f1_fn: 0.6027 - val_accuracy: 0.7361 - val_loss: 0.1172 - val_macro_f1_fn: 0.6198 Epoch 16/30 34/34 ━━━━━━━━━━━━━━━━━━━━ 1s 29ms/step - accuracy: 0.6125 - loss: 0.1148 - macro_f1_fn: 0.5544 - val_accuracy: 0.7222 - val_loss: 0.1303 - val_macro_f1_fn: 0.6228 Epoch 17/30 34/34 ━━━━━━━━━━━━━━━━━━━━ 2s 51ms/step - accuracy: 0.6763 - loss: 0.0961 - macro_f1_fn: 0.6100 - val_accuracy: 0.8056 - val_loss: 0.0986 - val_macro_f1_fn: 0.7007 Epoch 18/30 34/34 ━━━━━━━━━━━━━━━━━━━━ 1s 31ms/step - accuracy: 0.6633 - loss: 0.1009 - macro_f1_fn: 0.5864 - val_accuracy: 0.7778 - val_loss: 0.1170 - val_macro_f1_fn: 0.6747 Epoch 19/30 34/34 ━━━━━━━━━━━━━━━━━━━━ 1s 30ms/step - accuracy: 0.6110 - loss: 0.1112 - macro_f1_fn: 0.5510 - val_accuracy: 0.7361 - val_loss: 0.1648 - val_macro_f1_fn: 0.5852 Epoch 20/30 34/34 ━━━━━━━━━━━━━━━━━━━━ 1s 30ms/step - accuracy: 0.6383 - loss: 0.1008 - macro_f1_fn: 0.6131 - val_accuracy: 0.7361 - val_loss: 0.1053 - val_macro_f1_fn: 0.6497 Epoch 21/30 34/34 ━━━━━━━━━━━━━━━━━━━━ 1s 29ms/step - accuracy: 0.6206 - loss: 0.1114 - macro_f1_fn: 0.5351 - val_accuracy: 0.7778 - val_loss: 0.1201 - val_macro_f1_fn: 0.6489 Epoch 22/30 34/34 ━━━━━━━━━━━━━━━━━━━━ 1s 29ms/step - accuracy: 0.6858 - loss: 0.1001 - macro_f1_fn: 0.6240 - val_accuracy: 0.7639 - val_loss: 0.1215 - val_macro_f1_fn: 0.6561 Epoch 23/30 34/34 ━━━━━━━━━━━━━━━━━━━━ 2s 55ms/step - accuracy: 0.5527 - loss: 0.1271 - macro_f1_fn: 0.4947 - val_accuracy: 0.8333 - val_loss: 0.1142 - val_macro_f1_fn: 0.7217 Epoch 24/30 34/34 ━━━━━━━━━━━━━━━━━━━━ 2s 51ms/step - accuracy: 0.6807 - loss: 0.0966 - macro_f1_fn: 0.6460 - val_accuracy: 0.7778 - val_loss: 0.1189 - val_macro_f1_fn: 0.6714 Epoch 25/30 34/34 ━━━━━━━━━━━━━━━━━━━━ 1s 30ms/step - accuracy: 0.6708 - loss: 0.1046 - macro_f1_fn: 0.6088 - val_accuracy: 0.7500 - val_loss: 0.1277 - val_macro_f1_fn: 0.6151 Epoch 26/30 34/34 ━━━━━━━━━━━━━━━━━━━━ 1s 29ms/step - accuracy: 0.6291 - loss: 0.0991 - macro_f1_fn: 0.5921 - val_accuracy: 0.8194 - val_loss: 0.1040 - val_macro_f1_fn: 0.7002 Epoch 27/30 34/34 ━━━━━━━━━━━━━━━━━━━━ 1s 29ms/step - accuracy: 0.6365 - loss: 0.0955 - macro_f1_fn: 0.5792 - val_accuracy: 0.7639 - val_loss: 0.1301 - val_macro_f1_fn: 0.6675 Epoch 28/30 34/34 ━━━━━━━━━━━━━━━━━━━━ 1s 29ms/step - accuracy: 0.6877 - loss: 0.1069 - macro_f1_fn: 0.6460 - val_accuracy: 0.7778 - val_loss: 0.1171 - val_macro_f1_fn: 0.7117 Epoch 29/30 34/34 ━━━━━━━━━━━━━━━━━━━━ 1s 28ms/step - accuracy: 0.6761 - loss: 0.0919 - macro_f1_fn: 0.6006 - val_accuracy: 0.7778 - val_loss: 0.0921 - val_macro_f1_fn: 0.6525 Epoch 30/30 34/34 ━━━━━━━━━━━━━━━━━━━━ 1s 31ms/step - accuracy: 0.6473 - loss: 0.1041 - macro_f1_fn: 0.5914 - val_accuracy: 0.7778 - val_loss: 0.1208 - val_macro_f1_fn: 0.6065 ================================================== CROSS-VALIDATION SUMMARY ================================================== fold val_accuracy val_macro_f1 best_epoch 0 1 0.833333 0.734127 18 1 2 0.750000 0.662191 20 2 3 0.833333 0.771891 22 3 4 0.763889 0.637632 17 4 5 0.833333 0.721693 22 Summary Statistics: val_accuracy val_macro_f1 count 5.0000 5.0000 mean 0.8028 0.7055 std 0.0421 0.0547 min 0.7500 0.6376 25% 0.7639 0.6622 50% 0.8333 0.7217 75% 0.8333 0.7341 max 0.8333 0.7719 Variance Analysis: Accuracy StdDev: 0.0421 (OK) F1 StdDev: 0.0547 (OK)
In [26]:
history_dict.keys()
Out[26]:
dict_keys(['fold_1', 'fold_2', 'fold_3', 'fold_4', 'fold_5'])
In [27]:
metrics_list
Out[27]:
[{'fold': 1,
'best_epoch': 18,
'val_loss': 0.09619996696710587,
'val_accuracy': 0.8333333134651184,
'val_macro_f1': 0.7341269850730896,
'grouping_strategy': 'hierarchical',
'splitter': 'StratifiedGroupKFold (hierarchical)'},
{'fold': 2,
'best_epoch': 20,
'val_loss': 0.09801912307739258,
'val_accuracy': 0.75,
'val_macro_f1': 0.6621912717819214,
'grouping_strategy': 'hierarchical',
'splitter': 'StratifiedGroupKFold (hierarchical)'},
{'fold': 3,
'best_epoch': 22,
'val_loss': 0.0762423574924469,
'val_accuracy': 0.8333333134651184,
'val_macro_f1': 0.7718913555145264,
'grouping_strategy': 'hierarchical',
'splitter': 'StratifiedGroupKFold (hierarchical)'},
{'fold': 4,
'best_epoch': 17,
'val_loss': 0.10241618752479553,
'val_accuracy': 0.7638888955116272,
'val_macro_f1': 0.6376321911811829,
'grouping_strategy': 'hierarchical',
'splitter': 'StratifiedGroupKFold (hierarchical)'},
{'fold': 5,
'best_epoch': 22,
'val_loss': 0.11417350172996521,
'val_accuracy': 0.8333333134651184,
'val_macro_f1': 0.7216930985450745,
'grouping_strategy': 'hierarchical',
'splitter': 'StratifiedGroupKFold (hierarchical)'}]
In [28]:
# --- 1) Pick which fold to visualize ---------------------------------
prefer_metric = "val_macro_f1_fn"
if isinstance(metrics_list, list) and metrics_list:
# each item is expected like {"fold": i, "val_macro_f1_fn": 0.87, ...}
best_item = max(
metrics_list,
key=lambda d: d.get(prefer_metric, d.get("val_accuracy", float("-inf")))
)
fold_id = best_item.get("fold", metrics_list.index(best_item))
else:
# Fallback: just visualize fold 0
fold_id = 0
print(f"Visualizing diagnostics for fold: {fold_id}")
# --- 2) Get the History object for that fold --------------------------
from collections.abc import Mapping
if isinstance(history_dict, Mapping): # dict like {fold_id: History}
fold_history = history_dict[f"fold_{fold_id}"]
else:
raise ValueError("Unrecognized history_dict structure.")
plot_training_diagnostics(
history=fold_history,
cfg=cfg,
)
Visualizing diagnostics for fold: 1 Training curves saved to: /share/crsp/lab/pkaiser/ddlin/mids/DATASCI207_Bird_Sounds/data/working/plots/training_curves_256_aug_cw_tl_meta_film.png
In [29]:
# Custom objects needed to load the trained Keras models
custom_objects = {
'macro_f1_fn': macro_f1_fn,
# Ensure alpha/gamma match what was used in training/tuning
'categorical_focal_loss_a0.25_g2.0': categorical_focal_loss(alpha=0.25, gamma=2.0),
'AddClsToken': AddClsToken,
'SpecTimeMask': SpecTimeMask,
'SpecFreqMask': SpecFreqMask,
'preprocess_input': efn_preprocess, # EfficientNet preprocess_input
'efn_preprocess': efn_preprocess,
}
# Accumulate predictions from each fold model for ensembling
y_pred_prob_list = []
for fold in range(1, cfg.n_fold + 1):
fold_model_name = f'{cfg.model_save_name.replace(".keras", "")}_fold{fold}.keras'
model_path = cfg.MODELS_DIR / fold_model_name
if not model_path.exists():
print(f"Warning: Model for fold {fold} not found at {model_path}. Skipping.")
continue
print(f"Loading model for fold {fold} from {model_path}")
fold_model = keras.models.load_model(
model_path,
custom_objects=custom_objects,
compile=False,
safe_mode=False,
)
print(f"Model for fold {fold} loaded successfully.")
if cfg.use_metadata:
print("Evaluating with metadata features...")
y_true, y_pred_prob = evaluate_on_test_multimodal(
model=fold_model,
cfg=cfg,
spectrograms=filtered_spectrograms,
metadata_features=test_meta,
test_df=test_df,
)
else:
print("Evaluating without metadata features...")
# NOTE: evaluate_on_test signature does NOT accept metadata_features
# (see evaluation_utils.py). Just pass (model, cfg, spectrograms, test_df).
y_true, y_pred_prob = evaluate_on_test(
model=fold_model,
cfg=cfg,
spectrograms=filtered_spectrograms,
test_df=test_df,
)
y_true_final = y_true # same test set each fold
y_pred_prob_list.append(y_pred_prob)
Loading model for fold 1 from /share/crsp/lab/pkaiser/ddlin/mids/DATASCI207_Bird_Sounds/data/working/models/rescnn_model_256_aug_cw_tl_meta_film_fold1.keras
Model for fold 1 loaded successfully. Evaluating with metadata features... 9/9 ━━━━━━━━━━━━━━━━━━━━ 2s 33ms/step Test Loss: 1.6695 Test AUC: 0.5616 Test Accuracy: 0.3529 Loading model for fold 2 from /share/crsp/lab/pkaiser/ddlin/mids/DATASCI207_Bird_Sounds/data/working/models/rescnn_model_256_aug_cw_tl_meta_film_fold2.keras Model for fold 2 loaded successfully. Evaluating with metadata features... 9/9 ━━━━━━━━━━━━━━━━━━━━ 2s 42ms/step Test Loss: 1.8380 Test AUC: 0.4771 Test Accuracy: 0.2353 Loading model for fold 3 from /share/crsp/lab/pkaiser/ddlin/mids/DATASCI207_Bird_Sounds/data/working/models/rescnn_model_256_aug_cw_tl_meta_film_fold3.keras Model for fold 3 loaded successfully. Evaluating with metadata features... 9/9 ━━━━━━━━━━━━━━━━━━━━ 2s 36ms/step Test Loss: 1.6410 Test AUC: 0.5397 Test Accuracy: 0.2941 Loading model for fold 4 from /share/crsp/lab/pkaiser/ddlin/mids/DATASCI207_Bird_Sounds/data/working/models/rescnn_model_256_aug_cw_tl_meta_film_fold4.keras Model for fold 4 loaded successfully. Evaluating with metadata features... 9/9 ━━━━━━━━━━━━━━━━━━━━ 2s 35ms/step Test Loss: 1.6708 Test AUC: 0.5021 Test Accuracy: 0.2794 Loading model for fold 5 from /share/crsp/lab/pkaiser/ddlin/mids/DATASCI207_Bird_Sounds/data/working/models/rescnn_model_256_aug_cw_tl_meta_film_fold5.keras Model for fold 5 loaded successfully. Evaluating with metadata features... 9/9 ━━━━━━━━━━━━━━━━━━━━ 2s 33ms/step Test Loss: 1.7835 Test AUC: 0.5093 Test Accuracy: 0.3235
In [30]:
# Convert list to numpy array for easier handling
y_pred_prob = np.array(y_pred_prob_list).mean(axis=0)
# Plot training diagnostics
plot_test_evaluation(y_true, y_pred_prob, class_mapping=CLASS_MAPPING_DICT, cfg=cfg)
Test evaluation plot saved to: /share/crsp/lab/pkaiser/ddlin/mids/DATASCI207_Bird_Sounds/data/working/plots/test_evaluation_256_aug_cw_tl_meta_film.png
Per-class metrics saved to: /share/crsp/lab/pkaiser/ddlin/mids/DATASCI207_Bird_Sounds/data/working/plots/per_class_metrics_256_aug_cw_tl_meta_film.png
Classification report saved to: /share/crsp/lab/pkaiser/ddlin/mids/DATASCI207_Bird_Sounds/data/working/result_csvs/classification_report_256_aug_cw_tl_meta_film.csv
Classification Report:
precision recall f1-score support
Amphibia 0.14 0.15 0.15 13
Aves 0.17 0.06 0.09 16
Insecta 0.33 0.26 0.29 19
Mammalia 0.36 0.60 0.45 20
accuracy 0.29 68
macro avg 0.25 0.27 0.25 68
weighted avg 0.27 0.29 0.27 68
In [31]:
import mlflow
# Define a local path for MLflow artifacts
local_mlflow_path = "file:///tmp/mlruns" # Using /tmp is a safe, local choice
# Set the experiment AND the artifact location
mlflow.set_tracking_uri(local_mlflow_path)
mlflow.set_experiment("BirdCLEF2025_Multimodal")
# 2. Start the main run. All parameters, metrics, and artifacts will be logged under this run.
with mlflow.start_run(run_name=f"{cfg.model_type}_{cfg.input_dim}{get_suffix(cfg)}") as parent_run:
print(f"MLflow Run ID: {parent_run.info.run_id}")
# --- Log Configuration Parameters ---
# Convert the CFG object to a dictionary for logging
cfg_dict = {k: v for k, v in cfg.__dict__.items() if not k.startswith('__') and not callable(v)}
mlflow.log_params(cfg_dict)
# Also log the config as a JSON artifact for perfect reproducibility
cfg_json_path = cfg.OUTPUT_DIR / "config.json"
with open(cfg_json_path, 'w') as f:
import json
json.dump(cfg_dict, f, indent=4)
mlflow.log_artifact(str(cfg_json_path))
# --- [Existing code from your notebook] ---
# Assuming `metrics_list` (from CV) and `y_true`, `y_pred_prob` (from evaluation)
# are available from the cells above.
# --- Log Cross-Validation Metrics ---
if metrics_list:
cv_metrics_df = pd.DataFrame(metrics_list)
# Log summary statistics from the CV folds
cv_summary = cv_metrics_df[['val_accuracy', 'val_macro_f1']].describe().round(4)
mlflow.log_metric("cv_mean_val_accuracy", cv_summary.loc['mean', 'val_accuracy'])
mlflow.log_metric("cv_std_val_accuracy", cv_summary.loc['std', 'val_accuracy'])
mlflow.log_metric("cv_mean_val_macro_f1", cv_summary.loc['mean', 'val_macro_f1'])
mlflow.log_metric("cv_std_val_macro_f1", cv_summary.loc['std', 'val_macro_f1'])
# Log the full CV results as a CSV artifact
cv_csv_path = cfg.RESULTS_CSV_DIR / f"cv_summary{get_suffix(cfg)}.csv"
cv_metrics_df.to_csv(cv_csv_path, index=False)
mlflow.log_artifact(str(cv_csv_path))
# --- Log Final Ensembled Test Metrics ---
from sklearn.metrics import classification_report, roc_auc_score, log_loss
y_true_labels = np.argmax(y_true_final, axis=1)
y_pred_labels = np.argmax(y_pred_prob, axis=1)
# Calculate final metrics from the ensembled predictions
report = classification_report(y_true_labels, y_pred_labels, output_dict=True)
# Log overall metrics
mlflow.log_metric("test_accuracy", report['accuracy'])
mlflow.log_metric("test_macro_f1", report['macro avg']['f1-score'])
# Calculate and log multiclass AUC and LogLoss
try:
test_auc = roc_auc_score(y_true_final, y_pred_prob, multi_class='ovr', average='macro')
mlflow.log_metric("test_auc_ovr_macro", test_auc)
except ValueError as e:
print(f"Could not compute AUC: {e}")
test_loss = log_loss(y_true_final, y_pred_prob)
mlflow.log_metric("test_log_loss", test_loss)
print(f"\nLogged final test metrics to MLflow:")
print(f" Accuracy: {report['accuracy']:.4f}")
print(f" Macro F1: {report['macro avg']['f1-score']:.4f}")
print(f" AUC (OVR): {test_auc:.4f}")
print(f" Log Loss: {test_loss:.4f}")
# --- Log Artifacts (Plots and Reports) ---
print("\nLogging artifacts to MLflow...")
# The evaluation plots and reports are already saved to disk by your functions.
# We just need to tell MLflow where they are.
suffix = get_suffix(cfg)
# Log the classification report CSV that was already generated
class_report_path = cfg.RESULTS_CSV_DIR / f'classification_report_{cfg.input_dim}{suffix}.csv'
if class_report_path.exists():
mlflow.log_artifact(str(class_report_path), "evaluation_reports")
# Log the plots
eval_plot_path = cfg.PLOTS_DIR / f'test_evaluation_{cfg.input_dim}{suffix}.png'
if eval_plot_path.exists():
mlflow.log_artifact(str(eval_plot_path), "plots")
per_class_plot_path = cfg.PLOTS_DIR / f"per_class_metrics_{cfg.input_dim}{suffix}.png"
if per_class_plot_path.exists():
mlflow.log_artifact(str(per_class_plot_path), "plots")
# Log the training curves plot (from the best fold)
training_curves_path = cfg.PLOTS_DIR / f"training_curves_{cfg.input_dim}{suffix}.png"
if training_curves_path.exists():
mlflow.log_artifact(str(training_curves_path), "plots")
# Log the models directory path as a tag for reference
mlflow.set_tag("models_location", str(cfg.MODELS_DIR))
print("✅ MLflow logging complete.")
MLflow Run ID: 1d338d16c68b4d89ab21e674431cab8c Logged final test metrics to MLflow: Accuracy: 0.2941 Macro F1: 0.2465 AUC (OVR): 0.5538 Log Loss: 1.3992 Logging artifacts to MLflow... ✅ MLflow logging complete.
In [32]:
# Use uv run mlflow ui --backend-store-uri file:///tmp/mlruns to access the MLflow UI
# Open your browser and go to http://localhost:5000 to view the MLflow UI