From a1e046c1aee3f2b257b46225a509a2c469c5ffb1 Mon Sep 17 00:00:00 2001 From: SOFIA GARCIA s321387 Date: Thu, 6 Nov 2025 14:16:49 +0100 Subject: [PATCH] Supervised Learning models --- Code/Supervised_learning/MobileNetV1.py | 944 ++++++++++++++++++ .../{ => Supervised_learning}/Phenology_V1.py | 0 Code/Supervised_learning/README_ResNet50.md | 268 +++++ Code/Supervised_learning/ResNET.py | 741 ++++++++++++++ .../.github/prompts/speckit.analyze.prompt.md | 184 ++++ .../prompts/speckit.checklist.prompt.md | 294 ++++++ .../.github/prompts/speckit.clarify.prompt.md | 177 ++++ .../prompts/speckit.constitution.prompt.md | 78 ++ .../prompts/speckit.implement.prompt.md | 134 +++ .../.github/prompts/speckit.plan.prompt.md | 81 ++ .../.github/prompts/speckit.specify.prompt.md | 249 +++++ .../.github/prompts/speckit.tasks.prompt.md | 128 +++ Code/Supervised_learning/resnet/.gitignore | 31 + .../resnet/.specify/memory/constitution.md | 39 + .../powershell/check-prerequisites.ps1 | 148 +++ .../.specify/scripts/powershell/common.ps1 | 137 +++ .../scripts/powershell/create-new-feature.ps1 | 290 ++++++ .../scripts/powershell/setup-plan.ps1 | 62 ++ .../powershell/update-agent-context.ps1 | 439 ++++++++ .../.specify/templates/agent-file-template.md | 28 + .../.specify/templates/checklist-template.md | 40 + .../.specify/templates/plan-template.md | 108 ++ .../.specify/templates/spec-template.md | 115 +++ .../.specify/templates/tasks-template.md | 251 +++++ .../resnet/CONTRIBUTING.md | 203 ++++ Code/Supervised_learning/resnet/README.md | 224 +++++ Code/Supervised_learning/resnet/pytest.ini | 10 + .../resnet/requirements.txt | 13 + .../checklists/requirements.md | 35 + .../1-phenology-classifier/contracts/api.yaml | 60 ++ .../1-phenology-classifier/data-model.md | 41 + .../specs/1-phenology-classifier/plan.md | 88 ++ .../1-phenology-classifier/quickstart.md | 68 ++ .../specs/1-phenology-classifier/research.md | 39 + .../specs/1-phenology-classifier/spec.md | 94 ++ .../specs/1-phenology-classifier/tasks.md | 97 ++ .../resnet/src/__init__.py | 3 + Code/Supervised_learning/resnet/src/api.py | 174 ++++ .../resnet/src/data_loader.py | 171 ++++ .../resnet/src/evaluate.py | 259 +++++ .../resnet/src/inference.py | 226 +++++ Code/Supervised_learning/resnet/src/model.py | 106 ++ Code/Supervised_learning/resnet/src/train.py | 276 +++++ Code/Supervised_learning/resnet/src/utils.py | 183 ++++ .../resnet/tests/test_data_loader.py | 144 +++ .../resnet/tests/test_evaluate.py | 89 ++ .../resnet/tests/test_inference.py | 113 +++ .../resnet/tests/test_train.py | 93 ++ Code/Supervised_learning/train_resnet50.py | 298 ++++++ Code/Unsupervised_learning/PCA_V1_C.py | 433 ++++++++ Code/Unsupervised_learning/UMAP.py | 4 +- 51 files changed, 8508 insertions(+), 2 deletions(-) create mode 100644 Code/Supervised_learning/MobileNetV1.py rename Code/{ => Supervised_learning}/Phenology_V1.py (100%) create mode 100644 Code/Supervised_learning/README_ResNet50.md create mode 100644 Code/Supervised_learning/ResNET.py create mode 100644 Code/Supervised_learning/resnet/.github/prompts/speckit.analyze.prompt.md create mode 100644 Code/Supervised_learning/resnet/.github/prompts/speckit.checklist.prompt.md create mode 100644 Code/Supervised_learning/resnet/.github/prompts/speckit.clarify.prompt.md create mode 100644 Code/Supervised_learning/resnet/.github/prompts/speckit.constitution.prompt.md create mode 100644 Code/Supervised_learning/resnet/.github/prompts/speckit.implement.prompt.md create mode 100644 Code/Supervised_learning/resnet/.github/prompts/speckit.plan.prompt.md create mode 100644 Code/Supervised_learning/resnet/.github/prompts/speckit.specify.prompt.md create mode 100644 Code/Supervised_learning/resnet/.github/prompts/speckit.tasks.prompt.md create mode 100644 Code/Supervised_learning/resnet/.gitignore create mode 100644 Code/Supervised_learning/resnet/.specify/memory/constitution.md create mode 100644 Code/Supervised_learning/resnet/.specify/scripts/powershell/check-prerequisites.ps1 create mode 100644 Code/Supervised_learning/resnet/.specify/scripts/powershell/common.ps1 create mode 100644 Code/Supervised_learning/resnet/.specify/scripts/powershell/create-new-feature.ps1 create mode 100644 Code/Supervised_learning/resnet/.specify/scripts/powershell/setup-plan.ps1 create mode 100644 Code/Supervised_learning/resnet/.specify/scripts/powershell/update-agent-context.ps1 create mode 100644 Code/Supervised_learning/resnet/.specify/templates/agent-file-template.md create mode 100644 Code/Supervised_learning/resnet/.specify/templates/checklist-template.md create mode 100644 Code/Supervised_learning/resnet/.specify/templates/plan-template.md create mode 100644 Code/Supervised_learning/resnet/.specify/templates/spec-template.md create mode 100644 Code/Supervised_learning/resnet/.specify/templates/tasks-template.md create mode 100644 Code/Supervised_learning/resnet/CONTRIBUTING.md create mode 100644 Code/Supervised_learning/resnet/README.md create mode 100644 Code/Supervised_learning/resnet/pytest.ini create mode 100644 Code/Supervised_learning/resnet/requirements.txt create mode 100644 Code/Supervised_learning/resnet/specs/1-phenology-classifier/checklists/requirements.md create mode 100644 Code/Supervised_learning/resnet/specs/1-phenology-classifier/contracts/api.yaml create mode 100644 Code/Supervised_learning/resnet/specs/1-phenology-classifier/data-model.md create mode 100644 Code/Supervised_learning/resnet/specs/1-phenology-classifier/plan.md create mode 100644 Code/Supervised_learning/resnet/specs/1-phenology-classifier/quickstart.md create mode 100644 Code/Supervised_learning/resnet/specs/1-phenology-classifier/research.md create mode 100644 Code/Supervised_learning/resnet/specs/1-phenology-classifier/spec.md create mode 100644 Code/Supervised_learning/resnet/specs/1-phenology-classifier/tasks.md create mode 100644 Code/Supervised_learning/resnet/src/__init__.py create mode 100644 Code/Supervised_learning/resnet/src/api.py create mode 100644 Code/Supervised_learning/resnet/src/data_loader.py create mode 100644 Code/Supervised_learning/resnet/src/evaluate.py create mode 100644 Code/Supervised_learning/resnet/src/inference.py create mode 100644 Code/Supervised_learning/resnet/src/model.py create mode 100644 Code/Supervised_learning/resnet/src/train.py create mode 100644 Code/Supervised_learning/resnet/src/utils.py create mode 100644 Code/Supervised_learning/resnet/tests/test_data_loader.py create mode 100644 Code/Supervised_learning/resnet/tests/test_evaluate.py create mode 100644 Code/Supervised_learning/resnet/tests/test_inference.py create mode 100644 Code/Supervised_learning/resnet/tests/test_train.py create mode 100644 Code/Supervised_learning/train_resnet50.py create mode 100644 Code/Unsupervised_learning/PCA_V1_C.py diff --git a/Code/Supervised_learning/MobileNetV1.py b/Code/Supervised_learning/MobileNetV1.py new file mode 100644 index 0000000..09b0633 --- /dev/null +++ b/Code/Supervised_learning/MobileNetV1.py @@ -0,0 +1,944 @@ +""" +MobileNetV2 Transfer Learning para Clasificación de Fases Fenológicas - Nocciola +Adaptado para Visual Studio Code +Dataset: Nocciola GBIF +Objetivo: Predecir fase R (fenológica reproductive) +""" + +import os +import shutil +import random +import argparse +from pathlib import Path +import pandas as pd +import numpy as np +import matplotlib.pyplot as plt +import seaborn as sns +import tensorflow as tf +from tensorflow.keras import layers, models +from tensorflow.keras.applications import MobileNetV2 +from tensorflow.keras.preprocessing.image import ImageDataGenerator +from sklearn.metrics import classification_report, confusion_matrix +from sklearn.utils import class_weight +import json + +# ----------------- CONFIG ----------------- +PROJECT_PATH = r'C:\Users\sof12\Desktop\ML\Datasets\Nocciola\GBIF' +IMAGES_DIR = PROJECT_PATH # Las imágenes están en el directorio principal +CSV_PATH = os.path.join(PROJECT_PATH, 'assignments.csv') # CSV principal +OUTPUT_DIR = os.path.join(PROJECT_PATH, 'results_mobilenet_faseV_V1') +os.makedirs(OUTPUT_DIR, exist_ok=True) + +IMG_SIZE = (224, 224) # Recomendado para MobileNetV2 +BATCH_SIZE = 16 # Reducido para mejor estabilidad +SEED = 42 +SPLIT = {'train': 0.7, 'val': 0.15, 'test': 0.15} +FORCE_SPLIT = False + +# ----------------- Utilities ----------------- +def set_seed(seed=42): + """Establecer semilla para reproducibilidad""" + random.seed(seed) + np.random.seed(seed) + tf.random.set_seed(seed) + +def analyze_class_distribution(df, column_name='fase V'): + """Analizar distribución de clases y detectar desbalances""" + print(f"\n📊 === Análisis de Distribución de Clases ===") + + # Contar por clase + counts = df[column_name].value_counts() + total = len(df) + + print(f"📊 Total de muestras: {total}") + print(f"📊 Número de clases: {len(counts)}") + print(f"📊 Distribución por clase:") + + # Mostrar estadísticas detalladas + for clase, count in counts.items(): + percentage = (count / total) * 100 + print(f" - {clase}: {count} muestras ({percentage:.1f}%)") + + # Detectar clases problemáticas + min_samples = 5 # Umbral mínimo recomendado + small_classes = counts[counts < min_samples] + + if len(small_classes) > 0: + print(f"\n⚠️ Clases con menos de {min_samples} muestras:") + for clase, count in small_classes.items(): + print(f" - {clase}: {count} muestras") + + print(f"\n💡 Recomendaciones:") + print(f" 1. Considera recolectar más datos para estas clases") + print(f" 2. O fusionar clases similares") + print(f" 3. O usar técnicas de data augmentation específicas") + + return counts, small_classes + +def safe_read_csv(path): + """Leer CSV con manejo de encoding""" + if not os.path.exists(path): + raise FileNotFoundError(f'CSV no encontrado: {path}') + try: + df = pd.read_csv(path, encoding='utf-8') + except UnicodeDecodeError: + try: + df = pd.read_csv(path, encoding='latin-1') + except: + df = pd.read_csv(path, encoding='iso-8859-1') + return df + +def resolve_image_path(images_dir, img_id): + """Resolver la ruta completa de una imagen""" + if pd.isna(img_id) or str(img_id).strip() == '': + return None + + img_id = str(img_id).strip() + + # Verificar si ya incluye extensión y existe + direct_path = os.path.join(images_dir, img_id) + if os.path.exists(direct_path): + return direct_path + + # Probar con extensiones comunes + stem = os.path.splitext(img_id)[0] + for ext in ['.jpg', '.jpeg', '.png', '.JPG', '.JPEG', '.PNG']: + img_path = os.path.join(images_dir, stem + ext) + if os.path.exists(img_path): + return img_path + + return None + +def prepare_image_folders(df, images_dir, out_dir, split=SPLIT, seed=SEED): + """Crear estructura de carpetas para flow_from_directory""" + set_seed(seed) + + # Filtrar solo filas con fase R válida e imágenes existentes + print(f"📊 Datos iniciales: {len(df)} filas") + + # Filtrar filas con fase R válida + df_valid = df.dropna(subset=['fase V']).copy() + df_valid = df_valid[df_valid['fase V'].str.strip() != ''] + print(f"📊 Con fase V válida: {len(df_valid)} filas") + + # Verificar existencia de imágenes + valid_rows = [] + for _, row in df_valid.iterrows(): + img_path = resolve_image_path(images_dir, row['id_img']) + if img_path: + valid_rows.append(row) + else: + print(f"⚠️ Imagen no encontrada: {row['id_img']}") + + if not valid_rows: + raise ValueError("❌ No se encontraron imágenes válidas") + + df_final = pd.DataFrame(valid_rows) + print(f"📊 Con imágenes existentes: {len(df_final)} filas") + + # Mostrar distribución de clases + fase_counts = df_final['fase V'].value_counts() + print(f"\n📊 Distribución de fases R:") + for fase, count in fase_counts.items(): + print(f" - {fase}: {count} imágenes") + + # Remover clases con muy pocas muestras (menos de 3) + min_samples = 3 + valid_phases = fase_counts[fase_counts >= min_samples].index.tolist() + if len(valid_phases) < len(fase_counts): + excluded = fase_counts[fase_counts < min_samples].index.tolist() + print(f"⚠️ Excluyendo fases con menos de {min_samples} muestras: {excluded}") + df_final = df_final[df_final['fase V'].isin(valid_phases)] + print(f"📊 Después de filtrar: {len(df_final)} filas, {len(valid_phases)} clases") + + labels = df_final['fase V'].unique().tolist() + print(f"📊 Clases finales: {labels}") + + # Mezclar y dividir datos + df_shuffled = df_final.sample(frac=1, random_state=seed).reset_index(drop=True) + n = len(df_shuffled) + n_train = int(n * split['train']) + n_val = int(n * split['val']) + + train_df = df_shuffled.iloc[:n_train] + val_df = df_shuffled.iloc[n_train:n_train + n_val] + test_df = df_shuffled.iloc[n_train + n_val:] + + print(f"📊 División final:") + print(f" - Entrenamiento: {len(train_df)} imágenes") + print(f" - Validación: {len(val_df)} imágenes") + print(f" - Prueba: {len(test_df)} imágenes") + + # Crear estructura de carpetas + for part in ['train', 'val', 'test']: + for label in labels: + label_dir = os.path.join(out_dir, part, str(label)) + os.makedirs(label_dir, exist_ok=True) + + # Función para copiar imágenes + def copy_subset(subdf, subset_name): + copied, missing = 0, 0 + for _, row in subdf.iterrows(): + src = resolve_image_path(images_dir, row['id_img']) + if src: + fase = str(row['fase V']) + dst = os.path.join(out_dir, subset_name, fase, f"{row['id_img']}.jpg") + try: + shutil.copy2(src, dst) + copied += 1 + except Exception as e: + print(f"⚠️ Error copiando {src}: {e}") + missing += 1 + else: + missing += 1 + + print(f"✅ {subset_name}: {copied} imágenes copiadas, {missing} fallidas") + return copied + + # Copiar imágenes a las carpetas correspondientes + copy_subset(train_df, 'train') + copy_subset(val_df, 'val') + copy_subset(test_df, 'test') + + return train_df, val_df, test_df + +def main(): + """Función principal del pipeline""" + parser = argparse.ArgumentParser(description='MobileNetV2 Transfer Learning para Nocciola') + parser.add_argument('--csv_path', type=str, default=CSV_PATH, + help='Ruta al archivo CSV con metadatos') + parser.add_argument('--images_dir', type=str, default=IMAGES_DIR, + help='Directorio con las imágenes') + parser.add_argument('--output_dir', type=str, default=OUTPUT_DIR, + help='Directorio de salida para resultados') + parser.add_argument('--epochs', type=int, default=30, + help='Número de épocas de entrenamiento') + parser.add_argument('--force_split', action='store_true', + help='Forzar recreación del split de datos') + + args = parser.parse_args() + + print('\n🚀 === Inicio del pipeline MobileNetV2 para Nocciola ===') + print(f"📁 Directorio de imágenes: {args.images_dir}") + print(f"📄 Archivo CSV: {args.csv_path}") + print(f"📂 Directorio de salida: {args.output_dir}") + + # Establecer semilla + set_seed(SEED) + + # Crear directorio de salida + os.makedirs(args.output_dir, exist_ok=True) + + # Leer datos + print('\n📊 === Cargando datos ===') + df = safe_read_csv(args.csv_path) + print(f'📊 Total de registros en CSV: {len(df)}') + print(f'📊 Columnas disponibles: {list(df.columns)}') + + # Verificar columnas requeridas + required_cols = {'id_img', 'fase V'} + if not required_cols.issubset(set(df.columns)): + missing = required_cols - set(df.columns) + raise ValueError(f'❌ CSV debe contener las columnas: {missing}') + + # Analizar distribución de clases antes del procesamiento + analyze_class_distribution(df, 'fase V') + + # Preparar estructura de carpetas + SPLIT_DIR = os.path.join(args.output_dir, 'data_split') + + if args.force_split and os.path.exists(SPLIT_DIR): + print("🗑️ Eliminando split existente...") + shutil.rmtree(SPLIT_DIR) + + if not os.path.exists(SPLIT_DIR): + print("\n📁 === Creando nueva división de datos ===") + train_df, val_df, test_df = prepare_image_folders(df, args.images_dir, SPLIT_DIR) + + # Guardar información del split + train_df.to_csv(os.path.join(args.output_dir, 'train_split.csv'), index=False) + val_df.to_csv(os.path.join(args.output_dir, 'val_split.csv'), index=False) + test_df.to_csv(os.path.join(args.output_dir, 'test_split.csv'), index=False) + + else: + print("\n♻️ === Reutilizando división existente ===") + # Cargar información del split si existe + try: + train_df = pd.read_csv(os.path.join(args.output_dir, 'train_split.csv')) + val_df = pd.read_csv(os.path.join(args.output_dir, 'val_split.csv')) + test_df = pd.read_csv(os.path.join(args.output_dir, 'test_split.csv')) + except: + print("⚠️ No se pudieron cargar los archivos de split, recreando...") + train_df, val_df, test_df = prepare_image_folders(df, args.images_dir, SPLIT_DIR) + + # Crear generadores de datos + print("\n🔄 === Creando generadores de datos ===") + + # Data augmentation para entrenamiento + train_datagen = ImageDataGenerator( + rescale=1./255, + rotation_range=20, + width_shift_range=0.1, + height_shift_range=0.1, + shear_range=0.1, + zoom_range=0.1, + horizontal_flip=True, + fill_mode='nearest' + ) + + # Solo normalización para validación y test + val_test_datagen = ImageDataGenerator(rescale=1./255) + + # Crear generadores + train_gen = train_datagen.flow_from_directory( + os.path.join(SPLIT_DIR, 'train'), + target_size=IMG_SIZE, + batch_size=BATCH_SIZE, + class_mode='categorical', + seed=SEED + ) + + val_gen = val_test_datagen.flow_from_directory( + os.path.join(SPLIT_DIR, 'val'), + target_size=IMG_SIZE, + batch_size=BATCH_SIZE, + class_mode='categorical', + shuffle=False + ) + + test_gen = val_test_datagen.flow_from_directory( + os.path.join(SPLIT_DIR, 'test'), + target_size=IMG_SIZE, + batch_size=BATCH_SIZE, + class_mode='categorical', + shuffle=False + ) + + # Guardar mapeo de clases + class_indices = train_gen.class_indices + print(f'🏷️ Mapeo de clases: {class_indices}') + + with open(os.path.join(args.output_dir, 'class_indices.json'), 'w') as f: + json.dump(class_indices, f, indent=2) + + print(f"📊 Muestras por conjunto:") + print(f" - Entrenamiento: {train_gen.samples}") + print(f" - Validación: {val_gen.samples}") + print(f" - Prueba: {test_gen.samples}") + print(f" - Número de clases: {train_gen.num_classes}") + + # Crear y entrenar modelo + print("\n🤖 === Construcción del modelo ===") + + # Modelo base MobileNetV2 + base_model = MobileNetV2( + weights='imagenet', + include_top=False, + input_shape=(*IMG_SIZE, 3) + ) + base_model.trainable = False # Congelar inicialmente + + # Construir modelo secuencial + model = models.Sequential([ + base_model, + layers.GlobalAveragePooling2D(), + layers.Dropout(0.3), + layers.Dense(128, activation='relu'), + layers.Dropout(0.3), + layers.Dense(train_gen.num_classes, activation='softmax') + ]) + + # Compilar modelo + model.compile( + optimizer=tf.keras.optimizers.Adam(learning_rate=1e-3), + loss='categorical_crossentropy', + metrics=['accuracy'] + ) + + print("📋 Resumen del modelo:") + model.summary() + + # Calcular pesos de clase + print("\n⚖️ === Calculando pesos de clase ===") + try: + # Obtener etiquetas de entrenamiento + train_labels = [] + for i in range(len(train_gen)): + _, labels = train_gen[i] + train_labels.extend(np.argmax(labels, axis=1)) + if len(train_labels) >= train_gen.samples: + break + + # Calcular pesos balanceados + class_weights = class_weight.compute_class_weight( + 'balanced', + classes=np.unique(train_labels), + y=train_labels + ) + class_weight_dict = dict(zip(np.unique(train_labels), class_weights)) + print(f"⚖️ Pesos de clase: {class_weight_dict}") + + except Exception as e: + print(f"⚠️ Error calculando pesos de clase: {e}") + class_weight_dict = None + + # Callbacks para entrenamiento + early_stopping = tf.keras.callbacks.EarlyStopping( + monitor='val_loss', + patience=7, + restore_best_weights=True, + verbose=1 + ) + + model_checkpoint = tf.keras.callbacks.ModelCheckpoint( + os.path.join(args.output_dir, 'best_model.keras'), + save_best_only=True, + monitor='val_loss', + verbose=1 + ) + + reduce_lr = tf.keras.callbacks.ReduceLROnPlateau( + monitor='val_loss', + factor=0.2, + patience=3, + min_lr=1e-7, + verbose=1 + ) + + callbacks = [early_stopping, model_checkpoint, reduce_lr] + + # Entrenamiento inicial + print(f"\n🏋️ === Entrenamiento inicial ({args.epochs} épocas) ===") + + try: + history = model.fit( + train_gen, + validation_data=val_gen, + epochs=args.epochs, + callbacks=callbacks, + class_weight=class_weight_dict, + verbose=1 + ) + + print("✅ Entrenamiento inicial completado") + + except Exception as e: + print(f"❌ Error durante entrenamiento: {e}") + # Entrenar sin class_weight si hay problemas + print("🔄 Intentando entrenamiento sin pesos de clase...") + history = model.fit( + train_gen, + validation_data=val_gen, + epochs=args.epochs, + callbacks=callbacks, + verbose=1 + ) + + # Fine-tuning + print("\n🔧 === Fine-tuning ===") + + # Descongelar algunas capas del modelo base + base_model.trainable = True + fine_tune_at = 100 # Descongelar las últimas 100 capas + + for layer in base_model.layers[:fine_tune_at]: + layer.trainable = False + + # Recompilar con learning rate más bajo + model.compile( + optimizer=tf.keras.optimizers.Adam(learning_rate=1e-5), + loss='categorical_crossentropy', + metrics=['accuracy'] + ) + + # Continuar entrenamiento + fine_tune_epochs = 10 + total_epochs = len(history.history['loss']) + fine_tune_epochs + + try: + history_fine = model.fit( + train_gen, + validation_data=val_gen, + epochs=total_epochs, + initial_epoch=len(history.history['loss']), + callbacks=callbacks, + verbose=1 + ) + + print("✅ Fine-tuning completado") + + # Combinar historiales + for key in history.history: + if key in history_fine.history: + history.history[key].extend(history_fine.history[key]) + + except Exception as e: + print(f"⚠️ Error durante fine-tuning: {e}") + print("Continuando con modelo del entrenamiento inicial...") + + # Evaluación final + print("\n📊 === Evaluación en conjunto de prueba ===") + + # Cargar mejor modelo + try: + model.load_weights(os.path.join(args.output_dir, 'best_model.keras')) + print("✅ Cargado mejor modelo guardado") + except: + print("⚠️ Usando modelo actual") + + # Guardar modelo final + model.save(os.path.join(args.output_dir, 'final_model.keras')) + print("💾 Modelo final guardado") + + # Predicciones en test + test_gen.reset() + y_pred_prob = model.predict(test_gen, verbose=1) + y_pred = np.argmax(y_pred_prob, axis=1) + y_true = test_gen.classes + + # Mapeo de índices a nombres de clase + index_to_class = {v: k for k, v in class_indices.items()} + + # Obtener solo las clases que realmente aparecen en el conjunto de test + unique_test_classes = np.unique(np.concatenate([y_true, y_pred])) + test_class_names = [index_to_class[i] for i in unique_test_classes] + + print(f"📊 Clases en conjunto de test: {len(unique_test_classes)}") + print(f"📊 Todas las clases entrenadas: {len(class_indices)}") + print(f"📊 Clases presentes en test: {test_class_names}") + + # Verificar si hay clases faltantes + all_classes = set(range(len(class_indices))) + test_classes = set(unique_test_classes) + missing_classes = all_classes - test_classes + + if missing_classes: + missing_names = [index_to_class[i] for i in missing_classes] + print(f"⚠️ Clases sin muestras en test: {missing_names}") + + # Reporte de clasificación con clases filtradas + print("\n📋 === Reporte de Clasificación ===") + try: + report = classification_report( + y_true, y_pred, + labels=unique_test_classes, # Especificar las clases exactas + target_names=test_class_names, + output_dict=False, + zero_division=0 # Manejar divisiones por cero + ) + print(report) + + # Guardar reporte + with open(os.path.join(args.output_dir, 'classification_report.txt'), 'w') as f: + f.write(f"Clases evaluadas: {test_class_names}\n") + f.write(f"Clases faltantes en test: {[index_to_class[i] for i in missing_classes] if missing_classes else 'Ninguna'}\n\n") + f.write(report) + + except Exception as e: + print(f"❌ Error en classification_report: {e}") + print("📊 Generando reporte alternativo...") + + # Reporte manual si falla el automático + from collections import Counter + true_counts = Counter(y_true) + pred_counts = Counter(y_pred) + + print("\n📊 Distribución manual:") + print("Clase | Verdaderos | Predichos") + print("-" * 35) + for class_idx in unique_test_classes: + class_name = index_to_class[class_idx] + true_count = true_counts.get(class_idx, 0) + pred_count = pred_counts.get(class_idx, 0) + print(f"{class_name[:15]:15} | {true_count:10} | {pred_count:9}") + + # Calcular accuracy básico + accuracy = np.mean(y_true == y_pred) + print(f"\n📊 Accuracy general: {accuracy:.4f}") + + # Guardar reporte manual + with open(os.path.join(args.output_dir, 'classification_report.txt'), 'w') as f: + f.write("REPORTE MANUAL DE CLASIFICACIÓN\n") + f.write("=" * 40 + "\n\n") + f.write(f"Clases evaluadas: {test_class_names}\n") + f.write(f"Clases faltantes en test: {[index_to_class[i] for i in missing_classes] if missing_classes else 'Ninguna'}\n\n") + f.write("Distribución por clase:\n") + f.write("Clase | Verdaderos | Predichos\n") + f.write("-" * 35 + "\n") + for class_idx in unique_test_classes: + class_name = index_to_class[class_idx] + true_count = true_counts.get(class_idx, 0) + pred_count = pred_counts.get(class_idx, 0) + f.write(f"{class_name[:15]:15} | {true_count:10} | {pred_count:9}\n") + f.write(f"\nAccuracy general: {accuracy:.4f}\n") + + # Matriz de confusión con clases filtradas + cm = confusion_matrix(y_true, y_pred, labels=unique_test_classes) + print(f"\n🔢 Matriz de Confusión ({len(unique_test_classes)} clases):") + print(cm) + + np.savetxt(os.path.join(args.output_dir, 'confusion_matrix.csv'), + cm, delimiter=',', fmt='%d') + + # Visualizaciones con clases filtradas + print("\n📈 === Generando visualizaciones ===") + + # Gráfico de entrenamiento + plot_training_history(history, args.output_dir) + + # Matriz de confusión visual con clases filtradas + plot_confusion_matrix(cm, test_class_names, args.output_dir) + + # Ejemplos de predicciones con clases filtradas + plot_prediction_examples(test_gen, y_true, y_pred, test_class_names, args.output_dir, unique_test_classes) + + print(f"\n🎉 === Pipeline completado ===") + print(f"📁 Resultados guardados en: {args.output_dir}") + print(f"📊 Precisión final en test: {np.mean(y_true == y_pred):.4f}") + print(f"📊 Clases evaluadas: {len(unique_test_classes)}/{len(class_indices)}") + + # Información adicional sobre clases desbalanceadas + if missing_classes: + print(f"\n⚠️ === Información sobre Clases Desbalanceadas ===") + print(f"❌ Clases sin muestras en test: {len(missing_classes)}") + for missing_idx in missing_classes: + missing_name = index_to_class[missing_idx] + print(f" - {missing_name} (índice {missing_idx})") + print(f"💡 Sugerencia: Considera aumentar el dataset o fusionar clases similares") + +def plot_training_history(history, output_dir): + """Graficar historial de entrenamiento""" + try: + plt.figure(figsize=(12, 4)) + + # Accuracy + plt.subplot(1, 2, 1) + plt.plot(history.history['accuracy'], label='Entrenamiento') + if 'val_accuracy' in history.history: + plt.plot(history.history['val_accuracy'], label='Validación') + plt.title('Precisión del Modelo') + plt.xlabel('Época') + plt.ylabel('Precisión') + plt.legend() + plt.grid(True) + + # Loss + plt.subplot(1, 2, 2) + plt.plot(history.history['loss'], label='Entrenamiento') + if 'val_loss' in history.history: + plt.plot(history.history['val_loss'], label='Validación') + plt.title('Pérdida del Modelo') + plt.xlabel('Época') + plt.ylabel('Pérdida') + plt.legend() + plt.grid(True) + + plt.tight_layout() + plt.savefig(os.path.join(output_dir, 'training_history.png'), dpi=300, bbox_inches='tight') + plt.close() + print("✅ Gráfico de entrenamiento guardado") + + except Exception as e: + print(f"⚠️ Error creando gráfico de entrenamiento: {e}") + +def plot_confusion_matrix(cm, class_names, output_dir): + """Graficar matriz de confusión""" + try: + plt.figure(figsize=(10, 8)) + sns.heatmap(cm, annot=True, fmt='d', cmap='Blues', + xticklabels=class_names, yticklabels=class_names) + plt.title('Matriz de Confusión') + plt.ylabel('Etiqueta Verdadera') + plt.xlabel('Etiqueta Predicha') + plt.xticks(rotation=45, ha='right') + plt.yticks(rotation=0) + plt.tight_layout() + plt.savefig(os.path.join(output_dir, 'confusion_matrix.png'), dpi=300, bbox_inches='tight') + plt.close() + print("✅ Matriz de confusión guardada") + + except Exception as e: + print(f"⚠️ Error creando matriz de confusión: {e}") + +def plot_prediction_examples(test_gen, y_true, y_pred, class_names, output_dir, unique_classes=None, n_examples=12): + """Mostrar ejemplos de predicciones correctas e incorrectas""" + try: + # Obtener índices de predicciones correctas e incorrectas + correct_idx = np.where(y_true == y_pred)[0] + incorrect_idx = np.where(y_true != y_pred)[0] + + # Seleccionar ejemplos + n_correct = min(n_examples // 2, len(correct_idx)) + n_incorrect = min(n_examples // 2, len(incorrect_idx)) + + selected_correct = np.random.choice(correct_idx, n_correct, replace=False) if len(correct_idx) > 0 else [] + selected_incorrect = np.random.choice(incorrect_idx, n_incorrect, replace=False) if len(incorrect_idx) > 0 else [] + + selected_indices = np.concatenate([selected_correct, selected_incorrect]) + + if len(selected_indices) == 0: + print("⚠️ No hay ejemplos para mostrar") + return + + # Crear gráfico + n_show = len(selected_indices) + cols = 4 + rows = (n_show + cols - 1) // cols + + plt.figure(figsize=(15, 4 * rows)) + + for i, idx in enumerate(selected_indices): + plt.subplot(rows, cols, i + 1) + + # Obtener imagen + # Nota: esto es una aproximación, idealmente necesitaríamos acceder a las imágenes originales + img_path = test_gen.filepaths[idx] + img = plt.imread(img_path) + + plt.imshow(img) + plt.axis('off') + + true_label = class_names[y_true[idx]] + pred_label = class_names[y_pred[idx]] + + color = 'green' if y_true[idx] == y_pred[idx] else 'red' + plt.title(f'Real: {true_label}\nPredicción: {pred_label}', + color=color, fontsize=10) + + plt.suptitle('Ejemplos de Predicciones (Verde=Correcta, Rojo=Incorrecta)', fontsize=14) + plt.tight_layout() + plt.savefig(os.path.join(output_dir, 'prediction_examples.png'), dpi=300, bbox_inches='tight') + plt.close() + print("✅ Ejemplos de predicciones guardados") + + except Exception as e: + print(f"⚠️ Error creando ejemplos de predicciones: {e}") + +if __name__ == "__main__": + main() + +# ----------------- MAIN ----------------- +print('\n=== Start of the pipeline ===') +df = safe_read_csv(CSV_PATH) +print('Total registered images in the CSV:', len(df)) + +# Check columns +required_cols = {'id_img','fase V'} +if not required_cols.issubset(set(df.columns)): + raise ValueError(f'CSV must contain the columns: {required_cols}') + +# Prepare folders +SPLIT_DIR = os.path.join(PROJECT_PATH, 'results_nocc/split_fase V') +if FORCE_SPLIT: + shutil.rmtree(SPLIT_DIR, ignore_errors=True) + +if not os.path.exists(SPLIT_DIR): + print("Creating a new split...") + train_df, val_df, test_df = prepare_image_folders(df, IMAGES_DIR, SPLIT_DIR) +else: + print("Reusing existing split...") + # Load the dataframes from the created split directories + train_df = pd.DataFrame([(f.name.split('.')[0], Path(f).parent.name) for f in Path(os.path.join(SPLIT_DIR, 'train')).rglob('*.jpg')], columns=['id_img', 'fase']) + val_df = pd.DataFrame([(f.name.split('.')[0], Path(f).parent.name) for f in Path(os.path.join(SPLIT_DIR, 'val')).rglob('*.jpg')], columns=['id_img', 'fase']) + test_df = pd.DataFrame([(f.name.split('.')[0], Path(f).parent.name) for f in Path(os.path.join(SPLIT_DIR, 'test')).rglob('*.jpg')], columns=['id_img', 'fase']) + + +# Data generators +train_datagen = ImageDataGenerator(rescale=1./255, + rotation_range=20, + width_shift_range=0.1, + height_shift_range=0.1, + shear_range=0.1, + zoom_range=0.1, + horizontal_flip=True, + fill_mode='nearest') + +val_test_datagen = ImageDataGenerator(rescale=1./255) + +train_gen = train_datagen.flow_from_directory(os.path.join(SPLIT_DIR,'train'), + target_size=IMG_SIZE, + batch_size=BATCH_SIZE, + class_mode='categorical', + seed=SEED) + +val_gen = val_test_datagen.flow_from_directory(os.path.join(SPLIT_DIR,'val'), + target_size=IMG_SIZE, + batch_size=BATCH_SIZE, + class_mode='categorical', + shuffle=False) + +test_gen = val_test_datagen.flow_from_directory(os.path.join(SPLIT_DIR,'test'), + target_size=IMG_SIZE, + batch_size=BATCH_SIZE, + class_mode='categorical', + shuffle=False) + +# Save class mapping->index +class_indices = train_gen.class_indices +print('Class indices:', class_indices) +import json +with open(os.path.join(OUTPUT_DIR,'class_indices.txt'),'w') as f: + json.dump(class_indices, f) + +# ----------------- Modelo (Transfer Learning MobileNetV2) - Retraining from scratch ----------------- +print('\n=== Inicio del entrenamiento desde cero ===') + +# Define the MobileNetV2 base model +base_model = MobileNetV2(weights='imagenet', include_top=False, input_shape=(*IMG_SIZE,3)) + +# Set the base model to not be trainable initially +base_model.trainable = False + +# Build a new sequential model +model = models.Sequential([ + base_model, + layers.GlobalAveragePooling2D(), + layers.Dropout(0.3), + layers.Dense(128, activation='relu'), + layers.Dropout(0.3), + layers.Dense(train_gen.num_classes, activation='softmax') # Use the correct number of classes +]) + +# Compile the new model +model.compile(optimizer=tf.keras.optimizers.Adam(learning_rate=1e-3), + loss='categorical_crossentropy', + metrics=['accuracy']) + +model.summary() + +# Define callbacks +early = tf.keras.callbacks.EarlyStopping(monitor='val_loss', patience=5, restore_best_weights=True) +chk = tf.keras.callbacks.ModelCheckpoint(os.path.join(OUTPUT_DIR,'best_model.keras'), save_best_only=True) + +# Calculate class weights (re-calculate in case the dataframe changed) +from sklearn.utils import class_weight +class_weights = class_weight.compute_class_weight( + 'balanced', + classes=np.unique(train_df['fase']), + y=train_df['fase'] +) +class_weights = dict(zip(np.unique(train_gen.classes), class_weights)) +print("Class weights for training:", class_weights) + +# Train the new model +EPOCHS = 45 # Use the original number of epochs for the first phase +history = model.fit( + train_gen, + validation_data=val_gen, + epochs=EPOCHS, + callbacks=[early, chk], + class_weight=class_weights +) + +# --- FINE-TUNING --- +print('\n=== Start of fine-tuning phase ===') + +# Thawing some layers +base_model.trainable = True +fine_tune_at = 100 # This value could be adjusted, for example the last 100 layers +for layer in base_model.layers[:fine_tune_at]: + layer.trainable = False + +# Recompiling the model with a lower learning rate +model.compile(optimizer=tf.keras.optimizers.Adam(learning_rate=1e-5), + loss='categorical_crossentropy', + metrics=['accuracy']) + +model.summary() + +# Continuar entrenamiento +fine_tune_epochs = 10 +total_epochs = EPOCHS + fine_tune_epochs +history_fine_tune = model.fit( + train_gen, + validation_data=val_gen, + epochs=total_epochs, + initial_epoch=history.epoch[-1], + callbacks=[early, chk] # Using the same callbacks +) + +# ----------------- Evaluación en Test ----------------- +print('\n=== Evaluation in a test set ===') + +# Load the best weights saved during the training (from either initial or fine-tuning phase) +model.load_weights(os.path.join(OUTPUT_DIR,'best_model.keras')) + +# Predictions +y_pred_prob = model.predict(test_gen) +y_pred = np.argmax(y_pred_prob, axis=1) +y_true = test_gen.classes + +# Load the full class indices mapping +import json +with open(os.path.join(OUTPUT_DIR,'class_indices.txt'),'r') as f: + full_class_indices = json.load(f) + +# Get the corresponding class names for all classes from the loaded class_indices +index_to_class = {v: k for k, v in full_class_indices.items()} +all_class_names = [index_to_class[i] for i in sorted(index_to_class.keys())] + +# Get the unique class indices present in the test set +unique_test_indices = np.unique(y_true) + +# Get the corresponding class names for the unique test indices +test_labels_filtered = [index_to_class[i] for i in unique_test_indices] + + +# Reporte +report = classification_report(y_true, y_pred, labels=unique_test_indices, target_names=test_labels_filtered) # Use unique_test_indices for labels and test_labels_filtered for target_names +cm = confusion_matrix(y_true, y_pred, labels=unique_test_indices) # Specify labels for confusion matrix + +print('\nClassification Report:\n', report) +print('\nConfusion Matrix:\n', cm) + +with open(os.path.join(OUTPUT_DIR,'classification_report.txt'),'w') as f: + f.write(report) +np.savetxt(os.path.join(OUTPUT_DIR,'confusion_matrix.csv'), cm, delimiter=',', fmt='%d') + +# ----------------- Visualizations ----------------- + +def show_examples(test_gen, y_true, y_pred, labels, n=6): + filepaths = [] + for i in range(len(test_gen.filepaths)): + filepaths.append(test_gen.filepaths[i]) + # select examples + correct_idx = [i for i,(a,b) in enumerate(zip(y_true,y_pred)) if a==b] + wrong_idx = [i for i,(a,b) in enumerate(zip(y_true,y_pred)) if a!=b] + examples = (correct_idx[:n//2] if len(correct_idx)>0 else []) + (wrong_idx[:n//2] if len(wrong_idx)>0 else []) + + plt.figure(figsize=(15,8)) + for i, idx in enumerate(examples): + img = plt.imread(filepaths[idx]) + plt.subplot(2, n//2, i+1) + plt.imshow(img) + plt.axis('off') + plt.title(f'True: {labels[y_true[idx]]}\nPred: {labels[y_pred[idx]]}') + plt.suptitle('Examples: Right and Wrong') + plt.show() + +# Call the function with the correct variables from the evaluation step +show_examples(test_gen, y_true, y_pred, all_class_names, n=6) + +# +plt.figure(figsize=(8,4)) +plt.plot(history.history['accuracy'], label='train_acc') +plt.plot(history.history['val_accuracy'], label='val_acc') +# Include fine-tuning history if it exists +if 'history_fine_tune' in locals(): + plt.plot(history_fine_tune.history['accuracy'], label='fine_tune_train_acc') + plt.plot(history_fine_tune.history['val_accuracy'], label='fine_tune_val_acc') + +plt.title('Accuracy during training') +plt.xlabel('Epoch') +plt.ylabel('Accuracy') +plt.legend() +plt.grid() +plt.show() + +# Heatmap of the confusion matrix +plt.figure(figsize=(8,6)) +sns.heatmap(cm, annot=True, fmt='d', xticklabels=all_class_names, yticklabels=all_class_names, cmap='Blues') +plt.xlabel('Prediction') +plt.ylabel('True') +plt.title('Matriz de confusión') +plt.show() \ No newline at end of file diff --git a/Code/Phenology_V1.py b/Code/Supervised_learning/Phenology_V1.py similarity index 100% rename from Code/Phenology_V1.py rename to Code/Supervised_learning/Phenology_V1.py diff --git a/Code/Supervised_learning/README_ResNet50.md b/Code/Supervised_learning/README_ResNet50.md new file mode 100644 index 0000000..26d9ed9 --- /dev/null +++ b/Code/Supervised_learning/README_ResNet50.md @@ -0,0 +1,268 @@ +# ResNet50 Transfer Learning para Nocciola - Documentación + +## 🚀 **Implementación Completa de ResNet50** + +Este proyecto implementa transfer learning con ResNet50 para clasificación de fases fenológicas de nocciola, basado en la estructura exitosa de MobileNetV1.py pero optimizado específicamente para ResNet50. + +## 📁 **Estructura de Archivos** + +``` +Code/Supervised_learning/ +├── ResNET.py # Script principal ResNet50 ✅ NUEVO +├── train_resnet50.py # Script facilitado de entrenamiento ✅ NUEVO +├── MobileNetV1.py # Script MobileNetV2 (referencia exitosa) +└── README_ResNet50.md # Esta documentación ✅ NUEVO +``` + +## 🔧 **Características Específicas de ResNet50** + +### **Arquitectura Optimizada:** +```python +# Modelo base ResNet50 +base_model = ResNet50( + weights='imagenet', + include_top=False, + input_shape=(224, 224, 3) +) + +# Capas personalizadas optimizadas para ResNet50 +model = Sequential([ + base_model, + GlobalAveragePooling2D(), + BatchNormalization(), # ✅ Importante para ResNet50 + Dropout(0.5), # ✅ Dropout más alto + Dense(256, activation='relu'), # ✅ Más neuronas + BatchNormalization(), + Dropout(0.3), + Dense(num_classes, activation='softmax') +]) +``` + +### **Optimizaciones Específicas:** + +1. **Learning Rate Conservador:** + - Inicial: `5e-4` (vs `1e-3` en MobileNet) + - Fine-tuning: `1e-6` (muy bajo para estabilidad) + +2. **Data Augmentation Reducido:** + - Rotación: 15° (vs 20° en MobileNet) + - Shifts/Zoom: 0.08 (vs 0.1 en MobileNet) + - Más conservador para evitar overfitting + +3. **Fine-tuning Selectivo:** + - Solo últimas capas residuales (conv5_x) + - Punto de corte: capa 140 (de 175 total) + - Preserva features básicas, adapta características específicas + +4. **Callbacks Adaptados:** + - Paciencia EarlyStopping: 10 (vs 7 en MobileNet) + - Más épocas de fine-tuning: 15 (vs 10) + - Factor ReduceLR: 0.3 (más agresivo) + +## 📊 **Comparación ResNet50 vs MobileNetV2** + +| Característica | ResNet50 | MobileNetV2 | +|----------------|----------|-------------| +| **Parámetros** | ~25M | ~3.4M | +| **Memoria** | Alta | Baja | +| **Velocidad** | Lenta | Rápida | +| **Precisión** | Alta | Media-Alta | +| **Overfitting** | Más propenso | Menos propenso | +| **Dataset ideal** | Grande/Complejo | Pequeño/Mediano | + +## 🎯 **Casos de Uso Recomendados** + +### **Usar ResNet50 cuando:** +- ✅ Dataset tiene suficientes muestras (>1000) +- ✅ Patrones complejos en imágenes +- ✅ Precisión es más importante que velocidad +- ✅ Hardware robusto disponible +- ✅ Investigación/análisis detallado + +### **Usar MobileNetV2 cuando:** +- ✅ Dataset pequeño (<500 muestras) +- ✅ Velocidad es importante +- ✅ Recursos limitados +- ✅ Producción/móvil +- ✅ Prototipado rápido + +## 🚀 **Ejecución** + +### **Opción 1: Script Facilitado (Recomendado)** +```bash +# Entrenar solo ResNet50 +python train_resnet50.py --model resnet50 --epochs 25 + +# Comparar ambos modelos +python train_resnet50.py --model both --epochs 20 + +# Ver comparación detallada +python train_resnet50.py --compare +``` + +### **Opción 2: Ejecución Directa** +```bash +# ResNet50 básico +python ResNET.py --epochs 25 --force_split + +# ResNet50 con dataset específico +python ResNET.py --csv_path "assignments.csv" --epochs 30 +``` + +### **Parámetros Disponibles:** +- `--csv_path`: Ruta al CSV (default: assignments.csv) +- `--images_dir`: Directorio de imágenes +- `--output_dir`: Directorio de resultados (default: results_resnet50_faseV) +- `--epochs`: Épocas de entrenamiento (default: 25) +- `--force_split`: Recrear división de datos + +## 📈 **Proceso de Entrenamiento ResNet50** + +### **Fase 1: Entrenamiento Inicial** +1. Base model congelado (ImageNet weights) +2. Solo entrenar capas personalizadas +3. Learning rate conservador (5e-4) +4. Callbacks con paciencia aumentada + +### **Fase 2: Fine-tuning Selectivo** +5. Descongelar solo conv5_x layers (últimas capas residuales) +6. Learning rate muy bajo (1e-6) +7. 15 épocas adicionales +8. Monitoreo estricto de overfitting + +### **Evaluación Robusta:** +9. Manejo automático de clases desbalanceadas +10. Métricas específicas para ResNet50 +11. Visualizaciones diferenciadas + +## 📂 **Resultados Generados** + +### **Modelos:** +- `best_resnet50_model.keras`: Mejor modelo durante entrenamiento +- `final_resnet50_model.keras`: Modelo final completo + +### **Reportes:** +- `classification_report.txt`: Reporte detallado con marcador ResNet50 +- `confusion_matrix.csv`: Matriz numérica + +### **Visualizaciones:** +- `resnet50_training_history.png`: Gráficos de entrenamiento +- `resnet50_confusion_matrix.png`: Matriz visual +- `resnet50_prediction_examples.png`: Ejemplos de predicciones + +### **Data Splits:** +- `train_split.csv`, `val_split.csv`, `test_split.csv` +- `class_indices.json`: Mapeo de clases + +## ⚙️ **Configuración Técnica** + +### **Requisitos del Sistema:** +- RAM: Mínimo 8GB (recomendado 16GB) +- GPU: Opcional pero muy recomendada +- Espacio: ~3GB para resultados completos + +### **Dependencias:** +```bash +tensorflow>=2.8.0 +scikit-learn +pandas +numpy +matplotlib +seaborn +``` + +### **Configuración Interna:** +```python +IMG_SIZE = (224, 224) # Estándar ImageNet +BATCH_SIZE = 16 # Reducido para ResNet50 +SPLIT = {'train': 0.7, 'val': 0.15, 'test': 0.15} +``` + +## 🔍 **Diferencias Implementadas vs MobileNet** + +### **1. Arquitectura:** +- BatchNormalization adicional +- Dropout más agresivo (0.5 vs 0.3) +- Dense layer mayor (256 vs 128) + +### **2. Entrenamiento:** +- Learning rates más conservadores +- Fine-tuning más selectivo +- Más épocas de fine-tuning + +### **3. Data Augmentation:** +- Rotaciones menores (15° vs 20°) +- Shifts reducidos (0.08 vs 0.1) +- Menos agresivo para ResNet50 + +### **4. Callbacks:** +- Paciencia aumentada (10 vs 7) +- Factor ReduceLR más agresivo (0.3 vs 0.2) +- Monitoreo específico para ResNet + +### **5. Outputs:** +- Nombres diferenciados (`resnet50_*`) +- Reportes marcados con modelo +- Métricas específicas + +## 🎯 **Recomendaciones de Uso** + +### **Para Dataset Nocciola Actual:** +Dado que el dataset es relativamente pequeño (~500 muestras): + +1. **Primera opción:** MobileNetV2 (más adecuado) +2. **Segunda opción:** ResNet50 con regularización fuerte +3. **Comparación:** Entrenar ambos y comparar resultados + +### **Comando Recomendado:** +```bash +# Comparar ambos modelos con pocas épocas +python train_resnet50.py --model both --epochs 15 + +# Analizar resultados y elegir el mejor +``` + +## 🚨 **Solución de Problemas** + +### **Overfitting en ResNet50:** +- Reducir épocas de entrenamiento +- Aumentar dropout +- Usar dataset filtrado +- Más data augmentation + +### **Underfitting:** +- Aumentar épocas +- Reducir regularización +- Learning rate más alto +- Descongelar más capas + +### **Problemas de memoria:** +- Reducir BATCH_SIZE a 8 o 4 +- Usar gradient checkpointing +- Cerrar otras aplicaciones + +## 📊 **Interpretación de Resultados** + +### **Métricas Esperadas:** +- **ResNet50:** Mayor precision, posible overfitting +- **MobileNetV2:** Más generalizable, menos overfitting + +### **Comparación Visual:** +- Training curves más suaves en MobileNet +- Posible gap train/val en ResNet50 +- Matriz de confusión similar o mejor en ResNet50 + +### **Decisión Final:** +Elegir modelo basado en: +1. Accuracy en test set +2. Diferencia train/validation +3. Requisitos de producción +4. Interpretabilidad de errores + +--- + +## 🎉 **¡ResNet50 Implementado Exitosamente!** + +El modelo ResNet50 está completamente implementado usando la misma estructura robusta que MobileNetV1.py, con optimizaciones específicas para ResNet50 y manejo automático de clases desbalanceadas. + +**Para comenzar:** `python train_resnet50.py --model resnet50 --epochs 20` \ No newline at end of file diff --git a/Code/Supervised_learning/ResNET.py b/Code/Supervised_learning/ResNET.py new file mode 100644 index 0000000..1ba4c23 --- /dev/null +++ b/Code/Supervised_learning/ResNET.py @@ -0,0 +1,741 @@ +""" +ResNet50 Transfer Learning para Clasificación de Fases Fenológicas - Nocciola +Adaptado para Visual Studio Code basado en MobileNetV1.py exitoso +Dataset: Nocciola GBIF +Objetivo: Predecir fase (fenológica vegetativa) +""" + +# ============================================================================= +# LIBRARY IMPORTS +# ============================================================================= + +# System and file handling libraries +import os # Operating system interface (paths, directories) +import shutil # High-level file operations (copy, move, delete) +import argparse # Command-line argument parsing +import random # Random number generation for reproducibility +import pandas as pd # Data manipulation and analysis +import numpy as np # Numerical computing with multidimensional arrays +from pathlib import Path # Modern path handling +import json + +# Data visualization libraries +import matplotlib.pyplot as plt # Plotting and visualization +import seaborn as sns # Statistical data visualization + +# Deep learning libraries (TensorFlow and Keras) +import tensorflow as tf +from tensorflow.keras import layers, models +from tensorflow.keras.applications import ResNet50 +from tensorflow.keras.preprocessing.image import ImageDataGenerator +from sklearn.metrics import classification_report, confusion_matrix +from sklearn.utils import class_weight + +# ----------------- CONFIG ----------------- +PROJECT_PATH = r'C:\Users\sof12\Desktop\ML\Datasets\Nocciola\combi' +IMAGES_DIR = r'C:\Users\sof12\Desktop\ML\Datasets\Nocciola\combi' # Las imágenes están en el directorio principal +CSV_PATH = os.path.join(PROJECT_PATH, 'assignments.csv') # CSV principal +OUTPUT_DIR = os.path.join(PROJECT_PATH, 'results_resnet50') +os.makedirs(OUTPUT_DIR, exist_ok=True) + +IMG_SIZE = (224, 224) # Recomendado para ResNet50 (tamaño estándar ImageNet) +BATCH_SIZE = 16 # Reducido para ResNet50 (más pesado que MobileNet) +SEED = 42 +SPLIT = {'train': 0.7, 'val': 0.15, 'test': 0.15} +FORCE_SPLIT = False + +# ----------------- Utilities ----------------- +def set_seed(seed=42): + """Establecer semilla para reproducibilidad""" + random.seed(seed) + np.random.seed(seed) + tf.random.set_seed(seed) + +def analyze_class_distribution(df, column_name='fase V'): + """Analizar distribución de clases y detectar desbalances""" + print(f"\n📊 === Análisis de Distribución de Clases ===") + + # Contar por clase + counts = df[column_name].value_counts() + total = len(df) + + print(f"📊 Total de muestras: {total}") + print(f"📊 Número de clases: {len(counts)}") + print(f"📊 Distribución por clase:") + + # Mostrar estadísticas detalladas + for clase, count in counts.items(): + percentage = (count / total) * 100 + print(f" - {clase}: {count} muestras ({percentage:.1f}%)") + + # Detectar clases problemáticas + min_samples = 5 # Umbral mínimo recomendado + small_classes = counts[counts < min_samples] + + if len(small_classes) > 0: + print(f"\n⚠️ Clases con menos de {min_samples} muestras:") + for clase, count in small_classes.items(): + print(f" - {clase}: {count} muestras") + + print(f"\n💡 Recomendaciones:") + print(f" 1. Considera recolectar más datos para estas clases") + print(f" 2. O fusionar clases similares") + print(f" 3. O usar técnicas de data augmentation específicas") + + return counts, small_classes + +def safe_read_csv(path): + """Leer CSV con manejo de encoding""" + if not os.path.exists(path): + raise FileNotFoundError(f'CSV no encontrado: {path}') + try: + df = pd.read_csv(path, encoding='utf-8') + except UnicodeDecodeError: + try: + df = pd.read_csv(path, encoding='latin-1') + except: + df = pd.read_csv(path, encoding='cp1252') + + return df + +def resolve_image_path(images_dir, img_id): + """Resolver la ruta completa de una imagen""" + if pd.isna(img_id) or str(img_id).strip() == '': + return None + + img_id = str(img_id).strip() + + # Verificar si ya incluye extensión y existe + direct_path = os.path.join(images_dir, img_id) + if os.path.exists(direct_path): + return direct_path + + # Probar con extensiones comunes + stem = os.path.splitext(img_id)[0] + for ext in ['.jpg', '.jpeg', '.png', '.JPG', '.JPEG', '.PNG']: + img_path = os.path.join(images_dir, stem + ext) + if os.path.exists(img_path): + return img_path + + return None + +def prepare_image_folders(df, images_dir, out_dir, split=SPLIT, seed=SEED): + """Crear estructura de carpetas para flow_from_directory""" + set_seed(seed) + + # Filtrar solo filas con fase V válida e imágenes existentes + print(f"📊 Datos iniciales: {len(df)} filas") + + # Filtrar filas con fase V válida + df_valid = df.dropna(subset=['fase V']).copy() + df_valid = df_valid[df_valid['fase V'].str.strip() != ''] + print(f"📊 Con fase V válida: {len(df_valid)} filas") + + # Verificar existencia de imágenes + valid_rows = [] + for _, row in df_valid.iterrows(): + img_path = resolve_image_path(images_dir, row['id_img']) + if img_path: + valid_rows.append(row) + else: + print(f"⚠️ Imagen no encontrada: {row['id_img']}") + + if not valid_rows: + raise ValueError("❌ No se encontraron imágenes válidas") + + df_final = pd.DataFrame(valid_rows) + print(f"📊 Con imágenes existentes: {len(df_final)} filas") + + # Mostrar distribución de clases + fase_counts = df_final['fase V'].value_counts() + print(f"\n📊 Distribución de fases:") + for fase, count in fase_counts.items(): + print(f" - {fase}: {count} imágenes") + + # Remover clases con muy pocas muestras (menos de 3) + min_samples = 3 + valid_phases = fase_counts[fase_counts >= min_samples].index.tolist() + if len(valid_phases) < len(fase_counts): + excluded = fase_counts[fase_counts < min_samples].index.tolist() + print(f"⚠️ Excluyendo fases con menos de {min_samples} muestras: {excluded}") + df_final = df_final[df_final['fase V'].isin(valid_phases)] + print(f"📊 Después de filtrar: {len(df_final)} filas, {len(valid_phases)} clases") + + labels = df_final['fase V'].unique().tolist() + print(f"📊 Clases finales: {labels}") + + # Mezclar y dividir datos + df_shuffled = df_final.sample(frac=1, random_state=seed).reset_index(drop=True) + n = len(df_shuffled) + n_train = int(n * split['train']) + n_val = int(n * split['val']) + + train_df = df_shuffled.iloc[:n_train] + val_df = df_shuffled.iloc[n_train:n_train + n_val] + test_df = df_shuffled.iloc[n_train + n_val:] + + print(f"📊 División final:") + print(f" - Entrenamiento: {len(train_df)} imágenes") + print(f" - Validación: {len(val_df)} imágenes") + print(f" - Prueba: {len(test_df)} imágenes") + + # Crear estructura de carpetas + for part in ['train', 'val', 'test']: + for label in labels: + label_dir = os.path.join(out_dir, part, str(label)) + os.makedirs(label_dir, exist_ok=True) + + # Función para copiar imágenes + def copy_subset(subdf, subset_name): + copied, missing = 0, 0 + for _, row in subdf.iterrows(): + src = resolve_image_path(images_dir, row['id_img']) + if src: + fase = str(row['fase V']) + dst = os.path.join(out_dir, subset_name, fase, f"{row['id_img']}.jpg") + try: + shutil.copy2(src, dst) + copied += 1 + except Exception as e: + print(f"⚠️ Error copiando {src}: {e}") + missing += 1 + else: + missing += 1 + + print(f"✅ {subset_name}: {copied} imágenes copiadas, {missing} fallidas") + return copied + + # Copiar imágenes a las carpetas correspondientes + copy_subset(train_df, 'train') + copy_subset(val_df, 'val') + copy_subset(test_df, 'test') + + return train_df, val_df, test_df + +def main(): + """Función principal del pipeline ResNet50""" + parser = argparse.ArgumentParser(description='ResNet50 Transfer Learning para Nocciola') + parser.add_argument('--csv_path', type=str, default=CSV_PATH, + help='Ruta al archivo CSV con metadatos') + parser.add_argument('--images_dir', type=str, default=IMAGES_DIR, + help='Directorio con las imágenes') + parser.add_argument('--output_dir', type=str, default=OUTPUT_DIR, + help='Directorio de salida para resultados') + parser.add_argument('--epochs', type=int, default=25, + help='Número de épocas de entrenamiento') + parser.add_argument('--force_split', action='store_true', + help='Forzar recreación del split de datos') + + args = parser.parse_args() + + print('\n🚀 === Inicio del pipeline ResNet50 para Nocciola ===') + print(f"📁 Directorio de imágenes: {args.images_dir}") + print(f"📄 Archivo CSV: {args.csv_path}") + print(f"📂 Directorio de salida: {args.output_dir}") + + # Establecer semilla + set_seed(SEED) + + # Crear directorio de salida + os.makedirs(args.output_dir, exist_ok=True) + + # Leer datos + print('\n📊 === Cargando datos ===') + df = safe_read_csv(args.csv_path) + print(f'📊 Total de registros en CSV: {len(df)}') + print(f'📊 Columnas disponibles: {list(df.columns)}') + + # Verificar columnas requeridas + required_cols = {'id_img', 'fase V'} + if not required_cols.issubset(set(df.columns)): + missing = required_cols - set(df.columns) + raise ValueError(f'❌ CSV debe contener las columnas: {missing}') + + # Analizar distribución de clases antes del procesamiento + analyze_class_distribution(df, 'fase V') + + # Preparar estructura de carpetas + SPLIT_DIR = os.path.join(args.output_dir, 'data_split') + + if args.force_split and os.path.exists(SPLIT_DIR): + print("🗑️ Eliminando split existente...") + shutil.rmtree(SPLIT_DIR) + + if not os.path.exists(SPLIT_DIR): + print("\n📁 === Creando nueva división de datos ===") + train_df, val_df, test_df = prepare_image_folders(df, args.images_dir, SPLIT_DIR) + + # Guardar información del split + train_df.to_csv(os.path.join(args.output_dir, 'train_split.csv'), index=False) + val_df.to_csv(os.path.join(args.output_dir, 'val_split.csv'), index=False) + test_df.to_csv(os.path.join(args.output_dir, 'test_split.csv'), index=False) + + else: + print("\n♻️ === Reutilizando división existente ===") + # Cargar información del split si existe + try: + train_df = pd.read_csv(os.path.join(args.output_dir, 'train_split.csv')) + val_df = pd.read_csv(os.path.join(args.output_dir, 'val_split.csv')) + test_df = pd.read_csv(os.path.join(args.output_dir, 'test_split.csv')) + except: + print("⚠️ No se pudieron cargar los archivos de split, recreando...") + train_df, val_df, test_df = prepare_image_folders(df, args.images_dir, SPLIT_DIR) + + # Crear generadores de datos + print("\n🔄 === Creando generadores de datos ===") + + # Data augmentation para entrenamiento (más conservador para ResNet50) + train_datagen = ImageDataGenerator( + rescale=1./255, + rotation_range=15, # Menos rotación que MobileNet + width_shift_range=0.08, + height_shift_range=0.08, + shear_range=0.08, + zoom_range=0.08, + horizontal_flip=True, + fill_mode='nearest' + ) + + # Solo normalización para validación y test + val_test_datagen = ImageDataGenerator(rescale=1./255) + + # Crear generadores + train_gen = train_datagen.flow_from_directory( + os.path.join(SPLIT_DIR, 'train'), + target_size=IMG_SIZE, + batch_size=BATCH_SIZE, + class_mode='categorical', + seed=SEED + ) + + val_gen = val_test_datagen.flow_from_directory( + os.path.join(SPLIT_DIR, 'val'), + target_size=IMG_SIZE, + batch_size=BATCH_SIZE, + class_mode='categorical', + shuffle=False + ) + + test_gen = val_test_datagen.flow_from_directory( + os.path.join(SPLIT_DIR, 'test'), + target_size=IMG_SIZE, + batch_size=BATCH_SIZE, + class_mode='categorical', + shuffle=False + ) + + # Guardar mapeo de clases + class_indices = train_gen.class_indices + print(f'🏷️ Mapeo de clases: {class_indices}') + + with open(os.path.join(args.output_dir, 'class_indices.json'), 'w') as f: + json.dump(class_indices, f, indent=2) + + print(f"📊 Muestras por conjunto:") + print(f" - Entrenamiento: {train_gen.samples}") + print(f" - Validación: {val_gen.samples}") + print(f" - Prueba: {test_gen.samples}") + print(f" - Número de clases: {train_gen.num_classes}") + + # Crear y entrenar modelo ResNet50 + print("\n🤖 === Construcción del modelo ResNet50 ===") + + # Modelo base ResNet50 + base_model = ResNet50( + weights='imagenet', + include_top=False, + input_shape=(*IMG_SIZE, 3) + ) + base_model.trainable = False # Congelar inicialmente + + # Construir modelo secuencial optimizado para ResNet50 + model = models.Sequential([ + base_model, + layers.GlobalAveragePooling2D(), + layers.BatchNormalization(), # Importante para ResNet50 + layers.Dropout(0.5), # Dropout más alto para ResNet50 + layers.Dense(256, activation='relu'), + layers.BatchNormalization(), + layers.Dropout(0.3), + layers.Dense(train_gen.num_classes, activation='softmax') + ]) + + # Compilar modelo con learning rate más bajo para ResNet50 + model.compile( + optimizer=tf.keras.optimizers.Adam(learning_rate=5e-4), # LR más bajo + loss='categorical_crossentropy', + metrics=['accuracy'] + ) + + print("📋 Resumen del modelo ResNet50:") + model.summary() + + # Calcular pesos de clase + print("\n⚖️ === Calculando pesos de clase ===") + try: + # Obtener etiquetas de entrenamiento + train_labels = [] + for i in range(len(train_gen)): + _, labels = train_gen[i] + train_labels.extend(np.argmax(labels, axis=1)) + if len(train_labels) >= train_gen.samples: + break + + # Calcular pesos balanceados + class_weights = class_weight.compute_class_weight( + 'balanced', + classes=np.unique(train_labels), + y=train_labels + ) + class_weight_dict = dict(zip(np.unique(train_labels), class_weights)) + print(f"⚖️ Pesos de clase: {class_weight_dict}") + + except Exception as e: + print(f"⚠️ Error calculando pesos de clase: {e}") + class_weight_dict = None + + # Callbacks para entrenamiento ResNet50 + early_stopping = tf.keras.callbacks.EarlyStopping( + monitor='val_loss', + patience=10, # Más paciencia para ResNet50 + restore_best_weights=True, + verbose=1 + ) + + model_checkpoint = tf.keras.callbacks.ModelCheckpoint( + os.path.join(args.output_dir, 'best_resnet50_model.keras'), + save_best_only=True, + monitor='val_loss', + verbose=1 + ) + + reduce_lr = tf.keras.callbacks.ReduceLROnPlateau( + monitor='val_loss', + factor=0.3, # Reducción más agresiva + patience=5, # Menos paciencia para reducir LR + min_lr=1e-8, + verbose=1 + ) + + callbacks = [early_stopping, model_checkpoint, reduce_lr] + + # Entrenamiento inicial + print(f"\n🏋️ === Entrenamiento inicial ResNet50 ({args.epochs} épocas) ===") + + try: + history = model.fit( + train_gen, + validation_data=val_gen, + epochs=args.epochs, + callbacks=callbacks, + class_weight=class_weight_dict, + verbose=1 + ) + + print("✅ Entrenamiento inicial completado") + + except Exception as e: + print(f"❌ Error durante entrenamiento: {e}") + # Entrenar sin class_weight si hay problemas + print("🔄 Intentando entrenamiento sin pesos de clase...") + history = model.fit( + train_gen, + validation_data=val_gen, + epochs=args.epochs, + callbacks=callbacks, + verbose=1 + ) + + # Fine-tuning específico para ResNet50 + print("\n🔧 === Fine-tuning ResNet50 ===") + + # Descongelar capas específicas de ResNet50 + base_model.trainable = True + + # Para ResNet50, descongelar solo las últimas capas residuales + # ResNet50 tiene bloques conv5_x que son los más específicos + fine_tune_at = 140 # Aproximadamente desde conv5_block1 + + for layer in base_model.layers[:fine_tune_at]: + layer.trainable = False + + print(f"🔧 Capas entrenables: {len([l for l in base_model.layers if l.trainable])}/{len(base_model.layers)}") + + # Recompilar con learning rate muy bajo para fine-tuning + model.compile( + optimizer=tf.keras.optimizers.Adam(learning_rate=1e-6), # LR muy bajo para ResNet50 + loss='categorical_crossentropy', + metrics=['accuracy'] + ) + + # Continuar entrenamiento + fine_tune_epochs = 15 # Más épocas para fine-tuning + total_epochs = len(history.history['loss']) + fine_tune_epochs + + try: + history_fine = model.fit( + train_gen, + validation_data=val_gen, + epochs=total_epochs, + initial_epoch=len(history.history['loss']), + callbacks=callbacks, + verbose=1 + ) + + print("✅ Fine-tuning completado") + + # Combinar historiales + for key in history.history: + if key in history_fine.history: + history.history[key].extend(history_fine.history[key]) + + except Exception as e: + print(f"⚠️ Error durante fine-tuning: {e}") + print("Continuando con modelo del entrenamiento inicial...") + + # Evaluación final + print("\n📊 === Evaluación en conjunto de prueba ===") + + # Cargar mejor modelo + try: + model.load_weights(os.path.join(args.output_dir, 'best_resnet50_model.keras')) + print("✅ Cargado mejor modelo ResNet50 guardado") + except: + print("⚠️ Usando modelo actual") + + # Guardar modelo final + model.save(os.path.join(args.output_dir, 'final_resnet50_model.keras')) + print("💾 Modelo ResNet50 final guardado") + + # Predicciones en test + test_gen.reset() + y_pred_prob = model.predict(test_gen, verbose=1) + y_pred = np.argmax(y_pred_prob, axis=1) + y_true = test_gen.classes + + # Mapeo de índices a nombres de clase + index_to_class = {v: k for k, v in class_indices.items()} + + # Obtener solo las clases que realmente aparecen en el conjunto de test + unique_test_classes = np.unique(np.concatenate([y_true, y_pred])) + test_class_names = [index_to_class[i] for i in unique_test_classes] + + print(f"📊 Clases en conjunto de test: {len(unique_test_classes)}") + print(f"📊 Todas las clases entrenadas: {len(class_indices)}") + print(f"📊 Clases presentes en test: {test_class_names}") + + # Verificar si hay clases faltantes + all_classes = set(range(len(class_indices))) + test_classes = set(unique_test_classes) + missing_classes = all_classes - test_classes + + if missing_classes: + missing_names = [index_to_class[i] for i in missing_classes] + print(f"⚠️ Clases sin muestras en test: {missing_names}") + + # Reporte de clasificación con clases filtradas + print("\n📋 === Reporte de Clasificación ResNet50 ===") + try: + report = classification_report( + y_true, y_pred, + labels=unique_test_classes, + target_names=test_class_names, + output_dict=False, + zero_division=0 + ) + print(report) + + # Guardar reporte + with open(os.path.join(args.output_dir, 'classification_report.txt'), 'w') as f: + f.write(f"Modelo: ResNet50\n") + f.write(f"Clases evaluadas: {test_class_names}\n") + f.write(f"Clases faltantes en test: {[index_to_class[i] for i in missing_classes] if missing_classes else 'Ninguna'}\n\n") + f.write(report) + + except Exception as e: + print(f"❌ Error en classification_report: {e}") + print("📊 Generando reporte alternativo...") + + # Reporte manual si falla el automático + from collections import Counter + true_counts = Counter(y_true) + pred_counts = Counter(y_pred) + + print("\n📊 Distribución manual:") + print("Clase | Verdaderos | Predichos") + print("-" * 35) + for class_idx in unique_test_classes: + class_name = index_to_class[class_idx] + true_count = true_counts.get(class_idx, 0) + pred_count = pred_counts.get(class_idx, 0) + print(f"{class_name[:15]:15} | {true_count:10} | {pred_count:9}") + + # Calcular accuracy básico + accuracy = np.mean(y_true == y_pred) + print(f"\n📊 Accuracy general ResNet50: {accuracy:.4f}") + + # Guardar reporte manual + with open(os.path.join(args.output_dir, 'classification_report.txt'), 'w') as f: + f.write("REPORTE MANUAL DE CLASIFICACIÓN - ResNet50\n") + f.write("=" * 50 + "\n\n") + f.write(f"Clases evaluadas: {test_class_names}\n") + f.write(f"Clases faltantes en test: {[index_to_class[i] for i in missing_classes] if missing_classes else 'Ninguna'}\n\n") + f.write("Distribución por clase:\n") + f.write("Clase | Verdaderos | Predichos\n") + f.write("-" * 35 + "\n") + for class_idx in unique_test_classes: + class_name = index_to_class[class_idx] + true_count = true_counts.get(class_idx, 0) + pred_count = pred_counts.get(class_idx, 0) + f.write(f"{class_name[:15]:15} | {true_count:10} | {pred_count:9}\n") + f.write(f"\nAccuracy general ResNet50: {accuracy:.4f}\n") + + # Matriz de confusión con clases filtradas + cm = confusion_matrix(y_true, y_pred, labels=unique_test_classes) + print(f"\n🔢 Matriz de Confusión ResNet50 ({len(unique_test_classes)} clases):") + print(cm) + + np.savetxt(os.path.join(args.output_dir, 'confusion_matrix.csv'), + cm, delimiter=',', fmt='%d') + + # Visualizaciones con clases filtradas + print("\n📈 === Generando visualizaciones ResNet50 ===") + + # Gráfico de entrenamiento + plot_training_history(history, args.output_dir) + + # Matriz de confusión visual con clases filtradas + plot_confusion_matrix(cm, test_class_names, args.output_dir) + + # Ejemplos de predicciones con clases filtradas + plot_prediction_examples(test_gen, y_true, y_pred, test_class_names, args.output_dir, unique_test_classes) + + print(f"\n🎉 === Pipeline ResNet50 completado ===") + print(f"📁 Resultados guardados en: {args.output_dir}") + print(f"📊 Precisión final en test: {np.mean(y_true == y_pred):.4f}") + print(f"📊 Clases evaluadas: {len(unique_test_classes)}/{len(class_indices)}") + + # Información adicional sobre clases desbalanceadas + if missing_classes: + print(f"\n⚠️ === Información sobre Clases Desbalanceadas ===") + print(f"❌ Clases sin muestras en test: {len(missing_classes)}") + for missing_idx in missing_classes: + missing_name = index_to_class[missing_idx] + print(f" - {missing_name} (índice {missing_idx})") + print(f"💡 Sugerencia: Considera aumentar el dataset o fusionar clases similares") + +def plot_training_history(history, output_dir): + """Graficar historial de entrenamiento ResNet50""" + try: + plt.figure(figsize=(12, 4)) + + # Accuracy + plt.subplot(1, 2, 1) + plt.plot(history.history['accuracy'], label='Entrenamiento', linewidth=2) + if 'val_accuracy' in history.history: + plt.plot(history.history['val_accuracy'], label='Validación', linewidth=2) + plt.title('Precisión del Modelo ResNet50') + plt.xlabel('Época') + plt.ylabel('Precisión') + plt.legend() + plt.grid(True, alpha=0.3) + + # Loss + plt.subplot(1, 2, 2) + plt.plot(history.history['loss'], label='Entrenamiento', linewidth=2) + if 'val_loss' in history.history: + plt.plot(history.history['val_loss'], label='Validación', linewidth=2) + plt.title('Pérdida del Modelo ResNet50') + plt.xlabel('Época') + plt.ylabel('Pérdida') + plt.legend() + plt.grid(True, alpha=0.3) + + plt.tight_layout() + plt.savefig(os.path.join(output_dir, 'resnet50_training_history.png'), dpi=300, bbox_inches='tight') + plt.close() + print("✅ Gráfico de entrenamiento ResNet50 guardado") + + except Exception as e: + print(f"⚠️ Error creando gráfico de entrenamiento: {e}") + +def plot_confusion_matrix(cm, class_names, output_dir): + """Graficar matriz de confusión ResNet50""" + try: + plt.figure(figsize=(10, 8)) + sns.heatmap(cm, annot=True, fmt='d', cmap='Blues', + xticklabels=class_names, yticklabels=class_names, + cbar_kws={'label': 'Número de muestras'}) + plt.title('Matriz de Confusión - ResNet50') + plt.ylabel('Etiqueta Verdadera') + plt.xlabel('Etiqueta Predicha') + plt.xticks(rotation=45, ha='right') + plt.yticks(rotation=0) + plt.tight_layout() + plt.savefig(os.path.join(output_dir, 'resnet50_confusion_matrix.png'), dpi=300, bbox_inches='tight') + plt.close() + print("✅ Matriz de confusión ResNet50 guardada") + + except Exception as e: + print(f"⚠️ Error creando matriz de confusión: {e}") + +def plot_prediction_examples(test_gen, y_true, y_pred, class_names, output_dir, unique_classes=None, n_examples=12): + """Mostrar ejemplos de predicciones correctas e incorrectas para ResNet50""" + try: + # Obtener índices de predicciones correctas e incorrectas + correct_idx = np.where(y_true == y_pred)[0] + incorrect_idx = np.where(y_true != y_pred)[0] + + # Seleccionar ejemplos + n_correct = min(n_examples // 2, len(correct_idx)) + n_incorrect = min(n_examples // 2, len(incorrect_idx)) + + selected_correct = np.random.choice(correct_idx, n_correct, replace=False) if len(correct_idx) > 0 else [] + selected_incorrect = np.random.choice(incorrect_idx, n_incorrect, replace=False) if len(incorrect_idx) > 0 else [] + + selected_indices = np.concatenate([selected_correct, selected_incorrect]) + + if len(selected_indices) == 0: + print("⚠️ No hay ejemplos para mostrar") + return + + # Crear gráfico + n_show = len(selected_indices) + cols = 4 + rows = (n_show + cols - 1) // cols + + plt.figure(figsize=(15, 4 * rows)) + + for i, idx in enumerate(selected_indices): + plt.subplot(rows, cols, i + 1) + + # Obtener imagen + try: + img_path = test_gen.filepaths[idx] + img = plt.imread(img_path) + + plt.imshow(img) + plt.axis('off') + + true_label = class_names[y_true[idx]] + pred_label = class_names[y_pred[idx]] + + color = 'green' if y_true[idx] == y_pred[idx] else 'red' + plt.title(f'Real: {true_label}\nPredicción: {pred_label}', + color=color, fontsize=9, fontweight='bold') + except Exception as e: + plt.text(0.5, 0.5, f'Error cargando\nimagen {idx}', + ha='center', va='center', transform=plt.gca().transAxes) + plt.axis('off') + + plt.suptitle('Ejemplos de Predicciones ResNet50 (Verde=Correcta, Rojo=Incorrecta)', fontsize=14, fontweight='bold') + plt.tight_layout() + plt.savefig(os.path.join(output_dir, 'resnet50_prediction_examples.png'), dpi=300, bbox_inches='tight') + plt.close() + print("✅ Ejemplos de predicciones ResNet50 guardados") + + except Exception as e: + print(f"⚠️ Error creando ejemplos de predicciones: {e}") + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/Code/Supervised_learning/resnet/.github/prompts/speckit.analyze.prompt.md b/Code/Supervised_learning/resnet/.github/prompts/speckit.analyze.prompt.md new file mode 100644 index 0000000..542a3de --- /dev/null +++ b/Code/Supervised_learning/resnet/.github/prompts/speckit.analyze.prompt.md @@ -0,0 +1,184 @@ +--- +description: Perform a non-destructive cross-artifact consistency and quality analysis across spec.md, plan.md, and tasks.md after task generation. +--- + +## User Input + +```text +$ARGUMENTS +``` + +You **MUST** consider the user input before proceeding (if not empty). + +## Goal + +Identify inconsistencies, duplications, ambiguities, and underspecified items across the three core artifacts (`spec.md`, `plan.md`, `tasks.md`) before implementation. This command MUST run only after `/speckit.tasks` has successfully produced a complete `tasks.md`. + +## Operating Constraints + +**STRICTLY READ-ONLY**: Do **not** modify any files. Output a structured analysis report. Offer an optional remediation plan (user must explicitly approve before any follow-up editing commands would be invoked manually). + +**Constitution Authority**: The project constitution (`.specify/memory/constitution.md`) is **non-negotiable** within this analysis scope. Constitution conflicts are automatically CRITICAL and require adjustment of the spec, plan, or tasks—not dilution, reinterpretation, or silent ignoring of the principle. If a principle itself needs to change, that must occur in a separate, explicit constitution update outside `/speckit.analyze`. + +## Execution Steps + +### 1. Initialize Analysis Context + +Run `.specify/scripts/powershell/check-prerequisites.ps1 -Json -RequireTasks -IncludeTasks` once from repo root and parse JSON for FEATURE_DIR and AVAILABLE_DOCS. Derive absolute paths: + +- SPEC = FEATURE_DIR/spec.md +- PLAN = FEATURE_DIR/plan.md +- TASKS = FEATURE_DIR/tasks.md + +Abort with an error message if any required file is missing (instruct the user to run missing prerequisite command). +For single quotes in args like "I'm Groot", use escape syntax: e.g 'I'\''m Groot' (or double-quote if possible: "I'm Groot"). + +### 2. Load Artifacts (Progressive Disclosure) + +Load only the minimal necessary context from each artifact: + +**From spec.md:** + +- Overview/Context +- Functional Requirements +- Non-Functional Requirements +- User Stories +- Edge Cases (if present) + +**From plan.md:** + +- Architecture/stack choices +- Data Model references +- Phases +- Technical constraints + +**From tasks.md:** + +- Task IDs +- Descriptions +- Phase grouping +- Parallel markers [P] +- Referenced file paths + +**From constitution:** + +- Load `.specify/memory/constitution.md` for principle validation + +### 3. Build Semantic Models + +Create internal representations (do not include raw artifacts in output): + +- **Requirements inventory**: Each functional + non-functional requirement with a stable key (derive slug based on imperative phrase; e.g., "User can upload file" → `user-can-upload-file`) +- **User story/action inventory**: Discrete user actions with acceptance criteria +- **Task coverage mapping**: Map each task to one or more requirements or stories (inference by keyword / explicit reference patterns like IDs or key phrases) +- **Constitution rule set**: Extract principle names and MUST/SHOULD normative statements + +### 4. Detection Passes (Token-Efficient Analysis) + +Focus on high-signal findings. Limit to 50 findings total; aggregate remainder in overflow summary. + +#### A. Duplication Detection + +- Identify near-duplicate requirements +- Mark lower-quality phrasing for consolidation + +#### B. Ambiguity Detection + +- Flag vague adjectives (fast, scalable, secure, intuitive, robust) lacking measurable criteria +- Flag unresolved placeholders (TODO, TKTK, ???, ``, etc.) + +#### C. Underspecification + +- Requirements with verbs but missing object or measurable outcome +- User stories missing acceptance criteria alignment +- Tasks referencing files or components not defined in spec/plan + +#### D. Constitution Alignment + +- Any requirement or plan element conflicting with a MUST principle +- Missing mandated sections or quality gates from constitution + +#### E. Coverage Gaps + +- Requirements with zero associated tasks +- Tasks with no mapped requirement/story +- Non-functional requirements not reflected in tasks (e.g., performance, security) + +#### F. Inconsistency + +- Terminology drift (same concept named differently across files) +- Data entities referenced in plan but absent in spec (or vice versa) +- Task ordering contradictions (e.g., integration tasks before foundational setup tasks without dependency note) +- Conflicting requirements (e.g., one requires Next.js while other specifies Vue) + +### 5. Severity Assignment + +Use this heuristic to prioritize findings: + +- **CRITICAL**: Violates constitution MUST, missing core spec artifact, or requirement with zero coverage that blocks baseline functionality +- **HIGH**: Duplicate or conflicting requirement, ambiguous security/performance attribute, untestable acceptance criterion +- **MEDIUM**: Terminology drift, missing non-functional task coverage, underspecified edge case +- **LOW**: Style/wording improvements, minor redundancy not affecting execution order + +### 6. Produce Compact Analysis Report + +Output a Markdown report (no file writes) with the following structure: + +## Specification Analysis Report + +| ID | Category | Severity | Location(s) | Summary | Recommendation | +|----|----------|----------|-------------|---------|----------------| +| A1 | Duplication | HIGH | spec.md:L120-134 | Two similar requirements ... | Merge phrasing; keep clearer version | + +(Add one row per finding; generate stable IDs prefixed by category initial.) + +**Coverage Summary Table:** + +| Requirement Key | Has Task? | Task IDs | Notes | +|-----------------|-----------|----------|-------| + +**Constitution Alignment Issues:** (if any) + +**Unmapped Tasks:** (if any) + +**Metrics:** + +- Total Requirements +- Total Tasks +- Coverage % (requirements with >=1 task) +- Ambiguity Count +- Duplication Count +- Critical Issues Count + +### 7. Provide Next Actions + +At end of report, output a concise Next Actions block: + +- If CRITICAL issues exist: Recommend resolving before `/speckit.implement` +- If only LOW/MEDIUM: User may proceed, but provide improvement suggestions +- Provide explicit command suggestions: e.g., "Run /speckit.specify with refinement", "Run /speckit.plan to adjust architecture", "Manually edit tasks.md to add coverage for 'performance-metrics'" + +### 8. Offer Remediation + +Ask the user: "Would you like me to suggest concrete remediation edits for the top N issues?" (Do NOT apply them automatically.) + +## Operating Principles + +### Context Efficiency + +- **Minimal high-signal tokens**: Focus on actionable findings, not exhaustive documentation +- **Progressive disclosure**: Load artifacts incrementally; don't dump all content into analysis +- **Token-efficient output**: Limit findings table to 50 rows; summarize overflow +- **Deterministic results**: Rerunning without changes should produce consistent IDs and counts + +### Analysis Guidelines + +- **NEVER modify files** (this is read-only analysis) +- **NEVER hallucinate missing sections** (if absent, report them accurately) +- **Prioritize constitution violations** (these are always CRITICAL) +- **Use examples over exhaustive rules** (cite specific instances, not generic patterns) +- **Report zero issues gracefully** (emit success report with coverage statistics) + +## Context + +$ARGUMENTS diff --git a/Code/Supervised_learning/resnet/.github/prompts/speckit.checklist.prompt.md b/Code/Supervised_learning/resnet/.github/prompts/speckit.checklist.prompt.md new file mode 100644 index 0000000..b15f916 --- /dev/null +++ b/Code/Supervised_learning/resnet/.github/prompts/speckit.checklist.prompt.md @@ -0,0 +1,294 @@ +--- +description: Generate a custom checklist for the current feature based on user requirements. +--- + +## Checklist Purpose: "Unit Tests for English" + +**CRITICAL CONCEPT**: Checklists are **UNIT TESTS FOR REQUIREMENTS WRITING** - they validate the quality, clarity, and completeness of requirements in a given domain. + +**NOT for verification/testing**: + +- ❌ NOT "Verify the button clicks correctly" +- ❌ NOT "Test error handling works" +- ❌ NOT "Confirm the API returns 200" +- ❌ NOT checking if code/implementation matches the spec + +**FOR requirements quality validation**: + +- ✅ "Are visual hierarchy requirements defined for all card types?" (completeness) +- ✅ "Is 'prominent display' quantified with specific sizing/positioning?" (clarity) +- ✅ "Are hover state requirements consistent across all interactive elements?" (consistency) +- ✅ "Are accessibility requirements defined for keyboard navigation?" (coverage) +- ✅ "Does the spec define what happens when logo image fails to load?" (edge cases) + +**Metaphor**: If your spec is code written in English, the checklist is its unit test suite. You're testing whether the requirements are well-written, complete, unambiguous, and ready for implementation - NOT whether the implementation works. + +## User Input + +```text +$ARGUMENTS +``` + +You **MUST** consider the user input before proceeding (if not empty). + +## Execution Steps + +1. **Setup**: Run `.specify/scripts/powershell/check-prerequisites.ps1 -Json` from repo root and parse JSON for FEATURE_DIR and AVAILABLE_DOCS list. + - All file paths must be absolute. + - For single quotes in args like "I'm Groot", use escape syntax: e.g 'I'\''m Groot' (or double-quote if possible: "I'm Groot"). + +2. **Clarify intent (dynamic)**: Derive up to THREE initial contextual clarifying questions (no pre-baked catalog). They MUST: + - Be generated from the user's phrasing + extracted signals from spec/plan/tasks + - Only ask about information that materially changes checklist content + - Be skipped individually if already unambiguous in `$ARGUMENTS` + - Prefer precision over breadth + + Generation algorithm: + 1. Extract signals: feature domain keywords (e.g., auth, latency, UX, API), risk indicators ("critical", "must", "compliance"), stakeholder hints ("QA", "review", "security team"), and explicit deliverables ("a11y", "rollback", "contracts"). + 2. Cluster signals into candidate focus areas (max 4) ranked by relevance. + 3. Identify probable audience & timing (author, reviewer, QA, release) if not explicit. + 4. Detect missing dimensions: scope breadth, depth/rigor, risk emphasis, exclusion boundaries, measurable acceptance criteria. + 5. Formulate questions chosen from these archetypes: + - Scope refinement (e.g., "Should this include integration touchpoints with X and Y or stay limited to local module correctness?") + - Risk prioritization (e.g., "Which of these potential risk areas should receive mandatory gating checks?") + - Depth calibration (e.g., "Is this a lightweight pre-commit sanity list or a formal release gate?") + - Audience framing (e.g., "Will this be used by the author only or peers during PR review?") + - Boundary exclusion (e.g., "Should we explicitly exclude performance tuning items this round?") + - Scenario class gap (e.g., "No recovery flows detected—are rollback / partial failure paths in scope?") + + Question formatting rules: + - If presenting options, generate a compact table with columns: Option | Candidate | Why It Matters + - Limit to A–E options maximum; omit table if a free-form answer is clearer + - Never ask the user to restate what they already said + - Avoid speculative categories (no hallucination). If uncertain, ask explicitly: "Confirm whether X belongs in scope." + + Defaults when interaction impossible: + - Depth: Standard + - Audience: Reviewer (PR) if code-related; Author otherwise + - Focus: Top 2 relevance clusters + + Output the questions (label Q1/Q2/Q3). After answers: if ≥2 scenario classes (Alternate / Exception / Recovery / Non-Functional domain) remain unclear, you MAY ask up to TWO more targeted follow‑ups (Q4/Q5) with a one-line justification each (e.g., "Unresolved recovery path risk"). Do not exceed five total questions. Skip escalation if user explicitly declines more. + +3. **Understand user request**: Combine `$ARGUMENTS` + clarifying answers: + - Derive checklist theme (e.g., security, review, deploy, ux) + - Consolidate explicit must-have items mentioned by user + - Map focus selections to category scaffolding + - Infer any missing context from spec/plan/tasks (do NOT hallucinate) + +4. **Load feature context**: Read from FEATURE_DIR: + - spec.md: Feature requirements and scope + - plan.md (if exists): Technical details, dependencies + - tasks.md (if exists): Implementation tasks + + **Context Loading Strategy**: + - Load only necessary portions relevant to active focus areas (avoid full-file dumping) + - Prefer summarizing long sections into concise scenario/requirement bullets + - Use progressive disclosure: add follow-on retrieval only if gaps detected + - If source docs are large, generate interim summary items instead of embedding raw text + +5. **Generate checklist** - Create "Unit Tests for Requirements": + - Create `FEATURE_DIR/checklists/` directory if it doesn't exist + - Generate unique checklist filename: + - Use short, descriptive name based on domain (e.g., `ux.md`, `api.md`, `security.md`) + - Format: `[domain].md` + - If file exists, append to existing file + - Number items sequentially starting from CHK001 + - Each `/speckit.checklist` run creates a NEW file (never overwrites existing checklists) + + **CORE PRINCIPLE - Test the Requirements, Not the Implementation**: + Every checklist item MUST evaluate the REQUIREMENTS THEMSELVES for: + - **Completeness**: Are all necessary requirements present? + - **Clarity**: Are requirements unambiguous and specific? + - **Consistency**: Do requirements align with each other? + - **Measurability**: Can requirements be objectively verified? + - **Coverage**: Are all scenarios/edge cases addressed? + + **Category Structure** - Group items by requirement quality dimensions: + - **Requirement Completeness** (Are all necessary requirements documented?) + - **Requirement Clarity** (Are requirements specific and unambiguous?) + - **Requirement Consistency** (Do requirements align without conflicts?) + - **Acceptance Criteria Quality** (Are success criteria measurable?) + - **Scenario Coverage** (Are all flows/cases addressed?) + - **Edge Case Coverage** (Are boundary conditions defined?) + - **Non-Functional Requirements** (Performance, Security, Accessibility, etc. - are they specified?) + - **Dependencies & Assumptions** (Are they documented and validated?) + - **Ambiguities & Conflicts** (What needs clarification?) + + **HOW TO WRITE CHECKLIST ITEMS - "Unit Tests for English"**: + + ❌ **WRONG** (Testing implementation): + - "Verify landing page displays 3 episode cards" + - "Test hover states work on desktop" + - "Confirm logo click navigates home" + + ✅ **CORRECT** (Testing requirements quality): + - "Are the exact number and layout of featured episodes specified?" [Completeness] + - "Is 'prominent display' quantified with specific sizing/positioning?" [Clarity] + - "Are hover state requirements consistent across all interactive elements?" [Consistency] + - "Are keyboard navigation requirements defined for all interactive UI?" [Coverage] + - "Is the fallback behavior specified when logo image fails to load?" [Edge Cases] + - "Are loading states defined for asynchronous episode data?" [Completeness] + - "Does the spec define visual hierarchy for competing UI elements?" [Clarity] + + **ITEM STRUCTURE**: + Each item should follow this pattern: + - Question format asking about requirement quality + - Focus on what's WRITTEN (or not written) in the spec/plan + - Include quality dimension in brackets [Completeness/Clarity/Consistency/etc.] + - Reference spec section `[Spec §X.Y]` when checking existing requirements + - Use `[Gap]` marker when checking for missing requirements + + **EXAMPLES BY QUALITY DIMENSION**: + + Completeness: + - "Are error handling requirements defined for all API failure modes? [Gap]" + - "Are accessibility requirements specified for all interactive elements? [Completeness]" + - "Are mobile breakpoint requirements defined for responsive layouts? [Gap]" + + Clarity: + - "Is 'fast loading' quantified with specific timing thresholds? [Clarity, Spec §NFR-2]" + - "Are 'related episodes' selection criteria explicitly defined? [Clarity, Spec §FR-5]" + - "Is 'prominent' defined with measurable visual properties? [Ambiguity, Spec §FR-4]" + + Consistency: + - "Do navigation requirements align across all pages? [Consistency, Spec §FR-10]" + - "Are card component requirements consistent between landing and detail pages? [Consistency]" + + Coverage: + - "Are requirements defined for zero-state scenarios (no episodes)? [Coverage, Edge Case]" + - "Are concurrent user interaction scenarios addressed? [Coverage, Gap]" + - "Are requirements specified for partial data loading failures? [Coverage, Exception Flow]" + + Measurability: + - "Are visual hierarchy requirements measurable/testable? [Acceptance Criteria, Spec §FR-1]" + - "Can 'balanced visual weight' be objectively verified? [Measurability, Spec §FR-2]" + + **Scenario Classification & Coverage** (Requirements Quality Focus): + - Check if requirements exist for: Primary, Alternate, Exception/Error, Recovery, Non-Functional scenarios + - For each scenario class, ask: "Are [scenario type] requirements complete, clear, and consistent?" + - If scenario class missing: "Are [scenario type] requirements intentionally excluded or missing? [Gap]" + - Include resilience/rollback when state mutation occurs: "Are rollback requirements defined for migration failures? [Gap]" + + **Traceability Requirements**: + - MINIMUM: ≥80% of items MUST include at least one traceability reference + - Each item should reference: spec section `[Spec §X.Y]`, or use markers: `[Gap]`, `[Ambiguity]`, `[Conflict]`, `[Assumption]` + - If no ID system exists: "Is a requirement & acceptance criteria ID scheme established? [Traceability]" + + **Surface & Resolve Issues** (Requirements Quality Problems): + Ask questions about the requirements themselves: + - Ambiguities: "Is the term 'fast' quantified with specific metrics? [Ambiguity, Spec §NFR-1]" + - Conflicts: "Do navigation requirements conflict between §FR-10 and §FR-10a? [Conflict]" + - Assumptions: "Is the assumption of 'always available podcast API' validated? [Assumption]" + - Dependencies: "Are external podcast API requirements documented? [Dependency, Gap]" + - Missing definitions: "Is 'visual hierarchy' defined with measurable criteria? [Gap]" + + **Content Consolidation**: + - Soft cap: If raw candidate items > 40, prioritize by risk/impact + - Merge near-duplicates checking the same requirement aspect + - If >5 low-impact edge cases, create one item: "Are edge cases X, Y, Z addressed in requirements? [Coverage]" + + **🚫 ABSOLUTELY PROHIBITED** - These make it an implementation test, not a requirements test: + - ❌ Any item starting with "Verify", "Test", "Confirm", "Check" + implementation behavior + - ❌ References to code execution, user actions, system behavior + - ❌ "Displays correctly", "works properly", "functions as expected" + - ❌ "Click", "navigate", "render", "load", "execute" + - ❌ Test cases, test plans, QA procedures + - ❌ Implementation details (frameworks, APIs, algorithms) + + **✅ REQUIRED PATTERNS** - These test requirements quality: + - ✅ "Are [requirement type] defined/specified/documented for [scenario]?" + - ✅ "Is [vague term] quantified/clarified with specific criteria?" + - ✅ "Are requirements consistent between [section A] and [section B]?" + - ✅ "Can [requirement] be objectively measured/verified?" + - ✅ "Are [edge cases/scenarios] addressed in requirements?" + - ✅ "Does the spec define [missing aspect]?" + +6. **Structure Reference**: Generate the checklist following the canonical template in `.specify/templates/checklist-template.md` for title, meta section, category headings, and ID formatting. If template is unavailable, use: H1 title, purpose/created meta lines, `##` category sections containing `- [ ] CHK### ` lines with globally incrementing IDs starting at CHK001. + +7. **Report**: Output full path to created checklist, item count, and remind user that each run creates a new file. Summarize: + - Focus areas selected + - Depth level + - Actor/timing + - Any explicit user-specified must-have items incorporated + +**Important**: Each `/speckit.checklist` command invocation creates a checklist file using short, descriptive names unless file already exists. This allows: + +- Multiple checklists of different types (e.g., `ux.md`, `test.md`, `security.md`) +- Simple, memorable filenames that indicate checklist purpose +- Easy identification and navigation in the `checklists/` folder + +To avoid clutter, use descriptive types and clean up obsolete checklists when done. + +## Example Checklist Types & Sample Items + +**UX Requirements Quality:** `ux.md` + +Sample items (testing the requirements, NOT the implementation): + +- "Are visual hierarchy requirements defined with measurable criteria? [Clarity, Spec §FR-1]" +- "Is the number and positioning of UI elements explicitly specified? [Completeness, Spec §FR-1]" +- "Are interaction state requirements (hover, focus, active) consistently defined? [Consistency]" +- "Are accessibility requirements specified for all interactive elements? [Coverage, Gap]" +- "Is fallback behavior defined when images fail to load? [Edge Case, Gap]" +- "Can 'prominent display' be objectively measured? [Measurability, Spec §FR-4]" + +**API Requirements Quality:** `api.md` + +Sample items: + +- "Are error response formats specified for all failure scenarios? [Completeness]" +- "Are rate limiting requirements quantified with specific thresholds? [Clarity]" +- "Are authentication requirements consistent across all endpoints? [Consistency]" +- "Are retry/timeout requirements defined for external dependencies? [Coverage, Gap]" +- "Is versioning strategy documented in requirements? [Gap]" + +**Performance Requirements Quality:** `performance.md` + +Sample items: + +- "Are performance requirements quantified with specific metrics? [Clarity]" +- "Are performance targets defined for all critical user journeys? [Coverage]" +- "Are performance requirements under different load conditions specified? [Completeness]" +- "Can performance requirements be objectively measured? [Measurability]" +- "Are degradation requirements defined for high-load scenarios? [Edge Case, Gap]" + +**Security Requirements Quality:** `security.md` + +Sample items: + +- "Are authentication requirements specified for all protected resources? [Coverage]" +- "Are data protection requirements defined for sensitive information? [Completeness]" +- "Is the threat model documented and requirements aligned to it? [Traceability]" +- "Are security requirements consistent with compliance obligations? [Consistency]" +- "Are security failure/breach response requirements defined? [Gap, Exception Flow]" + +## Anti-Examples: What NOT To Do + +**❌ WRONG - These test implementation, not requirements:** + +```markdown +- [ ] CHK001 - Verify landing page displays 3 episode cards [Spec §FR-001] +- [ ] CHK002 - Test hover states work correctly on desktop [Spec §FR-003] +- [ ] CHK003 - Confirm logo click navigates to home page [Spec §FR-010] +- [ ] CHK004 - Check that related episodes section shows 3-5 items [Spec §FR-005] +``` + +**✅ CORRECT - These test requirements quality:** + +```markdown +- [ ] CHK001 - Are the number and layout of featured episodes explicitly specified? [Completeness, Spec §FR-001] +- [ ] CHK002 - Are hover state requirements consistently defined for all interactive elements? [Consistency, Spec §FR-003] +- [ ] CHK003 - Are navigation requirements clear for all clickable brand elements? [Clarity, Spec §FR-010] +- [ ] CHK004 - Is the selection criteria for related episodes documented? [Gap, Spec §FR-005] +- [ ] CHK005 - Are loading state requirements defined for asynchronous episode data? [Gap] +- [ ] CHK006 - Can "visual hierarchy" requirements be objectively measured? [Measurability, Spec §FR-001] +``` + +**Key Differences:** + +- Wrong: Tests if the system works correctly +- Correct: Tests if the requirements are written correctly +- Wrong: Verification of behavior +- Correct: Validation of requirement quality +- Wrong: "Does it do X?" +- Correct: "Is X clearly specified?" diff --git a/Code/Supervised_learning/resnet/.github/prompts/speckit.clarify.prompt.md b/Code/Supervised_learning/resnet/.github/prompts/speckit.clarify.prompt.md new file mode 100644 index 0000000..4700d29 --- /dev/null +++ b/Code/Supervised_learning/resnet/.github/prompts/speckit.clarify.prompt.md @@ -0,0 +1,177 @@ +--- +description: Identify underspecified areas in the current feature spec by asking up to 5 highly targeted clarification questions and encoding answers back into the spec. +--- + +## User Input + +```text +$ARGUMENTS +``` + +You **MUST** consider the user input before proceeding (if not empty). + +## Outline + +Goal: Detect and reduce ambiguity or missing decision points in the active feature specification and record the clarifications directly in the spec file. + +Note: This clarification workflow is expected to run (and be completed) BEFORE invoking `/speckit.plan`. If the user explicitly states they are skipping clarification (e.g., exploratory spike), you may proceed, but must warn that downstream rework risk increases. + +Execution steps: + +1. Run `.specify/scripts/powershell/check-prerequisites.ps1 -Json -PathsOnly` from repo root **once** (combined `--json --paths-only` mode / `-Json -PathsOnly`). Parse minimal JSON payload fields: + - `FEATURE_DIR` + - `FEATURE_SPEC` + - (Optionally capture `IMPL_PLAN`, `TASKS` for future chained flows.) + - If JSON parsing fails, abort and instruct user to re-run `/speckit.specify` or verify feature branch environment. + - For single quotes in args like "I'm Groot", use escape syntax: e.g 'I'\''m Groot' (or double-quote if possible: "I'm Groot"). + +2. Load the current spec file. Perform a structured ambiguity & coverage scan using this taxonomy. For each category, mark status: Clear / Partial / Missing. Produce an internal coverage map used for prioritization (do not output raw map unless no questions will be asked). + + Functional Scope & Behavior: + - Core user goals & success criteria + - Explicit out-of-scope declarations + - User roles / personas differentiation + + Domain & Data Model: + - Entities, attributes, relationships + - Identity & uniqueness rules + - Lifecycle/state transitions + - Data volume / scale assumptions + + Interaction & UX Flow: + - Critical user journeys / sequences + - Error/empty/loading states + - Accessibility or localization notes + + Non-Functional Quality Attributes: + - Performance (latency, throughput targets) + - Scalability (horizontal/vertical, limits) + - Reliability & availability (uptime, recovery expectations) + - Observability (logging, metrics, tracing signals) + - Security & privacy (authN/Z, data protection, threat assumptions) + - Compliance / regulatory constraints (if any) + + Integration & External Dependencies: + - External services/APIs and failure modes + - Data import/export formats + - Protocol/versioning assumptions + + Edge Cases & Failure Handling: + - Negative scenarios + - Rate limiting / throttling + - Conflict resolution (e.g., concurrent edits) + + Constraints & Tradeoffs: + - Technical constraints (language, storage, hosting) + - Explicit tradeoffs or rejected alternatives + + Terminology & Consistency: + - Canonical glossary terms + - Avoided synonyms / deprecated terms + + Completion Signals: + - Acceptance criteria testability + - Measurable Definition of Done style indicators + + Misc / Placeholders: + - TODO markers / unresolved decisions + - Ambiguous adjectives ("robust", "intuitive") lacking quantification + + For each category with Partial or Missing status, add a candidate question opportunity unless: + - Clarification would not materially change implementation or validation strategy + - Information is better deferred to planning phase (note internally) + +3. Generate (internally) a prioritized queue of candidate clarification questions (maximum 5). Do NOT output them all at once. Apply these constraints: + - Maximum of 10 total questions across the whole session. + - Each question must be answerable with EITHER: + - A short multiple‑choice selection (2–5 distinct, mutually exclusive options), OR + - A one-word / short‑phrase answer (explicitly constrain: "Answer in <=5 words"). + - Only include questions whose answers materially impact architecture, data modeling, task decomposition, test design, UX behavior, operational readiness, or compliance validation. + - Ensure category coverage balance: attempt to cover the highest impact unresolved categories first; avoid asking two low-impact questions when a single high-impact area (e.g., security posture) is unresolved. + - Exclude questions already answered, trivial stylistic preferences, or plan-level execution details (unless blocking correctness). + - Favor clarifications that reduce downstream rework risk or prevent misaligned acceptance tests. + - If more than 5 categories remain unresolved, select the top 5 by (Impact * Uncertainty) heuristic. + +4. Sequential questioning loop (interactive): + - Present EXACTLY ONE question at a time. + - For multiple‑choice questions: + - **Analyze all options** and determine the **most suitable option** based on: + - Best practices for the project type + - Common patterns in similar implementations + - Risk reduction (security, performance, maintainability) + - Alignment with any explicit project goals or constraints visible in the spec + - Present your **recommended option prominently** at the top with clear reasoning (1-2 sentences explaining why this is the best choice). + - Format as: `**Recommended:** Option [X] - ` + - Then render all options as a Markdown table: + + | Option | Description | + |--------|-------------| + | A |