commit d84470f7f77fa24bc70f46fc0ac3c4c659172683 Author: Mohsen Date: Tue Oct 27 14:13:32 2026 +0100 Initial Commit diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..20a4a40 --- /dev/null +++ b/.gitignore @@ -0,0 +1,24 @@ +# --- PRIVATE: 3A credentials, never commit --- +API/3a_credentials.json +API/mohsen_new_api.html +.env +*.env + +# --- cached 3A login token + local farmer fields --- +data/ + +# --- generated reports (station data, rebuilt by each run) --- +outputs/ + +# --- Python / tools --- +.venv/ +__pycache__/ +*.py[cod] +*.egg-info/ +.pytest_cache/ +build/ +dist/ + +# --- kept local only (your choice) --- +README.md +how to RUN.txt diff --git a/.vscode/extensions.json b/.vscode/extensions.json new file mode 100644 index 0000000..c707295 --- /dev/null +++ b/.vscode/extensions.json @@ -0,0 +1,9 @@ +{ + "recommendations": [ + "ms-python.python", + "ms-python.vscode-pylance", + "ms-python.debugpy", + "ms-toolsai.jupyter", + "redhat.vscode-yaml" + ] +} diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 0000000..4116d43 --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,334 @@ +{ + // Run and Debug (F5): pick a configuration in the Run and Debug panel. + // Every entry runs with -m (module), so the damage_engine imports work. + // Live entries use the 3A API (5 requests / 15 min): don't repeat them quickly. + "version": "0.2.0", + "configurations": [ + { + "name": "Pipeline: run every block + client reports (live API)", + "type": "debugpy", + "request": "launch", + "module": "damage_engine.main", + "cwd": "${workspaceFolder}", + "console": "integratedTerminal" + }, + { + "name": "Pipeline: run every block + client reports for a date (live API)", + "type": "debugpy", + "request": "launch", + "module": "damage_engine.main", + "args": ["--date", "${input:runDay}"], + "cwd": "${workspaceFolder}", + "console": "integratedTerminal" + }, + { + "name": "Brina: run + save report (live API)", + "type": "debugpy", + "request": "launch", + "module": "damage_engine.main", + "args": ["--once", "brina"], + "cwd": "${workspaceFolder}", + "console": "integratedTerminal" + }, + { + "name": "Brina: run + save report for a date (live API)", + "type": "debugpy", + "request": "launch", + "module": "damage_engine.main", + "args": ["--once", "brina", "--date", "${input:runDay}"], + "cwd": "${workspaceFolder}", + "console": "integratedTerminal" + }, + { + "name": "Brina: print only, no report (live API)", + "type": "debugpy", + "request": "launch", + "module": "damage_engine.detectors.brina", + "cwd": "${workspaceFolder}", + "console": "integratedTerminal" + }, + { + "name": "Flood: run + save report (live API)", + "type": "debugpy", + "request": "launch", + "module": "damage_engine.main", + "args": ["--once", "flood"], + "cwd": "${workspaceFolder}", + "console": "integratedTerminal" + }, + { + "name": "Flood: run + save report for a date (live API)", + "type": "debugpy", + "request": "launch", + "module": "damage_engine.main", + "args": ["--once", "flood", "--date", "${input:runDay}"], + "cwd": "${workspaceFolder}", + "console": "integratedTerminal" + }, + { + "name": "Flood: print only, no report (live API)", + "type": "debugpy", + "request": "launch", + "module": "damage_engine.detectors.flood", + "cwd": "${workspaceFolder}", + "console": "integratedTerminal" + }, + { + "name": "Freezing: run + save report (live API)", + "type": "debugpy", + "request": "launch", + "module": "damage_engine.main", + "args": ["--once", "freezing"], + "cwd": "${workspaceFolder}", + "console": "integratedTerminal" + }, + { + "name": "Freezing: run + save report for a date (live API)", + "type": "debugpy", + "request": "launch", + "module": "damage_engine.main", + "args": ["--once", "freezing", "--date", "${input:runDay}"], + "cwd": "${workspaceFolder}", + "console": "integratedTerminal" + }, + { + "name": "Freezing: print only, no report (live API)", + "type": "debugpy", + "request": "launch", + "module": "damage_engine.detectors.freezing", + "cwd": "${workspaceFolder}", + "console": "integratedTerminal" + }, + { + "name": "Strong wind: run + save report (live API)", + "type": "debugpy", + "request": "launch", + "module": "damage_engine.main", + "args": ["--once", "strong_wind"], + "cwd": "${workspaceFolder}", + "console": "integratedTerminal" + }, + { + "name": "Strong wind: run + save report for a date (live API)", + "type": "debugpy", + "request": "launch", + "module": "damage_engine.main", + "args": ["--once", "strong_wind", "--date", "${input:runDay}"], + "cwd": "${workspaceFolder}", + "console": "integratedTerminal" + }, + { + "name": "Strong wind: print only, no report (live API)", + "type": "debugpy", + "request": "launch", + "module": "damage_engine.detectors.strong_wind", + "cwd": "${workspaceFolder}", + "console": "integratedTerminal" + }, + { + "name": "Hot wind: run + save report (live API)", + "type": "debugpy", + "request": "launch", + "module": "damage_engine.main", + "args": ["--once", "hot_wind"], + "cwd": "${workspaceFolder}", + "console": "integratedTerminal" + }, + { + "name": "Hot wind: run + save report for a date (live API)", + "type": "debugpy", + "request": "launch", + "module": "damage_engine.main", + "args": ["--once", "hot_wind", "--date", "${input:runDay}"], + "cwd": "${workspaceFolder}", + "console": "integratedTerminal" + }, + { + "name": "Hot wind: print only, no report (live API)", + "type": "debugpy", + "request": "launch", + "module": "damage_engine.detectors.hot_wind", + "cwd": "${workspaceFolder}", + "console": "integratedTerminal" + }, + { + "name": "High temperature: run + save report (live API)", + "type": "debugpy", + "request": "launch", + "module": "damage_engine.main", + "args": ["--once", "high_temperature"], + "cwd": "${workspaceFolder}", + "console": "integratedTerminal" + }, + { + "name": "High temperature: run + save report for a date (live API)", + "type": "debugpy", + "request": "launch", + "module": "damage_engine.main", + "args": ["--once", "high_temperature", "--date", "${input:runDay}"], + "cwd": "${workspaceFolder}", + "console": "integratedTerminal" + }, + { + "name": "High temperature: print only, no report (live API)", + "type": "debugpy", + "request": "launch", + "module": "damage_engine.detectors.high_temperature", + "cwd": "${workspaceFolder}", + "console": "integratedTerminal" + }, + { + "name": "Snow: run + save report (live API)", + "type": "debugpy", + "request": "launch", + "module": "damage_engine.main", + "args": ["--once", "snow"], + "cwd": "${workspaceFolder}", + "console": "integratedTerminal" + }, + { + "name": "Snow: run + save report for a date (live API)", + "type": "debugpy", + "request": "launch", + "module": "damage_engine.main", + "args": ["--once", "snow", "--date", "${input:runDay}"], + "cwd": "${workspaceFolder}", + "console": "integratedTerminal" + }, + { + "name": "Snow: print only, no report (live API)", + "type": "debugpy", + "request": "launch", + "module": "damage_engine.detectors.snow", + "cwd": "${workspaceFolder}", + "console": "integratedTerminal" + }, + { + "name": "Hail: run + save report (live API)", + "type": "debugpy", + "request": "launch", + "module": "damage_engine.main", + "args": ["--once", "hail"], + "cwd": "${workspaceFolder}", + "console": "integratedTerminal" + }, + { + "name": "Hail: run + save report for a date (live API)", + "type": "debugpy", + "request": "launch", + "module": "damage_engine.main", + "args": ["--once", "hail", "--date", "${input:runDay}"], + "cwd": "${workspaceFolder}", + "console": "integratedTerminal" + }, + { + "name": "Hail: print only, no report (live API)", + "type": "debugpy", + "request": "launch", + "module": "damage_engine.detectors.hail", + "cwd": "${workspaceFolder}", + "console": "integratedTerminal" + }, + { + "name": "Hail: daytime check 13:00 today + save (live API)", + "type": "debugpy", + "request": "launch", + "module": "damage_engine.main", + "args": ["--hail-daytime", "13"], + "cwd": "${workspaceFolder}", + "console": "integratedTerminal" + }, + { + "name": "Hail: daytime check 19:00 today + save (live API)", + "type": "debugpy", + "request": "launch", + "module": "damage_engine.main", + "args": ["--hail-daytime", "19"], + "cwd": "${workspaceFolder}", + "console": "integratedTerminal" + }, + { + "name": "Drought: run + save report (live API)", + "type": "debugpy", + "request": "launch", + "module": "damage_engine.main", + "args": ["--once", "drought"], + "cwd": "${workspaceFolder}", + "console": "integratedTerminal" + }, + { + "name": "Drought: run + save report for a date (live API)", + "type": "debugpy", + "request": "launch", + "module": "damage_engine.main", + "args": ["--once", "drought", "--date", "${input:runDay}"], + "cwd": "${workspaceFolder}", + "console": "integratedTerminal" + }, + { + "name": "Drought: print only, no report (live API)", + "type": "debugpy", + "request": "launch", + "module": "damage_engine.detectors.drought", + "cwd": "${workspaceFolder}", + "console": "integratedTerminal" + }, + { + "name": "Scheduler: keep running (whole pipeline every day at 07:05)", + "type": "debugpy", + "request": "launch", + "module": "damage_engine.main", + "args": ["--serve"], + "cwd": "${workspaceFolder}", + "console": "integratedTerminal" + }, + { + "name": "Report: choose a field (no API)", + "type": "debugpy", + "request": "launch", + "module": "damage_engine.report.report", + "cwd": "${workspaceFolder}", + "console": "integratedTerminal" + }, + { + "name": "Report: all fields, block by block (no API)", + "type": "debugpy", + "request": "launch", + "module": "damage_engine.report.report", + "args": ["--all"], + "cwd": "${workspaceFolder}", + "console": "integratedTerminal" + }, + { + "name": "Station report: JSON + PDF for one station (no API)", + "type": "debugpy", + "request": "launch", + "module": "damage_engine.report.station_report", + "args": ["--station", "${input:stationId}"], + "cwd": "${workspaceFolder}", + "console": "integratedTerminal" + }, + { + "name": "Unit tests (no API)", + "type": "debugpy", + "request": "launch", + "module": "pytest", + "args": ["tests", "-q"], + "cwd": "${workspaceFolder}", + "console": "integratedTerminal" + } + ], + "inputs": [ + { + "id": "runDay", + "type": "promptString", + "description": "Run day (YYYY-MM-DD, within the last ~180 days)" + }, + { + "id": "stationId", + "type": "promptString", + "description": "3A station id (e.g. 566)", + "default": "566" + } + ] +} diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..8741395 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,11 @@ +{ + "python.defaultInterpreterPath": "${workspaceFolder}/.venv/Scripts/python.exe", + "python.terminal.activateEnvironment": true, + "python.analysis.extraPaths": ["${workspaceFolder}/src"], + "python.testing.pytestEnabled": true, + "python.testing.pytestArgs": ["tests"], + "files.exclude": { + "**/__pycache__": true, + "**/*.egg-info": true + } +} diff --git a/API/3a_stations_sensors.json b/API/3a_stations_sensors.json new file mode 100644 index 0000000..7c6103f --- /dev/null +++ b/API/3a_stations_sensors.json @@ -0,0 +1,965 @@ +{ + "source": "https://api.3acloud.it (getAvailableStations, getAvailableStationHourlySensors, getAvailableStationDailySensors)", + "checked": "2026-10-21", + "note": "Sensors = HOURLY sensors (what the blocks fetch). DailyOnlyParameters = columns the station offers only in daily data (/getWeatherDaily). Units from /getAvailableHourlyColumns.", + "NumStations": 14, + "Stations": [ + { + "StationId": 3, + "StationName": "Lagnasco", + "Municipality": "Lagnasco", + "Latitude_WGS84": "44.634745", + "Longitude_WGS84": "7.581672", + "Elevation_m": "335", + "TimeZone": "UTC+1", + "NumSensors": 6, + "Sensors": [ + { + "sensor": "TemperaturaAria", + "name": "Air temperature sensor", + "unit": "°C", + "api_parameters": [ + "TemperaturaAriaMax", + "TemperaturaAriaMedia", + "TemperaturaAriaMin" + ] + }, + { + "sensor": "UmiditaRelativa", + "name": "Relative humidity sensor", + "unit": "%", + "api_parameters": [ + "UmiditaRelativaMax", + "UmiditaRelativaMedia", + "UmiditaRelativaMin" + ] + }, + { + "sensor": "TBulb", + "name": "Wet-bulb temperature (mean, supplied by 3A)", + "unit": "°C", + "api_parameters": [ + "TBulbMed" + ] + }, + { + "sensor": "Pioggia", + "name": "Rain gauge (RainRateMax in mm/h)", + "unit": "mm", + "api_parameters": [ + "Pioggia" + ] + }, + { + "sensor": "BagnaturaFogliare", + "name": "Leaf wetness sensor", + "unit": "min", + "api_parameters": [ + "BagnaturaFogliare" + ] + }, + { + "sensor": "VelocitaVento2m", + "name": "Wind speed sensor (anemometer, 2 m)", + "unit": "m/s", + "api_parameters": [ + "VelocitaVento2mMax", + "VelocitaVento2mMedia" + ] + } + ], + "DailyOnlyParameters": [ + "EvapoPot" + ] + }, + { + "StationId": 5, + "StationName": "Castellar", + "Municipality": "Castellar", + "Latitude_WGS84": "44.626722", + "Longitude_WGS84": "7.444808", + "Elevation_m": "360", + "TimeZone": "UTC+1", + "NumSensors": 5, + "Sensors": [ + { + "sensor": "TemperaturaAria", + "name": "Air temperature sensor", + "unit": "°C", + "api_parameters": [ + "TemperaturaAriaMax", + "TemperaturaAriaMedia", + "TemperaturaAriaMin" + ] + }, + { + "sensor": "UmiditaRelativa", + "name": "Relative humidity sensor", + "unit": "%", + "api_parameters": [ + "UmiditaRelativaMax", + "UmiditaRelativaMedia", + "UmiditaRelativaMin" + ] + }, + { + "sensor": "TBulb", + "name": "Wet-bulb temperature (mean, supplied by 3A)", + "unit": "°C", + "api_parameters": [ + "TBulbMed" + ] + }, + { + "sensor": "Pioggia", + "name": "Rain gauge (RainRateMax in mm/h)", + "unit": "mm", + "api_parameters": [ + "Pioggia" + ] + }, + { + "sensor": "BagnaturaFogliare", + "name": "Leaf wetness sensor", + "unit": "min", + "api_parameters": [ + "BagnaturaFogliare" + ] + } + ], + "DailyOnlyParameters": [ + "EvapoPot" + ] + }, + { + "StationId": 6, + "StationName": "Cavour", + "Municipality": "Cavour", + "Latitude_WGS84": "44.802767", + "Longitude_WGS84": "7.370115", + "Elevation_m": "307", + "TimeZone": "UTC+1", + "NumSensors": 6, + "Sensors": [ + { + "sensor": "TemperaturaAria", + "name": "Air temperature sensor", + "unit": "°C", + "api_parameters": [ + "TemperaturaAriaMax", + "TemperaturaAriaMedia", + "TemperaturaAriaMin" + ] + }, + { + "sensor": "UmiditaRelativa", + "name": "Relative humidity sensor", + "unit": "%", + "api_parameters": [ + "UmiditaRelativaMax", + "UmiditaRelativaMedia", + "UmiditaRelativaMin" + ] + }, + { + "sensor": "TBulb", + "name": "Wet-bulb temperature (mean, supplied by 3A)", + "unit": "°C", + "api_parameters": [ + "TBulbMed" + ] + }, + { + "sensor": "Pioggia", + "name": "Rain gauge (RainRateMax in mm/h)", + "unit": "mm", + "api_parameters": [ + "Pioggia" + ] + }, + { + "sensor": "BagnaturaFogliare", + "name": "Leaf wetness sensor", + "unit": "min", + "api_parameters": [ + "BagnaturaFogliare" + ] + }, + { + "sensor": "VelocitaVento2m", + "name": "Wind speed sensor (anemometer, 2 m)", + "unit": "m/s", + "api_parameters": [ + "VelocitaVento2mMax", + "VelocitaVento2mMedia" + ] + } + ], + "DailyOnlyParameters": [ + "EvapoPot" + ] + }, + { + "StationId": 7, + "StationName": "CostigliolediSaluzzo", + "Municipality": "CostigliolediSaluzzo", + "Latitude_WGS84": "44.565536", + "Longitude_WGS84": "7.465652", + "Elevation_m": "470", + "TimeZone": "UTC+1", + "NumSensors": 5, + "Sensors": [ + { + "sensor": "TemperaturaAria", + "name": "Air temperature sensor", + "unit": "°C", + "api_parameters": [ + "TemperaturaAriaMax", + "TemperaturaAriaMedia", + "TemperaturaAriaMin" + ] + }, + { + "sensor": "UmiditaRelativa", + "name": "Relative humidity sensor", + "unit": "%", + "api_parameters": [ + "UmiditaRelativaMax", + "UmiditaRelativaMedia", + "UmiditaRelativaMin" + ] + }, + { + "sensor": "TBulb", + "name": "Wet-bulb temperature (mean, supplied by 3A)", + "unit": "°C", + "api_parameters": [ + "TBulbMed" + ] + }, + { + "sensor": "Pioggia", + "name": "Rain gauge (RainRateMax in mm/h)", + "unit": "mm", + "api_parameters": [ + "Pioggia" + ] + }, + { + "sensor": "BagnaturaFogliare", + "name": "Leaf wetness sensor", + "unit": "min", + "api_parameters": [ + "BagnaturaFogliare" + ] + } + ], + "DailyOnlyParameters": [ + "EvapoPot" + ] + }, + { + "StationId": 9, + "StationName": "Centallo", + "Municipality": "Centallo", + "Latitude_WGS84": "44.5001", + "Longitude_WGS84": "7.61836", + "Elevation_m": "413", + "TimeZone": "UTC+1", + "NumSensors": 4, + "Sensors": [ + { + "sensor": "TemperaturaAria", + "name": "Air temperature sensor", + "unit": "°C", + "api_parameters": [ + "TemperaturaAriaMax", + "TemperaturaAriaMedia", + "TemperaturaAriaMin" + ] + }, + { + "sensor": "UmiditaRelativa", + "name": "Relative humidity sensor", + "unit": "%", + "api_parameters": [ + "UmiditaRelativaMax", + "UmiditaRelativaMedia", + "UmiditaRelativaMin" + ] + }, + { + "sensor": "TBulb", + "name": "Wet-bulb temperature (mean, supplied by 3A)", + "unit": "°C", + "api_parameters": [ + "TBulbMed" + ] + }, + { + "sensor": "Pioggia", + "name": "Rain gauge (RainRateMax in mm/h)", + "unit": "mm", + "api_parameters": [ + "Pioggia" + ] + } + ], + "DailyOnlyParameters": [ + "EvapoPot" + ] + }, + { + "StationId": 10, + "StationName": "Revello", + "Municipality": "Revello", + "Latitude_WGS84": "44.6728", + "Longitude_WGS84": "7.3906", + "Elevation_m": "316", + "TimeZone": "UTC+1", + "NumSensors": 6, + "Sensors": [ + { + "sensor": "TemperaturaAria", + "name": "Air temperature sensor", + "unit": "°C", + "api_parameters": [ + "TemperaturaAriaMax", + "TemperaturaAriaMedia", + "TemperaturaAriaMin" + ] + }, + { + "sensor": "UmiditaRelativa", + "name": "Relative humidity sensor", + "unit": "%", + "api_parameters": [ + "UmiditaRelativaMax", + "UmiditaRelativaMedia", + "UmiditaRelativaMin" + ] + }, + { + "sensor": "TBulb", + "name": "Wet-bulb temperature (mean, supplied by 3A)", + "unit": "°C", + "api_parameters": [ + "TBulbMed" + ] + }, + { + "sensor": "Pioggia", + "name": "Rain gauge (RainRateMax in mm/h)", + "unit": "mm", + "api_parameters": [ + "Pioggia" + ] + }, + { + "sensor": "BagnaturaFogliare", + "name": "Leaf wetness sensor", + "unit": "min", + "api_parameters": [ + "BagnaturaFogliare" + ] + }, + { + "sensor": "VelocitaVento2m", + "name": "Wind speed sensor (anemometer, 2 m)", + "unit": "m/s", + "api_parameters": [ + "VelocitaVento2mMax", + "VelocitaVento2mMedia" + ] + } + ], + "DailyOnlyParameters": [ + "EvapoPot" + ] + }, + { + "StationId": 12, + "StationName": "Saluzzo", + "Municipality": "Saluzzo", + "Latitude_WGS84": "44.6513", + "Longitude_WGS84": "7.45032", + "Elevation_m": "350", + "TimeZone": "UTC+1", + "NumSensors": 4, + "Sensors": [ + { + "sensor": "TemperaturaAria", + "name": "Air temperature sensor", + "unit": "°C", + "api_parameters": [ + "TemperaturaAriaMax", + "TemperaturaAriaMedia", + "TemperaturaAriaMin" + ] + }, + { + "sensor": "UmiditaRelativa", + "name": "Relative humidity sensor", + "unit": "%", + "api_parameters": [ + "UmiditaRelativaMax", + "UmiditaRelativaMedia", + "UmiditaRelativaMin" + ] + }, + { + "sensor": "TBulb", + "name": "Wet-bulb temperature (mean, supplied by 3A)", + "unit": "°C", + "api_parameters": [ + "TBulbMed" + ] + }, + { + "sensor": "Pioggia", + "name": "Rain gauge (RainRateMax in mm/h)", + "unit": "mm", + "api_parameters": [ + "Pioggia" + ] + } + ], + "DailyOnlyParameters": [ + "BagnaturaFogliare", + "EvapoPot" + ] + }, + { + "StationId": 30, + "StationName": "FossanoMaddalene", + "Municipality": "Fossano", + "Latitude_WGS84": "44.502946", + "Longitude_WGS84": "7.656242", + "Elevation_m": "397", + "TimeZone": "UTC+1", + "NumSensors": 6, + "Sensors": [ + { + "sensor": "TemperaturaAria", + "name": "Air temperature sensor", + "unit": "°C", + "api_parameters": [ + "TemperaturaAriaMax", + "TemperaturaAriaMedia", + "TemperaturaAriaMin" + ] + }, + { + "sensor": "UmiditaRelativa", + "name": "Relative humidity sensor", + "unit": "%", + "api_parameters": [ + "UmiditaRelativaMax", + "UmiditaRelativaMedia", + "UmiditaRelativaMin" + ] + }, + { + "sensor": "TBulb", + "name": "Wet-bulb temperature (mean, supplied by 3A)", + "unit": "°C", + "api_parameters": [ + "TBulbMed" + ] + }, + { + "sensor": "Pioggia", + "name": "Rain gauge (RainRateMax in mm/h)", + "unit": "mm", + "api_parameters": [ + "Pioggia" + ] + }, + { + "sensor": "BagnaturaFogliare", + "name": "Leaf wetness sensor", + "unit": "min", + "api_parameters": [ + "BagnaturaFogliare" + ] + }, + { + "sensor": "VelocitaVento2m", + "name": "Wind speed sensor (anemometer, 2 m)", + "unit": "m/s", + "api_parameters": [ + "VelocitaVento2mMax", + "VelocitaVento2mMedia" + ] + } + ], + "DailyOnlyParameters": [ + "EvapoPot" + ] + }, + { + "StationId": 145, + "StationName": "Falicetto_viaPomarolo", + "Municipality": "Falicetto", + "Latitude_WGS84": "44.5922", + "Longitude_WGS84": "7.5342", + "Elevation_m": "360", + "TimeZone": "UTC+1", + "NumSensors": 5, + "Sensors": [ + { + "sensor": "TemperaturaAria", + "name": "Air temperature sensor", + "unit": "°C", + "api_parameters": [ + "TemperaturaAriaMax", + "TemperaturaAriaMedia", + "TemperaturaAriaMin" + ] + }, + { + "sensor": "UmiditaRelativa", + "name": "Relative humidity sensor", + "unit": "%", + "api_parameters": [ + "UmiditaRelativaMax", + "UmiditaRelativaMedia", + "UmiditaRelativaMin" + ] + }, + { + "sensor": "TBulb", + "name": "Wet-bulb temperature (mean, supplied by 3A)", + "unit": "°C", + "api_parameters": [ + "TBulbMed" + ] + }, + { + "sensor": "Pioggia", + "name": "Rain gauge (RainRateMax in mm/h)", + "unit": "mm", + "api_parameters": [ + "Pioggia" + ] + }, + { + "sensor": "BagnaturaFogliare", + "name": "Leaf wetness sensor", + "unit": "min", + "api_parameters": [ + "BagnaturaFogliare" + ] + } + ], + "DailyOnlyParameters": [ + "EvapoPot" + ] + }, + { + "StationId": 153, + "StationName": "Caraglio", + "Municipality": "Caraglio", + "Latitude_WGS84": "44.4379", + "Longitude_WGS84": "7.45428", + "Elevation_m": "575", + "TimeZone": "UTC+1", + "NumSensors": 6, + "Sensors": [ + { + "sensor": "TemperaturaAria", + "name": "Air temperature sensor", + "unit": "°C", + "api_parameters": [ + "TemperaturaAriaMax", + "TemperaturaAriaMedia", + "TemperaturaAriaMin" + ] + }, + { + "sensor": "UmiditaRelativa", + "name": "Relative humidity sensor", + "unit": "%", + "api_parameters": [ + "UmiditaRelativaMax", + "UmiditaRelativaMedia", + "UmiditaRelativaMin" + ] + }, + { + "sensor": "TBulb", + "name": "Wet-bulb temperature (mean, supplied by 3A)", + "unit": "°C", + "api_parameters": [ + "TBulbMed" + ] + }, + { + "sensor": "Pioggia", + "name": "Rain gauge (RainRateMax in mm/h)", + "unit": "mm", + "api_parameters": [ + "Pioggia" + ] + }, + { + "sensor": "BagnaturaFogliare", + "name": "Leaf wetness sensor", + "unit": "min", + "api_parameters": [ + "BagnaturaFogliare" + ] + }, + { + "sensor": "TemperaturaTerreno", + "name": "Soil temperature sensor 1", + "unit": "°C", + "api_parameters": [ + "TemperaturaTerrenoMedia" + ] + } + ], + "DailyOnlyParameters": [ + "EvapoPot" + ] + }, + { + "StationId": 293, + "StationName": "CuneoSanBenigno", + "Municipality": "Cuneo", + "Latitude_WGS84": "44.4690930", + "Longitude_WGS84": "7.5447420", + "Elevation_m": "460", + "TimeZone": "UTC+1", + "NumSensors": 6, + "Sensors": [ + { + "sensor": "TemperaturaAria", + "name": "Air temperature sensor", + "unit": "°C", + "api_parameters": [ + "TemperaturaAriaMax", + "TemperaturaAriaMedia", + "TemperaturaAriaMin" + ] + }, + { + "sensor": "UmiditaRelativa", + "name": "Relative humidity sensor", + "unit": "%", + "api_parameters": [ + "UmiditaRelativaMax", + "UmiditaRelativaMedia", + "UmiditaRelativaMin" + ] + }, + { + "sensor": "TBulb", + "name": "Wet-bulb temperature (mean, supplied by 3A)", + "unit": "°C", + "api_parameters": [ + "TBulbMed" + ] + }, + { + "sensor": "Pioggia", + "name": "Rain gauge (RainRateMax in mm/h)", + "unit": "mm", + "api_parameters": [ + "Pioggia", + "RainRateMax" + ] + }, + { + "sensor": "BagnaturaFogliare", + "name": "Leaf wetness sensor", + "unit": "min", + "api_parameters": [ + "BagnaturaFogliare" + ] + }, + { + "sensor": "Pressione", + "name": "Barometric pressure sensor", + "unit": "mbar", + "api_parameters": [ + "PressioneMax", + "PressioneMed", + "PressioneMin" + ] + } + ], + "DailyOnlyParameters": [ + "EvapoPot", + "RadNettaInt", + "RadiazioneSolareInt" + ] + }, + { + "StationId": 544, + "StationName": "RevelloSanPietro", + "Municipality": "Revello", + "Latitude_WGS84": "44.636467", + "Longitude_WGS84": "7.395118", + "Elevation_m": "300", + "TimeZone": "UTC+1", + "NumSensors": 8, + "Sensors": [ + { + "sensor": "TemperaturaAria", + "name": "Air temperature sensor", + "unit": "°C", + "api_parameters": [ + "TemperaturaAriaMax", + "TemperaturaAriaMedia", + "TemperaturaAriaMin" + ] + }, + { + "sensor": "UmiditaRelativa", + "name": "Relative humidity sensor", + "unit": "%", + "api_parameters": [ + "UmiditaRelativaMax", + "UmiditaRelativaMedia", + "UmiditaRelativaMin" + ] + }, + { + "sensor": "Pioggia", + "name": "Rain gauge (RainRateMax in mm/h)", + "unit": "mm", + "api_parameters": [ + "Pioggia" + ] + }, + { + "sensor": "BagnaturaFogliare", + "name": "Leaf wetness sensor", + "unit": "min", + "api_parameters": [ + "BagnaturaFogliare" + ] + }, + { + "sensor": "VelocitaVento2m", + "name": "Wind speed sensor (anemometer, 2 m)", + "unit": "m/s", + "api_parameters": [ + "VelocitaVento2mMax", + "VelocitaVento2mMedia" + ] + }, + { + "sensor": "RadiazioneSolare", + "name": "Solar radiation sensor (pyranometer, irradiance)", + "unit": "W/m²", + "api_parameters": [ + "RadiazioneSolareMax", + "RadiazioneSolareMin" + ] + }, + { + "sensor": "Suolo1", + "name": "Soil moisture sensor 1", + "unit": "%", + "api_parameters": [ + "Suolo1" + ] + }, + { + "sensor": "Suolo2", + "name": "Soil moisture sensor 2", + "unit": "%", + "api_parameters": [ + "Suolo2" + ] + } + ], + "DailyOnlyParameters": [ + "EvapoPot", + "RadiazioneSolareInt" + ] + }, + { + "StationId": 566, + "StationName": "CuneoGiordano", + "Municipality": "Cuneo", + "Latitude_WGS84": "44.450417", + "Longitude_WGS84": "7.544556", + "Elevation_m": "460", + "TimeZone": "UTC+1", + "NumSensors": 7, + "Sensors": [ + { + "sensor": "TemperaturaAria", + "name": "Air temperature sensor", + "unit": "°C", + "api_parameters": [ + "TemperaturaAriaMax", + "TemperaturaAriaMedia", + "TemperaturaAriaMin" + ] + }, + { + "sensor": "UmiditaRelativa", + "name": "Relative humidity sensor", + "unit": "%", + "api_parameters": [ + "UmiditaRelativaMax", + "UmiditaRelativaMedia", + "UmiditaRelativaMin" + ] + }, + { + "sensor": "TBulb", + "name": "Wet-bulb temperature (mean, supplied by 3A)", + "unit": "°C", + "api_parameters": [ + "TBulbMed" + ] + }, + { + "sensor": "Pioggia", + "name": "Rain gauge (RainRateMax in mm/h)", + "unit": "mm", + "api_parameters": [ + "Pioggia" + ] + }, + { + "sensor": "BagnaturaFogliare", + "name": "Leaf wetness sensor", + "unit": "min", + "api_parameters": [ + "BagnaturaFogliare" + ] + }, + { + "sensor": "VelocitaVento2m", + "name": "Wind speed sensor (anemometer, 2 m)", + "unit": "m/s", + "api_parameters": [ + "VelocitaVento2mMax", + "VelocitaVento2mMedia" + ] + }, + { + "sensor": "RadiazioneSolare", + "name": "Solar radiation sensor (pyranometer, irradiance)", + "unit": "W/m²", + "api_parameters": [ + "RadiazioneSolareMax", + "RadiazioneSolareMin" + ] + } + ], + "DailyOnlyParameters": [ + "EvapoPot", + "RadiazioneSolareInt" + ] + }, + { + "StationId": 600, + "StationName": "CascinaFrati", + "Municipality": "Revello", + "Latitude_WGS84": "44.668629", + "Longitude_WGS84": "7.379682", + "Elevation_m": "0", + "TimeZone": "", + "NumSensors": 10, + "Sensors": [ + { + "sensor": "TemperaturaAria", + "name": "Air temperature sensor", + "unit": "°C", + "api_parameters": [ + "TemperaturaAriaMax", + "TemperaturaAriaMedia", + "TemperaturaAriaMin" + ] + }, + { + "sensor": "UmiditaRelativa", + "name": "Relative humidity sensor", + "unit": "%", + "api_parameters": [ + "UmiditaRelativaMax", + "UmiditaRelativaMedia", + "UmiditaRelativaMin" + ] + }, + { + "sensor": "Pioggia", + "name": "Rain gauge (RainRateMax in mm/h)", + "unit": "mm", + "api_parameters": [ + "Pioggia" + ] + }, + { + "sensor": "BagnaturaFogliare", + "name": "Leaf wetness sensor", + "unit": "min", + "api_parameters": [ + "BagnaturaFogliare" + ] + }, + { + "sensor": "VelocitaVento2m", + "name": "Wind speed sensor (anemometer, 2 m)", + "unit": "m/s", + "api_parameters": [ + "VelocitaVento2mMax", + "VelocitaVento2mMedia" + ] + }, + { + "sensor": "RadiazioneSolare", + "name": "Solar radiation sensor (pyranometer, irradiance)", + "unit": "W/m²", + "api_parameters": [ + "RadiazioneSolareMax", + "RadiazioneSolareMin" + ] + }, + { + "sensor": "Suolo1", + "name": "Soil moisture sensor 1", + "unit": "%", + "api_parameters": [ + "Suolo1" + ] + }, + { + "sensor": "Suolo2", + "name": "Soil moisture sensor 2", + "unit": "%", + "api_parameters": [ + "Suolo2" + ] + }, + { + "sensor": "TemperaturaTerreno", + "name": "Soil temperature sensor 1", + "unit": "°C", + "api_parameters": [ + "TemperaturaTerrenoMedia" + ] + }, + { + "sensor": "TemperaturaTerreno2", + "name": "Soil temperature sensor 2", + "unit": "°C", + "api_parameters": [ + "TemperaturaTerrenoMedia2" + ] + } + ], + "DailyOnlyParameters": [ + "EvapoPot", + "RadiazioneSolareInt" + ] + } + ] +} diff --git a/API/fetch_3a.py b/API/fetch_3a.py new file mode 100644 index 0000000..8b5b630 --- /dev/null +++ b/API/fetch_3a.py @@ -0,0 +1,493 @@ +#!/usr/bin/env python3 +""" +fetch_3a.py - fetch stations, sensors and weather data from the 3A API +(https://api.3acloud.it, spec in 3a_api_dump/spec__serve_docs_base.json). + + python fetch_3a.py overview # stations count, sensors, columns, date ranges + python fetch_3a.py stations # list of stations (+ count) + python fetch_3a.py sensors # hourly/daily sensors per station + python fetch_3a.py catalogue # ask again (3 requests), rewrite 3a_stations_sensors.json + python fetch_3a.py columns # all hourly/daily column names + python fetch_3a.py daily --from 2024-01-01 --to 2024-12-31 + python fetch_3a.py hourly --from 2024-01-01 --to 2024-01-31 --stations 4,46 \ + --columns CodiceStazione,Stazione,Timestamp,TemperaturaAriaMedia,Pioggia + + options: --stations 1,2,3 (default: all stations) + --columns a,b,c (default: all available columns) + --chunk-days N (split long periods into N-day requests) + --refresh (ignore cached metadata and ask the server again) + +How auth works (from the spec): + POST /login with headers name, password, totp -> token + every other call sends header token: + +Credentials are not stored in this file: + THREEA_USER username (default: mohsen) + THREEA_PASSWORD password (else asked with a hidden prompt) + THREEA_TOTP_SECRET optional base32 TOTP secret -> codes generated automatically + (else you are asked to paste the 6-digit code when a login is needed) + +The token is cached in 3a_data/.token, so you only need a new TOTP when it expires. +The server allows only 5 requests per 15 minutes; the script waits automatically. +Results are written to 3a_data/ as .json and .csv. +""" + +import argparse +import getpass +import json +import os +import sys +import time +from datetime import datetime, timedelta +from pathlib import Path + +import pandas as pd +import requests + +BASE_URL = "https://api.3acloud.it" +OUT = Path(__file__).resolve().parent / "3a_data" +OUT.mkdir(exist_ok=True) +TOKEN_FILE = OUT / ".token" +TIMEOUT = 120 + +sys.stdout.reconfigure(encoding="utf-8", errors="replace") +session = requests.Session() +session.headers["output-fmt"] = "json" +_password = None + + +# ─────────────────────────── rate limit ─────────────────────────── + +_rate = {"remaining": None, "reset": None} + + +def _note_rate_limit(resp): + h = resp.headers + if "X-RateLimit-Remaining" in h: + _rate["remaining"] = int(h["X-RateLimit-Remaining"]) + if "X-RateLimit-Reset" in h: + _rate["reset"] = int(h["X-RateLimit-Reset"]) + + +def _wait_until(epoch, why): + while True: + left = int(epoch - time.time()) + 2 + if left <= 0: + break + print(f"\r {why}: waiting {left // 60:02d}:{left % 60:02d} ", end="", flush=True) + time.sleep(min(left, 5)) + print("\r" + " " * 60 + "\r", end="") + + +def _respect_rate_limit(): + if _rate["remaining"] == 0 and _rate["reset"] and _rate["reset"] > time.time(): + _wait_until(_rate["reset"], "rate limit (5 requests / 15 min)") + + +def _send(method, path, **kw): + """One HTTP request, waiting for the rate-limit window if needed.""" + while True: + _respect_rate_limit() + r = session.request(method, BASE_URL + path, timeout=TIMEOUT, **kw) + _note_rate_limit(r) + if r.status_code == 429: + reset = _rate["reset"] or time.time() + 15 * 60 + _wait_until(reset, "server said 429 Too Many Requests") + continue + return r + + +# ─────────────────────────── auth ─────────────────────────── + +CRED_FILE = Path(__file__).resolve().parent / "3a_credentials.json" + + +def _cred(key, env): + """Value from the environment, else from 3a_credentials.json (if filled in).""" + if os.environ.get(env): + return os.environ[env] + if CRED_FILE.exists(): + value = json.loads(CRED_FILE.read_text(encoding="utf-8")).get(key, "") + if value and not value.startswith("PUT_"): + return value + return None + + +def _ask_totp(): + secret = _cred("totp_secret", "THREEA_TOTP_SECRET") + if secret: + import pyotp # pip install pyotp + return pyotp.TOTP(secret.replace(" ", "")).now() + while True: + code = input("Paste a fresh 6-digit TOTP code (it changes every 30 s): ").strip().replace(" ", "") + if code.isdigit() and len(code) == 6: + return code + print(" That doesn't look like a 6-digit code, try again.") + + +def _extract_token(resp): + try: + data = resp.json() + except ValueError: + return resp.text.strip().strip('"') or None + if isinstance(data, str): + return data + if isinstance(data, dict): + for k in ("token", "access_token", "Token"): + if k in data: + return data[k] + return None + + +def login(): + global _password + user = _cred("user", "THREEA_USER") or "mohsen" + if _password is None: + _password = _cred("password", "THREEA_PASSWORD") or getpass.getpass( + f"Password for {user} (typing is hidden, press Enter when done): ") + for attempt in range(3): + totp = _ask_totp() + r = _send("POST", "/login", headers={"name": user, "password": _password, "totp": totp}) + token = _extract_token(r) if r.ok else None + if token: + session.headers["token"] = token + TOKEN_FILE.write_text(token) + print(" ✓ logged in, token cached") + return + print(f" ✗ login failed ({r.status_code}): {r.text[:200]}") + if r.status_code not in (401, 403): + break + print(" The TOTP may have expired or been mistyped.") + sys.exit("Could not log in.") + + +def _load_cached_token(): + if TOKEN_FILE.exists(): + session.headers["token"] = TOKEN_FILE.read_text().strip() + return True + return False + + +def call(method, path, body=None): + """Authenticated API call; logs in (asking for a TOTP) only when needed.""" + if "token" not in session.headers and not _load_cached_token(): + login() + r = _send(method, path, json=body) + if r.status_code in (401, 403): + print(f" token rejected ({r.status_code}: {r.text[:80]}) -> new login needed") + TOKEN_FILE.unlink(missing_ok=True) + session.headers.pop("token", None) + login() + r = _send(method, path, json=body) + if not r.ok: + sys.exit(f"{method} {path} failed ({r.status_code}): {r.text[:500]}") + try: + return r.json() + except ValueError: + return r.text + + +# ─────────────────────────── saving ─────────────────────────── + +def save(name, data): + (OUT / f"{name}.json").write_text(json.dumps(data, indent=2, ensure_ascii=False), encoding="utf-8") + df = to_frame(data) + if df is not None: + df.to_csv(OUT / f"{name}.csv", index=False, encoding="utf-8-sig") + return df + + +def to_frame(data): + if isinstance(data, list) and data and all(isinstance(x, dict) for x in data): + return pd.json_normalize(data) + if isinstance(data, dict): + # e.g. {"data": [...]} or {station_id: [...]} + lists = {k: v for k, v in data.items() if isinstance(v, list)} + if len(lists) == 1: + return to_frame(next(iter(lists.values()))) + if lists and all(not isinstance(x, dict) for v in lists.values() for x in v): + return pd.DataFrame([(k, x) for k, v in lists.items() for x in v], columns=["key", "value"]) + return None + + +def cached(name, method, path, body=None, refresh=False): + """Metadata changes rarely and requests are scarce, so reuse what's on disk.""" + f = OUT / f"{name}.json" + if f.exists() and not refresh: + return json.loads(f.read_text(encoding="utf-8")) + print(f"→ {method} {path}") + data = call(method, path, body) + save(name, data) + return data + + +# ─────────────────────────── commands ─────────────────────────── + +def get_stations(refresh=False): + data = cached("stations", "GET", "/getAvailableStations", refresh=refresh) + df = to_frame(data) + return data, df + + +def station_ids(args): + if args.stations: + return [int(s) for s in args.stations.split(",")] + _, df = get_stations(args.refresh) + col = next((c for c in ("StationId", "id") if df is not None and c in df.columns), None) + if col is None: + sys.exit("Could not read station ids from stations.json; pass --stations explicitly.") + return sorted(int(i) for i in df[col]) + + +def cmd_stations(args): + data, df = get_stations(args.refresh) + if df is None: + print(json.dumps(data, indent=2, ensure_ascii=False)[:3000]) + return + print(f"\nNumber of stations: {len(df)}") + with pd.option_context("display.max_rows", 500, "display.width", 200): + print(df.to_string(index=False)) + print(f"\nSaved → {OUT / 'stations.csv'}") + + +def cmd_sensors(args): + ids = station_ids(args) + for kind in ("Hourly", "Daily"): + data = cached(f"sensors_{kind.lower()}", "POST", f"/getAvailableStation{kind}Sensors", + {"stationsIds": ids}, args.refresh) + print(f"\n{kind} sensors:") + print(json.dumps(data, indent=1, ensure_ascii=False)[:4000]) + print(f" saved → {OUT / f'sensors_{kind.lower()}.json'}") + + +def cmd_summary(args): + """Station x sensor tables built from stations.json + sensors_*.json.""" + _, stations = get_stations(args.refresh) + names = dict(zip(stations["StationId"].astype(int), stations["StationName"])) + rows = [] + for kind in ("Hourly", "Daily"): + data = cached(f"sensors_{kind.lower()}", "POST", f"/getAvailableStation{kind}Sensors", + {"stationsIds": station_ids(args)}, args.refresh) + items = data if isinstance(data, list) else [data] + for item in items: + for sid, sensors in item.items(): + for s in sensors: + rows.append({"StationId": int(sid), "StationName": names.get(int(sid), ""), + "Resolution": kind.lower(), "Sensor": s.get("name"), + "Description": s.get("description"), "Unit": s.get("unit")}) + long = pd.DataFrame(rows).sort_values(["StationId", "Resolution", "Sensor"]) + long.to_csv(OUT / "station_sensors_list.csv", index=False, encoding="utf-8-sig") + + hourly = long[long.Resolution == "hourly"] + matrix = pd.crosstab([hourly.StationId, hourly.StationName], hourly.Sensor).clip(upper=1) + matrix = matrix.reindex(pd.MultiIndex.from_arrays( + [stations["StationId"].astype(int), stations["StationName"]]), fill_value=0) + matrix.insert(0, "NumSensors", matrix.sum(axis=1)) + matrix.to_csv(OUT / "station_sensors_matrix_hourly.csv", encoding="utf-8-sig") + + with pd.option_context("display.max_rows", 500, "display.width", 250): + print(f"\n{len(stations)} stations. Hourly sensors per station (1 = present):\n") + print(matrix.to_string()) + print("\nSensor catalogue (how many stations have each, hourly / daily):\n") + cat = long.groupby(["Sensor", "Description", "Unit", "Resolution"]).StationId.nunique() + print(cat.unstack("Resolution").fillna(0).astype(int).to_string()) + print(f"\nSaved → {OUT / 'station_sensors_list.csv'}\n {OUT / 'station_sensors_matrix_hourly.csv'}") + + +# ─────────────────────────── catalogue (3a_stations_sensors.json) ─────────────────────────── + +CATALOGUE_FILE = Path(__file__).resolve().parent / "3a_stations_sensors.json" + +# sensor group (name used by the engine's stations.py) -> label, unit, its hourly api_parameters +SENSOR_GROUPS = [ + ("TemperaturaAria", "Air temperature sensor", "°C", + ["TemperaturaAriaMax", "TemperaturaAriaMedia", "TemperaturaAriaMin"]), + ("UmiditaRelativa", "Relative humidity sensor", "%", + ["UmiditaRelativaMax", "UmiditaRelativaMedia", "UmiditaRelativaMin"]), + ("TBulb", "Wet-bulb temperature (mean, supplied by 3A)", "°C", ["TBulbMed"]), + ("Pioggia", "Rain gauge (RainRateMax in mm/h)", "mm", ["Pioggia", "RainRateMax"]), + ("BagnaturaFogliare", "Leaf wetness sensor", "min", ["BagnaturaFogliare"]), + ("VelocitaVento2m", "Wind speed sensor (anemometer, 2 m)", "m/s", + ["VelocitaVento2mMax", "VelocitaVento2mMedia"]), + ("Pressione", "Barometric pressure sensor", "mbar", ["PressioneMax", "PressioneMed", "PressioneMin"]), + ("RadiazioneSolare", "Solar radiation sensor (pyranometer, irradiance)", "W/m²", + ["RadiazioneSolareMax", "RadiazioneSolareMin"]), + ("Suolo1", "Soil moisture sensor 1", "%", ["Suolo1"]), + ("Suolo2", "Soil moisture sensor 2", "%", ["Suolo2"]), + ("TemperaturaTerreno", "Soil temperature sensor 1", "°C", ["TemperaturaTerrenoMedia"]), + ("TemperaturaTerreno2", "Soil temperature sensor 2", "°C", ["TemperaturaTerrenoMedia2"]), +] + + +def _sensors_by_station(data): + """{station id: {api_parameter: sensor entry}} from a getAvailableStation*Sensors answer.""" + out = {} + for item in (data if isinstance(data, list) else [data]): + for sid, sensors in item.items(): + out.setdefault(int(sid), {}).update({s["name"]: s for s in sensors}) + return out + + +def build_catalogue(stations, hourly, daily, checked): + """Catalogue dict from the three metadata answers (no API call). + + Sensors = HOURLY sensors grouped as in SENSOR_GROUPS; a parameter not in + any group gets its own group (named after it, label/unit from the API). + DailyOnlyParameters = columns offered only in daily data. + """ + hourly, daily = _sensors_by_station(hourly), _sensors_by_station(daily) + known = {p for g in SENSOR_GROUPS for p in g[3]} + result = [] + for st in sorted(stations, key=lambda s: int(s["StationId"])): + sid = int(st["StationId"]) + have = hourly.get(sid, {}) + sensors = [] + for key, label, unit, params in SENSOR_GROUPS: + present = [p for p in params if p in have] + if present: + sensors.append({"sensor": key, "name": label, "unit": unit, "api_parameters": present}) + for p in sorted(set(have) - known): + print(f" note: station {sid} has new hourly parameter {p!r}; listed as its own sensor") + sensors.append({"sensor": p, "name": have[p].get("description", p), + "unit": have[p].get("unit", ""), "api_parameters": [p]}) + entry = {k: st.get(k) for k in ("StationId", "StationName", "Municipality", "Latitude_WGS84", + "Longitude_WGS84", "Elevation_m", "TimeZone")} + entry.update(NumSensors=len(sensors), Sensors=sensors, + DailyOnlyParameters=sorted(set(daily.get(sid, {})) - set(have))) + result.append(entry) + return { + "source": "https://api.3acloud.it (getAvailableStations, getAvailableStationHourlySensors, " + "getAvailableStationDailySensors)", + "checked": checked, + "note": "Sensors = HOURLY sensors (what the blocks fetch). DailyOnlyParameters = columns the " + "station offers only in daily data (/getWeatherDaily). Units from /getAvailableHourlyColumns.", + "NumStations": len(result), + "Stations": result, + } + + +def cmd_catalogue(args): + """Ask the server again (3 requests) and rewrite 3a_stations_sensors.json.""" + stations, _ = get_stations(refresh=True) + ids = sorted(int(s["StationId"]) for s in stations) + sensors = {kind: cached(f"sensors_{kind.lower()}", "POST", f"/getAvailableStation{kind}Sensors", + {"stationsIds": ids}, refresh=True) for kind in ("Hourly", "Daily")} + old = json.loads(CATALOGUE_FILE.read_text(encoding="utf-8")) if CATALOGUE_FILE.exists() else {} + new = build_catalogue(stations, sensors["Hourly"], sensors["Daily"], datetime.now().strftime("%Y-%m-%d")) + params = lambda cat: {s["StationId"]: {p for x in s["Sensors"] for p in x["api_parameters"]} + for s in cat.get("Stations", [])} + before, after = params(old), params(new) + print("\nChanges in hourly parameters since the last catalogue:") + for sid in sorted(set(before) | set(after)): + added, removed = after.get(sid, set()) - before.get(sid, set()), before.get(sid, set()) - after.get(sid, set()) + if added or removed: + print(f" {sid}: added {sorted(added)} removed {sorted(removed)}") + if before == after: + print(" none") + CATALOGUE_FILE.write_text(json.dumps(new, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") + print(f"\nSaved → {CATALOGUE_FILE} (update README.md if anything changed)") + + +def get_columns(kind, refresh=False): + data = cached(f"columns_{kind.lower()}", "GET", f"/getAvailable{kind}Columns", refresh=refresh) + if isinstance(data, dict): + data = next((v for v in data.values() if isinstance(v, list)), data) + if isinstance(data, list) and data and isinstance(data[0], dict): + key = next(iter(data[0])) + data = [d[key] for d in data] + return data + + +def cmd_columns(args): + for kind in ("Hourly", "Daily"): + cols = get_columns(kind, args.refresh) + print(f"\n{kind} columns ({len(cols)}):\n " + ", ".join(map(str, cols))) + + +def cmd_overview(args): + cmd_stations(args) + cmd_sensors(args) + cmd_columns(args) + ids = station_ids(args) + print("\nAvailable periods:") + for kind in ("Daily", "Hourly"): + for edge in ("First", "Last"): + data = cached(f"{edge.lower()}_{kind.lower()}_timestamp", "POST", + f"/get{edge}Available{kind}Timestamp", {"stationsIds": ids}, args.refresh) + print(f" {edge:5s} {kind:6s}: {json.dumps(data, ensure_ascii=False)[:300]}") + + +def _parse_date(s): + for fmt in ("%Y-%m-%d %H:%M", "%Y-%m-%d"): + try: + return datetime.strptime(s, fmt) + except ValueError: + pass + sys.exit(f"Bad date {s!r}; use YYYY-MM-DD or 'YYYY-MM-DD HH:MM'") + + +def _periods(start, end, chunk_days): + if not chunk_days: + yield start, end + return + cur = start + while cur <= end: + nxt = min(cur + timedelta(days=chunk_days) - timedelta(minutes=1), end) + yield cur, nxt + cur = nxt + timedelta(minutes=1) + + +def cmd_weather(args, kind): + ids = station_ids(args) + columns = args.columns.split(",") if args.columns else get_columns(kind, args.refresh) + start, end = _parse_date(args.date_from), _parse_date(args.date_to) + if len(args.date_to) == 10: + end = end.replace(hour=23, minute=59) + + frames, raw = [], [] + for a, b in _periods(start, end, args.chunk_days): + body = {"stationsIds": ids, "dateFrom": a.strftime("%Y-%m-%d %H:%M"), + "dateTo": b.strftime("%Y-%m-%d %H:%M"), "columns": columns} + print(f"→ {kind} data {body['dateFrom']} … {body['dateTo']} for {len(ids)} station(s)") + data = call("POST", f"/getWeather{kind}", body) + raw.append(data) + df = to_frame(data) + if df is not None: + frames.append(df) + print(f" {len(df)} rows") + else: + print(" (response not tabular, kept in JSON)", str(data)[:200]) + + name = f"weather_{kind.lower()}_{start:%Y%m%d}_{end:%Y%m%d}" + (OUT / f"{name}.json").write_text(json.dumps(raw if len(raw) > 1 else raw[0], indent=1, + ensure_ascii=False), encoding="utf-8") + if frames: + df = pd.concat(frames, ignore_index=True) + df.to_csv(OUT / f"{name}.csv", index=False, encoding="utf-8-sig") + print(f"\n{len(df)} rows, {df.shape[1]} columns → {OUT / (name + '.csv')}") + print(df.head(10).to_string(index=False)) + + +def main(): + p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + p.add_argument("command", choices=["overview", "stations", "sensors", "summary", "catalogue", "columns", + "daily", "hourly", "login"]) + p.add_argument("--from", dest="date_from") + p.add_argument("--to", dest="date_to") + p.add_argument("--stations", help="comma-separated station ids (default: all)") + p.add_argument("--columns", help="comma-separated column names (default: all)") + p.add_argument("--chunk-days", type=int, help="split the period into N-day requests") + p.add_argument("--refresh", action="store_true", help="re-download cached metadata") + args = p.parse_args() + + if args.command in ("daily", "hourly") and not (args.date_from and args.date_to): + p.error("--from and --to are required for daily/hourly") + + {"overview": cmd_overview, "stations": cmd_stations, "sensors": cmd_sensors, "summary": cmd_summary, + "catalogue": cmd_catalogue, + "columns": cmd_columns, "login": lambda a: login(), + "daily": lambda a: cmd_weather(a, "Daily"), "hourly": lambda a: cmd_weather(a, "Hourly")}[args.command](args) + + +if __name__ == "__main__": + main() diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..fa9bb75 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,11 @@ +# Instructions for Claude + +1. Before any task, read `README.md`. It is the starting point: project + layout, rules, block status, known issues and the change log. +2. This project is private: never upload it, create git/GitHub files, or + print or send the 3A credentials (`API/3a_credentials.json`, + `API/mohsen_new_api.html`). +3. 3A API: 5 requests per 15 minutes, hourly data only for about the last + 180 days. Keep live calls to a minimum and prefer unit tests. +4. After every change, update `README.md`: the matching section (layout, + block status, known issues) and a dated entry in the *Change log*. diff --git a/Diagram.png b/Diagram.png new file mode 100644 index 0000000..23a15c9 Binary files /dev/null and b/Diagram.png differ diff --git a/docs/ImplementazioneAlgoritmiSensori.pdf b/docs/ImplementazioneAlgoritmiSensori.pdf new file mode 100644 index 0000000..90b77bb --- /dev/null +++ b/docs/ImplementazioneAlgoritmiSensori.pdf @@ -0,0 +1,364 @@ + 3a Soc. di Sviluppo per l’Ambiente e l’AgroAlimentare a r.l. + + ALGORITMI di STIMA di alcune GRANDEZZE + AGROMETEOROGLICHE + +1. RADIAZIONE GLOBALE (Rg) giornaliera + + Algoritmo Winslow et al. (2001) + +Radiazione solare extraterrestre: + +Ra(d)=C*(24/π)*d(d)*[sin(λ)*sin(δ(d))*ωs+ cos(λ)*cos(δ(d))*sin(ωs)] (1) + +Dove: + + il pedice (d) indica il passo temporale (day) + C=4.921 + +d(d ) = 1+ 2  i) + 0.033 cos( + 365 + +i= numero giuliano dell’anno (1 è il primo di gennaio) + +λ= latitudine (° o radianti), dipende cosa richiede la funzione sin o cos nel linguaggio di +programmazione usato. Di solito vuole l’angolo in radianti: + +[radianti]= [angolo in °] *π/180°. + +(d) = 0.409sin( 2 i −1.39) + 365 + + (2) + +La radiazione solare globale potenziale (a cielo sereno): + +Rso=(0.75+2*10-5*z)*Ra (3) + +Dove: + +z è l’altezza della stazione slm [m]; + +3a Srl P.IVA 07366180011 N. REA 154455 SASSARI +Sede Legale Via al Carmine, 2 - 07100 Sassari + +Sede di Torino Via Le Chiuse 68 - 10144 - Telefono: 011 437 8552 Fax : +39 11 437 8221 E-Mail:3a@green-planet.it + 3a Soc. di Sviluppo per l’Ambiente e l’AgroAlimentare a r.l. + + Ra la radiazione extraterrestre dell’eq (1). + +E infine la radiazione globale giornaliera: + (4) + +Dove: + Sd o Rso Radiazione solare potenziale eq (3) + τ Trasmissività + La trasmissività a cielo sereno τ (è un parametro caratteristico del sito, si calcola avendo a + disposizione un anno almeno di dati di radiazione e il calcolo va fatto una volta sola). + Per ciascun giorno si calcola il rapporto: + (5) + Dove Gx è il valore di radiazione globale misurato. + Per stimare il valore di τ si effettua la media dei 5 valori più alti ottenuti. + Se non si ha a disposizione uno storico di dati anziché una media dei max valori annuali dell’eq + (5), si inserisce un valore di riferimento pari a 0.75. + + dall’eq (2) + + T = escursione termica media dell’anno (T max (giorno) - T min (giorno)) + +3a Srl P.IVA 07366180011 N. REA 154455 SASSARI +Sede Legale Via al Carmine, 2 - 07100 Sassari + +Sede di Torino Via Le Chiuse 68 - 10144 - Telefono: 011 437 8552 Fax : +39 11 437 8221 E-Mail:3a@green-planet.it + 3a Soc. di Sviluppo per l’Ambiente e l’AgroAlimentare a r.l. + +T = temperatura media dell’anno + +2. RADIAZIONE NETTA (Rn) giornaliera + +metodo FAO + +Rn giornaliera viene stimata a partire da quella globale. Non avendo a disposizione un radiometro +globale prendiamo il valore stimato dall’algoritmo (4) della sezione 1. + +Radiazione solare netta a onda corta (Rns) [MJ m-2 giorno-1] + +Rns = (1-α) Rg (1) + +α albedo [] (pari a 0.23 per un tappeto erboso di riferimento ), + +Radiazione solare netta a onda lunga (Rnl) [MJ m-2 giorno-1]: + + (2) (2) + +σ costante Stefan-Boltzmann [4.903 10-9 MJ K-4 m-2 day-1], + +Tmax, K temperatura massima assoluta giornaliera [K =°C + 273.16], + +Tmin, K temperatura minima assoluta giornaliera [K = °C+ 273.16], + +ea pressione di vapore attuale [kPa] eq (2) della sezione 3. + +Rs/Rso è il rapporto tra la radiazione globale e quella potenziale (deve essere limitata <= 1.0), dalle +equazioni (4) e (1) della sezione 1. + +Rs o Rg [MJ m-2 day-1], equazioni (4) nell’algoritmo della sezione 1. + +Rso o S(d) [MJ m-2 day-1], equazione (1) nell’algoritmo della sezione 1. +E infine la radiazione netta complessiva sarà [MJ m-2 giorno-1]: + +Rn=Rns-Rnl (3) + +3. EVAPOTRASPIRAZIONE POTENZIALE (ET0) giornaliera + + Algoritmo Penman-Monteith, metodo FAO + +3a Srl P.IVA 07366180011 N. REA 154455 SASSARI +Sede Legale Via al Carmine, 2 - 07100 Sassari + +Sede di Torino Via Le Chiuse 68 - 10144 - Telefono: 011 437 8552 Fax : +39 11 437 8221 E-Mail:3a@green-planet.it + 3a Soc. di Sviluppo per l’Ambiente e l’AgroAlimentare a r.l. + + 0.408    (Rn − G) +   T 900  u  (es − ea ) + + 273 +ET0 = +  +   (1 + 0.34  u) + (1) + +dove, + +ET0 (mm) è l’evapotraspirazione; + +( ) Rn MJ / m2 è la radiazione netta giornaliera, dall’equazione (3) della sezione 2.; +( ) G MJ / m2  0 è il flusso di calore nel suolo; + +(KPa / C) = 4098 es (T ) è la pendenza della curva di pressione; + (T + 237.3)2 + +u(m/s) è la velocità del vento medio giornaliero; +T(°C) è la temperatura media dell’aria; + + (KPa/C) = 0.00163 P  0.163/  è la costante psicometrica; +  + +P la pressione in kPa; +(MJ / Kg ) = 2.5 − 0.00236T è il calore di vaporizzazione; + +es (T )(KPa) = 0.61 exp( 17.27 T ) è la pressione di vapore saturo; + T + 237.3 + +ea(KPa)= RH è la pressione di vapore (sarebbe meglio fare la media tra quella minima e + es (T )  100 + +quella massima giornaliera) ovvero: + + 0.611 exp( 17.27  Tmin )  RH max + 0.611 exp( 17.27  Tmax )  RH min + Tmin + 237.3 Tmax + 237.3 +ea(KPa)= + 2 100 + (2) + +RH è l’umidità media giornaliera; + +RHmax è l’umidità massima giornaliera; + +RHmin è l’umidità minima giornaliera; + +Tmin è la temperatura minima giornaliera; + +Tmax è la temperatura massima giornaliera. + +4. TEMPERATURA BULBO UMIDO (Tw) + +3a Srl P.IVA 07366180011 N. REA 154455 SASSARI +Sede Legale Via al Carmine, 2 - 07100 Sassari + +Sede di Torino Via Le Chiuse 68 - 10144 - Telefono: 011 437 8552 Fax : +39 11 437 8221 E-Mail:3a@green-planet.it + 3a Soc. di Sviluppo per l’Ambiente e l’AgroAlimentare a r.l. + +Tw = Td + T − Td -273.15 (1) + 1+ L2  s + + Rv  C p T 2 + +dove + +Rv=0.1c1aKl gèK la costante del gas per l’aria umida, + +Cp=0.2c4aKlgèK il calore specifico a pressione costante, + +Tw (°C) è la temperatura di bulbo umido, +Td (°K) è la temperatura di rugiada, +T (°K) è la temperatura media dell’aria (bulbo secco), +s(Kg/Kg) è l’umidità specifica, +L(cal/Kg) è il calore latente di evaporazione o sublimazione. +I valori utili per ricostruire la temperatura di bulbo umido variano a seconda che ci sia acqua o +ghiaccio. +Di seguito verranno riportati le formule e i valori per T > 0 e T<=0. + + a) Per T>0 +La temperatura misurata dalle stazioni in °C deve essere convertita in °K: +T (K )=T (C )+2.7 13 5 + +Il calore di evaporazione dell’acqua vale: + +L=59c7aKl g + +La formula per ottenere Td , ovvero la Temperatura di bulbo asciutto, è data da: + + 237.3  log e + +Td = 6.1 + 273.15 + e + 7.5 − log + 6.1 (2) + +dove e(mbar) è la pressione di vapore. Essa si ricava nel modo seguente: + +e=RHe*(T) + 100 + +dove RH(%) è l’umidità relativa e e0(T) è la tensione di vapore saturo alla temperatura T calcolata +con la formula: + + 7 .5 (T− 2.7 1)3 5 + +e*(T )=6 .110 T− 3.8 55 + +3a Srl P.IVA 07366180011 N. REA 154455 SASSARI +Sede Legale Via al Carmine, 2 - 07100 Sassari + +Sede di Torino Via Le Chiuse 68 - 10144 - Telefono: 011 437 8552 Fax : +39 11 437 8221 E-Mail:3a@green-planet.it + 3a Soc. di Sviluppo per l’Ambiente e l’AgroAlimentare a r.l. + +Il dato di umidità specifica è calcolato con la seguente equazione: + +s= 0.62e2 + P−0.37e8 + +con P pressione atmosferica (generalmente si può porre 1000 mbar). +Si possono dunque sostituire i valori trovati di T, Td, L ed s nell’equazione (1) e trovare la +temperatura di bulbo umido. + +b) Per T<=0 + +Il calcolo della temperatura di bulbo umido per T<=0 è analogo al caso precedente, ma alcune +grandezze assumono valori differenti. + +La temperatura misurata dalle stazioni in °C deve essere convertita in °K: + +T (K )=T (C )+2.7 13 5 + +Il calore di sublimazione vale: + +L=67c7aKl g + +La formula per ottenere Td è la seguente: + +26.55 log e +Td =9.5−log 6 e.1+27.135 +6.1 + (3) + +La tensione di vapore saturo alla temperatura T è calcolata per mezzo dell’equazione: + +e = RH  e*(T ) + 100 + + 9.5(T −273.15) + +e* (T ) = 6.110 T −7.65 + +Si possono dunque sostituire i valori trovati di T, Td, L ed s nell’equazione (1) e trovare la +temperatura di bulbo umido. + +5. TEMPERATURA DI RUGIADA (Td) + + Sulla base dell’equazione di Murray + Nel caso T>0 + +3a Srl P.IVA 07366180011 N. REA 154455 SASSARI +Sede Legale Via al Carmine, 2 - 07100 Sassari + +Sede di Torino Via Le Chiuse 68 - 10144 - Telefono: 011 437 8552 Fax : +39 11 437 8221 E-Mail:3a@green-planet.it + 3a Soc. di Sviluppo per l’Ambiente e l’AgroAlimentare a r.l. + + 237.3  log e + +Td = 6.1 + 273.15 + e + 7.5 − log + 6.1 + +Dove: +e = Pressione di vapore (hPa), +ea = Pressione di vapore saturo (hPa) + +e=RHe*(T) + 100 + + 7 .5 (T− 2.7 1)3 5 + +e*(T )=6 .110 T− 3.8 55 + +Per T<=0 + 26.55 log e + +Td =9.5−log 6 e.1+27.135 + 6.1 + +Dove: +e = Pressione di vapore (hPa), +ea = Pressione di vapore saturo (hPa) + +e=RHe*(T) + 100 + + 9.5(T −273.15) + +e* (T ) = 6.110 T −7.65 + +6. BAGNATURA FOGLIARE (Bf) + + Metodo CART (Classification And Regression Tree) (Kim, K.S., Taylor S.E., Gleason, M.L., Koehler, K.J., + 2002) + + DPD = T − Tdew + Dove: + DPD = depressione oraria di temperature dal punto di rugiada (°C), + T = temperatura oraria (°C), + Tdew = temperatura del punto di rugiada (°C). + +3a Srl P.IVA 07366180011 N. REA 154455 SASSARI +Sede Legale Via al Carmine, 2 - 07100 Sassari + +Sede di Torino Via Le Chiuse 68 - 10144 - Telefono: 011 437 8552 Fax : +39 11 437 8221 E-Mail:3a@green-planet.it + 3a Soc. di Sviluppo per l’Ambiente e l’AgroAlimentare a r.l. + + RWS = WS  (Ln(Ch − Z p ) − Ln(Z 0 )) + (Ln(ha − Z p ) − Ln(Z 0 )) + + Dove: + WS = velocità del vento oraria [m/s], + Ch = altezza della vegetazione (parametro esterno) [m], + Zp = altezza di spostamento zero (assunta essere 2/3 di Ch) [m], + Z0 = altezza di rugosità (assunta essere 1/10 di Ch) [m], + ha = altezza dell’anemometro [m]. + WS adj = (1.6064 T + 0.0036T 2 + 0.1531 RH − 0.4599 RWS  DPD − 0.0035T  RH ) + Dove: + RH = umidità dell’aria oraria + RH adj = (0.7921 T + 0.0046 RH − 2.3889 RWS − 0.0390T  RWS +1.0613 RWS  DPD) + Calcolo della Bagnatura fogliare (BF): + Bf= 60 + se + (DPD<4.7 & RWS <2.5 & WSadj>14.5) + oppure + (DPD<4.7 & RWS >=2.5 & RH>= 87.8 & RHadj>37) + Altrimenti Bf=0 + +3a Srl P.IVA 07366180011 N. REA 154455 SASSARI +Sede Legale Via al Carmine, 2 - 07100 Sassari + +Sede di Torino Via Le Chiuse 68 - 10144 - Telefono: 011 437 8552 Fax : +39 11 437 8221 E-Mail:3a@green-planet.it + diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..9d5e03d --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,27 @@ +# Makes src/damage_engine an installable package, so every file can import +# every other file with from damage_engine.. import ... +# Install once (inside the virtual environment): pip install -e . + +[build-system] +requires = ["setuptools>=68"] +build-backend = "setuptools.build_meta" + +[project] +name = "damage-engine" +version = "0.1.0" +requires-python = ">=3.10" +dependencies = [ + # add libraries here as you need them, e.g. "numpy", "pandas", "pyyaml" + "numpy", + "pandas", + "requests", + "pyotp", + "fpdf2", # station PDF reports (report/station_pdf.py), pure Python +] + +[tool.setuptools.packages.find] +where = ["src"] + +# logo and fonts used by the station PDF +[tool.setuptools.package-data] +"damage_engine.report" = ["assets/*.png", "assets/fonts/*"] diff --git a/src/damage_engine/CV_models/__init__.py b/src/damage_engine/CV_models/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/damage_engine/CV_models/brina_gelo_snow.py b/src/damage_engine/CV_models/brina_gelo_snow.py new file mode 100644 index 0000000..e69de29 diff --git a/src/damage_engine/CV_models/drought.py b/src/damage_engine/CV_models/drought.py new file mode 100644 index 0000000..e69de29 diff --git a/src/damage_engine/CV_models/flood.py b/src/damage_engine/CV_models/flood.py new file mode 100644 index 0000000..e69de29 diff --git a/src/damage_engine/CV_models/hail.py b/src/damage_engine/CV_models/hail.py new file mode 100644 index 0000000..e69de29 diff --git a/src/damage_engine/CV_models/strong_wind.py b/src/damage_engine/CV_models/strong_wind.py new file mode 100644 index 0000000..e69de29 diff --git a/src/damage_engine/__init__.py b/src/damage_engine/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/damage_engine/detectors/__init__.py b/src/damage_engine/detectors/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/damage_engine/detectors/brina.py b/src/damage_engine/detectors/brina.py new file mode 100644 index 0000000..d156ad4 --- /dev/null +++ b/src/damage_engine/detectors/brina.py @@ -0,0 +1,378 @@ +""" +Brina (hoar frost) detector. Runs every morning on the night just ended. + +Contract (UnipolSai Model 1506, p.7): freezing of dew / sublimation of water +vapour on the crop due to nocturnal radiation. The contract gives no number, +so every value below is a criterion for radiative frost with ice deposition. +All values are read from damage_engine.thresholds: + [CONTRACT] UnipolSai Model 1506, Ed. 01/2021 + [LITERATURE] WSU EB0913 / MSU "Critical spring temperatures for tree fruit bud stages" + [ENGINEERING] project default, the contract gives no number (to calibrate) + [PROJECT] pipeline setting, not a physical threshold + +No 3A station measures wet-bulb temperature. Brina uses the shared WetBulbMin +of damage_engine.sources.derived: the lowest wet-bulb of the hour, from the +hourly minimum temperature and maximum humidity, with the formula of the 3A +document [3A-DOC] (docs/ImplementazioneAlgoritmiSensori.pdf, section 4). Every +block uses that same value. The frost point is derived.dew_point_c, also from +the 3A document (section 5: ice constants for T <= 0 °C, i.e. the frost point), +on the hourly temperature used by Brina and the hourly mean humidity. +3A's own "TBulbMed" is the same formula on the hourly MEAN +temperature and humidity, so it is warmer than the coldest part of the hour. +It is NOT used for detection; on an event its lowest value is reported next to +ours (details["min_wet_bulb_3a_c"]) as a check. + +Night window - BRINA_SCHEDULE [PROJECT], from RUN_SCHEDULE "sunset -> sunrise": + 18:00 of the previous day (night_start_hour) -> 06:00 of the run day (run_hour). + Stat +An hour of the window is a brina hour when all of these hold (BRINA): + - it is really night: the hour's highest solar irradiance (RadiazioneSolareMax) + <= night_solar_max_wm2 = 10 W/m² [ENGINEERING]. + Why: brina forms only by nocturnal radiation (contract p.7), but the window + uses fixed clock hours, while sunset and sunrise move during the cover + season (budburst -> 10 May). In April-May 18:00-20:00 and often 06:00 are + still daylight. Where a pyranometer exists, these sunlit hours are dropped; + elsewhere the fixed window is the approximation of the night. + - air temperature <= t_air_max_c = 2.0 °C [ENGINEERING] + (2 m air may be slightly above 0 °C while buds freeze). + - wet-bulb <= t_wet_max_c = 0.0 °C [ENGINEERING] (supports ice deposition). + - frost on the buds, depending on the wind: + calm hour (wind <= calm_wind_max_ms = 2.0 m/s [ENGINEERING], or no wind data): + ice deposition, bud temperature <= min(0, frost point) + (BRINA["deposition"]), with bud = air - radiative_offset_c + = 2.0 °C [ENGINEERING] (on calm nights buds cool below the air + by radiation). + windy hour (wind > calm_wind_max_ms): air < 0 °C, the contract's frost + trigger (GELO["t_air_trigger_c"] = 0.0 °C, operator "<", + [CONTRACT p.7]), with bud = air (wind mixes the air). No ice + test: with buds at air temperature it would need 100 % humidity, + so windy sub-zero nights would almost never count. On a windy + hour the air and wet-bulb tests above always hold. +So every sub-zero night hour with wind above 2 m/s is a brina hour.ions +without a wind sensor: calm assumed, details["wind_checked"] = False. +Event when the brina hours reach min_duration_h = 0.5 h [LITERATURE] +(critical temperatures refer to ~30 min exposure), so one hourly record is enough. + +Severity, per field, from the night's lowest bud temperature: + - BBCH < APPLE_DORMANT_BELOW_BBCH = 51: dormant, severity none [ENGINEERING]. + - with a BBCH stage: stage from APPLE_BBCH_TO_STAGE [ENGINEERING, approx.], + T10/T90 bud kill temperatures from APPLE_CRITICAL_TEMPS_C [LITERATURE, WSU/MSU], + kill fraction from FROST_KILL_CURVE (0% at T10 + 1 °C, 100% at T90 - 2 °C) + [ENGINEERING], graded with FROST_KILL_BANDS (kill >= 0.01%/10%/50%/90% -> + light/moderate/severe/extreme) [ENGINEERING]. + - without a stage: FROST_FALLBACK_BANDS_C (bud temp < 0/-2/-4/-6 °C -> + light/moderate/severe/extreme) [ENGINEERING]. +Missing sensors / missing values - the detector still runs: + - an empty solar value: + the night test is not applied, every hour of the window counts as night. + - an empty wind value: + the hour counts as calm and the bud offset is used (worst case: coldest + buds); details["wind_checked"] = False tells the report. + - no hourly minimum temperature: the hourly mean temperature is used. + - no hourly maximum humidity: the hourly mean humidity is used for the + wet-bulb (derived.add_wet_bulb). + - no TBulbMed: the check value is left out; it is not + used for detection (we compute the minimum ourselves) + - temperature and humidity are essential (needed for wet-bulb and frost + point): a station without these sensors gives status "no_data" + ("station lacks required sensors"), hours missing either value are + skipped, and a night with no usable hour gives "no_data". + Missing columns are read as empty (_col), so no sensor causes an error. + Every case above is written to Analysis.data_notes (sensor missing at the + station, no data last night, or N of M hours missing, plus what was done), + so the report shows how each result was obtained. + +Cover window: COVER_WINDOWS["brina"] = budburst -> 10 May [CONTRACT Special Art.5 p.31]. +""" + +import argparse +import logging +import operator +from datetime import date, datetime, time, timedelta + +import numpy as np +import pandas as pd + +from damage_engine.schemas import EVENT, NO_DATA, NO_EVENT, Analysis, Severity +from damage_engine.severity import grade +from damage_engine.sources import derived +from damage_engine.sources.fields import Field, all_fields +from damage_engine.sources.stations import station_by_id, stations_with +from damage_engine.sources.threea import fetch_hourly +from damage_engine.thresholds import ( + APPLE_BBCH_TO_STAGE, + APPLE_CRITICAL_TEMPS_C, + APPLE_DORMANT_BELOW_BBCH, + BRINA, + BRINA_SCHEDULE, + COVER_WINDOWS, + FROST_FALLBACK_BANDS_C, + FROST_KILL_BANDS, + FROST_KILL_CURVE, + GELO, +) + +log = logging.getLogger(__name__) + +BLOCK = "brina" +RUN_HOUR = BRINA_SCHEDULE["run_hour"] +NIGHT_START_HOUR = BRINA_SCHEDULE["night_start_hour"] +HOURS_PER_RECORD = 1.0 # hourly data +SUB_ZERO_OP = {"<": operator.lt, "<=": operator.le}[GELO["t_air_trigger_op"]] # windy-hour test + +# ─────────────────────────── inputs (3A api_parameters) ─────────────────────────── + +REQUIRED_SENSORS = ("TemperaturaAria", "UmiditaRelativa") +SENSOR_WIND = "VelocitaVento2m" # optional sensors (3A sensor names) +SENSOR_SOLAR = "RadiazioneSolare" +COL_TEMP = "TemperaturaAriaMin" # hourly minimum air temperature, °C +COL_TEMP_FALLBACK = "TemperaturaAriaMedia" +COL_RH = "UmiditaRelativaMedia" # %, frost point +COL_RH_MAX = "UmiditaRelativaMax" # %, wet-bulb of the coldest moment (derived.WET_BULB_MIN) +COL_WIND = "VelocitaVento2mMedia" # m/s +COL_SOLAR = "RadiazioneSolareMax" # W/m², highest irradiance of the hour; stations 544, 566, 600 + # (not RadiazioneSolareInt: MJ/m², daily only, empty hourly) +COL_WET_BULB_3A = "TBulbMed" # °C, 3A hourly mean wet-bulb: reported only, not used to detect +COLUMNS = list(dict.fromkeys([COL_TEMP, COL_TEMP_FALLBACK, COL_RH, COL_RH_MAX, + *derived.WET_BULB_INPUTS, COL_WIND, COL_SOLAR, COL_WET_BULB_3A])) + + +# ─────────────────────────── physics ─────────────────────────── +# wet-bulb and frost point: damage_engine.sources.derived (3A document, shared by every block) + +def stage_for(bbch: int) -> str: + """Apple bud stage for a BBCH code (first stage whose highest code is >= bbch).""" + return next(APPLE_BBCH_TO_STAGE[c] for c in sorted(APPLE_BBCH_TO_STAGE) if bbch <= c) + + +def kill_fraction(bud_temp_c: float, stage: str) -> float: + """Fraction of buds killed: 0 at T10+offset, 10% at T10, 90% at T90, 100% at T90-offset.""" + t10, t90 = APPLE_CRITICAL_TEMPS_C[stage] + xs = [t90 - FROST_KILL_CURVE["full_kill_offset_c"], t90, t10, + t10 + FROST_KILL_CURVE["no_damage_offset_c"]] + return float(np.interp(bud_temp_c, xs, [1.0, 0.90, 0.10, 0.0])) + + +def field_severity(min_bud_temp_c: float, bbch: int | None): + """(Severity, score, details) for one field from the night's lowest bud temperature.""" + if bbch is None: + sev, score = grade(min_bud_temp_c, FROST_FALLBACK_BANDS_C) + return sev, score, {"severity_method": "fallback_min_bud_temp"} + if bbch < APPLE_DORMANT_BELOW_BBCH: + return Severity.NONE, 0.0, {"severity_method": "dormant", "bbch": bbch, "kill_fraction": 0.0} + stage = stage_for(bbch) + kill = kill_fraction(min_bud_temp_c, stage) + sev, score = grade(kill, FROST_KILL_BANDS) + return sev, score, {"severity_method": "kill_fraction", "bbch": bbch, "stage": stage, + "kill_fraction": round(kill, 3)} + + +def in_cover_window(run_day: date, bbch: int | None): + """Contract cover window (budburst -> end date). None when the stage is unknown.""" + _, end_mmdd = COVER_WINDOWS[BLOCK] + if run_day.strftime("%m-%d") > end_mmdd: + return False + if bbch is None: + return None + return bbch >= APPLE_DORMANT_BELOW_BBCH # approx.: buds out of dormancy = budburst + + +# ─────────────────────────── detection ─────────────────────────── + +def night_window(run_day: date): + end = datetime.combine(run_day, time(RUN_HOUR)) + start = datetime.combine(run_day - timedelta(days=1), time(NIGHT_START_HOUR)) + return start, end + + +def _col(df, name): + return df[name] if name in df else pd.Series(np.nan, index=df.index) + + +def brina_hours(df: pd.DataFrame) -> pd.DataFrame: + """Add temp/bud/wet-bulb/frost-point columns and a boolean `brina` column to hourly rows.""" + df = derived.add_wet_bulb(df) # WetBulbMin / WetBulbMax, same in every block + df["temp_c"] = _col(df, COL_TEMP).fillna(_col(df, COL_TEMP_FALLBACK)) + wind, solar = _col(df, COL_WIND), _col(df, COL_SOLAR) + night = solar.isna() | (solar <= BRINA["night_solar_max_wm2"]) # no pyranometer -> not applied + df["calm"] = wind.isna() | (wind <= BRINA["calm_wind_max_ms"]) # no wind data -> calm (worst case) + # buds cool below the 2 m air by radiation only on calm hours + df["bud_temp_c"] = df["temp_c"] - BRINA["radiative_offset_c"] * df["calm"].astype(float) + df["wet_bulb_c"] = df[derived.WET_BULB_MIN] # lowest wet-bulb of the hour + df["frost_point_c"] = derived.dew_point_c(df["temp_c"], df[COL_RH]) # over ice when <= 0 °C + deposition = df["bud_temp_c"] <= np.minimum(0.0, df["frost_point_c"]) # calm hours + sub_zero = SUB_ZERO_OP(df["temp_c"], GELO["t_air_trigger_c"]) # windy hours + df["brina"] = (night + & (df["temp_c"] <= BRINA["t_air_max_c"]) + & (df["wet_bulb_c"] <= BRINA["t_wet_max_c"]) + & deposition.where(df["calm"], sub_zero)) + return df + + +def _gap_note(values: pd.Series, label: str, has_sensor: bool | None, effect: str) -> str | None: + """One data note for an optional input: no sensor, no data, or some hours missing.""" + n, missing = len(values), int(values.isna().sum()) + if has_sensor is False: + return f"No {label} sensor at this station: {effect}." + if missing == n: + return f"No {label} data last night: {effect}." + if missing: + return f"{label.capitalize()} missing for {missing} of {n} hours: {effect} for those hours." + return None + + +def data_notes(rows: pd.DataFrame, sensors: set[str] | None = None) -> list[str]: + """Missing sensors / data of one station night and how they were handled. + + `sensors` = the station's 3A sensor names (None = unknown: only the data is checked). + """ + def has(sensor): + return None if sensors is None else sensor in sensors + + offset = BRINA["radiative_offset_c"] + notes = [ + _gap_note(_col(rows, COL_SOLAR), "solar radiation", has(SENSOR_SOLAR), + "night test not applied, every hour of the window counts as night"), + _gap_note(_col(rows, COL_WIND), "wind", has(SENSOR_WIND), + f"calm assumed, buds {offset:g} °C below the air (worst case)"), + ] + n = len(rows) + mean_used = int((_col(rows, COL_TEMP).isna() & _col(rows, COL_TEMP_FALLBACK).notna()).sum()) + if mean_used: + notes.append(f"Hourly minimum temperature missing for {mean_used} of {n} hours: hourly mean used.") + rh_mean_used = int((_col(rows, COL_RH_MAX).isna() & _col(rows, COL_RH).notna()).sum()) + if rh_mean_used: + notes.append(f"Hourly maximum humidity missing for {rh_mean_used} of {n} hours: " + f"hourly mean humidity used for the wet-bulb.") + skipped = int((rows["temp_c"].isna() | _col(rows, COL_RH).isna()).sum()) + if skipped: + notes.append(f"Temperature or humidity missing for {skipped} of {n} hours: those hours skipped.") + return [x for x in notes if x] + + +def analyse_station(rows: pd.DataFrame, sensors: set[str] | None = None) -> dict: + """Station-level result of one night: status, duration, times, measured extremes, data notes.""" + notes = data_notes(rows, sensors) + rows = rows.dropna(subset=["temp_c", COL_RH]) + if rows.empty: + return {"status": NO_DATA, "details": {"reason": "no temperature/humidity records"}, + "notes": notes} + hits = rows[rows["brina"]] + has_wind = rows[COL_WIND].notna().any() if COL_WIND in rows else False + base = {"wind_checked": bool(has_wind), "records": int(len(rows)), + "night_min_temp_c": round(float(rows["temp_c"].min()), 2)} + duration = len(hits) * HOURS_PER_RECORD + if duration < BRINA["min_duration_h"]: + return {"status": NO_EVENT, "details": base, "notes": notes} + check = {} + wet_3a = _col(hits, COL_WET_BULB_3A).dropna() + if not wet_3a.empty: # 3A's own wet-bulb, for comparison only + check["min_wet_bulb_3a_c"] = round(float(wet_3a.min()), 2) + wind = _col(hits, COL_WIND).dropna() + if not wind.empty: # wind decides only the bud offset + check["max_wind_ms"] = round(float(wind.max()), 1) + check["windy_hours"] = int((~hits["calm"]).sum()) * HOURS_PER_RECORD + return { + "status": EVENT, + "notes": notes, + "duration_h": duration, + "start": hits["Timestamp"].min().to_pydatetime(), + "end": hits["Timestamp"].max().to_pydatetime(), + "min_bud_temp_c": float(hits["bud_temp_c"].min()), + "details": {**base, + "min_temp_c": round(float(hits["temp_c"].min()), 2), + "min_bud_temp_c": round(float(hits["bud_temp_c"].min()), 2), + "min_wet_bulb_c": round(float(hits["wet_bulb_c"].min()), 2), + **check, + "max_rh_pct": round(float(hits[COL_RH].max()), 1)}, + } + + +def _summary(res, sev, sdet): + if res["status"] == NO_DATA: + return "No data from the station for last night." + if res["status"] == NO_EVENT: + return "No brina last night." + text = (f"Brina for {res['duration_h']:g} h, lowest bud temperature " + f"{res['details']['min_bud_temp_c']} °C: severity {sev.value}") + if "kill_fraction" in sdet: + text += f" (stage {sdet.get('stage', 'dormant')}, about {sdet['kill_fraction']:.0%} buds killed)" + return text + "." + + +def detect(df: pd.DataFrame, fields: list[Field], run_day: date, + station_names: dict[int, str] | None = None, + station_sensors: dict[int, set[str]] | None = None) -> list[Analysis]: + """One Analysis per field: the night of its station, graded with the field's stage. + + station_sensors: 3A sensor names per station, so the data notes can tell + "no sensor" from "no data"; without it only the data is checked. + """ + station_names = station_names or {} + station_sensors = station_sensors or {} + by_station = {} + if not df.empty: + for sid, rows in brina_hours(df).groupby("CodiceStazione"): + by_station[int(sid)] = analyse_station(rows, station_sensors.get(int(sid))) + out = [] + for f in fields: + res = by_station.get(f.station_id, {"status": NO_DATA, "details": {"reason": "no records"}, + "notes": ["No records from the station last night."]}) + sev, score, sdet = Severity.NONE, 0.0, {} + if res["status"] == EVENT: + sev, score, sdet = field_severity(res["min_bud_temp_c"], f.bbch) + details = {**res["details"], **sdet} + if res["status"] == EVENT: + details["in_cover_window"] = in_cover_window(run_day, f.bbch) + out.append(Analysis( + block=BLOCK, run_day=run_day, + field_id=f.id, field_name=f.name, + station_id=f.station_id, + station_name=station_names.get(f.station_id, str(f.station_id)), + status=res["status"], severity=sev, severity_score=score, + duration_h=res.get("duration_h", 0.0), + start=res.get("start"), end=res.get("end"), + summary=_summary(res, sev, sdet), details=details, + data_notes=res.get("notes", []), + )) + return out + + +def run(run_day: date | None = None) -> list[Analysis]: + """Fetch last night's hourly data for every field's station and detect brina.""" + run_day = run_day or date.today() + start, end = night_window(run_day) + stations = {s.id: s for s in stations_with(*REQUIRED_SENSORS)} + fields = all_fields() + usable = [f for f in fields if f.station_id in stations] + ids = sorted({f.station_id for f in usable}) + df = fetch_hourly(ids, start, end, COLUMNS) if ids else pd.DataFrame() + out = detect(df, usable, run_day, {i: s.name for i, s in stations.items()}, + {i: set(s.sensors) for i, s in stations.items()}) + for f in fields: + if f.station_id not in stations: + known = station_by_id(f.station_id) + if known is None: + note = "Station not in the 3A catalogue: Brina cannot be judged." + else: + lacking = [s for s in REQUIRED_SENSORS if not known.has(s)] + note = (f"No {' / '.join(lacking)} sensor at this station " + f"(needed for wet-bulb and frost point): Brina cannot be judged.") + out.append(Analysis(block=BLOCK, run_day=run_day, field_id=f.id, field_name=f.name, + station_id=f.station_id, + station_name=known.name if known else str(f.station_id), + status=NO_DATA, summary="The station has no temperature/humidity sensor.", + details={"reason": "station lacks required sensors"}, + data_notes=[note])) + n_events = sum(a.detected for a in out) + log.info("Brina %s: %d event(s) over %d field(s), %d station(s)", run_day, n_events, len(out), len(ids)) + return out + + +if __name__ == "__main__": + logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s") + p = argparse.ArgumentParser(description="Run the Brina detector once (prints, does not save).") + p.add_argument("--date", type=date.fromisoformat, help="morning to evaluate (default: today)") + args = p.parse_args() + for a in run(args.date): + print(a.to_dict()) diff --git a/src/damage_engine/detectors/drought.py b/src/damage_engine/detectors/drought.py new file mode 100644 index 0000000..56a119a --- /dev/null +++ b/src/damage_engine/detectors/drought.py @@ -0,0 +1,395 @@ +""" +Drought detector: a rain deficit against the climate average. Runs every month (on the +DROUGHT_SCHEDULE["run_day_of_month"] = 1st), on the last complete calendar months. + +What it detects: a meteorological drought (siccità) = much less rain than usual for the +time of year. The rain measured by the 3A station over the last DROUGHT["window_months"] += 3 complete calendar months [ENGINEERING] is compared with the climate average of the +same months (thresholds.RAIN_NORMALS_MM). Run on 1 Nov 2026, it checks Aug, Sep and Oct +2026. The block does not know about irrigation, so an event means "dry period", not +"crop damaged". No contract trigger is in thresholds.py for drought. + +Thresholds (all from damage_engine.thresholds): + - DROUGHT["window_months"] = 3 months checked [ENGINEERING] + - DROUGHT["rain_pct_of_normal_max"] = 60 %, operator "<=": EVENT when the rain of the + window is at most 60 % of its average (a deficit of 40 % or more) [ENGINEERING] + - DROUGHT["max_missing_days_pct"] = 20 %: more days without rain data -> no_data + [ENGINEERING] + - DROUGHT_BANDS_PCT: severity on the same % of average, lower is worse: + light <= 60, moderate <= 45, severe <= 30, extreme <= 15 % [ENGINEERING] + - RAIN_NORMALS_MM: monthly rain average, mm, per station, 1991-2020, CERRA + reanalysis read once from Open-Meteo (RAIN_NORMALS_SOURCE) [EXTERNAL] + It is a number in thresholds.py, edited by hand: the block never calls Open-Meteo + (only the 3A API is used at run time). The average of the window = the sum of its + months' averages. + - DROUGHT_SCHEDULE["run_day_of_month"] = 1: due() tells main.py when to run [PROJECT] + +Inputs (Diagram.png: "All sensors"; the block reads the ones that tell about water): + - Pioggia = daily rain, mm (3A daily data, all 14 stations): decides the event. + - EvapoPot = daily reference evapotranspiration ET0, mm (3A daily-only column, all + 14 stations, calculated by 3A): water balance rain - ET0, information only. + - Suolo1/2 = soil moisture, % (hourly sensor, stations 544 and 600 only): mean of the + last month, of its first and of its last 7 days, information only. +Requests: one daily request (all stations, the whole window) and, when a station has a +soil moisture sensor, one hourly request for the last month (within 3A's ~180 days). + +Missing sensors / missing values - the detector still runs (Analysis.data_notes): + - no rain gauge at the station, or no climate average for it in RAIN_NORMALS_MM -> + "no_data" (the comparison cannot be made). + - days without rain data: counted as 0 mm, so the rain is a lower bound (a note gives + the number of days). More than max_missing_days_pct of the days -> "no_data": the + deficit could be only missing data. + - days without ET0: the water balance uses only the days that have both values + (note); no ET0 at all -> no water balance (note). + - no soil moisture sensor (12 of 14 stations), no soil data, or the soil request + failed -> no soil values (note). Never changes the result. + Missing columns are read as empty, so no sensor causes an error. detect(..., + station_sensors=...) lets the notes tell "no sensor" from "no data". +""" + +import argparse +import logging +import operator +from datetime import date, datetime, time, timedelta + +import numpy as np +import pandas as pd + +from damage_engine.schemas import EVENT, NO_DATA, NO_EVENT, Analysis, Severity +from damage_engine.severity import grade +from damage_engine.sources.fields import Field, all_fields +from damage_engine.sources.stations import station_by_id, stations_with +from damage_engine.sources.threea import fetch_daily, fetch_hourly +from damage_engine.thresholds import ( + DROUGHT, + DROUGHT_BANDS_PCT, + DROUGHT_SCHEDULE, + RAIN_NORMALS_MM, + RAIN_NORMALS_SOURCE, +) + +log = logging.getLogger(__name__) + +BLOCK = "drought" +PHENOMENON = "drought" +RUN_DAY_OF_MONTH = DROUGHT_SCHEDULE["run_day_of_month"] +MONTHS = int(DROUGHT["window_months"]) +TRIGGER_PCT = DROUGHT["rain_pct_of_normal_max"] +TRIGGER_OP = DROUGHT["rain_pct_of_normal_op"] +OPS = {"<=": operator.le, "<": operator.lt} +SOIL_WEEK_DAYS = 7 # first / last days of the month compared (soil) + +# ─────────────────────────── inputs (3A api_parameters) ─────────────────────────── + +REQUIRED_SENSORS = ("Pioggia",) +SOIL_SENSORS = ("Suolo1", "Suolo2") # sensor name = api_parameter (soil moisture, %) +COL_DAY = "Data" # daily rows: date column +COL_RAIN = "Pioggia" # daily rain, mm +COL_ET0 = "EvapoPot" # daily reference evapotranspiration ET0, mm +DAILY_COLUMNS = [COL_RAIN, COL_ET0] + + +# ─────────────────────────── schedule and window ─────────────────────────── + +def due(run_day: date) -> bool: + """True on the day of the month the block runs (main.py skips it on other days).""" + return run_day.day == RUN_DAY_OF_MONTH + + +def _add_months(day: date, n: int) -> date: + m = day.year * 12 + day.month - 1 + n + return date(m // 12, m % 12 + 1, 1) + + +def window_months(run_day: date) -> list[date]: + """First day of each complete calendar month checked (the MONTHS before run_day's month).""" + this_month = run_day.replace(day=1) + return [_add_months(this_month, -MONTHS + i) for i in range(MONTHS)] + + +def window(run_day: date) -> tuple[datetime, datetime]: + """(start, end) of the window: from the 1st of the first month to the 1st of the run month.""" + months = window_months(run_day) + return datetime.combine(months[0], time()), datetime.combine(run_day.replace(day=1), time()) + + +def months_label(months: list[date]) -> str: + """'Aug-Oct 2026', 'Nov 2025-Jan 2026' or 'Oct 2026'.""" + first, last = months[0], months[-1] + if first == last: + return f"{first:%b %Y}" + if first.year == last.year: + return f"{first:%b}–{last:%b %Y}" + return f"{first:%b %Y}–{last:%b %Y}" + + +def normals_for(station_id: int, months: list[date]) -> list[float] | None: + """Climate average (mm) of each month for the station; None when the station has none.""" + row = RAIN_NORMALS_MM.get(int(station_id)) + return None if row is None else [float(row[m.month - 1]) for m in months] + + +# ─────────────────────────── detection ─────────────────────────── + +def _col(df, name): + return pd.to_numeric(df[name], errors="coerce") if name in df else pd.Series(np.nan, index=df.index) + + +def _round(x, digits=1): + return None if x is None or pd.isna(x) else round(float(x), digits) + + +def daily_grid(rows: pd.DataFrame, run_day: date) -> pd.DataFrame: + """One station's daily rows on the full grid of the window (missing day = empty row), + with day, month (1st of the month), rain_mm and et0_mm.""" + start, end = window(run_day) + grid = pd.date_range(start, end - timedelta(days=1), freq="D", name="day") + rows = rows.assign(day=pd.to_datetime(rows[COL_DAY]).dt.normalize()).drop_duplicates("day") + df = rows.set_index("day").reindex(grid).reset_index() + df["month"] = df["day"].dt.to_period("M").dt.to_timestamp() + df["rain_mm"] = _col(df, COL_RAIN) + df["et0_mm"] = _col(df, COL_ET0) + return df + + +def month_table(df: pd.DataFrame, months: list[date], normals: list[float]) -> list[dict]: + """Per month: rain (missing days = 0 mm), its average and % of it, ET0 and rain - ET0.""" + out = [] + for m, normal in zip(months, normals): + rows = df[df["month"] == pd.Timestamp(m)] + rain = float(rows["rain_mm"].sum()) + both = rows["rain_mm"].notna() & rows["et0_mm"].notna() + et0_ok = bool(rows["et0_mm"].notna().any()) + out.append({ + "month": f"{m:%Y-%m}", + "label": f"{m:%b %Y}", + "days": int(len(rows)), + "missing_days": int(rows["rain_mm"].isna().sum()), + "rain_mm": _round(rain), + "normal_mm": _round(normal), + "rain_pct_of_normal": _round(100 * rain / normal) if normal > 0 else None, + "et0_mm": _round(rows.loc[both, "et0_mm"].sum()) if et0_ok else None, + "water_balance_mm": (_round(rows.loc[both, "rain_mm"].sum() - rows.loc[both, "et0_mm"].sum()) + if et0_ok else None), + }) + return out + + +def soil_summary(rows: pd.DataFrame | None, run_day: date, sensors: set[str] | None) -> dict: + """Soil moisture of the last month, per sensor the station has: mean, first and last + SOIL_WEEK_DAYS days (% ). Empty dict when there is nothing to show.""" + if rows is None or rows.empty: + return {} + end = datetime.combine(run_day.replace(day=1), time()) + start = datetime.combine(window_months(run_day)[-1], time()) + ts = pd.to_datetime(rows["Timestamp"]) + rows = rows[(ts > start) & (ts <= end)].assign(Timestamp=ts) + out = {} + for name in SOIL_SENSORS: + if sensors is not None and name not in sensors: + continue + v = _col(rows, name) + if v.notna().sum() == 0: + continue + first = v[rows["Timestamp"] <= start + timedelta(days=SOIL_WEEK_DAYS)] + last = v[rows["Timestamp"] > end - timedelta(days=SOIL_WEEK_DAYS)] + out[name] = {"mean_pct": _round(v.mean()), "first_week_pct": _round(first.mean()), + "last_week_pct": _round(last.mean()), "hours": int(v.notna().sum())} + return out + + +def data_notes(table: list[dict], sensors: set[str] | None, soil: dict, + soil_error: str | None) -> list[str]: + """Missing data of one station's window and how it was handled.""" + notes = [f"Average rain of each month: {RAIN_NORMALS_SOURCE}, at this station's coordinates " + f"(modelled, not measured)."] + days = sum(m["days"] for m in table) + missing = sum(m["missing_days"] for m in table) + if missing: + which = ", ".join(f"{m['label']} {m['missing_days']}" for m in table if m["missing_days"]) + notes.append(f"Rain missing for {missing} of {days} days ({which}): counted as 0 mm, so the " + f"rain shown is a lower bound.") + if all(m["et0_mm"] is None for m in table): + notes.append(f"No evapotranspiration data ({COL_ET0}): water balance not shown.") + elif any(m["et0_mm"] is None for m in table): + notes.append(f"Evapotranspiration ({COL_ET0}) missing for some months: water balance only " + f"for the months with data.") + has_soil = sensors is None or any(s in sensors for s in SOIL_SENSORS) + if sensors is not None and not has_soil: + notes.append("No soil moisture sensor at this station: soil moisture not shown.") + elif soil_error: + notes.append(f"Soil moisture could not be read from 3A ({soil_error}): not shown.") + elif has_soil and not soil and sensors is not None: + notes.append("No soil moisture data in the last month: not shown.") + return notes + + +def analyse_station(daily: pd.DataFrame, run_day: date, station_id: int, + sensors: set[str] | None = None, soil_rows: pd.DataFrame | None = None, + soil_error: str | None = None) -> dict: + """Station-level result of the window: status, severity, details, notes.""" + months = window_months(run_day) + label = months_label(months) + normals = normals_for(station_id, months) + if normals is None: + return {"status": NO_DATA, "details": {"reason": "no climate average for this station"}, + "notes": [f"No average rain for station {station_id} in thresholds.RAIN_NORMALS_MM: " + f"the rain of {label} cannot be compared with its average."]} + grid = daily_grid(daily, run_day) + table = month_table(grid, months, normals) + soil = {} if soil_error else soil_summary(soil_rows, run_day, sensors) + notes = data_notes(table, sensors, soil, soil_error) + other = [n for n in notes if not n.startswith("Rain missing")] # for no_data results + days = sum(m["days"] for m in table) + missing = sum(m["missing_days"] for m in table) + if missing == days: + return {"status": NO_DATA, "details": {"reason": "no daily rain data in the window"}, + "notes": [f"No daily rain data for {label}: drought cannot be judged."] + other} + if 100 * missing / days > DROUGHT["max_missing_days_pct"]: + return {"status": NO_DATA, + "details": {"reason": "too many days without rain data", "missing_days": missing, + "days": days}, + "notes": [f"Rain missing for {missing} of {days} days of {label} (more than " + f"{DROUGHT['max_missing_days_pct']:g} %): a deficit could be only missing " + f"data, so drought is not judged."] + other} + rain = float(grid["rain_mm"].sum()) # unrounded (the table is rounded) + normal = sum(normals) + pct = 100 * rain / normal + both = grid["rain_mm"].notna() & grid["et0_mm"].notna() + et0 = [m for m in table if m["et0_mm"] is not None] + last = table[-1] + details = { + "months": table, + "window_label": label, + "window_months": MONTHS, + "days": days, + "missing_days": missing, + "rain_mm": _round(rain), + "normal_mm": _round(normal), + "rain_pct_of_normal": _round(pct), + "deficit_mm": _round(normal - rain), + "trigger_pct": TRIGGER_PCT, + "trigger_op": TRIGGER_OP, + "last_month_label": last["label"], + "last_month_rain_mm": last["rain_mm"], + "last_month_normal_mm": last["normal_mm"], + "last_month_pct_of_normal": last["rain_pct_of_normal"], + "et0_mm": _round(grid.loc[both, "et0_mm"].sum()) if et0 else None, + "water_balance_mm": (_round(grid.loc[both, "rain_mm"].sum() - grid.loc[both, "et0_mm"].sum()) + if et0 else None), + "water_balance_months": len(et0), + "soil_moisture": soil or None, + "normals_source": RAIN_NORMALS_SOURCE, + } + if not OPS[TRIGGER_OP](pct, TRIGGER_PCT): + return {"status": NO_EVENT, "details": details, "notes": notes} + severity, score = grade(pct, DROUGHT_BANDS_PCT) + start, end = window(run_day) + return {"status": EVENT, "details": details, "notes": notes, "severity": severity, + "score": score, "duration_h": float(days * 24), "start": start, "end": end} + + +def _summary(res) -> str: + if res["status"] == NO_DATA: + return res["notes"][0] if res.get("notes") else "No usable rain data from the station." + d = res["details"] + text = (f"Rain in {d['window_label']}: {d['rain_mm']:g} mm, {d['rain_pct_of_normal']:g} % of the " + f"average ({d['normal_mm']:g} mm)") + if res["status"] == NO_EVENT: + return text + f": no drought (drought at ≤ {TRIGGER_PCT:g} % of the average)." + text = (f"Dry period: {d['rain_mm']:g} mm of rain in {d['window_label']}, only " + f"{d['rain_pct_of_normal']:g} % of the average ({d['normal_mm']:g} mm, " + f"{d['deficit_mm']:g} mm below it).") + if d["last_month_pct_of_normal"] is not None: + text += f" {d['last_month_label']}: {d['last_month_pct_of_normal']:g} % of its average." + return text + + +def detect(daily: pd.DataFrame, fields: list[Field], run_day: date, + station_names: dict[int, str] | None = None, + station_sensors: dict[int, set[str]] | None = None, + soil: pd.DataFrame | None = None, soil_error: str | None = None) -> list[Analysis]: + """One Analysis per field: the rain of its station's window against the climate average. + + daily: daily rows (CodiceStazione, Data, Pioggia, EvapoPot) of the window; soil: hourly + rows (CodiceStazione, Timestamp, Suolo1, Suolo2) of the last month, or None. + """ + station_names = station_names or {} + station_sensors = station_sensors or {} + by_station = {} + groups = {int(k): v for k, v in daily.groupby("CodiceStazione")} if not daily.empty else {} + soil_groups = ({int(k): v for k, v in soil.groupby("CodiceStazione")} + if soil is not None and not soil.empty else {}) + for sid in {f.station_id for f in fields}: + rows = groups.get(sid, pd.DataFrame(columns=["CodiceStazione", COL_DAY, *DAILY_COLUMNS])) + by_station[sid] = analyse_station(rows, run_day, sid, station_sensors.get(sid), + soil_groups.get(sid), soil_error) + start, end = window(run_day) + period = {"window_start": start.isoformat(timespec="minutes"), # for the report: the + "window_end": end.isoformat(timespec="minutes"), # months checked + "window_label": months_label(window_months(run_day))} + out = [] + for f in fields: + res = by_station[f.station_id] + out.append(Analysis( + block=BLOCK, run_day=run_day, + field_id=f.id, field_name=f.name, + station_id=f.station_id, + station_name=station_names.get(f.station_id, str(f.station_id)), + status=res["status"], + severity=res.get("severity", Severity.NONE), + severity_score=res.get("score", 0.0), + duration_h=res.get("duration_h", 0.0), + start=res.get("start"), end=res.get("end"), + summary=_summary(res), details={**res["details"], **period}, + data_notes=list(res.get("notes", [])), + )) + return out + + +def run(run_day: date | None = None) -> list[Analysis]: + """Fetch the daily rain and ET0 of the window (and the last month's soil moisture where a + station has the sensor) and compare the rain with its climate average.""" + run_day = run_day or date.today() + start, end = window(run_day) + stations = {s.id: s for s in stations_with(*REQUIRED_SENSORS)} + fields = all_fields() + usable = [f for f in fields if f.station_id in stations] + ids = sorted({f.station_id for f in usable}) + daily = fetch_daily(ids, start, end - timedelta(minutes=1), DAILY_COLUMNS) if ids else pd.DataFrame() + soil_ids = [i for i in ids if any(stations[i].has(s) for s in SOIL_SENSORS)] + soil, soil_error = None, None + if soil_ids: + last_month = datetime.combine(window_months(run_day)[-1], time()) + try: + soil = fetch_hourly(soil_ids, last_month, end, list(SOIL_SENSORS)) + except Exception as e: # information only: never stops the block + log.warning("Drought: soil moisture request failed: %s", e) + soil_error = type(e).__name__ + out = detect(daily, usable, run_day, {i: s.name for i, s in stations.items()}, + {i: set(s.sensors) for i, s in stations.items()}, soil, soil_error) + for f in fields: + if f.station_id not in stations: + known = station_by_id(f.station_id) + note = ("Station not in the 3A catalogue: drought cannot be judged." if known is None + else "No rain gauge (Pioggia) at this station: drought cannot be judged.") + out.append(Analysis(block=BLOCK, run_day=run_day, field_id=f.id, field_name=f.name, + station_id=f.station_id, + station_name=known.name if known else str(f.station_id), + status=NO_DATA, summary="The station has no rain gauge.", + details={"reason": "station lacks required sensors"}, + data_notes=[note])) + n = sum(a.detected for a in out) + log.info("Drought %s (%s): dry period for %d of %d field(s), %d station(s)", + run_day, months_label(window_months(run_day)), n, len(out), len(ids)) + return out + + +if __name__ == "__main__": + logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s") + p = argparse.ArgumentParser(description="Run the Drought detector once (prints, does not save).") + p.add_argument("--date", type=date.fromisoformat, help="run day (default: today); the " + "complete months before it are checked") + args = p.parse_args() + for a in run(args.date): + print(a.to_dict()) diff --git a/src/damage_engine/detectors/flood.py b/src/damage_engine/detectors/flood.py new file mode 100644 index 0000000..087b51c --- /dev/null +++ b/src/damage_engine/detectors/flood.py @@ -0,0 +1,365 @@ +""" +Flood detector = contract "excessive rain" (eccesso di pioggia). Runs every +24 h on the day just ended. + +Contract (UnipolSai Model 1506, p.8): excessive rain is ANY ONE of three +criteria. The "Flood" box of Diagram.png is this peril: flooding by rivers +(alluvione) is not covered for apples, and the box only gets the rain gauge. +All values are read from damage_engine.thresholds; sources as tagged there: + [CONTRACT] UnipolSai Model 1506, Ed. 01/2021 + [ENGINEERING] project default, the contract gives no number (to calibrate) + [PROJECT] pipeline setting, not a physical threshold + +Input (Diagram.png): rain gauge only, 3A "Pioggia" (hourly rain, mm). + +Event day - FLOOD_SCHEDULE [PROJECT]: the 24 h ending at run_hour = 07:00 of +the run day (records after 07:00 of the previous day up to 07:00 of the run +day). lookback_days = 10 days of hourly rain are fetched (one API request) so +the 72 h and 10-day sums can be computed for windows ending in the event day. + +Criteria (EXCESSIVE_RAIN), each with the contract operator from its "_op" key: + - downpour: one hourly record >= downpour_1h_mm = 30 mm [CONTRACT p.8] + ("at least 30 mm over 1 hour"; hourly data are clock hours, so a downpour + split over two hours can be missed). + - intense: rain in 72 h >= intense_72h_mm = 80 mm [CONTRACT p.8] + ("at least 80 mm within 72 hours"), rolling 72 h sums. + - prolonged: 10-day rain > prolonged_10d_ratio = 1.5 x the 5-year average of + the same period AND >= prolonged_10d_min_mm = 80 mm [CONTRACT p.8] + ("exceeding the 5-year average by more than 50%, in no case less than 80 mm"). + Needs a reference average (detect(..., reference_10d_mm=...)). 3A gives + hourly data for only ~180 days and fetch_daily does not work yet, so there + is no average today: the criterion is not applied and a data note says so. +A criterion counts only for a window that ends on rain fallen during the event +day, so a dry day after heavy rain does not report the same event again. + +Claim helper: reported only, it does not decide the event. the latest 72 h window in the last +4 days whose rain reaches intense_72h_mm = 80 mm.: + - details["wettest_72h"]: the wettest 72 h window in those days. + +Severity, per field: rainfall / trigger of the criterion exceeded the most +(prolonged: trigger = max(1.5 x average, 80 mm)), graded with +EXCESSIVE_RAIN_BANDS_RATIO (ratio >= 1.0/1.5/2.0/3.0 -> light/moderate/severe/ +extreme) [ENGINEERING]. Same for every stage: the contract gives no +stage-dependent rain damage. + +Cover window: COVER_WINDOWS["excessive_rain"] = fruit set -> 31 Oct +[CONTRACT Special Art.5 p.31]; rain during flowering is excluded (Art.5 q), +which the window already does. Fruit set = BBCH above the highest +"post_bloom" code of APPLE_BBCH_TO_STAGE [ENGINEERING, approx.]. + +Missing sensors / missing values - the detector still runs: + - rain is essential: a station without a rain gauge (none since 2026-10-21, + all 14 stations have one) gives + "no_data" ("station lacks required sensors"), and so does an event day + with no rain record at all. + - missing hours are counted as 0 mm, so every total is a lower bound: an + event found is real, a "no event" may miss rain. Missing hours of the + event day and of the days before it are written to Analysis.data_notes. + - no 5-year reference average: prolonged criterion not applied (data note). + Missing rows are read as empty (the hourly grid is rebuilt), never an error. +""" + +import argparse +import logging +import operator +from datetime import date, datetime, time, timedelta + +import numpy as np +import pandas as pd + +from damage_engine.schemas import EVENT, NO_DATA, NO_EVENT, Analysis, Severity +from damage_engine.severity import grade +from damage_engine.sources.fields import Field, all_fields +from damage_engine.sources.stations import station_by_id, stations_with +from damage_engine.sources.threea import fetch_hourly +from damage_engine.thresholds import ( + APPLE_BBCH_TO_STAGE, + COVER_WINDOWS, + EXCESSIVE_RAIN, + EXCESSIVE_RAIN_BANDS_RATIO, + FLOOD_SCHEDULE, +) + +log = logging.getLogger(__name__) + +BLOCK = "flood" +PHENOMENON = "excessive_rain" # contract name, key in COVER_WINDOWS +RUN_HOUR = FLOOD_SCHEDULE["run_hour"] +LOOKBACK_DAYS = FLOOD_SCHEDULE["lookback_days"] +CLAIM_SEARCH_DAYS = FLOOD_SCHEDULE["claim_search_days"] +HOURS_PER_RECORD = 1.0 # hourly data +HOUR = timedelta(hours=1) + +# ─────────────────────────── inputs (3A api_parameters) ─────────────────────────── + +REQUIRED_SENSORS = ("Pioggia",) +COL_RAIN = "Pioggia" # hourly rain, mm +COLUMNS = [COL_RAIN] + +OPS = {">=": operator.ge, ">": operator.gt, "<=": operator.le, "<": operator.lt} + + +def _met(value, key, op_key): + """Contract comparison: value EXCESSIVE_RAIN[key], operator from EXCESSIVE_RAIN[op_key].""" + return value is not None and OPS[EXCESSIVE_RAIN[op_key]](value, EXCESSIVE_RAIN[key]) + + +# ─────────────────────────── windows ─────────────────────────── + +def event_window(run_day: date): + """(start, end): the event day is the records after start up to and including end.""" + end = datetime.combine(run_day, time(RUN_HOUR)) + return end - timedelta(days=1), end + + +def fetch_window(run_day: date): + """[start, end] to fetch: the event day plus the look-back for the 72 h / 10-day sums.""" + _, end = event_window(run_day) + return end - timedelta(days=LOOKBACK_DAYS) + HOUR, end + + +def fruit_set(bbch: int) -> bool: + """Fruit set (start of the excessive-rain cover): BBCH above the last post-bloom code.""" + return bbch > max(c for c, s in APPLE_BBCH_TO_STAGE.items() if s == "post_bloom") + + +def in_cover_window(run_day: date, bbch: int | None): + """Contract cover window (fruit set -> end date). None when the stage is unknown.""" + _, end_mmdd = COVER_WINDOWS[PHENOMENON] + if run_day.strftime("%m-%d") > end_mmdd: + return False + if bbch is None: + return None + return fruit_set(bbch) + + +# ─────────────────────────── detection ─────────────────────────── + +def _col(df, name): + return df[name] if name in df else pd.Series(np.nan, index=df.index) + + +def hourly_rain(rows: pd.DataFrame, run_day: date) -> pd.Series: + """Rain of one station on the full hourly grid of the fetch window (NaN = missing hour).""" + start, end = fetch_window(run_day) + grid = pd.date_range(start, end, freq="h") + rain = pd.Series(_col(rows, COL_RAIN).to_numpy(dtype=float), + index=pd.DatetimeIndex(rows["Timestamp"])) + rain = rain[~rain.index.duplicated()] + return rain.reindex(grid) + + +def _round(x): + return None if x is None else round(float(x), 1) + + +def data_notes(rain: pd.Series, run_day: date, reference_mm: float | None) -> list[str]: + """Missing data of one station and how it was handled.""" + day_start, _ = event_window(run_day) + day, before = rain[rain.index > day_start], rain[rain.index <= day_start] + notes = [] + if day.isna().any() and day.notna().any(): + notes.append(f"Rain missing for {int(day.isna().sum())} of {len(day)} hours of the event day: " + f"counted as 0 mm, so the totals are lower bounds.") + if before.isna().any(): + notes.append(f"Rain missing for {int(before.isna().sum())} of {len(before)} hours of the " + f"{LOOKBACK_DAYS - 1} days before the event day: counted as 0 mm, " + f"so the 72 h and 10-day sums are lower bounds.") + if reference_mm is None: + notes.append(f"No 5-year reference average for the 10-day rain: prolonged-rain criterion " + f"not applied (10-day total {rain.fillna(0).sum():.1f} mm).") + return notes + + +def _iso(t) -> str: + return t.to_pydatetime().isoformat(timespec="minutes") + + +def _window_info(watch: pd.Series, sum_72h: pd.Series, end, earlier: pd.Series) -> dict: + """Total, first/last rainy hour of the 72 h window ending at `end` (rain of the watch only). + + earlier: rain before the watch period; "partial" = some of it falls in this window's 72 h + (not counted, so the storm may have started before the days checked). + """ + window = watch[end - 71 * HOUR:end] # 72 hourly records + wet = window[window > 0].index + return {"mm": _round(sum_72h[end]), "first_rain": _iso(wet.min()), "last_rain": _iso(wet.max()), + "partial": bool(earlier[end - 71 * HOUR:].sum() > 0)} + + +def heavy_72h(filled: pd.Series) -> dict: + """Claim helper (Heavy Rain Watch): only the rain of the last CLAIM_SEARCH_DAYS counts + (asked by the user 2026-10-26, so the row never shows rain from before those days). + Latest 72 h window in that period that reaches intense_72h_mm, and the wettest one. + + filled: hourly rain of the look-back (missing hours = 0). + """ + out = {"last_heavy_72h": None, "wettest_72h": None} + search_from = filled.index[-1] - timedelta(days=CLAIM_SEARCH_DAYS) + inside = filled.index > search_from # records of the watch period + watch = filled.where(inside, 0.0) # rain before the period is not counted + earlier = filled.where(~inside, 0.0) + sum_72h = watch.rolling(72, min_periods=1).sum() + ends = sum_72h[inside] # windows ending in the watch period + if not ends.max() > 0: + return out + out["wettest_72h"] = _window_info(watch, sum_72h, ends.idxmax(), earlier) # first max: ends on rain + # round away float noise of the rolling sum (e.g. 79.99999999 for 80 mm) + hit = ends.round(6).apply(lambda v: _met(v, "intense_72h_mm", "intense_72h_op")) + if hit.any(): + last = hit[hit].index.max() + upto = hit[:last] + below = upto[~upto] + first = below.index.max() + HOUR if len(below) else upto.index[0] # start of that run + run = sum_72h[first:last] + info = _window_info(watch, sum_72h, run.idxmax(), earlier) + info["reached_at"] = _iso(first) + out["last_heavy_72h"] = info + return out + + +def analyse_station(rain: pd.Series, run_day: date, reference_mm: float | None = None) -> dict: + """Station-level result of one event day: status, criteria met, rain totals, data notes.""" + day_start, day_end = event_window(run_day) + notes = data_notes(rain, run_day, reference_mm) + day = rain[rain.index > day_start] + if day.notna().sum() == 0: + return {"status": NO_DATA, "details": {"reason": "no rain records in the event day"}, + "notes": [f"No rain data for the event day ({day_start:%Y-%m-%d %H:%M} -> " + f"{day_end:%Y-%m-%d %H:%M})."] + notes} + + filled = rain.fillna(0.0) + sum_72h = filled.rolling(72, min_periods=1).sum() # grid is complete: 72 rows = 72 h + rainy = day[day > 0].index # windows must end on rain of the event day + max_1h = float(day.max()) + max_72h = float(sum_72h[rainy].max()) if len(rainy) else None + rain_10d = float(filled.sum()) + + ratios = {} + if _met(max_1h, "downpour_1h_mm", "downpour_1h_op"): + ratios["downpour_1h"] = max_1h / EXCESSIVE_RAIN["downpour_1h_mm"] + if _met(max_72h, "intense_72h_mm", "intense_72h_op"): + ratios["intense_72h"] = max_72h / EXCESSIVE_RAIN["intense_72h_mm"] + if reference_mm and len(rainy): + ratio_avg = rain_10d / reference_mm + if (_met(ratio_avg, "prolonged_10d_ratio", "prolonged_10d_ratio_op") + and _met(rain_10d, "prolonged_10d_min_mm", "prolonged_10d_min_op")): + trigger = max(EXCESSIVE_RAIN["prolonged_10d_ratio"] * reference_mm, + EXCESSIVE_RAIN["prolonged_10d_min_mm"]) + ratios["prolonged_10d"] = rain_10d / trigger + + details = {"records": int(day.notna().sum()), + "rain_day_mm": _round(day.sum()), + "max_1h_mm": _round(max_1h), + "max_72h_mm": _round(max_72h), + "rain_72h_to_run_mm": _round(sum_72h.iloc[-1]), + "rain_10d_mm": _round(rain_10d), + "reference_10d_mm": _round(reference_mm), + **heavy_72h(filled)} + if not ratios: + return {"status": NO_EVENT, "details": details, "notes": notes} + worst = max(ratios, key=ratios.get) + return { + "status": EVENT, + "notes": notes, + "duration_h": len(rainy) * HOURS_PER_RECORD, + "start": rainy.min().to_pydatetime(), + "end": rainy.max().to_pydatetime(), + "ratio": ratios[worst], + "details": {**details, "criteria_met": sorted(ratios), "worst_criterion": worst, + "rain_trigger_ratio": round(ratios[worst], 3)}, + } + + +CRITERION_TEXT = { + "downpour_1h": lambda d: f"{d['max_1h_mm']} mm in 1 h", + "intense_72h": lambda d: f"{d['max_72h_mm']} mm in 72 h", + "prolonged_10d": lambda d: f"{d['rain_10d_mm']} mm in 10 days (average {d['reference_10d_mm']} mm)", +} + + +def _summary(res, sev): + if res["status"] == NO_DATA: + return "No rain data from the station for the last 24 h." + d = res["details"] + if res["status"] == NO_EVENT: + return (f"No excessive rain: {d['rain_day_mm']} mm in the last 24 h, " + f"{d['rain_72h_to_run_mm']} mm in 72 h.") + met = ", ".join(CRITERION_TEXT[c](d) for c in d["criteria_met"]) + return f"Excessive rain ({met}): severity {sev.value}." + + +def detect(df: pd.DataFrame, fields: list[Field], run_day: date, + station_names: dict[int, str] | None = None, + reference_10d_mm: dict[int, float] | None = None) -> list[Analysis]: + """One Analysis per field: the event day of its station. + + df: hourly rows (CodiceStazione, Timestamp, Pioggia) covering fetch_window(run_day). + reference_10d_mm: 5-year average rain of the same 10 days per station; without + it the prolonged-rain criterion is not applied. + """ + station_names = station_names or {} + reference_10d_mm = reference_10d_mm or {} + by_station = {} + if not df.empty: + for sid, rows in df.groupby("CodiceStazione"): + sid = int(sid) + by_station[sid] = analyse_station(hourly_rain(rows, run_day), run_day, + reference_10d_mm.get(sid)) + out = [] + for f in fields: + res = by_station.get(f.station_id, {"status": NO_DATA, "details": {"reason": "no records"}, + "notes": ["No rain records from the station for the last " + f"{LOOKBACK_DAYS} days."]}) + sev, score = Severity.NONE, 0.0 + details = dict(res["details"]) + if res["status"] == EVENT: + sev, score = grade(res["ratio"], EXCESSIVE_RAIN_BANDS_RATIO) + details["in_cover_window"] = in_cover_window(run_day, f.bbch) + out.append(Analysis( + block=BLOCK, run_day=run_day, + field_id=f.id, field_name=f.name, + station_id=f.station_id, + station_name=station_names.get(f.station_id, str(f.station_id)), + status=res["status"], severity=sev, severity_score=score, + duration_h=res.get("duration_h", 0.0), + start=res.get("start"), end=res.get("end"), + summary=_summary(res, sev), details=details, + data_notes=res.get("notes", []), + )) + return out + + +def run(run_day: date | None = None) -> list[Analysis]: + """Fetch the last LOOKBACK_DAYS of hourly rain for every field's station and detect.""" + run_day = run_day or date.today() + start, end = fetch_window(run_day) + stations = {s.id: s for s in stations_with(*REQUIRED_SENSORS)} + fields = all_fields() + usable = [f for f in fields if f.station_id in stations] + ids = sorted({f.station_id for f in usable}) + df = fetch_hourly(ids, start, end, COLUMNS) if ids else pd.DataFrame() + out = detect(df, usable, run_day, {i: s.name for i, s in stations.items()}) + for f in fields: + if f.station_id not in stations: + known = station_by_id(f.station_id) + note = ("Station not in the 3A catalogue: Flood cannot be judged." if known is None + else "No rain gauge (Pioggia) at this station: Flood cannot be judged.") + out.append(Analysis(block=BLOCK, run_day=run_day, field_id=f.id, field_name=f.name, + station_id=f.station_id, + station_name=known.name if known else str(f.station_id), + status=NO_DATA, summary="The station has no rain gauge.", + details={"reason": "station lacks required sensors"}, + data_notes=[note])) + n_events = sum(a.detected for a in out) + log.info("Flood %s: %d event(s) over %d field(s), %d station(s)", run_day, n_events, len(out), len(ids)) + return out + + +if __name__ == "__main__": + logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s") + p = argparse.ArgumentParser(description="Run the Flood detector once (prints, does not save).") + p.add_argument("--date", type=date.fromisoformat, help="run day to evaluate (default: today)") + args = p.parse_args() + for a in run(args.date): + print(a.to_dict()) diff --git a/src/damage_engine/detectors/freezing.py b/src/damage_engine/detectors/freezing.py new file mode 100644 index 0000000..ee18d02 --- /dev/null +++ b/src/damage_engine/detectors/freezing.py @@ -0,0 +1,404 @@ +""" +Freezing (gelo) detector: air below 0 °C in DAYLIGHT. Runs every 24 h. + +Split with Brina: Brina checks the night, Freezing checks +the daylight. For one run day both blocks look at the same 24 h and every hour +belongs to exactly one of them: + - Brina: the hours of its night window (BRINA_SCHEDULE: 18:00 -> 06:00, + records 18:00 ... 06:00) that are dark (RadiazioneSolareMax + <= BRINA["night_solar_max_wm2"], or no solar value). + - Freezing: every other hour = the day hours before the night window + (records 07:00 ... 17:00), plus the sunlit hours of the night + window (RadiazioneSolareMax > night_solar_max_wm2 = 10 W/m² + [ENGINEERING], the hours Brina drops as daytime; in April-May + often 18:00-20:00 and 06:00). +Both limits are Brina's settings in thresholds.py (BRINA_SCHEDULE [PROJECT], +BRINA["night_solar_max_wm2"]), read here and not copied, so a change there +moves both blocks and leaves no hour unchecked or checked twice. + +Contract (UnipolSai Model 1506, p.7): gelo = "a drop in temperature below 0 °C +caused by cold air masses". In daylight the sun stops the radiative cooling of +the crop (the cause of brina), so sub-zero air during the day comes from a cold +air mass. All values are read from damage_engine.thresholds; sources as tagged there: + [CONTRACT] UnipolSai Model 1506, Ed. 01/2021 + [LITERATURE] WSU EB0913 / MSU "Critical spring temperatures for tree fruit bud stages" + [ENGINEERING] project default, the contract gives no number (to calibrate) + [PROJECT] pipeline setting, not a physical threshold + +Inputs (Diagram.png): temperature, solar radiation, wet-bulb temperature, +humidity, wind speed. + - temperature: hourly minimum (TemperaturaAriaMin), for detection and severity. + - solar radiation: RadiazioneSolareMax (highest irradiance of the hour, W/m², + stations 544, 566, 600) finds the sunlit hours of the night window (above). + - wind (VelocitaVento2mMedia): reported only. The mean wind of the freezing + hours is compared with GELO["advective_wind_min_ms"] = 2.0 m/s [ENGINEERING] + (details["advective_wind"]): it shows whether wind backs the contract's + "cold air masses". Not needed to detect: in daylight any sub-zero hour + counts, with or without wind (and a windy sub-zero night hour is Brina's). + - wet-bulb and humidity: reported only. The lowest WetBulbMin of the freezing + hours (damage_engine.sources.derived, 3A document [3A-DOC] section 4, the + same value as in Brina): a wet bud (rain, dew) cools by evaporation towards + the wet-bulb, so it shows how cold wet buds could get. Not used to detect or + grade: the contract trigger is the air temperature, and a "wet-bulb <= 0 °C" + test would change nothing, because the wet-bulb is never warmer than the air + (equal at 100 % humidity): whenever the air is below 0 °C, so is the wet-bulb. + +A daylight hour is a freezing hour when (GELO): + - air temperature < t_air_trigger_c = 0.0 °C, operator t_air_trigger_op = "<" + [CONTRACT p.7, "a drop in temperature below 0 °C"]; exactly 0 °C does not count. +Event when the freezing hours reach min_duration_h = 0.5 h [LITERATURE] +(critical temperatures refer to ~30 min exposure), so one hourly record is enough. + +Bud temperature = air temperature: in daylight the buds do not cool below the +air by radiation (Brina's radiative_offset_c is for calm nights only). + +Severity, per field, from the lowest air temperature of the daylight hours +(same frost tables as Brina): + - BBCH < APPLE_DORMANT_BELOW_BBCH = 51: dormant, severity none [ENGINEERING]. + - with a BBCH stage: stage from APPLE_BBCH_TO_STAGE [ENGINEERING, approx.], + T10/T90 bud kill temperatures from APPLE_CRITICAL_TEMPS_C [LITERATURE, WSU/MSU], + kill fraction from FROST_KILL_CURVE (0% at T10 + 1 °C, 100% at T90 - 2 °C) + [ENGINEERING], graded with FROST_KILL_BANDS (kill >= 0.01%/10%/50%/90% -> + light/moderate/severe/extreme) [ENGINEERING]. + - without a stage: FROST_FALLBACK_BANDS_C (temperature < 0/-2/-4/-6 °C -> + light/moderate/severe/extreme) [ENGINEERING]. + +Missing sensors / missing values - the detector still runs: + - temperature is essential: a station without a temperature sensor gives + "no_data" ("station lacks required sensors"); daylight hours without a + temperature (no record, or an empty value) are not checked; a day with no + usable daylight hour gives "no_data". + - no hourly minimum temperature: the hourly mean temperature is used. + - no pyranometer (all stations except 544, 566 and 600), or no solar value + in an hour of the night window: that hour counts as night and is left to + Brina (Brina counts it as night too). Daylight = the fixed day hours only. + - no wind sensor (all stations except 3, 6, 10, 30, 544, 566, 600) or no wind + value: wind not reported (details["advective_wind"] = None); detection unchanged. + - no humidity value: no wet-bulb for that hour (reported only); detection unchanged. + Missing columns and missing hours are read as empty (_col, hourly grid), so no + sensor causes an error. Every case is written to Analysis.data_notes (sensor + missing at the station, no data, or N of M hours missing, plus what was done). + +Wind does not decide the block: a windy sub-zero day hour +is freezing, a windy sub-zero night hour is Brina's (Brina has no calm-air test). + +Cover window: COVER_WINDOWS["gelo"] = budburst -> 10 May [CONTRACT Special Art.5 p.31], +checked on the day of the first freezing hour. +""" + +import argparse +import logging +import operator +from datetime import date, datetime, time, timedelta + +import numpy as np +import pandas as pd + +from damage_engine.schemas import EVENT, NO_DATA, NO_EVENT, Analysis, Severity +from damage_engine.severity import grade +from damage_engine.sources import derived +from damage_engine.sources.fields import Field, all_fields +from damage_engine.sources.stations import station_by_id, stations_with +from damage_engine.sources.threea import fetch_hourly +from damage_engine.thresholds import ( + APPLE_BBCH_TO_STAGE, + APPLE_CRITICAL_TEMPS_C, + APPLE_DORMANT_BELOW_BBCH, + BRINA, + BRINA_SCHEDULE, + COVER_WINDOWS, + FROST_FALLBACK_BANDS_C, + FROST_KILL_BANDS, + FROST_KILL_CURVE, + GELO, +) + +log = logging.getLogger(__name__) + +BLOCK = "freezing" +PHENOMENON = "gelo" # contract name, key in COVER_WINDOWS +RUN_HOUR = BRINA_SCHEDULE["run_hour"] # the 24 h end where Brina's night ends +NIGHT_START_HOUR = BRINA_SCHEDULE["night_start_hour"] # Brina's night window starts here +NIGHT_SOLAR_MAX_WM2 = BRINA["night_solar_max_wm2"] # above = sunlit, Freezing's hour +HOURS_PER_RECORD = 1.0 # hourly data +HOUR = timedelta(hours=1) +FIXED_DAY_TEXT = f"records {RUN_HOUR + 1:02d}:00-{NIGHT_START_HOUR - 1:02d}:00" +NIGHT_TEXT = f"{NIGHT_START_HOUR:02d}:00-{RUN_HOUR:02d}:00" + +OPS = {"<": operator.lt, "<=": operator.le, ">": operator.gt, ">=": operator.ge} + +# ─────────────────────────── inputs (3A api_parameters) ─────────────────────────── + +REQUIRED_SENSORS = ("TemperaturaAria",) +SENSOR_WIND = "VelocitaVento2m" # optional sensors (3A sensor names) +SENSOR_SOLAR = "RadiazioneSolare" +SENSOR_RH = "UmiditaRelativa" +COL_TEMP = "TemperaturaAriaMin" # hourly minimum air temperature, °C +COL_TEMP_FALLBACK = "TemperaturaAriaMedia" +COL_RH = "UmiditaRelativaMedia" # % +COL_RH_MAX = "UmiditaRelativaMax" # %, wet-bulb of the coldest moment (derived.WET_BULB_MIN) +COL_WIND = "VelocitaVento2mMedia" # m/s +COL_SOLAR = "RadiazioneSolareMax" # W/m², highest irradiance of the hour; stations 544, 566, 600 +COLUMNS = list(dict.fromkeys([COL_TEMP, COL_TEMP_FALLBACK, COL_RH, COL_RH_MAX, + *derived.WET_BULB_INPUTS, COL_WIND, COL_SOLAR])) + + +# ─────────────────────────── frost severity (same tables as Brina) ─────────────────────────── + +def stage_for(bbch: int) -> str: + """Apple bud stage for a BBCH code (first stage whose highest code is >= bbch).""" + return next(APPLE_BBCH_TO_STAGE[c] for c in sorted(APPLE_BBCH_TO_STAGE) if bbch <= c) + + +def kill_fraction(bud_temp_c: float, stage: str) -> float: + """Fraction of buds killed: 0 at T10+offset, 10% at T10, 90% at T90, 100% at T90-offset.""" + t10, t90 = APPLE_CRITICAL_TEMPS_C[stage] + xs = [t90 - FROST_KILL_CURVE["full_kill_offset_c"], t90, t10, + t10 + FROST_KILL_CURVE["no_damage_offset_c"]] + return float(np.interp(bud_temp_c, xs, [1.0, 0.90, 0.10, 0.0])) + + +def field_severity(min_temp_c: float, bbch: int | None): + """(Severity, score, details) for one field from the lowest daylight air (= bud) temperature.""" + if bbch is None: + sev, score = grade(min_temp_c, FROST_FALLBACK_BANDS_C) + return sev, score, {"severity_method": "fallback_min_bud_temp"} + if bbch < APPLE_DORMANT_BELOW_BBCH: + return Severity.NONE, 0.0, {"severity_method": "dormant", "bbch": bbch, "kill_fraction": 0.0} + stage = stage_for(bbch) + kill = kill_fraction(min_temp_c, stage) + sev, score = grade(kill, FROST_KILL_BANDS) + return sev, score, {"severity_method": "kill_fraction", "bbch": bbch, "stage": stage, + "kill_fraction": round(kill, 3)} + + +def in_cover_window(day: date, bbch: int | None): + """Contract cover window (budburst -> end date). None when the stage is unknown.""" + _, end_mmdd = COVER_WINDOWS[PHENOMENON] + if day.strftime("%m-%d") > end_mmdd: + return False + if bbch is None: + return None + return bbch >= APPLE_DORMANT_BELOW_BBCH # approx.: buds out of dormancy = budburst + + +# ─────────────────────────── windows ─────────────────────────── + +def day_window(run_day: date): + """(start, end): the 24 h checked are the records after start up to and including end.""" + end = datetime.combine(run_day, time(RUN_HOUR)) + return end - timedelta(days=1), end + + +def night_start(run_day: date) -> datetime: + """First record of Brina's night window (the hours from here to the end are Brina's when dark).""" + return datetime.combine(run_day - timedelta(days=1), time(NIGHT_START_HOUR)) + + +# ─────────────────────────── detection ─────────────────────────── + +def _col(df, name): + return df[name] if name in df else pd.Series(np.nan, index=df.index) + + +def day_hours(rows: pd.DataFrame, run_day: date) -> pd.DataFrame: + """One station's rows on the full hourly grid of the 24 h (missing hour = empty row), with + temp_c, wet_bulb_c and the booleans night_window, sunlit, daylight and freezing.""" + start, end = day_window(run_day) + grid = pd.date_range(start + HOUR, end, freq="h", name="Timestamp") + rows = rows.assign(Timestamp=pd.to_datetime(rows["Timestamp"])).drop_duplicates("Timestamp") + df = rows.set_index("Timestamp").reindex(grid).reset_index() # rows outside the 24 h dropped + df = derived.add_wet_bulb(df) # WetBulbMin / WetBulbMax, same in every block + df["temp_c"] = _col(df, COL_TEMP).fillna(_col(df, COL_TEMP_FALLBACK)) + df["wet_bulb_c"] = df[derived.WET_BULB_MIN] # lowest wet-bulb of the hour + df["night_window"] = df["Timestamp"] >= night_start(run_day) + # exact opposite of Brina's night test (solar <= limit or no value = night) + df["sunlit"] = df["night_window"] & (_col(df, COL_SOLAR) > NIGHT_SOLAR_MAX_WM2) + df["daylight"] = ~df["night_window"] | df["sunlit"] + below = OPS[GELO["t_air_trigger_op"]](df["temp_c"], GELO["t_air_trigger_c"]) # NaN -> False + df["freezing"] = df["daylight"] & below + return df + + +def _gap_note(values: pd.Series, label: str, has_sensor: bool | None, effect: str) -> str | None: + """One data note for an optional input over the checked daylight hours.""" + n, missing = len(values), int(values.isna().sum()) + if has_sensor is False: + return f"No {label} sensor at this station: {effect}." + if n == 0: + return None + if missing == n: + return f"No {label} data in yesterday's daylight hours: {effect}." + if missing: + return f"{label.capitalize()} missing for {missing} of {n} daylight hours: {effect} for those hours." + return None + + +def data_notes(df: pd.DataFrame, sensors: set[str] | None = None) -> list[str]: + """Missing sensors / data of one station day and how they were handled. + + df = day_hours(); `sensors` = the station's 3A sensor names (None = unknown: + only the data is checked). + """ + def has(sensor): + return None if sensors is None else sensor in sensors + + notes = [] + night_solar = _col(df[df["night_window"]], COL_SOLAR) + fixed_only = f"daylight = the fixed day hours ({FIXED_DAY_TEXT}), the night window {NIGHT_TEXT} is left to Brina" + if has(SENSOR_SOLAR) is False: + notes.append(f"No solar radiation sensor at this station: {fixed_only}.") + elif night_solar.isna().all(): + notes.append(f"No solar radiation data in the night window: {fixed_only}.") + elif night_solar.isna().any(): + notes.append(f"Solar radiation missing for {int(night_solar.isna().sum())} of {len(night_solar)} " + f"hours of the night window: those hours counted as night (left to Brina).") + + day = df[df["daylight"]] + n_day = len(day) + no_temp = int(day["temp_c"].isna().sum()) + if no_temp: + notes.append(f"Air temperature missing for {no_temp} of {n_day} daylight hours: " + f"those hours not checked.") + checked = day[day["temp_c"].notna()] + mean_used = int((_col(checked, COL_TEMP).isna() & _col(checked, COL_TEMP_FALLBACK).notna()).sum()) + if mean_used: + notes.append(f"Hourly minimum temperature missing for {mean_used} of {len(checked)} daylight " + f"hours: hourly mean used.") + rh = _col(checked, COL_RH_MAX).fillna(_col(checked, COL_RH)) + notes += [ + _gap_note(_col(checked, COL_WIND), "wind", has(SENSOR_WIND), + "wind during the frost not reported (not needed to detect daylight freezing)"), + _gap_note(rh, "humidity", has(SENSOR_RH), + "wet-bulb not reported (not needed to detect daylight freezing)"), + ] + return [x for x in notes if x] + + +def _round(x, digits=2): + return None if x is None or pd.isna(x) else round(float(x), digits) + + +def analyse_station(df: pd.DataFrame, sensors: set[str] | None = None) -> dict: + """Station-level result of one day: status, duration, times, measured extremes, data notes.""" + notes = data_notes(df, sensors) + day = df[df["daylight"] & df["temp_c"].notna()] + if day.empty: + return {"status": NO_DATA, "details": {"reason": "no temperature records in the daylight hours"}, + "notes": notes} + hits = day[day["freezing"]] + base = {"records": int(len(day)), + "daylight_hours": int(df["daylight"].sum()), + "sunlit_hours_in_night_window": int(df["sunlit"].sum()), + "solar_checked": bool(_col(df[df["night_window"]], COL_SOLAR).notna().any()), + "day_min_temp_c": _round(day["temp_c"].min())} + duration = len(hits) * HOURS_PER_RECORD + if duration < GELO["min_duration_h"]: + return {"status": NO_EVENT, "details": base, "notes": notes} + wind = _col(hits, COL_WIND).dropna() + mean_wind = float(wind.mean()) if not wind.empty else None + return { + "status": EVENT, + "notes": notes, + "duration_h": duration, + "start": hits["Timestamp"].min().to_pydatetime(), + "end": hits["Timestamp"].max().to_pydatetime(), + "min_temp_c": float(hits["temp_c"].min()), + "details": {**base, + "mean_wind_ms": _round(mean_wind, 1), + "advective_wind": (None if mean_wind is None + else bool(mean_wind >= GELO["advective_wind_min_ms"])), + "min_wet_bulb_c": _round(hits["wet_bulb_c"].min())}, + } + + +def _summary(res, sev, sdet): + if res["status"] == NO_DATA: + return "No temperature data from the station for yesterday's daylight hours." + d = res["details"] + if res["status"] == NO_EVENT: + return f"No daylight freezing: lowest daytime air temperature {d['day_min_temp_c']} °C." + text = (f"Daylight freezing for {res['duration_h']:g} h, lowest air temperature " + f"{d['day_min_temp_c']} °C: severity {sev.value}") + if "kill_fraction" in sdet: + text += f" (stage {sdet.get('stage', 'dormant')}, about {sdet['kill_fraction']:.0%} buds killed)" + return text + "." + + +def detect(df: pd.DataFrame, fields: list[Field], run_day: date, + station_names: dict[int, str] | None = None, + station_sensors: dict[int, set[str]] | None = None) -> list[Analysis]: + """One Analysis per field: the daylight hours of its station, graded with the field's stage. + + df: hourly rows (CodiceStazione, Timestamp, COLUMNS) covering day_window(run_day); + rows outside it are ignored. station_sensors: 3A sensor names per station, so the + data notes can tell "no sensor" from "no data"; without it only the data is checked. + """ + station_names = station_names or {} + station_sensors = station_sensors or {} + by_station = {} + if not df.empty: + for sid, rows in df.groupby("CodiceStazione"): + sid = int(sid) + by_station[sid] = analyse_station(day_hours(rows, run_day), station_sensors.get(sid)) + out = [] + for f in fields: + res = by_station.get(f.station_id, {"status": NO_DATA, "details": {"reason": "no records"}, + "notes": ["No records from the station for yesterday's " + "daylight hours."]}) + sev, score, sdet = Severity.NONE, 0.0, {} + details = dict(res["details"]) + if res["status"] == EVENT: + sev, score, sdet = field_severity(res["min_temp_c"], f.bbch) + details.update(sdet) + details["in_cover_window"] = in_cover_window(res["start"].date(), f.bbch) + out.append(Analysis( + block=BLOCK, run_day=run_day, + field_id=f.id, field_name=f.name, + station_id=f.station_id, + station_name=station_names.get(f.station_id, str(f.station_id)), + status=res["status"], severity=sev, severity_score=score, + duration_h=res.get("duration_h", 0.0), + start=res.get("start"), end=res.get("end"), + summary=_summary(res, sev, sdet), details=details, + data_notes=res.get("notes", []), + )) + return out + + +def run(run_day: date | None = None) -> list[Analysis]: + """Fetch yesterday's 24 h of hourly data for every field's station and detect daylight freezing.""" + run_day = run_day or date.today() + start, end = day_window(run_day) + stations = {s.id: s for s in stations_with(*REQUIRED_SENSORS)} + fields = all_fields() + usable = [f for f in fields if f.station_id in stations] + ids = sorted({f.station_id for f in usable}) + df = fetch_hourly(ids, start + HOUR, end, COLUMNS) if ids else pd.DataFrame() + out = detect(df, usable, run_day, {i: s.name for i, s in stations.items()}, + {i: set(s.sensors) for i, s in stations.items()}) + for f in fields: + if f.station_id not in stations: + known = station_by_id(f.station_id) + note = ("Station not in the 3A catalogue: Freezing cannot be judged." if known is None + else "No air temperature sensor (TemperaturaAria) at this station: " + "Freezing cannot be judged.") + out.append(Analysis(block=BLOCK, run_day=run_day, field_id=f.id, field_name=f.name, + station_id=f.station_id, + station_name=known.name if known else str(f.station_id), + status=NO_DATA, summary="The station has no air temperature sensor.", + details={"reason": "station lacks required sensors"}, + data_notes=[note])) + n_events = sum(a.detected for a in out) + log.info("Freezing %s: %d event(s) over %d field(s), %d station(s)", + run_day, n_events, len(out), len(ids)) + return out + + +if __name__ == "__main__": + logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s") + p = argparse.ArgumentParser(description="Run the Freezing detector once (prints, does not save).") + p.add_argument("--date", type=date.fromisoformat, help="run morning to evaluate (default: today)") + args = p.parse_args() + for a in run(args.date): + print(a.to_dict()) diff --git a/src/damage_engine/detectors/hail.py b/src/damage_engine/detectors/hail.py new file mode 100644 index 0000000..9351004 --- /dev/null +++ b/src/damage_engine/detectors/hail.py @@ -0,0 +1,587 @@ +""" +Hail detector: flags POSSIBLE hail and asks the camera for a photo. Runs every morning on the +24 h ending at 07:00 of the run day and, on hail-risk days, also at 13:05 and 19:05 on the +6 h before (daytime checks, run_daytime(); hail melts fast, so the photo is asked for at once). + +No 3A station has a hail sensor, so this block cannot measure hail. It only checks whether +the weather data shows the signs of a hailstorm and, if so, writes a camera request +(schemas.CameraRequest in details["camera_request"], the same format as Snow); the Hail/Snow +CV model analyses the photo, confirms the hail and grades it (HAIL_SIZE_BANDS_MM is kept for +it). Goal: high recall - a photo too many costs little, a missed hailstorm is a missed claim +(user's choice 2026-10-26: one sign is enough for a photo). + +Contract (UnipolSai Model 1506, p.8): any hail counts, no size threshold, the 3 km rule does +not apply. All values are read from damage_engine.thresholds (HAIL, HAIL_RISK, HAIL_SCHEDULE); +sources: + [ENGINEERING] project default, the contract gives no number (to calibrate) + [PROJECT] pipeline setting, not a physical threshold + +Inputs (Diagram.png: temperature, rain gauge). 3A hourly values: + - Pioggia = rain of the hour, mm (sign A; "rain near" for signs B and C) + - RainRateMax = highest rain rate of the hour, mm/h (station 293 only, part of + its rain gauge): sign A + - TemperaturaAriaMin = lowest air temperature of the hour, °C (sign B; the hourly mean + TemperaturaAriaMedia is the fallback) + - TemperaturaAriaMedia = mean air temperature of the previous hour (sign B; dew point) + - TemperaturaAriaMax = highest air temperature of the hour (hail-risk day only; mean + fallback) + Extra inputs, not in the diagram: + - VelocitaVento2mMax = highest wind of the hour, m/s (stations 3, 6, 10, 30, 544, 566, + 600): sign C; VelocitaVento2mMedia is the fallback. Allowed because + HAIL["gust_ms"] is one of the three hail precursors of thresholds.py, + and the squall at the front of a hailstorm is one of its clearest signs. + - UmiditaRelativaMedia = mean humidity of the hour, %: only for the hail-risk day (dew point + = moist air that feeds thunderstorms, user's idea 2026-10-26); never + a sign. + +Windows - HAIL_SCHEDULE [PROJECT]: + - morning run, run_hour = 7: records after 07:00 of the previous day up to 07:00 of the run + day (24 hourly records; the hourly Timestamp is the end of the hour, as in the other + blocks). Photo at photo_hour = 09:00 of the run day. + - daytime checks (run_daytime, hail-risk days only, started by main.py): at each + daytime_run_hours = 13, 19 the daytime_window_h = 6 hours before it (records after 07:00 + up to 13:00, after 13:00 up to 19:00). Photo at the end of those hours (= at once). No night + check: the camera cannot take a photo in the dark; the night is in the next morning's 24 h, + which also re-checks the daytime hours (a second photo then, accepted). + HAIL["storm_window_h"] (1) more hours before each window are fetched, for the temperature + drop and the rain near the first hour (one API request per run). + +Signs, per hour (3A sends hourly values, so the 10-min / 30-min values of HAIL are read per +hour): + A burst rain: rain of the hour >= HAIL["rain_1h_mm"] = 20 mm [ENGINEERING], or (293 + only) RainRateMax >= 6 x HAIL["rain_10min_mm"] = 30 mm/h, the rate of 5 mm in 10 min + [ENGINEERING]. + B sudden temperature drop (cold outflow of the storm): mean temperature of the previous + hour minus the lowest temperature of this hour >= HAIL["t_drop_30min_c"] = 4 °C + [ENGINEERING]. With hourly data this is the closest to "4 °C in 30 min" (the drop can + span up to about 90 min, so it is a little more permissive). + C strong gust: hourly maximum wind >= HAIL["gust_ms"] = 13.9 m/s [ENGINEERING]. + B and C count only within HAIL["storm_window_h"] = 1 h [ENGINEERING] of an hour with rain + (hail melts in the rain gauge, so a hailstorm leaves rain); an hour with unknown rain counts + as possibly rainy (high recall). Operators: ">=" (HAIL has no contract "_op" keys). +Suspicion level (schemas.SUSPECT_*): "likely" = at least HAIL["min_indicators"] = 2 different +signs within storm_window_h hours of each other (one storm); "possible" = one sign; none +otherwise. Both levels ask for a photo (user's choice 2026-10-26). + +Hail-risk day (morning run only, details["hail_risk"]; main.py runs the daytime checks for the +stations where it is not False): inside HAIL_RISK["season"] = 1 May ... 30 Sep [ENGINEERING] +and (a hail sign in the 24 h, or highest air temperature >= HAIL_RISK["t_max_min_c"] = 25 °C +and highest hourly dew point >= HAIL_RISK["dew_point_min_c"] = 14 °C [ENGINEERING]). The dew +point is derived.dew_point_c (3A formula) on the hourly mean temperature and humidity. No +humidity data -> temperature alone; no data at all in the season -> None (unknown, checked). +3A has no forecast (and only the 3A API is allowed), so this means "today like yesterday". + +Result: status EVENT = "hail suspected, photo requested", with severity NONE (the CV model +grades it); NO_EVENT when no sign fired. The camera request holds station, photo time, level, +signs and reason. duration_h / start / end = the hours with a sign. Daytime checks are stored +as block DAYTIME_BLOCK ("hail_daytime"), one run per check (report.submit(..., slot="13h")), +so they never replace the morning run; the station report merges them into Hail. + +Cover window: COVER_WINDOWS["hail"] = fruit set -> 31 Oct [CONTRACT Special Art.5 p.31], +fruit set = BBCH above the last post_bloom code of APPLE_BBCH_TO_STAGE (> 69), the same rule +as Flood / Strong wind (copied: a detector may not import another detector). The photo is +asked for all year; details["in_cover_window"] says if the day is inside the cover (None +when the stage is unknown). HAIL_NET_COVER_END (25 May, orchards under anti-hail nets) is +not applied: fields.json does not say which orchards have nets (data note on events after +25 May). + +Missing sensors / missing values - the detector still runs (Analysis.data_notes): + - no rain gauge / no rain data (or in some hours): sign A not checked there; signs B and C + there counted as if it rained (worst case). + - RainRateMax exists only at 293: elsewhere sign A uses the hourly rain only (no note, + it is not a separate sensor). + - no temperature sensor / data (or in some hours, or in the hour before): sign B not + checked there; hourly minimum missing -> hourly mean (a drop may look smaller). + - no wind sensor (7 of 14 stations) / no wind data: sign C not checked; hourly maximum + missing -> hourly mean (never above the maximum, so a gust may be missed). + - no humidity data: hail-risk day on the temperature alone (morning run, note). + - no rain, temperature or wind value in the hours checked -> "no_data". + Missing columns and missing hours are read as empty (_col, hourly grid), so no sensor + causes an error. detect(..., station_sensors=...) lets the notes tell "no sensor" from + "no data". +""" + +import argparse +import logging +from datetime import date, datetime, time, timedelta + +import numpy as np +import pandas as pd + +from damage_engine.schemas import ( + EVENT, + NO_DATA, + NO_EVENT, + SUSPECT_LIKELY, + SUSPECT_POSSIBLE, + Analysis, + CameraRequest, +) +from damage_engine.sources import derived +from damage_engine.sources.fields import Field, all_fields +from damage_engine.sources.stations import all_stations, station_by_id +from damage_engine.sources.threea import fetch_hourly +from damage_engine.thresholds import ( + APPLE_BBCH_TO_STAGE, + COVER_WINDOWS, + HAIL, + HAIL_NET_COVER_END, + HAIL_RISK, + HAIL_SCHEDULE, +) + +log = logging.getLogger(__name__) + +BLOCK = "hail" +DAYTIME_BLOCK = "hail_daytime" # where the daytime checks are stored +PHENOMENON = "hail" # contract name, key in COVER_WINDOWS +RUN_HOUR = HAIL_SCHEDULE["run_hour"] +PHOTO_HOUR = HAIL_SCHEDULE["photo_hour"] +DAYTIME_HOURS = tuple(HAIL_SCHEDULE["daytime_run_hours"]) +DAYTIME_H = int(HAIL_SCHEDULE["daytime_window_h"]) +HOUR = timedelta(hours=1) +STORM_H = int(HAIL["storm_window_h"]) +LOOKBACK_H = max(1, STORM_H) # the hour before the window is needed for the drop +RATE_MMH = HAIL["rain_10min_mm"] * 60 / 10 # 5 mm in 10 min as a rate in mm/h (30) + +SIGNALS = { + "A": "burst rain", + "B": "sudden temperature drop near rain", + "C": "strong gust near rain", +} + +# ─────────────────────────── inputs (3A api_parameters) ─────────────────────────── + +RAIN_SENSOR = "Pioggia" +TEMP_SENSOR = "TemperaturaAria" +WIND_SENSOR = "VelocitaVento2m" +HUMIDITY_SENSOR = "UmiditaRelativa" +SENSORS = (RAIN_SENSOR, TEMP_SENSOR, WIND_SENSOR) # a station needs at least one of them +COL_RAIN = "Pioggia" # rain of the hour, mm +COL_RATE = "RainRateMax" # highest rain rate of the hour, mm/h (293 only) +COL_T_MIN = derived.T_MIN # lowest temperature of the hour, °C +COL_T_MEAN = derived.T_MEAN # mean temperature of the hour, °C +COL_T_MAX = derived.T_MAX # highest temperature of the hour, °C (risk day) +COL_RH_MEAN = derived.RH_MEAN # mean humidity of the hour, % (risk day: dew point) +COL_WIND = "VelocitaVento2mMax" # highest wind of the hour, m/s +COL_WIND_MEAN = "VelocitaVento2mMedia" +COLUMNS = [COL_RAIN, COL_RATE, COL_T_MIN, COL_T_MEAN, COL_T_MAX, COL_RH_MEAN, COL_WIND, + COL_WIND_MEAN] + + +# ─────────────────────────── cover window, season ─────────────────────────── + +def fruit_set(bbch: int) -> bool: + """Fruit set (start of the hail cover): BBCH above the last post-bloom code.""" + return bbch > max(c for c, s in APPLE_BBCH_TO_STAGE.items() if s == "post_bloom") + + +def in_cover_window(day: date, bbch: int | None): + """Contract cover window (fruit set -> end date). None when the stage is unknown.""" + _, end_mmdd = COVER_WINDOWS[PHENOMENON] + if day.strftime("%m-%d") > end_mmdd: + return False + if bbch is None: + return None + return fruit_set(bbch) + + +def in_season(day: date) -> bool: + """Inside the hail season of HAIL_RISK (the daytime checks run only then).""" + first, last = HAIL_RISK["season"] + return first <= day.strftime("%m-%d") <= last + + +# ─────────────────────────── windows ─────────────────────────── + +def day_window(run_day: date): + """(start, end): the 24 h checked are the records after start up to and including end.""" + end = datetime.combine(run_day, time(RUN_HOUR)) + return end - timedelta(days=1), end + + +def daytime_window(day: date, hour: int): + """(start, end) of the daytime check at `hour` of `day`: the DAYTIME_H hours before it.""" + end = datetime.combine(day, time(hour)) + return end - timedelta(hours=DAYTIME_H), end + + +def photo_time(run_day: date) -> datetime: + return datetime.combine(run_day, time(PHOTO_HOUR)) + + +# ─────────────────────────── detection ─────────────────────────── + +def _col(df, name): + return pd.to_numeric(df[name], errors="coerce") if name in df else pd.Series(np.nan, index=df.index) + + +def _near(flag: pd.Series) -> pd.Series: + """True where `flag` is True within STORM_H hours (before or after, on the hourly grid).""" + return flag.astype(float).rolling(2 * STORM_H + 1, center=True, min_periods=1).max().fillna(0) > 0 + + +def day_hours(rows: pd.DataFrame, run_day: date, window=None) -> pd.DataFrame: + """One station's rows on the full hourly grid (the hours checked plus LOOKBACK_H hours + before; missing hour = empty row), with in_window, rain_mm, rate_mmh, t_min_c, t_max_c, + drop_c, dew_c, wind_ms, near_rain, the signs sig_a, sig_b, sig_c and storm_signs (different + signs within STORM_H hours). window: (start, end) of a daytime check; None = the 24 h of + the morning run.""" + start, end = window or day_window(run_day) + grid = pd.date_range(start + HOUR - timedelta(hours=LOOKBACK_H), end, freq="h", name="Timestamp") + rows = rows.assign(Timestamp=pd.to_datetime(rows["Timestamp"])).drop_duplicates("Timestamp") + df = rows.set_index("Timestamp").reindex(grid).reset_index() + df["in_window"] = df["Timestamp"] > start + df["rain_mm"] = _col(df, COL_RAIN) + df["rate_mmh"] = _col(df, COL_RATE) + t_min, t_mean = _col(df, COL_T_MIN), _col(df, COL_T_MEAN) + df["t_min_c"] = t_min.fillna(t_mean) + df["t_fallback"] = t_min.isna() & t_mean.notna() + df["t_max_c"] = _col(df, COL_T_MAX).fillna(t_mean) + df["drop_c"] = t_mean.fillna(t_min).shift(1) - df["t_min_c"] + df["dew_c"] = derived.dew_point_c(t_mean, _col(df, COL_RH_MEAN)) + wind, wind_mean = _col(df, COL_WIND), _col(df, COL_WIND_MEAN) + df["wind_ms"] = wind.fillna(wind_mean) + df["wind_fallback"] = wind.isna() & wind_mean.notna() + rain_maybe = (df["rain_mm"] > 0) | (df["rate_mmh"] > 0) | df["rain_mm"].isna() + df["near_rain"] = _near(rain_maybe) + inside = df["in_window"] + df["raw_b"] = inside & (df["drop_c"] >= HAIL["t_drop_30min_c"]) # NaN -> False + df["raw_c"] = inside & (df["wind_ms"] >= HAIL["gust_ms"]) + df["sig_a"] = inside & ((df["rain_mm"] >= HAIL["rain_1h_mm"]) | (df["rate_mmh"] >= RATE_MMH)) + df["sig_b"] = df["raw_b"] & df["near_rain"] + df["sig_c"] = df["raw_c"] & df["near_rain"] + df["storm_signs"] = sum(_near(df[s]).astype(int) for s in ("sig_a", "sig_b", "sig_c")) + return df + + +def data_notes(w: pd.DataFrame, sensors: set[str] | None = None) -> list[str]: + """Missing data of one station's hours checked (w = the in-window rows of day_hours()) and + how it was handled. sensors: the station's 3A sensor names (None = unknown: only the data).""" + notes, n = [], len(w) + no_rain = int(w["rain_mm"].isna().sum()) + if sensors is not None and RAIN_SENSOR not in sensors: + notes.append(f"No rain gauge ({RAIN_SENSOR}) at this station: burst rain (sign A) not " + f"checked; temperature drop and gust counted as if it rained.") + elif no_rain == n: + notes.append(f"No rain data in the {n} hours: burst rain (sign A) not checked; temperature " + f"drop and gust counted as if it rained.") + elif no_rain: + notes.append(f"Rain missing for {no_rain} of {n} hours: burst rain (sign A) not checked " + f"there; near those hours a drop or gust counts as if it rained.") + no_temp = int(w["t_min_c"].isna().sum()) + if sensors is not None and TEMP_SENSOR not in sensors: + notes.append(f"No air temperature sensor ({TEMP_SENSOR}) at this station: temperature " + f"drop (sign B) not checked.") + elif no_temp == n: + notes.append(f"No air temperature data in the {n} hours: temperature drop (sign B) not " + f"checked.") + else: + no_drop = int(w["drop_c"].isna().sum()) + if no_drop: + notes.append(f"Temperature drop not checked for {no_drop} of {n} hours (temperature " + f"missing in that hour or in the hour before).") + fb = int(w["t_fallback"].sum()) + if fb: + notes.append(f"Hourly minimum temperature missing for {fb} hour(s): hourly mean used " + f"(a drop may look smaller).") + no_wind = int(w["wind_ms"].isna().sum()) + if sensors is not None and WIND_SENSOR not in sensors: + notes.append(f"No wind sensor ({WIND_SENSOR}) at this station: strong gust (sign C) not " + f"checked.") + elif no_wind == n: + notes.append(f"No wind data in the {n} hours: strong gust (sign C) not checked.") + else: + if no_wind: + notes.append(f"Wind missing for {no_wind} of {n} hours: strong gust (sign C) not " + f"checked there.") + fb = int(w["wind_fallback"].sum()) + if fb: + notes.append(f"Hourly maximum wind missing for {fb} hour(s): hourly mean used (never " + f"above the maximum, so a gust may be missed).") + return notes + + +def _round(x, digits=1): + return None if x is None or pd.isna(x) else round(float(x), digits) + + +def _iso(t): + return None if t is None or pd.isna(t) else pd.Timestamp(t).isoformat(timespec="minutes") + + +def _peak(w: pd.DataFrame, col: str, digits=1): + """(highest value, its hour) of a column, or (None, None) without data.""" + s = w[col].dropna() + if s.empty: + return None, None + i = s.idxmax() + return _round(s[i], digits), _iso(w.loc[i, "Timestamp"]) + + +def hail_risk(w: pd.DataFrame, run_day: date, signals: list[str]) -> tuple[bool | None, str, list[str]]: + """(risk, reason, notes) of the hail-risk day, from the morning run's 24 h (w).""" + if not in_season(run_day): + return False, "outside the hail season", [] + if signals: + return True, "a hail sign in the last 24 hours", [] + t_need, dew_need = HAIL_RISK["t_max_min_c"], HAIL_RISK["dew_point_min_c"] + t_max, dew = w["t_max_c"].max(), w["dew_c"].max() + if pd.isna(t_max): + return None, "no air temperature data (risk unknown: daytime checks run)", [] + if t_max < t_need: + return False, f"highest air temperature {t_max:.1f} °C (< {t_need:g} °C)", [] + if pd.isna(dew): + return (True, f"highest air temperature {t_max:.1f} °C, no humidity data", + ["No humidity data: the hail-risk day was decided on the air temperature alone."]) + if dew < dew_need: + return False, f"warm ({t_max:.1f} °C) but dry air (dew point {dew:.1f} °C < {dew_need:g} °C)", [] + return True, f"warm, moist air ({t_max:.1f} °C, dew point {dew:.1f} °C)", [] + + +def analyse_station(df: pd.DataFrame, run_day: date, station_id: int, + sensors: set[str] | None = None, photo_at: datetime | None = None, + daytime: bool = False) -> dict: + """Station-level result of the hours checked: status, signs, level, camera request, notes; + the morning run (daytime=False) also decides the hail-risk day.""" + w = df[df["in_window"]] + notes = data_notes(w, sensors) + has_value = w["rain_mm"].notna() | w["rate_mmh"].notna() | w["t_min_c"].notna() | w["wind_ms"].notna() + hours = len(w) + if not has_value.any(): + details = {"reason": f"no rain, temperature or wind records in the {hours} h", "hours": hours} + if not daytime: + details["hail_risk"] = None if in_season(run_day) else False + return {"status": NO_DATA, "details": details, "notes": notes} + a, b, c = w[w["sig_a"]], w[w["sig_b"]], w[w["sig_c"]] + sign_hours = w[w["sig_a"] | w["sig_b"] | w["sig_c"]] + rain_1h, rain_1h_at = _peak(w, "rain_mm") + rate, rate_at = _peak(w, "rate_mmh") + drop, drop_at = _peak(w, "drop_c") + gust, gust_at = _peak(w, "wind_ms") + storm = int(sign_hours["storm_signs"].max()) if len(sign_hours) else 0 + signals = [s for s, hit in (("A", len(a)), ("B", len(b)), ("C", len(c))) if hit] + if storm >= HAIL["min_indicators"]: + level = SUSPECT_LIKELY + elif signals: + level = SUSPECT_POSSIBLE + else: + level = None + rain_ok = bool(w["rain_mm"].notna().any()) + details = { + "hours": hours, + "records": int(has_value.sum()), + "rain_checked": rain_ok, + "rate_checked": bool(w["rate_mmh"].notna().any()), + "temp_checked": bool(w["drop_c"].notna().any()), + "wind_checked": bool(w["wind_ms"].notna().any()), + "rain_day_mm": _round(w["rain_mm"].sum()) if rain_ok else None, + "rain_hours": int((w["rain_mm"] > 0).sum()), + "max_rain_1h_mm": rain_1h, + "max_rain_1h_at": rain_1h_at, + "max_rain_rate_mmh": rate, + "max_rain_rate_at": rate_at, + "max_t_drop_c": drop, + "max_t_drop_at": drop_at, + "max_gust_ms": gust, + "max_gust_at": gust_at, + "burst_hours": int(len(a)), + "drop_hours": int(len(b)), + "gust_hours": int(len(c)), + "drop_dry_hours": int((w["raw_b"] & ~w["near_rain"]).sum()), # crossed, but no rain near + "gust_dry_hours": int((w["raw_c"] & ~w["near_rain"]).sum()), + "temp_fallback_hours": int(w["t_fallback"].sum()), + "wind_fallback_hours": int(w["wind_fallback"].sum()), + "storm_signs": storm, + "signals": signals, + "suspicion": level, + } + if daytime: + details["check"] = "daytime" + details["window_start"] = _iso(w["Timestamp"].min() - HOUR) + details["window_end"] = _iso(w["Timestamp"].max()) + else: + risk, why, risk_notes = hail_risk(w, run_day, signals) + notes += risk_notes + details.update({"max_temp_c": _round(w["t_max_c"].max()), "max_dew_point_c": _round(w["dew_c"].max()), + "hail_risk": risk, "hail_risk_reason": why}) + if level is None: + return {"status": NO_EVENT, "details": details, "notes": notes} + reason = "; ".join(SIGNALS[s] for s in signals) + request = CameraRequest(phenomenon=PHENOMENON, station_id=station_id, + photo_at=photo_at or photo_time(run_day), + level=level, signals=signals, reason=f"Hail {level}: {reason}.") + return { + "status": EVENT, + "notes": notes, + "duration_h": float(len(sign_hours)), + "start": sign_hours["Timestamp"].min().to_pydatetime(), + "end": sign_hours["Timestamp"].max().to_pydatetime(), + "details": {**details, "camera_request": request.to_dict()}, + } + + +def _hh(iso: str) -> str: + return datetime.fromisoformat(iso).strftime("%d %b %H:%M") + + +def _risk_text(d: dict) -> str: + """Morning run: one sentence on the daytime checks of the run day.""" + if d.get("hail_risk") is False or "hail_risk" not in d: + return "" + at = " and ".join(f"{h:02d}:00" for h in DAYTIME_HOURS) + return f" Hail-risk day: the station is checked again today at {at}." + + +def _summary(res) -> str: + d = res["details"] + hours = d.get("hours", 24) + span = "last 24 hours" if hours == 24 else f"{hours} hours checked" + if res["status"] == NO_DATA: + return f"No rain, temperature or wind data from the station for the {span}." + _risk_text(d) + if res["status"] == NO_EVENT: + parts = [] + if d["max_rain_1h_mm"] is not None: + parts.append(f"wettest hour {d['max_rain_1h_mm']:g} mm (sign at ≥ {HAIL['rain_1h_mm']:g} mm)") + if d["max_t_drop_c"] is not None: + parts.append(f"largest temperature drop {max(d['max_t_drop_c'], 0):g} °C " + f"(≥ {HAIL['t_drop_30min_c']:g} °C)") + if d["max_gust_ms"] is not None: + parts.append(f"highest wind {d['max_gust_ms']:g} m/s (≥ {HAIL['gust_ms']:g} m/s)") + if d["rain_checked"] and not d["rain_hours"] and not d["max_rain_rate_mmh"]: + text = f"No hail suspected: no rain in the {hours} hours." + else: + text = "No hail suspected" + (": " + ", ".join(parts) if parts else "") + "." + return text + _risk_text(d) + parts = [] + if d["burst_hours"]: + if d["max_rain_1h_mm"] is not None and d["max_rain_1h_mm"] >= HAIL["rain_1h_mm"]: + parts.append(f"burst rain of {d['max_rain_1h_mm']:g} mm in one hour ({_hh(d['max_rain_1h_at'])})") + else: + parts.append(f"rain rate up to {d['max_rain_rate_mmh']:g} mm/h ({_hh(d['max_rain_rate_at'])})") + if d["drop_hours"]: + parts.append(f"temperature drop of {d['max_t_drop_c']:g} °C near rain") + if d["gust_hours"]: + parts.append(f"gust of {d['max_gust_ms']:g} m/s near rain") + photo = datetime.fromisoformat(d["camera_request"]["photo_at"]) + when = "at once" if d.get("check") == "daytime" else "for" + return (f"Hail {d['suspicion']}: " + "; ".join(parts) + + f". Photo requested {when} {photo:%d %b %H:%M} to confirm." + _risk_text(d)) + + +def detect(df: pd.DataFrame, fields: list[Field], run_day: date, + station_names: dict[int, str] | None = None, + station_sensors: dict[int, set[str]] | None = None, + window=None) -> list[Analysis]: + """One Analysis per field; EVENT = hail suspected, photo requested. + + Morning run (window None): the 24 h of day_window(run_day), photo at photo_time(run_day), + plus the hail-risk day. Daytime check: window = daytime_window(day, hour), block + DAYTIME_BLOCK, photo at the window's end (at once), no risk day. + df: hourly rows (CodiceStazione, Timestamp, COLUMNS) covering the hours checked and the + LOOKBACK_H hours before; rows outside are ignored. station_sensors: 3A sensor names per + station, so the data notes can tell "no sensor" from "no data". + """ + station_names = station_names or {} + station_sensors = station_sensors or {} + daytime = window is not None + photo_at = window[1] if daytime else None + hours = DAYTIME_H if daytime else 24 + by_station = {} + if not df.empty: + for sid, rows in df.groupby("CodiceStazione"): + by_station[int(sid)] = analyse_station(day_hours(rows, run_day, window), run_day, int(sid), + station_sensors.get(int(sid)), photo_at, daytime) + missing = {"reason": "no records", "hours": hours} + if not daytime: + missing["hail_risk"] = None if in_season(run_day) else False + out = [] + for f in fields: + res = by_station.get(f.station_id, {"status": NO_DATA, "details": missing, + "notes": [f"No records from the station for the " + f"{'last 24' if hours == 24 else hours} hours."]}) + details = dict(res["details"]) + notes = list(res.get("notes", [])) + if res["status"] == EVENT: + day = res["start"].date() + details["in_cover_window"] = in_cover_window(day, f.bbch) + details["net_checked"] = False + if f.bbch is not None: + details["bbch"] = f.bbch + if day.strftime("%m-%d") > HAIL_NET_COVER_END and details["in_cover_window"] is not False: + end = datetime.strptime(HAIL_NET_COVER_END, "%m-%d") + notes.append(f"Anti-hail nets are not known for this field: under nets the hail " + f"cover ended on {end:%d %B}.") + out.append(Analysis( + block=DAYTIME_BLOCK if daytime else BLOCK, run_day=run_day, + field_id=f.id, field_name=f.name, + station_id=f.station_id, + station_name=station_names.get(f.station_id, str(f.station_id)), + status=res["status"], # severity stays NONE: the CV model grades + duration_h=res.get("duration_h", 0.0), + start=res.get("start"), end=res.get("end"), + summary=_summary({**res, "details": details}), details=details, + data_notes=notes, + )) + return out + + +def _run(run_day: date, window, station_ids: set[int] | None) -> list[Analysis]: + """Fetch the hours of `window` (None = the morning 24 h) plus LOOKBACK_H and detect.""" + start, end = window or day_window(run_day) + daytime = window is not None + stations = {s.id: s for s in all_stations() if any(s.has(x) for x in SENSORS)} + fields = [f for f in all_fields() if station_ids is None or f.station_id in station_ids] + usable = [f for f in fields if f.station_id in stations] + ids = sorted({f.station_id for f in usable}) + first = start + HOUR - timedelta(hours=LOOKBACK_H) + df = fetch_hourly(ids, first, end, COLUMNS) if ids else pd.DataFrame() + out = detect(df, usable, run_day, {i: s.name for i, s in stations.items()}, + {i: set(s.sensors) for i, s in stations.items()}, window) + for f in fields: + if f.station_id not in stations: + known = station_by_id(f.station_id) + note = ("Station not in the 3A catalogue: hail cannot be judged." if known is None + else "No rain gauge, air temperature or wind sensor at this station: hail " + "cannot be judged.") + details = {"reason": "station lacks required sensors"} + if not daytime: + details["hail_risk"] = False # nothing to check later either + out.append(Analysis(block=DAYTIME_BLOCK if daytime else BLOCK, run_day=run_day, + field_id=f.id, field_name=f.name, station_id=f.station_id, + station_name=known.name if known else str(f.station_id), + status=NO_DATA, summary="The station has no rain, temperature or " + "wind sensor.", + details=details, data_notes=[note])) + n = sum(a.detected for a in out) + log.info("Hail %s %s → %s: hail suspected (photo requested) for %d of %d field(s), %d station(s)", + "daytime check" if daytime else "morning run", start, end, n, len(out), len(ids)) + return out + + +def run(run_day: date | None = None) -> list[Analysis]: + """Morning run: fetch the 24 h (plus LOOKBACK_H hours) of hourly rain, temperature, wind + and humidity, flag possible hail and decide the hail-risk day of every station.""" + return _run(run_day or date.today(), None, None) + + +def run_daytime(day: date, hour: int, station_ids: set[int] | None = None) -> list[Analysis]: + """Daytime check at `hour` (one of DAYTIME_HOURS) of `day`: the DAYTIME_H hours before it, + for the fields of `station_ids` (None = every station). Photo at once when suspected. + main.py calls it on hail-risk days and stores it as DAYTIME_BLOCK with slot "h".""" + if hour not in DAYTIME_HOURS: + raise ValueError(f"daytime check hour must be one of {DAYTIME_HOURS}, not {hour}") + return _run(day, daytime_window(day, hour), station_ids) + + +if __name__ == "__main__": + logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s") + p = argparse.ArgumentParser(description="Run the Hail detector once (prints, does not save).") + p.add_argument("--date", type=date.fromisoformat, help="run day to evaluate (default: today)") + p.add_argument("--daytime", type=int, choices=DAYTIME_HOURS, + help="daytime check at this hour instead of the morning run") + args = p.parse_args() + results = (run_daytime(args.date or date.today(), args.daytime) if args.daytime + else run(args.date)) + for a in results: + print(a.to_dict()) diff --git a/src/damage_engine/detectors/high_temperature.py b/src/damage_engine/detectors/high_temperature.py new file mode 100644 index 0000000..fe02e17 --- /dev/null +++ b/src/damage_engine/detectors/high_temperature.py @@ -0,0 +1,407 @@ +""" +High temperature detector (contract "sunscald" / "sunstroke"): air of at least +40 °C at 2 m. Runs every 24 h, on the 24 h ending at 07:00 of the run day. + +Contract (UnipolSai Model 1506, p.8): sunscald = air temperature of at least 40 °C +at 2 m. Heatwave (15 days) is out of scope (thresholds.py). All values are read +from damage_engine.thresholds; sources as tagged there: + [CONTRACT] UnipolSai Model 1506, Ed. 01/2021 + [LITERATURE] WSU / Schrader et al. 2003, Racsko & Schrader 2012 (fruit sunburn) + [ENGINEERING] project default, the contract gives no number (to calibrate) + [PROJECT] pipeline setting, not a physical threshold + +Inputs (Diagram.png): air temperature, solar radiation, wet-bulb, humidity. 3A sends +hourly values: + - TemperaturaAriaMax = highest temperature of the hour, °C: compared with the + trigger (as Hot wind and Strong wind: a peak shorter than + one hour is still found) + - TemperaturaAriaMedia = mean temperature of the hour: fallback, reported + - RadiazioneSolareMax = highest irradiance of the hour, W/m² (stations 544, 566, + 600 only): fruit surface temperature (FST, below) + - UmiditaRelativaMin = lowest humidity of the hour, %: reported only + (UmiditaRelativaMedia fallback) + - wet-bulb: no station measures it; the shared WetBulbMax of sources/derived.py + (3A formula on the hourly maximum temperature + minimum humidity) is reported + for the hottest hour only. TemperaturaAriaMin / UmiditaRelativaMax are fetched + because derived.add_wet_bulb needs them. + Extra input, not in the diagram: VelocitaVento2mMedia (mean wind of the hour, + m/s). The FST formula of thresholds.py cools the fruit with wind, so without it + the FST would always be the calm-air (highest) value. Only the FST uses it. + +Event day - HIGH_TEMPERATURE_SCHEDULE["run_hour"] = 7 [PROJECT]: records after +07:00 of the previous day up to 07:00 of the run day (24 hourly records; the hourly +Timestamp is taken as the end of the hour, as in Flood, Strong wind, Hot wind). + +EVENT (user's choice 2026-10-22: the contract trigger): + an hour with air temperature (hourly maximum) >= HOT_TEMPERATURE["contract_t_max_c"] + = 40 °C, operator contract_t_max_op = ">=" [CONTRACT p.8, "at least 40 °C" at + 2 m]. Event = at least one such hour (no minimum duration in the contract; one + hourly record is the smallest unit). + +Fruit surface temperature (FST), thresholds.py formula, per hour with a solar value: + FST = air + fst_gain_max_c * (solar / 1000) / (1 + fst_wind_coeff * wind) + fst_gain_max_c = 15 °C [LITERATURE], fst_wind_coeff = 0.3 per m/s [ENGINEERING]. + The sun gain counts only when solar >= daytime_solar_min_wm2 = 300 W/m² + [ENGINEERING, "only sunlit records count"]; a weaker-sun hour has FST = air. + Air and solar are hourly maxima, so the FST is the peak of the hour. One hourly + record counts as 60 min, so browning_duration_min (60) and necrosis_duration_min + (10) [LITERATURE] are both taken as met by one hour (upper estimate). + +Severity, the same for every field and crop stage: + - with solar data: the highest FST of the 24 h graded with + HOT_TEMPERATURE_FST_BANDS_C = 46 / 49 / 52 / 55 °C -> light / moderate / severe + / extreme [LITERATURE, 55 ENGINEERING]. Below 46 °C: severity none (no + browning expected), the event stays an event (the contract trigger was met). + - without solar data (no pyranometer at 11 of 14 stations, or no value): the + highest air temperature of the event hours graded with HOT_WIND_BANDS_C = 40 / + 42 / 44 / 46 °C [ENGINEERING] (the air-temperature scale of the contract's + other 40 °C peril, read only). + +Sunburn risk, information only, never an event: hours +with air >= HOT_TEMPERATURE["risk_t_air_c"] = 32 °C [LITERATURE WSU] and FST >= the +first FST band edge (46 °C, browning). Below 40 °C air this damage is real but not +insurable under Model 1506. Shown in the summary of days without an event. +Needs solar data: details["sunburn_risk_hours"] = None without it. + +Cover window: COVER_WINDOWS["hot_temperature"] = fruit set -> 31 Oct [CONTRACT +Special Art.5 p.31], checked on the day of the first hot hour; fruit set = BBCH +above the last "post_bloom" code of APPLE_BBCH_TO_STAGE [ENGINEERING, approx.], the +same rule as Flood, Strong wind and Hot wind (a copy: a detector may not import +another detector). None when the stage is unknown. + +Missing sensors / missing values - the detector still runs: + - temperature is essential: a station without TemperaturaAria (none today) gives + "no_data"; hours without any temperature value are not checked; a day without + any usable hour gives "no_data". + - hourly maximum temperature missing but mean present: the mean is used for that + hour (never above the maximum, so a hit is still a hit, but a short peak may be + missed). + - no solar sensor or no solar value in the 24 h: no FST, severity on the air + temperature, sunburn risk not checked. Solar missing in some hours: no FST in + those hours (the highest FST may be underestimated). + - no wind value in an hour with sun gain (no anemometer, or no data): calm air, + wind = 0 (worst case: the highest FST). + - no humidity: humidity and wet-bulb not reported, detection unchanged. + Missing columns and missing hours are read as empty (_col, hourly grid), so no + sensor causes an error. Every case is written to Analysis.data_notes. detect(..., + station_sensors=...) lets the notes tell "no sensor" from "no data". +""" + +import argparse +import logging +import operator +from datetime import date, datetime, time, timedelta + +import numpy as np +import pandas as pd + +from damage_engine.schemas import EVENT, NO_DATA, NO_EVENT, Analysis, Severity +from damage_engine.severity import grade +from damage_engine.sources import derived +from damage_engine.sources.fields import Field, all_fields +from damage_engine.sources.stations import station_by_id, stations_with +from damage_engine.sources.threea import fetch_hourly +from damage_engine.thresholds import ( + APPLE_BBCH_TO_STAGE, + COVER_WINDOWS, + HIGH_TEMPERATURE_SCHEDULE, + HOT_TEMPERATURE, + HOT_TEMPERATURE_FST_BANDS_C, + HOT_WIND_BANDS_C, +) + +log = logging.getLogger(__name__) + +BLOCK = "high_temperature" +PHENOMENON = "hot_temperature" # contract name, key in COVER_WINDOWS +RUN_HOUR = HIGH_TEMPERATURE_SCHEDULE["run_hour"] +HOURS_PER_RECORD = 1 # hourly data +HOUR = timedelta(hours=1) +TRIGGER = "hourly_max" # which temperature is compared (stored in details) +AIR_BANDS = HOT_WIND_BANDS_C # severity without solar data (user's choice 2026-10-22) +SUNBURN_FST_C = HOT_TEMPERATURE_FST_BANDS_C["edges"][0] # 46 °C: browning starts + +OPS = {"<": operator.lt, "<=": operator.le, ">": operator.gt, ">=": operator.ge} + +# ─────────────────────────── inputs (3A api_parameters) ─────────────────────────── + +REQUIRED_SENSORS = ("TemperaturaAria",) +SOLAR_SENSOR = "RadiazioneSolare" +WIND_SENSOR = "VelocitaVento2m" +HUMIDITY_SENSOR = "UmiditaRelativa" +COL_T_MAX = derived.T_MAX # TemperaturaAriaMax: highest temperature of the hour, °C +COL_T_MEAN = derived.T_MEAN # TemperaturaAriaMedia: fallback, reported +COL_SOLAR = "RadiazioneSolareMax" # highest irradiance of the hour, W/m²: FST +COL_WIND = "VelocitaVento2mMedia" # mean wind of the hour, m/s: FST cooling only +COL_RH = derived.RH_MIN # UmiditaRelativaMin: lowest humidity of the hour, reported +COL_RH_MEAN = derived.RH_MEAN # fallback for the lowest humidity +COLUMNS = [*derived.WET_BULB_INPUTS, COL_SOLAR, COL_WIND] + + +# ─────────────────────────── cover window (same rule as Flood, Strong wind, Hot wind) ─────────────────────────── + +def fruit_set(bbch: int) -> bool: + """Fruit set (start of the hot temperature cover): BBCH above the last post-bloom code.""" + return bbch > max(c for c, s in APPLE_BBCH_TO_STAGE.items() if s == "post_bloom") + + +def in_cover_window(day: date, bbch: int | None): + """Contract cover window (fruit set -> end date). None when the stage is unknown.""" + _, end_mmdd = COVER_WINDOWS[PHENOMENON] + if day.strftime("%m-%d") > end_mmdd: + return False + if bbch is None: + return None + return fruit_set(bbch) + + +# ─────────────────────────── window ─────────────────────────── + +def day_window(run_day: date): + """(start, end): the 24 h checked are the records after start up to and including end.""" + end = datetime.combine(run_day, time(RUN_HOUR)) + return end - timedelta(days=1), end + + +# ─────────────────────────── fruit surface temperature ─────────────────────────── + +def fruit_surface_c(air_c, solar_wm2, wind_ms): + """FST (°C) of thresholds.py: air + sun gain, cooled by wind. The sun gain counts only + when solar >= daytime_solar_min_wm2; missing wind = calm (worst case). NaN without solar.""" + p = HOT_TEMPERATURE + air, solar = pd.Series(air_c, dtype=float), pd.Series(solar_wm2, dtype=float) + wind = pd.Series(wind_ms, dtype=float).fillna(0.0).clip(lower=0.0) + gain = p["fst_gain_max_c"] * (solar / 1000.0) / (1.0 + p["fst_wind_coeff"] * wind) + gain = gain.where(solar >= p["daytime_solar_min_wm2"], 0.0).where(solar.notna()) + return air + gain + + +# ─────────────────────────── detection ─────────────────────────── + +def _col(df, name): + return df[name] if name in df else pd.Series(np.nan, index=df.index) + + +def day_hours(rows: pd.DataFrame, run_day: date) -> pd.DataFrame: + """One station's rows on the full hourly grid of the 24 h (missing hour = empty row), with + temp_c (hourly max, mean filled in), temp_fallback, solar_wm2, wind_ms, sunlit, fst_c, + rh_pct, wet_bulb_c, hot (contract trigger) and risk (sunburn risk).""" + start, end = day_window(run_day) + grid = pd.date_range(start + HOUR, end, freq="h", name="Timestamp") + rows = derived.add_wet_bulb(rows) + rows = rows.assign(Timestamp=pd.to_datetime(rows["Timestamp"])).drop_duplicates("Timestamp") + df = rows.set_index("Timestamp").reindex(grid).reset_index() # rows outside the 24 h dropped + t_max = _col(df, COL_T_MAX) + df["temp_fallback"] = t_max.isna() & _col(df, COL_T_MEAN).notna() + df["temp_c"] = t_max.fillna(_col(df, COL_T_MEAN)) + df["solar_wm2"] = _col(df, COL_SOLAR) + df["wind_ms"] = _col(df, COL_WIND) + df["sunlit"] = df["solar_wm2"] >= HOT_TEMPERATURE["daytime_solar_min_wm2"] # NaN -> False + df["fst_c"] = fruit_surface_c(df["temp_c"], df["solar_wm2"], df["wind_ms"]).to_numpy() + df["rh_pct"] = _col(df, COL_RH).fillna(_col(df, COL_RH_MEAN)) + df["wet_bulb_c"] = _col(df, derived.WET_BULB_MAX) + p = HOT_TEMPERATURE + df["hot"] = OPS[p["contract_t_max_op"]](df["temp_c"], p["contract_t_max_c"]) # NaN -> False + df["risk"] = (df["temp_c"] >= p["risk_t_air_c"]) & (df["fst_c"] >= SUNBURN_FST_C) + return df + + +def data_notes(df: pd.DataFrame, sensors: set[str] | None = None) -> list[str]: + """Missing data of one station day (df = day_hours()) and how it was handled. + + sensors: the station's 3A sensor names (None = unknown: only the data is checked). + """ + notes, n = [], len(df) + no_temp = int(df["temp_c"].isna().sum()) + if no_temp == n: + return [f"No air temperature data in the {n} hours checked: high temperature cannot be judged."] + if no_temp: + notes.append(f"Air temperature missing for {no_temp} of {n} hours: those hours not checked.") + fallback = int(df["temp_fallback"].sum()) + if fallback: + notes.append(f"Hourly maximum temperature missing for {fallback} of {n} hours: hourly mean " + f"used (a short peak in those hours may be missed).") + has_temp = df["temp_c"].notna() + no_solar = int((has_temp & df["solar_wm2"].isna()).sum()) + without = ("fruit surface temperature not estimated, so the severity is graded on the air " + "temperature and the sunburn risk is not checked.") + if sensors is not None and SOLAR_SENSOR not in sensors: + notes.append(f"No solar radiation sensor ({SOLAR_SENSOR}) at this station: {without}") + elif no_solar == int(has_temp.sum()): + notes.append(f"No solar radiation data in the 24 hours: {without}") + else: + if no_solar: + notes.append(f"Solar radiation missing for {no_solar} of {n} hours: fruit surface " + f"temperature not estimated in those hours (it may be underestimated).") + calm = int((df["sunlit"] & df["wind_ms"].isna()).sum()) + if sensors is not None and WIND_SENSOR not in sensors: + notes.append(f"No wind sensor ({WIND_SENSOR}) at this station: fruit surface " + f"temperature computed for calm air (worst case).") + elif calm: + notes.append(f"Wind missing for {calm} sunlit hour(s): fruit surface temperature " + f"computed for calm air in those hours (worst case).") + if sensors is not None and HUMIDITY_SENSOR not in sensors: + notes.append(f"No humidity sensor ({HUMIDITY_SENSOR}) at this station: humidity and " + f"wet-bulb not reported (not used to detect).") + elif df.loc[has_temp, "rh_pct"].isna().all(): + notes.append("No humidity data in the 24 hours: humidity and wet-bulb not reported " + "(not used to detect).") + return notes + + +def _round(x, digits=1): + return None if x is None or pd.isna(x) else round(float(x), digits) + + +def _iso(t): + return None if t is None or pd.isna(t) else pd.Timestamp(t).isoformat(timespec="minutes") + + +def analyse_station(df: pd.DataFrame, sensors: set[str] | None = None) -> dict: + """Station-level result of one day: status, duration, times, temperatures, FST, notes.""" + notes = data_notes(df, sensors) + checked = df[df["temp_c"].notna()] + if checked.empty: + return {"status": NO_DATA, "details": {"reason": "no air temperature records in the 24 h"}, + "notes": notes} + top = checked.loc[checked["temp_c"].idxmax()] + with_fst = checked[checked["fst_c"].notna()] + solar = not with_fst.empty + fst_top = with_fst.loc[with_fst["fst_c"].idxmax()] if solar else None + risk = checked[checked["risk"]] + base = {"records": int(len(checked)), + "trigger": TRIGGER, + "max_temp_c": _round(top["temp_c"]), + "max_temp_at": _iso(top["Timestamp"]), + "max_mean_temp_c": _round(_col(checked, COL_T_MEAN).max()), + "fallback_hours": int(checked["temp_fallback"].sum()), + "humidity_at_max_temp_pct": _round(top["rh_pct"], 0), + "wet_bulb_at_max_temp_c": _round(top["wet_bulb_c"]), + "solar_checked": solar, + "sunlit_hours": int(checked["sunlit"].sum()) if solar else None, + "max_solar_wm2": _round(with_fst["solar_wm2"].max(), 0) if solar else None, + "max_fst_c": _round(fst_top["fst_c"]) if solar else None, + "max_fst_at": _iso(fst_top["Timestamp"]) if solar else None, + "solar_at_max_fst_wm2": _round(fst_top["solar_wm2"], 0) if solar else None, + "wind_at_max_fst_ms": _round(fst_top["wind_ms"]) if solar else None, + "sunburn_risk_hours": int(len(risk)) if solar else None, + "max_risk_fst_c": _round(risk["fst_c"].max()) if solar and len(risk) else None} + hits = checked[checked["hot"]] + if len(hits) == 0: + return {"status": NO_EVENT, "details": base, "notes": notes} + if solar: + method, value = "fst", float(with_fst["fst_c"].max()) + else: + method, value = "air", float(hits["temp_c"].max()) + return { + "status": EVENT, + "notes": notes, + "duration_h": len(hits) * HOURS_PER_RECORD, + "start": hits["Timestamp"].min().to_pydatetime(), + "end": hits["Timestamp"].max().to_pydatetime(), + "graded_value": value, + "details": {**base, + "hot_hours": int(len(hits)), + "severity_method": method, + "severity_value_c": _round(value)}, + } + + +def _summary(res, sev): + if res["status"] == NO_DATA: + return "No air temperature data from the station for the last 24 hours." + d = res["details"] + if res["status"] == NO_EVENT: + text = f"No high temperature: highest air temperature {d['max_temp_c']:g} °C." + if d["sunburn_risk_hours"]: + text += (f" Sunburn risk: fruit surface about {d['max_risk_fst_c']:g} °C in " + f"{d['sunburn_risk_hours']} hour(s) (below the contract's 40 °C air, " + f"not insurable).") + return text + text = f"High temperature for {res['duration_h']:g} h, highest air temperature {d['max_temp_c']:g} °C" + if d["severity_method"] == "fst": + text += f", fruit surface about {d['max_fst_c']:g} °C" + return text + f": severity {sev.value}." + + +def detect(df: pd.DataFrame, fields: list[Field], run_day: date, + station_names: dict[int, str] | None = None, + station_sensors: dict[int, set[str]] | None = None) -> list[Analysis]: + """One Analysis per field: the 24 h of its station, graded on the FST (or the air). + + df: hourly rows (CodiceStazione, Timestamp, COLUMNS) covering day_window(run_day); + rows outside it are ignored. station_sensors: 3A sensor names per station, so the data + notes can tell "no sensor" from "no data"; without it only the data is checked. + """ + station_names = station_names or {} + station_sensors = station_sensors or {} + by_station = {} + if not df.empty: + for sid, rows in df.groupby("CodiceStazione"): + by_station[int(sid)] = analyse_station(day_hours(rows, run_day), + station_sensors.get(int(sid))) + out = [] + for f in fields: + res = by_station.get(f.station_id, {"status": NO_DATA, "details": {"reason": "no records"}, + "notes": ["No records from the station for the last " + "24 hours."]}) + sev, score = Severity.NONE, 0.0 + details = dict(res["details"]) + if res["status"] == EVENT: + bands = HOT_TEMPERATURE_FST_BANDS_C if details["severity_method"] == "fst" else AIR_BANDS + sev, score = grade(res["graded_value"], bands) + details["in_cover_window"] = in_cover_window(res["start"].date(), f.bbch) + if f.bbch is not None: + details["bbch"] = f.bbch + out.append(Analysis( + block=BLOCK, run_day=run_day, + field_id=f.id, field_name=f.name, + station_id=f.station_id, + station_name=station_names.get(f.station_id, str(f.station_id)), + status=res["status"], severity=sev, severity_score=score, + duration_h=res.get("duration_h", 0.0), + start=res.get("start"), end=res.get("end"), + summary=_summary(res, sev), details=details, + data_notes=list(res.get("notes", [])), + )) + return out + + +def run(run_day: date | None = None) -> list[Analysis]: + """Fetch the last 24 h of hourly temperature, solar, wind and humidity and detect high + temperature.""" + run_day = run_day or date.today() + start, end = day_window(run_day) + stations = {s.id: s for s in stations_with(*REQUIRED_SENSORS)} + fields = all_fields() + usable = [f for f in fields if f.station_id in stations] + ids = sorted({f.station_id for f in usable}) + df = fetch_hourly(ids, start + HOUR, end, COLUMNS) if ids else pd.DataFrame() + out = detect(df, usable, run_day, {i: s.name for i, s in stations.items()}, + {i: set(s.sensors) for i, s in stations.items()}) + for f in fields: + if f.station_id not in stations: + known = station_by_id(f.station_id) + note = ("Station not in the 3A catalogue: high temperature cannot be judged." + if known is None + else "No air temperature sensor (TemperaturaAria) at this station: high " + "temperature cannot be judged.") + out.append(Analysis(block=BLOCK, run_day=run_day, field_id=f.id, field_name=f.name, + station_id=f.station_id, + station_name=known.name if known else str(f.station_id), + status=NO_DATA, summary="The station has no air temperature sensor.", + details={"reason": "station lacks required sensors"}, + data_notes=[note])) + n_events = sum(a.detected for a in out) + log.info("High temperature %s: %d event(s) over %d field(s), %d station(s)", + run_day, n_events, len(out), len(ids)) + return out + + +if __name__ == "__main__": + logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s") + p = argparse.ArgumentParser(description="Run the High temperature detector once (prints, does not save).") + p.add_argument("--date", type=date.fromisoformat, help="run day to evaluate (default: today)") + args = p.parse_args() + for a in run(args.date): + print(a.to_dict()) diff --git a/src/damage_engine/detectors/hot_wind.py b/src/damage_engine/detectors/hot_wind.py new file mode 100644 index 0000000..b8f966d --- /dev/null +++ b/src/damage_engine/detectors/hot_wind.py @@ -0,0 +1,359 @@ +""" +Hot wind detector (scirocco / libeccio): air of at least 40 °C carried by moving +air masses. Runs every 24 h, on the 24 h ending at 07:00 of the run day. + +Contract (UnipolSai Model 1506, p.8): hot wind = warm air masses from south-east +to south-west with a temperature of at least 40 °C at 2 m, causing withering in +the 30 days before harvest. All values are read from damage_engine.thresholds; +sources as tagged there: + [CONTRACT] UnipolSai Model 1506, Ed. 01/2021 + [ENGINEERING] project default, the contract gives no number (to calibrate) + [PROJECT] pipeline setting, not a physical threshold + +Inputs (Diagram.png): air temperature, wind speed, humidity. 3A sends hourly values: + - TemperaturaAriaMax = highest temperature of the hour, °C: compared with the + trigger (chosen by the user 2026-10-22, as for Strong + wind: a peak shorter than one hour is still found) + - TemperaturaAriaMedia = mean temperature of the hour: fallback, reported + - VelocitaVento2mMedia = mean wind of the hour, m/s: the "movement of air + masses" test; VelocitaVento2mMax is its fallback + - UmiditaRelativaMin = lowest humidity of the hour, %: reported only (dry air + speeds up withering; not a contract condition) + No 3A station measures wind DIRECTION, so the contract's SE-SW rule is never + applied. + +Event day - HOT_WIND_SCHEDULE["run_hour"] = 7 [PROJECT]: records after 07:00 of +the previous day up to 07:00 of the run day (24 hourly records; the hourly +Timestamp is taken as the end of the hour, as in Flood and Strong wind). + +An hour is a hot-wind hour when (HOT_WIND): + - air temperature (hourly maximum) >= t_max_trigger_c = 40 °C, operator + t_max_trigger_op = ">=" [CONTRACT p.8, "at least 40 °C" at 2 m], and + - the air moves: hourly mean wind >= wind_min_ms = 1.0 m/s [ENGINEERING; the + contract says "movement of air masses" and gives no speed]. Without a wind + value for the hour (no anemometer: 7 of 14 stations, or no data) the air is + taken as moving (worst case, as Brina; user's choice 2026-10-22). + - NOT applied: wind from dir_from_min_deg = 135° to dir_from_max_deg = 225° + (SE-SW) [CONTRACT p.8]: no direction sensor at any station (user's choice + 2026-10-22). details["direction_checked"] = False on every result. +Event when there is at least one hot-wind hour: neither the contract nor +thresholds.py gives a minimum duration, and one hourly record is the smallest unit. + +Severity, the same for every field and crop stage: the highest air temperature of +the hot-wind hours graded with HOT_WIND_BANDS_C [ENGINEERING]: >= 40 / 42 / 44 / +46 °C -> light / moderate / severe / extreme. The bands only grade an event that +already passed the trigger. + +Cover window: COVER_WINDOWS["hot_wind"] = fruit set -> 31 Oct [CONTRACT Special +Art.5 p.31], checked on the day of the first hot-wind hour; fruit set = BBCH above +the last "post_bloom" code of APPLE_BBCH_TO_STAGE [ENGINEERING, approx.], the same +rule as Flood and Strong wind (a copy: a detector may not import another detector). +The contract also limits hot wind to HOT_WIND["days_before_harvest"] = 30 days +before harvest [CONTRACT p.8]: fields have no harvest date yet, so this is not +checked (data note on events, details["harvest_checked"] = False). So +in_cover_window is never True: False when surely outside (after 31 Oct, or a +known stage before fruit set), otherwise None. + +Missing sensors / missing values - the detector still runs: + - temperature is essential: a station without TemperaturaAria (none today) gives + "no_data"; hours without any temperature value are not checked; a day without + any usable hour gives "no_data". + - hourly maximum temperature missing but mean present: the mean is used for that + hour (never above the maximum, so a hit is still a hit, but a short peak may + be missed). + - no wind sensor, or no wind value in an hour: air movement not checked, taken + as moving (worst case); details["wind_checked"] = False when the day has no + wind value at all. Hourly mean wind missing but maximum present: the maximum + is used (if even the maximum is below 1 m/s, the air was still). + - no humidity: not reported, detection unchanged. + Missing columns and missing hours are read as empty (_col, hourly grid), so no + sensor causes an error. Every case is written to Analysis.data_notes. detect(..., + station_sensors=...) lets the notes tell "no sensor" from "no data". +""" + +import argparse +import logging +import operator +from datetime import date, datetime, time, timedelta + +import numpy as np +import pandas as pd + +from damage_engine.schemas import EVENT, NO_DATA, NO_EVENT, Analysis, Severity +from damage_engine.severity import grade +from damage_engine.sources.fields import Field, all_fields +from damage_engine.sources.stations import station_by_id, stations_with +from damage_engine.sources.threea import fetch_hourly +from damage_engine.thresholds import ( + APPLE_BBCH_TO_STAGE, + COVER_WINDOWS, + HOT_WIND, + HOT_WIND_BANDS_C, + HOT_WIND_SCHEDULE, +) + +log = logging.getLogger(__name__) + +BLOCK = "hot_wind" +PHENOMENON = "hot_wind" # contract name, key in COVER_WINDOWS +RUN_HOUR = HOT_WIND_SCHEDULE["run_hour"] +HOURS_PER_RECORD = 1 # hourly data +HOUR = timedelta(hours=1) +TRIGGER = "hourly_max" # which temperature is compared (stored in details) + +OPS = {"<": operator.lt, "<=": operator.le, ">": operator.gt, ">=": operator.ge} + +# ─────────────────────────── inputs (3A api_parameters) ─────────────────────────── + +REQUIRED_SENSORS = ("TemperaturaAria",) +WIND_SENSOR = "VelocitaVento2m" +HUMIDITY_SENSOR = "UmiditaRelativa" +COL_T_MAX = "TemperaturaAriaMax" # highest temperature of the hour, °C: the trigger +COL_T_MEAN = "TemperaturaAriaMedia" # mean temperature of the hour: fallback, reported +COL_WIND = "VelocitaVento2mMedia" # mean wind of the hour, m/s: air movement test +COL_WIND_MAX = "VelocitaVento2mMax" # highest wind of the hour: fallback for the mean +COL_RH = "UmiditaRelativaMin" # lowest humidity of the hour, %: reported only +COL_RH_MEAN = "UmiditaRelativaMedia" # fallback for the lowest humidity +COLUMNS = [COL_T_MAX, COL_T_MEAN, COL_WIND, COL_WIND_MAX, COL_RH, COL_RH_MEAN] + + +# ─────────────────────────── cover window (same rule as Flood, Strong wind) ─────────────────────────── + +def fruit_set(bbch: int) -> bool: + """Fruit set (start of the hot wind cover): BBCH above the last post-bloom code.""" + return bbch > max(c for c, s in APPLE_BBCH_TO_STAGE.items() if s == "post_bloom") + + +def in_cover_window(day: date, bbch: int | None): + """Contract cover window (fruit set -> end date, and the 30 days before harvest). + + False when surely outside (after the end date, or before fruit set); otherwise None: + fields have no harvest date, so the 30-days-before-harvest rule cannot be checked. + """ + _, end_mmdd = COVER_WINDOWS[PHENOMENON] + if day.strftime("%m-%d") > end_mmdd: + return False + if bbch is not None and not fruit_set(bbch): + return False + return None + + +# ─────────────────────────── window ─────────────────────────── + +def day_window(run_day: date): + """(start, end): the 24 h checked are the records after start up to and including end.""" + end = datetime.combine(run_day, time(RUN_HOUR)) + return end - timedelta(days=1), end + + +# ─────────────────────────── detection ─────────────────────────── + +def _col(df, name): + return df[name] if name in df else pd.Series(np.nan, index=df.index) + + +def day_hours(rows: pd.DataFrame, run_day: date) -> pd.DataFrame: + """One station's rows on the full hourly grid of the 24 h (missing hour = empty row), with + temp_c (hourly max, mean filled in), temp_fallback, wind_ms (mean, max filled in), + rh_pct, hot (temperature trigger), moving (air movement, unknown = moving) and hot_wind.""" + start, end = day_window(run_day) + grid = pd.date_range(start + HOUR, end, freq="h", name="Timestamp") + rows = rows.assign(Timestamp=pd.to_datetime(rows["Timestamp"])).drop_duplicates("Timestamp") + df = rows.set_index("Timestamp").reindex(grid).reset_index() # rows outside the 24 h dropped + t_max = _col(df, COL_T_MAX) + df["temp_fallback"] = t_max.isna() & _col(df, COL_T_MEAN).notna() + df["temp_c"] = t_max.fillna(_col(df, COL_T_MEAN)) + wind = _col(df, COL_WIND) + df["wind_fallback"] = wind.isna() & _col(df, COL_WIND_MAX).notna() + df["wind_ms"] = wind.fillna(_col(df, COL_WIND_MAX)) + df["rh_pct"] = _col(df, COL_RH).fillna(_col(df, COL_RH_MEAN)) + df["hot"] = OPS[HOT_WIND["t_max_trigger_op"]](df["temp_c"], HOT_WIND["t_max_trigger_c"]) # NaN -> False + df["moving"] = df["wind_ms"].isna() | (df["wind_ms"] >= HOT_WIND["wind_min_ms"]) + df["hot_wind"] = df["hot"] & df["moving"] + return df + + +DIRECTION_NOTE = ("No wind-direction sensor at any 3A station: the contract's south-east to " + "south-west rule is not checked.") + + +def data_notes(df: pd.DataFrame, sensors: set[str] | None = None) -> list[str]: + """Missing data of one station day (df = day_hours()) and how it was handled. + + sensors: the station's 3A sensor names (None = unknown: only the data is checked). + """ + notes, n = [], len(df) + no_temp = int(df["temp_c"].isna().sum()) + if no_temp == n: + return [f"No air temperature data in the {n} hours checked: hot wind cannot be judged."] + if no_temp: + notes.append(f"Air temperature missing for {no_temp} of {n} hours: those hours not checked.") + fallback = int(df["temp_fallback"].sum()) + if fallback: + notes.append(f"Hourly maximum temperature missing for {fallback} of {n} hours: hourly mean " + f"used (a short peak in those hours may be missed).") + has_temp = df["temp_c"].notna() + no_wind = int((has_temp & df["wind_ms"].isna()).sum()) + if sensors is not None and WIND_SENSOR not in sensors: + notes.append(f"No wind sensor ({WIND_SENSOR}) at this station: air movement not checked, " + f"taken as moving (worst case).") + elif no_wind == int(has_temp.sum()): + notes.append("No wind data in the 24 hours: air movement not checked, taken as moving " + "(worst case).") + elif no_wind: + notes.append(f"Wind missing for {no_wind} of {n} hours: air taken as moving in those hours " + f"(worst case).") + wind_fb = int((has_temp & df["wind_fallback"]).sum()) + if wind_fb: + notes.append(f"Hourly mean wind missing for {wind_fb} of {n} hours: hourly maximum used.") + if sensors is not None and HUMIDITY_SENSOR not in sensors: + notes.append(f"No humidity sensor ({HUMIDITY_SENSOR}) at this station: humidity not " + f"reported (not used to detect).") + elif df.loc[has_temp, "rh_pct"].isna().all(): + notes.append("No humidity data in the 24 hours: humidity not reported (not used to detect).") + notes.append(DIRECTION_NOTE) + return notes + + +def _round(x, digits=1): + return None if x is None or pd.isna(x) else round(float(x), digits) + + +def _iso(t): + return None if t is None or pd.isna(t) else pd.Timestamp(t).isoformat(timespec="minutes") + + +def analyse_station(df: pd.DataFrame, sensors: set[str] | None = None) -> dict: + """Station-level result of one day: status, duration, times, highest temperature, notes.""" + notes = data_notes(df, sensors) + checked = df[df["temp_c"].notna()] + if checked.empty: + return {"status": NO_DATA, "details": {"reason": "no air temperature records in the 24 h"}, + "notes": notes} + top = checked.loc[checked["temp_c"].idxmax()] + hot = checked[checked["hot"]] + base = {"records": int(len(checked)), + "trigger": TRIGGER, + "max_temp_c": _round(top["temp_c"]), + "max_temp_at": _iso(top["Timestamp"]), + "max_mean_temp_c": _round(_col(checked, COL_T_MEAN).max()), + "fallback_hours": int(checked["temp_fallback"].sum()), + "hot_hours": int(len(hot)), + "still_hot_hours": int((hot["wind_ms"] < HOT_WIND["wind_min_ms"]).sum()), + "wind_checked": bool(checked["wind_ms"].notna().any()), + "wind_at_max_temp_ms": _round(top["wind_ms"]), + "humidity_at_max_temp_pct": _round(top["rh_pct"], 0), + "direction_checked": False} + hits = checked[checked["hot_wind"]] + if len(hits) == 0: + return {"status": NO_EVENT, "details": base, "notes": notes} + return { + "status": EVENT, + "notes": notes, + "duration_h": len(hits) * HOURS_PER_RECORD, + "start": hits["Timestamp"].min().to_pydatetime(), + "end": hits["Timestamp"].max().to_pydatetime(), + "max_temp_c": float(hits["temp_c"].max()), + "details": {**base, + "hot_wind_hours": int(len(hits)), + "max_hot_wind_temp_c": _round(hits["temp_c"].max()), + "wind_assumed_hours": int(hits["wind_ms"].isna().sum()), + "min_humidity_pct": _round(hits["rh_pct"].min(), 0), + "harvest_checked": False}, + } + + +def _summary(res, sev): + if res["status"] == NO_DATA: + return "No air temperature data from the station for the last 24 hours." + d = res["details"] + if res["status"] == NO_EVENT: + text = f"No hot wind: highest air temperature {d['max_temp_c']:g} °C." + if d["still_hot_hours"]: + text += f" {d['still_hot_hours']} hour(s) at 40 °C or more had still air." + return text + return (f"Hot wind for {res['duration_h']:g} h, highest air temperature " + f"{d['max_hot_wind_temp_c']:g} °C: severity {sev.value}.") + + +def detect(df: pd.DataFrame, fields: list[Field], run_day: date, + station_names: dict[int, str] | None = None, + station_sensors: dict[int, set[str]] | None = None) -> list[Analysis]: + """One Analysis per field: the 24 h of its station, graded with HOT_WIND_BANDS_C. + + df: hourly rows (CodiceStazione, Timestamp, COLUMNS) covering day_window(run_day); + rows outside it are ignored. station_sensors: 3A sensor names per station, so the data + notes can tell "no sensor" from "no data"; without it only the data is checked. + """ + station_names = station_names or {} + station_sensors = station_sensors or {} + by_station = {} + if not df.empty: + for sid, rows in df.groupby("CodiceStazione"): + by_station[int(sid)] = analyse_station(day_hours(rows, run_day), + station_sensors.get(int(sid))) + out = [] + for f in fields: + res = by_station.get(f.station_id, {"status": NO_DATA, "details": {"reason": "no records"}, + "notes": ["No records from the station for the last " + "24 hours."]}) + sev, score = Severity.NONE, 0.0 + details = dict(res["details"]) + notes = list(res.get("notes", [])) + if res["status"] == EVENT: + sev, score = grade(res["max_temp_c"], HOT_WIND_BANDS_C) + details["in_cover_window"] = in_cover_window(res["start"].date(), f.bbch) + if f.bbch is not None: + details["bbch"] = f.bbch + notes.append(f"Harvest date not known: the contract's " + f"{HOT_WIND['days_before_harvest']} days before harvest rule is not checked.") + out.append(Analysis( + block=BLOCK, run_day=run_day, + field_id=f.id, field_name=f.name, + station_id=f.station_id, + station_name=station_names.get(f.station_id, str(f.station_id)), + status=res["status"], severity=sev, severity_score=score, + duration_h=res.get("duration_h", 0.0), + start=res.get("start"), end=res.get("end"), + summary=_summary(res, sev), details=details, + data_notes=notes, + )) + return out + + +def run(run_day: date | None = None) -> list[Analysis]: + """Fetch the last 24 h of hourly temperature, wind and humidity and detect hot wind.""" + run_day = run_day or date.today() + start, end = day_window(run_day) + stations = {s.id: s for s in stations_with(*REQUIRED_SENSORS)} + fields = all_fields() + usable = [f for f in fields if f.station_id in stations] + ids = sorted({f.station_id for f in usable}) + df = fetch_hourly(ids, start + HOUR, end, COLUMNS) if ids else pd.DataFrame() + out = detect(df, usable, run_day, {i: s.name for i, s in stations.items()}, + {i: set(s.sensors) for i, s in stations.items()}) + for f in fields: + if f.station_id not in stations: + known = station_by_id(f.station_id) + note = ("Station not in the 3A catalogue: hot wind cannot be judged." if known is None + else "No air temperature sensor (TemperaturaAria) at this station: hot wind " + "cannot be judged.") + out.append(Analysis(block=BLOCK, run_day=run_day, field_id=f.id, field_name=f.name, + station_id=f.station_id, + station_name=known.name if known else str(f.station_id), + status=NO_DATA, summary="The station has no air temperature sensor.", + details={"reason": "station lacks required sensors"}, + data_notes=[note])) + n_events = sum(a.detected for a in out) + log.info("Hot wind %s: %d event(s) over %d field(s), %d station(s)", + run_day, n_events, len(out), len(ids)) + return out + + +if __name__ == "__main__": + logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s") + p = argparse.ArgumentParser(description="Run the Hot wind detector once (prints, does not save).") + p.add_argument("--date", type=date.fromisoformat, help="run day to evaluate (default: today)") + args = p.parse_args() + for a in run(args.date): + print(a.to_dict()) diff --git a/src/damage_engine/detectors/snow.py b/src/damage_engine/detectors/snow.py new file mode 100644 index 0000000..a85a1bc --- /dev/null +++ b/src/damage_engine/detectors/snow.py @@ -0,0 +1,424 @@ +""" +Snow detector: flags POSSIBLE snow and asks the camera for a photo. Runs every 24 h, +on the 24 h ending at 07:00 of the run day. + +No 3A station has a snow sensor, so this block cannot measure snow. It only checks whether snow +was possible from the weather data and, if so, writes a camera request +(schemas.CameraRequest in details["camera_request"]); the Hail/Snow CV model analyses the photo, confirms the snow and grades it. Goal: high recall - a photo too many costs little, a missed snowfall is a +missed claim. So when the data is unclear the block suspects snow. + +Contract (UnipolSai Model 1506, p.8): snow causing significant mechanical damage, no +depth threshold. All values are read from damage_engine.thresholds; sources as tagged: + [LITERATURE] rain / snow split on the wet-bulb temperature + [ENGINEERING] project default, the contract gives no number (to calibrate) + [PROJECT] pipeline setting, not a physical threshold + +Inputs (Diagram.png: rain, temperature, solar radiation, humidity). 3A hourly values: + - Pioggia = rain of the hour, mm (signals A, B; snow-water estimate) + - TemperaturaAria* = air temperature, °C (wet-bulb; air fallback) + - UmiditaRelativa* = humidity, % (wet-bulb; signal C uses UmiditaRelativaMax, + UmiditaRelativaMedia fallback) + - wet-bulb: the shared WetBulbMin of sources/derived.py (3A formula on the hourly + minimum temperature + maximum humidity) = the coldest moment of the hour, so an + hour that is partly snow counts as snow (worst case). + - RadiazioneSolareMax = highest irradiance of the hour, W/m² (stations 544, 566, + 600 only): signal D. + +Window - SNOW_SCHEDULE["run_hour"] = 7 [PROJECT]: records after 07:00 of the previous +day up to 07:00 of the run day (24 hourly records; the hourly Timestamp is the end of +the hour, as in the other blocks). SNOW["melt_lookahead_h"] more hours before it are +fetched so signal B sees cold hours just before the window (one API request). + +Cold hour: WetBulbMin <= SNOW["t_wet_snow_max_c"] = 1.0 °C [LITERATURE]. Without a +wet-bulb in that hour (no humidity) the air temperature (hourly minimum, mean fallback) +<= 1.0 °C is used instead. + +Signals, per hour of the 24 h: + A precipitation in a cold hour (Pioggia > 0): snow probably fell. strong + B rain in a NOT cold hour, up to SNOW["melt_lookahead_h"] = 12 h [ENGINEERING] + after a cold hour: an unheated rain gauge records snow only when it melts in + the funnel, as "rain" once it warms up. medium + C cold hour with humidity (hourly max) >= SNOW["saturated_rh_min_pct"] = 95 % + [ENGINEERING] and no rain recorded: snow the gauge may not have caught + (also fog or frost). weak + D (supporting, stations with a pyranometer) every midday record (records + ending SNOW["midday_hours"] = 11:00 ... 14:00 [ENGINEERING]) after the first + A/B/C hour has sun <= SNOW["covered_solar_max_wm2"] = 20 W/m² [ENGINEERING] + (overcast midday is about 50-200 W/m²): the sensor may be under snow. Never + triggers a photo on its own. +Suspicion level (schemas.SUSPECT_*): "likely" = A or B, or C with D; "possible" = C +only; none otherwise. + +Result: status EVENT = "snow suspected, photo requested", +with severity NONE (the CV model grades it); NO_EVENT when no signal fired. The camera +request holds station, photo time (SNOW_SCHEDULE["photo_hour"] = 09:00 of the run day +[PROJECT]: daylight, before the snow melts off the trees), level, signals and reason. +duration_h / start / end = the A, B, C hours. + +Snow-depth hint, information only: water of the A hours x SNOW["snow_liquid_ratio"] = +10 [ENGINEERING] (mm water -> cm snow: x 10 / 10) = est_new_snow_cm; counting the B +water too = est_new_snow_max_cm. SNOW_DEPTH_BANDS_CM and SNOW["foliage_level_bump"] +are not used here: they are kept for the CV model's grading. + +Cover window: COVER_WINDOWS["snow"] = budburst -> 31 Oct [ENGINEERING, snow is not +listed in Special Art.5 - verify], budburst = BBCH >= APPLE_DORMANT_BELOW_BBCH (51), as +Brina. The photo is asked for all year; details["in_cover_window"] says if the day is +inside the cover (None when the stage is unknown). + +Missing sensors / missing values - the detector still runs (Analysis.data_notes): + - temperature is essential: a station without TemperaturaAria gives "no_data"; hours + without temperature are not checked; no usable hour in the 24 h gives "no_data". + - no humidity (sensor or data, or in some hours): air <= 1 °C instead of the + wet-bulb in those hours; signal C not checked there. + - no rain gauge / no rain data (or in some hours): signals A and B not checked + there; an hour with unknown rain can still give C. + - no pyranometer / no solar data: signal D not checked (supporting only). + Missing columns and missing hours are read as empty (_col, hourly grid), so no + sensor causes an error. detect(..., station_sensors=...) lets the notes tell "no + sensor" from "no data". +""" + +import argparse +import logging +from datetime import date, datetime, time, timedelta + +import numpy as np +import pandas as pd + +from damage_engine.schemas import ( + EVENT, + NO_DATA, + NO_EVENT, + SUSPECT_LIKELY, + SUSPECT_POSSIBLE, + Analysis, + CameraRequest, +) +from damage_engine.sources import derived +from damage_engine.sources.fields import Field, all_fields +from damage_engine.sources.stations import station_by_id, stations_with +from damage_engine.sources.threea import fetch_hourly +from damage_engine.thresholds import APPLE_DORMANT_BELOW_BBCH, COVER_WINDOWS, SNOW, SNOW_SCHEDULE + +log = logging.getLogger(__name__) + +BLOCK = "snow" +PHENOMENON = "snow" # contract name, key in COVER_WINDOWS +RUN_HOUR = SNOW_SCHEDULE["run_hour"] +PHOTO_HOUR = SNOW_SCHEDULE["photo_hour"] +HOUR = timedelta(hours=1) +LOOKBACK_H = int(SNOW["melt_lookahead_h"]) +MIDDAY_FIRST, MIDDAY_LAST = SNOW["midday_hours"] + +SIGNALS = { + "A": "precipitation in a cold hour", + "B": "rain soon after a cold hour (snow melting in the rain gauge)", + "C": "cold saturated air without rain recorded", + "D": "midday sun very low after a suspect hour (solar sensor may be under snow)", +} + +# ─────────────────────────── inputs (3A api_parameters) ─────────────────────────── + +REQUIRED_SENSORS = ("TemperaturaAria",) +RAIN_SENSOR = "Pioggia" +HUMIDITY_SENSOR = "UmiditaRelativa" +SOLAR_SENSOR = "RadiazioneSolare" +COL_RAIN = "Pioggia" # rain of the hour, mm +COL_SOLAR = "RadiazioneSolareMax" # highest irradiance of the hour, W/m²: signal D +COL_T_MIN = derived.T_MIN # air fallback when there is no wet-bulb +COL_T_MEAN = derived.T_MEAN +COL_RH = derived.RH_MAX # highest humidity of the hour: signal C +COL_RH_MEAN = derived.RH_MEAN +COLUMNS = [*derived.WET_BULB_INPUTS, COL_RAIN, COL_SOLAR] + + +# ─────────────────────────── cover window ─────────────────────────── + +def in_cover_window(day: date, bbch: int | None): + """Contract cover window (budburst -> end date). None when the stage is unknown.""" + _, end_mmdd = COVER_WINDOWS[PHENOMENON] + if day.strftime("%m-%d") > end_mmdd: + return False + if bbch is None: + return None + return bbch >= APPLE_DORMANT_BELOW_BBCH + + +# ─────────────────────────── window ─────────────────────────── + +def day_window(run_day: date): + """(start, end): the 24 h checked are the records after start up to and including end.""" + end = datetime.combine(run_day, time(RUN_HOUR)) + return end - timedelta(days=1), end + + +def photo_time(run_day: date) -> datetime: + return datetime.combine(run_day, time(PHOTO_HOUR)) + + +# ─────────────────────────── detection ─────────────────────────── + +def _col(df, name): + return pd.to_numeric(df[name], errors="coerce") if name in df else pd.Series(np.nan, index=df.index) + + +def day_hours(rows: pd.DataFrame, run_day: date) -> pd.DataFrame: + """One station's rows on the full hourly grid (the 24 h plus LOOKBACK_H hours before; + missing hour = empty row), with in_window, cold_c (wet-bulb, air fallback), cold_basis, + cold, rain_mm, rh_pct, solar_wm2 and the signals sig_a, sig_b, sig_c.""" + start, end = day_window(run_day) + grid = pd.date_range(start + HOUR - timedelta(hours=LOOKBACK_H), end, freq="h", name="Timestamp") + rows = derived.add_wet_bulb(rows) + rows = rows.assign(Timestamp=pd.to_datetime(rows["Timestamp"])).drop_duplicates("Timestamp") + df = rows.set_index("Timestamp").reindex(grid).reset_index() + df["in_window"] = df["Timestamp"] > start + wet = _col(df, derived.WET_BULB_MIN) + air = _col(df, COL_T_MIN).fillna(_col(df, COL_T_MEAN)) + df["wet_bulb_c"] = wet + df["air_c"] = air + df["cold_c"] = wet.fillna(air) + df["cold_basis"] = np.where(wet.notna(), "wet_bulb", np.where(air.notna(), "air", None)) + df["cold"] = df["cold_c"] <= SNOW["t_wet_snow_max_c"] # NaN -> False + df["rain_mm"] = _col(df, COL_RAIN) + df["rh_pct"] = _col(df, COL_RH).fillna(_col(df, COL_RH_MEAN)) + df["solar_wm2"] = _col(df, COL_SOLAR) + wet_hour = df["rain_mm"] > 0 # NaN -> False + recent_cold = (df["cold"].astype(float).shift(1) + .rolling(LOOKBACK_H, min_periods=1).max().fillna(0) > 0) + df["sig_a"] = df["cold"] & wet_hour + df["sig_b"] = df["cold_c"].notna() & ~df["cold"] & recent_cold & wet_hour + df["sig_c"] = df["cold"] & ~wet_hour & (df["rh_pct"] >= SNOW["saturated_rh_min_pct"]) + return df + + +def data_notes(w: pd.DataFrame, sensors: set[str] | None = None) -> list[str]: + """Missing data of one station's 24 h (w = the in-window rows of day_hours()) and how it + was handled. sensors: the station's 3A sensor names (None = unknown: only the data).""" + notes, n = [], len(w) + no_temp = int(w["cold_c"].isna().sum()) + if no_temp == n: + return [f"No air temperature data in the {n} hours checked: snow cannot be judged."] + if no_temp: + notes.append(f"Air temperature missing for {no_temp} of {n} hours: those hours not checked.") + checked = w["cold_c"].notna() + t = SNOW["t_wet_snow_max_c"] + no_rh = int((checked & w["rh_pct"].isna()).sum()) + if sensors is not None and HUMIDITY_SENSOR not in sensors: + notes.append(f"No humidity sensor ({HUMIDITY_SENSOR}) at this station: air temperature " + f"≤ {t:g} °C used instead of the wet-bulb, and cold saturated air (signal C) " + f"not checked.") + elif no_rh == int(checked.sum()): + notes.append(f"No humidity data in the 24 hours: air temperature ≤ {t:g} °C used instead " + f"of the wet-bulb, and cold saturated air (signal C) not checked.") + elif no_rh: + notes.append(f"Humidity missing for {no_rh} of {n} hours: air temperature used instead of " + f"the wet-bulb in those hours (signal C not checked there).") + no_rain = int((checked & w["rain_mm"].isna()).sum()) + if sensors is not None and RAIN_SENSOR not in sensors: + notes.append(f"No rain gauge ({RAIN_SENSOR}) at this station: precipitation signals (A, B) " + f"not checked; snow can only be suspected from cold saturated air (C).") + elif no_rain == int(checked.sum()): + notes.append("No rain data in the 24 hours: precipitation signals (A, B) not checked; snow " + "can only be suspected from cold saturated air (C).") + elif no_rain: + notes.append(f"Rain missing for {no_rain} of {n} hours: precipitation signals (A, B) not " + f"checked in those hours.") + if sensors is not None and SOLAR_SENSOR not in sensors: + notes.append(f"No solar radiation sensor ({SOLAR_SENSOR}) at this station: the covered-" + f"sensor sign (signal D, supporting only) not checked.") + elif w.loc[checked, "solar_wm2"].isna().all(): + notes.append("No solar radiation data in the 24 hours: the covered-sensor sign (signal D, " + "supporting only) not checked.") + return notes + + +def _round(x, digits=1): + return None if x is None or pd.isna(x) else round(float(x), digits) + + +def _iso(t): + return None if t is None or pd.isna(t) else pd.Timestamp(t).isoformat(timespec="minutes") + + +def _midday(w: pd.DataFrame, after) -> pd.DataFrame: + """Midday records (ending MIDDAY_FIRST ... MIDDAY_LAST) after `after`, with a solar value.""" + ts = w["Timestamp"] + mid = ts.dt.hour.between(MIDDAY_FIRST, MIDDAY_LAST) & (ts.dt.minute == 0) + return w[mid & (ts > after) & w["solar_wm2"].notna()] + + +def analyse_station(df: pd.DataFrame, run_day: date, station_id: int, + sensors: set[str] | None = None) -> dict: + """Station-level result of one day: status, signals, level, camera request, notes.""" + w = df[df["in_window"]] + notes = data_notes(w, sensors) + checked = w[w["cold_c"].notna()] + if checked.empty: + return {"status": NO_DATA, "details": {"reason": "no air temperature records in the 24 h"}, + "notes": notes} + coldest = checked.loc[checked["cold_c"].idxmin()] + basis = set(checked["cold_basis"]) + a, b, c = checked[checked["sig_a"]], checked[checked["sig_b"]], checked[checked["sig_c"]] + suspect = checked[checked["sig_a"] | checked["sig_b"] | checked["sig_c"]] + ratio = SNOW["snow_liquid_ratio"] / 10.0 # mm of water -> cm of snow + snow_mm, melt_mm = float(a["rain_mm"].sum()), float(b["rain_mm"].sum()) + solar = bool(checked["solar_wm2"].notna().any()) + midday = _midday(checked, suspect["Timestamp"].min()) if solar and len(suspect) else None + covered = (None if midday is None or midday.empty + else bool((midday["solar_wm2"] <= SNOW["covered_solar_max_wm2"]).all())) + signals = [s for s, hit in (("A", len(a)), ("B", len(b)), ("C", len(c))) if hit] + if signals and covered: + signals.append("D") + if "A" in signals or "B" in signals or ("C" in signals and "D" in signals): + level = SUSPECT_LIKELY + elif signals: + level = SUSPECT_POSSIBLE + else: + level = None + details = { + "records": int(len(checked)), + "cold_basis": "wet_bulb" if basis == {"wet_bulb"} else ("air" if basis == {"air"} else "mixed"), + "air_fallback_hours": int((checked["cold_basis"] == "air").sum()), + "min_cold_c": _round(coldest["cold_c"]), + "min_cold_at": _iso(coldest["Timestamp"]), + "min_wet_bulb_c": _round(checked["wet_bulb_c"].min()), + "min_air_temp_c": _round(checked["air_c"].min()), + "cold_hours": int(checked["cold"].sum()), + "rain_checked": bool(checked["rain_mm"].notna().any()), + "humidity_checked": bool(checked["rh_pct"].notna().any()), + "solar_checked": solar, + "rain_day_mm": _round(checked["rain_mm"].sum()) if checked["rain_mm"].notna().any() else None, + "snow_hours": int(len(a)), + "snow_water_mm": _round(snow_mm), + "melt_hours": int(len(b)), + "melt_water_mm": _round(melt_mm), + "saturated_hours": int(len(c)), + "max_rh_cold_pct": _round(checked.loc[checked["cold"], "rh_pct"].max(), 0), + "midday_solar_max_wm2": None if midday is None or midday.empty else _round(midday["solar_wm2"].max(), 0), + "sensor_covered": covered, + "est_new_snow_cm": _round(snow_mm * ratio), + "est_new_snow_max_cm": _round((snow_mm + melt_mm) * ratio), + "signals": signals, + "suspicion": level, + } + if level is None: + return {"status": NO_EVENT, "details": details, "notes": notes} + reason = "; ".join(SIGNALS[s] for s in signals) + request = CameraRequest(phenomenon=PHENOMENON, station_id=station_id, photo_at=photo_time(run_day), + level=level, signals=signals, reason=f"Snow {level}: {reason}.") + return { + "status": EVENT, + "notes": notes, + "duration_h": float(len(suspect)), + "start": suspect["Timestamp"].min().to_pydatetime(), + "end": suspect["Timestamp"].max().to_pydatetime(), + "details": {**details, "camera_request": request.to_dict()}, + } + + +def _summary(res) -> str: + if res["status"] == NO_DATA: + return "No air temperature data from the station for the last 24 hours." + d = res["details"] + t = SNOW["t_wet_snow_max_c"] + what = "wet-bulb" if d["cold_basis"] != "air" else "air temperature" + if res["status"] == NO_EVENT: + if not d["cold_hours"]: + return f"No snow possible: lowest {what} {d['min_cold_c']:g} °C (snow needs ≤ {t:g} °C)." + return (f"Cold enough for snow in {d['cold_hours']} hour(s) (lowest {what} " + f"{d['min_cold_c']:g} °C), but no precipitation or saturated air: no snow suspected.") + parts = [] + if d["snow_hours"]: + parts.append(f"precipitation in {d['snow_hours']} cold hour(s) ({d['snow_water_mm']:g} mm)") + if d["melt_hours"]: + parts.append(f"{d['melt_water_mm']:g} mm of rain within {LOOKBACK_H} h after cold hours " + f"(snow may have melted in the rain gauge)") + if d["saturated_hours"]: + parts.append(f"{d['saturated_hours']} cold hour(s) of saturated air without rain recorded") + if d["sensor_covered"]: + parts.append("the solar sensor stayed dark at midday (it may be under snow)") + photo = datetime.fromisoformat(d["camera_request"]["photo_at"]) + text = f"Snow {d['suspicion']}: " + "; ".join(parts) + "." + if d["est_new_snow_cm"]: + text += f" About {d['est_new_snow_cm']:g} cm of new snow estimated from the rain gauge." + return text + f" Photo requested for {photo:%d %b %H:%M} to confirm." + + +def detect(df: pd.DataFrame, fields: list[Field], run_day: date, + station_names: dict[int, str] | None = None, + station_sensors: dict[int, set[str]] | None = None) -> list[Analysis]: + """One Analysis per field: the 24 h of its station; EVENT = snow suspected, photo requested. + + df: hourly rows (CodiceStazione, Timestamp, COLUMNS) covering the 24 h of day_window(run_day) + and the LOOKBACK_H hours before; rows outside are ignored. station_sensors: 3A sensor names + per station, so the data notes can tell "no sensor" from "no data". + """ + station_names = station_names or {} + station_sensors = station_sensors or {} + by_station = {} + if not df.empty: + for sid, rows in df.groupby("CodiceStazione"): + by_station[int(sid)] = analyse_station(day_hours(rows, run_day), run_day, int(sid), + station_sensors.get(int(sid))) + out = [] + for f in fields: + res = by_station.get(f.station_id, {"status": NO_DATA, "details": {"reason": "no records"}, + "notes": ["No records from the station for the last " + "24 hours."]}) + details = dict(res["details"]) + if res["status"] == EVENT: + details["in_cover_window"] = in_cover_window(res["start"].date(), f.bbch) + if f.bbch is not None: + details["bbch"] = f.bbch + out.append(Analysis( + block=BLOCK, run_day=run_day, + field_id=f.id, field_name=f.name, + station_id=f.station_id, + station_name=station_names.get(f.station_id, str(f.station_id)), + status=res["status"], # severity stays NONE: the CV model grades + duration_h=res.get("duration_h", 0.0), + start=res.get("start"), end=res.get("end"), + summary=_summary(res), details=details, + data_notes=list(res.get("notes", [])), + )) + return out + + +def run(run_day: date | None = None) -> list[Analysis]: + """Fetch the 24 h (plus LOOKBACK_H hours) of hourly rain, temperature, humidity and solar + radiation and flag possible snow.""" + run_day = run_day or date.today() + start, end = day_window(run_day) + stations = {s.id: s for s in stations_with(*REQUIRED_SENSORS)} + fields = all_fields() + usable = [f for f in fields if f.station_id in stations] + ids = sorted({f.station_id for f in usable}) + first = start + HOUR - timedelta(hours=LOOKBACK_H) + df = fetch_hourly(ids, first, end, COLUMNS) if ids else pd.DataFrame() + out = detect(df, usable, run_day, {i: s.name for i, s in stations.items()}, + {i: set(s.sensors) for i, s in stations.items()}) + for f in fields: + if f.station_id not in stations: + known = station_by_id(f.station_id) + note = ("Station not in the 3A catalogue: snow cannot be judged." if known is None + else "No air temperature sensor (TemperaturaAria) at this station: snow cannot " + "be judged.") + out.append(Analysis(block=BLOCK, run_day=run_day, field_id=f.id, field_name=f.name, + station_id=f.station_id, + station_name=known.name if known else str(f.station_id), + status=NO_DATA, summary="The station has no air temperature sensor.", + details={"reason": "station lacks required sensors"}, + data_notes=[note])) + n = sum(a.detected for a in out) + log.info("Snow %s: snow suspected (photo requested) for %d of %d field(s), %d station(s)", + run_day, n, len(out), len(ids)) + return out + + +if __name__ == "__main__": + logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s") + p = argparse.ArgumentParser(description="Run the Snow detector once (prints, does not save).") + p.add_argument("--date", type=date.fromisoformat, help="run day to evaluate (default: today)") + args = p.parse_args() + for a in run(args.date): + print(a.to_dict()) diff --git a/src/damage_engine/detectors/strong_wind.py b/src/damage_engine/detectors/strong_wind.py new file mode 100644 index 0000000..42a2ffa --- /dev/null +++ b/src/damage_engine/detectors/strong_wind.py @@ -0,0 +1,310 @@ +""" +Strong wind (storm) detector: wind of at least 50 km/h (13.9 m/s). Runs every +24 h, on the 24 h ending at 07:00 of the run day. + +Contract (UnipolSai Model 1506, p.8): strong wind = "at least force 7 on the +Beaufort scale - a speed of 50 km/h or more (14 m/s)", measured below 10 m height +(the 3A anemometers are at 2 m, so every reading is valid). The contract does not +say over what time the speed is measured. All values are read from +damage_engine.thresholds; sources as tagged there: + [CONTRACT] UnipolSai Model 1506, Ed. 01/2021 + [SCALE] Beaufort scale + [ENGINEERING] project default, the contract gives no number (to calibrate) + [PROJECT] pipeline setting, not a physical threshold + +Input (Diagram.png): wind speed only (3A sensor VelocitaVento2m, 3A sends hourly values only: + - VelocitaVento2mMax = highest wind of the hour, m/s + - VelocitaVento2mMedia = mean wind of the hour, m/s + STRONG_WIND["use_mean_or_gust"] = "hourly_max" [ENGINEERING, chosen by the user + 2026-10-22] picks the value that is compared with the trigger: the hourly + maximum, so a storm shorter than one hour is still found. 3A does not say if + its maximum is a 10-min mean (= the contract's measure) or a gust (higher than + the contract's measure): see README, Known issues. "hourly_mean" would use the + hourly mean instead. The other value is reported only. + +Event day - STRONG_WIND_SCHEDULE["run_hour"] = 7 [PROJECT]: records after 07:00 +of the previous day up to 07:00 of the run day (24 hourly records; the hourly +Timestamp is taken as the end of the hour, as in Flood). + +An hour is a strong-wind hour when (STRONG_WIND): + - wind (the value chosen above) >= wind_trigger_ms = 13.9 m/s, operator + wind_trigger_op = ">=" [CONTRACT p.8, "50 km/h or more"; 50 km/h and the + Beaufort 7 lower bound are both 13.9 m/s]. +Event when the strong-wind time reaches min_duration_min = 10 min [ENGINEERING] +(one 10-min averaging period). One hourly record counts as 60 min, so one hour +is enough. + +Severity, the same for every field and crop stage: the highest wind of the +strong-wind hours graded with STRONG_WIND_BANDS_MS [SCALE, Beaufort 7/8/9/10]: +>= 13.9 / 17.2 / 20.8 / 24.5 m/s -> light / moderate / severe / extreme. +The bands only grade an event that already passed the trigger. + +Cover window: COVER_WINDOWS["strong_wind"] = fruit set -> 31 Oct [CONTRACT Special +Art.5 p.31], checked on the day of the first strong-wind hour. Fruit set = BBCH +above the last "post_bloom" code of APPLE_BBCH_TO_STAGE [ENGINEERING, approx.], +the same rule as Flood (a copy: a detector may not import another detector). +Not checked by the block: the contract's rule that the effects must also show on +orchards within GENERAL["effect_radius_km"] = 3 km (no sensor can tell). + +Missing sensors / missing values - the detector still runs: + - wind is essential: a station without a wind sensor (VelocitaVento2m; 7 of the + 14 stations) gives "no_data" ("station lacks required sensors"); hours with + no wind value at all (no record, or both values empty) are not checked; a day + without any usable hour gives "no_data". + - hourly maximum missing but hourly mean present (trigger "hourly_max"): the + hourly mean is used for that hour. It is never above the maximum, so a hit is + still a hit, but a short storm in that hour may be missed. + - hourly mean missing (trigger "hourly_max"): only not reported for that hour. + - trigger "hourly_mean" with the mean missing: that hour is not checked (the + maximum would overstate the mean). + Missing columns and missing hours are read as empty (_col, hourly grid), so no + sensor causes an error. Every case is written to Analysis.data_notes (sensor + missing at the station, no data, or N of M hours missing, plus what was done). +""" + +import argparse +import logging +import operator +from datetime import date, datetime, time, timedelta + +import numpy as np +import pandas as pd + +from damage_engine.schemas import EVENT, NO_DATA, NO_EVENT, Analysis, Severity +from damage_engine.severity import grade +from damage_engine.sources.fields import Field, all_fields +from damage_engine.sources.stations import station_by_id, stations_with +from damage_engine.sources.threea import fetch_hourly +from damage_engine.thresholds import ( + APPLE_BBCH_TO_STAGE, + COVER_WINDOWS, + STRONG_WIND, + STRONG_WIND_BANDS_MS, + STRONG_WIND_SCHEDULE, +) + +log = logging.getLogger(__name__) + +BLOCK = "strong_wind" +PHENOMENON = "strong_wind" # contract name, key in COVER_WINDOWS +RUN_HOUR = STRONG_WIND_SCHEDULE["run_hour"] +MINUTES_PER_RECORD = 60 # hourly data +HOUR = timedelta(hours=1) +MS_TO_KMH = 3.6 + +OPS = {"<": operator.lt, "<=": operator.le, ">": operator.gt, ">=": operator.ge} + +# ─────────────────────────── inputs (3A api_parameters) ─────────────────────────── + +REQUIRED_SENSORS = ("VelocitaVento2m",) +COL_MAX = "VelocitaVento2mMax" # highest wind of the hour, m/s +COL_MEAN = "VelocitaVento2mMedia" # mean wind of the hour, m/s +COLUMNS = [COL_MAX, COL_MEAN] + +# STRONG_WIND["use_mean_or_gust"] -> (column compared with the trigger, fallback column) +TRIGGER_COLUMNS = { + "hourly_max": (COL_MAX, COL_MEAN), # the mean is never above the max: a safe lower bound + "hourly_mean": (COL_MEAN, None), # the max would overstate the mean: no fallback +} +TRIGGER_LABEL = {"hourly_max": "hourly maximum", "hourly_mean": "hourly mean"} + + +def trigger_columns() -> tuple[str, str | None]: + """(trigger column, fallback column) for STRONG_WIND["use_mean_or_gust"].""" + choice = STRONG_WIND["use_mean_or_gust"] + if choice not in TRIGGER_COLUMNS: + raise ValueError(f"STRONG_WIND['use_mean_or_gust'] = {choice!r}: " + f"expected one of {sorted(TRIGGER_COLUMNS)}") + return TRIGGER_COLUMNS[choice] + + +# ─────────────────────────── cover window (same rule as Flood) ─────────────────────────── + +def fruit_set(bbch: int) -> bool: + """Fruit set (start of the strong wind cover): BBCH above the last post-bloom code.""" + return bbch > max(c for c, s in APPLE_BBCH_TO_STAGE.items() if s == "post_bloom") + + +def in_cover_window(day: date, bbch: int | None): + """Contract cover window (fruit set -> end date). None when the stage is unknown.""" + _, end_mmdd = COVER_WINDOWS[PHENOMENON] + if day.strftime("%m-%d") > end_mmdd: + return False + if bbch is None: + return None + return fruit_set(bbch) + + +# ─────────────────────────── window ─────────────────────────── + +def day_window(run_day: date): + """(start, end): the 24 h checked are the records after start up to and including end.""" + end = datetime.combine(run_day, time(RUN_HOUR)) + return end - timedelta(days=1), end + + +# ─────────────────────────── detection ─────────────────────────── + +def _col(df, name): + return df[name] if name in df else pd.Series(np.nan, index=df.index) + + +def day_hours(rows: pd.DataFrame, run_day: date) -> pd.DataFrame: + """One station's rows on the full hourly grid of the 24 h (missing hour = empty row), with + wind_ms (the value compared with the trigger, fallback filled in), fallback_used and strong.""" + start, end = day_window(run_day) + grid = pd.date_range(start + HOUR, end, freq="h", name="Timestamp") + rows = rows.assign(Timestamp=pd.to_datetime(rows["Timestamp"])).drop_duplicates("Timestamp") + df = rows.set_index("Timestamp").reindex(grid).reset_index() # rows outside the 24 h dropped + main_col, fallback_col = trigger_columns() + wind = _col(df, main_col) + df["fallback_used"] = False + if fallback_col is not None: + df["fallback_used"] = wind.isna() & _col(df, fallback_col).notna() + wind = wind.fillna(_col(df, fallback_col)) + df["wind_ms"] = wind + df["strong"] = OPS[STRONG_WIND["wind_trigger_op"]](wind, STRONG_WIND["wind_trigger_ms"]) # NaN -> False + return df + + +def data_notes(df: pd.DataFrame) -> list[str]: + """Missing data of one station day (df = day_hours()) and how it was handled.""" + notes, n = [], len(df) + main_col, fallback_col = trigger_columns() + label = TRIGGER_LABEL[STRONG_WIND["use_mean_or_gust"]] + no_wind = int(df["wind_ms"].isna().sum()) + if no_wind == n: + return [f"No wind data in the {n} hours checked: strong wind cannot be judged."] + if no_wind: + notes.append(f"Wind missing for {no_wind} of {n} hours: those hours not checked.") + fallback = int(df["fallback_used"].sum()) + if fallback: + notes.append(f"Hourly maximum wind missing for {fallback} of {n} hours: hourly mean used " + f"(a short storm in those hours may be missed).") + if fallback_col is not None: # trigger = hourly max: the mean is reported only + no_mean = int((_col(df, main_col).notna() & _col(df, COL_MEAN).isna()).sum()) + if no_mean: + notes.append(f"Hourly mean wind missing for {no_mean} of {n} hours: not reported for " + f"those hours (the check uses the {label}).") + return notes + + +def _round(x, digits=1): + return None if x is None or pd.isna(x) else round(float(x), digits) + + +def _iso(t): + return None if t is None or pd.isna(t) else pd.Timestamp(t).isoformat(timespec="minutes") + + +def analyse_station(df: pd.DataFrame) -> dict: + """Station-level result of one day: status, duration, times, highest wind, data notes.""" + notes = data_notes(df) + checked = df[df["wind_ms"].notna()] + if checked.empty: + return {"status": NO_DATA, "details": {"reason": "no wind records in the 24 h"}, "notes": notes} + top = checked.loc[checked["wind_ms"].idxmax()] + base = {"records": int(len(checked)), + "trigger": STRONG_WIND["use_mean_or_gust"], + "max_wind_ms": _round(top["wind_ms"]), + "max_wind_kmh": _round(top["wind_ms"] * MS_TO_KMH), + "max_wind_at": _iso(top["Timestamp"]), + "max_mean_wind_ms": _round(_col(checked, COL_MEAN).max()), + "fallback_hours": int(checked["fallback_used"].sum())} + hits = checked[checked["strong"]] + minutes = len(hits) * MINUTES_PER_RECORD + if minutes < STRONG_WIND["min_duration_min"]: + return {"status": NO_EVENT, "details": base, "notes": notes} + return { + "status": EVENT, + "notes": notes, + "duration_h": minutes / 60, + "start": hits["Timestamp"].min().to_pydatetime(), + "end": hits["Timestamp"].max().to_pydatetime(), + "max_wind_ms": float(hits["wind_ms"].max()), + "details": {**base, "strong_wind_hours": int(len(hits))}, + } + + +def _summary(res, sev): + if res["status"] == NO_DATA: + return "No wind data from the station for the last 24 hours." + d = res["details"] + wind = f"{d['max_wind_ms']:g} m/s ({d['max_wind_kmh']:g} km/h)" + if res["status"] == NO_EVENT: + return f"No strong wind: highest wind {wind}." + return f"Strong wind for {res['duration_h']:g} h, highest wind {wind}: severity {sev.value}." + + +def detect(df: pd.DataFrame, fields: list[Field], run_day: date, + station_names: dict[int, str] | None = None) -> list[Analysis]: + """One Analysis per field: the 24 h of its station, graded with the Beaufort bands. + + df: hourly rows (CodiceStazione, Timestamp, COLUMNS) covering day_window(run_day); + rows outside it are ignored. + """ + station_names = station_names or {} + by_station = {} + if not df.empty: + for sid, rows in df.groupby("CodiceStazione"): + by_station[int(sid)] = analyse_station(day_hours(rows, run_day)) + out = [] + for f in fields: + res = by_station.get(f.station_id, {"status": NO_DATA, "details": {"reason": "no records"}, + "notes": ["No records from the station for the last " + "24 hours."]}) + sev, score = Severity.NONE, 0.0 + details = dict(res["details"]) + if res["status"] == EVENT: + sev, score = grade(res["max_wind_ms"], STRONG_WIND_BANDS_MS) + details["in_cover_window"] = in_cover_window(res["start"].date(), f.bbch) + if f.bbch is not None: + details["bbch"] = f.bbch + out.append(Analysis( + block=BLOCK, run_day=run_day, + field_id=f.id, field_name=f.name, + station_id=f.station_id, + station_name=station_names.get(f.station_id, str(f.station_id)), + status=res["status"], severity=sev, severity_score=score, + duration_h=res.get("duration_h", 0.0), + start=res.get("start"), end=res.get("end"), + summary=_summary(res, sev), details=details, + data_notes=res.get("notes", []), + )) + return out + + +def run(run_day: date | None = None) -> list[Analysis]: + """Fetch the last 24 h of hourly wind for every field's station and detect strong wind.""" + run_day = run_day or date.today() + start, end = day_window(run_day) + stations = {s.id: s for s in stations_with(*REQUIRED_SENSORS)} + fields = all_fields() + usable = [f for f in fields if f.station_id in stations] + ids = sorted({f.station_id for f in usable}) + df = fetch_hourly(ids, start + HOUR, end, COLUMNS) if ids else pd.DataFrame() + out = detect(df, usable, run_day, {i: s.name for i, s in stations.items()}) + for f in fields: + if f.station_id not in stations: + known = station_by_id(f.station_id) + note = ("Station not in the 3A catalogue: strong wind cannot be judged." if known is None + else "No wind sensor (VelocitaVento2m) at this station: strong wind cannot be judged.") + out.append(Analysis(block=BLOCK, run_day=run_day, field_id=f.id, field_name=f.name, + station_id=f.station_id, + station_name=known.name if known else str(f.station_id), + status=NO_DATA, summary="The station has no wind sensor.", + details={"reason": "station lacks required sensors"}, + data_notes=[note])) + n_events = sum(a.detected for a in out) + log.info("Strong wind %s: %d event(s) over %d field(s), %d station(s)", + run_day, n_events, len(out), len(ids)) + return out + + +if __name__ == "__main__": + logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s") + p = argparse.ArgumentParser(description="Run the Strong wind detector once (prints, does not save).") + p.add_argument("--date", type=date.fromisoformat, help="run day to evaluate (default: today)") + args = p.parse_args() + for a in run(args.date): + print(a.to_dict()) diff --git a/src/damage_engine/event_merge.py b/src/damage_engine/event_merge.py new file mode 100644 index 0000000..e69de29 diff --git a/src/damage_engine/hail-snow/__init__.py b/src/damage_engine/hail-snow/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/damage_engine/hail-snow/hail_snow.py b/src/damage_engine/hail-snow/hail_snow.py new file mode 100644 index 0000000..e69de29 diff --git a/src/damage_engine/main.py b/src/damage_engine/main.py new file mode 100644 index 0000000..6c917c7 --- /dev/null +++ b/src/damage_engine/main.py @@ -0,0 +1,258 @@ +""" +Entry point: runs the whole pipeline. + + python -m damage_engine.main # every block now, then every station report, exit + python -m damage_engine.main --date 2026-10-18 # the same for a past day + python -m damage_engine.main --once brina # one block only, then every station report + python -m damage_engine.main --serve # keep running: the whole pipeline every day + # (+ Hail's daytime checks on hail-risk days) + python -m damage_engine.main --hail-daytime 13 # Hail daytime check now (hail-risk stations) + +The blocks are the events of Diagram.png (station_report.EVENTS): for each one +the pipeline loads detectors/.py and calls its run(run_day). A block that +has no code yet (no run() function), crashes or cannot reach the 3A API does +not stop the others: its error is stored (report.submit_error) and shown in +every station report under "Checks that could not run". + +Blocks that do not run every day (Drought: monthly) define due(run_day); on the +other days the pipeline skips them and stores nothing, so the station reports keep +showing their latest run. --once runs a block on any day. + +After the blocks, one client report (JSON + PDF) is written for every station +(station_report.write), in outputs/station_reports//. + +Hail daytime checks (hail melts fast): the morning Hail run marks each station's hail-risk +day (details["hail_risk"]). At 13:05 and 19:05 (HAIL_SCHEDULE["daytime_run_hours"]) --serve +runs hail.run_daytime for the stations whose risk is not False, stores it as block +"hail_daytime" with slot "13h" / "19h" (never replacing the morning run) and rewrites those +stations' report of the day. No hail-risk station -> nothing is fetched or stored. No night +check: the camera cannot take a photo in the dark. + +Run day: the blocks look at windows ending at their RUN_HOUR (Brina 06:00, +Flood 07:00). Without --date, the run day is today once the latest RUN_HOUR has +passed, otherwise yesterday, so a run never reads an unfinished window. +""" + +import argparse +import importlib +import logging +import time +import traceback +from datetime import date, datetime, timedelta + +from damage_engine.report import report, station_report +from damage_engine.sources.fields import all_fields +from damage_engine.sources.stations import all_stations + +log = logging.getLogger("damage_engine") + +BLOCKS = list(station_report.EVENTS) # every event of the diagram = detectors/.py +DEFAULT_RUN_HOUR = 7 # used only if no block defines RUN_HOUR + + +class BlockNotReady(Exception): + """The block's file has no code to run yet.""" + + +def load_block(name): + """(module, run function) of detectors/.py; BlockNotReady if it has no code yet.""" + module_name = f"damage_engine.detectors.{name}" + try: + module = importlib.import_module(module_name) + except ModuleNotFoundError as e: + if e.name != module_name: + raise + raise BlockNotReady(f"Not available yet: the file detectors/{name}.py does not exist.") + func = getattr(module, "run", None) + if not callable(func): + raise BlockNotReady(f"Not available yet: the {name} check has no code " + f"(detectors/{name}.py has no run() function).") + return module, func + + +def pipeline_hour() -> int: + """Latest RUN_HOUR of the blocks that have code: the whole pipeline runs after it.""" + hours = [] + for name in BLOCKS: + try: + module, _ = load_block(name) + except Exception: + continue + hours.append(getattr(module, "RUN_HOUR", DEFAULT_RUN_HOUR)) + return max(hours, default=DEFAULT_RUN_HOUR) + + +def default_run_day(now: datetime | None = None) -> date: + now = now or datetime.now() + return now.date() if now.hour >= pipeline_hour() else now.date() - timedelta(days=1) + + +def run_block(name: str, run_day: date) -> str | None: + """Run one block and store its result; return None when it worked, else the error message.""" + try: + _, func = load_block(name) + analyses = func(run_day) + if not isinstance(analyses, list): + raise TypeError(f"run() returned {type(analyses).__name__}, not a list of Analysis") + except BlockNotReady as e: + log.warning("Block %s: %s", name, e) + report.submit_error(name, run_day, str(e)) + return str(e) + except Exception as e: + log.exception("Block %s failed for %s", name, run_day) + message = f"The check stopped with an error: {type(e).__name__}: {e}" + report.submit_error(name, run_day, message, traceback.format_exc()) + return message + report.submit(name, run_day, analyses) + log.info("Block %s: %d analysis(es) stored", name, len(analyses)) + return None + + +def station_ids() -> list[int]: + """Every client station: the 3A catalogue plus any station named in data/fields.json.""" + return sorted({s.id for s in all_stations()} | {f.station_id for f in all_fields()}) + + +def write_station_reports(run_day: date, ids: list[int] | None = None) -> dict[int, str | None]: + """One client report (JSON + PDF) per station (ids: only these); station -> None, or the + error message.""" + out = {} + for sid in (station_ids() if ids is None else ids): + try: + station_report.write(sid, run_day) + out[sid] = None + except Exception as e: + log.exception("Station report %s failed", sid) + out[sid] = f"{type(e).__name__}: {e}" + return out + + +def not_due(name: str, run_day: date) -> str | None: + """Why a block is skipped on run_day (its due() says no), or None when it runs.""" + try: + module, _ = load_block(name) + except Exception: + return None # run_block stores the problem + due = getattr(module, "due", None) + if callable(due) and not due(run_day): + return f"not due on {run_day} (see its due())" + return None + + +def run_pipeline(run_day: date, blocks: list[str] | None = None) -> dict: + """Run the blocks (default: all), then write every station report. Returns a summary. + + Without `blocks`, a block whose due(run_day) is False is skipped (summary "skipped"); + blocks named in `blocks` (--once) always run. + """ + log.info("Pipeline for run day %s", run_day) + skipped = {} if blocks else {n: why for n in BLOCKS if (why := not_due(n, run_day))} + block_errors = {name: run_block(name, run_day) for name in (blocks or BLOCKS) + if name not in skipped} + station_errors = write_station_reports(run_day) + _print_summary(run_day, block_errors, station_errors, skipped) + return {"blocks": block_errors, "skipped": skipped, "stations": station_errors} + + +def _print_summary(run_day, block_errors, station_errors, skipped=None): + lines = [f"\nPIPELINE - run day {run_day}", "Blocks:"] + for name, err in block_errors.items(): + lines.append(f" {name:<17} {'OK' if err is None else 'ERROR - ' + err}") + for name, why in (skipped or {}).items(): + lines.append(f" {name:<17} skipped - {why}") + ok = sum(e is None for e in station_errors.values()) + lines.append(f"Station reports: {ok} of {len(station_errors)} written to " + f"{station_report.STATION_REPORTS_DIR}") + lines += [f" station {sid}: ERROR - {err}" for sid, err in station_errors.items() if err] + print("\n".join(lines)) + + +def hail_risk_stations(day: date, root=None) -> set[int] | None: + """Stations whose Hail daytime checks run on `day`: hail_risk not False in that day's morning + Hail run. None = every station (no usable morning run, inside the hail season).""" + hail, _ = load_block("hail") + morning = next((r for r in report.day_runs("hail", day, root) if not r.get("slot")), None) + if morning is None or morning.get("error"): + return None if hail.in_season(day) else set() + return {a.station_id for a in morning["analyses"] if a.details.get("hail_risk") is not False} + + +def run_hail_daytime(day: date, hour: int) -> dict: + """One Hail daytime check: hail-risk stations only, stored as block "hail_daytime" with slot + "h", then those stations' report of the day is rewritten. Returns a summary.""" + hail, _ = load_block("hail") + ids = hail_risk_stations(day) + slot = f"{hour:02d}h" + if ids is not None and not ids: + log.info("Hail daytime check %s %s: no hail-risk station, nothing to check", day, slot) + print(f"\nHAIL DAYTIME CHECK - {day} {hour:02d}:00\nNo hail-risk station today: skipped.") + return {"error": None, "stations": {}, "checked": []} + try: + analyses = hail.run_daytime(day, hour, ids) + except Exception as e: + log.exception("Hail daytime check %s %s failed", day, slot) + message = f"The check stopped with an error: {type(e).__name__}: {e}" + report.submit_error(hail.DAYTIME_BLOCK, day, message, traceback.format_exc(), slot=slot) + error = message + else: + report.submit(hail.DAYTIME_BLOCK, day, analyses, slot=slot) + error = None + checked = sorted(ids) if ids is not None else station_ids() + stations = write_station_reports(day, checked) + print(f"\nHAIL DAYTIME CHECK - {day} {hour:02d}:00 ({len(checked)} station(s))\n " + f"{'OK' if error is None else 'ERROR - ' + error}") + return {"error": error, "stations": stations, "checked": checked} + + +def next_job(now: datetime) -> tuple[datetime, int | None]: + """(time, hour) of the next --serve job: hour None = the whole pipeline (after the latest block + RUN_HOUR), an hour = Hail's daytime check at that hour. All at minute 5.""" + try: + hail, _ = load_block("hail") + daytime = list(hail.DAYTIME_HOURS) + except Exception: + daytime = [] + jobs = [] + for hour, kind in [(pipeline_hour(), None), *[(h, h) for h in daytime]]: + at = now.replace(hour=hour, minute=5, second=0, microsecond=0) + if at <= now: + at += timedelta(days=1) + jobs.append((at, kind)) + return min(jobs, key=lambda j: j[0]) + + +def serve(): + """Keep running: the whole pipeline once a day after the latest block RUN_HOUR, and Hail's + daytime checks (13:05, 19:05) on hail-risk days.""" + while True: + at, hour = next_job(datetime.now()) + log.info("Next run: %s (%s)", at, "pipeline" if hour is None else "hail daytime check") + while (left := (at - datetime.now()).total_seconds()) > 0: + time.sleep(min(left, 60)) + if hour is None: + run_pipeline(at.date()) + else: + run_hail_daytime(at.date(), hour) + + +def main(): + logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s: %(message)s") + logging.getLogger("fontTools").setLevel(logging.WARNING) # PDF font subsetting is very chatty + p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + p.add_argument("--once", choices=BLOCKS, help="run only this block (then every station report)") + p.add_argument("--date", type=date.fromisoformat, help="run day (default: see Run day above)") + p.add_argument("--serve", action="store_true", help="keep running, whole pipeline every day") + p.add_argument("--hail-daytime", type=int, metavar="HOUR", + help="Hail daytime check at HOUR (13 or 19) for the hail-risk stations; with " + "--date for a past day (default: today)") + args = p.parse_args() + if args.serve: + serve() + elif args.hail_daytime is not None: + run_hail_daytime(args.date or date.today(), args.hail_daytime) + else: + run_pipeline(args.date or default_run_day(), [args.once] if args.once else None) + + +if __name__ == "__main__": + main() diff --git a/src/damage_engine/report/__init__.py b/src/damage_engine/report/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/damage_engine/report/assets/3a_logo.png b/src/damage_engine/report/assets/3a_logo.png new file mode 100644 index 0000000..a60f7dc Binary files /dev/null and b/src/damage_engine/report/assets/3a_logo.png differ diff --git a/src/damage_engine/report/assets/fonts/DejaVu-LICENSE.txt b/src/damage_engine/report/assets/fonts/DejaVu-LICENSE.txt new file mode 100644 index 0000000..df52c17 --- /dev/null +++ b/src/damage_engine/report/assets/fonts/DejaVu-LICENSE.txt @@ -0,0 +1,187 @@ +Fonts are (c) Bitstream (see below). DejaVu changes are in public domain. +Glyphs imported from Arev fonts are (c) Tavmjong Bah (see below) + + +Bitstream Vera Fonts Copyright +------------------------------ + +Copyright (c) 2003 by Bitstream, Inc. All Rights Reserved. Bitstream Vera is +a trademark of Bitstream, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of the fonts accompanying this license ("Fonts") and associated +documentation files (the "Font Software"), to reproduce and distribute the +Font Software, including without limitation the rights to use, copy, merge, +publish, distribute, and/or sell copies of the Font Software, and to permit +persons to whom the Font Software is furnished to do so, subject to the +following conditions: + +The above copyright and trademark notices and this permission notice shall +be included in all copies of one or more of the Font Software typefaces. + +The Font Software may be modified, altered, or added to, and in particular +the designs of glyphs or characters in the Fonts may be modified and +additional glyphs or characters may be added to the Fonts, only if the fonts +are renamed to names not containing either the words "Bitstream" or the word +"Vera". + +This License becomes null and void to the extent applicable to Fonts or Font +Software that has been modified and is distributed under the "Bitstream +Vera" names. + +The Font Software may be sold as part of a larger software package but no +copy of one or more of the Font Software typefaces may be sold by itself. + +THE FONT SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT OF COPYRIGHT, PATENT, +TRADEMARK, OR OTHER RIGHT. IN NO EVENT SHALL BITSTREAM OR THE GNOME +FOUNDATION BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, INCLUDING +ANY GENERAL, SPECIAL, INDIRECT, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, +WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF +THE USE OR INABILITY TO USE THE FONT SOFTWARE OR FROM OTHER DEALINGS IN THE +FONT SOFTWARE. + +Except as contained in this notice, the names of Gnome, the Gnome +Foundation, and Bitstream Inc., shall not be used in advertising or +otherwise to promote the sale, use or other dealings in this Font Software +without prior written authorization from the Gnome Foundation or Bitstream +Inc., respectively. For further information, contact: fonts at gnome dot +org. + +Arev Fonts Copyright +------------------------------ + +Copyright (c) 2006 by Tavmjong Bah. All Rights Reserved. + +Permission is hereby granted, free of charge, to any person obtaining +a copy of the fonts accompanying this license ("Fonts") and +associated documentation files (the "Font Software"), to reproduce +and distribute the modifications to the Bitstream Vera Font Software, +including without limitation the rights to use, copy, merge, publish, +distribute, and/or sell copies of the Font Software, and to permit +persons to whom the Font Software is furnished to do so, subject to +the following conditions: + +The above copyright and trademark notices and this permission notice +shall be included in all copies of one or more of the Font Software +typefaces. + +The Font Software may be modified, altered, or added to, and in +particular the designs of glyphs or characters in the Fonts may be +modified and additional glyphs or characters may be added to the +Fonts, only if the fonts are renamed to names not containing either +the words "Tavmjong Bah" or the word "Arev". + +This License becomes null and void to the extent applicable to Fonts +or Font Software that has been modified and is distributed under the +"Tavmjong Bah Arev" names. + +The Font Software may be sold as part of a larger software package but +no copy of one or more of the Font Software typefaces may be sold by +itself. + +THE FONT SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT +OF COPYRIGHT, PATENT, TRADEMARK, OR OTHER RIGHT. IN NO EVENT SHALL +TAVMJONG BAH BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +INCLUDING ANY GENERAL, SPECIAL, INDIRECT, INCIDENTAL, OR CONSEQUENTIAL +DAMAGES, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF THE USE OR INABILITY TO USE THE FONT SOFTWARE OR FROM +OTHER DEALINGS IN THE FONT SOFTWARE. + +Except as contained in this notice, the name of Tavmjong Bah shall not +be used in advertising or otherwise to promote the sale, use or other +dealings in this Font Software without prior written authorization +from Tavmjong Bah. For further information, contact: tavmjong @ free +. fr. + +TeX Gyre DJV Math +----------------- +Fonts are (c) Bitstream (see below). DejaVu changes are in public domain. + +Math extensions done by B. Jackowski, P. Strzelczyk and P. Pianowski +(on behalf of TeX users groups) are in public domain. + +Letters imported from Euler Fraktur from AMSfonts are (c) American +Mathematical Society (see below). +Bitstream Vera Fonts Copyright +Copyright (c) 2003 by Bitstream, Inc. All Rights Reserved. Bitstream Vera +is a trademark of Bitstream, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of the fonts accompanying this license (“Fonts”) and associated +documentation +files (the “Font Software”), to reproduce and distribute the Font Software, +including without limitation the rights to use, copy, merge, publish, +distribute, +and/or sell copies of the Font Software, and to permit persons to whom +the Font Software is furnished to do so, subject to the following +conditions: + +The above copyright and trademark notices and this permission notice +shall be +included in all copies of one or more of the Font Software typefaces. + +The Font Software may be modified, altered, or added to, and in particular +the designs of glyphs or characters in the Fonts may be modified and +additional +glyphs or characters may be added to the Fonts, only if the fonts are +renamed +to names not containing either the words “Bitstream” or the word “Vera”. + +This License becomes null and void to the extent applicable to Fonts or +Font Software +that has been modified and is distributed under the “Bitstream Vera” +names. + +The Font Software may be sold as part of a larger software package but +no copy +of one or more of the Font Software typefaces may be sold by itself. + +THE FONT SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS +OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT OF COPYRIGHT, PATENT, +TRADEMARK, OR OTHER RIGHT. IN NO EVENT SHALL BITSTREAM OR THE GNOME +FOUNDATION +BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, INCLUDING ANY GENERAL, +SPECIAL, INDIRECT, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, WHETHER IN AN +ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF THE USE OR +INABILITY TO USE +THE FONT SOFTWARE OR FROM OTHER DEALINGS IN THE FONT SOFTWARE. +Except as contained in this notice, the names of GNOME, the GNOME +Foundation, +and Bitstream Inc., shall not be used in advertising or otherwise to promote +the sale, use or other dealings in this Font Software without prior written +authorization from the GNOME Foundation or Bitstream Inc., respectively. +For further information, contact: fonts at gnome dot org. + +AMSFonts (v. 2.2) copyright + +The PostScript Type 1 implementation of the AMSFonts produced by and +previously distributed by Blue Sky Research and Y&Y, Inc. are now freely +available for general use. This has been accomplished through the +cooperation +of a consortium of scientific publishers with Blue Sky Research and Y&Y. +Members of this consortium include: + +Elsevier Science IBM Corporation Society for Industrial and Applied +Mathematics (SIAM) Springer-Verlag American Mathematical Society (AMS) + +In order to assure the authenticity of these fonts, copyright will be +held by +the American Mathematical Society. This is not meant to restrict in any way +the legitimate use of the fonts, such as (but not limited to) electronic +distribution of documents containing these fonts, inclusion of these fonts +into other public domain or commercial font collections or computer +applications, use of the outline data to create derivative fonts and/or +faces, etc. However, the AMS does require that the AMS copyright notice be +removed from any derivative versions of the fonts which have been altered in +any way. In addition, to ensure the fidelity of TeX documents using Computer +Modern fonts, Professor Donald Knuth, creator of the Computer Modern faces, +has requested that any alterations which yield different font metrics be +given a different name. + +$Id$ diff --git a/src/damage_engine/report/assets/fonts/DejaVuSans.ttf b/src/damage_engine/report/assets/fonts/DejaVuSans.ttf new file mode 100644 index 0000000..e5f7eec Binary files /dev/null and b/src/damage_engine/report/assets/fonts/DejaVuSans.ttf differ diff --git a/src/damage_engine/report/assets/fonts/NotoSans-Bold.ttf b/src/damage_engine/report/assets/fonts/NotoSans-Bold.ttf new file mode 100644 index 0000000..aae7546 Binary files /dev/null and b/src/damage_engine/report/assets/fonts/NotoSans-Bold.ttf differ diff --git a/src/damage_engine/report/assets/fonts/NotoSans-Italic.ttf b/src/damage_engine/report/assets/fonts/NotoSans-Italic.ttf new file mode 100644 index 0000000..7f53133 Binary files /dev/null and b/src/damage_engine/report/assets/fonts/NotoSans-Italic.ttf differ diff --git a/src/damage_engine/report/assets/fonts/NotoSans-OFL.txt b/src/damage_engine/report/assets/fonts/NotoSans-OFL.txt new file mode 100644 index 0000000..6843f31 --- /dev/null +++ b/src/damage_engine/report/assets/fonts/NotoSans-OFL.txt @@ -0,0 +1,93 @@ +Copyright 2022 The Noto Project Authors (https://github.com/notofonts/latin-greek-cyrillic) + +This Font Software is licensed under the SIL Open Font License, Version 1.1. +This license is copied below, and is also available with a FAQ at: +https://scripts.sil.org/OFL + + +----------------------------------------------------------- +SIL OPEN FONT LICENSE Version 1.1 - 26 February 2007 +----------------------------------------------------------- + +PREAMBLE +The goals of the Open Font License (OFL) are to stimulate worldwide +development of collaborative font projects, to support the font creation +efforts of academic and linguistic communities, and to provide a free and +open framework in which fonts may be shared and improved in partnership +with others. + +The OFL allows the licensed fonts to be used, studied, modified and +redistributed freely as long as they are not sold by themselves. The +fonts, including any derivative works, can be bundled, embedded, +redistributed and/or sold with any software provided that any reserved +names are not used by derivative works. The fonts and derivatives, +however, cannot be released under any other type of license. The +requirement for fonts to remain under this license does not apply +to any document created using the fonts or their derivatives. + +DEFINITIONS +"Font Software" refers to the set of files released by the Copyright +Holder(s) under this license and clearly marked as such. This may +include source files, build scripts and documentation. + +"Reserved Font Name" refers to any names specified as such after the +copyright statement(s). + +"Original Version" refers to the collection of Font Software components as +distributed by the Copyright Holder(s). + +"Modified Version" refers to any derivative made by adding to, deleting, +or substituting -- in part or in whole -- any of the components of the +Original Version, by changing formats or by porting the Font Software to a +new environment. + +"Author" refers to any designer, engineer, programmer, technical +writer or other person who contributed to the Font Software. + +PERMISSION & CONDITIONS +Permission is hereby granted, free of charge, to any person obtaining +a copy of the Font Software, to use, study, copy, merge, embed, modify, +redistribute, and sell modified and unmodified copies of the Font +Software, subject to the following conditions: + +1) Neither the Font Software nor any of its individual components, +in Original or Modified Versions, may be sold by itself. + +2) Original or Modified Versions of the Font Software may be bundled, +redistributed and/or sold with any software, provided that each copy +contains the above copyright notice and this license. These can be +included either as stand-alone text files, human-readable headers or +in the appropriate machine-readable metadata fields within text or +binary files as long as those fields can be easily viewed by the user. + +3) No Modified Version of the Font Software may use the Reserved Font +Name(s) unless explicit written permission is granted by the corresponding +Copyright Holder. This restriction only applies to the primary font name as +presented to the users. + +4) The name(s) of the Copyright Holder(s) or the Author(s) of the Font +Software shall not be used to promote, endorse or advertise any +Modified Version, except to acknowledge the contribution(s) of the +Copyright Holder(s) and the Author(s) or with their explicit written +permission. + +5) The Font Software, modified or unmodified, in part or in whole, +must be distributed entirely under this license, and must not be +distributed under any other license. The requirement for fonts to +remain under this license does not apply to any document created +using the Font Software. + +TERMINATION +This license becomes null and void if any of the above conditions are +not met. + +DISCLAIMER +THE FONT SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT +OF COPYRIGHT, PATENT, TRADEMARK, OR OTHER RIGHT. IN NO EVENT SHALL THE +COPYRIGHT HOLDER BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +INCLUDING ANY GENERAL, SPECIAL, INDIRECT, INCIDENTAL, OR CONSEQUENTIAL +DAMAGES, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF THE USE OR INABILITY TO USE THE FONT SOFTWARE OR FROM +OTHER DEALINGS IN THE FONT SOFTWARE. diff --git a/src/damage_engine/report/assets/fonts/NotoSans-Regular.ttf b/src/damage_engine/report/assets/fonts/NotoSans-Regular.ttf new file mode 100644 index 0000000..f27f4ff Binary files /dev/null and b/src/damage_engine/report/assets/fonts/NotoSans-Regular.ttf differ diff --git a/src/damage_engine/report/report.py b/src/damage_engine/report/report.py new file mode 100644 index 0000000..c9e845a --- /dev/null +++ b/src/damage_engine/report/report.py @@ -0,0 +1,301 @@ +""" +Report block: collects the Analysis list of every block run, writes a simple +report per block, keeps the last REPORT["history_runs"] runs of each block, +and builds the report by field: each field (sorted by name), then every block +that analysed it (brina, flood, ...), then the next field. + +A block run is sent here by main.py: + + from damage_engine.report import report + report.submit("brina", run_day, analyses) + report.submit_error("freezing", run_day, "No code yet: ...") # the block could not run + +A failed run is stored like a normal one, with an "error" message and no +analyses, so the block report, the field report and the station reports show it. + +Storage (one run = one block + one run day; running the same day again replaces it): + + outputs/reports//.json all Analysis of the run + outputs/reports//.txt readable block report + +A block that runs several times a day (Hail's daytime checks, block "hail_daytime") passes a +slot ("13h", "19h"): each check is its own run, _.json / .txt, and only the +same slot of the same day replaces it. The last HISTORY_RUNS run DAYS are kept (all slots). + outputs/reports/fields_report.txt report by field, newest run of every block + +For the farmer: + python -m damage_engine.report.report # choose a field (or all) interactively + python -m damage_engine.report.report --all [--date 2026-10-18] # every field, block by block + python -m damage_engine.report.report --fields # list fields + python -m damage_engine.report.report --field ST3 # latest report of a field + python -m damage_engine.report.report --field ST3 --history # dates of past reports + python -m damage_engine.report.report --field ST3 --date 2026-10-18 + python -m damage_engine.report.report --block brina [--date 2026-10-18] +""" + +import argparse +import json +from datetime import date, datetime +from pathlib import Path + +from damage_engine.schemas import EVENT, NO_DATA, NO_EVENT, Analysis, Severity +from damage_engine.thresholds import REPORT + +REPORTS_DIR = Path(__file__).resolve().parents[3] / "outputs" / "reports" +HISTORY_RUNS = REPORT["history_runs"] +FIELDS_REPORT_FILE = "fields_report.txt" + +STATUS_TEXT = {EVENT: "EVENT", NO_EVENT: "no event", NO_DATA: "no data"} + + +# ─────────────────────────── storage ─────────────────────────── + +def submit(block: str, run_day: date, analyses: list[Analysis], root: Path | None = None, + slot: str | None = None) -> str: + """Store one block run, write its block report, keep the last HISTORY_RUNS run days. + slot: for blocks that run several times a day (e.g. "13h"); None = one run per day.""" + return _store(block, run_day, {"analyses": [a.to_dict() for a in analyses]}, root, slot) + + +def submit_error(block: str, run_day: date, message: str, detail: str = "", + root: Path | None = None, slot: str | None = None) -> str: + """Store a block run that failed (no code yet, crash, API error). + + message: one sentence shown in every report; detail: e.g. the traceback, kept + only in outputs/reports//.json. + """ + return _store(block, run_day, {"error": message, "error_detail": detail, "analyses": []}, root, + slot) + + +def _store(block: str, run_day: date, body: dict, root: Path | None, slot: str | None = None) -> str: + folder = (root or REPORTS_DIR) / block + folder.mkdir(parents=True, exist_ok=True) + run = {"block": block, "run_day": run_day.isoformat(), + "created_at": datetime.now().isoformat(timespec="seconds"), **body} + name = run_day.isoformat() + if slot: + run["slot"] = slot + name += f"_{slot}" + (folder / f"{name}.json").write_text(json.dumps(run, indent=2, ensure_ascii=False), encoding="utf-8") + text = block_report(_parse(run)) + (folder / f"{name}.txt").write_text(text, encoding="utf-8") + _prune(folder) + (root or REPORTS_DIR).joinpath(FIELDS_REPORT_FILE).write_text(fields_report(root=root), + encoding="utf-8") + return text + + +def _prune(folder: Path): + """Keep the runs of the last HISTORY_RUNS run days (a file name starts with its day).""" + files = sorted(folder.glob("*.json"), reverse=True) + keep = sorted({f.name[:10] for f in files}, reverse=True)[:HISTORY_RUNS] + for old in files: + if old.name[:10] not in keep: + old.unlink() + old.with_suffix(".txt").unlink(missing_ok=True) + + +def _parse(run: dict) -> dict: + return {**run, "run_day": date.fromisoformat(run["run_day"]), + "analyses": [Analysis.from_dict(a) for a in run["analyses"]]} + + +def blocks(root: Path | None = None) -> list[str]: + root = root or REPORTS_DIR + return sorted(p.name for p in root.iterdir() if p.is_dir()) if root.exists() else [] + + +def runs(block: str, root: Path | None = None) -> list[dict]: + """Stored runs of one block, newest first.""" + folder = (root or REPORTS_DIR) / block + files = sorted(folder.glob("*.json"), reverse=True) if folder.exists() else [] + return [_parse(json.loads(f.read_text(encoding="utf-8"))) for f in files] + + +def latest_run(block: str, as_of: date | None = None, root: Path | None = None): + """Newest run of a block with run_day <= as_of (None = newest of all).""" + return next((r for r in runs(block, root) if as_of is None or r["run_day"] <= as_of), None) + + +def day_runs(block: str, day: date, root: Path | None = None) -> list[dict]: + """Every run of a block on one run day (all slots), oldest first.""" + return sorted((r for r in runs(block, root) if r["run_day"] == day), key=lambda r: r.get("slot") or "") + + +def fields(root: Path | None = None) -> dict[str, str]: + """field_id -> field name, for every field in any stored run.""" + out = {} + for b in blocks(root): + for r in runs(b, root): + for a in r["analyses"]: + out.setdefault(a.field_id, a.field_name) + return dict(sorted(out.items())) + + +def field_report_dates(field_id: str, root: Path | None = None) -> list[date]: + """Run days (newest first) on which any block analysed this field.""" + days = {r["run_day"] for b in blocks(root) for r in runs(b, root) + if any(a.field_id == field_id for a in r["analyses"])} + return sorted(days, reverse=True) + + +# ─────────────────────────── text reports ─────────────────────────── + +def _fmt_time(t): + return t.strftime("%Y-%m-%d %H:%M") if t else "-" + + +def block_report(run: dict) -> str: + """Simple report of one block run: counts and one line per field.""" + when = f"run day {run['run_day']}" + (f", check {run['slot']}" if run.get("slot") else "") + if run.get("error"): + return (f"{run['block'].upper()} report - {when}\n" + f"Created {run.get('created_at', '-')}\n\nERROR: {run['error']}\n") + items = run["analyses"] + events = [a for a in items if a.status == EVENT] + by_sev = {s: sum(a.severity == s for a in events) for s in Severity} + sev_text = ", ".join(f"{s.value} {n}" for s, n in by_sev.items() if n) or "-" + lines = [ + f"{run['block'].upper()} report - {when}", + f"Created {run.get('created_at', '-')} | {len(items)} field(s), " + f"{len({a.station_id for a in items})} station(s)", + f"Events: {len(events)} ({sev_text}) | no event: " + f"{sum(a.status == NO_EVENT for a in items)} | no data: {sum(a.status == NO_DATA for a in items)}", + "", + f"{'FIELD':<28} {'STATION':<22} {'STATUS':<9} {'SEVERITY':<9} {'SCORE':>5} {'HOURS':>5}", + ] + order = sorted(items, key=lambda a: (-a.severity.level, a.status != EVENT, a.field_id)) + for a in order: + lines.append(f"{(a.field_id + ' ' + a.field_name)[:28]:<28} " + f"{(str(a.station_id) + ' ' + a.station_name)[:22]:<22} " + f"{STATUS_TEXT[a.status]:<9} {a.severity.value:<9} " + f"{a.severity_score:>5.2f} {a.duration_h:>5g}") + noted = [a for a in order if a.data_notes] + if noted: + lines += ["", "Missing sensors / data:"] + for a in noted: + lines += [f" {a.field_id} (station {a.station_id} {a.station_name}): {n}" for n in a.data_notes] + return "\n".join(lines) + "\n" + + +def _field_section(block: str, run: dict | None, field_id: str) -> list[str]: + head = f"[{block.upper()}]" + if run is None: + return [f"{head} no run yet", ""] + if run.get("error"): + return [f"{head} run {run['run_day']}: ERROR - {run['error']}", ""] + mine = [a for a in run["analyses"] if a.field_id == field_id] + if not mine: + return [f"{head} run {run['run_day']}: this field was not analysed", ""] + lines = [] + for a in mine: + result = STATUS_TEXT[a.status] + if a.status == EVENT: + result += f" - severity {a.severity.value.upper()} (score {a.severity_score:.2f})" + lines += [f"{head} run {a.run_day}", + f" Station: {a.station_id} {a.station_name}", + f" Result: {result}"] + if a.status == EVENT: + lines.append(f" Time: {_fmt_time(a.start)} -> {_fmt_time(a.end)} ({a.duration_h:g} h)") + if a.summary: + lines.append(f" Summary: {a.summary}") + lines += [f" Data: {n}" for n in a.data_notes] + lines += [f" {k}: {v}" for k, v in a.details.items()] + lines.append("") + return lines + + +def field_report(field_id: str, as_of: date | None = None, root: Path | None = None) -> str: + """All blocks for one field together: for each block its newest run up to as_of.""" + name = fields(root).get(field_id, "?") + lines = [f"Field report: {field_id} - {name}", + f"As of: {as_of or 'latest'}", ""] + return "\n".join(lines + _field_sections(field_id, _latest_runs(as_of, root))) + + +def fields_report(as_of: date | None = None, root: Path | None = None) -> str: + """Report by field: each field (sorted by name), then all its block results, then the next field.""" + latest = _latest_runs(as_of, root) + known = sorted(fields(root).items(), key=lambda kv: (kv[1], kv[0])) + lines = ["REPORT BY FIELD", + f"As of: {as_of or 'latest'} | created {datetime.now():%Y-%m-%d %H:%M} | " + f"{len(known)} field(s) | blocks: {', '.join(latest) or '-'}", ""] + if not known: + lines.append("No block has run yet.") + rule = "=" * 72 + for fid, name in known: + lines += [rule, f"FIELD {name} [{fid}]", rule] + lines += _field_sections(fid, latest) + return "\n".join(lines) + "\n" + + +def _latest_runs(as_of: date | None, root: Path | None) -> dict[str, dict | None]: + """block -> its newest run up to as_of (blocks in alphabetical order).""" + return {b: latest_run(b, as_of, root) for b in blocks(root)} + + +def _field_sections(field_id: str, latest: dict[str, dict | None]) -> list[str]: + """Every block's section for one field.""" + lines = [] + for b, run in latest.items(): + lines += _field_section(b, run, field_id) + return lines or ["No block has run yet."] + + +# ─────────────────────────── command line ─────────────────────────── + +def _interactive(): + known = list(fields().items()) + if not known: + print("No reports yet.") + return + print(f"{0:>3}. all fields") + for i, (fid, name) in enumerate(known, 1): + print(f"{i:>3}. {fid} {name}") + pick = input("Field number: ").strip() + if pick == "0": + print("\n" + fields_report()) + return + if not pick.isdigit() or not 1 <= int(pick) <= len(known): + print("Unknown field.") + return + fid = known[int(pick) - 1][0] + print("\n" + field_report(fid)) + days = field_report_dates(fid)[1:] + if days and input("Show a past report? [y/N] ").strip().lower() == "y": + for i, d in enumerate(days, 1): + print(f"{i:>3}. {d}") + pick = input("Report number: ").strip() + if pick.isdigit() and 1 <= int(pick) <= len(days): + print("\n" + field_report(fid, days[int(pick) - 1])) + + +def main(): + p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + p.add_argument("--fields", action="store_true", help="list the fields that have reports") + p.add_argument("--all", action="store_true", help="report by field: every field, all its blocks") + p.add_argument("--field", help="field id") + p.add_argument("--history", action="store_true", help="with --field: list past report dates") + p.add_argument("--block", help="show a block report") + p.add_argument("--date", type=date.fromisoformat, help="report as of this run day") + args = p.parse_args() + if args.all: + print(fields_report(args.date)) + elif args.fields: + for fid, name in fields().items(): + print(f"{fid} {name}") + elif args.field and args.history: + for d in field_report_dates(args.field): + print(d) + elif args.field: + print(field_report(args.field, args.date)) + elif args.block: + run = latest_run(args.block, args.date) + print(block_report(run) if run else f"No {args.block} run stored.") + else: + _interactive() + + +if __name__ == "__main__": + main() diff --git a/src/damage_engine/report/station_pdf.py b/src/damage_engine/report/station_pdf.py new file mode 100644 index 0000000..c5c8af4 --- /dev/null +++ b/src/damage_engine/report/station_pdf.py @@ -0,0 +1,478 @@ +""" +Draws the station report (the dict built by station_report.build) as a PDF. +Pure Python (fpdf2), no LaTeX: it runs anywhere the pipeline runs. + +Assets in report/assets/: 3a_logo.png (from www.3acloud.it/favicon.ico), +Noto Sans (text) and DejaVu Sans (fallback for ≤ ≥ →), both free licences +(the licence files are next to the fonts). + + from damage_engine.report import station_pdf + station_pdf.render(content, Path("566.pdf")) +""" + +from datetime import date, datetime +from pathlib import Path + +from fpdf import FPDF +from fpdf.enums import XPos, YPos +from fpdf.fonts import FontFace + +ASSETS = Path(__file__).resolve().parent / "assets" +LOGO = ASSETS / "3a_logo.png" +FONTS = ASSETS / "fonts" + +# colours (RGB): text, 3A brand (from the logo), severity scale +INK, MUTED, LINE, PANEL = (33, 37, 41), (108, 117, 125), (218, 222, 226), (245, 247, 249) +ZEBRA = (250, 251, 252) +BLUE, GREEN, ORANGE = (88, 140, 201), (118, 146, 60), (227, 109, 38) +SEVERITY_COLOR = {"none": (118, 146, 60), "light": (214, 170, 20), "moderate": (227, 109, 38), + "severe": (200, 45, 45), "extreme": (120, 20, 30)} +NO_DATA_COLOR = (140, 147, 155) +ERROR_COLOR = (125, 60, 152) # a block that could not run (not a severity) + +STATUS_TEXT = {"crossed": "Crossed", "not_crossed": "Not crossed", "not_applied": "Not applied", + "not_recorded": "Not recorded", "info": "-"} +STATUS_COLOR = {"crossed": (200, 45, 45), "not_crossed": GREEN, "not_applied": MUTED, + "not_recorded": MUTED, "info": MUTED} +RESULT_TEXT = {"event": "Event", "no_event": "No event", "no_data": "No data", + "not_run": "Not available yet", "error": "Error"} + +LEFT, WIDTH = 16, 178 # page margin and content width (A4 = 210 mm) + + +def _dt(iso, with_year=False): + if not iso: + return "-" + t = datetime.fromisoformat(iso) + return t.strftime("%d %b %Y %H:%M" if with_year else "%d %b %H:%M") + + +def _day(iso): + return date.fromisoformat(iso).strftime("%d %B %Y") + + +def _val(x, unit): + return "-" if x is None else f"{x:g} {unit}".strip() + + +def _period(b): + """Period a block checked: its label for monthly blocks ("Jul–Sep 2026"), else start → end.""" + w = b.get("window") + if not w: + return "-" + return w.get("label") or f"{_dt(w['start'])} → {_dt(w['end'])}" + + +def _missing(b, sep="\n"): + """Missing sensors of one event, e.g. 'wind'. No counts of other stations.""" + return sep.join(m["name"] for m in b["missing_sensors"]) or "none" + + +def _suspected(field): + """Snow (later hail): possible from the weather data, a photo is asked for to confirm.""" + return bool(field and field.get("suspected")) + + +def _sev_color(field): + if _suspected(field): + return ORANGE + if field["status"] == "event": + return SEVERITY_COLOR[field["severity"]] + return SEVERITY_COLOR["none"] if field["status"] == "no_event" else NO_DATA_COLOR + + +class _Doc(FPDF): + def __init__(self, content): + super().__init__(orientation="P", unit="mm", format="A4") + self.c = content + self.set_margins(LEFT, 16, LEFT) + self.set_auto_page_break(True, margin=20) + self.add_font("Noto", "", FONTS / "NotoSans-Regular.ttf") + self.add_font("Noto", "B", FONTS / "NotoSans-Bold.ttf") + self.add_font("Noto", "I", FONTS / "NotoSans-Italic.ttf") + self.add_font("DejaVu", "", FONTS / "DejaVuSans.ttf") + self.set_fallback_fonts(["DejaVu"], exact_match=False) + st = content["station"] + self.set_title(f"Weather damage report - station {st['id']} {st['name']} - {content['report_day']}") + self.set_author("Damage Engine") + self.set_creator("Damage Engine (fpdf2)") + + # every page after the first gets a small header + def header(self): + if self.page_no() == 1: + return + st = self.c["station"] + self.image(LOGO, LEFT, 8, h=8) + self.font(8, color=MUTED) + self.set_xy(LEFT + 10, 9.5) + self.cell(WIDTH - 10, 5, f"Weather damage report · {st['name']} (station {st['id']}) · " + f"{_day(self.c['report_day'])}", align="R") + self._brand_bar(18, 0.6) + self.set_y(24) + + def footer(self): + self.set_y(-14) + self.set_draw_color(*LINE) + self.line(LEFT, self.get_y(), LEFT + WIDTH, self.get_y()) + self.font(7.5, color=MUTED) + self.set_y(-12) + st = self.c["station"] + self.cell(WIDTH / 2, 5, f"Data: 3A weather station {st['id']} {st['name']} · " + f"generated {_dt(self.c['created_at'], True)}") + self.cell(WIDTH / 2, 5, f"Page {self.page_no()} of {{nb}}", align="R") + + # ── helpers ── + def font(self, size, style="", color=INK): + self.set_font("Noto", style, size) + self.set_text_color(*color) + + def _brand_bar(self, y, h): + for i, col in enumerate((BLUE, GREEN, ORANGE)): + self.set_fill_color(*col) + self.rect(LEFT + i * WIDTH / 3, y, WIDTH / 3, h, style="F") + + def _label(self, x, y, w, label, value, size=11): + self.set_xy(x, y) + self.font(7, "B", MUTED) + self.cell(w, 4, label.upper(), new_x=XPos.LEFT, new_y=YPos.NEXT) + self.font(size, "B") + self.multi_cell(w, 5.2, value) + + def _chip(self, x, y, text, color, w=None): + self.font(8, "B", (255, 255, 255)) + w = w or self.get_string_width(text) + 6 + self.set_fill_color(*color) + self.rect(x, y, w, 5.6, style="F", round_corners=True, corner_radius=1.4) + self.set_xy(x, y + 0.2) + self.cell(w, 5.2, text, align="C") + return w + + def multi_cell(self, *args, **kw): + kw.setdefault("align", "L") # fpdf2 justifies by default: wide gaps + return super().multi_cell(*args, **kw) + + def table(self, *args, **kw): + self.set_fill_color(255, 255, 255) + return super().table(*args, **kw) + + def _ensure(self, h): + if self.will_page_break(h): + self.add_page() + + def _heading(self, text, color=BLUE): + self._ensure(30) + self.ln(3) + y = self.get_y() + self.set_fill_color(*color) + self.rect(LEFT, y + 1, 1.6, 6, style="F") + self.set_xy(LEFT + 4, y) + self.font(13, "B") + self.cell(WIDTH - 4, 8, text, new_x=XPos.LMARGIN, new_y=YPos.NEXT) + self.ln(1) + + # ── page 1 top ── + def title_block(self): + c, st = self.c, self.c["station"] + self.image(LOGO, LEFT, 13, h=17) + self.set_xy(LEFT + 21, 14) + self.font(18, "B") + self.cell(100, 8, "Weather Damage Report", new_x=XPos.LEFT, new_y=YPos.NEXT) + self.font(9, color=MUTED) + self.cell(100, 5, "Automatic assessment from 3A weather-station data") + self.set_xy(LEFT + WIDTH - 70, 14) + self.font(7, "B", MUTED) + self.cell(70, 4, "REPORT DAY", align="R", new_x=XPos.LEFT, new_y=YPos.NEXT) + self.font(12, "B") + self.cell(70, 6, _day(c["report_day"]), align="R", new_x=XPos.LEFT, new_y=YPos.NEXT) + self.font(8, color=MUTED) + self.cell(70, 5, f"Generated {_dt(c['created_at'], True)}", align="R") + self._brand_bar(34, 1.1) + + # station card + y = 40 + self.set_fill_color(*PANEL) + self.rect(LEFT, y, WIDTH, 22, style="F", round_corners=True, corner_radius=2) + windows = [b["window"] for b in c["blocks"] # monthly blocks (drought): own period + if b.get("window") and not b.get("monthly")] + months = [_period(b) for b in c["blocks"] if b.get("window") and b.get("monthly")] + period = (f"{_dt(min(w['start'] for w in windows))} → {_dt(max(w['end'] for w in windows))}" + if windows else (months[0] if months else "-")) + col = (WIDTH - 10) / 2 + self._label(LEFT + 5, y + 4, col, "Station", f"{st['name']} · ID {st['id']}") + self._label(LEFT + 5 + col, y + 4, col, "Period checked", period) + self.set_y(y + 26) + + # headline + worst = c["overall"]["worst_severity"] + notices = c["overall"].get("notices", []) + suspected = c["overall"].get("suspected", []) + if c["overall"]["events"]: + color = SEVERITY_COLOR[worst] + elif notices or suspected: + color = ORANGE + else: + color = SEVERITY_COLOR["none"] if "No damaging" in c["overall"]["headline"] else NO_DATA_COLOR + y = self.get_y() + self.set_xy(LEFT + 6, y + 1) + self.font(7, "B", MUTED) + self.cell(100, 4, "RESULT", new_x=XPos.LEFT, new_y=YPos.NEXT) + self.font(12, "B", color) + self.multi_cell(WIDTH - 6, 6.5, c["overall"]["headline"], new_x=XPos.LMARGIN, new_y=YPos.NEXT) + lines = [f"{n['title']}: {n['text']}" for n in notices] # a past event to claim (rain row 3) + lines += [s["text"] for s in suspected] # a photo asked for ("Snow likely: ...") + for line in lines: + self.set_x(LEFT + 6) + self.font(9, "B", ORANGE) + self.multi_cell(WIDTH - 6, 4.8, line, new_x=XPos.LMARGIN, new_y=YPos.NEXT) + self.set_fill_color(*color) + self.rect(LEFT, y, 2.2, max(self.get_y() - y, 12), style="F") + errors = c["overall"].get("errors", []) + if errors: + self.set_x(LEFT + 6) + self.font(8.5, "B", ERROR_COLOR) + self.multi_cell(WIDTH - 6, 4.6, f"{len(errors)} check(s) could not run: " + + ", ".join(e["title"] for e in errors) + + " (see \"Checks that could not run\").", + new_x=XPos.LMARGIN, new_y=YPos.NEXT) + self.ln(4) + + def overview(self): + self._heading("Overview") + self.set_draw_color(*LINE) + self.font(8) + head = FontFace(emphasis="BOLD", color=MUTED, fill_color=PANEL, size_pt=7) + several = len(self.c["fields"]) > 1 # one field per station: no field column + widths = (30, 28, 40, 26, 16, 12, 26) if several else (32, 44, 28, 16, 12, 46) + heads = ("PHENOMENON", "FIELD", "PERIOD CHECKED", "RESULT", "SEVERITY", "HOURS", + "MISSING SENSORS *") if several else ( + "PHENOMENON", "PERIOD CHECKED", "RESULT", "SEVERITY", "HOURS", "MISSING SENSORS *") + align = ["LEFT"] * len(heads) + align[-2] = "RIGHT" + with self.table(width=WIDTH, col_widths=widths, line_height=4.8, text_align=align, + headings_style=head, borders_layout="HORIZONTAL_LINES", padding=1.4, + cell_fill_color=ZEBRA, cell_fill_mode="ROWS") as t: + row = t.row() + for h in heads: + row.cell(h) + for b in self.c["blocks"]: + period = _period(b) + if b.get("watch_window"): # Excessive rain: also the last 4 days + w = b["watch_window"] + period += f"\n{w['label']}:\n{_dt(w['start'])} → {_dt(w['end'])}" + for f in b["fields"] or [None]: + row = t.row() + row.cell(b["title"], style=FontFace(emphasis="BOLD")) + if several: + row.cell(f["field_name"] if f else "-") + row.cell(period) + status = f["status"] if f else b["status"] + color = _sev_color(f) if f else ( + ERROR_COLOR if b["status"] == "error" else NO_DATA_COLOR) + result = RESULT_TEXT[status] + if b.get("notice") and status == "no_event": # past event found (rain row 3) + result, color = "No new event\n" + b["notice"]["short"], ORANGE + if _suspected(f): # snow: photo asked for + result = "Suspected\nphoto requested" + row.cell(result, style=FontFace(emphasis="BOLD", color=color)) + sev = (f["severity"].capitalize() if f and f["status"] == "event" + and not _suspected(f) else "-") + row.cell(sev, style=FontFace(emphasis="BOLD", color=color)) + row.cell((f.get("duration_text") or f"{f['duration_h']:g} h") + if f and f["status"] == "event" else "-") + row.cell(_missing(b), style=FontFace(color=ORANGE if b["missing_sensors"] else MUTED, + size_pt=7.5)) + self.font(7, "I", MUTED) + self.multi_cell(WIDTH, 3.8, "* Sensors the check uses that this station does not have. " + "\"Not available yet\": this check has not produced results yet. " + "\"Error\": the check could not run (see \"Checks that could not " + "run\").", + new_x=XPos.LMARGIN, new_y=YPos.NEXT) + self.ln(2) + + # ── one section per block ── + def block(self, b): + color = BLUE + if any(f["status"] == "event" for f in b["fields"]): + color = _sev_color(max((f for f in b["fields"] if f["status"] == "event"), + key=lambda f: f["severity_score"])) + self._ensure(60) # keep the heading with its result and table + self._heading(b["title"], color) + self.font(8.5, color=MUTED) + when = _period(b) + stale = " (latest run available)" if b.get("stale") else "" + self.multi_cell(WIDTH, 4.6, f"Period checked: {when} · run of {_day(b['run_day'])}{stale}", + new_x=XPos.LMARGIN, new_y=YPos.NEXT) + if b["missing_sensors"]: + self.set_text_color(*ORANGE) + self.multi_cell(WIDTH, 4.6, "Sensors missing at this station: " + _missing(b, ", "), + new_x=XPos.LMARGIN, new_y=YPos.NEXT) + self.ln(1) + for f in b["fields"]: + self._field_result(f, len(b["fields"]) > 1) + if b.get("notice"): + self.font(9.5, "B", ORANGE) + self.multi_cell(WIDTH, 5, b["notice"]["text"], new_x=XPos.LMARGIN, new_y=YPos.NEXT) + self.ln(2) + if b["checks"]: + self._checks(b["checks"]) + if b["data_notes"]: + self._notes(b["data_notes"]) + + def _field_result(self, f, show_name): + self._ensure(20) + x, y = LEFT, self.get_y() + suspected = _suspected(f) + w = self._chip(x, y, "SUSPECTED" if suspected else RESULT_TEXT[f["status"]].upper(), _sev_color(f)) + if suspected: + w += 2 + self._chip(x + w + 2, y, "PHOTO REQUESTED", _sev_color(f)) + elif f["status"] == "event": + w += 2 + self._chip(x + w + 2, y, f"SEVERITY {f['severity'].upper()}", _sev_color(f)) + self.set_xy(x + w + 3, y) + self.font(9.5, "B") + self.cell(WIDTH - w - 3, 5.6, f["field_name"] if show_name else "", + new_x=XPos.LMARGIN, new_y=YPos.NEXT) + self.ln(1.5) + self.font(10) + self.multi_cell(WIDTH, 5.2, f["summary"], new_x=XPos.LMARGIN, new_y=YPos.NEXT) + lines = [] + if suspected: + req = f["camera_request"] + lines.append(("Suspect hours", f"first {_dt(f['start'])}, last {_dt(f['end'])} · " + f"{f['duration_h']:g} h with a sign")) + lines.append(("Photo", f"requested for {_dt(req['photo_at'])} · {req['level']} " + f"(signs {', '.join(req['signals'])})")) + lines.append(("Severity", f["severity_basis"])) + elif f["status"] == "event" and f.get("period_text"): # monthly block (drought) + lines.append(("Period", f["period_text"])) + lines.append(("Severity", f"{f['severity'].capitalize()} (score {f['severity_score']:.2f} " + f"of 1) · {f['severity_basis']}")) + if f["severity_scale"]: + lines.append(("Scale", f["severity_scale"])) + elif f["status"] == "event": + lines.append(("Event hours", f"first {_dt(f['start'])}, last {_dt(f['end'])} · " + f"{f['duration_h']:g} h with event conditions")) + lines.append(("Severity", f"{f['severity'].capitalize()} (score {f['severity_score']:.2f} " + f"of 1) · {f['severity_basis']}")) + if f["severity_scale"]: + lines.append(("Scale", f["severity_scale"])) + if f["status"] == "event": + cover = {True: "inside the contract cover period", False: "outside the contract cover period", + None: "unknown (crop stage not recorded for this field)"}[f["in_cover_window"]] + cover = f.get("cover_text") or cover + lines.append(("Cover", cover)) + for k, v in lines: + self.font(8.5, "B", MUTED) + self.cell(22, 4.8, k) + self.font(8.5) + self.multi_cell(WIDTH - 22, 4.8, v, new_x=XPos.LMARGIN, new_y=YPos.NEXT) + self.ln(2) + + def _checks(self, checks): + self._ensure(12 + 7 * len(checks)) + self.font(7.5, "B", MUTED) + self.cell(WIDTH, 5, "MEASURED VALUES AGAINST THRESHOLDS", new_x=XPos.LMARGIN, new_y=YPos.NEXT) + self.set_draw_color(*LINE) + self.font(8.5) + head = FontFace(emphasis="BOLD", color=MUTED, fill_color=PANEL, size_pt=7.5) + with self.table(width=WIDTH, col_widths=(50, 44, 22, 24, 38), line_height=5, + text_align=("LEFT", "LEFT", "RIGHT", "LEFT", "LEFT"), + headings_style=head, borders_layout="HORIZONTAL_LINES", padding=1.4, + cell_fill_color=ZEBRA, cell_fill_mode="ROWS") as t: + row = t.row() + for h in ("CHECK", "THRESHOLD", "MEASURED", "RESULT", "MARGIN"): + row.cell(h) + for ch in checks: + row = t.row() + name = ch["check"] + (f"\n{ch['note']}" if ch["note"] else "") + row.cell(name) + row.cell(ch["threshold_text"]) + row.cell(_val(ch["measured"], ch["unit"]), style=FontFace(emphasis="BOLD")) + row.cell(STATUS_TEXT[ch["status"]], + style=FontFace(emphasis="BOLD", color=STATUS_COLOR[ch["status"]])) + margin = "-" + if ch["margin"] is not None: + margin = (f"past by {ch['margin']:g} {ch['unit']}" if ch["status"] == "crossed" + else f"{ch['margin']:g} {ch['unit']} away") + row.cell(margin) + self.font(7, "I", MUTED) + self.multi_cell(WIDTH, 3.8, "Measured = the extreme of the period (lowest / highest). " + "\"Not recorded\": the value is kept only for event hours. " + "\"Not applied\": the station has no sensor or data for this check.", + new_x=XPos.LMARGIN, new_y=YPos.NEXT) + self.ln(2) + + def _notes(self, notes): + self._ensure(10 + 5 * len(notes)) + y = self.get_y() + self.set_x(LEFT + 4) + self.font(7.5, "B", MUTED) + self.cell(WIDTH - 4, 5, "DATA BASIS", new_x=XPos.LMARGIN, new_y=YPos.NEXT) + self.font(8, color=MUTED) + for n in notes: + self.set_x(LEFT + 4) + self.multi_cell(WIDTH - 6, 4.3, f"• {n}", new_x=XPos.LMARGIN, new_y=YPos.NEXT) + self.set_fill_color(*LINE) + self.rect(LEFT, y + 1, 0.8, self.get_y() - y - 1, style="F") + self.ln(3) + + def errors(self): + """One table with every block that could not run and why.""" + failed = [b for b in self.c["blocks"] if b["status"] == "error"] + if not failed: + return + self._ensure(30 + 9 * len(failed)) + self._heading("Checks that could not run", ERROR_COLOR) + self.font(8.5, color=MUTED) + self.multi_cell(WIDTH, 4.6, "These checks gave no result for this report. The other " + "checks are not affected.", new_x=XPos.LMARGIN, new_y=YPos.NEXT) + self.ln(1) + self.set_draw_color(*LINE) + self.font(8.5) + head = FontFace(emphasis="BOLD", color=MUTED, fill_color=PANEL, size_pt=7.5) + with self.table(width=WIDTH, col_widths=(40, 26, 112), line_height=4.8, + text_align=("LEFT", "LEFT", "LEFT"), headings_style=head, + borders_layout="HORIZONTAL_LINES", padding=1.4, + cell_fill_color=ZEBRA, cell_fill_mode="ROWS") as t: + row = t.row() + for h in ("CHECK", "RUN OF", "PROBLEM"): + row.cell(h) + for b in failed: + row = t.row() + row.cell(b["title"], style=FontFace(emphasis="BOLD")) + row.cell(_day(b["run_day"])) + row.cell(b["error"], style=FontFace(color=ERROR_COLOR)) + self.ln(2) + + def closing(self): + self._ensure(30) + self.ln(4) + self.set_draw_color(*LINE) + self.line(LEFT, self.get_y(), LEFT + WIDTH, self.get_y()) + self.ln(2) + self.font(7.5, "B", MUTED) + self.cell(WIDTH, 5, "ABOUT THIS REPORT", new_x=XPos.LMARGIN, new_y=YPos.NEXT) + self.font(7.5, color=MUTED) + self.multi_cell(WIDTH, 4, ( + "This report is produced automatically from the hourly data of your 3A weather station. " + "Thresholds come from the crop insurance contract (UnipolSai Model 1506) and from " + "agronomic literature; where neither gives a number, a project default is used. " + "The severity estimates the likely damage from the weather alone: it is not a " + "field inspection or a loss assessment. All times are local station time."), + new_x=XPos.LMARGIN, new_y=YPos.NEXT) + + +def render(content: dict, path: Path) -> Path: + """Write the PDF of one station report and return its path.""" + pdf = _Doc(content) + pdf.add_page() + pdf.title_block() + pdf.overview() + for b in content["blocks"]: + if b["status"] not in ("not_run", "error"): # no results: overview (+ errors) only + pdf.block(b) + pdf.errors() + pdf.closing() + path = Path(path) + path.parent.mkdir(parents=True, exist_ok=True) + pdf.output(str(path)) + return path diff --git a/src/damage_engine/report/station_report.py b/src/damage_engine/report/station_report.py new file mode 100644 index 0000000..3d14dff --- /dev/null +++ b/src/damage_engine/report/station_report.py @@ -0,0 +1,900 @@ +""" +Station report: one report per station and day, for the client of that station +(each station = one client; a client never sees another station's report). +It lists EVERY event of Diagram.png (EVENTS, also blocks not written yet) in ONE +report and, for each check, shows the measured value, the threshold, whether it +was crossed and by how much. A block appears with its own section as soon as it +stores a run (report.submit); a block without a check function here gets a +generic table of its numeric details. For every event the report also says which +of its sensors this station lacks (nothing about other stations). + +Two files per station and day (a new run replaces that day's report only; the last +REPORT["history_runs"] = 10 days are kept): + + outputs/station_reports//.json content (the same data as the PDF) + outputs/station_reports//.pdf the client's report (station_pdf.py) + +The content is built here from the stored block runs (report.py) and the +thresholds (thresholds.py, read only). station_pdf.py only draws it, so the PDF +engine can change without touching this file. + +Limits: the blocks store only extremes and totals, not hour-by-hour values, so a +check shows the night/day extreme (e.g. lowest air temperature, wettest hour) and +the event's first and last hour. Values a block does not store are "not recorded". + + python -m damage_engine.report.station_report --station 566 # latest blocks + python -m damage_engine.report.station_report --station 566 --date 2026-10-20 +""" + +import argparse +import json +import operator +from datetime import date, datetime, time, timedelta +from pathlib import Path + +from damage_engine.report import report +from damage_engine.schemas import EVENT, NO_DATA, NO_EVENT, Analysis, Severity, camera_request +from damage_engine.sources.stations import all_stations, station_by_id +from damage_engine.thresholds import ( + BRINA, + BRINA_SCHEDULE, + DROUGHT, + DROUGHT_BANDS_PCT, + EXCESSIVE_RAIN, + EXCESSIVE_RAIN_BANDS_RATIO, + FLOOD_SCHEDULE, + FROST_FALLBACK_BANDS_C, + FROST_KILL_BANDS, + GELO, + HAIL, + HAIL_SCHEDULE, + HIGH_TEMPERATURE_SCHEDULE, + HOT_TEMPERATURE, + HOT_TEMPERATURE_FST_BANDS_C, + HOT_WIND, + HOT_WIND_BANDS_C, + HOT_WIND_SCHEDULE, + REPORT, + SNOW, + SNOW_SCHEDULE, + STRONG_WIND, + STRONG_WIND_BANDS_MS, + STRONG_WIND_SCHEDULE, +) + +STATION_REPORTS_DIR = report.REPORTS_DIR.parent / "station_reports" + +# check status +CROSSED = "crossed" # the measured value reached the threshold +NOT_CROSSED = "not_crossed" +NOT_APPLIED = "not_applied" # the station has no sensor / no data for this check +NOT_RECORDED = "not_recorded" # the block does not store this value (nights/days without event) +INFO = "info" # value shown for context, no threshold + +ERROR = "error" # block status: the block could not run (report.submit_error) + +OPS = {"<=": operator.le, "<": operator.lt, ">=": operator.ge, ">": operator.gt} +SYMBOL = {"<=": "≤", "<": "<", ">=": "≥", ">": ">"} + +# Every event of Diagram.png, in diagram order: block name (as given to report.submit) +# -> title and the 3A sensors it reads (Diagram.png inputs; Brina also uses solar +# radiation, see its header). Wet-bulb is not a sensor here: it is calculated from +# temperature and humidity. Hail also lists wind (its gust sign, see detectors/hail.py). +# sensors=None would mean "All sensors" (the diagram's drought +# input); Drought lists the sensors it reads: rain, and soil moisture for information +# (its ET0, EvapoPot, is a daily value calculated by 3A at every station, not a sensor). +_T, _H, _W, _S, _R = "TemperaturaAria", "UmiditaRelativa", "VelocitaVento2m", "RadiazioneSolare", "Pioggia" +EVENTS = { + "brina": {"title": "Brina (hoar frost)", "sensors": (_T, _H, _W, _S)}, + "strong_wind": {"title": "Strong wind (storm)", "sensors": (_W,)}, + "freezing": {"title": "Freezing (gelo)", "sensors": (_T, _S, _H, _W)}, + "flood": {"title": "Excessive rain", "sensors": (_R,)}, + "snow": {"title": "Snow", "sensors": (_R, _T, _S, _H)}, + "hail": {"title": "Hail", "sensors": (_T, _R, _W)}, + "hot_wind": {"title": "Hot wind", "sensors": (_W, _T, _H)}, + "high_temperature": {"title": "High temperature", "sensors": (_H, _S, _T)}, + "drought": {"title": "Drought", "sensors": (_R, "Suolo1", "Suolo2")}, +} +# Extra runs of a block stored as their own block, shown inside it in the client report: Hail's +# daytime checks on hail-risk days (detectors/hail.py DAYTIME_BLOCK, slots "13h", "19h"). +DAYTIME_BLOCK_OF = {"hail": "hail_daytime"} +CALCULATED_SENSORS = {"TBulb"} # 3A's wet-bulb, calculated by 3A: never "missing" +SENSOR_NAMES = { + "TemperaturaAria": "air temperature", "UmiditaRelativa": "humidity", "VelocitaVento2m": "wind", + "RadiazioneSolare": "solar radiation", "Pioggia": "rain gauge", "BagnaturaFogliare": "leaf wetness", + "Pressione": "air pressure", "Suolo1": "soil moisture 1", "Suolo2": "soil moisture 2", + "TemperaturaTerreno": "soil temperature", "TemperaturaTerreno2": "soil temperature 2", +} +HIDDEN_NOTES = ("5-year reference average",) # data notes not shown to the client +CRITERIA = {"downpour_1h": "rain in 1 hour", "intense_72h": "rain in 72 hours", + "prolonged_10d": "rain in 10 days"} + + +def _num(x): + return f"{x:g}" if isinstance(x, (int, float)) else str(x) + + +def _check(name, measured, unit, op=None, threshold=None, *, status=None, + threshold_text=None, note=""): + """One row: measured value against a threshold; status computed when not given.""" + if status is None: + if op is None: + status = INFO + elif measured is None: + status = NOT_RECORDED + else: + status = CROSSED if OPS[op](measured, threshold) else NOT_CROSSED + margin = None + if status in (CROSSED, NOT_CROSSED) and measured is not None and threshold is not None: + margin = round(abs(measured - threshold), 2) + if threshold_text is None: + threshold_text = f"{SYMBOL[op]} {_num(threshold)} {unit}" if op else "-" + return {"check": name, "measured": measured, "unit": unit, "operator": op, + "threshold": threshold, "threshold_text": threshold_text, + "status": status, "margin": margin, "note": note} + + +# ─────────────────────────── per-block checks ─────────────────────────── + +def _brina_checks(a: Analysis, station) -> list[dict]: + d, event = a.details, a.status == EVENT + has_solar = station is not None and station.has("RadiazioneSolare") + wind_ok = bool(d.get("wind_checked")) + seen = CROSSED if event else NOT_RECORDED # night held in every brina hour + max_wind = d.get("max_wind_ms") + offset = BRINA["radiative_offset_c"] + wet_note = (f"3A's own wet-bulb (from the hourly mean): {d['min_wet_bulb_3a_c']} °C" + if "min_wet_bulb_3a_c" in d else "") + return [ + _check("Darkness (solar radiation)", None, "W/m²", "<=", BRINA["night_solar_max_wm2"], + status=seen if has_solar else NOT_APPLIED, + note="" if has_solar else "No pyranometer: every hour of the night counts as dark."), + _check("Highest wind in brina hours", max_wind, "m/s", + status=INFO if max_wind is not None else (NOT_RECORDED if wind_ok else NOT_APPLIED), + threshold_text=f"≤ {BRINA['calm_wind_max_ms']:g} m/s = calm", + note=(f"Calm hours: bud {offset:g} °C below the air and ice test. Windier hours: " + f"bud = air, air below 0 °C instead of the ice test." if wind_ok + else f"No wind sensor or data: calm assumed, bud {offset:g} °C below the air " + f"(worst case).")), + _check("Lowest air temperature", d.get("night_min_temp_c"), "°C", "<=", BRINA["t_air_max_c"]), + _check("Lowest wet-bulb temperature", d.get("min_wet_bulb_c"), "°C", "<=", + BRINA["t_wet_max_c"], note=wet_note), + _check("Lowest bud temperature", d.get("min_bud_temp_c"), "°C", "<=", 0.0, + threshold_text="calm: ≤ 0 °C and ≤ frost point; windy: < 0 °C", + note=f"Bud = air − {offset:g} °C on calm hours, = air on windier hours."), + _check("Hours with brina conditions", a.duration_h, "h", ">=", BRINA["min_duration_h"]), + ] + + +def _freezing_checks(a: Analysis, station) -> list[dict]: + d, event, g = a.details, a.status == EVENT, GELO + run_h, night_h = BRINA_SCHEDULE["run_hour"], BRINA_SCHEDULE["night_start_hour"] + fixed = f"{run_h + 1:02d}:00-{night_h - 1:02d}:00" + solar_ok = bool(d.get("solar_checked")) + trigger = f"{g['t_air_trigger_c']:g} °C" + wind_status = None if d.get("mean_wind_ms") is not None else (NOT_APPLIED if event else NOT_RECORDED) + return [ + _check("Sunlit hours of the night (solar radiation)", d.get("sunlit_hours_in_night_window"), + "h", status=INFO if solar_ok else NOT_APPLIED, + threshold_text=f"> {BRINA['night_solar_max_wm2']:g} W/m²", + note=(f"Night hours with sun, checked here as daylight (day hours {fixed} always are)." + if solar_ok else f"No pyranometer: daylight = the day hours {fixed} only.")), + _check("Lowest daylight air temperature", d.get("day_min_temp_c"), "°C", + g["t_air_trigger_op"], g["t_air_trigger_c"]), + _check(f"Daylight hours below {trigger}", a.duration_h, "h", ">=", g["min_duration_h"]), + _check(f"Mean wind while below {trigger}", d.get("mean_wind_ms"), "m/s", ">=", + g["advective_wind_min_ms"], status=wind_status, + note=("For information: wind points to a cold air mass; not needed in daylight." + if wind_status is None or not event else "No wind sensor or data.")), + _check(f"Lowest wet-bulb while below {trigger}", d.get("min_wet_bulb_c"), "°C", + status=INFO if d.get("min_wet_bulb_c") is not None else NOT_RECORDED, + note="For information: how cold wet buds (rain, dew) could get."), + ] + + +def _flood_checks(a: Analysis, station) -> list[dict]: + d, met = a.details, set(a.details.get("criteria_met", [])) + r = EXCESSIVE_RAIN + rows = [_check("Rain in 1 hour (wettest hour)", d.get("max_1h_mm"), "mm", + r["downpour_1h_op"], r["downpour_1h_mm"])] + rows.append(_last_72h_check(d, met)) # row 2: the 72 h ending at the run hour + rows.append(_last_heavy_72h_check(d, a.run_day)) # row 3: for claims + text_10d =(f"{SYMBOL[r['prolonged_10d_ratio_op']]} {r['prolonged_10d_ratio']:g} × 5-year " + f"average and {SYMBOL[r['prolonged_10d_min_op']]} {r['prolonged_10d_min_mm']:g} mm") + ref = d.get("reference_10d_mm") + if ref is None: # criterion not applied: shown as a plain total + rows.append(_check("Rain in the last 10 days", d.get("rain_10d_mm"), "mm")) + else: + trigger = max(r["prolonged_10d_ratio"] * ref, r["prolonged_10d_min_mm"]) + rows.append(_check("Rain in 10 days", d.get("rain_10d_mm"), "mm", ">", trigger, + status=CROSSED if "prolonged_10d" in met else NOT_CROSSED, + threshold_text=f"{text_10d} (= {trigger:.1f} mm)")) + rows.append(_check("Rain in the event day (24 h)", d.get("rain_day_mm"), "mm")) + return rows + + +WIND_VALUE = {"hourly_max": "highest wind of the hour", "hourly_mean": "mean wind of the hour"} + + +def _strong_wind_checks(a: Analysis, station) -> list[dict]: + d, w = a.details, STRONG_WIND + op, ms = w["wind_trigger_op"], w["wind_trigger_ms"] + kmh = f"{ms * 3.6:.0f} km/h" + value = WIND_VALUE.get(d.get("trigger"), "wind") + top = d.get("max_wind_ms") + note = f"{SYMBOL[op]} {kmh}, Beaufort 7 (contract). Value used: {value}." + if d.get("max_wind_at") and top is not None: + note = f"At {_hour(d['max_wind_at'])}, {d.get('max_wind_kmh'):g} km/h. " + note + if d.get("fallback_hours"): + note += f" Hourly mean used for {d['fallback_hours']} hour(s) without a maximum." + minutes = a.duration_h * 60 + held = CROSSED if minutes >= w["min_duration_min"] else NOT_CROSSED + mean = d.get("max_mean_wind_ms") + return [ + _check("Highest wind in the 24 h", top, "m/s", op, ms, note=note), + _check(f"Time with wind {SYMBOL[op]} {ms:g} m/s", a.duration_h, "h", status=held, + threshold_text=f"≥ {w['min_duration_min']:g} min", + note="One hourly record counts as 60 min."), + _check("Highest hourly mean wind", mean, "m/s", status=INFO if mean is not None else NOT_RECORDED, + note="For information: the mean of the hour, not used to detect." + if d.get("trigger") != "hourly_mean" else "The value used to detect."), + ] + + +def _hot_wind_checks(a: Analysis, station) -> list[dict]: + d, w = a.details, HOT_WIND + op, deg = w["t_max_trigger_op"], w["t_max_trigger_c"] + wind_min = w["wind_min_ms"] + note = f"{SYMBOL[op]} {deg:g} °C at 2 m (contract). Value used: highest temperature of the hour." + if d.get("max_temp_at") and d.get("max_temp_c") is not None: + note = f"At {_hour(d['max_temp_at'])}. " + note + if d.get("fallback_hours"): + note += f" Hourly mean used for {d['fallback_hours']} hour(s) without a maximum." + hours = d.get("hot_wind_hours", 0) + hours_note = f"Moving air = mean wind ≥ {wind_min:g} m/s (project value; the contract gives no speed)." + if d.get("still_hot_hours"): + hours_note += f" {d['still_hot_hours']} hour(s) at ≥ {deg:g} °C had still air." + if d.get("wind_assumed_hours"): + hours_note += f" Air taken as moving in {d['wind_assumed_hours']} hour(s) without wind data." + wind = d.get("wind_at_max_temp_ms") + if wind is None: + wind_row = _check("Wind in the hottest hour", None, "m/s", ">=", wind_min, status=NOT_APPLIED, + note="No wind data: air taken as moving (worst case).") + else: + wind_row = _check("Wind in the hottest hour", wind, "m/s", ">=", wind_min, + note="Mean wind of the hour (project value; the contract gives no speed).") + rh = d.get("humidity_at_max_temp_pct") + lo, hi = w["dir_from_min_deg"], w["dir_from_max_deg"] + return [ + _check("Highest air temperature in the 24 h", d.get("max_temp_c"), "°C", op, deg, note=note), + _check(f"Hours at {SYMBOL[op]} {deg:g} °C with moving air", hours, "h", + status=CROSSED if a.status == EVENT else NOT_CROSSED, threshold_text="≥ 1 h", + note=hours_note), + wind_row, + _check("Wind from south-east to south-west", None, "", status=NOT_APPLIED, + threshold_text=f"{lo:g}°–{hi:g}°", + note="No 3A station measures wind direction: not checked."), + _check("Humidity in the hottest hour", rh, "%", status=INFO if rh is not None else NOT_RECORDED, + note="For information: the lowest humidity of the hour; dry air speeds up withering."), + ] + + +def _high_temperature_checks(a: Analysis, station) -> list[dict]: + d, p = a.details, HOT_TEMPERATURE + op, deg = p["contract_t_max_op"], p["contract_t_max_c"] + note = f"{SYMBOL[op]} {deg:g} °C at 2 m (contract). Value used: highest temperature of the hour." + if d.get("max_temp_at") and d.get("max_temp_c") is not None: + note = f"At {_hour(d['max_temp_at'])}. " + note + if d.get("fallback_hours"): + note += f" Hourly mean used for {d['fallback_hours']} hour(s) without a maximum." + sunburn = HOT_TEMPERATURE_FST_BANDS_C["edges"][0] + formula = (f"Estimated: air + {p['fst_gain_max_c']:g} °C × sun / 1000 W/m² ÷ " + f"(1 + {p['fst_wind_coeff']:g} × wind); sun counted from " + f"{p['daytime_solar_min_wm2']:g} W/m².") + risk_name = f"Sunburn risk hours (air ≥ {p['risk_t_air_c']:g} °C, fruit ≥ {sunburn:g} °C)" + if not d.get("solar_checked"): + no_sun = "No solar radiation data at this station: not estimated." + fst_row = _check("Highest fruit surface temperature", None, "°C", ">=", sunburn, + status=NOT_APPLIED, note=no_sun) + risk_row = _check(risk_name, None, "h", status=NOT_APPLIED, threshold_text="≥ 1 h", note=no_sun) + else: + fst_note = f"Sunburn (browning) starts at {sunburn:g} °C. {formula}" + if d.get("max_fst_at"): + wind = d.get("wind_at_max_fst_ms") + wind_text = f"wind {wind:g} m/s" if wind is not None else "no wind data, calm air taken" + fst_note = (f"At {_hour(d['max_fst_at'])}: sun {d.get('solar_at_max_fst_wm2'):g} W/m², " + f"{wind_text}. " + fst_note) + fst_row = _check("Highest fruit surface temperature", d.get("max_fst_c"), "°C", ">=", sunburn, + note=fst_note) + hours = d.get("sunburn_risk_hours") + risk_row = _check(risk_name, hours, "h", status=CROSSED if hours else NOT_CROSSED, + threshold_text="≥ 1 h", + note=f"For information: below {deg:g} °C air the contract does not cover " + f"sunburn.") + rh, wet = d.get("humidity_at_max_temp_pct"), d.get("wet_bulb_at_max_temp_c") + return [ + _check("Highest air temperature in the 24 h", d.get("max_temp_c"), "°C", op, deg, note=note), + _check(f"Hours at {SYMBOL[op]} {deg:g} °C", d.get("hot_hours", 0), "h", + status=CROSSED if a.status == EVENT else NOT_CROSSED, threshold_text="≥ 1 h"), + fst_row, + risk_row, + _check("Humidity in the hottest hour", rh, "%", status=INFO if rh is not None else NOT_RECORDED, + note="For information: the lowest humidity of the hour."), + _check("Wet-bulb in the hottest hour", wet, "°C", status=INFO if wet is not None else NOT_RECORDED, + note="For information: calculated from temperature and humidity (3A formula)."), + ] + + +def _snow_checks(a: Analysis, station) -> list[dict]: + """Snow: the signs that decide whether a photo is asked for (no station measures snow).""" + d, p = a.details, SNOW + t, hours = p["t_wet_snow_max_c"], p["melt_lookahead_h"] + what = "air temperature" if d.get("cold_basis") == "air" else "wet-bulb" + note = f"Snow is possible at ≤ {t:g} °C (coldest moment of the hour)." + if d.get("min_cold_at"): + note = f"At {_hour(d['min_cold_at'])}. " + note + if d.get("air_fallback_hours"): + note += f" Air temperature used for {d['air_fallback_hours']} hour(s) without humidity." + cold = d.get("cold_hours", 0) + rain_ok, rh_ok, solar_ok = d.get("rain_checked"), d.get("humidity_checked"), d.get("solar_checked") + no_rain = "No rain data at this station: not checked." + if not rain_ok: + a_row = _check("A · Precipitation in cold hours", None, "mm", status=NOT_APPLIED, + threshold_text="> 0 mm", note=no_rain) + b_row = _check(f"B · Rain within {hours:g} h after a cold hour", None, "mm", + status=NOT_APPLIED, threshold_text="> 0 mm", note=no_rain) + else: + a_row = _check("A · Precipitation in cold hours", d.get("snow_water_mm"), "mm", ">", 0.0, + note=f"{d.get('snow_hours', 0)} hour(s). Strong sign: snow probably fell.") + b_row = _check(f"B · Rain within {hours:g} h after a cold hour", d.get("melt_water_mm"), "mm", + ">", 0.0, note=f"{d.get('melt_hours', 0)} hour(s). Medium sign: an unheated " + f"rain gauge records snow only when it melts.") + rh = p["saturated_rh_min_pct"] + c_name = f"C · Cold hours with humidity ≥ {rh:g} % and no rain" + if rh_ok: + c_row = _check(c_name, d.get("saturated_hours"), "h", ">=", 1, threshold_text="≥ 1 h", + note="Weak sign: snow the rain gauge may have missed (or fog, frost).") + else: + c_row = _check(c_name, None, "h", status=NOT_APPLIED, threshold_text="≥ 1 h", + note="No humidity data at this station: not checked.") + d_name = "D · Highest midday sun after the first suspect hour" + lo, hi = p["midday_hours"] + covered = p["covered_solar_max_wm2"] + if not solar_ok: + d_row = _check(d_name, None, "W/m²", status=NOT_APPLIED, threshold_text=f"≤ {covered:g} W/m²", + note="No solar radiation data at this station: not checked.") + else: + val = d.get("midday_solar_max_wm2") + d_row = _check(d_name, val, "W/m²", "<=", covered, + note=(f"Supporting sign only (records {lo:02d}:00-{hi:02d}:00): the solar " + f"sensor may be under snow; an overcast midday is about 50-200 W/m²." + if val is not None else + "No midday hour after a suspect hour in the 24 h (or no suspect hour).")) + est, est_max = d.get("est_new_snow_cm"), d.get("est_new_snow_max_cm") + more = f"; up to {est_max:g} cm counting the rain of sign B" if est_max and est_max != est else "" + est_note = (f"Water of the cold hours × {p['snow_liquid_ratio']:g}{more}. " + f"Information only: the photo decides.") + return [ + _check(f"Lowest {what} in the 24 h", d.get("min_cold_c"), "°C", "<=", t, note=note), + _check(f"Hours at ≤ {t:g} °C (cold enough for snow)", cold, "h", + status=CROSSED if cold else NOT_CROSSED, threshold_text="≥ 1 h"), + a_row, b_row, c_row, d_row, + _check("Estimated new snow", est, "cm", status=INFO if rain_ok else NOT_APPLIED, + note=est_note if rain_ok else no_rain), + ] + + +def _hail_checks(a: Analysis, station) -> list[dict]: + """Hail: the signs of a hailstorm that decide whether a photo is asked for (no station + measures hail). B and C count only within storm_window_h of rain.""" + d, p = a.details, HAIL + near = f"Counts only within {p['storm_window_h']:g} h of rain (hail melts in the rain gauge)." + + def at(key): + return f"At {_hour(d[key])}. " if d.get(key) else "" + + def dry(key): + n = d.get(key, 0) + return f" {n} hour(s) without rain nearby: not counted." if n else "" + + rain_ok = d.get("rain_checked") + if rain_ok: + burst = _check("A · Rain in the wettest hour", d.get("max_rain_1h_mm"), "mm", ">=", + p["rain_1h_mm"], note=at("max_rain_1h_at") + "Burst rain of a storm.") + else: + burst = _check("A · Rain in the wettest hour", None, "mm", status=NOT_APPLIED, + threshold_text=f"≥ {p['rain_1h_mm']:g} mm", + note="No rain data at this station: not checked.") + rate_mmh = p["rain_10min_mm"] * 6 + if d.get("rate_checked"): + rate = _check("A · Highest rain rate", d.get("max_rain_rate_mmh"), "mm/h", ">=", rate_mmh, + note=at("max_rain_rate_at") + f"The rate of {p['rain_10min_mm']:g} mm in " + f"10 minutes.") + else: + rate = _check("A · Highest rain rate", None, "mm/h", status=NOT_APPLIED, + threshold_text=f"≥ {rate_mmh:g} mm/h", + note="This station's rain gauge sends no rain rate: not checked.") + if d.get("temp_checked"): + drop_val = d.get("max_t_drop_c") + drop_val = None if drop_val is None else max(drop_val, 0.0) + note = at("max_t_drop_at") + ("Mean of the hour before minus the lowest of the hour (cold " + "outflow of a storm). " + near + dry("drop_dry_hours")) + if d.get("temp_fallback_hours"): + note += f" Hourly mean used for {d['temp_fallback_hours']} hour(s) without a minimum." + drop = _check("B · Largest temperature drop", drop_val, "°C", ">=", p["t_drop_30min_c"], + note=note) + else: + drop = _check("B · Largest temperature drop", None, "°C", status=NOT_APPLIED, + threshold_text=f"≥ {p['t_drop_30min_c']:g} °C", + note="No air temperature data at this station: not checked.") + if d.get("wind_checked"): + note = at("max_gust_at") + "Highest wind of the hour. " + near + dry("gust_dry_hours") + if d.get("wind_fallback_hours"): + note += f" Hourly mean used for {d['wind_fallback_hours']} hour(s) without a maximum." + gust = _check("C · Highest wind (gust)", d.get("max_gust_ms"), "m/s", ">=", p["gust_ms"], + note=note) + else: + gust = _check("C · Highest wind (gust)", None, "m/s", status=NOT_APPLIED, + threshold_text=f"≥ {p['gust_ms']:g} m/s", + note="No wind data at this station: not checked.") + k = p["min_indicators"] + storm = _check("Signs in one storm", d.get("storm_signs", 0), "", status=INFO, + threshold_text=f"1 = possible, ≥ {k} = likely", + note=f"Signs within {p['storm_window_h']:g} h of each other. Any sign asks for a " + f"photo; the photo decides.") + rain = d.get("rain_day_mm") + total = _check("Rain in the 24 h", rain, "mm", status=INFO if rain_ok else NOT_APPLIED, + note=(f"{d.get('rain_hours', 0)} hour(s) with rain. For information." + if rain_ok else "No rain data at this station.")) + return [burst, rate, drop, gust, storm, total] + + +def _drought_checks(a: Analysis, station) -> list[dict]: + """Drought: rain of the last complete months against the climate average (thresholds.py + RAIN_NORMALS_MM), month by month, plus the water balance and soil moisture (information).""" + d, p = a.details, DROUGHT + op, trigger = p["rain_pct_of_normal_op"], p["rain_pct_of_normal_max"] + label, normal, rain = d.get("window_label", ""), d.get("normal_mm"), d.get("rain_mm") + gap = None if rain is None or normal is None else round(normal - rain, 1) + gap_text = "" if gap is None else (f"{gap:g} mm below the average." if gap > 0 + else f"{-gap:g} mm above the average.") + missing = f" {d['missing_days']} day(s) without data counted as 0 mm." if d.get("missing_days") else "" + rows = [ + _check(f"Rain in the last {d.get('window_months', len(d.get('months', [])))} months ({label})", + rain, "mm", status=INFO, threshold_text=f"average {_num(normal)} mm", + note=f"Average 1991-2020 of these months. {gap_text}{missing}".strip()), + _check("Rain as % of the average", d.get("rain_pct_of_normal"), "%", op, trigger, + threshold_text=(f"{SYMBOL[op]} {trigger:g} % of the average" + + (f" (= {normal * trigger / 100:.1f} mm)" if normal else "")), + note="Drought = much less rain than usual for these months (project value)."), + ] + for m in d.get("months", []): + note = f"{_num(m['rain_pct_of_normal'])} % of the average." + if m.get("missing_days"): + note += f" {m['missing_days']} day(s) without data." + rows.append(_check(f"Rain in {m['label']}", m["rain_mm"], "mm", status=INFO, + threshold_text=f"average {_num(m['normal_mm'])} mm", note=note)) + balance = d.get("water_balance_mm") + if balance is None: + rows.append(_check(f"Water balance: rain − ET0 ({label})", None, "mm", status=NOT_APPLIED, + note="No evapotranspiration data (3A EvapoPot) for these months.")) + else: + some = (f" Only {d['water_balance_months']} month(s) with data." + if d.get("water_balance_months", 0) < len(d.get("months", [])) else "") + rows.append(_check(f"Water balance: rain − ET0 ({label})", balance, "mm", status=INFO, + note=f"For information. ET0 (3A EvapoPot) {_num(d.get('et0_mm'))} mm. Below " + f"0: the crop needed more water than the rain gave (without " + f"irrigation).{some}")) + soil = d.get("soil_moisture") or {} + last = d.get("last_month_label", "the last month") + if soil: + for sensor, v in soil.items(): + rows.append(_check(f"{SENSOR_NAMES.get(sensor, sensor).capitalize()}, mean of {last}", + v["mean_pct"], "%", status=INFO, + note=f"For information. First 7 days {_num(v['first_week_pct'])} %, " + f"last 7 days {_num(v['last_week_pct'])} %.")) + else: + has = station is not None and any(station.has(s) for s in ("Suolo1", "Suolo2")) + rows.append(_check(f"Soil moisture, mean of {last}", None, "%", status=NOT_APPLIED, + note=("No soil moisture data for this month." if has + else "No soil moisture sensor at this station."))) + return rows + + +def _hour(iso: str) -> str: + return datetime.fromisoformat(iso).strftime("%d %b %H:%M") + + +def _last_72h_check(d: dict, met: set) -> dict: + """Rain of the last 72 h (ending at the run hour) against the contract's 72 h threshold. + + Flood's event still uses the contract rule (any 72 h window ending on rain of the event + day), so a note explains the two cases where this row and the event result differ. + """ + op, mm = EXCESSIVE_RAIN["intense_72h_op"], EXCESSIVE_RAIN["intense_72h_mm"] + rain = d.get("rain_72h_to_run_mm") + crossed = rain is not None and OPS[op](rain, mm) + note = "" + if crossed and "intense_72h" not in met: # the last 24 h were dry + note = "No rain in the last 24 h: not a new event (see the Heavy Rain Watch)." + elif not crossed and "intense_72h" in met: # an earlier 72 h window crossed it + note = "The event comes from an earlier 72 h window (see the Heavy Rain Watch)." + return _check("Rain in the last 72 hours", rain, "mm", op, mm, note=note) + + +def _last_heavy_72h_check(d: dict, run_day: date) -> dict: + """72-Hour Heavy Rain Watch (flood details), for claims: the latest 72 h window of the last + claim_search_days that reaches the 72 h threshold, else the wettest one; only the rain of + those days counts. The name carries those days (they follow the run day); the note gives + the first → last rainy hour of the window.""" + r, days = EXCESSIVE_RAIN, FLOOD_SCHEDULE["claim_search_days"] + op, mm = r["intense_72h_op"], r["intense_72h_mm"] + watch = _watch_window("flood", run_day) + name = f"72-Hour Heavy Rain Watch ({_hour(watch['start'])} → {_hour(watch['end'])})" + if "last_heavy_72h" not in d: # run stored before this check existed + return _check(name, None, "mm", op, mm) + heavy, wettest = d["last_heavy_72h"], d.get("wettest_72h") + window = heavy or wettest + if not window: + return _check(name, 0.0, "mm", op, mm, note=f"No rain in the last {days} days.") + first, last = _hour(window["first_rain"]), _hour(window["last_rain"]) + rain = f"Rain {first} → {last}" if first != last else f"Rain at {first}" + earlier = " Rain also fell just before these days (not counted)." if window.get("partial") else "" + if heavy: + return _check(name, heavy["mm"], "mm", op, mm, + note=f"{rain}; {mm:g} mm reached on {_hour(heavy['reached_at'])}.{earlier}") + return _check(name, wettest["mm"], "mm", op, mm, note=f"{rain}.{earlier}") + + +def _generic_checks(a: Analysis, station) -> list[dict]: + """Blocks without a check function yet: their numeric details, without thresholds.""" + return [_check(k.replace("_", " ").capitalize(), v, "") for k, v in a.details.items() + if isinstance(v, (int, float)) and not isinstance(v, bool)] + + +CHECKS = {"brina": _brina_checks, "flood": _flood_checks, "freezing": _freezing_checks, + "strong_wind": _strong_wind_checks, "hot_wind": _hot_wind_checks, + "high_temperature": _high_temperature_checks, "snow": _snow_checks, + "hail": _hail_checks, "drought": _drought_checks} +# Blocks that check whole months, not a day: their period is the label stored by the block +# ("Jul–Sep 2026"), it is left out of the top card's "Period checked", and an event shows its +# period and days instead of hours. +MONTHLY_BLOCKS = {"drought"} + + +def missing_sensors(block: str, station) -> list[dict]: + """Sensors the event reads that this station lacks (only sensors that exist in the 3A + network). The report of a station gives no information about other stations, so no + counts of who has them.""" + known = {sensor for st in all_stations() for sensor in st.sensors} + wanted = EVENTS[block]["sensors"] if block in EVENTS else () + if wanted is None: # "All sensors" + wanted = sorted((x for x in known if x not in CALCULATED_SENSORS), + key=lambda x: SENSOR_NAMES.get(x, x)) + mine = set(station.sensors) if station else set() + return [{"sensor": x, "name": SENSOR_NAMES.get(x, x)} + for x in wanted if x not in mine and x in known] + + +def _window(block: str, run_day: date): + """(start, end) of the period a block looked at.""" + if block == "brina": + return (datetime.combine(run_day - timedelta(days=1), time(BRINA_SCHEDULE["night_start_hour"])), + datetime.combine(run_day, time(BRINA_SCHEDULE["run_hour"]))) + if block == "flood": + end = datetime.combine(run_day, time(FLOOD_SCHEDULE["run_hour"])) + return end - timedelta(days=1), end + if block == "freezing": # the 24 h shared with Brina, ending where its night ends + end = datetime.combine(run_day, time(BRINA_SCHEDULE["run_hour"])) + return end - timedelta(days=1), end + if block == "strong_wind": + end = datetime.combine(run_day, time(STRONG_WIND_SCHEDULE["run_hour"])) + return end - timedelta(days=1), end + if block == "hot_wind": + end = datetime.combine(run_day, time(HOT_WIND_SCHEDULE["run_hour"])) + return end - timedelta(days=1), end + if block == "high_temperature": + end = datetime.combine(run_day, time(HIGH_TEMPERATURE_SCHEDULE["run_hour"])) + return end - timedelta(days=1), end + if block == "snow": + end = datetime.combine(run_day, time(SNOW_SCHEDULE["run_hour"])) + return end - timedelta(days=1), end + if block == "hail": + end = datetime.combine(run_day, time(HAIL_SCHEDULE["run_hour"])) + return end - timedelta(days=1), end + return None + + +def _watch_window(block: str, run_day: date) -> dict | None: + """Extra period shown in the overview and in rain row 3: Flood's Heavy Rain Watch counts + the rain of the last claim_search_days, not only the event day.""" + if block != "flood": + return None + days = FLOOD_SCHEDULE["claim_search_days"] + end = datetime.combine(run_day, time(FLOOD_SCHEDULE["run_hour"])) + return {"label": f"Last {days} days", "start": _iso(end - timedelta(days=days)), "end": _iso(end)} + + +# ─────────────────────────── severity text ─────────────────────────── + +def _scale(bands: dict, fmt) -> str: + higher, strict = bands["higher_is_worse"], bands.get("strict", False) + op = (">" if strict else "≥") if higher else ("<" if strict else "≤") + names = [s.value for s in Severity][1:] + return ", ".join(f"{n} {op} {fmt(e)}" for n, e in zip(names, bands["edges"])) + + +def _severity_basis(a: Analysis) -> tuple[str, str]: + """(how the severity was found, the severity scale) for one event.""" + d = a.details + if a.block in ("brina", "freezing"): # same frost tables; Freezing: bud = air in daylight + what = "lowest bud temperature" if a.block == "brina" else "lowest daylight air temperature" + method = d.get("severity_method") + if method == "kill_fraction": + return (f"Stage {str(d.get('stage')).replace('_', ' ')} (BBCH {d.get('bbch')}): " + f"about {d['kill_fraction']:.0%} " + f"of buds killed at the {what}.", + "Buds killed: " + _scale(FROST_KILL_BANDS, lambda e: f"{e * 100:g} %")) + if method == "dormant": + return f"Buds dormant (BBCH {d.get('bbch')}): no damage expected.", "" + scale = "Bud temperature: " if a.block == "brina" else "Air temperature: " + return (f"Crop stage not set: graded on the {what}.", + scale + _scale(FROST_FALLBACK_BANDS_C, lambda e: f"{e:g} °C")) + if a.block == "flood": + worst = d.get("worst_criterion") + return (f"Rain = {d.get('rain_trigger_ratio', 0):.2f} × its threshold for " + f"{CRITERIA.get(worst, worst)}, the check crossed the most.", + "Rain / threshold: " + _scale(EXCESSIVE_RAIN_BANDS_RATIO, lambda e: f"{e:g}×")) + if a.block == "strong_wind": # same for every crop stage + return ("Graded on the highest wind of the strong-wind hours (Beaufort scale), " + "the same for every crop stage.", + "Wind: " + _scale(STRONG_WIND_BANDS_MS, lambda e: f"{e:g} m/s") + + " (Beaufort 7, 8, 9, 10)") + if a.block == "hot_wind": # same for every crop stage + return ("Graded on the highest air temperature of the hot-wind hours, " + "the same for every crop stage.", + "Air temperature: " + _scale(HOT_WIND_BANDS_C, lambda e: f"{e:g} °C")) + if a.block == "high_temperature": # same for every crop stage + if d.get("severity_method") == "fst": + return (f"Graded on the highest estimated fruit surface temperature " + f"({d.get('severity_value_c'):g} °C), the same for every crop stage.", + "Fruit surface: " + _scale(HOT_TEMPERATURE_FST_BANDS_C, lambda e: f"{e:g} °C")) + return ("No solar radiation data: graded on the highest air temperature, " + "the same for every crop stage.", + "Air temperature: " + _scale(HOT_WIND_BANDS_C, lambda e: f"{e:g} °C")) + if a.block == "drought": # same for every crop stage + return (f"Graded on the rain of {d.get('window_label')} as % of its 1991-2020 average " + f"({d.get('rain_pct_of_normal'):g} %), the same for every crop stage.", + "Rain / average: " + _scale(DROUGHT_BANDS_PCT, lambda e: f"{e:g} %")) + if camera_request(d): # snow, hail: suspected, the photo grades it + return ("Not graded: no sensor measures it; the photo (CV model) will confirm and grade it.", + "") + return "", "" + + +# ─────────────────────────── content ─────────────────────────── + +def _iso(t): + return t.isoformat(timespec="minutes") if t else None + + +def _block_entry(block: str, run: dict | None, station_id: int, station, day: date) -> dict: + title = EVENTS[block]["title"] if block in EVENTS else block.replace("_", " ").capitalize() + entry = {"block": block, "title": title, "missing_sensors": missing_sensors(block, station)} + empty = {"fields": [], "checks": [], "data_notes": [], "details": {}} + if run and run.get("error"): # the block could not run (no code, crash, API) + window = _window(block, run["run_day"]) + return {**entry, **empty, "status": ERROR, "run_day": run["run_day"].isoformat(), + "stale": run["run_day"] != day, "error": run["error"][:400], + "window": {"start": _iso(window[0]), "end": _iso(window[1])} if window else None, + "watch_window": _watch_window(block, run["run_day"])} + mine = [a for a in (run or {}).get("analyses", []) if a.station_id == station_id] + if not mine: + return {**entry, "status": "not_run", "run_day": None, "fields": [], "checks": [], + "data_notes": [], "details": {}} + run_day = run["run_day"] + window = _window(block, run_day) + ref = next((a for a in mine if a.status != NO_DATA), mine[0]) # station-level values + label = None + if block in MONTHLY_BLOCKS and ref.details.get("window_start"): # the months the block checked + window = (datetime.fromisoformat(ref.details["window_start"]), + datetime.fromisoformat(ref.details["window_end"])) + label = ref.details.get("window_label") + checks = CHECKS.get(block, _generic_checks)(ref, station) if ref.status != NO_DATA else [] + notes = [n for n in dict.fromkeys(n for a in mine for n in a.data_notes) + if not any(h in n for h in HIDDEN_NOTES)] + fields = [_field_entry(block, a) for a in mine] + worst = max(mine, key=lambda a: a.severity.level) + status = worst.status if worst.status == EVENT else ref.status + return {**entry, "status": status, + "run_day": run_day.isoformat(), "stale": run_day != day, + "window": ({"start": _iso(window[0]), "end": _iso(window[1]), + **({"label": label} if label else {})} if window else None), + "monthly": block in MONTHLY_BLOCKS, + "watch_window": _watch_window(block, run_day), + "checks": checks, "fields": fields, "data_notes": notes, "details": ref.details, + "notice": _past_event_notice(block, status, ref.details)} + + +def _field_entry(block: str, a: Analysis) -> dict: + """One field's result in a block entry.""" + basis, scale = _severity_basis(a) if a.status == EVENT else ("", "") + return {"field_id": a.field_id, "field_name": a.field_name, "status": a.status, + "severity": a.severity.value, "severity_score": a.severity_score, + "duration_h": a.duration_h, "start": _iso(a.start), "end": _iso(a.end), + "summary": a.summary, "severity_basis": basis, "severity_scale": scale, + "in_cover_window": a.details.get("in_cover_window"), + "bbch": a.details.get("bbch"), + # suspected (snow, hail): no sensor, a photo is asked for to confirm + "suspected": a.status == EVENT and camera_request(a.details) is not None, + "camera_request": camera_request(a.details) if a.status == EVENT else None, + **_monthly_field(block, a)} + + +def _merge_daytime(entry: dict, runs: list[dict], station_id: int, day: date) -> dict: + """Hail: add the daytime checks of the report day (block DAYTIME_BLOCK, slots 13h / 19h) to + the Hail entry. The checks table stays the morning 24 h; a suspicion found by a daytime check + makes the field "suspected" with that check's photo request, and its summary is appended.""" + if not runs: + return entry + entry = {**entry, "fields": [dict(f) for f in entry["fields"]], "data_notes": list(entry["data_notes"])} + by_id = {f["field_id"]: f for f in entry["fields"]} + windows = [] + for run in runs: + slot = run.get("slot", "") + if run.get("error"): + entry["data_notes"].append(f"Daytime check {slot[:2]}:00 could not run: {run['error'][:300]}") + continue + for a in (x for x in run["analyses"] if x.station_id == station_id): + start, end = a.details.get("window_start"), a.details.get("window_end") + if start and end: + windows.append((start, end)) + f = by_id.get(a.field_id) + if f is None: # no morning result for this field + f = by_id[a.field_id] = _field_entry("hail", a) + f.update(status=NO_EVENT if a.status == EVENT else a.status, suspected=False, + camera_request=None, summary="", duration_h=0.0, start=None, end=None) + entry["fields"].append(f) + label = f"Daytime check {_hour(start)[-5:]} → {_hour(end)[-5:]}" if start and end else "Daytime check" + f["summary"] = (f["summary"] + " " if f["summary"] else "") + f"{label}: {a.summary}" + f.setdefault("daytime_checks", []).append( + {"slot": slot, "window_start": start, "window_end": end, "status": a.status, + "summary": a.summary, "camera_request": camera_request(a.details)}) + if a.status == EVENT: + if f["status"] != EVENT or not f.get("suspected"): + f.update(status=EVENT, start=_iso(a.start), end=_iso(a.end), duration_h=0.0, + severity_basis=_severity_basis(a)[0], severity_scale="", + in_cover_window=a.details.get("in_cover_window")) + f["suspected"] = True + f["camera_request"] = camera_request(a.details) # the newest photo request + f["start"] = min(filter(None, [f["start"], _iso(a.start)])) + f["end"] = max(filter(None, [f["end"], _iso(a.end)])) + f["duration_h"] = f["duration_h"] + a.duration_h + for n in a.data_notes: + note = f"{label}: {n}" + if note not in entry["data_notes"] and not any(h in n for h in HIDDEN_NOTES): + entry["data_notes"].append(note) + if windows: + entry["watch_window"] = {"label": "Daytime checks", "start": min(w[0] for w in windows), + "end": max(w[1] for w in windows)} + if any(f["status"] == EVENT for f in entry["fields"]): + entry["status"] = EVENT + elif entry["status"] == "not_run" and entry["fields"]: + entry["status"] = NO_EVENT if any(f["status"] == NO_EVENT for f in entry["fields"]) else NO_DATA + if entry.get("run_day") is None: + entry.update(run_day=day.isoformat(), stale=False) + return entry + + +def _monthly_field(block: str, a: Analysis) -> dict: + """Monthly blocks (drought): the event's period as text, and its cover line (drought has + no contract cover period in thresholds.COVER_WINDOWS).""" + if block not in MONTHLY_BLOCKS or a.status != EVENT: + return {} + days = a.details.get("days") + return {"period_text": f"{a.details.get('window_label')} ({days} days)" if days else + a.details.get("window_label"), + "duration_text": f"{days} d" if days else "-", + "cover_text": "no contract cover period for drought (not in thresholds.py)"} + + +def _past_event_notice(block: str, status: str, details: dict) -> dict | None: + """No new event, but a past one to claim (Flood's last_heavy_72h, rain row 3).""" + heavy = details.get("last_heavy_72h") if block == "flood" else None + if status != NO_EVENT or not heavy: + return None + first, last = datetime.fromisoformat(heavy["first_rain"]), datetime.fromisoformat(heavy["last_rain"]) + mm = EXCESSIVE_RAIN["intense_72h_mm"] + return {"short": f"Past: {heavy['mm']:g} mm in 72 h ({first:%d %b} → {last:%d %b})", + "text": (f"No new event, but {heavy['mm']:g} mm of rain fell within 72 h in the past " + f"days ({_hour(heavy['first_rain'])} → {_hour(heavy['last_rain'])}; " + f"{mm:g} mm reached on {_hour(heavy['reached_at'])}).")} + + +def _photo_text(request: dict) -> str: + """One line for the client: why a photo is asked for, and when.""" + when = datetime.fromisoformat(request["photo_at"]).strftime("%d %b %H:%M") + return f"{request['reason']} Photo requested for {when} to confirm." + + +def build(station_id: int, day: date | None = None, root: Path | None = None) -> dict: + """Report content of one station: every event, with its block's newest run up to `day`.""" + station = station_by_id(station_id) + stored = report.blocks(root) + merged = set(DAYTIME_BLOCK_OF.values()) # stored blocks shown inside another block + order = list(EVENTS) + [b for b in stored if b not in EVENTS and b not in merged] + latest = {b: report.latest_run(b, day, root) if b in stored else None for b in order} + day = day or max((r["run_day"] for r in latest.values() if r), default=date.today()) + blocks = [_block_entry(b, run, station_id, station, day) for b, run in latest.items()] + for i, b in enumerate(blocks): + extra = DAYTIME_BLOCK_OF.get(b["block"]) + if extra and extra in stored: + blocks[i] = _merge_daytime(b, report.day_runs(extra, day, root), station_id, day) + names = {a.station_name for r in latest.values() if r for a in r["analyses"] + if a.station_id == station_id} + name = station.name if station else (names.pop() if names else str(station_id)) + found = [(b, f) for b in blocks for f in b["fields"] if f["status"] == EVENT] + events = [(b, f) for b, f in found if not f.get("suspected")] + suspects = [(b, f) for b, f in found if f.get("suspected")] # photo asked for (snow, hail) + worst = max((Severity(f["severity"]) for _, f in events), key=lambda s: s.level, default=Severity.NONE) + suspected_titles = list(dict.fromkeys(b["title"] for b, _ in suspects)) + if events: + titles = list(dict.fromkeys(b["title"] for b, _ in events)) + headline = f"{' and '.join(titles)} detected · worst severity: {worst.value.upper()}" + if suspects: + headline += f" · {' and '.join(suspected_titles)} suspected" + elif suspects: + headline = f"{' and '.join(suspected_titles)} suspected · photo requested to confirm" + elif any(b.get("notice") for b in blocks): + headline = "No new damaging event · a past one was found (see below)." + elif any(b["status"] == NO_EVENT for b in blocks): + headline = "No damaging weather event detected." + else: + headline = "No usable data for this station." + fields = {f["field_id"]: f["field_name"] for b in blocks for f in b["fields"]} + return { + "station": {"id": station_id, "name": name, + "sensors": sorted(station.sensors) if station else []}, + "report_day": day.isoformat(), + "created_at": datetime.now().isoformat(timespec="minutes"), + "overall": {"events": len(events), "worst_severity": worst.value, "headline": headline, + "suspected": [{"block": b["block"], "title": b["title"], + "text": _photo_text(f["camera_request"])} for b, f in suspects], + "notices": [{"block": b["block"], "title": b["title"], "text": b["notice"]["text"]} + for b in blocks if b.get("notice")], + "errors": [{"block": b["block"], "title": b["title"], "error": b["error"]} + for b in blocks if b["status"] == ERROR]}, + "fields": [{"field_id": k, "field_name": v} for k, v in fields.items()], + "blocks": blocks, + } + + +def write(station_id: int, day: date | None = None, root: Path | None = None, + out_dir: Path | None = None) -> tuple[Path, Path]: + """Write .json and .pdf of one station. A report of the same day is replaced; + the other days stay, and only the last REPORT["history_runs"] (10) days are kept.""" + from damage_engine.report import station_pdf # needs fpdf2 only when writing + + content = build(station_id, day, root) + folder = (out_dir or STATION_REPORTS_DIR) / str(station_id) + folder.mkdir(parents=True, exist_ok=True) + json_path = folder / f"{content['report_day']}.json" + json_path.write_text(json.dumps(content, indent=2, ensure_ascii=False), encoding="utf-8") + pdf_path = station_pdf.render(content, json_path.with_suffix(".pdf")) + for old in sorted(folder.glob("*.json"), reverse=True)[REPORT["history_runs"]:]: + old.unlink() + old.with_suffix(".pdf").unlink(missing_ok=True) + return json_path, pdf_path + + +def main(): + p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + p.add_argument("--station", type=int, required=True, help="3A station id, e.g. 566") + p.add_argument("--date", type=date.fromisoformat, help="report day (default: newest block run)") + args = p.parse_args() + for path in write(args.station, args.date): + print(path) + + +if __name__ == "__main__": + main() diff --git a/src/damage_engine/schemas.py b/src/damage_engine/schemas.py new file mode 100644 index 0000000..cc3e5c6 --- /dev/null +++ b/src/damage_engine/schemas.py @@ -0,0 +1,107 @@ +""" +Shared data shapes. Every block may import this file. + +A block returns a list of Analysis: one per field and station it looked at, +also when nothing happened, so the report can say "no brina last night". +It carries the (phenomenon, severity, duration) message of the diagram plus +the metadata the report needs (field, station, run day, ...). +""" + +from dataclasses import asdict, dataclass, field +from datetime import date, datetime +from enum import Enum + +from damage_engine.thresholds import SEVERITY_LEVELS + + +class Severity(str, Enum): + """Severity levels; the names and their order come from thresholds.SEVERITY_LEVELS.""" + NONE = "none" + LIGHT = "light" + MODERATE = "moderate" + SEVERE = "severe" + EXTREME = "extreme" + + @classmethod + def from_level(cls, level: int) -> "Severity": + return cls[SEVERITY_LEVELS[level]] + + @property + def level(self) -> int: + return next(k for k, name in SEVERITY_LEVELS.items() if name == self.name) + + +# Analysis.status +EVENT = "event" # the block detected its phenomenon +NO_EVENT = "no_event" # data was fine, nothing detected +NO_DATA = "no_data" # the station could not be judged (no sensors / no records) + + +# Camera request: blocks without a sensor for their phenomenon (snow, hail) only flag that it +# was POSSIBLE and ask the camera for a photo; the Hail/Snow CV model confirms and grades it. +# The request is stored as a dict in Analysis.details["camera_request"] (CameraRequest.to_dict), +# so the camera / CV side reads it without importing the detector. +SUSPECT_POSSIBLE = "possible" # weak signals only +SUSPECT_LIKELY = "likely" # strong signals + + +@dataclass +class CameraRequest: + phenomenon: str # "snow" or "hail" + station_id: int + photo_at: datetime # when to take the photo (local time) + level: str # SUSPECT_POSSIBLE or SUSPECT_LIKELY + signals: list[str] # the signals that fired, e.g. ["A", "D"] + reason: str # one sentence: why the photo is asked for + + def to_dict(self): + d = asdict(self) + d["photo_at"] = self.photo_at.isoformat(timespec="minutes") + return d + + +def camera_request(details: dict) -> dict | None: + """The camera request stored in an Analysis' details, or None.""" + return details.get("camera_request") if details else None + + +@dataclass +class Analysis: + block: str # e.g. "brina" + run_day: date # day the block ran for + field_id: str + field_name: str + station_id: int + station_name: str + status: str # EVENT, NO_EVENT or NO_DATA + severity: Severity = Severity.NONE + severity_score: float = 0.0 # 0-1, see thresholds.SEVERITY_SCORE_AT_LEVEL_START + duration_h: float = 0.0 # hours the event conditions held + start: datetime | None = None + end: datetime | None = None + summary: str = "" # one sentence for the farmer + details: dict = field(default_factory=dict) # block-specific values (min temp, ...) + # missing sensors / missing data and how the block handled them, one sentence + # each; every block fills it and the report always shows it + data_notes: list[str] = field(default_factory=list) + + @property + def detected(self) -> bool: + return self.status == EVENT + + def to_dict(self): + d = asdict(self) + d["severity"] = self.severity.value + d["run_day"] = self.run_day.isoformat() + d["start"] = self.start.isoformat() if self.start else None + d["end"] = self.end.isoformat() if self.end else None + return d + + @classmethod + def from_dict(cls, d): + d = dict(d) + d["severity"] = Severity(d["severity"]) + d["run_day"] = date.fromisoformat(d["run_day"]) + d["start"] = datetime.fromisoformat(d["start"]) if d.get("start") else None + d["end"] = datetime.fromisoformat(d["end"]) if d.get("end") else None + return cls(**d) diff --git a/src/damage_engine/severity.py b/src/damage_engine/severity.py new file mode 100644 index 0000000..3ca7f8f --- /dev/null +++ b/src/damage_engine/severity.py @@ -0,0 +1,43 @@ +""" +Severity grading shared by every block. + +Turns a value into a Severity level and a 0-1 score using a band dict from +thresholds.py ({"edges": [...4 values...], "higher_is_worse": bool, "strict": bool}): + + from damage_engine.severity import grade + from damage_engine.thresholds import FROST_FALLBACK_BANDS_C + + grade(-3.1, FROST_FALLBACK_BANDS_C) # -> (Severity.MODERATE, 0.638) +""" + +from damage_engine.schemas import Severity +from damage_engine.thresholds import SEVERITY_SCORE_AT_LEVEL_START + + +def level_for(value, bands) -> int: + """0 (NONE) to 4 (EXTREME): number of band edges the value reaches.""" + higher, strict = bands["higher_is_worse"], bands.get("strict", False) + + def reached(edge): + if higher: + return value > edge if strict else value >= edge + return value < edge if strict else value <= edge + + return sum(1 for edge in bands["edges"] if reached(edge)) + + +def score_for(value, bands, level) -> float: + """s_sev: the level's start score, interpolated linearly up to the next level's start.""" + if level == 0: + return 0.0 + scores, edges = SEVERITY_SCORE_AT_LEVEL_START, bands["edges"] + if level >= len(edges): + return scores[len(edges)] + lo, hi = edges[level - 1], edges[level] + frac = min(max((value - lo) / (hi - lo), 0.0), 1.0) + return scores[level] + frac * (scores[level + 1] - scores[level]) + + +def grade(value, bands) -> tuple[Severity, float]: + level = level_for(value, bands) + return Severity.from_level(level), round(score_for(value, bands, level), 3) diff --git a/src/damage_engine/sources/__init__.py b/src/damage_engine/sources/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/damage_engine/sources/derived.py b/src/damage_engine/sources/derived.py new file mode 100644 index 0000000..10a8dcb --- /dev/null +++ b/src/damage_engine/sources/derived.py @@ -0,0 +1,108 @@ +""" +Quantities CALCULATED from 3A sensor data, one implementation for the whole +pipeline. Every block imports them from here and never keeps its own formula, +so a value (e.g. the wet-bulb temperature of an hour) is the same in every +block and in every report. + +Reference [3A-DOC]: 3A, "Algoritmi di stima di alcune grandezze +agrometeorologiche" (docs/ImplementazioneAlgoritmiSensori.pdf), section 4 +"Temperatura bulbo umido (Tw)". It is the formula 3A uses for its own TBulbMed: +on (hourly mean temperature, mean humidity) it gives 3A's TBulbMed in 49 of 49 +hours at station 566 (to 3A's 0.1 °C rounding, checked 2026-10-21). + +Wet-bulb of one hour, from the 3A hourly columns (add_wet_bulb): + WetBulbMin = Tw(TemperaturaAriaMin, UmiditaRelativaMax) lowest of the hour + WetBulbMax = Tw(TemperaturaAriaMax, UmiditaRelativaMin) highest of the hour +Why these pairs: within an hour the water vapour in the air changes little, so +the coldest moment is also the most humid one (and the warmest the driest). +The same pairing (Tmin with RHmax, Tmax with RHmin) is used by the 3A document +for the vapour pressure, section 3 eq. (2) (FAO-56). Checked on 566: the vapour +pressure of (Tmin, RHmax), (Tmean, RHmean) and (Tmax, RHmin) differs by < 1 hPa. + +Missing values: no hourly minimum / maximum temperature -> hourly mean +temperature; no hourly maximum / minimum humidity -> hourly mean humidity; no +temperature or no humidity at all -> NaN (the block decides what to do). + + from damage_engine.sources import derived + df = fetch_hourly(ids, start, end, [*derived.WET_BULB_INPUTS, ...]) + df = derived.add_wet_bulb(df) # adds WetBulbMin, WetBulbMax (°C) +""" + +import numpy as np +import pandas as pd + +K0 = 273.15 # 0 °C in K + +# constants of the 3A document, section 4 [3A-DOC] +RV = 0.11 # cal/(Kg K) gas constant of moist air +CP = 0.24 # cal/(Kg K) specific heat at constant pressure +L_EVAPORATION = 597.0 # cal/Kg latent heat, T > 0 °C (water) +L_SUBLIMATION = 677.0 # cal/Kg latent heat, T <= 0 °C (ice) +PRESSURE_HPA = 1000.0 # mbar atmospheric pressure ("generalmente si può porre 1000 mbar") +# saturation vapour pressure e*(T) = 6.1 * 10^(a*T / (T + b)), T in °C +MAGNUS_WATER = (7.5, 237.3) # T > 0 °C +MAGNUS_ICE = (9.5, 265.5) # T <= 0 °C + +# 3A hourly columns (api_parameters) used, and the columns added by add_wet_bulb +T_MIN, T_MEAN, T_MAX = "TemperaturaAriaMin", "TemperaturaAriaMedia", "TemperaturaAriaMax" +RH_MIN, RH_MEAN, RH_MAX = "UmiditaRelativaMin", "UmiditaRelativaMedia", "UmiditaRelativaMax" +WET_BULB_INPUTS = [T_MIN, T_MEAN, T_MAX, RH_MIN, RH_MEAN, RH_MAX] +WET_BULB_MIN = "WetBulbMin" # °C, lowest wet-bulb of the hour +WET_BULB_MAX = "WetBulbMax" # °C, highest wet-bulb of the hour + + +def _branch(t_c): + """(a, b, L) per value: ice constants for T <= 0 °C, water constants above.""" + ice = t_c <= 0 + a = np.where(ice, MAGNUS_ICE[0], MAGNUS_WATER[0]) + b = np.where(ice, MAGNUS_ICE[1], MAGNUS_WATER[1]) + latent = np.where(ice, L_SUBLIMATION, L_EVAPORATION) + return a, b, latent + + +def vapour_pressure_hpa(temp_c, rh_pct): + """Vapour pressure e = RH * e*(T) / 100 (hPa) [3A-DOC sec. 4].""" + t = np.asarray(temp_c, dtype=float) + rh = np.asarray(rh_pct, dtype=float) + a, b, _ = _branch(t) + return rh / 100.0 * 6.1 * 10.0 ** (a * t / (t + b)) + + +def dew_point_c(temp_c, rh_pct): + """Dew point (frost point for T <= 0 °C), °C [3A-DOC sec. 4 eq. 2-3, sec. 5].""" + t = np.asarray(temp_c, dtype=float) + a, b, _ = _branch(t) + with np.errstate(divide="ignore", invalid="ignore"): + x = np.log10(vapour_pressure_hpa(t, rh_pct) / 6.1) + return b * x / (a - x) + + +def wet_bulb_c(temp_c, rh_pct, pressure_hpa=PRESSURE_HPA): + """Wet-bulb temperature (°C) [3A-DOC sec. 4 eq. 1]: + + Tw = Td + (T - Td) / (1 + L² s / (Rv Cp T²)) - 273.15 (T, Td in K) + s = 0.622 e / (P - 0.378 e) (specific humidity) + Water constants for T > 0 °C, ice constants for T <= 0 °C. RH <= 0 gives NaN. + """ + t = np.asarray(temp_c, dtype=float) + _, _, latent = _branch(t) + e = vapour_pressure_hpa(t, rh_pct) + with np.errstate(divide="ignore", invalid="ignore"): + s = 0.622 * e / (pressure_hpa - 0.378 * e) + t_k, td_k = t + K0, dew_point_c(t, rh_pct) + K0 + return td_k + (t_k - td_k) / (1.0 + latent ** 2 * s / (RV * CP * t_k ** 2)) - K0 + + +def _col(df, name): + return pd.to_numeric(df[name], errors="coerce") if name in df else pd.Series(np.nan, index=df.index) + + +def add_wet_bulb(df: pd.DataFrame) -> pd.DataFrame: + """Copy of the hourly rows with WetBulbMin and WetBulbMax (°C) added (see module doc).""" + df = df.copy() + t_mean, rh_mean = _col(df, T_MEAN), _col(df, RH_MEAN) + t_lo, rh_hi = _col(df, T_MIN).fillna(t_mean), _col(df, RH_MAX).fillna(rh_mean) + t_hi, rh_lo = _col(df, T_MAX).fillna(t_mean), _col(df, RH_MIN).fillna(rh_mean) + df[WET_BULB_MIN] = np.round(wet_bulb_c(t_lo, rh_hi), 2) + df[WET_BULB_MAX] = np.round(wet_bulb_c(t_hi, rh_lo), 2) + return df diff --git a/src/damage_engine/sources/fields.py b/src/damage_engine/sources/fields.py new file mode 100644 index 0000000..823a887 --- /dev/null +++ b/src/damage_engine/sources/fields.py @@ -0,0 +1,53 @@ +""" +Field registry: which farmer fields exist and which station watches each one. + +Read from data/fields.json: + + {"fields": [ + {"field_id": "F001", "name": "Melo nord", "station_id": 3, "crop": "apple", "bbch": 65} + ]} + +"bbch" is the current phenological stage of the field (optional; update it +during the season). Without it, frost blocks use the fallback severity bands. +If data/fields.json does not exist, every station counts as one field with +id "ST", so the pipeline works before real fields are entered. +""" + +import json +from dataclasses import dataclass +from pathlib import Path + +from damage_engine.sources.stations import all_stations + +FIELDS_FILE = Path(__file__).resolve().parents[3] / "data" / "fields.json" + + +@dataclass(frozen=True) +class Field: + id: str + name: str + station_id: int + crop: str = "apple" + bbch: int | None = None + + +def all_fields(path: Path | None = None) -> list[Field]: + path = path or FIELDS_FILE + if not path.exists(): + return [Field(id=f"ST{s.id}", name=f"{s.name} (station {s.id})", station_id=s.id) + for s in all_stations()] + data = json.loads(path.read_text(encoding="utf-8")) + return [ + Field( + id=str(f["field_id"]), + name=f.get("name", str(f["field_id"])), + station_id=int(f["station_id"]), + crop=f.get("crop", "apple"), + bbch=int(f["bbch"]) if f.get("bbch") is not None else None, + ) + for f in data["fields"] + ] + + +def field_by_id(field_id, path: Path | None = None): + return next((f for f in all_fields(path) if f.id == str(field_id)), None) diff --git a/src/damage_engine/sources/stations.py b/src/damage_engine/sources/stations.py new file mode 100644 index 0000000..39085de --- /dev/null +++ b/src/damage_engine/sources/stations.py @@ -0,0 +1,62 @@ +""" +Station catalogue: which 3A stations exist and which sensors each one has. + +Read from API/3a_stations_sensors.json (built from the 3A metadata calls), so +blocks can pick their stations without spending rate-limited API requests: + + from damage_engine.sources.stations import stations_with + + stations_with("TemperaturaAria", "UmiditaRelativa") # -> [Station, ...] +""" + +import json +from dataclasses import dataclass, field +from functools import lru_cache +from pathlib import Path + +CATALOGUE_FILE = Path(__file__).resolve().parents[3] / "API" / "3a_stations_sensors.json" + + +@dataclass(frozen=True) +class Station: + id: int + name: str + latitude: float | None + longitude: float | None + elevation_m: float | None + sensors: dict[str, list[str]] = field(default_factory=dict) # sensor -> api_parameters + + def has(self, *sensors): + return all(s in self.sensors for s in sensors) + + +def _num(value): + try: + return float(value) + except (TypeError, ValueError): + return None + + +@lru_cache(maxsize=1) +def all_stations(): + data = json.loads(CATALOGUE_FILE.read_text(encoding="utf-8")) + return [ + Station( + id=int(s["StationId"]), + name=s["StationName"], + latitude=_num(s.get("Latitude_WGS84")), + longitude=_num(s.get("Longitude_WGS84")), + elevation_m=_num(s.get("Elevation_m")), + sensors={x["sensor"]: list(x["api_parameters"]) for x in s.get("Sensors", [])}, + ) + for s in data["Stations"] + ] + + +def stations_with(*sensors): + """Stations that have every one of the given sensors (3A sensor names).""" + return [s for s in all_stations() if s.has(*sensors)] + + +def station_by_id(station_id): + return next((s for s in all_stations() if s.id == int(station_id)), None) diff --git a/src/damage_engine/sources/threea.py b/src/damage_engine/sources/threea.py new file mode 100644 index 0000000..0d114a7 --- /dev/null +++ b/src/damage_engine/sources/threea.py @@ -0,0 +1,242 @@ +""" +Shared client for the 3A weather API (https://api.3acloud.it). + +Every block that needs sensor data uses this file, so login, token caching, +the rate limit and the response parsing live in one place: + + from damage_engine.sources.threea import fetch_hourly + + df = fetch_hourly(station_ids=[3, 6], start=..., end=..., + columns=["TemperaturaAriaMin", "UmiditaRelativaMedia"]) + +Auth (from the API spec): + POST /login with headers name, password, totp -> token + every other call sends the header token: + +Credentials are read, in this order, from + 1. environment variables THREEA_USER, THREEA_PASSWORD, THREEA_TOTP_SECRET + 2. a .env file in the project folder (same names, KEY=value lines) + 3. API/3a_credentials.json (keys user, password, totp_secret) +The TOTP secret is needed for unattended runs (e.g. Brina at 06:00): with it, +the 6-digit codes are generated automatically. + +The server allows 5 requests per 15 minutes; requests wait for the window. +""" + +import json +import logging +import os +import sys +import time +from datetime import datetime +from pathlib import Path + +import pandas as pd +import requests + +log = logging.getLogger(__name__) + +BASE_URL = "https://api.3acloud.it" +TIMEOUT_S = 120 +DATE_FMT = "%Y-%m-%d %H:%M" + +PROJECT_DIR = Path(__file__).resolve().parents[3] +CRED_FILE = PROJECT_DIR / "API" / "3a_credentials.json" +ENV_FILE = PROJECT_DIR / ".env" +CACHE_DIR = PROJECT_DIR / "data" / "3a_cache" +TOKEN_FILE = CACHE_DIR / ".token" + + +class ThreeAError(RuntimeError): + pass + + +# ─────────────────────────── credentials ─────────────────────────── + +def _read_env_file(): + if not ENV_FILE.exists(): + return {} + values = {} + for line in ENV_FILE.read_text(encoding="utf-8").splitlines(): + line = line.strip() + if line and not line.startswith("#") and "=" in line: + key, value = line.split("=", 1) + values[key.strip()] = value.strip().strip('"').strip("'") + return values + + +def _credential(env_name, json_key): + if os.environ.get(env_name): + return os.environ[env_name] + value = _read_env_file().get(env_name) + if value: + return value + if CRED_FILE.exists(): + value = json.loads(CRED_FILE.read_text(encoding="utf-8")).get(json_key, "") + if value and not value.startswith("PUT_"): + return value + return None + + +def _totp_code(): + secret = _credential("THREEA_TOTP_SECRET", "totp_secret") + if secret: + import pyotp + return pyotp.TOTP(secret.replace(" ", "")).now() + if not sys.stdin.isatty(): + raise ThreeAError("No TOTP secret configured and no terminal to ask for a code; " + "set THREEA_TOTP_SECRET for unattended runs.") + return input("Paste a fresh 6-digit TOTP code: ").strip().replace(" ", "") + + +# ─────────────────────────── client ─────────────────────────── + +class ThreeAClient: + """One authenticated session. Use the module-level `client()` to share it.""" + + def __init__(self): + self.session = requests.Session() + self.session.headers["output-fmt"] = "json" + self._rate_remaining = None + self._rate_reset = None + + # rate limit + def _wait_for_rate_limit(self): + if self._rate_remaining == 0 and self._rate_reset and self._rate_reset > time.time(): + wait = self._rate_reset - time.time() + 2 + log.info("3A rate limit reached, waiting %.0f s", wait) + time.sleep(wait) + + def _send(self, method, path, **kw): + while True: + self._wait_for_rate_limit() + r = self.session.request(method, BASE_URL + path, timeout=TIMEOUT_S, **kw) + if "X-RateLimit-Remaining" in r.headers: + self._rate_remaining = int(r.headers["X-RateLimit-Remaining"]) + if "X-RateLimit-Reset" in r.headers: + self._rate_reset = int(r.headers["X-RateLimit-Reset"]) + if r.status_code != 429: + return r + self._rate_remaining = 0 + self._rate_reset = self._rate_reset or time.time() + 15 * 60 + + # auth + def login(self): + user = _credential("THREEA_USER", "user") + password = _credential("THREEA_PASSWORD", "password") + if not (user and password): + raise ThreeAError("3A user/password not configured (see damage_engine/sources/threea.py).") + r = self._send("POST", "/login", + headers={"name": user, "password": password, "totp": _totp_code()}) + token = _extract_token(r) if r.ok else None + if not token: + raise ThreeAError(f"3A login failed ({r.status_code}): {r.text[:200]}") + self.session.headers["token"] = token + CACHE_DIR.mkdir(parents=True, exist_ok=True) + TOKEN_FILE.write_text(token) + log.info("3A login ok, token cached") + + def _ensure_token(self): + if "token" in self.session.headers: + return + if TOKEN_FILE.exists(): + self.session.headers["token"] = TOKEN_FILE.read_text().strip() + else: + self.login() + + def call(self, method, path, body=None): + """Authenticated call; logs in again once if the token was rejected.""" + self._ensure_token() + r = self._send(method, path, json=body) + if r.status_code in (401, 403): + log.info("3A token rejected (%s), logging in again", r.status_code) + TOKEN_FILE.unlink(missing_ok=True) + self.session.headers.pop("token", None) + self.login() + r = self._send(method, path, json=body) + if not r.ok: + raise ThreeAError(f"{method} {path} failed ({r.status_code}): {r.text[:500]}") + try: + return r.json() + except ValueError: + return r.text + + +def _extract_token(resp): + try: + data = resp.json() + except ValueError: + return resp.text.strip().strip('"') or None + if isinstance(data, str): + return data + if isinstance(data, dict): + for k in ("token", "access_token", "Token"): + if k in data: + return data[k] + return None + + +_client = None + + +def client(): + global _client + if _client is None: + _client = ThreeAClient() + return _client + + +# ─────────────────────────── data ─────────────────────────── + +def to_frame(data): + """Turn a 3A JSON response (list of rows, or {key: [rows]}) into a DataFrame.""" + if isinstance(data, list): + if not data: + return pd.DataFrame() + if all(isinstance(x, dict) for x in data): + return pd.json_normalize(data) + if isinstance(data, dict): + lists = [v for v in data.values() if isinstance(v, list)] + if len(lists) == 1: + return to_frame(lists[0]) + if lists and all(isinstance(x, dict) for v in lists for x in v): + return to_frame([x for v in lists for x in v]) + raise ThreeAError(f"Unexpected 3A response shape: {str(data)[:200]}") + + +# time column of each kind: daily rows have only "Data" (the API rejects "Timestamp") +TIME_COLUMN = {"Hourly": "Timestamp", "Daily": "Data"} + + +def _fetch_weather(kind, station_ids, start, end, columns): + time_col = TIME_COLUMN[kind] + cols = list(dict.fromkeys(["CodiceStazione", time_col, *columns])) + body = {"stationsIds": [int(s) for s in station_ids], + "dateFrom": start.strftime(DATE_FMT), "dateTo": end.strftime(DATE_FMT), + "columns": cols} + log.info("3A %s data %s … %s for %d station(s)", kind, body["dateFrom"], body["dateTo"], + len(body["stationsIds"])) + df = to_frame(client().call("POST", f"/getWeather{kind}", body)) + if df.empty: + return pd.DataFrame(columns=cols) + df[time_col] = pd.to_datetime(df[time_col]) + for c in columns: + if c in df.columns: + df[c] = pd.to_numeric(df[c], errors="coerce") + # the API pads the series with empty 10-minute rows; keep only rows with values + df = df.dropna(subset=[c for c in columns if c in df.columns], how="all") + return df.sort_values(["CodiceStazione", time_col]).reset_index(drop=True) + + +def fetch_hourly(station_ids, start: datetime, end: datetime, columns): + """Hourly rows for the stations in [start, end]; columns are 3A api_parameters.""" + return _fetch_weather("Hourly", station_ids, start, end, columns) + + +def fetch_daily(station_ids, start: datetime, end: datetime, columns): + """Daily rows for the stations in [start, end]; columns are 3A api_parameters. + + The date column is "Data" (not "Timestamp"): CodiceStazione, Data, . + Daily-only columns (e.g. EvapoPot) are in DailyOnlyParameters of 3a_stations_sensors.json. + """ + return _fetch_weather("Daily", station_ids, start, end, columns) diff --git a/src/damage_engine/thresholds.py b/src/damage_engine/thresholds.py new file mode 100644 index 0000000..19c00fb --- /dev/null +++ b/src/damage_engine/thresholds.py @@ -0,0 +1,574 @@ +# -*- coding: utf-8 -*- +""" +thresholds.py - THRESHOLD REFERENCE FOR THE DAMAGE PIPELINE (APPLE) +=================================================================== + +This file contains VALUES ONLY: no functions, no processing, no imports. +It is the ONLY place where thresholds live: every pipeline block reads them from here +and no block keeps its own copy: + + from damage_engine.thresholds import BRINA, SEVERITY_LEVELS + if t_air <= BRINA["t_air_max_c"]: ... + +To change pipeline behaviour, edit a number here; every block that reads it follows. +The numbers come from the sources tagged below - do not change them without a source. + +CROP: apple (melo) only. +PHENOMENA: brina, gelo (Freezing), strong wind, excessive rain (eccesso di pioggia, + the "Flood" box of the diagram), snow, hail, hot wind, hot temperature + (High temperature box), drought (detector only: a rain deficit against the + climate average; no contract trigger, see DROUGHT). +OUT OF SCOPE: drought claims and damage (future work), flooding by rivers (alluvione, not in + Model 1506 for this crop), heatwave, sunscald, sudden temperature change. + +CONTRACT: UnipolSai Assicurazioni S.p.A., "Agricultural Risks - Adverse Weather Conditions", + Model 1506, Ed. 01/2021 (English version). Page numbers refer to that document. + +SOURCE TAG in each comment: + [CONTRACT] written in Model 1506 - change only if the insurer's wording changes + [SCALE] standard meteorological scale (Beaufort, TORRO-based hail classes) + [LITERATURE] published agronomic data (WSU / MSU) + [ENGINEERING] project default, the contract gives no number -> CALIBRATE on own data + [PROJECT] pipeline setting (schedule, history size), not a physical threshold + [EXTERNAL] reference data from an outside service, copied here once by hand (the + pipeline never calls it: only the 3A API is used at run time); edit freely + +UNITS are in the key names: _c = deg C, _ms = m/s, _mm = mm, _cm = cm, _wm2 = W/m2, +_h = hours, _min = minutes, _days = days, _pct = percent, _deg = degrees (wind FROM). + +CONTRACT COMPARISONS: every contract trigger has a matching "_op" key giving the exact +operator from the wording ("<" for "below", ">" for "more than", ">=" for "at least"). +Detection must use these operators. + +SEVERITY BANDS: a list of 4 edges = value where LIGHT, MODERATE, SEVERE, EXTREME start. + "higher_is_worse": True -> value >= edge reaches that level (wind, rain, temperature) + "higher_is_worse": False -> value <= edge reaches that level (frost temperatures) + "strict": True -> use > / < instead of >= / <= +Bands only grade events that already passed detection (the "_op" check above). +The contract only defines the yes/no trigger; levels above it are never contractual. +""" + +CONTRACT_REF = ("UnipolSai Assicurazioni S.p.A., Model 1506 'Agricultural Risks - " + "Adverse Weather Conditions', Ed. 01/2021") +CROP = "apple" + + +# ============================================================================= +# SEVERITY SCALE (shared by all phenomena) +# ============================================================================= + +SEVERITY_LEVELS = {0: "NONE", 1: "LIGHT", 2: "MODERATE", 3: "SEVERE", 4: "EXTREME"} + +# s_sev (0-1) at the start of each level; interpolate linearly between edges [ENGINEERING] +SEVERITY_SCORE_AT_LEVEL_START = {1: 0.25, 2: 0.50, 3: 0.75, 4: 1.00} + + +# ============================================================================= +# GENERAL CONTRACT RULES +# ============================================================================= + +GENERAL = { + # Official agrometeorological data (e.g. ARPA) for the area containing the plot decide + # whether a threshold was reached; field sensors are indicative. [CONTRACT p.4] + "official_data_area_km2": 30, + # Effects must also appear on apple orchards within this radius, + # similar topography. Not required for hail. [CONTRACT p.7] + "effect_radius_km": 3, + # 'Reference period' for every average = last N years [CONTRACT p.7] + "reference_period_years": 5, + # No indemnity unless farm/municipality/product loss exceeds this [CONTRACT Art.7 p.16] + "damage_threshold_pct": 20, + # Claim notice deadlines [CONTRACT Art.10 p.20] + "claim_notice_working_days": 3, + "claim_notice_frost_h": 36, + # A 'for the record' claim can be converted to an assessment request + # within N days of the event... [CONTRACT Art.10 p.21] + "claim_conversion_days": 20, + # ...and only for events at least N days before maturity (Art.10 p.21 says + # 'maturity date'; the information pack p.7 says 'harvest date') [CONTRACT Art.10 p.21] + "claim_conversion_before_maturity_days": 20, +} + +# Cover starts at 12:00 on the Nth day after notification [CONTRACT Art.6 p.15] +COVER_START_LAG_DAYS = { + "hail": 3, + "strong_wind": 3, + "excessive_rain": 6, + "snow": 6, + "hot_temperature": 6, # as 'Sunstroke' + "brina": 12, + "gelo": 12, + "hot_wind": 30, +} + +# Cover window for apples: (phenological start, end date "MM-DD" at 12:00) [CONTRACT Special Art.5 p.31] +COVER_WINDOWS = { + "brina": ("budburst", "05-10"), + "gelo": ("budburst", "05-10"), + "excessive_rain": ("fruit_set", "10-31"), + "hail": ("fruit_set", "10-31"), + "strong_wind": ("fruit_set", "10-31"), + "hot_wind": ("fruit_set", "10-31"), + "hot_temperature": ("fruit_set", "10-31"), # as 'Sunstroke' + "snow": ("budburst", "10-31"), # [ENGINEERING] snow not listed in Art.5 - verify +} + +# Apples under anti-hail nets: hail cover ends on this date [CONTRACT Special Art.5 p.31] +HAIL_NET_COVER_END = "05-25" + +# Recommended run cadence (from the pipeline figure) +RUN_SCHEDULE = { + "brina": "daily, morning: previous night (sunset -> sunrise)", + "gelo": "every 24 h", + "strong_wind": "every 24 h", + "hail": "every 24 h", + "snow": "every 24 h", + "excessive_rain": "every 24 h", + "hot_temperature": "every 24 h", + "hot_wind": "every 24 h, only in the 30 days before harvest", + "drought": "every month (1st day): the last 3 complete calendar months", +} + + +# ============================================================================= +# FROST: APPLE CRITICAL TEMPERATURES (used by BRINA and GELO) +# ============================================================================= + +# Bud temperature (deg C) killing 10% and 90% of buds after ~30 min. +# [LITERATURE] WSU EB0913, reproduced in MSU 'Critical spring temperatures for tree fruit bud stages' +APPLE_CRITICAL_TEMPS_C = { + # stage (T10, T90) + "silver_tip": (-9.4, -16.7), + "green_tip": (-7.8, -12.2), + "half_inch_green": (-5.0, -9.4), + "tight_cluster": (-2.8, -6.1), + "first_pink": (-2.2, -4.4), + "full_pink": (-2.2, -3.9), + "first_bloom": (-2.2, -3.9), + "full_bloom": (-2.2, -3.9), + "post_bloom": (-2.2, -3.9), + "fruitlet": (-2.2, -3.9), # [ENGINEERING] post_bloom values reused +} + +# Highest BBCH code of each stage (BBCH below 51 = dormant, frost-hardy) [ENGINEERING] approx. mapping +APPLE_BBCH_TO_STAGE = { + 52: "silver_tip", + 53: "green_tip", + 54: "half_inch_green", + 56: "tight_cluster", + 57: "first_pink", + 59: "full_pink", + 61: "first_bloom", + 65: "full_bloom", + 69: "post_bloom", + 99: "fruitlet", +} +APPLE_DORMANT_BELOW_BBCH = 51 + +# Kill-fraction curve built on the table above [ENGINEERING] +# 0% kill at T10 + no_damage_offset, 10% at T10, 90% at T90, 100% at T90 - full_kill_offset +FROST_KILL_CURVE = { + "no_damage_offset_c": 1.0, + "full_kill_offset_c": 2.0, +} + +# Kill fraction where each severity level starts [ENGINEERING] +FROST_KILL_BANDS = {"edges": [0.0001, 0.10, 0.50, 0.90], "higher_is_worse": True} + +# Used only when the phenological stage is unknown (minimum temperature) [ENGINEERING] +# strict: LIGHT starts strictly below 0 C, matching the contract's "below 0 C" +FROST_FALLBACK_BANDS_C = {"edges": [0.0, -2.0, -4.0, -6.0], "higher_is_worse": False, "strict": True} + + +# ============================================================================= +# BRINA (hoar frost) +# ============================================================================= +# Contract (p.7): freezing of dew / sublimation of water vapour on the crop due to +# nocturnal radiation. NO temperature in the contract: all values are criteria for +# radiative frost with ice deposition. Severity: frost tables above. +# Brina and gelo share one guarantee, one excess (30%) and one window. + +BRINA = { + "night_solar_max_wm2": 10.0, # records with more radiation = daytime, ignored [ENGINEERING] + "calm_wind_max_ms": 2.0, # calm: bud offset + ice test; above: air < 0 C [ENGINEERING] + "t_air_max_c": 2.0, # 2 m air may be slightly above 0 C with brina [ENGINEERING] + "radiative_offset_c": 2.0, # bud temp = t_air - offset on calm nights [ENGINEERING] + "t_wet_max_c": 0.0, # wet-bulb <= this supports ice deposition [ENGINEERING] + "deposition": "bud_temp <= min(0, frost_point)", # calm hours: ice forms at the frost point + "min_duration_h": 0.5, # critical temps refer to ~30 min exposure [LITERATURE] +} + +# When the brina block runs and which night it looks at (RUN_SCHEDULE: sunset -> sunrise, +# approximated by fixed local hours) [PROJECT] +BRINA_SCHEDULE = { + "run_hour": 6, # local time; the night window ends here + "night_start_hour": 18, # window starts at this hour of the previous day +} + + +# ============================================================================= +# GELO (frost, "Freezing" box) +# ============================================================================= +# Contract (p.7): a drop in temperature below 0 C caused by cold air masses. + +GELO = { + "t_air_trigger_c": 0.0, # "a drop in temperature below 0 C" [CONTRACT p.7] + "t_air_trigger_op": "<", + "advective_wind_min_ms": 2.0, # mean wind while < 0 C in daylight: reported only [ENGINEERING] + "min_duration_h": 0.5, # ~30 min exposure [LITERATURE] +} + + +# ============================================================================= +# STRONG WIND (storm) +# ============================================================================= +# Contract (p.8): "at least force 7 on the Beaufort scale - a speed of 50 km/h or more +# (14 m/s)"; measurement valid below 10 m height. Contract does not say mean or gust. +# 13.9 m/s is used because 50 km/h and the Beaufort 7 lower bound are both 13.9 m/s; +# "14 m/s" is a rounded value. Set 14.0 to follow the bracketed figure instead +# (readings of 13.9 m/s would then no longer trigger). + +STRONG_WIND = { + "wind_trigger_ms": 13.9, # 50 km/h = Beaufort 7 lower bound [CONTRACT p.8] + "wind_trigger_op": ">=", # "at least" / "or more" + # 3A sends hourly wind only, so the contract's 10-min mean cannot be read directly. + # "hourly_max" = VelocitaVento2mMax (highest value of the hour: catches short storms; + # 3A does not say if it is a 10-min mean or a gust); "hourly_mean" = VelocitaVento2mMedia + # (the storm must last the whole hour). Chosen by the user 2026-10-22. [ENGINEERING] + "use_mean_or_gust": "hourly_max", + "min_duration_min": 10, # one 10-min averaging period [ENGINEERING] +} +STRONG_WIND_BANDS_MS = {"edges": [13.9, 17.2, 20.8, 24.5], "higher_is_worse": True} # Beaufort 7/8/9/10 [SCALE] + +# When the strong wind block runs and which hours it reads [PROJECT] +# Event day = the 24 h ending at run_hour (records after 07:00 of the previous day up to 07:00). +STRONG_WIND_SCHEDULE = { + "run_hour": 7, # local time; the event day ends here +} + + +# ============================================================================= +# HAIL +# ============================================================================= +# Contract (p.8): frozen water falling as ice pellets of varying sizes. ANY hail counts, +# no size threshold, 3-km rule does not apply. Weather sensors only give precursors; +# confirmation comes from the CV model / external hail data. +# No 3A station has a hail sensor, so the Hail block only flags POSSIBLE hail from these +# precursors and asks the camera for a photo, as Snow does (see detectors/hail.py). +# 3A data is hourly, so the 10-min and 30-min values are read per hour (comments below). + +HAIL = { + "rain_10min_mm": 5.0, # burst rainfall in 10 min; hourly data: compared as a + # rain rate (RainRateMax, station 293 only) >= 6 x this + # in mm/h [ENGINEERING] + "rain_1h_mm": 20.0, # burst rainfall in 1 h [ENGINEERING] + "t_drop_30min_c": 4.0, # temperature drop in 30 min (storm outflow); hourly + # data: previous hour's mean - this hour's minimum [ENGINEERING] + "gust_ms": 13.9, # squall gust (hourly maximum wind) [ENGINEERING] + "min_indicators": 2, # precursors (of the 3 above) for "likely" hail; 1 precursor + # = "possible", also asks for a photo (user's choice + # 2026-10-26: high recall, as Snow) [ENGINEERING] + "storm_window_h": 1, # a precursor counts only within N h of a rain hour (hail + # melts in the gauge); signs within N h of each other + # belong to the same storm [ENGINEERING] +} +HAIL_SIZE_BANDS_MM = {"edges": [5.0, 15.0, 25.0, 40.0], "higher_is_worse": True} # max stone diameter [SCALE] + +# Hail-risk day (user's idea 2026-10-26): hail melts fast, so on a risk day the Hail block +# also checks the day itself (daytime_run_hours below). Decided by the morning run from its own +# 24 h (no extra API request; 3A has no forecast): "today will be like yesterday". +# Risk = inside the season AND (a hail sign in the 24 h OR (highest air temperature >= t_max +# AND highest hourly dew point >= dew_point; no humidity data -> temperature alone)). +HAIL_RISK = { + "season": ("05-01", "09-30"), # "MM-DD" first and last day of the hail season [ENGINEERING] + "t_max_min_c": 25.0, # warm air (hourly maximum of the 24 h) [ENGINEERING] + "dew_point_min_c": 14.0, # moist air (dew point on the hourly mean temperature + # and humidity, 3A formula): fuel for thunderstorms [ENGINEERING] +} + +# When the hail block runs, which hours it reads and when the photo is asked for [PROJECT] +# Event day = the 24 h ending at run_hour (records after 07:00 of the previous day up to 07:00); +# storm_window_h more hours before it are fetched (temperature drop, rain near the first hour). +# Daytime checks (hail-risk days only): at each daytime_run_hours (+5 min) the daytime_window_h +# hours before it are checked (07-13, 13-19) and a photo is asked for at once. No night check: +# the camera cannot take a photo in the dark; the night is in the next morning's 24 h. +HAIL_SCHEDULE = { + "run_hour": 7, # local time; the event day ends here + "photo_hour": 9, # photo at this hour of the run day (daylight) + "daytime_run_hours": (13, 19), # extra checks on hail-risk days (local time) + "daytime_window_h": 6, # each daytime check reads the 6 h before it +} + + +# ============================================================================= +# SNOW (excessive snow) +# ============================================================================= +# Contract (p.8): snow causing significant MECHANICAL damage (breakage). No depth +# threshold. Needs a heated rain gauge or a snow-depth sensor. +# No 3A station has a snow sensor, so the Snow block only flags POSSIBLE snow from the +# weather data and asks the camera for a photo; the Hail/Snow CV model confirms and +# grades it (user's choice 2026-10-22). Signals A-D: see detectors/snow.py. + +SNOW = { + "t_wet_snow_max_c": 1.0, # precipitation counted as snow at wet-bulb <= this [LITERATURE] + "snow_liquid_ratio": 10.0, # cm of snow per cm of water (wet snow: 5-8) [ENGINEERING] + "foliage_level_bump": 1, # +levels when leaves/fruit are on the tree [ENGINEERING] + # suspicion signals (camera trigger), values suggested 2026-10-22, to calibrate: + "melt_lookahead_h": 12, # B: rain up to N h after a cold hour = snow melting + # in an unheated rain gauge [ENGINEERING] + "saturated_rh_min_pct": 95.0, # C: cold hour, no rain, humidity >= this [ENGINEERING] + "covered_solar_max_wm2": 20.0, # D: midday sun <= this after a suspect hour = + # pyranometer under snow (overcast: ~50-200) [ENGINEERING] + "midday_hours": (11, 14), # D: hourly records ending 11:00 ... 14:00 (local) [ENGINEERING] +} +# Depth bands: kept for the CV model (the Snow block does not grade severity) +SNOW_DEPTH_BANDS_CM = {"edges": [2.0, 10.0, 25.0, 50.0], "higher_is_worse": True} # [ENGINEERING] + +# When the snow block runs, which hours it reads and when the photo is asked for [PROJECT] +# Event day = the 24 h ending at run_hour (records after 07:00 of the previous day up to 07:00); +# melt_lookahead_h more hours before it are fetched for signal B. +SNOW_SCHEDULE = { + "run_hour": 7, # local time; the event day ends here + "photo_hour": 9, # photo at this hour of the run day (daylight, before melting) +} + + +# ============================================================================= +# EXCESSIVE RAIN (eccesso di pioggia) +# ============================================================================= +# Contract (p.8): any ONE of the three criteria below. +# Excluded: surface runoff, erosion, slope uprooting, rain during flowering (Art.5 q). +# Apple: rot/splitting only within 20 days before harvest (Art.2); earlier only root asphyxia. + +EXCESSIVE_RAIN = { + "downpour_1h_mm": 30.0, # "at least 30 mm over 1 hour" [CONTRACT p.8] + "downpour_1h_op": ">=", + "intense_72h_mm": 80.0, # "at least 80 mm within 72 hours" [CONTRACT p.8] + "intense_72h_op": ">=", + "prolonged_10d_ratio": 1.5, # "exceeding the 5-year average by more than 50%" [CONTRACT p.8] + "prolonged_10d_ratio_op": ">", + "prolonged_10d_min_mm": 80.0, # "in no case less than 80 mm" [CONTRACT p.8] + "prolonged_10d_min_op": ">=", + "rot_window_before_harvest_days": 20, # [CONTRACT Art.2 p.13] +} +# Severity = rainfall / the trigger of the criterion exceeded the most. +# Grades only events already detected with the operators above. +EXCESSIVE_RAIN_BANDS_RATIO = {"edges": [1.0, 1.5, 2.0, 3.0], "higher_is_worse": True} # [ENGINEERING] + +# When the flood (excessive rain) block runs and which hours it reads [PROJECT] +# Event day = the 24 h ending at run_hour; the 1 h / 72 h / 10-day criteria are checked +# for windows ending inside it, so look-back_days of hourly rain are fetched (1 request). +# 07:00, not 06:00: two jobs in the same hour would make the scheduler skip one. +FLOOD_SCHEDULE = { + "run_hour": 7, # local time; the event day ends here + "lookback_days": 10, # longest criterion: prolonged rain over 10 days + # claim row of the client report (Heavy Rain Watch): only the rain of the last N days + # counts, in 72 h windows (since 2026-10-26); needs N + 3 <= lookback_days to tell + # whether rain fell just before the N days + "claim_search_days": 4, +} + + +# ============================================================================= +# HOT TEMPERATURE ("High temperature" box) +# ============================================================================= +# Contract heat covers for apples (p.8): +# - Sunscald: air >= 40 C at 2 m -> the trigger used by this block +# - Heatwave: t_min > 28 C AND t_max > 40 C on 15 consecutive days, fruit set -> 30 Sep, +# 30-day start lag (Special Art.5 p.31, Art.6 p.15) -> OUT OF SCOPE +# - Hot wind: separate block below +# Sunburn starts earlier because the sunlit fruit surface is 10-15 C above air, so +# detection and severity use the estimated fruit surface temperature (FST): +# FST = t_air + fst_gain_max_c * (solar / 1000) / (1 + fst_wind_coeff * wind) +# Damage below 40 C air is real but not insurable under Model 1506. + +HOT_TEMPERATURE = { + "contract_t_max_c": 40.0, # Sunscald: "at least 40 C" at 2 m [CONTRACT p.8] + "contract_t_max_op": ">=", + "risk_t_air_c": 32.0, # sunburn risk rises above 90 F air [LITERATURE WSU] + "daytime_solar_min_wm2": 300.0, # only sunlit records count [ENGINEERING] + "fst_gain_max_c": 15.0, # FST above air at 1000 W/m2, calm (WSU: 10-15 C) [LITERATURE] + "fst_wind_coeff": 0.3, # wind cooling, 1/(m/s) - calibrate with fruit probe [ENGINEERING] + "browning_duration_min": 60, # FST held this long for browning classes [LITERATURE] + "necrosis_duration_min": 10, # FST held this long for necrosis classes [LITERATURE] +} +# Fruit surface temperature: 46 browning (sensitive cultivars), 49 browning (most cultivars), +# 52 necrosis [LITERATURE Schrader et al. 2003; Racsko & Schrader 2012], 55 extreme [ENGINEERING] +HOT_TEMPERATURE_FST_BANDS_C = {"edges": [46.0, 49.0, 52.0, 55.0], "higher_is_worse": True} + +# When the high temperature block runs and which hours it reads [PROJECT] +# Event day = the 24 h ending at run_hour (records after 07:00 of the previous day up to 07:00). +HIGH_TEMPERATURE_SCHEDULE = { + "run_hour": 7, # local time; the event day ends here +} + + +# ============================================================================= +# HOT WIND (scirocco / libeccio) +# ============================================================================= +# Contract (p.8): warm air masses from SE to SW with >= 40 C at 2 m, causing withering +# in the 30 days before harvest. Needs a wind-direction sensor. + +HOT_WIND = { + "t_max_trigger_c": 40.0, # "at least 40 C" at 2 m [CONTRACT p.8] + "t_max_trigger_op": ">=", + "dir_from_min_deg": 135.0, # south-east [CONTRACT p.8] + "dir_from_max_deg": 225.0, # south-west [CONTRACT p.8] + "wind_min_ms": 1.0, # 'movement of air masses', no speed in contract [ENGINEERING] + "days_before_harvest": 30, # [CONTRACT p.8] +} +HOT_WIND_BANDS_C = {"edges": [40.0, 42.0, 44.0, 46.0], "higher_is_worse": True} # air t_max [ENGINEERING] + +# When the hot wind block runs and which hours it reads [PROJECT] +# Event day = the 24 h ending at run_hour (records after 07:00 of the previous day up to 07:00). +HOT_WIND_SCHEDULE = { + "run_hour": 7, # local time; the event day ends here +} + + +# ============================================================================= +# DROUGHT (siccità) +# ============================================================================= +# No contract trigger in this file (Model 1506 is not transcribed here for drought), so +# every value below is a project default to calibrate. The block compares the rain +# measured by the 3A station over the last complete calendar months with the climate +# average of the same months (RAIN_NORMALS_MM): a meteorological drought (rain deficit). +# It does not know about irrigation, so it says "dry period", not "crop damaged". + +DROUGHT = { + "window_months": 3, # complete calendar months checked (rain total) [ENGINEERING] + "rain_pct_of_normal_max": 60.0, # event: rain of the window <= this % of the average [ENGINEERING] + "rain_pct_of_normal_op": "<=", + "max_missing_days_pct": 20.0, # more days without rain data than this % -> no_data + # (a deficit could be only missing data) [ENGINEERING] +} +# Severity = rain of the window as % of its average (lower is worse) [ENGINEERING] +DROUGHT_BANDS_PCT = {"edges": [60.0, 45.0, 30.0, 15.0], "higher_is_worse": False} + +# When the drought block runs [PROJECT] +# The pipeline runs every day; drought runs only on this day of the month (the diagram: +# "Run every month"). python -m damage_engine.main --once drought runs it on any day. +DROUGHT_SCHEDULE = { + "run_day_of_month": 1, +} + +# Climate average of the monthly rain, mm, Jan ... Dec, per 3A station id. [EXTERNAL] +# 1991-2020 mean of the monthly totals, CERRA reanalysis (Copernicus European Regional +# ReAnalysis, 5.5 km grid), read once from Open-Meteo's historical-weather archive +# (archive-api.open-meteo.com, models=cerra) at each station's coordinates, 2026-10-22. +# Modelled, not measured: about 15 % above the measured Cuneo average (~990 mm/yr, +# 2002-2020); ERA5 (31 km grid) was 40-60 % too high and is not used. Stations in the +# same grid cell share the same values (5 / 12 / 544, 10 / 600, 293 / 566). +# Replace a row with measured averages (e.g. ARPA Piemonte, or 3A's own history once it +# has enough years) when available. A station missing here gets "no_data" from Drought. +RAIN_NORMALS_SOURCE = "CERRA reanalysis 1991-2020 via Open-Meteo (read 2026-10-22)" +RAIN_NORMALS_MM = { + # Jan Feb Mar Apr May Jun Jul Aug Sep Oct Nov Dec + 3: ( 53.2, 63.3, 82.8, 132.5, 132.1, 110.8, 81.7, 90.2, 95.0, 99.3, 117.0, 63.5), # Lagnasco, 1121 mm/yr + 5: ( 53.1, 65.3, 85.2, 139.2, 139.2, 113.5, 77.8, 85.9, 103.6, 110.8, 124.5, 64.1), # Castellar, 1162 + 6: ( 53.9, 63.1, 82.9, 140.5, 138.3, 104.2, 73.3, 74.6, 101.7, 111.6, 131.2, 63.5), # Cavour, 1139 + 7: ( 52.6, 66.2, 91.5, 150.9, 163.0, 139.2, 111.2, 110.8, 109.6, 113.3, 122.4, 65.6), # Costigliole di Saluzzo, 1296 + 9: ( 50.3, 58.3, 77.4, 121.4, 118.6, 97.2, 73.0, 74.7, 84.1, 89.9, 108.9, 60.5), # Centallo, 1014 + 10: ( 49.3, 59.3, 81.6, 136.7, 141.7, 118.4, 80.2, 84.3, 101.2, 111.2, 120.9, 60.1), # Revello, 1145 + 12: ( 53.1, 65.3, 85.2, 139.2, 139.2, 113.5, 77.8, 85.9, 103.6, 110.8, 124.5, 64.1), # Saluzzo, 1162 + 30: ( 56.5, 64.8, 86.2, 132.3, 129.7, 107.2, 84.5, 91.2, 95.2, 100.9, 120.8, 67.3), # Fossano Maddalene, 1137 + 145: ( 52.0, 63.6, 84.0, 134.7, 133.8, 109.9, 79.4, 82.7, 95.7, 102.7, 117.0, 62.6), # Falicetto, 1118 + 153: ( 50.6, 58.2, 80.8, 137.0, 149.9, 129.2, 98.5, 108.4, 99.5, 105.4, 112.8, 63.2), # Caraglio, 1194 + 293: ( 52.7, 61.5, 82.2, 131.0, 127.2, 102.1, 80.5, 74.0, 87.2, 98.3, 113.2, 64.7), # Cuneo San Benigno, 1075 + 544: ( 53.1, 65.3, 85.2, 139.2, 139.2, 113.5, 77.8, 85.9, 103.6, 110.8, 124.5, 64.1), # Revello San Pietro, 1162 + 566: ( 52.7, 61.5, 82.2, 131.0, 127.2, 102.1, 80.5, 74.0, 87.2, 98.3, 113.2, 64.7), # Cuneo Giordano, 1075 + 600: ( 49.3, 59.3, 81.6, 136.7, 141.7, 118.4, 80.2, 84.3, 101.2, 111.2, 120.9, 60.1), # Cascina Frati, 1145 +} + + +# ============================================================================= +# EXCESS, LIMITS (apples) +# ============================================================================= + +# Policy package of the insured farm [CONTRACT Art.7 p.16] +# "A" -> one 30% ABSOLUTE excess for every peril (EXCESS_PCT is ignored) +# "B", "C", "F" -> fixed excess per peril (EXCESS_PCT) + combined excess (COMBINED_EXCESS) +POLICY_PACKAGE = "B" +PACKAGE_A_ABSOLUTE_EXCESS_PCT = 30 + +# Optional higher hail excess chosen by the insured (Packages B/C/F): None, 15, 20 or 30. +# If set above the table value, the strong-wind excess is raised to the same value. +# With 30%, combined damage with other perils stays fixed at 30%. [CONTRACT Art.7(c) p.18] +OPTIONAL_HAIL_EXCESS_PCT = None +OPTIONAL_HAIL_EXCESS_CHOICES = (15, 20, 30) + +# Fixed excess for a single event (Packages B/C/F, before any optional raise) [CONTRACT Art.7(a) p.17] +EXCESS_PCT = { + "hail": 15, + "strong_wind": 15, + "other": 30, # brina, gelo, excessive rain, snow, hot temperature, hot wind +} + +# Combined events [CONTRACT Art.7(b) p.18] +COMBINED_EXCESS = { + "hail_plus_wind": "strong_wind excess of the product", + "frost_plus_any_pct": 30, # brina/gelo + any other event on fruit + "hail_wind_plus_other_fixed_pct": 30, # if total <= 30% or hail/wind < 10% + "hail_wind_plus_other_total_above_pct": 30, + "hail_wind_plus_other_min_hw_pct": 10, + "hail_wind_plus_other_reduction_per_pt": 1, # -1 pt of excess per pt of hail/wind above 30% + "hail_wind_plus_other_floor_pct": 20, +} + +# Indemnity limit (% of insured value, None = no cap) and scoperto (%) [CONTRACT Art.7 p.19] +# Strong wind: apples fall under "All other products 0" -> read as no cap (verify). +# Combined damage (except hail+wind): limits of the predominant peril; tie -> most severe. +INDEMNITY_LIMITS = { + "hail": {"limit_pct": 80, "scoperto_pct": 10}, + "strong_wind": {"limit_pct": None, "scoperto_pct": 10}, + "frost": {"limit_pct": 50, "scoperto_pct": 10}, # brina, gelo + "other": {"limit_pct": 60, "scoperto_pct": 10}, +} + + +# ============================================================================= +# APPLE DAMAGE CLASSES (targets for the damage models) [CONTRACT Special Art.6 p.32] +# ============================================================================= +# A fruit is in the first class whose limits it respects. Quality loss applies to the +# produce remaining after the quantity loss. Class a allows no necrotic bruise +# (0.05 cm2 necrotic limit is for stone fruit only). + +APPLE_QUALITY_CLASSES = { + "a": {"elongated_max_cm": 2.0, "defect_max_cm2": 1.0, "bruise_max_cm2": 0.20, + "bruise_necrotic_max_cm2": 0.0, "flesh_depth_max_mm": 0.0, + "quantity_loss_pct": 0, "quality_loss_pct": 0}, + "b": {"elongated_max_cm": 4.0, "defect_max_cm2": 2.5, "bruise_max_cm2": 0.75, + "bruise_necrotic_max_cm2": 0.25, "flesh_depth_max_mm": 3.0, + "quantity_loss_pct": 0, "quality_loss_pct": 40}, + "c": {"description": "worse than b, industrial processing only", + "quantity_loss_pct": 0, "quality_loss_pct": 90}, + "d": {"description": "lost, destroyed or rotten", + "quantity_loss_pct": 100, "quality_loss_pct": 0}, +} + + +# ============================================================================= +# REPORT +# ============================================================================= + +REPORT = { + "history_runs": 10, # past runs kept per block [PROJECT] +} + + +# ============================================================================= +# INDEX: everything for one phenomenon, by name +# ============================================================================= + +PHENOMENA = { + "brina": {"params": BRINA, "bands": FROST_KILL_BANDS}, + "gelo": {"params": GELO, "bands": FROST_KILL_BANDS}, + "strong_wind": {"params": STRONG_WIND, "bands": STRONG_WIND_BANDS_MS}, + "hail": {"params": HAIL, "bands": HAIL_SIZE_BANDS_MM}, + "snow": {"params": SNOW, "bands": SNOW_DEPTH_BANDS_CM}, + "excessive_rain": {"params": EXCESSIVE_RAIN, "bands": EXCESSIVE_RAIN_BANDS_RATIO}, + "flood": {"params": EXCESSIVE_RAIN, "bands": EXCESSIVE_RAIN_BANDS_RATIO}, # "Flood" box = excessive rain + "hot_temperature": {"params": HOT_TEMPERATURE, "bands": HOT_TEMPERATURE_FST_BANDS_C}, + "hot_wind": {"params": HOT_WIND, "bands": HOT_WIND_BANDS_C}, + "drought": {"params": DROUGHT, "bands": DROUGHT_BANDS_PCT}, +} diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_brina.py b/tests/test_brina.py new file mode 100644 index 0000000..b1ec1c4 --- /dev/null +++ b/tests/test_brina.py @@ -0,0 +1,149 @@ +from datetime import date, datetime + +import pandas as pd + +from damage_engine.detectors import brina +from damage_engine.schemas import EVENT, NO_DATA, NO_EVENT, Severity +from damage_engine.sources.fields import Field + +RUN_DAY = date(2025, 3, 12) + + +def _night(station, temps, rh, wind=None, solar=None): + ts = pd.date_range("2025-03-11 18:00", periods=len(temps), freq="h") + df = pd.DataFrame({"CodiceStazione": station, "Timestamp": ts, + brina.COL_TEMP: temps, brina.COL_RH: rh, brina.COL_RH_MAX: rh}) + if wind is not None: + df[brina.COL_WIND] = wind + if solar is not None: + df[brina.COL_SOLAR] = solar + return df + + +def _field(station, bbch=None, fid=None): + return Field(id=fid or f"ST{station}", name=f"field {station}", station_id=station, bbch=bbch) + + +COLD_NIGHT = [3, 2, 1, 0.5, -0.5, -1.5, -2.5, -3, -2, -1, 0.5, 2, 3] + + +def test_night_window(): + assert brina.night_window(RUN_DAY) == (datetime(2025, 3, 11, 18), datetime(2025, 3, 12, 6)) + + +def test_cold_calm_humid_night_is_brina_fallback_severity(): + df = _night(3, COLD_NIGHT, [95] * 13, [0.5] * 13) + [a] = brina.detect(df, [_field(3)], RUN_DAY, {3: "Lagnasco"}) + assert a.status == EVENT and a.block == "brina" + assert a.field_id == "ST3" and a.station_name == "Lagnasco" + assert a.duration_h == 6 # not the 0.5 °C hours: wet-bulb +0.22 °C (3A formula) + assert a.details["min_bud_temp_c"] == -5 # -3 °C air - 2 °C radiative offset + assert a.severity == Severity.SEVERE # fallback bands: -4 >= x > -6 + assert a.details["severity_method"] == "fallback_min_bud_temp" + assert a.details["in_cover_window"] is None # stage unknown + + +def test_severity_uses_field_stage_when_known(): + df = _night(3, COLD_NIGHT, [95] * 13, [0.5] * 13) + bloom, dormant = brina.detect(df, [_field(3, 65, "A"), _field(3, 40, "B")], RUN_DAY) + assert bloom.details["stage"] == "full_bloom" + assert bloom.details["kill_fraction"] > 0.9 + assert bloom.severity == Severity.EXTREME + assert bloom.details["in_cover_window"] is True + assert dormant.status == EVENT and dormant.severity == Severity.NONE + + +def test_dry_or_daytime_night_is_not_brina(): + temps = [-3] * 13 + dry = _night(5, temps, [40] * 13) + sunny = _night(293, temps, [95] * 13, [0.5] * 13, [500.0] * 13) + out = brina.detect(pd.concat([dry, sunny]), [_field(5), _field(293)], RUN_DAY) + assert [a.status for a in out] == [NO_EVENT] * 2 + assert all(a.severity == Severity.NONE for a in out) + + +def test_windy_sub_zero_night_hour_is_brina_without_ice_test(): + windy_humid = _night(3, [-3] * 13, [95] * 13, [5.0] * 13) # no ice deposit at bud = air + windy_dry = _night(6, [-3] * 13, [40] * 13, [5.0] * 13) + windy_zero = _night(10, [0.0] * 13, [40] * 13, [5.0] * 13) # contract: "below 0 °C" + windy_mild = _night(30, [1.5] * 13, [40] * 13, [5.0] * 13) # air <= 2 °C, wet-bulb <= 0, not sub-zero + calm_humid = _night(544, [-3] * 13, [95] * 13, [0.5] * 13) # calm: ice test, buds 2 °C colder + calm_dry = _night(566, [-3] * 13, [40] * 13, [0.5] * 13) # calm and dry: no ice, no brina + out = brina.detect(pd.concat([windy_humid, windy_dry, windy_zero, windy_mild, calm_humid, calm_dry]), + [_field(i) for i in (3, 6, 10, 30, 544, 566)], RUN_DAY) + assert [a.status for a in out] == [EVENT, EVENT, NO_EVENT, NO_EVENT, EVENT, NO_EVENT] + hum, dry, calm = out[0], out[1], out[4] + assert hum.duration_h == 13 and hum.details["min_bud_temp_c"] == -3 # bud = air when windy + assert hum.details["windy_hours"] == 13 and hum.details["max_wind_ms"] == 5.0 + assert dry.severity == hum.severity == Severity.MODERATE # fallback: -3 °C + assert calm.details["min_bud_temp_c"] == -5 and calm.details["windy_hours"] == 0 + rows = brina.brina_hours(windy_mild) + assert (rows["wet_bulb_c"] <= 0).all() # only "below 0 °C" stops it + + +def test_station_without_wind_sensor_still_evaluated(): + [a] = brina.detect(_night(9, [-5] * 13, [92] * 13), [_field(9)], RUN_DAY, {9: "Centallo"}) + assert a.status == EVENT + assert a.details["wind_checked"] is False + assert a.severity == Severity.EXTREME # bud -7 °C < -6 °C + + +def test_field_without_station_records_is_no_data(): + [a] = brina.detect(_night(3, COLD_NIGHT, [95] * 13), [_field(77)], RUN_DAY) + assert a.status == NO_DATA + + +def test_out_of_cover_window_flagged(): + df = _night(3, COLD_NIGHT, [95] * 13, [0.5] * 13) + [a] = brina.detect(df, [_field(3, 65)], date(2025, 10, 19)) + assert a.details["in_cover_window"] is False + + +def test_missing_sensors_and_data_are_noted(): + df = _night(9, [-5] * 13, [92] * 13) # no wind, no solar columns + [a] = brina.detect(df, [_field(9)], RUN_DAY, station_sensors={9: {"TemperaturaAria", "UmiditaRelativa"}}) + assert any(n.startswith("No wind sensor at this station") for n in a.data_notes) + assert any(n.startswith("No solar radiation sensor at this station") for n in a.data_notes) + + +def test_partly_missing_hours_are_noted(): + solar = [0.0] * 10 + [None] * 3 + rh = [95] * 12 + [None] + df = _night(293, COLD_NIGHT, rh, [0.5] * 13, solar) + [a] = brina.detect(df, [_field(293)], RUN_DAY) + assert "Solar radiation missing for 3 of 13 hours" in " ".join(a.data_notes) + assert "Temperature or humidity missing for 1 of 13 hours" in " ".join(a.data_notes) + + +def test_complete_station_has_no_notes_and_no_records_is_noted(): + full = _night(3, COLD_NIGHT, [95] * 13, [0.5] * 13, [0.0] * 13) + ok, missing = brina.detect(full, [_field(3), _field(77)], RUN_DAY) + assert ok.data_notes == [] + assert missing.data_notes == ["No records from the station last night."] + + +def test_3a_wet_bulb_is_reported_but_not_used_to_detect(): + df = _night(566, COLD_NIGHT, [95] * 13, [0.5] * 13, [1.0] * 13) + df[brina.COL_WET_BULB_3A] = [5.0] * 13 # warm on purpose: must not block the event + [a] = brina.detect(df, [_field(566)], RUN_DAY) + assert a.status == EVENT and a.duration_h == 6 # same as without TBulbMed + assert a.details["min_wet_bulb_3a_c"] == 5.0 + [b] = brina.detect(_night(3, COLD_NIGHT, [95] * 13, [0.5] * 13), [_field(3)], RUN_DAY) + assert "min_wet_bulb_3a_c" not in b.details # station without TBulbMed + + +def test_solar_column_is_hourly_irradiance(): + assert brina.COL_SOLAR == "RadiazioneSolareMax" # W/m²; RadiazioneSolareInt is MJ/m², daily only + + +def test_wet_bulb_is_the_shared_lowest_of_the_hour(): + df = _night(3, COLD_NIGHT, [95] * 13, [0.5] * 13) + rows = brina.brina_hours(df) + assert rows["wet_bulb_c"].equals(rows["WetBulbMin"]) # one value for the whole pipeline + + +def test_missing_max_humidity_uses_mean_and_is_noted(): + df = _night(3, COLD_NIGHT, [95] * 13, [0.5] * 13).drop(columns=[brina.COL_RH_MAX]) + [a] = brina.detect(df, [_field(3)], RUN_DAY) + assert a.duration_h == 6 # RH max = RH mean here: same result + assert "Hourly maximum humidity missing for 13 of 13 hours" in " ".join(a.data_notes) diff --git a/tests/test_derived.py b/tests/test_derived.py new file mode 100644 index 0000000..8ea9539 --- /dev/null +++ b/tests/test_derived.py @@ -0,0 +1,59 @@ +import numpy as np +import pandas as pd +import pytest + +from damage_engine.sources import derived + +# real hours of station 566 (2026-10-19): hourly mean T (°C), mean RH (%), 3A's TBulbMed (°C) +STATION_566 = [(13.3, 99, 13.2), (13.8, 90, 12.8), (15.6, 82, 13.8), + (15.8, 78, 13.6), (15.9, 76, 13.4), (15.2, 78, 13.0)] + + +@pytest.mark.parametrize("t, rh, tbulb_3a", STATION_566) +def test_formula_reproduces_3a_tbulbmed(t, rh, tbulb_3a): + # same formula as 3A (document section 4): equal to 3A's value at its 0.1 °C rounding + assert abs(float(derived.wet_bulb_c(t, rh)) - tbulb_3a) <= 0.05 + + +def test_saturated_air_wet_bulb_equals_air_temperature(): + for t in (-5.0, 0.0, 10.0): + assert abs(float(derived.wet_bulb_c(t, 100)) - t) < 1e-6 + + +def test_wet_bulb_below_air_when_dry_and_ice_branch_below_zero(): + assert float(derived.wet_bulb_c(20, 50)) < 20 + assert float(derived.wet_bulb_c(-3, 80)) < -3 + # T <= 0 uses the ice constants: the result is continuous enough across 0 °C + assert abs(float(derived.wet_bulb_c(0.0, 90)) - float(derived.wet_bulb_c(0.01, 90))) < 0.2 + + +def test_dew_point_at_saturation_is_air_temperature(): + assert abs(float(derived.dew_point_c(12.0, 100)) - 12.0) < 1e-6 + + +def test_add_wet_bulb_pairs_tmin_rhmax_and_tmax_rhmin(): + df = pd.DataFrame({derived.T_MIN: [14.6], derived.T_MEAN: [15.6], derived.T_MAX: [17.5], + derived.RH_MIN: [70.0], derived.RH_MEAN: [82.0], derived.RH_MAX: [90.0]}) + out = derived.add_wet_bulb(df) + assert out[derived.WET_BULB_MIN][0] == round(float(derived.wet_bulb_c(14.6, 90)), 2) + assert out[derived.WET_BULB_MAX][0] == round(float(derived.wet_bulb_c(17.5, 70)), 2) + assert out[derived.WET_BULB_MIN][0] <= out[derived.WET_BULB_MAX][0] + assert derived.WET_BULB_MIN not in df # input not changed + + +def test_add_wet_bulb_falls_back_to_means_and_never_fails(): + df = pd.DataFrame({derived.T_MEAN: [5.0, None], derived.RH_MEAN: [90.0, 80.0]}) + out = derived.add_wet_bulb(df) # no min/max columns at all + expected = round(float(derived.wet_bulb_c(5.0, 90)), 2) + assert out[derived.WET_BULB_MIN][0] == expected == out[derived.WET_BULB_MAX][0] + assert np.isnan(out[derived.WET_BULB_MIN][1]) # no temperature -> NaN + assert derived.add_wet_bulb(pd.DataFrame({"x": [1]}))[derived.WET_BULB_MIN].isna().all() + + +def test_frost_point_below_zero_uses_ice_constants(): + assert abs(float(derived.dew_point_c(0, 100))) < 1e-9 + assert float(derived.dew_point_c(-3, 40)) < -10 + # T <= 0 °C: e = RH * 6.1 * 10^(9.5 T / (T + 265.5)), inverted with the same constants + e = 0.95 * 6.1 * 10 ** (9.5 * -3 / (-3 + 265.5)) + x = np.log10(e / 6.1) + assert abs(float(derived.dew_point_c(-3, 95)) - 265.5 * x / (9.5 - x)) < 1e-9 diff --git a/tests/test_drought.py b/tests/test_drought.py new file mode 100644 index 0000000..f16dee6 --- /dev/null +++ b/tests/test_drought.py @@ -0,0 +1,141 @@ +from datetime import date, datetime + +import numpy as np +import pandas as pd +import pytest + +from damage_engine.detectors import drought +from damage_engine.schemas import EVENT, NO_DATA, NO_EVENT, Severity +from damage_engine.sources.fields import Field +from damage_engine.thresholds import RAIN_NORMALS_MM + +RUN_DAY = date(2026, 11, 1) # checks Aug, Sep, Oct 2026 (92 days) +DAYS = pd.date_range("2026-08-01", "2026-10-31", freq="D") +NORMAL_566 = sum(RAIN_NORMALS_MM[566][7:10]) # 74.0 + 87.2 + 98.3 = 259.5 mm +ALL = {"Pioggia", "TemperaturaAria", "UmiditaRelativa"} +SOIL = ALL | {"Suolo1", "Suolo2"} + + +def _daily(station, rain_total, et0=2.0, drop=None, days=DAYS): + """Daily rows with rain_total spread evenly; `drop` = indexes of days without any row.""" + rain = [rain_total / len(days)] * len(days) + df = pd.DataFrame({"CodiceStazione": station, "Data": days, "Pioggia": rain, + "EvapoPot": [et0] * len(days)}) + return df.drop(index=list(drop)) if drop else df + + +def _one(df, station=566, sensors=ALL, soil=None, soil_error=None, day=RUN_DAY): + f = Field(id=f"ST{station}", name=f"field {station}", station_id=station) + [a] = drought.detect(df, [f], day, station_sensors={station: sensors}, soil=soil, + soil_error=soil_error) + return a + + +def test_schedule_window_and_labels(): + assert drought.due(date(2026, 11, 1)) and not drought.due(date(2026, 11, 2)) + assert drought.window_months(RUN_DAY) == [date(2026, 8, 1), date(2026, 9, 1), date(2026, 10, 1)] + assert drought.window_months(date(2026, 10, 22)) == [date(2026, 7, 1), date(2026, 8, 1), + date(2026, 9, 1)] + assert drought.window(RUN_DAY) == (datetime(2026, 8, 1), datetime(2026, 11, 1)) + assert drought.months_label(drought.window_months(RUN_DAY)) == "Aug–Oct 2026" + assert drought.months_label(drought.window_months(date(2026, 1, 1))) == "Oct–Dec 2025" + assert drought.months_label(drought.window_months(date(2026, 2, 1))) == "Nov 2025–Jan 2026" + assert drought.normals_for(566, drought.window_months(RUN_DAY)) == [74.0, 87.2, 98.3] + assert drought.normals_for(999, drought.window_months(RUN_DAY)) is None + + +def test_usual_rain_is_no_drought(): + a = _one(_daily(566, NORMAL_566)) + assert a.status == NO_EVENT and a.severity == Severity.NONE + d = a.details + assert (d["rain_mm"], d["normal_mm"], d["rain_pct_of_normal"]) == (259.5, 259.5, 100.0) + assert [m["label"] for m in d["months"]] == ["Aug 2026", "Sep 2026", "Oct 2026"] + assert [m["normal_mm"] for m in d["months"]] == [74.0, 87.2, 98.3] + assert d["window_label"] == "Aug–Oct 2026" and d["window_start"] == "2026-08-01T00:00" + assert "no drought" in a.summary and "100 % of the average" in a.summary + assert a.start is None and a.duration_h == 0 + + +def test_dry_months_are_a_drought_graded_on_percent_of_average(): + a = _one(_daily(566, 50.0)) # 19.3 % of 259.5 mm + assert a.status == EVENT and a.severity == Severity.SEVERE # <= 30 %, > 15 % + assert 0.75 <= a.severity_score < 1.0 + d = a.details + assert d["rain_pct_of_normal"] == 19.3 and d["deficit_mm"] == 209.5 + assert (a.start, a.end) == (datetime(2026, 8, 1), datetime(2026, 11, 1)) + assert a.duration_h == 92 * 24 + assert a.summary.startswith("Dry period: 50 mm of rain in Aug–Oct 2026, only 19.3 %") + assert "Oct 2026: 17.1 % of its average" in a.summary # 16.85 mm of 98.3 mm + + +@pytest.mark.parametrize("rain, status, severity", [ + (156.0, NO_EVENT, Severity.NONE), # 60.1 % + (155.0, EVENT, Severity.LIGHT), # 59.7 % + (100.0, EVENT, Severity.MODERATE), # 38.5 % + (30.0, EVENT, Severity.EXTREME), # 11.6 % +]) +def test_trigger_and_bands(rain, status, severity): + a = _one(_daily(566, rain)) + assert (a.status, a.severity) == (status, severity) + + +def test_one_dry_month_alone_is_not_enough(): + rain = [0.0] * 61 + [9.0] * 31 # Aug, Sep dry; Oct 279 mm + df = pd.DataFrame({"CodiceStazione": 566, "Data": DAYS, "Pioggia": rain, "EvapoPot": 2.0}) + a = _one(df) + assert a.status == NO_EVENT and a.details["rain_pct_of_normal"] == 107.5 + assert [m["rain_pct_of_normal"] for m in a.details["months"]] == [0.0, 0.0, 283.8] + + +def test_water_balance_from_et0(): + d = _one(_daily(566, 92.0, et0=3.0)).details + assert (d["et0_mm"], d["water_balance_mm"], d["water_balance_months"]) == (276.0, -184.0, 3) + assert d["months"][0]["water_balance_mm"] == -62.0 + + +def test_missing_days_are_a_lower_bound_until_too_many(): + a = _one(_daily(566, 92.0, drop=range(10))) # 10 of 92 days missing (10.9 %) + assert a.status == EVENT and a.details["missing_days"] == 10 and a.details["rain_mm"] == 82.0 + assert any("Rain missing for 10 of 92 days (Aug 2026 10)" in n for n in a.data_notes) + b = _one(_daily(566, 92.0, drop=range(20))) # 21.7 % > 20 % + assert b.status == NO_DATA and b.details["reason"] == "too many days without rain data" + assert "could be only missing data" in b.data_notes[0] and b.summary == b.data_notes[0] + + +def test_no_records_or_no_average_is_no_data(): + a = _one(pd.DataFrame()) + assert a.status == NO_DATA and "No daily rain data for Aug–Oct 2026" in a.data_notes[0] + b = _one(_daily(999, 50.0), station=999) + assert b.status == NO_DATA and "No average rain for station 999" in b.data_notes[0] + + +def test_missing_et0_column_and_no_soil_sensor_are_notes(): + df = _daily(566, 259.5).drop(columns=["EvapoPot"]) + a = _one(df) + assert a.status == NO_EVENT and a.details["water_balance_mm"] is None + assert any("No evapotranspiration data" in n for n in a.data_notes) + assert any("No soil moisture sensor" in n for n in a.data_notes) + assert any("CERRA" in n for n in a.data_notes) # where the average comes from + + +def _soil(station, s1, s2=None): + hours = pd.date_range("2026-10-01 01:00", "2026-11-01 00:00", freq="h") + first = hours <= pd.Timestamp("2026-10-08 00:00") + v1 = np.where(first, s1[0], s1[1]) + df = pd.DataFrame({"CodiceStazione": station, "Timestamp": hours, "Suolo1": v1}) + if s2 is not None: + df["Suolo2"] = s2 + return df + + +def test_soil_moisture_of_the_last_month_is_information_only(): + a = _one(_daily(600, 50.0), station=600, sensors=SOIL, soil=_soil(600, (30.0, 18.0), 25.0)) + soil = a.details["soil_moisture"] + assert soil["Suolo1"]["first_week_pct"] == 30.0 and soil["Suolo1"]["last_week_pct"] == 18.0 + assert soil["Suolo2"]["mean_pct"] == 25.0 + assert a.status == EVENT # decided by the rain only + b = _one(_daily(600, 50.0), station=600, sensors=SOIL, soil_error="ConnectionError") + assert b.details["soil_moisture"] is None and b.status == EVENT + assert any("could not be read" in n for n in b.data_notes) + c = _one(_daily(600, 50.0), station=600, sensors=SOIL) + assert any("No soil moisture data" in n for n in c.data_notes) diff --git a/tests/test_flood.py b/tests/test_flood.py new file mode 100644 index 0000000..f44fe08 --- /dev/null +++ b/tests/test_flood.py @@ -0,0 +1,189 @@ +from datetime import date, datetime + +import numpy as np +import pandas as pd + +from damage_engine.detectors import flood +from damage_engine.schemas import EVENT, NO_DATA, NO_EVENT, Severity +from damage_engine.sources.fields import Field + +RUN_DAY = date(2025, 6, 12) +HOURS = 24 * flood.LOOKBACK_DAYS # 240 hourly records in the fetch window + + +def _rain(station=3, rain=None): + """Hourly rain over the whole fetch window; `rain` = 240 values (default dry).""" + start, end = flood.fetch_window(RUN_DAY) + ts = pd.date_range(start, end, freq="h") + values = np.zeros(len(ts)) if rain is None else np.asarray(rain, dtype=float) + return pd.DataFrame({"CodiceStazione": station, "Timestamp": ts, flood.COL_RAIN: values}) + + +def _with_day(day_values, before=0.0): + """240 values: `before` mm/h in the 9 days before, then the 24 event-day values.""" + return [before] * (HOURS - 24) + list(day_values) + + +def _field(station=3, bbch=None): + return Field(id=f"ST{station}", name=f"field {station}", station_id=station, bbch=bbch) + + +def _one(df, bbch=None, **kw): + [a] = flood.detect(df, [_field(bbch=bbch)], RUN_DAY, {3: "Lagnasco"}, **kw) + return a + + +def test_windows(): + assert flood.event_window(RUN_DAY) == (datetime(2025, 6, 11, 7), datetime(2025, 6, 12, 7)) + start, end = flood.fetch_window(RUN_DAY) + assert (start, end) == (datetime(2025, 6, 2, 8), datetime(2025, 6, 12, 7)) + assert len(_rain()) == HOURS + + +def test_downpour_in_one_hour(): + day = [0.0] * 24 + day[10] = 35.0 + a = _one(_rain(rain=_with_day(day))) + assert a.status == EVENT and a.details["criteria_met"] == ["downpour_1h"] + assert a.severity == Severity.LIGHT and a.duration_h == 1 + assert a.start == datetime(2025, 6, 11, 18) + + +def test_contract_operator_at_least_30mm(): + day = [0.0] * 24 + day[0] = 30.0 + assert _one(_rain(rain=_with_day(day))).status == EVENT + day[0] = 29.9 + assert _one(_rain(rain=_with_day(day))).status == NO_EVENT + + +def test_severity_grows_with_ratio(): + day = [0.0] * 24 + day[5] = 95.0 # 95 / 30 = 3.2 -> extreme + a = _one(_rain(rain=_with_day(day))) + assert a.severity == Severity.EXTREME and a.severity_score == 1.0 + + +def test_intense_rain_over_72_hours(): + rain = [0.0] * (HOURS - 72) + [1.2] * 72 # 86.4 mm in 72 h, no downpour + a = _one(_rain(rain=rain)) + assert a.status == EVENT and a.details["criteria_met"] == ["intense_72h"] + assert a.details["max_72h_mm"] == 86.4 and a.severity == Severity.LIGHT + + +def test_dry_day_after_heavy_rain_is_not_repeated(): + rain = [0.0] * (HOURS - 96) + [1.2] * 72 + [0.0] * 24 + a = _one(_rain(rain=rain)) + assert a.status == NO_EVENT and a.details["max_72h_mm"] is None + assert a.details["rain_72h_to_run_mm"] > 0 + + +def test_light_rain_is_no_event(): + a = _one(_rain(rain=_with_day([0.5] * 24))) + assert a.status == NO_EVENT and "12.0 mm in the last 24 h" in a.summary + + +def test_prolonged_rain_only_with_reference_average(): + rain = [0.375] * HOURS # 90 mm in 10 days, 27 mm in 72 h + a = _one(_rain(rain=rain)) + assert a.status == NO_EVENT + assert any("No 5-year reference average" in n for n in a.data_notes) + a = _one(_rain(rain=rain), reference_10d_mm={3: 50.0}) # 90 > 1.5 x 50 and >= 80 + assert a.status == EVENT and a.details["criteria_met"] == ["prolonged_10d"] + assert a.details["rain_trigger_ratio"] == round(90 / 80, 3) # trigger = max(75, 80) + assert not any("reference average" in n for n in a.data_notes) + a = _one(_rain(rain=rain), reference_10d_mm={3: 70.0}) # 90 < 1.5 x 70 + assert a.status == NO_EVENT + + +def test_missing_hours_are_noted_and_counted_as_zero(): + rain = _with_day([0.0] * 24) + rain[3], rain[HOURS - 1] = np.nan, np.nan + a = _one(_rain(rain=rain).drop(index=10)) # a missing row too + assert a.status == NO_EVENT + assert any("1 of 24 hours of the event day" in n for n in a.data_notes) + assert any("2 of 216 hours" in n for n in a.data_notes) + + +def test_no_records_is_no_data(): + a = _one(_rain(station=5)) + assert a.status == NO_DATA and a.data_notes + rain = _with_day([np.nan] * 24, before=0.1) + a = _one(_rain(rain=rain)) + assert a.status == NO_DATA and "No rain data for the event day" in a.data_notes[0] + + +def _at(i): + """Timestamp (ISO, minutes) of record i of the fetch window (record 0 = 2025-06-02 08:00).""" + start, _ = flood.fetch_window(RUN_DAY) + return (start + i * flood.HOUR).isoformat(timespec="minutes") + + +def test_last_heavy_72h_found_days_later_for_claims(): + rain = [0.0] * HOURS + rain[150], rain[160], rain[180] = 30.0, 30.0, 30.0 # 90 mm in 31 h, 2-4 days ago + a = _one(_rain(rain=rain)) + assert a.status == NO_EVENT # the event day is dry: no new event + heavy = a.details["last_heavy_72h"] + assert heavy == {"mm": 90.0, "first_rain": _at(150), "last_rain": _at(180), + "reached_at": _at(180), "partial": False} + assert heavy["first_rain"] == "2025-06-08T14:00" and heavy["last_rain"] == "2025-06-09T20:00" + + +def test_last_heavy_72h_is_the_latest_not_the_wettest(): + rain = [0.0] * HOURS + rain[150], rain[230] = 100.0, 85.0 # both end inside the last 4 days + d = _one(_rain(rain=rain)).details + assert d["last_heavy_72h"]["mm"] == 85.0 and d["last_heavy_72h"]["reached_at"] == _at(230) + assert d["wettest_72h"] == {"mm": 100.0, "first_rain": _at(150), "last_rain": _at(150), + "partial": False} + + +def test_no_heavy_72h_gives_the_wettest_window(): + d = _one(_rain(rain=_with_day([0.5] * 24))).details + assert d["last_heavy_72h"] is None + assert d["wettest_72h"]["mm"] == 12.0 and d["wettest_72h"]["first_rain"] == _at(HOURS - 24) + d = _one(_rain(rain=_with_day([0.0] * 24))).details + assert d["last_heavy_72h"] is None and d["wettest_72h"] is None # no rain at all + + +def test_heavy_rain_watch_counts_only_the_rain_of_the_last_4_days(): + first = HOURS - 24 * flood.CLAIM_SEARCH_DAYS # first record of the watch period + assert flood.CLAIM_SEARCH_DAYS == 4 and first == 144 and _at(first) == "2025-06-08T08:00" + rain = [0.0] * HOURS + rain[first - 1] = 85.0 # 07:00 four days ago: just before the watch + d = _one(_rain(rain=rain)).details + assert d["last_heavy_72h"] is None and d["wettest_72h"] is None + rain[first] = 85.0 # first hour of the watch + heavy = _one(_rain(rain=rain)).details["last_heavy_72h"] + assert heavy["mm"] == 85.0 and heavy["first_rain"] == _at(first) # the earlier 85 mm left out + assert heavy["reached_at"] == _at(first) and heavy["partial"] is True # rain just before it + rain[first - 1] = 0.0 + assert _one(_rain(rain=rain)).details["last_heavy_72h"]["partial"] is False + + +def test_heavy_rain_watch_storm_partly_before_the_4_days(): + rain = [0.0] * HOURS + for i in range(110, 150): # 2 mm/h, 40 h; 6 h inside the watch + rain[i] = 2.0 + d = _one(_rain(rain=rain)).details + assert d["last_heavy_72h"] is None # 80 mm in all, only 12 mm counted + assert d["wettest_72h"] == {"mm": 12.0, "first_rain": _at(144), "last_rain": _at(149), + "partial": True} + + +def test_last_heavy_72h_uses_the_contract_operator_at_least_80mm(): + rain = [0.0] * (HOURS - 90) + [1.1] * 70 + [3.0] + [0.0] * 19 # 80 mm (float noise) + assert _one(_rain(rain=rain)).details["last_heavy_72h"]["mm"] == 80.0 + rain[HOURS - 20] = 2.9 # 79.9 mm + assert _one(_rain(rain=rain)).details["last_heavy_72h"] is None + + +def test_cover_window_starts_at_fruit_set(): + assert flood.in_cover_window(RUN_DAY, 65) is False # flowering: excluded + assert flood.in_cover_window(RUN_DAY, 72) is True + assert flood.in_cover_window(RUN_DAY, None) is None + assert flood.in_cover_window(date(2025, 11, 2), 85) is False + day = [0.0] * 24 + day[0] = 40.0 + assert _one(_rain(rain=_with_day(day)), bbch=72).details["in_cover_window"] is True diff --git a/tests/test_freezing.py b/tests/test_freezing.py new file mode 100644 index 0000000..2bffc18 --- /dev/null +++ b/tests/test_freezing.py @@ -0,0 +1,154 @@ +from datetime import date, datetime + +import pandas as pd + +from damage_engine.detectors import brina, freezing +from damage_engine.schemas import EVENT, NO_DATA, NO_EVENT, Severity +from damage_engine.sources import derived +from damage_engine.sources.fields import Field + +RUN_DAY = date(2025, 3, 12) +# the 24 h checked for RUN_DAY: index 0-10 = day hours 07:00-17:00 of 11 March, +# index 11-23 = Brina's night window 18:00 (11 March) -> 06:00 (12 March) +HOURS = pd.date_range("2025-03-11 07:00", "2025-03-12 06:00", freq="h") +DAY, NIGHT = 11, 13 +WARM_NIGHT = [3.0] * NIGHT +COLD_DAY = [1, 0.5, 0, -0.5, -1.5, -2.5, -1, 0, 1, 2, 3] + WARM_NIGHT # 4 h below 0 °C, 10:00-13:00 + + +def _day(station, temps, rh=80.0, wind=None, solar=None, ts=None): + ts = HOURS[:len(temps)] if ts is None else ts + df = pd.DataFrame({"CodiceStazione": station, "Timestamp": ts, freezing.COL_TEMP: temps, + freezing.COL_RH: rh, freezing.COL_RH_MAX: rh}) + if wind is not None: + df[freezing.COL_WIND] = wind + if solar is not None: + df[freezing.COL_SOLAR] = solar + return df + + +def _field(station, bbch=None, fid=None): + return Field(id=fid or f"ST{station}", name=f"field {station}", station_id=station, bbch=bbch) + + +def test_windows_match_brina(): + assert freezing.day_window(RUN_DAY) == (datetime(2025, 3, 11, 6), datetime(2025, 3, 12, 6)) + b_start, b_end = brina.night_window(RUN_DAY) + assert freezing.night_start(RUN_DAY) == b_start and freezing.day_window(RUN_DAY)[1] == b_end + + +def test_cold_day_is_freezing_fallback_severity(): + [a] = freezing.detect(_day(3, COLD_DAY), [_field(3)], RUN_DAY, {3: "Lagnasco"}) + assert a.status == EVENT and a.block == "freezing" and a.station_name == "Lagnasco" + assert a.duration_h == 4 # 0 °C hours do not count ("below 0 °C") + assert a.start == datetime(2025, 3, 11, 10) and a.end == datetime(2025, 3, 11, 13) + assert a.details["day_min_temp_c"] == -2.5 + assert a.severity == Severity.MODERATE # fallback: -2 > -2.5 >= -4 + assert a.details["severity_method"] == "fallback_min_bud_temp" + assert a.details["in_cover_window"] is None # stage unknown + + +def test_exactly_zero_is_not_freezing(): + [a] = freezing.detect(_day(3, [0.0] * 24), [_field(3)], RUN_DAY) + assert a.status == NO_EVENT and a.severity == Severity.NONE + + +def test_sub_zero_night_is_left_to_brina(): + [a] = freezing.detect(_day(3, [5.0] * DAY + [-3.0] * NIGHT), [_field(3)], RUN_DAY) + assert a.status == NO_EVENT + assert a.details["day_min_temp_c"] == 5.0 + assert a.details["sunlit_hours_in_night_window"] == 0 + + +def test_sunlit_hours_of_the_night_window_are_freezing(): + solar = [400.0] * DAY + [200, 50, 5] + [0.0] * 9 + [30] # 18:00, 19:00 and 06:00 sunlit + temps = [5.0] * DAY + [-1.0] * NIGHT + [a] = freezing.detect(_day(566, temps, solar=solar), [_field(566)], RUN_DAY) + assert a.status == EVENT and a.duration_h == 3 + assert a.details["sunlit_hours_in_night_window"] == 3 and a.details["daylight_hours"] == 14 + assert a.start == datetime(2025, 3, 11, 18) and a.end == datetime(2025, 3, 12, 6) + + +def test_every_hour_goes_to_brina_or_freezing_never_both(): + solar = [400.0] * DAY + [200, 50, 5] + [0.0] * 9 + [30] + for rh, wind in ((95.0, 0.5), (40.0, 5.0)): # brina all night: calm + humid, windy + dry + df = _day(566, [-3.0] * 24, rh=rh, wind=[wind] * 24, solar=solar) + frz = freezing.day_hours(df, RUN_DAY) + frz_hours = set(frz.loc[frz["freezing"], "Timestamp"]) + night = df[df["Timestamp"] >= brina.night_window(RUN_DAY)[0]] + br = brina.brina_hours(night) + brina_hours_ = set(br.loc[br["brina"], "Timestamp"]) + assert not frz_hours & brina_hours_ + assert frz_hours | brina_hours_ == set(HOURS) + + +def test_severity_uses_field_stage_when_known(): + bloom, dormant = freezing.detect(_day(3, COLD_DAY), [_field(3, 65, "A"), _field(3, 40, "B")], RUN_DAY) + assert bloom.details["stage"] == "full_bloom" + assert 0.10 < bloom.details["kill_fraction"] < 0.50 # -2.5 °C at full bloom + assert bloom.severity == Severity.MODERATE + assert bloom.details["in_cover_window"] is True + assert dormant.status == EVENT and dormant.severity == Severity.NONE + + +def test_out_of_cover_window_flagged(): + df = _day(3, COLD_DAY, ts=pd.date_range("2025-10-18 07:00", periods=24, freq="h")) + [a] = freezing.detect(df, [_field(3, 65)], date(2025, 10, 19)) + assert a.status == EVENT and a.details["in_cover_window"] is False + + +def test_wind_and_wet_bulb_are_reported_not_used_to_detect(): + windy = _day(3, COLD_DAY, wind=[3.0] * 24) + calm = _day(6, COLD_DAY, wind=[0.5] * 24) + no_wind = _day(9, COLD_DAY) + out = freezing.detect(pd.concat([windy, calm, no_wind]), [_field(3), _field(6), _field(9)], RUN_DAY) + assert [a.status for a in out] == [EVENT] * 3 and len({a.duration_h for a in out}) == 1 + assert [a.details["advective_wind"] for a in out] == [True, False, None] + assert out[0].details["mean_wind_ms"] == 3.0 and out[2].details["mean_wind_ms"] is None + rows = freezing.day_hours(windy, RUN_DAY) + assert rows["wet_bulb_c"].equals(rows[derived.WET_BULB_MIN]) # the shared value + assert out[0].details["min_wet_bulb_c"] == round(float(rows.loc[rows["freezing"], "wet_bulb_c"].min()), 2) + + +def test_rows_outside_the_24_hours_are_ignored(): + df = _day(3, [5.0] * 24) + extra = _day(3, [-10.0, -10.0], ts=pd.to_datetime(["2025-03-11 06:00", "2025-03-12 07:00"])) + [a] = freezing.detect(pd.concat([df, extra]), [_field(3)], RUN_DAY) + assert a.status == NO_EVENT and a.details["records"] == DAY + + +def test_no_records_or_no_daylight_temperature_is_no_data(): + night_only = _day(3, [5.0] * NIGHT, ts=HOURS[DAY:]) + missing, no_day = freezing.detect(night_only, [_field(77), _field(3)], RUN_DAY) + assert missing.status == NO_DATA + assert missing.data_notes == ["No records from the station for yesterday's daylight hours."] + assert no_day.status == NO_DATA + assert "Air temperature missing for 11 of 11 daylight hours" in " ".join(no_day.data_notes) + + +def test_missing_sensors_are_noted(): + [a] = freezing.detect(_day(9, COLD_DAY), [_field(9)], RUN_DAY, + station_sensors={9: {"TemperaturaAria", "UmiditaRelativa"}}) + assert a.status == EVENT + assert any(n.startswith("No wind sensor at this station") for n in a.data_notes) + assert any(n.startswith("No solar radiation sensor at this station: daylight = the fixed day " + "hours (records 07:00-17:00)") for n in a.data_notes) + + +def test_partly_missing_hours_are_noted(): + solar = [400.0] * DAY + [0.0] * 10 + [None] * 3 + df = _day(566, COLD_DAY, wind=[1.0] * 24, solar=solar).drop(index=[0, 1]) # no 07:00, 08:00 rows + df.loc[2, freezing.COL_TEMP] = None + df[freezing.COL_TEMP_FALLBACK] = 0.0 + [a] = freezing.detect(df, [_field(566)], RUN_DAY) + text = " ".join(a.data_notes) + assert "Solar radiation missing for 3 of 13 hours of the night window" in text + assert "Air temperature missing for 2 of 11 daylight hours" in text # the two missing rows + assert "Hourly minimum temperature missing for 1 of 9 daylight hours: hourly mean used" in text + + +def test_complete_station_has_no_notes(): + solar = [400.0] * DAY + [0.0] * NIGHT + [a] = freezing.detect(_day(566, COLD_DAY, wind=[1.0] * 24, solar=solar), [_field(566)], RUN_DAY) + assert a.status == EVENT and a.data_notes == [] + assert a.details["solar_checked"] is True diff --git a/tests/test_hail.py b/tests/test_hail.py new file mode 100644 index 0000000..d1fad86 --- /dev/null +++ b/tests/test_hail.py @@ -0,0 +1,290 @@ +from datetime import date, datetime + +import numpy as np +import pandas as pd + +from damage_engine.detectors import hail +from damage_engine.schemas import EVENT, NO_DATA, NO_EVENT, Severity, camera_request +from damage_engine.sources.fields import Field + +RUN_DAY = date(2026, 7, 10) +# fetched for RUN_DAY: 1 look-back record (07:00 of 9 Jul), then the 24 h checked +# (08:00 of 9 Jul ... 07:00 of 10 Jul) +HOURS = pd.date_range("2026-07-09 07:00", "2026-07-10 07:00", freq="h") +N = len(HOURS) # 25 +LB = 1 # index of the first record of the 24 h (08:00) +ALL = {"TemperaturaAria", "UmiditaRelativa", "Pioggia", "VelocitaVento2m"} +NO_WIND = {"TemperaturaAria", "UmiditaRelativa", "Pioggia"} + + +def _day(station=3, t=None, rain=None, wind=None, wind_mean=None, rate=None, t_min=None, rh=None): + """Hourly rows: mean temperature t (min = t - 0.5, max = t + 0.5 unless t_min given), rain, + wind, humidity rh (default 50 %).""" + t = [22.0] * N if t is None else t + nan = [np.nan] * N + return pd.DataFrame({ + "CodiceStazione": station, "Timestamp": HOURS, + hail.COL_T_MEAN: t, + hail.COL_T_MIN: [x - 0.5 for x in t] if t_min is None else t_min, + hail.COL_T_MAX: [x + 0.5 for x in t], + hail.COL_RH_MEAN: [50.0] * N if rh is None else rh, + hail.COL_RAIN: [0.0] * N if rain is None else rain, + hail.COL_RATE: nan if rate is None else rate, + hail.COL_WIND: [3.0] * N if wind is None else wind, + hail.COL_WIND_MEAN: [1.5] * N if wind_mean is None else wind_mean, + }) + + +def _set(base, at, values): + out = list(base) + out[at:at + len(values)] = values + return out + + +def _field(station=3, bbch=None): + return Field(id=f"ST{station}", name=f"field {station}", station_id=station, bbch=bbch) + + +def _one(df, sensors=ALL, bbch=None, day=RUN_DAY): + [a] = hail.detect(df, [_field(3, bbch)], day, station_sensors={3: sensors}) + return a + + +ZERO = [0.0] * N +H16 = LB + 8 # record ending 16:00 of 9 Jul + + +def test_window_photo_time_lookback_and_rate(): + assert hail.RUN_HOUR == 7 and hail.LOOKBACK_H == 1 and hail.STORM_H == 1 + assert hail.day_window(RUN_DAY) == (datetime(2026, 7, 9, 7), datetime(2026, 7, 10, 7)) + assert hail.photo_time(RUN_DAY) == datetime(2026, 7, 10, 9) + assert hail.RATE_MMH == 30.0 # 5 mm in 10 min + + +def test_dry_calm_day_no_hail(): + a = _one(_day()) + assert a.status == NO_EVENT and a.severity == Severity.NONE + assert a.details["signals"] == [] and a.details["suspicion"] is None + assert a.details["records"] == 24 and "camera_request" not in a.details + assert a.summary == "No hail suspected: no rain in the 24 hours." + assert a.data_notes == [] + + +def test_burst_rain_alone_is_possible_and_asks_for_a_photo(): + a = _one(_day(rain=_set(ZERO, H16, [20.0]))) # exactly 20 mm counts (>=) + assert a.status == EVENT and a.severity == Severity.NONE and a.severity_score == 0.0 + d = a.details + assert d["signals"] == ["A"] and d["suspicion"] == "possible" and d["storm_signs"] == 1 + assert (d["max_rain_1h_mm"], d["max_rain_1h_at"]) == (20.0, "2026-07-09T16:00") + assert (a.duration_h, a.start, a.end) == (1, datetime(2026, 7, 9, 16), datetime(2026, 7, 9, 16)) + req = camera_request(d) + assert req == {"phenomenon": "hail", "station_id": 3, "photo_at": "2026-07-10T09:00", + "level": "possible", "signals": ["A"], "reason": "Hail possible: burst rain."} + assert a.summary.startswith("Hail possible: burst rain of 20 mm in one hour (09 Jul 16:00)") + assert "Photo requested for 10 Jul 09:00" in a.summary + + +def test_rain_below_burst_no_sign(): + a = _one(_day(rain=_set(ZERO, H16, [19.9]))) + assert a.status == NO_EVENT + assert a.summary.startswith("No hail suspected: wettest hour 19.9 mm (sign at ≥ 20 mm)") + + +def test_all_three_signs_in_one_storm_are_likely(): + t = _set([28.0] * N, H16 + 1, [21.0, 20.0]) # 17:00 min 20.5 after 16:00 mean 28 + rain = _set(ZERO, H16 + 1, [25.0, 3.0]) + wind = _set([3.0] * N, H16, [15.0]) # gust at 16:00, rain starts at 17:00 + a = _one(_day(t=t, rain=rain, wind=wind)) + d = a.details + assert a.status == EVENT and d["suspicion"] == "likely" + assert d["signals"] == ["A", "B", "C"] and d["storm_signs"] == 3 + assert (d["max_t_drop_c"], d["max_t_drop_at"]) == (7.5, "2026-07-09T17:00") + assert (d["max_gust_ms"], d["max_gust_at"]) == (15.0, "2026-07-09T16:00") + assert (a.start, a.end, a.duration_h) == (datetime(2026, 7, 9, 16), datetime(2026, 7, 9, 17), 2) + assert camera_request(d)["reason"] == ("Hail likely: burst rain; sudden temperature drop near " + "rain; strong gust near rain.") + + +def test_drop_and_gust_without_rain_nearby_do_not_count(): + t = _set([28.0] * N, H16, [22.0]) # 6 °C drop, dry + wind = _set([3.0] * N, H16 + 6, [16.0]) # gust, dry + rain = _set(ZERO, H16 + 3, [2.0]) # rain 3 h after the drop + a = _one(_day(t=t, rain=rain, wind=wind)) + d = a.details + assert a.status == NO_EVENT + assert (d["drop_hours"], d["drop_dry_hours"]) == (0, 1) + assert (d["gust_hours"], d["gust_dry_hours"]) == (0, 1) + + +def test_drop_with_light_rain_next_hour_is_possible(): + t = _set([28.0] * N, H16, [22.0]) + a = _one(_day(t=t, rain=_set(ZERO, H16 + 1, [0.4]))) + d = a.details + assert a.status == EVENT and d["signals"] == ["B"] and d["suspicion"] == "possible" + assert "temperature drop of 6.5 °C near rain" in a.summary + + +def test_two_signs_hours_apart_are_two_storms_possible(): + rain = _set(ZERO, H16, [22.0]) + rain = _set(rain, H16 + 4, [1.0]) + wind = _set([3.0] * N, H16 + 4, [14.0]) # 4 h later: another storm + a = _one(_day(rain=rain, wind=wind)) + d = a.details + assert d["signals"] == ["A", "C"] and d["storm_signs"] == 1 and d["suspicion"] == "possible" + + +def test_rain_rate_at_293_gives_sign_a(): + rain = _set(ZERO, H16, [8.0]) + rate = _set([0.0] * N, H16, [36.0]) + [a] = hail.detect(_day(293, rain=rain, rate=rate), [_field(293)], RUN_DAY) + d = a.details + assert a.status == EVENT and d["signals"] == ["A"] and d["rate_checked"] + assert d["max_rain_rate_mmh"] == 36.0 + assert "rain rate up to 36 mm/h" in a.summary + + +def test_drop_in_first_hour_uses_the_lookback_record(): + t = _set([28.0] * N, LB, [22.0]) # 08:00 of 9 Jul, after 07:00 mean 28 + a = _one(_day(t=t, rain=_set(ZERO, LB, [1.0]))) + assert a.details["signals"] == ["B"] and a.start == datetime(2026, 7, 9, 8) + + +def test_no_wind_sensor_note_and_gust_not_checked(): + nan = [np.nan] * N + a = _one(_day(rain=_set(ZERO, H16, [21.0]), wind=nan, wind_mean=nan), sensors=NO_WIND) + assert a.details["wind_checked"] is False and a.details["max_gust_ms"] is None + assert any("No wind sensor (VelocitaVento2m)" in n for n in a.data_notes) + + +def test_wind_max_missing_uses_hourly_mean(): + wind = _set([3.0] * N, H16, [np.nan]) + wind_mean = _set([1.5] * N, H16, [14.5]) + a = _one(_day(rain=_set(ZERO, H16, [1.0]), wind=wind, wind_mean=wind_mean)) + assert a.details["signals"] == ["C"] and a.details["wind_fallback_hours"] == 1 + assert any("Hourly maximum wind missing for 1 hour(s)" in n for n in a.data_notes) + + +def test_no_rain_data_counts_drop_as_if_it_rained(): + t = _set([28.0] * N, H16, [22.0]) + a = _one(_day(t=t, rain=[np.nan] * N)) + assert a.status == EVENT and a.details["signals"] == ["B"] + assert a.details["rain_checked"] is False and a.details["rain_day_mm"] is None + assert any(n.startswith("No rain data in the 24 hours") for n in a.data_notes) + + +def test_missing_temperature_hours_noted(): + t = _set([25.0] * N, H16, [np.nan]) + a = _one(_day(t=t, t_min=_set([24.5] * N, H16, [np.nan]))) + # the drop is not known for 16:00 (no value) and 17:00 (no value in the hour before) + assert any("Temperature drop not checked for 2 of 24 hours" in n for n in a.data_notes) + + +def test_no_data_and_station_without_records(): + nan = [np.nan] * N + a = _one(_day(t=nan, t_min=nan, rain=nan, wind=nan, wind_mean=nan)) + assert a.status == NO_DATA + assert a.summary.startswith("No rain, temperature or wind data from the station for the last 24 hours.") + assert a.details["hail_risk"] is None # unknown in the season: checked later + [b] = hail.detect(pd.DataFrame(), [_field(6)], RUN_DAY) + assert b.status == NO_DATA and b.data_notes == ["No records from the station for the last 24 hours."] + + +def test_cover_window_fruit_set_and_nets_note(): + storm = _day(rain=_set(ZERO, H16, [25.0])) + assert _one(storm, bbch=72).details["in_cover_window"] is True + assert _one(storm, bbch=65).details["in_cover_window"] is False # before fruit set + a = _one(storm) + assert a.details["in_cover_window"] is None and a.details["net_checked"] is False + assert any("under nets the hail cover ended on 25 May" in n for n in a.data_notes) + assert hail.in_cover_window(date(2026, 11, 2), 80) is False + assert hail.in_cover_window(date(2026, 5, 20), 72) is True + + +def test_nets_note_not_before_25_may(): + hours = pd.date_range("2026-05-19 07:00", "2026-05-20 07:00", freq="h") + df = _day(rain=_set(ZERO, H16, [25.0])).assign(Timestamp=hours) + a = _one(df, day=date(2026, 5, 20)) + assert a.status == EVENT + assert not any("nets" in n for n in a.data_notes) + + +# ─────────────────────────── hail-risk day ─────────────────────────── + +def test_risk_day_warm_and_moist_in_season(): + t = _set([22.0] * N, LB + 6, [27.0, 28.0]) # afternoon up to 28.5 °C (max) + a = _one(_day(t=t, rh=[70.0] * N)) # dew point ~22 °C at 28 °C, 70 % + d = a.details + assert a.status == NO_EVENT and d["hail_risk"] is True + assert d["max_temp_c"] == 28.5 and d["max_dew_point_c"] >= 14 + assert d["hail_risk_reason"].startswith("warm, moist air") + assert a.summary.endswith("Hail-risk day: the station is checked again today at 13:00 and 19:00.") + + +def test_no_risk_when_dry_cool_or_out_of_season(): + hot = _set([22.0] * N, LB + 6, [28.0]) + dry = _one(_day(t=hot, rh=[25.0] * N)).details # dew point ~6 °C + assert dry["hail_risk"] is False and dry["hail_risk_reason"].startswith("warm (28.5 °C) but dry") + cool = _one(_day(rh=[90.0] * N)).details # 22.5 °C max + assert cool["hail_risk"] is False and "< 25 °C" in cool["hail_risk_reason"] + hours = pd.date_range("2026-10-25 07:00", "2026-10-26 07:00", freq="h") + october = _day(t=hot, rh=[80.0] * N, rain=_set(ZERO, H16, [25.0])).assign(Timestamp=hours) + d = _one(october, day=date(2026, 10, 26)).details + assert d["hail_risk"] is False and d["hail_risk_reason"] == "outside the hail season" + assert not hail.in_season(date(2026, 4, 30)) and hail.in_season(date(2026, 5, 1)) + + +def test_risk_day_after_a_hail_sign_and_without_humidity(): + a = _one(_day(rain=_set(ZERO, H16, [25.0]))) + assert a.details["hail_risk"] is True + assert a.details["hail_risk_reason"] == "a hail sign in the last 24 hours" + hot = _set([22.0] * N, LB + 6, [28.0]) + b = _one(_day(t=hot, rh=[np.nan] * N)) + assert b.details["hail_risk"] is True and b.details["max_dew_point_c"] is None + assert any("hail-risk day was decided on the air temperature alone" in n for n in b.data_notes) + + +# ─────────────────────────── daytime checks ─────────────────────────── + +DAY_HOURS = pd.date_range("2026-07-10 06:00", "2026-07-10 13:00", freq="h") # 1 look-back + 6 h + + +def _daytime(rain=None, t=None): + n = len(DAY_HOURS) + t = [24.0] * n if t is None else t + return pd.DataFrame({"CodiceStazione": 3, "Timestamp": DAY_HOURS, hail.COL_T_MEAN: t, + hail.COL_T_MIN: [x - 0.5 for x in t], hail.COL_T_MAX: [x + 0.5 for x in t], + hail.COL_RH_MEAN: [60.0] * n, + hail.COL_RAIN: [0.0] * n if rain is None else rain, + hail.COL_RATE: [np.nan] * n, hail.COL_WIND: [3.0] * n, + hail.COL_WIND_MEAN: [1.5] * n}) + + +def test_daytime_windows_and_hours(): + assert hail.DAYTIME_HOURS == (13, 19) and hail.DAYTIME_H == 6 + assert hail.daytime_window(RUN_DAY, 13) == (datetime(2026, 7, 10, 7), datetime(2026, 7, 10, 13)) + assert hail.daytime_window(RUN_DAY, 19) == (datetime(2026, 7, 10, 13), datetime(2026, 7, 10, 19)) + import pytest + with pytest.raises(ValueError): + hail.run_daytime(RUN_DAY, 1) # no night check + + +def test_daytime_check_asks_for_a_photo_at_once(): + rain = [0.0] * len(DAY_HOURS) + rain[4] = 23.0 # record ending 10:00 + window = hail.daytime_window(RUN_DAY, 13) + [a] = hail.detect(_daytime(rain=rain), [_field()], RUN_DAY, station_sensors={3: ALL}, window=window) + d = a.details + assert a.block == "hail_daytime" and a.status == EVENT and d["check"] == "daytime" + assert (d["window_start"], d["window_end"], d["hours"]) == ("2026-07-10T07:00", "2026-07-10T13:00", 6) + assert camera_request(d)["photo_at"] == "2026-07-10T13:00" # at once, not next morning + assert "hail_risk" not in d + assert "Photo requested at once 10 Jul 13:00" in a.summary + + +def test_daytime_check_quiet_and_no_records(): + window = hail.daytime_window(RUN_DAY, 13) + [a] = hail.detect(_daytime(), [_field()], RUN_DAY, station_sensors={3: ALL}, window=window) + assert a.status == NO_EVENT and a.summary == "No hail suspected: no rain in the 6 hours." + assert a.details["records"] == 6 + [b] = hail.detect(pd.DataFrame(), [_field(6)], RUN_DAY, window=window) + assert b.status == NO_DATA and b.data_notes == ["No records from the station for the 6 hours."] diff --git a/tests/test_high_temperature.py b/tests/test_high_temperature.py new file mode 100644 index 0000000..1f60217 --- /dev/null +++ b/tests/test_high_temperature.py @@ -0,0 +1,250 @@ +from datetime import date, datetime + +import numpy as np +import pandas as pd +import pytest + +from damage_engine.detectors import high_temperature as ht +from damage_engine.schemas import EVENT, NO_DATA, NO_EVENT, Severity +from damage_engine.sources import derived +from damage_engine.sources.fields import Field + +RUN_DAY = date(2026, 7, 20) +# the 24 h checked for RUN_DAY: records 08:00 of 19 Jul ... 07:00 of 20 Jul +HOURS = pd.date_range("2026-07-19 08:00", "2026-07-20 07:00", freq="h") +MILD = [30.0] * 24 +NOON = 6 # index of the 14:00 record +SUN = [800.0] * 11 + [0.0] * 13 # 08:00 ... 18:00 sunny, then night +BREEZE = [2.0] * 24 +ALL_SENSORS = {"TemperaturaAria", "UmiditaRelativa", "VelocitaVento2m", "RadiazioneSolare"} +NO_SOLAR = {"TemperaturaAria", "UmiditaRelativa"} + + +def _day(station, t_max, solar=None, wind=None, t_mean=None, rh=None, ts=None): + ts = HOURS[:len(t_max)] if ts is None else ts + n = len(ts) + nan = [np.nan] * n + t_mean = ([t - 2 if t is not None and not np.isnan(t) else np.nan for t in t_max] + if t_mean is None else t_mean) + rh = [30.0] * n if rh is None else rh + return pd.DataFrame({ + "CodiceStazione": station, "Timestamp": ts, + derived.T_MAX: t_max, derived.T_MEAN: t_mean, + derived.T_MIN: [t - 4 if not np.isnan(t) else np.nan for t in t_mean], + derived.RH_MIN: rh, derived.RH_MEAN: rh, derived.RH_MAX: rh, + ht.COL_SOLAR: nan if solar is None else solar, + ht.COL_WIND: nan if wind is None else wind, + }) + + +def _field(station, bbch=None): + return Field(id=f"ST{station}", name=f"field {station}", station_id=station, bbch=bbch) + + +def _with(values, at=NOON, base=MILD): + out = list(base) + out[at:at + len(values)] = values + return out + + +def test_window_is_the_24h_ending_at_07(): + assert ht.RUN_HOUR == 7 + assert ht.day_window(RUN_DAY) == (datetime(2026, 7, 19, 7), datetime(2026, 7, 20, 7)) + + +def test_fruit_surface_temperature_formula(): + fst = ht.fruit_surface_c([40.0, 40.0, 40.0, 40.0, 40.0, 40.0], + [1000.0, 1000.0, 299.0, 300.0, np.nan, 1000.0], + [0.0, 2.0, 0.0, 0.0, 0.0, np.nan]) + assert fst[0] == pytest.approx(55.0) # calm, full sun: + 15 °C + assert fst[1] == pytest.approx(40 + 15 / 1.6) # wind cools the fruit + assert fst[2] == 40.0 # below 300 W/m²: no sun gain + assert fst[3] == pytest.approx(44.5) # 300 W/m² counts + assert np.isnan(fst[4]) # no solar value: no FST + assert fst[5] == pytest.approx(55.0) # no wind value: calm (worst case) + + +def test_event_without_solar_is_graded_on_the_air(): + # 14:00-16:00: 40.5, 42.3, 41.0 °C hourly maximum, no pyranometer (11 of 14 stations) + df = _day(3, _with([40.5, 42.3, 41.0]), wind=BREEZE) + [a] = ht.detect(df, [_field(3)], RUN_DAY, {3: "Lagnasco"}, {3: NO_SOLAR | {"VelocitaVento2m"}}) + assert a.status == EVENT and a.block == "high_temperature" and a.station_name == "Lagnasco" + assert a.duration_h == 3 + assert a.start == datetime(2026, 7, 19, 14) and a.end == datetime(2026, 7, 19, 16) + d = a.details + assert d["max_temp_c"] == 42.3 and d["max_temp_at"] == "2026-07-19T15:00" + assert d["hot_hours"] == 3 and d["max_mean_temp_c"] == 40.3 + assert d["severity_method"] == "air" and d["severity_value_c"] == 42.3 + assert d["solar_checked"] is False and d["max_fst_c"] is None + assert d["sunburn_risk_hours"] is None + assert a.severity == Severity.MODERATE # air bands: 42 <= 42.3 < 44 + assert d["in_cover_window"] is None # stage unknown + assert a.summary == "High temperature for 3 h, highest air temperature 42.3 °C: severity moderate." + assert any(n.startswith("No solar radiation sensor (RadiazioneSolare) at this station") for n in a.data_notes) + + +def test_trigger_is_at_least_40(): + [hit] = ht.detect(_day(3, _with([40.0])), [_field(3)], RUN_DAY) + [miss] = ht.detect(_day(3, _with([39.9])), [_field(3)], RUN_DAY) + assert hit.status == EVENT and hit.severity == Severity.LIGHT and hit.duration_h == 1 + assert miss.status == NO_EVENT and miss.severity == Severity.NONE + assert miss.details["max_temp_c"] == 39.9 + assert miss.summary == "No high temperature: highest air temperature 39.9 °C." + + +@pytest.mark.parametrize("temp, level", [(42.0, Severity.MODERATE), (44.0, Severity.SEVERE), + (46.0, Severity.EXTREME)]) +def test_air_bands_without_solar(temp, level): + [a] = ht.detect(_day(3, _with([temp])), [_field(3)], RUN_DAY) + assert a.severity == level + + +def test_event_with_solar_is_graded_on_the_fruit_surface(): + # 14:00: 41 °C, 900 W/m², wind 1 m/s -> FST = 41 + 13.5 / 1.3 = 51.4 °C + solar, wind = _with([900.0], base=SUN), _with([1.0], base=BREEZE) + df = _day(566, _with([41.0]), solar=solar, wind=wind) + [a] = ht.detect(df, [_field(566)], RUN_DAY, station_sensors={566: ALL_SENSORS}) + d = a.details + assert a.status == EVENT and d["severity_method"] == "fst" + assert d["max_fst_c"] == 51.4 and d["max_fst_at"] == "2026-07-19T14:00" + assert d["solar_at_max_fst_wm2"] == 900 and d["wind_at_max_fst_ms"] == 1.0 + assert d["sunlit_hours"] == 11 and d["solar_checked"] is True + assert a.severity == Severity.MODERATE # 49 <= 51.4 < 52 + assert "fruit surface about 51.4 °C" in a.summary + assert a.data_notes == [] + + +def test_fst_below_46_is_an_event_without_severity(): + # 14:00: 40 °C but a weak sun and wind: FST = 40 + 6 / 1.9 = 43.2 °C + solar, wind = _with([400.0], base=[0.0] * 24), _with([3.0], base=BREEZE) + [a] = ht.detect(_day(566, _with([40.0]), solar=solar, wind=wind), [_field(566)], RUN_DAY) + assert a.status == EVENT and a.severity == Severity.NONE + assert a.details["severity_method"] == "fst" and a.details["max_fst_c"] == 43.2 + + +def test_severity_uses_the_hottest_fruit_of_the_day(): + # 13:00: 39 °C in full calm sun (FST 54) comes before the 40 °C hour (cloudy, FST 40) + t = _with([39.0, 40.0], at=NOON - 1) + solar = _with([1000.0, 100.0], at=NOON - 1, base=SUN) + wind = _with([0.0, 2.0], at=NOON - 1, base=BREEZE) + [a] = ht.detect(_day(566, t, solar=solar, wind=wind), [_field(566)], RUN_DAY) + assert a.status == EVENT and a.duration_h == 1 + assert a.details["max_fst_c"] == 54.0 and a.severity == Severity.SEVERE + + +def test_sunburn_risk_below_40_is_information_only(): + # 14:00: 35 °C, 1000 W/m², wind 0.5 -> FST 48.0 °C; 15:00: 31 °C (FST 46) is too cool + t = _with([35.0, 31.0]) + solar = _with([1000.0, 1000.0], base=SUN) + wind = _with([0.5, 0.0], base=BREEZE) + [a] = ht.detect(_day(566, t, solar=solar, wind=wind), [_field(566)], RUN_DAY) + assert a.status == NO_EVENT and a.severity == Severity.NONE + assert a.details["sunburn_risk_hours"] == 1 and a.details["max_risk_fst_c"] == 48.0 + assert a.summary == ("No high temperature: highest air temperature 35 °C. Sunburn risk: fruit " + "surface about 48 °C in 1 hour(s) (below the contract's 40 °C air, " + "not insurable).") + + +def test_no_sunburn_risk_on_a_mild_sunny_day(): + [a] = ht.detect(_day(566, MILD, solar=SUN, wind=BREEZE), [_field(566)], RUN_DAY) + assert a.status == NO_EVENT and a.details["sunburn_risk_hours"] == 0 + assert a.details["max_risk_fst_c"] is None + + +def test_missing_wind_in_sunlit_hours_is_calm(): + wind = _with([np.nan], base=BREEZE) + [a] = ht.detect(_day(566, _with([40.0]), solar=_with([1000.0], base=SUN), wind=wind), + [_field(566)], RUN_DAY, station_sensors={566: ALL_SENSORS}) + assert a.details["max_fst_c"] == 55.0 and a.severity == Severity.EXTREME + assert ("Wind missing for 1 sunlit hour(s): fruit surface temperature computed for calm " + "air in those hours (worst case).") in a.data_notes + + +def test_solar_station_without_anemometer_is_calm(): + [a] = ht.detect(_day(9, _with([40.0]), solar=SUN), [_field(9)], RUN_DAY, + station_sensors={9: NO_SOLAR | {"RadiazioneSolare"}}) + assert a.details["wind_at_max_fst_ms"] is None + assert ("No wind sensor (VelocitaVento2m) at this station: fruit surface temperature " + "computed for calm air (worst case).") in a.data_notes + + +def test_missing_solar_hours_are_noted(): + solar = list(SUN) + solar[NOON] = np.nan + [a] = ht.detect(_day(566, _with([41.0]), solar=solar, wind=BREEZE), [_field(566)], RUN_DAY, + station_sensors={566: ALL_SENSORS}) + assert a.status == EVENT and a.details["severity_method"] == "fst" + assert any(n.startswith("Solar radiation missing for 1 of 24 hours") for n in a.data_notes) + + +def test_solar_sensor_without_data_uses_the_air(): + [a] = ht.detect(_day(566, _with([42.0]), wind=BREEZE), [_field(566)], RUN_DAY, + station_sensors={566: ALL_SENSORS}) + assert a.details["severity_method"] == "air" and a.severity == Severity.MODERATE + assert any(n.startswith("No solar radiation data in the 24 hours") for n in a.data_notes) + + +def test_missing_maximum_temperature_uses_the_mean(): + t_max = _with([np.nan]) + t_mean = _with([40.2], base=[28.0] * 24) # only the mean of 14:00 is known + [a] = ht.detect(_day(3, t_max, t_mean=t_mean), [_field(3)], RUN_DAY) + assert a.status == EVENT and a.details["fallback_hours"] == 1 + assert any("Hourly maximum temperature missing for 1 of 24 hours: hourly mean used" in n + for n in a.data_notes) + + +def test_humidity_and_wet_bulb_of_the_hottest_hour(): + rh = _with([20.0], base=[50.0] * 24) + [a] = ht.detect(_day(3, _with([41.0]), rh=rh), [_field(3)], RUN_DAY) + assert a.details["humidity_at_max_temp_pct"] == 20 + assert a.details["wet_bulb_at_max_temp_c"] == round(float(derived.wet_bulb_c(41.0, 20.0)), 1) + + +def test_missing_humidity_is_noted_only(): + [a] = ht.detect(_day(3, _with([41.0]), rh=[np.nan] * 24), [_field(3)], RUN_DAY, + station_sensors={3: NO_SOLAR}) + assert a.status == EVENT and a.details["humidity_at_max_temp_pct"] is None + assert a.details["wet_bulb_at_max_temp_c"] is None + assert ("No humidity data in the 24 hours: humidity and wet-bulb not reported " + "(not used to detect).") in a.data_notes + + +def test_cover_window_fruit_set_to_31_october(): + df = _day(3, _with([41.0])) + [after, before] = ht.detect(df, [_field(3, bbch=75), _field(3, bbch=65)], RUN_DAY) + assert after.details["in_cover_window"] is True and after.details["bbch"] == 75 + assert before.details["in_cover_window"] is False # still in bloom + assert after.severity == before.severity # severity does not depend on the stage + assert ht.in_cover_window(date(2026, 11, 2), 80) is False + + +def test_hours_outside_the_24h_are_ignored(): + ts = pd.date_range("2026-07-19 07:00", "2026-07-20 08:00", freq="h") # 26 records + [a] = ht.detect(_day(3, [45.0] + MILD + [45.0], ts=ts), [_field(3)], RUN_DAY) + assert a.status == NO_EVENT and a.details["records"] == 24 + + +def test_missing_hours_are_noted_and_not_checked(): + df = _day(3, _with([41.0])).drop(index=[0, 1, 2]) # 3 hours with no record + [a] = ht.detect(df, [_field(3)], RUN_DAY) + assert a.status == EVENT and a.details["records"] == 21 + assert "Air temperature missing for 3 of 24 hours: those hours not checked." in a.data_notes + + +def test_no_temperature_values_is_no_data(): + df = _day(3, [np.nan] * 24, t_mean=[np.nan] * 24) + [a] = ht.detect(df, [_field(3)], RUN_DAY) + assert a.status == NO_DATA + assert a.data_notes == ["No air temperature data in the 24 hours checked: high temperature " + "cannot be judged."] + + +def test_missing_columns_do_not_break_the_block(): + df = pd.DataFrame({"CodiceStazione": 3, "Timestamp": HOURS}) # no columns at all + [a] = ht.detect(df, [_field(3)], RUN_DAY) + assert a.status == NO_DATA + + +def test_station_without_records_is_no_data(): + [a] = ht.detect(pd.DataFrame(), [_field(3)], RUN_DAY) + assert a.status == NO_DATA and a.data_notes == ["No records from the station for the last 24 hours."] diff --git a/tests/test_hot_wind.py b/tests/test_hot_wind.py new file mode 100644 index 0000000..8564597 --- /dev/null +++ b/tests/test_hot_wind.py @@ -0,0 +1,194 @@ +from datetime import date, datetime + +import numpy as np +import pandas as pd +import pytest + +from damage_engine.detectors import hot_wind +from damage_engine.schemas import EVENT, NO_DATA, NO_EVENT, Severity +from damage_engine.sources.fields import Field + +RUN_DAY = date(2026, 7, 20) +# the 24 h checked for RUN_DAY: records 08:00 of 19 Jul ... 07:00 of 20 Jul +HOURS = pd.date_range("2026-07-19 08:00", "2026-07-20 07:00", freq="h") +MILD = [30.0] * 24 +BREEZE = [2.5] * 24 +ALL_SENSORS = {"TemperaturaAria", "UmiditaRelativa", "VelocitaVento2m"} + + +def _day(station, t_max, wind=None, t_mean=None, wind_max=None, rh=None, ts=None): + ts = HOURS[:len(t_max)] if ts is None else ts + n = len(ts) + return pd.DataFrame({ + "CodiceStazione": station, "Timestamp": ts, + hot_wind.COL_T_MAX: t_max, + hot_wind.COL_T_MEAN: [t - 2 if t is not None and not np.isnan(t) else np.nan for t in t_max] + if t_mean is None else t_mean, + hot_wind.COL_WIND: BREEZE[:n] if wind is None else wind, + hot_wind.COL_WIND_MAX: [np.nan] * n if wind_max is None else wind_max, + hot_wind.COL_RH: [25.0] * n if rh is None else rh, + }) + + +def _field(station, bbch=None): + return Field(id=f"ST{station}", name=f"field {station}", station_id=station, bbch=bbch) + + +def _with_heat(values, at=6, base=MILD): + t = list(base) + t[at:at + len(values)] = values + return t + + +def test_window_is_the_24h_ending_at_07(): + assert hot_wind.RUN_HOUR == 7 + assert hot_wind.day_window(RUN_DAY) == (datetime(2026, 7, 19, 7), datetime(2026, 7, 20, 7)) + + +def test_hot_moving_air_is_an_event_graded_on_temperature(): + # 14:00-16:00: 40.5, 42.3, 41.0 °C hourly maximum, breeze 2.5 m/s + df = _day(3, _with_heat([40.5, 42.3, 41.0])) + [a] = hot_wind.detect(df, [_field(3)], RUN_DAY, {3: "Lagnasco"}, {3: ALL_SENSORS}) + assert a.status == EVENT and a.block == "hot_wind" and a.station_name == "Lagnasco" + assert a.duration_h == 3 + assert a.start == datetime(2026, 7, 19, 14) and a.end == datetime(2026, 7, 19, 16) + d = a.details + assert d["max_temp_c"] == 42.3 and d["max_temp_at"] == "2026-07-19T15:00" + assert d["max_hot_wind_temp_c"] == 42.3 and d["hot_wind_hours"] == 3 and d["hot_hours"] == 3 + assert d["max_mean_temp_c"] == 40.3 + assert d["wind_checked"] is True and d["wind_assumed_hours"] == 0 + assert d["wind_at_max_temp_ms"] == 2.5 and d["min_humidity_pct"] == 25 + assert d["direction_checked"] is False and d["harvest_checked"] is False + assert a.severity == Severity.MODERATE # 42 <= 42.3 < 44 + assert d["in_cover_window"] is None # stage and harvest unknown + assert "42.3 °C" in a.summary + assert hot_wind.DIRECTION_NOTE in a.data_notes + assert any("30 days before harvest" in n for n in a.data_notes) + + +def test_trigger_is_at_least_40(): + [hit] = hot_wind.detect(_day(3, _with_heat([40.0])), [_field(3)], RUN_DAY) + [miss] = hot_wind.detect(_day(3, _with_heat([39.9])), [_field(3)], RUN_DAY) + assert hit.status == EVENT and hit.severity == Severity.LIGHT and hit.duration_h == 1 + assert miss.status == NO_EVENT and miss.severity == Severity.NONE + assert miss.details["max_temp_c"] == 39.9 and miss.details["hot_hours"] == 0 + assert miss.summary == "No hot wind: highest air temperature 39.9 °C." + + +@pytest.mark.parametrize("temp, level", [(42.0, Severity.MODERATE), (44.0, Severity.SEVERE), + (46.0, Severity.EXTREME), (48.0, Severity.EXTREME)]) +def test_temperature_bands(temp, level): + [a] = hot_wind.detect(_day(3, _with_heat([temp])), [_field(3)], RUN_DAY) + assert a.severity == level + + +def test_still_air_is_not_hot_wind(): + wind = list(BREEZE) + wind[6] = 0.6 # 14:00: 41 °C, still air + [a] = hot_wind.detect(_day(3, _with_heat([41.0]), wind=wind), [_field(3)], RUN_DAY, + station_sensors={3: ALL_SENSORS}) + assert a.status == NO_EVENT + assert a.details["hot_hours"] == 1 and a.details["still_hot_hours"] == 1 + assert a.summary.endswith("1 hour(s) at 40 °C or more had still air.") + + +def test_wind_exactly_at_the_minimum_counts_as_moving(): + wind = list(BREEZE) + wind[6] = 1.0 + [a] = hot_wind.detect(_day(3, _with_heat([41.0]), wind=wind), [_field(3)], RUN_DAY) + assert a.status == EVENT + + +def test_severity_uses_the_hot_wind_hours_only(): + wind = list(BREEZE) + wind[7] = 0.2 # the 45 °C hour had still air + [a] = hot_wind.detect(_day(3, _with_heat([41.0, 45.0]), wind=wind), [_field(3)], RUN_DAY) + assert a.status == EVENT and a.duration_h == 1 + assert a.details["max_temp_c"] == 45.0 and a.details["max_hot_wind_temp_c"] == 41.0 + assert a.severity == Severity.LIGHT + + +def test_station_without_anemometer_assumes_moving_air(): + df = _day(9, _with_heat([41.0]), wind=[np.nan] * 24) + [a] = hot_wind.detect(df, [_field(9)], RUN_DAY, + station_sensors={9: {"TemperaturaAria", "UmiditaRelativa"}}) + assert a.status == EVENT + assert a.details["wind_checked"] is False and a.details["wind_assumed_hours"] == 1 + assert a.details["wind_at_max_temp_ms"] is None + assert ("No wind sensor (VelocitaVento2m) at this station: air movement not checked, " + "taken as moving (worst case).") in a.data_notes + + +def test_missing_wind_hours_are_taken_as_moving(): + wind = list(BREEZE) + wind[6] = np.nan + [a] = hot_wind.detect(_day(3, _with_heat([41.0]), wind=wind), [_field(3)], RUN_DAY, + station_sensors={3: ALL_SENSORS}) + assert a.status == EVENT and a.details["wind_assumed_hours"] == 1 + assert "Wind missing for 1 of 24 hours: air taken as moving in those hours (worst case)." in a.data_notes + + +def test_missing_mean_wind_uses_the_maximum(): + wind, wind_max = list(BREEZE), [np.nan] * 24 + wind[6], wind_max[6] = np.nan, 0.8 # even the maximum is still air + [a] = hot_wind.detect(_day(3, _with_heat([41.0]), wind=wind, wind_max=wind_max), [_field(3)], RUN_DAY) + assert a.status == NO_EVENT and a.details["still_hot_hours"] == 1 + assert "Hourly mean wind missing for 1 of 24 hours: hourly maximum used." in a.data_notes + + +def test_missing_maximum_temperature_uses_the_mean(): + t_max = _with_heat([np.nan]) + t_mean = [28.0] * 24 + t_mean[6] = 40.2 # only the mean of 14:00 is known + [a] = hot_wind.detect(_day(3, t_max, t_mean=t_mean), [_field(3)], RUN_DAY) + assert a.status == EVENT and a.details["fallback_hours"] == 1 + assert any("Hourly maximum temperature missing for 1 of 24 hours: hourly mean used" in n + for n in a.data_notes) + + +def test_hours_outside_the_24h_are_ignored(): + ts = pd.date_range("2026-07-19 07:00", "2026-07-20 08:00", freq="h") # 26 records + t = [45.0] + MILD + [45.0] + [a] = hot_wind.detect(_day(3, t, ts=ts, wind=[2.5] * 26, rh=[25.0] * 26), [_field(3)], RUN_DAY) + assert a.status == NO_EVENT and a.details["records"] == 24 + + +def test_cover_window_fruit_set_to_31_october(): + df = _day(3, _with_heat([41.0])) + [after, before] = hot_wind.detect(df, [_field(3, bbch=75), _field(3, bbch=65)], RUN_DAY) + assert after.details["in_cover_window"] is None # fruit set, harvest date unknown + assert before.details["in_cover_window"] is False # still in bloom + assert after.severity == before.severity # severity does not depend on the stage + assert hot_wind.in_cover_window(date(2026, 11, 2), 80) is False + + +def test_missing_humidity_is_noted_only(): + df = _day(3, _with_heat([41.0]), rh=[np.nan] * 24) + [a] = hot_wind.detect(df, [_field(3)], RUN_DAY, station_sensors={3: ALL_SENSORS}) + assert a.status == EVENT and a.details["min_humidity_pct"] is None + assert "No humidity data in the 24 hours: humidity not reported (not used to detect)." in a.data_notes + + +def test_missing_hours_are_noted_and_not_checked(): + df = _day(3, _with_heat([41.0])).drop(index=[0, 1, 2]) # 3 hours with no record + [a] = hot_wind.detect(df, [_field(3)], RUN_DAY) + assert a.status == EVENT and a.details["records"] == 21 + assert "Air temperature missing for 3 of 24 hours: those hours not checked." in a.data_notes + + +def test_no_temperature_values_is_no_data(): + df = _day(3, [np.nan] * 24, t_mean=[np.nan] * 24) + [a] = hot_wind.detect(df, [_field(3)], RUN_DAY) + assert a.status == NO_DATA + assert a.data_notes == ["No air temperature data in the 24 hours checked: hot wind cannot be judged."] + + +def test_missing_columns_do_not_break_the_block(): + df = pd.DataFrame({"CodiceStazione": 3, "Timestamp": HOURS}) # no columns at all + [a] = hot_wind.detect(df, [_field(3)], RUN_DAY) + assert a.status == NO_DATA + + +def test_station_without_records_is_no_data(): + [a] = hot_wind.detect(pd.DataFrame(), [_field(3)], RUN_DAY) + assert a.status == NO_DATA and a.data_notes == ["No records from the station for the last 24 hours."] diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py new file mode 100644 index 0000000..92aed32 --- /dev/null +++ b/tests/test_pipeline.py @@ -0,0 +1,182 @@ +"""The whole pipeline (main.py) offline: blocks are replaced, no API call.""" + +import json +import sys +import types +from datetime import date, datetime + +import pytest + +from damage_engine import main +from damage_engine.detectors import (brina, drought, flood, freezing, hail, high_temperature, hot_wind, + snow, strong_wind) +from damage_engine.report import report, station_report +from damage_engine.schemas import NO_EVENT, Analysis + +DAY = date(2026, 10, 20) + + +@pytest.fixture +def out(tmp_path, monkeypatch): + """Send every stored file to tmp_path and use one station (566).""" + monkeypatch.setattr(report, "REPORTS_DIR", tmp_path / "reports") + monkeypatch.setattr(station_report, "STATION_REPORTS_DIR", tmp_path / "station_reports") + monkeypatch.setattr(main, "station_ids", lambda: [566]) + monkeypatch.setattr(brina, "run", lambda day: [Analysis( + block="brina", run_day=day, field_id="ST566", field_name="field 566", station_id=566, + station_name="CuneoGiordano", status=NO_EVENT, details={"night_min_temp_c": 9.5})]) + monkeypatch.setattr(freezing, "run", lambda day: [Analysis( + block="freezing", run_day=day, field_id="ST566", field_name="field 566", station_id=566, + station_name="CuneoGiordano", status=NO_EVENT, details={"day_min_temp_c": 12.0})]) + monkeypatch.setattr(strong_wind, "run", lambda day: [Analysis( + block="strong_wind", run_day=day, field_id="ST566", field_name="field 566", station_id=566, + station_name="CuneoGiordano", status=NO_EVENT, details={"max_wind_ms": 6.2})]) + monkeypatch.setattr(hot_wind, "run", lambda day: [Analysis( + block="hot_wind", run_day=day, field_id="ST566", field_name="field 566", station_id=566, + station_name="CuneoGiordano", status=NO_EVENT, details={"max_temp_c": 24.0})]) + monkeypatch.setattr(high_temperature, "run", lambda day: [Analysis( + block="high_temperature", run_day=day, field_id="ST566", field_name="field 566", + station_id=566, station_name="CuneoGiordano", status=NO_EVENT, + details={"max_temp_c": 24.0})]) + monkeypatch.setattr(snow, "run", lambda day: [Analysis( + block="snow", run_day=day, field_id="ST566", field_name="field 566", station_id=566, + station_name="CuneoGiordano", status=NO_EVENT, details={"min_cold_c": 6.0})]) + monkeypatch.setattr(hail, "run", lambda day: [Analysis( + block="hail", run_day=day, field_id="ST566", field_name="field 566", station_id=566, + station_name="CuneoGiordano", status=NO_EVENT, details={"max_rain_1h_mm": 0.0})]) + monkeypatch.setattr(drought, "run", lambda day: [Analysis( + block="drought", run_day=day, field_id="ST566", field_name="field 566", station_id=566, + station_name="CuneoGiordano", status=NO_EVENT, details={"rain_pct_of_normal": 95.0})]) + + def broken(day): + raise ConnectionError("3A API not reachable") + monkeypatch.setattr(flood, "run", broken) + return tmp_path + + +def test_block_without_code_is_not_ready(monkeypatch): + # every block of the diagram has code now: a made-up empty detectors/ file stands in + monkeypatch.setitem(sys.modules, "damage_engine.detectors.empty_block", + types.ModuleType("damage_engine.detectors.empty_block")) + with pytest.raises(main.BlockNotReady, match="no run"): + main.load_block("empty_block") + with pytest.raises(main.BlockNotReady, match="does not exist"): + main.load_block("no_such_block") + + +def test_pipeline_runs_every_block_and_reports_errors(out): + summary = main.run_pipeline(DAY) + assert list(summary["blocks"]) == [b for b in main.BLOCKS if b != "drought"] + assert list(summary["skipped"]) == ["drought"] # monthly: not due on the 20th + assert summary["blocks"]["brina"] is None + assert "ConnectionError" in summary["blocks"]["flood"] + assert summary["blocks"]["freezing"] is None + assert summary["blocks"]["strong_wind"] is None + assert summary["blocks"]["hot_wind"] is None + assert summary["blocks"]["high_temperature"] is None + assert summary["blocks"]["snow"] is None + assert summary["blocks"]["hail"] is None + assert summary["stations"] == {566: None} + + stored = json.loads((out / "reports" / "flood" / f"{DAY}.json").read_text(encoding="utf-8")) + assert "Traceback" in stored["error_detail"] # full detail kept internally only + + folder = out / "station_reports" / "566" + content = json.loads((folder / f"{DAY}.json").read_text(encoding="utf-8")) + failed = {e["block"]: e["error"] for e in content["overall"]["errors"]} + assert set(failed) == {"flood"} # flood crashed + assert _drought_status(content) == "not_run" # skipped, nothing stored + assert "error_detail" not in json.dumps(content) # no traceback for the client + assert (folder / f"{DAY}.pdf").read_bytes()[:5] == b"%PDF-" + + +def _drought_status(content): + return next(b["status"] for b in content["blocks"] if b["block"] == "drought") + + +def test_once_runs_one_block(out): + summary = main.run_pipeline(DAY, ["brina"]) + assert summary["blocks"] == {"brina": None} + summary = main.run_pipeline(DAY, ["drought"]) # --once runs it on any day + assert summary["blocks"] == {"drought": None} and summary["skipped"] == {} + + +def test_monthly_block_runs_on_the_first_and_is_shown_afterwards(out): + first = date(2026, 11, 1) + summary = main.run_pipeline(first) + assert summary["blocks"]["drought"] is None and summary["skipped"] == {} + later = date(2026, 11, 5) + assert "drought" in main.run_pipeline(later)["skipped"] + content = json.loads((out / "station_reports" / "566" / f"{later}.json").read_text(encoding="utf-8")) + b = next(b for b in content["blocks"] if b["block"] == "drought") + assert b["status"] == "no_event" and b["run_day"] == "2026-11-01" and b["stale"] + + +def test_default_run_day_waits_for_the_latest_window(): + hour = main.pipeline_hour() # 7: Flood's window ends at 07:00 + assert hour == flood.RUN_HOUR + assert main.default_run_day(datetime(2026, 10, 21, hour)) == date(2026, 10, 21) + assert main.default_run_day(datetime(2026, 10, 21, hour - 1)) == date(2026, 10, 20) + + +# ─────────────────────────── Hail daytime checks ─────────────────────────── + +JULY = date(2026, 7, 10) + + +def _hail(station, risk): + return Analysis(block="hail", run_day=JULY, field_id=f"ST{station}", field_name=f"field {station}", + station_id=station, station_name=f"S{station}", status=NO_EVENT, + details={"hail_risk": risk}) + + +def test_hail_risk_stations_from_the_morning_run(out): + assert main.hail_risk_stations(JULY) is None # no morning run, in season: all + assert main.hail_risk_stations(date(2026, 11, 3)) == set() # outside the season: none + report.submit("hail", JULY, [_hail(3, True), _hail(6, False), _hail(10, None)]) + assert main.hail_risk_stations(JULY) == {3, 10} # unknown risk is checked too + + +def test_hail_daytime_check_runs_only_for_risk_stations_and_keeps_the_morning_run(out, monkeypatch): + report.submit("hail", JULY, [_hail(566, True), _hail(3, False)]) + calls = [] + + def fake(day, hour, ids): + calls.append((day, hour, ids)) + return [Analysis(block="hail_daytime", run_day=day, field_id="ST566", field_name="field 566", + station_id=566, station_name="CuneoGiordano", status=NO_EVENT, + summary="No hail suspected: no rain in the 6 hours.", + details={"check": "daytime", "window_start": "2026-07-10T07:00", + "window_end": "2026-07-10T13:00"})] + monkeypatch.setattr(hail, "run_daytime", fake) + summary = main.run_hail_daytime(JULY, 13) + assert calls == [(JULY, 13, {566})] and summary["error"] is None and summary["checked"] == [566] + assert [r.get("slot") for r in report.runs("hail")] == [None] # morning run kept + assert [r["slot"] for r in report.day_runs("hail_daytime", JULY)] == ["13h"] + content = json.loads((out / "station_reports" / "566" / f"{JULY}.json").read_text(encoding="utf-8")) + b = next(b for b in content["blocks"] if b["block"] == "hail") + assert b["watch_window"] == {"label": "Daytime checks", "start": "2026-07-10T07:00", + "end": "2026-07-10T13:00"} + assert not any(b["block"] == "hail_daytime" for b in content["blocks"]) + + +def test_hail_daytime_check_skipped_without_risk_and_errors_stored(out, monkeypatch): + report.submit("hail", JULY, [_hail(566, False)]) + monkeypatch.setattr(hail, "run_daytime", lambda *a: pytest.fail("must not run")) + assert main.run_hail_daytime(JULY, 19)["checked"] == [] + assert report.day_runs("hail_daytime", JULY) == [] # nothing stored + report.submit("hail", JULY, [_hail(566, True)]) + + def broken(*a): + raise ConnectionError("3A API not reachable") + monkeypatch.setattr(hail, "run_daytime", broken) + assert "ConnectionError" in main.run_hail_daytime(JULY, 19)["error"] + [run] = report.day_runs("hail_daytime", JULY) + assert run["slot"] == "19h" and "ConnectionError" in run["error"] + + +def test_serve_schedule_pipeline_then_daytime_checks(): + assert main.next_job(datetime(2026, 7, 10, 6, 0)) == (datetime(2026, 7, 10, 7, 5), None) + assert main.next_job(datetime(2026, 7, 10, 7, 5)) == (datetime(2026, 7, 10, 13, 5), 13) + assert main.next_job(datetime(2026, 7, 10, 14, 0)) == (datetime(2026, 7, 10, 19, 5), 19) + assert main.next_job(datetime(2026, 7, 10, 19, 30)) == (datetime(2026, 7, 11, 7, 5), None) diff --git a/tests/test_report.py b/tests/test_report.py new file mode 100644 index 0000000..b8ca0ba --- /dev/null +++ b/tests/test_report.py @@ -0,0 +1,106 @@ +from datetime import date, datetime, timedelta + +from damage_engine.report import report +from damage_engine.schemas import EVENT, NO_EVENT, Analysis, Severity +from damage_engine.severity import grade +from damage_engine.thresholds import FROST_FALLBACK_BANDS_C, FROST_KILL_BANDS + + +def _analysis(day, field="F1", status=NO_EVENT, severity=Severity.NONE, block="brina"): + start = datetime.combine(day, datetime.min.time()) if status == EVENT else None + return Analysis(block=block, run_day=day, field_id=field, field_name=f"name {field}", + station_id=3, station_name="Lagnasco", status=status, severity=severity, + severity_score=0.6 if status == EVENT else 0.0, + duration_h=4 if status == EVENT else 0, start=start, end=start, + summary="test", details={"min_bud_temp_c": -3.0}) + + +def test_grade_bands(): + assert grade(0.0, FROST_FALLBACK_BANDS_C)[0] == Severity.NONE # strict: below 0 °C only + sev, score = grade(-3.1, FROST_FALLBACK_BANDS_C) + assert sev == Severity.MODERATE and abs(score - 0.6375) < 1e-3 + assert grade(0.95, FROST_KILL_BANDS) == (Severity.EXTREME, 1.0) + + +def test_analysis_round_trip(): + a = _analysis(date(2025, 3, 12), status=EVENT, severity=Severity.SEVERE) + assert Analysis.from_dict(a.to_dict()) == a + + +def test_submit_writes_block_report_and_keeps_last_runs(tmp_path): + first = date(2025, 3, 1) + for i in range(report.HISTORY_RUNS + 2): + day = first + timedelta(days=i) + text = report.submit("brina", day, [_analysis(day)], root=tmp_path) + assert "BRINA report" in text + kept = report.runs("brina", tmp_path) + assert len(kept) == report.HISTORY_RUNS + assert kept[0]["run_day"] == first + timedelta(days=report.HISTORY_RUNS + 1) + assert len(list((tmp_path / "brina").glob("*.txt"))) == report.HISTORY_RUNS + + +def test_rerun_same_day_replaces_run(tmp_path): + day = date(2025, 3, 12) + report.submit("brina", day, [_analysis(day)], root=tmp_path) + report.submit("brina", day, [_analysis(day, status=EVENT, severity=Severity.LIGHT)], root=tmp_path) + [run] = report.runs("brina", tmp_path) + assert run["analyses"][0].status == EVENT + + +def test_field_report_combines_blocks_and_past_dates(tmp_path): + d1, d2 = date(2025, 3, 11), date(2025, 3, 12) + report.submit("brina", d1, [_analysis(d1, status=EVENT, severity=Severity.SEVERE), + _analysis(d1, field="F2")], root=tmp_path) + report.submit("brina", d2, [_analysis(d2), _analysis(d2, field="F2")], root=tmp_path) + report.submit("hail", d2, [_analysis(d2, block="hail")], root=tmp_path) + + assert report.fields(tmp_path) == {"F1": "name F1", "F2": "name F2"} + assert report.field_report_dates("F1", tmp_path) == [d2, d1] + + latest = report.field_report("F1", root=tmp_path) + assert "[BRINA] run 2025-03-12" in latest and "[HAIL] run 2025-03-12" in latest + + past = report.field_report("F1", d1, root=tmp_path) + assert "[BRINA] run 2025-03-11" in past and "SEVERE" in past + assert "[HAIL] no run yet" in past + + +def test_reports_show_data_notes(tmp_path): + day = date(2025, 3, 12) + a = _analysis(day) + a.data_notes = ["No wind sensor at this station: calm test not applied."] + text = report.submit("brina", day, [a], root=tmp_path) + assert "Missing sensors / data:" in text and "No wind sensor" in text + assert "Data: No wind sensor" in report.field_report("F1", root=tmp_path) + + +def test_fields_report_groups_blocks_under_each_field(tmp_path): + day = date(2025, 6, 12) + report.submit("flood", day, [_analysis(day, field="F2", block="flood"), + _analysis(day, field="F1", block="flood")], root=tmp_path) + report.submit("brina", day, [_analysis(day, field="F1", status=EVENT, severity=Severity.LIGHT), + _analysis(day, field="F2")], root=tmp_path) + text = (tmp_path / report.FIELDS_REPORT_FILE).read_text(encoding="utf-8") + assert text.startswith("REPORT BY FIELD") and "blocks: brina, flood" in text + f1, f2 = text.index("FIELD name F1 [F1]"), text.index("FIELD name F2 [F2]") + # field F1 first, with its brina and flood sections, then field F2 with its own + assert f1 < text.index("[BRINA]", f1) < text.index("[FLOOD]", f1) < f2 + assert f2 < text.index("[BRINA]", f2) < text.index("[FLOOD]", f2) + assert "LIGHT" in text[f1:f2] and "LIGHT" not in text[f2:] + + +def test_slots_keep_every_check_of_a_day_and_history_counts_days(tmp_path): + first = date(2025, 7, 1) + for i in range(report.HISTORY_RUNS + 1): + day = first + timedelta(days=i) + for slot in ("13h", "19h"): + text = report.submit("hail_daytime", day, [_analysis(day, block="hail_daytime")], + root=tmp_path, slot=slot) + assert "HAIL_DAYTIME report - run day 2025-07-11, check 19h" in text + last = first + timedelta(days=report.HISTORY_RUNS) + assert [r["slot"] for r in report.day_runs("hail_daytime", last, tmp_path)] == ["13h", "19h"] + days = {r["run_day"] for r in report.runs("hail_daytime", tmp_path)} + assert len(days) == report.HISTORY_RUNS and first not in days # 10 days, both slots each + assert len(list((tmp_path / "hail_daytime").glob("*.json"))) == 2 * report.HISTORY_RUNS + report.submit("hail_daytime", last, [], root=tmp_path, slot="13h") # same slot: replaced + assert [len(r["analyses"]) for r in report.day_runs("hail_daytime", last, tmp_path)] == [0, 1] diff --git a/tests/test_snow.py b/tests/test_snow.py new file mode 100644 index 0000000..3a43218 --- /dev/null +++ b/tests/test_snow.py @@ -0,0 +1,155 @@ +import json +from datetime import date, datetime + +import numpy as np +import pandas as pd + +from damage_engine.detectors import snow +from damage_engine.schemas import EVENT, NO_DATA, NO_EVENT, Analysis, Severity, camera_request +from damage_engine.sources import derived +from damage_engine.sources.fields import Field + +RUN_DAY = date(2026, 2, 10) +# fetched for RUN_DAY: 12 look-back records (20:00 of 8 Feb ... 07:00 of 9 Feb), then the 24 h +# checked (08:00 of 9 Feb ... 07:00 of 10 Feb) +HOURS = pd.date_range("2026-02-08 20:00", "2026-02-10 07:00", freq="h") +LB = 12 # index of the first record of the 24 h (08:00) +WARM = [8.0] * 36 +ALL = {"TemperaturaAria", "UmiditaRelativa", "Pioggia", "RadiazioneSolare"} +NO_SOLAR = {"TemperaturaAria", "UmiditaRelativa", "Pioggia"} + + +def _day(station, t, rh=None, rain=None, solar=None): + n = len(t) + nan = [np.nan] * n + rh = [70.0] * n if rh is None else rh + return pd.DataFrame({ + "CodiceStazione": station, "Timestamp": HOURS[:n], + derived.T_MIN: t, derived.T_MEAN: [x + 0.5 for x in t], derived.T_MAX: [x + 1 for x in t], + derived.RH_MIN: rh, derived.RH_MEAN: rh, derived.RH_MAX: rh, + snow.COL_RAIN: [0.0] * n if rain is None else rain, + snow.COL_SOLAR: nan if solar is None else solar, + }) + + +def _field(station, bbch=None): + return Field(id=f"ST{station}", name=f"field {station}", station_id=station, bbch=bbch) + + +def _set(base, at, values): + out = list(base) + out[at:at + len(values)] = values + return out + + +def _one(df, sensors=NO_SOLAR, bbch=None, day=RUN_DAY): + [a] = snow.detect(df, [_field(3, bbch)], day, station_sensors={3: sensors}) + return a + + +def test_window_photo_time_and_lookback(): + assert snow.RUN_HOUR == 7 and snow.LOOKBACK_H == 12 + assert snow.day_window(RUN_DAY) == (datetime(2026, 2, 9, 7), datetime(2026, 2, 10, 7)) + assert snow.photo_time(RUN_DAY) == datetime(2026, 2, 10, 9) + + +def test_warm_day_no_snow_possible(): + a = _one(_day(3, WARM, rain=_set([0.0] * 36, LB + 5, [4.0]))) + assert a.status == NO_EVENT and a.severity == Severity.NONE + assert a.details["cold_hours"] == 0 and a.details["signals"] == [] + assert "camera_request" not in a.details + assert a.summary.startswith("No snow possible: lowest wet-bulb") + + +def test_signal_a_cold_precipitation_is_likely_and_asks_for_a_photo(): + t = _set(WARM, LB + 15, [-1.0, -1.5, -1.0]) # 23:00, 00:00, 01:00 + rain = _set([0.0] * 36, LB + 15, [1.2, 2.0, 0.8]) + a = _one(_day(3, t, rh=[90.0] * 36, rain=rain)) + assert a.status == EVENT and a.severity == Severity.NONE and a.severity_score == 0.0 + d = a.details + assert d["signals"] == ["A"] and d["suspicion"] == "likely" + assert d["snow_water_mm"] == 4.0 and d["est_new_snow_cm"] == 4.0 # × 10, mm -> cm + assert (a.duration_h, a.start, a.end) == (3, datetime(2026, 2, 9, 23), datetime(2026, 2, 10, 1)) + req = camera_request(d) + assert req == {"phenomenon": "snow", "station_id": 3, "photo_at": "2026-02-10T09:00", + "level": "likely", "signals": ["A"], "reason": req["reason"]} + assert "Photo requested for 10 Feb 09:00" in a.summary and "4 cm" in a.summary + + +def test_signal_b_rain_soon_after_cold_hours_only_within_12h(): + t = _set(WARM, 0, [-2.0] * LB) # cold dry night before the window + rain = _set([0.0] * 36, LB + 2, [3.0]) # 10:00, 3 h after the last cold hour + a = _one(_day(3, t, rain=rain)) + assert a.status == EVENT and a.details["signals"] == ["B"] and a.details["suspicion"] == "likely" + assert a.details["melt_water_mm"] == 3.0 and a.details["est_new_snow_cm"] == 0.0 + assert a.details["est_new_snow_max_cm"] == 3.0 + late = _one(_day(3, t, rain=_set([0.0] * 36, LB + 20, [3.0]))) # 04:00: 21 h later + assert late.status == NO_EVENT + + +def test_signal_c_only_is_possible(): + t = _set(WARM, LB + 18, [-0.5] * 4) + rh = _set([70.0] * 36, LB + 18, [98.0] * 4) + a = _one(_day(3, t, rh=rh)) + assert a.status == EVENT and a.details["signals"] == ["C"] and a.details["suspicion"] == "possible" + assert a.details["saturated_hours"] == 4 and a.duration_h == 4 + + +def test_signal_d_raises_c_to_likely_but_never_triggers_alone(): + t = _set(WARM, LB, [-0.5] * 3) # 08:00 ... 10:00 cold and saturated + rh = _set([70.0] * 36, LB, [98.0] * 3) + dark = _set([0.0] * 36, LB + 3, [10.0] * 4) # midday records 11:00 ... 14:00 + a = _one(_day(3, t, rh=rh, solar=dark), sensors=ALL) + assert a.details["signals"] == ["C", "D"] and a.details["suspicion"] == "likely" + assert a.details["sensor_covered"] is True and a.details["midday_solar_max_wm2"] == 10 + grey = _one(_day(3, t, rh=rh, solar=_set(dark, LB + 3, [150.0] * 4)), sensors=ALL) + assert grey.details["signals"] == ["C"] and grey.details["sensor_covered"] is False + dry_cold = _one(_day(3, [-1.0] * 36, solar=dark), sensors=ALL) # cold, dark, no other sign + assert dry_cold.status == NO_EVENT and dry_cold.details["cold_hours"] == 24 + + +def test_no_humidity_uses_air_temperature(): + t = _set(WARM, LB + 15, [0.5]) + rain = _set([0.0] * 36, LB + 15, [2.0]) + sensors = {"TemperaturaAria", "Pioggia"} + a = _one(_day(3, t, rh=[np.nan] * 36, rain=rain), sensors=sensors) + assert a.status == EVENT and a.details["signals"] == ["A"] + assert a.details["cold_basis"] == "air" and a.details["air_fallback_hours"] == 24 + assert any("No humidity sensor" in n for n in a.data_notes) + dry = _one(_day(3, [-1.0] * 36, rh=[np.nan] * 36), sensors=sensors) # C cannot fire + assert dry.status == NO_EVENT + + +def test_no_rain_gauge_only_saturated_air_can_flag(): + rh = _set([70.0] * 36, LB + 18, [98.0] * 2) + sensors = {"TemperaturaAria", "UmiditaRelativa"} + a = _one(_day(3, [-1.0] * 36, rh=rh, rain=[np.nan] * 36), sensors=sensors) + assert a.status == EVENT and a.details["signals"] == ["C"] and a.details["rain_checked"] is False + assert any("No rain gauge" in n for n in a.data_notes) + + +def test_missing_temperature_and_missing_station(): + a = _one(_day(3, [np.nan] * 36)) + assert a.status == NO_DATA and "No air temperature data" in a.data_notes[0] + part = _one(_day(3, _set(WARM, LB, [np.nan] * 4))) + assert part.status == NO_EVENT and part.details["records"] == 20 + assert any("missing for 4 of 24" in n for n in part.data_notes) + [none] = snow.detect(_day(3, WARM), [_field(5)], RUN_DAY) + assert none.status == NO_DATA and "No records" in none.data_notes[0] + + +def test_solar_note_and_cover_window(): + a = _one(_day(3, WARM)) + assert any("No solar radiation sensor" in n for n in a.data_notes) + assert snow.in_cover_window(date(2026, 11, 15), 60) is False + assert snow.in_cover_window(date(2026, 4, 1), None) is None + assert snow.in_cover_window(date(2026, 4, 1), 40) is False # dormant + assert snow.in_cover_window(date(2026, 4, 1), 60) is True + + +def test_event_stores_and_reloads_as_json(): + t = _set(WARM, LB + 15, [-1.0]) + a = _one(_day(3, t, rain=_set([0.0] * 36, LB + 15, [1.0])), bbch=55) + assert a.details["in_cover_window"] is True and a.details["bbch"] == 55 + back = Analysis.from_dict(json.loads(json.dumps(a.to_dict()))) + assert camera_request(back.details)["photo_at"] == "2026-02-10T09:00" diff --git a/tests/test_station_report.py b/tests/test_station_report.py new file mode 100644 index 0000000..eb0dfee --- /dev/null +++ b/tests/test_station_report.py @@ -0,0 +1,507 @@ +import json +from datetime import date, datetime + +from damage_engine.report import report, station_report +from damage_engine.schemas import EVENT, NO_DATA, NO_EVENT, Analysis, Severity + +DAY = date(2026, 4, 22) + + +def _a(block, station, status=NO_EVENT, details=None, field=None, run_day=DAY, **kw): + return Analysis(block=block, run_day=run_day, field_id=field or f"ST{station}", + field_name=f"field {station}", station_id=station, station_name=f"S{station}", + status=status, details=details or {}, **kw) + + +BRINA_EVENT = dict(status=EVENT, severity=Severity.SEVERE, severity_score=0.79, duration_h=5, + start=datetime(2026, 4, 22, 1), end=datetime(2026, 4, 22, 5), + details={"wind_checked": True, "records": 13, "night_min_temp_c": -1.2, + "min_temp_c": -1.2, "min_bud_temp_c": -3.2, "min_wet_bulb_c": -1.58, + "severity_method": "kill_fraction", "bbch": 65, "stage": "full_bloom", + "kill_fraction": 0.57, "in_cover_window": True, + "max_wind_ms": 3.4, "windy_hours": 2}) +FLOOD_DRY = {"records": 24, "rain_day_mm": 0.0, "max_1h_mm": 0.0, "max_72h_mm": None, + "rain_72h_to_run_mm": 8.6, "rain_10d_mm": 8.8, "reference_10d_mm": None} + + +def _store(root): + report.submit("brina", DAY, [_a("brina", 566, **BRINA_EVENT), + _a("brina", 3, details={"night_min_temp_c": 9.0})], root=root) + report.submit("flood", DAY, [_a("flood", 566, details=FLOOD_DRY), + _a("flood", 3, details=FLOOD_DRY)], root=root) + + +def _check(block, name): + return next(c for c in block["checks"] if c["check"] == name) + + +def test_report_holds_only_its_station_and_every_block(tmp_path): + _store(tmp_path) + c = station_report.build(566, DAY, tmp_path) + assert [b["block"] for b in c["blocks"]] == list(station_report.EVENTS) # every event, diagram order + assert {b["block"] for b in c["blocks"] if b["status"] != "not_run"} == {"brina", "flood"} + assert {f["field_id"] for b in c["blocks"] for f in b["fields"]} == {"ST566"} + assert c["overall"]["events"] == 1 and c["overall"]["worst_severity"] == "severe" + + +def test_checks_show_value_threshold_and_margin(tmp_path): + _store(tmp_path) + c = station_report.build(566, DAY, tmp_path) + brina, flood = _block(c, "brina"), _block(c, "flood") + air = _check(brina, "Lowest air temperature") + assert (air["measured"], air["threshold"], air["status"], air["margin"]) == (-1.2, 2.0, "crossed", 3.2) + wind = _check(brina, "Highest wind in brina hours") # not a condition: info only + assert (wind["measured"], wind["status"]) == (3.4, "info") + rain = _check(flood, "Rain in 1 hour (wettest hour)") + assert (rain["status"], rain["margin"], rain["threshold_text"]) == ("not_crossed", 30.0, "≥ 30 mm") + assert _check(flood, "Rain in the last 10 days")["status"] == "info" # no 5-year average + + +def _block(content, name): + return next(b for b in content["blocks"] if b["block"] == name) + + +def test_no_event_night_marks_unstored_values(tmp_path): + _store(tmp_path) + brina = _block(station_report.build(3, DAY, tmp_path), "brina") + assert _check(brina, "Lowest wet-bulb temperature")["status"] == "not_recorded" + assert _check(brina, "Lowest air temperature")["status"] == "not_crossed" + assert _check(brina, "Highest wind in brina hours")["status"] == "not_applied" # no wind_checked + + +def test_no_data_has_no_checks(tmp_path): + report.submit("brina", DAY, [_a("brina", 600, status=NO_DATA, data_notes=["No sensor."])], + root=tmp_path) + c = station_report.build(600, DAY, tmp_path) + assert _block(c, "brina")["checks"] == [] and _block(c, "brina")["data_notes"] == ["No sensor."] + assert c["overall"]["headline"] == "No usable data for this station." + + +def test_write_makes_json_and_pdf(tmp_path): + _store(tmp_path / "reports") + json_path, pdf_path = station_report.write(566, DAY, tmp_path / "reports", tmp_path / "out") + assert json_path.name == "2026-04-22.json" and json_path.parent.name == "566" + assert json.loads(json_path.read_text(encoding="utf-8"))["station"]["id"] == 566 + assert pdf_path.read_bytes()[:5] == b"%PDF-" + + +def test_missing_sensors_compared_with_other_stations(tmp_path): + c = station_report.build(153, DAY, tmp_path) # Caraglio: no wind, no solar + brina = [m["sensor"] for m in _block(c, "brina")["missing_sensors"]] + assert brina == ["VelocitaVento2m", "RadiazioneSolare"] + wind = _block(c, "strong_wind")["missing_sensors"][0] + assert wind == {"sensor": "VelocitaVento2m", "name": wind["name"]} # no counts of other stations + assert _block(c, "flood")["missing_sensors"] == [] # every station has a rain gauge + drought = [m["sensor"] for m in _block(c, "drought")["missing_sensors"]] # rain + soil moisture + assert drought == ["Suolo1", "Suolo2"] + + +def test_five_year_note_hidden_and_new_block_gets_generic_table(tmp_path): + report.submit("flood", DAY, [_a("flood", 3, details=FLOOD_DRY, + data_notes=["No 5-year reference average for the 10-day rain: x.", + "Rain missing for 2 of 24 hours."])], root=tmp_path) + report.submit("new_block", DAY, [_a("new_block", 3, details={"max_size_mm": 21.5, + "wind_checked": True})], + root=tmp_path) + c = station_report.build(3, DAY, tmp_path) + assert _block(c, "flood")["data_notes"] == ["Rain missing for 2 of 24 hours."] + [size] = _block(c, "new_block")["checks"] + assert (size["check"], size["measured"], size["status"]) == ("Max size mm", 21.5, "info") + + +WIND_EVENT = dict(status=EVENT, severity=Severity.MODERATE, severity_score=0.6, duration_h=3, + start=datetime(2026, 4, 21, 18), end=datetime(2026, 4, 21, 20), + details={"records": 24, "trigger": "hourly_max", "max_wind_ms": 18.5, + "max_wind_kmh": 66.6, "max_wind_at": "2026-04-21T19:00", + "max_mean_wind_ms": 9.3, "fallback_hours": 0, "strong_wind_hours": 3, + "in_cover_window": None}) + + +def test_strong_wind_checks_period_and_severity_text(tmp_path): + report.submit("strong_wind", DAY, [ + _a("strong_wind", 566, **WIND_EVENT), + _a("strong_wind", 3, details={"records": 24, "trigger": "hourly_max", "max_wind_ms": 8.0, + "max_wind_kmh": 28.8, "max_wind_at": "2026-04-22T02:00", + "max_mean_wind_ms": None, "fallback_hours": 0})], root=tmp_path) + wind = _block(station_report.build(566, DAY, tmp_path), "strong_wind") + assert wind["window"] == {"start": "2026-04-21T07:00", "end": "2026-04-22T07:00"} + top = _check(wind, "Highest wind in the 24 h") + assert (top["status"], top["margin"], top["threshold_text"]) == ("crossed", 4.6, "≥ 13.9 m/s") + assert top["note"].startswith("At 21 Apr 19:00, 66.6 km/h. ≥ 50 km/h, Beaufort 7") + time_ = _check(wind, "Time with wind ≥ 13.9 m/s") + assert (time_["measured"], time_["status"], time_["threshold_text"]) == (3, "crossed", "≥ 10 min") + assert _check(wind, "Highest hourly mean wind")["status"] == "info" + field = wind["fields"][0] + assert field["severity_basis"].startswith("Graded on the highest wind") + assert field["severity_scale"] == ("Wind: light ≥ 13.9 m/s, moderate ≥ 17.2 m/s, severe ≥ 20.8 m/s, " + "extreme ≥ 24.5 m/s (Beaufort 7, 8, 9, 10)") + calm = _block(station_report.build(3, DAY, tmp_path), "strong_wind") + assert _check(calm, "Highest wind in the 24 h")["status"] == "not_crossed" + assert _check(calm, "Time with wind ≥ 13.9 m/s")["status"] == "not_crossed" + assert _check(calm, "Highest hourly mean wind")["status"] == "not_recorded" + + +FREEZING_EVENT = dict(status=EVENT, severity=Severity.MODERATE, severity_score=0.56, duration_h=4, + start=datetime(2026, 4, 21, 10), end=datetime(2026, 4, 21, 13), + details={"records": 13, "daylight_hours": 13, "sunlit_hours_in_night_window": 2, + "solar_checked": True, "day_min_temp_c": -2.5, "mean_wind_ms": 3.0, + "advective_wind": True, "min_wet_bulb_c": -3.47, + "severity_method": "fallback_min_bud_temp", "in_cover_window": None}) + + +def test_freezing_checks_period_and_severity_text(tmp_path): + report.submit("freezing", DAY, [_a("freezing", 566, **FREEZING_EVENT), + _a("freezing", 153, details={"records": 11, "daylight_hours": 11, + "sunlit_hours_in_night_window": 0, + "solar_checked": False, + "day_min_temp_c": 8.0})], root=tmp_path) + frz = _block(station_report.build(566, DAY, tmp_path), "freezing") + assert frz["window"] == {"start": "2026-04-21T06:00", "end": "2026-04-22T06:00"} + air = _check(frz, "Lowest daylight air temperature") + assert (air["status"], air["margin"], air["threshold_text"]) == ("crossed", 2.5, "< 0 °C") + assert _check(frz, "Daylight hours below 0 °C")["status"] == "crossed" + assert _check(frz, "Mean wind while below 0 °C")["status"] == "crossed" # ≥ 2 m/s + assert _check(frz, "Sunlit hours of the night (solar radiation)")["measured"] == 2 + assert frz["fields"][0]["severity_basis"] == ("Crop stage not set: graded on the lowest " + "daylight air temperature.") + quiet = _block(station_report.build(153, DAY, tmp_path), "freezing") # no pyranometer, warm day + assert _check(quiet, "Lowest daylight air temperature")["status"] == "not_crossed" + assert _check(quiet, "Sunlit hours of the night (solar radiation)")["status"] == "not_applied" + assert _check(quiet, "Mean wind while below 0 °C")["status"] == "not_recorded" + assert _check(quiet, "Lowest wet-bulb while below 0 °C")["status"] == "not_recorded" + + +HEAVY = {"mm": 90.0, "first_rain": "2026-04-18T12:00", "last_rain": "2026-04-19T18:00", + "reached_at": "2026-04-19T18:00", "partial": False} + + +def test_rain_row_3_is_the_last_72h_over_the_threshold(tmp_path): + report.submit("flood", DAY, [ + _a("flood", 566, details={**FLOOD_DRY, "last_heavy_72h": HEAVY, "wettest_72h": HEAVY}), + _a("flood", 3, details={**FLOOD_DRY, "last_heavy_72h": None, + "wettest_72h": {**HEAVY, "mm": 41.5, "partial": True}}), + _a("flood", 5, details=FLOOD_DRY), # run stored before this row existed + ], root=tmp_path) + row = _block(station_report.build(566, DAY, tmp_path), "flood")["checks"][2] + assert row["check"] == "72-Hour Heavy Rain Watch (18 Apr 07:00 → 22 Apr 07:00)" # the 4 days checked + assert (row["measured"], row["status"], row["margin"]) == (90.0, "crossed", 10.0) + assert row["note"] == "Rain 18 Apr 12:00 → 19 Apr 18:00; 80 mm reached on 19 Apr 18:00." + c = station_report.build(566, DAY, tmp_path) # no new event: past one in the overview + assert _block(c, "flood")["notice"]["short"] == "Past: 90 mm in 72 h (18 Apr → 19 Apr)" + assert c["overall"]["notices"][0]["text"].startswith("No new event, but 90 mm of rain fell") + assert c["overall"]["headline"].startswith("No new damaging event") + assert _block(station_report.build(3, DAY, tmp_path), "flood")["notice"] is None + none = _block(station_report.build(3, DAY, tmp_path), "flood")["checks"][2] + assert (none["measured"], none["status"], none["margin"]) == (41.5, "not_crossed", 38.5) + assert none["check"] == "72-Hour Heavy Rain Watch (18 Apr 07:00 → 22 Apr 07:00)" + assert none["note"] == ("Rain 18 Apr 12:00 → 19 Apr 18:00. " # the wettest window + "Rain also fell just before these days (not counted).") + old = _block(station_report.build(5, DAY, tmp_path), "flood")["checks"][2] + assert old["status"] == "not_recorded" and old["check"] == "72-Hour Heavy Rain Watch (18 Apr 07:00 → 22 Apr 07:00)" + + +def test_rain_row_3_name_is_the_watch_and_its_4_days(tmp_path): + report.submit("flood", DAY, [_a("flood", 3, details={**FLOOD_DRY, "last_heavy_72h": None, + "wettest_72h": {**HEAVY, "mm": 63.2}})], + root=tmp_path) + row = _block(station_report.build(3, DAY, tmp_path), "flood")["checks"][2] + assert (row["check"], row["note"]) == ("72-Hour Heavy Rain Watch (18 Apr 07:00 → 22 Apr 07:00)", "Rain 18 Apr 12:00 → 19 Apr 18:00.") + later = date(2026, 4, 26) + report.submit("flood", later, [_a("flood", 3, run_day=later, details={ + **FLOOD_DRY, "last_heavy_72h": None, "wettest_72h": None})], root=tmp_path) + row = _block(station_report.build(3, later, tmp_path), "flood")["checks"][2] + assert row["check"] == "72-Hour Heavy Rain Watch (22 Apr 07:00 → 26 Apr 07:00)" # follows the run day + assert row["note"] == "No rain in the last 4 days." + one = {"mm": 0.2, "first_rain": "2026-04-24T06:00", "last_rain": "2026-04-24T06:00", "partial": False} + report.submit("flood", later, [_a("flood", 3, run_day=later, details={ + **FLOOD_DRY, "last_heavy_72h": None, "wettest_72h": one})], root=tmp_path) + row = _block(station_report.build(3, later, tmp_path), "flood")["checks"][2] + assert row["note"] == "Rain at 24 Apr 06:00." # one rainy hour + + +def test_rain_row_2_is_the_last_72_hours(tmp_path): + intense = dict(status=EVENT, severity=Severity.LIGHT, severity_score=0.1, duration_h=1, + details={**FLOOD_DRY, "rain_day_mm": 1.0, "max_72h_mm": 85.0, + "rain_72h_to_run_mm": 70.0, "criteria_met": ["intense_72h"], + "worst_criterion": "intense_72h", "rain_trigger_ratio": 1.06}) + report.submit("flood", DAY, [ + _a("flood", 566, details=FLOOD_DRY), + _a("flood", 3, details={**FLOOD_DRY, "rain_72h_to_run_mm": 95.0}), # dry day after a storm + _a("flood", 5, **intense), # earlier window crossed + ], root=tmp_path) + rows = {s: _block(station_report.build(s, DAY, tmp_path), "flood")["checks"] for s in (566, 3, 5)} + assert [r["check"] for r in rows[566]][:2] == ["Rain in 1 hour (wettest hour)", "Rain in the last 72 hours"] + quiet, after_storm, earlier = (rows[s][1] for s in (566, 3, 5)) + assert (quiet["measured"], quiet["status"], quiet["margin"], quiet["note"]) == (8.6, "not_crossed", 71.4, "") + assert quiet["threshold_text"] == "≥ 80 mm" + assert (after_storm["status"], after_storm["margin"]) == ("crossed", 15.0) + assert after_storm["note"].startswith("No rain in the last 24 h: not a new event") + assert (earlier["measured"], earlier["status"]) == (70.0, "not_crossed") + assert earlier["note"].startswith("The event comes from an earlier 72 h window") + + +def test_overview_period_of_rain_includes_the_last_4_days(tmp_path): + _store(tmp_path) + c = station_report.build(566, DAY, tmp_path) + flood = _block(c, "flood") + assert flood["window"] == {"start": "2026-04-21T07:00", "end": "2026-04-22T07:00"} + assert flood["watch_window"] == {"label": "Last 4 days", "start": "2026-04-18T07:00", + "end": "2026-04-22T07:00"} + assert _block(c, "brina")["watch_window"] is None + + +def test_new_run_replaces_only_that_day_and_keeps_10_days(tmp_path): + _store(tmp_path / "reports") + folder = tmp_path / "out" / "566" + days = [date(2026, 4, d) for d in range(11, 23)] # 12 days: 11 → 22 April + for d in days: + station_report.write(566, d, tmp_path / "reports", tmp_path / "out") + kept = sorted(p.stem for p in folder.glob("*.json")) + assert kept == [d.isoformat() for d in days[-10:]] # the last 10 days stay + assert sorted(p.stem for p in folder.glob("*.pdf")) == kept + report.submit("flood", DAY, [_a("flood", 566, details={**FLOOD_DRY, "rain_day_mm": 7.5})], + root=tmp_path / "reports") # new run of the same day + station_report.write(566, DAY, tmp_path / "reports", tmp_path / "out") + assert sorted(p.stem for p in folder.glob("*.json")) == kept # no other day removed + new = json.loads((folder / "2026-04-22.json").read_text(encoding="utf-8")) + assert _block(new, "flood")["details"]["rain_day_mm"] == 7.5 # that day's report replaced + + +HOT_WIND_EVENT = dict(status=EVENT, severity=Severity.MODERATE, severity_score=0.55, duration_h=2, + start=datetime(2026, 4, 21, 14), end=datetime(2026, 4, 21, 15), + details={"records": 24, "trigger": "hourly_max", "max_temp_c": 42.3, + "max_temp_at": "2026-04-21T15:00", "max_mean_temp_c": 40.1, + "fallback_hours": 0, "hot_hours": 3, "still_hot_hours": 1, + "wind_checked": True, "wind_at_max_temp_ms": 2.4, + "humidity_at_max_temp_pct": 18, "direction_checked": False, + "hot_wind_hours": 2, "max_hot_wind_temp_c": 42.3, + "wind_assumed_hours": 0, "min_humidity_pct": 18, + "harvest_checked": False, "in_cover_window": None}) + + +def test_hot_wind_checks_period_and_severity_text(tmp_path): + report.submit("hot_wind", DAY, [ + _a("hot_wind", 566, **HOT_WIND_EVENT), + _a("hot_wind", 3, details={"records": 24, "trigger": "hourly_max", "max_temp_c": 31.0, + "max_temp_at": "2026-04-21T16:00", "max_mean_temp_c": 29.5, + "fallback_hours": 0, "hot_hours": 0, "still_hot_hours": 0, + "wind_checked": False, "wind_at_max_temp_ms": None, + "humidity_at_max_temp_pct": None, "direction_checked": False})], + root=tmp_path) + hot = _block(station_report.build(566, DAY, tmp_path), "hot_wind") + assert hot["window"] == {"start": "2026-04-21T07:00", "end": "2026-04-22T07:00"} + top = _check(hot, "Highest air temperature in the 24 h") + assert (top["status"], top["margin"], top["threshold_text"]) == ("crossed", 2.3, "≥ 40 °C") + assert top["note"].startswith("At 21 Apr 15:00. ≥ 40 °C at 2 m (contract)") + hours = _check(hot, "Hours at ≥ 40 °C with moving air") + assert (hours["measured"], hours["status"], hours["threshold_text"]) == (2, "crossed", "≥ 1 h") + assert "1 hour(s) at ≥ 40 °C had still air." in hours["note"] + wind = _check(hot, "Wind in the hottest hour") + assert (wind["status"], wind["threshold_text"]) == ("crossed", "≥ 1 m/s") + direction = _check(hot, "Wind from south-east to south-west") + assert (direction["status"], direction["threshold_text"]) == ("not_applied", "135°–225°") + assert _check(hot, "Humidity in the hottest hour")["status"] == "info" + field = hot["fields"][0] + assert field["severity_basis"].startswith("Graded on the highest air temperature") + assert field["severity_scale"] == ("Air temperature: light ≥ 40 °C, moderate ≥ 42 °C, " + "severe ≥ 44 °C, extreme ≥ 46 °C") + mild = _block(station_report.build(3, DAY, tmp_path), "hot_wind") + assert _check(mild, "Highest air temperature in the 24 h")["status"] == "not_crossed" + assert _check(mild, "Hours at ≥ 40 °C with moving air")["status"] == "not_crossed" + assert _check(mild, "Wind in the hottest hour")["status"] == "not_applied" + assert _check(mild, "Humidity in the hottest hour")["status"] == "not_recorded" + + +HIGH_TEMPERATURE_EVENT = dict( + status=EVENT, severity=Severity.MODERATE, severity_score=0.6, duration_h=2, + start=datetime(2026, 4, 21, 14), end=datetime(2026, 4, 21, 15), + details={"records": 24, "trigger": "hourly_max", "max_temp_c": 41.2, + "max_temp_at": "2026-04-21T15:00", "max_mean_temp_c": 39.5, "fallback_hours": 0, + "humidity_at_max_temp_pct": 22, "wet_bulb_at_max_temp_c": 24.1, + "solar_checked": True, "sunlit_hours": 10, "max_solar_wm2": 950, + "max_fst_c": 51.1, "max_fst_at": "2026-04-21T14:00", "solar_at_max_fst_wm2": 950, + "wind_at_max_fst_ms": 1.2, "sunburn_risk_hours": 4, "max_risk_fst_c": 51.1, + "hot_hours": 2, "severity_method": "fst", "severity_value_c": 51.1, + "in_cover_window": None}) + + +def test_high_temperature_checks_period_and_severity_text(tmp_path): + report.submit("high_temperature", DAY, [ + _a("high_temperature", 566, **HIGH_TEMPERATURE_EVENT), + _a("high_temperature", 3, status=EVENT, severity=Severity.LIGHT, duration_h=1, + start=datetime(2026, 4, 21, 15), end=datetime(2026, 4, 21, 15), + details={"records": 24, "trigger": "hourly_max", "max_temp_c": 40.4, + "max_temp_at": "2026-04-21T15:00", "fallback_hours": 0, + "humidity_at_max_temp_pct": None, "wet_bulb_at_max_temp_c": None, + "solar_checked": False, "max_fst_c": None, "sunburn_risk_hours": None, + "hot_hours": 1, "severity_method": "air", "severity_value_c": 40.4})], + root=tmp_path) + hot = _block(station_report.build(566, DAY, tmp_path), "high_temperature") + assert hot["window"] == {"start": "2026-04-21T07:00", "end": "2026-04-22T07:00"} + top = _check(hot, "Highest air temperature in the 24 h") + assert (top["status"], top["margin"], top["threshold_text"]) == ("crossed", 1.2, "≥ 40 °C") + assert top["note"].startswith("At 21 Apr 15:00. ≥ 40 °C at 2 m (contract)") + hours = _check(hot, "Hours at ≥ 40 °C") + assert (hours["measured"], hours["status"]) == (2, "crossed") + fst = _check(hot, "Highest fruit surface temperature") + assert (fst["status"], fst["margin"], fst["threshold_text"]) == ("crossed", 5.1, "≥ 46 °C") + assert fst["note"].startswith("At 21 Apr 14:00: sun 950 W/m², wind 1.2 m/s.") + risk = _check(hot, "Sunburn risk hours (air ≥ 32 °C, fruit ≥ 46 °C)") + assert (risk["measured"], risk["status"]) == (4, "crossed") + assert _check(hot, "Wet-bulb in the hottest hour")["status"] == "info" + field = hot["fields"][0] + assert field["severity_basis"].startswith("Graded on the highest estimated fruit surface temperature (51.1 °C)") + assert field["severity_scale"] == ("Fruit surface: light ≥ 46 °C, moderate ≥ 49 °C, " + "severe ≥ 52 °C, extreme ≥ 55 °C") + no_sun = _block(station_report.build(3, DAY, tmp_path), "high_temperature") + assert _check(no_sun, "Highest fruit surface temperature")["status"] == "not_applied" + assert _check(no_sun, "Sunburn risk hours (air ≥ 32 °C, fruit ≥ 46 °C)")["status"] == "not_applied" + assert _check(no_sun, "Humidity in the hottest hour")["status"] == "not_recorded" + assert no_sun["fields"][0]["severity_basis"].startswith("No solar radiation data: graded on the highest air") + assert no_sun["fields"][0]["severity_scale"].startswith("Air temperature: light ≥ 40 °C") + + +SNOW_SUSPECTED = dict(status=EVENT, duration_h=3, start=datetime(2026, 4, 21, 23), + end=datetime(2026, 4, 22, 1), + details={"records": 24, "cold_basis": "wet_bulb", "air_fallback_hours": 0, + "min_cold_c": -1.6, "min_cold_at": "2026-04-22T00:00", + "cold_hours": 3, "rain_checked": True, "humidity_checked": True, + "solar_checked": False, "snow_hours": 3, "snow_water_mm": 4.0, + "melt_hours": 0, "melt_water_mm": 0.0, "saturated_hours": 0, + "midday_solar_max_wm2": None, "est_new_snow_cm": 4.0, + "est_new_snow_max_cm": 4.0, "signals": ["A"], "suspicion": "likely", + "in_cover_window": True, + "camera_request": {"phenomenon": "snow", "station_id": 3, + "photo_at": "2026-04-22T09:00", "level": "likely", + "signals": ["A"], + "reason": "Snow likely: precipitation in a cold hour."}}) + + +def test_snow_is_shown_as_suspected_with_its_signs_and_photo(tmp_path): + report.submit("snow", DAY, [_a("snow", 3, **SNOW_SUSPECTED)], root=tmp_path) + c = station_report.build(3, DAY, tmp_path) + b = _block(c, "snow") + assert b["window"] == {"start": "2026-04-21T07:00", "end": "2026-04-22T07:00"} + [f] = b["fields"] + assert f["suspected"] and f["camera_request"]["photo_at"] == "2026-04-22T09:00" + assert f["severity_basis"].startswith("Not graded") + assert c["overall"]["events"] == 0 # not a detected event: a photo decides + assert c["overall"]["headline"] == "Snow suspected · photo requested to confirm" + assert "22 Apr 09:00" in c["overall"]["suspected"][0]["text"] + assert _check(b, "A · Precipitation in cold hours")["status"] == "crossed" + assert _check(b, "B · Rain within 12 h after a cold hour")["status"] == "not_crossed" + assert _check(b, "D · Highest midday sun after the first suspect hour")["status"] == "not_applied" + assert _check(b, "Lowest wet-bulb in the 24 h")["margin"] == 2.6 + json_path, pdf_path = station_report.write(3, DAY, tmp_path, out_dir=tmp_path / "out") + assert pdf_path.read_bytes()[:5] == b"%PDF-" + + +def test_hail_is_shown_as_suspected_with_its_signs_and_photo(tmp_path): + import pandas as pd + from damage_engine.detectors import hail + from damage_engine.sources.fields import Field + hours = pd.date_range("2026-04-21 07:00", "2026-04-22 07:00", freq="h") + t = [20.0] * len(hours) + t[10:12] = [14.0, 14.0] # 17:00: 6.5 °C drop + rain = [0.0] * len(hours) + rain[10:12] = [22.0, 2.0] # burst rain at 17:00 + df = pd.DataFrame({"CodiceStazione": 3, "Timestamp": hours, hail.COL_T_MEAN: t, + hail.COL_T_MIN: [x - 0.5 for x in t], hail.COL_RAIN: rain, + hail.COL_WIND: [3.0] * len(hours), hail.COL_WIND_MEAN: [1.0] * len(hours)}) + [a] = hail.detect(df, [Field(id="ST3", name="field 3", station_id=3)], DAY) + assert a.status == EVENT and a.details["suspicion"] == "likely" + report.submit("hail", DAY, [a], root=tmp_path) + c = station_report.build(3, DAY, tmp_path) + b = _block(c, "hail") + assert b["window"] == {"start": "2026-04-21T07:00", "end": "2026-04-22T07:00"} + [f] = b["fields"] + assert f["suspected"] and f["camera_request"]["photo_at"] == "2026-04-22T09:00" + assert f["severity_basis"].startswith("Not graded") + assert c["overall"]["events"] == 0 + assert c["overall"]["headline"] == "Hail suspected · photo requested to confirm" + burst = _check(b, "A · Rain in the wettest hour") + assert (burst["status"], burst["margin"]) == ("crossed", 2.0) + assert _check(b, "A · Highest rain rate")["status"] == "not_applied" + assert _check(b, "B · Largest temperature drop")["measured"] == 6.5 + assert _check(b, "C · Highest wind (gust)")["status"] == "not_crossed" + assert _check(b, "Signs in one storm")["measured"] == 2 + json_path, pdf_path = station_report.write(3, DAY, tmp_path, out_dir=tmp_path / "out") + assert pdf_path.read_bytes()[:5] == b"%PDF-" + + +DROUGHT_MONTHS = [ + {"month": "2026-01", "label": "Jan 2026", "days": 31, "missing_days": 0, "rain_mm": 10.0, + "normal_mm": 52.7, "rain_pct_of_normal": 19.0, "et0_mm": 20.0, "water_balance_mm": -10.0}, + {"month": "2026-02", "label": "Feb 2026", "days": 28, "missing_days": 2, "rain_mm": 20.0, + "normal_mm": 61.5, "rain_pct_of_normal": 32.5, "et0_mm": 30.0, "water_balance_mm": -10.0}, + {"month": "2026-03", "label": "Mar 2026", "days": 31, "missing_days": 0, "rain_mm": 30.0, + "normal_mm": 82.2, "rain_pct_of_normal": 36.5, "et0_mm": 50.0, "water_balance_mm": -20.0}, +] +DROUGHT_EVENT = dict(status=EVENT, severity=Severity.MODERATE, severity_score=0.6, + duration_h=90 * 24, start=datetime(2026, 1, 1), end=datetime(2026, 4, 1), + details={"months": DROUGHT_MONTHS, "window_label": "Jan–Mar 2026", + "window_months": 3, "days": 90, "missing_days": 2, "rain_mm": 60.0, + "normal_mm": 196.4, "rain_pct_of_normal": 30.5, "deficit_mm": 136.4, + "last_month_label": "Mar 2026", "et0_mm": 100.0, + "water_balance_mm": -40.0, "water_balance_months": 3, + "soil_moisture": None, "window_start": "2026-01-01T00:00", + "window_end": "2026-04-01T00:00"}) + + +def test_drought_shows_rain_against_the_average_month_by_month(tmp_path): + report.submit("drought", date(2026, 4, 1), [_a("drought", 566, **DROUGHT_EVENT)], root=tmp_path) + _store(tmp_path) # daily blocks of 22 Apr + c = station_report.build(566, DAY, tmp_path) + b = _block(c, "drought") + assert b["window"]["label"] == "Jan–Mar 2026" and b["monthly"] and b["stale"] + pct = _check(b, "Rain as % of the average") + assert (pct["measured"], pct["status"], pct["margin"]) == (30.5, "crossed", 29.5) + assert pct["threshold_text"] == "≤ 60 % of the average (= 117.8 mm)" + total = _check(b, "Rain in the last 3 months (Jan–Mar 2026)") + assert total["threshold_text"] == "average 196.4 mm" and "136.4 mm below" in total["note"] + feb = _check(b, "Rain in Feb 2026") + assert (feb["measured"], feb["threshold_text"], feb["status"]) == (20.0, "average 61.5 mm", "info") + assert _check(b, "Water balance: rain − ET0 (Jan–Mar 2026)")["measured"] == -40.0 + assert _check(b, "Soil moisture, mean of Mar 2026")["status"] == "not_applied" + [f] = b["fields"] + assert f["period_text"] == "Jan–Mar 2026 (90 days)" and f["duration_text"] == "90 d" + assert f["severity_scale"].startswith("Rain / average: light ≤ 60 %") + json_path, pdf_path = station_report.write(566, DAY, tmp_path, out_dir=tmp_path / "out") + assert pdf_path.read_bytes()[:5] == b"%PDF-" + + +def test_hail_daytime_suspicion_is_merged_into_the_hail_section(tmp_path): + report.submit("hail", DAY, [_a("hail", 3, summary="No hail suspected: no rain in the 24 hours. " + "Hail-risk day: the station is checked again " + "today at 13:00 and 19:00.", + details={"hail_risk": True, "storm_signs": 0})], root=tmp_path) + quiet = _a("hail_daytime", 3, summary="No hail suspected: no rain in the 6 hours.", + details={"check": "daytime", "window_start": "2026-04-22T07:00", + "window_end": "2026-04-22T13:00"}) + request = {"phenomenon": "hail", "station_id": 3, "photo_at": "2026-04-22T19:00", + "level": "likely", "signals": ["A", "B"], "reason": "Hail likely: burst rain; x."} + storm = _a("hail_daytime", 3, status=EVENT, duration_h=2, start=datetime(2026, 4, 22, 16), + end=datetime(2026, 4, 22, 17), summary="Hail likely: burst rain of 24 mm.", + details={"check": "daytime", "window_start": "2026-04-22T13:00", + "window_end": "2026-04-22T19:00", "camera_request": request, + "in_cover_window": None}, + data_notes=["No wind sensor (VelocitaVento2m) at this station: x."]) + report.submit("hail_daytime", DAY, [quiet], root=tmp_path, slot="13h") + report.submit("hail_daytime", DAY, [storm], root=tmp_path, slot="19h") + c = station_report.build(3, DAY, tmp_path) + assert not any(b["block"] == "hail_daytime" for b in c["blocks"]) + b = _block(c, "hail") + assert b["status"] == EVENT + assert b["watch_window"] == {"label": "Daytime checks", "start": "2026-04-22T07:00", + "end": "2026-04-22T19:00"} + [f] = b["fields"] + assert f["suspected"] and f["camera_request"]["photo_at"] == "2026-04-22T19:00" + assert (f["start"], f["end"], f["duration_h"]) == ("2026-04-22T16:00", "2026-04-22T17:00", 2) + assert "Daytime check 07:00 → 13:00: No hail suspected" in f["summary"] + assert "Daytime check 13:00 → 19:00: Hail likely" in f["summary"] + assert [d["slot"] for d in f["daytime_checks"]] == ["13h", "19h"] + assert b["data_notes"] == ["Daytime check 13:00 → 19:00: No wind sensor (VelocitaVento2m) at " + "this station: x."] + assert c["overall"]["headline"] == "Hail suspected · photo requested to confirm" + json_path, pdf_path = station_report.write(3, DAY, tmp_path, out_dir=tmp_path / "out") + assert pdf_path.read_bytes()[:5] == b"%PDF-" diff --git a/tests/test_strong_wind.py b/tests/test_strong_wind.py new file mode 100644 index 0000000..3703a6a --- /dev/null +++ b/tests/test_strong_wind.py @@ -0,0 +1,160 @@ +from datetime import date, datetime + +import numpy as np +import pandas as pd +import pytest + +from damage_engine import thresholds +from damage_engine.detectors import strong_wind +from damage_engine.schemas import EVENT, NO_DATA, NO_EVENT, Severity +from damage_engine.sources.fields import Field + +RUN_DAY = date(2026, 9, 15) +# the 24 h checked for RUN_DAY: records 08:00 of 14 Sep ... 07:00 of 15 Sep +HOURS = pd.date_range("2026-09-14 08:00", "2026-09-15 07:00", freq="h") +CALM = [3.0] * 24 + + +def _day(station, gust, mean=None, ts=None): + ts = HOURS[:len(gust)] if ts is None else ts + mean = [g / 2 if g is not None and not np.isnan(g) else np.nan for g in gust] if mean is None else mean + return pd.DataFrame({"CodiceStazione": station, "Timestamp": ts, + strong_wind.COL_MAX: gust, strong_wind.COL_MEAN: mean}) + + +def _field(station, bbch=None): + return Field(id=f"ST{station}", name=f"field {station}", station_id=station, bbch=bbch) + + +def _with_storm(values, at=10): + gust = list(CALM) + gust[at:at + len(values)] = values + return gust + + +def test_window_is_the_24h_ending_at_07(): + assert strong_wind.RUN_HOUR == 7 + assert strong_wind.day_window(RUN_DAY) == (datetime(2026, 9, 14, 7), datetime(2026, 9, 15, 7)) + + +def test_trigger_is_the_hourly_maximum(): + assert thresholds.STRONG_WIND["use_mean_or_gust"] == "hourly_max" + assert strong_wind.trigger_columns() == (strong_wind.COL_MAX, strong_wind.COL_MEAN) + + +def test_storm_is_an_event_graded_on_beaufort(): + # 18:00-20:00: 15.0, 18.5, 14.0 m/s hourly maximum, means far below the trigger + [a] = strong_wind.detect(_day(3, _with_storm([15.0, 18.5, 14.0])), [_field(3)], RUN_DAY, {3: "Lagnasco"}) + assert a.status == EVENT and a.block == "strong_wind" and a.station_name == "Lagnasco" + assert a.duration_h == 3 + assert a.start == datetime(2026, 9, 14, 18) and a.end == datetime(2026, 9, 14, 20) + assert a.details["max_wind_ms"] == 18.5 and a.details["max_wind_kmh"] == 66.6 + assert a.details["max_wind_at"] == "2026-09-14T19:00" + assert a.details["max_mean_wind_ms"] == 9.2 # rounded to 0.1 m/s + assert a.details["strong_wind_hours"] == 3 + assert a.severity == Severity.MODERATE # Beaufort 8: 17.2 <= 18.5 < 20.8 + assert a.details["in_cover_window"] is None # stage unknown + assert "18.5 m/s (66.6 km/h)" in a.summary + + +def test_trigger_is_at_least_13_9(): + [hit] = strong_wind.detect(_day(3, _with_storm([13.9])), [_field(3)], RUN_DAY) + [miss] = strong_wind.detect(_day(3, _with_storm([13.8])), [_field(3)], RUN_DAY) + assert hit.status == EVENT and hit.severity == Severity.LIGHT and hit.duration_h == 1 + assert miss.status == NO_EVENT and miss.severity == Severity.NONE + assert miss.details["max_wind_ms"] == 13.8 + assert miss.summary == "No strong wind: highest wind 13.8 m/s (49.7 km/h)." + + +@pytest.mark.parametrize("wind, level", [(17.2, Severity.MODERATE), (20.8, Severity.SEVERE), + (24.5, Severity.EXTREME), (30.0, Severity.EXTREME)]) +def test_beaufort_bands(wind, level): + [a] = strong_wind.detect(_day(3, _with_storm([wind])), [_field(3)], RUN_DAY) + assert a.severity == level + + +def test_hours_outside_the_24h_are_ignored(): + ts = pd.date_range("2026-09-14 07:00", "2026-09-15 08:00", freq="h") # 26 records + gust = [25.0] + CALM + [25.0] # storms at 07:00 (14th), 08:00 (15th) + [a] = strong_wind.detect(_day(3, gust, ts=ts), [_field(3)], RUN_DAY) + assert a.status == NO_EVENT and a.details["records"] == 24 + + +def test_cover_window_fruit_set_to_31_october(): + df = _day(3, _with_storm([16.0])) + [after, before] = strong_wind.detect(df, [_field(3, bbch=72), _field(3, bbch=65)], RUN_DAY) + assert after.details["in_cover_window"] is True # fruit set (BBCH > 69) + assert before.details["in_cover_window"] is False # still in bloom + assert after.severity == before.severity # severity does not depend on the stage + assert strong_wind.in_cover_window(date(2026, 11, 2), 75) is False + + +def test_missing_maximum_uses_the_mean(): + gust = _with_storm([np.nan, 16.0]) + mean = [1.5] * 24 + mean[10] = 15.0 # only the mean of 18:00 is known + [a] = strong_wind.detect(_day(3, gust, mean=mean), [_field(3)], RUN_DAY) + assert a.status == EVENT and a.duration_h == 2 and a.details["fallback_hours"] == 1 + assert any("Hourly maximum wind missing for 1 of 24 hours: hourly mean used" in n for n in a.data_notes) + + +def test_missing_hours_are_noted_and_not_checked(): + df = _day(3, _with_storm([20.0])).drop(index=[0, 1, 2]) # 3 hours with no record + [a] = strong_wind.detect(df, [_field(3)], RUN_DAY) + assert a.status == EVENT and a.details["records"] == 21 + assert "Wind missing for 3 of 24 hours: those hours not checked." in a.data_notes + + +def test_no_wind_values_is_no_data(): + df = _day(3, [np.nan] * 24, mean=[np.nan] * 24) + [a] = strong_wind.detect(df, [_field(3)], RUN_DAY) + assert a.status == NO_DATA + assert a.data_notes == ["No wind data in the 24 hours checked: strong wind cannot be judged."] + + +def test_missing_columns_do_not_break_the_block(): + df = pd.DataFrame({"CodiceStazione": 3, "Timestamp": HOURS}) # no wind columns at all + [a] = strong_wind.detect(df, [_field(3)], RUN_DAY) + assert a.status == NO_DATA + + +def test_station_without_records_is_no_data(): + [a] = strong_wind.detect(_day(3, CALM), [_field(6)], RUN_DAY) + assert a.status == NO_DATA and a.data_notes == ["No records from the station for the last 24 hours."] + + +def test_station_without_wind_sensor_is_no_data(monkeypatch): + station = type("S", (), {"id": 3, "name": "Lagnasco", "sensors": ["VelocitaVento2m"]})() + monkeypatch.setattr(strong_wind, "stations_with", lambda *s: [station]) + monkeypatch.setattr(strong_wind, "all_fields", lambda: [_field(3), _field(153)]) + monkeypatch.setattr(strong_wind, "fetch_hourly", lambda ids, start, end, cols: _day(3, CALM)) + out = {a.station_id: a for a in strong_wind.run(RUN_DAY)} + assert out[3].status == NO_EVENT + assert out[153].status == NO_DATA + assert out[153].data_notes == ["No wind sensor (VelocitaVento2m) at this station: " + "strong wind cannot be judged."] + + +def test_run_asks_for_the_24h_records(monkeypatch): + calls = [] + station = type("S", (), {"id": 3, "name": "Lagnasco", "sensors": ["VelocitaVento2m"]})() + monkeypatch.setattr(strong_wind, "stations_with", lambda *s: [station]) + monkeypatch.setattr(strong_wind, "all_fields", lambda: [_field(3)]) + monkeypatch.setattr(strong_wind, "fetch_hourly", + lambda ids, start, end, cols: calls.append((ids, start, end, cols)) or _day(3, CALM)) + strong_wind.run(RUN_DAY) + assert calls == [([3], datetime(2026, 9, 14, 8), datetime(2026, 9, 15, 7), strong_wind.COLUMNS)] + + +def test_unknown_trigger_setting_is_an_error(monkeypatch): + monkeypatch.setitem(thresholds.STRONG_WIND, "use_mean_or_gust", "gust") + with pytest.raises(ValueError, match="use_mean_or_gust"): + strong_wind.trigger_columns() + + +def test_hourly_mean_setting_uses_the_mean(monkeypatch): + monkeypatch.setitem(thresholds.STRONG_WIND, "use_mean_or_gust", "hourly_mean") + gust = _with_storm([20.0]) + mean = [2.0] * 24 + [a] = strong_wind.detect(_day(3, gust, mean=mean), [_field(3)], RUN_DAY) + assert a.status == NO_EVENT and a.details["max_wind_ms"] == 2.0 diff --git a/tests/test_threea.py b/tests/test_threea.py new file mode 100644 index 0000000..250ab12 --- /dev/null +++ b/tests/test_threea.py @@ -0,0 +1,38 @@ +from datetime import datetime + +from damage_engine.sources import threea + + +class FakeClient: + def __init__(self, answer): + self.answer, self.calls = answer, [] + + def call(self, method, path, body=None): + self.calls.append((method, path, body)) + return self.answer + + +def test_fetch_daily_asks_for_data_column_not_timestamp(monkeypatch): + answer = [ + {"CodiceStazione": 3, "Data": "2026-10-20 00:00:00.000", "Pioggia": "4.2", "EvapoPot": 1.1}, + {"CodiceStazione": 3, "Data": "2026-10-19 00:00:00.000", "Pioggia": 0, "EvapoPot": 1.3}, + {"CodiceStazione": 3, "Data": "2026-10-18 00:00:00.000", "Pioggia": None, "EvapoPot": None}, + ] + fake = FakeClient(answer) + monkeypatch.setattr(threea, "client", lambda: fake) + df = threea.fetch_daily([3], datetime(2026, 10, 18), datetime(2026, 10, 20), ["Pioggia", "EvapoPot"]) + method, path, body = fake.calls[0] + assert path == "/getWeatherDaily" + assert body["columns"] == ["CodiceStazione", "Data", "Pioggia", "EvapoPot"] + assert "Timestamp" not in body["columns"] + assert list(df["Data"].dt.day) == [19, 20] # empty row dropped, sorted by date + assert df["Pioggia"].tolist() == [0.0, 4.2] + + +def test_fetch_hourly_still_uses_timestamp(monkeypatch): + fake = FakeClient([{"CodiceStazione": 3, "Timestamp": "2026-10-20 06:00:00", "Pioggia": 1}]) + monkeypatch.setattr(threea, "client", lambda: fake) + df = threea.fetch_hourly([3], datetime(2026, 10, 20), datetime(2026, 10, 20, 6), ["Pioggia"]) + assert fake.calls[0][1] == "/getWeatherHourly" + assert fake.calls[0][2]["columns"][:2] == ["CodiceStazione", "Timestamp"] + assert len(df) == 1