"""Database-driven job discovery: find every (field, disease) pair to run today. The batch does not start from a field ID or from a crop list in config.yaml. It reads the diseases that can generate advice (`AI_agrosupport_an_pathogen_models.pat_mod_AI_model = 1`, named by `pat_mod_desc`) and the fields that have advice enabled for one of those diseases (`AI_agrosupport_agro_models.anmod_AI_advice = 1`). The cultivated crop comes from the field layer. Each matching agro-model row becomes one `Job`. """ from __future__ import annotations import logging from collections import Counter from dataclasses import dataclass import pyodbc from pipeline.config import WorklistSettings from pipeline.db import fetch_all from pipeline.errors import PipelineError from pipeline.vocab import Vocabulary logger = logging.getLogger(__name__) @dataclass(frozen=True) class Job: """One field/disease pair to run the pipeline for.""" field_id: int crop_english: str crop_italian: str model_name: str # e.g. "PERONOSPORA"; matched against anmod_model disease_english: str # canonical English disease name organic: bool station: int | None anmod_id: int _JOBS_SQL = """ SELECT DISTINCT l.cmplay_id, l.cmplay_colture, l.cmplay_imp, l.cmplay_station, c.colture_name, m.anmod_id, m.anmod_model FROM AI_agrosupport_agro_models m INNER JOIN ( SELECT DISTINCT LTRIM(RTRIM(pat_mod_desc)) AS model_name FROM AI_agrosupport_an_pathogen_models WHERE pat_mod_AI_model = 1 AND pat_mod_desc IS NOT NULL AND LTRIM(RTRIM(pat_mod_desc)) <> '' ) p ON UPPER(LTRIM(RTRIM(m.anmod_model))) = UPPER(p.model_name) INNER JOIN AI_agrosupport_cmp_layers l ON l.cmplay_id = m.anmod_cmplay LEFT JOIN AI_agrosupport_an_colture c ON c.colture_id = l.cmplay_colture WHERE m.anmod_AI_advice = 1 """ def build_jobs( conn: pyodbc.Connection, worklist: WorklistSettings, crop_vocab: Vocabulary, disease_vocab: Vocabulary, ) -> list[Job]: """Discover every (field, disease) job enabled in the database.""" allowlist = set(worklist.field_allowlist) rows = fetch_all(conn, _JOBS_SQL) jobs: list[Job] = [] unmapped_diseases: list[str] = [] unmapped_colture: list[str] = [] for row in rows: if allowlist and int(row.cmplay_id) not in allowlist: continue model_name = (row.anmod_model or "").strip() try: disease_english = disease_vocab.to_english(model_name, kind="disease") except PipelineError: unmapped_diseases.append(model_name or "(blank)") continue colture_name = (row.colture_name or "").strip() try: crop_english = crop_vocab.to_english(colture_name, kind="crop") except PipelineError: unmapped_colture.append(colture_name or "(blank)") continue jobs.append( Job( field_id=int(row.cmplay_id), crop_english=crop_english, crop_italian=colture_name, model_name=model_name, disease_english=disease_english, organic=row.cmplay_imp == 3, station=( int(row.cmplay_station) if row.cmplay_station is not None else None ), anmod_id=int(row.anmod_id), ) ) jobs.sort(key=lambda job: (job.field_id, job.model_name, job.anmod_id)) if unmapped_diseases: logger.warning( "%d disease model name(s) have no disease vocabulary mapping and will be " "ignored by the worklist: %s", len(set(unmapped_diseases)), sorted(set(unmapped_diseases)), ) if unmapped_colture: logger.warning( "%d colture name(s) have no crop vocabulary mapping and will be " "ignored by the worklist: %s", len(set(unmapped_colture)), sorted(set(unmapped_colture)), ) counts = Counter((job.crop_english, job.disease_english, job.model_name) for job in jobs) for crop, disease, model_name in sorted(counts): logger.info( "Worklist: crop=%s disease=%s (model=%s) -> %d job(s)", crop, disease, model_name, counts[(crop, disease, model_name)], ) return jobs