ebc8ad26b6
- Updated the job structure to include `anmod_id`, allowing for multiple mathematical models to be processed for each field and disease combination. - Enhanced the batch processing logic to discover and execute jobs for `(field, disease, anmod_id)` instead of just `(field, disease)`. - Modified the output file naming convention to include `anmod_id`, ensuring unique output for each model. - Adjusted database interactions to accommodate the new primary key structure based on `pathogen_model_id`. - Updated relevant documentation in `README.md` to reflect changes in job discovery, execution, and output generation. - Enhanced unit tests to validate the new functionality and ensure correct behavior across various scenarios.
144 lines
4.4 KiB
Python
144 lines
4.4 KiB
Python
"""Database-driven job discovery: find every advice-enabled agro-model row to run today.
|
|
|
|
The batch does not start from a field ID or from a crop list in config.yaml.
|
|
It reads the diseases that can generate advice
|
|
(`AI_agrosupport_an_pathogen_models.pat_mod_AI_model = 1`, named by
|
|
`pat_mod_desc`) and the fields that have advice enabled for one of those
|
|
diseases (`AI_agrosupport_agro_models.anmod_AI_advice = 1`). The cultivated
|
|
crop comes from the field layer. Each matching agro-model row becomes one
|
|
`Job`, including several mathematical models (`anmod_id`) for the same field
|
|
and disease.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import logging
|
|
from collections import Counter
|
|
from dataclasses import dataclass
|
|
|
|
import pyodbc
|
|
|
|
from pipeline.config import WorklistSettings
|
|
from pipeline.db import fetch_all
|
|
from pipeline.errors import PipelineError
|
|
from pipeline.vocab import Vocabulary
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class Job:
|
|
"""One field, disease, and mathematical model (`anmod_id`) to run."""
|
|
|
|
field_id: int
|
|
crop_english: str
|
|
crop_italian: str
|
|
model_name: str # e.g. "PERONOSPORA"; matched against anmod_model
|
|
disease_english: str # canonical English disease name
|
|
organic: bool
|
|
station: int | None
|
|
anmod_id: int
|
|
|
|
|
|
_JOBS_SQL = """
|
|
SELECT DISTINCT
|
|
l.cmplay_id,
|
|
l.cmplay_colture,
|
|
l.cmplay_imp,
|
|
l.cmplay_station,
|
|
c.colture_name,
|
|
m.anmod_id,
|
|
m.anmod_model
|
|
FROM AI_agrosupport_agro_models m
|
|
INNER JOIN (
|
|
SELECT DISTINCT LTRIM(RTRIM(pat_mod_desc)) AS model_name
|
|
FROM AI_agrosupport_an_pathogen_models
|
|
WHERE pat_mod_AI_model = 1
|
|
AND pat_mod_desc IS NOT NULL
|
|
AND LTRIM(RTRIM(pat_mod_desc)) <> ''
|
|
) p
|
|
ON UPPER(LTRIM(RTRIM(m.anmod_model))) = UPPER(p.model_name)
|
|
INNER JOIN AI_agrosupport_cmp_layers l
|
|
ON l.cmplay_id = m.anmod_cmplay
|
|
LEFT JOIN AI_agrosupport_an_colture c
|
|
ON c.colture_id = l.cmplay_colture
|
|
WHERE m.anmod_AI_advice = 1
|
|
"""
|
|
|
|
|
|
def build_jobs(
|
|
conn: pyodbc.Connection,
|
|
worklist: WorklistSettings,
|
|
crop_vocab: Vocabulary,
|
|
disease_vocab: Vocabulary,
|
|
) -> list[Job]:
|
|
"""Discover every (field, disease) job enabled in the database."""
|
|
allowlist = set(worklist.field_allowlist)
|
|
rows = fetch_all(conn, _JOBS_SQL)
|
|
|
|
jobs: list[Job] = []
|
|
unmapped_diseases: list[str] = []
|
|
unmapped_colture: list[str] = []
|
|
|
|
for row in rows:
|
|
if allowlist and int(row.cmplay_id) not in allowlist:
|
|
continue
|
|
|
|
model_name = (row.anmod_model or "").strip()
|
|
try:
|
|
disease_english = disease_vocab.to_english(model_name, kind="disease")
|
|
except PipelineError:
|
|
unmapped_diseases.append(model_name or "(blank)")
|
|
continue
|
|
|
|
colture_name = (row.colture_name or "").strip()
|
|
try:
|
|
crop_english = crop_vocab.to_english(colture_name, kind="crop")
|
|
except PipelineError:
|
|
unmapped_colture.append(colture_name or "(blank)")
|
|
continue
|
|
|
|
jobs.append(
|
|
Job(
|
|
field_id=int(row.cmplay_id),
|
|
crop_english=crop_english,
|
|
crop_italian=colture_name,
|
|
model_name=model_name,
|
|
disease_english=disease_english,
|
|
organic=row.cmplay_imp == 3,
|
|
station=(
|
|
int(row.cmplay_station) if row.cmplay_station is not None else None
|
|
),
|
|
anmod_id=int(row.anmod_id),
|
|
)
|
|
)
|
|
|
|
jobs.sort(key=lambda job: (job.field_id, job.model_name, job.anmod_id))
|
|
|
|
if unmapped_diseases:
|
|
logger.warning(
|
|
"%d disease model name(s) have no disease vocabulary mapping and will be "
|
|
"ignored by the worklist: %s",
|
|
len(set(unmapped_diseases)),
|
|
sorted(set(unmapped_diseases)),
|
|
)
|
|
if unmapped_colture:
|
|
logger.warning(
|
|
"%d colture name(s) have no crop vocabulary mapping and will be "
|
|
"ignored by the worklist: %s",
|
|
len(set(unmapped_colture)),
|
|
sorted(set(unmapped_colture)),
|
|
)
|
|
|
|
counts = Counter((job.crop_english, job.disease_english, job.model_name) for job in jobs)
|
|
for crop, disease, model_name in sorted(counts):
|
|
logger.info(
|
|
"Worklist: crop=%s disease=%s (model=%s) -> %d job(s)",
|
|
crop,
|
|
disease,
|
|
model_name,
|
|
counts[(crop, disease, model_name)],
|
|
)
|
|
|
|
return jobs
|