f77c0abebd
- Enhanced `.env.example` to include `SQL_APP_DATABASE` for managing additional advisory-related tables. - Refactored `config.yaml` to remove the `crops` block, shifting to a database-driven approach for job discovery. - Updated `README.md` to clarify the new database structure and job discovery process, emphasizing the role of the `AI_agrosupport_an_pathogen_models` and `AI_agrosupport_agro_models` tables. - Modified SQL queries across multiple files to accommodate the new database schema, ensuring correct table references and improving overall data handling. - Adjusted `pipeline` logic to utilize the new configuration, enhancing the efficiency of the advisory generation process.
100 lines
5.1 KiB
YAML
100 lines
5.1 KiB
YAML
# ── LLM configuration ─────────────────────────────────────────────────────────
|
|
# Two independent profiles, one per pipeline stage. Each accepts the same keys:
|
|
# provider: openai | anthropic | gemini
|
|
# openai_model / anthropic_model / gemini_model / max_tokens / temperature
|
|
# thinking_level (optional, Gemini 3 only): low | medium | high.
|
|
# Gemini counts thinking tokens inside max_tokens, so max_tokens is the
|
|
# combined thinking-plus-output cap. Omit thinking_level to use the model's default.
|
|
llm:
|
|
# Part_One: vector-search query synthesis (small structured JSON in,
|
|
# 1-2 short queries out). A lighter/cheaper model is sufficient here.
|
|
query_synthesis:
|
|
provider: gemini
|
|
openai_model: gpt-4o-mini
|
|
anthropic_model: claude-haiku-4-5
|
|
gemini_model: gemini-3.8-flash
|
|
max_tokens: 2048
|
|
temperature: 0.0
|
|
|
|
# Part_Two: the farmer-facing advisory generator. Receives the full
|
|
# json_for_advice_generation payload (weather, phenology, model output,
|
|
# applied treatments, product labels) and returns the structured advice
|
|
# JSON stored in the advisories table. Needs a more capable model.
|
|
advice_generation:
|
|
provider: gemini
|
|
openai_model: gpt-4o
|
|
anthropic_model: claude-opus-4-5
|
|
gemini_model: gemini-3.1-pro-preview
|
|
max_tokens: 13312
|
|
thinking_level: medium
|
|
temperature: 0.0
|
|
|
|
# ── Worklist filters ───────────────────────────────────────────────────────────
|
|
# `python -m pipeline batch` discovers jobs from the database, not from this
|
|
# file. A disease can generate advice when
|
|
# AI_agrosupport_an_pathogen_models.pat_mod_AI_model = 1; its name is
|
|
# pat_mod_desc. A field is included when AI_agrosupport_agro_models.anmod_AI_advice
|
|
# = 1 and anmod_model matches that name. The cultivated crop comes from
|
|
# AI_agrosupport_cmp_layers / AI_agrosupport_an_colture and vocab/crops.yaml.
|
|
# The English disease name comes from vocab/diseases.yaml.
|
|
#
|
|
# Each crop-disease pair needs a matching prompt directory under
|
|
# prompts/advice/<crop>__<disease slug>/ (see the pipeline README for the slug
|
|
# rules); pairs without one fall back to prompts/advice/_default/ and are
|
|
# logged at startup.
|
|
worklist:
|
|
# Restrict the batch to specific field IDs while testing; empty = no filter.
|
|
field_allowlist: []
|
|
|
|
# ── Concurrency & rate limiting ────────────────────────────────────────────────
|
|
# One "gemini" budget covers both direct LLM calls (query synthesis, advice
|
|
# generation) and Weaviate near_text search, because the ProductProfile
|
|
# collection is vectorised with text2vec-palm (gemini-embedding-001) using the
|
|
# same GEMINI_API_KEY — they share one quota.
|
|
concurrency:
|
|
workers: 12
|
|
limits:
|
|
sql: 6
|
|
gemini: 12
|
|
gemini_requests_per_minute: 60
|
|
|
|
# ── Retry policy for transient LLM / Weaviate / SQL failures ──────────────────
|
|
retry:
|
|
attempts: 3
|
|
initial_backoff_seconds: 2
|
|
max_backoff_seconds: 30
|
|
|
|
# ── Daily SLA ───────────────────────────────────────────────────────────────────
|
|
# The batch must not still be running after this local time; jobs not yet
|
|
# started by then are skipped (status skipped_deadline) rather than risking a
|
|
# late advisory. Schedule the run itself no earlier than ~07:00 so weather is
|
|
# fresh (see README).
|
|
schedule:
|
|
deadline: "09:00"
|
|
|
|
# ── Single-field debug mode (`python -m pipeline one`) ────────────────────────
|
|
# Not used by `batch`; kept for ad-hoc single-field runs and debugging.
|
|
field_id: 4012
|
|
disease_name: "PERONOSPORA"
|
|
|
|
# ── Vocabulary paths ──────────────────────────────────────────────────────────
|
|
vocab:
|
|
crops: vocab/crops.yaml
|
|
diseases: vocab/diseases.yaml
|
|
|
|
# ── Observability (optional; off by default) ──────────────────────────────────
|
|
# Sends OpenTelemetry/OpenInference spans to the self-hosted Phoenix container
|
|
# in docker-compose.yml (http://localhost:6006). Never required: if this is
|
|
# off, or Phoenix is down/unreachable, `batch` and `one` run exactly as if
|
|
# this block did not exist (see pipeline/observability.py).
|
|
observability:
|
|
enabled: false
|
|
project_name: ai-agro-support
|
|
endpoint: http://localhost:6006/v1/traces
|
|
# Keep prompt/payload text out of Phoenix spans. json_for_advice_generation
|
|
# carries the full product-label payload and is already persisted under
|
|
# output/<date>/*.json; spans only need model/token/status metadata.
|
|
hide_prompts: true
|
|
# Hard cap on every exported string attribute (OTel span attribute limit).
|
|
max_attribute_chars: 4096
|