f77c0abebd
- Enhanced `.env.example` to include `SQL_APP_DATABASE` for managing additional advisory-related tables. - Refactored `config.yaml` to remove the `crops` block, shifting to a database-driven approach for job discovery. - Updated `README.md` to clarify the new database structure and job discovery process, emphasizing the role of the `AI_agrosupport_an_pathogen_models` and `AI_agrosupport_agro_models` tables. - Modified SQL queries across multiple files to accommodate the new database schema, ensuring correct table references and improving overall data handling. - Adjusted `pipeline` logic to utilize the new configuration, enhancing the efficiency of the advisory generation process.
267 lines
8.5 KiB
Python
267 lines
8.5 KiB
Python
"""Load and merge config.yaml with environment variables."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
from dataclasses import dataclass
|
|
from pathlib import Path
|
|
|
|
import yaml
|
|
from dotenv import load_dotenv
|
|
|
|
ROOT = Path(__file__).resolve().parent.parent
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class LlmSettings:
|
|
provider: str
|
|
openai_model: str
|
|
anthropic_model: str
|
|
gemini_model: str
|
|
max_tokens: int
|
|
temperature: float
|
|
thinking_level: str | None
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class SqlSettings:
|
|
driver: str
|
|
server: str
|
|
database: str
|
|
app_database: str
|
|
username: str
|
|
password: str
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class WeaviateSettings:
|
|
host: str
|
|
http_port: int
|
|
grpc_port: int
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class WorklistSettings:
|
|
field_allowlist: tuple[int, ...]
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class ConcurrencySettings:
|
|
workers: int
|
|
sql_limit: int
|
|
gemini_limit: int
|
|
gemini_requests_per_minute: int
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class RetrySettings:
|
|
attempts: int
|
|
initial_backoff_seconds: float
|
|
max_backoff_seconds: float
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class ScheduleSettings:
|
|
deadline: str # "HH:MM", local time
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class ObservabilitySettings:
|
|
"""Opt-in OpenTelemetry/OpenInference tracing to a self-hosted Phoenix
|
|
instance (see docker-compose.yml). Disabled by default; see
|
|
pipeline/observability.py for the fail-open contract."""
|
|
|
|
enabled: bool
|
|
project_name: str
|
|
endpoint: str
|
|
hide_prompts: bool
|
|
max_attribute_chars: int
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class Settings:
|
|
root: Path
|
|
field_id: int | None
|
|
disease_name: str | None
|
|
worklist: WorklistSettings
|
|
concurrency: ConcurrencySettings
|
|
retry: RetrySettings
|
|
schedule: ScheduleSettings
|
|
observability: ObservabilitySettings
|
|
crops_vocab: Path
|
|
diseases_vocab: Path
|
|
llm_query_synthesis: LlmSettings
|
|
llm_advice_generation: LlmSettings
|
|
sql: SqlSettings
|
|
weaviate: WeaviateSettings
|
|
gemini_api_key: str
|
|
openai_api_key: str
|
|
anthropic_api_key: str
|
|
prompts_dir: Path
|
|
output_dir: Path
|
|
|
|
|
|
_THINKING_LEVELS = frozenset({"low", "medium", "high"})
|
|
|
|
|
|
def _parse_llm_settings(raw: dict, defaults: dict) -> LlmSettings:
|
|
"""Build an LlmSettings profile from a config sub-block, falling back to defaults."""
|
|
thinking_raw = raw["thinking_level"] if "thinking_level" in raw else defaults.get("thinking_level")
|
|
if thinking_raw is None or str(thinking_raw).strip() == "":
|
|
thinking_level = None
|
|
else:
|
|
thinking_level = str(thinking_raw).strip().lower()
|
|
if thinking_level not in _THINKING_LEVELS:
|
|
allowed = ", ".join(sorted(_THINKING_LEVELS))
|
|
raise ValueError(f"thinking_level must be one of {allowed}, got {thinking_raw!r}")
|
|
return LlmSettings(
|
|
provider=str(raw.get("provider", defaults["provider"])).strip().lower(),
|
|
openai_model=str(raw.get("openai_model", defaults["openai_model"])),
|
|
anthropic_model=str(raw.get("anthropic_model", defaults["anthropic_model"])),
|
|
gemini_model=str(raw.get("gemini_model", defaults["gemini_model"])),
|
|
max_tokens=int(raw.get("max_tokens", defaults["max_tokens"])),
|
|
temperature=float(raw.get("temperature", defaults["temperature"])),
|
|
thinking_level=thinking_level,
|
|
)
|
|
|
|
|
|
def _parse_worklist(raw: dict) -> WorklistSettings:
|
|
return WorklistSettings(
|
|
field_allowlist=tuple(int(v) for v in raw.get("field_allowlist", []) or []),
|
|
)
|
|
|
|
|
|
def _parse_concurrency(raw: dict) -> ConcurrencySettings:
|
|
limits = raw.get("limits", {}) or {}
|
|
return ConcurrencySettings(
|
|
workers=int(raw.get("workers", 8)),
|
|
sql_limit=int(limits.get("sql", 6)),
|
|
gemini_limit=int(limits.get("gemini", 4)),
|
|
gemini_requests_per_minute=int(raw.get("gemini_requests_per_minute", 60)),
|
|
)
|
|
|
|
|
|
def _parse_retry(raw: dict) -> RetrySettings:
|
|
return RetrySettings(
|
|
attempts=int(raw.get("attempts", 3)),
|
|
initial_backoff_seconds=float(raw.get("initial_backoff_seconds", 2)),
|
|
max_backoff_seconds=float(raw.get("max_backoff_seconds", 30)),
|
|
)
|
|
|
|
|
|
def _parse_schedule(raw: dict) -> ScheduleSettings:
|
|
return ScheduleSettings(deadline=str(raw.get("deadline", "09:00")))
|
|
|
|
|
|
def _env_bool(name: str) -> bool | None:
|
|
"""Read a boolean env var override; None if unset (falls back to config.yaml)."""
|
|
value = os.environ.get(name)
|
|
if value is None:
|
|
return None
|
|
return value.strip().lower() in {"1", "true", "yes", "on"}
|
|
|
|
|
|
def _parse_observability(raw: dict) -> ObservabilitySettings:
|
|
"""
|
|
Build the observability block, letting env vars override config.yaml so a
|
|
deployment can flip tracing on/off or point at a different Phoenix
|
|
instance without editing the checked-in file.
|
|
"""
|
|
raw = raw or {}
|
|
enabled = raw.get("enabled", False)
|
|
env_enabled = _env_bool("PHOENIX_TRACING_ENABLED")
|
|
if env_enabled is not None:
|
|
enabled = env_enabled
|
|
|
|
return ObservabilitySettings(
|
|
enabled=bool(enabled),
|
|
project_name=os.environ.get(
|
|
"PHOENIX_PROJECT_NAME", str(raw.get("project_name", "ai-agro-support"))
|
|
),
|
|
endpoint=os.environ.get(
|
|
"PHOENIX_COLLECTOR_ENDPOINT",
|
|
str(raw.get("endpoint", "http://localhost:6006/v1/traces")),
|
|
),
|
|
hide_prompts=bool(raw.get("hide_prompts", True)),
|
|
max_attribute_chars=int(raw.get("max_attribute_chars", 4096)),
|
|
)
|
|
|
|
|
|
def load_settings(
|
|
config_path: Path | None = None,
|
|
field_id_override: int | None = None,
|
|
disease_name_override: str | None = None,
|
|
) -> Settings:
|
|
"""Load YAML config and .env into a Settings object."""
|
|
load_dotenv(ROOT / ".env")
|
|
path = config_path or (ROOT / "config.yaml")
|
|
with path.open(encoding="utf-8") as fh:
|
|
raw = yaml.safe_load(fh)
|
|
|
|
llm_raw = raw.get("llm", {})
|
|
vocab = raw.get("vocab", {})
|
|
|
|
# field_id / disease_name only matter for `python -m pipeline one`. The
|
|
# batch worklist is discovered from the pathogen and agro-model tables.
|
|
field_id = field_id_override if field_id_override is not None else raw.get("field_id")
|
|
field_id = int(field_id) if field_id is not None else None
|
|
disease_name = disease_name_override or raw.get("disease_name")
|
|
disease_name = str(disease_name).strip() if disease_name is not None else None
|
|
|
|
query_synthesis_defaults = {
|
|
"provider": "gemini",
|
|
"openai_model": "gpt-4o-mini",
|
|
"anthropic_model": "claude-haiku-4-5",
|
|
"gemini_model": "gemini-3.8-flash",
|
|
"max_tokens": 8192,
|
|
"temperature": 0.0,
|
|
"thinking_level": None,
|
|
}
|
|
advice_generation_defaults = {
|
|
"provider": "gemini",
|
|
"openai_model": "gpt-4o",
|
|
"anthropic_model": "claude-opus-4-5",
|
|
"gemini_model": "gemini-3.1-pro-preview",
|
|
"max_tokens": 8192,
|
|
"temperature": 0.0,
|
|
"thinking_level": "medium",
|
|
}
|
|
|
|
return Settings(
|
|
root=ROOT,
|
|
field_id=field_id,
|
|
disease_name=disease_name,
|
|
worklist=_parse_worklist(raw.get("worklist", {})),
|
|
concurrency=_parse_concurrency(raw.get("concurrency", {})),
|
|
retry=_parse_retry(raw.get("retry", {})),
|
|
schedule=_parse_schedule(raw.get("schedule", {})),
|
|
observability=_parse_observability(raw.get("observability", {})),
|
|
crops_vocab=ROOT / vocab.get("crops", "vocab/crops.yaml"),
|
|
diseases_vocab=ROOT / vocab.get("diseases", "vocab/diseases.yaml"),
|
|
llm_query_synthesis=_parse_llm_settings(
|
|
llm_raw.get("query_synthesis", {}), query_synthesis_defaults
|
|
),
|
|
llm_advice_generation=_parse_llm_settings(
|
|
llm_raw.get("advice_generation", {}), advice_generation_defaults
|
|
),
|
|
sql=SqlSettings(
|
|
driver=os.environ.get("SQL_DRIVER", "ODBC Driver 17 for SQL Server"),
|
|
server=os.environ["SQL_SERVER"],
|
|
database=os.environ["SQL_DATABASE"],
|
|
app_database=os.environ.get("SQL_APP_DATABASE", "").strip()
|
|
or os.environ["SQL_DATABASE"],
|
|
username=os.environ["SQL_USERNAME"],
|
|
password=os.environ["SQL_PASSWORD"],
|
|
),
|
|
weaviate=WeaviateSettings(
|
|
host=os.environ.get("WEAVIATE_HOST", "localhost"),
|
|
http_port=int(os.environ.get("WEAVIATE_HTTP_PORT", "8080")),
|
|
grpc_port=int(os.environ.get("WEAVIATE_GRPC_PORT", "50051")),
|
|
),
|
|
gemini_api_key=os.environ.get("GEMINI_API_KEY", ""),
|
|
openai_api_key=os.environ.get("OPENAI_API_KEY", ""),
|
|
anthropic_api_key=os.environ.get("ANTHROPIC_API_KEY", ""),
|
|
prompts_dir=ROOT / "prompts",
|
|
output_dir=ROOT / "output",
|
|
)
|