e588d97e0f
- Updated .env.example and config.yaml to include observability settings. - Added a new Phoenix service in docker-compose.yml for self-hosted tracing. - Enhanced README.md with instructions on enabling and using observability features. - Implemented tracing in the pipeline, including job spans and LLM stage spans. - Introduced ObservabilitySettings class in config.py for better configuration management. - Updated job.py and resources.py to support tracing without affecting fault isolation. - Minor adjustments to other files for compatibility with the new observability features.
43 lines
1.2 KiB
YAML
43 lines
1.2 KiB
YAML
services:
|
|
weaviate:
|
|
command:
|
|
- --host
|
|
- 0.0.0.0
|
|
- --port
|
|
- "8080"
|
|
- --scheme
|
|
- http
|
|
image: cr.weaviate.io/semitechnologies/weaviate:1.32.0 # Required for text2vec-google taskType support
|
|
ports:
|
|
- 8080:8080
|
|
- 50051:50051
|
|
volumes:
|
|
- weaviate_data:/var/lib/weaviate
|
|
restart: on-failure:0
|
|
environment:
|
|
QUERY_DEFAULTS_LIMIT: 25
|
|
PERSISTENCE_DATA_PATH: "/var/lib/weaviate"
|
|
CLUSTER_HOSTNAME: "node1"
|
|
# Swap OpenAI to Google modules
|
|
ENABLE_MODULES: "text2vec-google"
|
|
DEFAULT_VECTORIZER_MODULE: "text2vec-google"
|
|
|
|
# Self-hosted tracing/eval UI (Arize Phoenix). Opt-in from the pipeline's
|
|
# side (config.yaml observability.enabled / PHOENIX_TRACING_ENABLED); this
|
|
# container can run continuously with no effect on `batch` when tracing is
|
|
# off. SQLite on a mounted volume is sufficient for a single-machine setup
|
|
# (see README "Observability"); do not add Postgres.
|
|
phoenix:
|
|
image: arizephoenix/phoenix:20.3.0
|
|
ports:
|
|
- 6006:6006 # UI + OTLP/HTTP collector (POST .../v1/traces)
|
|
environment:
|
|
PHOENIX_WORKING_DIR: /mnt/data
|
|
volumes:
|
|
- phoenix_data:/mnt/data
|
|
restart: unless-stopped
|
|
|
|
volumes:
|
|
weaviate_data:
|
|
phoenix_data:
|