diff --git a/README.md b/README.md index 0cccebb..933bb0c 100644 --- a/README.md +++ b/README.md @@ -50,7 +50,9 @@ The pipeline is **idempotent**: a hash manifest (`.processed_manifest.json`) tra python setup_weaviate.py ``` - The `ProductProfile` collection is configured to embed `situation_fit_summary` with `gemini-embedding-001` and the `RETRIEVAL_DOCUMENT` task type. If you change embedding settings on an existing database, recreate the collection: + The `ProductProfile` collection is configured to embed `situation_fit_summary` with `gemini-embedding-001` and the `RETRIEVAL_DOCUMENT` task type. `product_id` uses `tokenization: field` so filters such as `contains_any` match whole IDs (underscores included) rather than ID fragments. + + Tokenization and embedding settings cannot be changed in place. After changing them, recreate the collection **and** re-ingest with `--force`. Recreating without `--force` leaves Weaviate empty because the hash manifest would skip unchanged files: ```bash python setup_weaviate.py --recreate diff --git a/setup_weaviate.py b/setup_weaviate.py index 2be556e..155d5e5 100644 --- a/setup_weaviate.py +++ b/setup_weaviate.py @@ -7,7 +7,7 @@ Uses Google AI Studio (Gemini API) with: Usage: python setup_weaviate.py # create collection if it does not exist - python setup_weaviate.py --recreate # delete and recreate (required after embedding config changes) + python setup_weaviate.py --recreate # delete and recreate (required after embedding or tokenization changes) """ from __future__ import annotations @@ -30,7 +30,7 @@ except ModuleNotFoundError: sys.exit(1) from dotenv import load_dotenv -from weaviate.classes.config import Configure, DataType, Property +from weaviate.classes.config import Configure, DataType, Property, Tokenization COLLECTION_NAME = "ProductProfile" EMBEDDING_MODEL = "gemini-embedding-001" @@ -62,6 +62,37 @@ def connect_weaviate(cfg: Dict[str, str]) -> weaviate.WeaviateClient: ) +def _tokenization_name(value: object) -> str: + """Normalize a Weaviate tokenization enum or string to a lowercase name.""" + raw = getattr(value, "value", value) + return str(raw).split(".")[-1].lower() + + +def _warn_if_product_id_tokenization_drift(client: weaviate.WeaviateClient) -> None: + """Warn when an existing collection still tokenizes product_id as words.""" + collection = client.collections.get(COLLECTION_NAME) + config = collection.config.get() + actual = None + for prop in config.properties: + if prop.name == "product_id": + actual = getattr(prop, "tokenization", None) + break + + if actual is not None and _tokenization_name(actual) == _tokenization_name(Tokenization.FIELD): + return + + actual_label = _tokenization_name(actual) if actual is not None else "missing" + print( + f"[WARN] Collection '{COLLECTION_NAME}' product_id tokenization is '{actual_label}'; " + f"expected '{_tokenization_name(Tokenization.FIELD)}'. " + "Word tokenization splits underscored IDs, so contains_any filters can match unrelated products. " + "Tokenization cannot be changed in place. Recreate and re-ingest:\n" + " python setup_weaviate.py --recreate\n" + " python ingest.py --force", + file=sys.stderr, + ) + + def ensure_product_profile_collection( client: weaviate.WeaviateClient, *, @@ -76,6 +107,7 @@ def ensure_product_profile_collection( if recreate: client.collections.delete(COLLECTION_NAME) else: + _warn_if_product_id_tokenization_drift(client) return False client.collections.create( @@ -86,6 +118,7 @@ def ensure_product_profile_collection( data_type=DataType.TEXT, skip_vectorization=True, index_filterable=True, + tokenization=Tokenization.FIELD, ), Property( name="product_name", @@ -115,7 +148,7 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: parser.add_argument( "--recreate", action="store_true", - help="Delete and recreate the collection (use after changing embedding settings).", + help="Delete and recreate the collection (use after changing embedding or tokenization settings).", ) return parser.parse_args(argv)