Enhance README and setup_weaviate.py to clarify tokenization settings for the ProductProfile collection. Added warnings for tokenization drift and updated usage instructions for collection recreation after embedding or tokenization changes.

README.md

@@ -50,7 +50,9 @@ The pipeline is **idempotent**: a hash manifest (`.processed_manifest.json`) tra
    python setup_weaviate.py
    ```
 
-   The `ProductProfile` collection is configured to embed `situation_fit_summary` with `gemini-embedding-001` and the `RETRIEVAL_DOCUMENT` task type. If you change embedding settings on an existing database, recreate the collection:
+   The `ProductProfile` collection is configured to embed `situation_fit_summary` with `gemini-embedding-001` and the `RETRIEVAL_DOCUMENT` task type. `product_id` uses `tokenization: field` so filters such as `contains_any` match whole IDs (underscores included) rather than ID fragments.
+
+   Tokenization and embedding settings cannot be changed in place. After changing them, recreate the collection **and** re-ingest with `--force`. Recreating without `--force` leaves Weaviate empty because the hash manifest would skip unchanged files:
 
    ```bash
    python setup_weaviate.py --recreate

setup_weaviate.py

@@ -7,7 +7,7 @@ Uses Google AI Studio (Gemini API) with:
 
 Usage:
     python setup_weaviate.py              # create collection if it does not exist
-    python setup_weaviate.py --recreate    # delete and recreate (required after embedding config changes)
+    python setup_weaviate.py --recreate    # delete and recreate (required after embedding or tokenization changes)
 """
 
 from __future__ import annotations
@@ -30,7 +30,7 @@ except ModuleNotFoundError:
     sys.exit(1)
 
 from dotenv import load_dotenv
-from weaviate.classes.config import Configure, DataType, Property
+from weaviate.classes.config import Configure, DataType, Property, Tokenization
 
 COLLECTION_NAME = "ProductProfile"
 EMBEDDING_MODEL = "gemini-embedding-001"
@@ -62,6 +62,37 @@ def connect_weaviate(cfg: Dict[str, str]) -> weaviate.WeaviateClient:
     )
 
 
+def _tokenization_name(value: object) -> str:
+    """Normalize a Weaviate tokenization enum or string to a lowercase name."""
+    raw = getattr(value, "value", value)
+    return str(raw).split(".")[-1].lower()
+
+
+def _warn_if_product_id_tokenization_drift(client: weaviate.WeaviateClient) -> None:
+    """Warn when an existing collection still tokenizes product_id as words."""
+    collection = client.collections.get(COLLECTION_NAME)
+    config = collection.config.get()
+    actual = None
+    for prop in config.properties:
+        if prop.name == "product_id":
+            actual = getattr(prop, "tokenization", None)
+            break
+
+    if actual is not None and _tokenization_name(actual) == _tokenization_name(Tokenization.FIELD):
+        return
+
+    actual_label = _tokenization_name(actual) if actual is not None else "missing"
+    print(
+        f"[WARN] Collection '{COLLECTION_NAME}' product_id tokenization is '{actual_label}'; "
+        f"expected '{_tokenization_name(Tokenization.FIELD)}'. "
+        "Word tokenization splits underscored IDs, so contains_any filters can match unrelated products. "
+        "Tokenization cannot be changed in place. Recreate and re-ingest:\n"
+        "  python setup_weaviate.py --recreate\n"
+        "  python ingest.py --force",
+        file=sys.stderr,
+    )
+
+
 def ensure_product_profile_collection(
     client: weaviate.WeaviateClient,
     *,
@@ -76,6 +107,7 @@ def ensure_product_profile_collection(
         if recreate:
             client.collections.delete(COLLECTION_NAME)
         else:
+            _warn_if_product_id_tokenization_drift(client)
             return False
 
     client.collections.create(
@@ -86,6 +118,7 @@ def ensure_product_profile_collection(
                 data_type=DataType.TEXT,
                 skip_vectorization=True,
                 index_filterable=True,
+                tokenization=Tokenization.FIELD,
             ),
             Property(
                 name="product_name",
@@ -115,7 +148,7 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
     parser.add_argument(
         "--recreate",
         action="store_true",
-        help="Delete and recreate the collection (use after changing embedding settings).",
+        help="Delete and recreate the collection (use after changing embedding or tokenization settings).",
     )
     return parser.parse_args(argv)