diff --git a/README.md b/README.md index 933bb0c..f4507ba 100644 --- a/README.md +++ b/README.md @@ -117,7 +117,7 @@ Each `.jsonl` file contains one JSON object per line. See `data/chunks/delan_pro | JSON field | SQL column | Notes | |------------|------------|-------| -| `chunk_id` | `chunk_id` | Primary key | +| `chunk_id` | `label_chunk_id` | Primary key | | `product_id` | `product_id` | Foreign key to `products` | | `product_name` | `product_name` | | | `target_crops` | `target_crops` | Stored as JSON string via `json.dumps()` | diff --git a/ingest.py b/ingest.py index 67c6bf4..90821db 100644 --- a/ingest.py +++ b/ingest.py @@ -47,6 +47,7 @@ DEFAULT_CHUNKS_DIR = Path("data/chunks") # re-ingesting the same product always maps to the same object. WEAVIATE_UUID_NAMESPACE = uuid.UUID("1b7e2c3a-4f5d-4a8b-9c1e-6d2f3a8b9c0d") +# JSON keys for array-valued profile fields. ``frac_groups`` is stored in SQL as ``frac_codes``. ARRAY_FIELDS = ( "active_ingredients", "frac_groups", @@ -75,14 +76,15 @@ USE_FIELDS = ( "growth_stage_end", ) -CHUNK_INSERT_COLUMNS = ( - "chunk_id", - "product_id", - "product_name", - "target_crops", - "target_diseases", - "chunk_type", - "chunk_text", +# (SQL column, JSON key). JSON ``chunk_id`` is stored as ``label_chunk_id``. +CHUNK_COLUMN_SOURCES = ( + ("label_chunk_id", "chunk_id"), + ("product_id", "product_id"), + ("product_name", "product_name"), + ("target_crops", "target_crops"), + ("target_diseases", "target_diseases"), + ("chunk_type", "chunk_type"), + ("chunk_text", "chunk_text"), ) CHUNK_ARRAY_FIELDS = ("target_crops", "target_diseases") @@ -338,13 +340,13 @@ def upsert_product(cursor: pyodbc.Cursor, profile: Dict[str, Any]) -> None: SET product_name = ?, registration_number = ?, manufacturer = ?, - organic_certified = ?, + is_organic_certified = ?, systemicity = ?, - preventive_action = ?, - curative_action = ?, - eradicant_action = ?, + has_preventive_action = ?, + has_curative_action = ?, + has_eradicant_action = ?, active_ingredients = ?, - frac_groups = ?, + frac_codes = ?, target_crops = ?, target_diseases = ?, action_type = ? @@ -352,8 +354,8 @@ def upsert_product(cursor: pyodbc.Cursor, profile: Dict[str, Any]) -> None: ELSE INSERT INTO products ( product_id, product_name, registration_number, manufacturer, - organic_certified, systemicity, preventive_action, curative_action, - eradicant_action, active_ingredients, frac_groups, target_crops, + is_organic_certified, systemicity, has_preventive_action, has_curative_action, + has_eradicant_action, active_ingredients, frac_codes, target_crops, target_diseases, action_type ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) @@ -405,11 +407,11 @@ def replace_uses(cursor: pyodbc.Cursor, profile: Dict[str, Any]) -> None: def chunk_insert_values(chunk: Dict[str, Any]) -> tuple[Any, ...]: """Build parameterized insert values for a single label chunk row.""" values: list[Any] = [] - for column in CHUNK_INSERT_COLUMNS: - if column in CHUNK_ARRAY_FIELDS: - values.append(json.dumps(chunk.get(column, []))) + for _column, source_key in CHUNK_COLUMN_SOURCES: + if source_key in CHUNK_ARRAY_FIELDS: + values.append(json.dumps(chunk.get(source_key, []))) else: - values.append(chunk.get(column)) + values.append(chunk.get(source_key)) return tuple(values) @@ -421,11 +423,12 @@ def replace_label_chunks( """Replace all label-chunk rows for a product with the provided set.""" cursor.execute("DELETE FROM label_chunks WHERE product_id = ?", product_id) + insert_columns = [column for column, _source_key in CHUNK_COLUMN_SOURCES] insert_sql = f""" INSERT INTO label_chunks ( - {", ".join(CHUNK_INSERT_COLUMNS)} + {", ".join(insert_columns)} ) - VALUES ({", ".join(["?"] * len(CHUNK_INSERT_COLUMNS))}) + VALUES ({", ".join(["?"] * len(insert_columns))}) """ for chunk in chunks: