1

crawler_platform/app/core/extractor/ai_provider.py

@@ -149,6 +149,7 @@ class LLMJsonExtractor(AIExtractor):
                     {"role": "user", "content": prompt},
                 ],
                 "temperature": 0,
+                "response_format": extraction_response_format(),
             },
             timeout=self.timeout_seconds,
         )
@@ -266,6 +267,55 @@ Page text:
 """.strip()
 
 
+def extraction_response_format() -> dict[str, Any]:
+    return {
+        "type": "json_schema",
+        "json_schema": {
+            "name": "ontology_extraction",
+            "schema": {
+                "type": "object",
+                "properties": {
+                    "entities": {
+                        "type": "array",
+                        "items": {
+                            "type": "object",
+                            "properties": {
+                                "entity_type": {"type": "string"},
+                                "type": {"type": "string"},
+                                "name": {"type": "string"},
+                                "attributes": {"type": "object"},
+                                "confidence": {"type": "number"},
+                                "evidence_text": {"type": "string"},
+                            },
+                            "required": ["name"],
+                        },
+                    },
+                    "claims": {
+                        "type": "array",
+                        "items": {
+                            "type": "object",
+                            "properties": {
+                                "subject_name": {"type": "string"},
+                                "subject_type": {"type": "string"},
+                                "predicate": {"type": "string"},
+                                "object_name": {"type": ["string", "null"]},
+                                "object_type": {"type": ["string", "null"]},
+                                "object_value": {},
+                                "evidence_text": {"type": ["string", "null"]},
+                                "evidence_summary": {"type": ["string", "null"]},
+                                "confidence": {"type": "number"},
+                                "confidence_reason": {"type": ["string", "null"]},
+                            },
+                            "required": ["subject_name", "subject_type", "predicate"],
+                        },
+                    },
+                },
+                "required": ["entities", "claims"],
+            },
+        },
+    }
+
+
 def parse_json_content(content: str, retry=None) -> dict[str, Any]:
     try:
         return json.loads(content)
@@ -346,7 +396,7 @@ def parse_entities(items: list[dict[str, Any]]) -> list[ExtractedEntity]:
     entities: list[ExtractedEntity] = []
     for item in items:
         name = item.get("name")
-        entity_type = item.get("entity_type") or item.get("type")
+        entity_type = normalize_entity_type(item.get("entity_type") or item.get("type"))
         if not name or not entity_type:
             continue
         entities.append(
@@ -366,23 +416,73 @@ def parse_claims(items: list[dict[str, Any]]) -> list[ExtractedClaim]:
     claims: list[ExtractedClaim] = []
     for item in items:
         subject_name = item.get("subject_name")
-        subject_type = item.get("subject_type")
+        subject_type = normalize_entity_type(item.get("subject_type"))
         predicate = item.get("predicate")
         if not subject_name or not subject_type or not predicate:
             continue
-        claims.append(
+        object_name = item.get("object_name")
-            ExtractedClaim(
+        object_type = normalize_entity_type(item.get("object_type")) if item.get("object_type") else None
-                subject_name=str(subject_name),
+        object_value = item.get("object_value")
-                subject_type=str(subject_type),
+        if not object_name and isinstance(object_value, dict) and len(object_value) == 1:
-                predicate=str(predicate),
+            object_name = next(iter(object_value.keys()))
-                object_name=item.get("object_name"),
+            object_value = None
-                object_type=item.get("object_type"),
+        if object_name and not object_type:
-                object_value=item.get("object_value"),
+            object_type = infer_object_type(str(predicate))
-                evidence_text=item.get("evidence_text"),
+
-                evidence_summary=item.get("evidence_summary"),
+        object_names = split_object_names(object_name) if object_name and object_type else [object_name]
-                confidence=float(item.get("confidence") or 0.55),
+        for resolved_object_name in object_names:
-                confidence_reason=item.get("confidence_reason") or "AI extractor output",
+            claims.append(
-                metadata={"ai_extracted": True},
+                ExtractedClaim(
+                    subject_name=str(subject_name),
+                    subject_type=str(subject_type),
+                    predicate=str(predicate),
+                    object_name=resolved_object_name,
+                    object_type=object_type if resolved_object_name else None,
+                    object_value=None if resolved_object_name else object_value,
+                    evidence_text=item.get("evidence_text"),
+                    evidence_summary=item.get("evidence_summary"),
+                    confidence=float(item.get("confidence") or 0.55),
+                    confidence_reason=item.get("confidence_reason") or "AI extractor output",
+                    metadata={"ai_extracted": True},
+                )
             )
-        )
     return claims
+
+
+def normalize_entity_type(value: Any) -> str | None:
+    if not value:
+        return None
+    aliases = {
+        "perfume": "Perfume",
+        "product": "Perfume",
+        "brand": "Brand",
+        "note": "Note",
+        "accord": "Accord",
+        "mood": "Mood",
+        "season": "Season",
+        "occasion": "Occasion",
+        "review": "Review",
+        "price": "Price",
+    }
+    text = str(value).strip()
+    return aliases.get(text.lower(), text)
+
+
+def infer_object_type(predicate: str) -> str | None:
+    return {
+        "hasBrand": "Brand",
+        "hasTopNote": "Note",
+        "hasMiddleNote": "Note",
+        "hasBaseNote": "Note",
+        "hasAccord": "Accord",
+        "evokesMood": "Mood",
+        "suitableForSeason": "Season",
+        "suitableForOccasion": "Occasion",
+        "hasReviewKeyword": "Review",
+    }.get(predicate)
+
+
+def split_object_names(value: Any) -> list[str]:
+    text = str(value)
+    parts = re.split(r"[,/|·ㆍ]+|\band\b| 및 | 그리고 ", text, flags=re.IGNORECASE)
+    return [part.strip() for part in parts if part.strip()]