1

ontology_platform/crawler_platform/app/api/routes.py

@@ -44,6 +44,100 @@ from crawler_platform.app.core.research.memory_store import ResearchMemoryStore,
 
 DOMAIN_NAME_PATTERN = re.compile(r"^[a-z0-9][a-z0-9_-]{1,79}$")
 SITE_CRAWL_CANCEL_REQUESTS: set[int] = set()
+COMPARISON_KEYS = ("both_agree", "rule_only", "llm_only", "conflict", "rejected_by_validation")
+
+
+def extraction_log_summary(raw_output: dict[str, Any]) -> dict[str, Any]:
+    candidate_claims = raw_output.get("candidate_claims")
+    if not isinstance(candidate_claims, list):
+        candidate_claims = []
+
+    raw_comparison = raw_output.get("comparison")
+    if not isinstance(raw_comparison, dict):
+        raw_comparison = {}
+    comparison = {key: int(raw_comparison.get(key) or 0) for key in COMPARISON_KEYS}
+    if not any(comparison.values()):
+        comparison.update(comparison_from_candidate_claims(candidate_claims))
+    validation = raw_output.get("validation")
+    if isinstance(validation, dict) and comparison["rejected_by_validation"] == 0:
+        comparison["rejected_by_validation"] = number_or_default(validation.get("rejected_claim_count"), 0)
+
+    return {
+        "candidate_count": len(candidate_claims),
+        "comparison": comparison,
+        "rule_entity_count": number_or_none(raw_output.get("rule_entity_count")),
+        "rule_claim_count": number_or_derived(
+            raw_output.get("rule_claim_count"),
+            candidate_claims,
+            source="rule",
+        ),
+        "llm_entity_count": number_or_none(raw_output.get("llm_entity_count")),
+        "llm_claim_count": number_or_derived(
+            raw_output.get("llm_claim_count"),
+            candidate_claims,
+            source="llm",
+        ),
+        "agreement_claim_count": number_or_default(raw_output.get("agreement_claim_count"), comparison["both_agree"]),
+        "rule_only_claim_count": number_or_default(raw_output.get("rule_only_claim_count"), comparison["rule_only"]),
+        "llm_only_claim_count": number_or_default(raw_output.get("llm_only_claim_count"), comparison["llm_only"]),
+        "conflict_claim_count": number_or_default(raw_output.get("conflict_claim_count"), comparison["conflict"]),
+    }
+
+
+def comparison_from_candidate_claims(candidate_claims: list[Any]) -> dict[str, int]:
+    comparison = {key: 0 for key in COMPARISON_KEYS}
+    for claim in candidate_claims:
+        metadata = claim_metadata(claim)
+        agreement = str(metadata.get("agreement") or metadata.get("claim_kind") or "").lower()
+        if agreement == "rule_and_llm":
+            comparison["both_agree"] += 1
+        elif agreement == "rule_only":
+            comparison["rule_only"] += 1
+        elif agreement == "llm_only":
+            comparison["llm_only"] += 1
+        elif agreement == "conflict":
+            comparison["conflict"] += 1
+    return comparison
+
+
+def claim_metadata(claim: Any) -> dict[str, Any]:
+    if not isinstance(claim, dict):
+        return {}
+    metadata = claim.get("metadata")
+    return metadata if isinstance(metadata, dict) else {}
+
+
+def number_or_none(value: Any) -> int | None:
+    if isinstance(value, bool):
+        return None
+    if isinstance(value, (int, float)):
+        return int(value)
+    return None
+
+
+def number_or_default(value: Any, default: int) -> int:
+    parsed = number_or_none(value)
+    return default if parsed is None else parsed
+
+
+def number_or_derived(value: Any, candidate_claims: list[Any], *, source: str) -> int:
+    parsed = number_or_none(value)
+    if parsed is not None:
+        return parsed
+    return sum(1 for claim in candidate_claims if claim_matches_source(claim, source))
+
+
+def claim_matches_source(claim: Any, source: str) -> bool:
+    metadata = claim_metadata(claim)
+    extraction_source = str(metadata.get("extraction_source") or "").lower()
+    agreement = str(metadata.get("agreement") or "").lower()
+    if extraction_source == source:
+        return True
+    if source == "rule":
+        return agreement in {"rule_only", "rule_and_llm"}
+    if source == "llm":
+        return agreement in {"llm_only", "rule_and_llm"}
+    return False
 
 
 class CrawlRequest(BaseModel):
@@ -2490,35 +2584,37 @@ def register_routes(app, database_url: str) -> None:
                 .order_by(models.ExtractionLog.created_at.desc())
                 .limit(limit)
             ).all()
-            return [
-                {
+            payload = []
+            for log, page in rows:
+                raw_output = log.raw_output or {}
+                summary = extraction_log_summary(raw_output)
+                payload.append({
                     "id": log.id,
                     "page_url": page.url if page else None,
                     "extractor_name": log.extractor_name,
                     "provider": log.provider,
                     "error": log.error,
                     "created_at": log.created_at.isoformat(),
-                    "validation": (log.raw_output or {}).get("validation"),
-                    "page_context": (log.raw_output or {}).get("page_context"),
-                    "candidate_count": len((log.raw_output or {}).get("candidate_claims") or []),
-                    "extraction_mode": (log.raw_output or {}).get("extraction_mode"),
-                    "effective_extraction_mode": (log.raw_output or {}).get("effective_extraction_mode"),
-                    "comparison": (log.raw_output or {}).get("comparison"),
-                    "rule_entity_count": (log.raw_output or {}).get("rule_entity_count"),
-                    "rule_claim_count": (log.raw_output or {}).get("rule_claim_count"),
-                    "llm_entity_count": (log.raw_output or {}).get("llm_entity_count"),
-                    "llm_claim_count": (log.raw_output or {}).get("llm_claim_count"),
-                    "agreement_claim_count": (log.raw_output or {}).get("agreement_claim_count"),
-                    "rule_only_claim_count": (log.raw_output or {}).get("rule_only_claim_count"),
-                    "llm_only_claim_count": (log.raw_output or {}).get("llm_only_claim_count"),
-                    "conflict_claim_count": (log.raw_output or {}).get("conflict_claim_count"),
-                    "llm_skipped": (log.raw_output or {}).get("llm_skipped"),
-                    "llm_skip_reason": (log.raw_output or {}).get("llm_skip_reason"),
-                    "fallback": (log.raw_output or {}).get("fallback"),
+                    "validation": raw_output.get("validation"),
+                    "page_context": raw_output.get("page_context"),
+                    "candidate_count": summary["candidate_count"],
+                    "extraction_mode": raw_output.get("extraction_mode"),
+                    "effective_extraction_mode": raw_output.get("effective_extraction_mode"),
+                    "comparison": summary["comparison"],
+                    "rule_entity_count": summary["rule_entity_count"],
+                    "rule_claim_count": summary["rule_claim_count"],
+                    "llm_entity_count": summary["llm_entity_count"],
+                    "llm_claim_count": summary["llm_claim_count"],
+                    "agreement_claim_count": summary["agreement_claim_count"],
+                    "rule_only_claim_count": summary["rule_only_claim_count"],
+                    "llm_only_claim_count": summary["llm_only_claim_count"],
+                    "conflict_claim_count": summary["conflict_claim_count"],
+                    "llm_skipped": raw_output.get("llm_skipped"),
+                    "llm_skip_reason": raw_output.get("llm_skip_reason"),
+                    "fallback": raw_output.get("fallback"),
                     "raw_output": log.raw_output,
-                }
-                for log, page in rows
-            ]
+                })
+            return payload
 
     @app.patch("/claims/{claim_id}/confidence")
     def update_claim_confidence(claim_id: int, request: UpdateClaimConfidenceRequest):

ontology_platform/crawler_platform/app/core/crawler/page_classifier.py

@@ -3,6 +3,14 @@ from __future__ import annotations
 from urllib.parse import urlparse
 
 from crawler_platform.app.core.crawler.content_zone import is_claim_allowed_zone
+from crawler_platform.app.core.crawler.page_type_taxonomy import (
+    PageClassificationResult,
+    build_classification_result_from_legacy,
+    get_legacy_page_type,
+    normalize_page_type,
+)
+from crawler_platform.app.core.crawler.page_signal_extractor import extract_page_signals_from_page
+from crawler_platform.app.core.crawler.page_type_scorer import score_page_type
 
 
 PRODUCT_DETAIL_PREDICATES = {
@@ -95,6 +103,38 @@ def classify_page(
     return "UnknownPage"
 
 
+def classify_page_semantic(
+    url: str,
+    title: str | None = None,
+    text: str = "",
+    html: str | None = None,
+    source_zones: list[dict[str, object]] | None = None,
+) -> PageClassificationResult:
+    signals = extract_page_signals_from_page(
+        url=url,
+        title=title,
+        text=text,
+        html=html,
+        source_zones=source_zones,
+    )
+    result = score_page_type(url, signals)
+    if result.primary_page_type == "UnknownPage" and not result.alternatives:
+        legacy_page_type = classify_page(
+            url=url,
+            title=title,
+            text=text,
+            html=html,
+            source_zones=source_zones,
+        )
+        return build_classification_result_from_legacy(
+            url=url,
+            legacy_page_type=legacy_page_type,
+            confidence=0.35,
+            source="legacy_classifier_fallback",
+        )
+    return result
+
+
 def relation_allowed_for_page_type(page_type: str | None, predicate: str) -> bool:
     if page_type in PRODUCT_DETAIL_PAGE_TYPES:
         return True
@@ -109,26 +149,9 @@ def claim_allowed_for_context(page_type: str | None, predicate: str, zone_type:
     return relation_allowed_for_page_type(page_type, predicate) and is_claim_allowed_zone(zone_type)
 
 
-def normalize_page_type(value: str | None) -> str:
-    aliases = {
-        "product": "ProductPage",
-        "brand": "BrandStoryPage",
-        "review": "ReviewPage",
-        "listing": "CategoryPage",
-        "category": "CategoryPage",
-        "community": "BoardPage",
-        "board": "BoardPage",
-        "communitypage": "BoardPage",
-        "listingpage": "CategoryPage",
-        "promotionpage": "PromotionPage",
-    }
-    clean = str(value or "").strip()
-    return aliases.get(clean.lower(), aliases.get(clean, clean or "UnknownPage"))
-
-
-def should_analyze_page(page_type: str, analyze_page_types: set[str]) -> bool:
+def should_analyze_page(page_type: object, analyze_page_types: set[str] | None) -> bool:
     normalized_page_type = normalize_page_type(page_type)
-    normalized = {normalize_page_type(item) for item in analyze_page_types}
+    normalized = {normalize_page_type(item) for item in (analyze_page_types or set())}
     return normalized_page_type in normalized
 
 

ontology_platform/crawler_platform/app/core/extractor/hybrid.py

@@ -82,6 +82,13 @@ class HybridExtractor(Extractor):
                 "effective_extraction_mode": "rule_only",
                 "llm_skipped": True,
                 "llm_skip_reason": "rule_only mode",
+                **count_payload(
+                    rule_entity_count=len(rule_bundle.entities),
+                    rule_claim_count=len(rule_bundle.claims),
+                    llm_entity_count=0,
+                    llm_claim_count=0,
+                    comparison=comparison_payload(rule_only=len(rule_bundle.claims)),
+                ),
             }
             return rule_bundle
 
@@ -103,8 +110,13 @@ class HybridExtractor(Extractor):
                 **llm_bundle.raw_output,
                 "extraction_mode": self.mode,
                 "effective_extraction_mode": "llm_only",
-                "rule_entity_count": len(rule_bundle.entities),
-                "rule_claim_count": len(rule_bundle.claims),
+                **count_payload(
+                    rule_entity_count=len(rule_bundle.entities),
+                    rule_claim_count=len(rule_bundle.claims),
+                    llm_entity_count=len(llm_bundle.entities),
+                    llm_claim_count=len(llm_bundle.claims),
+                    comparison=comparison_payload(llm_only=len(llm_bundle.claims)),
+                ),
             }
             return llm_bundle
 
@@ -202,6 +214,13 @@ def fallback_bundle(rule_bundle: ExtractionBundle, extractor: HybridExtractor, e
         "ai_model": extractor.model,
         "ai_warning": reason,
         "fallback": "rule_based",
+        **count_payload(
+            rule_entity_count=len(bundle.entities),
+            rule_claim_count=len(bundle.claims),
+            llm_entity_count=0,
+            llm_claim_count=0,
+            comparison=comparison_payload(rule_only=len(bundle.claims)),
+        ),
     }
     for entity in bundle.entities:
         entity.metadata["ai_fallback_reason"] = reason
@@ -265,6 +284,44 @@ def llm_skipped_bundle(
     return bundle
 
 
+def comparison_payload(
+    *,
+    both_agree: int = 0,
+    rule_only: int = 0,
+    llm_only: int = 0,
+    conflict: int = 0,
+    rejected_by_validation: int = 0,
+) -> dict[str, int]:
+    return {
+        "both_agree": both_agree,
+        "rule_only": rule_only,
+        "llm_only": llm_only,
+        "conflict": conflict,
+        "rejected_by_validation": rejected_by_validation,
+    }
+
+
+def count_payload(
+    *,
+    rule_entity_count: int,
+    rule_claim_count: int,
+    llm_entity_count: int,
+    llm_claim_count: int,
+    comparison: dict[str, int],
+) -> dict[str, Any]:
+    return {
+        "rule_entity_count": rule_entity_count,
+        "rule_claim_count": rule_claim_count,
+        "llm_entity_count": llm_entity_count,
+        "llm_claim_count": llm_claim_count,
+        "agreement_claim_count": comparison["both_agree"],
+        "rule_only_claim_count": comparison["rule_only"],
+        "llm_only_claim_count": comparison["llm_only"],
+        "conflict_claim_count": comparison["conflict"],
+        "comparison": comparison,
+    }
+
+
 def llm_skip_reason(
     context: ExtractionPageContext | None,
     page_text: str,