참고소스 수정본

crawler_platform/app/core/ontology/definitions.py

@@ -43,7 +43,14 @@ DOMAIN_ONTOLOGIES: dict[str, Ontology] = {
         domain="perfume",
         entity_types=[
             "Perfume",
+            "Product",
             "Brand",
+            "Event",
+            "Article",
+            "Promotion",
+            "Category",
+            "Notice",
+            "Page",
             "Note",
             "Accord",
             "Mood",
@@ -52,6 +59,11 @@ DOMAIN_ONTOLOGIES: dict[str, Ontology] = {
             "Review",
             "Price",
             "ProductPage",
+            "CommunityPage",
+            "BrandStoryPage",
+            "ListingPage",
+            "PromotionPage",
+            "ReviewPage",
         ],
         predicates=[
             "hasBrand",
@@ -127,4 +139,3 @@ DOMAIN_ONTOLOGIES: dict[str, Ontology] = {
 
 def ontology_for_domain(domain: str) -> Ontology:
     return DOMAIN_ONTOLOGIES.get(domain, COMMON_ONTOLOGY)
-

crawler_platform/app/core/ontology/entity_normalizer.py

@@ -0,0 +1,76 @@
+from __future__ import annotations
+
+import re
+import unicodedata
+from typing import Any
+
+
+GENERIC_TYPE_ALIASES = {
+    "entity": "Entity",
+    "concept": "Concept",
+    "organization": "Organization",
+    "organisation": "Organization",
+    "org": "Organization",
+    "company": "Organization",
+    "person": "Person",
+    "people": "Person",
+    "place": "Place",
+    "location": "Place",
+    "event": "Event",
+    "document": "Document",
+    "article": "Document",
+    "paper": "Document",
+    "source": "Source",
+    "page": "Page",
+}
+
+
+def normalize_entity_type(value: Any, domain: str | None = None) -> str | None:
+    if not value:
+        return None
+    if domain:
+        from crawler_platform.app.adapters.registry import adapter_for_domain
+
+        adapter = adapter_for_domain(domain)
+        if adapter is not None:
+            adapter_type = adapter.normalize_entity_type(str(value))
+            if adapter_type:
+                return adapter_type
+    clean = compact_key(value)
+    return GENERIC_TYPE_ALIASES.get(clean, str(value).strip())
+
+
+def normalize_entity_name(value: Any, entity_type: str | None = None, domain: str | None = None) -> str:
+    text = normalize_text(value)
+    if domain:
+        from crawler_platform.app.adapters.registry import adapter_for_domain
+
+        adapter = adapter_for_domain(domain)
+        if adapter is not None:
+            adapter_name = adapter.normalize_entity_name(text, entity_type)
+            if adapter_name:
+                return adapter_name
+    return text
+
+
+def canonical_entity_key(value: Any, entity_type: str | None = None, domain: str | None = None) -> str:
+    normalized = normalize_entity_name(value, entity_type, domain)
+    return canonical_key(normalized)
+
+
+def normalize_text(value: Any) -> str:
+    text = unicodedata.normalize("NFKC", str(value or "")).strip()
+    text = re.sub(r"\s+", " ", text)
+    text = text.strip(" \t\r\n|/,:;")
+    return text
+
+
+def canonical_key(value: Any) -> str:
+    text = normalize_text(value).lower()
+    text = re.sub(r"[\[\]{}()<>\"'`]", "", text)
+    text = re.sub(r"\s+", " ", text)
+    return text.strip()
+
+
+def compact_key(value: Any) -> str:
+    return re.sub(r"[^0-9a-zA-Z]+", "", str(value or "")).strip().lower()

crawler_platform/app/core/ontology/gap_detector.py

@@ -0,0 +1,209 @@
+from __future__ import annotations
+
+from typing import Any
+
+from sqlalchemy import func, select
+from sqlalchemy.orm import Session
+
+from crawler_platform.app.core.database import models
+from crawler_platform.app.core.ontology.registry import stable_hash
+
+
+class KnowledgeGapDetector:
+    """Detects ontology coverage gaps that can drive future research loops."""
+
+    def __init__(self, session: Session):
+        self.session = session
+
+    def detect(self, project_id: int, limit: int = 100) -> list[models.KnowledgeGap]:
+        gaps: list[models.KnowledgeGap] = []
+        gaps.extend(self._entity_type_coverage_gaps(project_id))
+        gaps.extend(self._relation_type_usage_gaps(project_id))
+        gaps.extend(self._underconnected_entity_gaps(project_id))
+        gaps.extend(self._schema_proposal_gaps(project_id))
+        return sorted(gaps, key=lambda row: (-row.priority, row.updated_at), reverse=False)[:limit]
+
+    def list_open(self, project_id: int, limit: int = 100) -> list[dict[str, Any]]:
+        self.detect(project_id, limit=limit)
+        rows = self.session.scalars(
+            select(models.KnowledgeGap)
+            .where(models.KnowledgeGap.project_id == project_id, models.KnowledgeGap.status == "open")
+            .order_by(models.KnowledgeGap.priority.desc(), models.KnowledgeGap.updated_at.desc())
+            .limit(limit)
+        ).all()
+        return [gap_payload(row) for row in rows]
+
+    def _entity_type_coverage_gaps(self, project_id: int) -> list[models.KnowledgeGap]:
+        counts = dict(
+            self.session.execute(
+                select(models.Entity.entity_type, func.count(models.Entity.id))
+                .where(models.Entity.project_id == project_id)
+                .group_by(models.Entity.entity_type)
+            ).all()
+        )
+        rows = self.session.scalars(
+            select(models.OntologyEntityType).where(
+                models.OntologyEntityType.project_id == project_id,
+                models.OntologyEntityType.status == "active",
+                models.OntologyEntityType.domain != "core",
+            )
+        ).all()
+        gaps = []
+        for row in rows:
+            if int(counts.get(row.name, 0)) == 0:
+                gaps.append(
+                    self._upsert_gap(
+                        project_id,
+                        gap_type="entity_type_coverage",
+                        target_type="EntityType",
+                        target_name=row.name,
+                        description=f"Entity type '{row.name}' is registered but has no extracted entities.",
+                        priority=0.42,
+                        evidence={"entity_type_id": row.id, "count": 0},
+                    )
+                )
+        return gaps
+
+    def _relation_type_usage_gaps(self, project_id: int) -> list[models.KnowledgeGap]:
+        counts = dict(
+            self.session.execute(
+                select(models.OntologyTriple.predicate, func.count(models.OntologyTriple.id))
+                .where(models.OntologyTriple.project_id == project_id)
+                .group_by(models.OntologyTriple.predicate)
+            ).all()
+        )
+        rows = self.session.scalars(
+            select(models.OntologyRelationType).where(
+                models.OntologyRelationType.project_id == project_id,
+                models.OntologyRelationType.status == "active",
+                models.OntologyRelationType.domain != "core",
+            )
+        ).all()
+        gaps = []
+        for row in rows:
+            if int(counts.get(row.name, 0)) == 0:
+                gaps.append(
+                    self._upsert_gap(
+                        project_id,
+                        gap_type="relation_type_usage",
+                        target_type="RelationType",
+                        target_name=row.name,
+                        description=f"Relation type '{row.name}' is registered but has no graph triples.",
+                        priority=0.55,
+                        evidence={"relation_type_id": row.id, "count": 0},
+                    )
+                )
+        return gaps
+
+    def _underconnected_entity_gaps(self, project_id: int) -> list[models.KnowledgeGap]:
+        rows = self.session.scalars(
+            select(models.Entity)
+            .where(models.Entity.project_id == project_id, models.Entity.entity_type.notin_(["Page", "ProductPage"]))
+            .order_by(models.Entity.updated_at.desc())
+            .limit(50)
+        ).all()
+        gaps = []
+        for entity in rows:
+            count = self.session.scalar(
+                select(func.count(models.OntologyTriple.id)).where(
+                    models.OntologyTriple.project_id == project_id,
+                    (
+                        (models.OntologyTriple.subject_entity_id == entity.id)
+                        | (models.OntologyTriple.object_entity_id == entity.id)
+                    ),
+                    models.OntologyTriple.status.in_(["merged", "validated", "validated_literal"]),
+                )
+            )
+            if int(count or 0) == 0:
+                gaps.append(
+                    self._upsert_gap(
+                        project_id,
+                        gap_type="entity_connectivity",
+                        target_type=entity.entity_type,
+                        target_name=entity.name,
+                        description=f"Entity '{entity.name}' has no validated ontology triples.",
+                        priority=0.48,
+                        evidence={"entity_id": entity.id, "triple_count": 0},
+                    )
+                )
+        return gaps
+
+    def _schema_proposal_gaps(self, project_id: int) -> list[models.KnowledgeGap]:
+        rows = self.session.scalars(
+            select(models.OntologyProposal).where(
+                models.OntologyProposal.project_id == project_id,
+                models.OntologyProposal.status == "pending_review",
+            )
+        ).all()
+        return [
+            self._upsert_gap(
+                project_id,
+                gap_type="schema_governance",
+                target_type=row.proposal_type,
+                target_name=row.name,
+                description=f"Ontology schema proposal '{row.name}' is waiting for review.",
+                priority=0.86,
+                evidence={"proposal_id": row.id, "reason": row.reason},
+            )
+            for row in rows
+        ]
+
+    def _upsert_gap(
+        self,
+        project_id: int,
+        *,
+        gap_type: str,
+        target_type: str,
+        target_name: str,
+        description: str,
+        priority: float,
+        evidence: dict[str, Any],
+    ) -> models.KnowledgeGap:
+        gap_hash = stable_hash(
+            {
+                "project_id": project_id,
+                "gap_type": gap_type,
+                "target_type": target_type,
+                "target_name": target_name,
+            }
+        )
+        row = self.session.scalar(
+            select(models.KnowledgeGap).where(
+                models.KnowledgeGap.project_id == project_id,
+                models.KnowledgeGap.gap_hash == gap_hash,
+            )
+        )
+        if row is None:
+            row = models.KnowledgeGap(
+                project_id=project_id,
+                gap_type=gap_type,
+                target_type=target_type,
+                target_name=target_name,
+                description=description,
+                priority=min(max(priority, 0.0), 1.0),
+                gap_hash=gap_hash,
+                evidence=evidence,
+            )
+            self.session.add(row)
+            self.session.flush()
+            return row
+        row.description = description
+        row.priority = min(max(priority, 0.0), 1.0)
+        row.evidence = evidence
+        row.updated_at = models.utcnow()
+        return row
+
+
+def gap_payload(row: models.KnowledgeGap) -> dict[str, Any]:
+    return {
+        "id": row.id,
+        "gap_type": row.gap_type,
+        "target_type": row.target_type,
+        "target_name": row.target_name,
+        "description": row.description,
+        "priority": row.priority,
+        "status": row.status,
+        "evidence": row.evidence or {},
+        "metadata": row.metadata_json or {},
+        "updated_at": row.updated_at.isoformat() if row.updated_at else None,
+    }

crawler_platform/app/core/ontology/graph_merge.py

@@ -0,0 +1,30 @@
+from __future__ import annotations
+
+from typing import Any
+
+from crawler_platform.app.core.database import models
+from crawler_platform.app.core.ontology.relation_schema import minimum_confidence
+
+
+def should_merge_claim_to_graph(
+    claim: models.Claim,
+    ontology: dict[str, Any] | None = None,
+    domain: str | None = None,
+) -> tuple[bool, str | None]:
+    if claim.status != "validated_claim":
+        return False, f"claim status is not validated_claim: {claim.status}"
+    if claim.object_entity_id is None:
+        return False, "literal claims are stored as evidence claims, not graph relations"
+    threshold = minimum_confidence(claim.predicate, ontology, domain)
+    if claim.confidence < threshold:
+        return False, f"claim confidence {claim.confidence:.2f} is below graph threshold {threshold:.2f}"
+    metadata = claim.metadata_json or {}
+    if metadata.get("validation_status") != "validated_claim":
+        return False, "claim metadata validation_status is not validated_claim"
+    if metadata.get("review_required"):
+        return False, metadata.get("review_reason") or "claim requires human review"
+    if not metadata.get("evidence_found"):
+        return False, "claim has no matched evidence span"
+    if not metadata.get("source_zone_allowed"):
+        return False, "claim source zone is not graph-mergeable"
+    return True, None

crawler_platform/app/core/ontology/registry.py

@@ -0,0 +1,339 @@
+from __future__ import annotations
+
+import hashlib
+import json
+from typing import Any
+
+from sqlalchemy import select
+from sqlalchemy.orm import Session
+
+from crawler_platform.app.config.loader import ProjectConfig
+from crawler_platform.app.core.database import models
+from crawler_platform.app.core.ontology.relation_schema import relation_rule
+
+
+CORE_ENTITY_TYPES = {
+    "Entity",
+    "Concept",
+    "Event",
+    "Organization",
+    "Person",
+    "Place",
+    "Document",
+    "Source",
+    "Page",
+}
+
+CORE_RELATION_TYPES = {
+    "relatedTo": {
+        "allowed_subject_types": [],
+        "allowed_object_types": [],
+        "semantic_constraints": {"domain_agnostic": True},
+        "confidence_rules": {"min_confidence": 0.7},
+    },
+    "mentions": {
+        "allowed_subject_types": ["Document", "Page", "Source"],
+        "allowed_object_types": [],
+        "semantic_constraints": {"provenance_relation": True},
+        "confidence_rules": {"min_confidence": 0.6},
+    },
+    "sameAs": {
+        "allowed_subject_types": [],
+        "allowed_object_types": [],
+        "semantic_constraints": {"entity_resolution": True},
+        "confidence_rules": {"min_confidence": 0.9},
+    },
+    "partOf": {
+        "allowed_subject_types": [],
+        "allowed_object_types": [],
+        "semantic_constraints": {"transitive_candidate": True},
+        "confidence_rules": {"min_confidence": 0.75},
+    },
+}
+
+
+class OntologyRegistry:
+    """Project-scoped ontology schema registry.
+
+    The registry keeps schema-level objects in the database so entity and
+    relation types can evolve through proposals instead of remaining hidden in
+    code-only constants.
+    """
+
+    def __init__(self, session: Session):
+        self.session = session
+
+    def seed_from_config(self, project: models.Project, config: ProjectConfig) -> None:
+        metadata = {"origin": "project_config", "config_domain": config.domain}
+        for name in sorted(CORE_ENTITY_TYPES | set(config.target_entities) | set(config.ontology.get("entity_types") or [])):
+            self.upsert_entity_type(
+                project.id,
+                name=name,
+                domain=config.domain if name not in CORE_ENTITY_TYPES else "core",
+                metadata=metadata,
+            )
+        for name, spec in CORE_RELATION_TYPES.items():
+            self.upsert_relation_type(project.id, name=name, domain="core", metadata={"origin": "core"}, **spec)
+        configured_relation_types = config.ontology.get("relation_types") or {}
+        for predicate in sorted(set(config.ontology.get("predicates") or [])):
+            spec = configured_relation_types.get(predicate) if isinstance(configured_relation_types, dict) else None
+            self.upsert_relation_type(
+                project.id,
+                name=predicate,
+                domain=config.domain,
+                metadata=metadata,
+                **self._relation_metadata_from_config(predicate, spec, config.ontology, config.domain),
+            )
+
+    def upsert_entity_type(
+        self,
+        project_id: int,
+        *,
+        name: str,
+        domain: str = "generic",
+        description: str | None = None,
+        status: str = "active",
+        confidence: float = 1.0,
+        metadata: dict[str, Any] | None = None,
+    ) -> models.OntologyEntityType:
+        clean_name = normalize_schema_name(name)
+        row = self.session.scalar(
+            select(models.OntologyEntityType).where(
+                models.OntologyEntityType.project_id == project_id,
+                models.OntologyEntityType.name == clean_name,
+            )
+        )
+        if row is None:
+            row = models.OntologyEntityType(
+                project_id=project_id,
+                name=clean_name,
+                domain=domain,
+                description=description,
+                status=status,
+                confidence=confidence,
+                metadata_json=metadata or {},
+            )
+            self.session.add(row)
+            self.session.flush()
+            return row
+        row.domain = domain or row.domain
+        row.description = description or row.description
+        row.status = status or row.status
+        row.confidence = max(row.confidence or 0.0, confidence)
+        row.metadata_json = {**(row.metadata_json or {}), **(metadata or {})}
+        row.updated_at = models.utcnow()
+        return row
+
+    def upsert_relation_type(
+        self,
+        project_id: int,
+        *,
+        name: str,
+        domain: str = "generic",
+        description: str | None = None,
+        allowed_subject_types: list[str] | None = None,
+        allowed_object_types: list[str] | None = None,
+        allowed_page_types: list[str] | None = None,
+        allowed_source_zones: list[str] | None = None,
+        semantic_constraints: dict[str, Any] | None = None,
+        confidence_rules: dict[str, Any] | None = None,
+        status: str = "active",
+        confidence: float = 1.0,
+        metadata: dict[str, Any] | None = None,
+    ) -> models.OntologyRelationType:
+        clean_name = normalize_schema_name(name)
+        row = self.session.scalar(
+            select(models.OntologyRelationType).where(
+                models.OntologyRelationType.project_id == project_id,
+                models.OntologyRelationType.name == clean_name,
+            )
+        )
+        values = {
+            "allowed_subject_types": sorted(set(allowed_subject_types or [])),
+            "allowed_object_types": sorted(set(allowed_object_types or [])),
+            "allowed_page_types": sorted(set(allowed_page_types or [])),
+            "allowed_source_zones": sorted(set(allowed_source_zones or [])),
+            "semantic_constraints": semantic_constraints or {},
+            "confidence_rules": confidence_rules or {},
+        }
+        if row is None:
+            row = models.OntologyRelationType(
+                project_id=project_id,
+                name=clean_name,
+                domain=domain,
+                description=description,
+                status=status,
+                confidence=confidence,
+                metadata_json=metadata or {},
+                **values,
+            )
+            self.session.add(row)
+            self.session.flush()
+            return row
+        row.domain = domain or row.domain
+        row.description = description or row.description
+        row.status = status or row.status
+        row.confidence = max(row.confidence or 0.0, confidence)
+        row.allowed_subject_types = merge_unique(row.allowed_subject_types, values["allowed_subject_types"])
+        row.allowed_object_types = merge_unique(row.allowed_object_types, values["allowed_object_types"])
+        row.allowed_page_types = merge_unique(row.allowed_page_types, values["allowed_page_types"])
+        row.allowed_source_zones = merge_unique(row.allowed_source_zones, values["allowed_source_zones"])
+        row.semantic_constraints = {**(row.semantic_constraints or {}), **values["semantic_constraints"]}
+        row.confidence_rules = {**(row.confidence_rules or {}), **values["confidence_rules"]}
+        row.metadata_json = {**(row.metadata_json or {}), **(metadata or {})}
+        row.updated_at = models.utcnow()
+        return row
+
+    def relation_type(self, project_id: int, name: str) -> models.OntologyRelationType | None:
+        return self.session.scalar(
+            select(models.OntologyRelationType).where(
+                models.OntologyRelationType.project_id == project_id,
+                models.OntologyRelationType.name == normalize_schema_name(name),
+            )
+        )
+
+    def entity_type(self, project_id: int, name: str) -> models.OntologyEntityType | None:
+        return self.session.scalar(
+            select(models.OntologyEntityType).where(
+                models.OntologyEntityType.project_id == project_id,
+                models.OntologyEntityType.name == normalize_schema_name(name),
+            )
+        )
+
+    def propose_schema_change(
+        self,
+        project_id: int,
+        *,
+        proposal_type: str,
+        name: str,
+        reason: str,
+        evidence: str | None = None,
+        confidence: float = 0.5,
+        metadata: dict[str, Any] | None = None,
+    ) -> models.OntologyProposal:
+        proposal_hash = stable_hash(
+            {
+                "project_id": project_id,
+                "proposal_type": proposal_type,
+                "name": normalize_schema_name(name),
+                "metadata": metadata or {},
+            }
+        )
+        row = self.session.scalar(
+            select(models.OntologyProposal).where(
+                models.OntologyProposal.project_id == project_id,
+                models.OntologyProposal.proposal_hash == proposal_hash,
+            )
+        )
+        if row is None:
+            row = models.OntologyProposal(
+                project_id=project_id,
+                proposal_type=proposal_type,
+                name=normalize_schema_name(name),
+                reason=reason,
+                evidence=evidence,
+                confidence=min(max(confidence, 0.0), 1.0),
+                proposal_hash=proposal_hash,
+                metadata_json=metadata or {},
+            )
+            self.session.add(row)
+            self.session.flush()
+            return row
+        row.confidence = max(row.confidence, min(max(confidence, 0.0), 1.0))
+        row.reason = reason or row.reason
+        row.evidence = evidence or row.evidence
+        row.metadata_json = {**(row.metadata_json or {}), **(metadata or {})}
+        row.updated_at = models.utcnow()
+        return row
+
+    def registry_payload(self, project_id: int) -> dict[str, Any]:
+        entity_types = self.session.scalars(
+            select(models.OntologyEntityType)
+            .where(models.OntologyEntityType.project_id == project_id)
+            .order_by(models.OntologyEntityType.domain, models.OntologyEntityType.name)
+        ).all()
+        relation_types = self.session.scalars(
+            select(models.OntologyRelationType)
+            .where(models.OntologyRelationType.project_id == project_id)
+            .order_by(models.OntologyRelationType.domain, models.OntologyRelationType.name)
+        ).all()
+        return {
+            "entity_types": [entity_type_payload(row) for row in entity_types],
+            "relation_types": [relation_type_payload(row) for row in relation_types],
+        }
+
+    def _relation_metadata_from_config(
+        self,
+        predicate: str,
+        spec: dict[str, Any] | None,
+        ontology: dict[str, Any],
+        domain: str | None,
+    ) -> dict[str, Any]:
+        if spec:
+            return {
+                "description": spec.get("description"),
+                "allowed_subject_types": list(spec.get("allowed_subject_types") or spec.get("subject_types") or []),
+                "allowed_object_types": list(spec.get("allowed_object_types") or spec.get("object_types") or []),
+                "allowed_page_types": list(spec.get("allowed_page_types") or spec.get("page_types") or []),
+                "allowed_source_zones": list(spec.get("allowed_source_zones") or spec.get("source_zones") or []),
+                "semantic_constraints": dict(spec.get("semantic_constraints") or {}),
+                "confidence_rules": dict(spec.get("confidence_rules") or {}),
+            }
+        rule = relation_rule(predicate, ontology, domain)
+        if rule is None:
+            return {
+                "semantic_constraints": {"domain_defined": True, "requires_governance_for_new_constraints": True},
+                "confidence_rules": {"min_confidence": 0.8},
+            }
+        return {
+            "allowed_subject_types": sorted(rule.subject_types),
+            "allowed_object_types": sorted(rule.object_types),
+            "allowed_page_types": sorted(rule.page_types),
+            "allowed_source_zones": sorted(rule.source_zones),
+            "semantic_constraints": {"literal_value": rule.literal_value},
+            "confidence_rules": {"min_confidence": rule.min_confidence},
+        }
+
+
+def normalize_schema_name(value: str) -> str:
+    return " ".join(str(value or "").strip().split())
+
+
+def merge_unique(existing: Any, new_values: list[str]) -> list[str]:
+    return sorted({str(item) for item in (existing or []) if str(item)} | {str(item) for item in new_values if str(item)})
+
+
+def stable_hash(payload: dict[str, Any]) -> str:
+    return hashlib.sha256(json.dumps(payload, sort_keys=True, default=str).encode("utf-8")).hexdigest()
+
+
+def entity_type_payload(row: models.OntologyEntityType) -> dict[str, Any]:
+    return {
+        "id": row.id,
+        "name": row.name,
+        "domain": row.domain,
+        "description": row.description,
+        "status": row.status,
+        "version": row.version,
+        "confidence": row.confidence,
+        "metadata": row.metadata_json or {},
+    }
+
+
+def relation_type_payload(row: models.OntologyRelationType) -> dict[str, Any]:
+    return {
+        "id": row.id,
+        "name": row.name,
+        "domain": row.domain,
+        "description": row.description,
+        "allowed_subject_types": row.allowed_subject_types or [],
+        "allowed_object_types": row.allowed_object_types or [],
+        "allowed_page_types": row.allowed_page_types or [],
+        "allowed_source_zones": row.allowed_source_zones or [],
+        "semantic_constraints": row.semantic_constraints or {},
+        "confidence_rules": row.confidence_rules or {},
+        "status": row.status,
+        "version": row.version,
+        "confidence": row.confidence,
+        "metadata": row.metadata_json or {},
+    }

crawler_platform/app/core/ontology/relation_schema.py

@@ -0,0 +1,145 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any
+
+
+@dataclass(frozen=True, slots=True)
+class RelationRule:
+    predicate: str
+    subject_types: set[str]
+    object_types: set[str] = field(default_factory=set)
+    literal_value: bool = False
+    page_types: set[str] = field(default_factory=set)
+    source_zones: set[str] = field(default_factory=set)
+    min_confidence: float = 0.7
+
+
+CORE_RELATION_SCHEMA: dict[str, RelationRule] = {
+    "relatedTo": RelationRule("relatedTo", subject_types=set(), object_types=set(), page_types=set(), min_confidence=0.7),
+    "mentions": RelationRule("mentions", subject_types=set(), object_types=set(), page_types=set(), min_confidence=0.6),
+    "sameAs": RelationRule("sameAs", subject_types=set(), object_types=set(), page_types=set(), min_confidence=0.9),
+    "partOf": RelationRule("partOf", subject_types=set(), object_types=set(), page_types=set(), min_confidence=0.75),
+}
+
+
+def relation_rule(
+    predicate: str,
+    ontology: dict[str, Any] | None = None,
+    domain: str | None = None,
+) -> RelationRule | None:
+    configured_rule = configured_relation_rule(predicate, ontology)
+    if configured_rule is not None:
+        return configured_rule
+    if domain:
+        from crawler_platform.app.adapters.registry import adapter_for_domain
+
+        adapter = adapter_for_domain(domain)
+        if adapter is not None:
+            adapter_rule = adapter.relation_rule(predicate)
+            if adapter_rule is not None:
+                return adapter_rule
+    if predicate in CORE_RELATION_SCHEMA:
+        return CORE_RELATION_SCHEMA[predicate]
+    predicates = set((ontology or {}).get("predicates") or [])
+    if predicate in predicates:
+        return RelationRule(predicate=predicate, subject_types=set(), object_types=set(), page_types=set(), min_confidence=0.8)
+    return None
+
+
+def configured_relation_rule(predicate: str, ontology: dict[str, Any] | None = None) -> RelationRule | None:
+    relation_types = (ontology or {}).get("relation_types") or {}
+    if not isinstance(relation_types, dict) or predicate not in relation_types:
+        return None
+    spec = relation_types.get(predicate) or {}
+    constraints = dict(spec.get("semantic_constraints") or {})
+    confidence_rules = dict(spec.get("confidence_rules") or {})
+    literal_value = bool(
+        spec.get("literal_value")
+        or constraints.get("literal_value")
+        or spec.get("value_type") == "literal"
+        or spec.get("object_kind") == "literal"
+    )
+    return RelationRule(
+        predicate=predicate,
+        subject_types=set(spec.get("allowed_subject_types") or spec.get("subject_types") or []),
+        object_types=set(spec.get("allowed_object_types") or spec.get("object_types") or []),
+        literal_value=literal_value,
+        page_types=set(spec.get("allowed_page_types") or spec.get("page_types") or []),
+        source_zones=set(spec.get("allowed_source_zones") or spec.get("source_zones") or []),
+        min_confidence=float(confidence_rules.get("min_confidence") or spec.get("min_confidence") or 0.8),
+    )
+
+
+def relation_schema_compatible(
+    *,
+    predicate: str,
+    subject_type: str,
+    object_type: str | None,
+    has_literal_value: bool,
+    page_type: str | None,
+    source_zone: str | None,
+    ontology: dict[str, Any] | None = None,
+    domain: str | None = None,
+) -> str | None:
+    rule = relation_rule(predicate, ontology, domain)
+    if rule is None:
+        return "predicate has no relation schema"
+    if rule.subject_types and subject_type not in rule.subject_types:
+        return f"subject type {subject_type} is not allowed for {predicate}"
+    if rule.literal_value:
+        if not has_literal_value:
+            return f"{predicate} expects a literal object"
+    else:
+        if has_literal_value and rule.object_types:
+            return f"{predicate} expects a typed entity object"
+        if not has_literal_value and not object_type:
+            return f"{predicate} expects a typed entity object"
+        if object_type and rule.object_types and object_type not in rule.object_types:
+            return f"object type {object_type} is not allowed for {predicate}"
+    if page_type and rule.page_types and page_type not in rule.page_types:
+        return f"predicate {predicate} is not allowed for page type {page_type}"
+    if source_zone and rule.source_zones and source_zone not in rule.source_zones:
+        return f"source zone {source_zone} is not allowed for {predicate}"
+    return None
+
+
+def confidence_breakdown(
+    *,
+    llm_confidence: float,
+    evidence_found: bool,
+    ontology_compatible: bool,
+    source_zone_allowed: bool,
+    source_trust: float | None = None,
+) -> dict[str, float]:
+    schema_confidence = 1.0
+    evidence_confidence = 0.95 if evidence_found else 0.0
+    ontology_confidence = 0.95 if ontology_compatible else 0.0
+    zone_confidence = 0.9 if source_zone_allowed else 0.0
+    trust = source_trust if source_trust is not None else 0.8
+    final = (
+        llm_confidence * 0.35
+        + schema_confidence * 0.15
+        + evidence_confidence * 0.2
+        + ontology_confidence * 0.2
+        + zone_confidence * 0.05
+        + trust * 0.05
+    )
+    return {
+        "llm_confidence": round(llm_confidence, 4),
+        "schema_confidence": schema_confidence,
+        "evidence_confidence": evidence_confidence,
+        "ontology_confidence": ontology_confidence,
+        "source_zone_confidence": zone_confidence,
+        "source_trust": round(trust, 4),
+        "final_confidence": round(min(max(final, 0.0), 1.0), 4),
+    }
+
+
+def minimum_confidence(
+    predicate: str,
+    ontology: dict[str, Any] | None = None,
+    domain: str | None = None,
+) -> float:
+    rule = relation_rule(predicate, ontology, domain)
+    return rule.min_confidence if rule else 0.8

crawler_platform/app/core/ontology/structured_output.py

@@ -0,0 +1,111 @@
+from __future__ import annotations
+
+from typing import Any
+
+from pydantic import BaseModel, ConfigDict, Field, ValidationError, field_validator, model_validator
+
+
+class StructuredEntity(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    entity_type: str = Field(min_length=1)
+    name: str = Field(min_length=1)
+    attributes: dict[str, Any] = Field(default_factory=dict)
+    confidence: float = Field(ge=0.0, le=1.0)
+    evidence_text: str = Field(min_length=1)
+
+    @field_validator("entity_type", "name", "evidence_text")
+    @classmethod
+    def _strip_required_text(cls, value: str) -> str:
+        clean = value.strip()
+        if not clean:
+            raise ValueError("field cannot be blank")
+        return clean
+
+
+class StructuredClaim(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    subject_name: str = Field(min_length=1)
+    subject_type: str = Field(min_length=1)
+    predicate: str = Field(min_length=1)
+    object_name: str | None = None
+    object_type: str | None = None
+    object_value: Any | None = None
+    evidence_text: str = Field(min_length=1)
+    evidence_summary: str | None = None
+    confidence: float = Field(ge=0.0, le=1.0)
+    confidence_reason: str | None = None
+    source_zone: str | None = None
+
+    @field_validator("subject_name", "subject_type", "predicate", "evidence_text")
+    @classmethod
+    def _strip_required_text(cls, value: str) -> str:
+        clean = value.strip()
+        if not clean:
+            raise ValueError("field cannot be blank")
+        return clean
+
+    @field_validator("object_name", "object_type", "evidence_summary", "confidence_reason", "source_zone")
+    @classmethod
+    def _strip_optional_text(cls, value: str | None) -> str | None:
+        if value is None:
+            return None
+        clean = value.strip()
+        return clean or None
+
+    @model_validator(mode="after")
+    def _has_object(self) -> "StructuredClaim":
+        if self.object_name is None and self.object_value is None:
+            raise ValueError("claim must have object_name or object_value")
+        return self
+
+
+class StructuredExtraction(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    entities: list[StructuredEntity] = Field(default_factory=list)
+    claims: list[StructuredClaim] = Field(default_factory=list)
+
+
+def validate_structured_extraction(raw: dict[str, Any]) -> dict[str, Any]:
+    try:
+        parsed = StructuredExtraction.model_validate(raw)
+    except ValidationError as exc:
+        raise ValueError(f"schema_validation_failed: {exc.errors(include_url=False)}") from exc
+    return parsed.model_dump()
+
+
+def extraction_json_schema() -> dict[str, Any]:
+    schema = StructuredExtraction.model_json_schema()
+    _inline_defs(schema)
+    _disallow_additional_properties(schema)
+    return schema
+
+
+def _inline_defs(schema: dict[str, Any]) -> None:
+    defs = schema.pop("$defs", {})
+
+    def resolve(node: Any) -> Any:
+        if isinstance(node, dict):
+            ref = node.get("$ref")
+            if ref and ref.startswith("#/$defs/"):
+                name = ref.rsplit("/", 1)[-1]
+                return resolve(dict(defs[name]))
+            return {key: resolve(value) for key, value in node.items()}
+        if isinstance(node, list):
+            return [resolve(item) for item in node]
+        return node
+
+    schema.update(resolve(schema))
+
+
+def _disallow_additional_properties(node: Any) -> None:
+    if isinstance(node, dict):
+        if node.get("type") == "object":
+            node.setdefault("additionalProperties", False)
+        for value in node.values():
+            _disallow_additional_properties(value)
+    elif isinstance(node, list):
+        for item in node:
+            _disallow_additional_properties(item)

crawler_platform/app/core/ontology/triple_store.py

@@ -0,0 +1,183 @@
+from __future__ import annotations
+
+from typing import Any
+
+from sqlalchemy import select
+from sqlalchemy.orm import Session
+
+from crawler_platform.app.core.database import models
+from crawler_platform.app.core.ontology.registry import OntologyRegistry, stable_hash
+
+
+class OntologyTripleStore:
+    """Stores graph triples separately from extraction claims.
+
+    Claims remain source/evidence records. Triples are the project-level graph
+    facts that can accumulate support from many claims and sources.
+    """
+
+    def __init__(self, session: Session):
+        self.session = session
+        self.registry = OntologyRegistry(session)
+
+    def backfill_project(self, project_id: int, limit: int = 2000) -> int:
+        claims = self.session.scalars(
+            select(models.Claim)
+            .where(models.Claim.project_id == project_id, models.Claim.status == "validated_claim")
+            .order_by(models.Claim.last_seen_at.desc())
+            .limit(limit)
+        ).all()
+        count = 0
+        for claim in claims:
+            self.upsert_from_claim(claim)
+            count += 1
+        return count
+
+    def upsert_from_claim(self, claim: models.Claim) -> models.OntologyTriple:
+        subject = self.session.get(models.Entity, claim.subject_entity_id)
+        object_entity = self.session.get(models.Entity, claim.object_entity_id) if claim.object_entity_id else None
+        relation_type = self.registry.relation_type(claim.project_id, claim.predicate)
+        metadata = dict(claim.metadata_json or {})
+        if subject is None:
+            raise ValueError(f"Claim {claim.id} has no subject entity")
+        if self.registry.entity_type(claim.project_id, subject.entity_type) is None:
+            self.registry.propose_schema_change(
+                claim.project_id,
+                proposal_type="entity_type",
+                name=subject.entity_type,
+                reason="Validated claim references an entity type not present in ontology registry.",
+                evidence=subject.name,
+                confidence=claim.confidence,
+                metadata={"claim_id": claim.id},
+            )
+        if object_entity is not None and self.registry.entity_type(claim.project_id, object_entity.entity_type) is None:
+            self.registry.propose_schema_change(
+                claim.project_id,
+                proposal_type="entity_type",
+                name=object_entity.entity_type,
+                reason="Validated claim references an object entity type not present in ontology registry.",
+                evidence=object_entity.name,
+                confidence=claim.confidence,
+                metadata={"claim_id": claim.id},
+            )
+        if relation_type is None:
+            self.registry.propose_schema_change(
+                claim.project_id,
+                proposal_type="relation_type",
+                name=claim.predicate,
+                reason="Validated claim references a relation type not present in ontology registry.",
+                evidence=metadata.get("evidence_text") or claim.confidence_reason,
+                confidence=claim.confidence,
+                metadata={"claim_id": claim.id},
+            )
+
+        triple_hash = make_triple_hash(claim)
+        row = self.session.scalar(
+            select(models.OntologyTriple).where(
+                models.OntologyTriple.project_id == claim.project_id,
+                models.OntologyTriple.triple_hash == triple_hash,
+            )
+        )
+        source_item = {
+            "claim_id": claim.id,
+            "source_id": claim.source_id,
+            "page_id": claim.page_id,
+            "seen_at": models.utcnow().isoformat(),
+            "confidence": claim.confidence,
+            "status": claim.status,
+        }
+        triple_metadata = {
+            "source_history": [source_item],
+            "claim_metadata": {
+                "page_type": metadata.get("page_type"),
+                "source_zone": metadata.get("source_zone"),
+                "graph_merge_reason": metadata.get("graph_merge_reason"),
+                "review_reason": metadata.get("review_reason"),
+            },
+        }
+        status = triple_status_from_claim(claim)
+        if row is None:
+            row = models.OntologyTriple(
+                project_id=claim.project_id,
+                claim_id=claim.id,
+                source_id=claim.source_id,
+                page_id=claim.page_id,
+                subject_entity_id=claim.subject_entity_id,
+                subject_type=subject.entity_type,
+                predicate=claim.predicate,
+                relation_type_id=relation_type.id if relation_type else None,
+                object_entity_id=claim.object_entity_id,
+                object_type=object_entity.entity_type if object_entity else None,
+                object_value=claim.object_value,
+                value_type=claim.value_type,
+                triple_hash=triple_hash,
+                status=status,
+                confidence=claim.confidence,
+                metadata_json=triple_metadata,
+            )
+            self.session.add(row)
+            self.session.flush()
+            return row
+        existing_metadata = dict(row.metadata_json or {})
+        row.claim_id = claim.id
+        row.source_id = claim.source_id
+        row.page_id = claim.page_id
+        row.relation_type_id = relation_type.id if relation_type else row.relation_type_id
+        row.status = merge_status(row.status, status)
+        row.confidence = max(row.confidence, claim.confidence)
+        row.support_count += 1
+        row.last_seen_at = models.utcnow()
+        row.metadata_json = {
+            **existing_metadata,
+            "source_history": merge_source_history(existing_metadata.get("source_history") or [], [source_item]),
+            "claim_metadata": triple_metadata["claim_metadata"],
+        }
+        return row
+
+
+def make_triple_hash(claim: models.Claim) -> str:
+    return stable_hash(
+        {
+            "project_id": claim.project_id,
+            "subject_entity_id": claim.subject_entity_id,
+            "predicate": claim.predicate,
+            "object_entity_id": claim.object_entity_id,
+            "object_value": claim.object_value,
+            "value_type": claim.value_type,
+        }
+    )
+
+
+def triple_status_from_claim(claim: models.Claim) -> str:
+    metadata = claim.metadata_json or {}
+    if metadata.get("review_required") or metadata.get("conflict_status"):
+        return "review_required"
+    if claim.status != "validated_claim":
+        return "candidate"
+    if metadata.get("graph_merge_status") == "merged":
+        return "merged"
+    if claim.value_type == "literal":
+        return "validated_literal"
+    return "validated"
+
+
+def merge_status(existing: str, new_status: str) -> str:
+    rank = {
+        "candidate": 0,
+        "validated": 1,
+        "validated_literal": 1,
+        "review_required": 2,
+        "merged": 3,
+    }
+    return new_status if rank.get(new_status, 0) > rank.get(existing, 0) else existing
+
+
+def merge_source_history(existing: list[dict[str, Any]], new_items: list[dict[str, Any]]) -> list[dict[str, Any]]:
+    merged = [item for item in existing if isinstance(item, dict)]
+    seen = {(item.get("claim_id"), item.get("source_id"), item.get("page_id")) for item in merged}
+    for item in new_items:
+        key = (item.get("claim_id"), item.get("source_id"), item.get("page_id"))
+        if key not in seen:
+            merged.append(item)
+            seen.add(key)
+    return merged[-100:]