참고소스 수정본

crawler_platform/app/core/database/models.py

@@ -123,6 +123,115 @@ class Relation(Base):
     updated_at = Column(DateTime(timezone=True), default=utcnow, nullable=False)
 
 
+class OntologyEntityType(Base):
+    __tablename__ = "ontology_entity_types"
+    __table_args__ = (
+        UniqueConstraint("project_id", "name", name="uq_ontology_entity_type_project_name"),
+    )
+
+    id = Column(Integer, primary_key=True)
+    project_id = Column(Integer, ForeignKey("projects.id"), nullable=False, index=True)
+    name = Column(String(160), nullable=False, index=True)
+    domain = Column(String(120), nullable=False, default="generic", index=True)
+    description = Column(Text)
+    status = Column(String(40), nullable=False, default="active", index=True)
+    version = Column(String(40), nullable=False, default="1.0.0")
+    confidence = Column(Float, nullable=False, default=1.0)
+    metadata_json = Column(JSON, nullable=False, default=dict)
+    created_at = Column(DateTime(timezone=True), default=utcnow, nullable=False)
+    updated_at = Column(DateTime(timezone=True), default=utcnow, nullable=False)
+
+
+class OntologyRelationType(Base):
+    __tablename__ = "ontology_relation_types"
+    __table_args__ = (
+        UniqueConstraint("project_id", "name", name="uq_ontology_relation_type_project_name"),
+    )
+
+    id = Column(Integer, primary_key=True)
+    project_id = Column(Integer, ForeignKey("projects.id"), nullable=False, index=True)
+    name = Column(String(160), nullable=False, index=True)
+    domain = Column(String(120), nullable=False, default="generic", index=True)
+    description = Column(Text)
+    allowed_subject_types = Column(JSON, nullable=False, default=list)
+    allowed_object_types = Column(JSON, nullable=False, default=list)
+    allowed_page_types = Column(JSON, nullable=False, default=list)
+    allowed_source_zones = Column(JSON, nullable=False, default=list)
+    semantic_constraints = Column(JSON, nullable=False, default=dict)
+    confidence_rules = Column(JSON, nullable=False, default=dict)
+    status = Column(String(40), nullable=False, default="active", index=True)
+    version = Column(String(40), nullable=False, default="1.0.0")
+    confidence = Column(Float, nullable=False, default=1.0)
+    metadata_json = Column(JSON, nullable=False, default=dict)
+    created_at = Column(DateTime(timezone=True), default=utcnow, nullable=False)
+    updated_at = Column(DateTime(timezone=True), default=utcnow, nullable=False)
+
+
+class OntologyTriple(Base):
+    __tablename__ = "ontology_triples"
+    __table_args__ = (UniqueConstraint("project_id", "triple_hash", name="uq_ontology_triple_hash"),)
+
+    id = Column(Integer, primary_key=True)
+    project_id = Column(Integer, ForeignKey("projects.id"), nullable=False, index=True)
+    claim_id = Column(Integer, ForeignKey("claims.id"), nullable=True, index=True)
+    source_id = Column(Integer, ForeignKey("sources.id"), nullable=True, index=True)
+    page_id = Column(Integer, ForeignKey("pages.id"), nullable=True, index=True)
+    subject_entity_id = Column(Integer, ForeignKey("entities.id"), nullable=False, index=True)
+    subject_type = Column(String(160), nullable=False, index=True)
+    predicate = Column(String(160), nullable=False, index=True)
+    relation_type_id = Column(Integer, ForeignKey("ontology_relation_types.id"), nullable=True, index=True)
+    object_entity_id = Column(Integer, ForeignKey("entities.id"), nullable=True, index=True)
+    object_type = Column(String(160), nullable=True, index=True)
+    object_value = Column(JSON, nullable=True)
+    value_type = Column(String(80), nullable=False, default="entity")
+    triple_hash = Column(String(80), nullable=False, index=True)
+    status = Column(String(40), nullable=False, default="candidate", index=True)
+    confidence = Column(Float, nullable=False, default=0.5)
+    support_count = Column(Integer, nullable=False, default=1)
+    first_seen_at = Column(DateTime(timezone=True), default=utcnow, nullable=False)
+    last_seen_at = Column(DateTime(timezone=True), default=utcnow, nullable=False)
+    metadata_json = Column(JSON, nullable=False, default=dict)
+
+
+class OntologyProposal(Base):
+    __tablename__ = "ontology_proposals"
+    __table_args__ = (
+        UniqueConstraint("project_id", "proposal_hash", name="uq_ontology_proposal_hash"),
+    )
+
+    id = Column(Integer, primary_key=True)
+    project_id = Column(Integer, ForeignKey("projects.id"), nullable=False, index=True)
+    proposal_type = Column(String(80), nullable=False, index=True)
+    name = Column(String(160), nullable=False, index=True)
+    reason = Column(Text)
+    evidence = Column(Text)
+    status = Column(String(40), nullable=False, default="pending_review", index=True)
+    confidence = Column(Float, nullable=False, default=0.5)
+    proposal_hash = Column(String(80), nullable=False, index=True)
+    metadata_json = Column(JSON, nullable=False, default=dict)
+    created_at = Column(DateTime(timezone=True), default=utcnow, nullable=False)
+    updated_at = Column(DateTime(timezone=True), default=utcnow, nullable=False)
+
+
+class KnowledgeGap(Base):
+    __tablename__ = "knowledge_gaps"
+    __table_args__ = (UniqueConstraint("project_id", "gap_hash", name="uq_knowledge_gap_hash"),)
+
+    id = Column(Integer, primary_key=True)
+    project_id = Column(Integer, ForeignKey("projects.id"), nullable=False, index=True)
+    gap_type = Column(String(80), nullable=False, index=True)
+    target_type = Column(String(160), nullable=True, index=True)
+    target_name = Column(String(240), nullable=True, index=True)
+    description = Column(Text, nullable=False)
+    priority = Column(Float, nullable=False, default=0.5, index=True)
+    status = Column(String(40), nullable=False, default="open", index=True)
+    gap_hash = Column(String(80), nullable=False, index=True)
+    evidence = Column(JSON, nullable=False, default=dict)
+    metadata_json = Column(JSON, nullable=False, default=dict)
+    created_at = Column(DateTime(timezone=True), default=utcnow, nullable=False)
+    updated_at = Column(DateTime(timezone=True), default=utcnow, nullable=False)
+
+
 class Claim(Base):
     __tablename__ = "claims"
     __table_args__ = (UniqueConstraint("project_id", "claim_hash", name="uq_claim_hash"),)

crawler_platform/app/core/database/repository.py

@@ -11,10 +11,15 @@ from sqlalchemy.orm import Session
 from crawler_platform.app.config.loader import ProjectConfig, SourceConfig
 from crawler_platform.app.core.database import models
 from crawler_platform.app.core.extractor.base import ExtractedClaim, ExtractedEntity, ExtractionBundle
+from crawler_platform.app.core.extractor.validation import validate_extraction_bundle
+from crawler_platform.app.core.ontology.entity_normalizer import canonical_entity_key, normalize_entity_name
+from crawler_platform.app.core.ontology.graph_merge import should_merge_claim_to_graph
+from crawler_platform.app.core.ontology.registry import OntologyRegistry
+from crawler_platform.app.core.ontology.triple_store import OntologyTripleStore
 
 
 def canonicalize(value: str) -> str:
-    return " ".join(value.strip().lower().split())
+    return canonical_entity_key(value)
 
 
 def short_hash(text: str) -> str:
@@ -39,6 +44,8 @@ class KnowledgeRepository:
                 project.updated_at = models.utcnow()
         for source_config in config.sources:
             self.upsert_source(project, source_config)
+        OntologyRegistry(self.session).seed_from_config(project, config)
+        OntologyTripleStore(self.session).backfill_project(project.id)
         return project
 
     def upsert_source(self, project: models.Project, source_config: SourceConfig) -> models.Source:
@@ -89,6 +96,8 @@ class KnowledgeRepository:
             models.UserProfile,
             models.CrawlJob,
             models.ExtractionLog,
+            models.KnowledgeGap,
+            models.OntologyTriple,
             models.Evidence,
             models.Relation,
             models.Claim,
@@ -167,11 +176,24 @@ class KnowledgeRepository:
         source: models.Source,
         page: models.Page,
         bundle: ExtractionBundle,
+        project_config: ProjectConfig | None = None,
     ) -> list[models.Claim]:
+        if project_config is not None:
+            validation = validate_extraction_bundle(bundle, project_config)
+            bundle = validation.bundle
+            claim_status = validation.claim_status
+        else:
+            claim_status = "active"
+
+        if claim_status not in {"active", "validated_claim"}:
+            self._log_extraction(project_id, page, bundle)
+            return []
+
         entity_index: dict[tuple[str, str], models.Entity] = {}
         for extracted_entity in bundle.entities:
             entity = self._save_extracted_entity(project_id, source.id, extracted_entity)
-            entity_index[(extracted_entity.entity_type, canonicalize(extracted_entity.name))] = entity
+            normalized_name = normalize_entity_name(extracted_entity.name, extracted_entity.entity_type)
+            entity_index[(extracted_entity.entity_type, canonicalize(normalized_name))] = entity
 
         claims: list[models.Claim] = []
         for extracted_claim in bundle.claims:
@@ -185,6 +207,43 @@ class KnowledgeRepository:
                     entity_index,
                 )
             confidence = combine_confidence(extracted_claim.confidence, source.trust_level)
+            claim_metadata = {
+                **extracted_claim.metadata,
+                "source_trust": source.trust_level,
+            }
+            claim_metadata["source_history"] = [
+                {
+                    "source_id": source.id,
+                    "source_name": source.name,
+                    "source_type": source.type,
+                    "source_trust": source.trust_level,
+                    "page_id": page.id,
+                    "page_url": page.url,
+                    "seen_at": models.utcnow().isoformat(),
+                    "confidence": extracted_claim.confidence,
+                }
+            ]
+            conflict = self._find_conflicting_claim(
+                project_id,
+                subject.id,
+                extracted_claim.predicate,
+                object_entity.id if object_entity else None,
+                extracted_claim.object_value,
+            )
+            if conflict is not None:
+                claim_metadata = {
+                    **claim_metadata,
+                    "conflict_status": "conflicting_claim",
+                    "conflicts_with_claim_id": conflict.id,
+                    "review_required": True,
+                    "review_reason": "conflicting validated claim exists for same subject and predicate",
+                }
+            if "confidence_breakdown" in claim_metadata:
+                claim_metadata["confidence_breakdown"] = {
+                    **claim_metadata["confidence_breakdown"],
+                    "source_trust": round(source.trust_level, 4),
+                    "stored_confidence": confidence,
+                }
             claim_hash = make_claim_hash(
                 project_id=project_id,
                 source_id=source.id,
@@ -213,16 +272,25 @@ class KnowledgeRepository:
                     confidence=confidence,
                     confidence_reason=extracted_claim.confidence_reason,
                     extraction_method=bundle.extractor_name,
-                    metadata_json=extracted_claim.metadata,
+                    status=claim_status,
+                    metadata_json=claim_metadata,
                 )
                 self.session.add(claim)
                 self.session.flush()
             else:
+                existing_metadata = dict(claim.metadata_json or {})
+                existing_history = list(existing_metadata.get("source_history") or [])
                 claim.page_id = page.id
                 claim.last_seen_at = models.utcnow()
                 claim.confidence = max(claim.confidence, confidence)
                 claim.confidence_reason = extracted_claim.confidence_reason or claim.confidence_reason
-                claim.metadata_json = {**(claim.metadata_json or {}), **extracted_claim.metadata}
+                if claim_status == "active" or claim.status != "active":
+                    claim.status = claim_status
+                claim.metadata_json = {**existing_metadata, **claim_metadata}
+                claim.metadata_json["source_history"] = merge_source_history(
+                    existing_history,
+                    claim_metadata["source_history"],
+                )
             if extracted_claim.evidence_text:
                 self.session.add(
                     models.Evidence(
@@ -233,20 +301,27 @@ class KnowledgeRepository:
                         evidence_summary=extracted_claim.evidence_summary,
                     )
                 )
-            if object_entity:
+            can_merge, merge_reason = should_merge_claim_to_graph(
+                claim,
+                project_config.ontology if project_config is not None else None,
+                project_config.domain if project_config is not None else None,
+            )
+            claim.metadata_json = {
+                **(claim.metadata_json or {}),
+                "graph_merge_status": "merged" if can_merge else "skipped",
+                "graph_merge_reason": merge_reason,
+            }
+            if object_entity and can_merge:
                 self._upsert_relation(project_id, subject.id, extracted_claim.predicate, object_entity.id, confidence)
+            triple = OntologyTripleStore(self.session).upsert_from_claim(claim)
+            claim.metadata_json = {
+                **(claim.metadata_json or {}),
+                "ontology_triple_id": triple.id,
+                "ontology_triple_status": triple.status,
+            }
             claims.append(claim)
 
-        self.session.add(
-            models.ExtractionLog(
-                project_id=project_id,
-                page_id=page.id,
-                extractor_name=bundle.extractor_name,
-                provider=bundle.provider,
-                input_hash=page.content_hash,
-                raw_output=bundle.raw_output,
-            )
-        )
+        self._log_extraction(project_id, page, bundle)
         return claims
 
     def _save_extracted_entity(
@@ -255,11 +330,17 @@ class KnowledgeRepository:
         source_id: int,
         extracted_entity: ExtractedEntity,
     ) -> models.Entity:
+        normalized_type = extracted_entity.entity_type
+        normalized_name = normalize_entity_name(extracted_entity.name, normalized_type)
         entity = self.upsert_entity(
             project_id,
-            extracted_entity.entity_type,
-            extracted_entity.name,
-            extracted_entity.metadata,
+            normalized_type,
+            normalized_name,
+            {
+                **extracted_entity.metadata,
+                "raw_name": extracted_entity.name,
+                "normalization": "canonical_entity_resolver",
+            },
         )
         for name, value in extracted_entity.attributes.items():
             attribute = self.session.scalar(
@@ -292,9 +373,15 @@ class KnowledgeRepository:
         name: str,
         entity_index: dict[tuple[str, str], models.Entity],
     ) -> models.Entity:
-        key = (entity_type, canonicalize(name))
+        normalized_name = normalize_entity_name(name, entity_type)
+        key = (entity_type, canonicalize(normalized_name))
         if key not in entity_index:
-            entity_index[key] = self.upsert_entity(project_id, entity_type, name)
+            entity_index[key] = self.upsert_entity(
+                project_id,
+                entity_type,
+                normalized_name,
+                {"raw_name": name, "normalization": "canonical_entity_resolver"},
+            )
         return entity_index[key]
 
     def _upsert_relation(
@@ -329,6 +416,46 @@ class KnowledgeRepository:
             relation.updated_at = models.utcnow()
         return relation
 
+    def _find_conflicting_claim(
+        self,
+        project_id: int,
+        subject_id: int,
+        predicate: str,
+        object_entity_id: int | None,
+        object_value: Any | None,
+    ) -> models.Claim | None:
+        rows = self.session.scalars(
+            select(models.Claim).where(
+                models.Claim.project_id == project_id,
+                models.Claim.subject_entity_id == subject_id,
+                models.Claim.predicate == predicate,
+                models.Claim.status == "validated_claim",
+            )
+        ).all()
+        for row in rows:
+            if object_entity_id is not None:
+                if row.object_entity_id is not None and row.object_entity_id != object_entity_id:
+                    return row
+            elif row.object_value != object_value:
+                return row
+        return None
+
+    def _log_extraction(self, project_id: int, page: models.Page, bundle: ExtractionBundle) -> None:
+        self.session.add(
+            models.ExtractionLog(
+                project_id=project_id,
+                page_id=page.id,
+                extractor_name=bundle.extractor_name,
+                provider=bundle.provider,
+                input_hash=page.content_hash,
+                raw_output={
+                    **bundle.raw_output,
+                    "candidate_entities": [asdict(entity) for entity in bundle.entities[:50]],
+                    "candidate_claims": [asdict(claim) for claim in bundle.claims[:100]],
+                },
+            )
+        )
+
 
 def _project_config_to_dict(config: ProjectConfig) -> dict[str, Any]:
     return {
@@ -347,6 +474,17 @@ def combine_confidence(extraction_confidence: float, source_trust: float) -> flo
     return round(min(max((extraction_confidence * 0.7) + (source_trust * 0.3), 0.0), 1.0), 4)
 
 
+def merge_source_history(existing: list[dict[str, Any]], new_items: list[dict[str, Any]]) -> list[dict[str, Any]]:
+    merged = [item for item in existing if isinstance(item, dict)]
+    seen = {(item.get("source_id"), item.get("page_id"), item.get("seen_at")) for item in merged}
+    for item in new_items:
+        key = (item.get("source_id"), item.get("page_id"), item.get("seen_at"))
+        if key not in seen:
+            merged.append(item)
+            seen.add(key)
+    return merged[-50:]
+
+
 def make_claim_hash(
     project_id: int,
     source_id: int,