[crawler]

crawler_platform/app/core/database/__init__.py

@@ -0,0 +1,2 @@
+"""Database models and repositories."""
+

crawler_platform/app/core/database/models.py

@@ -0,0 +1,240 @@
+from __future__ import annotations
+
+from datetime import datetime, timezone
+
+from sqlalchemy import (
+    JSON,
+    Boolean,
+    Column,
+    DateTime,
+    Float,
+    ForeignKey,
+    Integer,
+    String,
+    Text,
+    UniqueConstraint,
+)
+from sqlalchemy.orm import declarative_base, relationship
+
+
+Base = declarative_base()
+
+
+def utcnow() -> datetime:
+    return datetime.now(timezone.utc)
+
+
+class Project(Base):
+    __tablename__ = "projects"
+
+    id = Column(Integer, primary_key=True)
+    name = Column(String(160), unique=True, nullable=False, index=True)
+    domain = Column(String(80), nullable=False, index=True)
+    config = Column(JSON, nullable=False, default=dict)
+    created_at = Column(DateTime(timezone=True), default=utcnow, nullable=False)
+    updated_at = Column(DateTime(timezone=True), default=utcnow, nullable=False)
+
+    sources = relationship("Source", back_populates="project", cascade="all, delete-orphan")
+
+
+class Source(Base):
+    __tablename__ = "sources"
+    __table_args__ = (UniqueConstraint("project_id", "name", name="uq_source_project_name"),)
+
+    id = Column(Integer, primary_key=True)
+    project_id = Column(Integer, ForeignKey("projects.id"), nullable=False, index=True)
+    name = Column(String(160), nullable=False)
+    type = Column(String(80), nullable=False, default="unknown")
+    base_url = Column(Text)
+    trust_level = Column(Float, nullable=False, default=0.5)
+    respect_robots_txt = Column(Boolean, nullable=False, default=True)
+    rate_limit_per_minute = Column(Integer, nullable=False, default=30)
+    update_policy = Column(JSON, nullable=False, default=dict)
+    created_at = Column(DateTime(timezone=True), default=utcnow, nullable=False)
+    updated_at = Column(DateTime(timezone=True), default=utcnow, nullable=False)
+
+    project = relationship("Project", back_populates="sources")
+
+
+class Page(Base):
+    __tablename__ = "pages"
+    __table_args__ = (UniqueConstraint("project_id", "url", name="uq_page_project_url"),)
+
+    id = Column(Integer, primary_key=True)
+    project_id = Column(Integer, ForeignKey("projects.id"), nullable=False, index=True)
+    source_id = Column(Integer, ForeignKey("sources.id"), nullable=False, index=True)
+    url = Column(Text, nullable=False)
+    canonical_url = Column(Text)
+    title = Column(Text)
+    content_hash = Column(String(80), index=True)
+    status_code = Column(Integer)
+    fetched_at = Column(DateTime(timezone=True), default=utcnow, nullable=False)
+    cleaned_text_summary = Column(Text)
+    raw_storage_ref = Column(Text)
+    metadata_json = Column(JSON, nullable=False, default=dict)
+
+
+class Entity(Base):
+    __tablename__ = "entities"
+    __table_args__ = (
+        UniqueConstraint("project_id", "entity_type", "canonical_name", name="uq_entity_identity"),
+    )
+
+    id = Column(Integer, primary_key=True)
+    project_id = Column(Integer, ForeignKey("projects.id"), nullable=False, index=True)
+    entity_type = Column(String(120), nullable=False, index=True)
+    name = Column(String(240), nullable=False)
+    canonical_name = Column(String(240), nullable=False, index=True)
+    external_ids = Column(JSON, nullable=False, default=dict)
+    metadata_json = Column(JSON, nullable=False, default=dict)
+    created_at = Column(DateTime(timezone=True), default=utcnow, nullable=False)
+    updated_at = Column(DateTime(timezone=True), default=utcnow, nullable=False)
+
+
+class Attribute(Base):
+    __tablename__ = "attributes"
+    __table_args__ = (
+        UniqueConstraint("entity_id", "name", "source_id", name="uq_attribute_entity_source"),
+    )
+
+    id = Column(Integer, primary_key=True)
+    project_id = Column(Integer, ForeignKey("projects.id"), nullable=False, index=True)
+    entity_id = Column(Integer, ForeignKey("entities.id"), nullable=False, index=True)
+    source_id = Column(Integer, ForeignKey("sources.id"), nullable=True, index=True)
+    name = Column(String(120), nullable=False, index=True)
+    value = Column(JSON, nullable=False)
+    confidence = Column(Float, nullable=False, default=0.5)
+    updated_at = Column(DateTime(timezone=True), default=utcnow, nullable=False)
+
+
+class Relation(Base):
+    __tablename__ = "relations"
+    __table_args__ = (
+        UniqueConstraint("project_id", "subject_entity_id", "predicate", "object_entity_id", name="uq_relation"),
+    )
+
+    id = Column(Integer, primary_key=True)
+    project_id = Column(Integer, ForeignKey("projects.id"), nullable=False, index=True)
+    subject_entity_id = Column(Integer, ForeignKey("entities.id"), nullable=False, index=True)
+    predicate = Column(String(160), nullable=False, index=True)
+    object_entity_id = Column(Integer, ForeignKey("entities.id"), nullable=False, index=True)
+    confidence = Column(Float, nullable=False, default=0.5)
+    support_count = Column(Integer, nullable=False, default=1)
+    updated_at = Column(DateTime(timezone=True), default=utcnow, nullable=False)
+
+
+class Claim(Base):
+    __tablename__ = "claims"
+    __table_args__ = (UniqueConstraint("project_id", "claim_hash", name="uq_claim_hash"),)
+
+    id = Column(Integer, primary_key=True)
+    project_id = Column(Integer, ForeignKey("projects.id"), nullable=False, index=True)
+    source_id = Column(Integer, ForeignKey("sources.id"), nullable=False, index=True)
+    page_id = Column(Integer, ForeignKey("pages.id"), nullable=True, index=True)
+    subject_entity_id = Column(Integer, ForeignKey("entities.id"), nullable=False, index=True)
+    predicate = Column(String(160), nullable=False, index=True)
+    object_entity_id = Column(Integer, ForeignKey("entities.id"), nullable=True, index=True)
+    object_value = Column(JSON, nullable=True)
+    value_type = Column(String(80), nullable=False, default="entity")
+    claim_hash = Column(String(80), nullable=False, index=True)
+    confidence = Column(Float, nullable=False, default=0.5)
+    confidence_reason = Column(Text)
+    extraction_method = Column(String(120), nullable=False, default="rule_based")
+    status = Column(String(40), nullable=False, default="active")
+    first_seen_at = Column(DateTime(timezone=True), default=utcnow, nullable=False)
+    last_seen_at = Column(DateTime(timezone=True), default=utcnow, nullable=False)
+    valid_until = Column(DateTime(timezone=True), nullable=True)
+    metadata_json = Column(JSON, nullable=False, default=dict)
+
+    evidence_items = relationship("Evidence", back_populates="claim", cascade="all, delete-orphan")
+
+
+class Evidence(Base):
+    __tablename__ = "evidence"
+
+    id = Column(Integer, primary_key=True)
+    project_id = Column(Integer, ForeignKey("projects.id"), nullable=False, index=True)
+    claim_id = Column(Integer, ForeignKey("claims.id"), nullable=False, index=True)
+    page_id = Column(Integer, ForeignKey("pages.id"), nullable=True, index=True)
+    evidence_text = Column(Text, nullable=False)
+    evidence_summary = Column(Text)
+    selector = Column(Text)
+    start_offset = Column(Integer)
+    end_offset = Column(Integer)
+    created_at = Column(DateTime(timezone=True), default=utcnow, nullable=False)
+
+    claim = relationship("Claim", back_populates="evidence_items")
+
+
+class ExtractionLog(Base):
+    __tablename__ = "extraction_logs"
+
+    id = Column(Integer, primary_key=True)
+    project_id = Column(Integer, ForeignKey("projects.id"), nullable=False, index=True)
+    page_id = Column(Integer, ForeignKey("pages.id"), nullable=True, index=True)
+    extractor_name = Column(String(160), nullable=False)
+    provider = Column(String(120), nullable=False, default="rule_based")
+    input_hash = Column(String(80))
+    raw_output = Column(JSON, nullable=False, default=dict)
+    error = Column(Text)
+    created_at = Column(DateTime(timezone=True), default=utcnow, nullable=False)
+
+
+class CrawlJob(Base):
+    __tablename__ = "crawl_jobs"
+
+    id = Column(Integer, primary_key=True)
+    project_id = Column(Integer, ForeignKey("projects.id"), nullable=False, index=True)
+    source_id = Column(Integer, ForeignKey("sources.id"), nullable=True, index=True)
+    url = Column(Text)
+    status = Column(String(40), nullable=False, default="pending")
+    priority = Column(Integer, nullable=False, default=100)
+    scheduled_at = Column(DateTime(timezone=True), default=utcnow, nullable=False)
+    started_at = Column(DateTime(timezone=True), nullable=True)
+    finished_at = Column(DateTime(timezone=True), nullable=True)
+    error = Column(Text)
+    metadata_json = Column(JSON, nullable=False, default=dict)
+
+
+class UserProfile(Base):
+    __tablename__ = "user_profiles"
+
+    id = Column(Integer, primary_key=True)
+    project_id = Column(Integer, ForeignKey("projects.id"), nullable=False, index=True)
+    external_user_id = Column(String(160), nullable=False, index=True)
+    display_name = Column(String(160))
+    context = Column(JSON, nullable=False, default=dict)
+    created_at = Column(DateTime(timezone=True), default=utcnow, nullable=False)
+    updated_at = Column(DateTime(timezone=True), default=utcnow, nullable=False)
+
+
+class UserPreference(Base):
+    __tablename__ = "user_preferences"
+
+    id = Column(Integer, primary_key=True)
+    project_id = Column(Integer, ForeignKey("projects.id"), nullable=False, index=True)
+    user_profile_id = Column(Integer, ForeignKey("user_profiles.id"), nullable=False, index=True)
+    likes = Column(JSON, nullable=False, default=list)
+    dislikes = Column(JSON, nullable=False, default=list)
+    preferred_moods = Column(JSON, nullable=False, default=list)
+    preferred_notes = Column(JSON, nullable=False, default=list)
+    avoided_notes = Column(JSON, nullable=False, default=list)
+    price_preference = Column(JSON, nullable=False, default=dict)
+    season_context = Column(String(80))
+    occasion_context = Column(String(120))
+    feedback_history = Column(JSON, nullable=False, default=list)
+    updated_at = Column(DateTime(timezone=True), default=utcnow, nullable=False)
+
+
+class FeedbackLog(Base):
+    __tablename__ = "feedback_logs"
+
+    id = Column(Integer, primary_key=True)
+    project_id = Column(Integer, ForeignKey("projects.id"), nullable=False, index=True)
+    user_profile_id = Column(Integer, ForeignKey("user_profiles.id"), nullable=False, index=True)
+    entity_id = Column(Integer, ForeignKey("entities.id"), nullable=True, index=True)
+    action = Column(String(80), nullable=False)
+    score = Column(Float)
+    reason = Column(Text)
+    metadata_json = Column(JSON, nullable=False, default=dict)
+    created_at = Column(DateTime(timezone=True), default=utcnow, nullable=False)

crawler_platform/app/core/database/repository.py

@@ -0,0 +1,333 @@
+from __future__ import annotations
+
+from dataclasses import asdict
+import hashlib
+import json
+from typing import Any
+
+from sqlalchemy import select
+from sqlalchemy.orm import Session
+
+from crawler_platform.app.config.loader import ProjectConfig, SourceConfig
+from crawler_platform.app.core.database import models
+from crawler_platform.app.core.extractor.base import ExtractedClaim, ExtractedEntity, ExtractionBundle
+
+
+def canonicalize(value: str) -> str:
+    return " ".join(value.strip().lower().split())
+
+
+def short_hash(text: str) -> str:
+    return hashlib.sha256(text.encode("utf-8")).hexdigest()
+
+
+class KnowledgeRepository:
+    def __init__(self, session: Session):
+        self.session = session
+
+    def upsert_project(self, config: ProjectConfig) -> models.Project:
+        project = self.session.scalar(select(models.Project).where(models.Project.name == config.project_name))
+        config_dict = _project_config_to_dict(config)
+        if project is None:
+            project = models.Project(name=config.project_name, domain=config.domain, config=config_dict)
+            self.session.add(project)
+            self.session.flush()
+        else:
+            project.domain = config.domain
+            project.config = config_dict
+            project.updated_at = models.utcnow()
+        for source_config in config.sources:
+            self.upsert_source(project, source_config)
+        return project
+
+    def upsert_source(self, project: models.Project, source_config: SourceConfig) -> models.Source:
+        source = self.session.scalar(
+            select(models.Source).where(
+                models.Source.project_id == project.id,
+                models.Source.name == source_config.name,
+            )
+        )
+        if source is None:
+            source = models.Source(project_id=project.id, name=source_config.name)
+            self.session.add(source)
+            self.session.flush()
+        source.type = source_config.type
+        source.base_url = source_config.base_url
+        source.trust_level = source_config.trust_level
+        source.respect_robots_txt = source_config.respect_robots_txt
+        source.rate_limit_per_minute = source_config.rate_limit_per_minute
+        source.updated_at = models.utcnow()
+        return source
+
+    def get_project(self, project_name: str) -> models.Project:
+        project = self.session.scalar(select(models.Project).where(models.Project.name == project_name))
+        if project is None:
+            raise KeyError(f"Project not found: {project_name}")
+        return project
+
+    def get_source(self, project_id: int, source_name: str) -> models.Source:
+        source = self.session.scalar(
+            select(models.Source).where(models.Source.project_id == project_id, models.Source.name == source_name)
+        )
+        if source is None:
+            raise KeyError(f"Source not found: {source_name}")
+        return source
+
+    def upsert_page(
+        self,
+        project_id: int,
+        source_id: int,
+        url: str,
+        title: str | None,
+        status_code: int | None,
+        cleaned_text: str,
+        metadata: dict[str, Any] | None = None,
+    ) -> models.Page:
+        page = self.session.scalar(select(models.Page).where(models.Page.project_id == project_id, models.Page.url == url))
+        digest = short_hash(cleaned_text)
+        summary = cleaned_text[:2000]
+        if page is None:
+            page = models.Page(project_id=project_id, source_id=source_id, url=url)
+            self.session.add(page)
+            self.session.flush()
+        page.title = title
+        page.status_code = status_code
+        page.content_hash = digest
+        page.cleaned_text_summary = summary
+        page.metadata_json = metadata or {}
+        page.fetched_at = models.utcnow()
+        return page
+
+    def upsert_entity(
+        self,
+        project_id: int,
+        entity_type: str,
+        name: str,
+        metadata: dict[str, Any] | None = None,
+    ) -> models.Entity:
+        canonical_name = canonicalize(name)
+        entity = self.session.scalar(
+            select(models.Entity).where(
+                models.Entity.project_id == project_id,
+                models.Entity.entity_type == entity_type,
+                models.Entity.canonical_name == canonical_name,
+            )
+        )
+        if entity is None:
+            entity = models.Entity(
+                project_id=project_id,
+                entity_type=entity_type,
+                name=name.strip(),
+                canonical_name=canonical_name,
+                metadata_json=metadata or {},
+            )
+            self.session.add(entity)
+            self.session.flush()
+        else:
+            entity.metadata_json = {**(entity.metadata_json or {}), **(metadata or {})}
+            entity.updated_at = models.utcnow()
+        return entity
+
+    def save_extraction_bundle(
+        self,
+        project_id: int,
+        source: models.Source,
+        page: models.Page,
+        bundle: ExtractionBundle,
+    ) -> list[models.Claim]:
+        entity_index: dict[tuple[str, str], models.Entity] = {}
+        for extracted_entity in bundle.entities:
+            entity = self._save_extracted_entity(project_id, source.id, extracted_entity)
+            entity_index[(extracted_entity.entity_type, canonicalize(extracted_entity.name))] = entity
+
+        claims: list[models.Claim] = []
+        for extracted_claim in bundle.claims:
+            subject = self._entity_for_claim(project_id, extracted_claim.subject_type, extracted_claim.subject_name, entity_index)
+            object_entity = None
+            if extracted_claim.object_name and extracted_claim.object_type:
+                object_entity = self._entity_for_claim(
+                    project_id,
+                    extracted_claim.object_type,
+                    extracted_claim.object_name,
+                    entity_index,
+                )
+            confidence = combine_confidence(extracted_claim.confidence, source.trust_level)
+            claim_hash = make_claim_hash(
+                project_id=project_id,
+                source_id=source.id,
+                subject_entity_id=subject.id,
+                predicate=extracted_claim.predicate,
+                object_entity_id=object_entity.id if object_entity else None,
+                object_value=extracted_claim.object_value,
+            )
+            claim = self.session.scalar(
+                select(models.Claim).where(
+                    models.Claim.project_id == project_id,
+                    models.Claim.claim_hash == claim_hash,
+                )
+            )
+            if claim is None:
+                claim = models.Claim(
+                    project_id=project_id,
+                    source_id=source.id,
+                    page_id=page.id,
+                    subject_entity_id=subject.id,
+                    predicate=extracted_claim.predicate,
+                    object_entity_id=object_entity.id if object_entity else None,
+                    object_value=extracted_claim.object_value,
+                    value_type="entity" if object_entity else "literal",
+                    claim_hash=claim_hash,
+                    confidence=confidence,
+                    confidence_reason=extracted_claim.confidence_reason,
+                    extraction_method=bundle.extractor_name,
+                    metadata_json=extracted_claim.metadata,
+                )
+                self.session.add(claim)
+                self.session.flush()
+            else:
+                claim.page_id = page.id
+                claim.last_seen_at = models.utcnow()
+                claim.confidence = max(claim.confidence, confidence)
+                claim.confidence_reason = extracted_claim.confidence_reason or claim.confidence_reason
+                claim.metadata_json = {**(claim.metadata_json or {}), **extracted_claim.metadata}
+            if extracted_claim.evidence_text:
+                self.session.add(
+                    models.Evidence(
+                        project_id=project_id,
+                        claim_id=claim.id,
+                        page_id=page.id,
+                        evidence_text=extracted_claim.evidence_text[:1000],
+                        evidence_summary=extracted_claim.evidence_summary,
+                    )
+                )
+            if object_entity:
+                self._upsert_relation(project_id, subject.id, extracted_claim.predicate, object_entity.id, confidence)
+            claims.append(claim)
+
+        self.session.add(
+            models.ExtractionLog(
+                project_id=project_id,
+                page_id=page.id,
+                extractor_name=bundle.extractor_name,
+                provider=bundle.provider,
+                input_hash=page.content_hash,
+                raw_output=bundle.raw_output,
+            )
+        )
+        return claims
+
+    def _save_extracted_entity(
+        self,
+        project_id: int,
+        source_id: int,
+        extracted_entity: ExtractedEntity,
+    ) -> models.Entity:
+        entity = self.upsert_entity(
+            project_id,
+            extracted_entity.entity_type,
+            extracted_entity.name,
+            extracted_entity.metadata,
+        )
+        for name, value in extracted_entity.attributes.items():
+            attribute = self.session.scalar(
+                select(models.Attribute).where(
+                    models.Attribute.entity_id == entity.id,
+                    models.Attribute.name == name,
+                    models.Attribute.source_id == source_id,
+                )
+            )
+            if attribute is None:
+                attribute = models.Attribute(
+                    project_id=project_id,
+                    entity_id=entity.id,
+                    source_id=source_id,
+                    name=name,
+                    value=value,
+                    confidence=extracted_entity.confidence,
+                )
+                self.session.add(attribute)
+            else:
+                attribute.value = value
+                attribute.confidence = max(attribute.confidence, extracted_entity.confidence)
+                attribute.updated_at = models.utcnow()
+        return entity
+
+    def _entity_for_claim(
+        self,
+        project_id: int,
+        entity_type: str,
+        name: str,
+        entity_index: dict[tuple[str, str], models.Entity],
+    ) -> models.Entity:
+        key = (entity_type, canonicalize(name))
+        if key not in entity_index:
+            entity_index[key] = self.upsert_entity(project_id, entity_type, name)
+        return entity_index[key]
+
+    def _upsert_relation(
+        self,
+        project_id: int,
+        subject_id: int,
+        predicate: str,
+        object_id: int,
+        confidence: float,
+    ) -> models.Relation:
+        relation = self.session.scalar(
+            select(models.Relation).where(
+                models.Relation.project_id == project_id,
+                models.Relation.subject_entity_id == subject_id,
+                models.Relation.predicate == predicate,
+                models.Relation.object_entity_id == object_id,
+            )
+        )
+        if relation is None:
+            relation = models.Relation(
+                project_id=project_id,
+                subject_entity_id=subject_id,
+                predicate=predicate,
+                object_entity_id=object_id,
+                confidence=confidence,
+            )
+            self.session.add(relation)
+            self.session.flush()
+        else:
+            relation.support_count += 1
+            relation.confidence = max(relation.confidence, confidence)
+            relation.updated_at = models.utcnow()
+        return relation
+
+
+def _project_config_to_dict(config: ProjectConfig) -> dict[str, Any]:
+    return {
+        "project_name": config.project_name,
+        "domain": config.domain,
+        "target_entities": config.target_entities,
+        "fields": config.fields,
+        "sources": [asdict(source) for source in config.sources],
+        "ontology": config.ontology,
+        "recommendation": config.recommendation,
+        "update_policy": config.update_policy,
+    }
+
+
+def combine_confidence(extraction_confidence: float, source_trust: float) -> float:
+    return round(min(max((extraction_confidence * 0.7) + (source_trust * 0.3), 0.0), 1.0), 4)
+
+
+def make_claim_hash(
+    project_id: int,
+    source_id: int,
+    subject_entity_id: int,
+    predicate: str,
+    object_entity_id: int | None,
+    object_value: Any | None,
+) -> str:
+    payload = {
+        "project_id": project_id,
+        "source_id": source_id,
+        "subject_entity_id": subject_entity_id,
+        "predicate": predicate,
+        "object_entity_id": object_entity_id,
+        "object_value": object_value,
+    }
+    return hashlib.sha256(json.dumps(payload, sort_keys=True, default=str).encode("utf-8")).hexdigest()

crawler_platform/app/core/database/session.py

@@ -0,0 +1,39 @@
+from __future__ import annotations
+
+from contextlib import contextmanager
+from typing import Iterator
+
+from sqlalchemy import create_engine
+from sqlalchemy.orm import Session, sessionmaker
+
+from crawler_platform.app.core.database.models import Base
+
+
+def make_engine(database_url: str = "sqlite:///crawler_platform.db"):
+    connect_args = {"check_same_thread": False} if database_url.startswith("sqlite") else {}
+    return create_engine(database_url, future=True, connect_args=connect_args)
+
+
+def init_db(database_url: str = "sqlite:///crawler_platform.db") -> None:
+    engine = make_engine(database_url)
+    Base.metadata.create_all(engine)
+
+
+def make_session_factory(database_url: str = "sqlite:///crawler_platform.db") -> sessionmaker[Session]:
+    engine = make_engine(database_url)
+    return sessionmaker(bind=engine, expire_on_commit=False, class_=Session, future=True)
+
+
+@contextmanager
+def session_scope(database_url: str = "sqlite:///crawler_platform.db") -> Iterator[Session]:
+    factory = make_session_factory(database_url)
+    session = factory()
+    try:
+        yield session
+        session.commit()
+    except Exception:
+        session.rollback()
+        raise
+    finally:
+        session.close()
+