AI/ontology_platform/crawler_platform/app/api/routes.py

from __future__ import annotations

import csv
import io
import json
from dataclasses import asdict
from typing import Any

from fastapi import BackgroundTasks, HTTPException, Response
from pydantic import BaseModel, Field
from sqlalchemy import select

from crawler_platform.app.config.loader import (
    ProjectConfig,
    SourceConfig,
    load_project_config,
    project_config_from_dict,
)
from crawler_platform.app.core.crawler.discovery import discover_links
from crawler_platform.app.core.crawler.fetchers import RobotsPolicy, make_fetcher
from crawler_platform.app.core.crawler.pipeline import CrawlPipeline
from crawler_platform.app.core.crawler.site_crawler import SiteCrawler
from crawler_platform.app.core.database import models
from crawler_platform.app.core.database.repository import (
    KnowledgeRepository,
    make_claim_hash,
)
from crawler_platform.app.core.database.session import session_scope
from crawler_platform.app.core.extractor.ai_provider import list_openai_compatible_models
from crawler_platform.app.core.extractor.factory import extractor_for_domain
from crawler_platform.app.core.ontology.definitions import DOMAIN_ONTOLOGIES, ontology_for_domain
from crawler_platform.app.core.ontology.gap_detector import KnowledgeGapDetector
from crawler_platform.app.core.ontology.mapper import ontology_to_dict
from crawler_platform.app.core.ontology.registry import OntologyRegistry
from crawler_platform.app.core.recommendation.scorer import PreferenceInput, RuleBasedRecommender
from crawler_platform.app.core.research.graph_query import SemanticGraphQuery
from crawler_platform.app.core.research.graph_research_loop import GraphResearchLoop
from crawler_platform.app.core.research.memory_store import ResearchMemoryStore, research_session_payload


class CrawlRequest(BaseModel):
    config_path: str
    source_name: str
    url: str
    extractor_provider: str = "lm_studio"
    extractor_model: str | None = None
    extractor_base_url: str | None = "http://localhost:1234/v1"
    check_robots_txt: bool = False
    respect_robots_txt: bool | None = None


class SiteCrawlRequest(CrawlRequest):
    max_depth: int = 2
    max_pages: int = 50
    same_domain_only: bool = True
    analyze_page_types: list[str] = Field(default_factory=lambda: ["ProductPage", "BrandStoryPage", "ReviewPage"])


class SiteCrawlByProjectRequest(BaseModel):
    """Site crawl invoked against an existing project (no filesystem config_path)."""

    project_name: str
    source_name: str
    url: str
    extractor_provider: str = "lm_studio"
    extractor_model: str | None = None
    extractor_base_url: str | None = "http://localhost:1234/v1"
    check_robots_txt: bool = False
    respect_robots_txt: bool | None = None
    max_depth: int = 2
    max_pages: int = 50
    same_domain_only: bool = True
    analyze_page_types: list[str] = Field(
        default_factory=lambda: ["ProductPage", "BrandStoryPage", "ReviewPage"]
    )

    def to_site_crawl_request(self, config_path_placeholder: str = "") -> "SiteCrawlRequest":
        """For internal handoff to existing crawl pipeline (config_path is not used)."""
        return SiteCrawlRequest(
            config_path=config_path_placeholder,
            source_name=self.source_name,
            url=self.url,
            extractor_provider=self.extractor_provider,
            extractor_model=self.extractor_model,
            extractor_base_url=self.extractor_base_url,
            check_robots_txt=self.check_robots_txt,
            respect_robots_txt=self.respect_robots_txt,
            max_depth=self.max_depth,
            max_pages=self.max_pages,
            same_domain_only=self.same_domain_only,
            analyze_page_types=list(self.analyze_page_types),
        )


class DiscoverRequest(BaseModel):
    config_path: str
    source_name: str
    url: str
    limit: int = 30
    check_robots_txt: bool = False
    respect_robots_txt: bool | None = None


class RecommendRequest(BaseModel):
    project_name: str
    target_entity_type: str = "Perfume"
    preferences: dict = Field(default_factory=dict)
    limit: int = 10


class CreateProjectRequest(BaseModel):
    config_path: str


class InlineSourceConfig(BaseModel):
    name: str
    type: str = "unknown"
    trust_level: float = 0.5
    base_url: str | None = None
    allowed_paths: list[str] = Field(default_factory=list)
    parser: str = "generic"
    fetcher: str = "requests"
    rate_limit_per_minute: int = 30
    respect_robots_txt: bool = False


class CreateProjectInlineRequest(BaseModel):
    """Create a project from inline JSON config (no filesystem dependency)."""

    project_name: str
    domain: str
    target_entities: list[str] = Field(default_factory=list)
    fields: list[str] = Field(default_factory=list)
    sources: list[InlineSourceConfig] = Field(default_factory=list)
    ontology: dict[str, Any] = Field(default_factory=dict)
    recommendation: dict[str, Any] = Field(default_factory=dict)
    update_policy: dict[str, Any] = Field(default_factory=dict)

    def to_project_config(self) -> ProjectConfig:
        return ProjectConfig(
            project_name=self.project_name,
            domain=self.domain,
            target_entities=list(self.target_entities),
            fields=list(self.fields),
            sources=[SourceConfig(**s.model_dump()) for s in self.sources],
            ontology=dict(self.ontology),
            recommendation=dict(self.recommendation),
            update_policy=dict(self.update_policy),
        )


class ResetProjectRequest(BaseModel):
    config_path: str
    project_name: str | None = None


def source_model_to_config(source: models.Source) -> SourceConfig:
    return SourceConfig(
        name=source.name,
        type=source.type,
        trust_level=source.trust_level,
        base_url=source.base_url,
        rate_limit_per_minute=source.rate_limit_per_minute,
        respect_robots_txt=source.respect_robots_txt,
    )


def project_config_from_project_row(session, project: models.Project) -> ProjectConfig:
    config_dict = dict(project.config or {})
    if not config_dict:
        raise HTTPException(
            status_code=400,
            detail=f"Project '{project.name}' has no stored config",
        )
    config = project_config_from_dict(config_dict)
    config.sources = [
        source_model_to_config(source)
        for source in session.scalars(
            select(models.Source).where(models.Source.project_id == project.id)
        ).all()
    ]
    return config


def project_config_to_dict(config: ProjectConfig) -> dict[str, Any]:
    return {
        "project_name": config.project_name,
        "domain": config.domain,
        "target_entities": list(config.target_entities),
        "fields": list(config.fields),
        "sources": [asdict(source) for source in config.sources],
        "ontology": dict(config.ontology),
        "recommendation": dict(config.recommendation),
        "update_policy": dict(config.update_policy),
    }


class CreateEntityRequest(BaseModel):
    entity_type: str
    name: str
    metadata: dict[str, Any] = Field(default_factory=dict)


class BulkCreateEntitiesRequest(BaseModel):
    entities: list[CreateEntityRequest]


class CreateClaimRequest(BaseModel):
    source_name: str
    subject_entity_id: int
    predicate: str
    object_entity_id: int | None = None
    object_value: Any = None
    confidence: float = 1.0
    confidence_reason: str | None = None
    evidence_text: str | None = None
    metadata: dict[str, Any] = Field(default_factory=dict)


class ResearchRunByProjectRequest(BaseModel):
    """Run autonomous research against an existing DB project (no config_path)."""

    project_name: str
    source_name: str
    url: str | None = None
    seed_entity_id: int | None = None
    goal: str = "Semantic ontology exploration"
    extractor_provider: str = "lm_studio"
    extractor_model: str | None = None
    extractor_base_url: str | None = "http://localhost:1234/v1"
    check_robots_txt: bool = False
    respect_robots_txt: bool | None = None
    max_depth: int = 2
    max_steps: int = 12
    max_branch: int = 8
    min_relevance: float = 0.35
    same_domain_only: bool = True
    analyze_page_types: list[str] = Field(
        default_factory=lambda: ["ProductPage", "BrandStoryPage", "ReviewPage"]
    )


class UpdateClaimConfidenceRequest(BaseModel):
    confidence: float
    reason: str | None = None


class UpdateClaimStatusRequest(BaseModel):
    status: str
    reason: str | None = None


class BulkClaimStatusRequest(BaseModel):
    claim_ids: list[int]
    status: str
    reason: str | None = None


class SchemaEntityTypeRequest(BaseModel):
    name: str
    domain: str = "generic"
    description: str | None = None
    status: str = "active"
    confidence: float = 1.0
    metadata: dict[str, Any] = Field(default_factory=dict)


class SchemaRelationTypeRequest(BaseModel):
    name: str
    domain: str = "generic"
    description: str | None = None
    allowed_subject_types: list[str] = Field(default_factory=list)
    allowed_object_types: list[str] = Field(default_factory=list)
    allowed_page_types: list[str] = Field(default_factory=list)
    allowed_source_zones: list[str] = Field(default_factory=list)
    semantic_constraints: dict[str, Any] = Field(default_factory=dict)
    confidence_rules: dict[str, Any] = Field(default_factory=dict)
    min_confidence: float | None = None
    status: str = "active"
    confidence: float = 1.0
    metadata: dict[str, Any] = Field(default_factory=dict)


class PipelineRerunRequest(BaseModel):
    stage: str


class MergeEntitiesRequest(BaseModel):
    project_name: str
    source_entity_id: int
    target_entity_id: int


class ExtractorModelsRequest(BaseModel):
    provider: str
    base_url: str | None = None


class ResearchRunRequest(CrawlRequest):
    project_name: str | None = None
    seed_entity_id: int | None = None
    goal: str = "Semantic ontology exploration"
    max_depth: int = 2
    max_steps: int = 12
    max_branch: int = 8
    min_relevance: float = 0.35
    same_domain_only: bool = True
    analyze_page_types: list[str] = Field(default_factory=lambda: ["ProductPage", "BrandStoryPage", "ReviewPage"])


def site_crawl_progress_payload(result, latest_page=None) -> dict[str, Any]:
    payload = asdict(result)
    if latest_page is not None:
        payload["latest_page"] = asdict(latest_page)
    return payload


_CLAIM_STATUS_ALIASES = {
    "accept": "validated_claim",
    "accepted": "validated_claim",
    "validated": "validated_claim",
    "validated_claim": "validated_claim",
    "reject": "rejected",
    "rejected": "rejected",
    "active": "active",
    "candidate": "active",
    "pending": "active",
}


def _normalize_claim_status(value: str | None) -> str | None:
    if not value:
        return None
    return _CLAIM_STATUS_ALIASES.get(value.strip().lower())


def _apply_claim_review(claim: models.Claim, status: str, reason: str | None) -> None:
    claim.status = status
    if reason:
        claim.confidence_reason = reason
    metadata = dict(claim.metadata_json or {})
    review = dict(metadata.get("review") or {})
    review["status"] = status
    review["reason"] = reason or review.get("reason")
    review["reviewed_at"] = models.utcnow().isoformat()
    metadata["review"] = review
    claim.metadata_json = metadata
    claim.last_seen_at = models.utcnow()


def turtle_id(value: Any) -> str:
    text = "".join(ch if ch.isalnum() else "_" for ch in str(value or "").strip())
    text = "_".join(part for part in text.split("_") if part)
    if not text:
        return "value"
    if text[0].isdigit():
        text = f"n_{text}"
    return text[:120]


def apply_crawl_request_overrides(config, request: CrawlRequest | DiscoverRequest) -> None:
    check_robots_txt = request.respect_robots_txt
    if check_robots_txt is None:
        check_robots_txt = request.check_robots_txt
    config.source_by_name(request.source_name).respect_robots_txt = check_robots_txt


def crawl_job_response(job: models.CrawlJob) -> dict[str, Any]:
    metadata = job.metadata_json or {}
    return {
        "job_id": job.id,
        "status": job.status,
        "url": job.url,
        "error": job.error,
        "scheduled_at": job.scheduled_at.isoformat() if job.scheduled_at else None,
        "started_at": job.started_at.isoformat() if job.started_at else None,
        "finished_at": job.finished_at.isoformat() if job.finished_at else None,
        "progress": metadata.get("progress", {}),
        "request": metadata.get("request", {}),
    }


def update_site_crawl_job_metadata(job: models.CrawlJob, **updates: Any) -> None:
    metadata = dict(job.metadata_json or {})
    metadata.update(updates)
    job.metadata_json = metadata


def is_site_crawl_cancel_requested(session, job_id: int) -> bool:
    session.expire_all()
    job = session.get(models.CrawlJob, job_id)
    return job is None or job.status == "cancel_requested"


def run_site_crawl_job(database_url: str, job_id: int, request_data: dict[str, Any]) -> None:
    inline_config = request_data.pop("__config_dict", None)
    request = SiteCrawlRequest(**request_data)
    try:
        if inline_config is not None:
            config = project_config_from_dict(inline_config)
        else:
            config = load_project_config(request.config_path)
        apply_crawl_request_overrides(config, request)
        with session_scope(database_url) as session:
            job = session.get(models.CrawlJob, job_id)
            if job is None:
                return
            if job.status == "cancel_requested":
                job.status = "canceled"
                job.finished_at = models.utcnow()
                return

            job.status = "running"
            job.started_at = models.utcnow()
            update_site_crawl_job_metadata(
                job,
                progress={
                    "seed_url": request.url,
                    "visited_count": 0,
                    "analyzed_count": 0,
                    "queued_count": 1,
                    "skipped_count": 0,
                    "errors": [],
                    "pages": [],
                },
            )
            session.commit()

            repo = KnowledgeRepository(session)
            crawler = SiteCrawler(
                repo,
                extractor_for_domain(
                    config.domain,
                    provider=request.extractor_provider,
                    model=request.extractor_model,
                    base_url=request.extractor_base_url,
                ),
            )

            def save_progress(result, latest_page) -> None:
                current_job = session.get(models.CrawlJob, job_id)
                if current_job is None:
                    return
                update_site_crawl_job_metadata(
                    current_job,
                    progress=site_crawl_progress_payload(result, latest_page),
                )
                session.commit()

            def should_stop() -> bool:
                return is_site_crawl_cancel_requested(session, job_id)

            result = crawler.crawl_site(
                config,
                request.source_name,
                request.url,
                max_depth=max(request.max_depth, 0),
                max_pages=max(min(request.max_pages, 500), 1),
                same_domain_only=request.same_domain_only,
                analyze_page_types=set(request.analyze_page_types),
                progress_callback=save_progress,
                should_stop=should_stop,
                parent_job_id=job_id,
            )

            finished_job = session.get(models.CrawlJob, job_id)
            if finished_job is not None:
                finished_job.status = "canceled" if finished_job.status == "cancel_requested" else "completed"
                finished_job.finished_at = models.utcnow()
                update_site_crawl_job_metadata(
                    finished_job,
                    progress=site_crawl_progress_payload(result),
                )
    except Exception as exc:
        with session_scope(database_url) as session:
            job = session.get(models.CrawlJob, job_id)
            if job is not None:
                job.status = "failed"
                job.error = str(exc)
                job.finished_at = models.utcnow()
                metadata = dict(job.metadata_json or {})
                progress = dict(metadata.get("progress") or {})
                progress["errors"] = [*progress.get("errors", []), str(exc)]
                update_site_crawl_job_metadata(job, progress=progress)


def register_routes(app, database_url: str) -> None:
    @app.get("/health")
    def health():
        return {"ok": True}

    @app.get("/projects")
    def projects():
        with session_scope(database_url) as session:
            rows = session.scalars(select(models.Project).order_by(models.Project.created_at.desc())).all()
            return [
                {
                    "id": project.id,
                    "name": project.name,
                    "domain": project.domain,
                    "created_at": project.created_at.isoformat(),
                    "updated_at": project.updated_at.isoformat(),
                }
                for project in rows
            ]

    @app.post("/projects")
    def create_project(request: CreateProjectRequest):
        config = load_project_config(request.config_path)
        with session_scope(database_url) as session:
            project = KnowledgeRepository(session).upsert_project(config)
            return {"id": project.id, "name": project.name, "domain": project.domain}

    @app.post("/projects/inline")
    def create_project_inline(request: CreateProjectInlineRequest):
        config = request.to_project_config()
        with session_scope(database_url) as session:
            project = KnowledgeRepository(session).upsert_project(config)
            return {"id": project.id, "name": project.name, "domain": project.domain}

    @app.get("/domains")
    def list_domains():
        """Available pre-defined ontology domains for project creation."""
        return [
            {
                "domain": ont.domain,
                "entity_types": list(ont.entity_types),
                "predicates": list(ont.predicates),
                "attribute_count": len(ont.attributes),
            }
            for ont in DOMAIN_ONTOLOGIES.values()
        ]

    @app.post("/projects/reset")
    def reset_project(request: ResetProjectRequest):
        config = load_project_config(request.config_path)
        if request.project_name and request.project_name != config.project_name:
            raise HTTPException(
                status_code=400,
                detail=(
                    f"Selected project '{request.project_name}' does not match "
                    f"config project '{config.project_name}'."
                ),
            )

        with session_scope(database_url) as session:
            repo = KnowledgeRepository(session)
            project = session.scalar(select(models.Project).where(models.Project.name == config.project_name))
            if project is None:
                project = repo.upsert_project(config)
                return {
                    "ok": True,
                    "name": project.name,
                    "domain": project.domain,
                    "created": True,
                    "reset": False,
                    "deleted": {},
                }

            deleted = repo.reset_project_runtime_data(project.id)
            project = repo.upsert_project(config)
            return {
                "ok": True,
                "name": project.name,
                "domain": project.domain,
                "created": False,
                "reset": True,
                "deleted": deleted,
            }

    @app.get("/projects/{project_name}")
    def project_detail(project_name: str):
        with session_scope(database_url) as session:
            project = KnowledgeRepository(session).get_project(project_name)
            sources = session.scalars(select(models.Source).where(models.Source.project_id == project.id)).all()
            return {
                "id": project.id,
                "name": project.name,
                "domain": project.domain,
                "config": project_config_to_dict(project_config_from_project_row(session, project)),
                "sources": [
                    {
                        "id": source.id,
                        "name": source.name,
                        "type": source.type,
                        "base_url": source.base_url,
                        "trust_level": source.trust_level,
                        "respect_robots_txt": source.respect_robots_txt,
                        "rate_limit_per_minute": source.rate_limit_per_minute,
                    }
                    for source in sources
                ],
            }

    @app.post("/projects/{project_name}/sources")
    def add_project_source(project_name: str, request: InlineSourceConfig):
        """Add or update a source on an existing project."""
        with session_scope(database_url) as session:
            repo = KnowledgeRepository(session)
            project = repo.get_project(project_name)
            source = repo.upsert_source(project, SourceConfig(**request.model_dump()))
            return {
                "id": source.id,
                "name": source.name,
                "type": source.type,
                "base_url": source.base_url,
                "trust_level": source.trust_level,
                "respect_robots_txt": source.respect_robots_txt,
                "rate_limit_per_minute": source.rate_limit_per_minute,
            }

    @app.delete("/projects/{project_name}/sources/{source_name}")
    def delete_project_source(project_name: str, source_name: str):
        with session_scope(database_url) as session:
            repo = KnowledgeRepository(session)
            project = repo.get_project(project_name)
            source = session.scalar(
                select(models.Source).where(
                    models.Source.project_id == project.id,
                    models.Source.name == source_name,
                )
            )
            if source is None:
                raise HTTPException(
                    status_code=404,
                    detail=f"Source '{source_name}' not found in project '{project_name}'",
                )
            session.delete(source)
            return {"ok": True, "deleted": source_name}

    @app.get("/ontology/{domain}")
    def ontology(domain: str):
        return ontology_to_dict(ontology_for_domain(domain))

    @app.get("/projects/{project_name}/ontology/registry")
    def ontology_registry(project_name: str):
        with session_scope(database_url) as session:
            project = KnowledgeRepository(session).get_project(project_name)
            return OntologyRegistry(session).registry_payload(project.id)

    @app.post("/projects/{project_name}/schema/entity-types")
    def create_schema_entity_type(project_name: str, request: SchemaEntityTypeRequest):
        with session_scope(database_url) as session:
            project = KnowledgeRepository(session).get_project(project_name)
            row = OntologyRegistry(session).upsert_entity_type(
                project.id,
                name=request.name,
                domain=request.domain,
                description=request.description,
                status=request.status,
                confidence=min(max(request.confidence, 0.0), 1.0),
                metadata={"origin": "manual_schema_designer", **request.metadata},
            )
            return {
                "id": row.id,
                "name": row.name,
                "domain": row.domain,
                "description": row.description,
                "status": row.status,
                "confidence": row.confidence,
            }

    @app.post("/projects/{project_name}/schema/relation-types")
    def create_schema_relation_type(project_name: str, request: SchemaRelationTypeRequest):
        confidence_rules = dict(request.confidence_rules or {})
        if request.min_confidence is not None:
            confidence_rules["min_confidence"] = min(max(request.min_confidence, 0.0), 1.0)
        with session_scope(database_url) as session:
            project = KnowledgeRepository(session).get_project(project_name)
            row = OntologyRegistry(session).upsert_relation_type(
                project.id,
                name=request.name,
                domain=request.domain,
                description=request.description,
                allowed_subject_types=request.allowed_subject_types,
                allowed_object_types=request.allowed_object_types,
                allowed_page_types=request.allowed_page_types,
                allowed_source_zones=request.allowed_source_zones,
                semantic_constraints=request.semantic_constraints,
                confidence_rules=confidence_rules,
                status=request.status,
                confidence=min(max(request.confidence, 0.0), 1.0),
                metadata={"origin": "manual_schema_designer", **request.metadata},
            )
            return {
                "id": row.id,
                "name": row.name,
                "domain": row.domain,
                "description": row.description,
                "allowed_subject_types": row.allowed_subject_types or [],
                "allowed_object_types": row.allowed_object_types or [],
                "status": row.status,
                "confidence": row.confidence,
            }

    @app.get("/projects/{project_name}/ontology/proposals")
    def ontology_proposals(project_name: str, limit: int = 100):
        with session_scope(database_url) as session:
            project = KnowledgeRepository(session).get_project(project_name)
            rows = session.scalars(
                select(models.OntologyProposal)
                .where(models.OntologyProposal.project_id == project.id)
                .order_by(models.OntologyProposal.updated_at.desc())
                .limit(max(min(limit, 300), 1))
            ).all()
            return [
                {
                    "id": row.id,
                    "proposal_type": row.proposal_type,
                    "name": row.name,
                    "reason": row.reason,
                    "evidence": row.evidence,
                    "status": row.status,
                    "confidence": row.confidence,
                    "metadata": row.metadata_json or {},
                    "updated_at": row.updated_at.isoformat() if row.updated_at else None,
                }
                for row in rows
            ]

    @app.get("/projects/{project_name}/ontology/triples")
    def ontology_triples(project_name: str, status: str | None = None, limit: int = 100):
        with session_scope(database_url) as session:
            project = KnowledgeRepository(session).get_project(project_name)
            query = select(models.OntologyTriple).where(models.OntologyTriple.project_id == project.id)
            if status:
                query = query.where(models.OntologyTriple.status == status)
            rows = session.scalars(
                query.order_by(models.OntologyTriple.last_seen_at.desc()).limit(max(min(limit, 300), 1))
            ).all()
            results = []
            for row in rows:
                subject = session.get(models.Entity, row.subject_entity_id)
                object_entity = session.get(models.Entity, row.object_entity_id) if row.object_entity_id else None
                results.append(
                    {
                        "id": row.id,
                        "claim_id": row.claim_id,
                        "subject": subject.name if subject else None,
                        "subject_type": row.subject_type,
                        "predicate": row.predicate,
                        "object": object_entity.name if object_entity else None,
                        "object_type": row.object_type,
                        "object_value": row.object_value,
                        "value_type": row.value_type,
                        "status": row.status,
                        "confidence": row.confidence,
                        "support_count": row.support_count,
                        "metadata": row.metadata_json or {},
                        "last_seen_at": row.last_seen_at.isoformat() if row.last_seen_at else None,
                    }
                )
            return results

    @app.get("/projects/{project_name}/export")
    def export_ontology(
        project_name: str,
        format: str = "json",
        status: str = "validated_claim",
        include_evidence: bool = True,
        limit: int = 1000,
    ):
        fmt = format.strip().lower()
        bounded_limit = max(min(limit, 5000), 1)
        with session_scope(database_url) as session:
            project = KnowledgeRepository(session).get_project(project_name)
            query = (
                select(models.Claim, models.Source, models.Page, models.Entity)
                .join(models.Source, models.Claim.source_id == models.Source.id)
                .join(models.Page, models.Claim.page_id == models.Page.id, isouter=True)
                .join(models.Entity, models.Claim.subject_entity_id == models.Entity.id)
                .where(models.Claim.project_id == project.id)
            )
            if status and status != "all":
                normalized = _normalize_claim_status(status) or status
                query = query.where(models.Claim.status == normalized)
            rows = session.execute(
                query.order_by(models.Claim.confidence.desc(), models.Claim.last_seen_at.desc()).limit(bounded_limit)
            ).all()
            items: list[dict[str, Any]] = []
            for claim, source, page, subject in rows:
                object_entity = session.get(models.Entity, claim.object_entity_id) if claim.object_entity_id else None
                evidence = None
                if include_evidence:
                    evidence = session.scalar(
                        select(models.Evidence)
                        .where(models.Evidence.claim_id == claim.id)
                        .order_by(models.Evidence.created_at.desc())
                    )
                items.append(
                    {
                        "claim_id": claim.id,
                        "subject": subject.name,
                        "subject_type": subject.entity_type,
                        "predicate": claim.predicate,
                        "object": object_entity.name if object_entity else claim.object_value,
                        "object_type": object_entity.entity_type if object_entity else claim.value_type,
                        "status": claim.status,
                        "confidence": claim.confidence,
                        "source": source.name,
                        "source_url": page.url if page else None,
                        "evidence_text": evidence.evidence_text if evidence else None,
                        "created_by": claim.extraction_method,
                        "last_seen_at": claim.last_seen_at.isoformat() if claim.last_seen_at else None,
                    }
                )

        filename = f"{project_name}_ontology.{fmt if fmt != 'turtle' else 'ttl'}"
        if fmt == "json":
            return {
                "project": project_name,
                "status": status,
                "count": len(items),
                "claims": items,
            }
        if fmt == "csv":
            output = io.StringIO()
            fieldnames = [
                "claim_id",
                "subject",
                "subject_type",
                "predicate",
                "object",
                "object_type",
                "status",
                "confidence",
                "source",
                "source_url",
                "evidence_text",
                "created_by",
                "last_seen_at",
            ]
            writer = csv.DictWriter(output, fieldnames=fieldnames)
            writer.writeheader()
            for item in items:
                row = {key: json.dumps(value, ensure_ascii=False) if isinstance(value, (dict, list)) else value for key, value in item.items()}
                writer.writerow(row)
            return Response(
                content=output.getvalue(),
                media_type="text/csv; charset=utf-8",
                headers={"Content-Disposition": f'attachment; filename="{filename}"'},
            )
        if fmt in {"ttl", "turtle"}:
            lines = [
                "@prefix ont: <https://example.local/ontology/> .",
                "@prefix claim: <https://example.local/claim/> .",
                "@prefix xsd: <http://www.w3.org/2001/XMLSchema#> .",
                "",
            ]
            for item in items:
                sid = turtle_id(item["subject"])
                predicate = turtle_id(item["predicate"])
                obj = item["object"]
                if isinstance(obj, str) and obj.strip():
                    object_repr = f'ont:{turtle_id(obj)}'
                else:
                    object_repr = json.dumps(obj, ensure_ascii=False)
                lines.extend(
                    [
                        f"ont:{sid} ont:{predicate} {object_repr} .",
                        f"claim:c{item['claim_id']} ont:confidence \"{item['confidence']}\"^^xsd:decimal .",
                    ]
                )
                if item.get("source_url"):
                    lines.append(f"claim:c{item['claim_id']} ont:sourceUrl {json.dumps(item['source_url'], ensure_ascii=False)} .")
                if item.get("evidence_text"):
                    lines.append(f"claim:c{item['claim_id']} ont:evidenceText {json.dumps(item['evidence_text'], ensure_ascii=False)} .")
                lines.append("")
            return Response(
                content="\n".join(lines),
                media_type="text/turtle; charset=utf-8",
                headers={"Content-Disposition": f'attachment; filename="{filename}"'},
            )
        raise HTTPException(status_code=400, detail=f"unsupported export format: {format}")

    @app.get("/projects/{project_name}/knowledge-gaps")
    def knowledge_gaps(project_name: str, limit: int = 100):
        with session_scope(database_url) as session:
            project = KnowledgeRepository(session).get_project(project_name)
            return KnowledgeGapDetector(session).list_open(project.id, max(min(limit, 300), 1))

    @app.post("/extractors/models")
    def extractor_models(request: ExtractorModelsRequest):
        try:
            if request.provider == "lm_studio":
                models = list_openai_compatible_models(request.base_url or "http://localhost:1234/v1")
                return {"ok": True, "models": models}
            if request.provider == "openai":
                import os

                models = list_openai_compatible_models(
                    request.base_url or "https://api.openai.com/v1",
                    os.getenv("OPENAI_API_KEY"),
                )
                return {"ok": True, "models": models}
            if request.provider == "ollama":
                return {"ok": False, "error": "Ollama model listing is not implemented yet. Enter the model manually."}
            return {"ok": True, "models": [{"id": "rule_based", "owned_by": "local"}]}
        except Exception as exc:
            return {"ok": False, "error": str(exc), "models": []}

    @app.post("/crawl")
    def crawl(request: CrawlRequest):
        config = load_project_config(request.config_path)
        apply_crawl_request_overrides(config, request)
        with session_scope(database_url) as session:
            repo = KnowledgeRepository(session)
            pipeline = CrawlPipeline(
                repo,
                extractor_for_domain(
                    config.domain,
                    provider=request.extractor_provider,
                    model=request.extractor_model,
                    base_url=request.extractor_base_url,
                ),
            )
            try:
                result = pipeline.crawl_url(config, request.source_name, request.url)
            except Exception as exc:
                raise HTTPException(status_code=400, detail=str(exc)) from exc
            return {
                "page_id": result.page_id,
                "claim_count": result.claim_count,
                "entity_count": result.entity_count,
                "crawl_status": result.crawl_status,
                "extraction_status": result.extraction_status,
                "page_type": result.page_type,
                "raw_text_length": result.raw_text_length,
                "clean_text_length": result.clean_text_length,
                "warnings": result.warnings or [],
            }

    @app.post("/crawl-site")
    def crawl_site(request: SiteCrawlRequest, background_tasks: BackgroundTasks):
        config = load_project_config(request.config_path)
        apply_crawl_request_overrides(config, request)
        with session_scope(database_url) as session:
            repo = KnowledgeRepository(session)
            project = repo.upsert_project(config)
            source = repo.get_source(project.id, request.source_name)
            job = models.CrawlJob(
                project_id=project.id,
                source_id=source.id,
                url=request.url,
                status="pending",
                metadata_json={
                    "kind": "site_crawl",
                    "request": request.model_dump(),
                    "progress": {
                        "seed_url": request.url,
                        "visited_count": 0,
                        "analyzed_count": 0,
                        "queued_count": 1,
                        "skipped_count": 0,
                        "errors": [],
                        "pages": [],
                    },
                },
            )
            session.add(job)
            session.flush()
            response = crawl_job_response(job)
        background_tasks.add_task(run_site_crawl_job, database_url, response["job_id"], request.model_dump())
        return response

    @app.post("/crawl-site/by-project")
    def crawl_site_by_project(
        request: SiteCrawlByProjectRequest, background_tasks: BackgroundTasks
    ):
        """Start a site crawl against an existing DB project (no config_path)."""
        with session_scope(database_url) as session:
            repo = KnowledgeRepository(session)
            project = repo.get_project(request.project_name)
            config = project_config_from_project_row(session, project)
            try:
                source = repo.get_source(project.id, request.source_name)
                config.source_by_name(request.source_name)
            except KeyError as exc:
                raise HTTPException(status_code=404, detail=str(exc)) from exc

            inner_request = request.to_site_crawl_request()
            apply_crawl_request_overrides(config, inner_request)

            job = models.CrawlJob(
                project_id=project.id,
                source_id=source.id,
                url=request.url,
                status="pending",
                metadata_json={
                    "kind": "site_crawl",
                    "request": inner_request.model_dump(),
                    "project_name": request.project_name,
                    "progress": {
                        "seed_url": request.url,
                        "visited_count": 0,
                        "analyzed_count": 0,
                        "queued_count": 1,
                        "skipped_count": 0,
                        "errors": [],
                        "pages": [],
                    },
                },
            )
            session.add(job)
            session.flush()
            response = crawl_job_response(job)

        task_payload = {
            **inner_request.model_dump(),
            "__config_dict": project_config_to_dict(config),
        }
        background_tasks.add_task(
            run_site_crawl_job, database_url, response["job_id"], task_payload
        )
        return response

    @app.get("/crawl-site/jobs/{job_id}")
    def crawl_site_job(job_id: int):
        with session_scope(database_url) as session:
            job = session.get(models.CrawlJob, job_id)
            if job is None or (job.metadata_json or {}).get("kind") != "site_crawl":
                raise HTTPException(status_code=404, detail="site crawl job not found")
            return crawl_job_response(job)

    @app.post("/crawl-site/jobs/{job_id}/cancel")
    def cancel_crawl_site_job(job_id: int):
        with session_scope(database_url) as session:
            job = session.get(models.CrawlJob, job_id)
            if job is None or (job.metadata_json or {}).get("kind") != "site_crawl":
                raise HTTPException(status_code=404, detail="site crawl job not found")
            if job.status in {"completed", "failed", "canceled"}:
                return crawl_job_response(job)
            job.status = "cancel_requested"
            job.error = "cancel requested by user"
            return crawl_job_response(job)

    @app.post("/discover")
    def discover(request: DiscoverRequest):
        config = load_project_config(request.config_path)
        apply_crawl_request_overrides(config, request)
        source_config = config.source_by_name(request.source_name)
        robots = RobotsPolicy()
        robots_decision = robots.check(request.url, source_config.respect_robots_txt)
        if not robots_decision.allowed:
            return {
                "ok": False,
                "error": f"{robots_decision.reason}: {request.url}",
                "robots_status": robots_decision.status,
                "robots_reason": robots_decision.reason,
                "links": [],
            }
        fetcher = make_fetcher(source_config.fetcher, source_config.rate_limit_per_minute)
        result = fetcher.fetch(request.url)
        links = discover_links(result.analysis_html, result.final_url or request.url, request.limit)
        return {
            "ok": True,
            "status_code": result.status_code,
            "final_url": result.final_url,
            "crawl_status": result.crawl_status,
            "robots_status": robots_decision.status,
            "robots_reason": robots_decision.reason,
            "warnings": result.warnings,
            "links": [asdict(link) for link in links],
        }

    @app.post("/research/run")
    def run_research(request: ResearchRunRequest):
        config = load_project_config(request.config_path)
        apply_crawl_request_overrides(config, request)
        if request.project_name and request.project_name != config.project_name:
            raise HTTPException(
                status_code=400,
                detail=f"Selected project '{request.project_name}' does not match config project '{config.project_name}'.",
            )
        with session_scope(database_url) as session:
            repo = KnowledgeRepository(session)
            loop = GraphResearchLoop(
                repo,
                extractor_for_domain(
                    config.domain,
                    provider=request.extractor_provider,
                    model=request.extractor_model,
                    base_url=request.extractor_base_url,
                ),
            )
            try:
                result = loop.run(
                    project_config=config,
                    source_name=request.source_name,
                    seed_url=request.url or None,
                    seed_entity_id=request.seed_entity_id,
                    goal=request.goal,
                    max_depth=max(request.max_depth, 0),
                    max_steps=max(min(request.max_steps, 50), 1),
                    max_branch=max(min(request.max_branch, 30), 1),
                    min_relevance=min(max(request.min_relevance, 0.0), 1.0),
                    same_domain_only=request.same_domain_only,
                    analyze_page_types=set(request.analyze_page_types),
                )
            except Exception as exc:
                raise HTTPException(status_code=400, detail=str(exc)) from exc
            return asdict(result)

    @app.post("/research/run/by-project")
    def run_research_by_project(request: ResearchRunByProjectRequest):
        """Run research against an existing DB project (no config_path)."""
        with session_scope(database_url) as session:
            repo = KnowledgeRepository(session)
            project = repo.get_project(request.project_name)
            config = project_config_from_project_row(session, project)
            try:
                source_config = config.source_by_name(request.source_name)
            except KeyError as exc:
                raise HTTPException(status_code=404, detail=str(exc)) from exc

            check_robots = request.respect_robots_txt
            if check_robots is None:
                check_robots = request.check_robots_txt
            source_config.respect_robots_txt = check_robots

            loop = GraphResearchLoop(
                repo,
                extractor_for_domain(
                    config.domain,
                    provider=request.extractor_provider,
                    model=request.extractor_model,
                    base_url=request.extractor_base_url,
                ),
            )
            try:
                result = loop.run(
                    project_config=config,
                    source_name=request.source_name,
                    seed_url=request.url or None,
                    seed_entity_id=request.seed_entity_id,
                    goal=request.goal,
                    max_depth=max(request.max_depth, 0),
                    max_steps=max(min(request.max_steps, 50), 1),
                    max_branch=max(min(request.max_branch, 30), 1),
                    min_relevance=min(max(request.min_relevance, 0.0), 1.0),
                    same_domain_only=request.same_domain_only,
                    analyze_page_types=set(request.analyze_page_types),
                )
            except Exception as exc:
                raise HTTPException(status_code=400, detail=str(exc)) from exc
            return asdict(result)

    @app.get("/projects/{project_name}/research/sessions")
    def research_sessions(project_name: str, limit: int = 25):
        with session_scope(database_url) as session:
            project = KnowledgeRepository(session).get_project(project_name)
            return ResearchMemoryStore(session).list_sessions(project.id, limit)

    @app.get("/research/sessions/{job_id}")
    def research_session(job_id: int):
        with session_scope(database_url) as session:
            job = session.get(models.CrawlJob, job_id)
            if job is None or (job.metadata_json or {}).get("kind") != "research_session":
                raise HTTPException(status_code=404, detail="research session not found")
            return research_session_payload(job)

    @app.get("/projects/{project_name}/graph/neighborhood")
    def graph_neighborhood(
        project_name: str,
        entity_id: int | None = None,
        limit: int = 120,
        include_candidates: bool = False,
        status: str | None = None,
    ):
        with session_scope(database_url) as session:
            project = KnowledgeRepository(session).get_project(project_name)
            if status and status != "all":
                normalized = _normalize_claim_status(status) or status
                statuses = [normalized]
            elif status == "all":
                statuses = None
            elif include_candidates:
                statuses = ["validated_claim", "active", "candidate_claim", "rule_candidate"]
            else:
                statuses = ["validated_claim"]
            return SemanticGraphQuery(session).neighborhood(
                project.id,
                entity_id,
                max(min(limit, 300), 1),
                statuses=statuses,
            )

    @app.get("/projects/{project_name}/graph/query")
    def graph_query(
        project_name: str,
        kind: str = "trend_summary",
        predicate: str = "hasAccord",
        tag: str | None = None,
        brand: str | None = None,
        limit: int = 100,
    ):
        with session_scope(database_url) as session:
            project = KnowledgeRepository(session).get_project(project_name)
            graph = SemanticGraphQuery(session)
            bounded_limit = max(min(limit, 300), 1)
            if kind == "brand_products":
                return graph.brand_products(project.id, brand, bounded_limit)
            if kind == "products_by_tag":
                return graph.products_by_tag(project.id, predicate, tag, bounded_limit)
            if kind == "relation_summary":
                return graph.relation_summary(project.id, bounded_limit)
            if kind == "entity_type_summary":
                return graph.entity_type_summary(project.id, bounded_limit)
            if kind == "trend_summary":
                return graph.trend_summary(project.id, bounded_limit)
            raise HTTPException(status_code=400, detail=f"unknown graph query kind: {kind}")

    @app.get("/projects/{project_name}/entities")
    def project_entities(project_name: str, entity_type: str | None = None, limit: int = 50):
        with session_scope(database_url) as session:
            project = KnowledgeRepository(session).get_project(project_name)
            query = select(models.Entity).where(models.Entity.project_id == project.id)
            if entity_type:
                query = query.where(models.Entity.entity_type == entity_type)
            entities = session.scalars(query.limit(limit)).all()
            return [
                {
                    "id": entity.id,
                    "type": entity.entity_type,
                    "name": entity.name,
                    "metadata": entity.metadata_json,
                }
                for entity in entities
            ]

    @app.post("/projects/{project_name}/entities")
    def create_entity(project_name: str, request: CreateEntityRequest):
        """Create or update a single entity directly (no extraction)."""
        with session_scope(database_url) as session:
            repo = KnowledgeRepository(session)
            project = repo.get_project(project_name)
            entity = repo.upsert_entity(
                project_id=project.id,
                entity_type=request.entity_type,
                name=request.name,
                metadata={
                    **request.metadata,
                    "input_method": request.metadata.get("input_method", "manual"),
                },
            )
            return {
                "id": entity.id,
                "type": entity.entity_type,
                "name": entity.name,
                "metadata": entity.metadata_json,
            }

    @app.post("/projects/{project_name}/entities/bulk")
    def bulk_create_entities(project_name: str, request: BulkCreateEntitiesRequest):
        """Create multiple entities in one call."""
        with session_scope(database_url) as session:
            repo = KnowledgeRepository(session)
            project = repo.get_project(project_name)
            created: list[dict[str, Any]] = []
            for item in request.entities:
                entity = repo.upsert_entity(
                    project_id=project.id,
                    entity_type=item.entity_type,
                    name=item.name,
                    metadata={
                        **item.metadata,
                        "input_method": item.metadata.get("input_method", "manual"),
                    },
                )
                created.append(
                    {
                        "id": entity.id,
                        "type": entity.entity_type,
                        "name": entity.name,
                    }
                )
            return {"created": len(created), "entities": created}

    @app.delete("/projects/{project_name}/entities/{entity_id}")
    def delete_entity(project_name: str, entity_id: int):
        with session_scope(database_url) as session:
            repo = KnowledgeRepository(session)
            project = repo.get_project(project_name)
            entity = session.get(models.Entity, entity_id)
            if entity is None or entity.project_id != project.id:
                raise HTTPException(
                    status_code=404,
                    detail=f"Entity {entity_id} not found in project '{project_name}'",
                )
            session.delete(entity)
            return {"ok": True, "deleted": entity_id}

    @app.post("/projects/{project_name}/claims")
    def create_claim(project_name: str, request: CreateClaimRequest):
        """Create a single claim directly (manual input)."""
        with session_scope(database_url) as session:
            repo = KnowledgeRepository(session)
            project = repo.get_project(project_name)
            try:
                source = repo.get_source(project.id, request.source_name)
            except KeyError as exc:
                raise HTTPException(status_code=404, detail=str(exc)) from exc

            subject = session.get(models.Entity, request.subject_entity_id)
            if subject is None or subject.project_id != project.id:
                raise HTTPException(
                    status_code=404,
                    detail=f"Subject entity {request.subject_entity_id} not found",
                )
            object_entity: models.Entity | None = None
            if request.object_entity_id is not None:
                object_entity = session.get(models.Entity, request.object_entity_id)
                if object_entity is None or object_entity.project_id != project.id:
                    raise HTTPException(
                        status_code=404,
                        detail=f"Object entity {request.object_entity_id} not found",
                    )

            claim_hash = make_claim_hash(
                project_id=project.id,
                source_id=source.id,
                subject_entity_id=subject.id,
                predicate=request.predicate,
                object_entity_id=object_entity.id if object_entity else None,
                object_value=request.object_value,
            )
            existing = session.scalar(
                select(models.Claim).where(
                    models.Claim.project_id == project.id,
                    models.Claim.claim_hash == claim_hash,
                )
            )
            if existing is not None:
                existing.confidence = max(existing.confidence, request.confidence)
                existing.last_seen_at = models.utcnow()
                if request.confidence_reason:
                    existing.confidence_reason = request.confidence_reason
                existing.metadata_json = {
                    **(existing.metadata_json or {}),
                    **request.metadata,
                    "input_method": "manual",
                }
                claim = existing
            else:
                claim = models.Claim(
                    project_id=project.id,
                    source_id=source.id,
                    page_id=None,
                    subject_entity_id=subject.id,
                    predicate=request.predicate,
                    object_entity_id=object_entity.id if object_entity else None,
                    object_value=request.object_value,
                    value_type="entity" if object_entity else "literal",
                    claim_hash=claim_hash,
                    confidence=max(0.0, min(1.0, request.confidence)),
                    confidence_reason=request.confidence_reason,
                    extraction_method="manual",
                    status="validated_claim",
                    metadata_json={**request.metadata, "input_method": "manual"},
                )
                session.add(claim)
                session.flush()
            if request.evidence_text:
                session.add(
                    models.Evidence(
                        project_id=project.id,
                        claim_id=claim.id,
                        page_id=None,
                        evidence_text=request.evidence_text,
                    )
                )
            return {
                "id": claim.id,
                "subject_entity_id": claim.subject_entity_id,
                "predicate": claim.predicate,
                "object_entity_id": claim.object_entity_id,
                "object_value": claim.object_value,
                "confidence": claim.confidence,
                "status": claim.status,
            }

    @app.delete("/projects/{project_name}/claims/{claim_id}")
    def delete_claim(project_name: str, claim_id: int):
        with session_scope(database_url) as session:
            repo = KnowledgeRepository(session)
            project = repo.get_project(project_name)
            claim = session.get(models.Claim, claim_id)
            if claim is None or claim.project_id != project.id:
                raise HTTPException(
                    status_code=404,
                    detail=f"Claim {claim_id} not found in project '{project_name}'",
                )
            session.delete(claim)
            return {"ok": True, "deleted": claim_id}

    @app.get("/projects/{project_name}/claims")
    def project_claims(
        project_name: str,
        limit: int = 100,
        include_candidates: bool = False,
        status: str | None = None,
    ):
        with session_scope(database_url) as session:
            project = KnowledgeRepository(session).get_project(project_name)
            query = (
                select(models.Claim, models.Source, models.Page, models.Entity)
                .join(models.Source, models.Claim.source_id == models.Source.id)
                .join(models.Page, models.Claim.page_id == models.Page.id, isouter=True)
                .join(models.Entity, models.Claim.subject_entity_id == models.Entity.id)
                .where(models.Claim.project_id == project.id)
            )
            if status:
                normalized = _normalize_claim_status(status)
                if normalized is None:
                    raise HTTPException(status_code=400, detail=f"unknown claim status: {status}")
                query = query.where(models.Claim.status == normalized)
            elif not include_candidates:
                query = query.where(models.Claim.status == "validated_claim")
            rows = session.execute(query.order_by(models.Claim.last_seen_at.desc()).limit(limit)).all()
            results: list[dict[str, Any]] = []
            for claim, source, page, subject in rows:
                object_name = None
                object_type = None
                if claim.object_entity_id:
                    object_entity = session.get(models.Entity, claim.object_entity_id)
                    object_name = object_entity.name if object_entity else None
                    object_type = object_entity.entity_type if object_entity else None
                evidence = session.scalar(
                    select(models.Evidence)
                    .where(models.Evidence.claim_id == claim.id)
                    .order_by(models.Evidence.created_at.desc())
                )
                results.append(
                    {
                        "id": claim.id,
                        "subject": subject.name,
                        "subject_type": subject.entity_type,
                        "predicate": claim.predicate,
                        "object": object_name,
                        "object_type": object_type,
                        "object_value": claim.object_value,
                        "source": source.name,
                        "page_url": page.url if page else None,
                        "confidence": claim.confidence,
                        "confidence_reason": claim.confidence_reason,
                        "status": claim.status,
                        "extraction_method": claim.extraction_method,
                        "evidence_text": evidence.evidence_text if evidence else None,
                        "evidence_summary": evidence.evidence_summary if evidence else None,
                        "page_type": (claim.metadata_json or {}).get("page_type")
                        or ((page.metadata_json or {}).get("page_type") if page else None),
                        "source_zone": (claim.metadata_json or {}).get("source_zone"),
                        "source_selector": (claim.metadata_json or {}).get("source_selector"),
                        "evidence_found": (claim.metadata_json or {}).get("evidence_found"),
                        "validation_status": (claim.metadata_json or {}).get("validation_status"),
                        "graph_merge_status": (claim.metadata_json or {}).get("graph_merge_status"),
                        "graph_merge_reason": (claim.metadata_json or {}).get("graph_merge_reason"),
                        "confidence_breakdown": (claim.metadata_json or {}).get("confidence_breakdown"),
                        "review_required": (claim.metadata_json or {}).get("review_required"),
                        "review_reason": (claim.metadata_json or {}).get("review_reason"),
                        "conflict_status": (claim.metadata_json or {}).get("conflict_status"),
                        "source_history": (claim.metadata_json or {}).get("source_history") or [],
                        "last_seen_at": claim.last_seen_at.isoformat(),
                    }
                )
            return results

    @app.get("/projects/{project_name}/pipeline")
    def project_pipeline(project_name: str):
        from sqlalchemy import func as sa_func, distinct
        with session_scope(database_url) as session:
            project = KnowledgeRepository(session).get_project(project_name)
            pid = project.id

            pages_total = session.scalar(
                select(sa_func.count(models.Page.id)).where(models.Page.project_id == pid)
            ) or 0
            ext_total = session.scalar(
                select(sa_func.count(models.ExtractionLog.id)).where(models.ExtractionLog.project_id == pid)
            ) or 0
            ext_pages = session.scalar(
                select(sa_func.count(distinct(models.ExtractionLog.page_id)))
                .where(models.ExtractionLog.project_id == pid, models.ExtractionLog.page_id.isnot(None))
            ) or 0
            ext_errors = session.scalar(
                select(sa_func.count(models.ExtractionLog.id))
                .where(models.ExtractionLog.project_id == pid, models.ExtractionLog.error.isnot(None))
            ) or 0

            status_rows = session.execute(
                select(models.Claim.status, sa_func.count(models.Claim.id))
                .where(models.Claim.project_id == pid)
                .group_by(models.Claim.status)
            ).all()
            claim_statuses = {row[0]: int(row[1]) for row in status_rows}
            claim_total = sum(claim_statuses.values())

            entity_count = session.scalar(
                select(sa_func.count(models.Entity.id)).where(models.Entity.project_id == pid)
            ) or 0
            triple_count = session.scalar(
                select(sa_func.count(models.OntologyTriple.id))
                .where(models.OntologyTriple.project_id == pid, models.OntologyTriple.status == "active")
            ) or 0

            entity_type_rows = session.execute(
                select(models.Entity.entity_type, sa_func.count(models.Entity.id))
                .where(models.Entity.project_id == pid)
                .group_by(models.Entity.entity_type)
                .order_by(sa_func.count(models.Entity.id).desc())
                .limit(10)
            ).all()

            recent_pages = session.scalars(
                select(models.Page).where(models.Page.project_id == pid)
                .order_by(models.Page.fetched_at.desc()).limit(5)
            ).all()
            recent_claims_rows = session.execute(
                select(models.Claim, models.Entity)
                .join(models.Entity, models.Claim.subject_entity_id == models.Entity.id)
                .where(models.Claim.project_id == pid)
                .order_by(models.Claim.last_seen_at.desc()).limit(5)
            ).all()

            return {
                "stages": [
                    {"key": "crawled", "count": pages_total},
                    {"key": "extracted", "count": ext_pages, "extra": {"events": ext_total, "errors": ext_errors}},
                    {"key": "claims", "count": claim_total, "extra": claim_statuses},
                    {"key": "validated", "count": claim_statuses.get("validated_claim", 0)},
                    {"key": "graph", "count": triple_count, "extra": {"entities": entity_count}},
                ],
                "entity_types": [{"type": row[0], "count": int(row[1])} for row in entity_type_rows],
                "recent_pages": [
                    {
                        "id": p.id,
                        "url": p.url,
                        "title": p.title,
                        "status_code": p.status_code,
                        "page_type": (p.metadata_json or {}).get("page_type"),
                        "fetched_at": p.fetched_at.isoformat() if p.fetched_at else None,
                    }
                    for p in recent_pages
                ],
                "recent_claims": [
                    {
                        "id": claim.id,
                        "subject": subject.name,
                        "predicate": claim.predicate,
                        "confidence": claim.confidence,
                        "status": claim.status,
                        "last_seen_at": claim.last_seen_at.isoformat() if claim.last_seen_at else None,
                    }
                    for claim, subject in recent_claims_rows
                ],
            }

    @app.post("/projects/{project_name}/pipeline/rerun")
    def rerun_pipeline_stage(
        project_name: str,
        request: PipelineRerunRequest,
        background_tasks: BackgroundTasks,
    ):
        crawl_stages = {
            "source-crawl",
            "page-clean",
            "page-classification",
            "entity-extraction",
            "claim-generation",
            "deduplication",
            "validation",
        }
        route_by_stage = {
            "human-review": f"/review/{project_name}",
            "ontology-commit": f"/graph/{project_name}",
            "export": f"/export/{project_name}",
        }
        if request.stage not in crawl_stages:
            return {
                "ok": True,
                "stage": request.stage,
                "action": "navigate",
                "route": route_by_stage.get(request.stage, f"/pipeline/{project_name}"),
                "message": "This stage is controlled from its workspace screen.",
            }

        with session_scope(database_url) as session:
            repo = KnowledgeRepository(session)
            project = repo.get_project(project_name)
            config = project_config_from_project_row(session, project)
            latest_job = session.scalar(
                select(models.CrawlJob)
                .where(models.CrawlJob.project_id == project.id)
                .order_by(models.CrawlJob.scheduled_at.desc())
            )
            latest_request = dict((latest_job.metadata_json or {}).get("request") or {}) if latest_job else {}
            source_name = latest_request.get("source_name")
            source = None
            if source_name:
                source = session.scalar(
                    select(models.Source).where(
                        models.Source.project_id == project.id,
                        models.Source.name == source_name,
                    )
                )
            if source is None:
                source = session.scalar(
                    select(models.Source)
                    .where(models.Source.project_id == project.id)
                    .order_by(models.Source.updated_at.desc())
                )
            if source is None:
                raise HTTPException(status_code=400, detail="No source is registered for this project.")
            url = latest_request.get("url") or source.base_url
            if not url:
                raise HTTPException(status_code=400, detail="No previous crawl URL or source base_url is available.")

            inner_request = SiteCrawlRequest(
                config_path="",
                source_name=source.name,
                url=url,
                extractor_provider=str(latest_request.get("extractor_provider") or "lm_studio"),
                extractor_model=latest_request.get("extractor_model"),
                extractor_base_url=latest_request.get("extractor_base_url") or "http://localhost:1234/v1",
                check_robots_txt=bool(latest_request.get("check_robots_txt") or False),
                respect_robots_txt=latest_request.get("respect_robots_txt"),
                max_depth=int(latest_request.get("max_depth") or 2),
                max_pages=int(latest_request.get("max_pages") or 50),
                same_domain_only=bool(latest_request.get("same_domain_only", True)),
                analyze_page_types=list(
                    latest_request.get("analyze_page_types")
                    or ["ProductPage", "BrandStoryPage", "ReviewPage"]
                ),
            )
            apply_crawl_request_overrides(config, inner_request)
            job = models.CrawlJob(
                project_id=project.id,
                source_id=source.id,
                url=url,
                status="pending",
                metadata_json={
                    "kind": "site_crawl",
                    "request": inner_request.model_dump(),
                    "project_name": project_name,
                    "rerun_stage": request.stage,
                    "progress": {
                        "seed_url": url,
                        "visited_count": 0,
                        "analyzed_count": 0,
                        "queued_count": 1,
                        "skipped_count": 0,
                        "errors": [],
                        "pages": [],
                    },
                },
            )
            session.add(job)
            session.flush()
            response = crawl_job_response(job)

        task_payload = {
            **inner_request.model_dump(),
            "__config_dict": project_config_to_dict(config),
        }
        background_tasks.add_task(
            run_site_crawl_job,
            database_url,
            response["job_id"],
            task_payload,
        )
        return {
            **response,
            "ok": True,
            "stage": request.stage,
            "action": "job_started",
        }

    @app.get("/projects/{project_name}/search")
    def project_search(project_name: str, q: str = "", limit: int = 10):
        q = (q or "").strip()
        if len(q) < 1:
            return {"entities": [], "claims": [], "pages": [], "predicates": []}
        like = f"%{q.lower()}%"
        bounded = max(min(limit, 50), 1)
        with session_scope(database_url) as session:
            project = KnowledgeRepository(session).get_project(project_name)
            pid = project.id

            from sqlalchemy import func as sa_func, or_
            entity_rows = session.scalars(
                select(models.Entity)
                .where(
                    models.Entity.project_id == pid,
                    or_(
                        sa_func.lower(models.Entity.name).like(like),
                        sa_func.lower(models.Entity.canonical_name).like(like),
                    ),
                )
                .limit(bounded)
            ).all()

            claim_rows = session.execute(
                select(models.Claim, models.Entity)
                .join(models.Entity, models.Claim.subject_entity_id == models.Entity.id)
                .where(
                    models.Claim.project_id == pid,
                    or_(
                        sa_func.lower(models.Entity.name).like(like),
                        sa_func.lower(models.Claim.predicate).like(like),
                    ),
                )
                .order_by(models.Claim.last_seen_at.desc())
                .limit(bounded)
            ).all()

            page_rows = session.scalars(
                select(models.Page)
                .where(
                    models.Page.project_id == pid,
                    or_(
                        sa_func.lower(models.Page.url).like(like),
                        sa_func.lower(models.Page.title).like(like),
                    ),
                )
                .order_by(models.Page.fetched_at.desc())
                .limit(bounded)
            ).all()

            predicate_rows = session.scalars(
                select(models.OntologyRelationType.name)
                .where(
                    models.OntologyRelationType.project_id == pid,
                    sa_func.lower(models.OntologyRelationType.name).like(like),
                )
                .limit(bounded)
            ).all()

            return {
                "entities": [
                    {"id": e.id, "name": e.name, "type": e.entity_type, "canonical_name": e.canonical_name}
                    for e in entity_rows
                ],
                "claims": [
                    {
                        "id": claim.id,
                        "subject": subject.name,
                        "predicate": claim.predicate,
                        "object_value": claim.object_value,
                        "confidence": claim.confidence,
                        "status": claim.status,
                    }
                    for claim, subject in claim_rows
                ],
                "pages": [
                    {"id": p.id, "url": p.url, "title": p.title}
                    for p in page_rows
                ],
                "predicates": list(predicate_rows),
            }

    @app.get("/projects/{project_name}/extraction-logs")
    def project_extraction_logs(project_name: str, limit: int = 50):
        with session_scope(database_url) as session:
            project = KnowledgeRepository(session).get_project(project_name)
            rows = session.execute(
                select(models.ExtractionLog, models.Page)
                .join(models.Page, models.ExtractionLog.page_id == models.Page.id, isouter=True)
                .where(models.ExtractionLog.project_id == project.id)
                .order_by(models.ExtractionLog.created_at.desc())
                .limit(limit)
            ).all()
            return [
                {
                    "id": log.id,
                    "page_url": page.url if page else None,
                    "extractor_name": log.extractor_name,
                    "provider": log.provider,
                    "error": log.error,
                    "created_at": log.created_at.isoformat(),
                    "validation": (log.raw_output or {}).get("validation"),
                    "page_context": (log.raw_output or {}).get("page_context"),
                    "candidate_count": len((log.raw_output or {}).get("candidate_claims") or []),
                    "raw_output": log.raw_output,
                }
                for log, page in rows
            ]

    @app.patch("/claims/{claim_id}/confidence")
    def update_claim_confidence(claim_id: int, request: UpdateClaimConfidenceRequest):
        confidence = min(max(request.confidence, 0.0), 1.0)
        with session_scope(database_url) as session:
            claim = session.get(models.Claim, claim_id)
            if claim is None:
                return {"ok": False, "error": "claim not found"}
            claim.confidence = confidence
            claim.confidence_reason = request.reason or "manual admin update"
            claim.last_seen_at = models.utcnow()
            return {"ok": True, "claim_id": claim.id, "confidence": claim.confidence}

    @app.patch("/claims/{claim_id}/status")
    def update_claim_status(claim_id: int, request: UpdateClaimStatusRequest):
        status = _normalize_claim_status(request.status)
        if status is None:
            raise HTTPException(status_code=400, detail=f"unknown claim status: {request.status}")
        with session_scope(database_url) as session:
            claim = session.get(models.Claim, claim_id)
            if claim is None:
                raise HTTPException(status_code=404, detail="claim not found")
            _apply_claim_review(claim, status, request.reason)
            return {"ok": True, "claim_id": claim.id, "status": claim.status}

    @app.post("/claims/bulk-status")
    def bulk_update_claim_status(request: BulkClaimStatusRequest):
        status = _normalize_claim_status(request.status)
        if status is None:
            raise HTTPException(status_code=400, detail=f"unknown claim status: {request.status}")
        if not request.claim_ids:
            return {"ok": True, "updated": 0}
        with session_scope(database_url) as session:
            rows = session.execute(
                select(models.Claim).where(models.Claim.id.in_(request.claim_ids))
            ).scalars().all()
            for claim in rows:
                _apply_claim_review(claim, status, request.reason)
            return {"ok": True, "updated": len(rows), "status": status}

    @app.post("/entities/merge")
    def merge_entities(request: MergeEntitiesRequest):
        with session_scope(database_url) as session:
            project = KnowledgeRepository(session).get_project(request.project_name)
            source = session.get(models.Entity, request.source_entity_id)
            target = session.get(models.Entity, request.target_entity_id)
            if source is None or target is None or source.project_id != project.id or target.project_id != project.id:
                return {"ok": False, "error": "entity not found in project"}
            session.query(models.Claim).filter(models.Claim.subject_entity_id == source.id).update(
                {models.Claim.subject_entity_id: target.id}
            )
            session.query(models.Claim).filter(models.Claim.object_entity_id == source.id).update(
                {models.Claim.object_entity_id: target.id}
            )
            session.query(models.Relation).filter(models.Relation.subject_entity_id == source.id).update(
                {models.Relation.subject_entity_id: target.id}
            )
            session.query(models.Relation).filter(models.Relation.object_entity_id == source.id).update(
                {models.Relation.object_entity_id: target.id}
            )
            source.metadata_json = {**(source.metadata_json or {}), "merged_into": target.id}
            source.updated_at = models.utcnow()
            return {"ok": True, "source_entity_id": source.id, "target_entity_id": target.id}

    @app.get("/projects/{project_name}/recommendation-tags")
    def recommendation_tags(project_name: str):
        with session_scope(database_url) as session:
            project = KnowledgeRepository(session).get_project(project_name)
            tag_predicates = {
                "hasTopNote",
                "hasMiddleNote",
                "hasBaseNote",
                "hasScentNote",
                "hasFlavorNote",
                "evokesMood",
                "suitableForSeason",
                "suitableForOccasion",
                "hasReviewKeyword",
            }
            rows = session.execute(
                select(models.Claim, models.Entity)
                .join(models.Entity, models.Claim.object_entity_id == models.Entity.id)
                .where(
                    models.Claim.project_id == project.id,
                    models.Claim.status == "validated_claim",
                    models.Claim.predicate.in_(tag_predicates),
                )
            ).all()
            grouped: dict[str, dict[str, Any]] = {}
            for claim, entity in rows:
                key = f"{claim.predicate}:{entity.canonical_name}"
                if key not in grouped:
                    grouped[key] = {
                        "predicate": claim.predicate,
                        "name": entity.name,
                        "type": entity.entity_type,
                        "support_count": 0,
                        "max_confidence": 0.0,
                    }
                grouped[key]["support_count"] += 1
                grouped[key]["max_confidence"] = max(grouped[key]["max_confidence"], claim.confidence)
            return sorted(grouped.values(), key=lambda item: (item["predicate"], -item["support_count"], item["name"]))

    @app.post("/recommend")
    def recommend(request: RecommendRequest):
        with session_scope(database_url) as session:
            repo = KnowledgeRepository(session)
            project = repo.get_project(request.project_name)
            recommender = RuleBasedRecommender(session)
            pref = PreferenceInput(**request.preferences)
            items = recommender.recommend(project.id, request.target_entity_type, pref, request.limit)
            return [asdict(item) for item in items]