AI/crawler_platform/app/api/routes.py

from __future__ import annotations

from dataclasses import asdict
from typing import Any

from fastapi import BackgroundTasks, HTTPException
from pydantic import BaseModel, Field
from sqlalchemy import select

from crawler_platform.app.config.loader import (
    ProjectConfig,
    SourceConfig,
    load_project_config,
    project_config_from_dict,
)
from crawler_platform.app.core.crawler.discovery import discover_links
from crawler_platform.app.core.crawler.fetchers import RobotsPolicy, make_fetcher
from crawler_platform.app.core.crawler.pipeline import CrawlPipeline
from crawler_platform.app.core.crawler.site_crawler import SiteCrawler
from crawler_platform.app.core.database import models
from crawler_platform.app.core.database.repository import KnowledgeRepository
from crawler_platform.app.core.database.session import session_scope
from crawler_platform.app.core.extractor.ai_provider import list_openai_compatible_models
from crawler_platform.app.core.extractor.factory import extractor_for_domain
from crawler_platform.app.core.ontology.definitions import DOMAIN_ONTOLOGIES, ontology_for_domain
from crawler_platform.app.core.ontology.gap_detector import KnowledgeGapDetector
from crawler_platform.app.core.ontology.mapper import ontology_to_dict
from crawler_platform.app.core.ontology.registry import OntologyRegistry
from crawler_platform.app.core.recommendation.scorer import PreferenceInput, RuleBasedRecommender
from crawler_platform.app.core.research.graph_query import SemanticGraphQuery
from crawler_platform.app.core.research.graph_research_loop import GraphResearchLoop
from crawler_platform.app.core.research.memory_store import ResearchMemoryStore, research_session_payload


class CrawlRequest(BaseModel):
    config_path: str
    source_name: str
    url: str
    extractor_provider: str = "lm_studio"
    extractor_model: str | None = None
    extractor_base_url: str | None = "http://localhost:1234/v1"
    check_robots_txt: bool = False
    respect_robots_txt: bool | None = None


class SiteCrawlRequest(CrawlRequest):
    max_depth: int = 2
    max_pages: int = 50
    same_domain_only: bool = True
    analyze_page_types: list[str] = Field(default_factory=lambda: ["ProductPage", "BrandStoryPage", "ReviewPage"])


class SiteCrawlByProjectRequest(BaseModel):
    """Site crawl invoked against an existing project (no filesystem config_path)."""

    project_name: str
    source_name: str
    url: str
    extractor_provider: str = "lm_studio"
    extractor_model: str | None = None
    extractor_base_url: str | None = "http://localhost:1234/v1"
    check_robots_txt: bool = False
    respect_robots_txt: bool | None = None
    max_depth: int = 2
    max_pages: int = 50
    same_domain_only: bool = True
    analyze_page_types: list[str] = Field(
        default_factory=lambda: ["ProductPage", "BrandStoryPage", "ReviewPage"]
    )

    def to_site_crawl_request(self, config_path_placeholder: str = "") -> "SiteCrawlRequest":
        """For internal handoff to existing crawl pipeline (config_path is not used)."""
        return SiteCrawlRequest(
            config_path=config_path_placeholder,
            source_name=self.source_name,
            url=self.url,
            extractor_provider=self.extractor_provider,
            extractor_model=self.extractor_model,
            extractor_base_url=self.extractor_base_url,
            check_robots_txt=self.check_robots_txt,
            respect_robots_txt=self.respect_robots_txt,
            max_depth=self.max_depth,
            max_pages=self.max_pages,
            same_domain_only=self.same_domain_only,
            analyze_page_types=list(self.analyze_page_types),
        )


class DiscoverRequest(BaseModel):
    config_path: str
    source_name: str
    url: str
    limit: int = 30
    check_robots_txt: bool = False
    respect_robots_txt: bool | None = None


class RecommendRequest(BaseModel):
    project_name: str
    target_entity_type: str = "Perfume"
    preferences: dict = Field(default_factory=dict)
    limit: int = 10


class CreateProjectRequest(BaseModel):
    config_path: str


class InlineSourceConfig(BaseModel):
    name: str
    type: str = "unknown"
    trust_level: float = 0.5
    base_url: str | None = None
    allowed_paths: list[str] = Field(default_factory=list)
    parser: str = "generic"
    fetcher: str = "requests"
    rate_limit_per_minute: int = 30
    respect_robots_txt: bool = False


class CreateProjectInlineRequest(BaseModel):
    """Create a project from inline JSON config (no filesystem dependency)."""

    project_name: str
    domain: str
    target_entities: list[str] = Field(default_factory=list)
    fields: list[str] = Field(default_factory=list)
    sources: list[InlineSourceConfig] = Field(default_factory=list)
    ontology: dict[str, Any] = Field(default_factory=dict)
    recommendation: dict[str, Any] = Field(default_factory=dict)
    update_policy: dict[str, Any] = Field(default_factory=dict)

    def to_project_config(self) -> ProjectConfig:
        return ProjectConfig(
            project_name=self.project_name,
            domain=self.domain,
            target_entities=list(self.target_entities),
            fields=list(self.fields),
            sources=[SourceConfig(**s.model_dump()) for s in self.sources],
            ontology=dict(self.ontology),
            recommendation=dict(self.recommendation),
            update_policy=dict(self.update_policy),
        )


class ResetProjectRequest(BaseModel):
    config_path: str
    project_name: str | None = None


class ResearchRunByProjectRequest(BaseModel):
    """Run autonomous research against an existing DB project (no config_path)."""

    project_name: str
    source_name: str
    url: str | None = None
    seed_entity_id: int | None = None
    goal: str = "Semantic ontology exploration"
    extractor_provider: str = "lm_studio"
    extractor_model: str | None = None
    extractor_base_url: str | None = "http://localhost:1234/v1"
    check_robots_txt: bool = False
    respect_robots_txt: bool | None = None
    max_depth: int = 2
    max_steps: int = 12
    max_branch: int = 8
    min_relevance: float = 0.35
    same_domain_only: bool = True
    analyze_page_types: list[str] = Field(
        default_factory=lambda: ["ProductPage", "BrandStoryPage", "ReviewPage"]
    )


class UpdateClaimConfidenceRequest(BaseModel):
    confidence: float
    reason: str | None = None


class UpdateClaimStatusRequest(BaseModel):
    status: str
    reason: str | None = None


class BulkClaimStatusRequest(BaseModel):
    claim_ids: list[int]
    status: str
    reason: str | None = None


class MergeEntitiesRequest(BaseModel):
    project_name: str
    source_entity_id: int
    target_entity_id: int


class ExtractorModelsRequest(BaseModel):
    provider: str
    base_url: str | None = None


class ResearchRunRequest(CrawlRequest):
    project_name: str | None = None
    seed_entity_id: int | None = None
    goal: str = "Semantic ontology exploration"
    max_depth: int = 2
    max_steps: int = 12
    max_branch: int = 8
    min_relevance: float = 0.35
    same_domain_only: bool = True
    analyze_page_types: list[str] = Field(default_factory=lambda: ["ProductPage", "BrandStoryPage", "ReviewPage"])


def site_crawl_progress_payload(result, latest_page=None) -> dict[str, Any]:
    payload = asdict(result)
    if latest_page is not None:
        payload["latest_page"] = asdict(latest_page)
    return payload


_CLAIM_STATUS_ALIASES = {
    "accept": "validated_claim",
    "accepted": "validated_claim",
    "validated": "validated_claim",
    "validated_claim": "validated_claim",
    "reject": "rejected",
    "rejected": "rejected",
    "active": "active",
    "candidate": "active",
    "pending": "active",
}


def _normalize_claim_status(value: str | None) -> str | None:
    if not value:
        return None
    return _CLAIM_STATUS_ALIASES.get(value.strip().lower())


def _apply_claim_review(claim: models.Claim, status: str, reason: str | None) -> None:
    claim.status = status
    if reason:
        claim.confidence_reason = reason
    metadata = dict(claim.metadata_json or {})
    review = dict(metadata.get("review") or {})
    review["status"] = status
    review["reason"] = reason or review.get("reason")
    review["reviewed_at"] = models.utcnow().isoformat()
    metadata["review"] = review
    claim.metadata_json = metadata
    claim.last_seen_at = models.utcnow()


def apply_crawl_request_overrides(config, request: CrawlRequest | DiscoverRequest) -> None:
    check_robots_txt = request.respect_robots_txt
    if check_robots_txt is None:
        check_robots_txt = request.check_robots_txt
    config.source_by_name(request.source_name).respect_robots_txt = check_robots_txt


def crawl_job_response(job: models.CrawlJob) -> dict[str, Any]:
    metadata = job.metadata_json or {}
    return {
        "job_id": job.id,
        "status": job.status,
        "url": job.url,
        "error": job.error,
        "scheduled_at": job.scheduled_at.isoformat() if job.scheduled_at else None,
        "started_at": job.started_at.isoformat() if job.started_at else None,
        "finished_at": job.finished_at.isoformat() if job.finished_at else None,
        "progress": metadata.get("progress", {}),
        "request": metadata.get("request", {}),
    }


def update_site_crawl_job_metadata(job: models.CrawlJob, **updates: Any) -> None:
    metadata = dict(job.metadata_json or {})
    metadata.update(updates)
    job.metadata_json = metadata


def is_site_crawl_cancel_requested(session, job_id: int) -> bool:
    session.expire_all()
    job = session.get(models.CrawlJob, job_id)
    return job is None or job.status == "cancel_requested"


def run_site_crawl_job(database_url: str, job_id: int, request_data: dict[str, Any]) -> None:
    inline_config = request_data.pop("__config_dict", None)
    request = SiteCrawlRequest(**request_data)
    try:
        if inline_config is not None:
            config = project_config_from_dict(inline_config)
        else:
            config = load_project_config(request.config_path)
        apply_crawl_request_overrides(config, request)
        with session_scope(database_url) as session:
            job = session.get(models.CrawlJob, job_id)
            if job is None:
                return
            if job.status == "cancel_requested":
                job.status = "canceled"
                job.finished_at = models.utcnow()
                return

            job.status = "running"
            job.started_at = models.utcnow()
            update_site_crawl_job_metadata(
                job,
                progress={
                    "seed_url": request.url,
                    "visited_count": 0,
                    "analyzed_count": 0,
                    "queued_count": 1,
                    "skipped_count": 0,
                    "errors": [],
                    "pages": [],
                },
            )
            session.commit()

            repo = KnowledgeRepository(session)
            crawler = SiteCrawler(
                repo,
                extractor_for_domain(
                    config.domain,
                    provider=request.extractor_provider,
                    model=request.extractor_model,
                    base_url=request.extractor_base_url,
                ),
            )

            def save_progress(result, latest_page) -> None:
                current_job = session.get(models.CrawlJob, job_id)
                if current_job is None:
                    return
                update_site_crawl_job_metadata(
                    current_job,
                    progress=site_crawl_progress_payload(result, latest_page),
                )
                session.commit()

            def should_stop() -> bool:
                return is_site_crawl_cancel_requested(session, job_id)

            result = crawler.crawl_site(
                config,
                request.source_name,
                request.url,
                max_depth=max(request.max_depth, 0),
                max_pages=max(min(request.max_pages, 500), 1),
                same_domain_only=request.same_domain_only,
                analyze_page_types=set(request.analyze_page_types),
                progress_callback=save_progress,
                should_stop=should_stop,
                parent_job_id=job_id,
            )

            finished_job = session.get(models.CrawlJob, job_id)
            if finished_job is not None:
                finished_job.status = "canceled" if finished_job.status == "cancel_requested" else "completed"
                finished_job.finished_at = models.utcnow()
                update_site_crawl_job_metadata(
                    finished_job,
                    progress=site_crawl_progress_payload(result),
                )
    except Exception as exc:
        with session_scope(database_url) as session:
            job = session.get(models.CrawlJob, job_id)
            if job is not None:
                job.status = "failed"
                job.error = str(exc)
                job.finished_at = models.utcnow()
                metadata = dict(job.metadata_json or {})
                progress = dict(metadata.get("progress") or {})
                progress["errors"] = [*progress.get("errors", []), str(exc)]
                update_site_crawl_job_metadata(job, progress=progress)


def register_routes(app, database_url: str) -> None:
    @app.get("/health")
    def health():
        return {"ok": True}

    @app.get("/projects")
    def projects():
        with session_scope(database_url) as session:
            rows = session.scalars(select(models.Project).order_by(models.Project.created_at.desc())).all()
            return [
                {
                    "id": project.id,
                    "name": project.name,
                    "domain": project.domain,
                    "created_at": project.created_at.isoformat(),
                    "updated_at": project.updated_at.isoformat(),
                }
                for project in rows
            ]

    @app.post("/projects")
    def create_project(request: CreateProjectRequest):
        config = load_project_config(request.config_path)
        with session_scope(database_url) as session:
            project = KnowledgeRepository(session).upsert_project(config)
            return {"id": project.id, "name": project.name, "domain": project.domain}

    @app.post("/projects/inline")
    def create_project_inline(request: CreateProjectInlineRequest):
        config = request.to_project_config()
        with session_scope(database_url) as session:
            project = KnowledgeRepository(session).upsert_project(config)
            return {"id": project.id, "name": project.name, "domain": project.domain}

    @app.get("/domains")
    def list_domains():
        """Available pre-defined ontology domains for project creation."""
        return [
            {
                "domain": ont.domain,
                "entity_types": list(ont.entity_types),
                "predicates": list(ont.predicates),
                "attribute_count": len(ont.attributes),
            }
            for ont in DOMAIN_ONTOLOGIES.values()
        ]

    @app.post("/projects/reset")
    def reset_project(request: ResetProjectRequest):
        config = load_project_config(request.config_path)
        if request.project_name and request.project_name != config.project_name:
            raise HTTPException(
                status_code=400,
                detail=(
                    f"Selected project '{request.project_name}' does not match "
                    f"config project '{config.project_name}'."
                ),
            )

        with session_scope(database_url) as session:
            repo = KnowledgeRepository(session)
            project = session.scalar(select(models.Project).where(models.Project.name == config.project_name))
            if project is None:
                project = repo.upsert_project(config)
                return {
                    "ok": True,
                    "name": project.name,
                    "domain": project.domain,
                    "created": True,
                    "reset": False,
                    "deleted": {},
                }

            deleted = repo.reset_project_runtime_data(project.id)
            project = repo.upsert_project(config)
            return {
                "ok": True,
                "name": project.name,
                "domain": project.domain,
                "created": False,
                "reset": True,
                "deleted": deleted,
            }

    @app.get("/projects/{project_name}")
    def project_detail(project_name: str):
        with session_scope(database_url) as session:
            project = KnowledgeRepository(session).get_project(project_name)
            sources = session.scalars(select(models.Source).where(models.Source.project_id == project.id)).all()
            return {
                "id": project.id,
                "name": project.name,
                "domain": project.domain,
                "config": project.config,
                "sources": [
                    {
                        "id": source.id,
                        "name": source.name,
                        "type": source.type,
                        "base_url": source.base_url,
                        "trust_level": source.trust_level,
                        "respect_robots_txt": source.respect_robots_txt,
                        "rate_limit_per_minute": source.rate_limit_per_minute,
                    }
                    for source in sources
                ],
            }

    @app.post("/projects/{project_name}/sources")
    def add_project_source(project_name: str, request: InlineSourceConfig):
        """Add or update a source on an existing project."""
        with session_scope(database_url) as session:
            repo = KnowledgeRepository(session)
            project = repo.get_project(project_name)
            source = repo.upsert_source(project, SourceConfig(**request.model_dump()))
            return {
                "id": source.id,
                "name": source.name,
                "type": source.type,
                "base_url": source.base_url,
                "trust_level": source.trust_level,
                "respect_robots_txt": source.respect_robots_txt,
                "rate_limit_per_minute": source.rate_limit_per_minute,
            }

    @app.delete("/projects/{project_name}/sources/{source_name}")
    def delete_project_source(project_name: str, source_name: str):
        with session_scope(database_url) as session:
            repo = KnowledgeRepository(session)
            project = repo.get_project(project_name)
            source = session.scalar(
                select(models.Source).where(
                    models.Source.project_id == project.id,
                    models.Source.name == source_name,
                )
            )
            if source is None:
                raise HTTPException(
                    status_code=404,
                    detail=f"Source '{source_name}' not found in project '{project_name}'",
                )
            session.delete(source)
            return {"ok": True, "deleted": source_name}

    @app.get("/ontology/{domain}")
    def ontology(domain: str):
        return ontology_to_dict(ontology_for_domain(domain))

    @app.get("/projects/{project_name}/ontology/registry")
    def ontology_registry(project_name: str):
        with session_scope(database_url) as session:
            project = KnowledgeRepository(session).get_project(project_name)
            return OntologyRegistry(session).registry_payload(project.id)

    @app.get("/projects/{project_name}/ontology/proposals")
    def ontology_proposals(project_name: str, limit: int = 100):
        with session_scope(database_url) as session:
            project = KnowledgeRepository(session).get_project(project_name)
            rows = session.scalars(
                select(models.OntologyProposal)
                .where(models.OntologyProposal.project_id == project.id)
                .order_by(models.OntologyProposal.updated_at.desc())
                .limit(max(min(limit, 300), 1))
            ).all()
            return [
                {
                    "id": row.id,
                    "proposal_type": row.proposal_type,
                    "name": row.name,
                    "reason": row.reason,
                    "evidence": row.evidence,
                    "status": row.status,
                    "confidence": row.confidence,
                    "metadata": row.metadata_json or {},
                    "updated_at": row.updated_at.isoformat() if row.updated_at else None,
                }
                for row in rows
            ]

    @app.get("/projects/{project_name}/ontology/triples")
    def ontology_triples(project_name: str, status: str | None = None, limit: int = 100):
        with session_scope(database_url) as session:
            project = KnowledgeRepository(session).get_project(project_name)
            query = select(models.OntologyTriple).where(models.OntologyTriple.project_id == project.id)
            if status:
                query = query.where(models.OntologyTriple.status == status)
            rows = session.scalars(
                query.order_by(models.OntologyTriple.last_seen_at.desc()).limit(max(min(limit, 300), 1))
            ).all()
            results = []
            for row in rows:
                subject = session.get(models.Entity, row.subject_entity_id)
                object_entity = session.get(models.Entity, row.object_entity_id) if row.object_entity_id else None
                results.append(
                    {
                        "id": row.id,
                        "claim_id": row.claim_id,
                        "subject": subject.name if subject else None,
                        "subject_type": row.subject_type,
                        "predicate": row.predicate,
                        "object": object_entity.name if object_entity else None,
                        "object_type": row.object_type,
                        "object_value": row.object_value,
                        "value_type": row.value_type,
                        "status": row.status,
                        "confidence": row.confidence,
                        "support_count": row.support_count,
                        "metadata": row.metadata_json or {},
                        "last_seen_at": row.last_seen_at.isoformat() if row.last_seen_at else None,
                    }
                )
            return results

    @app.get("/projects/{project_name}/knowledge-gaps")
    def knowledge_gaps(project_name: str, limit: int = 100):
        with session_scope(database_url) as session:
            project = KnowledgeRepository(session).get_project(project_name)
            return KnowledgeGapDetector(session).list_open(project.id, max(min(limit, 300), 1))

    @app.post("/extractors/models")
    def extractor_models(request: ExtractorModelsRequest):
        try:
            if request.provider == "lm_studio":
                models = list_openai_compatible_models(request.base_url or "http://localhost:1234/v1")
                return {"ok": True, "models": models}
            if request.provider == "openai":
                import os

                models = list_openai_compatible_models(
                    request.base_url or "https://api.openai.com/v1",
                    os.getenv("OPENAI_API_KEY"),
                )
                return {"ok": True, "models": models}
            if request.provider == "ollama":
                return {"ok": False, "error": "Ollama model listing is not implemented yet. Enter the model manually."}
            return {"ok": True, "models": [{"id": "rule_based", "owned_by": "local"}]}
        except Exception as exc:
            return {"ok": False, "error": str(exc), "models": []}

    @app.post("/crawl")
    def crawl(request: CrawlRequest):
        config = load_project_config(request.config_path)
        apply_crawl_request_overrides(config, request)
        with session_scope(database_url) as session:
            repo = KnowledgeRepository(session)
            pipeline = CrawlPipeline(
                repo,
                extractor_for_domain(
                    config.domain,
                    provider=request.extractor_provider,
                    model=request.extractor_model,
                    base_url=request.extractor_base_url,
                ),
            )
            try:
                result = pipeline.crawl_url(config, request.source_name, request.url)
            except Exception as exc:
                raise HTTPException(status_code=400, detail=str(exc)) from exc
            return {
                "page_id": result.page_id,
                "claim_count": result.claim_count,
                "entity_count": result.entity_count,
                "crawl_status": result.crawl_status,
                "extraction_status": result.extraction_status,
                "page_type": result.page_type,
                "raw_text_length": result.raw_text_length,
                "clean_text_length": result.clean_text_length,
                "warnings": result.warnings or [],
            }

    @app.post("/crawl-site")
    def crawl_site(request: SiteCrawlRequest, background_tasks: BackgroundTasks):
        config = load_project_config(request.config_path)
        apply_crawl_request_overrides(config, request)
        with session_scope(database_url) as session:
            repo = KnowledgeRepository(session)
            project = repo.upsert_project(config)
            source = repo.get_source(project.id, request.source_name)
            job = models.CrawlJob(
                project_id=project.id,
                source_id=source.id,
                url=request.url,
                status="pending",
                metadata_json={
                    "kind": "site_crawl",
                    "request": request.model_dump(),
                    "progress": {
                        "seed_url": request.url,
                        "visited_count": 0,
                        "analyzed_count": 0,
                        "queued_count": 1,
                        "skipped_count": 0,
                        "errors": [],
                        "pages": [],
                    },
                },
            )
            session.add(job)
            session.flush()
            response = crawl_job_response(job)
        background_tasks.add_task(run_site_crawl_job, database_url, response["job_id"], request.model_dump())
        return response

    @app.post("/crawl-site/by-project")
    def crawl_site_by_project(
        request: SiteCrawlByProjectRequest, background_tasks: BackgroundTasks
    ):
        """Start a site crawl against an existing DB project (no config_path)."""
        with session_scope(database_url) as session:
            repo = KnowledgeRepository(session)
            project = repo.get_project(request.project_name)
            config_dict = project.config or {}
            if not config_dict:
                raise HTTPException(
                    status_code=400,
                    detail=f"Project '{request.project_name}' has no stored config",
                )
            config = project_config_from_dict(config_dict)
            try:
                source = repo.get_source(project.id, request.source_name)
            except KeyError as exc:
                raise HTTPException(status_code=404, detail=str(exc)) from exc

            inner_request = request.to_site_crawl_request()
            apply_crawl_request_overrides(config, inner_request)

            job = models.CrawlJob(
                project_id=project.id,
                source_id=source.id,
                url=request.url,
                status="pending",
                metadata_json={
                    "kind": "site_crawl",
                    "request": inner_request.model_dump(),
                    "project_name": request.project_name,
                    "progress": {
                        "seed_url": request.url,
                        "visited_count": 0,
                        "analyzed_count": 0,
                        "queued_count": 1,
                        "skipped_count": 0,
                        "errors": [],
                        "pages": [],
                    },
                },
            )
            session.add(job)
            session.flush()
            response = crawl_job_response(job)

        task_payload = {**inner_request.model_dump(), "__config_dict": config_dict}
        background_tasks.add_task(
            run_site_crawl_job, database_url, response["job_id"], task_payload
        )
        return response

    @app.get("/crawl-site/jobs/{job_id}")
    def crawl_site_job(job_id: int):
        with session_scope(database_url) as session:
            job = session.get(models.CrawlJob, job_id)
            if job is None or (job.metadata_json or {}).get("kind") != "site_crawl":
                raise HTTPException(status_code=404, detail="site crawl job not found")
            return crawl_job_response(job)

    @app.post("/crawl-site/jobs/{job_id}/cancel")
    def cancel_crawl_site_job(job_id: int):
        with session_scope(database_url) as session:
            job = session.get(models.CrawlJob, job_id)
            if job is None or (job.metadata_json or {}).get("kind") != "site_crawl":
                raise HTTPException(status_code=404, detail="site crawl job not found")
            if job.status in {"completed", "failed", "canceled"}:
                return crawl_job_response(job)
            job.status = "cancel_requested"
            job.error = "cancel requested by user"
            return crawl_job_response(job)

    @app.post("/discover")
    def discover(request: DiscoverRequest):
        config = load_project_config(request.config_path)
        apply_crawl_request_overrides(config, request)
        source_config = config.source_by_name(request.source_name)
        robots = RobotsPolicy()
        robots_decision = robots.check(request.url, source_config.respect_robots_txt)
        if not robots_decision.allowed:
            return {
                "ok": False,
                "error": f"{robots_decision.reason}: {request.url}",
                "robots_status": robots_decision.status,
                "robots_reason": robots_decision.reason,
                "links": [],
            }
        fetcher = make_fetcher(source_config.fetcher, source_config.rate_limit_per_minute)
        result = fetcher.fetch(request.url)
        links = discover_links(result.analysis_html, result.final_url or request.url, request.limit)
        return {
            "ok": True,
            "status_code": result.status_code,
            "final_url": result.final_url,
            "crawl_status": result.crawl_status,
            "robots_status": robots_decision.status,
            "robots_reason": robots_decision.reason,
            "warnings": result.warnings,
            "links": [asdict(link) for link in links],
        }

    @app.post("/research/run")
    def run_research(request: ResearchRunRequest):
        config = load_project_config(request.config_path)
        apply_crawl_request_overrides(config, request)
        if request.project_name and request.project_name != config.project_name:
            raise HTTPException(
                status_code=400,
                detail=f"Selected project '{request.project_name}' does not match config project '{config.project_name}'.",
            )
        with session_scope(database_url) as session:
            repo = KnowledgeRepository(session)
            loop = GraphResearchLoop(
                repo,
                extractor_for_domain(
                    config.domain,
                    provider=request.extractor_provider,
                    model=request.extractor_model,
                    base_url=request.extractor_base_url,
                ),
            )
            try:
                result = loop.run(
                    project_config=config,
                    source_name=request.source_name,
                    seed_url=request.url or None,
                    seed_entity_id=request.seed_entity_id,
                    goal=request.goal,
                    max_depth=max(request.max_depth, 0),
                    max_steps=max(min(request.max_steps, 50), 1),
                    max_branch=max(min(request.max_branch, 30), 1),
                    min_relevance=min(max(request.min_relevance, 0.0), 1.0),
                    same_domain_only=request.same_domain_only,
                    analyze_page_types=set(request.analyze_page_types),
                )
            except Exception as exc:
                raise HTTPException(status_code=400, detail=str(exc)) from exc
            return asdict(result)

    @app.post("/research/run/by-project")
    def run_research_by_project(request: ResearchRunByProjectRequest):
        """Run research against an existing DB project (no config_path)."""
        with session_scope(database_url) as session:
            repo = KnowledgeRepository(session)
            project = repo.get_project(request.project_name)
            config_dict = project.config or {}
            if not config_dict:
                raise HTTPException(
                    status_code=400,
                    detail=f"Project '{request.project_name}' has no stored config",
                )
            config = project_config_from_dict(config_dict)
            try:
                source_config = config.source_by_name(request.source_name)
            except KeyError as exc:
                raise HTTPException(status_code=404, detail=str(exc)) from exc

            check_robots = request.respect_robots_txt
            if check_robots is None:
                check_robots = request.check_robots_txt
            source_config.respect_robots_txt = check_robots

            loop = GraphResearchLoop(
                repo,
                extractor_for_domain(
                    config.domain,
                    provider=request.extractor_provider,
                    model=request.extractor_model,
                    base_url=request.extractor_base_url,
                ),
            )
            try:
                result = loop.run(
                    project_config=config,
                    source_name=request.source_name,
                    seed_url=request.url or None,
                    seed_entity_id=request.seed_entity_id,
                    goal=request.goal,
                    max_depth=max(request.max_depth, 0),
                    max_steps=max(min(request.max_steps, 50), 1),
                    max_branch=max(min(request.max_branch, 30), 1),
                    min_relevance=min(max(request.min_relevance, 0.0), 1.0),
                    same_domain_only=request.same_domain_only,
                    analyze_page_types=set(request.analyze_page_types),
                )
            except Exception as exc:
                raise HTTPException(status_code=400, detail=str(exc)) from exc
            return asdict(result)

    @app.get("/projects/{project_name}/research/sessions")
    def research_sessions(project_name: str, limit: int = 25):
        with session_scope(database_url) as session:
            project = KnowledgeRepository(session).get_project(project_name)
            return ResearchMemoryStore(session).list_sessions(project.id, limit)

    @app.get("/research/sessions/{job_id}")
    def research_session(job_id: int):
        with session_scope(database_url) as session:
            job = session.get(models.CrawlJob, job_id)
            if job is None or (job.metadata_json or {}).get("kind") != "research_session":
                raise HTTPException(status_code=404, detail="research session not found")
            return research_session_payload(job)

    @app.get("/projects/{project_name}/graph/neighborhood")
    def graph_neighborhood(project_name: str, entity_id: int | None = None, limit: int = 120):
        with session_scope(database_url) as session:
            project = KnowledgeRepository(session).get_project(project_name)
            return SemanticGraphQuery(session).neighborhood(project.id, entity_id, max(min(limit, 300), 1))

    @app.get("/projects/{project_name}/graph/query")
    def graph_query(
        project_name: str,
        kind: str = "trend_summary",
        predicate: str = "hasAccord",
        tag: str | None = None,
        brand: str | None = None,
        limit: int = 100,
    ):
        with session_scope(database_url) as session:
            project = KnowledgeRepository(session).get_project(project_name)
            graph = SemanticGraphQuery(session)
            bounded_limit = max(min(limit, 300), 1)
            if kind == "brand_products":
                return graph.brand_products(project.id, brand, bounded_limit)
            if kind == "products_by_tag":
                return graph.products_by_tag(project.id, predicate, tag, bounded_limit)
            if kind == "relation_summary":
                return graph.relation_summary(project.id, bounded_limit)
            if kind == "entity_type_summary":
                return graph.entity_type_summary(project.id, bounded_limit)
            if kind == "trend_summary":
                return graph.trend_summary(project.id, bounded_limit)
            raise HTTPException(status_code=400, detail=f"unknown graph query kind: {kind}")

    @app.get("/projects/{project_name}/entities")
    def project_entities(project_name: str, entity_type: str | None = None, limit: int = 50):
        with session_scope(database_url) as session:
            project = KnowledgeRepository(session).get_project(project_name)
            query = select(models.Entity).where(models.Entity.project_id == project.id)
            if entity_type:
                query = query.where(models.Entity.entity_type == entity_type)
            entities = session.scalars(query.limit(limit)).all()
            return [
                {
                    "id": entity.id,
                    "type": entity.entity_type,
                    "name": entity.name,
                    "metadata": entity.metadata_json,
                }
                for entity in entities
            ]

    @app.get("/projects/{project_name}/claims")
    def project_claims(
        project_name: str,
        limit: int = 100,
        include_candidates: bool = False,
        status: str | None = None,
    ):
        with session_scope(database_url) as session:
            project = KnowledgeRepository(session).get_project(project_name)
            query = (
                select(models.Claim, models.Source, models.Page, models.Entity)
                .join(models.Source, models.Claim.source_id == models.Source.id)
                .join(models.Page, models.Claim.page_id == models.Page.id, isouter=True)
                .join(models.Entity, models.Claim.subject_entity_id == models.Entity.id)
                .where(models.Claim.project_id == project.id)
            )
            if status:
                normalized = _normalize_claim_status(status)
                if normalized is None:
                    raise HTTPException(status_code=400, detail=f"unknown claim status: {status}")
                query = query.where(models.Claim.status == normalized)
            elif not include_candidates:
                query = query.where(models.Claim.status == "validated_claim")
            rows = session.execute(query.order_by(models.Claim.last_seen_at.desc()).limit(limit)).all()
            results: list[dict[str, Any]] = []
            for claim, source, page, subject in rows:
                object_name = None
                if claim.object_entity_id:
                    object_entity = session.get(models.Entity, claim.object_entity_id)
                    object_name = object_entity.name if object_entity else None
                evidence = session.scalar(
                    select(models.Evidence)
                    .where(models.Evidence.claim_id == claim.id)
                    .order_by(models.Evidence.created_at.desc())
                )
                results.append(
                    {
                        "id": claim.id,
                        "subject": subject.name,
                        "subject_type": subject.entity_type,
                        "predicate": claim.predicate,
                        "object": object_name,
                        "object_value": claim.object_value,
                        "source": source.name,
                        "page_url": page.url if page else None,
                        "confidence": claim.confidence,
                        "confidence_reason": claim.confidence_reason,
                        "status": claim.status,
                        "evidence_text": evidence.evidence_text if evidence else None,
                        "evidence_summary": evidence.evidence_summary if evidence else None,
                        "page_type": (claim.metadata_json or {}).get("page_type")
                        or ((page.metadata_json or {}).get("page_type") if page else None),
                        "source_zone": (claim.metadata_json or {}).get("source_zone"),
                        "source_selector": (claim.metadata_json or {}).get("source_selector"),
                        "evidence_found": (claim.metadata_json or {}).get("evidence_found"),
                        "validation_status": (claim.metadata_json or {}).get("validation_status"),
                        "graph_merge_status": (claim.metadata_json or {}).get("graph_merge_status"),
                        "graph_merge_reason": (claim.metadata_json or {}).get("graph_merge_reason"),
                        "confidence_breakdown": (claim.metadata_json or {}).get("confidence_breakdown"),
                        "review_required": (claim.metadata_json or {}).get("review_required"),
                        "review_reason": (claim.metadata_json or {}).get("review_reason"),
                        "conflict_status": (claim.metadata_json or {}).get("conflict_status"),
                        "source_history": (claim.metadata_json or {}).get("source_history") or [],
                        "last_seen_at": claim.last_seen_at.isoformat(),
                    }
                )
            return results

    @app.get("/projects/{project_name}/pipeline")
    def project_pipeline(project_name: str):
        from sqlalchemy import func as sa_func, distinct
        with session_scope(database_url) as session:
            project = KnowledgeRepository(session).get_project(project_name)
            pid = project.id

            pages_total = session.scalar(
                select(sa_func.count(models.Page.id)).where(models.Page.project_id == pid)
            ) or 0
            ext_total = session.scalar(
                select(sa_func.count(models.ExtractionLog.id)).where(models.ExtractionLog.project_id == pid)
            ) or 0
            ext_pages = session.scalar(
                select(sa_func.count(distinct(models.ExtractionLog.page_id)))
                .where(models.ExtractionLog.project_id == pid, models.ExtractionLog.page_id.isnot(None))
            ) or 0
            ext_errors = session.scalar(
                select(sa_func.count(models.ExtractionLog.id))
                .where(models.ExtractionLog.project_id == pid, models.ExtractionLog.error.isnot(None))
            ) or 0

            status_rows = session.execute(
                select(models.Claim.status, sa_func.count(models.Claim.id))
                .where(models.Claim.project_id == pid)
                .group_by(models.Claim.status)
            ).all()
            claim_statuses = {row[0]: int(row[1]) for row in status_rows}
            claim_total = sum(claim_statuses.values())

            entity_count = session.scalar(
                select(sa_func.count(models.Entity.id)).where(models.Entity.project_id == pid)
            ) or 0
            triple_count = session.scalar(
                select(sa_func.count(models.OntologyTriple.id))
                .where(models.OntologyTriple.project_id == pid, models.OntologyTriple.status == "active")
            ) or 0

            entity_type_rows = session.execute(
                select(models.Entity.entity_type, sa_func.count(models.Entity.id))
                .where(models.Entity.project_id == pid)
                .group_by(models.Entity.entity_type)
                .order_by(sa_func.count(models.Entity.id).desc())
                .limit(10)
            ).all()

            recent_pages = session.scalars(
                select(models.Page).where(models.Page.project_id == pid)
                .order_by(models.Page.fetched_at.desc()).limit(5)
            ).all()
            recent_claims_rows = session.execute(
                select(models.Claim, models.Entity)
                .join(models.Entity, models.Claim.subject_entity_id == models.Entity.id)
                .where(models.Claim.project_id == pid)
                .order_by(models.Claim.last_seen_at.desc()).limit(5)
            ).all()

            return {
                "stages": [
                    {"key": "crawled", "count": pages_total},
                    {"key": "extracted", "count": ext_pages, "extra": {"events": ext_total, "errors": ext_errors}},
                    {"key": "claims", "count": claim_total, "extra": claim_statuses},
                    {"key": "validated", "count": claim_statuses.get("validated_claim", 0)},
                    {"key": "graph", "count": triple_count, "extra": {"entities": entity_count}},
                ],
                "entity_types": [{"type": row[0], "count": int(row[1])} for row in entity_type_rows],
                "recent_pages": [
                    {
                        "id": p.id,
                        "url": p.url,
                        "title": p.title,
                        "status_code": p.status_code,
                        "page_type": (p.metadata_json or {}).get("page_type"),
                        "fetched_at": p.fetched_at.isoformat() if p.fetched_at else None,
                    }
                    for p in recent_pages
                ],
                "recent_claims": [
                    {
                        "id": claim.id,
                        "subject": subject.name,
                        "predicate": claim.predicate,
                        "confidence": claim.confidence,
                        "status": claim.status,
                        "last_seen_at": claim.last_seen_at.isoformat() if claim.last_seen_at else None,
                    }
                    for claim, subject in recent_claims_rows
                ],
            }

    @app.get("/projects/{project_name}/search")
    def project_search(project_name: str, q: str = "", limit: int = 10):
        q = (q or "").strip()
        if len(q) < 1:
            return {"entities": [], "claims": [], "pages": [], "predicates": []}
        like = f"%{q.lower()}%"
        bounded = max(min(limit, 50), 1)
        with session_scope(database_url) as session:
            project = KnowledgeRepository(session).get_project(project_name)
            pid = project.id

            from sqlalchemy import func as sa_func, or_
            entity_rows = session.scalars(
                select(models.Entity)
                .where(
                    models.Entity.project_id == pid,
                    or_(
                        sa_func.lower(models.Entity.name).like(like),
                        sa_func.lower(models.Entity.canonical_name).like(like),
                    ),
                )
                .limit(bounded)
            ).all()

            claim_rows = session.execute(
                select(models.Claim, models.Entity)
                .join(models.Entity, models.Claim.subject_entity_id == models.Entity.id)
                .where(
                    models.Claim.project_id == pid,
                    or_(
                        sa_func.lower(models.Entity.name).like(like),
                        sa_func.lower(models.Claim.predicate).like(like),
                    ),
                )
                .order_by(models.Claim.last_seen_at.desc())
                .limit(bounded)
            ).all()

            page_rows = session.scalars(
                select(models.Page)
                .where(
                    models.Page.project_id == pid,
                    or_(
                        sa_func.lower(models.Page.url).like(like),
                        sa_func.lower(models.Page.title).like(like),
                    ),
                )
                .order_by(models.Page.fetched_at.desc())
                .limit(bounded)
            ).all()

            predicate_rows = session.scalars(
                select(models.OntologyRelationType.name)
                .where(
                    models.OntologyRelationType.project_id == pid,
                    sa_func.lower(models.OntologyRelationType.name).like(like),
                )
                .limit(bounded)
            ).all()

            return {
                "entities": [
                    {"id": e.id, "name": e.name, "type": e.entity_type, "canonical_name": e.canonical_name}
                    for e in entity_rows
                ],
                "claims": [
                    {
                        "id": claim.id,
                        "subject": subject.name,
                        "predicate": claim.predicate,
                        "object_value": claim.object_value,
                        "confidence": claim.confidence,
                        "status": claim.status,
                    }
                    for claim, subject in claim_rows
                ],
                "pages": [
                    {"id": p.id, "url": p.url, "title": p.title}
                    for p in page_rows
                ],
                "predicates": list(predicate_rows),
            }

    @app.get("/projects/{project_name}/extraction-logs")
    def project_extraction_logs(project_name: str, limit: int = 50):
        with session_scope(database_url) as session:
            project = KnowledgeRepository(session).get_project(project_name)
            rows = session.execute(
                select(models.ExtractionLog, models.Page)
                .join(models.Page, models.ExtractionLog.page_id == models.Page.id, isouter=True)
                .where(models.ExtractionLog.project_id == project.id)
                .order_by(models.ExtractionLog.created_at.desc())
                .limit(limit)
            ).all()
            return [
                {
                    "id": log.id,
                    "page_url": page.url if page else None,
                    "extractor_name": log.extractor_name,
                    "provider": log.provider,
                    "error": log.error,
                    "created_at": log.created_at.isoformat(),
                    "validation": (log.raw_output or {}).get("validation"),
                    "page_context": (log.raw_output or {}).get("page_context"),
                    "candidate_count": len((log.raw_output or {}).get("candidate_claims") or []),
                    "raw_output": log.raw_output,
                }
                for log, page in rows
            ]

    @app.patch("/claims/{claim_id}/confidence")
    def update_claim_confidence(claim_id: int, request: UpdateClaimConfidenceRequest):
        confidence = min(max(request.confidence, 0.0), 1.0)
        with session_scope(database_url) as session:
            claim = session.get(models.Claim, claim_id)
            if claim is None:
                return {"ok": False, "error": "claim not found"}
            claim.confidence = confidence
            claim.confidence_reason = request.reason or "manual admin update"
            claim.last_seen_at = models.utcnow()
            return {"ok": True, "claim_id": claim.id, "confidence": claim.confidence}

    @app.patch("/claims/{claim_id}/status")
    def update_claim_status(claim_id: int, request: UpdateClaimStatusRequest):
        status = _normalize_claim_status(request.status)
        if status is None:
            raise HTTPException(status_code=400, detail=f"unknown claim status: {request.status}")
        with session_scope(database_url) as session:
            claim = session.get(models.Claim, claim_id)
            if claim is None:
                raise HTTPException(status_code=404, detail="claim not found")
            _apply_claim_review(claim, status, request.reason)
            return {"ok": True, "claim_id": claim.id, "status": claim.status}

    @app.post("/claims/bulk-status")
    def bulk_update_claim_status(request: BulkClaimStatusRequest):
        status = _normalize_claim_status(request.status)
        if status is None:
            raise HTTPException(status_code=400, detail=f"unknown claim status: {request.status}")
        if not request.claim_ids:
            return {"ok": True, "updated": 0}
        with session_scope(database_url) as session:
            rows = session.execute(
                select(models.Claim).where(models.Claim.id.in_(request.claim_ids))
            ).scalars().all()
            for claim in rows:
                _apply_claim_review(claim, status, request.reason)
            return {"ok": True, "updated": len(rows), "status": status}

    @app.post("/entities/merge")
    def merge_entities(request: MergeEntitiesRequest):
        with session_scope(database_url) as session:
            project = KnowledgeRepository(session).get_project(request.project_name)
            source = session.get(models.Entity, request.source_entity_id)
            target = session.get(models.Entity, request.target_entity_id)
            if source is None or target is None or source.project_id != project.id or target.project_id != project.id:
                return {"ok": False, "error": "entity not found in project"}
            session.query(models.Claim).filter(models.Claim.subject_entity_id == source.id).update(
                {models.Claim.subject_entity_id: target.id}
            )
            session.query(models.Claim).filter(models.Claim.object_entity_id == source.id).update(
                {models.Claim.object_entity_id: target.id}
            )
            session.query(models.Relation).filter(models.Relation.subject_entity_id == source.id).update(
                {models.Relation.subject_entity_id: target.id}
            )
            session.query(models.Relation).filter(models.Relation.object_entity_id == source.id).update(
                {models.Relation.object_entity_id: target.id}
            )
            source.metadata_json = {**(source.metadata_json or {}), "merged_into": target.id}
            source.updated_at = models.utcnow()
            return {"ok": True, "source_entity_id": source.id, "target_entity_id": target.id}

    @app.get("/projects/{project_name}/recommendation-tags")
    def recommendation_tags(project_name: str):
        with session_scope(database_url) as session:
            project = KnowledgeRepository(session).get_project(project_name)
            tag_predicates = {
                "hasTopNote",
                "hasMiddleNote",
                "hasBaseNote",
                "hasScentNote",
                "hasFlavorNote",
                "evokesMood",
                "suitableForSeason",
                "suitableForOccasion",
                "hasReviewKeyword",
            }
            rows = session.execute(
                select(models.Claim, models.Entity)
                .join(models.Entity, models.Claim.object_entity_id == models.Entity.id)
                .where(
                    models.Claim.project_id == project.id,
                    models.Claim.status == "validated_claim",
                    models.Claim.predicate.in_(tag_predicates),
                )
            ).all()
            grouped: dict[str, dict[str, Any]] = {}
            for claim, entity in rows:
                key = f"{claim.predicate}:{entity.canonical_name}"
                if key not in grouped:
                    grouped[key] = {
                        "predicate": claim.predicate,
                        "name": entity.name,
                        "type": entity.entity_type,
                        "support_count": 0,
                        "max_confidence": 0.0,
                    }
                grouped[key]["support_count"] += 1
                grouped[key]["max_confidence"] = max(grouped[key]["max_confidence"], claim.confidence)
            return sorted(grouped.values(), key=lambda item: (item["predicate"], -item["support_count"], item["name"]))

    @app.post("/recommend")
    def recommend(request: RecommendRequest):
        with session_scope(database_url) as session:
            repo = KnowledgeRepository(session)
            project = repo.get_project(request.project_name)
            recommender = RuleBasedRecommender(session)
            pref = PreferenceInput(**request.preferences)
            items = recommender.recommend(project.id, request.target_entity_type, pref, request.limit)
            return [asdict(item) for item in items]