AI/ontology_platform/crawler_platform/app/api/routes.py

from __future__ import annotations

import csv
import io
import json
import re
import time
from dataclasses import asdict
from typing import Any

from fastapi import BackgroundTasks, HTTPException, Response
from pydantic import BaseModel, Field
from sqlalchemy import select
from sqlalchemy.exc import OperationalError

from crawler_platform.app.config.loader import (
    ProjectConfig,
    SourceConfig,
    load_project_config,
    project_config_from_dict,
)
from crawler_platform.app.core.crawler.discovery import discover_links
from crawler_platform.app.core.crawler.fetchers import RobotsPolicy, make_fetcher
from crawler_platform.app.core.crawler.pipeline import CrawlPipeline
from crawler_platform.app.core.crawler.site_crawler import SiteCrawler
from crawler_platform.app.core.database import models
from crawler_platform.app.core.database.repository import (
    KnowledgeRepository,
    make_claim_hash,
)
from crawler_platform.app.core.database.session import session_scope
from crawler_platform.app.core.extractor.ai_provider import list_openai_compatible_models
from crawler_platform.app.core.extractor.factory import extractor_for_domain
from crawler_platform.app.core.ontology.definitions import DOMAIN_ONTOLOGIES, Ontology, ontology_for_domain
from crawler_platform.app.core.ontology.domain_discovery import DomainDiscoveryService
from crawler_platform.app.core.ontology.gap_detector import KnowledgeGapDetector
from crawler_platform.app.core.ontology.mapper import ontology_to_dict
from crawler_platform.app.core.ontology.registry import OntologyRegistry
from crawler_platform.app.core.recommendation.scorer import PreferenceInput, RuleBasedRecommender
from crawler_platform.app.core.research.graph_query import SemanticGraphQuery
from crawler_platform.app.core.research.graph_research_loop import GraphResearchLoop
from crawler_platform.app.core.research.memory_store import ResearchMemoryStore, research_session_payload


DOMAIN_NAME_PATTERN = re.compile(r"^[a-z0-9][a-z0-9_-]{1,79}$")
SITE_CRAWL_CANCEL_REQUESTS: set[int] = set()


class CrawlRequest(BaseModel):
    config_path: str
    source_name: str
    url: str
    extraction_mode: str = "hybrid"
    extractor_provider: str = "lm_studio"
    extractor_model: str | None = None
    extractor_base_url: str | None = "http://localhost:1234/v1"
    fallback_to_rules: bool = True
    check_robots_txt: bool = False
    respect_robots_txt: bool | None = None


class SiteCrawlRequest(CrawlRequest):
    max_depth: int = 2
    max_pages: int = 50
    same_domain_only: bool = True
    analyze_page_types: list[str] = Field(default_factory=lambda: ["ProductPage", "BrandStoryPage", "ReviewPage"])


class SiteCrawlByProjectRequest(BaseModel):
    """Site crawl invoked against an existing project (no filesystem config_path)."""

    project_name: str
    source_name: str
    url: str
    extraction_mode: str = "hybrid"
    extractor_provider: str = "lm_studio"
    extractor_model: str | None = None
    extractor_base_url: str | None = "http://localhost:1234/v1"
    fallback_to_rules: bool = True
    check_robots_txt: bool = False
    respect_robots_txt: bool | None = None
    max_depth: int = 2
    max_pages: int = 50
    same_domain_only: bool = True
    analyze_page_types: list[str] = Field(
        default_factory=lambda: ["ProductPage", "BrandStoryPage", "ReviewPage"]
    )

    def to_site_crawl_request(self, config_path_placeholder: str = "") -> "SiteCrawlRequest":
        """For internal handoff to existing crawl pipeline (config_path is not used)."""
        return SiteCrawlRequest(
            config_path=config_path_placeholder,
            source_name=self.source_name,
            url=self.url,
            extraction_mode=self.extraction_mode,
            extractor_provider=self.extractor_provider,
            extractor_model=self.extractor_model,
            extractor_base_url=self.extractor_base_url,
            fallback_to_rules=self.fallback_to_rules,
            check_robots_txt=self.check_robots_txt,
            respect_robots_txt=self.respect_robots_txt,
            max_depth=self.max_depth,
            max_pages=self.max_pages,
            same_domain_only=self.same_domain_only,
            analyze_page_types=list(self.analyze_page_types),
        )


class DiscoverRequest(BaseModel):
    config_path: str
    source_name: str
    url: str
    limit: int = 30
    check_robots_txt: bool = False
    respect_robots_txt: bool | None = None


class RecommendRequest(BaseModel):
    project_name: str
    target_entity_type: str = "Perfume"
    preferences: dict = Field(default_factory=dict)
    limit: int = 10


class CreateProjectRequest(BaseModel):
    config_path: str


class InlineSourceConfig(BaseModel):
    name: str
    type: str = "unknown"
    trust_level: float = 0.5
    base_url: str | None = None
    allowed_paths: list[str] = Field(default_factory=list)
    parser: str = "generic"
    fetcher: str = "requests"
    rate_limit_per_minute: int = 30
    respect_robots_txt: bool = False


class CreateProjectInlineRequest(BaseModel):
    """Create a project from inline JSON config (no filesystem dependency)."""

    project_name: str
    domain: str
    target_entities: list[str] = Field(default_factory=list)
    fields: list[str] = Field(default_factory=list)
    sources: list[InlineSourceConfig] = Field(default_factory=list)
    ontology: dict[str, Any] = Field(default_factory=dict)
    recommendation: dict[str, Any] = Field(default_factory=dict)
    update_policy: dict[str, Any] = Field(default_factory=dict)

    def to_project_config(self) -> ProjectConfig:
        return ProjectConfig(
            project_name=self.project_name,
            domain=self.domain,
            target_entities=list(self.target_entities),
            fields=list(self.fields),
            sources=[SourceConfig(**s.model_dump()) for s in self.sources],
            ontology=dict(self.ontology),
            recommendation=dict(self.recommendation),
            update_policy=dict(self.update_policy),
        )


class DomainDefinitionRequest(BaseModel):
    domain: str
    description: str | None = None
    entity_types: list[str] = Field(default_factory=list)
    predicates: list[str] = Field(default_factory=list)
    attributes: list[str] = Field(default_factory=list)
    aliases: dict[str, str] = Field(default_factory=dict)


class DomainDefinitionUpdateRequest(BaseModel):
    description: str | None = None
    entity_types: list[str] = Field(default_factory=list)
    predicates: list[str] = Field(default_factory=list)
    attributes: list[str] = Field(default_factory=list)
    aliases: dict[str, str] = Field(default_factory=dict)


class DomainDiscoveryRequest(BaseModel):
    seed_urls: list[str] = Field(default_factory=list)
    max_pages: int = 20
    max_depth: int = 1
    same_domain_only: bool = True
    fetcher: str = "requests"
    respect_robots_txt: bool = False
    force_recrawl: bool = False


class DomainCandidateApplyRequest(BaseModel):
    candidate_ids: list[int] = Field(default_factory=list)


class DomainCandidateStatusRequest(BaseModel):
    status: str


class ResetProjectRequest(BaseModel):
    config_path: str
    project_name: str | None = None


def source_model_to_config(source: models.Source) -> SourceConfig:
    return SourceConfig(
        name=source.name,
        type=source.type,
        trust_level=source.trust_level,
        base_url=source.base_url,
        rate_limit_per_minute=source.rate_limit_per_minute,
        respect_robots_txt=source.respect_robots_txt,
    )


def normalize_domain_name(domain: str) -> str:
    normalized = domain.strip().lower()
    if not DOMAIN_NAME_PATTERN.match(normalized):
        raise HTTPException(
            status_code=400,
            detail="Domain must be 2-80 chars using lowercase letters, numbers, underscore, or hyphen.",
        )
    return normalized


def clean_string_list(values: list[str]) -> list[str]:
    cleaned: list[str] = []
    seen: set[str] = set()
    for value in values:
        item = value.strip()
        if not item or item in seen:
            continue
        cleaned.append(item)
        seen.add(item)
    return cleaned


def clean_aliases(values: dict[str, str]) -> dict[str, str]:
    return {
        key.strip(): value.strip()
        for key, value in values.items()
        if key.strip() and value.strip()
    }


def domain_row_to_ontology(row: models.DomainDefinition):
    return Ontology(
        domain=row.domain,
        entity_types=clean_string_list(list(row.entity_types or [])),
        predicates=clean_string_list(list(row.predicates or [])),
        attributes=clean_string_list(list(row.attributes or [])),
        aliases=clean_aliases(dict(row.aliases or {})),
    )


def domain_summary_payload(ont, *, is_builtin: bool, is_custom: bool, description: str | None = None):
    return {
        "domain": ont.domain,
        "description": description,
        "entity_types": list(ont.entity_types),
        "predicates": list(ont.predicates),
        "attribute_count": len(ont.attributes),
        "attributes": list(ont.attributes),
        "aliases": dict(ont.aliases),
        "is_builtin": is_builtin,
        "is_custom": is_custom,
    }


def upsert_domain_definition(
    session,
    domain: str,
    payload: DomainDefinitionUpdateRequest,
) -> models.DomainDefinition:
    normalized_domain = normalize_domain_name(domain)
    row = session.scalar(
        select(models.DomainDefinition).where(models.DomainDefinition.domain == normalized_domain)
    )
    if row is None:
        row = models.DomainDefinition(domain=normalized_domain)
        session.add(row)
        session.flush()
    row.description = payload.description.strip() if payload.description else None
    row.entity_types = clean_string_list(payload.entity_types)
    row.predicates = clean_string_list(payload.predicates)
    row.attributes = clean_string_list(payload.attributes)
    row.aliases = clean_aliases(payload.aliases)
    row.is_builtin_override = normalized_domain in DOMAIN_ONTOLOGIES
    row.updated_at = models.utcnow()
    return row


def ensure_domain_definition(session, domain: str) -> models.DomainDefinition:
    normalized_domain = normalize_domain_name(domain)
    row = session.scalar(
        select(models.DomainDefinition).where(models.DomainDefinition.domain == normalized_domain)
    )
    if row is not None:
        return row

    base = DOMAIN_ONTOLOGIES.get(normalized_domain)
    row = models.DomainDefinition(
        domain=normalized_domain,
        entity_types=list(base.entity_types) if base else [],
        predicates=list(base.predicates) if base else [],
        attributes=list(base.attributes) if base else [],
        aliases=dict(base.aliases) if base else {},
        is_builtin_override=normalized_domain in DOMAIN_ONTOLOGIES,
    )
    session.add(row)
    session.flush()
    return row


def ontology_for_domain_from_db(session, domain: str):
    normalized_domain = normalize_domain_name(domain)
    row = session.scalar(
        select(models.DomainDefinition).where(models.DomainDefinition.domain == normalized_domain)
    )
    if row is not None:
        return domain_row_to_ontology(row)
    return ontology_for_domain(normalized_domain)


def clean_seed_urls(values: list[str]) -> list[str]:
    cleaned: list[str] = []
    seen: set[str] = set()
    for value in values:
        url = value.strip()
        if not url or url in seen:
            continue
        if not (url.startswith("http://") or url.startswith("https://") or url.startswith("file://")):
            raise HTTPException(status_code=400, detail=f"Unsupported URL: {url}")
        cleaned.append(url)
        seen.add(url)
    if not cleaned:
        raise HTTPException(status_code=400, detail="At least one reference URL is required.")
    return cleaned


def domain_discovery_job_payload(job: models.DomainDiscoveryJob) -> dict[str, Any]:
    return {
        "job_id": job.id,
        "domain": job.domain,
        "status": job.status,
        "seed_urls": list(job.seed_urls or []),
        "max_pages": job.max_pages,
        "max_depth": job.max_depth,
        "same_domain_only": job.same_domain_only,
        "fetcher": job.fetcher,
        "respect_robots_txt": job.respect_robots_txt,
        "error": job.error,
        "progress": job.progress or {},
        "result_summary": job.result_summary or {},
        "created_at": job.created_at.isoformat() if job.created_at else None,
        "started_at": job.started_at.isoformat() if job.started_at else None,
        "finished_at": job.finished_at.isoformat() if job.finished_at else None,
    }


def domain_reference_source_payload(row: models.DomainReferenceSource) -> dict[str, Any]:
    return {
        "id": row.id,
        "domain": row.domain,
        "url": row.url,
        "label": row.label,
        "status": row.status,
        "last_crawled_at": row.last_crawled_at.isoformat() if row.last_crawled_at else None,
        "metadata": row.metadata_json or {},
        "created_at": row.created_at.isoformat() if row.created_at else None,
        "updated_at": row.updated_at.isoformat() if row.updated_at else None,
    }


def already_crawled_reference_urls(session, domain: str, seed_urls: list[str]) -> set[str]:
    if not seed_urls:
        return set()
    rows = session.scalars(
        select(models.DomainReferenceSource).where(
            models.DomainReferenceSource.domain == domain,
            models.DomainReferenceSource.url.in_(seed_urls),
            models.DomainReferenceSource.status == "active",
            models.DomainReferenceSource.last_crawled_at.is_not(None),
        )
    ).all()
    return {row.url for row in rows}


def domain_candidate_payload(row: models.DomainSchemaCandidate, evidence_rows: list[models.DomainCandidateEvidence]) -> dict[str, Any]:
    return {
        "id": row.id,
        "domain": row.domain,
        "job_id": row.job_id,
        "candidate_type": row.candidate_type,
        "name": row.name,
        "description": row.description,
        "confidence": row.confidence,
        "occurrence_count": row.occurrence_count,
        "status": row.status,
        "metadata": row.metadata_json or {},
        "created_at": row.created_at.isoformat() if row.created_at else None,
        "updated_at": row.updated_at.isoformat() if row.updated_at else None,
        "evidence": [
            {
                "id": evidence.id,
                "job_id": evidence.job_id,
                "url": evidence.url,
                "title": evidence.title,
                "snippet": evidence.snippet,
                "metadata": evidence.metadata_json or {},
                "created_at": evidence.created_at.isoformat() if evidence.created_at else None,
            }
            for evidence in evidence_rows
        ],
    }


def run_domain_discovery_job(database_url: str, job_id: int) -> None:
    try:
        with session_scope(database_url) as session:
            DomainDiscoveryService(session).run_job(job_id)
    except Exception as exc:
        with session_scope(database_url) as session:
            job = session.get(models.DomainDiscoveryJob, job_id)
            if job is not None:
                job.status = "failed"
                job.error = str(exc)
                job.finished_at = models.utcnow()
                progress = dict(job.progress or {})
                progress["errors"] = [*progress.get("errors", []), str(exc)]
                job.progress = progress


def recover_interrupted_domain_discovery_jobs(database_url: str) -> None:
    """Mark domain discovery jobs that cannot survive a server restart."""
    with session_scope(database_url) as session:
        rows = session.scalars(
            select(models.DomainDiscoveryJob).where(
                models.DomainDiscoveryJob.status.in_(
                    ["pending", "running", "cancel_requested"]
                )
            )
        ).all()
        for job in rows:
            progress = dict(job.progress or {})
            errors = list(progress.get("errors") or [])
            if job.status == "cancel_requested":
                job.status = "canceled"
                progress["completion_reason"] = "canceled"
            else:
                job.status = "failed"
                job.error = "Server restarted before this background analysis finished."
                progress["completion_reason"] = "interrupted"
                errors.append(job.error)
            progress["errors"] = errors[-10:]
            job.progress = progress
            job.finished_at = models.utcnow()


def project_config_from_project_row(session, project: models.Project) -> ProjectConfig:
    config_dict = dict(project.config or {})
    if not config_dict:
        raise HTTPException(
            status_code=400,
            detail=f"Project '{project.name}' has no stored config",
        )
    config = project_config_from_dict(config_dict)
    config.sources = [
        source_model_to_config(source)
        for source in session.scalars(
            select(models.Source).where(models.Source.project_id == project.id)
        ).all()
    ]
    return config


def project_config_to_dict(config: ProjectConfig) -> dict[str, Any]:
    return {
        "project_name": config.project_name,
        "domain": config.domain,
        "target_entities": list(config.target_entities),
        "fields": list(config.fields),
        "sources": [asdict(source) for source in config.sources],
        "ontology": dict(config.ontology),
        "recommendation": dict(config.recommendation),
        "update_policy": dict(config.update_policy),
    }


def delete_project_data(session, project_id: int, include_project: bool = False) -> dict[str, int]:
    repo = KnowledgeRepository(session)
    deleted = repo.reset_project_runtime_data(project_id)
    for table_model in [
        models.OntologyProposal,
        models.OntologyTriple,
        models.OntologyRelationType,
        models.OntologyEntityType,
        models.Source,
    ]:
        count = (
            session.query(table_model)
            .filter(table_model.project_id == project_id)
            .delete(synchronize_session=False)
        )
        deleted[table_model.__tablename__] = deleted.get(table_model.__tablename__, 0) + int(count or 0)
    if include_project:
        project_count = (
            session.query(models.Project)
            .filter(models.Project.id == project_id)
            .delete(synchronize_session=False)
        )
        deleted[models.Project.__tablename__] = int(project_count or 0)
    return deleted


class CreateEntityRequest(BaseModel):
    entity_type: str
    name: str
    metadata: dict[str, Any] = Field(default_factory=dict)


class BulkCreateEntitiesRequest(BaseModel):
    entities: list[CreateEntityRequest]


class CreateClaimRequest(BaseModel):
    source_name: str
    subject_entity_id: int
    predicate: str
    object_entity_id: int | None = None
    object_value: Any = None
    confidence: float = 1.0
    confidence_reason: str | None = None
    evidence_text: str | None = None
    metadata: dict[str, Any] = Field(default_factory=dict)


class ResearchRunByProjectRequest(BaseModel):
    """Run autonomous research against an existing DB project (no config_path)."""

    project_name: str
    source_name: str
    url: str | None = None
    seed_entity_id: int | None = None
    goal: str = "Semantic ontology exploration"
    extraction_mode: str = "hybrid"
    extractor_provider: str = "lm_studio"
    extractor_model: str | None = None
    extractor_base_url: str | None = "http://localhost:1234/v1"
    fallback_to_rules: bool = True
    check_robots_txt: bool = False
    respect_robots_txt: bool | None = None
    max_depth: int = 2
    max_steps: int = 12
    max_branch: int = 8
    min_relevance: float = 0.35
    same_domain_only: bool = True
    analyze_page_types: list[str] = Field(
        default_factory=lambda: ["ProductPage", "BrandStoryPage", "ReviewPage"]
    )


class UpdateClaimConfidenceRequest(BaseModel):
    confidence: float
    reason: str | None = None


class UpdateClaimStatusRequest(BaseModel):
    status: str
    reason: str | None = None


class BulkClaimStatusRequest(BaseModel):
    claim_ids: list[int]
    status: str
    reason: str | None = None


class SchemaEntityTypeRequest(BaseModel):
    name: str
    domain: str = "generic"
    description: str | None = None
    status: str = "active"
    confidence: float = 1.0
    metadata: dict[str, Any] = Field(default_factory=dict)


class SchemaRelationTypeRequest(BaseModel):
    name: str
    domain: str = "generic"
    description: str | None = None
    allowed_subject_types: list[str] = Field(default_factory=list)
    allowed_object_types: list[str] = Field(default_factory=list)
    allowed_page_types: list[str] = Field(default_factory=list)
    allowed_source_zones: list[str] = Field(default_factory=list)
    semantic_constraints: dict[str, Any] = Field(default_factory=dict)
    confidence_rules: dict[str, Any] = Field(default_factory=dict)
    min_confidence: float | None = None
    status: str = "active"
    confidence: float = 1.0
    metadata: dict[str, Any] = Field(default_factory=dict)


class PipelineRerunRequest(BaseModel):
    stage: str


class MergeEntitiesRequest(BaseModel):
    project_name: str
    source_entity_id: int
    target_entity_id: int


class ExtractorModelsRequest(BaseModel):
    provider: str
    base_url: str | None = None


class ResearchRunRequest(CrawlRequest):
    project_name: str | None = None
    seed_entity_id: int | None = None
    goal: str = "Semantic ontology exploration"
    max_depth: int = 2
    max_steps: int = 12
    max_branch: int = 8
    min_relevance: float = 0.35
    same_domain_only: bool = True
    analyze_page_types: list[str] = Field(default_factory=lambda: ["ProductPage", "BrandStoryPage", "ReviewPage"])


def site_crawl_progress_payload(result, latest_page=None) -> dict[str, Any]:
    payload = asdict(result)
    if latest_page is not None:
        payload["latest_page"] = asdict(latest_page)
    pages = payload.get("pages") or []
    payload["extraction_summary"] = {
        "llm_skipped_count": sum(1 for page in pages if page.get("llm_skipped")),
        "fallback_count": sum(1 for page in pages if page.get("fallback_used")),
        "conflict_claim_count": sum(int(page.get("conflict_claim_count") or 0) for page in pages),
        "agreement_claim_count": sum(int(page.get("agreement_claim_count") or 0) for page in pages),
        "llm_call_count": sum(
            1
            for page in pages
            if page.get("extraction_mode") in {"hybrid", "llm_only", "compare"}
            and not page.get("llm_skipped")
            and not page.get("fallback_used")
        ),
    }
    return payload


_CLAIM_STATUS_ALIASES = {
    "accept": "validated_claim",
    "accepted": "validated_claim",
    "validated": "validated_claim",
    "validated_claim": "validated_claim",
    "reject": "rejected",
    "rejected": "rejected",
    "active": "active",
    "candidate": "active",
    "pending": "active",
}


def _normalize_claim_status(value: str | None) -> str | None:
    if not value:
        return None
    return _CLAIM_STATUS_ALIASES.get(value.strip().lower())


def _apply_claim_review(claim: models.Claim, status: str, reason: str | None) -> None:
    claim.status = status
    if reason:
        claim.confidence_reason = reason
    metadata = dict(claim.metadata_json or {})
    review = dict(metadata.get("review") or {})
    review["status"] = status
    review["reason"] = reason or review.get("reason")
    review["reviewed_at"] = models.utcnow().isoformat()
    metadata["review"] = review
    claim.metadata_json = metadata
    claim.last_seen_at = models.utcnow()


def turtle_id(value: Any) -> str:
    text = "".join(ch if ch.isalnum() else "_" for ch in str(value or "").strip())
    text = "_".join(part for part in text.split("_") if part)
    if not text:
        return "value"
    if text[0].isdigit():
        text = f"n_{text}"
    return text[:120]


def apply_crawl_request_overrides(config, request: CrawlRequest | DiscoverRequest) -> None:
    check_robots_txt = request.respect_robots_txt
    if check_robots_txt is None:
        check_robots_txt = request.check_robots_txt
    config.source_by_name(request.source_name).respect_robots_txt = check_robots_txt


def crawl_job_response(job: models.CrawlJob) -> dict[str, Any]:
    metadata = job.metadata_json or {}
    return {
        "job_id": job.id,
        "status": job.status,
        "url": job.url,
        "error": job.error,
        "scheduled_at": job.scheduled_at.isoformat() if job.scheduled_at else None,
        "started_at": job.started_at.isoformat() if job.started_at else None,
        "finished_at": job.finished_at.isoformat() if job.finished_at else None,
        "progress": metadata.get("progress", {}),
        "request": metadata.get("request", {}),
    }


def update_site_crawl_job_metadata(job: models.CrawlJob, **updates: Any) -> None:
    metadata = dict(job.metadata_json or {})
    metadata.update(updates)
    job.metadata_json = metadata


def mark_orphan_site_crawl_jobs_canceled(database_url: str) -> None:
    with session_scope(database_url) as session:
        rows = session.scalars(
            select(models.CrawlJob).where(
                models.CrawlJob.status.in_(["running", "cancel_requested"]),
            )
        ).all()
        for job in rows:
            if (job.metadata_json or {}).get("kind") != "site_crawl":
                continue
            job.status = "canceled"
            job.error = job.error or "canceled after server restart/interrupted crawl"
            job.finished_at = models.utcnow()


def request_site_crawl_cancel(database_url: str, job_id: int) -> dict[str, Any]:
    SITE_CRAWL_CANCEL_REQUESTS.add(job_id)
    last_error: Exception | None = None
    for attempt in range(3):
        try:
            with session_scope(database_url) as session:
                job = session.get(models.CrawlJob, job_id)
                if job is None or (job.metadata_json or {}).get("kind") != "site_crawl":
                    raise HTTPException(status_code=404, detail="site crawl job not found")
                if job.status in {"completed", "failed", "canceled"}:
                    SITE_CRAWL_CANCEL_REQUESTS.discard(job_id)
                    return crawl_job_response(job)
                job.status = "cancel_requested"
                job.error = "cancel requested by user"
                payload = crawl_job_response(job)
            return payload
        except OperationalError as exc:
            last_error = exc
            if "database is locked" not in str(exc).lower():
                raise
            time.sleep(0.25 * (attempt + 1))
    return {
        "job_id": job_id,
        "status": "cancel_requested",
        "url": None,
        "error": f"cancel requested in memory; database was locked: {last_error}",
        "scheduled_at": None,
        "started_at": None,
        "finished_at": None,
        "progress": {},
        "request": {},
    }


def is_site_crawl_cancel_requested(session, job_id: int) -> bool:
    if job_id in SITE_CRAWL_CANCEL_REQUESTS:
        return True
    session.expire_all()
    job = session.get(models.CrawlJob, job_id)
    return job is None or job.status == "cancel_requested"


def run_site_crawl_job(database_url: str, job_id: int, request_data: dict[str, Any]) -> None:
    inline_config = request_data.pop("__config_dict", None)
    request = SiteCrawlRequest(**request_data)
    try:
        if inline_config is not None:
            config = project_config_from_dict(inline_config)
        else:
            config = load_project_config(request.config_path)
        apply_crawl_request_overrides(config, request)
        with session_scope(database_url) as session:
            job = session.get(models.CrawlJob, job_id)
            if job is None:
                return
            if job.status == "cancel_requested":
                job.status = "canceled"
                job.finished_at = models.utcnow()
                return

            job.status = "running"
            job.started_at = models.utcnow()
            update_site_crawl_job_metadata(
                job,
                progress={
                    "seed_url": request.url,
                    "visited_count": 0,
                    "analyzed_count": 0,
                    "queued_count": 1,
                    "skipped_count": 0,
                    "errors": [],
                    "pages": [],
                },
            )
            session.commit()

            repo = KnowledgeRepository(session)
            crawler = SiteCrawler(
                repo,
                extractor_for_domain(
                    config.domain,
                    provider=request.extractor_provider,
                    model=request.extractor_model,
                    base_url=request.extractor_base_url,
                    extraction_mode=request.extraction_mode,
                    fallback_to_rules=request.fallback_to_rules,
                ),
            )

            def save_progress(result, latest_page) -> None:
                current_job = session.get(models.CrawlJob, job_id)
                if current_job is None:
                    return
                update_site_crawl_job_metadata(
                    current_job,
                    progress=site_crawl_progress_payload(result, latest_page),
                )
                session.commit()

            def should_stop() -> bool:
                return is_site_crawl_cancel_requested(session, job_id)

            result = crawler.crawl_site(
                config,
                request.source_name,
                request.url,
                max_depth=max(request.max_depth, 0),
                max_pages=max(min(request.max_pages, 500), 1),
                same_domain_only=request.same_domain_only,
                analyze_page_types=set(request.analyze_page_types),
                progress_callback=save_progress,
                should_stop=should_stop,
                parent_job_id=job_id,
            )

            finished_job = session.get(models.CrawlJob, job_id)
            if finished_job is not None:
                finished_job.status = (
                    "canceled"
                    if finished_job.status == "cancel_requested" or job_id in SITE_CRAWL_CANCEL_REQUESTS
                    else "completed"
                )
                finished_job.finished_at = models.utcnow()
                update_site_crawl_job_metadata(
                    finished_job,
                    progress=site_crawl_progress_payload(result),
                )
            SITE_CRAWL_CANCEL_REQUESTS.discard(job_id)
    except Exception as exc:
        with session_scope(database_url) as session:
            job = session.get(models.CrawlJob, job_id)
            if job is not None:
                job.status = "failed"
                job.error = str(exc)
                job.finished_at = models.utcnow()
                metadata = dict(job.metadata_json or {})
                progress = dict(metadata.get("progress") or {})
                progress["errors"] = [*progress.get("errors", []), str(exc)]
                update_site_crawl_job_metadata(job, progress=progress)
        SITE_CRAWL_CANCEL_REQUESTS.discard(job_id)


def register_routes(app, database_url: str) -> None:
    recover_interrupted_domain_discovery_jobs(database_url)
    mark_orphan_site_crawl_jobs_canceled(database_url)

    @app.get("/health")
    def health():
        return {"ok": True}

    @app.get("/projects")
    def projects():
        with session_scope(database_url) as session:
            rows = session.scalars(select(models.Project).order_by(models.Project.created_at.desc())).all()
            return [
                {
                    "id": project.id,
                    "name": project.name,
                    "domain": project.domain,
                    "created_at": project.created_at.isoformat(),
                    "updated_at": project.updated_at.isoformat(),
                }
                for project in rows
            ]

    @app.post("/projects")
    def create_project(request: CreateProjectRequest):
        config = load_project_config(request.config_path)
        with session_scope(database_url) as session:
            project = KnowledgeRepository(session).upsert_project(config)
            return {"id": project.id, "name": project.name, "domain": project.domain}

    @app.post("/projects/inline")
    def create_project_inline(request: CreateProjectInlineRequest):
        config = request.to_project_config()
        with session_scope(database_url) as session:
            project = KnowledgeRepository(session).upsert_project(config)
            return {"id": project.id, "name": project.name, "domain": project.domain}

    @app.get("/domains")
    def list_domains():
        """Available ontology domains for project creation."""
        with session_scope(database_url) as session:
            rows = session.scalars(select(models.DomainDefinition)).all()
            custom_by_domain = {row.domain: row for row in rows}
            payloads = []
            for domain, built_in in DOMAIN_ONTOLOGIES.items():
                row = custom_by_domain.pop(domain, None)
                if row is not None:
                    payloads.append(
                        domain_summary_payload(
                            domain_row_to_ontology(row),
                            is_builtin=True,
                            is_custom=True,
                            description=row.description,
                        )
                    )
                else:
                    payloads.append(
                        domain_summary_payload(
                            built_in,
                            is_builtin=True,
                            is_custom=False,
                        )
                    )
            for row in sorted(custom_by_domain.values(), key=lambda item: item.domain):
                payloads.append(
                    domain_summary_payload(
                        domain_row_to_ontology(row),
                        is_builtin=False,
                        is_custom=True,
                        description=row.description,
                    )
                )
            return payloads

    @app.post("/domains")
    def create_domain(request: DomainDefinitionRequest):
        domain = normalize_domain_name(request.domain)
        with session_scope(database_url) as session:
            existing = session.scalar(
                select(models.DomainDefinition).where(models.DomainDefinition.domain == domain)
            )
            if existing is not None or domain in DOMAIN_ONTOLOGIES:
                raise HTTPException(status_code=409, detail=f"Domain '{domain}' already exists.")
            row = upsert_domain_definition(
                session,
                domain,
                DomainDefinitionUpdateRequest(
                    description=request.description,
                    entity_types=request.entity_types,
                    predicates=request.predicates,
                    attributes=request.attributes,
                    aliases=request.aliases,
                ),
            )
            return domain_summary_payload(
                domain_row_to_ontology(row),
                is_builtin=domain in DOMAIN_ONTOLOGIES,
                is_custom=True,
                description=row.description,
            )

    @app.put("/domains/{domain}")
    def update_domain(domain: str, request: DomainDefinitionUpdateRequest):
        with session_scope(database_url) as session:
            row = upsert_domain_definition(session, domain, request)
            return domain_summary_payload(
                domain_row_to_ontology(row),
                is_builtin=row.domain in DOMAIN_ONTOLOGIES,
                is_custom=True,
                description=row.description,
            )

    @app.get("/domains/{domain}/discovery/jobs")
    def list_domain_discovery_jobs(domain: str, limit: int = 20):
        normalized_domain = normalize_domain_name(domain)
        with session_scope(database_url) as session:
            rows = session.scalars(
                select(models.DomainDiscoveryJob)
                .where(models.DomainDiscoveryJob.domain == normalized_domain)
                .order_by(models.DomainDiscoveryJob.created_at.desc())
                .limit(max(min(limit, 100), 1))
            ).all()
            return [domain_discovery_job_payload(row) for row in rows]

    @app.get("/domains/{domain}/reference-sources")
    def list_domain_reference_sources(domain: str):
        normalized_domain = normalize_domain_name(domain)
        with session_scope(database_url) as session:
            rows = session.scalars(
                select(models.DomainReferenceSource)
                .where(models.DomainReferenceSource.domain == normalized_domain)
                .order_by(models.DomainReferenceSource.updated_at.desc())
            ).all()
            return [domain_reference_source_payload(row) for row in rows]

    @app.post("/domains/{domain}/discovery/jobs")
    def create_domain_discovery_job(
        domain: str,
        request: DomainDiscoveryRequest,
        background_tasks: BackgroundTasks,
    ):
        normalized_domain = normalize_domain_name(domain)
        seed_urls = clean_seed_urls(request.seed_urls)
        with session_scope(database_url) as session:
            ensure_domain_definition(session, normalized_domain)
            skipped_seed_urls: list[str] = []
            if not request.force_recrawl:
                skipped = already_crawled_reference_urls(session, normalized_domain, seed_urls)
                skipped_seed_urls = [url for url in seed_urls if url in skipped]
                seed_urls = [url for url in seed_urls if url not in skipped]

            if not seed_urls:
                job = models.DomainDiscoveryJob(
                    domain=normalized_domain,
                    status="completed",
                    seed_urls=[],
                    max_pages=max(min(request.max_pages, 200), 1),
                    max_depth=max(min(request.max_depth, 5), 0),
                    same_domain_only=request.same_domain_only,
                    fetcher=request.fetcher if request.fetcher in {"requests", "playwright", "browser"} else "requests",
                    respect_robots_txt=request.respect_robots_txt,
                    progress={
                        "visited_count": 0,
                        "queued_count": 0,
                        "candidate_count": 0,
                        "observation_count": 0,
                        "max_pages": max(min(request.max_pages, 200), 1),
                        "skipped_seed_urls": skipped_seed_urls,
                        "errors": [],
                        "pages": [],
                    },
                    result_summary={
                        "visited_count": 0,
                        "candidate_count": 0,
                        "skipped_seed_urls": skipped_seed_urls,
                    },
                    finished_at=models.utcnow(),
                )
                session.add(job)
                session.flush()
                return domain_discovery_job_payload(job)

            job = models.DomainDiscoveryJob(
                domain=normalized_domain,
                status="pending",
                seed_urls=seed_urls,
                max_pages=max(min(request.max_pages, 200), 1),
                max_depth=max(min(request.max_depth, 5), 0),
                same_domain_only=request.same_domain_only,
                fetcher=request.fetcher if request.fetcher in {"requests", "playwright", "browser"} else "requests",
                respect_robots_txt=request.respect_robots_txt,
                progress={
                    "visited_count": 0,
                    "queued_count": len(seed_urls),
                    "candidate_count": 0,
                    "observation_count": 0,
                    "max_pages": max(min(request.max_pages, 200), 1),
                    "skipped_seed_urls": skipped_seed_urls,
                    "errors": [],
                    "pages": [],
                },
            )
            session.add(job)
            session.flush()
            response = domain_discovery_job_payload(job)
        background_tasks.add_task(run_domain_discovery_job, database_url, response["job_id"])
        return response

    @app.get("/domains/{domain}/discovery/jobs/{job_id}")
    def domain_discovery_job(domain: str, job_id: int):
        normalized_domain = normalize_domain_name(domain)
        with session_scope(database_url) as session:
            job = session.get(models.DomainDiscoveryJob, job_id)
            if job is None or job.domain != normalized_domain:
                raise HTTPException(status_code=404, detail="domain discovery job not found")
            return domain_discovery_job_payload(job)

    @app.post("/domains/{domain}/discovery/jobs/{job_id}/cancel")
    def cancel_domain_discovery_job(domain: str, job_id: int):
        normalized_domain = normalize_domain_name(domain)
        with session_scope(database_url) as session:
            job = session.get(models.DomainDiscoveryJob, job_id)
            if job is None or job.domain != normalized_domain:
                raise HTTPException(status_code=404, detail="domain discovery job not found")
            if job.status in {"completed", "failed", "canceled"}:
                return domain_discovery_job_payload(job)
            job.status = "cancel_requested"
            job.error = "cancel requested by user"
            return domain_discovery_job_payload(job)

    @app.get("/domains/{domain}/candidates")
    def list_domain_candidates(domain: str, status: str | None = None, limit: int = 200):
        normalized_domain = normalize_domain_name(domain)
        with session_scope(database_url) as session:
            query = select(models.DomainSchemaCandidate).where(
                models.DomainSchemaCandidate.domain == normalized_domain
            )
            if status:
                query = query.where(models.DomainSchemaCandidate.status == status)
            rows = session.scalars(
                query.order_by(
                    models.DomainSchemaCandidate.status,
                    models.DomainSchemaCandidate.candidate_type,
                    models.DomainSchemaCandidate.confidence.desc(),
                    models.DomainSchemaCandidate.name,
                ).limit(max(min(limit, 500), 1))
            ).all()
            payload = []
            for row in rows:
                evidence = session.scalars(
                    select(models.DomainCandidateEvidence)
                    .where(models.DomainCandidateEvidence.candidate_id == row.id)
                    .order_by(models.DomainCandidateEvidence.created_at.desc())
                    .limit(3)
                ).all()
                payload.append(domain_candidate_payload(row, evidence))
            return payload

    @app.post("/domains/{domain}/candidates/apply")
    def apply_domain_candidates(domain: str, request: DomainCandidateApplyRequest):
        normalized_domain = normalize_domain_name(domain)
        if not request.candidate_ids:
            raise HTTPException(status_code=400, detail="No candidates selected.")
        with session_scope(database_url) as session:
            definition = ensure_domain_definition(session, normalized_domain)
            rows = session.scalars(
                select(models.DomainSchemaCandidate).where(
                    models.DomainSchemaCandidate.domain == normalized_domain,
                    models.DomainSchemaCandidate.id.in_(request.candidate_ids),
                )
            ).all()
            entity_types = set(definition.entity_types or [])
            predicates = set(definition.predicates or [])
            attributes = set(definition.attributes or [])
            aliases = dict(definition.aliases or {})
            applied = {"entity_type": 0, "predicate": 0, "attribute": 0, "alias": 0}
            for row in rows:
                if row.candidate_type == "entity_type":
                    entity_types.add(row.name)
                    applied["entity_type"] += 1
                elif row.candidate_type == "predicate":
                    predicates.add(row.name)
                    applied["predicate"] += 1
                elif row.candidate_type == "attribute":
                    attributes.add(row.name)
                    applied["attribute"] += 1
                elif row.candidate_type == "alias":
                    metadata = dict(row.metadata_json or {})
                    targets = metadata.get("targets") or []
                    target = metadata.get("target") or (targets[0] if targets else None)
                    if target:
                        aliases[row.name] = str(target)
                        applied["alias"] += 1
                row.status = "approved"
                row.updated_at = models.utcnow()
            definition.entity_types = sorted(entity_types)
            definition.predicates = sorted(predicates)
            definition.attributes = sorted(attributes)
            definition.aliases = aliases
            definition.is_builtin_override = normalized_domain in DOMAIN_ONTOLOGIES
            definition.updated_at = models.utcnow()
            return {
                "ok": True,
                "domain": normalized_domain,
                "applied": applied,
                "definition": domain_summary_payload(
                    domain_row_to_ontology(definition),
                    is_builtin=normalized_domain in DOMAIN_ONTOLOGIES,
                    is_custom=True,
                    description=definition.description,
                ),
            }

    @app.post("/domains/{domain}/candidates/{candidate_id}/status")
    def update_domain_candidate_status(domain: str, candidate_id: int, request: DomainCandidateStatusRequest):
        normalized_domain = normalize_domain_name(domain)
        if request.status not in {"pending_review", "approved", "rejected"}:
            raise HTTPException(status_code=400, detail="Unsupported candidate status.")
        with session_scope(database_url) as session:
            row = session.get(models.DomainSchemaCandidate, candidate_id)
            if row is None or row.domain != normalized_domain:
                raise HTTPException(status_code=404, detail="candidate not found")
            row.status = request.status
            row.updated_at = models.utcnow()
            evidence = session.scalars(
                select(models.DomainCandidateEvidence)
                .where(models.DomainCandidateEvidence.candidate_id == row.id)
                .order_by(models.DomainCandidateEvidence.created_at.desc())
                .limit(3)
            ).all()
            return domain_candidate_payload(row, evidence)

    @app.post("/projects/reset")
    def reset_project(request: ResetProjectRequest):
        config = load_project_config(request.config_path)
        if request.project_name and request.project_name != config.project_name:
            raise HTTPException(
                status_code=400,
                detail=(
                    f"Selected project '{request.project_name}' does not match "
                    f"config project '{config.project_name}'."
                ),
            )

        with session_scope(database_url) as session:
            repo = KnowledgeRepository(session)
            project = session.scalar(select(models.Project).where(models.Project.name == config.project_name))
            if project is None:
                project = repo.upsert_project(config)
                return {
                    "ok": True,
                    "name": project.name,
                    "domain": project.domain,
                    "created": True,
                    "reset": False,
                    "deleted": {},
                }

            deleted = repo.reset_project_runtime_data(project.id)
            project = repo.upsert_project(config)
            return {
                "ok": True,
                "name": project.name,
                "domain": project.domain,
                "created": False,
                "reset": True,
                "deleted": deleted,
            }

    @app.post("/projects/{project_name}/reset")
    def reset_project_by_name(project_name: str):
        with session_scope(database_url) as session:
            repo = KnowledgeRepository(session)
            project = repo.get_project(project_name)
            config = project_config_from_project_row(session, project)
            deleted = repo.reset_project_runtime_data(project.id)
            project = repo.upsert_project(config)
            return {
                "ok": True,
                "name": project.name,
                "domain": project.domain,
                "reset": True,
                "deleted": deleted,
            }

    @app.delete("/projects/{project_name}")
    def delete_project(project_name: str):
        with session_scope(database_url) as session:
            project = KnowledgeRepository(session).get_project(project_name)
            domain = project.domain
            deleted = delete_project_data(session, project.id, include_project=True)
            return {
                "ok": True,
                "name": project_name,
                "domain": domain,
                "deleted": deleted,
            }

    @app.get("/projects/{project_name}")
    def project_detail(project_name: str):
        with session_scope(database_url) as session:
            project = KnowledgeRepository(session).get_project(project_name)
            sources = session.scalars(select(models.Source).where(models.Source.project_id == project.id)).all()
            return {
                "id": project.id,
                "name": project.name,
                "domain": project.domain,
                "config": project_config_to_dict(project_config_from_project_row(session, project)),
                "sources": [
                    {
                        "id": source.id,
                        "name": source.name,
                        "type": source.type,
                        "base_url": source.base_url,
                        "trust_level": source.trust_level,
                        "respect_robots_txt": source.respect_robots_txt,
                        "rate_limit_per_minute": source.rate_limit_per_minute,
                    }
                    for source in sources
                ],
            }

    @app.post("/projects/{project_name}/sources")
    def add_project_source(project_name: str, request: InlineSourceConfig):
        """Add or update a source on an existing project."""
        with session_scope(database_url) as session:
            repo = KnowledgeRepository(session)
            project = repo.get_project(project_name)
            source = repo.upsert_source(project, SourceConfig(**request.model_dump()))
            return {
                "id": source.id,
                "name": source.name,
                "type": source.type,
                "base_url": source.base_url,
                "trust_level": source.trust_level,
                "respect_robots_txt": source.respect_robots_txt,
                "rate_limit_per_minute": source.rate_limit_per_minute,
            }

    @app.delete("/projects/{project_name}/sources/{source_name}")
    def delete_project_source(project_name: str, source_name: str):
        with session_scope(database_url) as session:
            repo = KnowledgeRepository(session)
            project = repo.get_project(project_name)
            source = session.scalar(
                select(models.Source).where(
                    models.Source.project_id == project.id,
                    models.Source.name == source_name,
                )
            )
            if source is None:
                raise HTTPException(
                    status_code=404,
                    detail=f"Source '{source_name}' not found in project '{project_name}'",
                )
            session.delete(source)
            return {"ok": True, "deleted": source_name}

    @app.get("/ontology/{domain}")
    def ontology(domain: str):
        with session_scope(database_url) as session:
            return ontology_to_dict(ontology_for_domain_from_db(session, domain))

    @app.get("/projects/{project_name}/ontology/registry")
    def ontology_registry(project_name: str):
        with session_scope(database_url) as session:
            project = KnowledgeRepository(session).get_project(project_name)
            return OntologyRegistry(session).registry_payload(project.id)

    @app.post("/projects/{project_name}/schema/entity-types")
    def create_schema_entity_type(project_name: str, request: SchemaEntityTypeRequest):
        with session_scope(database_url) as session:
            project = KnowledgeRepository(session).get_project(project_name)
            row = OntologyRegistry(session).upsert_entity_type(
                project.id,
                name=request.name,
                domain=request.domain,
                description=request.description,
                status=request.status,
                confidence=min(max(request.confidence, 0.0), 1.0),
                metadata={"origin": "manual_schema_designer", **request.metadata},
            )
            return {
                "id": row.id,
                "name": row.name,
                "domain": row.domain,
                "description": row.description,
                "status": row.status,
                "confidence": row.confidence,
            }

    @app.post("/projects/{project_name}/schema/relation-types")
    def create_schema_relation_type(project_name: str, request: SchemaRelationTypeRequest):
        confidence_rules = dict(request.confidence_rules or {})
        if request.min_confidence is not None:
            confidence_rules["min_confidence"] = min(max(request.min_confidence, 0.0), 1.0)
        with session_scope(database_url) as session:
            project = KnowledgeRepository(session).get_project(project_name)
            row = OntologyRegistry(session).upsert_relation_type(
                project.id,
                name=request.name,
                domain=request.domain,
                description=request.description,
                allowed_subject_types=request.allowed_subject_types,
                allowed_object_types=request.allowed_object_types,
                allowed_page_types=request.allowed_page_types,
                allowed_source_zones=request.allowed_source_zones,
                semantic_constraints=request.semantic_constraints,
                confidence_rules=confidence_rules,
                status=request.status,
                confidence=min(max(request.confidence, 0.0), 1.0),
                metadata={"origin": "manual_schema_designer", **request.metadata},
            )
            return {
                "id": row.id,
                "name": row.name,
                "domain": row.domain,
                "description": row.description,
                "allowed_subject_types": row.allowed_subject_types or [],
                "allowed_object_types": row.allowed_object_types or [],
                "status": row.status,
                "confidence": row.confidence,
            }

    @app.get("/projects/{project_name}/ontology/proposals")
    def ontology_proposals(project_name: str, limit: int = 100):
        with session_scope(database_url) as session:
            project = KnowledgeRepository(session).get_project(project_name)
            rows = session.scalars(
                select(models.OntologyProposal)
                .where(models.OntologyProposal.project_id == project.id)
                .order_by(models.OntologyProposal.updated_at.desc())
                .limit(max(min(limit, 300), 1))
            ).all()
            return [
                {
                    "id": row.id,
                    "proposal_type": row.proposal_type,
                    "name": row.name,
                    "reason": row.reason,
                    "evidence": row.evidence,
                    "status": row.status,
                    "confidence": row.confidence,
                    "metadata": row.metadata_json or {},
                    "updated_at": row.updated_at.isoformat() if row.updated_at else None,
                }
                for row in rows
            ]

    @app.get("/projects/{project_name}/ontology/triples")
    def ontology_triples(project_name: str, status: str | None = None, limit: int = 100):
        with session_scope(database_url) as session:
            project = KnowledgeRepository(session).get_project(project_name)
            query = select(models.OntologyTriple).where(models.OntologyTriple.project_id == project.id)
            if status:
                query = query.where(models.OntologyTriple.status == status)
            rows = session.scalars(
                query.order_by(models.OntologyTriple.last_seen_at.desc()).limit(max(min(limit, 300), 1))
            ).all()
            results = []
            for row in rows:
                subject = session.get(models.Entity, row.subject_entity_id)
                object_entity = session.get(models.Entity, row.object_entity_id) if row.object_entity_id else None
                results.append(
                    {
                        "id": row.id,
                        "claim_id": row.claim_id,
                        "subject": subject.name if subject else None,
                        "subject_type": row.subject_type,
                        "predicate": row.predicate,
                        "object": object_entity.name if object_entity else None,
                        "object_type": row.object_type,
                        "object_value": row.object_value,
                        "value_type": row.value_type,
                        "status": row.status,
                        "confidence": row.confidence,
                        "support_count": row.support_count,
                        "metadata": row.metadata_json or {},
                        "last_seen_at": row.last_seen_at.isoformat() if row.last_seen_at else None,
                    }
                )
            return results

    @app.get("/projects/{project_name}/export")
    def export_ontology(
        project_name: str,
        format: str = "json",
        status: str = "validated_claim",
        include_evidence: bool = True,
        limit: int = 1000,
    ):
        fmt = format.strip().lower()
        bounded_limit = max(min(limit, 5000), 1)
        with session_scope(database_url) as session:
            project = KnowledgeRepository(session).get_project(project_name)
            query = (
                select(models.Claim, models.Source, models.Page, models.Entity)
                .join(models.Source, models.Claim.source_id == models.Source.id)
                .join(models.Page, models.Claim.page_id == models.Page.id, isouter=True)
                .join(models.Entity, models.Claim.subject_entity_id == models.Entity.id)
                .where(models.Claim.project_id == project.id)
            )
            if status and status != "all":
                normalized = _normalize_claim_status(status) or status
                query = query.where(models.Claim.status == normalized)
            rows = session.execute(
                query.order_by(models.Claim.confidence.desc(), models.Claim.last_seen_at.desc()).limit(bounded_limit)
            ).all()
            items: list[dict[str, Any]] = []
            for claim, source, page, subject in rows:
                object_entity = session.get(models.Entity, claim.object_entity_id) if claim.object_entity_id else None
                evidence = None
                if include_evidence:
                    evidence = session.scalar(
                        select(models.Evidence)
                        .where(models.Evidence.claim_id == claim.id)
                        .order_by(models.Evidence.created_at.desc())
                    )
                items.append(
                    {
                        "claim_id": claim.id,
                        "subject": subject.name,
                        "subject_type": subject.entity_type,
                        "predicate": claim.predicate,
                        "object": object_entity.name if object_entity else claim.object_value,
                        "object_type": object_entity.entity_type if object_entity else claim.value_type,
                        "status": claim.status,
                        "confidence": claim.confidence,
                        "source": source.name,
                        "source_url": page.url if page else None,
                        "evidence_text": evidence.evidence_text if evidence else None,
                        "created_by": claim.extraction_method,
                        "last_seen_at": claim.last_seen_at.isoformat() if claim.last_seen_at else None,
                    }
                )

        filename = f"{project_name}_ontology.{fmt if fmt != 'turtle' else 'ttl'}"
        if fmt == "json":
            return {
                "project": project_name,
                "status": status,
                "count": len(items),
                "claims": items,
            }
        if fmt == "csv":
            output = io.StringIO()
            fieldnames = [
                "claim_id",
                "subject",
                "subject_type",
                "predicate",
                "object",
                "object_type",
                "status",
                "confidence",
                "source",
                "source_url",
                "evidence_text",
                "created_by",
                "last_seen_at",
            ]
            writer = csv.DictWriter(output, fieldnames=fieldnames)
            writer.writeheader()
            for item in items:
                row = {key: json.dumps(value, ensure_ascii=False) if isinstance(value, (dict, list)) else value for key, value in item.items()}
                writer.writerow(row)
            return Response(
                content=output.getvalue(),
                media_type="text/csv; charset=utf-8",
                headers={"Content-Disposition": f'attachment; filename="{filename}"'},
            )
        if fmt in {"ttl", "turtle"}:
            lines = [
                "@prefix ont: <https://example.local/ontology/> .",
                "@prefix claim: <https://example.local/claim/> .",
                "@prefix xsd: <http://www.w3.org/2001/XMLSchema#> .",
                "",
            ]
            for item in items:
                sid = turtle_id(item["subject"])
                predicate = turtle_id(item["predicate"])
                obj = item["object"]
                if isinstance(obj, str) and obj.strip():
                    object_repr = f'ont:{turtle_id(obj)}'
                else:
                    object_repr = json.dumps(obj, ensure_ascii=False)
                lines.extend(
                    [
                        f"ont:{sid} ont:{predicate} {object_repr} .",
                        f"claim:c{item['claim_id']} ont:confidence \"{item['confidence']}\"^^xsd:decimal .",
                    ]
                )
                if item.get("source_url"):
                    lines.append(f"claim:c{item['claim_id']} ont:sourceUrl {json.dumps(item['source_url'], ensure_ascii=False)} .")
                if item.get("evidence_text"):
                    lines.append(f"claim:c{item['claim_id']} ont:evidenceText {json.dumps(item['evidence_text'], ensure_ascii=False)} .")
                lines.append("")
            return Response(
                content="\n".join(lines),
                media_type="text/turtle; charset=utf-8",
                headers={"Content-Disposition": f'attachment; filename="{filename}"'},
            )
        raise HTTPException(status_code=400, detail=f"unsupported export format: {format}")

    @app.get("/projects/{project_name}/knowledge-gaps")
    def knowledge_gaps(project_name: str, limit: int = 100):
        with session_scope(database_url) as session:
            project = KnowledgeRepository(session).get_project(project_name)
            return KnowledgeGapDetector(session).list_open(project.id, max(min(limit, 300), 1))

    @app.post("/extractors/models")
    def extractor_models(request: ExtractorModelsRequest):
        try:
            if request.provider == "lm_studio":
                models = list_openai_compatible_models(request.base_url or "http://localhost:1234/v1")
                return {"ok": True, "models": models}
            if request.provider == "openai":
                import os

                models = list_openai_compatible_models(
                    request.base_url or "https://api.openai.com/v1",
                    os.getenv("OPENAI_API_KEY"),
                )
                return {"ok": True, "models": models}
            if request.provider == "ollama":
                return {"ok": False, "error": "Ollama model listing is not implemented yet. Enter the model manually."}
            return {"ok": True, "models": [{"id": "rule_based", "owned_by": "local"}]}
        except Exception as exc:
            return {"ok": False, "error": str(exc), "models": []}

    @app.post("/crawl")
    def crawl(request: CrawlRequest):
        config = load_project_config(request.config_path)
        apply_crawl_request_overrides(config, request)
        with session_scope(database_url) as session:
            repo = KnowledgeRepository(session)
            pipeline = CrawlPipeline(
                repo,
                extractor_for_domain(
                    config.domain,
                    provider=request.extractor_provider,
                    model=request.extractor_model,
                    base_url=request.extractor_base_url,
                    extraction_mode=request.extraction_mode,
                    fallback_to_rules=request.fallback_to_rules,
                ),
            )
            try:
                result = pipeline.crawl_url(config, request.source_name, request.url)
            except Exception as exc:
                raise HTTPException(status_code=400, detail=str(exc)) from exc
            return {
                "page_id": result.page_id,
                "claim_count": result.claim_count,
                "entity_count": result.entity_count,
                "extraction_mode": result.extraction_mode,
                "effective_extraction_mode": result.effective_extraction_mode,
                "llm_skipped": result.llm_skipped,
                "llm_skip_reason": result.llm_skip_reason,
                "fallback_used": result.fallback_used,
                "agreement_claim_count": result.agreement_claim_count,
                "conflict_claim_count": result.conflict_claim_count,
                "crawl_status": result.crawl_status,
                "extraction_status": result.extraction_status,
                "page_type": result.page_type,
                "raw_text_length": result.raw_text_length,
                "clean_text_length": result.clean_text_length,
                "warnings": result.warnings or [],
            }

    @app.post("/crawl-site")
    def crawl_site(request: SiteCrawlRequest, background_tasks: BackgroundTasks):
        config = load_project_config(request.config_path)
        apply_crawl_request_overrides(config, request)
        with session_scope(database_url) as session:
            repo = KnowledgeRepository(session)
            project = repo.upsert_project(config)
            source = repo.get_source(project.id, request.source_name)
            job = models.CrawlJob(
                project_id=project.id,
                source_id=source.id,
                url=request.url,
                status="pending",
                metadata_json={
                    "kind": "site_crawl",
                    "request": request.model_dump(),
                    "progress": {
                        "seed_url": request.url,
                        "visited_count": 0,
                        "analyzed_count": 0,
                        "queued_count": 1,
                        "skipped_count": 0,
                        "errors": [],
                        "pages": [],
                    },
                },
            )
            session.add(job)
            session.flush()
            response = crawl_job_response(job)
        background_tasks.add_task(run_site_crawl_job, database_url, response["job_id"], request.model_dump())
        return response

    @app.post("/crawl-site/by-project")
    def crawl_site_by_project(
        request: SiteCrawlByProjectRequest, background_tasks: BackgroundTasks
    ):
        """Start a site crawl against an existing DB project (no config_path)."""
        with session_scope(database_url) as session:
            repo = KnowledgeRepository(session)
            project = repo.get_project(request.project_name)
            config = project_config_from_project_row(session, project)
            try:
                source = repo.get_source(project.id, request.source_name)
                config.source_by_name(request.source_name)
            except KeyError as exc:
                raise HTTPException(status_code=404, detail=str(exc)) from exc

            inner_request = request.to_site_crawl_request()
            apply_crawl_request_overrides(config, inner_request)

            job = models.CrawlJob(
                project_id=project.id,
                source_id=source.id,
                url=request.url,
                status="pending",
                metadata_json={
                    "kind": "site_crawl",
                    "request": inner_request.model_dump(),
                    "project_name": request.project_name,
                    "progress": {
                        "seed_url": request.url,
                        "visited_count": 0,
                        "analyzed_count": 0,
                        "queued_count": 1,
                        "skipped_count": 0,
                        "errors": [],
                        "pages": [],
                    },
                },
            )
            session.add(job)
            session.flush()
            response = crawl_job_response(job)

        task_payload = {
            **inner_request.model_dump(),
            "__config_dict": project_config_to_dict(config),
        }
        background_tasks.add_task(
            run_site_crawl_job, database_url, response["job_id"], task_payload
        )
        return response

    @app.get("/crawl-site/jobs/{job_id}")
    def crawl_site_job(job_id: int):
        with session_scope(database_url) as session:
            job = session.get(models.CrawlJob, job_id)
            if job is None or (job.metadata_json or {}).get("kind") != "site_crawl":
                raise HTTPException(status_code=404, detail="site crawl job not found")
            return crawl_job_response(job)

    @app.post("/crawl-site/jobs/{job_id}/cancel")
    def cancel_crawl_site_job(job_id: int):
        return request_site_crawl_cancel(database_url, job_id)

    @app.post("/discover")
    def discover(request: DiscoverRequest):
        config = load_project_config(request.config_path)
        apply_crawl_request_overrides(config, request)
        source_config = config.source_by_name(request.source_name)
        robots = RobotsPolicy()
        robots_decision = robots.check(request.url, source_config.respect_robots_txt)
        if not robots_decision.allowed:
            return {
                "ok": False,
                "error": f"{robots_decision.reason}: {request.url}",
                "robots_status": robots_decision.status,
                "robots_reason": robots_decision.reason,
                "links": [],
            }
        fetcher = make_fetcher(source_config.fetcher, source_config.rate_limit_per_minute)
        result = fetcher.fetch(request.url)
        links = discover_links(result.analysis_html, result.final_url or request.url, request.limit)
        return {
            "ok": True,
            "status_code": result.status_code,
            "final_url": result.final_url,
            "crawl_status": result.crawl_status,
            "robots_status": robots_decision.status,
            "robots_reason": robots_decision.reason,
            "warnings": result.warnings,
            "links": [asdict(link) for link in links],
        }

    @app.post("/research/run")
    def run_research(request: ResearchRunRequest):
        config = load_project_config(request.config_path)
        apply_crawl_request_overrides(config, request)
        if request.project_name and request.project_name != config.project_name:
            raise HTTPException(
                status_code=400,
                detail=f"Selected project '{request.project_name}' does not match config project '{config.project_name}'.",
            )
        with session_scope(database_url) as session:
            repo = KnowledgeRepository(session)
            loop = GraphResearchLoop(
                repo,
                extractor_for_domain(
                    config.domain,
                    provider=request.extractor_provider,
                    model=request.extractor_model,
                    base_url=request.extractor_base_url,
                    extraction_mode=request.extraction_mode,
                    fallback_to_rules=request.fallback_to_rules,
                ),
            )
            try:
                result = loop.run(
                    project_config=config,
                    source_name=request.source_name,
                    seed_url=request.url or None,
                    seed_entity_id=request.seed_entity_id,
                    goal=request.goal,
                    max_depth=max(request.max_depth, 0),
                    max_steps=max(min(request.max_steps, 50), 1),
                    max_branch=max(min(request.max_branch, 30), 1),
                    min_relevance=min(max(request.min_relevance, 0.0), 1.0),
                    same_domain_only=request.same_domain_only,
                    analyze_page_types=set(request.analyze_page_types),
                )
            except Exception as exc:
                raise HTTPException(status_code=400, detail=str(exc)) from exc
            return asdict(result)

    @app.post("/research/run/by-project")
    def run_research_by_project(request: ResearchRunByProjectRequest):
        """Run research against an existing DB project (no config_path)."""
        with session_scope(database_url) as session:
            repo = KnowledgeRepository(session)
            project = repo.get_project(request.project_name)
            config = project_config_from_project_row(session, project)
            try:
                source_config = config.source_by_name(request.source_name)
            except KeyError as exc:
                raise HTTPException(status_code=404, detail=str(exc)) from exc

            check_robots = request.respect_robots_txt
            if check_robots is None:
                check_robots = request.check_robots_txt
            source_config.respect_robots_txt = check_robots

            loop = GraphResearchLoop(
                repo,
                extractor_for_domain(
                    config.domain,
                    provider=request.extractor_provider,
                    model=request.extractor_model,
                    base_url=request.extractor_base_url,
                    extraction_mode=request.extraction_mode,
                    fallback_to_rules=request.fallback_to_rules,
                ),
            )
            try:
                result = loop.run(
                    project_config=config,
                    source_name=request.source_name,
                    seed_url=request.url or None,
                    seed_entity_id=request.seed_entity_id,
                    goal=request.goal,
                    max_depth=max(request.max_depth, 0),
                    max_steps=max(min(request.max_steps, 50), 1),
                    max_branch=max(min(request.max_branch, 30), 1),
                    min_relevance=min(max(request.min_relevance, 0.0), 1.0),
                    same_domain_only=request.same_domain_only,
                    analyze_page_types=set(request.analyze_page_types),
                )
            except Exception as exc:
                raise HTTPException(status_code=400, detail=str(exc)) from exc
            return asdict(result)

    @app.get("/projects/{project_name}/research/sessions")
    def research_sessions(project_name: str, limit: int = 25):
        with session_scope(database_url) as session:
            project = KnowledgeRepository(session).get_project(project_name)
            return ResearchMemoryStore(session).list_sessions(project.id, limit)

    @app.get("/research/sessions/{job_id}")
    def research_session(job_id: int):
        with session_scope(database_url) as session:
            job = session.get(models.CrawlJob, job_id)
            if job is None or (job.metadata_json or {}).get("kind") != "research_session":
                raise HTTPException(status_code=404, detail="research session not found")
            return research_session_payload(job)

    @app.get("/projects/{project_name}/graph/neighborhood")
    def graph_neighborhood(
        project_name: str,
        entity_id: int | None = None,
        limit: int = 120,
        include_candidates: bool = False,
        status: str | None = None,
    ):
        with session_scope(database_url) as session:
            project = KnowledgeRepository(session).get_project(project_name)
            if status and status != "all":
                normalized = _normalize_claim_status(status) or status
                statuses = [normalized]
            elif status == "all":
                statuses = None
            elif include_candidates:
                statuses = ["validated_claim", "active", "candidate_claim", "rule_candidate"]
            else:
                statuses = ["validated_claim"]
            return SemanticGraphQuery(session).neighborhood(
                project.id,
                entity_id,
                max(min(limit, 300), 1),
                statuses=statuses,
            )

    @app.get("/projects/{project_name}/graph/query")
    def graph_query(
        project_name: str,
        kind: str = "trend_summary",
        predicate: str = "hasAccord",
        tag: str | None = None,
        brand: str | None = None,
        limit: int = 100,
    ):
        with session_scope(database_url) as session:
            project = KnowledgeRepository(session).get_project(project_name)
            graph = SemanticGraphQuery(session)
            bounded_limit = max(min(limit, 300), 1)
            if kind == "brand_products":
                return graph.brand_products(project.id, brand, bounded_limit)
            if kind == "products_by_tag":
                return graph.products_by_tag(project.id, predicate, tag, bounded_limit)
            if kind == "relation_summary":
                return graph.relation_summary(project.id, bounded_limit)
            if kind == "entity_type_summary":
                return graph.entity_type_summary(project.id, bounded_limit)
            if kind == "trend_summary":
                return graph.trend_summary(project.id, bounded_limit)
            raise HTTPException(status_code=400, detail=f"unknown graph query kind: {kind}")

    @app.get("/projects/{project_name}/entities")
    def project_entities(project_name: str, entity_type: str | None = None, limit: int = 50):
        with session_scope(database_url) as session:
            project = KnowledgeRepository(session).get_project(project_name)
            query = select(models.Entity).where(models.Entity.project_id == project.id)
            if entity_type:
                query = query.where(models.Entity.entity_type == entity_type)
            entities = session.scalars(query.limit(limit)).all()
            return [
                {
                    "id": entity.id,
                    "type": entity.entity_type,
                    "name": entity.name,
                    "metadata": entity.metadata_json,
                }
                for entity in entities
            ]

    @app.post("/projects/{project_name}/entities")
    def create_entity(project_name: str, request: CreateEntityRequest):
        """Create or update a single entity directly (no extraction)."""
        with session_scope(database_url) as session:
            repo = KnowledgeRepository(session)
            project = repo.get_project(project_name)
            entity = repo.upsert_entity(
                project_id=project.id,
                entity_type=request.entity_type,
                name=request.name,
                metadata={
                    **request.metadata,
                    "input_method": request.metadata.get("input_method", "manual"),
                },
            )
            return {
                "id": entity.id,
                "type": entity.entity_type,
                "name": entity.name,
                "metadata": entity.metadata_json,
            }

    @app.post("/projects/{project_name}/entities/bulk")
    def bulk_create_entities(project_name: str, request: BulkCreateEntitiesRequest):
        """Create multiple entities in one call."""
        with session_scope(database_url) as session:
            repo = KnowledgeRepository(session)
            project = repo.get_project(project_name)
            created: list[dict[str, Any]] = []
            for item in request.entities:
                entity = repo.upsert_entity(
                    project_id=project.id,
                    entity_type=item.entity_type,
                    name=item.name,
                    metadata={
                        **item.metadata,
                        "input_method": item.metadata.get("input_method", "manual"),
                    },
                )
                created.append(
                    {
                        "id": entity.id,
                        "type": entity.entity_type,
                        "name": entity.name,
                    }
                )
            return {"created": len(created), "entities": created}

    @app.delete("/projects/{project_name}/entities/{entity_id}")
    def delete_entity(project_name: str, entity_id: int):
        with session_scope(database_url) as session:
            repo = KnowledgeRepository(session)
            project = repo.get_project(project_name)
            entity = session.get(models.Entity, entity_id)
            if entity is None or entity.project_id != project.id:
                raise HTTPException(
                    status_code=404,
                    detail=f"Entity {entity_id} not found in project '{project_name}'",
                )
            session.delete(entity)
            return {"ok": True, "deleted": entity_id}

    @app.post("/projects/{project_name}/claims")
    def create_claim(project_name: str, request: CreateClaimRequest):
        """Create a single claim directly (manual input)."""
        with session_scope(database_url) as session:
            repo = KnowledgeRepository(session)
            project = repo.get_project(project_name)
            try:
                source = repo.get_source(project.id, request.source_name)
            except KeyError as exc:
                raise HTTPException(status_code=404, detail=str(exc)) from exc

            subject = session.get(models.Entity, request.subject_entity_id)
            if subject is None or subject.project_id != project.id:
                raise HTTPException(
                    status_code=404,
                    detail=f"Subject entity {request.subject_entity_id} not found",
                )
            object_entity: models.Entity | None = None
            if request.object_entity_id is not None:
                object_entity = session.get(models.Entity, request.object_entity_id)
                if object_entity is None or object_entity.project_id != project.id:
                    raise HTTPException(
                        status_code=404,
                        detail=f"Object entity {request.object_entity_id} not found",
                    )

            claim_hash = make_claim_hash(
                project_id=project.id,
                source_id=source.id,
                subject_entity_id=subject.id,
                predicate=request.predicate,
                object_entity_id=object_entity.id if object_entity else None,
                object_value=request.object_value,
            )
            existing = session.scalar(
                select(models.Claim).where(
                    models.Claim.project_id == project.id,
                    models.Claim.claim_hash == claim_hash,
                )
            )
            if existing is not None:
                existing.confidence = max(existing.confidence, request.confidence)
                existing.last_seen_at = models.utcnow()
                if request.confidence_reason:
                    existing.confidence_reason = request.confidence_reason
                existing.metadata_json = {
                    **(existing.metadata_json or {}),
                    **request.metadata,
                    "input_method": "manual",
                }
                claim = existing
            else:
                claim = models.Claim(
                    project_id=project.id,
                    source_id=source.id,
                    page_id=None,
                    subject_entity_id=subject.id,
                    predicate=request.predicate,
                    object_entity_id=object_entity.id if object_entity else None,
                    object_value=request.object_value,
                    value_type="entity" if object_entity else "literal",
                    claim_hash=claim_hash,
                    confidence=max(0.0, min(1.0, request.confidence)),
                    confidence_reason=request.confidence_reason,
                    extraction_method="manual",
                    status="validated_claim",
                    metadata_json={**request.metadata, "input_method": "manual"},
                )
                session.add(claim)
                session.flush()
            if request.evidence_text:
                session.add(
                    models.Evidence(
                        project_id=project.id,
                        claim_id=claim.id,
                        page_id=None,
                        evidence_text=request.evidence_text,
                    )
                )
            return {
                "id": claim.id,
                "subject_entity_id": claim.subject_entity_id,
                "predicate": claim.predicate,
                "object_entity_id": claim.object_entity_id,
                "object_value": claim.object_value,
                "confidence": claim.confidence,
                "status": claim.status,
            }

    @app.delete("/projects/{project_name}/claims/{claim_id}")
    def delete_claim(project_name: str, claim_id: int):
        with session_scope(database_url) as session:
            repo = KnowledgeRepository(session)
            project = repo.get_project(project_name)
            claim = session.get(models.Claim, claim_id)
            if claim is None or claim.project_id != project.id:
                raise HTTPException(
                    status_code=404,
                    detail=f"Claim {claim_id} not found in project '{project_name}'",
                )
            session.delete(claim)
            return {"ok": True, "deleted": claim_id}

    @app.get("/projects/{project_name}/claims")
    def project_claims(
        project_name: str,
        limit: int = 100,
        include_candidates: bool = False,
        status: str | None = None,
    ):
        with session_scope(database_url) as session:
            project = KnowledgeRepository(session).get_project(project_name)
            query = (
                select(models.Claim, models.Source, models.Page, models.Entity)
                .join(models.Source, models.Claim.source_id == models.Source.id)
                .join(models.Page, models.Claim.page_id == models.Page.id, isouter=True)
                .join(models.Entity, models.Claim.subject_entity_id == models.Entity.id)
                .where(models.Claim.project_id == project.id)
            )
            if status:
                normalized = _normalize_claim_status(status)
                if normalized is None:
                    raise HTTPException(status_code=400, detail=f"unknown claim status: {status}")
                query = query.where(models.Claim.status == normalized)
            elif not include_candidates:
                query = query.where(models.Claim.status == "validated_claim")
            rows = session.execute(query.order_by(models.Claim.last_seen_at.desc()).limit(limit)).all()
            results: list[dict[str, Any]] = []
            for claim, source, page, subject in rows:
                object_name = None
                object_type = None
                if claim.object_entity_id:
                    object_entity = session.get(models.Entity, claim.object_entity_id)
                    object_name = object_entity.name if object_entity else None
                    object_type = object_entity.entity_type if object_entity else None
                evidence = session.scalar(
                    select(models.Evidence)
                    .where(models.Evidence.claim_id == claim.id)
                    .order_by(models.Evidence.created_at.desc())
                )
                results.append(
                    {
                        "id": claim.id,
                        "subject": subject.name,
                        "subject_type": subject.entity_type,
                        "predicate": claim.predicate,
                        "object": object_name,
                        "object_type": object_type,
                        "object_value": claim.object_value,
                        "source": source.name,
                        "page_url": page.url if page else None,
                        "confidence": claim.confidence,
                        "confidence_reason": claim.confidence_reason,
                        "status": claim.status,
                        "extraction_method": claim.extraction_method,
                        "evidence_text": evidence.evidence_text if evidence else None,
                        "evidence_summary": evidence.evidence_summary if evidence else None,
                        "page_type": (claim.metadata_json or {}).get("page_type")
                        or ((page.metadata_json or {}).get("page_type") if page else None),
                        "source_zone": (claim.metadata_json or {}).get("source_zone"),
                        "source_selector": (claim.metadata_json or {}).get("source_selector"),
                        "evidence_found": (claim.metadata_json or {}).get("evidence_found"),
                        "validation_status": (claim.metadata_json or {}).get("validation_status"),
                        "graph_merge_status": (claim.metadata_json or {}).get("graph_merge_status"),
                        "graph_merge_reason": (claim.metadata_json or {}).get("graph_merge_reason"),
                        "confidence_breakdown": (claim.metadata_json or {}).get("confidence_breakdown"),
                        "agreement": (claim.metadata_json or {}).get("agreement"),
                        "extraction_source": (claim.metadata_json or {}).get("extraction_source"),
                        "claim_kind": (claim.metadata_json or {}).get("claim_kind"),
                        "rule_confidence": (claim.metadata_json or {}).get("rule_confidence"),
                        "llm_confidence": (claim.metadata_json or {}).get("llm_confidence"),
                        "review_required": (claim.metadata_json or {}).get("review_required"),
                        "review_reason": (claim.metadata_json or {}).get("review_reason"),
                        "conflict_status": (claim.metadata_json or {}).get("conflict_status"),
                        "source_history": (claim.metadata_json or {}).get("source_history") or [],
                        "last_seen_at": claim.last_seen_at.isoformat(),
                    }
                )
            return results

    @app.get("/projects/{project_name}/pipeline")
    def project_pipeline(project_name: str):
        from sqlalchemy import func as sa_func, distinct
        with session_scope(database_url) as session:
            project = KnowledgeRepository(session).get_project(project_name)
            pid = project.id

            pages_total = session.scalar(
                select(sa_func.count(models.Page.id)).where(models.Page.project_id == pid)
            ) or 0
            ext_total = session.scalar(
                select(sa_func.count(models.ExtractionLog.id)).where(models.ExtractionLog.project_id == pid)
            ) or 0
            ext_pages = session.scalar(
                select(sa_func.count(distinct(models.ExtractionLog.page_id)))
                .where(models.ExtractionLog.project_id == pid, models.ExtractionLog.page_id.isnot(None))
            ) or 0
            ext_errors = session.scalar(
                select(sa_func.count(models.ExtractionLog.id))
                .where(models.ExtractionLog.project_id == pid, models.ExtractionLog.error.isnot(None))
            ) or 0

            status_rows = session.execute(
                select(models.Claim.status, sa_func.count(models.Claim.id))
                .where(models.Claim.project_id == pid)
                .group_by(models.Claim.status)
            ).all()
            claim_statuses = {row[0]: int(row[1]) for row in status_rows}
            claim_total = sum(claim_statuses.values())

            entity_count = session.scalar(
                select(sa_func.count(models.Entity.id)).where(models.Entity.project_id == pid)
            ) or 0
            triple_count = session.scalar(
                select(sa_func.count(models.OntologyTriple.id))
                .where(models.OntologyTriple.project_id == pid, models.OntologyTriple.status == "active")
            ) or 0

            entity_type_rows = session.execute(
                select(models.Entity.entity_type, sa_func.count(models.Entity.id))
                .where(models.Entity.project_id == pid)
                .group_by(models.Entity.entity_type)
                .order_by(sa_func.count(models.Entity.id).desc())
                .limit(10)
            ).all()

            recent_pages = session.scalars(
                select(models.Page).where(models.Page.project_id == pid)
                .order_by(models.Page.fetched_at.desc()).limit(5)
            ).all()
            recent_claims_rows = session.execute(
                select(models.Claim, models.Entity)
                .join(models.Entity, models.Claim.subject_entity_id == models.Entity.id)
                .where(models.Claim.project_id == pid)
                .order_by(models.Claim.last_seen_at.desc()).limit(5)
            ).all()

            return {
                "stages": [
                    {"key": "crawled", "count": pages_total},
                    {"key": "extracted", "count": ext_pages, "extra": {"events": ext_total, "errors": ext_errors}},
                    {"key": "claims", "count": claim_total, "extra": claim_statuses},
                    {"key": "validated", "count": claim_statuses.get("validated_claim", 0)},
                    {"key": "graph", "count": triple_count, "extra": {"entities": entity_count}},
                ],
                "entity_types": [{"type": row[0], "count": int(row[1])} for row in entity_type_rows],
                "recent_pages": [
                    {
                        "id": p.id,
                        "url": p.url,
                        "title": p.title,
                        "status_code": p.status_code,
                        "page_type": (p.metadata_json or {}).get("page_type"),
                        "fetched_at": p.fetched_at.isoformat() if p.fetched_at else None,
                    }
                    for p in recent_pages
                ],
                "recent_claims": [
                    {
                        "id": claim.id,
                        "subject": subject.name,
                        "predicate": claim.predicate,
                        "confidence": claim.confidence,
                        "status": claim.status,
                        "last_seen_at": claim.last_seen_at.isoformat() if claim.last_seen_at else None,
                    }
                    for claim, subject in recent_claims_rows
                ],
            }

    @app.post("/projects/{project_name}/pipeline/rerun")
    def rerun_pipeline_stage(
        project_name: str,
        request: PipelineRerunRequest,
        background_tasks: BackgroundTasks,
    ):
        crawl_stages = {
            "source-crawl",
            "page-clean",
            "page-classification",
            "entity-extraction",
            "claim-generation",
            "deduplication",
            "validation",
        }
        route_by_stage = {
            "human-review": f"/review/{project_name}",
            "ontology-commit": f"/graph/{project_name}",
            "export": f"/export/{project_name}",
        }
        if request.stage not in crawl_stages:
            return {
                "ok": True,
                "stage": request.stage,
                "action": "navigate",
                "route": route_by_stage.get(request.stage, f"/pipeline/{project_name}"),
                "message": "This stage is controlled from its workspace screen.",
            }

        with session_scope(database_url) as session:
            repo = KnowledgeRepository(session)
            project = repo.get_project(project_name)
            config = project_config_from_project_row(session, project)
            latest_job = session.scalar(
                select(models.CrawlJob)
                .where(models.CrawlJob.project_id == project.id)
                .order_by(models.CrawlJob.scheduled_at.desc())
            )
            latest_request = dict((latest_job.metadata_json or {}).get("request") or {}) if latest_job else {}
            source_name = latest_request.get("source_name")
            source = None
            if source_name:
                source = session.scalar(
                    select(models.Source).where(
                        models.Source.project_id == project.id,
                        models.Source.name == source_name,
                    )
                )
            if source is None:
                source = session.scalar(
                    select(models.Source)
                    .where(models.Source.project_id == project.id)
                    .order_by(models.Source.updated_at.desc())
                )
            if source is None:
                raise HTTPException(status_code=400, detail="No source is registered for this project.")
            url = latest_request.get("url") or source.base_url
            if not url:
                raise HTTPException(status_code=400, detail="No previous crawl URL or source base_url is available.")

            inner_request = SiteCrawlRequest(
                config_path="",
                source_name=source.name,
                url=url,
                extraction_mode=str(latest_request.get("extraction_mode") or "hybrid"),
                extractor_provider=str(latest_request.get("extractor_provider") or "lm_studio"),
                extractor_model=latest_request.get("extractor_model"),
                extractor_base_url=latest_request.get("extractor_base_url") or "http://localhost:1234/v1",
                fallback_to_rules=bool(latest_request.get("fallback_to_rules", True)),
                check_robots_txt=bool(latest_request.get("check_robots_txt") or False),
                respect_robots_txt=latest_request.get("respect_robots_txt"),
                max_depth=int(latest_request.get("max_depth") or 2),
                max_pages=int(latest_request.get("max_pages") or 50),
                same_domain_only=bool(latest_request.get("same_domain_only", True)),
                analyze_page_types=list(
                    latest_request.get("analyze_page_types")
                    or ["ProductPage", "BrandStoryPage", "ReviewPage"]
                ),
            )
            apply_crawl_request_overrides(config, inner_request)
            job = models.CrawlJob(
                project_id=project.id,
                source_id=source.id,
                url=url,
                status="pending",
                metadata_json={
                    "kind": "site_crawl",
                    "request": inner_request.model_dump(),
                    "project_name": project_name,
                    "rerun_stage": request.stage,
                    "progress": {
                        "seed_url": url,
                        "visited_count": 0,
                        "analyzed_count": 0,
                        "queued_count": 1,
                        "skipped_count": 0,
                        "errors": [],
                        "pages": [],
                    },
                },
            )
            session.add(job)
            session.flush()
            response = crawl_job_response(job)

        task_payload = {
            **inner_request.model_dump(),
            "__config_dict": project_config_to_dict(config),
        }
        background_tasks.add_task(
            run_site_crawl_job,
            database_url,
            response["job_id"],
            task_payload,
        )
        return {
            **response,
            "ok": True,
            "stage": request.stage,
            "action": "job_started",
        }

    @app.get("/projects/{project_name}/search")
    def project_search(project_name: str, q: str = "", limit: int = 10):
        q = (q or "").strip()
        if len(q) < 1:
            return {"entities": [], "claims": [], "pages": [], "predicates": []}
        like = f"%{q.lower()}%"
        bounded = max(min(limit, 50), 1)
        with session_scope(database_url) as session:
            project = KnowledgeRepository(session).get_project(project_name)
            pid = project.id

            from sqlalchemy import func as sa_func, or_
            entity_rows = session.scalars(
                select(models.Entity)
                .where(
                    models.Entity.project_id == pid,
                    or_(
                        sa_func.lower(models.Entity.name).like(like),
                        sa_func.lower(models.Entity.canonical_name).like(like),
                    ),
                )
                .limit(bounded)
            ).all()

            claim_rows = session.execute(
                select(models.Claim, models.Entity)
                .join(models.Entity, models.Claim.subject_entity_id == models.Entity.id)
                .where(
                    models.Claim.project_id == pid,
                    or_(
                        sa_func.lower(models.Entity.name).like(like),
                        sa_func.lower(models.Claim.predicate).like(like),
                    ),
                )
                .order_by(models.Claim.last_seen_at.desc())
                .limit(bounded)
            ).all()

            page_rows = session.scalars(
                select(models.Page)
                .where(
                    models.Page.project_id == pid,
                    or_(
                        sa_func.lower(models.Page.url).like(like),
                        sa_func.lower(models.Page.title).like(like),
                    ),
                )
                .order_by(models.Page.fetched_at.desc())
                .limit(bounded)
            ).all()

            predicate_rows = session.scalars(
                select(models.OntologyRelationType.name)
                .where(
                    models.OntologyRelationType.project_id == pid,
                    sa_func.lower(models.OntologyRelationType.name).like(like),
                )
                .limit(bounded)
            ).all()

            return {
                "entities": [
                    {"id": e.id, "name": e.name, "type": e.entity_type, "canonical_name": e.canonical_name}
                    for e in entity_rows
                ],
                "claims": [
                    {
                        "id": claim.id,
                        "subject": subject.name,
                        "predicate": claim.predicate,
                        "object_value": claim.object_value,
                        "confidence": claim.confidence,
                        "status": claim.status,
                    }
                    for claim, subject in claim_rows
                ],
                "pages": [
                    {"id": p.id, "url": p.url, "title": p.title}
                    for p in page_rows
                ],
                "predicates": list(predicate_rows),
            }

    @app.get("/projects/{project_name}/extraction-logs")
    def project_extraction_logs(project_name: str, limit: int = 50):
        with session_scope(database_url) as session:
            project = KnowledgeRepository(session).get_project(project_name)
            rows = session.execute(
                select(models.ExtractionLog, models.Page)
                .join(models.Page, models.ExtractionLog.page_id == models.Page.id, isouter=True)
                .where(models.ExtractionLog.project_id == project.id)
                .order_by(models.ExtractionLog.created_at.desc())
                .limit(limit)
            ).all()
            return [
                {
                    "id": log.id,
                    "page_url": page.url if page else None,
                    "extractor_name": log.extractor_name,
                    "provider": log.provider,
                    "error": log.error,
                    "created_at": log.created_at.isoformat(),
                    "validation": (log.raw_output or {}).get("validation"),
                    "page_context": (log.raw_output or {}).get("page_context"),
                    "candidate_count": len((log.raw_output or {}).get("candidate_claims") or []),
                    "extraction_mode": (log.raw_output or {}).get("extraction_mode"),
                    "effective_extraction_mode": (log.raw_output or {}).get("effective_extraction_mode"),
                    "comparison": (log.raw_output or {}).get("comparison"),
                    "rule_entity_count": (log.raw_output or {}).get("rule_entity_count"),
                    "rule_claim_count": (log.raw_output or {}).get("rule_claim_count"),
                    "llm_entity_count": (log.raw_output or {}).get("llm_entity_count"),
                    "llm_claim_count": (log.raw_output or {}).get("llm_claim_count"),
                    "agreement_claim_count": (log.raw_output or {}).get("agreement_claim_count"),
                    "rule_only_claim_count": (log.raw_output or {}).get("rule_only_claim_count"),
                    "llm_only_claim_count": (log.raw_output or {}).get("llm_only_claim_count"),
                    "conflict_claim_count": (log.raw_output or {}).get("conflict_claim_count"),
                    "llm_skipped": (log.raw_output or {}).get("llm_skipped"),
                    "llm_skip_reason": (log.raw_output or {}).get("llm_skip_reason"),
                    "fallback": (log.raw_output or {}).get("fallback"),
                    "raw_output": log.raw_output,
                }
                for log, page in rows
            ]

    @app.patch("/claims/{claim_id}/confidence")
    def update_claim_confidence(claim_id: int, request: UpdateClaimConfidenceRequest):
        confidence = min(max(request.confidence, 0.0), 1.0)
        with session_scope(database_url) as session:
            claim = session.get(models.Claim, claim_id)
            if claim is None:
                return {"ok": False, "error": "claim not found"}
            claim.confidence = confidence
            claim.confidence_reason = request.reason or "manual admin update"
            claim.last_seen_at = models.utcnow()
            return {"ok": True, "claim_id": claim.id, "confidence": claim.confidence}

    @app.patch("/claims/{claim_id}/status")
    def update_claim_status(claim_id: int, request: UpdateClaimStatusRequest):
        status = _normalize_claim_status(request.status)
        if status is None:
            raise HTTPException(status_code=400, detail=f"unknown claim status: {request.status}")
        with session_scope(database_url) as session:
            claim = session.get(models.Claim, claim_id)
            if claim is None:
                raise HTTPException(status_code=404, detail="claim not found")
            _apply_claim_review(claim, status, request.reason)
            return {"ok": True, "claim_id": claim.id, "status": claim.status}

    @app.post("/claims/bulk-status")
    def bulk_update_claim_status(request: BulkClaimStatusRequest):
        status = _normalize_claim_status(request.status)
        if status is None:
            raise HTTPException(status_code=400, detail=f"unknown claim status: {request.status}")
        if not request.claim_ids:
            return {"ok": True, "updated": 0}
        with session_scope(database_url) as session:
            rows = session.execute(
                select(models.Claim).where(models.Claim.id.in_(request.claim_ids))
            ).scalars().all()
            for claim in rows:
                _apply_claim_review(claim, status, request.reason)
            return {"ok": True, "updated": len(rows), "status": status}

    @app.post("/entities/merge")
    def merge_entities(request: MergeEntitiesRequest):
        with session_scope(database_url) as session:
            project = KnowledgeRepository(session).get_project(request.project_name)
            source = session.get(models.Entity, request.source_entity_id)
            target = session.get(models.Entity, request.target_entity_id)
            if source is None or target is None or source.project_id != project.id or target.project_id != project.id:
                return {"ok": False, "error": "entity not found in project"}
            session.query(models.Claim).filter(models.Claim.subject_entity_id == source.id).update(
                {models.Claim.subject_entity_id: target.id}
            )
            session.query(models.Claim).filter(models.Claim.object_entity_id == source.id).update(
                {models.Claim.object_entity_id: target.id}
            )
            session.query(models.Relation).filter(models.Relation.subject_entity_id == source.id).update(
                {models.Relation.subject_entity_id: target.id}
            )
            session.query(models.Relation).filter(models.Relation.object_entity_id == source.id).update(
                {models.Relation.object_entity_id: target.id}
            )
            source.metadata_json = {**(source.metadata_json or {}), "merged_into": target.id}
            source.updated_at = models.utcnow()
            return {"ok": True, "source_entity_id": source.id, "target_entity_id": target.id}

    @app.get("/projects/{project_name}/recommendation-tags")
    def recommendation_tags(project_name: str):
        with session_scope(database_url) as session:
            project = KnowledgeRepository(session).get_project(project_name)
            tag_predicates = {
                "hasTopNote",
                "hasMiddleNote",
                "hasBaseNote",
                "hasScentNote",
                "hasFlavorNote",
                "evokesMood",
                "suitableForSeason",
                "suitableForOccasion",
                "hasReviewKeyword",
            }
            rows = session.execute(
                select(models.Claim, models.Entity)
                .join(models.Entity, models.Claim.object_entity_id == models.Entity.id)
                .where(
                    models.Claim.project_id == project.id,
                    models.Claim.status == "validated_claim",
                    models.Claim.predicate.in_(tag_predicates),
                )
            ).all()
            grouped: dict[str, dict[str, Any]] = {}
            for claim, entity in rows:
                key = f"{claim.predicate}:{entity.canonical_name}"
                if key not in grouped:
                    grouped[key] = {
                        "predicate": claim.predicate,
                        "name": entity.name,
                        "type": entity.entity_type,
                        "support_count": 0,
                        "max_confidence": 0.0,
                    }
                grouped[key]["support_count"] += 1
                grouped[key]["max_confidence"] = max(grouped[key]["max_confidence"], claim.confidence)
            return sorted(grouped.values(), key=lambda item: (item["predicate"], -item["support_count"], item["name"]))

    @app.post("/recommend")
    def recommend(request: RecommendRequest):
        with session_scope(database_url) as session:
            repo = KnowledgeRepository(session)
            project = repo.get_project(request.project_name)
            recommender = RuleBasedRecommender(session)
            pref = PreferenceInput(**request.preferences)
            items = recommender.recommend(project.id, request.target_entity_type, pref, request.limit)
            return [asdict(item) for item in items]