참고소스 수정본

crawler_platform/app/api/routes.py

@@ -3,7 +3,7 @@ from __future__ import annotations
 from dataclasses import asdict
 from typing import Any
 
-from fastapi import HTTPException
+from fastapi import BackgroundTasks, HTTPException
 from pydantic import BaseModel, Field
 from sqlalchemy import select
 
@@ -18,24 +18,29 @@ from crawler_platform.app.core.database.session import session_scope
 from crawler_platform.app.core.extractor.ai_provider import list_openai_compatible_models
 from crawler_platform.app.core.extractor.factory import extractor_for_domain
 from crawler_platform.app.core.ontology.definitions import ontology_for_domain
+from crawler_platform.app.core.ontology.gap_detector import KnowledgeGapDetector
 from crawler_platform.app.core.ontology.mapper import ontology_to_dict
+from crawler_platform.app.core.ontology.registry import OntologyRegistry
 from crawler_platform.app.core.recommendation.scorer import PreferenceInput, RuleBasedRecommender
+from crawler_platform.app.core.research.graph_query import SemanticGraphQuery
+from crawler_platform.app.core.research.graph_research_loop import GraphResearchLoop
+from crawler_platform.app.core.research.memory_store import ResearchMemoryStore, research_session_payload
 
 
 class CrawlRequest(BaseModel):
     config_path: str
     source_name: str
     url: str
-    extractor_provider: str = "rule_based"
+    extractor_provider: str = "lm_studio"
     extractor_model: str | None = None
-    extractor_base_url: str | None = None
+    extractor_base_url: str | None = "http://localhost:1234/v1"
 
 
 class SiteCrawlRequest(CrawlRequest):
     max_depth: int = 2
     max_pages: int = 50
     same_domain_only: bool = True
-    analyze_page_types: list[str] = Field(default_factory=lambda: ["product", "brand", "review"])
+    analyze_page_types: list[str] = Field(default_factory=lambda: ["ProductPage", "BrandStoryPage", "ReviewPage"])
 
 
 class DiscoverRequest(BaseModel):
@@ -66,6 +71,17 @@ class UpdateClaimConfidenceRequest(BaseModel):
     reason: str | None = None
 
 
+class UpdateClaimStatusRequest(BaseModel):
+    status: str
+    reason: str | None = None
+
+
+class BulkClaimStatusRequest(BaseModel):
+    claim_ids: list[int]
+    status: str
+    reason: str | None = None
+
+
 class MergeEntitiesRequest(BaseModel):
     project_name: str
     source_entity_id: int
@@ -77,6 +93,172 @@ class ExtractorModelsRequest(BaseModel):
     base_url: str | None = None
 
 
+class ResearchRunRequest(CrawlRequest):
+    project_name: str | None = None
+    seed_entity_id: int | None = None
+    goal: str = "Semantic ontology exploration"
+    max_depth: int = 2
+    max_steps: int = 12
+    max_branch: int = 8
+    min_relevance: float = 0.35
+    same_domain_only: bool = True
+    analyze_page_types: list[str] = Field(default_factory=lambda: ["ProductPage", "BrandStoryPage", "ReviewPage"])
+
+
+def site_crawl_progress_payload(result, latest_page=None) -> dict[str, Any]:
+    payload = asdict(result)
+    if latest_page is not None:
+        payload["latest_page"] = asdict(latest_page)
+    return payload
+
+
+_CLAIM_STATUS_ALIASES = {
+    "accept": "validated_claim",
+    "accepted": "validated_claim",
+    "validated": "validated_claim",
+    "validated_claim": "validated_claim",
+    "reject": "rejected",
+    "rejected": "rejected",
+    "active": "active",
+    "candidate": "active",
+    "pending": "active",
+}
+
+
+def _normalize_claim_status(value: str | None) -> str | None:
+    if not value:
+        return None
+    return _CLAIM_STATUS_ALIASES.get(value.strip().lower())
+
+
+def _apply_claim_review(claim: models.Claim, status: str, reason: str | None) -> None:
+    claim.status = status
+    if reason:
+        claim.confidence_reason = reason
+    metadata = dict(claim.metadata_json or {})
+    review = dict(metadata.get("review") or {})
+    review["status"] = status
+    review["reason"] = reason or review.get("reason")
+    review["reviewed_at"] = models.utcnow().isoformat()
+    metadata["review"] = review
+    claim.metadata_json = metadata
+    claim.last_seen_at = models.utcnow()
+
+
+def crawl_job_response(job: models.CrawlJob) -> dict[str, Any]:
+    metadata = job.metadata_json or {}
+    return {
+        "job_id": job.id,
+        "status": job.status,
+        "url": job.url,
+        "error": job.error,
+        "scheduled_at": job.scheduled_at.isoformat() if job.scheduled_at else None,
+        "started_at": job.started_at.isoformat() if job.started_at else None,
+        "finished_at": job.finished_at.isoformat() if job.finished_at else None,
+        "progress": metadata.get("progress", {}),
+        "request": metadata.get("request", {}),
+    }
+
+
+def update_site_crawl_job_metadata(job: models.CrawlJob, **updates: Any) -> None:
+    metadata = dict(job.metadata_json or {})
+    metadata.update(updates)
+    job.metadata_json = metadata
+
+
+def is_site_crawl_cancel_requested(session, job_id: int) -> bool:
+    session.expire_all()
+    job = session.get(models.CrawlJob, job_id)
+    return job is None or job.status == "cancel_requested"
+
+
+def run_site_crawl_job(database_url: str, job_id: int, request_data: dict[str, Any]) -> None:
+    request = SiteCrawlRequest(**request_data)
+    try:
+        config = load_project_config(request.config_path)
+        with session_scope(database_url) as session:
+            job = session.get(models.CrawlJob, job_id)
+            if job is None:
+                return
+            if job.status == "cancel_requested":
+                job.status = "canceled"
+                job.finished_at = models.utcnow()
+                return
+
+            job.status = "running"
+            job.started_at = models.utcnow()
+            update_site_crawl_job_metadata(
+                job,
+                progress={
+                    "seed_url": request.url,
+                    "visited_count": 0,
+                    "analyzed_count": 0,
+                    "queued_count": 1,
+                    "skipped_count": 0,
+                    "errors": [],
+                    "pages": [],
+                },
+            )
+            session.commit()
+
+            repo = KnowledgeRepository(session)
+            crawler = SiteCrawler(
+                repo,
+                extractor_for_domain(
+                    config.domain,
+                    provider=request.extractor_provider,
+                    model=request.extractor_model,
+                    base_url=request.extractor_base_url,
+                ),
+            )
+
+            def save_progress(result, latest_page) -> None:
+                current_job = session.get(models.CrawlJob, job_id)
+                if current_job is None:
+                    return
+                update_site_crawl_job_metadata(
+                    current_job,
+                    progress=site_crawl_progress_payload(result, latest_page),
+                )
+                session.commit()
+
+            def should_stop() -> bool:
+                return is_site_crawl_cancel_requested(session, job_id)
+
+            result = crawler.crawl_site(
+                config,
+                request.source_name,
+                request.url,
+                max_depth=max(request.max_depth, 0),
+                max_pages=max(min(request.max_pages, 500), 1),
+                same_domain_only=request.same_domain_only,
+                analyze_page_types=set(request.analyze_page_types),
+                progress_callback=save_progress,
+                should_stop=should_stop,
+                parent_job_id=job_id,
+            )
+
+            finished_job = session.get(models.CrawlJob, job_id)
+            if finished_job is not None:
+                finished_job.status = "canceled" if finished_job.status == "cancel_requested" else "completed"
+                finished_job.finished_at = models.utcnow()
+                update_site_crawl_job_metadata(
+                    finished_job,
+                    progress=site_crawl_progress_payload(result),
+                )
+    except Exception as exc:
+        with session_scope(database_url) as session:
+            job = session.get(models.CrawlJob, job_id)
+            if job is not None:
+                job.status = "failed"
+                job.error = str(exc)
+                job.finished_at = models.utcnow()
+                metadata = dict(job.metadata_json or {})
+                progress = dict(metadata.get("progress") or {})
+                progress["errors"] = [*progress.get("errors", []), str(exc)]
+                update_site_crawl_job_metadata(job, progress=progress)
+
+
 def register_routes(app, database_url: str) -> None:
     @app.get("/health")
     def health():
@@ -168,6 +350,77 @@ def register_routes(app, database_url: str) -> None:
     def ontology(domain: str):
         return ontology_to_dict(ontology_for_domain(domain))
 
+    @app.get("/projects/{project_name}/ontology/registry")
+    def ontology_registry(project_name: str):
+        with session_scope(database_url) as session:
+            project = KnowledgeRepository(session).get_project(project_name)
+            return OntologyRegistry(session).registry_payload(project.id)
+
+    @app.get("/projects/{project_name}/ontology/proposals")
+    def ontology_proposals(project_name: str, limit: int = 100):
+        with session_scope(database_url) as session:
+            project = KnowledgeRepository(session).get_project(project_name)
+            rows = session.scalars(
+                select(models.OntologyProposal)
+                .where(models.OntologyProposal.project_id == project.id)
+                .order_by(models.OntologyProposal.updated_at.desc())
+                .limit(max(min(limit, 300), 1))
+            ).all()
+            return [
+                {
+                    "id": row.id,
+                    "proposal_type": row.proposal_type,
+                    "name": row.name,
+                    "reason": row.reason,
+                    "evidence": row.evidence,
+                    "status": row.status,
+                    "confidence": row.confidence,
+                    "metadata": row.metadata_json or {},
+                    "updated_at": row.updated_at.isoformat() if row.updated_at else None,
+                }
+                for row in rows
+            ]
+
+    @app.get("/projects/{project_name}/ontology/triples")
+    def ontology_triples(project_name: str, status: str | None = None, limit: int = 100):
+        with session_scope(database_url) as session:
+            project = KnowledgeRepository(session).get_project(project_name)
+            query = select(models.OntologyTriple).where(models.OntologyTriple.project_id == project.id)
+            if status:
+                query = query.where(models.OntologyTriple.status == status)
+            rows = session.scalars(
+                query.order_by(models.OntologyTriple.last_seen_at.desc()).limit(max(min(limit, 300), 1))
+            ).all()
+            results = []
+            for row in rows:
+                subject = session.get(models.Entity, row.subject_entity_id)
+                object_entity = session.get(models.Entity, row.object_entity_id) if row.object_entity_id else None
+                results.append(
+                    {
+                        "id": row.id,
+                        "claim_id": row.claim_id,
+                        "subject": subject.name if subject else None,
+                        "subject_type": row.subject_type,
+                        "predicate": row.predicate,
+                        "object": object_entity.name if object_entity else None,
+                        "object_type": row.object_type,
+                        "object_value": row.object_value,
+                        "value_type": row.value_type,
+                        "status": row.status,
+                        "confidence": row.confidence,
+                        "support_count": row.support_count,
+                        "metadata": row.metadata_json or {},
+                        "last_seen_at": row.last_seen_at.isoformat() if row.last_seen_at else None,
+                    }
+                )
+            return results
+
+    @app.get("/projects/{project_name}/knowledge-gaps")
+    def knowledge_gaps(project_name: str, limit: int = 100):
+        with session_scope(database_url) as session:
+            project = KnowledgeRepository(session).get_project(project_name)
+            return KnowledgeGapDetector(session).list_open(project.id, max(min(limit, 300), 1))
+
     @app.post("/extractors/models")
     def extractor_models(request: ExtractorModelsRequest):
         try:
@@ -206,35 +459,69 @@ def register_routes(app, database_url: str) -> None:
                 result = pipeline.crawl_url(config, request.source_name, request.url)
             except Exception as exc:
                 raise HTTPException(status_code=400, detail=str(exc)) from exc
-            return {"page_id": result.page_id, "claim_count": result.claim_count, "entity_count": result.entity_count}
+            return {
+                "page_id": result.page_id,
+                "claim_count": result.claim_count,
+                "entity_count": result.entity_count,
+                "crawl_status": result.crawl_status,
+                "extraction_status": result.extraction_status,
+                "page_type": result.page_type,
+                "raw_text_length": result.raw_text_length,
+                "clean_text_length": result.clean_text_length,
+                "warnings": result.warnings or [],
+            }
 
     @app.post("/crawl-site")
-    def crawl_site(request: SiteCrawlRequest):
+    def crawl_site(request: SiteCrawlRequest, background_tasks: BackgroundTasks):
         config = load_project_config(request.config_path)
         with session_scope(database_url) as session:
             repo = KnowledgeRepository(session)
-            crawler = SiteCrawler(
-                repo,
-                extractor_for_domain(
-                    config.domain,
-                    provider=request.extractor_provider,
-                    model=request.extractor_model,
-                    base_url=request.extractor_base_url,
-                ),
+            project = repo.upsert_project(config)
+            source = repo.get_source(project.id, request.source_name)
+            job = models.CrawlJob(
+                project_id=project.id,
+                source_id=source.id,
+                url=request.url,
+                status="pending",
+                metadata_json={
+                    "kind": "site_crawl",
+                    "request": request.model_dump(),
+                    "progress": {
+                        "seed_url": request.url,
+                        "visited_count": 0,
+                        "analyzed_count": 0,
+                        "queued_count": 1,
+                        "skipped_count": 0,
+                        "errors": [],
+                        "pages": [],
+                    },
+                },
             )
-            try:
-                result = crawler.crawl_site(
-                    config,
-                    request.source_name,
-                    request.url,
-                    max_depth=max(request.max_depth, 0),
-                    max_pages=max(min(request.max_pages, 500), 1),
-                    same_domain_only=request.same_domain_only,
-                    analyze_page_types=set(request.analyze_page_types),
-                )
-            except Exception as exc:
-                raise HTTPException(status_code=400, detail=str(exc)) from exc
-            return asdict(result)
+            session.add(job)
+            session.flush()
+            response = crawl_job_response(job)
+        background_tasks.add_task(run_site_crawl_job, database_url, response["job_id"], request.model_dump())
+        return response
+
+    @app.get("/crawl-site/jobs/{job_id}")
+    def crawl_site_job(job_id: int):
+        with session_scope(database_url) as session:
+            job = session.get(models.CrawlJob, job_id)
+            if job is None or (job.metadata_json or {}).get("kind") != "site_crawl":
+                raise HTTPException(status_code=404, detail="site crawl job not found")
+            return crawl_job_response(job)
+
+    @app.post("/crawl-site/jobs/{job_id}/cancel")
+    def cancel_crawl_site_job(job_id: int):
+        with session_scope(database_url) as session:
+            job = session.get(models.CrawlJob, job_id)
+            if job is None or (job.metadata_json or {}).get("kind") != "site_crawl":
+                raise HTTPException(status_code=404, detail="site crawl job not found")
+            if job.status in {"completed", "failed", "canceled"}:
+                return crawl_job_response(job)
+            job.status = "cancel_requested"
+            job.error = "cancel requested by user"
+            return crawl_job_response(job)
 
     @app.post("/discover")
     def discover(request: DiscoverRequest):
@@ -245,14 +532,98 @@ def register_routes(app, database_url: str) -> None:
             return {"ok": False, "error": "robots.txt does not allow discovery for this URL", "links": []}
         fetcher = make_fetcher(source_config.fetcher, source_config.rate_limit_per_minute)
         result = fetcher.fetch(request.url)
-        links = discover_links(result.html, result.final_url or request.url, request.limit)
+        links = discover_links(result.analysis_html, result.final_url or request.url, request.limit)
         return {
             "ok": True,
             "status_code": result.status_code,
             "final_url": result.final_url,
+            "crawl_status": result.crawl_status,
+            "warnings": result.warnings,
             "links": [asdict(link) for link in links],
         }
 
+    @app.post("/research/run")
+    def run_research(request: ResearchRunRequest):
+        config = load_project_config(request.config_path)
+        if request.project_name and request.project_name != config.project_name:
+            raise HTTPException(
+                status_code=400,
+                detail=f"Selected project '{request.project_name}' does not match config project '{config.project_name}'.",
+            )
+        with session_scope(database_url) as session:
+            repo = KnowledgeRepository(session)
+            loop = GraphResearchLoop(
+                repo,
+                extractor_for_domain(
+                    config.domain,
+                    provider=request.extractor_provider,
+                    model=request.extractor_model,
+                    base_url=request.extractor_base_url,
+                ),
+            )
+            try:
+                result = loop.run(
+                    project_config=config,
+                    source_name=request.source_name,
+                    seed_url=request.url or None,
+                    seed_entity_id=request.seed_entity_id,
+                    goal=request.goal,
+                    max_depth=max(request.max_depth, 0),
+                    max_steps=max(min(request.max_steps, 50), 1),
+                    max_branch=max(min(request.max_branch, 30), 1),
+                    min_relevance=min(max(request.min_relevance, 0.0), 1.0),
+                    same_domain_only=request.same_domain_only,
+                    analyze_page_types=set(request.analyze_page_types),
+                )
+            except Exception as exc:
+                raise HTTPException(status_code=400, detail=str(exc)) from exc
+            return asdict(result)
+
+    @app.get("/projects/{project_name}/research/sessions")
+    def research_sessions(project_name: str, limit: int = 25):
+        with session_scope(database_url) as session:
+            project = KnowledgeRepository(session).get_project(project_name)
+            return ResearchMemoryStore(session).list_sessions(project.id, limit)
+
+    @app.get("/research/sessions/{job_id}")
+    def research_session(job_id: int):
+        with session_scope(database_url) as session:
+            job = session.get(models.CrawlJob, job_id)
+            if job is None or (job.metadata_json or {}).get("kind") != "research_session":
+                raise HTTPException(status_code=404, detail="research session not found")
+            return research_session_payload(job)
+
+    @app.get("/projects/{project_name}/graph/neighborhood")
+    def graph_neighborhood(project_name: str, entity_id: int | None = None, limit: int = 120):
+        with session_scope(database_url) as session:
+            project = KnowledgeRepository(session).get_project(project_name)
+            return SemanticGraphQuery(session).neighborhood(project.id, entity_id, max(min(limit, 300), 1))
+
+    @app.get("/projects/{project_name}/graph/query")
+    def graph_query(
+        project_name: str,
+        kind: str = "trend_summary",
+        predicate: str = "hasAccord",
+        tag: str | None = None,
+        brand: str | None = None,
+        limit: int = 100,
+    ):
+        with session_scope(database_url) as session:
+            project = KnowledgeRepository(session).get_project(project_name)
+            graph = SemanticGraphQuery(session)
+            bounded_limit = max(min(limit, 300), 1)
+            if kind == "brand_products":
+                return graph.brand_products(project.id, brand, bounded_limit)
+            if kind == "products_by_tag":
+                return graph.products_by_tag(project.id, predicate, tag, bounded_limit)
+            if kind == "relation_summary":
+                return graph.relation_summary(project.id, bounded_limit)
+            if kind == "entity_type_summary":
+                return graph.entity_type_summary(project.id, bounded_limit)
+            if kind == "trend_summary":
+                return graph.trend_summary(project.id, bounded_limit)
+            raise HTTPException(status_code=400, detail=f"unknown graph query kind: {kind}")
+
     @app.get("/projects/{project_name}/entities")
     def project_entities(project_name: str, entity_type: str | None = None, limit: int = 50):
         with session_scope(database_url) as session:
@@ -272,18 +643,29 @@ def register_routes(app, database_url: str) -> None:
             ]
 
     @app.get("/projects/{project_name}/claims")
-    def project_claims(project_name: str, limit: int = 100):
+    def project_claims(
+        project_name: str,
+        limit: int = 100,
+        include_candidates: bool = False,
+        status: str | None = None,
+    ):
         with session_scope(database_url) as session:
             project = KnowledgeRepository(session).get_project(project_name)
-            rows = session.execute(
+            query = (
                 select(models.Claim, models.Source, models.Page, models.Entity)
                 .join(models.Source, models.Claim.source_id == models.Source.id)
                 .join(models.Page, models.Claim.page_id == models.Page.id, isouter=True)
                 .join(models.Entity, models.Claim.subject_entity_id == models.Entity.id)
                 .where(models.Claim.project_id == project.id)
-                .order_by(models.Claim.last_seen_at.desc())
-                .limit(limit)
-            ).all()
+            )
+            if status:
+                normalized = _normalize_claim_status(status)
+                if normalized is None:
+                    raise HTTPException(status_code=400, detail=f"unknown claim status: {status}")
+                query = query.where(models.Claim.status == normalized)
+            elif not include_candidates:
+                query = query.where(models.Claim.status == "validated_claim")
+            rows = session.execute(query.order_by(models.Claim.last_seen_at.desc()).limit(limit)).all()
             results: list[dict[str, Any]] = []
             for claim, source, page, subject in rows:
                 object_name = None
@@ -307,12 +689,227 @@ def register_routes(app, database_url: str) -> None:
                         "page_url": page.url if page else None,
                         "confidence": claim.confidence,
                         "confidence_reason": claim.confidence_reason,
+                        "status": claim.status,
                         "evidence_text": evidence.evidence_text if evidence else None,
+                        "evidence_summary": evidence.evidence_summary if evidence else None,
+                        "page_type": (claim.metadata_json or {}).get("page_type")
+                        or ((page.metadata_json or {}).get("page_type") if page else None),
+                        "source_zone": (claim.metadata_json or {}).get("source_zone"),
+                        "source_selector": (claim.metadata_json or {}).get("source_selector"),
+                        "evidence_found": (claim.metadata_json or {}).get("evidence_found"),
+                        "validation_status": (claim.metadata_json or {}).get("validation_status"),
+                        "graph_merge_status": (claim.metadata_json or {}).get("graph_merge_status"),
+                        "graph_merge_reason": (claim.metadata_json or {}).get("graph_merge_reason"),
+                        "confidence_breakdown": (claim.metadata_json or {}).get("confidence_breakdown"),
+                        "review_required": (claim.metadata_json or {}).get("review_required"),
+                        "review_reason": (claim.metadata_json or {}).get("review_reason"),
+                        "conflict_status": (claim.metadata_json or {}).get("conflict_status"),
+                        "source_history": (claim.metadata_json or {}).get("source_history") or [],
                         "last_seen_at": claim.last_seen_at.isoformat(),
                     }
                 )
             return results
 
+    @app.get("/projects/{project_name}/pipeline")
+    def project_pipeline(project_name: str):
+        from sqlalchemy import func as sa_func, distinct
+        with session_scope(database_url) as session:
+            project = KnowledgeRepository(session).get_project(project_name)
+            pid = project.id
+
+            pages_total = session.scalar(
+                select(sa_func.count(models.Page.id)).where(models.Page.project_id == pid)
+            ) or 0
+            ext_total = session.scalar(
+                select(sa_func.count(models.ExtractionLog.id)).where(models.ExtractionLog.project_id == pid)
+            ) or 0
+            ext_pages = session.scalar(
+                select(sa_func.count(distinct(models.ExtractionLog.page_id)))
+                .where(models.ExtractionLog.project_id == pid, models.ExtractionLog.page_id.isnot(None))
+            ) or 0
+            ext_errors = session.scalar(
+                select(sa_func.count(models.ExtractionLog.id))
+                .where(models.ExtractionLog.project_id == pid, models.ExtractionLog.error.isnot(None))
+            ) or 0
+
+            status_rows = session.execute(
+                select(models.Claim.status, sa_func.count(models.Claim.id))
+                .where(models.Claim.project_id == pid)
+                .group_by(models.Claim.status)
+            ).all()
+            claim_statuses = {row[0]: int(row[1]) for row in status_rows}
+            claim_total = sum(claim_statuses.values())
+
+            entity_count = session.scalar(
+                select(sa_func.count(models.Entity.id)).where(models.Entity.project_id == pid)
+            ) or 0
+            triple_count = session.scalar(
+                select(sa_func.count(models.OntologyTriple.id))
+                .where(models.OntologyTriple.project_id == pid, models.OntologyTriple.status == "active")
+            ) or 0
+
+            entity_type_rows = session.execute(
+                select(models.Entity.entity_type, sa_func.count(models.Entity.id))
+                .where(models.Entity.project_id == pid)
+                .group_by(models.Entity.entity_type)
+                .order_by(sa_func.count(models.Entity.id).desc())
+                .limit(10)
+            ).all()
+
+            recent_pages = session.scalars(
+                select(models.Page).where(models.Page.project_id == pid)
+                .order_by(models.Page.fetched_at.desc()).limit(5)
+            ).all()
+            recent_claims_rows = session.execute(
+                select(models.Claim, models.Entity)
+                .join(models.Entity, models.Claim.subject_entity_id == models.Entity.id)
+                .where(models.Claim.project_id == pid)
+                .order_by(models.Claim.last_seen_at.desc()).limit(5)
+            ).all()
+
+            return {
+                "stages": [
+                    {"key": "crawled", "count": pages_total},
+                    {"key": "extracted", "count": ext_pages, "extra": {"events": ext_total, "errors": ext_errors}},
+                    {"key": "claims", "count": claim_total, "extra": claim_statuses},
+                    {"key": "validated", "count": claim_statuses.get("validated_claim", 0)},
+                    {"key": "graph", "count": triple_count, "extra": {"entities": entity_count}},
+                ],
+                "entity_types": [{"type": row[0], "count": int(row[1])} for row in entity_type_rows],
+                "recent_pages": [
+                    {
+                        "id": p.id,
+                        "url": p.url,
+                        "title": p.title,
+                        "status_code": p.status_code,
+                        "page_type": (p.metadata_json or {}).get("page_type"),
+                        "fetched_at": p.fetched_at.isoformat() if p.fetched_at else None,
+                    }
+                    for p in recent_pages
+                ],
+                "recent_claims": [
+                    {
+                        "id": claim.id,
+                        "subject": subject.name,
+                        "predicate": claim.predicate,
+                        "confidence": claim.confidence,
+                        "status": claim.status,
+                        "last_seen_at": claim.last_seen_at.isoformat() if claim.last_seen_at else None,
+                    }
+                    for claim, subject in recent_claims_rows
+                ],
+            }
+
+    @app.get("/projects/{project_name}/search")
+    def project_search(project_name: str, q: str = "", limit: int = 10):
+        q = (q or "").strip()
+        if len(q) < 1:
+            return {"entities": [], "claims": [], "pages": [], "predicates": []}
+        like = f"%{q.lower()}%"
+        bounded = max(min(limit, 50), 1)
+        with session_scope(database_url) as session:
+            project = KnowledgeRepository(session).get_project(project_name)
+            pid = project.id
+
+            from sqlalchemy import func as sa_func, or_
+            entity_rows = session.scalars(
+                select(models.Entity)
+                .where(
+                    models.Entity.project_id == pid,
+                    or_(
+                        sa_func.lower(models.Entity.name).like(like),
+                        sa_func.lower(models.Entity.canonical_name).like(like),
+                    ),
+                )
+                .limit(bounded)
+            ).all()
+
+            claim_rows = session.execute(
+                select(models.Claim, models.Entity)
+                .join(models.Entity, models.Claim.subject_entity_id == models.Entity.id)
+                .where(
+                    models.Claim.project_id == pid,
+                    or_(
+                        sa_func.lower(models.Entity.name).like(like),
+                        sa_func.lower(models.Claim.predicate).like(like),
+                    ),
+                )
+                .order_by(models.Claim.last_seen_at.desc())
+                .limit(bounded)
+            ).all()
+
+            page_rows = session.scalars(
+                select(models.Page)
+                .where(
+                    models.Page.project_id == pid,
+                    or_(
+                        sa_func.lower(models.Page.url).like(like),
+                        sa_func.lower(models.Page.title).like(like),
+                    ),
+                )
+                .order_by(models.Page.fetched_at.desc())
+                .limit(bounded)
+            ).all()
+
+            predicate_rows = session.scalars(
+                select(models.OntologyRelationType.name)
+                .where(
+                    models.OntologyRelationType.project_id == pid,
+                    sa_func.lower(models.OntologyRelationType.name).like(like),
+                )
+                .limit(bounded)
+            ).all()
+
+            return {
+                "entities": [
+                    {"id": e.id, "name": e.name, "type": e.entity_type, "canonical_name": e.canonical_name}
+                    for e in entity_rows
+                ],
+                "claims": [
+                    {
+                        "id": claim.id,
+                        "subject": subject.name,
+                        "predicate": claim.predicate,
+                        "object_value": claim.object_value,
+                        "confidence": claim.confidence,
+                        "status": claim.status,
+                    }
+                    for claim, subject in claim_rows
+                ],
+                "pages": [
+                    {"id": p.id, "url": p.url, "title": p.title}
+                    for p in page_rows
+                ],
+                "predicates": list(predicate_rows),
+            }
+
+    @app.get("/projects/{project_name}/extraction-logs")
+    def project_extraction_logs(project_name: str, limit: int = 50):
+        with session_scope(database_url) as session:
+            project = KnowledgeRepository(session).get_project(project_name)
+            rows = session.execute(
+                select(models.ExtractionLog, models.Page)
+                .join(models.Page, models.ExtractionLog.page_id == models.Page.id, isouter=True)
+                .where(models.ExtractionLog.project_id == project.id)
+                .order_by(models.ExtractionLog.created_at.desc())
+                .limit(limit)
+            ).all()
+            return [
+                {
+                    "id": log.id,
+                    "page_url": page.url if page else None,
+                    "extractor_name": log.extractor_name,
+                    "provider": log.provider,
+                    "error": log.error,
+                    "created_at": log.created_at.isoformat(),
+                    "validation": (log.raw_output or {}).get("validation"),
+                    "page_context": (log.raw_output or {}).get("page_context"),
+                    "candidate_count": len((log.raw_output or {}).get("candidate_claims") or []),
+                    "raw_output": log.raw_output,
+                }
+                for log, page in rows
+            ]
+
     @app.patch("/claims/{claim_id}/confidence")
     def update_claim_confidence(claim_id: int, request: UpdateClaimConfidenceRequest):
         confidence = min(max(request.confidence, 0.0), 1.0)
@@ -325,6 +922,33 @@ def register_routes(app, database_url: str) -> None:
             claim.last_seen_at = models.utcnow()
             return {"ok": True, "claim_id": claim.id, "confidence": claim.confidence}
 
+    @app.patch("/claims/{claim_id}/status")
+    def update_claim_status(claim_id: int, request: UpdateClaimStatusRequest):
+        status = _normalize_claim_status(request.status)
+        if status is None:
+            raise HTTPException(status_code=400, detail=f"unknown claim status: {request.status}")
+        with session_scope(database_url) as session:
+            claim = session.get(models.Claim, claim_id)
+            if claim is None:
+                raise HTTPException(status_code=404, detail="claim not found")
+            _apply_claim_review(claim, status, request.reason)
+            return {"ok": True, "claim_id": claim.id, "status": claim.status}
+
+    @app.post("/claims/bulk-status")
+    def bulk_update_claim_status(request: BulkClaimStatusRequest):
+        status = _normalize_claim_status(request.status)
+        if status is None:
+            raise HTTPException(status_code=400, detail=f"unknown claim status: {request.status}")
+        if not request.claim_ids:
+            return {"ok": True, "updated": 0}
+        with session_scope(database_url) as session:
+            rows = session.execute(
+                select(models.Claim).where(models.Claim.id.in_(request.claim_ids))
+            ).scalars().all()
+            for claim in rows:
+                _apply_claim_review(claim, status, request.reason)
+            return {"ok": True, "updated": len(rows), "status": status}
+
     @app.post("/entities/merge")
     def merge_entities(request: MergeEntitiesRequest):
         with session_scope(database_url) as session:
@@ -367,7 +991,11 @@ def register_routes(app, database_url: str) -> None:
             rows = session.execute(
                 select(models.Claim, models.Entity)
                 .join(models.Entity, models.Claim.object_entity_id == models.Entity.id)
-                .where(models.Claim.project_id == project.id, models.Claim.predicate.in_(tag_predicates))
+                .where(
+                    models.Claim.project_id == project.id,
+                    models.Claim.status == "validated_claim",
+                    models.Claim.predicate.in_(tag_predicates),
+                )
             ).all()
             grouped: dict[str, dict[str, Any]] = {}
             for claim, entity in rows: