from __future__ import annotations from dataclasses import asdict from typing import Any from fastapi import BackgroundTasks, HTTPException from pydantic import BaseModel, Field from sqlalchemy import select from crawler_platform.app.config.loader import ( ProjectConfig, SourceConfig, load_project_config, project_config_from_dict, ) from crawler_platform.app.core.crawler.discovery import discover_links from crawler_platform.app.core.crawler.fetchers import RobotsPolicy, make_fetcher from crawler_platform.app.core.crawler.pipeline import CrawlPipeline from crawler_platform.app.core.crawler.site_crawler import SiteCrawler from crawler_platform.app.core.database import models from crawler_platform.app.core.database.repository import KnowledgeRepository from crawler_platform.app.core.database.session import session_scope from crawler_platform.app.core.extractor.ai_provider import list_openai_compatible_models from crawler_platform.app.core.extractor.factory import extractor_for_domain from crawler_platform.app.core.ontology.definitions import DOMAIN_ONTOLOGIES, ontology_for_domain from crawler_platform.app.core.ontology.gap_detector import KnowledgeGapDetector from crawler_platform.app.core.ontology.mapper import ontology_to_dict from crawler_platform.app.core.ontology.registry import OntologyRegistry from crawler_platform.app.core.recommendation.scorer import PreferenceInput, RuleBasedRecommender from crawler_platform.app.core.research.graph_query import SemanticGraphQuery from crawler_platform.app.core.research.graph_research_loop import GraphResearchLoop from crawler_platform.app.core.research.memory_store import ResearchMemoryStore, research_session_payload class CrawlRequest(BaseModel): config_path: str source_name: str url: str extractor_provider: str = "lm_studio" extractor_model: str | None = None extractor_base_url: str | None = "http://localhost:1234/v1" check_robots_txt: bool = False respect_robots_txt: bool | None = None class SiteCrawlRequest(CrawlRequest): max_depth: int = 2 max_pages: int = 50 same_domain_only: bool = True analyze_page_types: list[str] = Field(default_factory=lambda: ["ProductPage", "BrandStoryPage", "ReviewPage"]) class SiteCrawlByProjectRequest(BaseModel): """Site crawl invoked against an existing project (no filesystem config_path).""" project_name: str source_name: str url: str extractor_provider: str = "lm_studio" extractor_model: str | None = None extractor_base_url: str | None = "http://localhost:1234/v1" check_robots_txt: bool = False respect_robots_txt: bool | None = None max_depth: int = 2 max_pages: int = 50 same_domain_only: bool = True analyze_page_types: list[str] = Field( default_factory=lambda: ["ProductPage", "BrandStoryPage", "ReviewPage"] ) def to_site_crawl_request(self, config_path_placeholder: str = "") -> "SiteCrawlRequest": """For internal handoff to existing crawl pipeline (config_path is not used).""" return SiteCrawlRequest( config_path=config_path_placeholder, source_name=self.source_name, url=self.url, extractor_provider=self.extractor_provider, extractor_model=self.extractor_model, extractor_base_url=self.extractor_base_url, check_robots_txt=self.check_robots_txt, respect_robots_txt=self.respect_robots_txt, max_depth=self.max_depth, max_pages=self.max_pages, same_domain_only=self.same_domain_only, analyze_page_types=list(self.analyze_page_types), ) class DiscoverRequest(BaseModel): config_path: str source_name: str url: str limit: int = 30 check_robots_txt: bool = False respect_robots_txt: bool | None = None class RecommendRequest(BaseModel): project_name: str target_entity_type: str = "Perfume" preferences: dict = Field(default_factory=dict) limit: int = 10 class CreateProjectRequest(BaseModel): config_path: str class InlineSourceConfig(BaseModel): name: str type: str = "unknown" trust_level: float = 0.5 base_url: str | None = None allowed_paths: list[str] = Field(default_factory=list) parser: str = "generic" fetcher: str = "requests" rate_limit_per_minute: int = 30 respect_robots_txt: bool = False class CreateProjectInlineRequest(BaseModel): """Create a project from inline JSON config (no filesystem dependency).""" project_name: str domain: str target_entities: list[str] = Field(default_factory=list) fields: list[str] = Field(default_factory=list) sources: list[InlineSourceConfig] = Field(default_factory=list) ontology: dict[str, Any] = Field(default_factory=dict) recommendation: dict[str, Any] = Field(default_factory=dict) update_policy: dict[str, Any] = Field(default_factory=dict) def to_project_config(self) -> ProjectConfig: return ProjectConfig( project_name=self.project_name, domain=self.domain, target_entities=list(self.target_entities), fields=list(self.fields), sources=[SourceConfig(**s.model_dump()) for s in self.sources], ontology=dict(self.ontology), recommendation=dict(self.recommendation), update_policy=dict(self.update_policy), ) class ResetProjectRequest(BaseModel): config_path: str project_name: str | None = None class ResearchRunByProjectRequest(BaseModel): """Run autonomous research against an existing DB project (no config_path).""" project_name: str source_name: str url: str | None = None seed_entity_id: int | None = None goal: str = "Semantic ontology exploration" extractor_provider: str = "lm_studio" extractor_model: str | None = None extractor_base_url: str | None = "http://localhost:1234/v1" check_robots_txt: bool = False respect_robots_txt: bool | None = None max_depth: int = 2 max_steps: int = 12 max_branch: int = 8 min_relevance: float = 0.35 same_domain_only: bool = True analyze_page_types: list[str] = Field( default_factory=lambda: ["ProductPage", "BrandStoryPage", "ReviewPage"] ) class UpdateClaimConfidenceRequest(BaseModel): confidence: float reason: str | None = None class UpdateClaimStatusRequest(BaseModel): status: str reason: str | None = None class BulkClaimStatusRequest(BaseModel): claim_ids: list[int] status: str reason: str | None = None class MergeEntitiesRequest(BaseModel): project_name: str source_entity_id: int target_entity_id: int class ExtractorModelsRequest(BaseModel): provider: str base_url: str | None = None class ResearchRunRequest(CrawlRequest): project_name: str | None = None seed_entity_id: int | None = None goal: str = "Semantic ontology exploration" max_depth: int = 2 max_steps: int = 12 max_branch: int = 8 min_relevance: float = 0.35 same_domain_only: bool = True analyze_page_types: list[str] = Field(default_factory=lambda: ["ProductPage", "BrandStoryPage", "ReviewPage"]) def site_crawl_progress_payload(result, latest_page=None) -> dict[str, Any]: payload = asdict(result) if latest_page is not None: payload["latest_page"] = asdict(latest_page) return payload _CLAIM_STATUS_ALIASES = { "accept": "validated_claim", "accepted": "validated_claim", "validated": "validated_claim", "validated_claim": "validated_claim", "reject": "rejected", "rejected": "rejected", "active": "active", "candidate": "active", "pending": "active", } def _normalize_claim_status(value: str | None) -> str | None: if not value: return None return _CLAIM_STATUS_ALIASES.get(value.strip().lower()) def _apply_claim_review(claim: models.Claim, status: str, reason: str | None) -> None: claim.status = status if reason: claim.confidence_reason = reason metadata = dict(claim.metadata_json or {}) review = dict(metadata.get("review") or {}) review["status"] = status review["reason"] = reason or review.get("reason") review["reviewed_at"] = models.utcnow().isoformat() metadata["review"] = review claim.metadata_json = metadata claim.last_seen_at = models.utcnow() def apply_crawl_request_overrides(config, request: CrawlRequest | DiscoverRequest) -> None: check_robots_txt = request.respect_robots_txt if check_robots_txt is None: check_robots_txt = request.check_robots_txt config.source_by_name(request.source_name).respect_robots_txt = check_robots_txt def crawl_job_response(job: models.CrawlJob) -> dict[str, Any]: metadata = job.metadata_json or {} return { "job_id": job.id, "status": job.status, "url": job.url, "error": job.error, "scheduled_at": job.scheduled_at.isoformat() if job.scheduled_at else None, "started_at": job.started_at.isoformat() if job.started_at else None, "finished_at": job.finished_at.isoformat() if job.finished_at else None, "progress": metadata.get("progress", {}), "request": metadata.get("request", {}), } def update_site_crawl_job_metadata(job: models.CrawlJob, **updates: Any) -> None: metadata = dict(job.metadata_json or {}) metadata.update(updates) job.metadata_json = metadata def is_site_crawl_cancel_requested(session, job_id: int) -> bool: session.expire_all() job = session.get(models.CrawlJob, job_id) return job is None or job.status == "cancel_requested" def run_site_crawl_job(database_url: str, job_id: int, request_data: dict[str, Any]) -> None: inline_config = request_data.pop("__config_dict", None) request = SiteCrawlRequest(**request_data) try: if inline_config is not None: config = project_config_from_dict(inline_config) else: config = load_project_config(request.config_path) apply_crawl_request_overrides(config, request) with session_scope(database_url) as session: job = session.get(models.CrawlJob, job_id) if job is None: return if job.status == "cancel_requested": job.status = "canceled" job.finished_at = models.utcnow() return job.status = "running" job.started_at = models.utcnow() update_site_crawl_job_metadata( job, progress={ "seed_url": request.url, "visited_count": 0, "analyzed_count": 0, "queued_count": 1, "skipped_count": 0, "errors": [], "pages": [], }, ) session.commit() repo = KnowledgeRepository(session) crawler = SiteCrawler( repo, extractor_for_domain( config.domain, provider=request.extractor_provider, model=request.extractor_model, base_url=request.extractor_base_url, ), ) def save_progress(result, latest_page) -> None: current_job = session.get(models.CrawlJob, job_id) if current_job is None: return update_site_crawl_job_metadata( current_job, progress=site_crawl_progress_payload(result, latest_page), ) session.commit() def should_stop() -> bool: return is_site_crawl_cancel_requested(session, job_id) result = crawler.crawl_site( config, request.source_name, request.url, max_depth=max(request.max_depth, 0), max_pages=max(min(request.max_pages, 500), 1), same_domain_only=request.same_domain_only, analyze_page_types=set(request.analyze_page_types), progress_callback=save_progress, should_stop=should_stop, parent_job_id=job_id, ) finished_job = session.get(models.CrawlJob, job_id) if finished_job is not None: finished_job.status = "canceled" if finished_job.status == "cancel_requested" else "completed" finished_job.finished_at = models.utcnow() update_site_crawl_job_metadata( finished_job, progress=site_crawl_progress_payload(result), ) except Exception as exc: with session_scope(database_url) as session: job = session.get(models.CrawlJob, job_id) if job is not None: job.status = "failed" job.error = str(exc) job.finished_at = models.utcnow() metadata = dict(job.metadata_json or {}) progress = dict(metadata.get("progress") or {}) progress["errors"] = [*progress.get("errors", []), str(exc)] update_site_crawl_job_metadata(job, progress=progress) def register_routes(app, database_url: str) -> None: @app.get("/health") def health(): return {"ok": True} @app.get("/projects") def projects(): with session_scope(database_url) as session: rows = session.scalars(select(models.Project).order_by(models.Project.created_at.desc())).all() return [ { "id": project.id, "name": project.name, "domain": project.domain, "created_at": project.created_at.isoformat(), "updated_at": project.updated_at.isoformat(), } for project in rows ] @app.post("/projects") def create_project(request: CreateProjectRequest): config = load_project_config(request.config_path) with session_scope(database_url) as session: project = KnowledgeRepository(session).upsert_project(config) return {"id": project.id, "name": project.name, "domain": project.domain} @app.post("/projects/inline") def create_project_inline(request: CreateProjectInlineRequest): config = request.to_project_config() with session_scope(database_url) as session: project = KnowledgeRepository(session).upsert_project(config) return {"id": project.id, "name": project.name, "domain": project.domain} @app.get("/domains") def list_domains(): """Available pre-defined ontology domains for project creation.""" return [ { "domain": ont.domain, "entity_types": list(ont.entity_types), "predicates": list(ont.predicates), "attribute_count": len(ont.attributes), } for ont in DOMAIN_ONTOLOGIES.values() ] @app.post("/projects/reset") def reset_project(request: ResetProjectRequest): config = load_project_config(request.config_path) if request.project_name and request.project_name != config.project_name: raise HTTPException( status_code=400, detail=( f"Selected project '{request.project_name}' does not match " f"config project '{config.project_name}'." ), ) with session_scope(database_url) as session: repo = KnowledgeRepository(session) project = session.scalar(select(models.Project).where(models.Project.name == config.project_name)) if project is None: project = repo.upsert_project(config) return { "ok": True, "name": project.name, "domain": project.domain, "created": True, "reset": False, "deleted": {}, } deleted = repo.reset_project_runtime_data(project.id) project = repo.upsert_project(config) return { "ok": True, "name": project.name, "domain": project.domain, "created": False, "reset": True, "deleted": deleted, } @app.get("/projects/{project_name}") def project_detail(project_name: str): with session_scope(database_url) as session: project = KnowledgeRepository(session).get_project(project_name) sources = session.scalars(select(models.Source).where(models.Source.project_id == project.id)).all() return { "id": project.id, "name": project.name, "domain": project.domain, "config": project.config, "sources": [ { "id": source.id, "name": source.name, "type": source.type, "base_url": source.base_url, "trust_level": source.trust_level, "respect_robots_txt": source.respect_robots_txt, "rate_limit_per_minute": source.rate_limit_per_minute, } for source in sources ], } @app.post("/projects/{project_name}/sources") def add_project_source(project_name: str, request: InlineSourceConfig): """Add or update a source on an existing project.""" with session_scope(database_url) as session: repo = KnowledgeRepository(session) project = repo.get_project(project_name) source = repo.upsert_source(project, SourceConfig(**request.model_dump())) return { "id": source.id, "name": source.name, "type": source.type, "base_url": source.base_url, "trust_level": source.trust_level, "respect_robots_txt": source.respect_robots_txt, "rate_limit_per_minute": source.rate_limit_per_minute, } @app.delete("/projects/{project_name}/sources/{source_name}") def delete_project_source(project_name: str, source_name: str): with session_scope(database_url) as session: repo = KnowledgeRepository(session) project = repo.get_project(project_name) source = session.scalar( select(models.Source).where( models.Source.project_id == project.id, models.Source.name == source_name, ) ) if source is None: raise HTTPException( status_code=404, detail=f"Source '{source_name}' not found in project '{project_name}'", ) session.delete(source) return {"ok": True, "deleted": source_name} @app.get("/ontology/{domain}") def ontology(domain: str): return ontology_to_dict(ontology_for_domain(domain)) @app.get("/projects/{project_name}/ontology/registry") def ontology_registry(project_name: str): with session_scope(database_url) as session: project = KnowledgeRepository(session).get_project(project_name) return OntologyRegistry(session).registry_payload(project.id) @app.get("/projects/{project_name}/ontology/proposals") def ontology_proposals(project_name: str, limit: int = 100): with session_scope(database_url) as session: project = KnowledgeRepository(session).get_project(project_name) rows = session.scalars( select(models.OntologyProposal) .where(models.OntologyProposal.project_id == project.id) .order_by(models.OntologyProposal.updated_at.desc()) .limit(max(min(limit, 300), 1)) ).all() return [ { "id": row.id, "proposal_type": row.proposal_type, "name": row.name, "reason": row.reason, "evidence": row.evidence, "status": row.status, "confidence": row.confidence, "metadata": row.metadata_json or {}, "updated_at": row.updated_at.isoformat() if row.updated_at else None, } for row in rows ] @app.get("/projects/{project_name}/ontology/triples") def ontology_triples(project_name: str, status: str | None = None, limit: int = 100): with session_scope(database_url) as session: project = KnowledgeRepository(session).get_project(project_name) query = select(models.OntologyTriple).where(models.OntologyTriple.project_id == project.id) if status: query = query.where(models.OntologyTriple.status == status) rows = session.scalars( query.order_by(models.OntologyTriple.last_seen_at.desc()).limit(max(min(limit, 300), 1)) ).all() results = [] for row in rows: subject = session.get(models.Entity, row.subject_entity_id) object_entity = session.get(models.Entity, row.object_entity_id) if row.object_entity_id else None results.append( { "id": row.id, "claim_id": row.claim_id, "subject": subject.name if subject else None, "subject_type": row.subject_type, "predicate": row.predicate, "object": object_entity.name if object_entity else None, "object_type": row.object_type, "object_value": row.object_value, "value_type": row.value_type, "status": row.status, "confidence": row.confidence, "support_count": row.support_count, "metadata": row.metadata_json or {}, "last_seen_at": row.last_seen_at.isoformat() if row.last_seen_at else None, } ) return results @app.get("/projects/{project_name}/knowledge-gaps") def knowledge_gaps(project_name: str, limit: int = 100): with session_scope(database_url) as session: project = KnowledgeRepository(session).get_project(project_name) return KnowledgeGapDetector(session).list_open(project.id, max(min(limit, 300), 1)) @app.post("/extractors/models") def extractor_models(request: ExtractorModelsRequest): try: if request.provider == "lm_studio": models = list_openai_compatible_models(request.base_url or "http://localhost:1234/v1") return {"ok": True, "models": models} if request.provider == "openai": import os models = list_openai_compatible_models( request.base_url or "https://api.openai.com/v1", os.getenv("OPENAI_API_KEY"), ) return {"ok": True, "models": models} if request.provider == "ollama": return {"ok": False, "error": "Ollama model listing is not implemented yet. Enter the model manually."} return {"ok": True, "models": [{"id": "rule_based", "owned_by": "local"}]} except Exception as exc: return {"ok": False, "error": str(exc), "models": []} @app.post("/crawl") def crawl(request: CrawlRequest): config = load_project_config(request.config_path) apply_crawl_request_overrides(config, request) with session_scope(database_url) as session: repo = KnowledgeRepository(session) pipeline = CrawlPipeline( repo, extractor_for_domain( config.domain, provider=request.extractor_provider, model=request.extractor_model, base_url=request.extractor_base_url, ), ) try: result = pipeline.crawl_url(config, request.source_name, request.url) except Exception as exc: raise HTTPException(status_code=400, detail=str(exc)) from exc return { "page_id": result.page_id, "claim_count": result.claim_count, "entity_count": result.entity_count, "crawl_status": result.crawl_status, "extraction_status": result.extraction_status, "page_type": result.page_type, "raw_text_length": result.raw_text_length, "clean_text_length": result.clean_text_length, "warnings": result.warnings or [], } @app.post("/crawl-site") def crawl_site(request: SiteCrawlRequest, background_tasks: BackgroundTasks): config = load_project_config(request.config_path) apply_crawl_request_overrides(config, request) with session_scope(database_url) as session: repo = KnowledgeRepository(session) project = repo.upsert_project(config) source = repo.get_source(project.id, request.source_name) job = models.CrawlJob( project_id=project.id, source_id=source.id, url=request.url, status="pending", metadata_json={ "kind": "site_crawl", "request": request.model_dump(), "progress": { "seed_url": request.url, "visited_count": 0, "analyzed_count": 0, "queued_count": 1, "skipped_count": 0, "errors": [], "pages": [], }, }, ) session.add(job) session.flush() response = crawl_job_response(job) background_tasks.add_task(run_site_crawl_job, database_url, response["job_id"], request.model_dump()) return response @app.post("/crawl-site/by-project") def crawl_site_by_project( request: SiteCrawlByProjectRequest, background_tasks: BackgroundTasks ): """Start a site crawl against an existing DB project (no config_path).""" with session_scope(database_url) as session: repo = KnowledgeRepository(session) project = repo.get_project(request.project_name) config_dict = project.config or {} if not config_dict: raise HTTPException( status_code=400, detail=f"Project '{request.project_name}' has no stored config", ) config = project_config_from_dict(config_dict) try: source = repo.get_source(project.id, request.source_name) except KeyError as exc: raise HTTPException(status_code=404, detail=str(exc)) from exc inner_request = request.to_site_crawl_request() apply_crawl_request_overrides(config, inner_request) job = models.CrawlJob( project_id=project.id, source_id=source.id, url=request.url, status="pending", metadata_json={ "kind": "site_crawl", "request": inner_request.model_dump(), "project_name": request.project_name, "progress": { "seed_url": request.url, "visited_count": 0, "analyzed_count": 0, "queued_count": 1, "skipped_count": 0, "errors": [], "pages": [], }, }, ) session.add(job) session.flush() response = crawl_job_response(job) task_payload = {**inner_request.model_dump(), "__config_dict": config_dict} background_tasks.add_task( run_site_crawl_job, database_url, response["job_id"], task_payload ) return response @app.get("/crawl-site/jobs/{job_id}") def crawl_site_job(job_id: int): with session_scope(database_url) as session: job = session.get(models.CrawlJob, job_id) if job is None or (job.metadata_json or {}).get("kind") != "site_crawl": raise HTTPException(status_code=404, detail="site crawl job not found") return crawl_job_response(job) @app.post("/crawl-site/jobs/{job_id}/cancel") def cancel_crawl_site_job(job_id: int): with session_scope(database_url) as session: job = session.get(models.CrawlJob, job_id) if job is None or (job.metadata_json or {}).get("kind") != "site_crawl": raise HTTPException(status_code=404, detail="site crawl job not found") if job.status in {"completed", "failed", "canceled"}: return crawl_job_response(job) job.status = "cancel_requested" job.error = "cancel requested by user" return crawl_job_response(job) @app.post("/discover") def discover(request: DiscoverRequest): config = load_project_config(request.config_path) apply_crawl_request_overrides(config, request) source_config = config.source_by_name(request.source_name) robots = RobotsPolicy() robots_decision = robots.check(request.url, source_config.respect_robots_txt) if not robots_decision.allowed: return { "ok": False, "error": f"{robots_decision.reason}: {request.url}", "robots_status": robots_decision.status, "robots_reason": robots_decision.reason, "links": [], } fetcher = make_fetcher(source_config.fetcher, source_config.rate_limit_per_minute) result = fetcher.fetch(request.url) links = discover_links(result.analysis_html, result.final_url or request.url, request.limit) return { "ok": True, "status_code": result.status_code, "final_url": result.final_url, "crawl_status": result.crawl_status, "robots_status": robots_decision.status, "robots_reason": robots_decision.reason, "warnings": result.warnings, "links": [asdict(link) for link in links], } @app.post("/research/run") def run_research(request: ResearchRunRequest): config = load_project_config(request.config_path) apply_crawl_request_overrides(config, request) if request.project_name and request.project_name != config.project_name: raise HTTPException( status_code=400, detail=f"Selected project '{request.project_name}' does not match config project '{config.project_name}'.", ) with session_scope(database_url) as session: repo = KnowledgeRepository(session) loop = GraphResearchLoop( repo, extractor_for_domain( config.domain, provider=request.extractor_provider, model=request.extractor_model, base_url=request.extractor_base_url, ), ) try: result = loop.run( project_config=config, source_name=request.source_name, seed_url=request.url or None, seed_entity_id=request.seed_entity_id, goal=request.goal, max_depth=max(request.max_depth, 0), max_steps=max(min(request.max_steps, 50), 1), max_branch=max(min(request.max_branch, 30), 1), min_relevance=min(max(request.min_relevance, 0.0), 1.0), same_domain_only=request.same_domain_only, analyze_page_types=set(request.analyze_page_types), ) except Exception as exc: raise HTTPException(status_code=400, detail=str(exc)) from exc return asdict(result) @app.post("/research/run/by-project") def run_research_by_project(request: ResearchRunByProjectRequest): """Run research against an existing DB project (no config_path).""" with session_scope(database_url) as session: repo = KnowledgeRepository(session) project = repo.get_project(request.project_name) config_dict = project.config or {} if not config_dict: raise HTTPException( status_code=400, detail=f"Project '{request.project_name}' has no stored config", ) config = project_config_from_dict(config_dict) try: source_config = config.source_by_name(request.source_name) except KeyError as exc: raise HTTPException(status_code=404, detail=str(exc)) from exc check_robots = request.respect_robots_txt if check_robots is None: check_robots = request.check_robots_txt source_config.respect_robots_txt = check_robots loop = GraphResearchLoop( repo, extractor_for_domain( config.domain, provider=request.extractor_provider, model=request.extractor_model, base_url=request.extractor_base_url, ), ) try: result = loop.run( project_config=config, source_name=request.source_name, seed_url=request.url or None, seed_entity_id=request.seed_entity_id, goal=request.goal, max_depth=max(request.max_depth, 0), max_steps=max(min(request.max_steps, 50), 1), max_branch=max(min(request.max_branch, 30), 1), min_relevance=min(max(request.min_relevance, 0.0), 1.0), same_domain_only=request.same_domain_only, analyze_page_types=set(request.analyze_page_types), ) except Exception as exc: raise HTTPException(status_code=400, detail=str(exc)) from exc return asdict(result) @app.get("/projects/{project_name}/research/sessions") def research_sessions(project_name: str, limit: int = 25): with session_scope(database_url) as session: project = KnowledgeRepository(session).get_project(project_name) return ResearchMemoryStore(session).list_sessions(project.id, limit) @app.get("/research/sessions/{job_id}") def research_session(job_id: int): with session_scope(database_url) as session: job = session.get(models.CrawlJob, job_id) if job is None or (job.metadata_json or {}).get("kind") != "research_session": raise HTTPException(status_code=404, detail="research session not found") return research_session_payload(job) @app.get("/projects/{project_name}/graph/neighborhood") def graph_neighborhood(project_name: str, entity_id: int | None = None, limit: int = 120): with session_scope(database_url) as session: project = KnowledgeRepository(session).get_project(project_name) return SemanticGraphQuery(session).neighborhood(project.id, entity_id, max(min(limit, 300), 1)) @app.get("/projects/{project_name}/graph/query") def graph_query( project_name: str, kind: str = "trend_summary", predicate: str = "hasAccord", tag: str | None = None, brand: str | None = None, limit: int = 100, ): with session_scope(database_url) as session: project = KnowledgeRepository(session).get_project(project_name) graph = SemanticGraphQuery(session) bounded_limit = max(min(limit, 300), 1) if kind == "brand_products": return graph.brand_products(project.id, brand, bounded_limit) if kind == "products_by_tag": return graph.products_by_tag(project.id, predicate, tag, bounded_limit) if kind == "relation_summary": return graph.relation_summary(project.id, bounded_limit) if kind == "entity_type_summary": return graph.entity_type_summary(project.id, bounded_limit) if kind == "trend_summary": return graph.trend_summary(project.id, bounded_limit) raise HTTPException(status_code=400, detail=f"unknown graph query kind: {kind}") @app.get("/projects/{project_name}/entities") def project_entities(project_name: str, entity_type: str | None = None, limit: int = 50): with session_scope(database_url) as session: project = KnowledgeRepository(session).get_project(project_name) query = select(models.Entity).where(models.Entity.project_id == project.id) if entity_type: query = query.where(models.Entity.entity_type == entity_type) entities = session.scalars(query.limit(limit)).all() return [ { "id": entity.id, "type": entity.entity_type, "name": entity.name, "metadata": entity.metadata_json, } for entity in entities ] @app.get("/projects/{project_name}/claims") def project_claims( project_name: str, limit: int = 100, include_candidates: bool = False, status: str | None = None, ): with session_scope(database_url) as session: project = KnowledgeRepository(session).get_project(project_name) query = ( select(models.Claim, models.Source, models.Page, models.Entity) .join(models.Source, models.Claim.source_id == models.Source.id) .join(models.Page, models.Claim.page_id == models.Page.id, isouter=True) .join(models.Entity, models.Claim.subject_entity_id == models.Entity.id) .where(models.Claim.project_id == project.id) ) if status: normalized = _normalize_claim_status(status) if normalized is None: raise HTTPException(status_code=400, detail=f"unknown claim status: {status}") query = query.where(models.Claim.status == normalized) elif not include_candidates: query = query.where(models.Claim.status == "validated_claim") rows = session.execute(query.order_by(models.Claim.last_seen_at.desc()).limit(limit)).all() results: list[dict[str, Any]] = [] for claim, source, page, subject in rows: object_name = None if claim.object_entity_id: object_entity = session.get(models.Entity, claim.object_entity_id) object_name = object_entity.name if object_entity else None evidence = session.scalar( select(models.Evidence) .where(models.Evidence.claim_id == claim.id) .order_by(models.Evidence.created_at.desc()) ) results.append( { "id": claim.id, "subject": subject.name, "subject_type": subject.entity_type, "predicate": claim.predicate, "object": object_name, "object_value": claim.object_value, "source": source.name, "page_url": page.url if page else None, "confidence": claim.confidence, "confidence_reason": claim.confidence_reason, "status": claim.status, "evidence_text": evidence.evidence_text if evidence else None, "evidence_summary": evidence.evidence_summary if evidence else None, "page_type": (claim.metadata_json or {}).get("page_type") or ((page.metadata_json or {}).get("page_type") if page else None), "source_zone": (claim.metadata_json or {}).get("source_zone"), "source_selector": (claim.metadata_json or {}).get("source_selector"), "evidence_found": (claim.metadata_json or {}).get("evidence_found"), "validation_status": (claim.metadata_json or {}).get("validation_status"), "graph_merge_status": (claim.metadata_json or {}).get("graph_merge_status"), "graph_merge_reason": (claim.metadata_json or {}).get("graph_merge_reason"), "confidence_breakdown": (claim.metadata_json or {}).get("confidence_breakdown"), "review_required": (claim.metadata_json or {}).get("review_required"), "review_reason": (claim.metadata_json or {}).get("review_reason"), "conflict_status": (claim.metadata_json or {}).get("conflict_status"), "source_history": (claim.metadata_json or {}).get("source_history") or [], "last_seen_at": claim.last_seen_at.isoformat(), } ) return results @app.get("/projects/{project_name}/pipeline") def project_pipeline(project_name: str): from sqlalchemy import func as sa_func, distinct with session_scope(database_url) as session: project = KnowledgeRepository(session).get_project(project_name) pid = project.id pages_total = session.scalar( select(sa_func.count(models.Page.id)).where(models.Page.project_id == pid) ) or 0 ext_total = session.scalar( select(sa_func.count(models.ExtractionLog.id)).where(models.ExtractionLog.project_id == pid) ) or 0 ext_pages = session.scalar( select(sa_func.count(distinct(models.ExtractionLog.page_id))) .where(models.ExtractionLog.project_id == pid, models.ExtractionLog.page_id.isnot(None)) ) or 0 ext_errors = session.scalar( select(sa_func.count(models.ExtractionLog.id)) .where(models.ExtractionLog.project_id == pid, models.ExtractionLog.error.isnot(None)) ) or 0 status_rows = session.execute( select(models.Claim.status, sa_func.count(models.Claim.id)) .where(models.Claim.project_id == pid) .group_by(models.Claim.status) ).all() claim_statuses = {row[0]: int(row[1]) for row in status_rows} claim_total = sum(claim_statuses.values()) entity_count = session.scalar( select(sa_func.count(models.Entity.id)).where(models.Entity.project_id == pid) ) or 0 triple_count = session.scalar( select(sa_func.count(models.OntologyTriple.id)) .where(models.OntologyTriple.project_id == pid, models.OntologyTriple.status == "active") ) or 0 entity_type_rows = session.execute( select(models.Entity.entity_type, sa_func.count(models.Entity.id)) .where(models.Entity.project_id == pid) .group_by(models.Entity.entity_type) .order_by(sa_func.count(models.Entity.id).desc()) .limit(10) ).all() recent_pages = session.scalars( select(models.Page).where(models.Page.project_id == pid) .order_by(models.Page.fetched_at.desc()).limit(5) ).all() recent_claims_rows = session.execute( select(models.Claim, models.Entity) .join(models.Entity, models.Claim.subject_entity_id == models.Entity.id) .where(models.Claim.project_id == pid) .order_by(models.Claim.last_seen_at.desc()).limit(5) ).all() return { "stages": [ {"key": "crawled", "count": pages_total}, {"key": "extracted", "count": ext_pages, "extra": {"events": ext_total, "errors": ext_errors}}, {"key": "claims", "count": claim_total, "extra": claim_statuses}, {"key": "validated", "count": claim_statuses.get("validated_claim", 0)}, {"key": "graph", "count": triple_count, "extra": {"entities": entity_count}}, ], "entity_types": [{"type": row[0], "count": int(row[1])} for row in entity_type_rows], "recent_pages": [ { "id": p.id, "url": p.url, "title": p.title, "status_code": p.status_code, "page_type": (p.metadata_json or {}).get("page_type"), "fetched_at": p.fetched_at.isoformat() if p.fetched_at else None, } for p in recent_pages ], "recent_claims": [ { "id": claim.id, "subject": subject.name, "predicate": claim.predicate, "confidence": claim.confidence, "status": claim.status, "last_seen_at": claim.last_seen_at.isoformat() if claim.last_seen_at else None, } for claim, subject in recent_claims_rows ], } @app.get("/projects/{project_name}/search") def project_search(project_name: str, q: str = "", limit: int = 10): q = (q or "").strip() if len(q) < 1: return {"entities": [], "claims": [], "pages": [], "predicates": []} like = f"%{q.lower()}%" bounded = max(min(limit, 50), 1) with session_scope(database_url) as session: project = KnowledgeRepository(session).get_project(project_name) pid = project.id from sqlalchemy import func as sa_func, or_ entity_rows = session.scalars( select(models.Entity) .where( models.Entity.project_id == pid, or_( sa_func.lower(models.Entity.name).like(like), sa_func.lower(models.Entity.canonical_name).like(like), ), ) .limit(bounded) ).all() claim_rows = session.execute( select(models.Claim, models.Entity) .join(models.Entity, models.Claim.subject_entity_id == models.Entity.id) .where( models.Claim.project_id == pid, or_( sa_func.lower(models.Entity.name).like(like), sa_func.lower(models.Claim.predicate).like(like), ), ) .order_by(models.Claim.last_seen_at.desc()) .limit(bounded) ).all() page_rows = session.scalars( select(models.Page) .where( models.Page.project_id == pid, or_( sa_func.lower(models.Page.url).like(like), sa_func.lower(models.Page.title).like(like), ), ) .order_by(models.Page.fetched_at.desc()) .limit(bounded) ).all() predicate_rows = session.scalars( select(models.OntologyRelationType.name) .where( models.OntologyRelationType.project_id == pid, sa_func.lower(models.OntologyRelationType.name).like(like), ) .limit(bounded) ).all() return { "entities": [ {"id": e.id, "name": e.name, "type": e.entity_type, "canonical_name": e.canonical_name} for e in entity_rows ], "claims": [ { "id": claim.id, "subject": subject.name, "predicate": claim.predicate, "object_value": claim.object_value, "confidence": claim.confidence, "status": claim.status, } for claim, subject in claim_rows ], "pages": [ {"id": p.id, "url": p.url, "title": p.title} for p in page_rows ], "predicates": list(predicate_rows), } @app.get("/projects/{project_name}/extraction-logs") def project_extraction_logs(project_name: str, limit: int = 50): with session_scope(database_url) as session: project = KnowledgeRepository(session).get_project(project_name) rows = session.execute( select(models.ExtractionLog, models.Page) .join(models.Page, models.ExtractionLog.page_id == models.Page.id, isouter=True) .where(models.ExtractionLog.project_id == project.id) .order_by(models.ExtractionLog.created_at.desc()) .limit(limit) ).all() return [ { "id": log.id, "page_url": page.url if page else None, "extractor_name": log.extractor_name, "provider": log.provider, "error": log.error, "created_at": log.created_at.isoformat(), "validation": (log.raw_output or {}).get("validation"), "page_context": (log.raw_output or {}).get("page_context"), "candidate_count": len((log.raw_output or {}).get("candidate_claims") or []), "raw_output": log.raw_output, } for log, page in rows ] @app.patch("/claims/{claim_id}/confidence") def update_claim_confidence(claim_id: int, request: UpdateClaimConfidenceRequest): confidence = min(max(request.confidence, 0.0), 1.0) with session_scope(database_url) as session: claim = session.get(models.Claim, claim_id) if claim is None: return {"ok": False, "error": "claim not found"} claim.confidence = confidence claim.confidence_reason = request.reason or "manual admin update" claim.last_seen_at = models.utcnow() return {"ok": True, "claim_id": claim.id, "confidence": claim.confidence} @app.patch("/claims/{claim_id}/status") def update_claim_status(claim_id: int, request: UpdateClaimStatusRequest): status = _normalize_claim_status(request.status) if status is None: raise HTTPException(status_code=400, detail=f"unknown claim status: {request.status}") with session_scope(database_url) as session: claim = session.get(models.Claim, claim_id) if claim is None: raise HTTPException(status_code=404, detail="claim not found") _apply_claim_review(claim, status, request.reason) return {"ok": True, "claim_id": claim.id, "status": claim.status} @app.post("/claims/bulk-status") def bulk_update_claim_status(request: BulkClaimStatusRequest): status = _normalize_claim_status(request.status) if status is None: raise HTTPException(status_code=400, detail=f"unknown claim status: {request.status}") if not request.claim_ids: return {"ok": True, "updated": 0} with session_scope(database_url) as session: rows = session.execute( select(models.Claim).where(models.Claim.id.in_(request.claim_ids)) ).scalars().all() for claim in rows: _apply_claim_review(claim, status, request.reason) return {"ok": True, "updated": len(rows), "status": status} @app.post("/entities/merge") def merge_entities(request: MergeEntitiesRequest): with session_scope(database_url) as session: project = KnowledgeRepository(session).get_project(request.project_name) source = session.get(models.Entity, request.source_entity_id) target = session.get(models.Entity, request.target_entity_id) if source is None or target is None or source.project_id != project.id or target.project_id != project.id: return {"ok": False, "error": "entity not found in project"} session.query(models.Claim).filter(models.Claim.subject_entity_id == source.id).update( {models.Claim.subject_entity_id: target.id} ) session.query(models.Claim).filter(models.Claim.object_entity_id == source.id).update( {models.Claim.object_entity_id: target.id} ) session.query(models.Relation).filter(models.Relation.subject_entity_id == source.id).update( {models.Relation.subject_entity_id: target.id} ) session.query(models.Relation).filter(models.Relation.object_entity_id == source.id).update( {models.Relation.object_entity_id: target.id} ) source.metadata_json = {**(source.metadata_json or {}), "merged_into": target.id} source.updated_at = models.utcnow() return {"ok": True, "source_entity_id": source.id, "target_entity_id": target.id} @app.get("/projects/{project_name}/recommendation-tags") def recommendation_tags(project_name: str): with session_scope(database_url) as session: project = KnowledgeRepository(session).get_project(project_name) tag_predicates = { "hasTopNote", "hasMiddleNote", "hasBaseNote", "hasScentNote", "hasFlavorNote", "evokesMood", "suitableForSeason", "suitableForOccasion", "hasReviewKeyword", } rows = session.execute( select(models.Claim, models.Entity) .join(models.Entity, models.Claim.object_entity_id == models.Entity.id) .where( models.Claim.project_id == project.id, models.Claim.status == "validated_claim", models.Claim.predicate.in_(tag_predicates), ) ).all() grouped: dict[str, dict[str, Any]] = {} for claim, entity in rows: key = f"{claim.predicate}:{entity.canonical_name}" if key not in grouped: grouped[key] = { "predicate": claim.predicate, "name": entity.name, "type": entity.entity_type, "support_count": 0, "max_confidence": 0.0, } grouped[key]["support_count"] += 1 grouped[key]["max_confidence"] = max(grouped[key]["max_confidence"], claim.confidence) return sorted(grouped.values(), key=lambda item: (item["predicate"], -item["support_count"], item["name"])) @app.post("/recommend") def recommend(request: RecommendRequest): with session_scope(database_url) as session: repo = KnowledgeRepository(session) project = repo.get_project(request.project_name) recommender = RuleBasedRecommender(session) pref = PreferenceInput(**request.preferences) items = recommender.recommend(project.id, request.target_entity_type, pref, request.limit) return [asdict(item) for item in items]