from __future__ import annotations from dataclasses import asdict from typing import Any from fastapi import HTTPException from pydantic import BaseModel, Field from sqlalchemy import select from crawler_platform.app.config.loader import load_project_config from crawler_platform.app.core.crawler.discovery import discover_links from crawler_platform.app.core.crawler.fetchers import RobotsPolicy, make_fetcher from crawler_platform.app.core.crawler.pipeline import CrawlPipeline from crawler_platform.app.core.database import models from crawler_platform.app.core.database.repository import KnowledgeRepository from crawler_platform.app.core.database.session import session_scope from crawler_platform.app.core.extractor.ai_provider import list_openai_compatible_models from crawler_platform.app.core.extractor.factory import extractor_for_domain from crawler_platform.app.core.ontology.definitions import ontology_for_domain from crawler_platform.app.core.ontology.mapper import ontology_to_dict from crawler_platform.app.core.recommendation.scorer import PreferenceInput, RuleBasedRecommender class CrawlRequest(BaseModel): config_path: str source_name: str url: str extractor_provider: str = "rule_based" extractor_model: str | None = None extractor_base_url: str | None = None class DiscoverRequest(BaseModel): config_path: str source_name: str url: str limit: int = 30 class RecommendRequest(BaseModel): project_name: str target_entity_type: str = "Perfume" preferences: dict = Field(default_factory=dict) limit: int = 10 class CreateProjectRequest(BaseModel): config_path: str class UpdateClaimConfidenceRequest(BaseModel): confidence: float reason: str | None = None class MergeEntitiesRequest(BaseModel): project_name: str source_entity_id: int target_entity_id: int class ExtractorModelsRequest(BaseModel): provider: str base_url: str | None = None def register_routes(app, database_url: str) -> None: @app.get("/health") def health(): return {"ok": True} @app.get("/projects") def projects(): with session_scope(database_url) as session: rows = session.scalars(select(models.Project).order_by(models.Project.created_at.desc())).all() return [ { "id": project.id, "name": project.name, "domain": project.domain, "created_at": project.created_at.isoformat(), "updated_at": project.updated_at.isoformat(), } for project in rows ] @app.post("/projects") def create_project(request: CreateProjectRequest): config = load_project_config(request.config_path) with session_scope(database_url) as session: project = KnowledgeRepository(session).upsert_project(config) return {"id": project.id, "name": project.name, "domain": project.domain} @app.get("/projects/{project_name}") def project_detail(project_name: str): with session_scope(database_url) as session: project = KnowledgeRepository(session).get_project(project_name) sources = session.scalars(select(models.Source).where(models.Source.project_id == project.id)).all() return { "id": project.id, "name": project.name, "domain": project.domain, "config": project.config, "sources": [ { "id": source.id, "name": source.name, "type": source.type, "trust_level": source.trust_level, "respect_robots_txt": source.respect_robots_txt, "rate_limit_per_minute": source.rate_limit_per_minute, } for source in sources ], } @app.get("/ontology/{domain}") def ontology(domain: str): return ontology_to_dict(ontology_for_domain(domain)) @app.post("/extractors/models") def extractor_models(request: ExtractorModelsRequest): try: if request.provider == "lm_studio": models = list_openai_compatible_models(request.base_url or "http://localhost:1234/v1") return {"ok": True, "models": models} if request.provider == "openai": import os models = list_openai_compatible_models( request.base_url or "https://api.openai.com/v1", os.getenv("OPENAI_API_KEY"), ) return {"ok": True, "models": models} if request.provider == "ollama": return {"ok": False, "error": "Ollama model listing is not implemented yet. Enter the model manually."} return {"ok": True, "models": [{"id": "rule_based", "owned_by": "local"}]} except Exception as exc: return {"ok": False, "error": str(exc), "models": []} @app.post("/crawl") def crawl(request: CrawlRequest): config = load_project_config(request.config_path) with session_scope(database_url) as session: repo = KnowledgeRepository(session) pipeline = CrawlPipeline( repo, extractor_for_domain( config.domain, provider=request.extractor_provider, model=request.extractor_model, base_url=request.extractor_base_url, ), ) try: result = pipeline.crawl_url(config, request.source_name, request.url) except Exception as exc: raise HTTPException(status_code=400, detail=str(exc)) from exc return {"page_id": result.page_id, "claim_count": result.claim_count, "entity_count": result.entity_count} @app.post("/discover") def discover(request: DiscoverRequest): config = load_project_config(request.config_path) source_config = config.source_by_name(request.source_name) robots = RobotsPolicy() if not robots.allowed(request.url, source_config.respect_robots_txt): return {"ok": False, "error": "robots.txt does not allow discovery for this URL", "links": []} fetcher = make_fetcher(source_config.fetcher, source_config.rate_limit_per_minute) result = fetcher.fetch(request.url) links = discover_links(result.html, result.final_url or request.url, request.limit) return { "ok": True, "status_code": result.status_code, "final_url": result.final_url, "links": [asdict(link) for link in links], } @app.get("/projects/{project_name}/entities") def project_entities(project_name: str, entity_type: str | None = None, limit: int = 50): with session_scope(database_url) as session: project = KnowledgeRepository(session).get_project(project_name) query = select(models.Entity).where(models.Entity.project_id == project.id) if entity_type: query = query.where(models.Entity.entity_type == entity_type) entities = session.scalars(query.limit(limit)).all() return [ { "id": entity.id, "type": entity.entity_type, "name": entity.name, "metadata": entity.metadata_json, } for entity in entities ] @app.get("/projects/{project_name}/claims") def project_claims(project_name: str, limit: int = 100): with session_scope(database_url) as session: project = KnowledgeRepository(session).get_project(project_name) rows = session.execute( select(models.Claim, models.Source, models.Page, models.Entity) .join(models.Source, models.Claim.source_id == models.Source.id) .join(models.Page, models.Claim.page_id == models.Page.id, isouter=True) .join(models.Entity, models.Claim.subject_entity_id == models.Entity.id) .where(models.Claim.project_id == project.id) .order_by(models.Claim.last_seen_at.desc()) .limit(limit) ).all() results: list[dict[str, Any]] = [] for claim, source, page, subject in rows: object_name = None if claim.object_entity_id: object_entity = session.get(models.Entity, claim.object_entity_id) object_name = object_entity.name if object_entity else None evidence = session.scalar( select(models.Evidence) .where(models.Evidence.claim_id == claim.id) .order_by(models.Evidence.created_at.desc()) ) results.append( { "id": claim.id, "subject": subject.name, "subject_type": subject.entity_type, "predicate": claim.predicate, "object": object_name, "object_value": claim.object_value, "source": source.name, "page_url": page.url if page else None, "confidence": claim.confidence, "confidence_reason": claim.confidence_reason, "evidence_text": evidence.evidence_text if evidence else None, "last_seen_at": claim.last_seen_at.isoformat(), } ) return results @app.patch("/claims/{claim_id}/confidence") def update_claim_confidence(claim_id: int, request: UpdateClaimConfidenceRequest): confidence = min(max(request.confidence, 0.0), 1.0) with session_scope(database_url) as session: claim = session.get(models.Claim, claim_id) if claim is None: return {"ok": False, "error": "claim not found"} claim.confidence = confidence claim.confidence_reason = request.reason or "manual admin update" claim.last_seen_at = models.utcnow() return {"ok": True, "claim_id": claim.id, "confidence": claim.confidence} @app.post("/entities/merge") def merge_entities(request: MergeEntitiesRequest): with session_scope(database_url) as session: project = KnowledgeRepository(session).get_project(request.project_name) source = session.get(models.Entity, request.source_entity_id) target = session.get(models.Entity, request.target_entity_id) if source is None or target is None or source.project_id != project.id or target.project_id != project.id: return {"ok": False, "error": "entity not found in project"} session.query(models.Claim).filter(models.Claim.subject_entity_id == source.id).update( {models.Claim.subject_entity_id: target.id} ) session.query(models.Claim).filter(models.Claim.object_entity_id == source.id).update( {models.Claim.object_entity_id: target.id} ) session.query(models.Relation).filter(models.Relation.subject_entity_id == source.id).update( {models.Relation.subject_entity_id: target.id} ) session.query(models.Relation).filter(models.Relation.object_entity_id == source.id).update( {models.Relation.object_entity_id: target.id} ) source.metadata_json = {**(source.metadata_json or {}), "merged_into": target.id} source.updated_at = models.utcnow() return {"ok": True, "source_entity_id": source.id, "target_entity_id": target.id} @app.get("/projects/{project_name}/recommendation-tags") def recommendation_tags(project_name: str): with session_scope(database_url) as session: project = KnowledgeRepository(session).get_project(project_name) tag_predicates = { "hasTopNote", "hasMiddleNote", "hasBaseNote", "hasScentNote", "hasFlavorNote", "evokesMood", "suitableForSeason", "suitableForOccasion", "hasReviewKeyword", } rows = session.execute( select(models.Claim, models.Entity) .join(models.Entity, models.Claim.object_entity_id == models.Entity.id) .where(models.Claim.project_id == project.id, models.Claim.predicate.in_(tag_predicates)) ).all() grouped: dict[str, dict[str, Any]] = {} for claim, entity in rows: key = f"{claim.predicate}:{entity.canonical_name}" if key not in grouped: grouped[key] = { "predicate": claim.predicate, "name": entity.name, "type": entity.entity_type, "support_count": 0, "max_confidence": 0.0, } grouped[key]["support_count"] += 1 grouped[key]["max_confidence"] = max(grouped[key]["max_confidence"], claim.confidence) return sorted(grouped.values(), key=lambda item: (item["predicate"], -item["support_count"], item["name"])) @app.post("/recommend") def recommend(request: RecommendRequest): with session_scope(database_url) as session: repo = KnowledgeRepository(session) project = repo.get_project(request.project_name) recommender = RuleBasedRecommender(session) pref = PreferenceInput(**request.preferences) items = recommender.recommend(project.id, request.target_entity_type, pref, request.limit) return [asdict(item) for item in items]