1024 lines
45 KiB
Python
1024 lines
45 KiB
Python
from __future__ import annotations
|
|
|
|
from dataclasses import asdict
|
|
from typing import Any
|
|
|
|
from fastapi import BackgroundTasks, HTTPException
|
|
from pydantic import BaseModel, Field
|
|
from sqlalchemy import select
|
|
|
|
from crawler_platform.app.config.loader import load_project_config
|
|
from crawler_platform.app.core.crawler.discovery import discover_links
|
|
from crawler_platform.app.core.crawler.fetchers import RobotsPolicy, make_fetcher
|
|
from crawler_platform.app.core.crawler.pipeline import CrawlPipeline
|
|
from crawler_platform.app.core.crawler.site_crawler import SiteCrawler
|
|
from crawler_platform.app.core.database import models
|
|
from crawler_platform.app.core.database.repository import KnowledgeRepository
|
|
from crawler_platform.app.core.database.session import session_scope
|
|
from crawler_platform.app.core.extractor.ai_provider import list_openai_compatible_models
|
|
from crawler_platform.app.core.extractor.factory import extractor_for_domain
|
|
from crawler_platform.app.core.ontology.definitions import ontology_for_domain
|
|
from crawler_platform.app.core.ontology.gap_detector import KnowledgeGapDetector
|
|
from crawler_platform.app.core.ontology.mapper import ontology_to_dict
|
|
from crawler_platform.app.core.ontology.registry import OntologyRegistry
|
|
from crawler_platform.app.core.recommendation.scorer import PreferenceInput, RuleBasedRecommender
|
|
from crawler_platform.app.core.research.graph_query import SemanticGraphQuery
|
|
from crawler_platform.app.core.research.graph_research_loop import GraphResearchLoop
|
|
from crawler_platform.app.core.research.memory_store import ResearchMemoryStore, research_session_payload
|
|
|
|
|
|
class CrawlRequest(BaseModel):
|
|
config_path: str
|
|
source_name: str
|
|
url: str
|
|
extractor_provider: str = "lm_studio"
|
|
extractor_model: str | None = None
|
|
extractor_base_url: str | None = "http://localhost:1234/v1"
|
|
|
|
|
|
class SiteCrawlRequest(CrawlRequest):
|
|
max_depth: int = 2
|
|
max_pages: int = 50
|
|
same_domain_only: bool = True
|
|
analyze_page_types: list[str] = Field(default_factory=lambda: ["ProductPage", "BrandStoryPage", "ReviewPage"])
|
|
|
|
|
|
class DiscoverRequest(BaseModel):
|
|
config_path: str
|
|
source_name: str
|
|
url: str
|
|
limit: int = 30
|
|
|
|
|
|
class RecommendRequest(BaseModel):
|
|
project_name: str
|
|
target_entity_type: str = "Perfume"
|
|
preferences: dict = Field(default_factory=dict)
|
|
limit: int = 10
|
|
|
|
|
|
class CreateProjectRequest(BaseModel):
|
|
config_path: str
|
|
|
|
|
|
class ResetProjectRequest(BaseModel):
|
|
config_path: str
|
|
project_name: str | None = None
|
|
|
|
|
|
class UpdateClaimConfidenceRequest(BaseModel):
|
|
confidence: float
|
|
reason: str | None = None
|
|
|
|
|
|
class UpdateClaimStatusRequest(BaseModel):
|
|
status: str
|
|
reason: str | None = None
|
|
|
|
|
|
class BulkClaimStatusRequest(BaseModel):
|
|
claim_ids: list[int]
|
|
status: str
|
|
reason: str | None = None
|
|
|
|
|
|
class MergeEntitiesRequest(BaseModel):
|
|
project_name: str
|
|
source_entity_id: int
|
|
target_entity_id: int
|
|
|
|
|
|
class ExtractorModelsRequest(BaseModel):
|
|
provider: str
|
|
base_url: str | None = None
|
|
|
|
|
|
class ResearchRunRequest(CrawlRequest):
|
|
project_name: str | None = None
|
|
seed_entity_id: int | None = None
|
|
goal: str = "Semantic ontology exploration"
|
|
max_depth: int = 2
|
|
max_steps: int = 12
|
|
max_branch: int = 8
|
|
min_relevance: float = 0.35
|
|
same_domain_only: bool = True
|
|
analyze_page_types: list[str] = Field(default_factory=lambda: ["ProductPage", "BrandStoryPage", "ReviewPage"])
|
|
|
|
|
|
def site_crawl_progress_payload(result, latest_page=None) -> dict[str, Any]:
|
|
payload = asdict(result)
|
|
if latest_page is not None:
|
|
payload["latest_page"] = asdict(latest_page)
|
|
return payload
|
|
|
|
|
|
_CLAIM_STATUS_ALIASES = {
|
|
"accept": "validated_claim",
|
|
"accepted": "validated_claim",
|
|
"validated": "validated_claim",
|
|
"validated_claim": "validated_claim",
|
|
"reject": "rejected",
|
|
"rejected": "rejected",
|
|
"active": "active",
|
|
"candidate": "active",
|
|
"pending": "active",
|
|
}
|
|
|
|
|
|
def _normalize_claim_status(value: str | None) -> str | None:
|
|
if not value:
|
|
return None
|
|
return _CLAIM_STATUS_ALIASES.get(value.strip().lower())
|
|
|
|
|
|
def _apply_claim_review(claim: models.Claim, status: str, reason: str | None) -> None:
|
|
claim.status = status
|
|
if reason:
|
|
claim.confidence_reason = reason
|
|
metadata = dict(claim.metadata_json or {})
|
|
review = dict(metadata.get("review") or {})
|
|
review["status"] = status
|
|
review["reason"] = reason or review.get("reason")
|
|
review["reviewed_at"] = models.utcnow().isoformat()
|
|
metadata["review"] = review
|
|
claim.metadata_json = metadata
|
|
claim.last_seen_at = models.utcnow()
|
|
|
|
|
|
def crawl_job_response(job: models.CrawlJob) -> dict[str, Any]:
|
|
metadata = job.metadata_json or {}
|
|
return {
|
|
"job_id": job.id,
|
|
"status": job.status,
|
|
"url": job.url,
|
|
"error": job.error,
|
|
"scheduled_at": job.scheduled_at.isoformat() if job.scheduled_at else None,
|
|
"started_at": job.started_at.isoformat() if job.started_at else None,
|
|
"finished_at": job.finished_at.isoformat() if job.finished_at else None,
|
|
"progress": metadata.get("progress", {}),
|
|
"request": metadata.get("request", {}),
|
|
}
|
|
|
|
|
|
def update_site_crawl_job_metadata(job: models.CrawlJob, **updates: Any) -> None:
|
|
metadata = dict(job.metadata_json or {})
|
|
metadata.update(updates)
|
|
job.metadata_json = metadata
|
|
|
|
|
|
def is_site_crawl_cancel_requested(session, job_id: int) -> bool:
|
|
session.expire_all()
|
|
job = session.get(models.CrawlJob, job_id)
|
|
return job is None or job.status == "cancel_requested"
|
|
|
|
|
|
def run_site_crawl_job(database_url: str, job_id: int, request_data: dict[str, Any]) -> None:
|
|
request = SiteCrawlRequest(**request_data)
|
|
try:
|
|
config = load_project_config(request.config_path)
|
|
with session_scope(database_url) as session:
|
|
job = session.get(models.CrawlJob, job_id)
|
|
if job is None:
|
|
return
|
|
if job.status == "cancel_requested":
|
|
job.status = "canceled"
|
|
job.finished_at = models.utcnow()
|
|
return
|
|
|
|
job.status = "running"
|
|
job.started_at = models.utcnow()
|
|
update_site_crawl_job_metadata(
|
|
job,
|
|
progress={
|
|
"seed_url": request.url,
|
|
"visited_count": 0,
|
|
"analyzed_count": 0,
|
|
"queued_count": 1,
|
|
"skipped_count": 0,
|
|
"errors": [],
|
|
"pages": [],
|
|
},
|
|
)
|
|
session.commit()
|
|
|
|
repo = KnowledgeRepository(session)
|
|
crawler = SiteCrawler(
|
|
repo,
|
|
extractor_for_domain(
|
|
config.domain,
|
|
provider=request.extractor_provider,
|
|
model=request.extractor_model,
|
|
base_url=request.extractor_base_url,
|
|
),
|
|
)
|
|
|
|
def save_progress(result, latest_page) -> None:
|
|
current_job = session.get(models.CrawlJob, job_id)
|
|
if current_job is None:
|
|
return
|
|
update_site_crawl_job_metadata(
|
|
current_job,
|
|
progress=site_crawl_progress_payload(result, latest_page),
|
|
)
|
|
session.commit()
|
|
|
|
def should_stop() -> bool:
|
|
return is_site_crawl_cancel_requested(session, job_id)
|
|
|
|
result = crawler.crawl_site(
|
|
config,
|
|
request.source_name,
|
|
request.url,
|
|
max_depth=max(request.max_depth, 0),
|
|
max_pages=max(min(request.max_pages, 500), 1),
|
|
same_domain_only=request.same_domain_only,
|
|
analyze_page_types=set(request.analyze_page_types),
|
|
progress_callback=save_progress,
|
|
should_stop=should_stop,
|
|
parent_job_id=job_id,
|
|
)
|
|
|
|
finished_job = session.get(models.CrawlJob, job_id)
|
|
if finished_job is not None:
|
|
finished_job.status = "canceled" if finished_job.status == "cancel_requested" else "completed"
|
|
finished_job.finished_at = models.utcnow()
|
|
update_site_crawl_job_metadata(
|
|
finished_job,
|
|
progress=site_crawl_progress_payload(result),
|
|
)
|
|
except Exception as exc:
|
|
with session_scope(database_url) as session:
|
|
job = session.get(models.CrawlJob, job_id)
|
|
if job is not None:
|
|
job.status = "failed"
|
|
job.error = str(exc)
|
|
job.finished_at = models.utcnow()
|
|
metadata = dict(job.metadata_json or {})
|
|
progress = dict(metadata.get("progress") or {})
|
|
progress["errors"] = [*progress.get("errors", []), str(exc)]
|
|
update_site_crawl_job_metadata(job, progress=progress)
|
|
|
|
|
|
def register_routes(app, database_url: str) -> None:
|
|
@app.get("/health")
|
|
def health():
|
|
return {"ok": True}
|
|
|
|
@app.get("/projects")
|
|
def projects():
|
|
with session_scope(database_url) as session:
|
|
rows = session.scalars(select(models.Project).order_by(models.Project.created_at.desc())).all()
|
|
return [
|
|
{
|
|
"id": project.id,
|
|
"name": project.name,
|
|
"domain": project.domain,
|
|
"created_at": project.created_at.isoformat(),
|
|
"updated_at": project.updated_at.isoformat(),
|
|
}
|
|
for project in rows
|
|
]
|
|
|
|
@app.post("/projects")
|
|
def create_project(request: CreateProjectRequest):
|
|
config = load_project_config(request.config_path)
|
|
with session_scope(database_url) as session:
|
|
project = KnowledgeRepository(session).upsert_project(config)
|
|
return {"id": project.id, "name": project.name, "domain": project.domain}
|
|
|
|
@app.post("/projects/reset")
|
|
def reset_project(request: ResetProjectRequest):
|
|
config = load_project_config(request.config_path)
|
|
if request.project_name and request.project_name != config.project_name:
|
|
raise HTTPException(
|
|
status_code=400,
|
|
detail=(
|
|
f"Selected project '{request.project_name}' does not match "
|
|
f"config project '{config.project_name}'."
|
|
),
|
|
)
|
|
|
|
with session_scope(database_url) as session:
|
|
repo = KnowledgeRepository(session)
|
|
project = session.scalar(select(models.Project).where(models.Project.name == config.project_name))
|
|
if project is None:
|
|
project = repo.upsert_project(config)
|
|
return {
|
|
"ok": True,
|
|
"name": project.name,
|
|
"domain": project.domain,
|
|
"created": True,
|
|
"reset": False,
|
|
"deleted": {},
|
|
}
|
|
|
|
deleted = repo.reset_project_runtime_data(project.id)
|
|
project = repo.upsert_project(config)
|
|
return {
|
|
"ok": True,
|
|
"name": project.name,
|
|
"domain": project.domain,
|
|
"created": False,
|
|
"reset": True,
|
|
"deleted": deleted,
|
|
}
|
|
|
|
@app.get("/projects/{project_name}")
|
|
def project_detail(project_name: str):
|
|
with session_scope(database_url) as session:
|
|
project = KnowledgeRepository(session).get_project(project_name)
|
|
sources = session.scalars(select(models.Source).where(models.Source.project_id == project.id)).all()
|
|
return {
|
|
"id": project.id,
|
|
"name": project.name,
|
|
"domain": project.domain,
|
|
"config": project.config,
|
|
"sources": [
|
|
{
|
|
"id": source.id,
|
|
"name": source.name,
|
|
"type": source.type,
|
|
"trust_level": source.trust_level,
|
|
"respect_robots_txt": source.respect_robots_txt,
|
|
"rate_limit_per_minute": source.rate_limit_per_minute,
|
|
}
|
|
for source in sources
|
|
],
|
|
}
|
|
|
|
@app.get("/ontology/{domain}")
|
|
def ontology(domain: str):
|
|
return ontology_to_dict(ontology_for_domain(domain))
|
|
|
|
@app.get("/projects/{project_name}/ontology/registry")
|
|
def ontology_registry(project_name: str):
|
|
with session_scope(database_url) as session:
|
|
project = KnowledgeRepository(session).get_project(project_name)
|
|
return OntologyRegistry(session).registry_payload(project.id)
|
|
|
|
@app.get("/projects/{project_name}/ontology/proposals")
|
|
def ontology_proposals(project_name: str, limit: int = 100):
|
|
with session_scope(database_url) as session:
|
|
project = KnowledgeRepository(session).get_project(project_name)
|
|
rows = session.scalars(
|
|
select(models.OntologyProposal)
|
|
.where(models.OntologyProposal.project_id == project.id)
|
|
.order_by(models.OntologyProposal.updated_at.desc())
|
|
.limit(max(min(limit, 300), 1))
|
|
).all()
|
|
return [
|
|
{
|
|
"id": row.id,
|
|
"proposal_type": row.proposal_type,
|
|
"name": row.name,
|
|
"reason": row.reason,
|
|
"evidence": row.evidence,
|
|
"status": row.status,
|
|
"confidence": row.confidence,
|
|
"metadata": row.metadata_json or {},
|
|
"updated_at": row.updated_at.isoformat() if row.updated_at else None,
|
|
}
|
|
for row in rows
|
|
]
|
|
|
|
@app.get("/projects/{project_name}/ontology/triples")
|
|
def ontology_triples(project_name: str, status: str | None = None, limit: int = 100):
|
|
with session_scope(database_url) as session:
|
|
project = KnowledgeRepository(session).get_project(project_name)
|
|
query = select(models.OntologyTriple).where(models.OntologyTriple.project_id == project.id)
|
|
if status:
|
|
query = query.where(models.OntologyTriple.status == status)
|
|
rows = session.scalars(
|
|
query.order_by(models.OntologyTriple.last_seen_at.desc()).limit(max(min(limit, 300), 1))
|
|
).all()
|
|
results = []
|
|
for row in rows:
|
|
subject = session.get(models.Entity, row.subject_entity_id)
|
|
object_entity = session.get(models.Entity, row.object_entity_id) if row.object_entity_id else None
|
|
results.append(
|
|
{
|
|
"id": row.id,
|
|
"claim_id": row.claim_id,
|
|
"subject": subject.name if subject else None,
|
|
"subject_type": row.subject_type,
|
|
"predicate": row.predicate,
|
|
"object": object_entity.name if object_entity else None,
|
|
"object_type": row.object_type,
|
|
"object_value": row.object_value,
|
|
"value_type": row.value_type,
|
|
"status": row.status,
|
|
"confidence": row.confidence,
|
|
"support_count": row.support_count,
|
|
"metadata": row.metadata_json or {},
|
|
"last_seen_at": row.last_seen_at.isoformat() if row.last_seen_at else None,
|
|
}
|
|
)
|
|
return results
|
|
|
|
@app.get("/projects/{project_name}/knowledge-gaps")
|
|
def knowledge_gaps(project_name: str, limit: int = 100):
|
|
with session_scope(database_url) as session:
|
|
project = KnowledgeRepository(session).get_project(project_name)
|
|
return KnowledgeGapDetector(session).list_open(project.id, max(min(limit, 300), 1))
|
|
|
|
@app.post("/extractors/models")
|
|
def extractor_models(request: ExtractorModelsRequest):
|
|
try:
|
|
if request.provider == "lm_studio":
|
|
models = list_openai_compatible_models(request.base_url or "http://localhost:1234/v1")
|
|
return {"ok": True, "models": models}
|
|
if request.provider == "openai":
|
|
import os
|
|
|
|
models = list_openai_compatible_models(
|
|
request.base_url or "https://api.openai.com/v1",
|
|
os.getenv("OPENAI_API_KEY"),
|
|
)
|
|
return {"ok": True, "models": models}
|
|
if request.provider == "ollama":
|
|
return {"ok": False, "error": "Ollama model listing is not implemented yet. Enter the model manually."}
|
|
return {"ok": True, "models": [{"id": "rule_based", "owned_by": "local"}]}
|
|
except Exception as exc:
|
|
return {"ok": False, "error": str(exc), "models": []}
|
|
|
|
@app.post("/crawl")
|
|
def crawl(request: CrawlRequest):
|
|
config = load_project_config(request.config_path)
|
|
with session_scope(database_url) as session:
|
|
repo = KnowledgeRepository(session)
|
|
pipeline = CrawlPipeline(
|
|
repo,
|
|
extractor_for_domain(
|
|
config.domain,
|
|
provider=request.extractor_provider,
|
|
model=request.extractor_model,
|
|
base_url=request.extractor_base_url,
|
|
),
|
|
)
|
|
try:
|
|
result = pipeline.crawl_url(config, request.source_name, request.url)
|
|
except Exception as exc:
|
|
raise HTTPException(status_code=400, detail=str(exc)) from exc
|
|
return {
|
|
"page_id": result.page_id,
|
|
"claim_count": result.claim_count,
|
|
"entity_count": result.entity_count,
|
|
"crawl_status": result.crawl_status,
|
|
"extraction_status": result.extraction_status,
|
|
"page_type": result.page_type,
|
|
"raw_text_length": result.raw_text_length,
|
|
"clean_text_length": result.clean_text_length,
|
|
"warnings": result.warnings or [],
|
|
}
|
|
|
|
@app.post("/crawl-site")
|
|
def crawl_site(request: SiteCrawlRequest, background_tasks: BackgroundTasks):
|
|
config = load_project_config(request.config_path)
|
|
with session_scope(database_url) as session:
|
|
repo = KnowledgeRepository(session)
|
|
project = repo.upsert_project(config)
|
|
source = repo.get_source(project.id, request.source_name)
|
|
job = models.CrawlJob(
|
|
project_id=project.id,
|
|
source_id=source.id,
|
|
url=request.url,
|
|
status="pending",
|
|
metadata_json={
|
|
"kind": "site_crawl",
|
|
"request": request.model_dump(),
|
|
"progress": {
|
|
"seed_url": request.url,
|
|
"visited_count": 0,
|
|
"analyzed_count": 0,
|
|
"queued_count": 1,
|
|
"skipped_count": 0,
|
|
"errors": [],
|
|
"pages": [],
|
|
},
|
|
},
|
|
)
|
|
session.add(job)
|
|
session.flush()
|
|
response = crawl_job_response(job)
|
|
background_tasks.add_task(run_site_crawl_job, database_url, response["job_id"], request.model_dump())
|
|
return response
|
|
|
|
@app.get("/crawl-site/jobs/{job_id}")
|
|
def crawl_site_job(job_id: int):
|
|
with session_scope(database_url) as session:
|
|
job = session.get(models.CrawlJob, job_id)
|
|
if job is None or (job.metadata_json or {}).get("kind") != "site_crawl":
|
|
raise HTTPException(status_code=404, detail="site crawl job not found")
|
|
return crawl_job_response(job)
|
|
|
|
@app.post("/crawl-site/jobs/{job_id}/cancel")
|
|
def cancel_crawl_site_job(job_id: int):
|
|
with session_scope(database_url) as session:
|
|
job = session.get(models.CrawlJob, job_id)
|
|
if job is None or (job.metadata_json or {}).get("kind") != "site_crawl":
|
|
raise HTTPException(status_code=404, detail="site crawl job not found")
|
|
if job.status in {"completed", "failed", "canceled"}:
|
|
return crawl_job_response(job)
|
|
job.status = "cancel_requested"
|
|
job.error = "cancel requested by user"
|
|
return crawl_job_response(job)
|
|
|
|
@app.post("/discover")
|
|
def discover(request: DiscoverRequest):
|
|
config = load_project_config(request.config_path)
|
|
source_config = config.source_by_name(request.source_name)
|
|
robots = RobotsPolicy()
|
|
if not robots.allowed(request.url, source_config.respect_robots_txt):
|
|
return {"ok": False, "error": "robots.txt does not allow discovery for this URL", "links": []}
|
|
fetcher = make_fetcher(source_config.fetcher, source_config.rate_limit_per_minute)
|
|
result = fetcher.fetch(request.url)
|
|
links = discover_links(result.analysis_html, result.final_url or request.url, request.limit)
|
|
return {
|
|
"ok": True,
|
|
"status_code": result.status_code,
|
|
"final_url": result.final_url,
|
|
"crawl_status": result.crawl_status,
|
|
"warnings": result.warnings,
|
|
"links": [asdict(link) for link in links],
|
|
}
|
|
|
|
@app.post("/research/run")
|
|
def run_research(request: ResearchRunRequest):
|
|
config = load_project_config(request.config_path)
|
|
if request.project_name and request.project_name != config.project_name:
|
|
raise HTTPException(
|
|
status_code=400,
|
|
detail=f"Selected project '{request.project_name}' does not match config project '{config.project_name}'.",
|
|
)
|
|
with session_scope(database_url) as session:
|
|
repo = KnowledgeRepository(session)
|
|
loop = GraphResearchLoop(
|
|
repo,
|
|
extractor_for_domain(
|
|
config.domain,
|
|
provider=request.extractor_provider,
|
|
model=request.extractor_model,
|
|
base_url=request.extractor_base_url,
|
|
),
|
|
)
|
|
try:
|
|
result = loop.run(
|
|
project_config=config,
|
|
source_name=request.source_name,
|
|
seed_url=request.url or None,
|
|
seed_entity_id=request.seed_entity_id,
|
|
goal=request.goal,
|
|
max_depth=max(request.max_depth, 0),
|
|
max_steps=max(min(request.max_steps, 50), 1),
|
|
max_branch=max(min(request.max_branch, 30), 1),
|
|
min_relevance=min(max(request.min_relevance, 0.0), 1.0),
|
|
same_domain_only=request.same_domain_only,
|
|
analyze_page_types=set(request.analyze_page_types),
|
|
)
|
|
except Exception as exc:
|
|
raise HTTPException(status_code=400, detail=str(exc)) from exc
|
|
return asdict(result)
|
|
|
|
@app.get("/projects/{project_name}/research/sessions")
|
|
def research_sessions(project_name: str, limit: int = 25):
|
|
with session_scope(database_url) as session:
|
|
project = KnowledgeRepository(session).get_project(project_name)
|
|
return ResearchMemoryStore(session).list_sessions(project.id, limit)
|
|
|
|
@app.get("/research/sessions/{job_id}")
|
|
def research_session(job_id: int):
|
|
with session_scope(database_url) as session:
|
|
job = session.get(models.CrawlJob, job_id)
|
|
if job is None or (job.metadata_json or {}).get("kind") != "research_session":
|
|
raise HTTPException(status_code=404, detail="research session not found")
|
|
return research_session_payload(job)
|
|
|
|
@app.get("/projects/{project_name}/graph/neighborhood")
|
|
def graph_neighborhood(project_name: str, entity_id: int | None = None, limit: int = 120):
|
|
with session_scope(database_url) as session:
|
|
project = KnowledgeRepository(session).get_project(project_name)
|
|
return SemanticGraphQuery(session).neighborhood(project.id, entity_id, max(min(limit, 300), 1))
|
|
|
|
@app.get("/projects/{project_name}/graph/query")
|
|
def graph_query(
|
|
project_name: str,
|
|
kind: str = "trend_summary",
|
|
predicate: str = "hasAccord",
|
|
tag: str | None = None,
|
|
brand: str | None = None,
|
|
limit: int = 100,
|
|
):
|
|
with session_scope(database_url) as session:
|
|
project = KnowledgeRepository(session).get_project(project_name)
|
|
graph = SemanticGraphQuery(session)
|
|
bounded_limit = max(min(limit, 300), 1)
|
|
if kind == "brand_products":
|
|
return graph.brand_products(project.id, brand, bounded_limit)
|
|
if kind == "products_by_tag":
|
|
return graph.products_by_tag(project.id, predicate, tag, bounded_limit)
|
|
if kind == "relation_summary":
|
|
return graph.relation_summary(project.id, bounded_limit)
|
|
if kind == "entity_type_summary":
|
|
return graph.entity_type_summary(project.id, bounded_limit)
|
|
if kind == "trend_summary":
|
|
return graph.trend_summary(project.id, bounded_limit)
|
|
raise HTTPException(status_code=400, detail=f"unknown graph query kind: {kind}")
|
|
|
|
@app.get("/projects/{project_name}/entities")
|
|
def project_entities(project_name: str, entity_type: str | None = None, limit: int = 50):
|
|
with session_scope(database_url) as session:
|
|
project = KnowledgeRepository(session).get_project(project_name)
|
|
query = select(models.Entity).where(models.Entity.project_id == project.id)
|
|
if entity_type:
|
|
query = query.where(models.Entity.entity_type == entity_type)
|
|
entities = session.scalars(query.limit(limit)).all()
|
|
return [
|
|
{
|
|
"id": entity.id,
|
|
"type": entity.entity_type,
|
|
"name": entity.name,
|
|
"metadata": entity.metadata_json,
|
|
}
|
|
for entity in entities
|
|
]
|
|
|
|
@app.get("/projects/{project_name}/claims")
|
|
def project_claims(
|
|
project_name: str,
|
|
limit: int = 100,
|
|
include_candidates: bool = False,
|
|
status: str | None = None,
|
|
):
|
|
with session_scope(database_url) as session:
|
|
project = KnowledgeRepository(session).get_project(project_name)
|
|
query = (
|
|
select(models.Claim, models.Source, models.Page, models.Entity)
|
|
.join(models.Source, models.Claim.source_id == models.Source.id)
|
|
.join(models.Page, models.Claim.page_id == models.Page.id, isouter=True)
|
|
.join(models.Entity, models.Claim.subject_entity_id == models.Entity.id)
|
|
.where(models.Claim.project_id == project.id)
|
|
)
|
|
if status:
|
|
normalized = _normalize_claim_status(status)
|
|
if normalized is None:
|
|
raise HTTPException(status_code=400, detail=f"unknown claim status: {status}")
|
|
query = query.where(models.Claim.status == normalized)
|
|
elif not include_candidates:
|
|
query = query.where(models.Claim.status == "validated_claim")
|
|
rows = session.execute(query.order_by(models.Claim.last_seen_at.desc()).limit(limit)).all()
|
|
results: list[dict[str, Any]] = []
|
|
for claim, source, page, subject in rows:
|
|
object_name = None
|
|
if claim.object_entity_id:
|
|
object_entity = session.get(models.Entity, claim.object_entity_id)
|
|
object_name = object_entity.name if object_entity else None
|
|
evidence = session.scalar(
|
|
select(models.Evidence)
|
|
.where(models.Evidence.claim_id == claim.id)
|
|
.order_by(models.Evidence.created_at.desc())
|
|
)
|
|
results.append(
|
|
{
|
|
"id": claim.id,
|
|
"subject": subject.name,
|
|
"subject_type": subject.entity_type,
|
|
"predicate": claim.predicate,
|
|
"object": object_name,
|
|
"object_value": claim.object_value,
|
|
"source": source.name,
|
|
"page_url": page.url if page else None,
|
|
"confidence": claim.confidence,
|
|
"confidence_reason": claim.confidence_reason,
|
|
"status": claim.status,
|
|
"evidence_text": evidence.evidence_text if evidence else None,
|
|
"evidence_summary": evidence.evidence_summary if evidence else None,
|
|
"page_type": (claim.metadata_json or {}).get("page_type")
|
|
or ((page.metadata_json or {}).get("page_type") if page else None),
|
|
"source_zone": (claim.metadata_json or {}).get("source_zone"),
|
|
"source_selector": (claim.metadata_json or {}).get("source_selector"),
|
|
"evidence_found": (claim.metadata_json or {}).get("evidence_found"),
|
|
"validation_status": (claim.metadata_json or {}).get("validation_status"),
|
|
"graph_merge_status": (claim.metadata_json or {}).get("graph_merge_status"),
|
|
"graph_merge_reason": (claim.metadata_json or {}).get("graph_merge_reason"),
|
|
"confidence_breakdown": (claim.metadata_json or {}).get("confidence_breakdown"),
|
|
"review_required": (claim.metadata_json or {}).get("review_required"),
|
|
"review_reason": (claim.metadata_json or {}).get("review_reason"),
|
|
"conflict_status": (claim.metadata_json or {}).get("conflict_status"),
|
|
"source_history": (claim.metadata_json or {}).get("source_history") or [],
|
|
"last_seen_at": claim.last_seen_at.isoformat(),
|
|
}
|
|
)
|
|
return results
|
|
|
|
@app.get("/projects/{project_name}/pipeline")
|
|
def project_pipeline(project_name: str):
|
|
from sqlalchemy import func as sa_func, distinct
|
|
with session_scope(database_url) as session:
|
|
project = KnowledgeRepository(session).get_project(project_name)
|
|
pid = project.id
|
|
|
|
pages_total = session.scalar(
|
|
select(sa_func.count(models.Page.id)).where(models.Page.project_id == pid)
|
|
) or 0
|
|
ext_total = session.scalar(
|
|
select(sa_func.count(models.ExtractionLog.id)).where(models.ExtractionLog.project_id == pid)
|
|
) or 0
|
|
ext_pages = session.scalar(
|
|
select(sa_func.count(distinct(models.ExtractionLog.page_id)))
|
|
.where(models.ExtractionLog.project_id == pid, models.ExtractionLog.page_id.isnot(None))
|
|
) or 0
|
|
ext_errors = session.scalar(
|
|
select(sa_func.count(models.ExtractionLog.id))
|
|
.where(models.ExtractionLog.project_id == pid, models.ExtractionLog.error.isnot(None))
|
|
) or 0
|
|
|
|
status_rows = session.execute(
|
|
select(models.Claim.status, sa_func.count(models.Claim.id))
|
|
.where(models.Claim.project_id == pid)
|
|
.group_by(models.Claim.status)
|
|
).all()
|
|
claim_statuses = {row[0]: int(row[1]) for row in status_rows}
|
|
claim_total = sum(claim_statuses.values())
|
|
|
|
entity_count = session.scalar(
|
|
select(sa_func.count(models.Entity.id)).where(models.Entity.project_id == pid)
|
|
) or 0
|
|
triple_count = session.scalar(
|
|
select(sa_func.count(models.OntologyTriple.id))
|
|
.where(models.OntologyTriple.project_id == pid, models.OntologyTriple.status == "active")
|
|
) or 0
|
|
|
|
entity_type_rows = session.execute(
|
|
select(models.Entity.entity_type, sa_func.count(models.Entity.id))
|
|
.where(models.Entity.project_id == pid)
|
|
.group_by(models.Entity.entity_type)
|
|
.order_by(sa_func.count(models.Entity.id).desc())
|
|
.limit(10)
|
|
).all()
|
|
|
|
recent_pages = session.scalars(
|
|
select(models.Page).where(models.Page.project_id == pid)
|
|
.order_by(models.Page.fetched_at.desc()).limit(5)
|
|
).all()
|
|
recent_claims_rows = session.execute(
|
|
select(models.Claim, models.Entity)
|
|
.join(models.Entity, models.Claim.subject_entity_id == models.Entity.id)
|
|
.where(models.Claim.project_id == pid)
|
|
.order_by(models.Claim.last_seen_at.desc()).limit(5)
|
|
).all()
|
|
|
|
return {
|
|
"stages": [
|
|
{"key": "crawled", "count": pages_total},
|
|
{"key": "extracted", "count": ext_pages, "extra": {"events": ext_total, "errors": ext_errors}},
|
|
{"key": "claims", "count": claim_total, "extra": claim_statuses},
|
|
{"key": "validated", "count": claim_statuses.get("validated_claim", 0)},
|
|
{"key": "graph", "count": triple_count, "extra": {"entities": entity_count}},
|
|
],
|
|
"entity_types": [{"type": row[0], "count": int(row[1])} for row in entity_type_rows],
|
|
"recent_pages": [
|
|
{
|
|
"id": p.id,
|
|
"url": p.url,
|
|
"title": p.title,
|
|
"status_code": p.status_code,
|
|
"page_type": (p.metadata_json or {}).get("page_type"),
|
|
"fetched_at": p.fetched_at.isoformat() if p.fetched_at else None,
|
|
}
|
|
for p in recent_pages
|
|
],
|
|
"recent_claims": [
|
|
{
|
|
"id": claim.id,
|
|
"subject": subject.name,
|
|
"predicate": claim.predicate,
|
|
"confidence": claim.confidence,
|
|
"status": claim.status,
|
|
"last_seen_at": claim.last_seen_at.isoformat() if claim.last_seen_at else None,
|
|
}
|
|
for claim, subject in recent_claims_rows
|
|
],
|
|
}
|
|
|
|
@app.get("/projects/{project_name}/search")
|
|
def project_search(project_name: str, q: str = "", limit: int = 10):
|
|
q = (q or "").strip()
|
|
if len(q) < 1:
|
|
return {"entities": [], "claims": [], "pages": [], "predicates": []}
|
|
like = f"%{q.lower()}%"
|
|
bounded = max(min(limit, 50), 1)
|
|
with session_scope(database_url) as session:
|
|
project = KnowledgeRepository(session).get_project(project_name)
|
|
pid = project.id
|
|
|
|
from sqlalchemy import func as sa_func, or_
|
|
entity_rows = session.scalars(
|
|
select(models.Entity)
|
|
.where(
|
|
models.Entity.project_id == pid,
|
|
or_(
|
|
sa_func.lower(models.Entity.name).like(like),
|
|
sa_func.lower(models.Entity.canonical_name).like(like),
|
|
),
|
|
)
|
|
.limit(bounded)
|
|
).all()
|
|
|
|
claim_rows = session.execute(
|
|
select(models.Claim, models.Entity)
|
|
.join(models.Entity, models.Claim.subject_entity_id == models.Entity.id)
|
|
.where(
|
|
models.Claim.project_id == pid,
|
|
or_(
|
|
sa_func.lower(models.Entity.name).like(like),
|
|
sa_func.lower(models.Claim.predicate).like(like),
|
|
),
|
|
)
|
|
.order_by(models.Claim.last_seen_at.desc())
|
|
.limit(bounded)
|
|
).all()
|
|
|
|
page_rows = session.scalars(
|
|
select(models.Page)
|
|
.where(
|
|
models.Page.project_id == pid,
|
|
or_(
|
|
sa_func.lower(models.Page.url).like(like),
|
|
sa_func.lower(models.Page.title).like(like),
|
|
),
|
|
)
|
|
.order_by(models.Page.fetched_at.desc())
|
|
.limit(bounded)
|
|
).all()
|
|
|
|
predicate_rows = session.scalars(
|
|
select(models.OntologyRelationType.name)
|
|
.where(
|
|
models.OntologyRelationType.project_id == pid,
|
|
sa_func.lower(models.OntologyRelationType.name).like(like),
|
|
)
|
|
.limit(bounded)
|
|
).all()
|
|
|
|
return {
|
|
"entities": [
|
|
{"id": e.id, "name": e.name, "type": e.entity_type, "canonical_name": e.canonical_name}
|
|
for e in entity_rows
|
|
],
|
|
"claims": [
|
|
{
|
|
"id": claim.id,
|
|
"subject": subject.name,
|
|
"predicate": claim.predicate,
|
|
"object_value": claim.object_value,
|
|
"confidence": claim.confidence,
|
|
"status": claim.status,
|
|
}
|
|
for claim, subject in claim_rows
|
|
],
|
|
"pages": [
|
|
{"id": p.id, "url": p.url, "title": p.title}
|
|
for p in page_rows
|
|
],
|
|
"predicates": list(predicate_rows),
|
|
}
|
|
|
|
@app.get("/projects/{project_name}/extraction-logs")
|
|
def project_extraction_logs(project_name: str, limit: int = 50):
|
|
with session_scope(database_url) as session:
|
|
project = KnowledgeRepository(session).get_project(project_name)
|
|
rows = session.execute(
|
|
select(models.ExtractionLog, models.Page)
|
|
.join(models.Page, models.ExtractionLog.page_id == models.Page.id, isouter=True)
|
|
.where(models.ExtractionLog.project_id == project.id)
|
|
.order_by(models.ExtractionLog.created_at.desc())
|
|
.limit(limit)
|
|
).all()
|
|
return [
|
|
{
|
|
"id": log.id,
|
|
"page_url": page.url if page else None,
|
|
"extractor_name": log.extractor_name,
|
|
"provider": log.provider,
|
|
"error": log.error,
|
|
"created_at": log.created_at.isoformat(),
|
|
"validation": (log.raw_output or {}).get("validation"),
|
|
"page_context": (log.raw_output or {}).get("page_context"),
|
|
"candidate_count": len((log.raw_output or {}).get("candidate_claims") or []),
|
|
"raw_output": log.raw_output,
|
|
}
|
|
for log, page in rows
|
|
]
|
|
|
|
@app.patch("/claims/{claim_id}/confidence")
|
|
def update_claim_confidence(claim_id: int, request: UpdateClaimConfidenceRequest):
|
|
confidence = min(max(request.confidence, 0.0), 1.0)
|
|
with session_scope(database_url) as session:
|
|
claim = session.get(models.Claim, claim_id)
|
|
if claim is None:
|
|
return {"ok": False, "error": "claim not found"}
|
|
claim.confidence = confidence
|
|
claim.confidence_reason = request.reason or "manual admin update"
|
|
claim.last_seen_at = models.utcnow()
|
|
return {"ok": True, "claim_id": claim.id, "confidence": claim.confidence}
|
|
|
|
@app.patch("/claims/{claim_id}/status")
|
|
def update_claim_status(claim_id: int, request: UpdateClaimStatusRequest):
|
|
status = _normalize_claim_status(request.status)
|
|
if status is None:
|
|
raise HTTPException(status_code=400, detail=f"unknown claim status: {request.status}")
|
|
with session_scope(database_url) as session:
|
|
claim = session.get(models.Claim, claim_id)
|
|
if claim is None:
|
|
raise HTTPException(status_code=404, detail="claim not found")
|
|
_apply_claim_review(claim, status, request.reason)
|
|
return {"ok": True, "claim_id": claim.id, "status": claim.status}
|
|
|
|
@app.post("/claims/bulk-status")
|
|
def bulk_update_claim_status(request: BulkClaimStatusRequest):
|
|
status = _normalize_claim_status(request.status)
|
|
if status is None:
|
|
raise HTTPException(status_code=400, detail=f"unknown claim status: {request.status}")
|
|
if not request.claim_ids:
|
|
return {"ok": True, "updated": 0}
|
|
with session_scope(database_url) as session:
|
|
rows = session.execute(
|
|
select(models.Claim).where(models.Claim.id.in_(request.claim_ids))
|
|
).scalars().all()
|
|
for claim in rows:
|
|
_apply_claim_review(claim, status, request.reason)
|
|
return {"ok": True, "updated": len(rows), "status": status}
|
|
|
|
@app.post("/entities/merge")
|
|
def merge_entities(request: MergeEntitiesRequest):
|
|
with session_scope(database_url) as session:
|
|
project = KnowledgeRepository(session).get_project(request.project_name)
|
|
source = session.get(models.Entity, request.source_entity_id)
|
|
target = session.get(models.Entity, request.target_entity_id)
|
|
if source is None or target is None or source.project_id != project.id or target.project_id != project.id:
|
|
return {"ok": False, "error": "entity not found in project"}
|
|
session.query(models.Claim).filter(models.Claim.subject_entity_id == source.id).update(
|
|
{models.Claim.subject_entity_id: target.id}
|
|
)
|
|
session.query(models.Claim).filter(models.Claim.object_entity_id == source.id).update(
|
|
{models.Claim.object_entity_id: target.id}
|
|
)
|
|
session.query(models.Relation).filter(models.Relation.subject_entity_id == source.id).update(
|
|
{models.Relation.subject_entity_id: target.id}
|
|
)
|
|
session.query(models.Relation).filter(models.Relation.object_entity_id == source.id).update(
|
|
{models.Relation.object_entity_id: target.id}
|
|
)
|
|
source.metadata_json = {**(source.metadata_json or {}), "merged_into": target.id}
|
|
source.updated_at = models.utcnow()
|
|
return {"ok": True, "source_entity_id": source.id, "target_entity_id": target.id}
|
|
|
|
@app.get("/projects/{project_name}/recommendation-tags")
|
|
def recommendation_tags(project_name: str):
|
|
with session_scope(database_url) as session:
|
|
project = KnowledgeRepository(session).get_project(project_name)
|
|
tag_predicates = {
|
|
"hasTopNote",
|
|
"hasMiddleNote",
|
|
"hasBaseNote",
|
|
"hasScentNote",
|
|
"hasFlavorNote",
|
|
"evokesMood",
|
|
"suitableForSeason",
|
|
"suitableForOccasion",
|
|
"hasReviewKeyword",
|
|
}
|
|
rows = session.execute(
|
|
select(models.Claim, models.Entity)
|
|
.join(models.Entity, models.Claim.object_entity_id == models.Entity.id)
|
|
.where(
|
|
models.Claim.project_id == project.id,
|
|
models.Claim.status == "validated_claim",
|
|
models.Claim.predicate.in_(tag_predicates),
|
|
)
|
|
).all()
|
|
grouped: dict[str, dict[str, Any]] = {}
|
|
for claim, entity in rows:
|
|
key = f"{claim.predicate}:{entity.canonical_name}"
|
|
if key not in grouped:
|
|
grouped[key] = {
|
|
"predicate": claim.predicate,
|
|
"name": entity.name,
|
|
"type": entity.entity_type,
|
|
"support_count": 0,
|
|
"max_confidence": 0.0,
|
|
}
|
|
grouped[key]["support_count"] += 1
|
|
grouped[key]["max_confidence"] = max(grouped[key]["max_confidence"], claim.confidence)
|
|
return sorted(grouped.values(), key=lambda item: (item["predicate"], -item["support_count"], item["name"]))
|
|
|
|
@app.post("/recommend")
|
|
def recommend(request: RecommendRequest):
|
|
with session_scope(database_url) as session:
|
|
repo = KnowledgeRepository(session)
|
|
project = repo.get_project(request.project_name)
|
|
recommender = RuleBasedRecommender(session)
|
|
pref = PreferenceInput(**request.preferences)
|
|
items = recommender.recommend(project.id, request.target_entity_type, pref, request.limit)
|
|
return [asdict(item) for item in items]
|