92 lines
2.9 KiB
Python
92 lines
2.9 KiB
Python
"""Phase 3 crawl acquisition job routes."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from typing import Annotated
|
|
|
|
from fastapi import APIRouter, Body, Depends, HTTPException
|
|
from pydantic import BaseModel, Field, model_validator
|
|
from sqlalchemy.orm import Session
|
|
|
|
from ont_platform.api.db_deps import get_db
|
|
from ont_platform.core.crawler import CachePolicy, CrawlProfile, RobotsPolicy
|
|
from ont_platform.core.crawler.jobs import CrawlJobRequest, CrawlJobRunner, job_to_dict
|
|
from ont_platform.storage.models import ExtractionJob
|
|
|
|
router = APIRouter(prefix="/api/v1/crawl", tags=["crawl"])
|
|
|
|
|
|
class CrawlJobStartRequest(BaseModel):
|
|
project_id: str = "default"
|
|
url: str | None = None
|
|
html: str | None = None
|
|
profile: CrawlProfile = CrawlProfile.FAST_STATIC
|
|
max_pages: int = Field(default=50, ge=1, le=50)
|
|
max_depth: int = Field(default=1, ge=0, le=5)
|
|
robots_policy: RobotsPolicy = RobotsPolicy.RESPECT
|
|
cache_policy: CachePolicy = CachePolicy.ENABLED
|
|
|
|
@model_validator(mode="after")
|
|
def _require_input(self) -> CrawlJobStartRequest:
|
|
if not self.url and not self.html:
|
|
raise ValueError("url or html is required")
|
|
return self
|
|
|
|
|
|
@router.post("/jobs")
|
|
async def start_crawl_job(
|
|
request: Annotated[CrawlJobStartRequest, Body()],
|
|
db: Annotated[Session, Depends(get_db)],
|
|
) -> dict:
|
|
runner = CrawlJobRunner(db)
|
|
try:
|
|
job = await runner.run(
|
|
CrawlJobRequest(
|
|
project_id=request.project_id,
|
|
url=request.url,
|
|
html=request.html,
|
|
profile=request.profile,
|
|
max_pages=request.max_pages,
|
|
max_depth=request.max_depth,
|
|
robots_policy=request.robots_policy,
|
|
cache_policy=request.cache_policy,
|
|
)
|
|
)
|
|
except ValueError as exc:
|
|
raise HTTPException(status_code=400, detail=str(exc)) from exc
|
|
except Exception as exc:
|
|
db.commit()
|
|
raise HTTPException(status_code=500, detail=f"Crawl job failed: {exc}") from exc
|
|
|
|
db.commit()
|
|
return {"status": "success", "job": job_to_dict(job)}
|
|
|
|
|
|
@router.get("/jobs/{job_id}")
|
|
def get_crawl_job(
|
|
job_id: str,
|
|
db: Annotated[Session, Depends(get_db)],
|
|
) -> dict:
|
|
job = db.get(ExtractionJob, job_id)
|
|
if job is None or job.job_type != "crawl":
|
|
raise HTTPException(status_code=404, detail=f"Crawl job not found: {job_id}")
|
|
return {"job": job_to_dict(job)}
|
|
|
|
|
|
@router.post("/jobs/{job_id}/cancel")
|
|
def cancel_crawl_job(
|
|
job_id: str,
|
|
db: Annotated[Session, Depends(get_db)],
|
|
) -> dict:
|
|
job = db.get(ExtractionJob, job_id)
|
|
if job is None or job.job_type != "crawl":
|
|
raise HTTPException(status_code=404, detail=f"Crawl job not found: {job_id}")
|
|
if job.status in {"completed", "failed", "canceled"}:
|
|
return {"status": "noop", "job": job_to_dict(job)}
|
|
job.status = "canceled"
|
|
db.commit()
|
|
return {"status": "success", "job": job_to_dict(job)}
|
|
|
|
|
|
__all__ = ["router"]
|