docs

ontology_platform/ont_platform/api/routes/crawl.py

@@ -0,0 +1,91 @@
+"""Phase 3 crawl acquisition job routes."""
+
+from __future__ import annotations
+
+from typing import Annotated
+
+from fastapi import APIRouter, Body, Depends, HTTPException
+from pydantic import BaseModel, Field, model_validator
+from sqlalchemy.orm import Session
+
+from ont_platform.api.db_deps import get_db
+from ont_platform.core.crawler import CachePolicy, CrawlProfile, RobotsPolicy
+from ont_platform.core.crawler.jobs import CrawlJobRequest, CrawlJobRunner, job_to_dict
+from ont_platform.storage.models import ExtractionJob
+
+router = APIRouter(prefix="/api/v1/crawl", tags=["crawl"])
+
+
+class CrawlJobStartRequest(BaseModel):
+    project_id: str = "default"
+    url: str | None = None
+    html: str | None = None
+    profile: CrawlProfile = CrawlProfile.FAST_STATIC
+    max_pages: int = Field(default=50, ge=1, le=50)
+    max_depth: int = Field(default=1, ge=0, le=5)
+    robots_policy: RobotsPolicy = RobotsPolicy.RESPECT
+    cache_policy: CachePolicy = CachePolicy.ENABLED
+
+    @model_validator(mode="after")
+    def _require_input(self) -> CrawlJobStartRequest:
+        if not self.url and not self.html:
+            raise ValueError("url or html is required")
+        return self
+
+
+@router.post("/jobs")
+async def start_crawl_job(
+    request: Annotated[CrawlJobStartRequest, Body()],
+    db: Annotated[Session, Depends(get_db)],
+) -> dict:
+    runner = CrawlJobRunner(db)
+    try:
+        job = await runner.run(
+            CrawlJobRequest(
+                project_id=request.project_id,
+                url=request.url,
+                html=request.html,
+                profile=request.profile,
+                max_pages=request.max_pages,
+                max_depth=request.max_depth,
+                robots_policy=request.robots_policy,
+                cache_policy=request.cache_policy,
+            )
+        )
+    except ValueError as exc:
+        raise HTTPException(status_code=400, detail=str(exc)) from exc
+    except Exception as exc:
+        db.commit()
+        raise HTTPException(status_code=500, detail=f"Crawl job failed: {exc}") from exc
+
+    db.commit()
+    return {"status": "success", "job": job_to_dict(job)}
+
+
+@router.get("/jobs/{job_id}")
+def get_crawl_job(
+    job_id: str,
+    db: Annotated[Session, Depends(get_db)],
+) -> dict:
+    job = db.get(ExtractionJob, job_id)
+    if job is None or job.job_type != "crawl":
+        raise HTTPException(status_code=404, detail=f"Crawl job not found: {job_id}")
+    return {"job": job_to_dict(job)}
+
+
+@router.post("/jobs/{job_id}/cancel")
+def cancel_crawl_job(
+    job_id: str,
+    db: Annotated[Session, Depends(get_db)],
+) -> dict:
+    job = db.get(ExtractionJob, job_id)
+    if job is None or job.job_type != "crawl":
+        raise HTTPException(status_code=404, detail=f"Crawl job not found: {job_id}")
+    if job.status in {"completed", "failed", "canceled"}:
+        return {"status": "noop", "job": job_to_dict(job)}
+    job.status = "canceled"
+    db.commit()
+    return {"status": "success", "job": job_to_dict(job)}
+
+
+__all__ = ["router"]