ontology

crawler_platform/app/api/routes.py

@@ -34,6 +34,8 @@ class CrawlRequest(BaseModel):
     extractor_provider: str = "lm_studio"
     extractor_model: str | None = None
     extractor_base_url: str | None = "http://localhost:1234/v1"
+    check_robots_txt: bool = False
+    respect_robots_txt: bool | None = None
 
 
 class SiteCrawlRequest(CrawlRequest):
@@ -48,6 +50,8 @@ class DiscoverRequest(BaseModel):
     source_name: str
     url: str
     limit: int = 30
+    check_robots_txt: bool = False
+    respect_robots_txt: bool | None = None
 
 
 class RecommendRequest(BaseModel):
@@ -145,6 +149,13 @@ def _apply_claim_review(claim: models.Claim, status: str, reason: str | None) ->
     claim.last_seen_at = models.utcnow()
 
 
+def apply_crawl_request_overrides(config, request: CrawlRequest | DiscoverRequest) -> None:
+    check_robots_txt = request.respect_robots_txt
+    if check_robots_txt is None:
+        check_robots_txt = request.check_robots_txt
+    config.source_by_name(request.source_name).respect_robots_txt = check_robots_txt
+
+
 def crawl_job_response(job: models.CrawlJob) -> dict[str, Any]:
     metadata = job.metadata_json or {}
     return {
@@ -176,6 +187,7 @@ def run_site_crawl_job(database_url: str, job_id: int, request_data: dict[str, A
     request = SiteCrawlRequest(**request_data)
     try:
         config = load_project_config(request.config_path)
+        apply_crawl_request_overrides(config, request)
         with session_scope(database_url) as session:
             job = session.get(models.CrawlJob, job_id)
             if job is None:
@@ -444,6 +456,7 @@ def register_routes(app, database_url: str) -> None:
     @app.post("/crawl")
     def crawl(request: CrawlRequest):
         config = load_project_config(request.config_path)
+        apply_crawl_request_overrides(config, request)
         with session_scope(database_url) as session:
             repo = KnowledgeRepository(session)
             pipeline = CrawlPipeline(
@@ -474,6 +487,7 @@ def register_routes(app, database_url: str) -> None:
     @app.post("/crawl-site")
     def crawl_site(request: SiteCrawlRequest, background_tasks: BackgroundTasks):
         config = load_project_config(request.config_path)
+        apply_crawl_request_overrides(config, request)
         with session_scope(database_url) as session:
             repo = KnowledgeRepository(session)
             project = repo.upsert_project(config)
@@ -526,10 +540,18 @@ def register_routes(app, database_url: str) -> None:
     @app.post("/discover")
     def discover(request: DiscoverRequest):
         config = load_project_config(request.config_path)
+        apply_crawl_request_overrides(config, request)
         source_config = config.source_by_name(request.source_name)
         robots = RobotsPolicy()
-        if not robots.allowed(request.url, source_config.respect_robots_txt):
-            return {"ok": False, "error": "robots.txt does not allow discovery for this URL", "links": []}
+        robots_decision = robots.check(request.url, source_config.respect_robots_txt)
+        if not robots_decision.allowed:
+            return {
+                "ok": False,
+                "error": f"{robots_decision.reason}: {request.url}",
+                "robots_status": robots_decision.status,
+                "robots_reason": robots_decision.reason,
+                "links": [],
+            }
         fetcher = make_fetcher(source_config.fetcher, source_config.rate_limit_per_minute)
         result = fetcher.fetch(request.url)
         links = discover_links(result.analysis_html, result.final_url or request.url, request.limit)
@@ -538,6 +560,8 @@ def register_routes(app, database_url: str) -> None:
             "status_code": result.status_code,
             "final_url": result.final_url,
             "crawl_status": result.crawl_status,
+            "robots_status": robots_decision.status,
+            "robots_reason": robots_decision.reason,
             "warnings": result.warnings,
             "links": [asdict(link) for link in links],
         }
@@ -545,6 +569,7 @@ def register_routes(app, database_url: str) -> None:
     @app.post("/research/run")
     def run_research(request: ResearchRunRequest):
         config = load_project_config(request.config_path)
+        apply_crawl_request_overrides(config, request)
         if request.project_name and request.project_name != config.project_name:
             raise HTTPException(
                 status_code=400,