docs

crawler_platform/app/api/routes.py

@@ -1,9 +1,12 @@
 from __future__ import annotations
 
+import csv
+import io
+import json
 from dataclasses import asdict
 from typing import Any
 
-from fastapi import BackgroundTasks, HTTPException
+from fastapi import BackgroundTasks, HTTPException, Response
 from pydantic import BaseModel, Field
 from sqlalchemy import select
 
@@ -253,6 +256,35 @@ class BulkClaimStatusRequest(BaseModel):
     reason: str | None = None
 
 
+class SchemaEntityTypeRequest(BaseModel):
+    name: str
+    domain: str = "generic"
+    description: str | None = None
+    status: str = "active"
+    confidence: float = 1.0
+    metadata: dict[str, Any] = Field(default_factory=dict)
+
+
+class SchemaRelationTypeRequest(BaseModel):
+    name: str
+    domain: str = "generic"
+    description: str | None = None
+    allowed_subject_types: list[str] = Field(default_factory=list)
+    allowed_object_types: list[str] = Field(default_factory=list)
+    allowed_page_types: list[str] = Field(default_factory=list)
+    allowed_source_zones: list[str] = Field(default_factory=list)
+    semantic_constraints: dict[str, Any] = Field(default_factory=dict)
+    confidence_rules: dict[str, Any] = Field(default_factory=dict)
+    min_confidence: float | None = None
+    status: str = "active"
+    confidence: float = 1.0
+    metadata: dict[str, Any] = Field(default_factory=dict)
+
+
+class PipelineRerunRequest(BaseModel):
+    stage: str
+
+
 class MergeEntitiesRequest(BaseModel):
     project_name: str
     source_entity_id: int
@@ -316,6 +348,16 @@ def _apply_claim_review(claim: models.Claim, status: str, reason: str | None) ->
     claim.last_seen_at = models.utcnow()
 
 
+def turtle_id(value: Any) -> str:
+    text = "".join(ch if ch.isalnum() else "_" for ch in str(value or "").strip())
+    text = "_".join(part for part in text.split("_") if part)
+    if not text:
+        return "value"
+    if text[0].isdigit():
+        text = f"n_{text}"
+    return text[:120]
+
+
 def apply_crawl_request_overrides(config, request: CrawlRequest | DiscoverRequest) -> None:
     check_robots_txt = request.respect_robots_txt
     if check_robots_txt is None:
@@ -596,6 +638,61 @@ def register_routes(app, database_url: str) -> None:
             project = KnowledgeRepository(session).get_project(project_name)
             return OntologyRegistry(session).registry_payload(project.id)
 
+    @app.post("/projects/{project_name}/schema/entity-types")
+    def create_schema_entity_type(project_name: str, request: SchemaEntityTypeRequest):
+        with session_scope(database_url) as session:
+            project = KnowledgeRepository(session).get_project(project_name)
+            row = OntologyRegistry(session).upsert_entity_type(
+                project.id,
+                name=request.name,
+                domain=request.domain,
+                description=request.description,
+                status=request.status,
+                confidence=min(max(request.confidence, 0.0), 1.0),
+                metadata={"origin": "manual_schema_designer", **request.metadata},
+            )
+            return {
+                "id": row.id,
+                "name": row.name,
+                "domain": row.domain,
+                "description": row.description,
+                "status": row.status,
+                "confidence": row.confidence,
+            }
+
+    @app.post("/projects/{project_name}/schema/relation-types")
+    def create_schema_relation_type(project_name: str, request: SchemaRelationTypeRequest):
+        confidence_rules = dict(request.confidence_rules or {})
+        if request.min_confidence is not None:
+            confidence_rules["min_confidence"] = min(max(request.min_confidence, 0.0), 1.0)
+        with session_scope(database_url) as session:
+            project = KnowledgeRepository(session).get_project(project_name)
+            row = OntologyRegistry(session).upsert_relation_type(
+                project.id,
+                name=request.name,
+                domain=request.domain,
+                description=request.description,
+                allowed_subject_types=request.allowed_subject_types,
+                allowed_object_types=request.allowed_object_types,
+                allowed_page_types=request.allowed_page_types,
+                allowed_source_zones=request.allowed_source_zones,
+                semantic_constraints=request.semantic_constraints,
+                confidence_rules=confidence_rules,
+                status=request.status,
+                confidence=min(max(request.confidence, 0.0), 1.0),
+                metadata={"origin": "manual_schema_designer", **request.metadata},
+            )
+            return {
+                "id": row.id,
+                "name": row.name,
+                "domain": row.domain,
+                "description": row.description,
+                "allowed_subject_types": row.allowed_subject_types or [],
+                "allowed_object_types": row.allowed_object_types or [],
+                "status": row.status,
+                "confidence": row.confidence,
+            }
+
     @app.get("/projects/{project_name}/ontology/proposals")
     def ontology_proposals(project_name: str, limit: int = 100):
         with session_scope(database_url) as session:
@@ -655,6 +752,127 @@ def register_routes(app, database_url: str) -> None:
                 )
             return results
 
+    @app.get("/projects/{project_name}/export")
+    def export_ontology(
+        project_name: str,
+        format: str = "json",
+        status: str = "validated_claim",
+        include_evidence: bool = True,
+        limit: int = 1000,
+    ):
+        fmt = format.strip().lower()
+        bounded_limit = max(min(limit, 5000), 1)
+        with session_scope(database_url) as session:
+            project = KnowledgeRepository(session).get_project(project_name)
+            query = (
+                select(models.Claim, models.Source, models.Page, models.Entity)
+                .join(models.Source, models.Claim.source_id == models.Source.id)
+                .join(models.Page, models.Claim.page_id == models.Page.id, isouter=True)
+                .join(models.Entity, models.Claim.subject_entity_id == models.Entity.id)
+                .where(models.Claim.project_id == project.id)
+            )
+            if status and status != "all":
+                normalized = _normalize_claim_status(status) or status
+                query = query.where(models.Claim.status == normalized)
+            rows = session.execute(
+                query.order_by(models.Claim.confidence.desc(), models.Claim.last_seen_at.desc()).limit(bounded_limit)
+            ).all()
+            items: list[dict[str, Any]] = []
+            for claim, source, page, subject in rows:
+                object_entity = session.get(models.Entity, claim.object_entity_id) if claim.object_entity_id else None
+                evidence = None
+                if include_evidence:
+                    evidence = session.scalar(
+                        select(models.Evidence)
+                        .where(models.Evidence.claim_id == claim.id)
+                        .order_by(models.Evidence.created_at.desc())
+                    )
+                items.append(
+                    {
+                        "claim_id": claim.id,
+                        "subject": subject.name,
+                        "subject_type": subject.entity_type,
+                        "predicate": claim.predicate,
+                        "object": object_entity.name if object_entity else claim.object_value,
+                        "object_type": object_entity.entity_type if object_entity else claim.value_type,
+                        "status": claim.status,
+                        "confidence": claim.confidence,
+                        "source": source.name,
+                        "source_url": page.url if page else None,
+                        "evidence_text": evidence.evidence_text if evidence else None,
+                        "created_by": claim.extraction_method,
+                        "last_seen_at": claim.last_seen_at.isoformat() if claim.last_seen_at else None,
+                    }
+                )
+
+        filename = f"{project_name}_ontology.{fmt if fmt != 'turtle' else 'ttl'}"
+        if fmt == "json":
+            return {
+                "project": project_name,
+                "status": status,
+                "count": len(items),
+                "claims": items,
+            }
+        if fmt == "csv":
+            output = io.StringIO()
+            fieldnames = [
+                "claim_id",
+                "subject",
+                "subject_type",
+                "predicate",
+                "object",
+                "object_type",
+                "status",
+                "confidence",
+                "source",
+                "source_url",
+                "evidence_text",
+                "created_by",
+                "last_seen_at",
+            ]
+            writer = csv.DictWriter(output, fieldnames=fieldnames)
+            writer.writeheader()
+            for item in items:
+                row = {key: json.dumps(value, ensure_ascii=False) if isinstance(value, (dict, list)) else value for key, value in item.items()}
+                writer.writerow(row)
+            return Response(
+                content=output.getvalue(),
+                media_type="text/csv; charset=utf-8",
+                headers={"Content-Disposition": f'attachment; filename="{filename}"'},
+            )
+        if fmt in {"ttl", "turtle"}:
+            lines = [
+                "@prefix ont: <https://example.local/ontology/> .",
+                "@prefix claim: <https://example.local/claim/> .",
+                "@prefix xsd: <http://www.w3.org/2001/XMLSchema#> .",
+                "",
+            ]
+            for item in items:
+                sid = turtle_id(item["subject"])
+                predicate = turtle_id(item["predicate"])
+                obj = item["object"]
+                if isinstance(obj, str) and obj.strip():
+                    object_repr = f'ont:{turtle_id(obj)}'
+                else:
+                    object_repr = json.dumps(obj, ensure_ascii=False)
+                lines.extend(
+                    [
+                        f"ont:{sid} ont:{predicate} {object_repr} .",
+                        f"claim:c{item['claim_id']} ont:confidence \"{item['confidence']}\"^^xsd:decimal .",
+                    ]
+                )
+                if item.get("source_url"):
+                    lines.append(f"claim:c{item['claim_id']} ont:sourceUrl {json.dumps(item['source_url'], ensure_ascii=False)} .")
+                if item.get("evidence_text"):
+                    lines.append(f"claim:c{item['claim_id']} ont:evidenceText {json.dumps(item['evidence_text'], ensure_ascii=False)} .")
+                lines.append("")
+            return Response(
+                content="\n".join(lines),
+                media_type="text/turtle; charset=utf-8",
+                headers={"Content-Disposition": f'attachment; filename="{filename}"'},
+            )
+        raise HTTPException(status_code=400, detail=f"unsupported export format: {format}")
+
     @app.get("/projects/{project_name}/knowledge-gaps")
     def knowledge_gaps(project_name: str, limit: int = 100):
         with session_scope(database_url) as session:
@@ -942,10 +1160,30 @@ def register_routes(app, database_url: str) -> None:
             return research_session_payload(job)
 
     @app.get("/projects/{project_name}/graph/neighborhood")
-    def graph_neighborhood(project_name: str, entity_id: int | None = None, limit: int = 120):
+    def graph_neighborhood(
+        project_name: str,
+        entity_id: int | None = None,
+        limit: int = 120,
+        include_candidates: bool = False,
+        status: str | None = None,
+    ):
         with session_scope(database_url) as session:
             project = KnowledgeRepository(session).get_project(project_name)
-            return SemanticGraphQuery(session).neighborhood(project.id, entity_id, max(min(limit, 300), 1))
+            if status and status != "all":
+                normalized = _normalize_claim_status(status) or status
+                statuses = [normalized]
+            elif status == "all":
+                statuses = None
+            elif include_candidates:
+                statuses = ["validated_claim", "active", "candidate_claim", "rule_candidate"]
+            else:
+                statuses = ["validated_claim"]
+            return SemanticGraphQuery(session).neighborhood(
+                project.id,
+                entity_id,
+                max(min(limit, 300), 1),
+                statuses=statuses,
+            )
 
     @app.get("/projects/{project_name}/graph/query")
     def graph_query(
@@ -1182,9 +1420,11 @@ def register_routes(app, database_url: str) -> None:
             results: list[dict[str, Any]] = []
             for claim, source, page, subject in rows:
                 object_name = None
+                object_type = None
                 if claim.object_entity_id:
                     object_entity = session.get(models.Entity, claim.object_entity_id)
                     object_name = object_entity.name if object_entity else None
+                    object_type = object_entity.entity_type if object_entity else None
                 evidence = session.scalar(
                     select(models.Evidence)
                     .where(models.Evidence.claim_id == claim.id)
@@ -1197,12 +1437,14 @@ def register_routes(app, database_url: str) -> None:
                         "subject_type": subject.entity_type,
                         "predicate": claim.predicate,
                         "object": object_name,
+                        "object_type": object_type,
                         "object_value": claim.object_value,
                         "source": source.name,
                         "page_url": page.url if page else None,
                         "confidence": claim.confidence,
                         "confidence_reason": claim.confidence_reason,
                         "status": claim.status,
+                        "extraction_method": claim.extraction_method,
                         "evidence_text": evidence.evidence_text if evidence else None,
                         "evidence_summary": evidence.evidence_summary if evidence else None,
                         "page_type": (claim.metadata_json or {}).get("page_type")
@@ -1313,6 +1555,126 @@ def register_routes(app, database_url: str) -> None:
                 ],
             }
 
+    @app.post("/projects/{project_name}/pipeline/rerun")
+    def rerun_pipeline_stage(
+        project_name: str,
+        request: PipelineRerunRequest,
+        background_tasks: BackgroundTasks,
+    ):
+        crawl_stages = {
+            "source-crawl",
+            "page-clean",
+            "page-classification",
+            "entity-extraction",
+            "claim-generation",
+            "deduplication",
+            "validation",
+        }
+        route_by_stage = {
+            "human-review": f"/review/{project_name}",
+            "ontology-commit": f"/graph/{project_name}",
+            "export": f"/export/{project_name}",
+        }
+        if request.stage not in crawl_stages:
+            return {
+                "ok": True,
+                "stage": request.stage,
+                "action": "navigate",
+                "route": route_by_stage.get(request.stage, f"/pipeline/{project_name}"),
+                "message": "This stage is controlled from its workspace screen.",
+            }
+
+        with session_scope(database_url) as session:
+            repo = KnowledgeRepository(session)
+            project = repo.get_project(project_name)
+            config = project_config_from_project_row(session, project)
+            latest_job = session.scalar(
+                select(models.CrawlJob)
+                .where(models.CrawlJob.project_id == project.id)
+                .order_by(models.CrawlJob.scheduled_at.desc())
+            )
+            latest_request = dict((latest_job.metadata_json or {}).get("request") or {}) if latest_job else {}
+            source_name = latest_request.get("source_name")
+            source = None
+            if source_name:
+                source = session.scalar(
+                    select(models.Source).where(
+                        models.Source.project_id == project.id,
+                        models.Source.name == source_name,
+                    )
+                )
+            if source is None:
+                source = session.scalar(
+                    select(models.Source)
+                    .where(models.Source.project_id == project.id)
+                    .order_by(models.Source.updated_at.desc())
+                )
+            if source is None:
+                raise HTTPException(status_code=400, detail="No source is registered for this project.")
+            url = latest_request.get("url") or source.base_url
+            if not url:
+                raise HTTPException(status_code=400, detail="No previous crawl URL or source base_url is available.")
+
+            inner_request = SiteCrawlRequest(
+                config_path="",
+                source_name=source.name,
+                url=url,
+                extractor_provider=str(latest_request.get("extractor_provider") or "lm_studio"),
+                extractor_model=latest_request.get("extractor_model"),
+                extractor_base_url=latest_request.get("extractor_base_url") or "http://localhost:1234/v1",
+                check_robots_txt=bool(latest_request.get("check_robots_txt") or False),
+                respect_robots_txt=latest_request.get("respect_robots_txt"),
+                max_depth=int(latest_request.get("max_depth") or 2),
+                max_pages=int(latest_request.get("max_pages") or 50),
+                same_domain_only=bool(latest_request.get("same_domain_only", True)),
+                analyze_page_types=list(
+                    latest_request.get("analyze_page_types")
+                    or ["ProductPage", "BrandStoryPage", "ReviewPage"]
+                ),
+            )
+            apply_crawl_request_overrides(config, inner_request)
+            job = models.CrawlJob(
+                project_id=project.id,
+                source_id=source.id,
+                url=url,
+                status="pending",
+                metadata_json={
+                    "kind": "site_crawl",
+                    "request": inner_request.model_dump(),
+                    "project_name": project_name,
+                    "rerun_stage": request.stage,
+                    "progress": {
+                        "seed_url": url,
+                        "visited_count": 0,
+                        "analyzed_count": 0,
+                        "queued_count": 1,
+                        "skipped_count": 0,
+                        "errors": [],
+                        "pages": [],
+                    },
+                },
+            )
+            session.add(job)
+            session.flush()
+            response = crawl_job_response(job)
+
+        task_payload = {
+            **inner_request.model_dump(),
+            "__config_dict": project_config_to_dict(config),
+        }
+        background_tasks.add_task(
+            run_site_crawl_job,
+            database_url,
+            response["job_id"],
+            task_payload,
+        )
+        return {
+            **response,
+            "ok": True,
+            "stage": request.stage,
+            "action": "job_started",
+        }
+
     @app.get("/projects/{project_name}/search")
     def project_search(project_name: str, q: str = "", limit: int = 10):
         q = (q or "").strip()