lasta/AI
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

graph

Browse Source
This commit is contained in:
lasta
2026-05-22 00:22:03 +09:00
parent 8d77bc659f
commit d841fb823a
49 changed files with 2732 additions and 3763 deletions
Download Patch File Download Diff File Expand all files Collapse all files

131
ontology_platform/crawler_platform/app/api/routes.py
View File

@@ -4,12 +4,14 @@ import csv
import io
import json
import re
import time
from dataclasses import asdict
from typing import Any
from fastapi import BackgroundTasks, HTTPException, Response
from pydantic import BaseModel, Field
from sqlalchemy import select
from sqlalchemy.exc import OperationalError
from crawler_platform.app.config.loader import (
ProjectConfig,
@@ -41,15 +43,18 @@ from crawler_platform.app.core.research.memory_store import ResearchMemoryStore,
DOMAIN_NAME_PATTERN = re.compile(r"^[a-z0-9][a-z0-9_-]{1,79}$")
SITE_CRAWL_CANCEL_REQUESTS: set[int] = set()
class CrawlRequest(BaseModel):
config_path: str
source_name: str
url: str
extraction_mode: str = "hybrid"
extractor_provider: str = "lm_studio"
extractor_model: str | None = None
extractor_base_url: str | None = "http://localhost:1234/v1"
fallback_to_rules: bool = True
check_robots_txt: bool = False
respect_robots_txt: bool | None = None
@@ -67,9 +72,11 @@ class SiteCrawlByProjectRequest(BaseModel):
project_name: str
source_name: str
url: str
extraction_mode: str = "hybrid"
extractor_provider: str = "lm_studio"
extractor_model: str | None = None
extractor_base_url: str | None = "http://localhost:1234/v1"
fallback_to_rules: bool = True
check_robots_txt: bool = False
respect_robots_txt: bool | None = None
max_depth: int = 2
@@ -85,9 +92,11 @@ class SiteCrawlByProjectRequest(BaseModel):
config_path=config_path_placeholder,
source_name=self.source_name,
url=self.url,
extraction_mode=self.extraction_mode,
extractor_provider=self.extractor_provider,
extractor_model=self.extractor_model,
extractor_base_url=self.extractor_base_url,
fallback_to_rules=self.fallback_to_rules,
check_robots_txt=self.check_robots_txt,
respect_robots_txt=self.respect_robots_txt,
max_depth=self.max_depth,
@@ -535,9 +544,11 @@ class ResearchRunByProjectRequest(BaseModel):
url: str | None = None
seed_entity_id: int | None = None
goal: str = "Semantic ontology exploration"
extraction_mode: str = "hybrid"
extractor_provider: str = "lm_studio"
extractor_model: str | None = None
extractor_base_url: str | None = "http://localhost:1234/v1"
fallback_to_rules: bool = True
check_robots_txt: bool = False
respect_robots_txt: bool | None = None
max_depth: int = 2
@@ -622,6 +633,20 @@ def site_crawl_progress_payload(result, latest_page=None) -> dict[str, Any]:
payload = asdict(result)
if latest_page is not None:
payload["latest_page"] = asdict(latest_page)
pages = payload.get("pages") or []
payload["extraction_summary"] = {
"llm_skipped_count": sum(1 for page in pages if page.get("llm_skipped")),
"fallback_count": sum(1 for page in pages if page.get("fallback_used")),
"conflict_claim_count": sum(int(page.get("conflict_claim_count") or 0) for page in pages),
"agreement_claim_count": sum(int(page.get("agreement_claim_count") or 0) for page in pages),
"llm_call_count": sum(
1
for page in pages
if page.get("extraction_mode") in {"hybrid", "llm_only", "compare"}
and not page.get("llm_skipped")
and not page.get("fallback_used")
),
}
return payload
@@ -696,7 +721,58 @@ def update_site_crawl_job_metadata(job: models.CrawlJob, **updates: Any) -> None
job.metadata_json = metadata
def mark_orphan_site_crawl_jobs_canceled(database_url: str) -> None:
with session_scope(database_url) as session:
rows = session.scalars(
select(models.CrawlJob).where(
models.CrawlJob.status.in_(["running", "cancel_requested"]),
)
).all()
for job in rows:
if (job.metadata_json or {}).get("kind") != "site_crawl":
continue
job.status = "canceled"
job.error = job.error or "canceled after server restart/interrupted crawl"
job.finished_at = models.utcnow()
def request_site_crawl_cancel(database_url: str, job_id: int) -> dict[str, Any]:
SITE_CRAWL_CANCEL_REQUESTS.add(job_id)
last_error: Exception | None = None
for attempt in range(3):
try:
with session_scope(database_url) as session:
job = session.get(models.CrawlJob, job_id)
if job is None or (job.metadata_json or {}).get("kind") != "site_crawl":
raise HTTPException(status_code=404, detail="site crawl job not found")
if job.status in {"completed", "failed", "canceled"}:
SITE_CRAWL_CANCEL_REQUESTS.discard(job_id)
return crawl_job_response(job)
job.status = "cancel_requested"
job.error = "cancel requested by user"
payload = crawl_job_response(job)
return payload
except OperationalError as exc:
last_error = exc
if "database is locked" not in str(exc).lower():
raise
time.sleep(0.25 * (attempt + 1))
return {
"job_id": job_id,
"status": "cancel_requested",
"url": None,
"error": f"cancel requested in memory; database was locked: {last_error}",
"scheduled_at": None,
"started_at": None,
"finished_at": None,
"progress": {},
"request": {},
}
def is_site_crawl_cancel_requested(session, job_id: int) -> bool:
if job_id in SITE_CRAWL_CANCEL_REQUESTS:
return True
session.expire_all()
job = session.get(models.CrawlJob, job_id)
return job is None or job.status == "cancel_requested"
@@ -744,6 +820,8 @@ def run_site_crawl_job(database_url: str, job_id: int, request_data: dict[str, A
provider=request.extractor_provider,
model=request.extractor_model,
base_url=request.extractor_base_url,
extraction_mode=request.extraction_mode,
fallback_to_rules=request.fallback_to_rules,
),
)
@@ -775,12 +853,17 @@ def run_site_crawl_job(database_url: str, job_id: int, request_data: dict[str, A
finished_job = session.get(models.CrawlJob, job_id)
if finished_job is not None:
finished_job.status = "canceled" if finished_job.status == "cancel_requested" else "completed"
finished_job.status = (
"canceled"
if finished_job.status == "cancel_requested" or job_id in SITE_CRAWL_CANCEL_REQUESTS
else "completed"
)
finished_job.finished_at = models.utcnow()
update_site_crawl_job_metadata(
finished_job,
progress=site_crawl_progress_payload(result),
)
SITE_CRAWL_CANCEL_REQUESTS.discard(job_id)
except Exception as exc:
with session_scope(database_url) as session:
job = session.get(models.CrawlJob, job_id)
@@ -792,10 +875,12 @@ def run_site_crawl_job(database_url: str, job_id: int, request_data: dict[str, A
progress = dict(metadata.get("progress") or {})
progress["errors"] = [*progress.get("errors", []), str(exc)]
update_site_crawl_job_metadata(job, progress=progress)
SITE_CRAWL_CANCEL_REQUESTS.discard(job_id)
def register_routes(app, database_url: str) -> None:
recover_interrupted_domain_discovery_jobs(database_url)
mark_orphan_site_crawl_jobs_canceled(database_url)
@app.get("/health")
def health():
@@ -1536,6 +1621,8 @@ def register_routes(app, database_url: str) -> None:
provider=request.extractor_provider,
model=request.extractor_model,
base_url=request.extractor_base_url,
extraction_mode=request.extraction_mode,
fallback_to_rules=request.fallback_to_rules,
),
)
try:
@@ -1546,6 +1633,13 @@ def register_routes(app, database_url: str) -> None:
"page_id": result.page_id,
"claim_count": result.claim_count,
"entity_count": result.entity_count,
"extraction_mode": result.extraction_mode,
"effective_extraction_mode": result.effective_extraction_mode,
"llm_skipped": result.llm_skipped,
"llm_skip_reason": result.llm_skip_reason,
"fallback_used": result.fallback_used,
"agreement_claim_count": result.agreement_claim_count,
"conflict_claim_count": result.conflict_claim_count,
"crawl_status": result.crawl_status,
"extraction_status": result.extraction_status,
"page_type": result.page_type,
@@ -1648,15 +1742,7 @@ def register_routes(app, database_url: str) -> None:
@app.post("/crawl-site/jobs/{job_id}/cancel")
def cancel_crawl_site_job(job_id: int):
with session_scope(database_url) as session:
job = session.get(models.CrawlJob, job_id)
if job is None or (job.metadata_json or {}).get("kind") != "site_crawl":
raise HTTPException(status_code=404, detail="site crawl job not found")
if job.status in {"completed", "failed", "canceled"}:
return crawl_job_response(job)
job.status = "cancel_requested"
job.error = "cancel requested by user"
return crawl_job_response(job)
return request_site_crawl_cancel(database_url, job_id)
@app.post("/discover")
def discover(request: DiscoverRequest):
@@ -1705,6 +1791,8 @@ def register_routes(app, database_url: str) -> None:
provider=request.extractor_provider,
model=request.extractor_model,
base_url=request.extractor_base_url,
extraction_mode=request.extraction_mode,
fallback_to_rules=request.fallback_to_rules,
),
)
try:
@@ -1749,6 +1837,8 @@ def register_routes(app, database_url: str) -> None:
provider=request.extractor_provider,
model=request.extractor_model,
base_url=request.extractor_base_url,
extraction_mode=request.extraction_mode,
fallback_to_rules=request.fallback_to_rules,
),
)
try:
@@ -2080,6 +2170,11 @@ def register_routes(app, database_url: str) -> None:
"graph_merge_status": (claim.metadata_json or {}).get("graph_merge_status"),
"graph_merge_reason": (claim.metadata_json or {}).get("graph_merge_reason"),
"confidence_breakdown": (claim.metadata_json or {}).get("confidence_breakdown"),
"agreement": (claim.metadata_json or {}).get("agreement"),
"extraction_source": (claim.metadata_json or {}).get("extraction_source"),
"claim_kind": (claim.metadata_json or {}).get("claim_kind"),
"rule_confidence": (claim.metadata_json or {}).get("rule_confidence"),
"llm_confidence": (claim.metadata_json or {}).get("llm_confidence"),
"review_required": (claim.metadata_json or {}).get("review_required"),
"review_reason": (claim.metadata_json or {}).get("review_reason"),
"conflict_status": (claim.metadata_json or {}).get("conflict_status"),
@@ -2243,9 +2338,11 @@ def register_routes(app, database_url: str) -> None:
config_path="",
source_name=source.name,
url=url,
extraction_mode=str(latest_request.get("extraction_mode") or "hybrid"),
extractor_provider=str(latest_request.get("extractor_provider") or "lm_studio"),
extractor_model=latest_request.get("extractor_model"),
extractor_base_url=latest_request.get("extractor_base_url") or "http://localhost:1234/v1",
fallback_to_rules=bool(latest_request.get("fallback_to_rules", True)),
check_robots_txt=bool(latest_request.get("check_robots_txt") or False),
respect_robots_txt=latest_request.get("respect_robots_txt"),
max_depth=int(latest_request.get("max_depth") or 2),
@@ -2404,6 +2501,20 @@ def register_routes(app, database_url: str) -> None:
"validation": (log.raw_output or {}).get("validation"),
"page_context": (log.raw_output or {}).get("page_context"),
"candidate_count": len((log.raw_output or {}).get("candidate_claims") or []),
"extraction_mode": (log.raw_output or {}).get("extraction_mode"),
"effective_extraction_mode": (log.raw_output or {}).get("effective_extraction_mode"),
"comparison": (log.raw_output or {}).get("comparison"),
"rule_entity_count": (log.raw_output or {}).get("rule_entity_count"),
"rule_claim_count": (log.raw_output or {}).get("rule_claim_count"),
"llm_entity_count": (log.raw_output or {}).get("llm_entity_count"),
"llm_claim_count": (log.raw_output or {}).get("llm_claim_count"),
"agreement_claim_count": (log.raw_output or {}).get("agreement_claim_count"),
"rule_only_claim_count": (log.raw_output or {}).get("rule_only_claim_count"),
"llm_only_claim_count": (log.raw_output or {}).get("llm_only_claim_count"),
"conflict_claim_count": (log.raw_output or {}).get("conflict_claim_count"),
"llm_skipped": (log.raw_output or {}).get("llm_skipped"),
"llm_skip_reason": (log.raw_output or {}).get("llm_skip_reason"),
"fallback": (log.raw_output or {}).get("fallback"),
"raw_output": log.raw_output,
}
for log, page in rows