from __future__ import annotations
import csv
import io
import json
import re
import time
from dataclasses import asdict
from typing import Any
from fastapi import BackgroundTasks, HTTPException, Response
from pydantic import BaseModel, Field
from sqlalchemy import select
from sqlalchemy.exc import OperationalError
from crawler_platform.app.config.loader import (
ProjectConfig,
SourceConfig,
load_project_config,
project_config_from_dict,
)
from crawler_platform.app.core.crawler.discovery import discover_links
from crawler_platform.app.core.crawler.fetchers import RobotsPolicy, make_fetcher
from crawler_platform.app.core.crawler.pipeline import CrawlPipeline
from crawler_platform.app.core.crawler.site_crawler import SiteCrawler
from crawler_platform.app.core.database import models
from crawler_platform.app.core.database.repository import (
KnowledgeRepository,
make_claim_hash,
)
from crawler_platform.app.core.database.session import session_scope
from crawler_platform.app.core.extractor.ai_provider import list_openai_compatible_models
from crawler_platform.app.core.extractor.factory import extractor_for_domain
from crawler_platform.app.core.ontology.definitions import DOMAIN_ONTOLOGIES, Ontology, ontology_for_domain
from crawler_platform.app.core.ontology.domain_discovery import DomainDiscoveryService
from crawler_platform.app.core.ontology.gap_detector import KnowledgeGapDetector
from crawler_platform.app.core.ontology.mapper import ontology_to_dict
from crawler_platform.app.core.ontology.registry import OntologyRegistry
from crawler_platform.app.core.recommendation.scorer import PreferenceInput, RuleBasedRecommender
from crawler_platform.app.core.research.graph_query import SemanticGraphQuery
from crawler_platform.app.core.research.graph_research_loop import GraphResearchLoop
from crawler_platform.app.core.research.memory_store import ResearchMemoryStore, research_session_payload
DOMAIN_NAME_PATTERN = re.compile(r"^[a-z0-9][a-z0-9_-]{1,79}$")
SITE_CRAWL_CANCEL_REQUESTS: set[int] = set()
COMPARISON_KEYS = ("both_agree", "rule_only", "llm_only", "conflict", "rejected_by_validation")
def extraction_log_summary(raw_output: dict[str, Any]) -> dict[str, Any]:
candidate_claims = raw_output.get("candidate_claims")
if not isinstance(candidate_claims, list):
candidate_claims = []
raw_comparison = raw_output.get("comparison")
if not isinstance(raw_comparison, dict):
raw_comparison = {}
comparison = {key: int(raw_comparison.get(key) or 0) for key in COMPARISON_KEYS}
if not any(comparison.values()):
comparison.update(comparison_from_candidate_claims(candidate_claims))
validation = raw_output.get("validation")
if isinstance(validation, dict) and comparison["rejected_by_validation"] == 0:
comparison["rejected_by_validation"] = number_or_default(validation.get("rejected_claim_count"), 0)
return {
"candidate_count": len(candidate_claims),
"comparison": comparison,
"rule_entity_count": number_or_none(raw_output.get("rule_entity_count")),
"rule_claim_count": number_or_derived(
raw_output.get("rule_claim_count"),
candidate_claims,
source="rule",
),
"llm_entity_count": number_or_none(raw_output.get("llm_entity_count")),
"llm_claim_count": number_or_derived(
raw_output.get("llm_claim_count"),
candidate_claims,
source="llm",
),
"agreement_claim_count": number_or_default(raw_output.get("agreement_claim_count"), comparison["both_agree"]),
"rule_only_claim_count": number_or_default(raw_output.get("rule_only_claim_count"), comparison["rule_only"]),
"llm_only_claim_count": number_or_default(raw_output.get("llm_only_claim_count"), comparison["llm_only"]),
"conflict_claim_count": number_or_default(raw_output.get("conflict_claim_count"), comparison["conflict"]),
}
def comparison_from_candidate_claims(candidate_claims: list[Any]) -> dict[str, int]:
comparison = {key: 0 for key in COMPARISON_KEYS}
for claim in candidate_claims:
metadata = claim_metadata(claim)
agreement = str(metadata.get("agreement") or metadata.get("claim_kind") or "").lower()
if agreement == "rule_and_llm":
comparison["both_agree"] += 1
elif agreement == "rule_only":
comparison["rule_only"] += 1
elif agreement == "llm_only":
comparison["llm_only"] += 1
elif agreement == "conflict":
comparison["conflict"] += 1
return comparison
def claim_metadata(claim: Any) -> dict[str, Any]:
if not isinstance(claim, dict):
return {}
metadata = claim.get("metadata")
return metadata if isinstance(metadata, dict) else {}
def number_or_none(value: Any) -> int | None:
if isinstance(value, bool):
return None
if isinstance(value, (int, float)):
return int(value)
return None
def number_or_default(value: Any, default: int) -> int:
parsed = number_or_none(value)
return default if parsed is None else parsed
def number_or_derived(value: Any, candidate_claims: list[Any], *, source: str) -> int:
parsed = number_or_none(value)
if parsed is not None:
return parsed
return sum(1 for claim in candidate_claims if claim_matches_source(claim, source))
def claim_matches_source(claim: Any, source: str) -> bool:
metadata = claim_metadata(claim)
extraction_source = str(metadata.get("extraction_source") or "").lower()
agreement = str(metadata.get("agreement") or "").lower()
if extraction_source == source:
return True
if source == "rule":
return agreement in {"rule_only", "rule_and_llm"}
if source == "llm":
return agreement in {"llm_only", "rule_and_llm"}
return False
class CrawlRequest(BaseModel):
config_path: str
source_name: str
url: str
extraction_mode: str = "hybrid"
extractor_provider: str = "lm_studio"
extractor_model: str | None = None
extractor_base_url: str | None = "http://localhost:1234/v1"
fallback_to_rules: bool = True
check_robots_txt: bool = False
respect_robots_txt: bool | None = None
class SiteCrawlRequest(CrawlRequest):
max_depth: int = 2
max_pages: int = 50
same_domain_only: bool = True
analyze_page_types: list[str] = Field(default_factory=lambda: ["ProductPage", "BrandStoryPage", "ReviewPage"])
class SiteCrawlByProjectRequest(BaseModel):
"""Site crawl invoked against an existing project (no filesystem config_path)."""
project_name: str
source_name: str
url: str
extraction_mode: str = "hybrid"
extractor_provider: str = "lm_studio"
extractor_model: str | None = None
extractor_base_url: str | None = "http://localhost:1234/v1"
fallback_to_rules: bool = True
check_robots_txt: bool = False
respect_robots_txt: bool | None = None
max_depth: int = 2
max_pages: int = 50
same_domain_only: bool = True
analyze_page_types: list[str] = Field(
default_factory=lambda: ["ProductPage", "BrandStoryPage", "ReviewPage"]
)
def to_site_crawl_request(self, config_path_placeholder: str = "") -> "SiteCrawlRequest":
"""For internal handoff to existing crawl pipeline (config_path is not used)."""
return SiteCrawlRequest(
config_path=config_path_placeholder,
source_name=self.source_name,
url=self.url,
extraction_mode=self.extraction_mode,
extractor_provider=self.extractor_provider,
extractor_model=self.extractor_model,
extractor_base_url=self.extractor_base_url,
fallback_to_rules=self.fallback_to_rules,
check_robots_txt=self.check_robots_txt,
respect_robots_txt=self.respect_robots_txt,
max_depth=self.max_depth,
max_pages=self.max_pages,
same_domain_only=self.same_domain_only,
analyze_page_types=list(self.analyze_page_types),
)
class DiscoverRequest(BaseModel):
config_path: str
source_name: str
url: str
limit: int = 30
check_robots_txt: bool = False
respect_robots_txt: bool | None = None
class RecommendRequest(BaseModel):
project_name: str
target_entity_type: str = "Perfume"
preferences: dict = Field(default_factory=dict)
limit: int = 10
class CreateProjectRequest(BaseModel):
config_path: str
class InlineSourceConfig(BaseModel):
name: str
type: str = "unknown"
trust_level: float = 0.5
base_url: str | None = None
allowed_paths: list[str] = Field(default_factory=list)
parser: str = "generic"
fetcher: str = "requests"
rate_limit_per_minute: int = 30
respect_robots_txt: bool = False
class CreateProjectInlineRequest(BaseModel):
"""Create a project from inline JSON config (no filesystem dependency)."""
project_name: str
domain: str
target_entities: list[str] = Field(default_factory=list)
fields: list[str] = Field(default_factory=list)
sources: list[InlineSourceConfig] = Field(default_factory=list)
ontology: dict[str, Any] = Field(default_factory=dict)
recommendation: dict[str, Any] = Field(default_factory=dict)
update_policy: dict[str, Any] = Field(default_factory=dict)
def to_project_config(self) -> ProjectConfig:
return ProjectConfig(
project_name=self.project_name,
domain=self.domain,
target_entities=list(self.target_entities),
fields=list(self.fields),
sources=[SourceConfig(**s.model_dump()) for s in self.sources],
ontology=dict(self.ontology),
recommendation=dict(self.recommendation),
update_policy=dict(self.update_policy),
)
class DomainDefinitionRequest(BaseModel):
domain: str
description: str | None = None
entity_types: list[str] = Field(default_factory=list)
predicates: list[str] = Field(default_factory=list)
attributes: list[str] = Field(default_factory=list)
aliases: dict[str, str] = Field(default_factory=dict)
class DomainDefinitionUpdateRequest(BaseModel):
description: str | None = None
entity_types: list[str] = Field(default_factory=list)
predicates: list[str] = Field(default_factory=list)
attributes: list[str] = Field(default_factory=list)
aliases: dict[str, str] = Field(default_factory=dict)
class DomainDiscoveryRequest(BaseModel):
seed_urls: list[str] = Field(default_factory=list)
max_pages: int = 20
max_depth: int = 1
same_domain_only: bool = True
fetcher: str = "requests"
respect_robots_txt: bool = False
force_recrawl: bool = False
class DomainCandidateApplyRequest(BaseModel):
candidate_ids: list[int] = Field(default_factory=list)
class DomainCandidateStatusRequest(BaseModel):
status: str
class ResetProjectRequest(BaseModel):
config_path: str
project_name: str | None = None
def source_model_to_config(source: models.Source) -> SourceConfig:
return SourceConfig(
name=source.name,
type=source.type,
trust_level=source.trust_level,
base_url=source.base_url,
rate_limit_per_minute=source.rate_limit_per_minute,
respect_robots_txt=source.respect_robots_txt,
)
def normalize_domain_name(domain: str) -> str:
normalized = domain.strip().lower()
if not DOMAIN_NAME_PATTERN.match(normalized):
raise HTTPException(
status_code=400,
detail="Domain must be 2-80 chars using lowercase letters, numbers, underscore, or hyphen.",
)
return normalized
def clean_string_list(values: list[str]) -> list[str]:
cleaned: list[str] = []
seen: set[str] = set()
for value in values:
item = value.strip()
if not item or item in seen:
continue
cleaned.append(item)
seen.add(item)
return cleaned
def clean_aliases(values: dict[str, str]) -> dict[str, str]:
return {
key.strip(): value.strip()
for key, value in values.items()
if key.strip() and value.strip()
}
def domain_row_to_ontology(row: models.DomainDefinition):
return Ontology(
domain=row.domain,
entity_types=clean_string_list(list(row.entity_types or [])),
predicates=clean_string_list(list(row.predicates or [])),
attributes=clean_string_list(list(row.attributes or [])),
aliases=clean_aliases(dict(row.aliases or {})),
)
def domain_summary_payload(ont, *, is_builtin: bool, is_custom: bool, description: str | None = None):
return {
"domain": ont.domain,
"description": description,
"entity_types": list(ont.entity_types),
"predicates": list(ont.predicates),
"attribute_count": len(ont.attributes),
"attributes": list(ont.attributes),
"aliases": dict(ont.aliases),
"is_builtin": is_builtin,
"is_custom": is_custom,
}
def upsert_domain_definition(
session,
domain: str,
payload: DomainDefinitionUpdateRequest,
) -> models.DomainDefinition:
normalized_domain = normalize_domain_name(domain)
row = session.scalar(
select(models.DomainDefinition).where(models.DomainDefinition.domain == normalized_domain)
)
if row is None:
row = models.DomainDefinition(domain=normalized_domain)
session.add(row)
session.flush()
row.description = payload.description.strip() if payload.description else None
row.entity_types = clean_string_list(payload.entity_types)
row.predicates = clean_string_list(payload.predicates)
row.attributes = clean_string_list(payload.attributes)
row.aliases = clean_aliases(payload.aliases)
row.is_builtin_override = normalized_domain in DOMAIN_ONTOLOGIES
row.updated_at = models.utcnow()
return row
def ensure_domain_definition(session, domain: str) -> models.DomainDefinition:
normalized_domain = normalize_domain_name(domain)
row = session.scalar(
select(models.DomainDefinition).where(models.DomainDefinition.domain == normalized_domain)
)
if row is not None:
return row
base = DOMAIN_ONTOLOGIES.get(normalized_domain)
row = models.DomainDefinition(
domain=normalized_domain,
entity_types=list(base.entity_types) if base else [],
predicates=list(base.predicates) if base else [],
attributes=list(base.attributes) if base else [],
aliases=dict(base.aliases) if base else {},
is_builtin_override=normalized_domain in DOMAIN_ONTOLOGIES,
)
session.add(row)
session.flush()
return row
def ontology_for_domain_from_db(session, domain: str):
normalized_domain = normalize_domain_name(domain)
row = session.scalar(
select(models.DomainDefinition).where(models.DomainDefinition.domain == normalized_domain)
)
if row is not None:
return domain_row_to_ontology(row)
return ontology_for_domain(normalized_domain)
def clean_seed_urls(values: list[str]) -> list[str]:
cleaned: list[str] = []
seen: set[str] = set()
for value in values:
url = value.strip()
if not url or url in seen:
continue
if not (url.startswith("http://") or url.startswith("https://") or url.startswith("file://")):
raise HTTPException(status_code=400, detail=f"Unsupported URL: {url}")
cleaned.append(url)
seen.add(url)
if not cleaned:
raise HTTPException(status_code=400, detail="At least one reference URL is required.")
return cleaned
def domain_discovery_job_payload(job: models.DomainDiscoveryJob) -> dict[str, Any]:
return {
"job_id": job.id,
"domain": job.domain,
"status": job.status,
"seed_urls": list(job.seed_urls or []),
"max_pages": job.max_pages,
"max_depth": job.max_depth,
"same_domain_only": job.same_domain_only,
"fetcher": job.fetcher,
"respect_robots_txt": job.respect_robots_txt,
"error": job.error,
"progress": job.progress or {},
"result_summary": job.result_summary or {},
"created_at": job.created_at.isoformat() if job.created_at else None,
"started_at": job.started_at.isoformat() if job.started_at else None,
"finished_at": job.finished_at.isoformat() if job.finished_at else None,
}
def domain_reference_source_payload(row: models.DomainReferenceSource) -> dict[str, Any]:
return {
"id": row.id,
"domain": row.domain,
"url": row.url,
"label": row.label,
"status": row.status,
"last_crawled_at": row.last_crawled_at.isoformat() if row.last_crawled_at else None,
"metadata": row.metadata_json or {},
"created_at": row.created_at.isoformat() if row.created_at else None,
"updated_at": row.updated_at.isoformat() if row.updated_at else None,
}
def already_crawled_reference_urls(session, domain: str, seed_urls: list[str]) -> set[str]:
if not seed_urls:
return set()
rows = session.scalars(
select(models.DomainReferenceSource).where(
models.DomainReferenceSource.domain == domain,
models.DomainReferenceSource.url.in_(seed_urls),
models.DomainReferenceSource.status == "active",
models.DomainReferenceSource.last_crawled_at.is_not(None),
)
).all()
return {row.url for row in rows}
def domain_candidate_payload(row: models.DomainSchemaCandidate, evidence_rows: list[models.DomainCandidateEvidence]) -> dict[str, Any]:
return {
"id": row.id,
"domain": row.domain,
"job_id": row.job_id,
"candidate_type": row.candidate_type,
"name": row.name,
"description": row.description,
"confidence": row.confidence,
"occurrence_count": row.occurrence_count,
"status": row.status,
"metadata": row.metadata_json or {},
"created_at": row.created_at.isoformat() if row.created_at else None,
"updated_at": row.updated_at.isoformat() if row.updated_at else None,
"evidence": [
{
"id": evidence.id,
"job_id": evidence.job_id,
"url": evidence.url,
"title": evidence.title,
"snippet": evidence.snippet,
"metadata": evidence.metadata_json or {},
"created_at": evidence.created_at.isoformat() if evidence.created_at else None,
}
for evidence in evidence_rows
],
}
def run_domain_discovery_job(database_url: str, job_id: int) -> None:
try:
with session_scope(database_url) as session:
DomainDiscoveryService(session).run_job(job_id)
except Exception as exc:
with session_scope(database_url) as session:
job = session.get(models.DomainDiscoveryJob, job_id)
if job is not None:
job.status = "failed"
job.error = str(exc)
job.finished_at = models.utcnow()
progress = dict(job.progress or {})
progress["errors"] = [*progress.get("errors", []), str(exc)]
job.progress = progress
def recover_interrupted_domain_discovery_jobs(database_url: str) -> None:
"""Mark domain discovery jobs that cannot survive a server restart."""
with session_scope(database_url) as session:
rows = session.scalars(
select(models.DomainDiscoveryJob).where(
models.DomainDiscoveryJob.status.in_(
["pending", "running", "cancel_requested"]
)
)
).all()
for job in rows:
progress = dict(job.progress or {})
errors = list(progress.get("errors") or [])
if job.status == "cancel_requested":
job.status = "canceled"
progress["completion_reason"] = "canceled"
else:
job.status = "failed"
job.error = "Server restarted before this background analysis finished."
progress["completion_reason"] = "interrupted"
errors.append(job.error)
progress["errors"] = errors[-10:]
job.progress = progress
job.finished_at = models.utcnow()
def project_config_from_project_row(session, project: models.Project) -> ProjectConfig:
config_dict = dict(project.config or {})
if not config_dict:
raise HTTPException(
status_code=400,
detail=f"Project '{project.name}' has no stored config",
)
config = project_config_from_dict(config_dict)
config.sources = [
source_model_to_config(source)
for source in session.scalars(
select(models.Source).where(models.Source.project_id == project.id)
).all()
]
return config
def project_config_to_dict(config: ProjectConfig) -> dict[str, Any]:
return {
"project_name": config.project_name,
"domain": config.domain,
"target_entities": list(config.target_entities),
"fields": list(config.fields),
"sources": [asdict(source) for source in config.sources],
"ontology": dict(config.ontology),
"recommendation": dict(config.recommendation),
"update_policy": dict(config.update_policy),
}
def delete_project_data(session, project_id: int, include_project: bool = False) -> dict[str, int]:
repo = KnowledgeRepository(session)
deleted = repo.reset_project_runtime_data(project_id)
for table_model in [
models.OntologyProposal,
models.OntologyTriple,
models.OntologyRelationType,
models.OntologyEntityType,
models.Source,
]:
count = (
session.query(table_model)
.filter(table_model.project_id == project_id)
.delete(synchronize_session=False)
)
deleted[table_model.__tablename__] = deleted.get(table_model.__tablename__, 0) + int(count or 0)
if include_project:
project_count = (
session.query(models.Project)
.filter(models.Project.id == project_id)
.delete(synchronize_session=False)
)
deleted[models.Project.__tablename__] = int(project_count or 0)
return deleted
class CreateEntityRequest(BaseModel):
entity_type: str
name: str
metadata: dict[str, Any] = Field(default_factory=dict)
class BulkCreateEntitiesRequest(BaseModel):
entities: list[CreateEntityRequest]
class CreateClaimRequest(BaseModel):
source_name: str
subject_entity_id: int
predicate: str
object_entity_id: int | None = None
object_value: Any = None
confidence: float = 1.0
confidence_reason: str | None = None
evidence_text: str | None = None
metadata: dict[str, Any] = Field(default_factory=dict)
class ResearchRunByProjectRequest(BaseModel):
"""Run autonomous research against an existing DB project (no config_path)."""
project_name: str
source_name: str
url: str | None = None
seed_entity_id: int | None = None
goal: str = "Semantic ontology exploration"
extraction_mode: str = "hybrid"
extractor_provider: str = "lm_studio"
extractor_model: str | None = None
extractor_base_url: str | None = "http://localhost:1234/v1"
fallback_to_rules: bool = True
check_robots_txt: bool = False
respect_robots_txt: bool | None = None
max_depth: int = 2
max_steps: int = 12
max_branch: int = 8
min_relevance: float = 0.35
same_domain_only: bool = True
analyze_page_types: list[str] = Field(
default_factory=lambda: ["ProductPage", "BrandStoryPage", "ReviewPage"]
)
class UpdateClaimConfidenceRequest(BaseModel):
confidence: float
reason: str | None = None
class UpdateClaimStatusRequest(BaseModel):
status: str
reason: str | None = None
class BulkClaimStatusRequest(BaseModel):
claim_ids: list[int]
status: str
reason: str | None = None
class SchemaEntityTypeRequest(BaseModel):
name: str
domain: str = "generic"
description: str | None = None
status: str = "active"
confidence: float = 1.0
metadata: dict[str, Any] = Field(default_factory=dict)
class SchemaRelationTypeRequest(BaseModel):
name: str
domain: str = "generic"
description: str | None = None
allowed_subject_types: list[str] = Field(default_factory=list)
allowed_object_types: list[str] = Field(default_factory=list)
allowed_page_types: list[str] = Field(default_factory=list)
allowed_source_zones: list[str] = Field(default_factory=list)
semantic_constraints: dict[str, Any] = Field(default_factory=dict)
confidence_rules: dict[str, Any] = Field(default_factory=dict)
min_confidence: float | None = None
status: str = "active"
confidence: float = 1.0
metadata: dict[str, Any] = Field(default_factory=dict)
class PipelineRerunRequest(BaseModel):
stage: str
class MergeEntitiesRequest(BaseModel):
project_name: str
source_entity_id: int
target_entity_id: int
class ExtractorModelsRequest(BaseModel):
provider: str
base_url: str | None = None
class ResearchRunRequest(CrawlRequest):
project_name: str | None = None
seed_entity_id: int | None = None
goal: str = "Semantic ontology exploration"
max_depth: int = 2
max_steps: int = 12
max_branch: int = 8
min_relevance: float = 0.35
same_domain_only: bool = True
analyze_page_types: list[str] = Field(default_factory=lambda: ["ProductPage", "BrandStoryPage", "ReviewPage"])
def site_crawl_progress_payload(result, latest_page=None) -> dict[str, Any]:
payload = asdict(result)
if latest_page is not None:
payload["latest_page"] = asdict(latest_page)
pages = payload.get("pages") or []
payload["extraction_summary"] = {
"llm_skipped_count": sum(1 for page in pages if page.get("llm_skipped")),
"fallback_count": sum(1 for page in pages if page.get("fallback_used")),
"conflict_claim_count": sum(int(page.get("conflict_claim_count") or 0) for page in pages),
"agreement_claim_count": sum(int(page.get("agreement_claim_count") or 0) for page in pages),
"llm_call_count": sum(
1
for page in pages
if page.get("extraction_mode") in {"hybrid", "llm_only", "compare"}
and not page.get("llm_skipped")
and not page.get("fallback_used")
),
}
return payload
_CLAIM_STATUS_ALIASES = {
"accept": "validated_claim",
"accepted": "validated_claim",
"validated": "validated_claim",
"validated_claim": "validated_claim",
"reject": "rejected",
"rejected": "rejected",
"active": "active",
"candidate": "active",
"pending": "active",
}
def _normalize_claim_status(value: str | None) -> str | None:
if not value:
return None
return _CLAIM_STATUS_ALIASES.get(value.strip().lower())
def _apply_claim_review(claim: models.Claim, status: str, reason: str | None) -> None:
claim.status = status
if reason:
claim.confidence_reason = reason
metadata = dict(claim.metadata_json or {})
review = dict(metadata.get("review") or {})
review["status"] = status
review["reason"] = reason or review.get("reason")
review["reviewed_at"] = models.utcnow().isoformat()
metadata["review"] = review
claim.metadata_json = metadata
claim.last_seen_at = models.utcnow()
def turtle_id(value: Any) -> str:
text = "".join(ch if ch.isalnum() else "_" for ch in str(value or "").strip())
text = "_".join(part for part in text.split("_") if part)
if not text:
return "value"
if text[0].isdigit():
text = f"n_{text}"
return text[:120]
def apply_crawl_request_overrides(config, request: CrawlRequest | DiscoverRequest) -> None:
check_robots_txt = request.respect_robots_txt
if check_robots_txt is None:
check_robots_txt = request.check_robots_txt
config.source_by_name(request.source_name).respect_robots_txt = check_robots_txt
def crawl_job_response(job: models.CrawlJob) -> dict[str, Any]:
metadata = job.metadata_json or {}
return {
"job_id": job.id,
"status": job.status,
"url": job.url,
"error": job.error,
"scheduled_at": job.scheduled_at.isoformat() if job.scheduled_at else None,
"started_at": job.started_at.isoformat() if job.started_at else None,
"finished_at": job.finished_at.isoformat() if job.finished_at else None,
"progress": metadata.get("progress", {}),
"request": metadata.get("request", {}),
}
def update_site_crawl_job_metadata(job: models.CrawlJob, **updates: Any) -> None:
metadata = dict(job.metadata_json or {})
metadata.update(updates)
job.metadata_json = metadata
def mark_orphan_site_crawl_jobs_canceled(database_url: str) -> None:
with session_scope(database_url) as session:
rows = session.scalars(
select(models.CrawlJob).where(
models.CrawlJob.status.in_(["running", "cancel_requested"]),
)
).all()
for job in rows:
if (job.metadata_json or {}).get("kind") != "site_crawl":
continue
job.status = "canceled"
job.error = job.error or "canceled after server restart/interrupted crawl"
job.finished_at = models.utcnow()
def request_site_crawl_cancel(database_url: str, job_id: int) -> dict[str, Any]:
SITE_CRAWL_CANCEL_REQUESTS.add(job_id)
last_error: Exception | None = None
for attempt in range(3):
try:
with session_scope(database_url) as session:
job = session.get(models.CrawlJob, job_id)
if job is None or (job.metadata_json or {}).get("kind") != "site_crawl":
raise HTTPException(status_code=404, detail="site crawl job not found")
if job.status in {"completed", "failed", "canceled"}:
SITE_CRAWL_CANCEL_REQUESTS.discard(job_id)
return crawl_job_response(job)
job.status = "cancel_requested"
job.error = "cancel requested by user"
payload = crawl_job_response(job)
return payload
except OperationalError as exc:
last_error = exc
if "database is locked" not in str(exc).lower():
raise
time.sleep(0.25 * (attempt + 1))
return {
"job_id": job_id,
"status": "cancel_requested",
"url": None,
"error": f"cancel requested in memory; database was locked: {last_error}",
"scheduled_at": None,
"started_at": None,
"finished_at": None,
"progress": {},
"request": {},
}
def is_site_crawl_cancel_requested(session, job_id: int) -> bool:
if job_id in SITE_CRAWL_CANCEL_REQUESTS:
return True
session.expire_all()
job = session.get(models.CrawlJob, job_id)
return job is None or job.status == "cancel_requested"
def run_site_crawl_job(database_url: str, job_id: int, request_data: dict[str, Any]) -> None:
inline_config = request_data.pop("__config_dict", None)
request = SiteCrawlRequest(**request_data)
try:
if inline_config is not None:
config = project_config_from_dict(inline_config)
else:
config = load_project_config(request.config_path)
apply_crawl_request_overrides(config, request)
with session_scope(database_url) as session:
job = session.get(models.CrawlJob, job_id)
if job is None:
return
if job.status == "cancel_requested":
job.status = "canceled"
job.finished_at = models.utcnow()
return
job.status = "running"
job.started_at = models.utcnow()
update_site_crawl_job_metadata(
job,
progress={
"seed_url": request.url,
"visited_count": 0,
"analyzed_count": 0,
"queued_count": 1,
"skipped_count": 0,
"errors": [],
"pages": [],
},
)
session.commit()
repo = KnowledgeRepository(session)
crawler = SiteCrawler(
repo,
extractor_for_domain(
config.domain,
provider=request.extractor_provider,
model=request.extractor_model,
base_url=request.extractor_base_url,
extraction_mode=request.extraction_mode,
fallback_to_rules=request.fallback_to_rules,
),
)
def save_progress(result, latest_page) -> None:
current_job = session.get(models.CrawlJob, job_id)
if current_job is None:
return
update_site_crawl_job_metadata(
current_job,
progress=site_crawl_progress_payload(result, latest_page),
)
session.commit()
def should_stop() -> bool:
return is_site_crawl_cancel_requested(session, job_id)
result = crawler.crawl_site(
config,
request.source_name,
request.url,
max_depth=max(request.max_depth, 0),
max_pages=max(min(request.max_pages, 500), 1),
same_domain_only=request.same_domain_only,
analyze_page_types=set(request.analyze_page_types),
progress_callback=save_progress,
should_stop=should_stop,
parent_job_id=job_id,
)
finished_job = session.get(models.CrawlJob, job_id)
if finished_job is not None:
finished_job.status = (
"canceled"
if finished_job.status == "cancel_requested" or job_id in SITE_CRAWL_CANCEL_REQUESTS
else "completed"
)
finished_job.finished_at = models.utcnow()
update_site_crawl_job_metadata(
finished_job,
progress=site_crawl_progress_payload(result),
)
SITE_CRAWL_CANCEL_REQUESTS.discard(job_id)
except Exception as exc:
with session_scope(database_url) as session:
job = session.get(models.CrawlJob, job_id)
if job is not None:
job.status = "failed"
job.error = str(exc)
job.finished_at = models.utcnow()
metadata = dict(job.metadata_json or {})
progress = dict(metadata.get("progress") or {})
progress["errors"] = [*progress.get("errors", []), str(exc)]
update_site_crawl_job_metadata(job, progress=progress)
SITE_CRAWL_CANCEL_REQUESTS.discard(job_id)
def register_routes(app, database_url: str) -> None:
recover_interrupted_domain_discovery_jobs(database_url)
mark_orphan_site_crawl_jobs_canceled(database_url)
@app.get("/health")
def health():
return {"ok": True}
@app.get("/projects")
def projects():
with session_scope(database_url) as session:
rows = session.scalars(select(models.Project).order_by(models.Project.created_at.desc())).all()
return [
{
"id": project.id,
"name": project.name,
"domain": project.domain,
"created_at": project.created_at.isoformat(),
"updated_at": project.updated_at.isoformat(),
}
for project in rows
]
@app.post("/projects")
def create_project(request: CreateProjectRequest):
config = load_project_config(request.config_path)
with session_scope(database_url) as session:
project = KnowledgeRepository(session).upsert_project(config)
return {"id": project.id, "name": project.name, "domain": project.domain}
@app.post("/projects/inline")
def create_project_inline(request: CreateProjectInlineRequest):
config = request.to_project_config()
with session_scope(database_url) as session:
project = KnowledgeRepository(session).upsert_project(config)
return {"id": project.id, "name": project.name, "domain": project.domain}
@app.get("/domains")
def list_domains():
"""Available ontology domains for project creation."""
with session_scope(database_url) as session:
rows = session.scalars(select(models.DomainDefinition)).all()
custom_by_domain = {row.domain: row for row in rows}
payloads = []
for domain, built_in in DOMAIN_ONTOLOGIES.items():
row = custom_by_domain.pop(domain, None)
if row is not None:
payloads.append(
domain_summary_payload(
domain_row_to_ontology(row),
is_builtin=True,
is_custom=True,
description=row.description,
)
)
else:
payloads.append(
domain_summary_payload(
built_in,
is_builtin=True,
is_custom=False,
)
)
for row in sorted(custom_by_domain.values(), key=lambda item: item.domain):
payloads.append(
domain_summary_payload(
domain_row_to_ontology(row),
is_builtin=False,
is_custom=True,
description=row.description,
)
)
return payloads
@app.post("/domains")
def create_domain(request: DomainDefinitionRequest):
domain = normalize_domain_name(request.domain)
with session_scope(database_url) as session:
existing = session.scalar(
select(models.DomainDefinition).where(models.DomainDefinition.domain == domain)
)
if existing is not None or domain in DOMAIN_ONTOLOGIES:
raise HTTPException(status_code=409, detail=f"Domain '{domain}' already exists.")
row = upsert_domain_definition(
session,
domain,
DomainDefinitionUpdateRequest(
description=request.description,
entity_types=request.entity_types,
predicates=request.predicates,
attributes=request.attributes,
aliases=request.aliases,
),
)
return domain_summary_payload(
domain_row_to_ontology(row),
is_builtin=domain in DOMAIN_ONTOLOGIES,
is_custom=True,
description=row.description,
)
@app.put("/domains/{domain}")
def update_domain(domain: str, request: DomainDefinitionUpdateRequest):
with session_scope(database_url) as session:
row = upsert_domain_definition(session, domain, request)
return domain_summary_payload(
domain_row_to_ontology(row),
is_builtin=row.domain in DOMAIN_ONTOLOGIES,
is_custom=True,
description=row.description,
)
@app.get("/domains/{domain}/discovery/jobs")
def list_domain_discovery_jobs(domain: str, limit: int = 20):
normalized_domain = normalize_domain_name(domain)
with session_scope(database_url) as session:
rows = session.scalars(
select(models.DomainDiscoveryJob)
.where(models.DomainDiscoveryJob.domain == normalized_domain)
.order_by(models.DomainDiscoveryJob.created_at.desc())
.limit(max(min(limit, 100), 1))
).all()
return [domain_discovery_job_payload(row) for row in rows]
@app.get("/domains/{domain}/reference-sources")
def list_domain_reference_sources(domain: str):
normalized_domain = normalize_domain_name(domain)
with session_scope(database_url) as session:
rows = session.scalars(
select(models.DomainReferenceSource)
.where(models.DomainReferenceSource.domain == normalized_domain)
.order_by(models.DomainReferenceSource.updated_at.desc())
).all()
return [domain_reference_source_payload(row) for row in rows]
@app.post("/domains/{domain}/discovery/jobs")
def create_domain_discovery_job(
domain: str,
request: DomainDiscoveryRequest,
background_tasks: BackgroundTasks,
):
normalized_domain = normalize_domain_name(domain)
seed_urls = clean_seed_urls(request.seed_urls)
with session_scope(database_url) as session:
ensure_domain_definition(session, normalized_domain)
skipped_seed_urls: list[str] = []
if not request.force_recrawl:
skipped = already_crawled_reference_urls(session, normalized_domain, seed_urls)
skipped_seed_urls = [url for url in seed_urls if url in skipped]
seed_urls = [url for url in seed_urls if url not in skipped]
if not seed_urls:
job = models.DomainDiscoveryJob(
domain=normalized_domain,
status="completed",
seed_urls=[],
max_pages=max(min(request.max_pages, 200), 1),
max_depth=max(min(request.max_depth, 5), 0),
same_domain_only=request.same_domain_only,
fetcher=request.fetcher if request.fetcher in {"requests", "playwright", "browser"} else "requests",
respect_robots_txt=request.respect_robots_txt,
progress={
"visited_count": 0,
"queued_count": 0,
"candidate_count": 0,
"observation_count": 0,
"max_pages": max(min(request.max_pages, 200), 1),
"skipped_seed_urls": skipped_seed_urls,
"errors": [],
"pages": [],
},
result_summary={
"visited_count": 0,
"candidate_count": 0,
"skipped_seed_urls": skipped_seed_urls,
},
finished_at=models.utcnow(),
)
session.add(job)
session.flush()
return domain_discovery_job_payload(job)
job = models.DomainDiscoveryJob(
domain=normalized_domain,
status="pending",
seed_urls=seed_urls,
max_pages=max(min(request.max_pages, 200), 1),
max_depth=max(min(request.max_depth, 5), 0),
same_domain_only=request.same_domain_only,
fetcher=request.fetcher if request.fetcher in {"requests", "playwright", "browser"} else "requests",
respect_robots_txt=request.respect_robots_txt,
progress={
"visited_count": 0,
"queued_count": len(seed_urls),
"candidate_count": 0,
"observation_count": 0,
"max_pages": max(min(request.max_pages, 200), 1),
"skipped_seed_urls": skipped_seed_urls,
"errors": [],
"pages": [],
},
)
session.add(job)
session.flush()
response = domain_discovery_job_payload(job)
background_tasks.add_task(run_domain_discovery_job, database_url, response["job_id"])
return response
@app.get("/domains/{domain}/discovery/jobs/{job_id}")
def domain_discovery_job(domain: str, job_id: int):
normalized_domain = normalize_domain_name(domain)
with session_scope(database_url) as session:
job = session.get(models.DomainDiscoveryJob, job_id)
if job is None or job.domain != normalized_domain:
raise HTTPException(status_code=404, detail="domain discovery job not found")
return domain_discovery_job_payload(job)
@app.post("/domains/{domain}/discovery/jobs/{job_id}/cancel")
def cancel_domain_discovery_job(domain: str, job_id: int):
normalized_domain = normalize_domain_name(domain)
with session_scope(database_url) as session:
job = session.get(models.DomainDiscoveryJob, job_id)
if job is None or job.domain != normalized_domain:
raise HTTPException(status_code=404, detail="domain discovery job not found")
if job.status in {"completed", "failed", "canceled"}:
return domain_discovery_job_payload(job)
job.status = "cancel_requested"
job.error = "cancel requested by user"
return domain_discovery_job_payload(job)
@app.get("/domains/{domain}/candidates")
def list_domain_candidates(domain: str, status: str | None = None, limit: int = 200):
normalized_domain = normalize_domain_name(domain)
with session_scope(database_url) as session:
query = select(models.DomainSchemaCandidate).where(
models.DomainSchemaCandidate.domain == normalized_domain
)
if status:
query = query.where(models.DomainSchemaCandidate.status == status)
rows = session.scalars(
query.order_by(
models.DomainSchemaCandidate.status,
models.DomainSchemaCandidate.candidate_type,
models.DomainSchemaCandidate.confidence.desc(),
models.DomainSchemaCandidate.name,
).limit(max(min(limit, 500), 1))
).all()
payload = []
for row in rows:
evidence = session.scalars(
select(models.DomainCandidateEvidence)
.where(models.DomainCandidateEvidence.candidate_id == row.id)
.order_by(models.DomainCandidateEvidence.created_at.desc())
.limit(3)
).all()
payload.append(domain_candidate_payload(row, evidence))
return payload
@app.post("/domains/{domain}/candidates/apply")
def apply_domain_candidates(domain: str, request: DomainCandidateApplyRequest):
normalized_domain = normalize_domain_name(domain)
if not request.candidate_ids:
raise HTTPException(status_code=400, detail="No candidates selected.")
with session_scope(database_url) as session:
definition = ensure_domain_definition(session, normalized_domain)
rows = session.scalars(
select(models.DomainSchemaCandidate).where(
models.DomainSchemaCandidate.domain == normalized_domain,
models.DomainSchemaCandidate.id.in_(request.candidate_ids),
)
).all()
entity_types = set(definition.entity_types or [])
predicates = set(definition.predicates or [])
attributes = set(definition.attributes or [])
aliases = dict(definition.aliases or {})
applied = {"entity_type": 0, "predicate": 0, "attribute": 0, "alias": 0}
for row in rows:
if row.candidate_type == "entity_type":
entity_types.add(row.name)
applied["entity_type"] += 1
elif row.candidate_type == "predicate":
predicates.add(row.name)
applied["predicate"] += 1
elif row.candidate_type == "attribute":
attributes.add(row.name)
applied["attribute"] += 1
elif row.candidate_type == "alias":
metadata = dict(row.metadata_json or {})
targets = metadata.get("targets") or []
target = metadata.get("target") or (targets[0] if targets else None)
if target:
aliases[row.name] = str(target)
applied["alias"] += 1
row.status = "approved"
row.updated_at = models.utcnow()
definition.entity_types = sorted(entity_types)
definition.predicates = sorted(predicates)
definition.attributes = sorted(attributes)
definition.aliases = aliases
definition.is_builtin_override = normalized_domain in DOMAIN_ONTOLOGIES
definition.updated_at = models.utcnow()
return {
"ok": True,
"domain": normalized_domain,
"applied": applied,
"definition": domain_summary_payload(
domain_row_to_ontology(definition),
is_builtin=normalized_domain in DOMAIN_ONTOLOGIES,
is_custom=True,
description=definition.description,
),
}
@app.post("/domains/{domain}/candidates/{candidate_id}/status")
def update_domain_candidate_status(domain: str, candidate_id: int, request: DomainCandidateStatusRequest):
normalized_domain = normalize_domain_name(domain)
if request.status not in {"pending_review", "approved", "rejected"}:
raise HTTPException(status_code=400, detail="Unsupported candidate status.")
with session_scope(database_url) as session:
row = session.get(models.DomainSchemaCandidate, candidate_id)
if row is None or row.domain != normalized_domain:
raise HTTPException(status_code=404, detail="candidate not found")
row.status = request.status
row.updated_at = models.utcnow()
evidence = session.scalars(
select(models.DomainCandidateEvidence)
.where(models.DomainCandidateEvidence.candidate_id == row.id)
.order_by(models.DomainCandidateEvidence.created_at.desc())
.limit(3)
).all()
return domain_candidate_payload(row, evidence)
@app.post("/projects/reset")
def reset_project(request: ResetProjectRequest):
config = load_project_config(request.config_path)
if request.project_name and request.project_name != config.project_name:
raise HTTPException(
status_code=400,
detail=(
f"Selected project '{request.project_name}' does not match "
f"config project '{config.project_name}'."
),
)
with session_scope(database_url) as session:
repo = KnowledgeRepository(session)
project = session.scalar(select(models.Project).where(models.Project.name == config.project_name))
if project is None:
project = repo.upsert_project(config)
return {
"ok": True,
"name": project.name,
"domain": project.domain,
"created": True,
"reset": False,
"deleted": {},
}
deleted = repo.reset_project_runtime_data(project.id)
project = repo.upsert_project(config)
return {
"ok": True,
"name": project.name,
"domain": project.domain,
"created": False,
"reset": True,
"deleted": deleted,
}
@app.post("/projects/{project_name}/reset")
def reset_project_by_name(project_name: str):
with session_scope(database_url) as session:
repo = KnowledgeRepository(session)
project = repo.get_project(project_name)
config = project_config_from_project_row(session, project)
deleted = repo.reset_project_runtime_data(project.id)
project = repo.upsert_project(config)
return {
"ok": True,
"name": project.name,
"domain": project.domain,
"reset": True,
"deleted": deleted,
}
@app.delete("/projects/{project_name}")
def delete_project(project_name: str):
with session_scope(database_url) as session:
project = KnowledgeRepository(session).get_project(project_name)
domain = project.domain
deleted = delete_project_data(session, project.id, include_project=True)
return {
"ok": True,
"name": project_name,
"domain": domain,
"deleted": deleted,
}
@app.get("/projects/{project_name}")
def project_detail(project_name: str):
with session_scope(database_url) as session:
project = KnowledgeRepository(session).get_project(project_name)
sources = session.scalars(select(models.Source).where(models.Source.project_id == project.id)).all()
return {
"id": project.id,
"name": project.name,
"domain": project.domain,
"config": project_config_to_dict(project_config_from_project_row(session, project)),
"sources": [
{
"id": source.id,
"name": source.name,
"type": source.type,
"base_url": source.base_url,
"trust_level": source.trust_level,
"respect_robots_txt": source.respect_robots_txt,
"rate_limit_per_minute": source.rate_limit_per_minute,
}
for source in sources
],
}
@app.post("/projects/{project_name}/sources")
def add_project_source(project_name: str, request: InlineSourceConfig):
"""Add or update a source on an existing project."""
with session_scope(database_url) as session:
repo = KnowledgeRepository(session)
project = repo.get_project(project_name)
source = repo.upsert_source(project, SourceConfig(**request.model_dump()))
return {
"id": source.id,
"name": source.name,
"type": source.type,
"base_url": source.base_url,
"trust_level": source.trust_level,
"respect_robots_txt": source.respect_robots_txt,
"rate_limit_per_minute": source.rate_limit_per_minute,
}
@app.delete("/projects/{project_name}/sources/{source_name}")
def delete_project_source(project_name: str, source_name: str):
with session_scope(database_url) as session:
repo = KnowledgeRepository(session)
project = repo.get_project(project_name)
source = session.scalar(
select(models.Source).where(
models.Source.project_id == project.id,
models.Source.name == source_name,
)
)
if source is None:
raise HTTPException(
status_code=404,
detail=f"Source '{source_name}' not found in project '{project_name}'",
)
session.delete(source)
return {"ok": True, "deleted": source_name}
@app.get("/ontology/{domain}")
def ontology(domain: str):
with session_scope(database_url) as session:
return ontology_to_dict(ontology_for_domain_from_db(session, domain))
@app.get("/projects/{project_name}/ontology/registry")
def ontology_registry(project_name: str):
with session_scope(database_url) as session:
project = KnowledgeRepository(session).get_project(project_name)
return OntologyRegistry(session).registry_payload(project.id)
@app.post("/projects/{project_name}/schema/entity-types")
def create_schema_entity_type(project_name: str, request: SchemaEntityTypeRequest):
with session_scope(database_url) as session:
project = KnowledgeRepository(session).get_project(project_name)
row = OntologyRegistry(session).upsert_entity_type(
project.id,
name=request.name,
domain=request.domain,
description=request.description,
status=request.status,
confidence=min(max(request.confidence, 0.0), 1.0),
metadata={"origin": "manual_schema_designer", **request.metadata},
)
return {
"id": row.id,
"name": row.name,
"domain": row.domain,
"description": row.description,
"status": row.status,
"confidence": row.confidence,
}
@app.post("/projects/{project_name}/schema/relation-types")
def create_schema_relation_type(project_name: str, request: SchemaRelationTypeRequest):
confidence_rules = dict(request.confidence_rules or {})
if request.min_confidence is not None:
confidence_rules["min_confidence"] = min(max(request.min_confidence, 0.0), 1.0)
with session_scope(database_url) as session:
project = KnowledgeRepository(session).get_project(project_name)
row = OntologyRegistry(session).upsert_relation_type(
project.id,
name=request.name,
domain=request.domain,
description=request.description,
allowed_subject_types=request.allowed_subject_types,
allowed_object_types=request.allowed_object_types,
allowed_page_types=request.allowed_page_types,
allowed_source_zones=request.allowed_source_zones,
semantic_constraints=request.semantic_constraints,
confidence_rules=confidence_rules,
status=request.status,
confidence=min(max(request.confidence, 0.0), 1.0),
metadata={"origin": "manual_schema_designer", **request.metadata},
)
return {
"id": row.id,
"name": row.name,
"domain": row.domain,
"description": row.description,
"allowed_subject_types": row.allowed_subject_types or [],
"allowed_object_types": row.allowed_object_types or [],
"status": row.status,
"confidence": row.confidence,
}
@app.get("/projects/{project_name}/ontology/proposals")
def ontology_proposals(project_name: str, limit: int = 100):
with session_scope(database_url) as session:
project = KnowledgeRepository(session).get_project(project_name)
rows = session.scalars(
select(models.OntologyProposal)
.where(models.OntologyProposal.project_id == project.id)
.order_by(models.OntologyProposal.updated_at.desc())
.limit(max(min(limit, 300), 1))
).all()
return [
{
"id": row.id,
"proposal_type": row.proposal_type,
"name": row.name,
"reason": row.reason,
"evidence": row.evidence,
"status": row.status,
"confidence": row.confidence,
"metadata": row.metadata_json or {},
"updated_at": row.updated_at.isoformat() if row.updated_at else None,
}
for row in rows
]
@app.get("/projects/{project_name}/ontology/triples")
def ontology_triples(project_name: str, status: str | None = None, limit: int = 100):
with session_scope(database_url) as session:
project = KnowledgeRepository(session).get_project(project_name)
query = select(models.OntologyTriple).where(models.OntologyTriple.project_id == project.id)
if status:
query = query.where(models.OntologyTriple.status == status)
rows = session.scalars(
query.order_by(models.OntologyTriple.last_seen_at.desc()).limit(max(min(limit, 300), 1))
).all()
results = []
for row in rows:
subject = session.get(models.Entity, row.subject_entity_id)
object_entity = session.get(models.Entity, row.object_entity_id) if row.object_entity_id else None
results.append(
{
"id": row.id,
"claim_id": row.claim_id,
"subject": subject.name if subject else None,
"subject_type": row.subject_type,
"predicate": row.predicate,
"object": object_entity.name if object_entity else None,
"object_type": row.object_type,
"object_value": row.object_value,
"value_type": row.value_type,
"status": row.status,
"confidence": row.confidence,
"support_count": row.support_count,
"metadata": row.metadata_json or {},
"last_seen_at": row.last_seen_at.isoformat() if row.last_seen_at else None,
}
)
return results
@app.get("/projects/{project_name}/export")
def export_ontology(
project_name: str,
format: str = "json",
status: str = "validated_claim",
include_evidence: bool = True,
limit: int = 1000,
):
fmt = format.strip().lower()
bounded_limit = max(min(limit, 5000), 1)
with session_scope(database_url) as session:
project = KnowledgeRepository(session).get_project(project_name)
query = (
select(models.Claim, models.Source, models.Page, models.Entity)
.join(models.Source, models.Claim.source_id == models.Source.id)
.join(models.Page, models.Claim.page_id == models.Page.id, isouter=True)
.join(models.Entity, models.Claim.subject_entity_id == models.Entity.id)
.where(models.Claim.project_id == project.id)
)
if status and status != "all":
normalized = _normalize_claim_status(status) or status
query = query.where(models.Claim.status == normalized)
rows = session.execute(
query.order_by(models.Claim.confidence.desc(), models.Claim.last_seen_at.desc()).limit(bounded_limit)
).all()
items: list[dict[str, Any]] = []
for claim, source, page, subject in rows:
object_entity = session.get(models.Entity, claim.object_entity_id) if claim.object_entity_id else None
evidence = None
if include_evidence:
evidence = session.scalar(
select(models.Evidence)
.where(models.Evidence.claim_id == claim.id)
.order_by(models.Evidence.created_at.desc())
)
items.append(
{
"claim_id": claim.id,
"subject": subject.name,
"subject_type": subject.entity_type,
"predicate": claim.predicate,
"object": object_entity.name if object_entity else claim.object_value,
"object_type": object_entity.entity_type if object_entity else claim.value_type,
"status": claim.status,
"confidence": claim.confidence,
"source": source.name,
"source_url": page.url if page else None,
"evidence_text": evidence.evidence_text if evidence else None,
"created_by": claim.extraction_method,
"last_seen_at": claim.last_seen_at.isoformat() if claim.last_seen_at else None,
}
)
filename = f"{project_name}_ontology.{fmt if fmt != 'turtle' else 'ttl'}"
if fmt == "json":
return {
"project": project_name,
"status": status,
"count": len(items),
"claims": items,
}
if fmt == "csv":
output = io.StringIO()
fieldnames = [
"claim_id",
"subject",
"subject_type",
"predicate",
"object",
"object_type",
"status",
"confidence",
"source",
"source_url",
"evidence_text",
"created_by",
"last_seen_at",
]
writer = csv.DictWriter(output, fieldnames=fieldnames)
writer.writeheader()
for item in items:
row = {key: json.dumps(value, ensure_ascii=False) if isinstance(value, (dict, list)) else value for key, value in item.items()}
writer.writerow(row)
return Response(
content=output.getvalue(),
media_type="text/csv; charset=utf-8",
headers={"Content-Disposition": f'attachment; filename="{filename}"'},
)
if fmt in {"ttl", "turtle"}:
lines = [
"@prefix ont: .",
"@prefix claim: .",
"@prefix xsd: .",
"",
]
for item in items:
sid = turtle_id(item["subject"])
predicate = turtle_id(item["predicate"])
obj = item["object"]
if isinstance(obj, str) and obj.strip():
object_repr = f'ont:{turtle_id(obj)}'
else:
object_repr = json.dumps(obj, ensure_ascii=False)
lines.extend(
[
f"ont:{sid} ont:{predicate} {object_repr} .",
f"claim:c{item['claim_id']} ont:confidence \"{item['confidence']}\"^^xsd:decimal .",
]
)
if item.get("source_url"):
lines.append(f"claim:c{item['claim_id']} ont:sourceUrl {json.dumps(item['source_url'], ensure_ascii=False)} .")
if item.get("evidence_text"):
lines.append(f"claim:c{item['claim_id']} ont:evidenceText {json.dumps(item['evidence_text'], ensure_ascii=False)} .")
lines.append("")
return Response(
content="\n".join(lines),
media_type="text/turtle; charset=utf-8",
headers={"Content-Disposition": f'attachment; filename="{filename}"'},
)
raise HTTPException(status_code=400, detail=f"unsupported export format: {format}")
@app.get("/projects/{project_name}/knowledge-gaps")
def knowledge_gaps(project_name: str, limit: int = 100):
with session_scope(database_url) as session:
project = KnowledgeRepository(session).get_project(project_name)
return KnowledgeGapDetector(session).list_open(project.id, max(min(limit, 300), 1))
@app.post("/extractors/models")
def extractor_models(request: ExtractorModelsRequest):
try:
if request.provider == "lm_studio":
models = list_openai_compatible_models(request.base_url or "http://localhost:1234/v1")
return {"ok": True, "models": models}
if request.provider == "openai":
import os
models = list_openai_compatible_models(
request.base_url or "https://api.openai.com/v1",
os.getenv("OPENAI_API_KEY"),
)
return {"ok": True, "models": models}
if request.provider == "ollama":
return {"ok": False, "error": "Ollama model listing is not implemented yet. Enter the model manually."}
return {"ok": True, "models": [{"id": "rule_based", "owned_by": "local"}]}
except Exception as exc:
return {"ok": False, "error": str(exc), "models": []}
@app.post("/crawl")
def crawl(request: CrawlRequest):
config = load_project_config(request.config_path)
apply_crawl_request_overrides(config, request)
with session_scope(database_url) as session:
repo = KnowledgeRepository(session)
pipeline = CrawlPipeline(
repo,
extractor_for_domain(
config.domain,
provider=request.extractor_provider,
model=request.extractor_model,
base_url=request.extractor_base_url,
extraction_mode=request.extraction_mode,
fallback_to_rules=request.fallback_to_rules,
),
)
try:
result = pipeline.crawl_url(config, request.source_name, request.url)
except Exception as exc:
raise HTTPException(status_code=400, detail=str(exc)) from exc
return {
"page_id": result.page_id,
"claim_count": result.claim_count,
"entity_count": result.entity_count,
"extraction_mode": result.extraction_mode,
"effective_extraction_mode": result.effective_extraction_mode,
"llm_skipped": result.llm_skipped,
"llm_skip_reason": result.llm_skip_reason,
"fallback_used": result.fallback_used,
"agreement_claim_count": result.agreement_claim_count,
"conflict_claim_count": result.conflict_claim_count,
"crawl_status": result.crawl_status,
"extraction_status": result.extraction_status,
"page_type": result.page_type,
"raw_text_length": result.raw_text_length,
"clean_text_length": result.clean_text_length,
"warnings": result.warnings or [],
}
@app.post("/crawl-site")
def crawl_site(request: SiteCrawlRequest, background_tasks: BackgroundTasks):
config = load_project_config(request.config_path)
apply_crawl_request_overrides(config, request)
with session_scope(database_url) as session:
repo = KnowledgeRepository(session)
project = repo.upsert_project(config)
source = repo.get_source(project.id, request.source_name)
job = models.CrawlJob(
project_id=project.id,
source_id=source.id,
url=request.url,
status="pending",
metadata_json={
"kind": "site_crawl",
"request": request.model_dump(),
"progress": {
"seed_url": request.url,
"visited_count": 0,
"analyzed_count": 0,
"queued_count": 1,
"skipped_count": 0,
"errors": [],
"pages": [],
},
},
)
session.add(job)
session.flush()
response = crawl_job_response(job)
background_tasks.add_task(run_site_crawl_job, database_url, response["job_id"], request.model_dump())
return response
@app.post("/crawl-site/by-project")
def crawl_site_by_project(
request: SiteCrawlByProjectRequest, background_tasks: BackgroundTasks
):
"""Start a site crawl against an existing DB project (no config_path)."""
with session_scope(database_url) as session:
repo = KnowledgeRepository(session)
project = repo.get_project(request.project_name)
config = project_config_from_project_row(session, project)
try:
source = repo.get_source(project.id, request.source_name)
config.source_by_name(request.source_name)
except KeyError as exc:
raise HTTPException(status_code=404, detail=str(exc)) from exc
inner_request = request.to_site_crawl_request()
apply_crawl_request_overrides(config, inner_request)
job = models.CrawlJob(
project_id=project.id,
source_id=source.id,
url=request.url,
status="pending",
metadata_json={
"kind": "site_crawl",
"request": inner_request.model_dump(),
"project_name": request.project_name,
"progress": {
"seed_url": request.url,
"visited_count": 0,
"analyzed_count": 0,
"queued_count": 1,
"skipped_count": 0,
"errors": [],
"pages": [],
},
},
)
session.add(job)
session.flush()
response = crawl_job_response(job)
task_payload = {
**inner_request.model_dump(),
"__config_dict": project_config_to_dict(config),
}
background_tasks.add_task(
run_site_crawl_job, database_url, response["job_id"], task_payload
)
return response
@app.get("/crawl-site/jobs/{job_id}")
def crawl_site_job(job_id: int):
with session_scope(database_url) as session:
job = session.get(models.CrawlJob, job_id)
if job is None or (job.metadata_json or {}).get("kind") != "site_crawl":
raise HTTPException(status_code=404, detail="site crawl job not found")
return crawl_job_response(job)
@app.post("/crawl-site/jobs/{job_id}/cancel")
def cancel_crawl_site_job(job_id: int):
return request_site_crawl_cancel(database_url, job_id)
@app.post("/discover")
def discover(request: DiscoverRequest):
config = load_project_config(request.config_path)
apply_crawl_request_overrides(config, request)
source_config = config.source_by_name(request.source_name)
robots = RobotsPolicy()
robots_decision = robots.check(request.url, source_config.respect_robots_txt)
if not robots_decision.allowed:
return {
"ok": False,
"error": f"{robots_decision.reason}: {request.url}",
"robots_status": robots_decision.status,
"robots_reason": robots_decision.reason,
"links": [],
}
fetcher = make_fetcher(source_config.fetcher, source_config.rate_limit_per_minute)
result = fetcher.fetch(request.url)
links = discover_links(result.analysis_html, result.final_url or request.url, request.limit)
return {
"ok": True,
"status_code": result.status_code,
"final_url": result.final_url,
"crawl_status": result.crawl_status,
"robots_status": robots_decision.status,
"robots_reason": robots_decision.reason,
"warnings": result.warnings,
"links": [asdict(link) for link in links],
}
@app.post("/research/run")
def run_research(request: ResearchRunRequest):
config = load_project_config(request.config_path)
apply_crawl_request_overrides(config, request)
if request.project_name and request.project_name != config.project_name:
raise HTTPException(
status_code=400,
detail=f"Selected project '{request.project_name}' does not match config project '{config.project_name}'.",
)
with session_scope(database_url) as session:
repo = KnowledgeRepository(session)
loop = GraphResearchLoop(
repo,
extractor_for_domain(
config.domain,
provider=request.extractor_provider,
model=request.extractor_model,
base_url=request.extractor_base_url,
extraction_mode=request.extraction_mode,
fallback_to_rules=request.fallback_to_rules,
),
)
try:
result = loop.run(
project_config=config,
source_name=request.source_name,
seed_url=request.url or None,
seed_entity_id=request.seed_entity_id,
goal=request.goal,
max_depth=max(request.max_depth, 0),
max_steps=max(min(request.max_steps, 50), 1),
max_branch=max(min(request.max_branch, 30), 1),
min_relevance=min(max(request.min_relevance, 0.0), 1.0),
same_domain_only=request.same_domain_only,
analyze_page_types=set(request.analyze_page_types),
)
except Exception as exc:
raise HTTPException(status_code=400, detail=str(exc)) from exc
return asdict(result)
@app.post("/research/run/by-project")
def run_research_by_project(request: ResearchRunByProjectRequest):
"""Run research against an existing DB project (no config_path)."""
with session_scope(database_url) as session:
repo = KnowledgeRepository(session)
project = repo.get_project(request.project_name)
config = project_config_from_project_row(session, project)
try:
source_config = config.source_by_name(request.source_name)
except KeyError as exc:
raise HTTPException(status_code=404, detail=str(exc)) from exc
check_robots = request.respect_robots_txt
if check_robots is None:
check_robots = request.check_robots_txt
source_config.respect_robots_txt = check_robots
loop = GraphResearchLoop(
repo,
extractor_for_domain(
config.domain,
provider=request.extractor_provider,
model=request.extractor_model,
base_url=request.extractor_base_url,
extraction_mode=request.extraction_mode,
fallback_to_rules=request.fallback_to_rules,
),
)
try:
result = loop.run(
project_config=config,
source_name=request.source_name,
seed_url=request.url or None,
seed_entity_id=request.seed_entity_id,
goal=request.goal,
max_depth=max(request.max_depth, 0),
max_steps=max(min(request.max_steps, 50), 1),
max_branch=max(min(request.max_branch, 30), 1),
min_relevance=min(max(request.min_relevance, 0.0), 1.0),
same_domain_only=request.same_domain_only,
analyze_page_types=set(request.analyze_page_types),
)
except Exception as exc:
raise HTTPException(status_code=400, detail=str(exc)) from exc
return asdict(result)
@app.get("/projects/{project_name}/research/sessions")
def research_sessions(project_name: str, limit: int = 25):
with session_scope(database_url) as session:
project = KnowledgeRepository(session).get_project(project_name)
return ResearchMemoryStore(session).list_sessions(project.id, limit)
@app.get("/research/sessions/{job_id}")
def research_session(job_id: int):
with session_scope(database_url) as session:
job = session.get(models.CrawlJob, job_id)
if job is None or (job.metadata_json or {}).get("kind") != "research_session":
raise HTTPException(status_code=404, detail="research session not found")
return research_session_payload(job)
@app.get("/projects/{project_name}/graph/neighborhood")
def graph_neighborhood(
project_name: str,
entity_id: int | None = None,
limit: int = 120,
include_candidates: bool = False,
status: str | None = None,
):
with session_scope(database_url) as session:
project = KnowledgeRepository(session).get_project(project_name)
if status and status != "all":
normalized = _normalize_claim_status(status) or status
statuses = [normalized]
elif status == "all":
statuses = None
elif include_candidates:
statuses = ["validated_claim", "active", "candidate_claim", "rule_candidate"]
else:
statuses = ["validated_claim"]
return SemanticGraphQuery(session).neighborhood(
project.id,
entity_id,
max(min(limit, 300), 1),
statuses=statuses,
)
@app.get("/projects/{project_name}/graph/query")
def graph_query(
project_name: str,
kind: str = "trend_summary",
predicate: str = "hasAccord",
tag: str | None = None,
brand: str | None = None,
limit: int = 100,
):
with session_scope(database_url) as session:
project = KnowledgeRepository(session).get_project(project_name)
graph = SemanticGraphQuery(session)
bounded_limit = max(min(limit, 300), 1)
if kind == "brand_products":
return graph.brand_products(project.id, brand, bounded_limit)
if kind == "products_by_tag":
return graph.products_by_tag(project.id, predicate, tag, bounded_limit)
if kind == "relation_summary":
return graph.relation_summary(project.id, bounded_limit)
if kind == "entity_type_summary":
return graph.entity_type_summary(project.id, bounded_limit)
if kind == "trend_summary":
return graph.trend_summary(project.id, bounded_limit)
raise HTTPException(status_code=400, detail=f"unknown graph query kind: {kind}")
@app.get("/projects/{project_name}/entities")
def project_entities(project_name: str, entity_type: str | None = None, limit: int = 50):
with session_scope(database_url) as session:
project = KnowledgeRepository(session).get_project(project_name)
query = select(models.Entity).where(models.Entity.project_id == project.id)
if entity_type:
query = query.where(models.Entity.entity_type == entity_type)
entities = session.scalars(query.limit(limit)).all()
return [
{
"id": entity.id,
"type": entity.entity_type,
"name": entity.name,
"metadata": entity.metadata_json,
}
for entity in entities
]
@app.post("/projects/{project_name}/entities")
def create_entity(project_name: str, request: CreateEntityRequest):
"""Create or update a single entity directly (no extraction)."""
with session_scope(database_url) as session:
repo = KnowledgeRepository(session)
project = repo.get_project(project_name)
entity = repo.upsert_entity(
project_id=project.id,
entity_type=request.entity_type,
name=request.name,
metadata={
**request.metadata,
"input_method": request.metadata.get("input_method", "manual"),
},
)
return {
"id": entity.id,
"type": entity.entity_type,
"name": entity.name,
"metadata": entity.metadata_json,
}
@app.post("/projects/{project_name}/entities/bulk")
def bulk_create_entities(project_name: str, request: BulkCreateEntitiesRequest):
"""Create multiple entities in one call."""
with session_scope(database_url) as session:
repo = KnowledgeRepository(session)
project = repo.get_project(project_name)
created: list[dict[str, Any]] = []
for item in request.entities:
entity = repo.upsert_entity(
project_id=project.id,
entity_type=item.entity_type,
name=item.name,
metadata={
**item.metadata,
"input_method": item.metadata.get("input_method", "manual"),
},
)
created.append(
{
"id": entity.id,
"type": entity.entity_type,
"name": entity.name,
}
)
return {"created": len(created), "entities": created}
@app.delete("/projects/{project_name}/entities/{entity_id}")
def delete_entity(project_name: str, entity_id: int):
with session_scope(database_url) as session:
repo = KnowledgeRepository(session)
project = repo.get_project(project_name)
entity = session.get(models.Entity, entity_id)
if entity is None or entity.project_id != project.id:
raise HTTPException(
status_code=404,
detail=f"Entity {entity_id} not found in project '{project_name}'",
)
session.delete(entity)
return {"ok": True, "deleted": entity_id}
@app.post("/projects/{project_name}/claims")
def create_claim(project_name: str, request: CreateClaimRequest):
"""Create a single claim directly (manual input)."""
with session_scope(database_url) as session:
repo = KnowledgeRepository(session)
project = repo.get_project(project_name)
try:
source = repo.get_source(project.id, request.source_name)
except KeyError as exc:
raise HTTPException(status_code=404, detail=str(exc)) from exc
subject = session.get(models.Entity, request.subject_entity_id)
if subject is None or subject.project_id != project.id:
raise HTTPException(
status_code=404,
detail=f"Subject entity {request.subject_entity_id} not found",
)
object_entity: models.Entity | None = None
if request.object_entity_id is not None:
object_entity = session.get(models.Entity, request.object_entity_id)
if object_entity is None or object_entity.project_id != project.id:
raise HTTPException(
status_code=404,
detail=f"Object entity {request.object_entity_id} not found",
)
claim_hash = make_claim_hash(
project_id=project.id,
source_id=source.id,
subject_entity_id=subject.id,
predicate=request.predicate,
object_entity_id=object_entity.id if object_entity else None,
object_value=request.object_value,
)
existing = session.scalar(
select(models.Claim).where(
models.Claim.project_id == project.id,
models.Claim.claim_hash == claim_hash,
)
)
if existing is not None:
existing.confidence = max(existing.confidence, request.confidence)
existing.last_seen_at = models.utcnow()
if request.confidence_reason:
existing.confidence_reason = request.confidence_reason
existing.metadata_json = {
**(existing.metadata_json or {}),
**request.metadata,
"input_method": "manual",
}
claim = existing
else:
claim = models.Claim(
project_id=project.id,
source_id=source.id,
page_id=None,
subject_entity_id=subject.id,
predicate=request.predicate,
object_entity_id=object_entity.id if object_entity else None,
object_value=request.object_value,
value_type="entity" if object_entity else "literal",
claim_hash=claim_hash,
confidence=max(0.0, min(1.0, request.confidence)),
confidence_reason=request.confidence_reason,
extraction_method="manual",
status="validated_claim",
metadata_json={**request.metadata, "input_method": "manual"},
)
session.add(claim)
session.flush()
if request.evidence_text:
session.add(
models.Evidence(
project_id=project.id,
claim_id=claim.id,
page_id=None,
evidence_text=request.evidence_text,
)
)
return {
"id": claim.id,
"subject_entity_id": claim.subject_entity_id,
"predicate": claim.predicate,
"object_entity_id": claim.object_entity_id,
"object_value": claim.object_value,
"confidence": claim.confidence,
"status": claim.status,
}
@app.delete("/projects/{project_name}/claims/{claim_id}")
def delete_claim(project_name: str, claim_id: int):
with session_scope(database_url) as session:
repo = KnowledgeRepository(session)
project = repo.get_project(project_name)
claim = session.get(models.Claim, claim_id)
if claim is None or claim.project_id != project.id:
raise HTTPException(
status_code=404,
detail=f"Claim {claim_id} not found in project '{project_name}'",
)
session.delete(claim)
return {"ok": True, "deleted": claim_id}
@app.get("/projects/{project_name}/claims")
def project_claims(
project_name: str,
limit: int = 100,
include_candidates: bool = False,
status: str | None = None,
):
with session_scope(database_url) as session:
project = KnowledgeRepository(session).get_project(project_name)
query = (
select(models.Claim, models.Source, models.Page, models.Entity)
.join(models.Source, models.Claim.source_id == models.Source.id)
.join(models.Page, models.Claim.page_id == models.Page.id, isouter=True)
.join(models.Entity, models.Claim.subject_entity_id == models.Entity.id)
.where(models.Claim.project_id == project.id)
)
if status:
normalized = _normalize_claim_status(status)
if normalized is None:
raise HTTPException(status_code=400, detail=f"unknown claim status: {status}")
query = query.where(models.Claim.status == normalized)
elif not include_candidates:
query = query.where(models.Claim.status == "validated_claim")
rows = session.execute(query.order_by(models.Claim.last_seen_at.desc()).limit(limit)).all()
results: list[dict[str, Any]] = []
for claim, source, page, subject in rows:
object_name = None
object_type = None
if claim.object_entity_id:
object_entity = session.get(models.Entity, claim.object_entity_id)
object_name = object_entity.name if object_entity else None
object_type = object_entity.entity_type if object_entity else None
evidence = session.scalar(
select(models.Evidence)
.where(models.Evidence.claim_id == claim.id)
.order_by(models.Evidence.created_at.desc())
)
results.append(
{
"id": claim.id,
"subject": subject.name,
"subject_type": subject.entity_type,
"predicate": claim.predicate,
"object": object_name,
"object_type": object_type,
"object_value": claim.object_value,
"source": source.name,
"page_url": page.url if page else None,
"confidence": claim.confidence,
"confidence_reason": claim.confidence_reason,
"status": claim.status,
"extraction_method": claim.extraction_method,
"evidence_text": evidence.evidence_text if evidence else None,
"evidence_summary": evidence.evidence_summary if evidence else None,
"page_type": (claim.metadata_json or {}).get("page_type")
or ((page.metadata_json or {}).get("page_type") if page else None),
"source_zone": (claim.metadata_json or {}).get("source_zone"),
"source_selector": (claim.metadata_json or {}).get("source_selector"),
"evidence_found": (claim.metadata_json or {}).get("evidence_found"),
"validation_status": (claim.metadata_json or {}).get("validation_status"),
"graph_merge_status": (claim.metadata_json or {}).get("graph_merge_status"),
"graph_merge_reason": (claim.metadata_json or {}).get("graph_merge_reason"),
"confidence_breakdown": (claim.metadata_json or {}).get("confidence_breakdown"),
"agreement": (claim.metadata_json or {}).get("agreement"),
"extraction_source": (claim.metadata_json or {}).get("extraction_source"),
"claim_kind": (claim.metadata_json or {}).get("claim_kind"),
"rule_confidence": (claim.metadata_json or {}).get("rule_confidence"),
"llm_confidence": (claim.metadata_json or {}).get("llm_confidence"),
"review_required": (claim.metadata_json or {}).get("review_required"),
"review_reason": (claim.metadata_json or {}).get("review_reason"),
"conflict_status": (claim.metadata_json or {}).get("conflict_status"),
"source_history": (claim.metadata_json or {}).get("source_history") or [],
"last_seen_at": claim.last_seen_at.isoformat(),
}
)
return results
@app.get("/projects/{project_name}/pipeline")
def project_pipeline(project_name: str):
from sqlalchemy import func as sa_func, distinct
with session_scope(database_url) as session:
project = KnowledgeRepository(session).get_project(project_name)
pid = project.id
pages_total = session.scalar(
select(sa_func.count(models.Page.id)).where(models.Page.project_id == pid)
) or 0
ext_total = session.scalar(
select(sa_func.count(models.ExtractionLog.id)).where(models.ExtractionLog.project_id == pid)
) or 0
ext_pages = session.scalar(
select(sa_func.count(distinct(models.ExtractionLog.page_id)))
.where(models.ExtractionLog.project_id == pid, models.ExtractionLog.page_id.isnot(None))
) or 0
ext_errors = session.scalar(
select(sa_func.count(models.ExtractionLog.id))
.where(models.ExtractionLog.project_id == pid, models.ExtractionLog.error.isnot(None))
) or 0
status_rows = session.execute(
select(models.Claim.status, sa_func.count(models.Claim.id))
.where(models.Claim.project_id == pid)
.group_by(models.Claim.status)
).all()
claim_statuses = {row[0]: int(row[1]) for row in status_rows}
claim_total = sum(claim_statuses.values())
entity_count = session.scalar(
select(sa_func.count(models.Entity.id)).where(models.Entity.project_id == pid)
) or 0
triple_count = session.scalar(
select(sa_func.count(models.OntologyTriple.id))
.where(models.OntologyTriple.project_id == pid, models.OntologyTriple.status == "active")
) or 0
entity_type_rows = session.execute(
select(models.Entity.entity_type, sa_func.count(models.Entity.id))
.where(models.Entity.project_id == pid)
.group_by(models.Entity.entity_type)
.order_by(sa_func.count(models.Entity.id).desc())
.limit(10)
).all()
recent_pages = session.scalars(
select(models.Page).where(models.Page.project_id == pid)
.order_by(models.Page.fetched_at.desc()).limit(5)
).all()
recent_claims_rows = session.execute(
select(models.Claim, models.Entity)
.join(models.Entity, models.Claim.subject_entity_id == models.Entity.id)
.where(models.Claim.project_id == pid)
.order_by(models.Claim.last_seen_at.desc()).limit(5)
).all()
return {
"stages": [
{"key": "crawled", "count": pages_total},
{"key": "extracted", "count": ext_pages, "extra": {"events": ext_total, "errors": ext_errors}},
{"key": "claims", "count": claim_total, "extra": claim_statuses},
{"key": "validated", "count": claim_statuses.get("validated_claim", 0)},
{"key": "graph", "count": triple_count, "extra": {"entities": entity_count}},
],
"entity_types": [{"type": row[0], "count": int(row[1])} for row in entity_type_rows],
"recent_pages": [
{
"id": p.id,
"url": p.url,
"title": p.title,
"status_code": p.status_code,
"page_type": (p.metadata_json or {}).get("page_type"),
"fetched_at": p.fetched_at.isoformat() if p.fetched_at else None,
}
for p in recent_pages
],
"recent_claims": [
{
"id": claim.id,
"subject": subject.name,
"predicate": claim.predicate,
"confidence": claim.confidence,
"status": claim.status,
"last_seen_at": claim.last_seen_at.isoformat() if claim.last_seen_at else None,
}
for claim, subject in recent_claims_rows
],
}
@app.post("/projects/{project_name}/pipeline/rerun")
def rerun_pipeline_stage(
project_name: str,
request: PipelineRerunRequest,
background_tasks: BackgroundTasks,
):
crawl_stages = {
"source-crawl",
"page-clean",
"page-classification",
"entity-extraction",
"claim-generation",
"deduplication",
"validation",
}
route_by_stage = {
"human-review": f"/review/{project_name}",
"ontology-commit": f"/graph/{project_name}",
"export": f"/export/{project_name}",
}
if request.stage not in crawl_stages:
return {
"ok": True,
"stage": request.stage,
"action": "navigate",
"route": route_by_stage.get(request.stage, f"/pipeline/{project_name}"),
"message": "This stage is controlled from its workspace screen.",
}
with session_scope(database_url) as session:
repo = KnowledgeRepository(session)
project = repo.get_project(project_name)
config = project_config_from_project_row(session, project)
latest_job = session.scalar(
select(models.CrawlJob)
.where(models.CrawlJob.project_id == project.id)
.order_by(models.CrawlJob.scheduled_at.desc())
)
latest_request = dict((latest_job.metadata_json or {}).get("request") or {}) if latest_job else {}
source_name = latest_request.get("source_name")
source = None
if source_name:
source = session.scalar(
select(models.Source).where(
models.Source.project_id == project.id,
models.Source.name == source_name,
)
)
if source is None:
source = session.scalar(
select(models.Source)
.where(models.Source.project_id == project.id)
.order_by(models.Source.updated_at.desc())
)
if source is None:
raise HTTPException(status_code=400, detail="No source is registered for this project.")
url = latest_request.get("url") or source.base_url
if not url:
raise HTTPException(status_code=400, detail="No previous crawl URL or source base_url is available.")
inner_request = SiteCrawlRequest(
config_path="",
source_name=source.name,
url=url,
extraction_mode=str(latest_request.get("extraction_mode") or "hybrid"),
extractor_provider=str(latest_request.get("extractor_provider") or "lm_studio"),
extractor_model=latest_request.get("extractor_model"),
extractor_base_url=latest_request.get("extractor_base_url") or "http://localhost:1234/v1",
fallback_to_rules=bool(latest_request.get("fallback_to_rules", True)),
check_robots_txt=bool(latest_request.get("check_robots_txt") or False),
respect_robots_txt=latest_request.get("respect_robots_txt"),
max_depth=int(latest_request.get("max_depth") or 2),
max_pages=int(latest_request.get("max_pages") or 50),
same_domain_only=bool(latest_request.get("same_domain_only", True)),
analyze_page_types=list(
latest_request.get("analyze_page_types")
or ["ProductPage", "BrandStoryPage", "ReviewPage"]
),
)
apply_crawl_request_overrides(config, inner_request)
job = models.CrawlJob(
project_id=project.id,
source_id=source.id,
url=url,
status="pending",
metadata_json={
"kind": "site_crawl",
"request": inner_request.model_dump(),
"project_name": project_name,
"rerun_stage": request.stage,
"progress": {
"seed_url": url,
"visited_count": 0,
"analyzed_count": 0,
"queued_count": 1,
"skipped_count": 0,
"errors": [],
"pages": [],
},
},
)
session.add(job)
session.flush()
response = crawl_job_response(job)
task_payload = {
**inner_request.model_dump(),
"__config_dict": project_config_to_dict(config),
}
background_tasks.add_task(
run_site_crawl_job,
database_url,
response["job_id"],
task_payload,
)
return {
**response,
"ok": True,
"stage": request.stage,
"action": "job_started",
}
@app.get("/projects/{project_name}/search")
def project_search(project_name: str, q: str = "", limit: int = 10):
q = (q or "").strip()
if len(q) < 1:
return {"entities": [], "claims": [], "pages": [], "predicates": []}
like = f"%{q.lower()}%"
bounded = max(min(limit, 50), 1)
with session_scope(database_url) as session:
project = KnowledgeRepository(session).get_project(project_name)
pid = project.id
from sqlalchemy import func as sa_func, or_
entity_rows = session.scalars(
select(models.Entity)
.where(
models.Entity.project_id == pid,
or_(
sa_func.lower(models.Entity.name).like(like),
sa_func.lower(models.Entity.canonical_name).like(like),
),
)
.limit(bounded)
).all()
claim_rows = session.execute(
select(models.Claim, models.Entity)
.join(models.Entity, models.Claim.subject_entity_id == models.Entity.id)
.where(
models.Claim.project_id == pid,
or_(
sa_func.lower(models.Entity.name).like(like),
sa_func.lower(models.Claim.predicate).like(like),
),
)
.order_by(models.Claim.last_seen_at.desc())
.limit(bounded)
).all()
page_rows = session.scalars(
select(models.Page)
.where(
models.Page.project_id == pid,
or_(
sa_func.lower(models.Page.url).like(like),
sa_func.lower(models.Page.title).like(like),
),
)
.order_by(models.Page.fetched_at.desc())
.limit(bounded)
).all()
predicate_rows = session.scalars(
select(models.OntologyRelationType.name)
.where(
models.OntologyRelationType.project_id == pid,
sa_func.lower(models.OntologyRelationType.name).like(like),
)
.limit(bounded)
).all()
return {
"entities": [
{"id": e.id, "name": e.name, "type": e.entity_type, "canonical_name": e.canonical_name}
for e in entity_rows
],
"claims": [
{
"id": claim.id,
"subject": subject.name,
"predicate": claim.predicate,
"object_value": claim.object_value,
"confidence": claim.confidence,
"status": claim.status,
}
for claim, subject in claim_rows
],
"pages": [
{"id": p.id, "url": p.url, "title": p.title}
for p in page_rows
],
"predicates": list(predicate_rows),
}
@app.get("/projects/{project_name}/extraction-logs")
def project_extraction_logs(project_name: str, limit: int = 50):
with session_scope(database_url) as session:
project = KnowledgeRepository(session).get_project(project_name)
rows = session.execute(
select(models.ExtractionLog, models.Page)
.join(models.Page, models.ExtractionLog.page_id == models.Page.id, isouter=True)
.where(models.ExtractionLog.project_id == project.id)
.order_by(models.ExtractionLog.created_at.desc())
.limit(limit)
).all()
payload = []
for log, page in rows:
raw_output = log.raw_output or {}
summary = extraction_log_summary(raw_output)
payload.append({
"id": log.id,
"page_url": page.url if page else None,
"extractor_name": log.extractor_name,
"provider": log.provider,
"error": log.error,
"created_at": log.created_at.isoformat(),
"validation": raw_output.get("validation"),
"page_context": raw_output.get("page_context"),
"candidate_count": summary["candidate_count"],
"extraction_mode": raw_output.get("extraction_mode"),
"effective_extraction_mode": raw_output.get("effective_extraction_mode"),
"comparison": summary["comparison"],
"rule_entity_count": summary["rule_entity_count"],
"rule_claim_count": summary["rule_claim_count"],
"llm_entity_count": summary["llm_entity_count"],
"llm_claim_count": summary["llm_claim_count"],
"agreement_claim_count": summary["agreement_claim_count"],
"rule_only_claim_count": summary["rule_only_claim_count"],
"llm_only_claim_count": summary["llm_only_claim_count"],
"conflict_claim_count": summary["conflict_claim_count"],
"llm_skipped": raw_output.get("llm_skipped"),
"llm_skip_reason": raw_output.get("llm_skip_reason"),
"fallback": raw_output.get("fallback"),
"raw_output": log.raw_output,
})
return payload
@app.patch("/claims/{claim_id}/confidence")
def update_claim_confidence(claim_id: int, request: UpdateClaimConfidenceRequest):
confidence = min(max(request.confidence, 0.0), 1.0)
with session_scope(database_url) as session:
claim = session.get(models.Claim, claim_id)
if claim is None:
return {"ok": False, "error": "claim not found"}
claim.confidence = confidence
claim.confidence_reason = request.reason or "manual admin update"
claim.last_seen_at = models.utcnow()
return {"ok": True, "claim_id": claim.id, "confidence": claim.confidence}
@app.patch("/claims/{claim_id}/status")
def update_claim_status(claim_id: int, request: UpdateClaimStatusRequest):
status = _normalize_claim_status(request.status)
if status is None:
raise HTTPException(status_code=400, detail=f"unknown claim status: {request.status}")
with session_scope(database_url) as session:
claim = session.get(models.Claim, claim_id)
if claim is None:
raise HTTPException(status_code=404, detail="claim not found")
_apply_claim_review(claim, status, request.reason)
return {"ok": True, "claim_id": claim.id, "status": claim.status}
@app.post("/claims/bulk-status")
def bulk_update_claim_status(request: BulkClaimStatusRequest):
status = _normalize_claim_status(request.status)
if status is None:
raise HTTPException(status_code=400, detail=f"unknown claim status: {request.status}")
if not request.claim_ids:
return {"ok": True, "updated": 0}
with session_scope(database_url) as session:
rows = session.execute(
select(models.Claim).where(models.Claim.id.in_(request.claim_ids))
).scalars().all()
for claim in rows:
_apply_claim_review(claim, status, request.reason)
return {"ok": True, "updated": len(rows), "status": status}
@app.post("/entities/merge")
def merge_entities(request: MergeEntitiesRequest):
with session_scope(database_url) as session:
project = KnowledgeRepository(session).get_project(request.project_name)
source = session.get(models.Entity, request.source_entity_id)
target = session.get(models.Entity, request.target_entity_id)
if source is None or target is None or source.project_id != project.id or target.project_id != project.id:
return {"ok": False, "error": "entity not found in project"}
session.query(models.Claim).filter(models.Claim.subject_entity_id == source.id).update(
{models.Claim.subject_entity_id: target.id}
)
session.query(models.Claim).filter(models.Claim.object_entity_id == source.id).update(
{models.Claim.object_entity_id: target.id}
)
session.query(models.Relation).filter(models.Relation.subject_entity_id == source.id).update(
{models.Relation.subject_entity_id: target.id}
)
session.query(models.Relation).filter(models.Relation.object_entity_id == source.id).update(
{models.Relation.object_entity_id: target.id}
)
source.metadata_json = {**(source.metadata_json or {}), "merged_into": target.id}
source.updated_at = models.utcnow()
return {"ok": True, "source_entity_id": source.id, "target_entity_id": target.id}
@app.get("/projects/{project_name}/recommendation-tags")
def recommendation_tags(project_name: str):
with session_scope(database_url) as session:
project = KnowledgeRepository(session).get_project(project_name)
tag_predicates = {
"hasTopNote",
"hasMiddleNote",
"hasBaseNote",
"hasScentNote",
"hasFlavorNote",
"evokesMood",
"suitableForSeason",
"suitableForOccasion",
"hasReviewKeyword",
}
rows = session.execute(
select(models.Claim, models.Entity)
.join(models.Entity, models.Claim.object_entity_id == models.Entity.id)
.where(
models.Claim.project_id == project.id,
models.Claim.status == "validated_claim",
models.Claim.predicate.in_(tag_predicates),
)
).all()
grouped: dict[str, dict[str, Any]] = {}
for claim, entity in rows:
key = f"{claim.predicate}:{entity.canonical_name}"
if key not in grouped:
grouped[key] = {
"predicate": claim.predicate,
"name": entity.name,
"type": entity.entity_type,
"support_count": 0,
"max_confidence": 0.0,
}
grouped[key]["support_count"] += 1
grouped[key]["max_confidence"] = max(grouped[key]["max_confidence"], claim.confidence)
return sorted(grouped.values(), key=lambda item: (item["predicate"], -item["support_count"], item["name"]))
@app.post("/recommend")
def recommend(request: RecommendRequest):
with session_scope(database_url) as session:
repo = KnowledgeRepository(session)
project = repo.get_project(request.project_name)
recommender = RuleBasedRecommender(session)
pref = PreferenceInput(**request.preferences)
items = recommender.recommend(project.id, request.target_entity_type, pref, request.limit)
return [asdict(item) for item in items]