lasta/AI
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

docs

Browse Source
This commit is contained in:
LASTA_DEV01\lasta
2026-05-19 20:31:52 +09:00
parent 00407e7a08
commit e260e5f218
104 changed files with 12898 additions and 1709 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
ontology_platform/ont_platform/api/db_deps.py
View File

@@ -1,11 +1,13 @@
"""Database dependencies for FastAPI."""
from typing import Generator
from collections.abc import Generator
from pathlib import Path
from sqlalchemy import create_engine
from sqlalchemy.orm import Session, sessionmaker
from ont_platform.config import load_settings
from ont_platform.storage.models import Base
# Initialize database engine (lazy singleton)
_engine = None
@@ -18,12 +20,14 @@ def get_db_engine():
if _engine is None:
settings = load_settings()
database_url = settings.database_url
_ensure_sqlite_parent(database_url)
_engine = create_engine(
database_url,
connect_args={"timeout": 30} if "sqlite" in database_url else {},
pool_pre_ping=True,
echo=False,
)
Base.metadata.create_all(bind=_engine)
return _engine
@@ -46,4 +50,13 @@ def get_db() -> Generator[Session, None, None]:
db.close()
def _ensure_sqlite_parent(database_url: str) -> None:
if not database_url.startswith("sqlite:///"):
return
db_path = database_url.removeprefix("sqlite:///")
if db_path in {":memory:", ""}:
return
Path(db_path).expanduser().parent.mkdir(parents=True, exist_ok=True)
__all__ = ["get_db", "get_db_engine", "get_session_factory"]

47
ontology_platform/ont_platform/api/main.py
View File

@@ -44,7 +44,6 @@ from ont_platform.api.deps import ( # noqa: E402
get_app_context,
initialize_app_context,
)
from ont_platform.api.routes import extraction_router # noqa: E402
platform_config = importlib.import_module("ont_platform.config")
@@ -78,6 +77,49 @@ ONTOCAST_VERSION = _resolve_ontocast_version()
PLATFORM_VERSION = "0.0.1"
def _include_phase_routers(app: FastAPI) -> None:
"""Attach routers whose dependencies are enabled for the configured phase."""
settings = platform_config.load_settings()
enabled_routes: list[str] = []
if settings.phase >= platform_config.Phase.TRAFILATURA:
try:
from ont_platform.api.routes import get_extraction_router
except ImportError as exc:
raise RuntimeError(
"Phase 1 route loading requires the Phase 1 extraction dependencies. "
"Install the Phase 1 dependency set or run with PHASE=0."
) from exc
app.include_router(get_extraction_router())
enabled_routes.append("extraction")
if settings.phase >= platform_config.Phase.CANDIDATE_REVIEW:
from ont_platform.api.routes import get_review_router
app.include_router(get_review_router())
enabled_routes.append("review")
if settings.phase >= platform_config.Phase.CRAWL4AI:
from ont_platform.api.routes import get_crawl_router
app.include_router(get_crawl_router())
enabled_routes.append("crawl")
if settings.phase >= platform_config.Phase.NEO4J_GRAPHRAG:
from ont_platform.api.routes import get_graph_router
app.include_router(get_graph_router())
enabled_routes.append("graph")
if settings.phase >= platform_config.Phase.MULTI_AGENT:
from ont_platform.api.routes import get_maintenance_router
app.include_router(get_maintenance_router())
enabled_routes.append("maintenance")
app.state.enabled_phase_routes = enabled_routes
@asynccontextmanager
async def lifespan(app: FastAPI) -> AsyncIterator[None]:
"""FastAPI lifespan: build ToolBox + workflow once on startup."""
@@ -369,8 +411,7 @@ def create_app() -> FastAPI:
},
)
# ─── Phase 0 routes ───────────────────────────────────────────────
app.include_router(extraction_router)
_include_phase_routers(app)
return app

45
ontology_platform/ont_platform/api/routes/__init__.py
View File

@@ -1,5 +1,44 @@
"""API routes."""
"""API route loaders.
from .extraction import router as extraction_router
Future-phase routers stay behind lazy loader functions so importing the
Phase 0 app does not require optional dependencies such as Trafilatura.
"""
__all__ = ["extraction_router"]
def get_extraction_router():
from .extraction import router
return router
def get_review_router():
from .review import router
return router
def get_crawl_router():
from .crawl import router
return router
def get_graph_router():
from .graph import router
return router
def get_maintenance_router():
from .maintenance import router
return router
__all__ = [
"get_crawl_router",
"get_extraction_router",
"get_graph_router",
"get_maintenance_router",
"get_review_router",
]

91
ontology_platform/ont_platform/api/routes/crawl.py Normal file
View File

@@ -0,0 +1,91 @@
"""Phase 3 crawl acquisition job routes."""
from __future__ import annotations
from typing import Annotated
from fastapi import APIRouter, Body, Depends, HTTPException
from pydantic import BaseModel, Field, model_validator
from sqlalchemy.orm import Session
from ont_platform.api.db_deps import get_db
from ont_platform.core.crawler import CachePolicy, CrawlProfile, RobotsPolicy
from ont_platform.core.crawler.jobs import CrawlJobRequest, CrawlJobRunner, job_to_dict
from ont_platform.storage.models import ExtractionJob
router = APIRouter(prefix="/api/v1/crawl", tags=["crawl"])
class CrawlJobStartRequest(BaseModel):
project_id: str = "default"
url: str | None = None
html: str | None = None
profile: CrawlProfile = CrawlProfile.FAST_STATIC
max_pages: int = Field(default=50, ge=1, le=50)
max_depth: int = Field(default=1, ge=0, le=5)
robots_policy: RobotsPolicy = RobotsPolicy.RESPECT
cache_policy: CachePolicy = CachePolicy.ENABLED
@model_validator(mode="after")
def _require_input(self) -> CrawlJobStartRequest:
if not self.url and not self.html:
raise ValueError("url or html is required")
return self
@router.post("/jobs")
async def start_crawl_job(
request: Annotated[CrawlJobStartRequest, Body()],
db: Annotated[Session, Depends(get_db)],
) -> dict:
runner = CrawlJobRunner(db)
try:
job = await runner.run(
CrawlJobRequest(
project_id=request.project_id,
url=request.url,
html=request.html,
profile=request.profile,
max_pages=request.max_pages,
max_depth=request.max_depth,
robots_policy=request.robots_policy,
cache_policy=request.cache_policy,
)
)
except ValueError as exc:
raise HTTPException(status_code=400, detail=str(exc)) from exc
except Exception as exc:
db.commit()
raise HTTPException(status_code=500, detail=f"Crawl job failed: {exc}") from exc
db.commit()
return {"status": "success", "job": job_to_dict(job)}
@router.get("/jobs/{job_id}")
def get_crawl_job(
job_id: str,
db: Annotated[Session, Depends(get_db)],
) -> dict:
job = db.get(ExtractionJob, job_id)
if job is None or job.job_type != "crawl":
raise HTTPException(status_code=404, detail=f"Crawl job not found: {job_id}")
return {"job": job_to_dict(job)}
@router.post("/jobs/{job_id}/cancel")
def cancel_crawl_job(
job_id: str,
db: Annotated[Session, Depends(get_db)],
) -> dict:
job = db.get(ExtractionJob, job_id)
if job is None or job.job_type != "crawl":
raise HTTPException(status_code=404, detail=f"Crawl job not found: {job_id}")
if job.status in {"completed", "failed", "canceled"}:
return {"status": "noop", "job": job_to_dict(job)}
job.status = "canceled"
db.commit()
return {"status": "success", "job": job_to_dict(job)}
__all__ = ["router"]

172
ontology_platform/ont_platform/api/routes/extraction.py
View File

@@ -1,71 +1,159 @@
"""
Phase 0 Extraction routes: Fast JSON Extraction MVP.
"""Phase 1 URL/HTML ingestion routes."""
No database storage - just extract and return JSON candidates.
Goal: 10-30 seconds per URL.
"""
from __future__ import annotations
from fastapi import APIRouter, HTTPException
import time
from typing import Annotated
from fastapi import APIRouter, Body, HTTPException, Query
from pydantic import BaseModel, Field, model_validator
from ont_platform.core.extractors.web_extractor import extract_web_content
from ont_platform.core.extraction.lightweight_extractor import LightweightExtractor
from ont_platform.core.extractors.web_extractor import extract_web_content
from ont_platform.models.content_unit import PlatformContentUnit
from ont_platform.storage.dedup_cache import get_default_dedup_cache
router = APIRouter(prefix="/api/v1", tags=["extraction"])
router = APIRouter(tags=["extraction"])
@router.post("/extract/url")
async def extract_url(url: str):
"""
Extract candidates from URL (Phase 0 MVP).
class UrlIngestRequest(BaseModel):
"""URL/HTML request accepted by Phase 1 ingestion endpoints."""
Returns:
{
"url": "...",
"title": "...",
"entities": [...],
"relations": [...],
"extraction_time_sec": 0.5,
"warnings": [...]
}
"""
if not url:
raise HTTPException(status_code=400, detail="url is required")
url: str | None = None
html: str | None = None
project_id: str = "default"
language: str | None = None
skip_if_duplicate: bool = True
ontology_user_instruction: str = ""
facts_user_instruction: str = ""
@model_validator(mode="after")
def _require_url_or_html(self) -> UrlIngestRequest:
if not self.url and not self.html:
raise ValueError("url or html is required")
return self
class UrlIngestResponse(BaseModel):
status: str
url: str | None
title: str | None
author: str | None
published_date: str | None
language: str | None
text_length: int
source_document: dict = Field(default_factory=dict)
evidence_spans: list[dict] = Field(default_factory=list)
content_unit: dict = Field(default_factory=dict)
dedup: dict = Field(default_factory=dict)
entities: list = Field(default_factory=list)
relations: list = Field(default_factory=list)
extraction_time_sec: float
entity_count: int
relation_count: int
warnings: list[str] = Field(default_factory=list)
def _build_payload(
payload: UrlIngestRequest | None,
query_url: str | None,
query_project_id: str,
) -> UrlIngestRequest:
if payload is None:
try:
return UrlIngestRequest(url=query_url, project_id=query_project_id)
except ValueError as exc:
raise HTTPException(status_code=400, detail=str(exc)) from exc
data = payload.model_dump()
if query_url:
data["url"] = query_url
if query_project_id != "default" and payload.project_id == "default":
data["project_id"] = query_project_id
try:
return UrlIngestRequest(**data)
except ValueError as exc:
raise HTTPException(status_code=400, detail=str(exc)) from exc
@router.post("/api/v1/extract/url", response_model=UrlIngestResponse)
@router.post("/api/v1/process/url", response_model=UrlIngestResponse)
@router.post("/process/url", response_model=UrlIngestResponse)
async def extract_url(
payload: Annotated[UrlIngestRequest | None, Body()] = None,
url: Annotated[str | None, Query()] = None,
project_id: Annotated[str, Query()] = "default",
):
"""Extract a SourceDocument-ready payload from URL or supplied HTML."""
request = _build_payload(payload, query_url=url, query_project_id=project_id)
start_time = time.time()
try:
# Step 1: Extract web content with Trafilatura
extracted = extract_web_content(url=url)
# Step 2: Extract JSON candidates with lightweight extractor
lightweight = LightweightExtractor(use_llm=False)
candidates = lightweight.extract(
text=extracted.text,
project_id="default", # Phase 0: no projects yet
document_id="temp",
extracted = extract_web_content(
html=request.html,
url=request.url,
lang=request.language,
)
source_document = extracted.to_source_document_dict(project_id=request.project_id)
evidence_spans = [
span.to_dict()
for span in extracted.evidence_spans(
project_id=request.project_id,
document_id=source_document["id"],
)
]
content_unit = PlatformContentUnit.from_extracted(extracted).to_dict()
dedup_result = get_default_dedup_cache().check_and_remember(
project_id=request.project_id,
document_id=source_document["id"],
content_hash=extracted.content_hash,
fingerprint=extracted.fingerprint,
)
entities: list = []
relations: list = []
warnings: list[str] = []
if dedup_result.is_duplicate and request.skip_if_duplicate:
warnings.append("Duplicate source document skipped by fingerprint.")
else:
lightweight = LightweightExtractor(use_llm=False)
candidates = lightweight.extract(
text=extracted.text,
project_id=request.project_id,
document_id=source_document["id"],
)
entities = candidates.entities
relations = candidates.relations
warnings = candidates.warnings
extraction_time = time.time() - start_time
# Return just the JSON (entities/relations are already dicts)
return {
"url": url,
"status": "success",
"url": request.url,
"title": extracted.title,
"author": extracted.author,
"published_date": extracted.publish_date,
"language": extracted.language,
"text_length": len(extracted.text),
"entities": candidates.entities,
"relations": candidates.relations,
"source_document": source_document,
"evidence_spans": evidence_spans,
"content_unit": content_unit,
"dedup": dedup_result.to_dict(),
"entities": entities,
"relations": relations,
"extraction_time_sec": round(extraction_time, 2),
"entity_count": len(candidates.entities),
"relation_count": len(candidates.relations),
"warnings": candidates.warnings,
"entity_count": len(entities),
"relation_count": len(relations),
"warnings": warnings,
}
except Exception as e:
raise HTTPException(status_code=500, detail=f"Extraction failed: {str(e)}")
except HTTPException:
raise
except Exception as exc:
raise HTTPException(status_code=500, detail=f"Extraction failed: {exc}") from exc
__all__ = ["router"]

79
ontology_platform/ont_platform/api/routes/graph.py Normal file
View File

@@ -0,0 +1,79 @@
"""Phase 5 projection and GraphRAG search routes."""
from __future__ import annotations
from typing import Annotated, Any
from fastapi import APIRouter, Body, Depends, HTTPException, Query
from pydantic import BaseModel, Field
from sqlalchemy.orm import Session
from ont_platform.api.db_deps import get_db
from ont_platform.config import load_settings
from ont_platform.core.graph.cypher_guard import ReadOnlyCypherGuard, UnsafeCypherError
from ont_platform.core.graph.search import CandidateGraphSearchService
from ont_platform.core.projection.rdf_to_neo4j import RDFToNeo4jProjector
router = APIRouter(prefix="/api/v1/graph", tags=["graph"])
class ProjectionPreviewRequest(BaseModel):
project_id: str = "default"
triples: list[tuple[str, str, str]]
provenance: dict[str, Any] = Field(default_factory=dict)
class ReadOnlyCypherRequest(BaseModel):
query: str
limit: int | None = None
@router.post("/projection/preview")
async def preview_projection(request: Annotated[ProjectionPreviewRequest, Body()]) -> dict:
projector = RDFToNeo4jProjector(project_id=request.project_id)
result = await projector.preview_projection(
request.triples,
provenance=request.provenance,
)
return {"status": "success", "projection": result.to_dict()}
@router.post("/cypher/read")
def sanitize_read_only_cypher(request: Annotated[ReadOnlyCypherRequest, Body()]) -> dict:
settings = load_settings()
guard = ReadOnlyCypherGuard(max_limit=settings.text2cypher_result_limit)
try:
sanitized = guard.sanitize(request.query, limit=request.limit)
except UnsafeCypherError as exc:
raise HTTPException(status_code=400, detail=str(exc)) from exc
return {
"status": "success",
"read_only": True,
"query": sanitized.query,
"limit": sanitized.limit,
}
@router.get("/search")
def search_graph(
db: Annotated[Session, Depends(get_db)],
q: Annotated[str, Query(min_length=1)],
project_id: Annotated[str, Query()] = "default",
limit: Annotated[int, Query(ge=1, le=100)] = 20,
) -> dict:
settings = load_settings()
effective_limit = min(limit, settings.graph_search_result_limit)
results = CandidateGraphSearchService(db).search(
project_id=project_id,
query=q,
limit=effective_limit,
)
return {
"status": "success",
"query": q,
"result_count": len(results),
"results": [result.to_dict() for result in results],
}
__all__ = ["router"]

115
ontology_platform/ont_platform/api/routes/maintenance.py Normal file
View File

@@ -0,0 +1,115 @@
"""Phase 6 maintenance loop routes."""
from __future__ import annotations
from typing import Annotated
from fastapi import APIRouter, Body, Depends, HTTPException, Query
from pydantic import BaseModel, Field
from sqlalchemy.orm import Session
from ont_platform.api.db_deps import get_db
from ont_platform.core.maintenance import (
MaintenanceLoopService,
MaintenancePermissionError,
MaintenanceProposalNotFoundError,
maintenance_proposal_to_dict,
maintenance_run_to_dict,
)
from ont_platform.storage.models import MaintenanceProposalStatus
router = APIRouter(prefix="/api/v1/maintenance", tags=["maintenance"])
class MaintenanceRunRequest(BaseModel):
project_id: str = "default"
requested_by: str = "system"
actor_role: str = "admin"
low_confidence_threshold: float = Field(default=0.65, ge=0.0, le=1.0)
class ProposalReviewRequest(BaseModel):
reviewed_by: str
actor_role: str = "admin"
approve: bool
reason: str | None = None
@router.post("/runs")
async def start_maintenance_run(
request: Annotated[MaintenanceRunRequest, Body()],
db: Annotated[Session, Depends(get_db)],
) -> dict:
service = MaintenanceLoopService(db)
try:
run = await service.run(
project_id=request.project_id,
requested_by=request.requested_by,
actor_role=request.actor_role,
low_confidence_threshold=request.low_confidence_threshold,
)
except MaintenancePermissionError as exc:
raise HTTPException(status_code=403, detail=str(exc)) from exc
except Exception as exc:
db.commit()
raise HTTPException(status_code=500, detail=f"Maintenance run failed: {exc}") from exc
db.commit()
return {"status": "success", "run": maintenance_run_to_dict(run)}
@router.get("/runs")
def list_maintenance_runs(
db: Annotated[Session, Depends(get_db)],
project_id: Annotated[str, Query()] = "default",
limit: Annotated[int, Query(ge=1, le=100)] = 50,
) -> dict:
service = MaintenanceLoopService(db)
runs = service.list_runs(project_id=project_id, limit=limit)
return {"runs": [maintenance_run_to_dict(run) for run in runs]}
@router.get("/proposals")
def list_maintenance_proposals(
db: Annotated[Session, Depends(get_db)],
project_id: Annotated[str, Query()] = "default",
status: Annotated[str | None, Query()] = None,
limit: Annotated[int, Query(ge=1, le=200)] = 100,
) -> dict:
service = MaintenanceLoopService(db)
try:
proposals = service.list_proposals(
project_id=project_id,
status=MaintenanceProposalStatus(status) if status else None,
limit=limit,
)
except ValueError as exc:
raise HTTPException(status_code=400, detail=f"Invalid proposal status: {status}") from exc
return {"proposals": [maintenance_proposal_to_dict(proposal) for proposal in proposals]}
@router.post("/proposals/{proposal_id}/review")
async def review_maintenance_proposal(
proposal_id: str,
request: Annotated[ProposalReviewRequest, Body()],
db: Annotated[Session, Depends(get_db)],
) -> dict:
service = MaintenanceLoopService(db)
try:
proposal = await service.review_proposal(
proposal_id=proposal_id,
reviewed_by=request.reviewed_by,
actor_role=request.actor_role,
approve=request.approve,
reason=request.reason,
)
except MaintenancePermissionError as exc:
raise HTTPException(status_code=403, detail=str(exc)) from exc
except MaintenanceProposalNotFoundError as exc:
raise HTTPException(status_code=404, detail=str(exc)) from exc
except ValueError as exc:
raise HTTPException(status_code=400, detail=str(exc)) from exc
db.commit()
return {"status": "success", "proposal": maintenance_proposal_to_dict(proposal)}
__all__ = ["router"]

335
ontology_platform/ont_platform/api/routes/review.py Normal file
View File

@@ -0,0 +1,335 @@
"""Phase 2 candidate review queue API."""
from __future__ import annotations
from typing import Annotated, Any
from fastapi import APIRouter, Body, Depends, HTTPException, Query
from pydantic import BaseModel, Field
from sqlalchemy.orm import Session
from ont_platform.api.db_deps import get_db
from ont_platform.core.review import CandidatePromotionService, ReviewService
from ont_platform.core.review.review_service import (
EvidenceRequiredError,
InvalidReviewTransitionError,
review_decision_to_dict,
)
from ont_platform.storage.candidate_repository import CandidateNotFoundError, CandidateRepository
from ont_platform.storage.models import CandidateEntity, CandidateKind, CandidateRelation
router = APIRouter(prefix="/api/v1/review", tags=["review"])
class CandidateIngestRequest(BaseModel):
project_id: str = "default"
document_id: str
entities: list[dict[str, Any]] = Field(default_factory=list)
relations: list[dict[str, Any]] = Field(default_factory=list)
evidence_spans: list[dict[str, Any]] = Field(default_factory=list)
source_trust: float = Field(default=0.5, ge=0.0, le=1.0)
validation_passed: bool = True
validation_errors: list[str] = Field(default_factory=list)
validation_issues: list[dict[str, Any]] = Field(default_factory=list)
class ReviewDecisionRequest(BaseModel):
reviewed_by: str = "user"
reason: str | None = None
class BulkApproveRequest(ReviewDecisionRequest):
candidate_kind: CandidateKind
candidate_ids: list[str]
@router.post("/ingest/lightweight")
def ingest_lightweight_candidates(
request: CandidateIngestRequest,
db: Annotated[Session, Depends(get_db)],
) -> dict[str, Any]:
repository = CandidateRepository(db)
batch = repository.save_lightweight_result(
project_id=request.project_id,
document_id=request.document_id,
result=request.model_dump(),
source_trust=request.source_trust,
validation_passed=request.validation_passed,
)
db.commit()
return {"status": "success", "source_type": "lightweight", **batch.to_dict()}
@router.post("/ingest/ontocast")
def ingest_ontocast_candidates(
request: CandidateIngestRequest,
db: Annotated[Session, Depends(get_db)],
) -> dict[str, Any]:
repository = CandidateRepository(db)
batch = repository.save_ontocast_result(
project_id=request.project_id,
document_id=request.document_id,
result=request.model_dump(),
source_trust=request.source_trust,
validation_passed=request.validation_passed,
)
db.commit()
return {"status": "success", "source_type": "ontocast", **batch.to_dict()}
@router.get("/candidates")
def list_candidates(
db: Annotated[Session, Depends(get_db)],
project_id: Annotated[str, Query()] = "default",
status: Annotated[str | None, Query()] = None,
source_type: Annotated[str | None, Query()] = None,
) -> dict[str, Any]:
repository = CandidateRepository(db)
candidates = repository.list_candidates(
project_id=project_id,
status=status,
source_type=source_type,
)
return {
"entities": [_candidate_to_dict(entity, CandidateKind.ENTITY) for entity in candidates["entities"]],
"relations": [
_candidate_to_dict(relation, CandidateKind.RELATION)
for relation in candidates["relations"]
],
}
@router.get("/validation/issues")
def list_validation_issues(
db: Annotated[Session, Depends(get_db)],
project_id: Annotated[str, Query()] = "default",
document_id: Annotated[str | None, Query()] = None,
candidate_id: Annotated[str | None, Query()] = None,
) -> dict[str, Any]:
repository = CandidateRepository(db)
issues = repository.list_validation_issues(
project_id=project_id,
document_id=document_id,
candidate_id=candidate_id,
)
return {"issues": [_validation_issue_to_dict(issue) for issue in issues]}
@router.get("/candidates/{candidate_kind}/{candidate_id}")
def get_candidate_detail(
candidate_kind: CandidateKind,
candidate_id: str,
db: Annotated[Session, Depends(get_db)],
) -> dict[str, Any]:
repository = CandidateRepository(db)
try:
candidate = repository.get_candidate(
candidate_kind=candidate_kind,
candidate_id=candidate_id,
)
except CandidateNotFoundError as exc:
raise HTTPException(status_code=404, detail=str(exc)) from exc
return {
"candidate": _candidate_to_dict(candidate, candidate_kind),
"history": [
review_decision_to_dict(decision)
for decision in repository.review_history(
candidate_kind=candidate_kind,
candidate_id=candidate_id,
)
],
}
@router.post("/candidates/{candidate_kind}/{candidate_id}/approve")
def approve_candidate(
candidate_kind: CandidateKind,
candidate_id: str,
request: Annotated[ReviewDecisionRequest, Body()],
db: Annotated[Session, Depends(get_db)],
) -> dict[str, Any]:
decision = _apply_review_decision(
db=db,
action="approve",
candidate_kind=candidate_kind,
candidate_id=candidate_id,
reviewed_by=request.reviewed_by,
reason=request.reason,
)
db.commit()
return {"status": "success", "decision": review_decision_to_dict(decision)}
@router.post("/candidates/{candidate_kind}/{candidate_id}/auto-approve")
def auto_approve_candidate(
candidate_kind: CandidateKind,
candidate_id: str,
db: Annotated[Session, Depends(get_db)],
request: Annotated[ReviewDecisionRequest, Body()],
) -> dict[str, Any]:
decision = _apply_review_decision(
db=db,
action="auto_approve",
candidate_kind=candidate_kind,
candidate_id=candidate_id,
reviewed_by=request.reviewed_by,
reason=request.reason,
)
db.commit()
return {"status": "success", "decision": review_decision_to_dict(decision)}
@router.post("/candidates/{candidate_kind}/{candidate_id}/reject")
def reject_candidate(
candidate_kind: CandidateKind,
candidate_id: str,
request: Annotated[ReviewDecisionRequest, Body()],
db: Annotated[Session, Depends(get_db)],
) -> dict[str, Any]:
decision = _apply_review_decision(
db=db,
action="reject",
candidate_kind=candidate_kind,
candidate_id=candidate_id,
reviewed_by=request.reviewed_by,
reason=request.reason,
)
db.commit()
return {"status": "success", "decision": review_decision_to_dict(decision)}
@router.post("/candidates/bulk-approve")
def bulk_approve_candidates(
request: BulkApproveRequest,
db: Annotated[Session, Depends(get_db)],
) -> dict[str, Any]:
repository = CandidateRepository(db)
service = ReviewService(repository)
try:
decisions = service.bulk_approve(
candidate_kind=request.candidate_kind,
candidate_ids=request.candidate_ids,
reviewed_by=request.reviewed_by,
reason=request.reason,
)
except (CandidateNotFoundError, EvidenceRequiredError, InvalidReviewTransitionError) as exc:
raise HTTPException(status_code=400, detail=str(exc)) from exc
db.commit()
return {
"status": "success",
"decisions": [review_decision_to_dict(decision) for decision in decisions],
}
@router.post("/promote")
def build_promotion_plan(
db: Annotated[Session, Depends(get_db)],
project_id: Annotated[str, Query()] = "default",
mark_promoted: Annotated[bool, Query()] = False,
) -> dict[str, Any]:
repository = CandidateRepository(db)
plan = CandidatePromotionService(repository).build_commit_plan(
project_id=project_id,
mark_promoted=mark_promoted,
)
if mark_promoted:
db.commit()
return {"status": "success", "promotion_plan": plan.to_dict()}
def _apply_review_decision(
*,
db: Session,
action: str,
candidate_kind: CandidateKind,
candidate_id: str,
reviewed_by: str,
reason: str | None,
):
repository = CandidateRepository(db)
service = ReviewService(repository)
try:
if action == "approve":
return service.approve(
candidate_kind=candidate_kind,
candidate_id=candidate_id,
reviewed_by=reviewed_by,
reason=reason,
)
if action == "auto_approve":
return service.auto_approve(
candidate_kind=candidate_kind,
candidate_id=candidate_id,
reviewed_by=reviewed_by,
reason=reason,
)
return service.reject(
candidate_kind=candidate_kind,
candidate_id=candidate_id,
reviewed_by=reviewed_by,
reason=reason,
)
except CandidateNotFoundError as exc:
raise HTTPException(status_code=404, detail=str(exc)) from exc
except EvidenceRequiredError as exc:
raise HTTPException(status_code=422, detail=str(exc)) from exc
except InvalidReviewTransitionError as exc:
raise HTTPException(status_code=400, detail=str(exc)) from exc
def _candidate_to_dict(
candidate: CandidateEntity | CandidateRelation,
candidate_kind: CandidateKind,
) -> dict[str, Any]:
data = {
"id": candidate.id,
"candidate_kind": candidate_kind.value,
"project_id": candidate.project_id,
"document_id": candidate.document_id,
"source_type": candidate.source_type.value,
"created_by": candidate.created_by,
"confidence": candidate.confidence,
"source_trust": candidate.source_trust,
"validation_passed": candidate.validation_passed,
"evidence_ids": candidate.evidence_ids or [],
"review_status": candidate.review_status.value,
"reviewed_by": candidate.reviewed_by,
"review_reason": candidate.review_reason,
"metadata": candidate.metadata_ or {},
}
if isinstance(candidate, CandidateEntity):
data.update(
{
"label": candidate.label,
"entity_type": candidate.entity_type,
"description": candidate.description,
}
)
else:
data.update(
{
"source_entity_id": candidate.source_entity_id,
"predicate": candidate.predicate,
"target_entity_id": candidate.target_entity_id,
}
)
return data
def _validation_issue_to_dict(issue) -> dict[str, Any]:
return {
"id": issue.id,
"project_id": issue.project_id,
"document_id": issue.document_id,
"candidate_id": issue.candidate_id,
"candidate_kind": issue.candidate_kind.value if issue.candidate_kind else None,
"severity": issue.severity.value,
"code": issue.code,
"message": issue.message,
"source": issue.source,
"metadata": issue.metadata_ or {},
"created_at": issue.created_at.isoformat() if issue.created_at else None,
}
__all__ = ["router"]

5
ontology_platform/ont_platform/auth/rbac.py
View File

@@ -45,6 +45,8 @@ class Permission(str, Enum):
VIEW_AUDIT_LOG = "view:audit"
VIEW_BILLING = "view:billing"
MANAGE_ORGANIZATION = "manage:org"
RUN_MAINTENANCE = "run:maintenance"
APPROVE_MAINTENANCE = "approve:maintenance"
class RBAC:
@@ -70,6 +72,8 @@ class RBAC:
Permission.VIEW_AUDIT_LOG,
Permission.VIEW_BILLING,
Permission.MANAGE_ORGANIZATION,
Permission.RUN_MAINTENANCE,
Permission.APPROVE_MAINTENANCE,
},
Role.EDITOR: {
# 읽기, 쓰기, 분석
@@ -85,6 +89,7 @@ class RBAC:
Permission.VIEW_ANALYTICS,
Permission.RUN_LLM_QUERY,
Permission.VIEW_BILLING,
Permission.RUN_MAINTENANCE,
},
Role.VIEWER: {
# 읽기, 분석, LLM만

7
ontology_platform/ont_platform/billing/calculator.py
View File

@@ -243,13 +243,14 @@ class CostCalculator:
예측 정보
"""
if days_into_month is None:
days_into_month = datetime.utcnow().day
days_into_month = datetime.now(UTC).day
# 현재 월 사용량
cutoff_time = datetime(
datetime.utcnow().year,
datetime.utcnow().month,
datetime.now(UTC).year,
datetime.now(UTC).month,
1,
tzinfo=UTC,
)
current_month_usages = [

32
ontology_platform/ont_platform/config.py
View File

@@ -58,10 +58,11 @@ class Phase(IntEnum):
BASE = 0 # OntoCast only, filesystem storage
TRAFILATURA = 1
CRAWL4AI = 2
GUARDRAILS = 3
NEO4J_GRAPHRAG = 4
MULTI_AGENT = 5
CANDIDATE_REVIEW = 2
CRAWL4AI = 3
GUARDRAILS = 4
NEO4J_GRAPHRAG = 5
MULTI_AGENT = 6
StorageBackend = Literal["filesystem", "fuseki", "neo4j"]
@@ -94,6 +95,10 @@ class PlatformSettings(BaseSettings):
"regardless of any Neo4j/Fuseki credentials in the environment."
),
)
database_url: str = Field(
default="sqlite:///./data/ontology_platform.db",
description="SQLAlchemy database URL for Phase 2 candidate/review storage.",
)
# ─── Paths (mirror ONTOCAST_* but with platform defaults) ────────
working_directory: Path = Field(
@@ -116,12 +121,24 @@ class PlatformSettings(BaseSettings):
default="respect",
description="robots.txt 준수 정책. Phase 2 Crawl4AI 통합에서 사용.",
)
crawler_default_profile: Literal[
"fast_static",
"dynamic_page",
"full_capture",
"structured_extract",
"deep_discovery",
] = Field(default="fast_static")
crawler_cache_policy: Literal["enabled", "disabled", "bypass"] = Field(default="enabled")
crawler_max_pages: int = Field(default=50, ge=1, le=50)
crawler_max_depth: int = Field(default=1, ge=0, le=5)
text2cypher_result_limit: int = Field(default=100, ge=1, le=1000)
graph_search_result_limit: int = Field(default=20, ge=1, le=100)
daily_llm_call_limit: int = Field(default=10_000)
daily_llm_token_limit: int = Field(default=10_000_000)
# ─── Phase 0 enforcement ─────────────────────────────────────────
@model_validator(mode="after")
def _enforce_phase_storage_consistency(self) -> "PlatformSettings":
def _enforce_phase_storage_consistency(self) -> PlatformSettings:
"""Phase 0 forces filesystem; later phases may opt into other backends.
Anything other than 'filesystem' before Phase 4 is treated as a
@@ -130,7 +147,8 @@ class PlatformSettings(BaseSettings):
"""
if self.phase < Phase.NEO4J_GRAPHRAG and self.storage_backend != "filesystem":
raise ValueError(
f"storage_backend={self.storage_backend!r} requires Phase 4+, "
f"storage_backend={self.storage_backend!r} requires Phase 4+/Phase 5+ "
f"(Phase 5 in the current roadmap), "
f"but PHASE={int(self.phase)}. See docs/통합설계서.md §5."
)
# Ensure working directory exists for filesystem mode.
@@ -248,10 +266,10 @@ def load_settings() -> PlatformSettings:
# Provider-grade LLM/embedding config helpers can be added in later phases.
# For Phase 0, OntoCast's own ``LLMConfig`` is sufficient.
__all__ = [
"LLMConfig",
"Phase",
"PlatformSettings",
"StorageBackend",
"build_ontocast_config",
"load_settings",
"LLMConfig",
]

16
ontology_platform/ont_platform/core/crawler/__init__.py
View File

@@ -1,17 +1,25 @@
"""Web crawler module (Phase 0 onwards)."""
from .crawl4ai_adapter import (
Crawl4AIAdapter,
BasicCrawler,
CrawlerConfig,
CachePolicy,
Crawl4AIAdapter,
CrawlBatchResult,
CrawlProfile,
CrawlResult,
CrawlerConfig,
RobotsPolicy,
crawl_url,
)
__all__ = [
"Crawl4AIAdapter",
"BasicCrawler",
"CrawlerConfig",
"CachePolicy",
"Crawl4AIAdapter",
"CrawlBatchResult",
"CrawlProfile",
"CrawlResult",
"CrawlerConfig",
"RobotsPolicy",
"crawl_url",
]

450
ontology_platform/ont_platform/core/crawler/crawl4ai_adapter.py
View File

@@ -1,281 +1,307 @@
"""
Crawl4AI adapter for Phase 2+ (dynamic page support).
"""Crawl4AI acquisition adapter with static fallback.
Phase 0-1: HTTP fetch + Trafilatura (BasicCrawler)
Phase 2+: Crawl4AI for dynamic/JS-heavy pages with profile selection
This adapter provides unified interface with intelligent profile selection.
The platform treats Crawl4AI as an optional acquisition engine. Importing this
module must not require Crawl4AI to be installed; dynamic profiles try to load
it lazily and fall back to the basic HTTP crawler when it is unavailable.
"""
from __future__ import annotations
import asyncio
import logging
from enum import Enum
from typing import Optional, Literal
from dataclasses import dataclass, field
from enum import StrEnum
from typing import Any
from urllib.parse import urljoin, urlparse
from urllib.robotparser import RobotFileParser
import requests
from crawl4ai import AsyncWebCrawler, CacheMode, CrawlerRunConfig
from bs4 import BeautifulSoup
logger = logging.getLogger(__name__)
class CrawlProfile(str, Enum):
"""Crawl4AI profile selection (Phase 2+)."""
class CrawlProfile(StrEnum):
"""Supported acquisition profiles."""
FAST_STATIC = "fast_static" # HTTP only, Trafilatura post-process
DYNAMIC_PAGE = "dynamic_page" # Playwright + JS wait
FULL_CAPTURE = "full_capture" # screenshot/PDF/MHTML
STRUCTURED_EXTRACT = "structured_extract" # CSS/XPath schema
DEEP_DISCOVERY = "deep_discovery" # URL Seeder + BFS/DFS
FAST_STATIC = "fast_static"
DYNAMIC_PAGE = "dynamic_page"
FULL_CAPTURE = "full_capture"
STRUCTURED_EXTRACT = "structured_extract"
DEEP_DISCOVERY = "deep_discovery"
class RobotsPolicy(StrEnum):
STRICT = "strict"
RESPECT = "respect"
IGNORE = "ignore"
class CachePolicy(StrEnum):
ENABLED = "enabled"
DISABLED = "disabled"
BYPASS = "bypass"
@dataclass
class CrawlResult:
"""Result of a crawl operation."""
"""Result of one fetched page."""
def __init__(
self,
url: str,
html: str,
status_code: int = 200,
headers: Optional[dict] = None,
markdown: Optional[str] = None,
profile_used: Optional[str] = None,
):
self.url = url
self.html = html
self.status_code = status_code
self.headers = headers or {}
self.markdown = markdown
self.profile_used = profile_used
url: str
html: str
status_code: int = 200
headers: dict[str, str] = field(default_factory=dict)
markdown: str | None = None
profile_used: str | None = None
requested_profile: str | None = None
metadata: dict[str, Any] = field(default_factory=dict)
@dataclass
class CrawlBatchResult:
"""Result of a seed crawl or deep-discovery job."""
seed_url: str
pages: list[CrawlResult] = field(default_factory=list)
discovered_urls: list[str] = field(default_factory=list)
warnings: list[str] = field(default_factory=list)
@property
def page_count(self) -> int:
return len(self.pages)
@dataclass
class CrawlerConfig:
"""Configuration for crawler."""
"""Configuration for crawler behavior."""
def __init__(
self,
timeout: int = 15,
user_agent: Optional[str] = None,
follow_redirects: bool = True,
cache_mode: CacheMode = CacheMode.ENABLED,
check_cache_freshness: bool = True,
):
self.timeout = timeout
self.user_agent = user_agent or (
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
"AppleWebKit/537.36 (KHTML, like Gecko) "
"Chrome/120.0.0.0 Safari/537.36"
)
self.follow_redirects = follow_redirects
self.cache_mode = cache_mode
self.check_cache_freshness = check_cache_freshness
timeout: int = 15
user_agent: str = (
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
"AppleWebKit/537.36 (KHTML, like Gecko) "
"Chrome/120.0.0.0 Safari/537.36"
)
follow_redirects: bool = True
cache_policy: CachePolicy = CachePolicy.ENABLED
robots_policy: RobotsPolicy = RobotsPolicy.RESPECT
default_profile: CrawlProfile = CrawlProfile.FAST_STATIC
max_pages: int = 50
max_depth: int = 1
class BasicCrawler:
"""Phase 0-1: Basic HTTP crawler (fallback for dynamic_page errors)."""
"""HTTP crawler used for static pages and fallback paths."""
def __init__(self, config: Optional[CrawlerConfig] = None):
"""Initialize crawler with optional config."""
def __init__(self, config: CrawlerConfig | None = None):
self.config = config or CrawlerConfig()
self.session = requests.Session()
self.session.headers.update({"User-Agent": self.config.user_agent})
def fetch(self, url: str) -> CrawlResult:
"""
Fetch URL content using basic HTTP.
Args:
url: URL to fetch
Returns:
CrawlResult with HTML content
"""
try:
response = self.session.get(
url,
timeout=self.config.timeout,
allow_redirects=self.config.follow_redirects,
)
response.raise_for_status()
return CrawlResult(
url=response.url,
html=response.text,
status_code=response.status_code,
headers=dict(response.headers),
profile_used="basic_http",
)
except requests.RequestException as e:
logger.error(f"Failed to fetch {url}: {e}")
raise
self._enforce_robots(url)
response = self.session.get(
url,
timeout=self.config.timeout,
allow_redirects=self.config.follow_redirects,
)
response.raise_for_status()
return CrawlResult(
url=response.url,
html=response.text,
status_code=response.status_code,
headers=dict(response.headers),
profile_used="basic_http",
requested_profile=CrawlProfile.FAST_STATIC.value,
)
async def fetch_async(self, url: str) -> CrawlResult:
"""Async wrapper for fetch."""
return await asyncio.to_thread(self.fetch, url)
def close(self):
"""Close session resources."""
def extract_links(self, html: str, base_url: str) -> list[str]:
parsed_base = urlparse(base_url)
urls: list[str] = []
for anchor in BeautifulSoup(html, "html.parser").find_all("a", href=True):
candidate = urljoin(base_url, anchor["href"])
parsed = urlparse(candidate)
if parsed.scheme not in {"http", "https"}:
continue
if parsed.netloc != parsed_base.netloc:
continue
normalized = parsed._replace(fragment="", query="").geturl()
if normalized not in urls:
urls.append(normalized)
return urls
def close(self) -> None:
self.session.close()
def _enforce_robots(self, url: str) -> None:
if self.config.robots_policy == RobotsPolicy.IGNORE:
return
parsed = urlparse(url)
robots_url = f"{parsed.scheme}://{parsed.netloc}/robots.txt"
parser = RobotFileParser()
parser.set_url(robots_url)
try:
parser.read()
except Exception as exc: # noqa: BLE001 - robots failures are policy-dependent.
if self.config.robots_policy == RobotsPolicy.STRICT:
raise PermissionError(f"robots.txt could not be read for {url}: {exc}") from exc
logger.info("robots.txt unavailable for %s; continuing with respect policy", url)
return
if not parser.can_fetch(self.config.user_agent, url):
raise PermissionError(f"robots.txt disallows fetching {url}")
class Crawl4AIAdapter:
"""
Unified adapter for crawling with intelligent profile selection.
"""Unified acquisition adapter for Phase 3 jobs."""
Phase 2+: Uses Crawl4AI with fallback to BasicCrawler.
"""
def __init__(self, config: Optional[CrawlerConfig] = None):
"""Initialize adapter."""
def __init__(self, config: CrawlerConfig | None = None):
self.config = config or CrawlerConfig()
self.basic_crawler = BasicCrawler(self.config)
self.crawl4ai: Optional[AsyncWebCrawler] = None
async def _get_crawl4ai(self) -> AsyncWebCrawler:
"""Lazy-initialize Crawl4AI crawler."""
if self.crawl4ai is None:
self.crawl4ai = AsyncWebCrawler(
cache_mode=self.config.cache_mode,
)
return self.crawl4ai
def _select_profile(self, url: str) -> CrawlProfile:
"""
Intelligent profile selection based on URL characteristics.
Phase 2 decision rules:
- If domain is known JS-heavy → dynamic_page
- If URL has sitemap → deep_discovery (not yet)
- Default → fast_static (HTTP only)
"""
# TODO: Implement domain detection (robots.txt, Known JS-heavy list)
# For Phase 2 MVP: use fast_static by default
return CrawlProfile.FAST_STATIC
self._crawl4ai: Any | None = None
async def crawl(
self,
url: str,
profile: Optional[CrawlProfile] = None,
profile: CrawlProfile | str | None = None,
) -> CrawlResult:
"""
Crawl URL content with optional profile override.
selected_profile = CrawlProfile(profile) if profile else self.config.default_profile
Phase 2: Automatic profile selection + Crawl4AI support.
if selected_profile == CrawlProfile.FAST_STATIC:
result = await self.basic_crawler.fetch_async(url)
result.requested_profile = selected_profile.value
return result
Args:
url: URL to crawl
profile: Optional profile override
Returns:
CrawlResult with content (HTML + optional markdown)
"""
# Select profile
selected_profile = profile or self._select_profile(url)
if selected_profile == CrawlProfile.DEEP_DISCOVERY:
batch = await self.crawl_seed(url, profile=CrawlProfile.DEEP_DISCOVERY)
if not batch.pages:
raise RuntimeError(f"No pages fetched for {url}")
return batch.pages[0]
try:
if selected_profile == CrawlProfile.FAST_STATIC:
# Phase 0-1: Use BasicCrawler for static content
return await self.basic_crawler.fetch_async(url)
return await self._crawl_with_crawl4ai(url, selected_profile)
except ModuleNotFoundError:
logger.info("Crawl4AI is not installed; falling back to static HTTP for %s", url)
except Exception as exc: # noqa: BLE001 - acquisition fallback is intentional.
logger.warning("Crawl4AI %s failed for %s: %s", selected_profile.value, url, exc)
elif selected_profile == CrawlProfile.DYNAMIC_PAGE:
# Phase 2: Use Crawl4AI for JS-rendered content
return await self._crawl_dynamic(url)
result = await self.basic_crawler.fetch_async(url)
result.requested_profile = selected_profile.value
result.metadata["fallback_from"] = selected_profile.value
return result
elif selected_profile == CrawlProfile.FULL_CAPTURE:
return await self._crawl_full_capture(url)
async def crawl_seed(
self,
seed_url: str,
*,
profile: CrawlProfile | str | None = None,
max_pages: int | None = None,
max_depth: int | None = None,
) -> CrawlBatchResult:
"""Fetch a seed URL and optionally same-domain links up to limits."""
elif selected_profile == CrawlProfile.DEEP_DISCOVERY:
# Phase 2+: Not yet implemented
logger.warning(f"deep_discovery not yet implemented, using fast_static for {url}")
return await self.basic_crawler.fetch_async(url)
selected_profile = CrawlProfile(profile) if profile else self.config.default_profile
page_limit = min(max_pages or self.config.max_pages, 50)
depth_limit = max_depth if max_depth is not None else self.config.max_depth
batch = CrawlBatchResult(seed_url=seed_url)
queue: list[tuple[str, int]] = [(seed_url, 0)]
seen: set[str] = set()
else:
# Fallback
return await self.basic_crawler.fetch_async(url)
except Exception as e:
logger.error(f"Crawl failed with profile {selected_profile}: {e}")
# Fallback to basic HTTP
while queue and len(batch.pages) < page_limit:
url, depth = queue.pop(0)
if url in seen:
continue
seen.add(url)
try:
logger.info(f"Falling back to basic HTTP for {url}")
return await self.basic_crawler.fetch_async(url)
except Exception as fallback_err:
logger.error(f"Fallback also failed: {fallback_err}")
raise
result = await self.crawl(
url,
profile=CrawlProfile.FAST_STATIC
if selected_profile == CrawlProfile.DEEP_DISCOVERY
else selected_profile,
)
except Exception as exc: # noqa: BLE001
batch.warnings.append(f"{url}: {exc}")
continue
async def _crawl_dynamic(self, url: str) -> CrawlResult:
"""Crawl JavaScript-rendered page using Crawl4AI + Playwright."""
crawler = await self._get_crawl4ai()
batch.pages.append(result)
if depth >= depth_limit:
continue
config = CrawlerRunConfig(
cache_mode=self.config.cache_mode,
screenshot=False,
markdown_generator=None, # Use default markdown
links = self.basic_crawler.extract_links(result.html, result.url)
for link in links:
if link not in seen and len(seen) + len(queue) < page_limit:
queue.append((link, depth + 1))
batch.discovered_urls.append(link)
return batch
async def _crawl_with_crawl4ai(self, url: str, profile: CrawlProfile) -> CrawlResult:
AsyncWebCrawler, CrawlerRunConfig, CacheMode = _load_crawl4ai()
crawler = await self._get_crawl4ai(AsyncWebCrawler, CacheMode)
run_config = CrawlerRunConfig(
cache_mode=_to_crawl4ai_cache_mode(CacheMode, self.config.cache_policy),
screenshot=profile == CrawlProfile.FULL_CAPTURE,
)
result = await crawler.arun(url, config=run_config)
return CrawlResult(
url=getattr(result, "url", url) or url,
html=getattr(result, "html", None) or "",
status_code=200 if getattr(result, "html", None) else 500,
markdown=getattr(result, "markdown", None),
profile_used=profile.value,
requested_profile=profile.value,
)
try:
result = await crawler.arun(url, config=config)
async def _get_crawl4ai(self, AsyncWebCrawler: Any, CacheMode: Any) -> Any:
if self._crawl4ai is None:
kwargs: dict[str, Any] = {}
cache_mode = _to_crawl4ai_cache_mode(CacheMode, self.config.cache_policy)
if cache_mode is not None:
kwargs["cache_mode"] = cache_mode
self._crawl4ai = AsyncWebCrawler(**kwargs)
return self._crawl4ai
return CrawlResult(
url=url,
html=result.html or "",
status_code=200 if result.html else 500,
markdown=result.markdown,
profile_used="dynamic_page",
)
except Exception as e:
logger.error(f"Crawl4AI dynamic crawl failed for {url}: {e}")
raise
async def _crawl_full_capture(self, url: str) -> CrawlResult:
"""Crawl with full capture (screenshot, PDF, MHTML)."""
crawler = await self._get_crawl4ai()
config = CrawlerRunConfig(
cache_mode=self.config.cache_mode,
screenshot=True, # Capture screenshot
)
try:
result = await crawler.arun(url, config=config)
return CrawlResult(
url=url,
html=result.html or "",
status_code=200 if result.html else 500,
markdown=result.markdown,
profile_used="full_capture",
)
except Exception as e:
logger.error(f"Crawl4AI full capture failed for {url}: {e}")
raise
async def close(self):
"""Clean up resources."""
async def close(self) -> None:
self.basic_crawler.close()
if self.crawl4ai is not None:
await self.crawl4ai.close()
if self._crawl4ai is not None:
await self._crawl4ai.close()
def _load_crawl4ai() -> tuple[Any, Any, Any]:
try:
from crawl4ai import AsyncWebCrawler, CacheMode, CrawlerRunConfig
except ModuleNotFoundError as exc:
raise ModuleNotFoundError("crawl4ai is not installed") from exc
return AsyncWebCrawler, CrawlerRunConfig, CacheMode
def _to_crawl4ai_cache_mode(CacheMode: Any, cache_policy: CachePolicy) -> Any | None:
if cache_policy == CachePolicy.DISABLED:
return getattr(CacheMode, "DISABLED", None)
if cache_policy == CachePolicy.BYPASS:
return getattr(CacheMode, "BYPASS", getattr(CacheMode, "DISABLED", None))
return getattr(CacheMode, "ENABLED", None)
async def crawl_url(url: str) -> CrawlResult:
"""Convenience function for quick crawling."""
adapter = Crawl4AIAdapter()
try:
return await adapter.crawl(url)
finally:
adapter.close()
await adapter.close()
if __name__ == "__main__":
# Simple test
import asyncio
async def test():
result = await crawl_url("https://example.com")
print(f"✓ Fetched {result.url}")
print(f" Status: {result.status_code}")
print(f" HTML length: {len(result.html)}")
asyncio.run(test())
__all__ = [
"BasicCrawler",
"CachePolicy",
"Crawl4AIAdapter",
"CrawlBatchResult",
"CrawlProfile",
"CrawlResult",
"CrawlerConfig",
"RobotsPolicy",
"crawl_url",
]

283
ontology_platform/ont_platform/core/crawler/jobs.py Normal file
View File

@@ -0,0 +1,283 @@
"""Phase 3 crawl job orchestration.
Jobs are persisted in the Phase 2 SQL store and executed synchronously for
now. That gives the API a stable start/status/cancel contract without adding a
queue worker before the acceptance gate needs one.
"""
from __future__ import annotations
from dataclasses import dataclass
from datetime import datetime
from typing import Any
from sqlalchemy.orm import Session
from ont_platform.core.crawler.crawl4ai_adapter import (
CachePolicy,
Crawl4AIAdapter,
CrawlBatchResult,
CrawlProfile,
CrawlerConfig,
RobotsPolicy,
)
from ont_platform.core.extraction.lightweight_extractor import LightweightExtractor
from ont_platform.core.extractors.web_extractor import ExtractedWebContent, extract_web_content
from ont_platform.storage.candidate_repository import CandidateRepository
from ont_platform.storage.models import EvidenceSpan, ExtractionJob, SourceDocument
@dataclass(frozen=True)
class CrawlJobRequest:
"""Input accepted by the Phase 3 job runner."""
project_id: str
url: str | None = None
html: str | None = None
profile: CrawlProfile = CrawlProfile.FAST_STATIC
max_pages: int = 50
max_depth: int = 1
robots_policy: RobotsPolicy = RobotsPolicy.RESPECT
cache_policy: CachePolicy = CachePolicy.ENABLED
class CrawlJobRunner:
"""Runs acquisition, Trafilatura normalization, and candidate import."""
def __init__(self, db: Session):
self.db = db
async def run(self, request: CrawlJobRequest) -> ExtractionJob:
if not request.url and not request.html:
raise ValueError("url or html is required")
job = ExtractionJob(
id=f"crawl_{_id_suffix()}",
project_id=request.project_id,
job_type="crawl",
status="running",
input_url=request.url,
started_at=datetime.utcnow(),
metadata_={
"progress": _progress(
pages_total=1,
pages_completed=0,
profile=request.profile.value,
robots_policy=request.robots_policy.value,
cache_policy=request.cache_policy.value,
),
"documents": [],
"warnings": [],
},
)
self.db.add(job)
self.db.flush()
try:
batch = await self._acquire(request)
metadata = dict(job.metadata_ or {})
progress = dict(metadata.get("progress") or {})
progress["pages_total"] = max(batch.page_count, 1)
metadata["progress"] = progress
job.metadata_ = metadata
document_ids: list[str] = []
entity_count = 0
relation_count = 0
for page in batch.pages:
extracted = extract_web_content(html=page.html, url=page.url)
source_document = self._save_source_document(extracted, request.project_id)
evidence_spans = self._save_evidence_spans(
extracted=extracted,
project_id=request.project_id,
document_id=source_document.id,
)
candidates = LightweightExtractor(use_llm=False).extract(
text=extracted.text,
project_id=request.project_id,
document_id=source_document.id,
)
CandidateRepository(self.db).save_lightweight_result(
project_id=request.project_id,
document_id=source_document.id,
result={
"entities": candidates.entities,
"relations": candidates.relations,
"evidence_spans": [span_to_dict(span) for span in evidence_spans],
"warnings": candidates.warnings,
},
source_trust=0.6,
validation_passed=True,
)
document_ids.append(source_document.id)
entity_count += len(candidates.entities)
relation_count += len(candidates.relations)
metadata = dict(job.metadata_ or {})
progress = dict(metadata.get("progress") or {})
progress["pages_completed"] = int(progress.get("pages_completed", 0)) + 1
documents = list(metadata.get("documents") or [])
documents.append(
{
"id": source_document.id,
"url": source_document.source_url,
"title": source_document.title,
"profile_used": page.profile_used,
}
)
metadata["progress"] = progress
metadata["documents"] = documents
job.metadata_ = metadata
job.status = "completed"
job.document_id = document_ids[0] if document_ids else None
job.entity_count = entity_count
job.relation_count = relation_count
job.completed_at = datetime.utcnow()
metadata = dict(job.metadata_ or {})
metadata["warnings"] = [*list(metadata.get("warnings") or []), *batch.warnings]
job.metadata_ = metadata
self.db.flush()
return job
except Exception as exc:
job.status = "failed"
job.error_message = str(exc)
job.completed_at = datetime.utcnow()
self.db.flush()
raise
async def _acquire(self, request: CrawlJobRequest) -> CrawlBatchResult:
if request.html:
return CrawlBatchResult(
seed_url=request.url or "inline:html",
pages=[
_inline_page(
url=request.url or "inline:html",
html=request.html,
requested_profile=request.profile.value,
)
],
)
config = CrawlerConfig(
robots_policy=request.robots_policy,
cache_policy=request.cache_policy,
default_profile=request.profile,
max_pages=request.max_pages,
max_depth=request.max_depth,
)
adapter = Crawl4AIAdapter(config=config)
try:
return await adapter.crawl_seed(
request.url or "",
profile=request.profile,
max_pages=request.max_pages,
max_depth=request.max_depth,
)
finally:
await adapter.close()
def _save_source_document(
self,
extracted: ExtractedWebContent,
project_id: str,
) -> SourceDocument:
existing = self.db.get(SourceDocument, extracted.document_id)
if existing is not None:
return existing
model = extracted.to_source_document(project_id=project_id)
self.db.add(model)
self.db.flush()
return model
def _save_evidence_spans(
self,
*,
extracted: ExtractedWebContent,
project_id: str,
document_id: str,
) -> list[EvidenceSpan]:
saved: list[EvidenceSpan] = []
for span in extracted.evidence_spans(project_id=project_id, document_id=document_id):
existing = self.db.get(EvidenceSpan, span.id)
if existing is not None:
saved.append(existing)
continue
model = EvidenceSpan(
id=span.id,
document_id=document_id,
project_id=project_id,
text=span.text,
start_offset=span.start_offset,
end_offset=span.end_offset,
)
self.db.add(model)
saved.append(model)
self.db.flush()
return saved
def job_to_dict(job: ExtractionJob) -> dict[str, Any]:
return {
"id": job.id,
"project_id": job.project_id,
"job_type": job.job_type,
"status": job.status,
"input_url": job.input_url,
"document_id": job.document_id,
"entity_count": job.entity_count,
"relation_count": job.relation_count,
"error_message": job.error_message,
"started_at": job.started_at.isoformat() if job.started_at else None,
"completed_at": job.completed_at.isoformat() if job.completed_at else None,
"created_at": job.created_at.isoformat() if job.created_at else None,
"metadata": job.metadata_ or {},
}
def span_to_dict(span: EvidenceSpan) -> dict[str, Any]:
return {
"id": span.id,
"document_id": span.document_id,
"project_id": span.project_id,
"text": span.text,
"start_offset": span.start_offset,
"end_offset": span.end_offset,
}
def _inline_page(url: str, html: str, requested_profile: str):
from ont_platform.core.crawler.crawl4ai_adapter import CrawlResult
return CrawlResult(
url=url,
html=html,
status_code=200,
profile_used="inline_html",
requested_profile=requested_profile,
)
def _progress(
*,
pages_total: int,
pages_completed: int,
profile: str,
robots_policy: str,
cache_policy: str,
) -> dict[str, Any]:
return {
"pages_total": pages_total,
"pages_completed": pages_completed,
"profile": profile,
"robots_policy": robots_policy,
"cache_policy": cache_policy,
}
def _id_suffix() -> str:
import uuid
return uuid.uuid4().hex
__all__ = ["CrawlJobRequest", "CrawlJobRunner", "job_to_dict"]

89
ontology_platform/ont_platform/core/extraction/schemas.py
View File

@@ -4,9 +4,9 @@ Pydantic schemas for extraction and validation.
Defines the structure of extracted candidates for API and validation.
"""
from typing import Optional
from datetime import datetime
from pydantic import BaseModel, Field
from pydantic import AliasChoices, BaseModel, ConfigDict, Field
class EvidenceSpanSchema(BaseModel):
@@ -27,11 +27,11 @@ class CandidateEntitySchema(BaseModel):
id: str
label: str
entity_type: str = Field(..., description="Entity type (concept, person, org, etc.)")
description: Optional[str] = None
description: str | None = None
confidence: float = Field(..., ge=0.0, le=1.0)
source_trust: Optional[float] = Field(default=0.5, ge=0.0, le=1.0)
evidence_ids: Optional[list[str]] = []
aliases: Optional[list[str]] = []
source_trust: float | None = Field(default=0.5, ge=0.0, le=1.0)
evidence_ids: list[str] | None = []
aliases: list[str] | None = []
class Config:
from_attributes = True
@@ -45,8 +45,8 @@ class CandidateRelationSchema(BaseModel):
predicate: str
target_entity_id: str
confidence: float = Field(..., ge=0.0, le=1.0)
source_trust: Optional[float] = Field(default=0.5, ge=0.0, le=1.0)
evidence_ids: Optional[list[str]] = []
source_trust: float | None = Field(default=0.5, ge=0.0, le=1.0)
evidence_ids: list[str] | None = []
class Config:
from_attributes = True
@@ -57,7 +57,7 @@ class LightweightExtractionResult(BaseModel):
entities: list[CandidateEntitySchema] = []
relations: list[CandidateRelationSchema] = []
evidence_spans: list[EvidenceSpanSchema] = []
evidence_spans: list[EvidenceSpanSchema] = Field(default_factory=list)
warnings: list[str] = []
class Config:
@@ -69,20 +69,27 @@ class SourceDocumentSchema(BaseModel):
id: str
project_id: str
source_url: Optional[str] = None
file_path: Optional[str] = None
source_url: str | None = None
canonical_url: str | None = None
file_path: str | None = None
document_type: str # "html", "pdf", "markdown", "docx", "inline_text"
title: Optional[str] = None
author: Optional[str] = None
publish_date: Optional[str] = None
language: Optional[str] = None
sitename: Optional[str] = None
title: str | None = None
author: str | None = None
publish_date: str | None = None
language: str | None = None
sitename: str | None = None
description: str | None = None
text_length: int | None = None
content_hash: str
fingerprint: Optional[str] = None
retrieved_at: str # ISO-8601
fingerprint: str | None = None
retrieved_at: datetime | str # ISO-8601
extracted_by: str
metadata: dict = Field(
default_factory=dict,
validation_alias=AliasChoices("metadata_", "metadata"),
)
class Config:
from_attributes = True
@@ -95,13 +102,38 @@ class ExtractionJobSchema(BaseModel):
project_id: str
job_type: str
status: str
input_url: Optional[str] = None
input_file: Optional[str] = None
document_id: Optional[str] = None
input_url: str | None = None
input_file: str | None = None
document_id: str | None = None
entity_count: int = 0
relation_count: int = 0
error_message: Optional[str] = None
error_message: str | None = None
created_at: str
metadata: dict = Field(
default_factory=dict,
validation_alias=AliasChoices("metadata_", "metadata"),
)
class Config:
from_attributes = True
class ValidationIssueSchema(BaseModel):
"""Stored validation issue."""
id: str
project_id: str
document_id: str | None = None
candidate_id: str | None = None
candidate_kind: str | None = None
severity: str
code: str
message: str
source: str
metadata: dict = Field(
default_factory=dict,
validation_alias=AliasChoices("metadata_", "metadata"),
)
class Config:
from_attributes = True
@@ -110,20 +142,21 @@ class ExtractionJobSchema(BaseModel):
class ExtractRequestSchema(BaseModel):
"""Request to extract from URL or text."""
url: Optional[str] = None
project_id: str
class Config:
json_schema_extra = {
model_config = ConfigDict(
json_schema_extra={
"example": {"url": "https://example.com", "project_id": "proj_123"}
}
)
url: str | None = None
project_id: str
class CandidateListResponseSchema(BaseModel):
"""Response listing candidates."""
document_id: str
document_title: Optional[str]
document_title: str | None
entity_count: int
relation_count: int
entities: list[CandidateEntitySchema]

475
ontology_platform/ont_platform/core/extractors/web_extractor.py
View File

@@ -1,22 +1,129 @@
"""
Web content extraction using Trafilatura.
"""Trafilatura adapter for URL/HTML ingestion.
Handles HTML/URL content extraction with metadata preservation for ontology candidate extraction.
This module owns the Phase 1 boundary between arbitrary web input and the
platform's SourceDocument/EvidenceSpan contract. OntoCast stays untouched:
the platform prepares clean text, provenance metadata, hashes, and evidence
spans before any downstream workflow receives the document.
"""
from dataclasses import dataclass
from typing import Optional
from __future__ import annotations
import hashlib
from datetime import datetime
import re
from dataclasses import dataclass, field
from datetime import UTC, datetime
from typing import Any
import trafilatura
from trafilatura import extract
from trafilatura.metadata import extract_metadata
from lxml import etree
try: # Phase 1 dependency; keep import optional for lower-phase smoke tests.
import trafilatura
from trafilatura.settings import Extractor
HAS_TRAFILATURA = True
except ModuleNotFoundError: # pragma: no cover - exercised when dependency is absent.
trafilatura = None # type: ignore[assignment]
Extractor = None # type: ignore[assignment]
HAS_TRAFILATURA = False
try:
from bs4 import BeautifulSoup
except ModuleNotFoundError: # pragma: no cover - beautifulsoup4 is in the base requirements.
BeautifulSoup = None # type: ignore[assignment]
def _now_iso() -> str:
return datetime.now(UTC).isoformat()
def _normalize_text(text: str) -> str:
return re.sub(r"\s+", " ", text).strip()
def _content_hash(text: str) -> str:
return hashlib.sha256(_normalize_text(text).encode("utf-8")).hexdigest()
def _stable_fingerprint(text: str) -> str:
"""Stable exact-content fingerprint for Phase 1 dedup.
Trafilatura 2.0's ``Document.fingerprint`` is not always populated for
local HTML fixtures, so Phase 1 uses a deterministic normalized-text hash.
"""
normalized = _normalize_text(text).lower()
digest = hashlib.sha1(normalized.encode("utf-8")).hexdigest()
return f"sha1:{digest}"
def _html_language(html: str) -> str | None:
match = re.search(r"<html\b[^>]*\blang=[\"']?([A-Za-z0-9_-]+)", html, re.IGNORECASE)
if not match:
return None
return match.group(1).split("-")[0].lower()
def _canonicalize_url(url: str | None) -> str | None:
if not url:
return None
from urllib.parse import urlsplit, urlunsplit
parts = urlsplit(url)
return urlunsplit((parts.scheme, parts.netloc, parts.path, "", ""))
def _json_safe_metadata(raw: dict[str, Any]) -> dict[str, Any]:
safe: dict[str, Any] = {}
for key, value in raw.items():
if key in {"body", "comments", "commentsbody"}:
continue
if value is None or isinstance(value, str | int | float | bool):
safe[key] = value
elif isinstance(value, list):
safe[key] = [item for item in value if isinstance(item, str | int | float | bool)]
return safe
def _serialize_body_xml(body: Any) -> str | None:
if body is None:
return None
if isinstance(body, str):
return body
try:
return etree.tostring(body, encoding="unicode")
except (TypeError, ValueError):
return None
def _document_id(content_hash: str) -> str:
return f"doc_{content_hash[:16]}"
@dataclass(frozen=True)
class EvidenceSpanData:
"""Serializable evidence span produced from cleaned source text."""
id: str
document_id: str
project_id: str
text: str
start_offset: int
end_offset: int
def to_dict(self) -> dict[str, Any]:
return {
"id": self.id,
"document_id": self.document_id,
"project_id": self.project_id,
"text": self.text,
"start_offset": self.start_offset,
"end_offset": self.end_offset,
}
@dataclass
class ExtractedWebContent:
"""Result of web content extraction."""
"""Result of Phase 1 web ingestion."""
url: str | None
text: str
@@ -25,150 +132,316 @@ class ExtractedWebContent:
publish_date: str | None
language: str | None
sitename: str | None
# Additional metadata
description: str | None
canonical_url: str | None
fingerprint: str | None
fingerprint: str
content_hash: str
retrieved_at: str
source: str # "trafilatura"
source: str = "trafilatura"
body_xml: str | None = None
raw_html: str | None = None
metadata: dict[str, Any] = field(default_factory=dict)
# Raw metadata
metadata: dict
@property
def document_id(self) -> str:
return _document_id(self.content_hash)
def to_source_document(self, project_id: str = "default", document_id: str | None = None):
"""Build an unsaved SQLAlchemy SourceDocument model."""
from ont_platform.storage.models import SourceDocument
retrieved_at = datetime.fromisoformat(self.retrieved_at)
return SourceDocument(
id=document_id or self.document_id,
project_id=project_id,
source_url=self.url,
canonical_url=self.canonical_url,
document_type="html",
title=self.title,
author=self.author,
publish_date=self.publish_date,
language=self.language,
sitename=self.sitename,
description=self.description,
text=self.text,
raw_html=self.raw_html,
body_xml=self.body_xml,
content_hash=self.content_hash,
fingerprint=self.fingerprint,
retrieved_at=retrieved_at,
extracted_by=self.source,
metadata_=self.metadata,
)
def to_source_document_dict(self, project_id: str = "default") -> dict[str, Any]:
return {
"id": self.document_id,
"project_id": project_id,
"source_url": self.url,
"canonical_url": self.canonical_url,
"document_type": "html",
"title": self.title,
"author": self.author,
"publish_date": self.publish_date,
"language": self.language,
"sitename": self.sitename,
"description": self.description,
"text_length": len(self.text),
"content_hash": self.content_hash,
"fingerprint": self.fingerprint,
"retrieved_at": self.retrieved_at,
"extracted_by": self.source,
"metadata": self.metadata,
}
def evidence_spans(
self,
project_id: str = "default",
document_id: str | None = None,
min_chars: int = 40,
) -> list[EvidenceSpanData]:
"""Create paragraph-level evidence spans with offsets into ``text``."""
doc_id = document_id or self.document_id
spans: list[EvidenceSpanData] = []
cursor = 0
paragraphs = [part.strip() for part in re.split(r"\n\s*\n", self.text) if part.strip()]
if not paragraphs and self.text.strip():
paragraphs = [self.text.strip()]
for index, paragraph in enumerate(paragraphs, start=1):
if len(paragraph) < min_chars and paragraphs != [paragraph]:
continue
start = self.text.find(paragraph, cursor)
if start < 0:
start = cursor
end = start + len(paragraph)
cursor = end
span_hash = hashlib.sha1(f"{doc_id}:{index}:{start}:{end}".encode()).hexdigest()
spans.append(
EvidenceSpanData(
id=f"ev_{span_hash[:16]}",
document_id=doc_id,
project_id=project_id,
text=paragraph,
start_offset=start,
end_offset=end,
)
)
return spans
class WebExtractor:
"""Web content extractor using Trafilatura."""
"""Web content extractor using Trafilatura 2.x."""
def __init__(self):
"""Initialize extractor."""
def __init__(self) -> None:
self.source = "trafilatura"
def extract_from_html(
self,
html: str,
source_url: str | None = None,
lang: str | None = None,
) -> ExtractedWebContent:
"""
Extract content from HTML string.
if not html or not html.strip():
raise ValueError("html is required")
Args:
html: Raw HTML content
source_url: Optional source URL for metadata
if not HAS_TRAFILATURA:
return self._fallback_extract_from_html(html, source_url=source_url, lang=lang)
Returns:
ExtractedWebContent with text and metadata
"""
# Extract main content
text = extract(html, include_comments=False, output_format="txt")
options = Extractor(
output_format="python",
url=source_url,
with_metadata=True,
comments=False,
tables=True,
formatting=True,
links=True,
images=True,
dedup=True,
)
doc = trafilatura.bare_extraction(html, options=options)
text = getattr(doc, "text", None) if doc is not None else None
if not text:
text = trafilatura.extract(
html,
url=source_url,
include_comments=False,
include_tables=True,
include_formatting=True,
include_links=True,
include_images=True,
deduplicate=True,
with_metadata=True,
output_format="txt",
)
if not text:
raise ValueError("Could not extract text from HTML")
# Extract metadata (returns Document object in trafilatura 2.0+)
doc = extract_metadata(html)
raw_metadata = doc.as_dict() if doc is not None else {}
metadata = _json_safe_metadata(raw_metadata)
metadata["source"] = self.source
# Calculate content hash
content_hash = hashlib.sha256(text.encode()).hexdigest()
content_hash = _content_hash(text)
fingerprint = getattr(doc, "fingerprint", None) if doc is not None else None
if not fingerprint:
fingerprint = _stable_fingerprint(text)
# Extract fingerprint (near-duplicate detection)
fingerprint = self._get_fingerprint(text)
canonical_url = metadata.get("url") or source_url
canonical_url = _canonicalize_url(canonical_url)
language = metadata.get("language") or lang or _html_language(html)
# Convert Document object to dict (trafilatura 2.0+)
metadata_dict = {}
if doc:
metadata_dict = {
"title": getattr(doc, "title", None),
"author": getattr(doc, "author", None),
"date": getattr(doc, "date", None),
"language": getattr(doc, "language", None),
"sitename": getattr(doc, "sitename", None),
"url": getattr(doc, "url", None),
}
return ExtractedWebContent(
url=source_url,
text=text.strip(),
title=metadata.get("title"),
author=metadata.get("author"),
publish_date=metadata.get("date"),
language=language,
sitename=metadata.get("sitename") or metadata.get("hostname"),
description=metadata.get("description"),
canonical_url=canonical_url,
fingerprint=fingerprint,
content_hash=content_hash,
retrieved_at=_now_iso(),
source=self.source,
body_xml=_serialize_body_xml(getattr(doc, "body", None)),
raw_html=html,
metadata=metadata,
)
def _fallback_extract_from_html(
self,
html: str,
source_url: str | None = None,
lang: str | None = None,
) -> ExtractedWebContent:
"""Small, deterministic extractor used only when Trafilatura is absent.
It preserves the same SourceDocument contract so tests and lower-phase
route imports do not fail in minimal environments. Production installs
should still use Trafilatura.
"""
if BeautifulSoup is None:
raise RuntimeError("trafilatura or beautifulsoup4 is required for HTML extraction")
soup = BeautifulSoup(html, "html.parser")
for tag in soup(["script", "style", "noscript", "nav", "footer", "header", "aside"]):
tag.decompose()
title = _meta_content(soup, "title") or (soup.title.get_text(strip=True) if soup.title else None)
author = _meta_content(soup, "author")
description = _meta_content(soup, "description")
sitename = _meta_property(soup, "og:site_name")
publish_date = _meta_property(soup, "article:published_time") or _meta_content(soup, "date")
canonical = None
canonical_tag = soup.find("link", rel=lambda value: value and "canonical" in value)
if canonical_tag is not None:
canonical = canonical_tag.get("href")
canonical_url = _canonicalize_url(canonical or source_url)
main = soup.find("article") or soup.body or soup
blocks = [
_normalize_text(node.get_text(" ", strip=True))
for node in main.find_all(["h1", "h2", "h3", "p", "li"])
]
blocks = [block for block in blocks if block]
if not blocks:
blocks = [_normalize_text(main.get_text(" ", strip=True))]
text = "\n\n".join(blocks).strip()
if not text:
raise ValueError("Could not extract text from HTML")
metadata: dict[str, Any] = {
"source": self.source,
"title": title,
"author": author,
"date": publish_date,
"description": description,
"sitename": sitename,
"url": canonical_url,
"fallback": "beautifulsoup",
}
metadata = _json_safe_metadata(metadata)
content_hash = _content_hash(text)
return ExtractedWebContent(
url=source_url,
text=text,
title=metadata_dict.get("title"),
author=metadata_dict.get("author"),
publish_date=metadata_dict.get("date"),
language=metadata_dict.get("language"),
sitename=metadata_dict.get("sitename"),
canonical_url=metadata_dict.get("url") or source_url,
fingerprint=fingerprint,
title=title,
author=author,
publish_date=publish_date,
language=lang or _html_language(html),
sitename=sitename,
description=description,
canonical_url=canonical_url,
fingerprint=_stable_fingerprint(text),
content_hash=content_hash,
retrieved_at=datetime.utcnow().isoformat(),
retrieved_at=_now_iso(),
source=self.source,
metadata=metadata_dict,
body_xml=str(main),
raw_html=html,
metadata=metadata,
)
def extract_from_url(
self,
url: str,
timeout: int = 10,
) -> ExtractedWebContent:
"""
Extract content from URL (requires network access).
if not url:
raise ValueError("url is required")
Args:
url: HTTP(S) URL
timeout: Request timeout in seconds (not used with trafilatura 2.0+)
Returns:
ExtractedWebContent with text and metadata
"""
try:
downloaded = trafilatura.fetch_url(url)
if HAS_TRAFILATURA:
downloaded = trafilatura.fetch_url(url)
else:
import requests
response = requests.get(url, timeout=15)
response.raise_for_status()
downloaded = response.text
if not downloaded:
raise ValueError(f"Could not fetch URL: {url}")
return self.extract_from_html(downloaded, source_url=url)
except Exception as e:
raise RuntimeError(f"Failed to extract from {url}: {e}")
except Exception as exc:
raise RuntimeError(f"Failed to extract from {url}: {exc}") from exc
@staticmethod
def _get_fingerprint(text: str) -> str:
"""
Calculate SimHash-like fingerprint for near-duplicate detection.
Simple implementation: hash of first 1000 chars + length.
For production, use trafilatura.content_fingerprint() or simhash.
def _meta_content(soup: Any, name: str) -> str | None:
tag = soup.find("meta", attrs={"name": name})
return tag.get("content") if tag is not None else None
Args:
text: Content text
Returns:
Fingerprint string
"""
sample = text[:1000] if len(text) > 1000 else text
sample_hash = hashlib.md5(sample.encode()).hexdigest()[:16]
length_hash = hashlib.md5(str(len(text)).encode()).hexdigest()[:8]
return f"{sample_hash}_{length_hash}"
def _meta_property(soup: Any, prop: str) -> str | None:
tag = soup.find("meta", attrs={"property": prop})
return tag.get("content") if tag is not None else None
def extract_web_content(
html: str | None = None,
url: str | None = None,
lang: str | None = None,
) -> ExtractedWebContent:
"""
Convenience function for web extraction.
"""Extract a Phase 1 SourceDocument-ready payload from HTML or URL."""
Args:
html: Raw HTML (if available)
url: URL to fetch (if html not provided)
Returns:
ExtractedWebContent
Raises:
ValueError: If neither html nor url provided, or extraction fails
"""
if not html and not url:
raise ValueError("Either html or url must be provided")
extractor = WebExtractor()
if html:
return extractor.extract_from_html(html, source_url=url)
else:
return extractor.extract_from_url(url)
return extractor.extract_from_html(html, source_url=url, lang=lang)
return extractor.extract_from_url(url or "")
__all__ = [
"EvidenceSpanData",
"ExtractedWebContent",
"WebExtractor",
"extract_web_content",
]

7
ontology_platform/ont_platform/core/graph/__init__.py
View File

@@ -2,6 +2,8 @@
from .neo4j_adapter import Neo4jAdapter, Neo4jConfig
from .rdf_converter import RDFToPropertyGraphConverter
from .cypher_guard import ReadOnlyCypherGuard, SanitizedCypher, UnsafeCypherError
from .search import CandidateGraphSearchService, GraphSearchResult
from .entity_resolver import EntityResolver, EntityCluster
from .subgraph_retriever import SubgraphRetriever
from .pattern_matcher import PatternMatcher, PathResult, CycleResult
@@ -11,6 +13,11 @@ __all__ = [
"Neo4jAdapter",
"Neo4jConfig",
"RDFToPropertyGraphConverter",
"ReadOnlyCypherGuard",
"SanitizedCypher",
"UnsafeCypherError",
"CandidateGraphSearchService",
"GraphSearchResult",
"EntityResolver",
"EntityCluster",
"SubgraphRetriever",

61
ontology_platform/ont_platform/core/graph/cypher_guard.py Normal file
View File

@@ -0,0 +1,61 @@
"""Read-only Text2Cypher guard for GraphRAG search."""
from __future__ import annotations
import re
from dataclasses import dataclass
class UnsafeCypherError(ValueError):
"""Raised when a generated Cypher query attempts writes or unsafe calls."""
@dataclass(frozen=True)
class SanitizedCypher:
query: str
limit: int
read_only: bool = True
class ReadOnlyCypherGuard:
"""Allowlist and limit enforcement for generated Cypher."""
_write_keywords = re.compile(
r"\b(CREATE|MERGE|SET|DELETE|DETACH|REMOVE|DROP|ALTER|LOAD\s+CSV|CALL\s+dbms|CALL\s+apoc)\b",
re.IGNORECASE,
)
_allowed_start = re.compile(r"^\s*(MATCH|OPTIONAL\s+MATCH|WITH|UNWIND|RETURN)\b", re.IGNORECASE)
_limit_clause = re.compile(r"\bLIMIT\s+(\d+)\b", re.IGNORECASE)
def __init__(self, max_limit: int = 100):
self.max_limit = max_limit
def sanitize(self, query: str, *, limit: int | None = None) -> SanitizedCypher:
cleaned = self._strip_comments(query).strip().rstrip(";")
if not cleaned:
raise UnsafeCypherError("Cypher query is empty")
if self._write_keywords.search(cleaned):
raise UnsafeCypherError("Only read-only Cypher is allowed")
if not self._allowed_start.search(cleaned):
raise UnsafeCypherError("Cypher must start with a read-only clause")
effective_limit = min(limit or self.max_limit, self.max_limit)
match = self._limit_clause.search(cleaned)
if match:
requested = int(match.group(1))
if requested > effective_limit:
cleaned = self._limit_clause.sub(f"LIMIT {effective_limit}", cleaned, count=1)
else:
cleaned = f"{cleaned}\nLIMIT {effective_limit}"
return SanitizedCypher(query=cleaned, limit=effective_limit)
@staticmethod
def _strip_comments(query: str) -> str:
lines = []
for line in query.splitlines():
lines.append(re.sub(r"//.*$", "", line))
return re.sub(r"/\*.*?\*/", "", "\n".join(lines), flags=re.DOTALL)
__all__ = ["ReadOnlyCypherGuard", "SanitizedCypher", "UnsafeCypherError"]

123
ontology_platform/ont_platform/core/graph/search.py Normal file
View File

@@ -0,0 +1,123 @@
"""Graph search result shaping with provenance."""
from __future__ import annotations
from dataclasses import dataclass, field
from typing import Any
from sqlalchemy import select
from sqlalchemy.orm import Session
from ont_platform.storage.models import CandidateEntity, CandidateRelation, EvidenceSpan, SourceDocument
@dataclass
class GraphSearchResult:
id: str
label: str
result_type: str
score: float
provenance: dict[str, Any] = field(default_factory=dict)
properties: dict[str, Any] = field(default_factory=dict)
def to_dict(self) -> dict[str, Any]:
return {
"id": self.id,
"label": self.label,
"result_type": self.result_type,
"score": self.score,
"provenance": self.provenance,
"properties": self.properties,
}
class CandidateGraphSearchService:
"""Searches the reviewed candidate projection when Neo4j is unavailable."""
def __init__(self, db: Session):
self.db = db
def search(self, *, project_id: str, query: str, limit: int = 20) -> list[GraphSearchResult]:
normalized_query = query.lower().strip()
if not normalized_query:
return []
results: list[GraphSearchResult] = []
entity_stmt = (
select(CandidateEntity)
.where(CandidateEntity.project_id == project_id)
.order_by(CandidateEntity.confidence.desc())
)
for entity in self.db.scalars(entity_stmt):
haystack = f"{entity.label} {entity.entity_type} {entity.description or ''}".lower()
if normalized_query not in haystack:
continue
results.append(
GraphSearchResult(
id=entity.id,
label=entity.label,
result_type="entity",
score=float(entity.confidence or 0.0),
provenance=self._provenance(entity.document_id, entity.evidence_ids or []),
properties={
"entity_type": entity.entity_type,
"review_status": entity.review_status.value,
"validation_passed": entity.validation_passed,
},
)
)
if len(results) >= limit:
return results
relation_stmt = (
select(CandidateRelation)
.where(CandidateRelation.project_id == project_id)
.order_by(CandidateRelation.confidence.desc())
)
for relation in self.db.scalars(relation_stmt):
haystack = f"{relation.predicate} {relation.source_entity_id} {relation.target_entity_id}".lower()
if normalized_query not in haystack:
continue
results.append(
GraphSearchResult(
id=relation.id,
label=relation.predicate,
result_type="relation",
score=float(relation.confidence or 0.0),
provenance=self._provenance(relation.document_id, relation.evidence_ids or []),
properties={
"source_entity_id": relation.source_entity_id,
"target_entity_id": relation.target_entity_id,
"review_status": relation.review_status.value,
"validation_passed": relation.validation_passed,
},
)
)
if len(results) >= limit:
return results
return results
def _provenance(self, document_id: str, evidence_ids: list[str]) -> dict[str, Any]:
document = self.db.get(SourceDocument, document_id)
evidence = []
if evidence_ids:
stmt = select(EvidenceSpan).where(EvidenceSpan.id.in_(evidence_ids))
evidence = [
{
"id": span.id,
"text": span.text,
"start_offset": span.start_offset,
"end_offset": span.end_offset,
}
for span in self.db.scalars(stmt)
]
return {
"document_id": document_id,
"source_url": document.source_url if document else None,
"title": document.title if document else None,
"evidence_spans": evidence,
}
__all__ = ["CandidateGraphSearchService", "GraphSearchResult"]

17
ontology_platform/ont_platform/core/maintenance/__init__.py Normal file
View File

@@ -0,0 +1,17 @@
"""Maintenance loop services for Phase 6."""
from .service import (
MaintenanceLoopService,
MaintenancePermissionError,
MaintenanceProposalNotFoundError,
maintenance_proposal_to_dict,
maintenance_run_to_dict,
)
__all__ = [
"MaintenanceLoopService",
"MaintenancePermissionError",
"MaintenanceProposalNotFoundError",
"maintenance_proposal_to_dict",
"maintenance_run_to_dict",
]

574
ontology_platform/ont_platform/core/maintenance/service.py Normal file
View File

@@ -0,0 +1,574 @@
"""Non-destructive maintenance loop for Phase 6.
This module borrows the workflow pattern of multi-role graph maintenance, not
any Knowledge Agent implementation. Every role emits observations or proposals;
the graph and reviewed candidates are never changed directly by the loop.
"""
from __future__ import annotations
import uuid
from collections import defaultdict
from dataclasses import dataclass
from datetime import datetime
from typing import Any
from sqlalchemy import select
from sqlalchemy.orm import Session
from ont_platform.audit.logger import AuditLogger
from ont_platform.audit.models import AuditAction, ResourceType
from ont_platform.auth.rbac import Permission, RBAC
from ont_platform.billing.calculator import CostCalculator
from ont_platform.billing.models import OperationType
from ont_platform.storage.candidate_repository import CandidateRepository
from ont_platform.storage.models import (
CandidateEntity,
CandidateKind,
CandidateRelation,
MaintenanceProposal,
MaintenanceProposalStatus,
MaintenanceRole,
MaintenanceRun,
MaintenanceRunStatus,
SourceDocument,
)
class MaintenancePermissionError(PermissionError):
"""Raised when an actor lacks a Phase 6 maintenance permission."""
class MaintenanceProposalNotFoundError(LookupError):
"""Raised when a proposal does not exist."""
@dataclass(frozen=True)
class MaintenanceFinding:
code: str
message: str
target_kind: str | None = None
target_id: str | None = None
severity: str = "info"
metadata: dict[str, Any] | None = None
def to_dict(self) -> dict[str, Any]:
return {
"code": self.code,
"message": self.message,
"target_kind": self.target_kind,
"target_id": self.target_id,
"severity": self.severity,
"metadata": self.metadata or {},
}
@dataclass(frozen=True)
class ProposalDraft:
role: MaintenanceRole
proposal_type: str
title: str
description: str
target_kind: str | None = None
target_id: str | None = None
risk_level: str = "low"
metadata: dict[str, Any] | None = None
class MaintenanceLoopService:
"""Coordinates Analyst/Researcher/Curator/Auditor/Fixer/Advisor roles."""
def __init__(
self,
db: Session,
*,
audit_logger: AuditLogger | None = None,
cost_calculator: CostCalculator | None = None,
event_broadcaster: Any | None = None,
) -> None:
self.db = db
self.audit_logger = audit_logger or AuditLogger()
self.cost_calculator = cost_calculator or CostCalculator()
self.event_broadcaster = event_broadcaster
self.rbac = RBAC()
async def run(
self,
*,
project_id: str,
requested_by: str,
actor_role: str,
low_confidence_threshold: float = 0.65,
) -> MaintenanceRun:
self._require(actor_role, Permission.RUN_MAINTENANCE)
run = MaintenanceRun(
id=f"maint_{uuid.uuid4().hex}",
project_id=project_id,
requested_by=requested_by,
status=MaintenanceRunStatus.RUNNING,
metadata_={"role_order": [role.value for role in MaintenanceRole]},
)
self.db.add(run)
self.db.flush()
try:
context = self._load_context(
project_id=project_id,
low_confidence_threshold=low_confidence_threshold,
)
analyst = self._analyst(context)
researcher = self._researcher(context, analyst)
curator = self._curator(context, researcher)
auditor = self._auditor(context)
fixer_drafts = self._fixer(context, analyst, auditor)
proposals = self._save_proposals(run, fixer_drafts)
advisor = await self._advisor(
project_id=project_id,
user_id=requested_by,
run_id=run.id,
proposal_count=len(proposals),
finding_count=len(analyst["findings"]) + len(auditor["findings"]),
)
audit_entry = await self.audit_logger.log_action(
org_id=project_id,
user_id=requested_by,
action=AuditAction.ANALYZE,
resource_type=ResourceType.GRAPH,
resource_id=project_id,
metadata={
"run_id": run.id,
"proposal_count": len(proposals),
"non_destructive": True,
},
)
realtime_events = await self._broadcast_completed(project_id, run.id, len(proposals))
role_reports = {
"analyst": analyst,
"researcher": researcher,
"curator": curator,
"auditor": auditor,
"fixer": {
"proposal_count": len(proposals),
"proposal_ids": [proposal.id for proposal in proposals],
"mode": "proposal_only",
},
"advisor": advisor,
}
run.status = MaintenanceRunStatus.COMPLETED
run.completed_at = datetime.utcnow()
run.summary = {
"finding_count": len(analyst["findings"]) + len(auditor["findings"]),
"proposal_count": len(proposals),
"direct_mutations": 0,
"approval_gate": "required",
}
run.budget_summary = advisor["budget"]
run.audit_summary = {
"audit_log_id": audit_entry.id,
"action": audit_entry.action.value,
"resource_type": audit_entry.resource_type.value,
}
run.metadata_ = {
**(run.metadata_ or {}),
"role_reports": role_reports,
"realtime_events": realtime_events,
}
self.db.flush()
return run
except Exception as exc:
run.status = MaintenanceRunStatus.FAILED
run.error_message = str(exc)
run.completed_at = datetime.utcnow()
self.db.flush()
raise
async def review_proposal(
self,
*,
proposal_id: str,
reviewed_by: str,
actor_role: str,
approve: bool,
reason: str | None = None,
) -> MaintenanceProposal:
self._require(actor_role, Permission.APPROVE_MAINTENANCE)
proposal = self.db.get(MaintenanceProposal, proposal_id)
if proposal is None:
raise MaintenanceProposalNotFoundError(f"Maintenance proposal not found: {proposal_id}")
if proposal.status != MaintenanceProposalStatus.PENDING_REVIEW:
raise ValueError(f"Proposal is already {proposal.status.value}")
if approve:
proposal.status = MaintenanceProposalStatus.APPROVED
proposal.approved_by = reviewed_by
proposal.approved_at = datetime.utcnow()
else:
proposal.status = MaintenanceProposalStatus.REJECTED
proposal.rejection_reason = reason
await self.audit_logger.log_action(
org_id=proposal.project_id,
user_id=reviewed_by,
action=AuditAction.UPDATE,
resource_type=ResourceType.GRAPH,
resource_id=proposal.target_id or proposal.id,
metadata={
"proposal_id": proposal.id,
"proposal_status": proposal.status.value,
"proposal_type": proposal.proposal_type,
"non_destructive": True,
},
)
self.db.flush()
return proposal
def list_runs(self, *, project_id: str, limit: int = 50) -> list[MaintenanceRun]:
stmt = (
select(MaintenanceRun)
.where(MaintenanceRun.project_id == project_id)
.order_by(MaintenanceRun.created_at.desc())
.limit(limit)
)
return list(self.db.scalars(stmt))
def list_proposals(
self,
*,
project_id: str,
status: MaintenanceProposalStatus | str | None = None,
limit: int = 100,
) -> list[MaintenanceProposal]:
stmt = select(MaintenanceProposal).where(MaintenanceProposal.project_id == project_id)
if status is not None:
stmt = stmt.where(MaintenanceProposal.status == MaintenanceProposalStatus(status))
stmt = stmt.order_by(MaintenanceProposal.created_at.desc()).limit(limit)
return list(self.db.scalars(stmt))
def _load_context(self, *, project_id: str, low_confidence_threshold: float) -> dict[str, Any]:
repository = CandidateRepository(self.db)
candidates = repository.list_candidates(project_id=project_id)
entities = list(candidates["entities"])
relations = list(candidates["relations"])
issues = repository.list_validation_issues(project_id=project_id)
documents = list(
self.db.scalars(select(SourceDocument).where(SourceDocument.project_id == project_id))
)
missing_evidence = [
candidate
for candidate in [*entities, *relations]
if not repository.candidate_has_valid_evidence(candidate)
]
low_confidence = [
candidate
for candidate in [*entities, *relations]
if float(candidate.confidence or 0.0) < low_confidence_threshold
]
duplicates = self._find_duplicate_entities(entities)
return {
"project_id": project_id,
"entities": entities,
"relations": relations,
"issues": issues,
"documents": documents,
"missing_evidence": missing_evidence,
"low_confidence": low_confidence,
"duplicates": duplicates,
}
def _analyst(self, context: dict[str, Any]) -> dict[str, Any]:
findings: list[MaintenanceFinding] = []
for candidate in context["missing_evidence"]:
findings.append(
MaintenanceFinding(
code="missing_evidence",
message=f"{candidate.id} has no valid evidence span",
target_kind=_candidate_kind(candidate),
target_id=candidate.id,
severity="warning",
)
)
for candidate in context["low_confidence"]:
findings.append(
MaintenanceFinding(
code="low_confidence",
message=f"{candidate.id} confidence is {candidate.confidence}",
target_kind=_candidate_kind(candidate),
target_id=candidate.id,
severity="info",
metadata={"confidence": candidate.confidence},
)
)
for duplicate in context["duplicates"]:
findings.append(
MaintenanceFinding(
code="duplicate_candidate",
message=f"Duplicate label cluster: {duplicate['label']}",
target_kind="entity",
target_id=duplicate["canonical_id"],
severity="warning",
metadata=duplicate,
)
)
return {
"role": MaintenanceRole.ANALYST.value,
"entity_count": len(context["entities"]),
"relation_count": len(context["relations"]),
"document_count": len(context["documents"]),
"findings": [finding.to_dict() for finding in findings],
}
def _researcher(self, context: dict[str, Any], analyst: dict[str, Any]) -> dict[str, Any]:
targets = [
finding
for finding in analyst["findings"]
if finding["code"] in {"missing_evidence", "low_confidence"}
]
plans = [
{
"target_id": target["target_id"],
"target_kind": target["target_kind"],
"query_hint": self._label_for_target(context, target["target_id"]),
"goal": "find corroborating source evidence",
}
for target in targets[:10]
]
return {
"role": MaintenanceRole.RESEARCHER.value,
"source_discovery_plans": plans,
"external_code_used": False,
}
def _curator(self, context: dict[str, Any], researcher: dict[str, Any]) -> dict[str, Any]:
proposals = []
for plan in researcher["source_discovery_plans"]:
proposals.append(
{
"target_id": plan["target_id"],
"quality_checks": ["source_url_required", "evidence_text_required", "dedup_check"],
"suggested_ingestion_profile": "fast_static",
}
)
return {
"role": MaintenanceRole.CURATOR.value,
"ingestion_suggestions": proposals,
"source_quality_policy": "provenance_first",
}
def _auditor(self, context: dict[str, Any]) -> dict[str, Any]:
findings = [
MaintenanceFinding(
code=issue.code,
message=issue.message,
target_kind=issue.candidate_kind.value if issue.candidate_kind else None,
target_id=issue.candidate_id,
severity=issue.severity.value,
metadata={"issue_id": issue.id, "source": issue.source},
)
for issue in context["issues"]
]
return {
"role": MaintenanceRole.AUDITOR.value,
"validation_issue_count": len(context["issues"]),
"findings": [finding.to_dict() for finding in findings],
}
def _fixer(
self,
context: dict[str, Any],
analyst: dict[str, Any],
auditor: dict[str, Any],
) -> list[ProposalDraft]:
drafts: list[ProposalDraft] = []
for finding in analyst["findings"]:
if finding["code"] == "missing_evidence":
drafts.append(
ProposalDraft(
role=MaintenanceRole.FIXER,
proposal_type="request_evidence",
title=f"Attach evidence for {finding['target_id']}",
description="Create an evidence-backed candidate update through review.",
target_kind=finding["target_kind"],
target_id=finding["target_id"],
risk_level="medium",
metadata={"finding": finding, "direct_mutation": False},
)
)
elif finding["code"] == "duplicate_candidate":
drafts.append(
ProposalDraft(
role=MaintenanceRole.FIXER,
proposal_type="merge_duplicate_candidate",
title=f"Review duplicate cluster {finding['metadata']['label']}",
description="Prepare a human-reviewed merge plan; do not merge automatically.",
target_kind="entity",
target_id=finding["target_id"],
risk_level="high",
metadata={"finding": finding, "direct_mutation": False},
)
)
for finding in auditor["findings"]:
drafts.append(
ProposalDraft(
role=MaintenanceRole.AUDITOR,
proposal_type="resolve_validation_issue",
title=f"Resolve validation issue {finding['code']}",
description=finding["message"],
target_kind=finding["target_kind"],
target_id=finding["target_id"],
risk_level="medium",
metadata={"finding": finding, "direct_mutation": False},
)
)
return drafts
async def _advisor(
self,
*,
project_id: str,
user_id: str,
run_id: str,
proposal_count: int,
finding_count: int,
) -> dict[str, Any]:
usage = await self.cost_calculator.record_usage(
org_id=project_id,
user_id=user_id,
operation_type=OperationType.ANALYSIS,
quantity=1,
metadata={"run_id": run_id, "proposal_count": proposal_count},
)
forecast = await self.cost_calculator.get_cost_forecast(project_id)
return {
"role": MaintenanceRole.ADVISOR.value,
"budget": {
"usage": usage.to_dict(),
"forecast": forecast,
},
"trend": {
"finding_count": finding_count,
"proposal_count": proposal_count,
"repeated_issue_signal": finding_count > 0,
},
}
def _save_proposals(
self,
run: MaintenanceRun,
drafts: list[ProposalDraft],
) -> list[MaintenanceProposal]:
saved: list[MaintenanceProposal] = []
for draft in drafts:
proposal = MaintenanceProposal(
id=f"mprop_{uuid.uuid4().hex}",
run_id=run.id,
project_id=run.project_id,
role=draft.role,
proposal_type=draft.proposal_type,
title=draft.title,
description=draft.description,
target_kind=draft.target_kind,
target_id=draft.target_id,
risk_level=draft.risk_level,
requires_human_approval=True,
status=MaintenanceProposalStatus.PENDING_REVIEW,
metadata_=draft.metadata or {},
)
self.db.add(proposal)
saved.append(proposal)
self.db.flush()
return saved
async def _broadcast_completed(self, project_id: str, run_id: str, proposal_count: int) -> dict[str, Any]:
if self.event_broadcaster is None:
return {"enabled": False, "sent": 0}
sent = await self.event_broadcaster.broadcast_notification(
org_id=project_id,
title="Maintenance run completed",
message=f"{proposal_count} proposals are pending review.",
severity="info",
)
return {"enabled": True, "sent": sent, "run_id": run_id}
def _find_duplicate_entities(self, entities: list[CandidateEntity]) -> list[dict[str, Any]]:
buckets: dict[str, list[CandidateEntity]] = defaultdict(list)
for entity in entities:
buckets[_normalize_label(entity.label)].append(entity)
duplicates: list[dict[str, Any]] = []
for label, members in buckets.items():
if len(members) < 2:
continue
canonical = sorted(members, key=lambda item: (-(item.confidence or 0.0), item.id))[0]
duplicates.append(
{
"label": label,
"canonical_id": canonical.id,
"duplicate_ids": [member.id for member in members if member.id != canonical.id],
"member_count": len(members),
}
)
return duplicates
def _label_for_target(self, context: dict[str, Any], target_id: str | None) -> str:
if not target_id:
return ""
for candidate in [*context["entities"], *context["relations"]]:
if candidate.id != target_id:
continue
if isinstance(candidate, CandidateEntity):
return candidate.label
return candidate.predicate
return target_id
def _require(self, role: str, permission: Permission) -> None:
if not self.rbac.has_permission(role, permission.value):
raise MaintenancePermissionError(f"Permission denied: {permission.value}")
def maintenance_run_to_dict(run: MaintenanceRun) -> dict[str, Any]:
return {
"id": run.id,
"project_id": run.project_id,
"status": run.status.value,
"requested_by": run.requested_by,
"started_at": run.started_at.isoformat() if run.started_at else None,
"completed_at": run.completed_at.isoformat() if run.completed_at else None,
"error_message": run.error_message,
"summary": run.summary or {},
"budget_summary": run.budget_summary or {},
"audit_summary": run.audit_summary or {},
"metadata": run.metadata_ or {},
}
def maintenance_proposal_to_dict(proposal: MaintenanceProposal) -> dict[str, Any]:
return {
"id": proposal.id,
"run_id": proposal.run_id,
"project_id": proposal.project_id,
"role": proposal.role.value,
"proposal_type": proposal.proposal_type,
"title": proposal.title,
"description": proposal.description,
"target_kind": proposal.target_kind,
"target_id": proposal.target_id,
"risk_level": proposal.risk_level,
"requires_human_approval": proposal.requires_human_approval,
"status": proposal.status.value,
"approved_by": proposal.approved_by,
"approved_at": proposal.approved_at.isoformat() if proposal.approved_at else None,
"rejection_reason": proposal.rejection_reason,
"metadata": proposal.metadata_ or {},
"created_at": proposal.created_at.isoformat() if proposal.created_at else None,
}
def _candidate_kind(candidate: CandidateEntity | CandidateRelation) -> str:
return CandidateKind.ENTITY.value if isinstance(candidate, CandidateEntity) else CandidateKind.RELATION.value
def _normalize_label(value: str) -> str:
return " ".join(value.lower().replace("_", " ").replace("-", " ").split())

5
ontology_platform/ont_platform/core/projection/__init__.py
View File

@@ -0,0 +1,5 @@
"""Projection helpers for Phase 5."""
from .rdf_to_neo4j import ProjectionContract, ProjectionResult, RDFToNeo4jProjector
__all__ = ["ProjectionContract", "ProjectionResult", "RDFToNeo4jProjector"]

87
ontology_platform/ont_platform/core/projection/rdf_to_neo4j.py Normal file
View File

@@ -0,0 +1,87 @@
"""RDF canonical store to Neo4j projection contract."""
from __future__ import annotations
import hashlib
from dataclasses import dataclass, field
from datetime import datetime
from typing import Any
from ont_platform.core.graph.rdf_converter import RDFToPropertyGraphConverter
@dataclass(frozen=True)
class ProjectionContract:
"""Declares store responsibility for Phase 5."""
canonical_store: str = "rdf_fuseki"
projection_store: str = "neo4j"
mode: str = "projection_search_only"
@dataclass
class ProjectionResult:
nodes: list[dict[str, Any]]
relationships: list[dict[str, Any]]
source_graph_hash: str
contract: ProjectionContract = field(default_factory=ProjectionContract)
warnings: list[str] = field(default_factory=list)
generated_at: str = field(default_factory=lambda: datetime.utcnow().isoformat())
def to_dict(self) -> dict[str, Any]:
return {
"contract": {
"canonical_store": self.contract.canonical_store,
"projection_store": self.contract.projection_store,
"mode": self.contract.mode,
},
"nodes": self.nodes,
"relationships": self.relationships,
"node_count": len(self.nodes),
"relationship_count": len(self.relationships),
"source_graph_hash": self.source_graph_hash,
"warnings": self.warnings,
"generated_at": self.generated_at,
}
class RDFToNeo4jProjector:
"""Builds Neo4j projection payloads from canonical RDF triples."""
def __init__(self, namespace_base: str = "http://example.org/", project_id: str | None = None):
self.namespace_base = namespace_base
self.project_id = project_id
async def preview_projection(
self,
triples: list[tuple[str, str, str]],
*,
provenance: dict[str, Any] | None = None,
) -> ProjectionResult:
converter = RDFToPropertyGraphConverter(
namespace_base=self.namespace_base,
project_id=self.project_id,
)
graph = await converter.convert_triples_to_graph(triples)
nodes = [
{**node, "store_role": "projection", "provenance": provenance or {}}
for node in graph["nodes"]
]
relationships = [
{**rel, "store_role": "projection", "provenance": provenance or {}}
for rel in graph["edges"]
]
return ProjectionResult(
nodes=nodes,
relationships=relationships,
source_graph_hash=_triples_hash(triples),
warnings=graph.get("warnings", []),
)
def _triples_hash(triples: list[tuple[str, str, str]]) -> str:
normalized = "\n".join("\t".join(triple) for triple in sorted(triples))
return hashlib.sha256(normalized.encode("utf-8")).hexdigest()
__all__ = ["ProjectionContract", "ProjectionResult", "RDFToNeo4jProjector"]

18
ontology_platform/ont_platform/core/review/__init__.py Normal file
View File

@@ -0,0 +1,18 @@
"""Phase 2 review queue and promotion services."""
from ont_platform.core.review.promotion import CandidatePromotionService, PromotionPlan
from ont_platform.core.review.review_service import (
EvidenceRequiredError,
InvalidReviewTransitionError,
ReviewPolicy,
ReviewService,
)
__all__ = [
"CandidatePromotionService",
"EvidenceRequiredError",
"InvalidReviewTransitionError",
"PromotionPlan",
"ReviewPolicy",
"ReviewService",
]

118
ontology_platform/ont_platform/core/review/promotion.py Normal file
View File

@@ -0,0 +1,118 @@
"""Promotion gate for moving reviewed candidates toward graph commit."""
from __future__ import annotations
from dataclasses import dataclass, field
from datetime import datetime
from typing import Any
from ont_platform.storage.candidate_repository import CandidateRepository
from ont_platform.storage.models import CandidateEntity, CandidateRelation, ReviewStatus
APPROVED_STATUSES = {ReviewStatus.APPROVED, ReviewStatus.AUTO_APPROVED}
@dataclass
class PromotionPlan:
"""Candidates allowed or blocked from graph commit."""
entities: list[CandidateEntity] = field(default_factory=list)
relations: list[CandidateRelation] = field(default_factory=list)
blocked: list[dict[str, Any]] = field(default_factory=list)
def to_dict(self) -> dict[str, Any]:
return {
"entity_count": len(self.entities),
"relation_count": len(self.relations),
"blocked_count": len(self.blocked),
"entities": [_candidate_to_dict(entity, "entity") for entity in self.entities],
"relations": [_candidate_to_dict(relation, "relation") for relation in self.relations],
"blocked": self.blocked,
}
class CandidatePromotionService:
"""Builds a commit plan while enforcing evidence provenance."""
def __init__(self, repository: CandidateRepository) -> None:
self.repository = repository
def build_commit_plan(self, *, project_id: str, mark_promoted: bool = False) -> PromotionPlan:
candidates = self.repository.list_candidates(project_id=project_id)
plan = PromotionPlan()
for entity in candidates["entities"]:
self._place_candidate(entity, "entity", plan)
for relation in candidates["relations"]:
self._place_candidate(relation, "relation", plan)
if mark_promoted:
now = datetime.utcnow()
for candidate in [*plan.entities, *plan.relations]:
candidate.promoted_at = now
self.repository.db.flush()
return plan
def _place_candidate(
self,
candidate: CandidateEntity | CandidateRelation,
kind: str,
plan: PromotionPlan,
) -> None:
status = ReviewStatus(candidate.review_status)
if status not in APPROVED_STATUSES:
return
if not self.repository.candidate_has_valid_evidence(candidate):
plan.blocked.append(
{
"candidate_kind": kind,
"candidate_id": candidate.id,
"reason": "missing_or_invalid_evidence",
"review_status": status.value,
}
)
return
if not bool(candidate.validation_passed):
plan.blocked.append(
{
"candidate_kind": kind,
"candidate_id": candidate.id,
"reason": "validation_failed",
"review_status": status.value,
}
)
return
if kind == "entity":
plan.entities.append(candidate)
else:
plan.relations.append(candidate)
def _candidate_to_dict(candidate: CandidateEntity | CandidateRelation, kind: str) -> dict[str, Any]:
common = {
"id": candidate.id,
"candidate_kind": kind,
"project_id": candidate.project_id,
"document_id": candidate.document_id,
"review_status": candidate.review_status.value,
"source_type": candidate.source_type.value,
"confidence": candidate.confidence,
"source_trust": candidate.source_trust,
"validation_passed": candidate.validation_passed,
"evidence_ids": candidate.evidence_ids or [],
}
if isinstance(candidate, CandidateEntity):
common.update({"label": candidate.label, "entity_type": candidate.entity_type})
else:
common.update(
{
"source_entity_id": candidate.source_entity_id,
"predicate": candidate.predicate,
"target_entity_id": candidate.target_entity_id,
}
)
return common
__all__ = ["CandidatePromotionService", "PromotionPlan"]

211
ontology_platform/ont_platform/core/review/review_service.py Normal file
View File

@@ -0,0 +1,211 @@
"""Review transition rules for Phase 2 candidates."""
from __future__ import annotations
from dataclasses import dataclass
from typing import Any
from ont_platform.storage.candidate_repository import CandidateRepository
from ont_platform.storage.models import (
CandidateEntity,
CandidateKind,
CandidateRelation,
ReviewDecision,
ReviewStatus,
)
class InvalidReviewTransitionError(ValueError):
"""Raised when a review status transition is not allowed."""
class EvidenceRequiredError(ValueError):
"""Raised when a candidate lacks valid evidence for approval."""
@dataclass(frozen=True)
class ReviewPolicy:
"""Policy for review transitions and automatic approval."""
auto_approve_min_confidence: float = 0.85
auto_approve_min_source_trust: float = 0.8
@property
def allowed_transitions(self) -> dict[ReviewStatus, set[ReviewStatus]]:
return {
ReviewStatus.PENDING: {
ReviewStatus.APPROVED,
ReviewStatus.AUTO_APPROVED,
ReviewStatus.REJECTED,
},
ReviewStatus.APPROVED: {ReviewStatus.REJECTED},
ReviewStatus.AUTO_APPROVED: {ReviewStatus.REJECTED},
ReviewStatus.REJECTED: set(),
}
def validate_transition(
self,
*,
current_status: ReviewStatus,
new_status: ReviewStatus,
) -> None:
allowed = self.allowed_transitions[current_status]
if new_status not in allowed:
raise InvalidReviewTransitionError(
f"Cannot transition candidate from {current_status.value} to {new_status.value}"
)
def qualifies_for_auto_approval(
self,
candidate: CandidateEntity | CandidateRelation,
) -> bool:
return (
bool(candidate.validation_passed)
and float(candidate.confidence or 0.0) >= self.auto_approve_min_confidence
and float(candidate.source_trust or 0.0) >= self.auto_approve_min_source_trust
)
class ReviewService:
"""Applies Phase 2 review rules over candidate repository records."""
def __init__(
self,
repository: CandidateRepository,
policy: ReviewPolicy | None = None,
) -> None:
self.repository = repository
self.policy = policy or ReviewPolicy()
def approve(
self,
*,
candidate_kind: CandidateKind | str,
candidate_id: str,
reviewed_by: str,
reason: str | None = None,
) -> ReviewDecision:
candidate, kind = self._get_candidate(candidate_kind, candidate_id)
self._validate_approval(candidate, ReviewStatus.APPROVED)
return self.repository.set_review_status(
candidate=candidate,
candidate_kind=kind,
new_status=ReviewStatus.APPROVED,
reviewed_by=reviewed_by,
reason=reason,
)
def reject(
self,
*,
candidate_kind: CandidateKind | str,
candidate_id: str,
reviewed_by: str,
reason: str | None = None,
) -> ReviewDecision:
candidate, kind = self._get_candidate(candidate_kind, candidate_id)
self.policy.validate_transition(
current_status=ReviewStatus(candidate.review_status),
new_status=ReviewStatus.REJECTED,
)
return self.repository.set_review_status(
candidate=candidate,
candidate_kind=kind,
new_status=ReviewStatus.REJECTED,
reviewed_by=reviewed_by,
reason=reason,
)
def auto_approve(
self,
*,
candidate_kind: CandidateKind | str,
candidate_id: str,
reviewed_by: str = "policy:auto_approve",
reason: str | None = None,
) -> ReviewDecision:
candidate, kind = self._get_candidate(candidate_kind, candidate_id)
self._validate_approval(candidate, ReviewStatus.AUTO_APPROVED)
if not self.policy.qualifies_for_auto_approval(candidate):
raise InvalidReviewTransitionError(
"Candidate does not satisfy auto-approval confidence, trust, and validation policy"
)
return self.repository.set_review_status(
candidate=candidate,
candidate_kind=kind,
new_status=ReviewStatus.AUTO_APPROVED,
reviewed_by=reviewed_by,
reason=reason,
metadata={
"auto_approve_min_confidence": self.policy.auto_approve_min_confidence,
"auto_approve_min_source_trust": self.policy.auto_approve_min_source_trust,
},
)
def bulk_approve(
self,
*,
candidate_kind: CandidateKind | str,
candidate_ids: list[str],
reviewed_by: str,
reason: str | None = None,
) -> list[ReviewDecision]:
return [
self.approve(
candidate_kind=candidate_kind,
candidate_id=candidate_id,
reviewed_by=reviewed_by,
reason=reason,
)
for candidate_id in candidate_ids
]
def _validate_approval(
self,
candidate: CandidateEntity | CandidateRelation,
new_status: ReviewStatus,
) -> None:
self.policy.validate_transition(
current_status=ReviewStatus(candidate.review_status),
new_status=new_status,
)
if not self.repository.candidate_has_valid_evidence(candidate):
raise EvidenceRequiredError(
f"Candidate {candidate.id} cannot be approved without valid evidence"
)
def _get_candidate(
self,
candidate_kind: CandidateKind | str,
candidate_id: str,
) -> tuple[CandidateEntity | CandidateRelation, CandidateKind]:
kind = CandidateKind(candidate_kind)
candidate = self.repository.get_candidate(
candidate_kind=kind,
candidate_id=candidate_id,
)
return candidate, kind
def review_decision_to_dict(decision: ReviewDecision) -> dict[str, Any]:
return {
"id": decision.id,
"project_id": decision.project_id,
"candidate_id": decision.candidate_id,
"candidate_kind": decision.candidate_kind.value,
"previous_status": decision.previous_status.value if decision.previous_status else None,
"new_status": decision.new_status.value,
"reviewed_by": decision.reviewed_by,
"reason": decision.reason,
"metadata": decision.metadata_ or {},
"created_at": decision.created_at.isoformat() if decision.created_at else None,
}
__all__ = [
"EvidenceRequiredError",
"InvalidReviewTransitionError",
"ReviewPolicy",
"ReviewService",
"review_decision_to_dict",
]

5
ontology_platform/ont_platform/core/validation/__init__.py
View File

@@ -12,9 +12,10 @@ from .models import (
OntologyExtractionResult,
Evidence,
EntityType,
ValidationIssueData,
)
from .guards import OntologyGuard, get_default_guard, validate
from .validators import BaseValidator, LightweightValidator, ValidatorFactory
from .validators import BaseValidator, GuardrailsFacadeValidator, LightweightValidator, ValidatorFactory
from .ontocast_validator import OntoCastValidator, SPARQLValidator, GraphUpdate
__all__ = [
@@ -24,12 +25,14 @@ __all__ = [
"OntologyExtractionResult",
"Evidence",
"EntityType",
"ValidationIssueData",
# Guards
"OntologyGuard",
"get_default_guard",
"validate",
# Validators
"BaseValidator",
"GuardrailsFacadeValidator",
"LightweightValidator",
"OntoCastValidator",
"SPARQLValidator",

9
ontology_platform/ont_platform/core/validation/guards.py
View File

@@ -8,7 +8,7 @@ Guardrails (Phase 3 upgraded) or OntoCast (Phase 3 Option B).
from typing import Optional
import logging
from .models import OntologyExtractionResult
from .models import OntologyExtractionResult, ValidationIssueData
from .validators import BaseValidator, ValidatorFactory
logger = logging.getLogger(__name__)
@@ -80,6 +80,13 @@ class OntologyGuard:
warnings=[f"Validation failed: {str(e)}"],
validation_passed=False,
validation_errors=[str(e)],
validation_issues=[
ValidationIssueData(
code="validator_exception",
message=str(e),
source=self.validator_type,
)
],
)

13
ontology_platform/ont_platform/core/validation/models.py
View File

@@ -27,6 +27,18 @@ class Evidence(BaseModel):
confidence: float = Field(default=0.8, ge=0.0, le=1.0)
class ValidationIssueData(BaseModel):
"""Structured validation issue suitable for review storage."""
severity: Literal["error", "warning"] = "error"
code: str = "validation_error"
message: str
candidate_id: str | None = None
candidate_kind: Literal["entity", "relation"] | None = None
source: str = "lightweight"
metadata: dict = Field(default_factory=dict)
class OntologyEntity(BaseModel):
"""Entity in ontology extraction result."""
id: str = Field(..., description="Unique entity ID (E_xxxxx)")
@@ -75,6 +87,7 @@ class OntologyExtractionResult(BaseModel):
warnings: List[str] = Field(default_factory=list)
validation_passed: bool = Field(default=True)
validation_errors: List[str] = Field(default_factory=list)
validation_issues: List[ValidationIssueData] = Field(default_factory=list)
@field_validator("relations")
@classmethod

61
ontology_platform/ont_platform/core/validation/validators.py
View File

@@ -10,7 +10,7 @@ from abc import ABC, abstractmethod
from typing import Optional, List, Tuple
from pydantic import ValidationError
from .models import OntologyExtractionResult, OntologyEntity, OntologyRelation
from .models import OntologyExtractionResult, OntologyEntity, OntologyRelation, ValidationIssueData
class BaseValidator(ABC):
@@ -61,6 +61,7 @@ class LightweightValidator(BaseValidator):
entities = []
relations = []
validation_errors = []
validation_issues: list[ValidationIssueData] = []
warnings = list(result.get("warnings", []))
# Phase 1: Validate entities
@@ -71,6 +72,15 @@ class LightweightValidator(BaseValidator):
except ValidationError as e:
error_msg = f"Entity {ent_dict.get('id', '?')}: {str(e)}"
validation_errors.append(error_msg)
validation_issues.append(
ValidationIssueData(
code="entity_schema_violation",
message=error_msg,
candidate_id=ent_dict.get("id"),
candidate_kind="entity",
metadata={"error_count": len(e.errors())},
)
)
if self.strict:
raise
warnings.append(error_msg)
@@ -85,10 +95,24 @@ class LightweightValidator(BaseValidator):
raise ValueError(f"Source entity {relation.source_id} not found")
if relation.target_id not in entity_ids:
raise ValueError(f"Target entity {relation.target_id} not found")
if relation.source_id == relation.target_id:
raise ValueError(f"Self-relation not allowed: {relation.id}")
relations.append(relation)
except (ValidationError, ValueError) as e:
error_msg = f"Relation {rel_dict.get('id', '?')}: {str(e)}"
validation_errors.append(error_msg)
validation_issues.append(
ValidationIssueData(
code=(
"relation_endpoint_missing"
if "not found" in str(e)
else "relation_schema_violation"
),
message=error_msg,
candidate_id=rel_dict.get("id"),
candidate_kind="relation",
)
)
if self.strict:
raise
warnings.append(error_msg)
@@ -99,6 +123,14 @@ class LightweightValidator(BaseValidator):
if duplicates:
error_msg = f"Duplicate entity IDs: {duplicates}"
validation_errors.append(error_msg)
validation_issues.append(
ValidationIssueData(
code="duplicate_entity_id",
message=error_msg,
candidate_kind="entity",
metadata={"duplicates": sorted(set(duplicates))},
)
)
warnings.append(error_msg)
# Phase 4: Check for meaningless entities
@@ -113,9 +145,31 @@ class LightweightValidator(BaseValidator):
warnings=warnings,
validation_passed=len(validation_errors) == 0,
validation_errors=validation_errors,
validation_issues=validation_issues,
)
class GuardrailsFacadeValidator(BaseValidator):
"""Guardrails-shaped facade with lightweight validation fallback.
The platform can install real ``guardrails-ai`` later without changing
OntoCast. For the current gate this facade provides the same policy
boundary and issue shape while avoiding Hub/telemetry side effects.
"""
def __init__(self, strict: bool = False, on_fail: str = "refrain"):
self.strict = strict
self.on_fail = on_fail
self.lightweight = LightweightValidator(strict=strict)
async def validate(self, result: dict) -> OntologyExtractionResult:
validated = await self.lightweight.validate(result)
for issue in validated.validation_issues:
issue.source = "guardrails_facade"
issue.metadata = {**issue.metadata, "on_fail": self.on_fail}
return validated
class ValidatorFactory:
"""Factory for creating validators (supports multiple implementations)."""
@@ -146,7 +200,10 @@ class ValidatorFactory:
strict=kwargs.get("strict", False),
)
elif validator_type == ValidatorFactory.GUARDRAILS:
raise NotImplementedError("Guardrails validator requires 'pip install guardrails-ai'")
return GuardrailsFacadeValidator(
strict=kwargs.get("strict", False),
on_fail=kwargs.get("on_fail", "refrain"),
)
elif validator_type == ValidatorFactory.ONTOCAST:
# Phase 3 Option B: OntoCast validator
from .ontocast_validator import OntoCastValidator

54
ontology_platform/ont_platform/storage/__init__.py
View File

@@ -1,7 +1,51 @@
"""Storage module (Phase 1+).
"""Storage module for source documents and candidate review queues."""
Phase 0: No database storage yet.
Phase 1: Add SQLAlchemy models for candidate storage.
"""
from ont_platform.storage.candidate_repository import (
CandidateBatch,
CandidateNotFoundError,
CandidateRepository,
)
from ont_platform.storage.models import (
Base,
CandidateEntity,
CandidateKind,
CandidateRelation,
CandidateSource,
EvidenceSpan,
MaintenanceProposal,
MaintenanceProposalStatus,
MaintenanceRole,
MaintenanceRun,
MaintenanceRunStatus,
ProjectionStatus,
ProjectionSyncState,
ReviewDecision,
ReviewStatus,
SourceDocument,
ValidationIssue,
ValidationSeverity,
)
__all__ = []
__all__ = [
"Base",
"CandidateBatch",
"CandidateEntity",
"CandidateKind",
"CandidateNotFoundError",
"CandidateRelation",
"CandidateRepository",
"CandidateSource",
"EvidenceSpan",
"MaintenanceProposal",
"MaintenanceProposalStatus",
"MaintenanceRole",
"MaintenanceRun",
"MaintenanceRunStatus",
"ProjectionStatus",
"ProjectionSyncState",
"ReviewDecision",
"ReviewStatus",
"SourceDocument",
"ValidationIssue",
"ValidationSeverity",
]

445
ontology_platform/ont_platform/storage/candidate_repository.py Normal file
View File

@@ -0,0 +1,445 @@
"""Repository for Phase 2 candidate and review queue storage."""
from __future__ import annotations
import uuid
from dataclasses import dataclass, field
from typing import Any
from sqlalchemy import select
from sqlalchemy.orm import Session
from ont_platform.core.extraction.lightweight_extractor import ExtractionResult
from ont_platform.storage.models import (
CandidateEntity,
CandidateKind,
CandidateRelation,
CandidateSource,
EvidenceSpan,
ReviewDecision,
ReviewStatus,
ValidationIssue,
ValidationSeverity,
)
@dataclass
class CandidateBatch:
"""Candidates persisted from one extraction result."""
entities: list[CandidateEntity] = field(default_factory=list)
relations: list[CandidateRelation] = field(default_factory=list)
evidence_spans: list[EvidenceSpan] = field(default_factory=list)
def to_dict(self) -> dict[str, Any]:
return {
"entity_count": len(self.entities),
"relation_count": len(self.relations),
"evidence_span_count": len(self.evidence_spans),
"entity_ids": [entity.id for entity in self.entities],
"relation_ids": [relation.id for relation in self.relations],
}
class CandidateNotFoundError(LookupError):
"""Raised when a candidate cannot be found."""
class CandidateRepository:
"""SQLAlchemy-backed review queue repository."""
def __init__(self, db: Session) -> None:
self.db = db
def save_lightweight_result(
self,
*,
project_id: str,
document_id: str,
result: ExtractionResult | dict[str, Any],
source_trust: float = 0.5,
validation_passed: bool = True,
) -> CandidateBatch:
payload = _result_to_dict(result)
return self._save_candidate_payload(
project_id=project_id,
document_id=document_id,
payload=payload,
source_type=CandidateSource.LIGHTWEIGHT,
created_by="lightweight",
source_trust=source_trust,
validation_passed=validation_passed,
)
def save_ontocast_result(
self,
*,
project_id: str,
document_id: str,
result: dict[str, Any],
source_trust: float = 0.7,
validation_passed: bool = False,
) -> CandidateBatch:
return self._save_candidate_payload(
project_id=project_id,
document_id=document_id,
payload=result,
source_type=CandidateSource.ONTOCAST,
created_by="ontocast",
source_trust=source_trust,
validation_passed=validation_passed,
)
def get_candidate(
self,
*,
candidate_kind: CandidateKind | str,
candidate_id: str,
) -> CandidateEntity | CandidateRelation:
kind = CandidateKind(candidate_kind)
model = _model_for_kind(kind)
candidate = self.db.get(model, candidate_id)
if candidate is None:
raise CandidateNotFoundError(f"{kind.value} candidate not found: {candidate_id}")
return candidate
def list_candidates(
self,
*,
project_id: str,
status: ReviewStatus | str | None = None,
source_type: CandidateSource | str | None = None,
) -> dict[str, list[CandidateEntity] | list[CandidateRelation]]:
entity_stmt = select(CandidateEntity).where(CandidateEntity.project_id == project_id)
relation_stmt = select(CandidateRelation).where(CandidateRelation.project_id == project_id)
if status is not None:
review_status = ReviewStatus(status)
entity_stmt = entity_stmt.where(CandidateEntity.review_status == review_status)
relation_stmt = relation_stmt.where(CandidateRelation.review_status == review_status)
if source_type is not None:
candidate_source = CandidateSource(source_type)
entity_stmt = entity_stmt.where(CandidateEntity.source_type == candidate_source)
relation_stmt = relation_stmt.where(CandidateRelation.source_type == candidate_source)
return {
"entities": list(self.db.scalars(entity_stmt.order_by(CandidateEntity.created_at))),
"relations": list(self.db.scalars(relation_stmt.order_by(CandidateRelation.created_at))),
}
def evidence_ids_exist(
self,
*,
project_id: str,
document_id: str,
evidence_ids: list[str],
) -> bool:
if not evidence_ids:
return False
stmt = select(EvidenceSpan.id).where(
EvidenceSpan.project_id == project_id,
EvidenceSpan.document_id == document_id,
EvidenceSpan.id.in_(evidence_ids),
)
found = set(self.db.scalars(stmt))
return found == set(evidence_ids)
def candidate_has_valid_evidence(self, candidate: CandidateEntity | CandidateRelation) -> bool:
evidence_ids = list(candidate.evidence_ids or [])
return self.evidence_ids_exist(
project_id=candidate.project_id,
document_id=candidate.document_id,
evidence_ids=evidence_ids,
)
def set_review_status(
self,
*,
candidate: CandidateEntity | CandidateRelation,
candidate_kind: CandidateKind,
new_status: ReviewStatus,
reviewed_by: str,
reason: str | None = None,
metadata: dict[str, Any] | None = None,
) -> ReviewDecision:
previous_status = candidate.review_status
candidate.review_status = new_status
candidate.reviewed_by = reviewed_by
candidate.reviewed_at = _utcnow()
candidate.review_reason = reason
decision = ReviewDecision(
id=f"decision_{uuid.uuid4().hex}",
project_id=candidate.project_id,
candidate_id=candidate.id,
candidate_kind=candidate_kind,
previous_status=previous_status,
new_status=new_status,
reviewed_by=reviewed_by,
reason=reason,
metadata_=metadata or {},
)
self.db.add(decision)
self.db.flush()
return decision
def review_history(
self,
*,
candidate_kind: CandidateKind | str,
candidate_id: str,
) -> list[ReviewDecision]:
kind = CandidateKind(candidate_kind)
stmt = (
select(ReviewDecision)
.where(
ReviewDecision.candidate_kind == kind,
ReviewDecision.candidate_id == candidate_id,
)
.order_by(ReviewDecision.created_at)
)
return list(self.db.scalars(stmt))
def record_validation_issues(
self,
*,
project_id: str,
document_id: str | None,
issues: list[dict[str, Any] | str],
candidate_id: str | None = None,
candidate_kind: CandidateKind | str | None = None,
source: str = "validation",
) -> list[ValidationIssue]:
"""Persist validation failures for review UI/API tracing."""
saved: list[ValidationIssue] = []
normalized_kind = CandidateKind(candidate_kind) if candidate_kind else None
for issue in issues:
issue_data = _normalize_validation_issue(issue)
model = ValidationIssue(
id=issue_data.get("id") or f"issue_{uuid.uuid4().hex}",
project_id=project_id,
document_id=document_id,
candidate_id=issue_data.get("candidate_id") or candidate_id,
candidate_kind=(
CandidateKind(issue_data["candidate_kind"])
if issue_data.get("candidate_kind")
else normalized_kind
),
severity=ValidationSeverity(issue_data.get("severity", "error")),
code=issue_data.get("code") or "validation_error",
message=issue_data.get("message") or str(issue),
source=issue_data.get("source") or source,
metadata_=issue_data.get("metadata") or {},
)
self.db.add(model)
saved.append(model)
self.db.flush()
return saved
def list_validation_issues(
self,
*,
project_id: str,
document_id: str | None = None,
candidate_id: str | None = None,
) -> list[ValidationIssue]:
stmt = select(ValidationIssue).where(ValidationIssue.project_id == project_id)
if document_id is not None:
stmt = stmt.where(ValidationIssue.document_id == document_id)
if candidate_id is not None:
stmt = stmt.where(ValidationIssue.candidate_id == candidate_id)
return list(self.db.scalars(stmt.order_by(ValidationIssue.created_at)))
def _save_candidate_payload(
self,
*,
project_id: str,
document_id: str,
payload: dict[str, Any],
source_type: CandidateSource,
created_by: str,
source_trust: float,
validation_passed: bool,
) -> CandidateBatch:
evidence_spans = self._save_evidence_spans(
project_id=project_id,
document_id=document_id,
spans=payload.get("evidence_spans") or [],
)
entities = [
self._save_entity(
project_id=project_id,
document_id=document_id,
entity=entity,
source_type=source_type,
created_by=created_by,
source_trust=source_trust,
validation_passed=validation_passed,
)
for entity in payload.get("entities") or []
]
relations = [
self._save_relation(
project_id=project_id,
document_id=document_id,
relation=relation,
source_type=source_type,
created_by=created_by,
source_trust=source_trust,
validation_passed=validation_passed,
)
for relation in payload.get("relations") or []
]
issue_payload = payload.get("validation_issues") or payload.get("validation_errors") or []
if issue_payload:
self.record_validation_issues(
project_id=project_id,
document_id=document_id,
issues=issue_payload,
source="candidate_ingest",
)
self.db.flush()
return CandidateBatch(
entities=entities,
relations=relations,
evidence_spans=evidence_spans,
)
def _save_evidence_spans(
self,
*,
project_id: str,
document_id: str,
spans: list[dict[str, Any]],
) -> list[EvidenceSpan]:
saved: list[EvidenceSpan] = []
for span in spans:
span_id = span.get("id") or f"ev_{uuid.uuid4().hex[:16]}"
existing = self.db.get(EvidenceSpan, span_id)
if existing is not None:
saved.append(existing)
continue
model = EvidenceSpan(
id=span_id,
document_id=span.get("document_id") or document_id,
project_id=span.get("project_id") or project_id,
text=span.get("text") or "",
start_offset=span.get("start_offset", 0),
end_offset=span.get("end_offset", 0),
)
self.db.add(model)
saved.append(model)
return saved
def _save_entity(
self,
*,
project_id: str,
document_id: str,
entity: dict[str, Any],
source_type: CandidateSource,
created_by: str,
source_trust: float,
validation_passed: bool,
) -> CandidateEntity:
entity_id = entity.get("id") or f"E_{uuid.uuid4().hex[:8]}"
existing = self.db.get(CandidateEntity, entity_id)
if existing is not None:
return existing
model = CandidateEntity(
id=entity_id,
project_id=project_id,
document_id=document_id,
label=entity.get("label") or entity.get("name") or entity_id,
entity_type=entity.get("entity_type") or entity.get("type") or "concept",
description=entity.get("description"),
source_type=source_type,
created_by=created_by,
confidence=float(entity.get("confidence", 0.5)),
source_trust=float(entity.get("source_trust", source_trust)),
validation_passed=bool(entity.get("validation_passed", validation_passed)),
evidence_ids=list(entity.get("evidence_ids") or []),
aliases=list(entity.get("aliases") or []),
review_status=ReviewStatus.PENDING,
metadata_={"source_type": source_type.value, "raw": entity},
)
self.db.add(model)
return model
def _save_relation(
self,
*,
project_id: str,
document_id: str,
relation: dict[str, Any],
source_type: CandidateSource,
created_by: str,
source_trust: float,
validation_passed: bool,
) -> CandidateRelation:
relation_id = relation.get("id") or f"R_{uuid.uuid4().hex[:8]}"
existing = self.db.get(CandidateRelation, relation_id)
if existing is not None:
return existing
model = CandidateRelation(
id=relation_id,
project_id=project_id,
document_id=document_id,
source_entity_id=relation.get("source_entity_id") or relation.get("source") or "",
predicate=relation.get("predicate") or relation.get("type") or "related_to",
target_entity_id=relation.get("target_entity_id") or relation.get("target") or "",
source_type=source_type,
created_by=created_by,
confidence=float(relation.get("confidence", 0.5)),
source_trust=float(relation.get("source_trust", source_trust)),
validation_passed=bool(relation.get("validation_passed", validation_passed)),
evidence_ids=list(relation.get("evidence_ids") or []),
review_status=ReviewStatus.PENDING,
metadata_={"source_type": source_type.value, "raw": relation},
)
self.db.add(model)
return model
def _result_to_dict(result: ExtractionResult | dict[str, Any]) -> dict[str, Any]:
if isinstance(result, dict):
return result
return {
"entities": result.entities,
"relations": result.relations,
"evidence_spans": result.evidence_spans,
"warnings": result.warnings,
}
def _model_for_kind(candidate_kind: CandidateKind):
return CandidateEntity if candidate_kind == CandidateKind.ENTITY else CandidateRelation
def _utcnow():
from datetime import datetime
return datetime.utcnow()
def _normalize_validation_issue(issue: dict[str, Any] | str) -> dict[str, Any]:
if isinstance(issue, dict):
data = dict(issue)
if "msg" in data and "message" not in data:
data["message"] = data["msg"]
return data
return {
"severity": "error",
"code": "validation_error",
"message": issue,
}
__all__ = [
"CandidateBatch",
"CandidateNotFoundError",
"CandidateRepository",
]

88
ontology_platform/ont_platform/storage/dedup_cache.py Normal file
View File

@@ -0,0 +1,88 @@
"""Phase 1 in-memory document deduplication cache."""
from __future__ import annotations
from dataclasses import dataclass
from threading import RLock
@dataclass(frozen=True)
class DedupResult:
"""Result of checking whether a source document was already seen."""
is_duplicate: bool
key: str
document_id: str
existing_document_id: str | None = None
@property
def skipped_processing(self) -> bool:
return self.is_duplicate
def to_dict(self) -> dict[str, str | bool | None]:
return {
"is_duplicate": self.is_duplicate,
"key": self.key,
"document_id": self.document_id,
"existing_document_id": self.existing_document_id,
"skipped_processing": self.skipped_processing,
}
class InMemoryDedupCache:
"""Small process-local cache used until Phase 2 introduces durable storage."""
def __init__(self) -> None:
self._lock = RLock()
self._seen: dict[str, str] = {}
def check_and_remember(
self,
*,
project_id: str,
document_id: str,
content_hash: str,
fingerprint: str | None = None,
) -> DedupResult:
if not content_hash and not fingerprint:
raise ValueError("content_hash or fingerprint is required")
key_value = fingerprint or content_hash
key = f"{project_id}:{key_value}"
with self._lock:
existing_document_id = self._seen.get(key)
if existing_document_id is not None:
return DedupResult(
is_duplicate=True,
key=key,
document_id=document_id,
existing_document_id=existing_document_id,
)
self._seen[key] = document_id
return DedupResult(is_duplicate=False, key=key, document_id=document_id)
def clear(self) -> None:
with self._lock:
self._seen.clear()
def __len__(self) -> int:
return len(self._seen)
_DEFAULT_CACHE = InMemoryDedupCache()
def get_default_dedup_cache() -> InMemoryDedupCache:
return _DEFAULT_CACHE
def reset_default_dedup_cache() -> None:
_DEFAULT_CACHE.clear()
__all__ = [
"DedupResult",
"InMemoryDedupCache",
"get_default_dedup_cache",
"reset_default_dedup_cache",
]

195
ontology_platform/ont_platform/storage/models.py
View File

@@ -5,16 +5,16 @@ Holds extracted entity/relation candidates before final RDF conversion.
"""
from datetime import datetime
from enum import Enum
from typing import Any
from enum import StrEnum
from sqlalchemy import JSON, Column, DateTime, Float, Integer, String, Text, Enum as SQLEnum
from sqlalchemy import JSON, Boolean, Column, DateTime, Float, Integer, String, Text
from sqlalchemy import Enum as SQLEnum
from sqlalchemy.orm import declarative_base
Base = declarative_base()
class ReviewStatus(str, Enum):
class ReviewStatus(StrEnum):
"""Review status of a candidate."""
PENDING = "pending" # Awaiting human review
@@ -23,6 +23,63 @@ class ReviewStatus(str, Enum):
REJECTED = "rejected" # Rejected by human
class CandidateSource(StrEnum):
"""Source path that produced a candidate."""
LIGHTWEIGHT = "lightweight"
ONTOCAST = "ontocast"
class CandidateKind(StrEnum):
"""Reviewable candidate kind."""
ENTITY = "entity"
RELATION = "relation"
class ValidationSeverity(StrEnum):
"""Severity of validation issue."""
ERROR = "error"
WARNING = "warning"
class ProjectionStatus(StrEnum):
"""Status of an RDF-to-Neo4j projection sync."""
PENDING = "pending"
RUNNING = "running"
COMPLETED = "completed"
FAILED = "failed"
class MaintenanceRunStatus(StrEnum):
"""Status of a maintenance loop run."""
RUNNING = "running"
COMPLETED = "completed"
FAILED = "failed"
class MaintenanceProposalStatus(StrEnum):
"""Human review state for maintenance proposals."""
PENDING_REVIEW = "pending_review"
APPROVED = "approved"
REJECTED = "rejected"
class MaintenanceRole(StrEnum):
"""Maintenance loop role name."""
ANALYST = "analyst"
RESEARCHER = "researcher"
CURATOR = "curator"
AUDITOR = "auditor"
FIXER = "fixer"
ADVISOR = "advisor"
class SourceDocument(Base):
"""Source document metadata."""
@@ -31,6 +88,7 @@ class SourceDocument(Base):
id = Column(String(255), primary_key=True)
project_id = Column(String(255), nullable=False, index=True)
source_url = Column(String(2048), nullable=True, index=True)
canonical_url = Column(String(2048), nullable=True)
file_path = Column(String(2048), nullable=True)
document_type = Column(String(50)) # "html", "pdf", "markdown", "docx", "inline_text"
@@ -39,15 +97,18 @@ class SourceDocument(Base):
publish_date = Column(String(50), nullable=True) # ISO-8601
language = Column(String(10), nullable=True)
sitename = Column(String(255), nullable=True)
description = Column(Text, nullable=True)
text = Column(Text)
raw_html = Column(Text, nullable=True)
body_xml = Column(Text, nullable=True)
content_hash = Column(String(64), unique=True, nullable=False, index=True)
fingerprint = Column(String(100), nullable=True, index=True)
retrieved_at = Column(DateTime, default=datetime.utcnow)
extracted_by = Column(String(100), default="trafilatura") # Source tool
metadata = Column(JSON, nullable=True) # Raw metadata
metadata_ = Column("metadata", JSON, nullable=True) # Raw metadata
created_at = Column(DateTime, default=datetime.utcnow)
@@ -80,9 +141,12 @@ class CandidateEntity(Base):
label = Column(String(512), nullable=False)
entity_type = Column(String(100), nullable=False) # "concept", "person", "org", etc.
description = Column(Text, nullable=True)
source_type = Column(SQLEnum(CandidateSource), default=CandidateSource.LIGHTWEIGHT, nullable=False, index=True)
created_by = Column(String(100), default="lightweight", nullable=False)
confidence = Column(Float, default=0.5) # 0.0 ~ 1.0
source_trust = Column(Float, default=0.5) # Trust in source
validation_passed = Column(Boolean, default=False, nullable=False)
evidence_ids = Column(JSON, nullable=True) # List of evidence span IDs
aliases = Column(JSON, nullable=True) # List of alternative names
@@ -92,10 +156,11 @@ class CandidateEntity(Base):
reviewed_at = Column(DateTime, nullable=True)
review_reason = Column(Text, nullable=True)
metadata = Column(JSON, nullable=True) # Raw LLM output, domain-specific fields
metadata_ = Column("metadata", JSON, nullable=True) # Raw LLM output, domain-specific fields
created_at = Column(DateTime, default=datetime.utcnow)
updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)
promoted_at = Column(DateTime, nullable=True)
class CandidateRelation(Base):
@@ -110,9 +175,12 @@ class CandidateRelation(Base):
source_entity_id = Column(String(255), nullable=False, index=True)
predicate = Column(String(255), nullable=False)
target_entity_id = Column(String(255), nullable=False, index=True)
source_type = Column(SQLEnum(CandidateSource), default=CandidateSource.LIGHTWEIGHT, nullable=False, index=True)
created_by = Column(String(100), default="lightweight", nullable=False)
confidence = Column(Float, default=0.5) # 0.0 ~ 1.0
source_trust = Column(Float, default=0.5)
validation_passed = Column(Boolean, default=False, nullable=False)
evidence_ids = Column(JSON, nullable=True) # List of evidence span IDs
@@ -121,10 +189,51 @@ class CandidateRelation(Base):
reviewed_at = Column(DateTime, nullable=True)
review_reason = Column(Text, nullable=True)
metadata = Column(JSON, nullable=True) # Raw LLM output
metadata_ = Column("metadata", JSON, nullable=True) # Raw LLM output
created_at = Column(DateTime, default=datetime.utcnow)
updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)
promoted_at = Column(DateTime, nullable=True)
class ReviewDecision(Base):
"""Audit trail for review status changes."""
__tablename__ = "review_decisions"
id = Column(String(255), primary_key=True)
project_id = Column(String(255), nullable=False, index=True)
candidate_id = Column(String(255), nullable=False, index=True)
candidate_kind = Column(SQLEnum(CandidateKind), nullable=False, index=True)
previous_status = Column(SQLEnum(ReviewStatus), nullable=True)
new_status = Column(SQLEnum(ReviewStatus), nullable=False, index=True)
reviewed_by = Column(String(255), nullable=False)
reason = Column(Text, nullable=True)
metadata_ = Column("metadata", JSON, nullable=True)
created_at = Column(DateTime, default=datetime.utcnow)
class ValidationIssue(Base):
"""Structured validation issue stored for review and audit."""
__tablename__ = "validation_issues"
id = Column(String(255), primary_key=True)
project_id = Column(String(255), nullable=False, index=True)
document_id = Column(String(255), nullable=True, index=True)
candidate_id = Column(String(255), nullable=True, index=True)
candidate_kind = Column(SQLEnum(CandidateKind), nullable=True, index=True)
severity = Column(SQLEnum(ValidationSeverity), default=ValidationSeverity.ERROR, nullable=False)
code = Column(String(100), nullable=False, index=True)
message = Column(Text, nullable=False)
source = Column(String(100), default="validation", nullable=False)
metadata_ = Column("metadata", JSON, nullable=True)
created_at = Column(DateTime, default=datetime.utcnow)
class ExtractionJob(Base):
@@ -149,6 +258,76 @@ class ExtractionJob(Base):
started_at = Column(DateTime, nullable=True)
completed_at = Column(DateTime, nullable=True)
metadata = Column(JSON, nullable=True)
metadata_ = Column("metadata", JSON, nullable=True)
created_at = Column(DateTime, default=datetime.utcnow)
class ProjectionSyncState(Base):
"""RDF canonical store to Neo4j projection/search sync state."""
__tablename__ = "projection_sync_states"
id = Column(String(255), primary_key=True)
project_id = Column(String(255), nullable=False, index=True)
canonical_store = Column(String(100), default="rdf_fuseki", nullable=False)
projection_store = Column(String(100), default="neo4j", nullable=False)
status = Column(SQLEnum(ProjectionStatus), default=ProjectionStatus.PENDING, index=True)
last_sync_at = Column(DateTime, nullable=True)
source_graph_hash = Column(String(128), nullable=True, index=True)
error_message = Column(Text, nullable=True)
metadata_ = Column("metadata", JSON, nullable=True)
created_at = Column(DateTime, default=datetime.utcnow)
updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)
class MaintenanceRun(Base):
"""One non-destructive maintenance loop run."""
__tablename__ = "maintenance_runs"
id = Column(String(255), primary_key=True)
project_id = Column(String(255), nullable=False, index=True)
status = Column(SQLEnum(MaintenanceRunStatus), default=MaintenanceRunStatus.RUNNING, index=True)
requested_by = Column(String(255), default="system", nullable=False)
started_at = Column(DateTime, default=datetime.utcnow)
completed_at = Column(DateTime, nullable=True)
error_message = Column(Text, nullable=True)
summary = Column(JSON, nullable=True)
budget_summary = Column(JSON, nullable=True)
audit_summary = Column(JSON, nullable=True)
metadata_ = Column("metadata", JSON, nullable=True)
created_at = Column(DateTime, default=datetime.utcnow)
class MaintenanceProposal(Base):
"""Proposal created by the maintenance loop.
Proposals do not mutate graph/candidate state. They must be reviewed and
approved before any downstream execution layer can act on them.
"""
__tablename__ = "maintenance_proposals"
id = Column(String(255), primary_key=True)
run_id = Column(String(255), nullable=False, index=True)
project_id = Column(String(255), nullable=False, index=True)
role = Column(SQLEnum(MaintenanceRole), nullable=False, index=True)
proposal_type = Column(String(100), nullable=False, index=True)
title = Column(String(512), nullable=False)
description = Column(Text, nullable=True)
target_kind = Column(String(100), nullable=True, index=True)
target_id = Column(String(255), nullable=True, index=True)
risk_level = Column(String(50), default="low", nullable=False)
requires_human_approval = Column(Boolean, default=True, nullable=False)
status = Column(
SQLEnum(MaintenanceProposalStatus),
default=MaintenanceProposalStatus.PENDING_REVIEW,
index=True,
)
approved_by = Column(String(255), nullable=True)
approved_at = Column(DateTime, nullable=True)
rejection_reason = Column(Text, nullable=True)
metadata_ = Column("metadata", JSON, nullable=True)
created_at = Column(DateTime, default=datetime.utcnow)
updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)