334 lines
12 KiB
Python
334 lines
12 KiB
Python
"""
|
|
Database models for candidate storage.
|
|
|
|
Holds extracted entity/relation candidates before final RDF conversion.
|
|
"""
|
|
|
|
from datetime import datetime
|
|
from enum import StrEnum
|
|
|
|
from sqlalchemy import JSON, Boolean, Column, DateTime, Float, Integer, String, Text
|
|
from sqlalchemy import Enum as SQLEnum
|
|
from sqlalchemy.orm import declarative_base
|
|
|
|
Base = declarative_base()
|
|
|
|
|
|
class ReviewStatus(StrEnum):
|
|
"""Review status of a candidate."""
|
|
|
|
PENDING = "pending" # Awaiting human review
|
|
APPROVED = "approved" # Approved by human
|
|
AUTO_APPROVED = "auto_approved" # Approved by policy
|
|
REJECTED = "rejected" # Rejected by human
|
|
|
|
|
|
class CandidateSource(StrEnum):
|
|
"""Source path that produced a candidate."""
|
|
|
|
LIGHTWEIGHT = "lightweight"
|
|
ONTOCAST = "ontocast"
|
|
|
|
|
|
class CandidateKind(StrEnum):
|
|
"""Reviewable candidate kind."""
|
|
|
|
ENTITY = "entity"
|
|
RELATION = "relation"
|
|
|
|
|
|
class ValidationSeverity(StrEnum):
|
|
"""Severity of validation issue."""
|
|
|
|
ERROR = "error"
|
|
WARNING = "warning"
|
|
|
|
|
|
class ProjectionStatus(StrEnum):
|
|
"""Status of an RDF-to-Neo4j projection sync."""
|
|
|
|
PENDING = "pending"
|
|
RUNNING = "running"
|
|
COMPLETED = "completed"
|
|
FAILED = "failed"
|
|
|
|
|
|
class MaintenanceRunStatus(StrEnum):
|
|
"""Status of a maintenance loop run."""
|
|
|
|
RUNNING = "running"
|
|
COMPLETED = "completed"
|
|
FAILED = "failed"
|
|
|
|
|
|
class MaintenanceProposalStatus(StrEnum):
|
|
"""Human review state for maintenance proposals."""
|
|
|
|
PENDING_REVIEW = "pending_review"
|
|
APPROVED = "approved"
|
|
REJECTED = "rejected"
|
|
|
|
|
|
class MaintenanceRole(StrEnum):
|
|
"""Maintenance loop role name."""
|
|
|
|
ANALYST = "analyst"
|
|
RESEARCHER = "researcher"
|
|
CURATOR = "curator"
|
|
AUDITOR = "auditor"
|
|
FIXER = "fixer"
|
|
ADVISOR = "advisor"
|
|
|
|
|
|
class SourceDocument(Base):
|
|
"""Source document metadata."""
|
|
|
|
__tablename__ = "source_documents"
|
|
|
|
id = Column(String(255), primary_key=True)
|
|
project_id = Column(String(255), nullable=False, index=True)
|
|
source_url = Column(String(2048), nullable=True, index=True)
|
|
canonical_url = Column(String(2048), nullable=True)
|
|
file_path = Column(String(2048), nullable=True)
|
|
document_type = Column(String(50)) # "html", "pdf", "markdown", "docx", "inline_text"
|
|
|
|
title = Column(String(512), nullable=True)
|
|
author = Column(String(255), nullable=True)
|
|
publish_date = Column(String(50), nullable=True) # ISO-8601
|
|
language = Column(String(10), nullable=True)
|
|
sitename = Column(String(255), nullable=True)
|
|
description = Column(Text, nullable=True)
|
|
|
|
text = Column(Text)
|
|
raw_html = Column(Text, nullable=True)
|
|
body_xml = Column(Text, nullable=True)
|
|
content_hash = Column(String(64), unique=True, nullable=False, index=True)
|
|
fingerprint = Column(String(100), nullable=True, index=True)
|
|
|
|
retrieved_at = Column(DateTime, default=datetime.utcnow)
|
|
extracted_by = Column(String(100), default="trafilatura") # Source tool
|
|
|
|
metadata_ = Column("metadata", JSON, nullable=True) # Raw metadata
|
|
|
|
created_at = Column(DateTime, default=datetime.utcnow)
|
|
|
|
|
|
class EvidenceSpan(Base):
|
|
"""Evidence text span from source document."""
|
|
|
|
__tablename__ = "evidence_spans"
|
|
|
|
id = Column(String(255), primary_key=True)
|
|
document_id = Column(String(255), nullable=False, index=True)
|
|
project_id = Column(String(255), nullable=False, index=True)
|
|
|
|
text = Column(Text)
|
|
start_offset = Column(Integer)
|
|
end_offset = Column(Integer)
|
|
|
|
created_at = Column(DateTime, default=datetime.utcnow)
|
|
|
|
|
|
class CandidateEntity(Base):
|
|
"""Extracted entity candidate."""
|
|
|
|
__tablename__ = "candidate_entities"
|
|
|
|
id = Column(String(255), primary_key=True)
|
|
project_id = Column(String(255), nullable=False, index=True)
|
|
document_id = Column(String(255), nullable=False, index=True)
|
|
|
|
label = Column(String(512), nullable=False)
|
|
entity_type = Column(String(100), nullable=False) # "concept", "person", "org", etc.
|
|
description = Column(Text, nullable=True)
|
|
source_type = Column(SQLEnum(CandidateSource), default=CandidateSource.LIGHTWEIGHT, nullable=False, index=True)
|
|
created_by = Column(String(100), default="lightweight", nullable=False)
|
|
|
|
confidence = Column(Float, default=0.5) # 0.0 ~ 1.0
|
|
source_trust = Column(Float, default=0.5) # Trust in source
|
|
validation_passed = Column(Boolean, default=False, nullable=False)
|
|
|
|
evidence_ids = Column(JSON, nullable=True) # List of evidence span IDs
|
|
aliases = Column(JSON, nullable=True) # List of alternative names
|
|
|
|
review_status = Column(SQLEnum(ReviewStatus), default=ReviewStatus.PENDING, index=True)
|
|
reviewed_by = Column(String(255), nullable=True)
|
|
reviewed_at = Column(DateTime, nullable=True)
|
|
review_reason = Column(Text, nullable=True)
|
|
|
|
metadata_ = Column("metadata", JSON, nullable=True) # Raw LLM output, domain-specific fields
|
|
|
|
created_at = Column(DateTime, default=datetime.utcnow)
|
|
updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)
|
|
promoted_at = Column(DateTime, nullable=True)
|
|
|
|
|
|
class CandidateRelation(Base):
|
|
"""Extracted relation candidate."""
|
|
|
|
__tablename__ = "candidate_relations"
|
|
|
|
id = Column(String(255), primary_key=True)
|
|
project_id = Column(String(255), nullable=False, index=True)
|
|
document_id = Column(String(255), nullable=False, index=True)
|
|
|
|
source_entity_id = Column(String(255), nullable=False, index=True)
|
|
predicate = Column(String(255), nullable=False)
|
|
target_entity_id = Column(String(255), nullable=False, index=True)
|
|
source_type = Column(SQLEnum(CandidateSource), default=CandidateSource.LIGHTWEIGHT, nullable=False, index=True)
|
|
created_by = Column(String(100), default="lightweight", nullable=False)
|
|
|
|
confidence = Column(Float, default=0.5) # 0.0 ~ 1.0
|
|
source_trust = Column(Float, default=0.5)
|
|
validation_passed = Column(Boolean, default=False, nullable=False)
|
|
|
|
evidence_ids = Column(JSON, nullable=True) # List of evidence span IDs
|
|
|
|
review_status = Column(SQLEnum(ReviewStatus), default=ReviewStatus.PENDING, index=True)
|
|
reviewed_by = Column(String(255), nullable=True)
|
|
reviewed_at = Column(DateTime, nullable=True)
|
|
review_reason = Column(Text, nullable=True)
|
|
|
|
metadata_ = Column("metadata", JSON, nullable=True) # Raw LLM output
|
|
|
|
created_at = Column(DateTime, default=datetime.utcnow)
|
|
updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)
|
|
promoted_at = Column(DateTime, nullable=True)
|
|
|
|
|
|
class ReviewDecision(Base):
|
|
"""Audit trail for review status changes."""
|
|
|
|
__tablename__ = "review_decisions"
|
|
|
|
id = Column(String(255), primary_key=True)
|
|
project_id = Column(String(255), nullable=False, index=True)
|
|
candidate_id = Column(String(255), nullable=False, index=True)
|
|
candidate_kind = Column(SQLEnum(CandidateKind), nullable=False, index=True)
|
|
|
|
previous_status = Column(SQLEnum(ReviewStatus), nullable=True)
|
|
new_status = Column(SQLEnum(ReviewStatus), nullable=False, index=True)
|
|
|
|
reviewed_by = Column(String(255), nullable=False)
|
|
reason = Column(Text, nullable=True)
|
|
metadata_ = Column("metadata", JSON, nullable=True)
|
|
|
|
created_at = Column(DateTime, default=datetime.utcnow)
|
|
|
|
|
|
class ValidationIssue(Base):
|
|
"""Structured validation issue stored for review and audit."""
|
|
|
|
__tablename__ = "validation_issues"
|
|
|
|
id = Column(String(255), primary_key=True)
|
|
project_id = Column(String(255), nullable=False, index=True)
|
|
document_id = Column(String(255), nullable=True, index=True)
|
|
candidate_id = Column(String(255), nullable=True, index=True)
|
|
candidate_kind = Column(SQLEnum(CandidateKind), nullable=True, index=True)
|
|
|
|
severity = Column(SQLEnum(ValidationSeverity), default=ValidationSeverity.ERROR, nullable=False)
|
|
code = Column(String(100), nullable=False, index=True)
|
|
message = Column(Text, nullable=False)
|
|
source = Column(String(100), default="validation", nullable=False)
|
|
|
|
metadata_ = Column("metadata", JSON, nullable=True)
|
|
created_at = Column(DateTime, default=datetime.utcnow)
|
|
|
|
|
|
class ExtractionJob(Base):
|
|
"""Extraction job metadata."""
|
|
|
|
__tablename__ = "extraction_jobs"
|
|
|
|
id = Column(String(255), primary_key=True)
|
|
project_id = Column(String(255), nullable=False, index=True)
|
|
|
|
job_type = Column(String(50)) # "extract", "validate", "review", etc.
|
|
status = Column(String(50), index=True) # "pending", "running", "completed", "failed"
|
|
|
|
input_url = Column(String(2048), nullable=True)
|
|
input_file = Column(String(2048), nullable=True)
|
|
|
|
document_id = Column(String(255), nullable=True)
|
|
entity_count = Column(Integer, default=0)
|
|
relation_count = Column(Integer, default=0)
|
|
error_message = Column(Text, nullable=True)
|
|
|
|
started_at = Column(DateTime, nullable=True)
|
|
completed_at = Column(DateTime, nullable=True)
|
|
|
|
metadata_ = Column("metadata", JSON, nullable=True)
|
|
|
|
created_at = Column(DateTime, default=datetime.utcnow)
|
|
|
|
|
|
class ProjectionSyncState(Base):
|
|
"""RDF canonical store to Neo4j projection/search sync state."""
|
|
|
|
__tablename__ = "projection_sync_states"
|
|
|
|
id = Column(String(255), primary_key=True)
|
|
project_id = Column(String(255), nullable=False, index=True)
|
|
canonical_store = Column(String(100), default="rdf_fuseki", nullable=False)
|
|
projection_store = Column(String(100), default="neo4j", nullable=False)
|
|
status = Column(SQLEnum(ProjectionStatus), default=ProjectionStatus.PENDING, index=True)
|
|
last_sync_at = Column(DateTime, nullable=True)
|
|
source_graph_hash = Column(String(128), nullable=True, index=True)
|
|
error_message = Column(Text, nullable=True)
|
|
metadata_ = Column("metadata", JSON, nullable=True)
|
|
created_at = Column(DateTime, default=datetime.utcnow)
|
|
updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)
|
|
|
|
|
|
class MaintenanceRun(Base):
|
|
"""One non-destructive maintenance loop run."""
|
|
|
|
__tablename__ = "maintenance_runs"
|
|
|
|
id = Column(String(255), primary_key=True)
|
|
project_id = Column(String(255), nullable=False, index=True)
|
|
status = Column(SQLEnum(MaintenanceRunStatus), default=MaintenanceRunStatus.RUNNING, index=True)
|
|
requested_by = Column(String(255), default="system", nullable=False)
|
|
started_at = Column(DateTime, default=datetime.utcnow)
|
|
completed_at = Column(DateTime, nullable=True)
|
|
error_message = Column(Text, nullable=True)
|
|
summary = Column(JSON, nullable=True)
|
|
budget_summary = Column(JSON, nullable=True)
|
|
audit_summary = Column(JSON, nullable=True)
|
|
metadata_ = Column("metadata", JSON, nullable=True)
|
|
created_at = Column(DateTime, default=datetime.utcnow)
|
|
|
|
|
|
class MaintenanceProposal(Base):
|
|
"""Proposal created by the maintenance loop.
|
|
|
|
Proposals do not mutate graph/candidate state. They must be reviewed and
|
|
approved before any downstream execution layer can act on them.
|
|
"""
|
|
|
|
__tablename__ = "maintenance_proposals"
|
|
|
|
id = Column(String(255), primary_key=True)
|
|
run_id = Column(String(255), nullable=False, index=True)
|
|
project_id = Column(String(255), nullable=False, index=True)
|
|
role = Column(SQLEnum(MaintenanceRole), nullable=False, index=True)
|
|
proposal_type = Column(String(100), nullable=False, index=True)
|
|
title = Column(String(512), nullable=False)
|
|
description = Column(Text, nullable=True)
|
|
target_kind = Column(String(100), nullable=True, index=True)
|
|
target_id = Column(String(255), nullable=True, index=True)
|
|
risk_level = Column(String(50), default="low", nullable=False)
|
|
requires_human_approval = Column(Boolean, default=True, nullable=False)
|
|
status = Column(
|
|
SQLEnum(MaintenanceProposalStatus),
|
|
default=MaintenanceProposalStatus.PENDING_REVIEW,
|
|
index=True,
|
|
)
|
|
approved_by = Column(String(255), nullable=True)
|
|
approved_at = Column(DateTime, nullable=True)
|
|
rejection_reason = Column(Text, nullable=True)
|
|
metadata_ = Column("metadata", JSON, nullable=True)
|
|
created_at = Column(DateTime, default=datetime.utcnow)
|
|
updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)
|