AI/ontology_platform/ont_platform/storage/models.py

"""
Database models for candidate storage.

Holds extracted entity/relation candidates before final RDF conversion.
"""

from datetime import datetime
from enum import StrEnum

from sqlalchemy import JSON, Boolean, Column, DateTime, Float, Integer, String, Text
from sqlalchemy import Enum as SQLEnum
from sqlalchemy.orm import declarative_base

Base = declarative_base()


class ReviewStatus(StrEnum):
    """Review status of a candidate."""

    PENDING = "pending"  # Awaiting human review
    APPROVED = "approved"  # Approved by human
    AUTO_APPROVED = "auto_approved"  # Approved by policy
    REJECTED = "rejected"  # Rejected by human


class CandidateSource(StrEnum):
    """Source path that produced a candidate."""

    LIGHTWEIGHT = "lightweight"
    ONTOCAST = "ontocast"


class CandidateKind(StrEnum):
    """Reviewable candidate kind."""

    ENTITY = "entity"
    RELATION = "relation"


class ValidationSeverity(StrEnum):
    """Severity of validation issue."""

    ERROR = "error"
    WARNING = "warning"


class ProjectionStatus(StrEnum):
    """Status of an RDF-to-Neo4j projection sync."""

    PENDING = "pending"
    RUNNING = "running"
    COMPLETED = "completed"
    FAILED = "failed"


class MaintenanceRunStatus(StrEnum):
    """Status of a maintenance loop run."""

    RUNNING = "running"
    COMPLETED = "completed"
    FAILED = "failed"


class MaintenanceProposalStatus(StrEnum):
    """Human review state for maintenance proposals."""

    PENDING_REVIEW = "pending_review"
    APPROVED = "approved"
    REJECTED = "rejected"


class MaintenanceRole(StrEnum):
    """Maintenance loop role name."""

    ANALYST = "analyst"
    RESEARCHER = "researcher"
    CURATOR = "curator"
    AUDITOR = "auditor"
    FIXER = "fixer"
    ADVISOR = "advisor"


class SourceDocument(Base):
    """Source document metadata."""

    __tablename__ = "source_documents"

    id = Column(String(255), primary_key=True)
    project_id = Column(String(255), nullable=False, index=True)
    source_url = Column(String(2048), nullable=True, index=True)
    canonical_url = Column(String(2048), nullable=True)
    file_path = Column(String(2048), nullable=True)
    document_type = Column(String(50))  # "html", "pdf", "markdown", "docx", "inline_text"

    title = Column(String(512), nullable=True)
    author = Column(String(255), nullable=True)
    publish_date = Column(String(50), nullable=True)  # ISO-8601
    language = Column(String(10), nullable=True)
    sitename = Column(String(255), nullable=True)
    description = Column(Text, nullable=True)

    text = Column(Text)
    raw_html = Column(Text, nullable=True)
    body_xml = Column(Text, nullable=True)
    content_hash = Column(String(64), unique=True, nullable=False, index=True)
    fingerprint = Column(String(100), nullable=True, index=True)

    retrieved_at = Column(DateTime, default=datetime.utcnow)
    extracted_by = Column(String(100), default="trafilatura")  # Source tool

    metadata_ = Column("metadata", JSON, nullable=True)  # Raw metadata

    created_at = Column(DateTime, default=datetime.utcnow)


class EvidenceSpan(Base):
    """Evidence text span from source document."""

    __tablename__ = "evidence_spans"

    id = Column(String(255), primary_key=True)
    document_id = Column(String(255), nullable=False, index=True)
    project_id = Column(String(255), nullable=False, index=True)

    text = Column(Text)
    start_offset = Column(Integer)
    end_offset = Column(Integer)

    created_at = Column(DateTime, default=datetime.utcnow)


class CandidateEntity(Base):
    """Extracted entity candidate."""

    __tablename__ = "candidate_entities"

    id = Column(String(255), primary_key=True)
    project_id = Column(String(255), nullable=False, index=True)
    document_id = Column(String(255), nullable=False, index=True)

    label = Column(String(512), nullable=False)
    entity_type = Column(String(100), nullable=False)  # "concept", "person", "org", etc.
    description = Column(Text, nullable=True)
    source_type = Column(SQLEnum(CandidateSource), default=CandidateSource.LIGHTWEIGHT, nullable=False, index=True)
    created_by = Column(String(100), default="lightweight", nullable=False)

    confidence = Column(Float, default=0.5)  # 0.0 ~ 1.0
    source_trust = Column(Float, default=0.5)  # Trust in source
    validation_passed = Column(Boolean, default=False, nullable=False)

    evidence_ids = Column(JSON, nullable=True)  # List of evidence span IDs
    aliases = Column(JSON, nullable=True)  # List of alternative names

    review_status = Column(SQLEnum(ReviewStatus), default=ReviewStatus.PENDING, index=True)
    reviewed_by = Column(String(255), nullable=True)
    reviewed_at = Column(DateTime, nullable=True)
    review_reason = Column(Text, nullable=True)

    metadata_ = Column("metadata", JSON, nullable=True)  # Raw LLM output, domain-specific fields

    created_at = Column(DateTime, default=datetime.utcnow)
    updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)
    promoted_at = Column(DateTime, nullable=True)


class CandidateRelation(Base):
    """Extracted relation candidate."""

    __tablename__ = "candidate_relations"

    id = Column(String(255), primary_key=True)
    project_id = Column(String(255), nullable=False, index=True)
    document_id = Column(String(255), nullable=False, index=True)

    source_entity_id = Column(String(255), nullable=False, index=True)
    predicate = Column(String(255), nullable=False)
    target_entity_id = Column(String(255), nullable=False, index=True)
    source_type = Column(SQLEnum(CandidateSource), default=CandidateSource.LIGHTWEIGHT, nullable=False, index=True)
    created_by = Column(String(100), default="lightweight", nullable=False)

    confidence = Column(Float, default=0.5)  # 0.0 ~ 1.0
    source_trust = Column(Float, default=0.5)
    validation_passed = Column(Boolean, default=False, nullable=False)

    evidence_ids = Column(JSON, nullable=True)  # List of evidence span IDs

    review_status = Column(SQLEnum(ReviewStatus), default=ReviewStatus.PENDING, index=True)
    reviewed_by = Column(String(255), nullable=True)
    reviewed_at = Column(DateTime, nullable=True)
    review_reason = Column(Text, nullable=True)

    metadata_ = Column("metadata", JSON, nullable=True)  # Raw LLM output

    created_at = Column(DateTime, default=datetime.utcnow)
    updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)
    promoted_at = Column(DateTime, nullable=True)


class ReviewDecision(Base):
    """Audit trail for review status changes."""

    __tablename__ = "review_decisions"

    id = Column(String(255), primary_key=True)
    project_id = Column(String(255), nullable=False, index=True)
    candidate_id = Column(String(255), nullable=False, index=True)
    candidate_kind = Column(SQLEnum(CandidateKind), nullable=False, index=True)

    previous_status = Column(SQLEnum(ReviewStatus), nullable=True)
    new_status = Column(SQLEnum(ReviewStatus), nullable=False, index=True)

    reviewed_by = Column(String(255), nullable=False)
    reason = Column(Text, nullable=True)
    metadata_ = Column("metadata", JSON, nullable=True)

    created_at = Column(DateTime, default=datetime.utcnow)


class ValidationIssue(Base):
    """Structured validation issue stored for review and audit."""

    __tablename__ = "validation_issues"

    id = Column(String(255), primary_key=True)
    project_id = Column(String(255), nullable=False, index=True)
    document_id = Column(String(255), nullable=True, index=True)
    candidate_id = Column(String(255), nullable=True, index=True)
    candidate_kind = Column(SQLEnum(CandidateKind), nullable=True, index=True)

    severity = Column(SQLEnum(ValidationSeverity), default=ValidationSeverity.ERROR, nullable=False)
    code = Column(String(100), nullable=False, index=True)
    message = Column(Text, nullable=False)
    source = Column(String(100), default="validation", nullable=False)

    metadata_ = Column("metadata", JSON, nullable=True)
    created_at = Column(DateTime, default=datetime.utcnow)


class ExtractionJob(Base):
    """Extraction job metadata."""

    __tablename__ = "extraction_jobs"

    id = Column(String(255), primary_key=True)
    project_id = Column(String(255), nullable=False, index=True)

    job_type = Column(String(50))  # "extract", "validate", "review", etc.
    status = Column(String(50), index=True)  # "pending", "running", "completed", "failed"

    input_url = Column(String(2048), nullable=True)
    input_file = Column(String(2048), nullable=True)

    document_id = Column(String(255), nullable=True)
    entity_count = Column(Integer, default=0)
    relation_count = Column(Integer, default=0)
    error_message = Column(Text, nullable=True)

    started_at = Column(DateTime, nullable=True)
    completed_at = Column(DateTime, nullable=True)

    metadata_ = Column("metadata", JSON, nullable=True)

    created_at = Column(DateTime, default=datetime.utcnow)


class ProjectionSyncState(Base):
    """RDF canonical store to Neo4j projection/search sync state."""

    __tablename__ = "projection_sync_states"

    id = Column(String(255), primary_key=True)
    project_id = Column(String(255), nullable=False, index=True)
    canonical_store = Column(String(100), default="rdf_fuseki", nullable=False)
    projection_store = Column(String(100), default="neo4j", nullable=False)
    status = Column(SQLEnum(ProjectionStatus), default=ProjectionStatus.PENDING, index=True)
    last_sync_at = Column(DateTime, nullable=True)
    source_graph_hash = Column(String(128), nullable=True, index=True)
    error_message = Column(Text, nullable=True)
    metadata_ = Column("metadata", JSON, nullable=True)
    created_at = Column(DateTime, default=datetime.utcnow)
    updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)


class MaintenanceRun(Base):
    """One non-destructive maintenance loop run."""

    __tablename__ = "maintenance_runs"

    id = Column(String(255), primary_key=True)
    project_id = Column(String(255), nullable=False, index=True)
    status = Column(SQLEnum(MaintenanceRunStatus), default=MaintenanceRunStatus.RUNNING, index=True)
    requested_by = Column(String(255), default="system", nullable=False)
    started_at = Column(DateTime, default=datetime.utcnow)
    completed_at = Column(DateTime, nullable=True)
    error_message = Column(Text, nullable=True)
    summary = Column(JSON, nullable=True)
    budget_summary = Column(JSON, nullable=True)
    audit_summary = Column(JSON, nullable=True)
    metadata_ = Column("metadata", JSON, nullable=True)
    created_at = Column(DateTime, default=datetime.utcnow)


class MaintenanceProposal(Base):
    """Proposal created by the maintenance loop.

    Proposals do not mutate graph/candidate state. They must be reviewed and
    approved before any downstream execution layer can act on them.
    """

    __tablename__ = "maintenance_proposals"

    id = Column(String(255), primary_key=True)
    run_id = Column(String(255), nullable=False, index=True)
    project_id = Column(String(255), nullable=False, index=True)
    role = Column(SQLEnum(MaintenanceRole), nullable=False, index=True)
    proposal_type = Column(String(100), nullable=False, index=True)
    title = Column(String(512), nullable=False)
    description = Column(Text, nullable=True)
    target_kind = Column(String(100), nullable=True, index=True)
    target_id = Column(String(255), nullable=True, index=True)
    risk_level = Column(String(50), default="low", nullable=False)
    requires_human_approval = Column(Boolean, default=True, nullable=False)
    status = Column(
        SQLEnum(MaintenanceProposalStatus),
        default=MaintenanceProposalStatus.PENDING_REVIEW,
        index=True,
    )
    approved_by = Column(String(255), nullable=True)
    approved_at = Column(DateTime, nullable=True)
    rejection_reason = Column(Text, nullable=True)
    metadata_ = Column("metadata", JSON, nullable=True)
    created_at = Column(DateTime, default=datetime.utcnow)
    updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)