docs
This commit is contained in:
LASTA_DEV01\lasta
@@ -5,16 +5,16 @@ Holds extracted entity/relation candidates before final RDF conversion.
|
||||
"""
|
||||
|
||||
from datetime import datetime
|
||||
from enum import Enum
|
||||
from typing import Any
|
||||
from enum import StrEnum
|
||||
|
||||
from sqlalchemy import JSON, Column, DateTime, Float, Integer, String, Text, Enum as SQLEnum
|
||||
from sqlalchemy import JSON, Boolean, Column, DateTime, Float, Integer, String, Text
|
||||
from sqlalchemy import Enum as SQLEnum
|
||||
from sqlalchemy.orm import declarative_base
|
||||
|
||||
Base = declarative_base()
|
||||
|
||||
|
||||
class ReviewStatus(str, Enum):
|
||||
class ReviewStatus(StrEnum):
|
||||
"""Review status of a candidate."""
|
||||
|
||||
PENDING = "pending" # Awaiting human review
|
||||
@@ -23,6 +23,63 @@ class ReviewStatus(str, Enum):
|
||||
REJECTED = "rejected" # Rejected by human
|
||||
|
||||
|
||||
class CandidateSource(StrEnum):
|
||||
"""Source path that produced a candidate."""
|
||||
|
||||
LIGHTWEIGHT = "lightweight"
|
||||
ONTOCAST = "ontocast"
|
||||
|
||||
|
||||
class CandidateKind(StrEnum):
|
||||
"""Reviewable candidate kind."""
|
||||
|
||||
ENTITY = "entity"
|
||||
RELATION = "relation"
|
||||
|
||||
|
||||
class ValidationSeverity(StrEnum):
|
||||
"""Severity of validation issue."""
|
||||
|
||||
ERROR = "error"
|
||||
WARNING = "warning"
|
||||
|
||||
|
||||
class ProjectionStatus(StrEnum):
|
||||
"""Status of an RDF-to-Neo4j projection sync."""
|
||||
|
||||
PENDING = "pending"
|
||||
RUNNING = "running"
|
||||
COMPLETED = "completed"
|
||||
FAILED = "failed"
|
||||
|
||||
|
||||
class MaintenanceRunStatus(StrEnum):
|
||||
"""Status of a maintenance loop run."""
|
||||
|
||||
RUNNING = "running"
|
||||
COMPLETED = "completed"
|
||||
FAILED = "failed"
|
||||
|
||||
|
||||
class MaintenanceProposalStatus(StrEnum):
|
||||
"""Human review state for maintenance proposals."""
|
||||
|
||||
PENDING_REVIEW = "pending_review"
|
||||
APPROVED = "approved"
|
||||
REJECTED = "rejected"
|
||||
|
||||
|
||||
class MaintenanceRole(StrEnum):
|
||||
"""Maintenance loop role name."""
|
||||
|
||||
ANALYST = "analyst"
|
||||
RESEARCHER = "researcher"
|
||||
CURATOR = "curator"
|
||||
AUDITOR = "auditor"
|
||||
FIXER = "fixer"
|
||||
ADVISOR = "advisor"
|
||||
|
||||
|
||||
class SourceDocument(Base):
|
||||
"""Source document metadata."""
|
||||
|
||||
@@ -31,6 +88,7 @@ class SourceDocument(Base):
|
||||
id = Column(String(255), primary_key=True)
|
||||
project_id = Column(String(255), nullable=False, index=True)
|
||||
source_url = Column(String(2048), nullable=True, index=True)
|
||||
canonical_url = Column(String(2048), nullable=True)
|
||||
file_path = Column(String(2048), nullable=True)
|
||||
document_type = Column(String(50)) # "html", "pdf", "markdown", "docx", "inline_text"
|
||||
|
||||
@@ -39,15 +97,18 @@ class SourceDocument(Base):
|
||||
publish_date = Column(String(50), nullable=True) # ISO-8601
|
||||
language = Column(String(10), nullable=True)
|
||||
sitename = Column(String(255), nullable=True)
|
||||
description = Column(Text, nullable=True)
|
||||
|
||||
text = Column(Text)
|
||||
raw_html = Column(Text, nullable=True)
|
||||
body_xml = Column(Text, nullable=True)
|
||||
content_hash = Column(String(64), unique=True, nullable=False, index=True)
|
||||
fingerprint = Column(String(100), nullable=True, index=True)
|
||||
|
||||
retrieved_at = Column(DateTime, default=datetime.utcnow)
|
||||
extracted_by = Column(String(100), default="trafilatura") # Source tool
|
||||
|
||||
metadata = Column(JSON, nullable=True) # Raw metadata
|
||||
metadata_ = Column("metadata", JSON, nullable=True) # Raw metadata
|
||||
|
||||
created_at = Column(DateTime, default=datetime.utcnow)
|
||||
|
||||
@@ -80,9 +141,12 @@ class CandidateEntity(Base):
|
||||
label = Column(String(512), nullable=False)
|
||||
entity_type = Column(String(100), nullable=False) # "concept", "person", "org", etc.
|
||||
description = Column(Text, nullable=True)
|
||||
source_type = Column(SQLEnum(CandidateSource), default=CandidateSource.LIGHTWEIGHT, nullable=False, index=True)
|
||||
created_by = Column(String(100), default="lightweight", nullable=False)
|
||||
|
||||
confidence = Column(Float, default=0.5) # 0.0 ~ 1.0
|
||||
source_trust = Column(Float, default=0.5) # Trust in source
|
||||
validation_passed = Column(Boolean, default=False, nullable=False)
|
||||
|
||||
evidence_ids = Column(JSON, nullable=True) # List of evidence span IDs
|
||||
aliases = Column(JSON, nullable=True) # List of alternative names
|
||||
@@ -92,10 +156,11 @@ class CandidateEntity(Base):
|
||||
reviewed_at = Column(DateTime, nullable=True)
|
||||
review_reason = Column(Text, nullable=True)
|
||||
|
||||
metadata = Column(JSON, nullable=True) # Raw LLM output, domain-specific fields
|
||||
metadata_ = Column("metadata", JSON, nullable=True) # Raw LLM output, domain-specific fields
|
||||
|
||||
created_at = Column(DateTime, default=datetime.utcnow)
|
||||
updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)
|
||||
promoted_at = Column(DateTime, nullable=True)
|
||||
|
||||
|
||||
class CandidateRelation(Base):
|
||||
@@ -110,9 +175,12 @@ class CandidateRelation(Base):
|
||||
source_entity_id = Column(String(255), nullable=False, index=True)
|
||||
predicate = Column(String(255), nullable=False)
|
||||
target_entity_id = Column(String(255), nullable=False, index=True)
|
||||
source_type = Column(SQLEnum(CandidateSource), default=CandidateSource.LIGHTWEIGHT, nullable=False, index=True)
|
||||
created_by = Column(String(100), default="lightweight", nullable=False)
|
||||
|
||||
confidence = Column(Float, default=0.5) # 0.0 ~ 1.0
|
||||
source_trust = Column(Float, default=0.5)
|
||||
validation_passed = Column(Boolean, default=False, nullable=False)
|
||||
|
||||
evidence_ids = Column(JSON, nullable=True) # List of evidence span IDs
|
||||
|
||||
@@ -121,10 +189,51 @@ class CandidateRelation(Base):
|
||||
reviewed_at = Column(DateTime, nullable=True)
|
||||
review_reason = Column(Text, nullable=True)
|
||||
|
||||
metadata = Column(JSON, nullable=True) # Raw LLM output
|
||||
metadata_ = Column("metadata", JSON, nullable=True) # Raw LLM output
|
||||
|
||||
created_at = Column(DateTime, default=datetime.utcnow)
|
||||
updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)
|
||||
promoted_at = Column(DateTime, nullable=True)
|
||||
|
||||
|
||||
class ReviewDecision(Base):
|
||||
"""Audit trail for review status changes."""
|
||||
|
||||
__tablename__ = "review_decisions"
|
||||
|
||||
id = Column(String(255), primary_key=True)
|
||||
project_id = Column(String(255), nullable=False, index=True)
|
||||
candidate_id = Column(String(255), nullable=False, index=True)
|
||||
candidate_kind = Column(SQLEnum(CandidateKind), nullable=False, index=True)
|
||||
|
||||
previous_status = Column(SQLEnum(ReviewStatus), nullable=True)
|
||||
new_status = Column(SQLEnum(ReviewStatus), nullable=False, index=True)
|
||||
|
||||
reviewed_by = Column(String(255), nullable=False)
|
||||
reason = Column(Text, nullable=True)
|
||||
metadata_ = Column("metadata", JSON, nullable=True)
|
||||
|
||||
created_at = Column(DateTime, default=datetime.utcnow)
|
||||
|
||||
|
||||
class ValidationIssue(Base):
|
||||
"""Structured validation issue stored for review and audit."""
|
||||
|
||||
__tablename__ = "validation_issues"
|
||||
|
||||
id = Column(String(255), primary_key=True)
|
||||
project_id = Column(String(255), nullable=False, index=True)
|
||||
document_id = Column(String(255), nullable=True, index=True)
|
||||
candidate_id = Column(String(255), nullable=True, index=True)
|
||||
candidate_kind = Column(SQLEnum(CandidateKind), nullable=True, index=True)
|
||||
|
||||
severity = Column(SQLEnum(ValidationSeverity), default=ValidationSeverity.ERROR, nullable=False)
|
||||
code = Column(String(100), nullable=False, index=True)
|
||||
message = Column(Text, nullable=False)
|
||||
source = Column(String(100), default="validation", nullable=False)
|
||||
|
||||
metadata_ = Column("metadata", JSON, nullable=True)
|
||||
created_at = Column(DateTime, default=datetime.utcnow)
|
||||
|
||||
|
||||
class ExtractionJob(Base):
|
||||
@@ -149,6 +258,76 @@ class ExtractionJob(Base):
|
||||
started_at = Column(DateTime, nullable=True)
|
||||
completed_at = Column(DateTime, nullable=True)
|
||||
|
||||
metadata = Column(JSON, nullable=True)
|
||||
metadata_ = Column("metadata", JSON, nullable=True)
|
||||
|
||||
created_at = Column(DateTime, default=datetime.utcnow)
|
||||
|
||||
|
||||
class ProjectionSyncState(Base):
|
||||
"""RDF canonical store to Neo4j projection/search sync state."""
|
||||
|
||||
__tablename__ = "projection_sync_states"
|
||||
|
||||
id = Column(String(255), primary_key=True)
|
||||
project_id = Column(String(255), nullable=False, index=True)
|
||||
canonical_store = Column(String(100), default="rdf_fuseki", nullable=False)
|
||||
projection_store = Column(String(100), default="neo4j", nullable=False)
|
||||
status = Column(SQLEnum(ProjectionStatus), default=ProjectionStatus.PENDING, index=True)
|
||||
last_sync_at = Column(DateTime, nullable=True)
|
||||
source_graph_hash = Column(String(128), nullable=True, index=True)
|
||||
error_message = Column(Text, nullable=True)
|
||||
metadata_ = Column("metadata", JSON, nullable=True)
|
||||
created_at = Column(DateTime, default=datetime.utcnow)
|
||||
updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)
|
||||
|
||||
|
||||
class MaintenanceRun(Base):
|
||||
"""One non-destructive maintenance loop run."""
|
||||
|
||||
__tablename__ = "maintenance_runs"
|
||||
|
||||
id = Column(String(255), primary_key=True)
|
||||
project_id = Column(String(255), nullable=False, index=True)
|
||||
status = Column(SQLEnum(MaintenanceRunStatus), default=MaintenanceRunStatus.RUNNING, index=True)
|
||||
requested_by = Column(String(255), default="system", nullable=False)
|
||||
started_at = Column(DateTime, default=datetime.utcnow)
|
||||
completed_at = Column(DateTime, nullable=True)
|
||||
error_message = Column(Text, nullable=True)
|
||||
summary = Column(JSON, nullable=True)
|
||||
budget_summary = Column(JSON, nullable=True)
|
||||
audit_summary = Column(JSON, nullable=True)
|
||||
metadata_ = Column("metadata", JSON, nullable=True)
|
||||
created_at = Column(DateTime, default=datetime.utcnow)
|
||||
|
||||
|
||||
class MaintenanceProposal(Base):
|
||||
"""Proposal created by the maintenance loop.
|
||||
|
||||
Proposals do not mutate graph/candidate state. They must be reviewed and
|
||||
approved before any downstream execution layer can act on them.
|
||||
"""
|
||||
|
||||
__tablename__ = "maintenance_proposals"
|
||||
|
||||
id = Column(String(255), primary_key=True)
|
||||
run_id = Column(String(255), nullable=False, index=True)
|
||||
project_id = Column(String(255), nullable=False, index=True)
|
||||
role = Column(SQLEnum(MaintenanceRole), nullable=False, index=True)
|
||||
proposal_type = Column(String(100), nullable=False, index=True)
|
||||
title = Column(String(512), nullable=False)
|
||||
description = Column(Text, nullable=True)
|
||||
target_kind = Column(String(100), nullable=True, index=True)
|
||||
target_id = Column(String(255), nullable=True, index=True)
|
||||
risk_level = Column(String(50), default="low", nullable=False)
|
||||
requires_human_approval = Column(Boolean, default=True, nullable=False)
|
||||
status = Column(
|
||||
SQLEnum(MaintenanceProposalStatus),
|
||||
default=MaintenanceProposalStatus.PENDING_REVIEW,
|
||||
index=True,
|
||||
)
|
||||
approved_by = Column(String(255), nullable=True)
|
||||
approved_at = Column(DateTime, nullable=True)
|
||||
rejection_reason = Column(Text, nullable=True)
|
||||
metadata_ = Column("metadata", JSON, nullable=True)
|
||||
created_at = Column(DateTime, default=datetime.utcnow)
|
||||
updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)
|
||||
|
||||
Reference in New Issue
Block a user