155 lines
5.3 KiB
Python
155 lines
5.3 KiB
Python
"""
|
|
Database models for candidate storage.
|
|
|
|
Holds extracted entity/relation candidates before final RDF conversion.
|
|
"""
|
|
|
|
from datetime import datetime
|
|
from enum import Enum
|
|
from typing import Any
|
|
|
|
from sqlalchemy import JSON, Column, DateTime, Float, Integer, String, Text, Enum as SQLEnum
|
|
from sqlalchemy.orm import declarative_base
|
|
|
|
Base = declarative_base()
|
|
|
|
|
|
class ReviewStatus(str, Enum):
|
|
"""Review status of a candidate."""
|
|
|
|
PENDING = "pending" # Awaiting human review
|
|
APPROVED = "approved" # Approved by human
|
|
AUTO_APPROVED = "auto_approved" # Approved by policy
|
|
REJECTED = "rejected" # Rejected by human
|
|
|
|
|
|
class SourceDocument(Base):
|
|
"""Source document metadata."""
|
|
|
|
__tablename__ = "source_documents"
|
|
|
|
id = Column(String(255), primary_key=True)
|
|
project_id = Column(String(255), nullable=False, index=True)
|
|
source_url = Column(String(2048), nullable=True, index=True)
|
|
file_path = Column(String(2048), nullable=True)
|
|
document_type = Column(String(50)) # "html", "pdf", "markdown", "docx", "inline_text"
|
|
|
|
title = Column(String(512), nullable=True)
|
|
author = Column(String(255), nullable=True)
|
|
publish_date = Column(String(50), nullable=True) # ISO-8601
|
|
language = Column(String(10), nullable=True)
|
|
sitename = Column(String(255), nullable=True)
|
|
|
|
text = Column(Text)
|
|
content_hash = Column(String(64), unique=True, nullable=False, index=True)
|
|
fingerprint = Column(String(100), nullable=True, index=True)
|
|
|
|
retrieved_at = Column(DateTime, default=datetime.utcnow)
|
|
extracted_by = Column(String(100), default="trafilatura") # Source tool
|
|
|
|
metadata = Column(JSON, nullable=True) # Raw metadata
|
|
|
|
created_at = Column(DateTime, default=datetime.utcnow)
|
|
|
|
|
|
class EvidenceSpan(Base):
|
|
"""Evidence text span from source document."""
|
|
|
|
__tablename__ = "evidence_spans"
|
|
|
|
id = Column(String(255), primary_key=True)
|
|
document_id = Column(String(255), nullable=False, index=True)
|
|
project_id = Column(String(255), nullable=False, index=True)
|
|
|
|
text = Column(Text)
|
|
start_offset = Column(Integer)
|
|
end_offset = Column(Integer)
|
|
|
|
created_at = Column(DateTime, default=datetime.utcnow)
|
|
|
|
|
|
class CandidateEntity(Base):
|
|
"""Extracted entity candidate."""
|
|
|
|
__tablename__ = "candidate_entities"
|
|
|
|
id = Column(String(255), primary_key=True)
|
|
project_id = Column(String(255), nullable=False, index=True)
|
|
document_id = Column(String(255), nullable=False, index=True)
|
|
|
|
label = Column(String(512), nullable=False)
|
|
entity_type = Column(String(100), nullable=False) # "concept", "person", "org", etc.
|
|
description = Column(Text, nullable=True)
|
|
|
|
confidence = Column(Float, default=0.5) # 0.0 ~ 1.0
|
|
source_trust = Column(Float, default=0.5) # Trust in source
|
|
|
|
evidence_ids = Column(JSON, nullable=True) # List of evidence span IDs
|
|
aliases = Column(JSON, nullable=True) # List of alternative names
|
|
|
|
review_status = Column(SQLEnum(ReviewStatus), default=ReviewStatus.PENDING, index=True)
|
|
reviewed_by = Column(String(255), nullable=True)
|
|
reviewed_at = Column(DateTime, nullable=True)
|
|
review_reason = Column(Text, nullable=True)
|
|
|
|
metadata = Column(JSON, nullable=True) # Raw LLM output, domain-specific fields
|
|
|
|
created_at = Column(DateTime, default=datetime.utcnow)
|
|
updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)
|
|
|
|
|
|
class CandidateRelation(Base):
|
|
"""Extracted relation candidate."""
|
|
|
|
__tablename__ = "candidate_relations"
|
|
|
|
id = Column(String(255), primary_key=True)
|
|
project_id = Column(String(255), nullable=False, index=True)
|
|
document_id = Column(String(255), nullable=False, index=True)
|
|
|
|
source_entity_id = Column(String(255), nullable=False, index=True)
|
|
predicate = Column(String(255), nullable=False)
|
|
target_entity_id = Column(String(255), nullable=False, index=True)
|
|
|
|
confidence = Column(Float, default=0.5) # 0.0 ~ 1.0
|
|
source_trust = Column(Float, default=0.5)
|
|
|
|
evidence_ids = Column(JSON, nullable=True) # List of evidence span IDs
|
|
|
|
review_status = Column(SQLEnum(ReviewStatus), default=ReviewStatus.PENDING, index=True)
|
|
reviewed_by = Column(String(255), nullable=True)
|
|
reviewed_at = Column(DateTime, nullable=True)
|
|
review_reason = Column(Text, nullable=True)
|
|
|
|
metadata = Column(JSON, nullable=True) # Raw LLM output
|
|
|
|
created_at = Column(DateTime, default=datetime.utcnow)
|
|
updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)
|
|
|
|
|
|
class ExtractionJob(Base):
|
|
"""Extraction job metadata."""
|
|
|
|
__tablename__ = "extraction_jobs"
|
|
|
|
id = Column(String(255), primary_key=True)
|
|
project_id = Column(String(255), nullable=False, index=True)
|
|
|
|
job_type = Column(String(50)) # "extract", "validate", "review", etc.
|
|
status = Column(String(50), index=True) # "pending", "running", "completed", "failed"
|
|
|
|
input_url = Column(String(2048), nullable=True)
|
|
input_file = Column(String(2048), nullable=True)
|
|
|
|
document_id = Column(String(255), nullable=True)
|
|
entity_count = Column(Integer, default=0)
|
|
relation_count = Column(Integer, default=0)
|
|
error_message = Column(Text, nullable=True)
|
|
|
|
started_at = Column(DateTime, nullable=True)
|
|
completed_at = Column(DateTime, nullable=True)
|
|
|
|
metadata = Column(JSON, nullable=True)
|
|
|
|
created_at = Column(DateTime, default=datetime.utcnow)
|