Phase 4 구현 완료: Neo4j 벡터 검색 + 그래프 저장소

ontology_platform/ont_platform/storage/__init__.py

@@ -0,0 +1,7 @@
+"""Storage module (Phase 1+).
+
+Phase 0: No database storage yet.
+Phase 1: Add SQLAlchemy models for candidate storage.
+"""
+
+__all__ = []

ontology_platform/ont_platform/storage/init_db.py

@@ -0,0 +1,37 @@
+"""Database initialization script."""
+
+import logging
+from sqlalchemy import create_engine
+
+from ont_platform.config import load_settings
+from ont_platform.storage.models import Base
+
+logger = logging.getLogger(__name__)
+
+
+def init_db():
+    """Create all tables in the database."""
+    settings = load_settings()
+    database_url = settings.database_url
+
+    engine = create_engine(
+        database_url,
+        connect_args={"timeout": 30} if "sqlite" in database_url else {},
+    )
+
+    logger.info(f"Creating tables in {database_url}")
+    Base.metadata.create_all(bind=engine)
+    logger.info("Database tables created successfully")
+
+
+if __name__ == "__main__":
+    import sys
+
+    logging.basicConfig(level=logging.INFO)
+    try:
+        init_db()
+        print("✓ Database initialized")
+        sys.exit(0)
+    except Exception as e:
+        print(f"✗ Error: {e}")
+        sys.exit(1)

ontology_platform/ont_platform/storage/models.py

@@ -0,0 +1,154 @@
+"""
+Database models for candidate storage.
+
+Holds extracted entity/relation candidates before final RDF conversion.
+"""
+
+from datetime import datetime
+from enum import Enum
+from typing import Any
+
+from sqlalchemy import JSON, Column, DateTime, Float, Integer, String, Text, Enum as SQLEnum
+from sqlalchemy.orm import declarative_base
+
+Base = declarative_base()
+
+
+class ReviewStatus(str, Enum):
+    """Review status of a candidate."""
+
+    PENDING = "pending"  # Awaiting human review
+    APPROVED = "approved"  # Approved by human
+    AUTO_APPROVED = "auto_approved"  # Approved by policy
+    REJECTED = "rejected"  # Rejected by human
+
+
+class SourceDocument(Base):
+    """Source document metadata."""
+
+    __tablename__ = "source_documents"
+
+    id = Column(String(255), primary_key=True)
+    project_id = Column(String(255), nullable=False, index=True)
+    source_url = Column(String(2048), nullable=True, index=True)
+    file_path = Column(String(2048), nullable=True)
+    document_type = Column(String(50))  # "html", "pdf", "markdown", "docx", "inline_text"
+
+    title = Column(String(512), nullable=True)
+    author = Column(String(255), nullable=True)
+    publish_date = Column(String(50), nullable=True)  # ISO-8601
+    language = Column(String(10), nullable=True)
+    sitename = Column(String(255), nullable=True)
+
+    text = Column(Text)
+    content_hash = Column(String(64), unique=True, nullable=False, index=True)
+    fingerprint = Column(String(100), nullable=True, index=True)
+
+    retrieved_at = Column(DateTime, default=datetime.utcnow)
+    extracted_by = Column(String(100), default="trafilatura")  # Source tool
+
+    metadata = Column(JSON, nullable=True)  # Raw metadata
+
+    created_at = Column(DateTime, default=datetime.utcnow)
+
+
+class EvidenceSpan(Base):
+    """Evidence text span from source document."""
+
+    __tablename__ = "evidence_spans"
+
+    id = Column(String(255), primary_key=True)
+    document_id = Column(String(255), nullable=False, index=True)
+    project_id = Column(String(255), nullable=False, index=True)
+
+    text = Column(Text)
+    start_offset = Column(Integer)
+    end_offset = Column(Integer)
+
+    created_at = Column(DateTime, default=datetime.utcnow)
+
+
+class CandidateEntity(Base):
+    """Extracted entity candidate."""
+
+    __tablename__ = "candidate_entities"
+
+    id = Column(String(255), primary_key=True)
+    project_id = Column(String(255), nullable=False, index=True)
+    document_id = Column(String(255), nullable=False, index=True)
+
+    label = Column(String(512), nullable=False)
+    entity_type = Column(String(100), nullable=False)  # "concept", "person", "org", etc.
+    description = Column(Text, nullable=True)
+
+    confidence = Column(Float, default=0.5)  # 0.0 ~ 1.0
+    source_trust = Column(Float, default=0.5)  # Trust in source
+
+    evidence_ids = Column(JSON, nullable=True)  # List of evidence span IDs
+    aliases = Column(JSON, nullable=True)  # List of alternative names
+
+    review_status = Column(SQLEnum(ReviewStatus), default=ReviewStatus.PENDING, index=True)
+    reviewed_by = Column(String(255), nullable=True)
+    reviewed_at = Column(DateTime, nullable=True)
+    review_reason = Column(Text, nullable=True)
+
+    metadata = Column(JSON, nullable=True)  # Raw LLM output, domain-specific fields
+
+    created_at = Column(DateTime, default=datetime.utcnow)
+    updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)
+
+
+class CandidateRelation(Base):
+    """Extracted relation candidate."""
+
+    __tablename__ = "candidate_relations"
+
+    id = Column(String(255), primary_key=True)
+    project_id = Column(String(255), nullable=False, index=True)
+    document_id = Column(String(255), nullable=False, index=True)
+
+    source_entity_id = Column(String(255), nullable=False, index=True)
+    predicate = Column(String(255), nullable=False)
+    target_entity_id = Column(String(255), nullable=False, index=True)
+
+    confidence = Column(Float, default=0.5)  # 0.0 ~ 1.0
+    source_trust = Column(Float, default=0.5)
+
+    evidence_ids = Column(JSON, nullable=True)  # List of evidence span IDs
+
+    review_status = Column(SQLEnum(ReviewStatus), default=ReviewStatus.PENDING, index=True)
+    reviewed_by = Column(String(255), nullable=True)
+    reviewed_at = Column(DateTime, nullable=True)
+    review_reason = Column(Text, nullable=True)
+
+    metadata = Column(JSON, nullable=True)  # Raw LLM output
+
+    created_at = Column(DateTime, default=datetime.utcnow)
+    updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)
+
+
+class ExtractionJob(Base):
+    """Extraction job metadata."""
+
+    __tablename__ = "extraction_jobs"
+
+    id = Column(String(255), primary_key=True)
+    project_id = Column(String(255), nullable=False, index=True)
+
+    job_type = Column(String(50))  # "extract", "validate", "review", etc.
+    status = Column(String(50), index=True)  # "pending", "running", "completed", "failed"
+
+    input_url = Column(String(2048), nullable=True)
+    input_file = Column(String(2048), nullable=True)
+
+    document_id = Column(String(255), nullable=True)
+    entity_count = Column(Integer, default=0)
+    relation_count = Column(Integer, default=0)
+    error_message = Column(Text, nullable=True)
+
+    started_at = Column(DateTime, nullable=True)
+    completed_at = Column(DateTime, nullable=True)
+
+    metadata = Column(JSON, nullable=True)
+
+    created_at = Column(DateTime, default=datetime.utcnow)