Phase 4 구현 완료: Neo4j 벡터 검색 + 그래프 저장소

ontology_platform/ont_platform/api/db_deps.py

@@ -0,0 +1,49 @@
+"""Database dependencies for FastAPI."""
+
+from typing import Generator
+
+from sqlalchemy import create_engine
+from sqlalchemy.orm import Session, sessionmaker
+
+from ont_platform.config import load_settings
+
+# Initialize database engine (lazy singleton)
+_engine = None
+_SessionLocal = None
+
+
+def get_db_engine():
+    """Get or create database engine."""
+    global _engine
+    if _engine is None:
+        settings = load_settings()
+        database_url = settings.database_url
+        _engine = create_engine(
+            database_url,
+            connect_args={"timeout": 30} if "sqlite" in database_url else {},
+            pool_pre_ping=True,
+            echo=False,
+        )
+    return _engine
+
+
+def get_session_factory():
+    """Get or create session factory."""
+    global _SessionLocal
+    if _SessionLocal is None:
+        engine = get_db_engine()
+        _SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
+    return _SessionLocal
+
+
+def get_db() -> Generator[Session, None, None]:
+    """FastAPI dependency for database session."""
+    SessionLocal = get_session_factory()
+    db = SessionLocal()
+    try:
+        yield db
+    finally:
+        db.close()
+
+
+__all__ = ["get_db", "get_db_engine", "get_session_factory"]

ontology_platform/ont_platform/api/main.py

@@ -44,6 +44,7 @@ from ont_platform.api.deps import (  # noqa: E402
     get_app_context,
     initialize_app_context,
 )
+from ont_platform.api.routes import extraction_router  # noqa: E402
 
 platform_config = importlib.import_module("ont_platform.config")
 
@@ -368,6 +369,9 @@ def create_app() -> FastAPI:
             },
         )
 
+    # ─── Phase 0 routes ───────────────────────────────────────────────
+    app.include_router(extraction_router)
+
     return app
 
 

ontology_platform/ont_platform/api/phase0_app.py

@@ -0,0 +1,252 @@
+"""Phase 0-4 FastAPI application.
+
+Phase 0: Basic URL extraction
+Phase 2: Crawl4AI profile support for dynamic pages
+Phase 3: Validation (lightweight + OntoCast)
+Phase 4: Neo4j vector search
+"""
+
+from fastapi import FastAPI, APIRouter, HTTPException, Query
+from typing import Optional, Literal, List
+import time
+import asyncio
+import logging
+
+from ont_platform.core.extractors.web_extractor import extract_web_content
+from ont_platform.core.extraction.lightweight_extractor import LightweightExtractor
+from ont_platform.core.crawler.crawl4ai_adapter import (
+    Crawl4AIAdapter,
+    CrawlProfile,
+)
+from ont_platform.core.validation import OntologyGuard
+from ont_platform.core.graph.neo4j_adapter import Neo4jAdapter, Neo4jConfig
+
+logger = logging.getLogger(__name__)
+
+app = FastAPI(
+    title="Ontology Platform - Phase 0-4",
+    description="Extraction + Validation + Graph Search. 10-30 seconds per URL.",
+    version="0.4.0",
+)
+
+extraction_router = APIRouter(prefix="/api/v1/extract", tags=["extraction"])
+search_router = APIRouter(prefix="/api/v1/search", tags=["search"])
+
+# Phase 3: Initialize validation guard
+guard = OntologyGuard(validator_type="lightweight", strict=False)
+
+# Phase 4: Neo4j adapter (lazy initialization)
+_neo4j_adapter: Optional[Neo4jAdapter] = None
+
+async def get_neo4j_adapter() -> Neo4jAdapter:
+    """Get or create Neo4j adapter instance."""
+    global _neo4j_adapter
+    if _neo4j_adapter is None:
+        _neo4j_adapter = Neo4jAdapter()
+        if not await _neo4j_adapter.connect():
+            logger.warning("Neo4j not available, search will be unavailable")
+        else:
+            try:
+                await _neo4j_adapter.initialize_embedder()
+            except Exception as e:
+                logger.warning(f"Failed to initialize embedder: {e}")
+    return _neo4j_adapter
+
+
+@extraction_router.post("/url")
+async def extract_url(
+    url: str,
+    profile: Optional[Literal["fast_static", "dynamic_page"]] = Query(None),
+):
+    """
+    Extract candidates from URL (Phase 0-2).
+
+    Phase 0-1: Default fast_static (HTTP only)
+    Phase 2: Supports dynamic_page for JS-rendered content
+    """
+    if not url:
+        raise HTTPException(status_code=400, detail="url is required")
+
+    start_time = time.time()
+
+    try:
+        # Phase 2: Use Crawl4AI for dynamic pages
+        if profile == "dynamic_page":
+            adapter = Crawl4AIAdapter()
+            try:
+                crawl_result = await adapter.crawl(url, profile=CrawlProfile.DYNAMIC_PAGE)
+                profile_used = crawl_result.profile_used
+                html_content = crawl_result.html
+            finally:
+                await adapter.close()
+
+            # Extract from crawled HTML
+            extracted = extract_web_content(html=html_content, url=url)
+        else:
+            # Phase 0-1: Default fast_static (HTTP only)
+            extracted = extract_web_content(url=url)
+            profile_used = "trafilatura"
+
+        # Step 2: Extract JSON candidates with lightweight extractor
+        lightweight = LightweightExtractor(use_llm=False)
+        candidates = lightweight.extract(
+            text=extracted.text,
+            project_id="default",
+            document_id="temp",
+        )
+
+        # Phase 3: Validate extraction results
+        raw_result = {
+            "entities": candidates.entities,
+            "relations": candidates.relations,
+            "warnings": candidates.warnings,
+        }
+        validated = await guard.validate(raw_result)
+
+        extraction_time = time.time() - start_time
+
+        # Return JSON with validation info
+        return {
+            "url": url,
+            "title": extracted.title,
+            "author": extracted.author,
+            "published_date": extracted.publish_date,
+            "language": extracted.language,
+            "text_length": len(extracted.text),
+            "profile_used": profile_used,
+            "entities": [e.dict() for e in validated.entities],
+            "relations": [r.dict() for r in validated.relations],
+            "extraction_time_sec": round(extraction_time, 2),
+            "entity_count": len(validated.entities),
+            "relation_count": len(validated.relations),
+            "warnings": validated.warnings,
+            "validation_passed": validated.validation_passed,
+            "validation_errors": validated.validation_errors,
+        }
+
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=f"Extraction failed: {str(e)}")
+
+
+@search_router.post("/vector")
+async def vector_search(
+    query: str = Query(..., description="Search query"),
+    limit: int = Query(10, ge=1, le=100),
+    threshold: float = Query(0.5, ge=0.0, le=1.0),
+):
+    """
+    Vector search in Neo4j (Phase 4).
+
+    Returns top-k similar entities using vector embeddings.
+    """
+    try:
+        adapter = await get_neo4j_adapter()
+        results = await adapter.vector_search(
+            query_text=query,
+            limit=limit,
+            threshold=threshold,
+        )
+
+        return {
+            "query": query,
+            "results": results,
+            "result_count": len(results),
+            "limit": limit,
+            "threshold": threshold,
+        }
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=f"Search failed: {str(e)}")
+
+
+@search_router.get("/stats")
+async def graph_stats():
+    """
+    Get Neo4j graph statistics (Phase 4).
+
+    Returns node and edge counts.
+    """
+    try:
+        adapter = await get_neo4j_adapter()
+        stats = await adapter.get_stats()
+        return {
+            "status": "connected" if stats else "disconnected",
+            "stats": stats,
+        }
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=f"Stats retrieval failed: {str(e)}")
+
+
+@search_router.get("/entity/{entity_id}")
+async def get_entity_neighbors(
+    entity_id: str,
+    depth: int = Query(1, ge=1, le=2),
+):
+    """
+    Get entity and its neighbors in the graph (Phase 4).
+    """
+    try:
+        adapter = await get_neo4j_adapter()
+        result = await adapter.get_entity_neighbors(entity_id, depth=depth)
+
+        if not result:
+            raise HTTPException(status_code=404, detail=f"Entity {entity_id} not found")
+
+        return result
+    except HTTPException:
+        raise
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=f"Query failed: {str(e)}")
+
+
+@search_router.post("/ingest")
+async def ingest_extraction_result(
+    extraction_result: dict = None,
+):
+    """
+    Ingest extraction results into Neo4j graph (Phase 4).
+
+    Takes validated entities and relations from extraction output,
+    creates nodes and edges in Neo4j with vector embeddings.
+
+    Expected input:
+    {
+        "entities": [
+            {"id": "E_1", "label": "...", "type": "...", "confidence": 0.9}
+        ],
+        "relations": [
+            {"source_id": "E_1", "target_id": "E_2", "predicate": "...", "confidence": 0.8}
+        ]
+    }
+    """
+    try:
+        if not extraction_result or ("entities" not in extraction_result and "relations" not in extraction_result):
+            raise HTTPException(status_code=400, detail="Missing entities or relations in input")
+
+        adapter = await get_neo4j_adapter()
+        entities_ingested = 0
+        relations_ingested = 0
+
+        # Ingest entities if present
+        if extraction_result.get("entities"):
+            entities_ingested = await adapter.create_entity_nodes(extraction_result["entities"])
+
+        # Ingest relations if present
+        if extraction_result.get("relations"):
+            relations_ingested = await adapter.create_relation_edges(extraction_result["relations"])
+
+        return {
+            "status": "success",
+            "entities_ingested": entities_ingested,
+            "relations_ingested": relations_ingested,
+            "total_ingested": entities_ingested + relations_ingested,
+        }
+
+    except HTTPException:
+        raise
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=f"Ingestion failed: {str(e)}")
+
+
+# Register routers
+app.include_router(extraction_router)
+app.include_router(search_router)

ontology_platform/ont_platform/api/routes/__init__.py

@@ -0,0 +1,5 @@
+"""API routes."""
+
+from .extraction import router as extraction_router
+
+__all__ = ["extraction_router"]

ontology_platform/ont_platform/api/routes/extraction.py

@@ -0,0 +1,71 @@
+"""
+Phase 0 Extraction routes: Fast JSON Extraction MVP.
+
+No database storage - just extract and return JSON candidates.
+Goal: 10-30 seconds per URL.
+"""
+
+from fastapi import APIRouter, HTTPException
+import time
+
+from ont_platform.core.extractors.web_extractor import extract_web_content
+from ont_platform.core.extraction.lightweight_extractor import LightweightExtractor
+
+router = APIRouter(prefix="/api/v1", tags=["extraction"])
+
+
+@router.post("/extract/url")
+async def extract_url(url: str):
+    """
+    Extract candidates from URL (Phase 0 MVP).
+
+    Returns:
+    {
+        "url": "...",
+        "title": "...",
+        "entities": [...],
+        "relations": [...],
+        "extraction_time_sec": 0.5,
+        "warnings": [...]
+    }
+    """
+    if not url:
+        raise HTTPException(status_code=400, detail="url is required")
+
+    start_time = time.time()
+
+    try:
+        # Step 1: Extract web content with Trafilatura
+        extracted = extract_web_content(url=url)
+
+        # Step 2: Extract JSON candidates with lightweight extractor
+        lightweight = LightweightExtractor(use_llm=False)
+        candidates = lightweight.extract(
+            text=extracted.text,
+            project_id="default",  # Phase 0: no projects yet
+            document_id="temp",
+        )
+
+        extraction_time = time.time() - start_time
+
+        # Return just the JSON (entities/relations are already dicts)
+        return {
+            "url": url,
+            "title": extracted.title,
+            "author": extracted.author,
+            "published_date": extracted.publish_date,
+            "language": extracted.language,
+            "text_length": len(extracted.text),
+            "entities": candidates.entities,
+            "relations": candidates.relations,
+            "extraction_time_sec": round(extraction_time, 2),
+            "entity_count": len(candidates.entities),
+            "relation_count": len(candidates.relations),
+            "warnings": candidates.warnings,
+        }
+
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=f"Extraction failed: {str(e)}")
+
+
+__all__ = ["router"]

ontology_platform/ont_platform/core/crawler/__init__.py

@@ -0,0 +1,17 @@
+"""Web crawler module (Phase 0 onwards)."""
+
+from .crawl4ai_adapter import (
+    Crawl4AIAdapter,
+    BasicCrawler,
+    CrawlerConfig,
+    CrawlResult,
+    crawl_url,
+)
+
+__all__ = [
+    "Crawl4AIAdapter",
+    "BasicCrawler",
+    "CrawlerConfig",
+    "CrawlResult",
+    "crawl_url",
+]

ontology_platform/ont_platform/core/crawler/crawl4ai_adapter.py

@@ -0,0 +1,281 @@
+"""
+Crawl4AI adapter for Phase 2+ (dynamic page support).
+
+Phase 0-1: HTTP fetch + Trafilatura (BasicCrawler)
+Phase 2+: Crawl4AI for dynamic/JS-heavy pages with profile selection
+
+This adapter provides unified interface with intelligent profile selection.
+"""
+
+import asyncio
+import logging
+from enum import Enum
+from typing import Optional, Literal
+
+import requests
+from crawl4ai import AsyncWebCrawler, CacheMode, CrawlerRunConfig
+
+logger = logging.getLogger(__name__)
+
+
+class CrawlProfile(str, Enum):
+    """Crawl4AI profile selection (Phase 2+)."""
+
+    FAST_STATIC = "fast_static"  # HTTP only, Trafilatura post-process
+    DYNAMIC_PAGE = "dynamic_page"  # Playwright + JS wait
+    FULL_CAPTURE = "full_capture"  # screenshot/PDF/MHTML
+    STRUCTURED_EXTRACT = "structured_extract"  # CSS/XPath schema
+    DEEP_DISCOVERY = "deep_discovery"  # URL Seeder + BFS/DFS
+
+
+class CrawlResult:
+    """Result of a crawl operation."""
+
+    def __init__(
+        self,
+        url: str,
+        html: str,
+        status_code: int = 200,
+        headers: Optional[dict] = None,
+        markdown: Optional[str] = None,
+        profile_used: Optional[str] = None,
+    ):
+        self.url = url
+        self.html = html
+        self.status_code = status_code
+        self.headers = headers or {}
+        self.markdown = markdown
+        self.profile_used = profile_used
+
+
+class CrawlerConfig:
+    """Configuration for crawler."""
+
+    def __init__(
+        self,
+        timeout: int = 15,
+        user_agent: Optional[str] = None,
+        follow_redirects: bool = True,
+        cache_mode: CacheMode = CacheMode.ENABLED,
+        check_cache_freshness: bool = True,
+    ):
+        self.timeout = timeout
+        self.user_agent = user_agent or (
+            "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
+            "AppleWebKit/537.36 (KHTML, like Gecko) "
+            "Chrome/120.0.0.0 Safari/537.36"
+        )
+        self.follow_redirects = follow_redirects
+        self.cache_mode = cache_mode
+        self.check_cache_freshness = check_cache_freshness
+
+
+class BasicCrawler:
+    """Phase 0-1: Basic HTTP crawler (fallback for dynamic_page errors)."""
+
+    def __init__(self, config: Optional[CrawlerConfig] = None):
+        """Initialize crawler with optional config."""
+        self.config = config or CrawlerConfig()
+        self.session = requests.Session()
+        self.session.headers.update({"User-Agent": self.config.user_agent})
+
+    def fetch(self, url: str) -> CrawlResult:
+        """
+        Fetch URL content using basic HTTP.
+
+        Args:
+            url: URL to fetch
+
+        Returns:
+            CrawlResult with HTML content
+        """
+        try:
+            response = self.session.get(
+                url,
+                timeout=self.config.timeout,
+                allow_redirects=self.config.follow_redirects,
+            )
+            response.raise_for_status()
+
+            return CrawlResult(
+                url=response.url,
+                html=response.text,
+                status_code=response.status_code,
+                headers=dict(response.headers),
+                profile_used="basic_http",
+            )
+
+        except requests.RequestException as e:
+            logger.error(f"Failed to fetch {url}: {e}")
+            raise
+
+    async def fetch_async(self, url: str) -> CrawlResult:
+        """Async wrapper for fetch."""
+        return await asyncio.to_thread(self.fetch, url)
+
+    def close(self):
+        """Close session resources."""
+        self.session.close()
+
+
+class Crawl4AIAdapter:
+    """
+    Unified adapter for crawling with intelligent profile selection.
+
+    Phase 2+: Uses Crawl4AI with fallback to BasicCrawler.
+    """
+
+    def __init__(self, config: Optional[CrawlerConfig] = None):
+        """Initialize adapter."""
+        self.config = config or CrawlerConfig()
+        self.basic_crawler = BasicCrawler(self.config)
+        self.crawl4ai: Optional[AsyncWebCrawler] = None
+
+    async def _get_crawl4ai(self) -> AsyncWebCrawler:
+        """Lazy-initialize Crawl4AI crawler."""
+        if self.crawl4ai is None:
+            self.crawl4ai = AsyncWebCrawler(
+                cache_mode=self.config.cache_mode,
+            )
+        return self.crawl4ai
+
+    def _select_profile(self, url: str) -> CrawlProfile:
+        """
+        Intelligent profile selection based on URL characteristics.
+
+        Phase 2 decision rules:
+        - If domain is known JS-heavy → dynamic_page
+        - If URL has sitemap → deep_discovery (not yet)
+        - Default → fast_static (HTTP only)
+        """
+        # TODO: Implement domain detection (robots.txt, Known JS-heavy list)
+        # For Phase 2 MVP: use fast_static by default
+        return CrawlProfile.FAST_STATIC
+
+    async def crawl(
+        self,
+        url: str,
+        profile: Optional[CrawlProfile] = None,
+    ) -> CrawlResult:
+        """
+        Crawl URL content with optional profile override.
+
+        Phase 2: Automatic profile selection + Crawl4AI support.
+
+        Args:
+            url: URL to crawl
+            profile: Optional profile override
+
+        Returns:
+            CrawlResult with content (HTML + optional markdown)
+        """
+        # Select profile
+        selected_profile = profile or self._select_profile(url)
+
+        try:
+            if selected_profile == CrawlProfile.FAST_STATIC:
+                # Phase 0-1: Use BasicCrawler for static content
+                return await self.basic_crawler.fetch_async(url)
+
+            elif selected_profile == CrawlProfile.DYNAMIC_PAGE:
+                # Phase 2: Use Crawl4AI for JS-rendered content
+                return await self._crawl_dynamic(url)
+
+            elif selected_profile == CrawlProfile.FULL_CAPTURE:
+                return await self._crawl_full_capture(url)
+
+            elif selected_profile == CrawlProfile.DEEP_DISCOVERY:
+                # Phase 2+: Not yet implemented
+                logger.warning(f"deep_discovery not yet implemented, using fast_static for {url}")
+                return await self.basic_crawler.fetch_async(url)
+
+            else:
+                # Fallback
+                return await self.basic_crawler.fetch_async(url)
+
+        except Exception as e:
+            logger.error(f"Crawl failed with profile {selected_profile}: {e}")
+            # Fallback to basic HTTP
+            try:
+                logger.info(f"Falling back to basic HTTP for {url}")
+                return await self.basic_crawler.fetch_async(url)
+            except Exception as fallback_err:
+                logger.error(f"Fallback also failed: {fallback_err}")
+                raise
+
+    async def _crawl_dynamic(self, url: str) -> CrawlResult:
+        """Crawl JavaScript-rendered page using Crawl4AI + Playwright."""
+        crawler = await self._get_crawl4ai()
+
+        config = CrawlerRunConfig(
+            cache_mode=self.config.cache_mode,
+            screenshot=False,
+            markdown_generator=None,  # Use default markdown
+        )
+
+        try:
+            result = await crawler.arun(url, config=config)
+
+            return CrawlResult(
+                url=url,
+                html=result.html or "",
+                status_code=200 if result.html else 500,
+                markdown=result.markdown,
+                profile_used="dynamic_page",
+            )
+
+        except Exception as e:
+            logger.error(f"Crawl4AI dynamic crawl failed for {url}: {e}")
+            raise
+
+    async def _crawl_full_capture(self, url: str) -> CrawlResult:
+        """Crawl with full capture (screenshot, PDF, MHTML)."""
+        crawler = await self._get_crawl4ai()
+
+        config = CrawlerRunConfig(
+            cache_mode=self.config.cache_mode,
+            screenshot=True,  # Capture screenshot
+        )
+
+        try:
+            result = await crawler.arun(url, config=config)
+
+            return CrawlResult(
+                url=url,
+                html=result.html or "",
+                status_code=200 if result.html else 500,
+                markdown=result.markdown,
+                profile_used="full_capture",
+            )
+
+        except Exception as e:
+            logger.error(f"Crawl4AI full capture failed for {url}: {e}")
+            raise
+
+    async def close(self):
+        """Clean up resources."""
+        self.basic_crawler.close()
+        if self.crawl4ai is not None:
+            await self.crawl4ai.close()
+
+
+async def crawl_url(url: str) -> CrawlResult:
+    """Convenience function for quick crawling."""
+    adapter = Crawl4AIAdapter()
+    try:
+        return await adapter.crawl(url)
+    finally:
+        adapter.close()
+
+
+if __name__ == "__main__":
+    # Simple test
+    import asyncio
+
+    async def test():
+        result = await crawl_url("https://example.com")
+        print(f"✓ Fetched {result.url}")
+        print(f"  Status: {result.status_code}")
+        print(f"  HTML length: {len(result.html)}")
+
+    asyncio.run(test())

ontology_platform/ont_platform/core/extraction/__init__.py

@@ -0,0 +1,5 @@
+"""Lightweight JSON extraction module (Phase 0)."""
+
+from .lightweight_extractor import LightweightExtractor, ExtractionResult
+
+__all__ = ["LightweightExtractor", "ExtractionResult"]

ontology_platform/ont_platform/core/extraction/lightweight_extractor.py

@@ -0,0 +1,148 @@
+"""
+Phase 0: Lightweight JSON extraction of entities and relations.
+
+Simple rule-based extraction (no LLM yet).
+Returns plain Python dicts, no Pydantic models.
+Goal: 10-30 seconds per URL.
+"""
+
+import re
+import uuid
+from dataclasses import dataclass
+
+
+@dataclass
+class ExtractionResult:
+    """Simple extraction result container."""
+
+    entities: list  # list of dicts
+    relations: list  # list of dicts
+    evidence_spans: list  # list of dicts
+    warnings: list  # list of warning strings
+
+
+class LightweightExtractor:
+    """Extract entity/relation candidates from text (Phase 0 MVP)."""
+
+    def __init__(self, use_llm: bool = False):
+        """Initialize extractor.
+
+        Args:
+            use_llm: Ignored in Phase 0 (rule-based only)
+        """
+        self.use_llm = use_llm
+
+    def extract(
+        self,
+        text: str,
+        project_id: str,
+        document_id: str,
+    ) -> ExtractionResult:
+        """
+        Extract candidates from text.
+
+        Phase 0: Simple rule-based extraction
+        - Find capitalized words (proper nouns)
+        - No relations for now
+
+        Args:
+            text: Document text
+            project_id: Project ID (for later use)
+            document_id: Source document ID (for later use)
+
+        Returns:
+            ExtractionResult with entities and metadata
+        """
+        entities = []
+        evidence_spans = []
+        warnings = []
+
+        # Find named entities (capitalized sequences)
+        entity_matches = self._find_named_entities(text)
+
+        for match in entity_matches:
+            entity_id = f"E_{uuid.uuid4().hex[:8]}"
+            evidence_id = f"EV_{uuid.uuid4().hex[:8]}"
+
+            # Create entity dict
+            entities.append({
+                "id": entity_id,
+                "label": match["text"],
+                "type": "concept",  # Phase 0: no type inference
+                "confidence": 0.6,  # Phase 0: constant confidence
+                "evidence_ids": [evidence_id],
+            })
+
+            # Create evidence span dict
+            evidence_spans.append({
+                "id": evidence_id,
+                "text": match["text"],
+                "start_offset": match["start"],
+                "end_offset": match["end"],
+            })
+
+        # Phase 0: No relation extraction yet
+        relations = []
+
+        # Validate and warn
+        warnings = self._validate_extraction(entities, relations)
+
+        return ExtractionResult(
+            entities=entities,
+            relations=relations,
+            evidence_spans=evidence_spans,
+            warnings=warnings,
+        )
+
+    @staticmethod
+    def _find_named_entities(text: str) -> list[dict]:
+        """
+        Find named entities using simple regex.
+
+        Phase 0 MVP: Capitalize letter sequences, proper nouns.
+        Phase 1+: Use NER model or LLM.
+
+        Returns:
+            List of {text, start, end} dicts
+        """
+        matches = []
+
+        # Pattern: Capitalized words (proper nouns)
+        pattern = r"\b[A-Z][a-z]+(?:\s+[A-Z][a-z]+)*\b"
+
+        for match in re.finditer(pattern, text):
+            word = match.group()
+            # Filter out common words
+            if word not in {"The", "This", "That", "These", "Those"}:
+                matches.append({
+                    "text": word,
+                    "start": match.start(),
+                    "end": match.end(),
+                })
+
+        return matches
+
+    @staticmethod
+    def _validate_extraction(entities: list, relations: list) -> list[str]:
+        """Validate extraction results.
+
+        Phase 0: Basic checks only.
+        Phase 2+: Use Guardrails for stronger validation.
+
+        Returns:
+            List of warning messages
+        """
+        warnings = []
+
+        # Check for meaningless entities
+        meaningless_terms = {"value", "keyword", "type", "name", "item", "thing"}
+        for entity in entities:
+            if entity["label"].lower() in meaningless_terms:
+                warnings.append(f"Low-confidence entity: {entity['label']}")
+
+        # Check for very short entities
+        for entity in entities:
+            if len(entity["label"]) < 2:
+                warnings.append(f"Very short entity: {entity['label']}")
+
+        return warnings

ontology_platform/ont_platform/core/extraction/schemas.py

@@ -0,0 +1,131 @@
+"""
+Pydantic schemas for extraction and validation.
+
+Defines the structure of extracted candidates for API and validation.
+"""
+
+from typing import Optional
+from datetime import datetime
+from pydantic import BaseModel, Field
+
+
+class EvidenceSpanSchema(BaseModel):
+    """Evidence text span."""
+
+    id: str
+    text: str
+    start_offset: int
+    end_offset: int
+
+    class Config:
+        from_attributes = True
+
+
+class CandidateEntitySchema(BaseModel):
+    """Extracted entity candidate."""
+
+    id: str
+    label: str
+    entity_type: str = Field(..., description="Entity type (concept, person, org, etc.)")
+    description: Optional[str] = None
+    confidence: float = Field(..., ge=0.0, le=1.0)
+    source_trust: Optional[float] = Field(default=0.5, ge=0.0, le=1.0)
+    evidence_ids: Optional[list[str]] = []
+    aliases: Optional[list[str]] = []
+
+    class Config:
+        from_attributes = True
+
+
+class CandidateRelationSchema(BaseModel):
+    """Extracted relation candidate."""
+
+    id: str
+    source_entity_id: str
+    predicate: str
+    target_entity_id: str
+    confidence: float = Field(..., ge=0.0, le=1.0)
+    source_trust: Optional[float] = Field(default=0.5, ge=0.0, le=1.0)
+    evidence_ids: Optional[list[str]] = []
+
+    class Config:
+        from_attributes = True
+
+
+class LightweightExtractionResult(BaseModel):
+    """Result of lightweight JSON extraction."""
+
+    entities: list[CandidateEntitySchema] = []
+    relations: list[CandidateRelationSchema] = []
+    evidence_spans: list[EvidenceSpanSchema] = []
+    warnings: list[str] = []
+
+    class Config:
+        from_attributes = True
+
+
+class SourceDocumentSchema(BaseModel):
+    """Source document metadata."""
+
+    id: str
+    project_id: str
+    source_url: Optional[str] = None
+    file_path: Optional[str] = None
+    document_type: str  # "html", "pdf", "markdown", "docx", "inline_text"
+
+    title: Optional[str] = None
+    author: Optional[str] = None
+    publish_date: Optional[str] = None
+    language: Optional[str] = None
+    sitename: Optional[str] = None
+
+    content_hash: str
+    fingerprint: Optional[str] = None
+    retrieved_at: str  # ISO-8601
+    extracted_by: str
+
+    class Config:
+        from_attributes = True
+
+
+class ExtractionJobSchema(BaseModel):
+    """Extraction job information."""
+
+    id: str
+    project_id: str
+    job_type: str
+    status: str
+    input_url: Optional[str] = None
+    input_file: Optional[str] = None
+    document_id: Optional[str] = None
+    entity_count: int = 0
+    relation_count: int = 0
+    error_message: Optional[str] = None
+    created_at: str
+
+    class Config:
+        from_attributes = True
+
+
+class ExtractRequestSchema(BaseModel):
+    """Request to extract from URL or text."""
+
+    url: Optional[str] = None
+    project_id: str
+
+    class Config:
+        json_schema_extra = {
+            "example": {"url": "https://example.com", "project_id": "proj_123"}
+        }
+
+
+class CandidateListResponseSchema(BaseModel):
+    """Response listing candidates."""
+
+    document_id: str
+    document_title: Optional[str]
+    entity_count: int
+    relation_count: int
+    entities: list[CandidateEntitySchema]
+    relations: list[CandidateRelationSchema]
+    extracted_at: str

ontology_platform/ont_platform/core/extractors/web_extractor.py

@@ -0,0 +1,174 @@
+"""
+Web content extraction using Trafilatura.
+
+Handles HTML/URL content extraction with metadata preservation for ontology candidate extraction.
+"""
+
+from dataclasses import dataclass
+from typing import Optional
+import hashlib
+from datetime import datetime
+
+import trafilatura
+from trafilatura import extract
+from trafilatura.metadata import extract_metadata
+
+
+@dataclass
+class ExtractedWebContent:
+    """Result of web content extraction."""
+
+    url: str | None
+    text: str
+    title: str | None
+    author: str | None
+    publish_date: str | None
+    language: str | None
+    sitename: str | None
+
+    # Additional metadata
+    canonical_url: str | None
+    fingerprint: str | None
+    content_hash: str
+    retrieved_at: str
+    source: str  # "trafilatura"
+
+    # Raw metadata
+    metadata: dict
+
+
+class WebExtractor:
+    """Web content extractor using Trafilatura."""
+
+    def __init__(self):
+        """Initialize extractor."""
+        self.source = "trafilatura"
+
+    def extract_from_html(
+        self,
+        html: str,
+        source_url: str | None = None,
+    ) -> ExtractedWebContent:
+        """
+        Extract content from HTML string.
+
+        Args:
+            html: Raw HTML content
+            source_url: Optional source URL for metadata
+
+        Returns:
+            ExtractedWebContent with text and metadata
+        """
+        # Extract main content
+        text = extract(html, include_comments=False, output_format="txt")
+        if not text:
+            raise ValueError("Could not extract text from HTML")
+
+        # Extract metadata (returns Document object in trafilatura 2.0+)
+        doc = extract_metadata(html)
+
+        # Calculate content hash
+        content_hash = hashlib.sha256(text.encode()).hexdigest()
+
+        # Extract fingerprint (near-duplicate detection)
+        fingerprint = self._get_fingerprint(text)
+
+        # Convert Document object to dict (trafilatura 2.0+)
+        metadata_dict = {}
+        if doc:
+            metadata_dict = {
+                "title": getattr(doc, "title", None),
+                "author": getattr(doc, "author", None),
+                "date": getattr(doc, "date", None),
+                "language": getattr(doc, "language", None),
+                "sitename": getattr(doc, "sitename", None),
+                "url": getattr(doc, "url", None),
+            }
+
+        return ExtractedWebContent(
+            url=source_url,
+            text=text,
+            title=metadata_dict.get("title"),
+            author=metadata_dict.get("author"),
+            publish_date=metadata_dict.get("date"),
+            language=metadata_dict.get("language"),
+            sitename=metadata_dict.get("sitename"),
+            canonical_url=metadata_dict.get("url") or source_url,
+            fingerprint=fingerprint,
+            content_hash=content_hash,
+            retrieved_at=datetime.utcnow().isoformat(),
+            source=self.source,
+            metadata=metadata_dict,
+        )
+
+    def extract_from_url(
+        self,
+        url: str,
+        timeout: int = 10,
+    ) -> ExtractedWebContent:
+        """
+        Extract content from URL (requires network access).
+
+        Args:
+            url: HTTP(S) URL
+            timeout: Request timeout in seconds (not used with trafilatura 2.0+)
+
+        Returns:
+            ExtractedWebContent with text and metadata
+        """
+        try:
+            downloaded = trafilatura.fetch_url(url)
+            if not downloaded:
+                raise ValueError(f"Could not fetch URL: {url}")
+
+            return self.extract_from_html(downloaded, source_url=url)
+        except Exception as e:
+            raise RuntimeError(f"Failed to extract from {url}: {e}")
+
+    @staticmethod
+    def _get_fingerprint(text: str) -> str:
+        """
+        Calculate SimHash-like fingerprint for near-duplicate detection.
+
+        Simple implementation: hash of first 1000 chars + length.
+        For production, use trafilatura.content_fingerprint() or simhash.
+
+        Args:
+            text: Content text
+
+        Returns:
+            Fingerprint string
+        """
+        sample = text[:1000] if len(text) > 1000 else text
+        sample_hash = hashlib.md5(sample.encode()).hexdigest()[:16]
+        length_hash = hashlib.md5(str(len(text)).encode()).hexdigest()[:8]
+        return f"{sample_hash}_{length_hash}"
+
+
+
+def extract_web_content(
+    html: str | None = None,
+    url: str | None = None,
+) -> ExtractedWebContent:
+    """
+    Convenience function for web extraction.
+
+    Args:
+        html: Raw HTML (if available)
+        url: URL to fetch (if html not provided)
+
+    Returns:
+        ExtractedWebContent
+
+    Raises:
+        ValueError: If neither html nor url provided, or extraction fails
+    """
+    if not html and not url:
+        raise ValueError("Either html or url must be provided")
+
+    extractor = WebExtractor()
+
+    if html:
+        return extractor.extract_from_html(html, source_url=url)
+    else:
+        return extractor.extract_from_url(url)

ontology_platform/ont_platform/core/graph/neo4j_adapter.py

@@ -0,0 +1,372 @@
+"""Neo4j adapter for Phase 4: Graph projection and search.
+
+Provides:
+- Connection management
+- Basic RDF → Property Graph conversion
+- Vector embedding and indexing
+- Search APIs (vector search)
+
+Design: Lightweight, extensible for Phase 5+ enhancements.
+"""
+
+import logging
+from typing import Optional, List, Dict, Any
+import asyncio
+from neo4j import AsyncGraphDatabase
+from sentence_transformers import SentenceTransformer
+
+logger = logging.getLogger(__name__)
+
+
+class Neo4jConfig:
+    """Neo4j connection configuration."""
+
+    def __init__(
+        self,
+        uri: str = "bolt://localhost:7687",
+        username: str = "neo4j",
+        password: str = "ontology123",
+        database: str = "neo4j",
+    ):
+        self.uri = uri
+        self.username = username
+        self.password = password
+        self.database = database
+
+
+class Neo4jAdapter:
+    """
+    Neo4j adapter for Phase 4 (Lite).
+
+    Capabilities:
+    - Entity and relation node creation
+    - Basic RDF-like property management
+    - Vector embedding for search
+    - Simple vector search
+
+    Future (Phase 5+):
+    - RDF → Property Graph full projection
+    - GraphRAG integration
+    - Complex queries and analytics
+    """
+
+    def __init__(self, config: Optional[Neo4jConfig] = None):
+        """
+        Initialize Neo4j adapter.
+
+        Args:
+            config: Neo4j connection config (default: localhost:7687)
+        """
+        self.config = config or Neo4jConfig()
+        self._driver: Optional[AsyncDriver] = None
+        self._embedder: Optional[SentenceTransformer] = None
+
+    async def connect(self) -> bool:
+        """
+        Establish Neo4j connection.
+
+        Returns:
+            True if connection successful
+        """
+        try:
+            self._driver = AsyncGraphDatabase.driver(
+                self.config.uri,
+                auth=(self.config.username, self.config.password),
+            )
+            # Test connection
+            async with self._driver.session(database=self.config.database) as session:
+                await session.run("RETURN 1")
+            logger.info(f"Connected to Neo4j at {self.config.uri}")
+            return True
+        except Exception as e:
+            logger.error(f"Failed to connect to Neo4j: {e}")
+            return False
+
+    async def initialize_embedder(self, model_name: str = "all-MiniLM-L6-v2"):
+        """
+        Initialize embedding model.
+
+        Args:
+            model_name: SentenceTransformer model (default: all-MiniLM-L6-v2)
+        """
+        try:
+            self._embedder = SentenceTransformer(model_name)
+            logger.info(f"Loaded embedding model: {model_name}")
+        except Exception as e:
+            logger.error(f"Failed to load embedder: {e}")
+            raise
+
+    def _get_embeddings(self, texts: List[str]) -> List[List[float]]:
+        """
+        Get embeddings for texts.
+
+        Args:
+            texts: List of text strings
+
+        Returns:
+            List of embedding vectors
+        """
+        if not self._embedder:
+            raise RuntimeError("Embedder not initialized. Call initialize_embedder() first.")
+        return self._embedder.encode(texts, convert_to_tensor=False).tolist()
+
+    async def create_entity_nodes(
+        self,
+        entities: List[Dict[str, Any]],
+        entity_type: str = "Entity",
+    ) -> int:
+        """
+        Create entity nodes in Neo4j.
+
+        Args:
+            entities: List of entity dicts with id, label, properties
+            entity_type: Node label (default: Entity)
+
+        Returns:
+            Number of nodes created
+        """
+        if not self._driver:
+            raise RuntimeError("Not connected to Neo4j")
+
+        created = 0
+        async with self._driver.session(database=self.config.database) as session:
+            for entity in entities:
+                try:
+                    # Get embedding for label
+                    embedding = self._get_embeddings([entity.get("label", "")])[0]
+
+                    query = f"""
+                    MERGE (e:{entity_type} {{id: $id}})
+                    SET e.label = $label,
+                        e.type = $entity_type,
+                        e.confidence = $confidence,
+                        e.embedding = $embedding
+                    RETURN e
+                    """
+                    result = await session.run(
+                        query,
+                        id=entity.get("id"),
+                        label=entity.get("label"),
+                        entity_type=entity.get("type", "concept"),
+                        confidence=entity.get("confidence", 0.5),
+                        embedding=embedding,
+                    )
+                    await result.consume()
+                    created += 1
+                except Exception as e:
+                    logger.warning(f"Failed to create entity {entity.get('id')}: {e}")
+
+        logger.info(f"Created {created} entity nodes")
+        return created
+
+    async def create_relation_edges(
+        self,
+        relations: List[Dict[str, Any]],
+    ) -> int:
+        """
+        Create relation edges between entities.
+
+        Args:
+            relations: List of relation dicts with source_id, predicate, target_id
+
+        Returns:
+            Number of edges created
+        """
+        if not self._driver:
+            raise RuntimeError("Not connected to Neo4j")
+
+        created = 0
+        async with self._driver.session(database=self.config.database) as session:
+            for relation in relations:
+                try:
+                    query = """
+                    MATCH (source {id: $source_id})
+                    MATCH (target {id: $target_id})
+                    MERGE (source)-[r:RELATES {predicate: $predicate}]->(target)
+                    SET r.confidence = $confidence
+                    RETURN r
+                    """
+                    result = await session.run(
+                        query,
+                        source_id=relation.get("source_id"),
+                        target_id=relation.get("target_id"),
+                        predicate=relation.get("predicate"),
+                        confidence=relation.get("confidence", 0.5),
+                    )
+                    await result.consume()
+                    created += 1
+                except Exception as e:
+                    logger.warning(f"Failed to create relation: {e}")
+
+        logger.info(f"Created {created} relation edges")
+        return created
+
+    async def vector_search(
+        self,
+        query_text: str,
+        limit: int = 10,
+        threshold: float = 0.5,
+    ) -> List[Dict[str, Any]]:
+        """
+        Search for entities using vector similarity.
+
+        Args:
+            query_text: Query text
+            limit: Number of results to return
+            threshold: Minimum similarity threshold (0-1)
+
+        Returns:
+            List of matching entities with similarity scores
+        """
+        if not self._driver:
+            raise RuntimeError("Not connected to Neo4j")
+
+        if not self._embedder:
+            raise RuntimeError("Embedder not initialized")
+
+        # Get query embedding
+        query_embedding = self._get_embeddings([query_text])[0]
+
+        results = []
+        async with self._driver.session(database=self.config.database) as session:
+            # Simple similarity search using cosine distance
+            # Neo4j 5.18+ has built-in vector functions
+            query = """
+            MATCH (n:Entity)
+            WHERE n.embedding IS NOT NULL
+            WITH n, gds.similarity.cosine(n.embedding, $query_embedding) AS similarity
+            WHERE similarity >= $threshold
+            ORDER BY similarity DESC
+            LIMIT $limit
+            RETURN {
+                id: n.id,
+                label: n.label,
+                type: n.type,
+                confidence: n.confidence,
+                similarity: similarity
+            } AS result
+            """
+            try:
+                result = await session.run(
+                    query,
+                    query_embedding=query_embedding,
+                    threshold=threshold,
+                    limit=limit,
+                )
+                async for record in result:
+                    results.append(record["result"])
+            except Exception as e:
+                logger.warning(f"Vector search failed: {e}")
+                # Fallback: simple label search
+                fallback_query = """
+                MATCH (n:Entity)
+                WHERE n.label CONTAINS $query_text
+                LIMIT $limit
+                RETURN {
+                    id: n.id,
+                    label: n.label,
+                    type: n.type,
+                    confidence: n.confidence,
+                    similarity: 0.0
+                } AS result
+                """
+                result = await session.run(
+                    fallback_query,
+                    query_text=query_text,
+                    limit=limit,
+                )
+                async for record in result:
+                    results.append(record["result"])
+
+        logger.info(f"Vector search found {len(results)} results")
+        return results
+
+    async def get_entity_neighbors(
+        self,
+        entity_id: str,
+        depth: int = 1,
+    ) -> Dict[str, Any]:
+        """
+        Get neighbors of an entity (connected nodes).
+
+        Args:
+            entity_id: Entity ID
+            depth: Traversal depth (1-2)
+
+        Returns:
+            Entity and its neighbors
+        """
+        if not self._driver:
+            raise RuntimeError("Not connected to Neo4j")
+
+        async with self._driver.session(database=self.config.database) as session:
+            # Get entity itself
+            entity_query = "MATCH (n:Entity {id: $id}) RETURN n LIMIT 1"
+            entity_result = await session.run(entity_query, id=entity_id)
+            entity_record = await entity_result.single()
+
+            if not entity_record:
+                return {}
+
+            # Get neighbors
+            neighbors_query = f"""
+            MATCH (e:Entity {{id: $id}})-[r:RELATES*1..{depth}]-(neighbor)
+            RETURN {{
+                source: e.label,
+                target: neighbor.label,
+                predicate: type(r),
+                confidence: r.confidence
+            }} AS relation
+            """
+            relations_result = await session.run(neighbors_query, id=entity_id)
+            relations = []
+            async for record in relations_result:
+                relations.append(record["relation"])
+
+            return {
+                "entity": entity_id,
+                "label": entity_record["n"]["label"],
+                "type": entity_record["n"]["type"],
+                "neighbors": len(set(r["target"] for r in relations)),
+                "relations": relations,
+            }
+
+    async def get_stats(self) -> Dict[str, Any]:
+        """Get graph statistics."""
+        if not self._driver:
+            raise RuntimeError("Not connected to Neo4j")
+
+        async with self._driver.session(database=self.config.database) as session:
+            # Count nodes
+            nodes_result = await session.run("MATCH (n) RETURN count(n) AS count")
+            nodes_count = (await nodes_result.single())["count"]
+
+            # Count edges
+            edges_result = await session.run("MATCH ()-[r]->() RETURN count(r) AS count")
+            edges_count = (await edges_result.single())["count"]
+
+            # Count entities
+            entities_result = await session.run("MATCH (e:Entity) RETURN count(e) AS count")
+            entities_count = (await entities_result.single())["count"]
+
+            return {
+                "total_nodes": nodes_count,
+                "total_edges": edges_count,
+                "entity_nodes": entities_count,
+            }
+
+    async def close(self):
+        """Close Neo4j connection."""
+        if self._driver:
+            await self._driver.close()
+            logger.info("Neo4j connection closed")
+
+    async def __aenter__(self):
+        """Async context manager entry."""
+        if await self.connect():
+            await self.initialize_embedder()
+        return self
+
+    async def __aexit__(self, exc_type, exc_val, exc_tb):
+        """Async context manager exit."""
+        await self.close()

ontology_platform/ont_platform/core/validation/__init__.py

@@ -0,0 +1,38 @@
+"""Ontology validation module (Phase 3+).
+
+Supports multiple validation backends:
+- Phase 3 MVP (A): Lightweight Pydantic validation
+- Phase 3+ : Guardrails integration (prepared)
+- Phase 3 Option B (Hybrid): OntoCast SPARQL validation (prepared)
+"""
+
+from .models import (
+    OntologyEntity,
+    OntologyRelation,
+    OntologyExtractionResult,
+    Evidence,
+    EntityType,
+)
+from .guards import OntologyGuard, get_default_guard, validate
+from .validators import BaseValidator, LightweightValidator, ValidatorFactory
+from .ontocast_validator import OntoCastValidator, SPARQLValidator, GraphUpdate
+
+__all__ = [
+    # Models
+    "OntologyEntity",
+    "OntologyRelation",
+    "OntologyExtractionResult",
+    "Evidence",
+    "EntityType",
+    # Guards
+    "OntologyGuard",
+    "get_default_guard",
+    "validate",
+    # Validators
+    "BaseValidator",
+    "LightweightValidator",
+    "OntoCastValidator",
+    "SPARQLValidator",
+    "GraphUpdate",
+    "ValidatorFactory",
+]

ontology_platform/ont_platform/core/validation/guards.py

@@ -0,0 +1,109 @@
+"""Ontology Guards API (Phase 3+).
+
+High-level validation interface supporting multiple validation backends.
+Designed to be easily upgradable from lightweight (Phase 3 MVP) to
+Guardrails (Phase 3 upgraded) or OntoCast (Phase 3 Option B).
+"""
+
+from typing import Optional
+import logging
+
+from .models import OntologyExtractionResult
+from .validators import BaseValidator, ValidatorFactory
+
+logger = logging.getLogger(__name__)
+
+
+class OntologyGuard:
+    """
+    High-level guard for ontology extraction validation.
+
+    Wraps multiple validator backends and provides unified interface.
+    """
+
+    def __init__(
+        self,
+        validator_type: str = "lightweight",
+        strict: bool = False,
+        **kwargs,
+    ):
+        """
+        Initialize guard.
+
+        Args:
+            validator_type: "lightweight", "guardrails", or "ontocast"
+            strict: If True, raise on validation errors; if False, collect as warnings
+            **kwargs: Validator-specific config
+        """
+        self.validator_type = validator_type
+        self.strict = strict
+        try:
+            self.validator = ValidatorFactory.create(
+                validator_type=validator_type,
+                strict=strict,
+                **kwargs,
+            )
+            logger.info(f"Initialized {validator_type} validator")
+        except NotImplementedError as e:
+            logger.warning(f"{e}, falling back to lightweight")
+            self.validator = ValidatorFactory.create(
+                validator_type="lightweight",
+                strict=strict,
+            )
+
+    async def validate(self, result: dict) -> OntologyExtractionResult:
+        """
+        Validate extraction result.
+
+        Args:
+            result: Raw extraction result (dicts from LightweightExtractor)
+
+        Returns:
+            OntologyExtractionResult with validation status and errors
+        """
+        try:
+            validated = await self.validator.validate(result)
+            if not validated.validation_passed:
+                logger.warning(
+                    f"Validation warnings: {len(validated.validation_errors)} errors, "
+                    f"{len(validated.warnings)} total warnings"
+                )
+            return validated
+        except Exception as e:
+            logger.error(f"Validation failed: {e}")
+            if self.strict:
+                raise
+            # Fallback: return result with error markers
+            return OntologyExtractionResult(
+                entities=[],
+                relations=[],
+                warnings=[f"Validation failed: {str(e)}"],
+                validation_passed=False,
+                validation_errors=[str(e)],
+            )
+
+
+# Default instance
+_default_guard: Optional[OntologyGuard] = None
+
+
+def get_default_guard() -> OntologyGuard:
+    """Get or create default guard instance."""
+    global _default_guard
+    if _default_guard is None:
+        _default_guard = OntologyGuard(validator_type="lightweight", strict=False)
+    return _default_guard
+
+
+async def validate(result: dict) -> OntologyExtractionResult:
+    """
+    Convenience function: validate using default guard.
+
+    Args:
+        result: Raw extraction result
+
+    Returns:
+        OntologyExtractionResult
+    """
+    guard = get_default_guard()
+    return await guard.validate(result)

ontology_platform/ont_platform/core/validation/models.py

@@ -0,0 +1,99 @@
+"""Ontology validation data models (Phase 3+).
+
+Pydantic models for LLM extraction output validation.
+Designed to work with both lightweight validators (Phase 3 MVP)
+and Guardrails (Phase 3 upgraded).
+"""
+
+from pydantic import BaseModel, Field, field_validator
+from typing import Optional, List, Literal
+from enum import Enum
+
+
+class EntityType(str, Enum):
+    """Entity types in ontology extraction."""
+    CLASS = "class"
+    INDIVIDUAL = "individual"
+    OBJECT_PROPERTY = "object_property"
+    DATA_PROPERTY = "data_property"
+    CONCEPT = "concept"  # Phase 0-2 lightweight type
+
+
+class Evidence(BaseModel):
+    """Evidence for an extracted entity or relation."""
+    text: str = Field(..., description="Evidence text snippet")
+    source_url: Optional[str] = None
+    offset: Optional[tuple[int, int]] = None  # (start, end) character offsets
+    confidence: float = Field(default=0.8, ge=0.0, le=1.0)
+
+
+class OntologyEntity(BaseModel):
+    """Entity in ontology extraction result."""
+    id: str = Field(..., description="Unique entity ID (E_xxxxx)")
+    label: str = Field(..., min_length=1, max_length=500)
+    type: Literal["class", "individual", "object_property", "data_property", "concept"]
+    description: Optional[str] = Field(None, max_length=1000)
+    aliases: List[str] = Field(default_factory=list)
+    confidence: float = Field(..., ge=0.0, le=1.0)
+    evidence: List[Evidence] = Field(default_factory=list)
+
+    @field_validator("id")
+    @classmethod
+    def validate_entity_id(cls, v: str) -> str:
+        """Validate entity ID format (E_xxxxxxxx)."""
+        if not v.startswith("E_") or len(v) < 3:
+            raise ValueError(f"Entity ID must start with E_: {v}")
+        return v
+
+
+class OntologyRelation(BaseModel):
+    """Relation in ontology extraction result."""
+    id: str = Field(..., description="Unique relation ID (R_xxxxx)")
+    source_id: str = Field(..., description="Source entity ID")
+    predicate: str = Field(..., min_length=1, max_length=200)
+    target_id: str = Field(..., description="Target entity ID")
+    confidence: float = Field(..., ge=0.0, le=1.0)
+    evidence: List[Evidence] = Field(default_factory=list)
+
+    @field_validator("id")
+    @classmethod
+    def validate_relation_id(cls, v: str) -> str:
+        """Validate relation ID format (R_xxxxxxxx)."""
+        if not v.startswith("R_") or len(v) < 3:
+            raise ValueError(f"Relation ID must start with R_: {v}")
+        return v
+
+
+class OntologyExtractionResult(BaseModel):
+    """Result of ontology extraction with validation.
+
+    Phase 3 MVP: Lightweight validation with Pydantic
+    Phase 3+: Can be upgraded to Guardrails with reask/fix policies
+    """
+    entities: List[OntologyEntity] = Field(default_factory=list)
+    relations: List[OntologyRelation] = Field(default_factory=list)
+    warnings: List[str] = Field(default_factory=list)
+    validation_passed: bool = Field(default=True)
+    validation_errors: List[str] = Field(default_factory=list)
+
+    @field_validator("relations")
+    @classmethod
+    def validate_relation_endpoints(cls, v: List[OntologyRelation], info) -> List[OntologyRelation]:
+        """Validate that relation endpoints exist in entities."""
+        if info.data.get("entities"):
+            entity_ids = {e.id for e in info.data["entities"]}
+            for rel in v:
+                if rel.source_id not in entity_ids:
+                    raise ValueError(f"Relation {rel.id}: source entity {rel.source_id} not found")
+                if rel.target_id not in entity_ids:
+                    raise ValueError(f"Relation {rel.id}: target entity {rel.target_id} not found")
+        return v
+
+    @field_validator("relations")
+    @classmethod
+    def no_self_relations(cls, v: List[OntologyRelation]) -> List[OntologyRelation]:
+        """Relations cannot be self-loops."""
+        for rel in v:
+            if rel.source_id == rel.target_id:
+                raise ValueError(f"Self-relation not allowed: {rel.id}")
+        return v

ontology_platform/ont_platform/core/validation/ontocast_validator.py

@@ -0,0 +1,240 @@
+"""OntoCast GraphUpdate validation (Phase 3 Option B, Hybrid approach).
+
+This module provides validation for OntoCast's SPARQL operations (GraphUpdate).
+Designed to be integrated gradually without disrupting Phase 0-2.
+
+Key features:
+- SPARQL query syntax validation
+- Data consistency checks
+- Safe operation ordering (INSERT → UPDATE → DELETE)
+- Future Critic loop integration point
+"""
+
+import logging
+import re
+from typing import Optional, List, Dict, Any
+from pydantic import BaseModel, Field
+
+logger = logging.getLogger(__name__)
+
+
+class SPARQLOperation(BaseModel):
+    """Single SPARQL operation for validation."""
+    operation_type: str  # "INSERT", "UPDATE", "DELETE"
+    query: str
+    description: Optional[str] = None
+
+
+class GraphUpdate(BaseModel):
+    """OntoCast GraphUpdate model (simplified for validation)."""
+    operations: List[SPARQLOperation] = Field(default_factory=list)
+    namespaces: Dict[str, str] = Field(default_factory=dict)
+    validation_passed: bool = False
+    validation_errors: List[str] = Field(default_factory=list)
+    validation_warnings: List[str] = Field(default_factory=list)
+
+
+class SPARQLValidator:
+    """Basic SPARQL query validator."""
+
+    # Common SPARQL keywords
+    SPARQL_KEYWORDS = {
+        "INSERT", "DELETE", "UPDATE", "SELECT", "CONSTRUCT", "DESCRIBE",
+        "ASK", "WHERE", "FILTER", "OPTIONAL", "UNION", "GRAPH", "SERVICE"
+    }
+
+    # Required RDF prefixes
+    COMMON_PREFIXES = {
+        "rdf": "http://www.w3.org/1999/02/22-rdf-syntax-ns#",
+        "rdfs": "http://www.w3.org/2000/01/rdf-schema#",
+        "owl": "http://www.w3.org/2002/07/owl#",
+        "xsd": "http://www.w3.org/2001/XMLSchema#",
+    }
+
+    @staticmethod
+    def validate_sparql_syntax(query: str) -> tuple[bool, List[str]]:
+        """
+        Basic SPARQL syntax validation.
+
+        Checks:
+        - Query is not empty
+        - Has valid SPARQL keywords
+        - No obvious syntax errors
+        - Balanced brackets/braces
+
+        Returns:
+            (is_valid, error_messages)
+        """
+        errors = []
+
+        # Check empty
+        if not query or not query.strip():
+            errors.append("SPARQL query cannot be empty")
+            return False, errors
+
+        # Check for SPARQL keywords
+        uppercase_query = query.upper()
+        has_keyword = any(kw in uppercase_query for kw in SPARQLValidator.SPARQL_KEYWORDS)
+        if not has_keyword:
+            errors.append("Query does not contain recognized SPARQL keywords")
+
+        # Check bracket balance
+        if query.count("{") != query.count("}"):
+            errors.append("Unbalanced curly braces in SPARQL query")
+
+        if query.count("[") != query.count("]"):
+            errors.append("Unbalanced square brackets in SPARQL query")
+
+        if query.count("(") != query.count(")"):
+            errors.append("Unbalanced parentheses in SPARQL query")
+
+        # Check for obvious SQL injection patterns (safety)
+        dangerous_patterns = [
+            r";\s*(DROP|TRUNCATE|EXEC)",  # SQL commands
+            r"'|\".*?;",  # Quoted semicolons
+        ]
+        for pattern in dangerous_patterns:
+            if re.search(pattern, query, re.IGNORECASE):
+                errors.append(f"Potentially dangerous pattern detected: {pattern}")
+
+        return len(errors) == 0, errors
+
+    @staticmethod
+    def validate_operation_order(operations: List[SPARQLOperation]) -> tuple[bool, List[str]]:
+        """
+        Validate SPARQL operation ordering.
+
+        Safe order: INSERT → UPDATE → DELETE
+
+        Returns:
+            (is_valid, error_messages)
+        """
+        errors = []
+        order_map = {"INSERT": 0, "UPDATE": 1, "DELETE": 2}
+        last_priority = -1
+
+        for op in operations:
+            op_type = op.operation_type.upper()
+            priority = order_map.get(op_type, -1)
+
+            if priority == -1:
+                errors.append(f"Unknown operation type: {op_type}")
+            elif priority < last_priority:
+                errors.append(
+                    f"Unsafe operation order: {op_type} after "
+                    f"{[k for k, v in order_map.items() if v == last_priority][0]}"
+                )
+            last_priority = priority
+
+        return len(errors) == 0, errors
+
+    @staticmethod
+    def validate_prefix_declarations(
+        operations: List[SPARQLOperation],
+        declared_prefixes: Dict[str, str]
+    ) -> tuple[bool, List[str]]:
+        """
+        Validate that used prefixes are declared.
+
+        Returns:
+            (is_valid, error_messages)
+        """
+        errors = []
+
+        # Extract prefix usage from queries
+        used_prefixes = set()
+        for op in operations:
+            # Simple pattern: word:something
+            matches = re.findall(r"(\w+):\w+", op.query)
+            used_prefixes.update(matches)
+
+        # Check against declared
+        for prefix in used_prefixes:
+            if prefix not in declared_prefixes and prefix not in SPARQLValidator.COMMON_PREFIXES:
+                errors.append(f"Prefix '{prefix}' used but not declared")
+
+        return len(errors) == 0, errors
+
+
+class OntoCastValidator:
+    """
+    OntoCast GraphUpdate validator (Phase 3 Option B).
+
+    Hybrid approach:
+    - Lightweight validation now (SPARQL syntax, operation order)
+    - Future: Critic loop integration (Phase 4+)
+    - Future: Full RDF consistency checks when Fuseki is available
+    """
+
+    def __init__(self, strict: bool = False):
+        """
+        Initialize OntoCast validator.
+
+        Args:
+            strict: If True, reject on first error; if False, collect warnings
+        """
+        self.strict = strict
+        self.sparql_validator = SPARQLValidator()
+
+    async def validate(self, update: Dict[str, Any]) -> GraphUpdate:
+        """
+        Validate OntoCast GraphUpdate.
+
+        Args:
+            update: Raw GraphUpdate dict with operations and namespaces
+
+        Returns:
+            GraphUpdate with validation status and errors
+        """
+        errors = []
+        warnings = []
+
+        # Parse operations
+        operations = []
+        for op_dict in update.get("operations", []):
+            try:
+                op = SPARQLOperation(**op_dict)
+                operations.append(op)
+            except Exception as e:
+                msg = f"Invalid operation: {str(e)}"
+                errors.append(msg)
+                if self.strict:
+                    raise
+
+        namespaces = update.get("namespaces", {})
+
+        # Phase 1: SPARQL syntax validation
+        for op in operations:
+            is_valid, syntax_errors = self.sparql_validator.validate_sparql_syntax(op.query)
+            if not is_valid:
+                errors.extend(syntax_errors)
+                if self.strict:
+                    break
+
+        # Phase 2: Operation order validation
+        is_ordered, order_errors = self.sparql_validator.validate_operation_order(operations)
+        if not is_ordered:
+            errors.extend(order_errors)
+            if self.strict:
+                raise ValueError(f"Invalid operation order: {order_errors}")
+
+        # Phase 3: Prefix validation
+        is_prefixed, prefix_errors = self.sparql_validator.validate_prefix_declarations(
+            operations, namespaces
+        )
+        if not is_prefixed:
+            warnings.extend(prefix_errors)
+
+        # Phase 4: Operation count sanity check
+        if len(operations) == 0:
+            warnings.append("GraphUpdate contains no operations")
+        elif len(operations) > 100:
+            warnings.append(f"GraphUpdate contains {len(operations)} operations (very large)")
+
+        return GraphUpdate(
+            operations=operations,
+            namespaces=namespaces,
+            validation_passed=len(errors) == 0,
+            validation_errors=errors,
+            validation_warnings=warnings,
+        )

ontology_platform/ont_platform/core/validation/validators.py

@@ -0,0 +1,157 @@
+"""Validation logic for ontology extraction (Phase 3+).
+
+Abstract validator interface designed to support both:
+- Lightweight validation (Phase 3 MVP, Pydantic-based)
+- Guardrails integration (Phase 3 upgraded)
+- OntoCast integration (Phase 3 Option B)
+"""
+
+from abc import ABC, abstractmethod
+from typing import Optional, List, Tuple
+from pydantic import ValidationError
+
+from .models import OntologyExtractionResult, OntologyEntity, OntologyRelation
+
+
+class BaseValidator(ABC):
+    """Abstract base for ontology validators."""
+
+    @abstractmethod
+    async def validate(self, result: dict) -> OntologyExtractionResult:
+        """
+        Validate extraction result.
+
+        Args:
+            result: Raw extraction result (dicts)
+
+        Returns:
+            OntologyExtractionResult with validation status
+        """
+        pass
+
+
+class LightweightValidator(BaseValidator):
+    """Phase 3 MVP: Pydantic-based lightweight validation."""
+
+    def __init__(self, strict: bool = False):
+        """
+        Initialize lightweight validator.
+
+        Args:
+            strict: If False, collect warnings; if True, raise on first error
+        """
+        self.strict = strict
+
+    async def validate(self, result: dict) -> OntologyExtractionResult:
+        """
+        Validate extraction result using Pydantic models.
+
+        Phase 3 MVP approach:
+        1. Convert dicts to Pydantic models
+        2. Run field validators
+        3. Collect validation errors as warnings (non-strict)
+        4. Return validated result
+
+        Args:
+            result: Raw dict with entities, relations, warnings
+
+        Returns:
+            OntologyExtractionResult with validation_passed flag
+        """
+        entities = []
+        relations = []
+        validation_errors = []
+        warnings = list(result.get("warnings", []))
+
+        # Phase 1: Validate entities
+        for ent_dict in result.get("entities", []):
+            try:
+                entity = OntologyEntity(**ent_dict)
+                entities.append(entity)
+            except ValidationError as e:
+                error_msg = f"Entity {ent_dict.get('id', '?')}: {str(e)}"
+                validation_errors.append(error_msg)
+                if self.strict:
+                    raise
+                warnings.append(error_msg)
+
+        # Phase 2: Validate relations
+        for rel_dict in result.get("relations", []):
+            try:
+                relation = OntologyRelation(**rel_dict)
+                # Check that endpoints exist
+                entity_ids = {e.id for e in entities}
+                if relation.source_id not in entity_ids:
+                    raise ValueError(f"Source entity {relation.source_id} not found")
+                if relation.target_id not in entity_ids:
+                    raise ValueError(f"Target entity {relation.target_id} not found")
+                relations.append(relation)
+            except (ValidationError, ValueError) as e:
+                error_msg = f"Relation {rel_dict.get('id', '?')}: {str(e)}"
+                validation_errors.append(error_msg)
+                if self.strict:
+                    raise
+                warnings.append(error_msg)
+
+        # Phase 3: Check for duplicate entity IDs
+        entity_ids = [e.id for e in entities]
+        duplicates = [eid for eid in entity_ids if entity_ids.count(eid) > 1]
+        if duplicates:
+            error_msg = f"Duplicate entity IDs: {duplicates}"
+            validation_errors.append(error_msg)
+            warnings.append(error_msg)
+
+        # Phase 4: Check for meaningless entities
+        meaningless_terms = {"value", "keyword", "type", "name", "item", "thing"}
+        for entity in entities:
+            if entity.label.lower() in meaningless_terms and entity.confidence < 0.7:
+                warnings.append(f"Low-confidence meaningless entity: {entity.label}")
+
+        return OntologyExtractionResult(
+            entities=entities,
+            relations=relations,
+            warnings=warnings,
+            validation_passed=len(validation_errors) == 0,
+            validation_errors=validation_errors,
+        )
+
+
+class ValidatorFactory:
+    """Factory for creating validators (supports multiple implementations)."""
+
+    LIGHTWEIGHT = "lightweight"
+    GUARDRAILS = "guardrails"  # Phase 3+ (future)
+    ONTOCAST = "ontocast"  # Phase 3 Option B (prepared)
+
+    @staticmethod
+    def create(
+        validator_type: str = LIGHTWEIGHT,
+        **kwargs,
+    ) -> BaseValidator:
+        """
+        Create validator instance.
+
+        Args:
+            validator_type: Type of validator ("lightweight", "guardrails", "ontocast")
+            **kwargs: Additional config for specific validator
+
+        Returns:
+            BaseValidator instance
+
+        Raises:
+            ValueError: If validator_type not supported
+        """
+        if validator_type == ValidatorFactory.LIGHTWEIGHT:
+            return LightweightValidator(
+                strict=kwargs.get("strict", False),
+            )
+        elif validator_type == ValidatorFactory.GUARDRAILS:
+            raise NotImplementedError("Guardrails validator requires 'pip install guardrails-ai'")
+        elif validator_type == ValidatorFactory.ONTOCAST:
+            # Phase 3 Option B: OntoCast validator
+            from .ontocast_validator import OntoCastValidator
+            return OntoCastValidator(
+                strict=kwargs.get("strict", False),
+            )
+        else:
+            raise ValueError(f"Unknown validator type: {validator_type}")

ontology_platform/ont_platform/storage/__init__.py

@@ -0,0 +1,7 @@
+"""Storage module (Phase 1+).
+
+Phase 0: No database storage yet.
+Phase 1: Add SQLAlchemy models for candidate storage.
+"""
+
+__all__ = []

ontology_platform/ont_platform/storage/init_db.py

@@ -0,0 +1,37 @@
+"""Database initialization script."""
+
+import logging
+from sqlalchemy import create_engine
+
+from ont_platform.config import load_settings
+from ont_platform.storage.models import Base
+
+logger = logging.getLogger(__name__)
+
+
+def init_db():
+    """Create all tables in the database."""
+    settings = load_settings()
+    database_url = settings.database_url
+
+    engine = create_engine(
+        database_url,
+        connect_args={"timeout": 30} if "sqlite" in database_url else {},
+    )
+
+    logger.info(f"Creating tables in {database_url}")
+    Base.metadata.create_all(bind=engine)
+    logger.info("Database tables created successfully")
+
+
+if __name__ == "__main__":
+    import sys
+
+    logging.basicConfig(level=logging.INFO)
+    try:
+        init_db()
+        print("✓ Database initialized")
+        sys.exit(0)
+    except Exception as e:
+        print(f"✗ Error: {e}")
+        sys.exit(1)

ontology_platform/ont_platform/storage/models.py

@@ -0,0 +1,154 @@
+"""
+Database models for candidate storage.
+
+Holds extracted entity/relation candidates before final RDF conversion.
+"""
+
+from datetime import datetime
+from enum import Enum
+from typing import Any
+
+from sqlalchemy import JSON, Column, DateTime, Float, Integer, String, Text, Enum as SQLEnum
+from sqlalchemy.orm import declarative_base
+
+Base = declarative_base()
+
+
+class ReviewStatus(str, Enum):
+    """Review status of a candidate."""
+
+    PENDING = "pending"  # Awaiting human review
+    APPROVED = "approved"  # Approved by human
+    AUTO_APPROVED = "auto_approved"  # Approved by policy
+    REJECTED = "rejected"  # Rejected by human
+
+
+class SourceDocument(Base):
+    """Source document metadata."""
+
+    __tablename__ = "source_documents"
+
+    id = Column(String(255), primary_key=True)
+    project_id = Column(String(255), nullable=False, index=True)
+    source_url = Column(String(2048), nullable=True, index=True)
+    file_path = Column(String(2048), nullable=True)
+    document_type = Column(String(50))  # "html", "pdf", "markdown", "docx", "inline_text"
+
+    title = Column(String(512), nullable=True)
+    author = Column(String(255), nullable=True)
+    publish_date = Column(String(50), nullable=True)  # ISO-8601
+    language = Column(String(10), nullable=True)
+    sitename = Column(String(255), nullable=True)
+
+    text = Column(Text)
+    content_hash = Column(String(64), unique=True, nullable=False, index=True)
+    fingerprint = Column(String(100), nullable=True, index=True)
+
+    retrieved_at = Column(DateTime, default=datetime.utcnow)
+    extracted_by = Column(String(100), default="trafilatura")  # Source tool
+
+    metadata = Column(JSON, nullable=True)  # Raw metadata
+
+    created_at = Column(DateTime, default=datetime.utcnow)
+
+
+class EvidenceSpan(Base):
+    """Evidence text span from source document."""
+
+    __tablename__ = "evidence_spans"
+
+    id = Column(String(255), primary_key=True)
+    document_id = Column(String(255), nullable=False, index=True)
+    project_id = Column(String(255), nullable=False, index=True)
+
+    text = Column(Text)
+    start_offset = Column(Integer)
+    end_offset = Column(Integer)
+
+    created_at = Column(DateTime, default=datetime.utcnow)
+
+
+class CandidateEntity(Base):
+    """Extracted entity candidate."""
+
+    __tablename__ = "candidate_entities"
+
+    id = Column(String(255), primary_key=True)
+    project_id = Column(String(255), nullable=False, index=True)
+    document_id = Column(String(255), nullable=False, index=True)
+
+    label = Column(String(512), nullable=False)
+    entity_type = Column(String(100), nullable=False)  # "concept", "person", "org", etc.
+    description = Column(Text, nullable=True)
+
+    confidence = Column(Float, default=0.5)  # 0.0 ~ 1.0
+    source_trust = Column(Float, default=0.5)  # Trust in source
+
+    evidence_ids = Column(JSON, nullable=True)  # List of evidence span IDs
+    aliases = Column(JSON, nullable=True)  # List of alternative names
+
+    review_status = Column(SQLEnum(ReviewStatus), default=ReviewStatus.PENDING, index=True)
+    reviewed_by = Column(String(255), nullable=True)
+    reviewed_at = Column(DateTime, nullable=True)
+    review_reason = Column(Text, nullable=True)
+
+    metadata = Column(JSON, nullable=True)  # Raw LLM output, domain-specific fields
+
+    created_at = Column(DateTime, default=datetime.utcnow)
+    updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)
+
+
+class CandidateRelation(Base):
+    """Extracted relation candidate."""
+
+    __tablename__ = "candidate_relations"
+
+    id = Column(String(255), primary_key=True)
+    project_id = Column(String(255), nullable=False, index=True)
+    document_id = Column(String(255), nullable=False, index=True)
+
+    source_entity_id = Column(String(255), nullable=False, index=True)
+    predicate = Column(String(255), nullable=False)
+    target_entity_id = Column(String(255), nullable=False, index=True)
+
+    confidence = Column(Float, default=0.5)  # 0.0 ~ 1.0
+    source_trust = Column(Float, default=0.5)
+
+    evidence_ids = Column(JSON, nullable=True)  # List of evidence span IDs
+
+    review_status = Column(SQLEnum(ReviewStatus), default=ReviewStatus.PENDING, index=True)
+    reviewed_by = Column(String(255), nullable=True)
+    reviewed_at = Column(DateTime, nullable=True)
+    review_reason = Column(Text, nullable=True)
+
+    metadata = Column(JSON, nullable=True)  # Raw LLM output
+
+    created_at = Column(DateTime, default=datetime.utcnow)
+    updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)
+
+
+class ExtractionJob(Base):
+    """Extraction job metadata."""
+
+    __tablename__ = "extraction_jobs"
+
+    id = Column(String(255), primary_key=True)
+    project_id = Column(String(255), nullable=False, index=True)
+
+    job_type = Column(String(50))  # "extract", "validate", "review", etc.
+    status = Column(String(50), index=True)  # "pending", "running", "completed", "failed"
+
+    input_url = Column(String(2048), nullable=True)
+    input_file = Column(String(2048), nullable=True)
+
+    document_id = Column(String(255), nullable=True)
+    entity_count = Column(Integer, default=0)
+    relation_count = Column(Integer, default=0)
+    error_message = Column(Text, nullable=True)
+
+    started_at = Column(DateTime, nullable=True)
+    completed_at = Column(DateTime, nullable=True)
+
+    metadata = Column(JSON, nullable=True)
+
+    created_at = Column(DateTime, default=datetime.utcnow)