Phase 4 구현 완료: Neo4j 벡터 검색 + 그래프 저장소
This commit is contained in:
lasta
252
ontology_platform/ont_platform/api/phase0_app.py
Normal file
252
ontology_platform/ont_platform/api/phase0_app.py
Normal file
@@ -0,0 +1,252 @@
|
||||
"""Phase 0-4 FastAPI application.
|
||||
|
||||
Phase 0: Basic URL extraction
|
||||
Phase 2: Crawl4AI profile support for dynamic pages
|
||||
Phase 3: Validation (lightweight + OntoCast)
|
||||
Phase 4: Neo4j vector search
|
||||
"""
|
||||
|
||||
from fastapi import FastAPI, APIRouter, HTTPException, Query
|
||||
from typing import Optional, Literal, List
|
||||
import time
|
||||
import asyncio
|
||||
import logging
|
||||
|
||||
from ont_platform.core.extractors.web_extractor import extract_web_content
|
||||
from ont_platform.core.extraction.lightweight_extractor import LightweightExtractor
|
||||
from ont_platform.core.crawler.crawl4ai_adapter import (
|
||||
Crawl4AIAdapter,
|
||||
CrawlProfile,
|
||||
)
|
||||
from ont_platform.core.validation import OntologyGuard
|
||||
from ont_platform.core.graph.neo4j_adapter import Neo4jAdapter, Neo4jConfig
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
app = FastAPI(
|
||||
title="Ontology Platform - Phase 0-4",
|
||||
description="Extraction + Validation + Graph Search. 10-30 seconds per URL.",
|
||||
version="0.4.0",
|
||||
)
|
||||
|
||||
extraction_router = APIRouter(prefix="/api/v1/extract", tags=["extraction"])
|
||||
search_router = APIRouter(prefix="/api/v1/search", tags=["search"])
|
||||
|
||||
# Phase 3: Initialize validation guard
|
||||
guard = OntologyGuard(validator_type="lightweight", strict=False)
|
||||
|
||||
# Phase 4: Neo4j adapter (lazy initialization)
|
||||
_neo4j_adapter: Optional[Neo4jAdapter] = None
|
||||
|
||||
async def get_neo4j_adapter() -> Neo4jAdapter:
|
||||
"""Get or create Neo4j adapter instance."""
|
||||
global _neo4j_adapter
|
||||
if _neo4j_adapter is None:
|
||||
_neo4j_adapter = Neo4jAdapter()
|
||||
if not await _neo4j_adapter.connect():
|
||||
logger.warning("Neo4j not available, search will be unavailable")
|
||||
else:
|
||||
try:
|
||||
await _neo4j_adapter.initialize_embedder()
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to initialize embedder: {e}")
|
||||
return _neo4j_adapter
|
||||
|
||||
|
||||
@extraction_router.post("/url")
|
||||
async def extract_url(
|
||||
url: str,
|
||||
profile: Optional[Literal["fast_static", "dynamic_page"]] = Query(None),
|
||||
):
|
||||
"""
|
||||
Extract candidates from URL (Phase 0-2).
|
||||
|
||||
Phase 0-1: Default fast_static (HTTP only)
|
||||
Phase 2: Supports dynamic_page for JS-rendered content
|
||||
"""
|
||||
if not url:
|
||||
raise HTTPException(status_code=400, detail="url is required")
|
||||
|
||||
start_time = time.time()
|
||||
|
||||
try:
|
||||
# Phase 2: Use Crawl4AI for dynamic pages
|
||||
if profile == "dynamic_page":
|
||||
adapter = Crawl4AIAdapter()
|
||||
try:
|
||||
crawl_result = await adapter.crawl(url, profile=CrawlProfile.DYNAMIC_PAGE)
|
||||
profile_used = crawl_result.profile_used
|
||||
html_content = crawl_result.html
|
||||
finally:
|
||||
await adapter.close()
|
||||
|
||||
# Extract from crawled HTML
|
||||
extracted = extract_web_content(html=html_content, url=url)
|
||||
else:
|
||||
# Phase 0-1: Default fast_static (HTTP only)
|
||||
extracted = extract_web_content(url=url)
|
||||
profile_used = "trafilatura"
|
||||
|
||||
# Step 2: Extract JSON candidates with lightweight extractor
|
||||
lightweight = LightweightExtractor(use_llm=False)
|
||||
candidates = lightweight.extract(
|
||||
text=extracted.text,
|
||||
project_id="default",
|
||||
document_id="temp",
|
||||
)
|
||||
|
||||
# Phase 3: Validate extraction results
|
||||
raw_result = {
|
||||
"entities": candidates.entities,
|
||||
"relations": candidates.relations,
|
||||
"warnings": candidates.warnings,
|
||||
}
|
||||
validated = await guard.validate(raw_result)
|
||||
|
||||
extraction_time = time.time() - start_time
|
||||
|
||||
# Return JSON with validation info
|
||||
return {
|
||||
"url": url,
|
||||
"title": extracted.title,
|
||||
"author": extracted.author,
|
||||
"published_date": extracted.publish_date,
|
||||
"language": extracted.language,
|
||||
"text_length": len(extracted.text),
|
||||
"profile_used": profile_used,
|
||||
"entities": [e.dict() for e in validated.entities],
|
||||
"relations": [r.dict() for r in validated.relations],
|
||||
"extraction_time_sec": round(extraction_time, 2),
|
||||
"entity_count": len(validated.entities),
|
||||
"relation_count": len(validated.relations),
|
||||
"warnings": validated.warnings,
|
||||
"validation_passed": validated.validation_passed,
|
||||
"validation_errors": validated.validation_errors,
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=500, detail=f"Extraction failed: {str(e)}")
|
||||
|
||||
|
||||
@search_router.post("/vector")
|
||||
async def vector_search(
|
||||
query: str = Query(..., description="Search query"),
|
||||
limit: int = Query(10, ge=1, le=100),
|
||||
threshold: float = Query(0.5, ge=0.0, le=1.0),
|
||||
):
|
||||
"""
|
||||
Vector search in Neo4j (Phase 4).
|
||||
|
||||
Returns top-k similar entities using vector embeddings.
|
||||
"""
|
||||
try:
|
||||
adapter = await get_neo4j_adapter()
|
||||
results = await adapter.vector_search(
|
||||
query_text=query,
|
||||
limit=limit,
|
||||
threshold=threshold,
|
||||
)
|
||||
|
||||
return {
|
||||
"query": query,
|
||||
"results": results,
|
||||
"result_count": len(results),
|
||||
"limit": limit,
|
||||
"threshold": threshold,
|
||||
}
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=500, detail=f"Search failed: {str(e)}")
|
||||
|
||||
|
||||
@search_router.get("/stats")
|
||||
async def graph_stats():
|
||||
"""
|
||||
Get Neo4j graph statistics (Phase 4).
|
||||
|
||||
Returns node and edge counts.
|
||||
"""
|
||||
try:
|
||||
adapter = await get_neo4j_adapter()
|
||||
stats = await adapter.get_stats()
|
||||
return {
|
||||
"status": "connected" if stats else "disconnected",
|
||||
"stats": stats,
|
||||
}
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=500, detail=f"Stats retrieval failed: {str(e)}")
|
||||
|
||||
|
||||
@search_router.get("/entity/{entity_id}")
|
||||
async def get_entity_neighbors(
|
||||
entity_id: str,
|
||||
depth: int = Query(1, ge=1, le=2),
|
||||
):
|
||||
"""
|
||||
Get entity and its neighbors in the graph (Phase 4).
|
||||
"""
|
||||
try:
|
||||
adapter = await get_neo4j_adapter()
|
||||
result = await adapter.get_entity_neighbors(entity_id, depth=depth)
|
||||
|
||||
if not result:
|
||||
raise HTTPException(status_code=404, detail=f"Entity {entity_id} not found")
|
||||
|
||||
return result
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=500, detail=f"Query failed: {str(e)}")
|
||||
|
||||
|
||||
@search_router.post("/ingest")
|
||||
async def ingest_extraction_result(
|
||||
extraction_result: dict = None,
|
||||
):
|
||||
"""
|
||||
Ingest extraction results into Neo4j graph (Phase 4).
|
||||
|
||||
Takes validated entities and relations from extraction output,
|
||||
creates nodes and edges in Neo4j with vector embeddings.
|
||||
|
||||
Expected input:
|
||||
{
|
||||
"entities": [
|
||||
{"id": "E_1", "label": "...", "type": "...", "confidence": 0.9}
|
||||
],
|
||||
"relations": [
|
||||
{"source_id": "E_1", "target_id": "E_2", "predicate": "...", "confidence": 0.8}
|
||||
]
|
||||
}
|
||||
"""
|
||||
try:
|
||||
if not extraction_result or ("entities" not in extraction_result and "relations" not in extraction_result):
|
||||
raise HTTPException(status_code=400, detail="Missing entities or relations in input")
|
||||
|
||||
adapter = await get_neo4j_adapter()
|
||||
entities_ingested = 0
|
||||
relations_ingested = 0
|
||||
|
||||
# Ingest entities if present
|
||||
if extraction_result.get("entities"):
|
||||
entities_ingested = await adapter.create_entity_nodes(extraction_result["entities"])
|
||||
|
||||
# Ingest relations if present
|
||||
if extraction_result.get("relations"):
|
||||
relations_ingested = await adapter.create_relation_edges(extraction_result["relations"])
|
||||
|
||||
return {
|
||||
"status": "success",
|
||||
"entities_ingested": entities_ingested,
|
||||
"relations_ingested": relations_ingested,
|
||||
"total_ingested": entities_ingested + relations_ingested,
|
||||
}
|
||||
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=500, detail=f"Ingestion failed: {str(e)}")
|
||||
|
||||
|
||||
# Register routers
|
||||
app.include_router(extraction_router)
|
||||
app.include_router(search_router)
|
||||
Reference in New Issue
Block a user