253 lines
8.0 KiB
Python
253 lines
8.0 KiB
Python
|
"""Phase 0-4 FastAPI application.
|
||
|
|
|
||
|
|
Phase 0: Basic URL extraction
|
||
|
|
Phase 2: Crawl4AI profile support for dynamic pages
|
||
|
|
Phase 3: Validation (lightweight + OntoCast)
|
||
|
|
Phase 4: Neo4j vector search
|
||
|
|
"""
|
||
|
|
|
||
|
|
from fastapi import FastAPI, APIRouter, HTTPException, Query
|
||
|
|
from typing import Optional, Literal, List
|
||
|
|
import time
|
||
|
|
import asyncio
|
||
|
|
import logging
|
||
|
|
|
||
|
|
from ont_platform.core.extractors.web_extractor import extract_web_content
|
||
|
|
from ont_platform.core.extraction.lightweight_extractor import LightweightExtractor
|
||
|
|
from ont_platform.core.crawler.crawl4ai_adapter import (
|
||
|
|
Crawl4AIAdapter,
|
||
|
|
CrawlProfile,
|
||
|
|
)
|
||
|
|
from ont_platform.core.validation import OntologyGuard
|
||
|
|
from ont_platform.core.graph.neo4j_adapter import Neo4jAdapter, Neo4jConfig
|
||
|
|
|
||
|
|
logger = logging.getLogger(__name__)
|
||
|
|
|
||
|
|
app = FastAPI(
|
||
|
|
title="Ontology Platform - Phase 0-4",
|
||
|
|
description="Extraction + Validation + Graph Search. 10-30 seconds per URL.",
|
||
|
|
version="0.4.0",
|
||
|
|
)
|
||
|
|
|
||
|
|
extraction_router = APIRouter(prefix="/api/v1/extract", tags=["extraction"])
|
||
|
|
search_router = APIRouter(prefix="/api/v1/search", tags=["search"])
|
||
|
|
|
||
|
|
# Phase 3: Initialize validation guard
|
||
|
|
guard = OntologyGuard(validator_type="lightweight", strict=False)
|
||
|
|
|
||
|
|
# Phase 4: Neo4j adapter (lazy initialization)
|
||
|
|
_neo4j_adapter: Optional[Neo4jAdapter] = None
|
||
|
|
|
||
|
|
async def get_neo4j_adapter() -> Neo4jAdapter:
|
||
|
|
"""Get or create Neo4j adapter instance."""
|
||
|
|
global _neo4j_adapter
|
||
|
|
if _neo4j_adapter is None:
|
||
|
|
_neo4j_adapter = Neo4jAdapter()
|
||
|
|
if not await _neo4j_adapter.connect():
|
||
|
|
logger.warning("Neo4j not available, search will be unavailable")
|
||
|
|
else:
|
||
|
|
try:
|
||
|
|
await _neo4j_adapter.initialize_embedder()
|
||
|
|
except Exception as e:
|
||
|
|
logger.warning(f"Failed to initialize embedder: {e}")
|
||
|
|
return _neo4j_adapter
|
||
|
|
|
||
|
|
|
||
|
|
@extraction_router.post("/url")
|
||
|
|
async def extract_url(
|
||
|
|
url: str,
|
||
|
|
profile: Optional[Literal["fast_static", "dynamic_page"]] = Query(None),
|
||
|
|
):
|
||
|
|
"""
|
||
|
|
Extract candidates from URL (Phase 0-2).
|
||
|
|
|
||
|
|
Phase 0-1: Default fast_static (HTTP only)
|
||
|
|
Phase 2: Supports dynamic_page for JS-rendered content
|
||
|
|
"""
|
||
|
|
if not url:
|
||
|
|
raise HTTPException(status_code=400, detail="url is required")
|
||
|
|
|
||
|
|
start_time = time.time()
|
||
|
|
|
||
|
|
try:
|
||
|
|
# Phase 2: Use Crawl4AI for dynamic pages
|
||
|
|
if profile == "dynamic_page":
|
||
|
|
adapter = Crawl4AIAdapter()
|
||
|
|
try:
|
||
|
|
crawl_result = await adapter.crawl(url, profile=CrawlProfile.DYNAMIC_PAGE)
|
||
|
|
profile_used = crawl_result.profile_used
|
||
|
|
html_content = crawl_result.html
|
||
|
|
finally:
|
||
|
|
await adapter.close()
|
||
|
|
|
||
|
|
# Extract from crawled HTML
|
||
|
|
extracted = extract_web_content(html=html_content, url=url)
|
||
|
|
else:
|
||
|
|
# Phase 0-1: Default fast_static (HTTP only)
|
||
|
|
extracted = extract_web_content(url=url)
|
||
|
|
profile_used = "trafilatura"
|
||
|
|
|
||
|
|
# Step 2: Extract JSON candidates with lightweight extractor
|
||
|
|
lightweight = LightweightExtractor(use_llm=False)
|
||
|
|
candidates = lightweight.extract(
|
||
|
|
text=extracted.text,
|
||
|
|
project_id="default",
|
||
|
|
document_id="temp",
|
||
|
|
)
|
||
|
|
|
||
|
|
# Phase 3: Validate extraction results
|
||
|
|
raw_result = {
|
||
|
|
"entities": candidates.entities,
|
||
|
|
"relations": candidates.relations,
|
||
|
|
"warnings": candidates.warnings,
|
||
|
|
}
|
||
|
|
validated = await guard.validate(raw_result)
|
||
|
|
|
||
|
|
extraction_time = time.time() - start_time
|
||
|
|
|
||
|
|
# Return JSON with validation info
|
||
|
|
return {
|
||
|
|
"url": url,
|
||
|
|
"title": extracted.title,
|
||
|
|
"author": extracted.author,
|
||
|
|
"published_date": extracted.publish_date,
|
||
|
|
"language": extracted.language,
|
||
|
|
"text_length": len(extracted.text),
|
||
|
|
"profile_used": profile_used,
|
||
|
|
"entities": [e.dict() for e in validated.entities],
|
||
|
|
"relations": [r.dict() for r in validated.relations],
|
||
|
|
"extraction_time_sec": round(extraction_time, 2),
|
||
|
|
"entity_count": len(validated.entities),
|
||
|
|
"relation_count": len(validated.relations),
|
||
|
|
"warnings": validated.warnings,
|
||
|
|
"validation_passed": validated.validation_passed,
|
||
|
|
"validation_errors": validated.validation_errors,
|
||
|
|
}
|
||
|
|
|
||
|
|
except Exception as e:
|
||
|
|
raise HTTPException(status_code=500, detail=f"Extraction failed: {str(e)}")
|
||
|
|
|
||
|
|
|
||
|
|
@search_router.post("/vector")
|
||
|
|
async def vector_search(
|
||
|
|
query: str = Query(..., description="Search query"),
|
||
|
|
limit: int = Query(10, ge=1, le=100),
|
||
|
|
threshold: float = Query(0.5, ge=0.0, le=1.0),
|
||
|
|
):
|
||
|
|
"""
|
||
|
|
Vector search in Neo4j (Phase 4).
|
||
|
|
|
||
|
|
Returns top-k similar entities using vector embeddings.
|
||
|
|
"""
|
||
|
|
try:
|
||
|
|
adapter = await get_neo4j_adapter()
|
||
|
|
results = await adapter.vector_search(
|
||
|
|
query_text=query,
|
||
|
|
limit=limit,
|
||
|
|
threshold=threshold,
|
||
|
|
)
|
||
|
|
|
||
|
|
return {
|
||
|
|
"query": query,
|
||
|
|
"results": results,
|
||
|
|
"result_count": len(results),
|
||
|
|
"limit": limit,
|
||
|
|
"threshold": threshold,
|
||
|
|
}
|
||
|
|
except Exception as e:
|
||
|
|
raise HTTPException(status_code=500, detail=f"Search failed: {str(e)}")
|
||
|
|
|
||
|
|
|
||
|
|
@search_router.get("/stats")
|
||
|
|
async def graph_stats():
|
||
|
|
"""
|
||
|
|
Get Neo4j graph statistics (Phase 4).
|
||
|
|
|
||
|
|
Returns node and edge counts.
|
||
|
|
"""
|
||
|
|
try:
|
||
|
|
adapter = await get_neo4j_adapter()
|
||
|
|
stats = await adapter.get_stats()
|
||
|
|
return {
|
||
|
|
"status": "connected" if stats else "disconnected",
|
||
|
|
"stats": stats,
|
||
|
|
}
|
||
|
|
except Exception as e:
|
||
|
|
raise HTTPException(status_code=500, detail=f"Stats retrieval failed: {str(e)}")
|
||
|
|
|
||
|
|
|
||
|
|
@search_router.get("/entity/{entity_id}")
|
||
|
|
async def get_entity_neighbors(
|
||
|
|
entity_id: str,
|
||
|
|
depth: int = Query(1, ge=1, le=2),
|
||
|
|
):
|
||
|
|
"""
|
||
|
|
Get entity and its neighbors in the graph (Phase 4).
|
||
|
|
"""
|
||
|
|
try:
|
||
|
|
adapter = await get_neo4j_adapter()
|
||
|
|
result = await adapter.get_entity_neighbors(entity_id, depth=depth)
|
||
|
|
|
||
|
|
if not result:
|
||
|
|
raise HTTPException(status_code=404, detail=f"Entity {entity_id} not found")
|
||
|
|
|
||
|
|
return result
|
||
|
|
except HTTPException:
|
||
|
|
raise
|
||
|
|
except Exception as e:
|
||
|
|
raise HTTPException(status_code=500, detail=f"Query failed: {str(e)}")
|
||
|
|
|
||
|
|
|
||
|
|
@search_router.post("/ingest")
|
||
|
|
async def ingest_extraction_result(
|
||
|
|
extraction_result: dict = None,
|
||
|
|
):
|
||
|
|
"""
|
||
|
|
Ingest extraction results into Neo4j graph (Phase 4).
|
||
|
|
|
||
|
|
Takes validated entities and relations from extraction output,
|
||
|
|
creates nodes and edges in Neo4j with vector embeddings.
|
||
|
|
|
||
|
|
Expected input:
|
||
|
|
{
|
||
|
|
"entities": [
|
||
|
|
{"id": "E_1", "label": "...", "type": "...", "confidence": 0.9}
|
||
|
|
],
|
||
|
|
"relations": [
|
||
|
|
{"source_id": "E_1", "target_id": "E_2", "predicate": "...", "confidence": 0.8}
|
||
|
|
]
|
||
|
|
}
|
||
|
|
"""
|
||
|
|
try:
|
||
|
|
if not extraction_result or ("entities" not in extraction_result and "relations" not in extraction_result):
|
||
|
|
raise HTTPException(status_code=400, detail="Missing entities or relations in input")
|
||
|
|
|
||
|
|
adapter = await get_neo4j_adapter()
|
||
|
|
entities_ingested = 0
|
||
|
|
relations_ingested = 0
|
||
|
|
|
||
|
|
# Ingest entities if present
|
||
|
|
if extraction_result.get("entities"):
|
||
|
|
entities_ingested = await adapter.create_entity_nodes(extraction_result["entities"])
|
||
|
|
|
||
|
|
# Ingest relations if present
|
||
|
|
if extraction_result.get("relations"):
|
||
|
|
relations_ingested = await adapter.create_relation_edges(extraction_result["relations"])
|
||
|
|
|
||
|
|
return {
|
||
|
|
"status": "success",
|
||
|
|
"entities_ingested": entities_ingested,
|
||
|
|
"relations_ingested": relations_ingested,
|
||
|
|
"total_ingested": entities_ingested + relations_ingested,
|
||
|
|
}
|
||
|
|
|
||
|
|
except HTTPException:
|
||
|
|
raise
|
||
|
|
except Exception as e:
|
||
|
|
raise HTTPException(status_code=500, detail=f"Ingestion failed: {str(e)}")
|
||
|
|
|
||
|
|
|
||
|
|
# Register routers
|
||
|
|
app.include_router(extraction_router)
|
||
|
|
app.include_router(search_router)
|