"""Phase 0-4 FastAPI application. Phase 0: Basic URL extraction Phase 2: Crawl4AI profile support for dynamic pages Phase 3: Validation (lightweight + OntoCast) Phase 4: Neo4j vector search """ from fastapi import FastAPI, APIRouter, HTTPException, Query from typing import Optional, Literal, List import time import asyncio import logging from ont_platform.core.extractors.web_extractor import extract_web_content from ont_platform.core.extraction.lightweight_extractor import LightweightExtractor from ont_platform.core.crawler.crawl4ai_adapter import ( Crawl4AIAdapter, CrawlProfile, ) from ont_platform.core.validation import OntologyGuard from ont_platform.core.graph.neo4j_adapter import Neo4jAdapter, Neo4jConfig logger = logging.getLogger(__name__) app = FastAPI( title="Ontology Platform - Phase 0-4", description="Extraction + Validation + Graph Search. 10-30 seconds per URL.", version="0.4.0", ) extraction_router = APIRouter(prefix="/api/v1/extract", tags=["extraction"]) search_router = APIRouter(prefix="/api/v1/search", tags=["search"]) # Phase 3: Initialize validation guard guard = OntologyGuard(validator_type="lightweight", strict=False) # Phase 4: Neo4j adapter (lazy initialization) _neo4j_adapter: Optional[Neo4jAdapter] = None async def get_neo4j_adapter() -> Neo4jAdapter: """Get or create Neo4j adapter instance.""" global _neo4j_adapter if _neo4j_adapter is None: _neo4j_adapter = Neo4jAdapter() if not await _neo4j_adapter.connect(): logger.warning("Neo4j not available, search will be unavailable") else: try: await _neo4j_adapter.initialize_embedder() except Exception as e: logger.warning(f"Failed to initialize embedder: {e}") return _neo4j_adapter @extraction_router.post("/url") async def extract_url( url: str, profile: Optional[Literal["fast_static", "dynamic_page"]] = Query(None), ): """ Extract candidates from URL (Phase 0-2). Phase 0-1: Default fast_static (HTTP only) Phase 2: Supports dynamic_page for JS-rendered content """ if not url: raise HTTPException(status_code=400, detail="url is required") start_time = time.time() try: # Phase 2: Use Crawl4AI for dynamic pages if profile == "dynamic_page": adapter = Crawl4AIAdapter() try: crawl_result = await adapter.crawl(url, profile=CrawlProfile.DYNAMIC_PAGE) profile_used = crawl_result.profile_used html_content = crawl_result.html finally: await adapter.close() # Extract from crawled HTML extracted = extract_web_content(html=html_content, url=url) else: # Phase 0-1: Default fast_static (HTTP only) extracted = extract_web_content(url=url) profile_used = "trafilatura" # Step 2: Extract JSON candidates with lightweight extractor lightweight = LightweightExtractor(use_llm=False) candidates = lightweight.extract( text=extracted.text, project_id="default", document_id="temp", ) # Phase 3: Validate extraction results raw_result = { "entities": candidates.entities, "relations": candidates.relations, "warnings": candidates.warnings, } validated = await guard.validate(raw_result) extraction_time = time.time() - start_time # Return JSON with validation info return { "url": url, "title": extracted.title, "author": extracted.author, "published_date": extracted.publish_date, "language": extracted.language, "text_length": len(extracted.text), "profile_used": profile_used, "entities": [e.dict() for e in validated.entities], "relations": [r.dict() for r in validated.relations], "extraction_time_sec": round(extraction_time, 2), "entity_count": len(validated.entities), "relation_count": len(validated.relations), "warnings": validated.warnings, "validation_passed": validated.validation_passed, "validation_errors": validated.validation_errors, } except Exception as e: raise HTTPException(status_code=500, detail=f"Extraction failed: {str(e)}") @search_router.post("/vector") async def vector_search( query: str = Query(..., description="Search query"), limit: int = Query(10, ge=1, le=100), threshold: float = Query(0.5, ge=0.0, le=1.0), ): """ Vector search in Neo4j (Phase 4). Returns top-k similar entities using vector embeddings. """ try: adapter = await get_neo4j_adapter() results = await adapter.vector_search( query_text=query, limit=limit, threshold=threshold, ) return { "query": query, "results": results, "result_count": len(results), "limit": limit, "threshold": threshold, } except Exception as e: raise HTTPException(status_code=500, detail=f"Search failed: {str(e)}") @search_router.get("/stats") async def graph_stats(): """ Get Neo4j graph statistics (Phase 4). Returns node and edge counts. """ try: adapter = await get_neo4j_adapter() stats = await adapter.get_stats() return { "status": "connected" if stats else "disconnected", "stats": stats, } except Exception as e: raise HTTPException(status_code=500, detail=f"Stats retrieval failed: {str(e)}") @search_router.get("/entity/{entity_id}") async def get_entity_neighbors( entity_id: str, depth: int = Query(1, ge=1, le=2), ): """ Get entity and its neighbors in the graph (Phase 4). """ try: adapter = await get_neo4j_adapter() result = await adapter.get_entity_neighbors(entity_id, depth=depth) if not result: raise HTTPException(status_code=404, detail=f"Entity {entity_id} not found") return result except HTTPException: raise except Exception as e: raise HTTPException(status_code=500, detail=f"Query failed: {str(e)}") @search_router.post("/ingest") async def ingest_extraction_result( extraction_result: dict = None, ): """ Ingest extraction results into Neo4j graph (Phase 4). Takes validated entities and relations from extraction output, creates nodes and edges in Neo4j with vector embeddings. Expected input: { "entities": [ {"id": "E_1", "label": "...", "type": "...", "confidence": 0.9} ], "relations": [ {"source_id": "E_1", "target_id": "E_2", "predicate": "...", "confidence": 0.8} ] } """ try: if not extraction_result or ("entities" not in extraction_result and "relations" not in extraction_result): raise HTTPException(status_code=400, detail="Missing entities or relations in input") adapter = await get_neo4j_adapter() entities_ingested = 0 relations_ingested = 0 # Ingest entities if present if extraction_result.get("entities"): entities_ingested = await adapter.create_entity_nodes(extraction_result["entities"]) # Ingest relations if present if extraction_result.get("relations"): relations_ingested = await adapter.create_relation_edges(extraction_result["relations"]) return { "status": "success", "entities_ingested": entities_ingested, "relations_ingested": relations_ingested, "total_ingested": entities_ingested + relations_ingested, } except HTTPException: raise except Exception as e: raise HTTPException(status_code=500, detail=f"Ingestion failed: {str(e)}") # Register routers app.include_router(extraction_router) app.include_router(search_router)