98 lines
3.1 KiB
Python
98 lines
3.1 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
Phase 0 extraction test: Extract candidates from a URL.
|
|
|
|
Usage:
|
|
python test_phase0_extraction.py <URL>
|
|
|
|
Example:
|
|
python test_phase0_extraction.py https://example.com
|
|
"""
|
|
|
|
import sys
|
|
import time
|
|
from pathlib import Path
|
|
|
|
# Add ont_platform to path
|
|
sys.path.insert(0, str(Path(__file__).parent / "ontology_platform"))
|
|
|
|
from ont_platform.core.extractors.web_extractor import extract_web_content
|
|
from ont_platform.core.extraction.lightweight_extractor import LightweightExtractor
|
|
|
|
|
|
def test_extraction(url: str):
|
|
"""Test Phase 0 extraction on a URL."""
|
|
print(f"\n🔗 Extracting from: {url}\n")
|
|
|
|
start_time = time.time()
|
|
|
|
try:
|
|
# Step 1: Extract web content
|
|
print("📄 Step 1: Extracting web content with Trafilatura...")
|
|
extracted = extract_web_content(url=url)
|
|
|
|
print(f" ✓ Title: {extracted.title}")
|
|
print(f" ✓ Length: {len(extracted.text)} chars")
|
|
print(f" ✓ Language: {extracted.language}")
|
|
|
|
# Step 2: Extract JSON candidates
|
|
print("\n🎯 Step 2: Extracting JSON candidates...")
|
|
lightweight = LightweightExtractor(use_llm=False)
|
|
candidates = lightweight.extract(
|
|
text=extracted.text,
|
|
project_id="test",
|
|
document_id="test_doc",
|
|
)
|
|
|
|
print(f" ✓ Entities: {len(candidates.entities)}")
|
|
print(f" ✓ Relations: {len(candidates.relations)}")
|
|
if candidates.warnings:
|
|
print(f" ⚠️ Warnings: {len(candidates.warnings)}")
|
|
for w in candidates.warnings[:3]:
|
|
print(f" - {w}")
|
|
|
|
elapsed = time.time() - start_time
|
|
|
|
# Display results
|
|
print(f"\n📊 Results:")
|
|
print(f" Total time: {elapsed:.2f} seconds")
|
|
print(f"\n Entities ({len(candidates.entities)}):")
|
|
for e in candidates.entities[:5]:
|
|
print(f" - {e.label} ({e.entity_type}) [confidence: {e.confidence:.2f}]")
|
|
if len(candidates.entities) > 5:
|
|
print(f" ... and {len(candidates.entities) - 5} more")
|
|
|
|
print(f"\n Relations ({len(candidates.relations)}):")
|
|
for r in candidates.relations[:3]:
|
|
print(
|
|
f" - {r.source_entity_id} --{r.predicate}--> {r.target_entity_id}"
|
|
)
|
|
if len(candidates.relations) > 3:
|
|
print(f" ... and {len(candidates.relations) - 3} more")
|
|
|
|
# Check if within Phase 0 goal
|
|
if elapsed <= 30:
|
|
print(f"\n✅ Phase 0 Goal Achieved: {elapsed:.2f}s <= 30s")
|
|
else:
|
|
print(f"\n⚠️ Phase 0 Goal Not Met: {elapsed:.2f}s > 30s")
|
|
|
|
return True
|
|
|
|
except Exception as e:
|
|
print(f"\n❌ Error: {e}")
|
|
import traceback
|
|
|
|
traceback.print_exc()
|
|
return False
|
|
|
|
|
|
if __name__ == "__main__":
|
|
if len(sys.argv) < 2:
|
|
print("Usage: python test_phase0_extraction.py <URL>")
|
|
print("Example: python test_phase0_extraction.py https://www.wikipedia.org/wiki/Python_(programming_language)")
|
|
sys.exit(1)
|
|
|
|
url = sys.argv[1]
|
|
success = test_extraction(url)
|
|
sys.exit(0 if success else 1)
|