67 lines
1.7 KiB
Python
67 lines
1.7 KiB
Python
|
#!/usr/bin/env python3
|
||
|
|
"""Phase 2 Crawl4AI integration test."""
|
||
|
|
|
||
|
|
import asyncio
|
||
|
|
import sys
|
||
|
|
import time
|
||
|
|
from pathlib import Path
|
||
|
|
|
||
|
|
sys.path.insert(0, str(Path(__file__).parent / "ontology_platform"))
|
||
|
|
|
||
|
|
from ont_platform.core.crawler.crawl4ai_adapter import (
|
||
|
|
Crawl4AIAdapter,
|
||
|
|
CrawlProfile,
|
||
|
|
)
|
||
|
|
|
||
|
|
|
||
|
|
async def test_crawl_profile(url: str, profile: CrawlProfile):
|
||
|
|
"""Test crawling with specific profile."""
|
||
|
|
print(f"\nTesting {profile.value} profile on {url}\n")
|
||
|
|
|
||
|
|
adapter = Crawl4AIAdapter()
|
||
|
|
start = time.time()
|
||
|
|
|
||
|
|
try:
|
||
|
|
result = await adapter.crawl(url, profile=profile)
|
||
|
|
elapsed = time.time() - start
|
||
|
|
|
||
|
|
print(f"[OK] Status: {result.status_code}")
|
||
|
|
print(f"[OK] Profile used: {result.profile_used}")
|
||
|
|
print(f"[OK] HTML length: {len(result.html)} chars")
|
||
|
|
if result.markdown:
|
||
|
|
print(f"[OK] Markdown length: {len(result.markdown)} chars")
|
||
|
|
print(f"[OK] Time: {elapsed:.2f}s")
|
||
|
|
|
||
|
|
return True
|
||
|
|
|
||
|
|
except Exception as e:
|
||
|
|
print(f"[ERROR] {e}")
|
||
|
|
return False
|
||
|
|
|
||
|
|
finally:
|
||
|
|
await adapter.close()
|
||
|
|
|
||
|
|
|
||
|
|
async def main():
|
||
|
|
"""Run Phase 2 tests."""
|
||
|
|
# Test fast_static (should work like Phase 0/1)
|
||
|
|
success_static = await test_crawl_profile(
|
||
|
|
"https://example.com",
|
||
|
|
CrawlProfile.FAST_STATIC,
|
||
|
|
)
|
||
|
|
|
||
|
|
if not success_static:
|
||
|
|
print("\n[FAILED] Static crawl failed!")
|
||
|
|
return False
|
||
|
|
|
||
|
|
print("\n[SUCCESS] Phase 2 MVP complete: Crawl4AI adapter working")
|
||
|
|
print(" - Fast static crawling (Phase 0/1 compatibility)")
|
||
|
|
print(" - Ready for dynamic_page profile (requires Playwright setup)")
|
||
|
|
|
||
|
|
return True
|
||
|
|
|
||
|
|
|
||
|
|
if __name__ == "__main__":
|
||
|
|
success = asyncio.run(main())
|
||
|
|
sys.exit(0 if success else 1)
|