88 lines
2.8 KiB
Python
88 lines
2.8 KiB
Python
"""Phase 1 URL/HTML ingestion API tests."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import importlib
|
|
from contextlib import asynccontextmanager
|
|
from pathlib import Path
|
|
|
|
from fastapi.testclient import TestClient
|
|
from ont_platform.storage.dedup_cache import reset_default_dedup_cache
|
|
|
|
main_module = importlib.import_module("ont_platform.api.main")
|
|
|
|
|
|
@asynccontextmanager
|
|
async def _noop_lifespan(app):
|
|
yield
|
|
|
|
|
|
def _phase1_client(monkeypatch) -> TestClient:
|
|
monkeypatch.setenv("PHASE", "1")
|
|
reset_default_dedup_cache()
|
|
app = main_module.create_app()
|
|
app.router.lifespan_context = _noop_lifespan
|
|
return TestClient(app, raise_server_exceptions=True, backend="asyncio")
|
|
|
|
|
|
def test_extract_url_accepts_html_payload_and_returns_source_document(
|
|
monkeypatch,
|
|
fixtures_dir: Path,
|
|
) -> None:
|
|
html = (fixtures_dir / "korean" / "news_yonhap.html").read_text(encoding="utf-8")
|
|
|
|
with _phase1_client(monkeypatch) as client:
|
|
response = client.post(
|
|
"/api/v1/extract/url",
|
|
json={
|
|
"url": "https://example.test/news/data-quality",
|
|
"html": html,
|
|
"project_id": "proj_phase1",
|
|
},
|
|
)
|
|
|
|
assert response.status_code == 200, response.text
|
|
body = response.json()
|
|
assert body["status"] == "success"
|
|
assert body["source_document"]["source_url"] == "https://example.test/news/data-quality"
|
|
assert body["source_document"]["language"] == "ko"
|
|
assert body["source_document"]["content_hash"]
|
|
assert body["evidence_spans"]
|
|
assert body["content_unit"]["doc_iri"].startswith("urn:source:doc_")
|
|
assert body["dedup"]["is_duplicate"] is False
|
|
|
|
|
|
def test_process_url_skips_duplicate_payload_by_fingerprint(
|
|
monkeypatch,
|
|
fixtures_dir: Path,
|
|
) -> None:
|
|
html = (fixtures_dir / "korean" / "blog_naver.html").read_text(encoding="utf-8")
|
|
|
|
with _phase1_client(monkeypatch) as client:
|
|
first = client.post(
|
|
"/process/url",
|
|
json={
|
|
"url": "https://example.test/blog/original",
|
|
"html": html,
|
|
"project_id": "proj_phase1",
|
|
},
|
|
)
|
|
second = client.post(
|
|
"/process/url",
|
|
json={
|
|
"url": "https://example.test/blog/mirror",
|
|
"html": html,
|
|
"project_id": "proj_phase1",
|
|
},
|
|
)
|
|
|
|
assert first.status_code == 200, first.text
|
|
assert second.status_code == 200, second.text
|
|
first_body = first.json()
|
|
second_body = second.json()
|
|
assert first_body["dedup"]["is_duplicate"] is False
|
|
assert second_body["dedup"]["is_duplicate"] is True
|
|
assert second_body["dedup"]["existing_document_id"] == first_body["source_document"]["id"]
|
|
assert second_body["entity_count"] == 0
|
|
assert "Duplicate source document skipped" in second_body["warnings"][0]
|