docs

2026-05-19 20:31:52 +09:00
parent 00407e7a08
commit e260e5f218
104 changed files with 12898 additions and 1709 deletions
--- a/ontology_platform/tests/unit/test_web_extractor.py
+++ b/ontology_platform/tests/unit/test_web_extractor.py
@@ -0,0 +1,64 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+from ont_platform.core.extraction.schemas import SourceDocumentSchema
+from ont_platform.core.extractors.web_extractor import WebExtractor, extract_web_content
+
+FIXTURES = Path(__file__).resolve().parents[1] / "fixtures" / "korean"
+
+
+def test_extract_from_korean_html_preserves_document_contract() -> None:
+    html = (FIXTURES / "news_yonhap.html").read_text(encoding="utf-8")
+
+    extracted = WebExtractor().extract_from_html(
+        html,
+        source_url="https://example.test/news/data-quality?utm=tracking",
+    )
+
+    assert "공공 데이터 품질 관리 체계" in extracted.text
+    assert extracted.title == "정부, 공공 데이터 품질 관리 체계 확대"
+    assert extracted.language == "ko"
+    assert extracted.canonical_url == "https://example.test/news/data-quality"
+    assert extracted.content_hash
+    assert extracted.fingerprint.startswith("sha1:")
+    assert extracted.body_xml
+    assert extracted.metadata["source"] == "trafilatura"
+
+
+def test_same_clean_body_gets_same_hash_and_fingerprint() -> None:
+    body = """
+    <article>
+      <h1>중복 문서</h1>
+      <p>Alice works at Acme in Berlin. The ontology platform keeps evidence spans.</p>
+      <p>Alice works at Acme in Berlin. The ontology platform keeps evidence spans.</p>
+      <p>Alice works at Acme in Berlin. The ontology platform keeps evidence spans.</p>
+    </article>
+    """
+    first_html = f"<html><head><title>중복 문서</title></head><body>{body}</body></html>"
+    second_html = f"<html><head><title>중복 문서</title></head><body><nav>menu</nav>{body}</body></html>"
+
+    first = extract_web_content(html=first_html, url="https://example.test/a")
+    second = extract_web_content(html=second_html, url="https://example.test/b")
+
+    assert first.content_hash == second.content_hash
+    assert first.fingerprint == second.fingerprint
+    assert first.document_id == second.document_id
+
+
+def test_extracted_content_maps_to_source_document_and_evidence_spans() -> None:
+    html = (FIXTURES / "blog_naver.html").read_text(encoding="utf-8")
+    extracted = extract_web_content(html=html, url="https://example.test/blog/ontology-build-log")
+
+    source_document = extracted.to_source_document(project_id="proj_1")
+    spans = extracted.evidence_spans(project_id="proj_1", document_id=source_document.id)
+
+    assert source_document.project_id == "proj_1"
+    assert source_document.source_url == "https://example.test/blog/ontology-build-log"
+    assert source_document.content_hash == extracted.content_hash
+    assert source_document.metadata_["source"] == "trafilatura"
+    schema = SourceDocumentSchema.model_validate(source_document)
+    assert schema.metadata["source"] == "trafilatura"
+    assert spans
+    assert spans[0].start_offset >= 0
+    assert spans[0].end_offset <= len(extracted.text)