from __future__ import annotations from pathlib import Path from ont_platform.core.extraction.schemas import SourceDocumentSchema from ont_platform.core.extractors.web_extractor import WebExtractor, extract_web_content FIXTURES = Path(__file__).resolve().parents[1] / "fixtures" / "korean" def test_extract_from_korean_html_preserves_document_contract() -> None: html = (FIXTURES / "news_yonhap.html").read_text(encoding="utf-8") extracted = WebExtractor().extract_from_html( html, source_url="https://example.test/news/data-quality?utm=tracking", ) assert "공공 데이터 품질 관리 체계" in extracted.text assert extracted.title == "정부, 공공 데이터 품질 관리 체계 확대" assert extracted.language == "ko" assert extracted.canonical_url == "https://example.test/news/data-quality" assert extracted.content_hash assert extracted.fingerprint.startswith("sha1:") assert extracted.body_xml assert extracted.metadata["source"] == "trafilatura" def test_same_clean_body_gets_same_hash_and_fingerprint() -> None: body = """

중복 문서

Alice works at Acme in Berlin. The ontology platform keeps evidence spans.

Alice works at Acme in Berlin. The ontology platform keeps evidence spans.

Alice works at Acme in Berlin. The ontology platform keeps evidence spans.

""" first_html = f"중복 문서{body}" second_html = f"중복 문서{body}" first = extract_web_content(html=first_html, url="https://example.test/a") second = extract_web_content(html=second_html, url="https://example.test/b") assert first.content_hash == second.content_hash assert first.fingerprint == second.fingerprint assert first.document_id == second.document_id def test_extracted_content_maps_to_source_document_and_evidence_spans() -> None: html = (FIXTURES / "blog_naver.html").read_text(encoding="utf-8") extracted = extract_web_content(html=html, url="https://example.test/blog/ontology-build-log") source_document = extracted.to_source_document(project_id="proj_1") spans = extracted.evidence_spans(project_id="proj_1", document_id=source_document.id) assert source_document.project_id == "proj_1" assert source_document.source_url == "https://example.test/blog/ontology-build-log" assert source_document.content_hash == extracted.content_hash assert source_document.metadata_["source"] == "trafilatura" schema = SourceDocumentSchema.model_validate(source_document) assert schema.metadata["source"] == "trafilatura" assert spans assert spans[0].start_offset >= 0 assert spans[0].end_offset <= len(extracted.text)