351 lines
12 KiB
Python
351 lines
12 KiB
Python
|
import json
|
||
|
|
|
||
|
|
from crawler_platform.app.config.loader import load_project_config
|
||
|
|
from crawler_platform.app.core.database import models
|
||
|
|
from crawler_platform.app.core.database.repository import KnowledgeRepository
|
||
|
|
from crawler_platform.app.core.database.session import make_engine
|
||
|
|
from crawler_platform.app.core.crawler.html_cleaner import clean_html_with_metadata
|
||
|
|
from crawler_platform.app.core.extractor.base import (
|
||
|
|
ExtractedClaim,
|
||
|
|
ExtractedEntity,
|
||
|
|
ExtractionPageContext,
|
||
|
|
ExtractionBundle,
|
||
|
|
)
|
||
|
|
from crawler_platform.app.core.extractor.ai_provider import parse_json_content
|
||
|
|
from crawler_platform.app.core.extractor.validation import attach_page_context, validate_extraction_bundle
|
||
|
|
|
||
|
|
|
||
|
|
def test_cleaner_prefers_product_content_and_removes_boilerplate():
|
||
|
|
html = """
|
||
|
|
<html>
|
||
|
|
<head><title>Sample Product</title></head>
|
||
|
|
<body>
|
||
|
|
<header>CAFE24 Login Cart Low price Product count</header>
|
||
|
|
<nav>Home Board Event Shipping</nav>
|
||
|
|
<main class="product-detail">
|
||
|
|
<h1>Neroli Summer Eau de Parfum</h1>
|
||
|
|
<p>Brand: Example Maison</p>
|
||
|
|
<p>Top notes: Bergamot, Neroli</p>
|
||
|
|
<p>Base notes: Musk</p>
|
||
|
|
<p>Price $89</p>
|
||
|
|
</main>
|
||
|
|
<footer>Powered by CAFE24 Copyright Shipping Country List</footer>
|
||
|
|
</body>
|
||
|
|
</html>
|
||
|
|
"""
|
||
|
|
|
||
|
|
cleaned = clean_html_with_metadata(html)
|
||
|
|
|
||
|
|
assert "Neroli Summer Eau de Parfum" in cleaned.text
|
||
|
|
assert "Bergamot" in cleaned.text
|
||
|
|
assert "CAFE24" not in cleaned.text
|
||
|
|
assert "Low price" not in cleaned.text
|
||
|
|
assert cleaned.page_type == "ProductPage"
|
||
|
|
assert cleaned.extraction_status == "success"
|
||
|
|
assert any(zone["zone_type"] == "product_detail" for zone in cleaned.source_zones)
|
||
|
|
|
||
|
|
|
||
|
|
def test_cleaner_keeps_raw_and_clean_text_separate_with_noise_debug():
|
||
|
|
html = """
|
||
|
|
<html>
|
||
|
|
<head><title>Sample Product</title></head>
|
||
|
|
<body>
|
||
|
|
<header>CAFE24 Login Cart Low price Product count</header>
|
||
|
|
<main class="product-detail">
|
||
|
|
<h1>Neroli Summer Eau de Parfum</h1>
|
||
|
|
<p>Brand: Example Maison</p>
|
||
|
|
<p>Top notes: Bergamot, Neroli</p>
|
||
|
|
<p>Price $89</p>
|
||
|
|
</main>
|
||
|
|
<section class="shipping">Shipping delivery return exchange country list Korea USA Japan Canada</section>
|
||
|
|
</body>
|
||
|
|
</html>
|
||
|
|
"""
|
||
|
|
|
||
|
|
cleaned = clean_html_with_metadata(html, url="https://example.com/product/detail.html")
|
||
|
|
|
||
|
|
assert "CAFE24" in cleaned.raw_text
|
||
|
|
assert "CAFE24" not in cleaned.text
|
||
|
|
assert "Shipping delivery" not in cleaned.text
|
||
|
|
assert cleaned.clean_markdown
|
||
|
|
assert cleaned.metadata["removed_noise_zones_count"] >= 1
|
||
|
|
|
||
|
|
|
||
|
|
def test_cleaner_handles_empty_dom_attributes():
|
||
|
|
html = """
|
||
|
|
<html>
|
||
|
|
<body>
|
||
|
|
<div id class aria-label>
|
||
|
|
<p>Brand: Example Maison</p>
|
||
|
|
<p>Top notes: Bergamot, Neroli</p>
|
||
|
|
</div>
|
||
|
|
</body>
|
||
|
|
</html>
|
||
|
|
"""
|
||
|
|
|
||
|
|
cleaned = clean_html_with_metadata(html)
|
||
|
|
|
||
|
|
assert "Example Maison" in cleaned.text
|
||
|
|
|
||
|
|
|
||
|
|
def test_cleaner_skips_decomposed_nodes_during_boilerplate_scan():
|
||
|
|
from bs4 import BeautifulSoup
|
||
|
|
|
||
|
|
soup = BeautifulSoup("<div id='footer'><span class='child'>x</span></div>", "html.parser")
|
||
|
|
tag = soup.span
|
||
|
|
assert tag is not None
|
||
|
|
tag.decompose()
|
||
|
|
|
||
|
|
from crawler_platform.app.core.crawler.html_cleaner import node_attr
|
||
|
|
|
||
|
|
assert node_attr(tag, "class") == ""
|
||
|
|
|
||
|
|
|
||
|
|
def test_validation_rejects_placeholders_and_marks_rule_candidates():
|
||
|
|
config = load_project_config("configs/perfume_subscription.yaml")
|
||
|
|
bundle = ExtractionBundle(
|
||
|
|
entities=[
|
||
|
|
ExtractedEntity("Brand", "CAFE24"),
|
||
|
|
ExtractedEntity("Perfume", "Neroli Summer"),
|
||
|
|
ExtractedEntity("Accord", "Green"),
|
||
|
|
],
|
||
|
|
claims=[
|
||
|
|
ExtractedClaim("Neroli Summer", "Perfume", "hasBrand", "CAFE24", "Brand", evidence_text="Brand: CAFE24"),
|
||
|
|
ExtractedClaim(
|
||
|
|
"Neroli Summer",
|
||
|
|
"Perfume",
|
||
|
|
"hasTopNote",
|
||
|
|
"Bergamot",
|
||
|
|
"Note",
|
||
|
|
evidence_text="Top notes: Bergamot",
|
||
|
|
confidence=0.82,
|
||
|
|
),
|
||
|
|
ExtractedClaim("Neroli Summer", "Perfume", "hasAccord", "Green", "Accord", evidence_text="Accord: Green"),
|
||
|
|
],
|
||
|
|
extractor_name="perfume_rule_based",
|
||
|
|
provider="rule_based",
|
||
|
|
raw_output={},
|
||
|
|
)
|
||
|
|
|
||
|
|
result = validate_extraction_bundle(bundle, config)
|
||
|
|
|
||
|
|
assert result.claim_status == "rule_candidate"
|
||
|
|
assert [claim.object_name for claim in result.bundle.claims] == ["Bergamot"]
|
||
|
|
assert all(claim.metadata["validation_status"] == "rule_candidate" for claim in result.bundle.claims)
|
||
|
|
assert len(result.rejected_claims) == 2
|
||
|
|
|
||
|
|
|
||
|
|
def test_validation_rejects_product_detail_claims_on_category_page_context():
|
||
|
|
config = load_project_config("configs/perfume_subscription.yaml")
|
||
|
|
bundle = ExtractionBundle(
|
||
|
|
entities=[ExtractedEntity("Perfume", "Neroli Summer", evidence_text="Neroli Summer")],
|
||
|
|
claims=[
|
||
|
|
ExtractedClaim(
|
||
|
|
"Neroli Summer",
|
||
|
|
"Perfume",
|
||
|
|
"hasPrice",
|
||
|
|
object_value={"amount": 89, "currency": "USD"},
|
||
|
|
evidence_text="Neroli Summer Price $89",
|
||
|
|
)
|
||
|
|
],
|
||
|
|
extractor_name="llm_json_extractor",
|
||
|
|
provider="lm_studio",
|
||
|
|
raw_output={"extraction_mode": "primary"},
|
||
|
|
)
|
||
|
|
context = ExtractionPageContext(
|
||
|
|
url="https://example.com/category/perfume",
|
||
|
|
final_url="https://example.com/category/perfume",
|
||
|
|
title="Perfume list",
|
||
|
|
page_type="CategoryPage",
|
||
|
|
clean_text="Neroli Summer Price $89",
|
||
|
|
source_zones=[
|
||
|
|
{
|
||
|
|
"zone_type": "product_summary",
|
||
|
|
"selector": ".product-list",
|
||
|
|
"text": "Neroli Summer Price $89",
|
||
|
|
"claim_allowed": True,
|
||
|
|
}
|
||
|
|
],
|
||
|
|
)
|
||
|
|
|
||
|
|
result = validate_extraction_bundle(attach_page_context(bundle, context), config)
|
||
|
|
|
||
|
|
assert result.claim_status == "candidate_claim"
|
||
|
|
assert result.bundle.claims == []
|
||
|
|
assert result.rejected_claims[0]["reason"] == "predicate hasPrice is not allowed for page type CategoryPage"
|
||
|
|
|
||
|
|
|
||
|
|
def test_repository_logs_rule_candidates_without_persisting_ontology_rows():
|
||
|
|
config = load_project_config("configs/perfume_subscription.yaml")
|
||
|
|
engine = make_engine("sqlite:///:memory:")
|
||
|
|
models.Base.metadata.create_all(engine)
|
||
|
|
|
||
|
|
from sqlalchemy.orm import Session
|
||
|
|
|
||
|
|
session = Session(engine)
|
||
|
|
repo = KnowledgeRepository(session)
|
||
|
|
project = repo.upsert_project(config)
|
||
|
|
source = repo.get_source(project.id, "official_brand_site")
|
||
|
|
page = repo.upsert_page(project.id, source.id, "https://example.com/product", "Product", 200, "Neroli Summer")
|
||
|
|
bundle = ExtractionBundle(
|
||
|
|
entities=[ExtractedEntity("Perfume", "Neroli Summer")],
|
||
|
|
claims=[
|
||
|
|
ExtractedClaim(
|
||
|
|
"Neroli Summer",
|
||
|
|
"Perfume",
|
||
|
|
"hasTopNote",
|
||
|
|
"Bergamot",
|
||
|
|
"Note",
|
||
|
|
evidence_text="Top notes: Bergamot",
|
||
|
|
)
|
||
|
|
],
|
||
|
|
extractor_name="perfume_rule_based",
|
||
|
|
provider="rule_based",
|
||
|
|
raw_output={},
|
||
|
|
)
|
||
|
|
|
||
|
|
claims = repo.save_extraction_bundle(project.id, source, page, bundle, config)
|
||
|
|
session.commit()
|
||
|
|
|
||
|
|
assert claims == []
|
||
|
|
assert session.query(models.Claim).count() == 0
|
||
|
|
assert session.query(models.Entity).count() == 0
|
||
|
|
assert session.query(models.ExtractionLog).count() == 1
|
||
|
|
session.close()
|
||
|
|
|
||
|
|
|
||
|
|
def test_parse_json_content_accepts_markdown_fenced_json():
|
||
|
|
raw = """```json
|
||
|
|
{"entities": [], "claims": []}
|
||
|
|
```"""
|
||
|
|
|
||
|
|
assert parse_json_content(raw) == {"entities": [], "claims": []}
|
||
|
|
|
||
|
|
|
||
|
|
def test_parse_json_content_requires_structured_claim_schema():
|
||
|
|
raw = {
|
||
|
|
"entities": [
|
||
|
|
{
|
||
|
|
"entity_type": "Perfume",
|
||
|
|
"name": "Neroli Summer",
|
||
|
|
"attributes": {},
|
||
|
|
"confidence": 0.9,
|
||
|
|
"evidence_text": "Neroli Summer",
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"claims": [
|
||
|
|
{
|
||
|
|
"subject_name": "Neroli Summer",
|
||
|
|
"subject_type": "Perfume",
|
||
|
|
"predicate": "hasTopNote",
|
||
|
|
"object_name": "Bergamot",
|
||
|
|
"object_type": "FragranceNote",
|
||
|
|
"object_value": None,
|
||
|
|
"evidence_text": "Top notes: Bergamot",
|
||
|
|
"evidence_summary": "explicit note listing",
|
||
|
|
"confidence": 0.9,
|
||
|
|
"confidence_reason": "directly stated",
|
||
|
|
"source_zone": "product_description",
|
||
|
|
}
|
||
|
|
],
|
||
|
|
}
|
||
|
|
|
||
|
|
parsed = parse_json_content(json.dumps(raw))
|
||
|
|
|
||
|
|
assert parsed["claims"][0]["source_zone"] == "product_description"
|
||
|
|
|
||
|
|
|
||
|
|
def test_parse_json_content_rejects_claim_without_evidence():
|
||
|
|
raw = {
|
||
|
|
"entities": [],
|
||
|
|
"claims": [
|
||
|
|
{
|
||
|
|
"subject_name": "Neroli Summer",
|
||
|
|
"subject_type": "Perfume",
|
||
|
|
"predicate": "hasTopNote",
|
||
|
|
"object_name": "Bergamot",
|
||
|
|
"object_type": "Note",
|
||
|
|
"object_value": None,
|
||
|
|
"evidence_text": "",
|
||
|
|
"evidence_summary": "missing evidence",
|
||
|
|
"confidence": 0.9,
|
||
|
|
"confidence_reason": "bad",
|
||
|
|
"source_zone": "product_description",
|
||
|
|
}
|
||
|
|
],
|
||
|
|
}
|
||
|
|
|
||
|
|
try:
|
||
|
|
parse_json_content(json.dumps(raw))
|
||
|
|
except ValueError as exc:
|
||
|
|
assert "schema_validation_failed" in str(exc)
|
||
|
|
else:
|
||
|
|
raise AssertionError("expected schema validation failure")
|
||
|
|
|
||
|
|
|
||
|
|
def test_repository_merges_only_validated_typed_claims_to_graph():
|
||
|
|
config = load_project_config("configs/perfume_subscription.yaml")
|
||
|
|
engine = make_engine("sqlite:///:memory:")
|
||
|
|
models.Base.metadata.create_all(engine)
|
||
|
|
|
||
|
|
from sqlalchemy.orm import Session
|
||
|
|
|
||
|
|
session = Session(engine)
|
||
|
|
repo = KnowledgeRepository(session)
|
||
|
|
project = repo.upsert_project(config)
|
||
|
|
source = repo.get_source(project.id, "official_brand_site")
|
||
|
|
page = repo.upsert_page(
|
||
|
|
project.id,
|
||
|
|
source.id,
|
||
|
|
"https://example.com/product/detail.html",
|
||
|
|
"Neroli Summer",
|
||
|
|
200,
|
||
|
|
"Neroli Summer\nTop notes: Bergamot",
|
||
|
|
)
|
||
|
|
bundle = ExtractionBundle(
|
||
|
|
entities=[
|
||
|
|
ExtractedEntity("Perfume", "Neroli Summer", evidence_text="Neroli Summer", confidence=0.9),
|
||
|
|
ExtractedEntity("FragranceNote", "Bergamot", evidence_text="Top notes: Bergamot", confidence=0.9),
|
||
|
|
],
|
||
|
|
claims=[
|
||
|
|
ExtractedClaim(
|
||
|
|
"Neroli Summer",
|
||
|
|
"Perfume",
|
||
|
|
"hasTopNote",
|
||
|
|
"Bergamot",
|
||
|
|
"FragranceNote",
|
||
|
|
evidence_text="Top notes: Bergamot",
|
||
|
|
confidence=0.92,
|
||
|
|
confidence_reason="directly stated",
|
||
|
|
)
|
||
|
|
],
|
||
|
|
extractor_name="llm_json_extractor",
|
||
|
|
provider="lm_studio",
|
||
|
|
raw_output={"extraction_mode": "primary"},
|
||
|
|
)
|
||
|
|
context = ExtractionPageContext(
|
||
|
|
url="https://example.com/product/detail.html",
|
||
|
|
final_url="https://example.com/product/detail.html",
|
||
|
|
title="Neroli Summer",
|
||
|
|
page_type="ProductPage",
|
||
|
|
clean_text="Neroli Summer\nTop notes: Bergamot",
|
||
|
|
source_zones=[
|
||
|
|
{
|
||
|
|
"zone_type": "product_description",
|
||
|
|
"selector": ".description",
|
||
|
|
"text": "Top notes: Bergamot",
|
||
|
|
"claim_allowed": True,
|
||
|
|
}
|
||
|
|
],
|
||
|
|
)
|
||
|
|
|
||
|
|
claims = repo.save_extraction_bundle(project.id, source, page, attach_page_context(bundle, context), config)
|
||
|
|
session.commit()
|
||
|
|
|
||
|
|
assert len(claims) == 1
|
||
|
|
assert claims[0].status == "validated_claim"
|
||
|
|
assert claims[0].metadata_json["graph_merge_status"] == "merged"
|
||
|
|
assert session.query(models.Relation).count() == 1
|
||
|
|
assert session.query(models.Entity).filter(models.Entity.entity_type == "Note").count() == 1
|
||
|
|
session.close()
|