import json
from crawler_platform.app.config.loader import load_project_config
from crawler_platform.app.core.database import models
from crawler_platform.app.core.database.repository import KnowledgeRepository
from crawler_platform.app.core.database.session import make_engine
from crawler_platform.app.core.crawler.html_cleaner import clean_html_with_metadata
from crawler_platform.app.core.extractor.base import (
ExtractedClaim,
ExtractedEntity,
ExtractionPageContext,
ExtractionBundle,
)
from crawler_platform.app.core.extractor.ai_provider import parse_json_content
from crawler_platform.app.core.extractor.ai_provider import LLMJsonExtractor
from crawler_platform.app.core.extractor.validation import attach_page_context, validate_extraction_bundle
from crawler_platform.app.domains.perfume.extractor import PerfumeRuleBasedExtractor
def test_cleaner_prefers_product_content_and_removes_boilerplate():
html = """
Sample Product
CAFE24 Login Cart Low price Product count
Home Board Event Shipping
Neroli Summer Eau de Parfum
Brand: Example Maison
Top notes: Bergamot, Neroli
Base notes: Musk
Price $89
Powered by CAFE24 Copyright Shipping Country List
"""
cleaned = clean_html_with_metadata(html)
assert "Neroli Summer Eau de Parfum" in cleaned.text
assert "Bergamot" in cleaned.text
assert "CAFE24" not in cleaned.text
assert "Low price" not in cleaned.text
assert cleaned.page_type == "ProductPage"
assert cleaned.extraction_status == "success"
assert any(zone["zone_type"] == "product_detail" for zone in cleaned.source_zones)
def test_cleaner_keeps_raw_and_clean_text_separate_with_noise_debug():
html = """
Sample Product
CAFE24 Login Cart Low price Product count
Neroli Summer Eau de Parfum
Brand: Example Maison
Top notes: Bergamot, Neroli
Price $89
Shipping delivery return exchange country list Korea USA Japan Canada
"""
cleaned = clean_html_with_metadata(html, url="https://example.com/product/detail.html")
assert "CAFE24" in cleaned.raw_text
assert "CAFE24" not in cleaned.text
assert "Shipping delivery" not in cleaned.text
assert cleaned.clean_markdown
assert cleaned.metadata["removed_noise_zones_count"] >= 1
def test_cleaner_handles_empty_dom_attributes():
html = """
Brand: Example Maison
Top notes: Bergamot, Neroli
"""
cleaned = clean_html_with_metadata(html)
assert "Example Maison" in cleaned.text
def test_cleaner_skips_decomposed_nodes_during_boilerplate_scan():
from bs4 import BeautifulSoup
soup = BeautifulSoup("", "html.parser")
tag = soup.span
assert tag is not None
tag.decompose()
from crawler_platform.app.core.crawler.html_cleaner import node_attr
assert node_attr(tag, "class") == ""
def test_validation_rejects_placeholders_and_marks_rule_candidates():
config = load_project_config("configs/perfume_subscription.yaml")
bundle = ExtractionBundle(
entities=[
ExtractedEntity("Brand", "CAFE24"),
ExtractedEntity("Perfume", "Neroli Summer"),
ExtractedEntity("Accord", "Green"),
],
claims=[
ExtractedClaim("Neroli Summer", "Perfume", "hasBrand", "CAFE24", "Brand", evidence_text="Brand: CAFE24"),
ExtractedClaim(
"Neroli Summer",
"Perfume",
"hasTopNote",
"Bergamot",
"Note",
evidence_text="Top notes: Bergamot",
confidence=0.82,
),
ExtractedClaim("Neroli Summer", "Perfume", "hasAccord", "Green", "Accord", evidence_text="Accord: Green"),
],
extractor_name="perfume_rule_based",
provider="rule_based",
raw_output={},
)
result = validate_extraction_bundle(bundle, config)
assert result.claim_status == "rule_candidate"
assert [claim.object_name for claim in result.bundle.claims] == ["Bergamot"]
assert all(claim.metadata["validation_status"] == "rule_candidate" for claim in result.bundle.claims)
assert len(result.rejected_claims) == 2
def test_validation_rejects_product_detail_claims_on_category_page_context():
config = load_project_config("configs/perfume_subscription.yaml")
bundle = ExtractionBundle(
entities=[ExtractedEntity("Perfume", "Neroli Summer", evidence_text="Neroli Summer")],
claims=[
ExtractedClaim(
"Neroli Summer",
"Perfume",
"hasPrice",
object_value={"amount": 89, "currency": "USD"},
evidence_text="Neroli Summer Price $89",
)
],
extractor_name="llm_json_extractor",
provider="lm_studio",
raw_output={"extraction_mode": "primary"},
)
context = ExtractionPageContext(
url="https://example.com/category/perfume",
final_url="https://example.com/category/perfume",
title="Perfume list",
page_type="CategoryPage",
clean_text="Neroli Summer Price $89",
source_zones=[
{
"zone_type": "product_summary",
"selector": ".product-list",
"text": "Neroli Summer Price $89",
"claim_allowed": True,
}
],
)
result = validate_extraction_bundle(attach_page_context(bundle, context), config)
assert result.claim_status == "candidate_claim"
assert result.bundle.claims == []
assert result.rejected_claims[0]["reason"] == "predicate hasPrice is not allowed for page type CategoryPage"
def test_repository_persists_rule_candidates_without_merging_graph_relations():
config = load_project_config("configs/perfume_subscription.yaml")
engine = make_engine("sqlite:///:memory:")
models.Base.metadata.create_all(engine)
from sqlalchemy.orm import Session
session = Session(engine)
repo = KnowledgeRepository(session)
project = repo.upsert_project(config)
source = repo.get_source(project.id, "official_brand_site")
page = repo.upsert_page(project.id, source.id, "https://example.com/product", "Product", 200, "Neroli Summer")
bundle = ExtractionBundle(
entities=[ExtractedEntity("Perfume", "Neroli Summer")],
claims=[
ExtractedClaim(
"Neroli Summer",
"Perfume",
"hasTopNote",
"Bergamot",
"Note",
evidence_text="Top notes: Bergamot",
confidence=0.9,
)
],
extractor_name="perfume_rule_based",
provider="rule_based",
raw_output={},
)
context = ExtractionPageContext(
url="https://example.com/product",
final_url="https://example.com/product",
title="Product",
page_type="ProductPage",
clean_text="Neroli Summer\nTop notes: Bergamot",
source_zones=[
{
"zone_type": "product_description",
"selector": ".description",
"text": "Top notes: Bergamot",
"claim_allowed": True,
}
],
)
claims = repo.save_extraction_bundle(project.id, source, page, attach_page_context(bundle, context), config)
session.commit()
assert len(claims) == 1
assert claims[0].status == "rule_candidate"
assert session.query(models.Claim).count() == 1
assert session.query(models.Entity).count() == 2
assert session.query(models.ExtractionLog).count() == 1
assert session.query(models.Relation).count() == 0
session.close()
def test_parse_json_content_accepts_markdown_fenced_json():
raw = """```json
{"entities": [], "claims": []}
```"""
assert parse_json_content(raw) == {"entities": [], "claims": []}
def test_lm_studio_extractor_merges_rule_claims_even_when_ai_claim_is_invalid():
config = load_project_config("configs/perfume_subscription.yaml")
class StubExtractor(LLMJsonExtractor):
def complete_json(self, page_text, project_config, compact=False, context=None):
return {
"entities": [
{
"entity_type": "Brand",
"name": "912",
"attributes": {},
"confidence": 0.9,
"evidence_text": "912",
}
],
"claims": [
{
"subject_name": "912",
"subject_type": "Brand",
"predicate": "hasBrand",
"object_name": "912",
"object_type": "Brand",
"object_value": None,
"evidence_text": "912",
"evidence_summary": "bad self-brand claim",
"confidence": 0.9,
"confidence_reason": "stub",
"source_zone": "product_title",
}
],
}
text = "Neroli Summer\nTop notes: Bergamot\nPrice $89"
bundle = StubExtractor("perfume", "lm_studio", model="stub").extract(text, config)
assert any(claim.predicate == "hasTopNote" for claim in bundle.claims)
def test_the912_product_rule_fallback_uses_product_title_brand_and_price():
config = load_project_config("configs/perfume_subscription.yaml")
text = """
새로운 향수의 시작, 클론 향수
[2+1 기획] 912 클론 니치향수 모음 40ml
4.8
114,414
일반 구매가격
72,000원
할인 적용금액
37,000원
최종 구매금액
35,000원
현재 위치
전체 상품
니치향수
오드 퍼퓸
"""
bundle = PerfumeRuleBasedExtractor().extract(text, config)
context = ExtractionPageContext(
url="https://the912.co.kr/product/detail.html?product_no=513",
final_url="https://the912.co.kr/product/detail.html?product_no=513",
title="[2+1 기획] 912 클론 니치향수 모음 40ml - 912",
page_type="ProductPage",
clean_text=text,
source_zones=[
{
"zone_type": "product_title",
"selector": "h1",
"text": "[2+1 기획] 912 클론 니치향수 모음 40ml",
"claim_allowed": True,
},
{
"zone_type": "product_summary",
"selector": ".detail",
"text": "일반 구매가격\n72,000원\n할인 적용금액\n37,000원\n최종 구매금액\n35,000원",
"claim_allowed": True,
},
],
)
result = validate_extraction_bundle(attach_page_context(bundle, context), config)
predicates = {claim.predicate for claim in result.bundle.claims}
perfume_names = {entity.name for entity in result.bundle.entities if entity.entity_type == "Perfume"}
assert "[2+1 기획] 912 클론 니치향수 모음 40ml" in perfume_names
assert {"hasBrand", "hasPrice"} <= predicates
def test_parse_json_content_requires_structured_claim_schema():
raw = {
"entities": [
{
"entity_type": "Perfume",
"name": "Neroli Summer",
"attributes": {},
"confidence": 0.9,
"evidence_text": "Neroli Summer",
}
],
"claims": [
{
"subject_name": "Neroli Summer",
"subject_type": "Perfume",
"predicate": "hasTopNote",
"object_name": "Bergamot",
"object_type": "FragranceNote",
"object_value": None,
"evidence_text": "Top notes: Bergamot",
"evidence_summary": "explicit note listing",
"confidence": 0.9,
"confidence_reason": "directly stated",
"source_zone": "product_description",
}
],
}
parsed = parse_json_content(json.dumps(raw))
assert parsed["claims"][0]["source_zone"] == "product_description"
def test_parse_json_content_rejects_claim_without_evidence():
raw = {
"entities": [],
"claims": [
{
"subject_name": "Neroli Summer",
"subject_type": "Perfume",
"predicate": "hasTopNote",
"object_name": "Bergamot",
"object_type": "Note",
"object_value": None,
"evidence_text": "",
"evidence_summary": "missing evidence",
"confidence": 0.9,
"confidence_reason": "bad",
"source_zone": "product_description",
}
],
}
try:
parse_json_content(json.dumps(raw))
except ValueError as exc:
assert "schema_validation_failed" in str(exc)
else:
raise AssertionError("expected schema validation failure")
def test_repository_merges_only_validated_typed_claims_to_graph():
config = load_project_config("configs/perfume_subscription.yaml")
engine = make_engine("sqlite:///:memory:")
models.Base.metadata.create_all(engine)
from sqlalchemy.orm import Session
session = Session(engine)
repo = KnowledgeRepository(session)
project = repo.upsert_project(config)
source = repo.get_source(project.id, "official_brand_site")
page = repo.upsert_page(
project.id,
source.id,
"https://example.com/product/detail.html",
"Neroli Summer",
200,
"Neroli Summer\nTop notes: Bergamot",
)
bundle = ExtractionBundle(
entities=[
ExtractedEntity("Perfume", "Neroli Summer", evidence_text="Neroli Summer", confidence=0.9),
ExtractedEntity("FragranceNote", "Bergamot", evidence_text="Top notes: Bergamot", confidence=0.9),
],
claims=[
ExtractedClaim(
"Neroli Summer",
"Perfume",
"hasTopNote",
"Bergamot",
"FragranceNote",
evidence_text="Top notes: Bergamot",
confidence=0.92,
confidence_reason="directly stated",
)
],
extractor_name="llm_json_extractor",
provider="lm_studio",
raw_output={"extraction_mode": "primary"},
)
context = ExtractionPageContext(
url="https://example.com/product/detail.html",
final_url="https://example.com/product/detail.html",
title="Neroli Summer",
page_type="ProductPage",
clean_text="Neroli Summer\nTop notes: Bergamot",
source_zones=[
{
"zone_type": "product_description",
"selector": ".description",
"text": "Top notes: Bergamot",
"claim_allowed": True,
}
],
)
claims = repo.save_extraction_bundle(project.id, source, page, attach_page_context(bundle, context), config)
session.commit()
assert len(claims) == 1
assert claims[0].status == "validated_claim"
assert claims[0].metadata_json["graph_merge_status"] == "merged"
assert session.query(models.Relation).count() == 1
assert session.query(models.Entity).filter(models.Entity.entity_type == "Note").count() == 1
session.close()