ontology
This commit is contained in:
LASTA_DEV01\lasta
@@ -12,7 +12,9 @@ from crawler_platform.app.core.extractor.base import (
|
||||
ExtractionBundle,
|
||||
)
|
||||
from crawler_platform.app.core.extractor.ai_provider import parse_json_content
|
||||
from crawler_platform.app.core.extractor.ai_provider import LLMJsonExtractor
|
||||
from crawler_platform.app.core.extractor.validation import attach_page_context, validate_extraction_bundle
|
||||
from crawler_platform.app.domains.perfume.extractor import PerfumeRuleBasedExtractor
|
||||
|
||||
|
||||
def test_cleaner_prefers_product_content_and_removes_boilerplate():
|
||||
@@ -175,7 +177,7 @@ def test_validation_rejects_product_detail_claims_on_category_page_context():
|
||||
assert result.rejected_claims[0]["reason"] == "predicate hasPrice is not allowed for page type CategoryPage"
|
||||
|
||||
|
||||
def test_repository_logs_rule_candidates_without_persisting_ontology_rows():
|
||||
def test_repository_persists_rule_candidates_without_merging_graph_relations():
|
||||
config = load_project_config("configs/perfume_subscription.yaml")
|
||||
engine = make_engine("sqlite:///:memory:")
|
||||
models.Base.metadata.create_all(engine)
|
||||
@@ -194,23 +196,41 @@ def test_repository_logs_rule_candidates_without_persisting_ontology_rows():
|
||||
"Neroli Summer",
|
||||
"Perfume",
|
||||
"hasTopNote",
|
||||
"Bergamot",
|
||||
"Note",
|
||||
evidence_text="Top notes: Bergamot",
|
||||
)
|
||||
"Bergamot",
|
||||
"Note",
|
||||
evidence_text="Top notes: Bergamot",
|
||||
confidence=0.9,
|
||||
)
|
||||
],
|
||||
extractor_name="perfume_rule_based",
|
||||
provider="rule_based",
|
||||
raw_output={},
|
||||
)
|
||||
context = ExtractionPageContext(
|
||||
url="https://example.com/product",
|
||||
final_url="https://example.com/product",
|
||||
title="Product",
|
||||
page_type="ProductPage",
|
||||
clean_text="Neroli Summer\nTop notes: Bergamot",
|
||||
source_zones=[
|
||||
{
|
||||
"zone_type": "product_description",
|
||||
"selector": ".description",
|
||||
"text": "Top notes: Bergamot",
|
||||
"claim_allowed": True,
|
||||
}
|
||||
],
|
||||
)
|
||||
|
||||
claims = repo.save_extraction_bundle(project.id, source, page, bundle, config)
|
||||
claims = repo.save_extraction_bundle(project.id, source, page, attach_page_context(bundle, context), config)
|
||||
session.commit()
|
||||
|
||||
assert claims == []
|
||||
assert session.query(models.Claim).count() == 0
|
||||
assert session.query(models.Entity).count() == 0
|
||||
assert len(claims) == 1
|
||||
assert claims[0].status == "rule_candidate"
|
||||
assert session.query(models.Claim).count() == 1
|
||||
assert session.query(models.Entity).count() == 2
|
||||
assert session.query(models.ExtractionLog).count() == 1
|
||||
assert session.query(models.Relation).count() == 0
|
||||
session.close()
|
||||
|
||||
|
||||
@@ -222,6 +242,93 @@ def test_parse_json_content_accepts_markdown_fenced_json():
|
||||
assert parse_json_content(raw) == {"entities": [], "claims": []}
|
||||
|
||||
|
||||
def test_lm_studio_extractor_merges_rule_claims_even_when_ai_claim_is_invalid():
|
||||
config = load_project_config("configs/perfume_subscription.yaml")
|
||||
|
||||
class StubExtractor(LLMJsonExtractor):
|
||||
def complete_json(self, page_text, project_config, compact=False, context=None):
|
||||
return {
|
||||
"entities": [
|
||||
{
|
||||
"entity_type": "Brand",
|
||||
"name": "912",
|
||||
"attributes": {},
|
||||
"confidence": 0.9,
|
||||
"evidence_text": "912",
|
||||
}
|
||||
],
|
||||
"claims": [
|
||||
{
|
||||
"subject_name": "912",
|
||||
"subject_type": "Brand",
|
||||
"predicate": "hasBrand",
|
||||
"object_name": "912",
|
||||
"object_type": "Brand",
|
||||
"object_value": None,
|
||||
"evidence_text": "912",
|
||||
"evidence_summary": "bad self-brand claim",
|
||||
"confidence": 0.9,
|
||||
"confidence_reason": "stub",
|
||||
"source_zone": "product_title",
|
||||
}
|
||||
],
|
||||
}
|
||||
|
||||
text = "Neroli Summer\nTop notes: Bergamot\nPrice $89"
|
||||
bundle = StubExtractor("perfume", "lm_studio", model="stub").extract(text, config)
|
||||
|
||||
assert any(claim.predicate == "hasTopNote" for claim in bundle.claims)
|
||||
|
||||
|
||||
def test_the912_product_rule_fallback_uses_product_title_brand_and_price():
|
||||
config = load_project_config("configs/perfume_subscription.yaml")
|
||||
text = """
|
||||
새로운 향수의 시작, 클론 향수
|
||||
[2+1 기획] 912 클론 니치향수 모음 40ml
|
||||
4.8
|
||||
114,414
|
||||
일반 구매가격
|
||||
72,000원
|
||||
할인 적용금액
|
||||
37,000원
|
||||
최종 구매금액
|
||||
35,000원
|
||||
현재 위치
|
||||
전체 상품
|
||||
니치향수
|
||||
오드 퍼퓸
|
||||
"""
|
||||
bundle = PerfumeRuleBasedExtractor().extract(text, config)
|
||||
context = ExtractionPageContext(
|
||||
url="https://the912.co.kr/product/detail.html?product_no=513",
|
||||
final_url="https://the912.co.kr/product/detail.html?product_no=513",
|
||||
title="[2+1 기획] 912 클론 니치향수 모음 40ml - 912",
|
||||
page_type="ProductPage",
|
||||
clean_text=text,
|
||||
source_zones=[
|
||||
{
|
||||
"zone_type": "product_title",
|
||||
"selector": "h1",
|
||||
"text": "[2+1 기획] 912 클론 니치향수 모음 40ml",
|
||||
"claim_allowed": True,
|
||||
},
|
||||
{
|
||||
"zone_type": "product_summary",
|
||||
"selector": ".detail",
|
||||
"text": "일반 구매가격\n72,000원\n할인 적용금액\n37,000원\n최종 구매금액\n35,000원",
|
||||
"claim_allowed": True,
|
||||
},
|
||||
],
|
||||
)
|
||||
|
||||
result = validate_extraction_bundle(attach_page_context(bundle, context), config)
|
||||
predicates = {claim.predicate for claim in result.bundle.claims}
|
||||
perfume_names = {entity.name for entity in result.bundle.entities if entity.entity_type == "Perfume"}
|
||||
|
||||
assert "[2+1 기획] 912 클론 니치향수 모음 40ml" in perfume_names
|
||||
assert {"hasBrand", "hasPrice"} <= predicates
|
||||
|
||||
|
||||
def test_parse_json_content_requires_structured_claim_schema():
|
||||
raw = {
|
||||
"entities": [
|
||||
|
||||
Reference in New Issue
Block a user