참고소스 수정본
This commit is contained in:
LASTA_DEV01\lasta
350
tests/test_content_quality.py
Normal file
350
tests/test_content_quality.py
Normal file
@@ -0,0 +1,350 @@
|
||||
import json
|
||||
|
||||
from crawler_platform.app.config.loader import load_project_config
|
||||
from crawler_platform.app.core.database import models
|
||||
from crawler_platform.app.core.database.repository import KnowledgeRepository
|
||||
from crawler_platform.app.core.database.session import make_engine
|
||||
from crawler_platform.app.core.crawler.html_cleaner import clean_html_with_metadata
|
||||
from crawler_platform.app.core.extractor.base import (
|
||||
ExtractedClaim,
|
||||
ExtractedEntity,
|
||||
ExtractionPageContext,
|
||||
ExtractionBundle,
|
||||
)
|
||||
from crawler_platform.app.core.extractor.ai_provider import parse_json_content
|
||||
from crawler_platform.app.core.extractor.validation import attach_page_context, validate_extraction_bundle
|
||||
|
||||
|
||||
def test_cleaner_prefers_product_content_and_removes_boilerplate():
|
||||
html = """
|
||||
<html>
|
||||
<head><title>Sample Product</title></head>
|
||||
<body>
|
||||
<header>CAFE24 Login Cart Low price Product count</header>
|
||||
<nav>Home Board Event Shipping</nav>
|
||||
<main class="product-detail">
|
||||
<h1>Neroli Summer Eau de Parfum</h1>
|
||||
<p>Brand: Example Maison</p>
|
||||
<p>Top notes: Bergamot, Neroli</p>
|
||||
<p>Base notes: Musk</p>
|
||||
<p>Price $89</p>
|
||||
</main>
|
||||
<footer>Powered by CAFE24 Copyright Shipping Country List</footer>
|
||||
</body>
|
||||
</html>
|
||||
"""
|
||||
|
||||
cleaned = clean_html_with_metadata(html)
|
||||
|
||||
assert "Neroli Summer Eau de Parfum" in cleaned.text
|
||||
assert "Bergamot" in cleaned.text
|
||||
assert "CAFE24" not in cleaned.text
|
||||
assert "Low price" not in cleaned.text
|
||||
assert cleaned.page_type == "ProductPage"
|
||||
assert cleaned.extraction_status == "success"
|
||||
assert any(zone["zone_type"] == "product_detail" for zone in cleaned.source_zones)
|
||||
|
||||
|
||||
def test_cleaner_keeps_raw_and_clean_text_separate_with_noise_debug():
|
||||
html = """
|
||||
<html>
|
||||
<head><title>Sample Product</title></head>
|
||||
<body>
|
||||
<header>CAFE24 Login Cart Low price Product count</header>
|
||||
<main class="product-detail">
|
||||
<h1>Neroli Summer Eau de Parfum</h1>
|
||||
<p>Brand: Example Maison</p>
|
||||
<p>Top notes: Bergamot, Neroli</p>
|
||||
<p>Price $89</p>
|
||||
</main>
|
||||
<section class="shipping">Shipping delivery return exchange country list Korea USA Japan Canada</section>
|
||||
</body>
|
||||
</html>
|
||||
"""
|
||||
|
||||
cleaned = clean_html_with_metadata(html, url="https://example.com/product/detail.html")
|
||||
|
||||
assert "CAFE24" in cleaned.raw_text
|
||||
assert "CAFE24" not in cleaned.text
|
||||
assert "Shipping delivery" not in cleaned.text
|
||||
assert cleaned.clean_markdown
|
||||
assert cleaned.metadata["removed_noise_zones_count"] >= 1
|
||||
|
||||
|
||||
def test_cleaner_handles_empty_dom_attributes():
|
||||
html = """
|
||||
<html>
|
||||
<body>
|
||||
<div id class aria-label>
|
||||
<p>Brand: Example Maison</p>
|
||||
<p>Top notes: Bergamot, Neroli</p>
|
||||
</div>
|
||||
</body>
|
||||
</html>
|
||||
"""
|
||||
|
||||
cleaned = clean_html_with_metadata(html)
|
||||
|
||||
assert "Example Maison" in cleaned.text
|
||||
|
||||
|
||||
def test_cleaner_skips_decomposed_nodes_during_boilerplate_scan():
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
soup = BeautifulSoup("<div id='footer'><span class='child'>x</span></div>", "html.parser")
|
||||
tag = soup.span
|
||||
assert tag is not None
|
||||
tag.decompose()
|
||||
|
||||
from crawler_platform.app.core.crawler.html_cleaner import node_attr
|
||||
|
||||
assert node_attr(tag, "class") == ""
|
||||
|
||||
|
||||
def test_validation_rejects_placeholders_and_marks_rule_candidates():
|
||||
config = load_project_config("configs/perfume_subscription.yaml")
|
||||
bundle = ExtractionBundle(
|
||||
entities=[
|
||||
ExtractedEntity("Brand", "CAFE24"),
|
||||
ExtractedEntity("Perfume", "Neroli Summer"),
|
||||
ExtractedEntity("Accord", "Green"),
|
||||
],
|
||||
claims=[
|
||||
ExtractedClaim("Neroli Summer", "Perfume", "hasBrand", "CAFE24", "Brand", evidence_text="Brand: CAFE24"),
|
||||
ExtractedClaim(
|
||||
"Neroli Summer",
|
||||
"Perfume",
|
||||
"hasTopNote",
|
||||
"Bergamot",
|
||||
"Note",
|
||||
evidence_text="Top notes: Bergamot",
|
||||
confidence=0.82,
|
||||
),
|
||||
ExtractedClaim("Neroli Summer", "Perfume", "hasAccord", "Green", "Accord", evidence_text="Accord: Green"),
|
||||
],
|
||||
extractor_name="perfume_rule_based",
|
||||
provider="rule_based",
|
||||
raw_output={},
|
||||
)
|
||||
|
||||
result = validate_extraction_bundle(bundle, config)
|
||||
|
||||
assert result.claim_status == "rule_candidate"
|
||||
assert [claim.object_name for claim in result.bundle.claims] == ["Bergamot"]
|
||||
assert all(claim.metadata["validation_status"] == "rule_candidate" for claim in result.bundle.claims)
|
||||
assert len(result.rejected_claims) == 2
|
||||
|
||||
|
||||
def test_validation_rejects_product_detail_claims_on_category_page_context():
|
||||
config = load_project_config("configs/perfume_subscription.yaml")
|
||||
bundle = ExtractionBundle(
|
||||
entities=[ExtractedEntity("Perfume", "Neroli Summer", evidence_text="Neroli Summer")],
|
||||
claims=[
|
||||
ExtractedClaim(
|
||||
"Neroli Summer",
|
||||
"Perfume",
|
||||
"hasPrice",
|
||||
object_value={"amount": 89, "currency": "USD"},
|
||||
evidence_text="Neroli Summer Price $89",
|
||||
)
|
||||
],
|
||||
extractor_name="llm_json_extractor",
|
||||
provider="lm_studio",
|
||||
raw_output={"extraction_mode": "primary"},
|
||||
)
|
||||
context = ExtractionPageContext(
|
||||
url="https://example.com/category/perfume",
|
||||
final_url="https://example.com/category/perfume",
|
||||
title="Perfume list",
|
||||
page_type="CategoryPage",
|
||||
clean_text="Neroli Summer Price $89",
|
||||
source_zones=[
|
||||
{
|
||||
"zone_type": "product_summary",
|
||||
"selector": ".product-list",
|
||||
"text": "Neroli Summer Price $89",
|
||||
"claim_allowed": True,
|
||||
}
|
||||
],
|
||||
)
|
||||
|
||||
result = validate_extraction_bundle(attach_page_context(bundle, context), config)
|
||||
|
||||
assert result.claim_status == "candidate_claim"
|
||||
assert result.bundle.claims == []
|
||||
assert result.rejected_claims[0]["reason"] == "predicate hasPrice is not allowed for page type CategoryPage"
|
||||
|
||||
|
||||
def test_repository_logs_rule_candidates_without_persisting_ontology_rows():
|
||||
config = load_project_config("configs/perfume_subscription.yaml")
|
||||
engine = make_engine("sqlite:///:memory:")
|
||||
models.Base.metadata.create_all(engine)
|
||||
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
session = Session(engine)
|
||||
repo = KnowledgeRepository(session)
|
||||
project = repo.upsert_project(config)
|
||||
source = repo.get_source(project.id, "official_brand_site")
|
||||
page = repo.upsert_page(project.id, source.id, "https://example.com/product", "Product", 200, "Neroli Summer")
|
||||
bundle = ExtractionBundle(
|
||||
entities=[ExtractedEntity("Perfume", "Neroli Summer")],
|
||||
claims=[
|
||||
ExtractedClaim(
|
||||
"Neroli Summer",
|
||||
"Perfume",
|
||||
"hasTopNote",
|
||||
"Bergamot",
|
||||
"Note",
|
||||
evidence_text="Top notes: Bergamot",
|
||||
)
|
||||
],
|
||||
extractor_name="perfume_rule_based",
|
||||
provider="rule_based",
|
||||
raw_output={},
|
||||
)
|
||||
|
||||
claims = repo.save_extraction_bundle(project.id, source, page, bundle, config)
|
||||
session.commit()
|
||||
|
||||
assert claims == []
|
||||
assert session.query(models.Claim).count() == 0
|
||||
assert session.query(models.Entity).count() == 0
|
||||
assert session.query(models.ExtractionLog).count() == 1
|
||||
session.close()
|
||||
|
||||
|
||||
def test_parse_json_content_accepts_markdown_fenced_json():
|
||||
raw = """```json
|
||||
{"entities": [], "claims": []}
|
||||
```"""
|
||||
|
||||
assert parse_json_content(raw) == {"entities": [], "claims": []}
|
||||
|
||||
|
||||
def test_parse_json_content_requires_structured_claim_schema():
|
||||
raw = {
|
||||
"entities": [
|
||||
{
|
||||
"entity_type": "Perfume",
|
||||
"name": "Neroli Summer",
|
||||
"attributes": {},
|
||||
"confidence": 0.9,
|
||||
"evidence_text": "Neroli Summer",
|
||||
}
|
||||
],
|
||||
"claims": [
|
||||
{
|
||||
"subject_name": "Neroli Summer",
|
||||
"subject_type": "Perfume",
|
||||
"predicate": "hasTopNote",
|
||||
"object_name": "Bergamot",
|
||||
"object_type": "FragranceNote",
|
||||
"object_value": None,
|
||||
"evidence_text": "Top notes: Bergamot",
|
||||
"evidence_summary": "explicit note listing",
|
||||
"confidence": 0.9,
|
||||
"confidence_reason": "directly stated",
|
||||
"source_zone": "product_description",
|
||||
}
|
||||
],
|
||||
}
|
||||
|
||||
parsed = parse_json_content(json.dumps(raw))
|
||||
|
||||
assert parsed["claims"][0]["source_zone"] == "product_description"
|
||||
|
||||
|
||||
def test_parse_json_content_rejects_claim_without_evidence():
|
||||
raw = {
|
||||
"entities": [],
|
||||
"claims": [
|
||||
{
|
||||
"subject_name": "Neroli Summer",
|
||||
"subject_type": "Perfume",
|
||||
"predicate": "hasTopNote",
|
||||
"object_name": "Bergamot",
|
||||
"object_type": "Note",
|
||||
"object_value": None,
|
||||
"evidence_text": "",
|
||||
"evidence_summary": "missing evidence",
|
||||
"confidence": 0.9,
|
||||
"confidence_reason": "bad",
|
||||
"source_zone": "product_description",
|
||||
}
|
||||
],
|
||||
}
|
||||
|
||||
try:
|
||||
parse_json_content(json.dumps(raw))
|
||||
except ValueError as exc:
|
||||
assert "schema_validation_failed" in str(exc)
|
||||
else:
|
||||
raise AssertionError("expected schema validation failure")
|
||||
|
||||
|
||||
def test_repository_merges_only_validated_typed_claims_to_graph():
|
||||
config = load_project_config("configs/perfume_subscription.yaml")
|
||||
engine = make_engine("sqlite:///:memory:")
|
||||
models.Base.metadata.create_all(engine)
|
||||
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
session = Session(engine)
|
||||
repo = KnowledgeRepository(session)
|
||||
project = repo.upsert_project(config)
|
||||
source = repo.get_source(project.id, "official_brand_site")
|
||||
page = repo.upsert_page(
|
||||
project.id,
|
||||
source.id,
|
||||
"https://example.com/product/detail.html",
|
||||
"Neroli Summer",
|
||||
200,
|
||||
"Neroli Summer\nTop notes: Bergamot",
|
||||
)
|
||||
bundle = ExtractionBundle(
|
||||
entities=[
|
||||
ExtractedEntity("Perfume", "Neroli Summer", evidence_text="Neroli Summer", confidence=0.9),
|
||||
ExtractedEntity("FragranceNote", "Bergamot", evidence_text="Top notes: Bergamot", confidence=0.9),
|
||||
],
|
||||
claims=[
|
||||
ExtractedClaim(
|
||||
"Neroli Summer",
|
||||
"Perfume",
|
||||
"hasTopNote",
|
||||
"Bergamot",
|
||||
"FragranceNote",
|
||||
evidence_text="Top notes: Bergamot",
|
||||
confidence=0.92,
|
||||
confidence_reason="directly stated",
|
||||
)
|
||||
],
|
||||
extractor_name="llm_json_extractor",
|
||||
provider="lm_studio",
|
||||
raw_output={"extraction_mode": "primary"},
|
||||
)
|
||||
context = ExtractionPageContext(
|
||||
url="https://example.com/product/detail.html",
|
||||
final_url="https://example.com/product/detail.html",
|
||||
title="Neroli Summer",
|
||||
page_type="ProductPage",
|
||||
clean_text="Neroli Summer\nTop notes: Bergamot",
|
||||
source_zones=[
|
||||
{
|
||||
"zone_type": "product_description",
|
||||
"selector": ".description",
|
||||
"text": "Top notes: Bergamot",
|
||||
"claim_allowed": True,
|
||||
}
|
||||
],
|
||||
)
|
||||
|
||||
claims = repo.save_extraction_bundle(project.id, source, page, attach_page_context(bundle, context), config)
|
||||
session.commit()
|
||||
|
||||
assert len(claims) == 1
|
||||
assert claims[0].status == "validated_claim"
|
||||
assert claims[0].metadata_json["graph_merge_status"] == "merged"
|
||||
assert session.query(models.Relation).count() == 1
|
||||
assert session.query(models.Entity).filter(models.Entity.entity_type == "Note").count() == 1
|
||||
session.close()
|
||||
208
tests/test_generic_ontology_core.py
Normal file
208
tests/test_generic_ontology_core.py
Normal file
@@ -0,0 +1,208 @@
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from crawler_platform.app.config.loader import ProjectConfig, SourceConfig, load_project_config
|
||||
from crawler_platform.app.core.database import models
|
||||
from crawler_platform.app.core.database.repository import KnowledgeRepository
|
||||
from crawler_platform.app.core.database.session import make_engine
|
||||
from crawler_platform.app.core.extractor.base import ExtractedClaim, ExtractedEntity, ExtractionBundle, ExtractionPageContext
|
||||
from crawler_platform.app.core.extractor.validation import attach_page_context
|
||||
from crawler_platform.app.core.ontology.gap_detector import KnowledgeGapDetector
|
||||
from crawler_platform.app.core.ontology.registry import OntologyRegistry
|
||||
from crawler_platform.app.core.research.graph_query import SemanticGraphQuery
|
||||
|
||||
|
||||
def make_repo():
|
||||
config = load_project_config("configs/perfume_subscription.yaml")
|
||||
engine = make_engine("sqlite:///:memory:")
|
||||
models.Base.metadata.create_all(engine)
|
||||
session = Session(engine)
|
||||
repo = KnowledgeRepository(session)
|
||||
project = repo.upsert_project(config)
|
||||
source = repo.get_source(project.id, "official_brand_site")
|
||||
return config, session, repo, project, source
|
||||
|
||||
|
||||
def save_validated_note_claim(repo, project, source, config):
|
||||
page = repo.upsert_page(
|
||||
project.id,
|
||||
source.id,
|
||||
"https://example.com/product/cotton",
|
||||
"Cotton Example",
|
||||
200,
|
||||
"Cotton Example\nTop notes: Bergamot",
|
||||
)
|
||||
bundle = ExtractionBundle(
|
||||
entities=[
|
||||
ExtractedEntity("Perfume", "Cotton Example", evidence_text="Cotton Example", confidence=0.95),
|
||||
ExtractedEntity("Note", "Bergamot", evidence_text="Top notes: Bergamot", confidence=0.95),
|
||||
],
|
||||
claims=[
|
||||
ExtractedClaim(
|
||||
"Cotton Example",
|
||||
"Perfume",
|
||||
"hasTopNote",
|
||||
"Bergamot",
|
||||
"Note",
|
||||
evidence_text="Top notes: Bergamot",
|
||||
confidence=0.96,
|
||||
confidence_reason="direct evidence",
|
||||
)
|
||||
],
|
||||
extractor_name="llm_json_extractor",
|
||||
provider="lm_studio",
|
||||
raw_output={"extraction_mode": "primary"},
|
||||
)
|
||||
context = ExtractionPageContext(
|
||||
url=page.url,
|
||||
final_url=page.url,
|
||||
title=page.title,
|
||||
page_type="ProductPage",
|
||||
clean_text=page.cleaned_text_summary,
|
||||
source_zones=[
|
||||
{
|
||||
"zone_type": "product_description",
|
||||
"selector": "main",
|
||||
"text": "Top notes: Bergamot",
|
||||
"claim_allowed": True,
|
||||
}
|
||||
],
|
||||
)
|
||||
return repo.save_extraction_bundle(project.id, source, page, attach_page_context(bundle, context), config)
|
||||
|
||||
|
||||
def test_project_upsert_seeds_schema_registry():
|
||||
_config, session, _repo, project, _source = make_repo()
|
||||
|
||||
registry = OntologyRegistry(session).registry_payload(project.id)
|
||||
|
||||
entity_names = {row["name"] for row in registry["entity_types"]}
|
||||
relation_names = {row["name"] for row in registry["relation_types"]}
|
||||
assert {"Entity", "Product", "Perfume"}.issubset(entity_names)
|
||||
assert {"relatedTo", "hasTopNote", "hasBrand"}.issubset(relation_names)
|
||||
session.close()
|
||||
|
||||
|
||||
def test_validated_claim_creates_separate_ontology_triple():
|
||||
config, session, repo, project, source = make_repo()
|
||||
|
||||
claims = save_validated_note_claim(repo, project, source, config)
|
||||
|
||||
triples = session.query(models.OntologyTriple).all()
|
||||
assert len(claims) == 1
|
||||
assert len(triples) == 1
|
||||
assert triples[0].predicate == "hasTopNote"
|
||||
assert triples[0].status == "merged"
|
||||
assert claims[0].metadata_json["ontology_triple_id"] == triples[0].id
|
||||
session.close()
|
||||
|
||||
|
||||
def test_schema_proposal_and_knowledge_gap_are_governance_records():
|
||||
_config, session, _repo, project, _source = make_repo()
|
||||
registry = OntologyRegistry(session)
|
||||
|
||||
proposal = registry.propose_schema_change(
|
||||
project.id,
|
||||
proposal_type="relation_type",
|
||||
name="influencedBy",
|
||||
reason="New cross-domain relation observed in evidence.",
|
||||
evidence="A research paper influenced a product strategy.",
|
||||
confidence=0.73,
|
||||
)
|
||||
gaps = KnowledgeGapDetector(session).list_open(project.id)
|
||||
|
||||
assert proposal.status == "pending_review"
|
||||
assert any(gap["gap_type"] == "schema_governance" and gap["target_name"] == "influencedBy" for gap in gaps)
|
||||
session.close()
|
||||
|
||||
|
||||
def test_generic_graph_query_uses_ontology_triples():
|
||||
config, session, repo, project, source = make_repo()
|
||||
save_validated_note_claim(repo, project, source, config)
|
||||
|
||||
rows = SemanticGraphQuery(session).relation_summary(project.id)
|
||||
|
||||
assert rows[0]["predicate"] == "hasTopNote"
|
||||
assert rows[0]["support_count"] == 1
|
||||
session.close()
|
||||
|
||||
|
||||
def test_config_driven_academic_relation_without_perfume_adapter():
|
||||
config = ProjectConfig(
|
||||
project_name="academic_demo",
|
||||
domain="academic",
|
||||
target_entities=["ResearchPaper", "Person"],
|
||||
fields=["title", "author"],
|
||||
sources=[SourceConfig(name="research_site", type="paper_index", trust_level=0.9)],
|
||||
ontology={
|
||||
"entity_types": ["ResearchPaper", "Person"],
|
||||
"predicates": ["authoredBy"],
|
||||
"relation_types": {
|
||||
"authoredBy": {
|
||||
"allowed_subject_types": ["ResearchPaper"],
|
||||
"allowed_object_types": ["Person"],
|
||||
"allowed_page_types": ["ArticlePage"],
|
||||
"allowed_source_zones": ["article_body"],
|
||||
"confidence_rules": {"min_confidence": 0.82},
|
||||
}
|
||||
},
|
||||
},
|
||||
)
|
||||
engine = make_engine("sqlite:///:memory:")
|
||||
models.Base.metadata.create_all(engine)
|
||||
session = Session(engine)
|
||||
repo = KnowledgeRepository(session)
|
||||
project = repo.upsert_project(config)
|
||||
source = repo.get_source(project.id, "research_site")
|
||||
page = repo.upsert_page(
|
||||
project.id,
|
||||
source.id,
|
||||
"https://example.org/papers/semantic-systems",
|
||||
"Semantic Systems",
|
||||
200,
|
||||
"Semantic Systems was authored by Ada Kim.",
|
||||
)
|
||||
bundle = ExtractionBundle(
|
||||
entities=[
|
||||
ExtractedEntity("ResearchPaper", "Semantic Systems", evidence_text="Semantic Systems", confidence=0.95),
|
||||
ExtractedEntity("Person", "Ada Kim", evidence_text="authored by Ada Kim", confidence=0.95),
|
||||
],
|
||||
claims=[
|
||||
ExtractedClaim(
|
||||
"Semantic Systems",
|
||||
"ResearchPaper",
|
||||
"authoredBy",
|
||||
"Ada Kim",
|
||||
"Person",
|
||||
evidence_text="Semantic Systems was authored by Ada Kim.",
|
||||
confidence=0.97,
|
||||
confidence_reason="directly stated",
|
||||
)
|
||||
],
|
||||
extractor_name="llm_json_extractor",
|
||||
provider="lm_studio",
|
||||
raw_output={"extraction_mode": "primary"},
|
||||
)
|
||||
context = ExtractionPageContext(
|
||||
url=page.url,
|
||||
final_url=page.url,
|
||||
title=page.title,
|
||||
page_type="ArticlePage",
|
||||
clean_text=page.cleaned_text_summary,
|
||||
source_zones=[
|
||||
{
|
||||
"zone_type": "article_body",
|
||||
"selector": "article",
|
||||
"text": "Semantic Systems was authored by Ada Kim.",
|
||||
"claim_allowed": True,
|
||||
}
|
||||
],
|
||||
)
|
||||
|
||||
claims = repo.save_extraction_bundle(project.id, source, page, attach_page_context(bundle, context), config)
|
||||
|
||||
triples = session.query(models.OntologyTriple).all()
|
||||
assert len(claims) == 1
|
||||
assert triples[0].predicate == "authoredBy"
|
||||
assert triples[0].status == "merged"
|
||||
assert session.query(models.OntologyRelationType).filter_by(name="authoredBy").one().domain == "academic"
|
||||
session.close()
|
||||
151
tests/test_research.py
Normal file
151
tests/test_research.py
Normal file
@@ -0,0 +1,151 @@
|
||||
from pathlib import Path
|
||||
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from crawler_platform.app.config.loader import load_project_config
|
||||
from crawler_platform.app.core.database import models
|
||||
from crawler_platform.app.core.database.repository import KnowledgeRepository
|
||||
from crawler_platform.app.core.database.session import make_engine
|
||||
from crawler_platform.app.core.extractor.base import ExtractedClaim, ExtractedEntity, ExtractionBundle, ExtractionPageContext
|
||||
from crawler_platform.app.core.extractor.validation import attach_page_context
|
||||
from crawler_platform.app.core.research.graph_query import SemanticGraphQuery
|
||||
from crawler_platform.app.core.research.graph_research_loop import GraphResearchLoop
|
||||
from crawler_platform.app.core.research.relevance_engine import RelevanceEngine
|
||||
from crawler_platform.app.domains.perfume.extractor import PerfumeRuleBasedExtractor
|
||||
|
||||
|
||||
def make_repo():
|
||||
config = load_project_config("configs/perfume_subscription.yaml")
|
||||
engine = make_engine("sqlite:///:memory:")
|
||||
models.Base.metadata.create_all(engine)
|
||||
session = Session(engine)
|
||||
repo = KnowledgeRepository(session)
|
||||
project = repo.upsert_project(config)
|
||||
source = repo.get_source(project.id, "official_brand_site")
|
||||
return config, session, repo, project, source
|
||||
|
||||
|
||||
def seed_validated_claim(repo, project, source, config):
|
||||
page = repo.upsert_page(
|
||||
project.id,
|
||||
source.id,
|
||||
"https://example.com/product/neroli",
|
||||
"Neroli Summer",
|
||||
200,
|
||||
"Neroli Summer\nTop notes: Bergamot",
|
||||
)
|
||||
bundle = ExtractionBundle(
|
||||
entities=[
|
||||
ExtractedEntity("Perfume", "Neroli Summer", evidence_text="Neroli Summer", confidence=0.9),
|
||||
ExtractedEntity("FragranceNote", "Bergamot", evidence_text="Top notes: Bergamot", confidence=0.9),
|
||||
],
|
||||
claims=[
|
||||
ExtractedClaim(
|
||||
"Neroli Summer",
|
||||
"Perfume",
|
||||
"hasTopNote",
|
||||
"Bergamot",
|
||||
"FragranceNote",
|
||||
evidence_text="Top notes: Bergamot",
|
||||
confidence=0.93,
|
||||
confidence_reason="directly stated",
|
||||
)
|
||||
],
|
||||
extractor_name="llm_json_extractor",
|
||||
provider="lm_studio",
|
||||
raw_output={"extraction_mode": "primary"},
|
||||
)
|
||||
context = ExtractionPageContext(
|
||||
url=page.url,
|
||||
final_url=page.url,
|
||||
title=page.title,
|
||||
page_type="ProductPage",
|
||||
clean_text=page.cleaned_text_summary,
|
||||
source_zones=[
|
||||
{
|
||||
"zone_type": "product_description",
|
||||
"selector": "main",
|
||||
"text": "Top notes: Bergamot",
|
||||
"claim_allowed": True,
|
||||
}
|
||||
],
|
||||
)
|
||||
repo.save_extraction_bundle(project.id, source, page, attach_page_context(bundle, context), config)
|
||||
|
||||
|
||||
def test_relevance_engine_prefers_graph_related_product_urls():
|
||||
config, session, repo, project, source = make_repo()
|
||||
seed_validated_claim(repo, project, source, config)
|
||||
|
||||
engine = RelevanceEngine(session)
|
||||
product = engine.score_url(
|
||||
project_id=project.id,
|
||||
url="https://example.com/product/neroli-bergamot",
|
||||
label="Neroli Summer Bergamot perfume",
|
||||
source_trust=source.trust_level,
|
||||
)
|
||||
login = engine.score_url(
|
||||
project_id=project.id,
|
||||
url="https://example.com/member/login.html",
|
||||
label="Login",
|
||||
source_trust=source.trust_level,
|
||||
)
|
||||
|
||||
assert product.score > login.score
|
||||
assert product.breakdown["entity_overlap"] > 0
|
||||
session.close()
|
||||
|
||||
|
||||
def test_semantic_graph_query_returns_trend_summary():
|
||||
config, session, repo, project, source = make_repo()
|
||||
seed_validated_claim(repo, project, source, config)
|
||||
|
||||
rows = SemanticGraphQuery(session).trend_summary(project.id)
|
||||
|
||||
assert rows[0]["name"] == "Bergamot"
|
||||
assert rows[0]["predicate"] == "hasTopNote" or rows[0]["support_count"] >= 1
|
||||
session.close()
|
||||
|
||||
|
||||
def test_graph_research_loop_records_session_memory():
|
||||
config, session, repo, project, _source = make_repo()
|
||||
fixture = Path("tests/fixtures/sample_perfume.html").resolve()
|
||||
loop = GraphResearchLoop(repo, PerfumeRuleBasedExtractor())
|
||||
|
||||
result = loop.run(
|
||||
project_config=config,
|
||||
source_name="official_brand_site",
|
||||
seed_url=str(fixture),
|
||||
goal="Fixture semantic exploration",
|
||||
max_depth=0,
|
||||
max_steps=1,
|
||||
min_relevance=0.0,
|
||||
)
|
||||
session.commit()
|
||||
|
||||
assert result.explored_count == 1
|
||||
assert result.history
|
||||
assert result.memory["visited_targets"]
|
||||
jobs = session.query(models.CrawlJob).all()
|
||||
assert any((job.metadata_json or {}).get("kind") == "research_session" for job in jobs)
|
||||
session.close()
|
||||
|
||||
|
||||
def test_graph_research_loop_can_start_from_knowledge_gaps():
|
||||
config, session, repo, _project, _source = make_repo()
|
||||
loop = GraphResearchLoop(repo, PerfumeRuleBasedExtractor())
|
||||
|
||||
result = loop.run(
|
||||
project_config=config,
|
||||
source_name="official_brand_site",
|
||||
goal="Gap-driven semantic exploration",
|
||||
max_depth=0,
|
||||
max_steps=1,
|
||||
max_branch=3,
|
||||
min_relevance=0.0,
|
||||
)
|
||||
|
||||
assert result.explored_count == 1
|
||||
assert result.history
|
||||
assert result.history[0]["item"]["target_type"] == "knowledge_gap"
|
||||
session.close()
|
||||
@@ -1,11 +1,58 @@
|
||||
from crawler_platform.app.core.crawler.site_crawler import classify_page, normalize_url
|
||||
from crawler_platform.app.core.crawler.site_crawler import (
|
||||
classify_page,
|
||||
is_failed_fetch_status,
|
||||
normalize_url,
|
||||
should_analyze_page,
|
||||
)
|
||||
from crawler_platform.app.core.crawler.fetchers import detect_crawl_status
|
||||
|
||||
|
||||
def test_classify_perfume_product_page():
|
||||
text = "Top notes: Bergamot, Neroli\nMiddle notes: Jasmine\nBase notes: Musk\nPrice $89"
|
||||
|
||||
assert classify_page("https://example.com/products/neroli", "Neroli Summer", text) == "product"
|
||||
assert classify_page("https://example.com/products/neroli", "Neroli Summer", text) == "ProductPage"
|
||||
|
||||
|
||||
def test_normalize_url_removes_fragment_and_trailing_slash():
|
||||
assert normalize_url("https://example.com/path/#details") == "https://example.com/path"
|
||||
|
||||
|
||||
def test_failed_fetch_status_detection():
|
||||
assert is_failed_fetch_status(None)
|
||||
assert is_failed_fetch_status(404)
|
||||
assert is_failed_fetch_status(500)
|
||||
assert not is_failed_fetch_status(200)
|
||||
assert not is_failed_fetch_status(302)
|
||||
|
||||
|
||||
def test_crawl_status_ignores_captcha_mentions_inside_scripts():
|
||||
html = "<html><body><h1>Product</h1><script>var path='recaptcha';</script></body></html>"
|
||||
|
||||
status, warnings = detect_crawl_status(200, html)
|
||||
|
||||
assert status == "success"
|
||||
assert warnings == []
|
||||
|
||||
|
||||
def test_should_analyze_page_supports_legacy_names():
|
||||
assert should_analyze_page("ProductPage", {"product"})
|
||||
assert not should_analyze_page("CommunityPage", {"product", "brand", "review"})
|
||||
|
||||
|
||||
def test_classify_board_page_before_content_analysis():
|
||||
assert classify_page("https://example.com/board/free/read.html", "Notice", "Price $89") == "NoticePage"
|
||||
|
||||
|
||||
def test_classify_promotion_homepage_before_product_analysis():
|
||||
text = "BLACK FRIDAY SALE\n회원가입 쿠폰\n무료배송 이벤트\n제품 보기"
|
||||
|
||||
assert classify_page("https://example.com/", "Brand", text) == "PromotionPage"
|
||||
|
||||
|
||||
def test_classify_product_list_and_search_as_non_detail_pages():
|
||||
text = "Price $89\nTop notes: Bergamot\nAdd to cart"
|
||||
|
||||
assert classify_page("https://example.com/product/list.html?cate_no=24", "Perfume", text) == "CategoryPage"
|
||||
assert classify_page("https://example.com/product/search.html?keyword=cotton", "Search", text) == "SearchPage"
|
||||
assert not should_analyze_page("CategoryPage", {"ProductPage", "BrandStoryPage", "ReviewPage"})
|
||||
assert not should_analyze_page("SearchPage", {"ProductPage", "BrandStoryPage", "ReviewPage"})
|
||||
|
||||
Reference in New Issue
Block a user