ontology

tests/test_content_quality.py

@@ -12,7 +12,9 @@ from crawler_platform.app.core.extractor.base import (
     ExtractionBundle,
 )
 from crawler_platform.app.core.extractor.ai_provider import parse_json_content
+from crawler_platform.app.core.extractor.ai_provider import LLMJsonExtractor
 from crawler_platform.app.core.extractor.validation import attach_page_context, validate_extraction_bundle
+from crawler_platform.app.domains.perfume.extractor import PerfumeRuleBasedExtractor
 
 
 def test_cleaner_prefers_product_content_and_removes_boilerplate():
@@ -175,7 +177,7 @@ def test_validation_rejects_product_detail_claims_on_category_page_context():
     assert result.rejected_claims[0]["reason"] == "predicate hasPrice is not allowed for page type CategoryPage"
 
 
-def test_repository_logs_rule_candidates_without_persisting_ontology_rows():
+def test_repository_persists_rule_candidates_without_merging_graph_relations():
     config = load_project_config("configs/perfume_subscription.yaml")
     engine = make_engine("sqlite:///:memory:")
     models.Base.metadata.create_all(engine)
@@ -194,23 +196,41 @@ def test_repository_logs_rule_candidates_without_persisting_ontology_rows():
                 "Neroli Summer",
                 "Perfume",
                 "hasTopNote",
-                "Bergamot",
-                "Note",
-                evidence_text="Top notes: Bergamot",
-            )
+                    "Bergamot",
+                    "Note",
+                    evidence_text="Top notes: Bergamot",
+                    confidence=0.9,
+                )
         ],
         extractor_name="perfume_rule_based",
         provider="rule_based",
         raw_output={},
     )
+    context = ExtractionPageContext(
+        url="https://example.com/product",
+        final_url="https://example.com/product",
+        title="Product",
+        page_type="ProductPage",
+        clean_text="Neroli Summer\nTop notes: Bergamot",
+        source_zones=[
+            {
+                "zone_type": "product_description",
+                "selector": ".description",
+                "text": "Top notes: Bergamot",
+                "claim_allowed": True,
+            }
+        ],
+    )
 
-    claims = repo.save_extraction_bundle(project.id, source, page, bundle, config)
+    claims = repo.save_extraction_bundle(project.id, source, page, attach_page_context(bundle, context), config)
     session.commit()
 
-    assert claims == []
-    assert session.query(models.Claim).count() == 0
-    assert session.query(models.Entity).count() == 0
+    assert len(claims) == 1
+    assert claims[0].status == "rule_candidate"
+    assert session.query(models.Claim).count() == 1
+    assert session.query(models.Entity).count() == 2
     assert session.query(models.ExtractionLog).count() == 1
+    assert session.query(models.Relation).count() == 0
     session.close()
 
 
@@ -222,6 +242,93 @@ def test_parse_json_content_accepts_markdown_fenced_json():
     assert parse_json_content(raw) == {"entities": [], "claims": []}
 
 
+def test_lm_studio_extractor_merges_rule_claims_even_when_ai_claim_is_invalid():
+    config = load_project_config("configs/perfume_subscription.yaml")
+
+    class StubExtractor(LLMJsonExtractor):
+        def complete_json(self, page_text, project_config, compact=False, context=None):
+            return {
+                "entities": [
+                    {
+                        "entity_type": "Brand",
+                        "name": "912",
+                        "attributes": {},
+                        "confidence": 0.9,
+                        "evidence_text": "912",
+                    }
+                ],
+                "claims": [
+                    {
+                        "subject_name": "912",
+                        "subject_type": "Brand",
+                        "predicate": "hasBrand",
+                        "object_name": "912",
+                        "object_type": "Brand",
+                        "object_value": None,
+                        "evidence_text": "912",
+                        "evidence_summary": "bad self-brand claim",
+                        "confidence": 0.9,
+                        "confidence_reason": "stub",
+                        "source_zone": "product_title",
+                    }
+                ],
+            }
+
+    text = "Neroli Summer\nTop notes: Bergamot\nPrice $89"
+    bundle = StubExtractor("perfume", "lm_studio", model="stub").extract(text, config)
+
+    assert any(claim.predicate == "hasTopNote" for claim in bundle.claims)
+
+
+def test_the912_product_rule_fallback_uses_product_title_brand_and_price():
+    config = load_project_config("configs/perfume_subscription.yaml")
+    text = """
+    새로운 향수의 시작, 클론 향수
+    [2+1 기획] 912 클론 니치향수 모음 40ml
+    4.8
+    114,414
+    일반 구매가격
+    72,000원
+    할인 적용금액
+    37,000원
+    최종 구매금액
+    35,000원
+    현재 위치
+    전체 상품
+    니치향수
+    오드 퍼퓸
+    """
+    bundle = PerfumeRuleBasedExtractor().extract(text, config)
+    context = ExtractionPageContext(
+        url="https://the912.co.kr/product/detail.html?product_no=513",
+        final_url="https://the912.co.kr/product/detail.html?product_no=513",
+        title="[2+1 기획] 912 클론 니치향수 모음 40ml - 912",
+        page_type="ProductPage",
+        clean_text=text,
+        source_zones=[
+            {
+                "zone_type": "product_title",
+                "selector": "h1",
+                "text": "[2+1 기획] 912 클론 니치향수 모음 40ml",
+                "claim_allowed": True,
+            },
+            {
+                "zone_type": "product_summary",
+                "selector": ".detail",
+                "text": "일반 구매가격\n72,000원\n할인 적용금액\n37,000원\n최종 구매금액\n35,000원",
+                "claim_allowed": True,
+            },
+        ],
+    )
+
+    result = validate_extraction_bundle(attach_page_context(bundle, context), config)
+    predicates = {claim.predicate for claim in result.bundle.claims}
+    perfume_names = {entity.name for entity in result.bundle.entities if entity.entity_type == "Perfume"}
+
+    assert "[2+1 기획] 912 클론 니치향수 모음 40ml" in perfume_names
+    assert {"hasBrand", "hasPrice"} <= predicates
+
+
 def test_parse_json_content_requires_structured_claim_schema():
     raw = {
         "entities": [

tests/test_site_crawler.py

@@ -4,7 +4,9 @@ from crawler_platform.app.core.crawler.site_crawler import (
     normalize_url,
     should_analyze_page,
 )
-from crawler_platform.app.core.crawler.fetchers import detect_crawl_status
+from crawler_platform.app.core.crawler.discovery import discover_links, normalize_cafe24_product_url
+from crawler_platform.app.core.crawler.fetchers import FallbackFetcher, FetchResult, RobotsPolicy, detect_crawl_status
+from crawler_platform.app.api.routes import CrawlRequest
 
 
 def test_classify_perfume_product_page():
@@ -39,6 +41,67 @@ def test_should_analyze_page_supports_legacy_names():
     assert not should_analyze_page("CommunityPage", {"product", "brand", "review"})
 
 
+def test_robots_policy_disabled_skips_check():
+    decision = RobotsPolicy().check("https://example.com/products/1", respect_robots_txt=False)
+
+    assert decision.allowed
+    assert decision.status == "disabled"
+    assert not decision.checked
+
+
+def test_crawl_request_defaults_to_no_robots_check():
+    request = CrawlRequest(
+        config_path="configs/perfume_subscription.yaml",
+        source_name="official_brand_site",
+        url="https://example.com",
+    )
+
+    assert request.check_robots_txt is False
+
+
+def test_robots_policy_allows_when_robots_unavailable(monkeypatch):
+    class UnavailableRobotsParser:
+        def set_url(self, url):
+            self.url = url
+
+        def read(self):
+            raise OSError("network unavailable")
+
+    monkeypatch.setattr(
+        "crawler_platform.app.core.crawler.fetchers.RobotFileParser",
+        UnavailableRobotsParser,
+    )
+
+    decision = RobotsPolicy().check("https://example.com/products/1")
+
+    assert decision.allowed
+    assert decision.status == "unavailable"
+    assert "allowing crawl" in decision.reason
+
+
+def test_robots_policy_reports_block_reason(monkeypatch):
+    class BlockingRobotsParser:
+        def set_url(self, url):
+            self.url = url
+
+        def read(self):
+            return None
+
+        def can_fetch(self, user_agent, url):
+            return False
+
+    monkeypatch.setattr(
+        "crawler_platform.app.core.crawler.fetchers.RobotFileParser",
+        BlockingRobotsParser,
+    )
+
+    decision = RobotsPolicy().check("https://example.com/private")
+
+    assert not decision.allowed
+    assert decision.status == "blocked"
+    assert "blocks crawling" in decision.reason
+
+
 def test_classify_board_page_before_content_analysis():
     assert classify_page("https://example.com/board/free/read.html", "Notice", "Price $89") == "NoticePage"
 
@@ -56,3 +119,44 @@ def test_classify_product_list_and_search_as_non_detail_pages():
     assert classify_page("https://example.com/product/search.html?keyword=cotton", "Search", text) == "SearchPage"
     assert not should_analyze_page("CategoryPage", {"ProductPage", "BrandStoryPage", "ReviewPage"})
     assert not should_analyze_page("SearchPage", {"ProductPage", "BrandStoryPage", "ReviewPage"})
+
+
+def test_cafe24_product_detail_with_category_segment_is_product_page():
+    url = "https://the912.co.kr/product/21-기획-912-클론-니치향수-모음-40ml/513/category/1/display/2/"
+
+    assert classify_page(url, "912 clone perfume", "") == "ProductPage"
+
+
+def test_discovery_skips_utility_pages_seen_on_the912():
+    html = """
+    <a href="/myshop/wish_list.html">wish</a>
+    <a href="/board/faq/list.html?board_no=3">faq</a>
+    <a href="/event/list.html?cate_no=42">event</a>
+    <a href="/product/search.html?keyword=">search</a>
+    <a href="/product/sample/123/category/1/display/2/">product</a>
+    """
+
+    links = discover_links(html, "https://the912.co.kr", limit=10)
+
+    assert [link.url for link in links] == ["https://the912.co.kr/product/detail.html?product_no=123"]
+
+
+def test_cafe24_product_urls_are_canonicalized_for_dedupe():
+    url = "https://the912.co.kr/product/21-기획-912-클론-니치향수-모음-40ml/513/category/1/display/2/?icid=x"
+
+    assert normalize_cafe24_product_url(url) == "https://the912.co.kr/product/detail.html?product_no=513"
+
+
+def test_fallback_fetcher_continues_when_primary_fails():
+    class FailingFetcher:
+        def fetch(self, url):
+            raise PermissionError("[WinError 5] access denied")
+
+    class WorkingFetcher:
+        def fetch(self, url):
+            return FetchResult(url=url, status_code=200, html="<html><body>ok</body></html>")
+
+    result = FallbackFetcher(FailingFetcher(), WorkingFetcher(), fallback_label="requests").fetch("https://example.com")
+
+    assert result.status_code == 200
+    assert any("primary fetcher failed" in warning for warning in result.warnings)