ontology

tests/test_site_crawler.py

@@ -4,7 +4,9 @@ from crawler_platform.app.core.crawler.site_crawler import (
     normalize_url,
     should_analyze_page,
 )
-from crawler_platform.app.core.crawler.fetchers import detect_crawl_status
+from crawler_platform.app.core.crawler.discovery import discover_links, normalize_cafe24_product_url
+from crawler_platform.app.core.crawler.fetchers import FallbackFetcher, FetchResult, RobotsPolicy, detect_crawl_status
+from crawler_platform.app.api.routes import CrawlRequest
 
 
 def test_classify_perfume_product_page():
@@ -39,6 +41,67 @@ def test_should_analyze_page_supports_legacy_names():
     assert not should_analyze_page("CommunityPage", {"product", "brand", "review"})
 
 
+def test_robots_policy_disabled_skips_check():
+    decision = RobotsPolicy().check("https://example.com/products/1", respect_robots_txt=False)
+
+    assert decision.allowed
+    assert decision.status == "disabled"
+    assert not decision.checked
+
+
+def test_crawl_request_defaults_to_no_robots_check():
+    request = CrawlRequest(
+        config_path="configs/perfume_subscription.yaml",
+        source_name="official_brand_site",
+        url="https://example.com",
+    )
+
+    assert request.check_robots_txt is False
+
+
+def test_robots_policy_allows_when_robots_unavailable(monkeypatch):
+    class UnavailableRobotsParser:
+        def set_url(self, url):
+            self.url = url
+
+        def read(self):
+            raise OSError("network unavailable")
+
+    monkeypatch.setattr(
+        "crawler_platform.app.core.crawler.fetchers.RobotFileParser",
+        UnavailableRobotsParser,
+    )
+
+    decision = RobotsPolicy().check("https://example.com/products/1")
+
+    assert decision.allowed
+    assert decision.status == "unavailable"
+    assert "allowing crawl" in decision.reason
+
+
+def test_robots_policy_reports_block_reason(monkeypatch):
+    class BlockingRobotsParser:
+        def set_url(self, url):
+            self.url = url
+
+        def read(self):
+            return None
+
+        def can_fetch(self, user_agent, url):
+            return False
+
+    monkeypatch.setattr(
+        "crawler_platform.app.core.crawler.fetchers.RobotFileParser",
+        BlockingRobotsParser,
+    )
+
+    decision = RobotsPolicy().check("https://example.com/private")
+
+    assert not decision.allowed
+    assert decision.status == "blocked"
+    assert "blocks crawling" in decision.reason
+
+
 def test_classify_board_page_before_content_analysis():
     assert classify_page("https://example.com/board/free/read.html", "Notice", "Price $89") == "NoticePage"
 
@@ -56,3 +119,44 @@ def test_classify_product_list_and_search_as_non_detail_pages():
     assert classify_page("https://example.com/product/search.html?keyword=cotton", "Search", text) == "SearchPage"
     assert not should_analyze_page("CategoryPage", {"ProductPage", "BrandStoryPage", "ReviewPage"})
     assert not should_analyze_page("SearchPage", {"ProductPage", "BrandStoryPage", "ReviewPage"})
+
+
+def test_cafe24_product_detail_with_category_segment_is_product_page():
+    url = "https://the912.co.kr/product/21-기획-912-클론-니치향수-모음-40ml/513/category/1/display/2/"
+
+    assert classify_page(url, "912 clone perfume", "") == "ProductPage"
+
+
+def test_discovery_skips_utility_pages_seen_on_the912():
+    html = """
+    <a href="/myshop/wish_list.html">wish</a>
+    <a href="/board/faq/list.html?board_no=3">faq</a>
+    <a href="/event/list.html?cate_no=42">event</a>
+    <a href="/product/search.html?keyword=">search</a>
+    <a href="/product/sample/123/category/1/display/2/">product</a>
+    """
+
+    links = discover_links(html, "https://the912.co.kr", limit=10)
+
+    assert [link.url for link in links] == ["https://the912.co.kr/product/detail.html?product_no=123"]
+
+
+def test_cafe24_product_urls_are_canonicalized_for_dedupe():
+    url = "https://the912.co.kr/product/21-기획-912-클론-니치향수-모음-40ml/513/category/1/display/2/?icid=x"
+
+    assert normalize_cafe24_product_url(url) == "https://the912.co.kr/product/detail.html?product_no=513"
+
+
+def test_fallback_fetcher_continues_when_primary_fails():
+    class FailingFetcher:
+        def fetch(self, url):
+            raise PermissionError("[WinError 5] access denied")
+
+    class WorkingFetcher:
+        def fetch(self, url):
+            return FetchResult(url=url, status_code=200, html="<html><body>ok</body></html>")
+
+    result = FallbackFetcher(FailingFetcher(), WorkingFetcher(), fallback_label="requests").fetch("https://example.com")
+
+    assert result.status_code == 200
+    assert any("primary fetcher failed" in warning for warning in result.warnings)