from crawler_platform.app.core.crawler.site_crawler import (
classify_page,
is_failed_fetch_status,
normalize_url,
should_analyze_page,
)
from crawler_platform.app.core.crawler.discovery import discover_links, normalize_cafe24_product_url
from crawler_platform.app.core.crawler.fetchers import FallbackFetcher, FetchResult, RobotsPolicy, detect_crawl_status
from crawler_platform.app.api.routes import CrawlRequest
def test_classify_perfume_product_page():
text = "Top notes: Bergamot, Neroli\nMiddle notes: Jasmine\nBase notes: Musk\nPrice $89"
assert classify_page("https://example.com/products/neroli", "Neroli Summer", text) == "ProductPage"
def test_normalize_url_removes_fragment_and_trailing_slash():
assert normalize_url("https://example.com/path/#details") == "https://example.com/path"
def test_failed_fetch_status_detection():
assert is_failed_fetch_status(None)
assert is_failed_fetch_status(404)
assert is_failed_fetch_status(500)
assert not is_failed_fetch_status(200)
assert not is_failed_fetch_status(302)
def test_crawl_status_ignores_captcha_mentions_inside_scripts():
html = "Product
"
status, warnings = detect_crawl_status(200, html)
assert status == "success"
assert warnings == []
def test_should_analyze_page_supports_legacy_names():
assert should_analyze_page("ProductPage", {"product"})
assert not should_analyze_page("CommunityPage", {"product", "brand", "review"})
def test_robots_policy_disabled_skips_check():
decision = RobotsPolicy().check("https://example.com/products/1", respect_robots_txt=False)
assert decision.allowed
assert decision.status == "disabled"
assert not decision.checked
def test_crawl_request_defaults_to_no_robots_check():
request = CrawlRequest(
config_path="configs/perfume_subscription.yaml",
source_name="official_brand_site",
url="https://example.com",
)
assert request.check_robots_txt is False
def test_robots_policy_allows_when_robots_unavailable(monkeypatch):
class UnavailableRobotsParser:
def set_url(self, url):
self.url = url
def read(self):
raise OSError("network unavailable")
monkeypatch.setattr(
"crawler_platform.app.core.crawler.fetchers.RobotFileParser",
UnavailableRobotsParser,
)
decision = RobotsPolicy().check("https://example.com/products/1")
assert decision.allowed
assert decision.status == "unavailable"
assert "allowing crawl" in decision.reason
def test_robots_policy_reports_block_reason(monkeypatch):
class BlockingRobotsParser:
def set_url(self, url):
self.url = url
def read(self):
return None
def can_fetch(self, user_agent, url):
return False
monkeypatch.setattr(
"crawler_platform.app.core.crawler.fetchers.RobotFileParser",
BlockingRobotsParser,
)
decision = RobotsPolicy().check("https://example.com/private")
assert not decision.allowed
assert decision.status == "blocked"
assert "blocks crawling" in decision.reason
def test_classify_board_page_before_content_analysis():
assert classify_page("https://example.com/board/free/read.html", "Notice", "Price $89") == "NoticePage"
def test_classify_promotion_homepage_before_product_analysis():
text = "BLACK FRIDAY SALE\n회원가입 쿠폰\n무료배송 이벤트\n제품 보기"
assert classify_page("https://example.com/", "Brand", text) == "PromotionPage"
def test_classify_product_list_and_search_as_non_detail_pages():
text = "Price $89\nTop notes: Bergamot\nAdd to cart"
assert classify_page("https://example.com/product/list.html?cate_no=24", "Perfume", text) == "CategoryPage"
assert classify_page("https://example.com/product/search.html?keyword=cotton", "Search", text) == "SearchPage"
assert not should_analyze_page("CategoryPage", {"ProductPage", "BrandStoryPage", "ReviewPage"})
assert not should_analyze_page("SearchPage", {"ProductPage", "BrandStoryPage", "ReviewPage"})
def test_cafe24_product_detail_with_category_segment_is_product_page():
url = "https://the912.co.kr/product/21-기획-912-클론-니치향수-모음-40ml/513/category/1/display/2/"
assert classify_page(url, "912 clone perfume", "") == "ProductPage"
def test_discovery_skips_utility_pages_seen_on_the912():
html = """
wish
faq
event
search
product
"""
links = discover_links(html, "https://the912.co.kr", limit=10)
assert [link.url for link in links] == ["https://the912.co.kr/product/detail.html?product_no=123"]
def test_cafe24_product_urls_are_canonicalized_for_dedupe():
url = "https://the912.co.kr/product/21-기획-912-클론-니치향수-모음-40ml/513/category/1/display/2/?icid=x"
assert normalize_cafe24_product_url(url) == "https://the912.co.kr/product/detail.html?product_no=513"
def test_fallback_fetcher_continues_when_primary_fails():
class FailingFetcher:
def fetch(self, url):
raise PermissionError("[WinError 5] access denied")
class WorkingFetcher:
def fetch(self, url):
return FetchResult(url=url, status_code=200, html="ok")
result = FallbackFetcher(FailingFetcher(), WorkingFetcher(), fallback_label="requests").fetch("https://example.com")
assert result.status_code == 200
assert any("primary fetcher failed" in warning for warning in result.warnings)