from crawler_platform.app.core.crawler.site_crawler import (
classify_page,
is_failed_fetch_status,
normalize_url,
should_analyze_page,
)
from crawler_platform.app.core.crawler.fetchers import detect_crawl_status
def test_classify_perfume_product_page():
text = "Top notes: Bergamot, Neroli\nMiddle notes: Jasmine\nBase notes: Musk\nPrice $89"
assert classify_page("https://example.com/products/neroli", "Neroli Summer", text) == "ProductPage"
def test_normalize_url_removes_fragment_and_trailing_slash():
assert normalize_url("https://example.com/path/#details") == "https://example.com/path"
def test_failed_fetch_status_detection():
assert is_failed_fetch_status(None)
assert is_failed_fetch_status(404)
assert is_failed_fetch_status(500)
assert not is_failed_fetch_status(200)
assert not is_failed_fetch_status(302)
def test_crawl_status_ignores_captcha_mentions_inside_scripts():
html = "Product
"
status, warnings = detect_crawl_status(200, html)
assert status == "success"
assert warnings == []
def test_should_analyze_page_supports_legacy_names():
assert should_analyze_page("ProductPage", {"product"})
assert not should_analyze_page("CommunityPage", {"product", "brand", "review"})
def test_classify_board_page_before_content_analysis():
assert classify_page("https://example.com/board/free/read.html", "Notice", "Price $89") == "NoticePage"
def test_classify_promotion_homepage_before_product_analysis():
text = "BLACK FRIDAY SALE\n회원가입 쿠폰\n무료배송 이벤트\n제품 보기"
assert classify_page("https://example.com/", "Brand", text) == "PromotionPage"
def test_classify_product_list_and_search_as_non_detail_pages():
text = "Price $89\nTop notes: Bergamot\nAdd to cart"
assert classify_page("https://example.com/product/list.html?cate_no=24", "Perfume", text) == "CategoryPage"
assert classify_page("https://example.com/product/search.html?keyword=cotton", "Search", text) == "SearchPage"
assert not should_analyze_page("CategoryPage", {"ProductPage", "BrandStoryPage", "ReviewPage"})
assert not should_analyze_page("SearchPage", {"ProductPage", "BrandStoryPage", "ReviewPage"})