2026-05-12 19:40:31 +09:00
|
|
|
from crawler_platform.app.core.crawler.site_crawler import (
|
|
|
|
|
classify_page,
|
|
|
|
|
is_failed_fetch_status,
|
|
|
|
|
normalize_url,
|
|
|
|
|
should_analyze_page,
|
|
|
|
|
)
|
|
|
|
|
from crawler_platform.app.core.crawler.fetchers import detect_crawl_status
|
2026-05-11 13:02:11 +09:00
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_classify_perfume_product_page():
|
|
|
|
|
text = "Top notes: Bergamot, Neroli\nMiddle notes: Jasmine\nBase notes: Musk\nPrice $89"
|
|
|
|
|
|
2026-05-12 19:40:31 +09:00
|
|
|
assert classify_page("https://example.com/products/neroli", "Neroli Summer", text) == "ProductPage"
|
2026-05-11 13:02:11 +09:00
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_normalize_url_removes_fragment_and_trailing_slash():
|
|
|
|
|
assert normalize_url("https://example.com/path/#details") == "https://example.com/path"
|
2026-05-12 19:40:31 +09:00
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_failed_fetch_status_detection():
|
|
|
|
|
assert is_failed_fetch_status(None)
|
|
|
|
|
assert is_failed_fetch_status(404)
|
|
|
|
|
assert is_failed_fetch_status(500)
|
|
|
|
|
assert not is_failed_fetch_status(200)
|
|
|
|
|
assert not is_failed_fetch_status(302)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_crawl_status_ignores_captcha_mentions_inside_scripts():
|
|
|
|
|
html = "<html><body><h1>Product</h1><script>var path='recaptcha';</script></body></html>"
|
|
|
|
|
|
|
|
|
|
status, warnings = detect_crawl_status(200, html)
|
|
|
|
|
|
|
|
|
|
assert status == "success"
|
|
|
|
|
assert warnings == []
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_should_analyze_page_supports_legacy_names():
|
|
|
|
|
assert should_analyze_page("ProductPage", {"product"})
|
|
|
|
|
assert not should_analyze_page("CommunityPage", {"product", "brand", "review"})
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_classify_board_page_before_content_analysis():
|
|
|
|
|
assert classify_page("https://example.com/board/free/read.html", "Notice", "Price $89") == "NoticePage"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_classify_promotion_homepage_before_product_analysis():
|
|
|
|
|
text = "BLACK FRIDAY SALE\n회원가입 쿠폰\n무료배송 이벤트\n제품 보기"
|
|
|
|
|
|
|
|
|
|
assert classify_page("https://example.com/", "Brand", text) == "PromotionPage"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_classify_product_list_and_search_as_non_detail_pages():
|
|
|
|
|
text = "Price $89\nTop notes: Bergamot\nAdd to cart"
|
|
|
|
|
|
|
|
|
|
assert classify_page("https://example.com/product/list.html?cate_no=24", "Perfume", text) == "CategoryPage"
|
|
|
|
|
assert classify_page("https://example.com/product/search.html?keyword=cotton", "Search", text) == "SearchPage"
|
|
|
|
|
assert not should_analyze_page("CategoryPage", {"ProductPage", "BrandStoryPage", "ReviewPage"})
|
|
|
|
|
assert not should_analyze_page("SearchPage", {"ProductPage", "BrandStoryPage", "ReviewPage"})
|
