ontology

crawler_platform/app/core/crawler/discovery.py

@@ -1,7 +1,7 @@
 from __future__ import annotations
 
 from dataclasses import dataclass
-from urllib.parse import parse_qs, unquote, urljoin, urlparse
+from urllib.parse import parse_qs, urlencode, unquote, urljoin, urlparse, urlunparse
 
 from bs4 import BeautifulSoup
 
@@ -21,7 +21,7 @@ def discover_links(html: str, base_url: str, limit: int = 30) -> list[Discovered
         raw_href = anchor.get("href", "")
         if should_skip_raw_href(raw_href):
             continue
-        url = normalize_search_redirect(urljoin(base_url, raw_href))
+        url = normalize_cafe24_product_url(normalize_search_redirect(urljoin(base_url, raw_href)))
         if not url or url in seen or not url.startswith(("http://", "https://")) or should_skip_url(url):
             continue
         seen.add(url)
@@ -58,12 +58,18 @@ def should_skip_url(url: str) -> bool:
     skip_path_tokens = [
         "/member/",
         "/order/",
+        "/myshop/",
+        "/event/list",
         "/exec/front/newcoupon/",
+        "/board/free/list",
+        "/board/faq/list",
         "/board/free/modify",
         "/board/free/reply",
     ]
     if any(token in path for token in skip_path_tokens):
         return True
+    if path.endswith("/product/search.html"):
+        return True
     if "facebook.com/" in path or "instagram.com/" in path:
         return True
     if "coupon_no=" in query:
@@ -82,6 +88,33 @@ def normalize_search_redirect(url: str) -> str:
     return url
 
 
+def normalize_cafe24_product_url(url: str) -> str:
+    parsed = urlparse(url)
+    path = unquote(parsed.path)
+    parts = [part for part in path.strip("/").split("/") if part]
+    query = parse_qs(parsed.query)
+
+    if path.endswith("/product/detail.html") and query.get("product_no"):
+        return urlunparse(
+            parsed._replace(
+                query=urlencode({"product_no": query["product_no"][0]}),
+                fragment="",
+            )
+        )
+
+    if parts and parts[0] == "product":
+        product_no = next((part for part in parts[1:] if part.isdigit()), None)
+        if product_no:
+            return urlunparse(
+                parsed._replace(
+                    path="/product/detail.html",
+                    query=urlencode({"product_no": product_no}),
+                    fragment="",
+                )
+            )
+    return url
+
+
 def classify_url(url: str) -> str:
     host = urlparse(url).netloc.lower()
     if "smartstore.naver.com" in host or "brand.naver.com" in host:

crawler_platform/app/core/crawler/fetchers.py

@@ -42,6 +42,16 @@ class FetchResult:
         return self.rendered_html or self.raw_html or self.html
 
 
+@dataclass(slots=True)
+class RobotsDecision:
+    allowed: bool
+    checked: bool
+    status: str
+    reason: str
+    robots_url: str | None = None
+    user_agent: str = DEFAULT_USER_AGENT
+
+
 class RateLimiter:
     def __init__(self, per_minute: int = 30):
         self.delay = 60 / max(per_minute, 1)
@@ -60,11 +70,26 @@ class RobotsPolicy:
         self._cache: dict[str, RobotFileParser] = {}
 
     def allowed(self, url: str, respect_robots_txt: bool = True) -> bool:
+        return self.check(url, respect_robots_txt).allowed
+
+    def check(self, url: str, respect_robots_txt: bool = True) -> RobotsDecision:
         if not respect_robots_txt:
-            return True
+            return RobotsDecision(
+                allowed=True,
+                checked=False,
+                status="disabled",
+                reason="robots.txt check disabled by source/request config",
+                user_agent=self.user_agent,
+            )
         parsed = urlparse(url)
         if parsed.scheme in {"", "file"}:
-            return True
+            return RobotsDecision(
+                allowed=True,
+                checked=False,
+                status="local",
+                reason="robots.txt is not applicable to local or scheme-less URLs",
+                user_agent=self.user_agent,
+            )
         robots_url = f"{parsed.scheme}://{parsed.netloc}/robots.txt"
         parser = self._cache.get(robots_url)
         if parser is None:
@@ -72,10 +97,29 @@ class RobotsPolicy:
             parser.set_url(robots_url)
             try:
                 parser.read()
-            except Exception:
-                return False
+            except Exception as exc:
+                return RobotsDecision(
+                    allowed=True,
+                    checked=True,
+                    status="unavailable",
+                    reason=f"robots.txt unavailable ({exc.__class__.__name__}); allowing crawl",
+                    robots_url=robots_url,
+                    user_agent=self.user_agent,
+                )
             self._cache[robots_url] = parser
-        return parser.can_fetch(self.user_agent, url)
+        allowed = parser.can_fetch(self.user_agent, url)
+        return RobotsDecision(
+            allowed=allowed,
+            checked=True,
+            status="allowed" if allowed else "blocked",
+            reason=(
+                "robots.txt allows crawling"
+                if allowed
+                else f"robots.txt blocks crawling for user-agent {self.user_agent}"
+            ),
+            robots_url=robots_url,
+            user_agent=self.user_agent,
+        )
 
 
 class BaseFetcher:
@@ -83,6 +127,23 @@ class BaseFetcher:
         raise NotImplementedError
 
 
+class FallbackFetcher(BaseFetcher):
+    def __init__(self, primary: BaseFetcher, fallback: BaseFetcher, fallback_label: str = "fallback"):
+        self.primary = primary
+        self.fallback = fallback
+        self.fallback_label = fallback_label
+
+    def fetch(self, url: str) -> FetchResult:
+        try:
+            return self.primary.fetch(url)
+        except Exception as exc:
+            result = self.fallback.fetch(url)
+            result.warnings.append(
+                f"primary fetcher failed ({exc.__class__.__name__}: {exc}); used {self.fallback_label}"
+            )
+            return result
+
+
 class RequestsFetcher(BaseFetcher):
     def __init__(
         self,
@@ -170,7 +231,11 @@ class PlaywrightFetcher(BaseFetcher):
 
 def make_fetcher(kind: str, rate_limit_per_minute: int = 30) -> BaseFetcher:
     if kind in {"playwright", "browser"}:
-        return PlaywrightFetcher()
+        return FallbackFetcher(
+            PlaywrightFetcher(),
+            RequestsFetcher(rate_limit_per_minute=rate_limit_per_minute),
+            fallback_label="requests",
+        )
     return RequestsFetcher(rate_limit_per_minute=rate_limit_per_minute)
 
 

crawler_platform/app/core/crawler/page_classifier.py

@@ -49,7 +49,7 @@ def classify_page(
     if any(token in path for token in ["shopinfo", "company", "about", "brand-story", "brand_story"]):
         return "BrandStoryPage"
     if any(token in path for token in ["product/detail", "/product/", "/products/", "/goods/", "/item/"]):
-        if _looks_like_category_path(path, combined):
+        if _looks_like_category_path(path, combined) and not _looks_like_product_detail_path(path):
             return "CategoryPage"
         return "ProductPage"
     if any(token in path for token in ["category", "/collections", "/collection", "/shop/", "/list"]):
@@ -142,6 +142,14 @@ def _has_product_detail_signal(text: str) -> bool:
     return note_count >= 1 or commerce_count >= 2
 
 
+def _looks_like_product_detail_path(path: str) -> bool:
+    parts = [part for part in path.strip("/").split("/") if part]
+    if "product" not in parts:
+        return False
+    product_index = parts.index("product")
+    return any(part.isdigit() for part in parts[product_index + 1 :])
+
+
 def _looks_like_category_path(path: str, text: str) -> bool:
     category_tokens = ["category", "cate_no", "display_group", "sort_method"]
     if any(token in path or token in text for token in category_tokens):

crawler_platform/app/core/crawler/pipeline.py

@@ -22,6 +22,8 @@ class CrawlResult:
     clean_text_length: int = 0
     raw_text_length: int = 0
     warnings: list[str] | None = None
+    robots_status: str = "unchecked"
+    robots_reason: str | None = None
 
 
 class CrawlPipeline:
@@ -39,8 +41,9 @@ class CrawlPipeline:
 
     def crawl_url(self, project_config: ProjectConfig, source_name: str, url: str) -> CrawlResult:
         source_config = project_config.source_by_name(source_name)
-        if not self.robots_policy.allowed(url, source_config.respect_robots_txt):
-            raise PermissionError(f"robots.txt does not allow crawling: {url}")
+        robots_decision = self.robots_policy.check(url, source_config.respect_robots_txt)
+        if not robots_decision.allowed:
+            raise PermissionError(f"{robots_decision.reason}: {url}")
 
         fetcher = make_fetcher(source_config.fetcher, source_config.rate_limit_per_minute)
         fetch_result = fetcher.fetch(url)
@@ -87,6 +90,8 @@ class CrawlPipeline:
                 clean_text_length=len(parsed.text or ""),
                 raw_text_length=len(parsed.raw_text or ""),
                 warnings=warnings,
+                robots_status=robots_decision.status,
+                robots_reason=robots_decision.reason,
             )
 
         context = ExtractionPageContext(
@@ -117,4 +122,6 @@ class CrawlPipeline:
             clean_text_length=len(parsed.text or ""),
             raw_text_length=len(parsed.raw_text or ""),
             warnings=warnings,
+            robots_status=robots_decision.status,
+            robots_reason=robots_decision.reason,
         )

crawler_platform/app/core/crawler/site_crawler.py

@@ -7,7 +7,7 @@ from urllib.parse import urldefrag, urlparse
 
 from crawler_platform.app.config.loader import ProjectConfig
 from crawler_platform.app.core.crawler.discovery import discover_links
-from crawler_platform.app.core.crawler.fetchers import RobotsPolicy, make_fetcher
+from crawler_platform.app.core.crawler.fetchers import RobotsDecision, RobotsPolicy, make_fetcher
 from crawler_platform.app.core.crawler.page_classifier import (
     classify_page as classify_page_type,
     should_analyze_page as should_analyze_page_type,
@@ -35,6 +35,8 @@ class SiteCrawlPageResult:
     clean_text_length: int = 0
     removed_noise_zones_count: int = 0
     warnings: list[str] = field(default_factory=list)
+    robots_status: str = "unchecked"
+    robots_reason: str | None = None
     error: str | None = None
 
 
@@ -120,14 +122,23 @@ class SiteCrawler:
                     progress_callback,
                 )
                 continue
-            if not self.robots_policy.allowed(url, source_config.respect_robots_txt):
-                error = f"robots.txt does not allow crawling: {url}"
+            robots_decision = self.robots_policy.check(url, source_config.respect_robots_txt)
+            if not robots_decision.allowed:
+                error = f"{robots_decision.reason}: {url}"
                 self._finish_job(job, "blocked", error)
                 result.skipped_count += 1
                 result.errors.append(error)
                 self._record_page_result(
                     result,
-                    SiteCrawlPageResult(url=url, depth=depth, status="blocked", page_type="unknown", error=error),
+                    SiteCrawlPageResult(
+                        url=url,
+                        depth=depth,
+                        status="blocked",
+                        page_type="unknown",
+                        robots_status=robots_decision.status,
+                        robots_reason=robots_decision.reason,
+                        error=error,
+                    ),
                     len(queue),
                     progress_callback,
                 )
@@ -150,6 +161,7 @@ class SiteCrawler:
                     analyze_page_types=analyze_page_types,
                     result=result,
                     job=job,
+                    robots_decision=robots_decision,
                     progress_callback=progress_callback,
                 )
             except Exception as exc:
@@ -184,6 +196,7 @@ class SiteCrawler:
         analyze_page_types: set[str],
         result: SiteCrawlResult,
         job: models.CrawlJob,
+        robots_decision: RobotsDecision,
         progress_callback: Callable[[SiteCrawlResult, SiteCrawlPageResult], None] | None,
     ) -> None:
         fetch_result = fetcher.fetch(url)
@@ -215,6 +228,8 @@ class SiteCrawler:
                     page_type="unknown",
                     page_id=page.id,
                     crawl_status=fetch_result.crawl_status,
+                    robots_status=robots_decision.status,
+                    robots_reason=robots_decision.reason,
                     warnings=fetch_result.warnings,
                     error=error,
                 ),
@@ -277,6 +292,8 @@ class SiteCrawler:
             "clean_text_length": len(parsed.text or ""),
             "removed_noise_zones_count": int(parsed.metadata.get("removed_noise_zones_count") or 0),
             "warnings": warnings,
+            "robots_status": robots_decision.status,
+            "robots_reason": robots_decision.reason,
         }
 
         if parsed.extraction_status == "failed":