참고소스 수정본

crawler_platform/app/core/crawler/content_zone.py

@@ -0,0 +1,60 @@
+from __future__ import annotations
+
+from dataclasses import asdict, dataclass
+
+
+CLAIM_ALLOWED_ZONE_TYPES = {
+    "main_content",
+    "document_body",
+    "article_body",
+    "research_body",
+    "product_title",
+    "product_summary",
+    "product_description",
+    "product_detail",
+    "brand_story_body",
+    "notice_body",
+    "event_body",
+}
+
+CLAIM_BLOCKED_ZONE_TYPES = {
+    "header",
+    "footer",
+    "nav",
+    "menu",
+    "category_filter",
+    "sort_control",
+    "shipping_policy",
+    "payment_policy",
+    "exchange_policy",
+    "recommendation",
+    "related_products",
+    "login_join",
+    "copyright",
+    "platform_credit",
+    "unknown",
+}
+
+
+@dataclass(slots=True)
+class ContentZone:
+    zone_type: str
+    text: str
+    selector: str | None = None
+    confidence: float = 0.0
+    claim_allowed: bool = False
+    reason: str | None = None
+
+    def to_dict(self, max_text_length: int | None = None) -> dict[str, object]:
+        data = asdict(self)
+        if max_text_length is not None and len(self.text) > max_text_length:
+            data["text"] = self.text[:max_text_length]
+        return data
+
+
+def is_claim_allowed_zone(zone_type: str | None) -> bool:
+    return bool(zone_type and zone_type in CLAIM_ALLOWED_ZONE_TYPES)
+
+
+def zone_dicts(zones: list[ContentZone], max_text_length: int | None = None) -> list[dict[str, object]]:
+    return [zone.to_dict(max_text_length=max_text_length) for zone in zones]

crawler_platform/app/core/crawler/discovery.py

@@ -1,7 +1,7 @@
 from __future__ import annotations
 
 from dataclasses import dataclass
-from urllib.parse import parse_qs, urljoin, urlparse
+from urllib.parse import parse_qs, unquote, urljoin, urlparse
 
 from bs4 import BeautifulSoup
 
@@ -19,8 +19,10 @@ def discover_links(html: str, base_url: str, limit: int = 30) -> list[Discovered
     results: list[DiscoveredUrl] = []
     for anchor in soup.find_all("a", href=True):
         raw_href = anchor.get("href", "")
+        if should_skip_raw_href(raw_href):
+            continue
         url = normalize_search_redirect(urljoin(base_url, raw_href))
-        if not url or url in seen or not url.startswith(("http://", "https://")):
+        if not url or url in seen or not url.startswith(("http://", "https://")) or should_skip_url(url):
             continue
         seen.add(url)
         label = anchor.get_text(" ", strip=True)[:160] or urlparse(url).netloc
@@ -30,6 +32,45 @@ def discover_links(html: str, base_url: str, limit: int = 30) -> list[Discovered
     return results
 
 
+def should_skip_raw_href(raw_href: str) -> bool:
+    href = (raw_href or "").strip()
+    lowered = href.lower()
+    if not href or href in {"#", "/", "javascript:;", "javascript:void(0)"}:
+        return True
+    if "{" in href or "}" in href:
+        return True
+    if lowered.startswith(("mailto:", "tel:", "sms:", "javascript:")):
+        return True
+    if lowered.startswith(("facebook.com/", "instagram.com/", "kakao.com/", "pf.kakao.com/")):
+        return True
+    return False
+
+
+def should_skip_url(url: str) -> bool:
+    parsed = urlparse(url)
+    host = parsed.netloc.lower()
+    path = unquote(parsed.path.lower())
+    query = unquote(parsed.query.lower())
+    if any(token in host for token in ["facebook.com", "instagram.com", "kakao.com", "pf.kakao.com"]):
+        return True
+    if "{" in path or "}" in path or "%7b" in url.lower() or "%7d" in url.lower():
+        return True
+    skip_path_tokens = [
+        "/member/",
+        "/order/",
+        "/exec/front/newcoupon/",
+        "/board/free/modify",
+        "/board/free/reply",
+    ]
+    if any(token in path for token in skip_path_tokens):
+        return True
+    if "facebook.com/" in path or "instagram.com/" in path:
+        return True
+    if "coupon_no=" in query:
+        return True
+    return False
+
+
 def normalize_search_redirect(url: str) -> str:
     parsed = urlparse(url)
     query = parse_qs(parsed.query)

crawler_platform/app/core/crawler/fetchers.py

@@ -2,6 +2,7 @@ from __future__ import annotations
 
 from dataclasses import dataclass, field
 from pathlib import Path
+import re
 import time
 from urllib.parse import urlparse
 from urllib.robotparser import RobotFileParser
@@ -17,6 +18,28 @@ class FetchResult:
     html: str
     final_url: str | None = None
     headers: dict[str, str] = field(default_factory=dict)
+    raw_html: str | None = None
+    rendered_html: str | None = None
+    title: str | None = None
+    crawl_status: str = "success"
+    warnings: list[str] = field(default_factory=list)
+
+    def __post_init__(self) -> None:
+        if self.raw_html is None:
+            self.raw_html = self.html
+        if self.rendered_html is None:
+            self.rendered_html = self.html
+        detected_status, detected_warnings = detect_crawl_status(
+            status_code=self.status_code,
+            html=self.rendered_html or self.raw_html or self.html,
+        )
+        if self.crawl_status == "success":
+            self.crawl_status = detected_status
+        self.warnings.extend(detected_warnings)
+
+    @property
+    def analysis_html(self) -> str:
+        return self.rendered_html or self.raw_html or self.html
 
 
 class RateLimiter:
@@ -76,7 +99,8 @@ class RequestsFetcher(BaseFetcher):
     def fetch(self, url: str) -> FetchResult:
         local_path = _local_path_from_url(url)
         if local_path:
-            return FetchResult(url=url, status_code=200, html=local_path.read_text(encoding="utf-8"), final_url=url)
+            html = local_path.read_text(encoding="utf-8")
+            return FetchResult(url=url, status_code=200, html=html, final_url=url, raw_html=html, rendered_html=html)
 
         import requests
 
@@ -107,21 +131,45 @@ class PlaywrightFetcher(BaseFetcher):
         self.timeout_ms = timeout_ms
 
     def fetch(self, url: str) -> FetchResult:
+        local_path = _local_path_from_url(url)
+        if local_path:
+            html = local_path.read_text(encoding="utf-8")
+            return FetchResult(url=url, status_code=200, html=html, final_url=url, raw_html=html, rendered_html=html)
+
         from playwright.sync_api import sync_playwright
 
+        warnings: list[str] = []
         with sync_playwright() as playwright:
             browser = playwright.chromium.launch(headless=True)
             page = browser.new_page(user_agent=self.user_agent)
-            response = page.goto(url, wait_until="networkidle", timeout=self.timeout_ms)
-            html = page.content()
-            final_url = page.url
-            status = response.status if response else None
-            browser.close()
-        return FetchResult(url=url, status_code=status, html=html, final_url=final_url)
+            try:
+                response = page.goto(url, wait_until="domcontentloaded", timeout=self.timeout_ms)
+                try:
+                    page.wait_for_load_state("networkidle", timeout=min(self.timeout_ms, 8000))
+                except Exception as exc:
+                    warnings.append(f"networkidle wait timed out: {exc}")
+                for _ in range(3):
+                    page.mouse.wheel(0, 1200)
+                    page.wait_for_timeout(250)
+                title = page.title()
+                html = page.content()
+                final_url = page.url
+                status = response.status if response else None
+            finally:
+                browser.close()
+        return FetchResult(
+            url=url,
+            status_code=status,
+            html=html,
+            final_url=final_url,
+            rendered_html=html,
+            title=title,
+            warnings=warnings,
+        )
 
 
 def make_fetcher(kind: str, rate_limit_per_minute: int = 30) -> BaseFetcher:
-    if kind == "playwright":
+    if kind in {"playwright", "browser"}:
         return PlaywrightFetcher()
     return RequestsFetcher(rate_limit_per_minute=rate_limit_per_minute)
 
@@ -134,3 +182,48 @@ def _local_path_from_url(url: str) -> Path | None:
     if path.exists() and path.is_file():
         return path
     return None
+
+
+def detect_crawl_status(status_code: int | None, html: str | None) -> tuple[str, list[str]]:
+    warnings: list[str] = []
+    text = visible_text_for_status_detection(html or "")
+    lowered = " ".join(text.lower().split())
+    if status_code is None:
+        warnings.append("missing status code")
+        return "crawl_failed", warnings
+    if status_code in {401, 403, 407, 429}:
+        warnings.append(f"blocked status code: {status_code}")
+        return "blocked", warnings
+    if status_code >= 400:
+        warnings.append(f"unavailable status code: {status_code}")
+        return "unavailable", warnings
+    if not lowered:
+        warnings.append("empty html")
+        return "empty", warnings
+    blocked_patterns = ["access denied", "request blocked", "bot detection", "unusual traffic"]
+    unavailable_patterns = ["page unavailable", "page not found", "temporarily unavailable", "service unavailable"]
+    if any(pattern in lowered for pattern in blocked_patterns):
+        warnings.append("blocked page pattern detected")
+        return "blocked", warnings
+    if ("captcha" in lowered or "recaptcha" in lowered) and any(
+        token in lowered for token in ["verify", "verification", "robot", "blocked", "challenge"]
+    ):
+        warnings.append("captcha challenge detected")
+        return "blocked", warnings
+    if any(pattern in lowered for pattern in unavailable_patterns):
+        warnings.append("unavailable page pattern detected")
+        return "unavailable", warnings
+    return "success", warnings
+
+
+def visible_text_for_status_detection(html: str) -> str:
+    try:
+        from bs4 import BeautifulSoup
+    except ImportError:
+        return re.sub(r"<[^>]+>", " ", html)
+
+    soup = BeautifulSoup(html, "html.parser")
+    for selector in ["script", "style", "noscript", "svg", "iframe"]:
+        for tag in soup.select(selector):
+            tag.decompose()
+    return soup.get_text(" ", strip=True)

crawler_platform/app/core/crawler/html_cleaner.py

@@ -1,25 +1,288 @@
 from __future__ import annotations
 
+from dataclasses import dataclass, field
 import re
+from typing import Any
+
+
+BOILERPLATE_SELECTORS = [
+    "script",
+    "style",
+    "noscript",
+    "svg",
+    "iframe",
+    "header",
+    "footer",
+    "nav",
+    "aside",
+    "form",
+    "[role='navigation']",
+    "[role='banner']",
+    "[role='contentinfo']",
+    ".header",
+    ".footer",
+    ".nav",
+    ".navigation",
+    ".gnb",
+    ".lnb",
+    ".menu",
+    ".breadcrumb",
+    ".pagination",
+    ".paging",
+    ".toolbar",
+    ".sort",
+    ".search",
+    ".login",
+    ".cart",
+    ".basket",
+    ".coupon",
+    ".event",
+    ".banner",
+    ".promotion",
+    ".recommend",
+    ".related",
+    ".recent",
+    ".review-list",
+    ".board",
+    ".notice",
+    ".cs",
+    ".customer",
+    ".shipping",
+    ".delivery",
+    "#header",
+    "#footer",
+    "#nav",
+    "#gnb",
+    "#lnb",
+    "#sidebar",
+    "#aside",
+    "#event",
+    "#banner",
+    "#board",
+    "#notice",
+]
+
+MAIN_CONTENT_SELECTORS = [
+    "main",
+    "article",
+    "[role='main']",
+    "#contents",
+    "#content",
+    "#container",
+    "#main",
+    ".contents",
+    ".content",
+    ".container",
+    ".product-detail",
+    ".prd-detail",
+    ".detailArea",
+    ".xans-product-detail",
+    ".xans-product-additional",
+    ".ec-base-product",
+    ".description",
+    ".summary",
+]
+
+NOISY_LINE_TERMS = {
+    "cafe24",
+    "powered by cafe24",
+    "hosting by cafe24",
+    "home",
+    "login",
+    "logout",
+    "cart",
+    "basket",
+    "checkout",
+    "my page",
+    "search",
+    "sort",
+    "low price",
+    "high price",
+    "new item",
+    "best item",
+    "product count",
+    "privacy policy",
+    "terms",
+    "company",
+    "customer center",
+    "notice",
+    "q&a",
+    "faq",
+    "review",
+    "event",
+    "copyright",
+}
+
+NOISY_LINE_PATTERNS = [
+    re.compile(pattern, re.IGNORECASE)
+    for pattern in [
+        r"^\d+\s*/\s*\d+$",
+        r"^page\s+\d+",
+        r"^(prev|previous|next|first|last)$",
+        r"^(add to cart|buy now|wish list)$",
+        r"^(usd|krw|eur|jpy)$",
+        r"shipping|delivery|return|exchange|refund",
+        r"country|language|currency",
+        r"facebook|instagram|youtube|kakao|naver",
+        r"cafe24|copyright|all rights reserved",
+    ]
+]
+
+
+@dataclass(slots=True)
+class CleanedHtml:
+    title: str | None
+    text: str
+    metadata: dict[str, Any] = field(default_factory=dict)
+    raw_text: str = ""
+    main_content: str = ""
+    clean_markdown: str = ""
+    source_zones: list[dict[str, object]] = field(default_factory=list)
+    noise_zones: list[dict[str, object]] = field(default_factory=list)
+    page_type: str = "UnknownPage"
+    extraction_status: str = "failed"
+    extraction_warnings: list[str] = field(default_factory=list)
 
 
 def clean_html(html: str) -> tuple[str | None, str]:
-    try:
-        from bs4 import BeautifulSoup
-    except ImportError:
-        text = re.sub(r"<[^>]+>", " ", html)
-        return None, normalize_whitespace(text)
+    cleaned = clean_html_with_metadata(html)
+    return cleaned.title, cleaned.text
 
-    soup = BeautifulSoup(html, "html.parser")
-    for tag in soup(["script", "style", "noscript", "svg", "iframe"]):
-        tag.decompose()
-    title = soup.title.get_text(" ", strip=True) if soup.title else None
-    main = soup.find("main") or soup.body or soup
-    text = main.get_text("\n", strip=True)
-    return title, normalize_whitespace(text)
+
+def clean_html_with_metadata(html: str, url: str = "", page_type: str | None = None) -> CleanedHtml:
+    from crawler_platform.app.core.crawler.page_cleaner import PageCleaner
+
+    cleaned = PageCleaner().clean(html, url=url, page_type=page_type)
+    source_zones = [zone.to_dict(max_text_length=1000) for zone in cleaned.source_zones]
+    noise_zones = [zone.to_dict(max_text_length=500) for zone in cleaned.noise_zones]
+    return CleanedHtml(
+        title=cleaned.title,
+        text=cleaned.clean_text,
+        metadata=cleaned.metadata,
+        raw_text=cleaned.raw_text,
+        main_content=cleaned.main_content,
+        clean_markdown=cleaned.clean_markdown,
+        source_zones=source_zones,
+        noise_zones=noise_zones,
+        page_type=cleaned.page_type,
+        extraction_status=cleaned.extraction_status,
+        extraction_warnings=cleaned.extraction_warnings,
+    )
+
+
+def remove_boilerplate_nodes(soup) -> None:
+    for selector in BOILERPLATE_SELECTORS:
+        for tag in soup.select(selector):
+            tag.decompose()
+    for tag in list(soup.find_all(True)):
+        if getattr(tag, "attrs", None) is None:
+            continue
+        token_text = " ".join([node_attr(tag, "id"), node_classes(tag), node_attr(tag, "aria-label")]).lower()
+        if any(token in token_text for token in ["footer", "header", "nav", "menu", "shipping", "delivery", "cafe24"]):
+            tag.decompose()
+
+
+def content_candidates(soup) -> list:
+    candidates = []
+    for selector in MAIN_CONTENT_SELECTORS:
+        candidates.extend(soup.select(selector))
+    if soup.body:
+        candidates.append(soup.body)
+    candidates.append(soup)
+    return [candidate for candidate in candidates if candidate is not None]
+
+
+def choose_main_node(candidates: list, soup) -> tuple[Any | None, str]:
+    scored = [(content_score(candidate), candidate) for candidate in candidates]
+    scored = [(score, candidate) for score, candidate in scored if score > 0]
+    if scored:
+        scored.sort(key=lambda item: item[0], reverse=True)
+        return scored[0][1], "selector_or_density"
+    return soup.body or soup, "body_fallback"
+
+
+def content_score(node) -> float:
+    text = normalize_content_lines(node.get_text("\n", strip=True) if node else "")
+    if len(text) < 40:
+        return 0
+    lower = text.lower()
+    product_signals = sum(
+        1
+        for token in [
+            "brand",
+            "price",
+            "notes",
+            "top notes",
+            "middle notes",
+            "base notes",
+            "description",
+            "ingredient",
+            "option",
+            "product",
+        ]
+        if token in lower
+    )
+    link_count = len(node.find_all("a")) if hasattr(node, "find_all") else 0
+    text_len = max(len(text), 1)
+    link_penalty = min(link_count * 30 / text_len, 0.7)
+    return len(text) * (1 + product_signals * 0.25) * (1 - link_penalty)
+
+
+def normalize_content_lines(text: str) -> str:
+    normalized = normalize_whitespace(text)
+    lines: list[str] = []
+    seen: set[str] = set()
+    for raw in normalized.splitlines():
+        line = cleanup_line(raw)
+        if not line or is_noisy_line(line):
+            continue
+        key = line.lower()
+        if key in seen:
+            continue
+        seen.add(key)
+        lines.append(line)
+    return "\n".join(lines)
+
+
+def cleanup_line(value: str) -> str:
+    return re.sub(r"\s+", " ", value.replace("\xa0", " ")).strip(" -:|")
+
+
+def is_noisy_line(line: str) -> bool:
+    lowered = line.lower().strip()
+    if not lowered or lowered in NOISY_LINE_TERMS:
+        return True
+    if len(lowered) <= 1:
+        return True
+    if len(lowered) <= 3 and not any(ch.isdigit() for ch in lowered):
+        return True
+    if any(pattern.search(lowered) for pattern in NOISY_LINE_PATTERNS):
+        return True
+    if lowered.count("|") >= 3:
+        return True
+    return False
 
 
 def normalize_whitespace(text: str) -> str:
     lines = [" ".join(line.split()) for line in text.splitlines()]
     return "\n".join(line for line in lines if line)
 
+
+def node_attr(node, name: str) -> str:
+    if node is None or not hasattr(node, "get"):
+        return ""
+    if getattr(node, "attrs", None) is None:
+        return ""
+    try:
+        value = node.get(name)
+    except AttributeError:
+        return ""
+    if value is None:
+        return ""
+    if isinstance(value, list):
+        return " ".join(str(item) for item in value if item)
+    return str(value)
+
+
+def node_classes(node) -> str:
+    return node_attr(node, "class")

crawler_platform/app/core/crawler/page_classifier.py

@@ -0,0 +1,149 @@
+from __future__ import annotations
+
+from urllib.parse import urlparse
+
+from crawler_platform.app.core.crawler.content_zone import is_claim_allowed_zone
+
+
+PRODUCT_DETAIL_PREDICATES = {
+    "hasBrand",
+    "hasTopNote",
+    "hasMiddleNote",
+    "hasBaseNote",
+    "hasAccord",
+    "evokesMood",
+    "suitableForSeason",
+    "suitableForOccasion",
+    "soldBy",
+    "hasPrice",
+    "hasReviewKeyword",
+}
+
+PRODUCT_DETAIL_PAGE_TYPES = {"ProductPage"}
+CONTENT_PAGE_TYPES = {"BrandStoryPage", "NoticePage", "EventPage", "PromotionPage"}
+NON_MERGE_PAGE_TYPES = {"UnknownPage", "SearchPage", "CategoryPage", "BoardPage"}
+
+
+def classify_page(
+    url: str,
+    title: str | None = None,
+    text: str = "",
+    html: str | None = None,
+    source_zones: list[dict[str, object]] | None = None,
+) -> str:
+    path = urlparse(url).path.lower()
+    query = urlparse(url).query.lower()
+    combined = f"{url}\n{title or ''}\n{text[:5000]}".lower()
+    html_lower = (html or "")[:8000].lower()
+
+    if "/product/list" in path or path.endswith("/product/list.html"):
+        return "CategoryPage"
+    if "/product/search" in path or path.endswith("/product/search.html"):
+        return "SearchPage"
+    if any(token in path for token in ["/search", "/find"]) or "keyword=" in query or "search" in query:
+        return "SearchPage"
+    if any(token in path for token in ["/board/", "board/free", "board/product", "/article/"]):
+        if any(token in combined for token in ["notice", "공지"]):
+            return "NoticePage"
+        return "BoardPage"
+    if any(token in path for token in ["shopinfo", "company", "about", "brand-story", "brand_story"]):
+        return "BrandStoryPage"
+    if any(token in path for token in ["product/detail", "/product/", "/products/", "/goods/", "/item/"]):
+        if _looks_like_category_path(path, combined):
+            return "CategoryPage"
+        return "ProductPage"
+    if any(token in path for token in ["category", "/collections", "/collection", "/shop/", "/list"]):
+        return "CategoryPage"
+    if any(token in path for token in ["event", "promotion", "promo", "sale"]):
+        return "PromotionPage"
+
+    if source_zones and any(_zone_type(zone) in {"product_detail", "product_description"} for zone in source_zones):
+        return "ProductPage"
+
+    product_tokens = [
+        "add to cart",
+        "buy now",
+        "price",
+        "top notes",
+        "middle notes",
+        "base notes",
+        "fragrance notes",
+        "장바구니",
+        "구매",
+        "가격",
+        "탑노트",
+        "베이스노트",
+    ]
+    listing_tokens = ["sort", "low price", "high price", "product count", "상품수", "낮은가격", "높은가격"]
+    brand_tokens = ["about us", "brand story", "our story", "philosophy", "official", "브랜드", "소개"]
+    promotion_tokens = ["event", "sale", "coupon", "promotion", "black friday", "회원가입", "쿠폰", "증정", "무료배송"]
+    notice_tokens = ["notice", "공지", "announcement"]
+    board_tokens = ["q&a", "faq", "review", "게시판", "문의"]
+
+    if sum(1 for token in promotion_tokens if token in combined) >= 2:
+        return "PromotionPage"
+    if any(token in combined for token in notice_tokens):
+        return "NoticePage"
+    if any(token in combined for token in board_tokens):
+        return "BoardPage"
+    if any(token in combined for token in brand_tokens):
+        return "BrandStoryPage"
+    if sum(1 for token in listing_tokens if token in combined) >= 2 and not _has_product_detail_signal(combined):
+        return "CategoryPage"
+    if _has_product_detail_signal(combined) or _has_product_detail_signal(html_lower):
+        return "ProductPage"
+    return "UnknownPage"
+
+
+def relation_allowed_for_page_type(page_type: str | None, predicate: str) -> bool:
+    if page_type in PRODUCT_DETAIL_PAGE_TYPES:
+        return True
+    if predicate in PRODUCT_DETAIL_PREDICATES:
+        return False
+    if page_type in NON_MERGE_PAGE_TYPES:
+        return False
+    return page_type in CONTENT_PAGE_TYPES
+
+
+def claim_allowed_for_context(page_type: str | None, predicate: str, zone_type: str | None) -> bool:
+    return relation_allowed_for_page_type(page_type, predicate) and is_claim_allowed_zone(zone_type)
+
+
+def normalize_page_type(value: str | None) -> str:
+    aliases = {
+        "product": "ProductPage",
+        "brand": "BrandStoryPage",
+        "review": "ReviewPage",
+        "listing": "CategoryPage",
+        "category": "CategoryPage",
+        "community": "BoardPage",
+        "board": "BoardPage",
+        "communitypage": "BoardPage",
+        "listingpage": "CategoryPage",
+        "promotionpage": "PromotionPage",
+    }
+    clean = str(value or "").strip()
+    return aliases.get(clean.lower(), aliases.get(clean, clean or "UnknownPage"))
+
+
+def should_analyze_page(page_type: str, analyze_page_types: set[str]) -> bool:
+    normalized_page_type = normalize_page_type(page_type)
+    normalized = {normalize_page_type(item) for item in analyze_page_types}
+    return normalized_page_type in normalized
+
+
+def _zone_type(zone: dict[str, object]) -> str:
+    return str(zone.get("zone_type") or "")
+
+
+def _has_product_detail_signal(text: str) -> bool:
+    note_count = sum(1 for token in ["top notes", "middle notes", "base notes", "탑노트", "미들노트", "베이스노트"] if token in text)
+    commerce_count = sum(1 for token in ["price", "add to cart", "buy now", "장바구니", "구매", "가격"] if token in text)
+    return note_count >= 1 or commerce_count >= 2
+
+
+def _looks_like_category_path(path: str, text: str) -> bool:
+    category_tokens = ["category", "cate_no", "display_group", "sort_method"]
+    if any(token in path or token in text for token in category_tokens):
+        return True
+    return sum(1 for token in ["low price", "high price", "product count", "상품수", "낮은가격"] if token in text) >= 2

crawler_platform/app/core/crawler/page_cleaner.py

@@ -0,0 +1,606 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+import re
+from typing import Any
+
+from crawler_platform.app.core.crawler.content_zone import ContentZone, zone_dicts
+from crawler_platform.app.core.crawler.page_classifier import classify_page
+
+
+SCRIPT_STYLE_SELECTORS = ["script", "style", "noscript", "svg", "iframe"]
+
+NOISE_SELECTORS: dict[str, list[str]] = {
+    "header": ["header", "[role='banner']", ".header", "#header", ".topbar"],
+    "footer": ["footer", "[role='contentinfo']", ".footer", "#footer"],
+    "nav": ["nav", "[role='navigation']", ".nav", ".navigation", ".gnb", ".lnb", "#gnb", "#lnb"],
+    "menu": [".menu", ".breadcrumb", ".pagination", ".paging", ".toolbar", ".tabs"],
+    "category_filter": [".filter", ".category", ".categories", ".xans-product-menupackage"],
+    "sort_control": [".sort", ".order", ".ec-base-paginate"],
+    "shipping_policy": [".shipping", ".delivery", ".policy", ".guide", ".return", ".exchange", ".refund"],
+    "recommendation": [".recommend", ".related", ".recent", ".best", ".new", ".also", ".relation"],
+    "login_join": [".login", ".join", ".account", ".member", ".cart", ".basket"],
+    "platform_credit": [".cafe24", ".hosting", ".powered"],
+}
+
+ZONE_SELECTORS: dict[str, list[str]] = {
+    "product_title": [
+        "h1",
+        ".product-title",
+        ".product_name",
+        ".product-name",
+        ".name",
+        ".headingArea h2",
+        ".xans-product-detail .headingArea",
+    ],
+    "product_summary": [
+        ".summary",
+        ".prd-summary",
+        ".prdSummary",
+        ".simple_desc",
+        ".xans-product-detaildesign",
+    ],
+    "product_description": [
+        ".description",
+        ".desc",
+        ".product-description",
+        ".prdDesc",
+        ".detail-description",
+    ],
+    "product_detail": [
+        "main",
+        "article",
+        "[role='main']",
+        ".product-detail",
+        ".prd-detail",
+    ],
+    "brand_story_body": [
+        "main",
+        "article",
+        ".brand-story",
+        ".brand_story",
+        ".about",
+        ".company",
+        ".story",
+        "#about",
+        "#company",
+    ],
+    "notice_body": ["main", "article", ".notice", ".boardView", ".view", ".post", ".article"],
+    "event_body": ["main", "article", ".event", ".promotion", ".promo", ".event-view", ".post", ".article"],
+}
+
+ZONE_PRIORITY_BY_PAGE_TYPE = {
+    "ProductPage": ["product_title", "product_summary", "product_description", "product_detail"],
+    "BrandStoryPage": ["brand_story_body"],
+    "NoticePage": ["notice_body"],
+    "BoardPage": ["notice_body"],
+    "EventPage": ["event_body"],
+    "PromotionPage": ["event_body"],
+    "CategoryPage": ["product_title", "product_summary"],
+}
+
+STRUCTURAL_NOISE_TOKENS = {
+    "footer",
+    "header",
+    "nav",
+    "menu",
+    "breadcrumb",
+    "pagination",
+    "shipping",
+    "delivery",
+    "exchange",
+    "refund",
+    "policy",
+    "recommend",
+    "related",
+    "recent",
+    "login",
+    "join",
+    "cart",
+    "basket",
+    "cafe24",
+    "copyright",
+}
+
+NOISY_LINE_TERMS = {
+    "cafe24",
+    "powered by cafe24",
+    "hosting by cafe24",
+    "home",
+    "login",
+    "logout",
+    "cart",
+    "basket",
+    "checkout",
+    "my page",
+    "search",
+    "sort",
+    "low price",
+    "high price",
+    "new item",
+    "best item",
+    "product count",
+    "privacy policy",
+    "terms",
+    "company",
+    "customer center",
+    "notice",
+    "q&a",
+    "faq",
+    "review",
+    "event",
+    "copyright",
+    "상품수",
+    "낮은가격",
+    "높은가격",
+}
+
+NOISY_LINE_PATTERNS = [
+    re.compile(pattern, re.IGNORECASE)
+    for pattern in [
+        r"^\d+\s*/\s*\d+$",
+        r"^page\s+\d+",
+        r"^(prev|previous|next|first|last)$",
+        r"^(add to cart|buy now|wish list)$",
+        r"^(usd|krw|eur|jpy)$",
+        r"shipping|delivery|return|exchange|refund",
+        r"country|language|currency",
+        r"facebook|instagram|youtube|kakao|naver",
+        r"cafe24|copyright|all rights reserved",
+    ]
+]
+
+POLICY_TERMS = {
+    "shipping",
+    "delivery",
+    "return",
+    "exchange",
+    "refund",
+    "privacy",
+    "terms",
+    "country",
+    "language",
+    "배송",
+    "교환",
+    "반품",
+    "환불",
+}
+
+
+@dataclass(slots=True)
+class CleanedPage:
+    title: str | None
+    raw_text: str
+    main_content: str
+    clean_text: str
+    clean_markdown: str
+    page_type: str
+    source_zones: list[ContentZone] = field(default_factory=list)
+    noise_zones: list[ContentZone] = field(default_factory=list)
+    extraction_status: str = "failed"
+    extraction_warnings: list[str] = field(default_factory=list)
+    metadata: dict[str, Any] = field(default_factory=dict)
+
+
+class PageCleaner:
+    def clean(self, html: str, url: str = "", page_type: str | None = None) -> CleanedPage:
+        try:
+            from bs4 import BeautifulSoup
+        except ImportError:
+            text = normalize_content_lines(re.sub(r"<[^>]+>", " ", html))
+            resolved_page_type = page_type or classify_page(url, None, text)
+            return CleanedPage(
+                title=None,
+                raw_text=text,
+                main_content=text,
+                clean_text=text,
+                clean_markdown=text,
+                page_type=resolved_page_type,
+                extraction_status="partial" if text else "failed",
+                extraction_warnings=["beautifulsoup unavailable; used regex fallback"],
+                metadata={"cleaner": "regex_fallback", "page_type": resolved_page_type},
+            )
+
+        soup = BeautifulSoup(html or "", "html.parser")
+        title = soup.title.get_text(" ", strip=True) if soup.title else None
+        raw_text = normalize_whitespace(soup.get_text("\n", strip=True))
+        resolved_page_type = page_type or classify_page(url, title, raw_text, html)
+
+        work = BeautifulSoup(html or "", "html.parser")
+        for selector in SCRIPT_STYLE_SELECTORS:
+            for tag in work.select(selector):
+                tag.decompose()
+
+        noise_zones = self._remove_noise_zones(work, resolved_page_type)
+        source_zones = self._source_zones(work, resolved_page_type)
+        if not source_zones:
+            fallback_zone = self._fallback_main_zone(work)
+            if fallback_zone:
+                source_zones = [fallback_zone]
+        if not source_zones and raw_text:
+            recovered_text = normalize_content_lines(raw_text)
+            if _is_contentful(recovered_text):
+                source_zones = [
+                    ContentZone(
+                        zone_type="unknown",
+                        selector="raw_text",
+                        text=recovered_text,
+                        confidence=0.2,
+                        claim_allowed=False,
+                        reason="raw text fallback after noise removal stripped content",
+                    )
+                ]
+
+        main_content = normalize_content_lines("\n".join(zone.text for zone in source_zones))
+        clean_text = normalize_content_lines(main_content)
+        clean_markdown = build_clean_markdown(source_zones)
+        warnings: list[str] = []
+        if not source_zones:
+            warnings.append("no content zones detected")
+        if len(clean_text) < 80:
+            warnings.append("clean text is short")
+        extraction_status = self._status_for(clean_text, source_zones)
+        metadata = {
+            "cleaner": "page_cleaner_v1",
+            "page_type": resolved_page_type,
+            "raw_text_length": len(raw_text),
+            "main_content_length": len(main_content),
+            "clean_text_length": len(clean_text),
+            "clean_markdown_length": len(clean_markdown),
+            "source_zone_count": len(source_zones),
+            "noise_zone_count": len(noise_zones),
+            "removed_noise_zones_count": len(noise_zones),
+            "source_zones": zone_dicts(source_zones, max_text_length=500),
+            "noise_zones": zone_dicts(noise_zones, max_text_length=240),
+            "extraction_status": extraction_status,
+            "extraction_warnings": warnings,
+        }
+        return CleanedPage(
+            title=title,
+            raw_text=raw_text,
+            main_content=main_content,
+            clean_text=clean_text,
+            clean_markdown=clean_markdown,
+            page_type=resolved_page_type,
+            source_zones=source_zones,
+            noise_zones=noise_zones,
+            extraction_status=extraction_status,
+            extraction_warnings=warnings,
+            metadata=metadata,
+        )
+
+    def _remove_noise_zones(self, soup, page_type: str) -> list[ContentZone]:
+        zones: list[ContentZone] = []
+        for zone_type, selectors in NOISE_SELECTORS.items():
+            if _preserve_noise_type_for_page(zone_type, page_type):
+                continue
+            for selector in selectors:
+                for tag in list(soup.select(selector)):
+                    text = normalize_whitespace(tag.get_text("\n", strip=True))
+                    if text:
+                        zones.append(ContentZone(zone_type=zone_type, selector=selector, text=text, confidence=0.9))
+                    tag.decompose()
+
+        total_text_len = len(normalize_whitespace(soup.get_text("\n", strip=True)))
+        for tag in list(soup.find_all(True)):
+            if not _tag_alive(tag):
+                continue
+            if tag.name in {"html", "body", "head", "[document]"}:
+                continue
+            token_text = " ".join([node_attr(tag, "id"), node_classes(tag), node_attr(tag, "aria-label")]).lower()
+            text = normalize_whitespace(tag.get_text("\n", strip=True))
+            if not text:
+                continue
+            structural_match = any(token in token_text for token in STRUCTURAL_NOISE_TOKENS)
+            policy_match = _looks_like_policy_block(text)
+            sort_match = _looks_like_sort_or_filter(text)
+            if (policy_match or sort_match) and not structural_match and _contains_content_selector(tag):
+                continue
+            if not structural_match and total_text_len > 0 and len(text) >= max(800, total_text_len * 0.5):
+                continue
+            if structural_match or policy_match or sort_match:
+                zone_type = _zone_type_for_noise(token_text, text)
+                if _preserve_noise_type_for_page(zone_type, page_type):
+                    continue
+                zones.append(
+                    ContentZone(
+                        zone_type=zone_type,
+                        selector=css_hint(tag),
+                        text=text,
+                        confidence=0.72,
+                        reason="structural or policy-like block",
+                    )
+                )
+                tag.decompose()
+        return _dedupe_zones(zones)
+
+    def _source_zones(self, soup, page_type: str) -> list[ContentZone]:
+        zones: list[ContentZone] = []
+        zone_types = ZONE_PRIORITY_BY_PAGE_TYPE.get(page_type) or ["product_detail", "brand_story_body", "notice_body"]
+        for zone_type in zone_types:
+            for selector in ZONE_SELECTORS.get(zone_type, []):
+                for tag in soup.select(selector):
+                    text = normalize_content_lines(tag.get_text("\n", strip=True))
+                    if not _is_contentful(text):
+                        continue
+                    zones.append(
+                        ContentZone(
+                            zone_type=zone_type,
+                            selector=selector,
+                            text=text,
+                            confidence=_zone_confidence(zone_type, text),
+                            claim_allowed=zone_type in {
+                                "product_title",
+                                "product_summary",
+                                "product_description",
+                                "product_detail",
+                                "brand_story_body",
+                                "notice_body",
+                                "event_body",
+                            },
+                        )
+                    )
+        return _dedupe_zones(zones)
+
+    def _fallback_main_zone(self, soup) -> ContentZone | None:
+        candidates = []
+        for selector in ["main", "article", "[role='main']", "body"]:
+            candidates.extend(soup.select(selector))
+        candidates.append(soup)
+        scored = []
+        for candidate in candidates:
+            text = normalize_content_lines(candidate.get_text("\n", strip=True))
+            if _is_contentful(text):
+                scored.append((content_score(candidate, text), candidate, text))
+        if not scored:
+            return None
+        scored.sort(key=lambda item: item[0], reverse=True)
+        _score, node, text = scored[0]
+        return ContentZone(
+            zone_type="unknown",
+            selector=css_hint(node),
+            text=text,
+            confidence=0.35,
+            claim_allowed=False,
+            reason="fallback body/main candidate",
+        )
+
+    def _status_for(self, clean_text: str, source_zones: list[ContentZone]) -> str:
+        if not clean_text or len(clean_text) < 20:
+            return "failed"
+        if not source_zones or all(zone.zone_type == "unknown" for zone in source_zones):
+            return "partial"
+        if len(clean_text) < 80:
+            return "partial"
+        return "success"
+
+
+def build_clean_markdown(source_zones: list[ContentZone]) -> str:
+    blocks = []
+    for zone in source_zones:
+        text = normalize_content_lines(zone.text)
+        if not text:
+            continue
+        heading = zone.zone_type.replace("_", " ").title()
+        blocks.append(f"## {heading}\n{text}")
+    return "\n\n".join(blocks)
+
+
+def normalize_content_lines(text: str) -> str:
+    normalized = normalize_whitespace(text)
+    raw_lines = normalized.splitlines()
+    lines: list[str] = []
+    seen: set[str] = set()
+    for idx, raw in enumerate(raw_lines):
+        line = cleanup_line(raw)
+        if not line or is_noisy_line(line) or is_contextual_noise_line(raw_lines, idx, line):
+            continue
+        key = line.lower()
+        if key in seen:
+            continue
+        seen.add(key)
+        lines.append(line)
+    return "\n".join(lines)
+
+
+def normalize_whitespace(text: str) -> str:
+    lines = [" ".join(line.split()) for line in text.splitlines()]
+    return "\n".join(line for line in lines if line)
+
+
+def cleanup_line(value: str) -> str:
+    return re.sub(r"\s+", " ", value.replace("\xa0", " ")).strip(" -:|")
+
+
+def is_noisy_line(line: str) -> bool:
+    lowered = line.lower().strip()
+    if not lowered or lowered in NOISY_LINE_TERMS:
+        return True
+    if lowered in {
+        "기본 정보",
+        "소비자가",
+        "상품정보",
+        "상품 간략설명",
+        "목록 내 상품 간단 설명",
+        "상세페이지 참고",
+        "상품 옵션",
+        "옵션 선택",
+        "사이즈 가이드",
+        "배송 예정일",
+        "상품 목록",
+        "구매하기",
+        "sold out",
+        "유의사항",
+    }:
+        return True
+    if "{#" in lowered or "{$" in lowered or "display_" in lowered or "*display" in lowered:
+        return True
+    if len(lowered) <= 1:
+        return True
+    if len(lowered) <= 3 and not any(ch.isdigit() for ch in lowered):
+        return True
+    if any(pattern.search(lowered) for pattern in NOISY_LINE_PATTERNS):
+        return True
+    if lowered.count("|") >= 3:
+        return True
+    return False
+
+
+def is_contextual_noise_line(raw_lines: list[str], idx: int, line: str) -> bool:
+    lowered_line = line.lower()
+    window = "\n".join(raw_lines[max(0, idx - 2) : min(len(raw_lines), idx + 3)]).lower()
+    if re.fullmatch(r"\d{1,3}(?:,\d{3})*\s*원", line) and any(
+        token in window for token in ["배송", "무료", "이상 구매", "shipping", "delivery", "free"]
+    ):
+        return True
+    if any(token in lowered_line for token in ["배송비", "무료배송", "이상 구매 시 무료", "회원 가입", "카카오톡 채널", "적립금"]):
+        return True
+    if lowered_line.startswith(("상품에 대해 궁금", "상품의 사용후기", "로그인 후 적립")):
+        return True
+    return False
+
+
+def content_score(node, text: str) -> float:
+    link_count = len(node.find_all("a")) if hasattr(node, "find_all") else 0
+    text_len = max(len(text), 1)
+    link_penalty = min(link_count * 30 / text_len, 0.7)
+    semantic_bonus = sum(
+        1
+        for token in ["brand", "price", "notes", "description", "ingredient", "product", "story", "notice"]
+        if token in text.lower()
+    )
+    return text_len * (1 + semantic_bonus * 0.2) * (1 - link_penalty)
+
+
+def node_attr(node, name: str) -> str:
+    if node is None or not hasattr(node, "get"):
+        return ""
+    if getattr(node, "attrs", None) is None:
+        return ""
+    try:
+        value = node.get(name)
+    except (AttributeError, TypeError):
+        return ""
+    if value is None:
+        return ""
+    if isinstance(value, list):
+        return " ".join(str(item) for item in value if item)
+    return str(value)
+
+
+def node_classes(node) -> str:
+    return node_attr(node, "class")
+
+
+def css_hint(node) -> str | None:
+    if node is None or not getattr(node, "name", None):
+        return None
+    node_id = node_attr(node, "id")
+    if node_id:
+        return f"#{node_id}"
+    classes = node_classes(node).split()
+    if classes:
+        return f"{node.name}.{classes[0]}"
+    return str(node.name)
+
+
+def _preserve_noise_type_for_page(zone_type: str, page_type: str) -> bool:
+    if page_type in {"EventPage", "PromotionPage"} and zone_type in {"recommendation"}:
+        return False
+    if page_type in {"EventPage", "PromotionPage"} and zone_type in {"category_filter", "sort_control"}:
+        return False
+    if page_type == "NoticePage" and zone_type == "menu":
+        return False
+    return False
+
+
+def _looks_like_policy_block(text: str) -> bool:
+    lowered = text.lower()
+    if len(text) < 120:
+        return False
+    term_count = sum(1 for term in POLICY_TERMS if term in lowered)
+    comma_or_line_count = text.count("\n") + text.count(",")
+    return term_count >= 2 and comma_or_line_count >= 4
+
+
+def _looks_like_sort_or_filter(text: str) -> bool:
+    lowered = text.lower()
+    sort_terms = ["sort", "low price", "high price", "product count", "상품수", "낮은가격", "높은가격"]
+    return len(text) < 500 and sum(1 for term in sort_terms if term in lowered) >= 2
+
+
+def _zone_type_for_noise(token_text: str, text: str) -> str:
+    lowered = f"{token_text}\n{text}".lower()
+    if "shipping" in lowered or "delivery" in lowered or "배송" in lowered:
+        return "shipping_policy"
+    if "exchange" in lowered or "return" in lowered or "refund" in lowered or "교환" in lowered:
+        return "exchange_policy"
+    if "sort" in lowered or "low price" in lowered or "상품수" in lowered:
+        return "sort_control"
+    if "recommend" in lowered or "related" in lowered:
+        return "recommendation"
+    if "cafe24" in lowered or "powered" in lowered:
+        return "platform_credit"
+    if "login" in lowered or "cart" in lowered or "join" in lowered:
+        return "login_join"
+    if "footer" in lowered or "copyright" in lowered:
+        return "footer"
+    if "header" in lowered:
+        return "header"
+    if "nav" in lowered or "menu" in lowered:
+        return "nav"
+    return "unknown"
+
+
+def _is_contentful(text: str) -> bool:
+    return bool(text and len(text) >= 8 and not is_noisy_line(text))
+
+
+def _zone_confidence(zone_type: str, text: str) -> float:
+    base = {
+        "product_title": 0.86,
+        "product_summary": 0.78,
+        "product_description": 0.78,
+        "product_detail": 0.74,
+        "brand_story_body": 0.72,
+        "notice_body": 0.68,
+        "event_body": 0.66,
+    }.get(zone_type, 0.45)
+    if len(text) > 160:
+        base += 0.05
+    return min(base, 0.95)
+
+
+def _dedupe_zones(zones: list[ContentZone]) -> list[ContentZone]:
+    result: list[ContentZone] = []
+    seen: set[str] = set()
+    for zone in sorted(zones, key=lambda item: (item.confidence, len(item.text)), reverse=True):
+        key_source = normalize_content_lines(zone.text) if zone.claim_allowed else normalize_whitespace(zone.text)
+        key = key_source.lower()
+        if not key or key in seen:
+            continue
+        if any(key in existing or existing in key for existing in seen if min(len(key), len(existing)) > 80):
+            continue
+        seen.add(key)
+        result.append(zone)
+    result.sort(key=lambda item: (-item.confidence, item.zone_type))
+    return result
+
+
+def _tag_alive(tag) -> bool:
+    return bool(getattr(tag, "name", None))
+
+
+def _contains_content_selector(tag) -> bool:
+    if not hasattr(tag, "select"):
+        return False
+    content_selectors = [
+        "h1",
+        ".detailArea",
+        ".infoArea",
+        "#prdDetail",
+        ".xans-product-detail",
+        ".product-detail",
+        ".product-description",
+    ]
+    return any(tag.select(selector) for selector in content_selectors)

crawler_platform/app/core/crawler/pipeline.py

@@ -3,10 +3,12 @@ from __future__ import annotations
 from dataclasses import dataclass
 
 from crawler_platform.app.config.loader import ProjectConfig
+from crawler_platform.app.core.crawler.page_classifier import classify_page
 from crawler_platform.app.core.crawler.fetchers import RobotsPolicy, make_fetcher
 from crawler_platform.app.core.crawler.plugins import ParserRegistry, default_parser_registry
 from crawler_platform.app.core.database.repository import KnowledgeRepository
-from crawler_platform.app.core.extractor.base import Extractor
+from crawler_platform.app.core.extractor.base import ExtractionPageContext, Extractor
+from crawler_platform.app.core.extractor.validation import attach_page_context
 
 
 @dataclass(slots=True)
@@ -14,6 +16,12 @@ class CrawlResult:
     page_id: int
     claim_count: int
     entity_count: int
+    crawl_status: str = "success"
+    extraction_status: str = "success"
+    page_type: str = "UnknownPage"
+    clean_text_length: int = 0
+    raw_text_length: int = 0
+    warnings: list[str] | None = None
 
 
 class CrawlPipeline:
@@ -37,19 +45,76 @@ class CrawlPipeline:
         fetcher = make_fetcher(source_config.fetcher, source_config.rate_limit_per_minute)
         fetch_result = fetcher.fetch(url)
         parser = self.parser_registry.get(source_config.parser)
-        parsed = parser.parse(fetch_result.html, fetch_result.final_url or url)
-        bundle = self.extractor.extract(parsed.text, project_config)
+        parsed = parser.parse(fetch_result.analysis_html, fetch_result.final_url or url)
+        page_type = classify_page(
+            fetch_result.final_url or url,
+            parsed.title or fetch_result.title,
+            parsed.raw_text or parsed.text,
+            fetch_result.analysis_html,
+            parsed.source_zones or [],
+        )
 
         project = self.repository.upsert_project(project_config)
         source = self.repository.get_source(project.id, source_name)
+        warnings = [*fetch_result.warnings, *(parsed.extraction_warnings or [])]
+        metadata = {
+            **parsed.metadata,
+            "final_url": fetch_result.final_url,
+            "crawl_status": fetch_result.crawl_status,
+            "crawl_warnings": fetch_result.warnings,
+            "page_type": page_type,
+            "raw_text_length": len(parsed.raw_text or ""),
+            "clean_text_length": len(parsed.text or ""),
+            "main_content_preview": (parsed.main_content or parsed.text)[:800],
+        }
         page = self.repository.upsert_page(
             project_id=project.id,
             source_id=source.id,
             url=url,
-            title=parsed.title,
+            title=parsed.title or fetch_result.title,
             status_code=fetch_result.status_code,
             cleaned_text=parsed.text,
-            metadata={**parsed.metadata, "final_url": fetch_result.final_url},
+            metadata=metadata,
+        )
+        if fetch_result.crawl_status != "success" or parsed.extraction_status == "failed":
+            return CrawlResult(
+                page_id=page.id,
+                claim_count=0,
+                entity_count=0,
+                crawl_status=fetch_result.crawl_status,
+                extraction_status=parsed.extraction_status,
+                page_type=page_type,
+                clean_text_length=len(parsed.text or ""),
+                raw_text_length=len(parsed.raw_text or ""),
+                warnings=warnings,
+            )
+
+        context = ExtractionPageContext(
+            url=url,
+            final_url=fetch_result.final_url,
+            title=parsed.title or fetch_result.title,
+            page_type=page_type,
+            clean_text=parsed.text,
+            raw_text=parsed.raw_text,
+            main_content=parsed.main_content,
+            clean_markdown=parsed.clean_markdown,
+            source_zones=parsed.source_zones or [],
+            crawl_status=fetch_result.crawl_status,
+            extraction_status=parsed.extraction_status,
+            warnings=warnings,
+            metadata=metadata,
+        )
+        bundle = self.extractor.extract_from_context(context, project_config)
+        bundle = attach_page_context(bundle, context)
+        claims = self.repository.save_extraction_bundle(project.id, source, page, bundle, project_config)
+        return CrawlResult(
+            page_id=page.id,
+            claim_count=len(claims),
+            entity_count=len(bundle.entities),
+            crawl_status=fetch_result.crawl_status,
+            extraction_status=parsed.extraction_status,
+            page_type=page_type,
+            clean_text_length=len(parsed.text or ""),
+            raw_text_length=len(parsed.raw_text or ""),
+            warnings=warnings,
         )
-        claims = self.repository.save_extraction_bundle(project.id, source, page, bundle)
-        return CrawlResult(page_id=page.id, claim_count=len(claims), entity_count=len(bundle.entities))

crawler_platform/app/core/crawler/plugins.py

@@ -9,6 +9,14 @@ class ParsedPage:
     title: str | None
     text: str
     metadata: dict[str, object]
+    raw_text: str = ""
+    main_content: str = ""
+    clean_markdown: str = ""
+    source_zones: list[dict[str, object]] | None = None
+    noise_zones: list[dict[str, object]] | None = None
+    page_type: str = "UnknownPage"
+    extraction_status: str = "failed"
+    extraction_warnings: list[str] | None = None
 
 
 class SiteParser(Protocol):
@@ -35,14 +43,25 @@ class GenericProductParser:
     name = "generic"
 
     def parse(self, html: str, url: str) -> ParsedPage:
-        from crawler_platform.app.core.crawler.html_cleaner import clean_html
+        from crawler_platform.app.core.crawler.html_cleaner import clean_html_with_metadata
 
-        title, text = clean_html(html)
-        return ParsedPage(title=title, text=text, metadata={"parser": self.name, "url": url})
+        cleaned = clean_html_with_metadata(html, url=url)
+        return ParsedPage(
+            title=cleaned.title,
+            text=cleaned.text,
+            metadata={"parser": self.name, "url": url, **cleaned.metadata},
+            raw_text=cleaned.raw_text,
+            main_content=cleaned.main_content,
+            clean_markdown=cleaned.clean_markdown,
+            source_zones=cleaned.source_zones,
+            noise_zones=cleaned.noise_zones,
+            page_type=cleaned.page_type,
+            extraction_status=cleaned.extraction_status,
+            extraction_warnings=cleaned.extraction_warnings,
+        )
 
 
 def default_parser_registry() -> ParserRegistry:
     registry = ParserRegistry()
     registry.register(GenericProductParser())
     return registry
-

crawler_platform/app/core/crawler/site_crawler.py

@@ -2,15 +2,21 @@ from __future__ import annotations
 
 from collections import deque
 from dataclasses import dataclass, field
+from typing import Callable
 from urllib.parse import urldefrag, urlparse
 
 from crawler_platform.app.config.loader import ProjectConfig
 from crawler_platform.app.core.crawler.discovery import discover_links
 from crawler_platform.app.core.crawler.fetchers import RobotsPolicy, make_fetcher
+from crawler_platform.app.core.crawler.page_classifier import (
+    classify_page as classify_page_type,
+    should_analyze_page as should_analyze_page_type,
+)
 from crawler_platform.app.core.crawler.plugins import ParserRegistry, default_parser_registry
 from crawler_platform.app.core.database import models
 from crawler_platform.app.core.database.repository import KnowledgeRepository
-from crawler_platform.app.core.extractor.base import Extractor
+from crawler_platform.app.core.extractor.base import ExtractionPageContext, Extractor
+from crawler_platform.app.core.extractor.validation import attach_page_context
 
 
 @dataclass(slots=True)
@@ -23,6 +29,12 @@ class SiteCrawlPageResult:
     claim_count: int = 0
     entity_count: int = 0
     discovered_count: int = 0
+    crawl_status: str = "success"
+    extraction_status: str = "unknown"
+    raw_text_length: int = 0
+    clean_text_length: int = 0
+    removed_noise_zones_count: int = 0
+    warnings: list[str] = field(default_factory=list)
     error: str | None = None
 
 
@@ -59,12 +71,15 @@ class SiteCrawler:
         max_pages: int = 50,
         same_domain_only: bool = True,
         analyze_page_types: set[str] | None = None,
+        progress_callback: Callable[[SiteCrawlResult, SiteCrawlPageResult], None] | None = None,
+        should_stop: Callable[[], bool] | None = None,
+        parent_job_id: int | None = None,
     ) -> SiteCrawlResult:
         source_config = project_config.source_by_name(source_name)
         fetcher = make_fetcher(source_config.fetcher, source_config.rate_limit_per_minute)
         parser = self.parser_registry.get(source_config.parser)
         seed_host = normalized_host(seed_url)
-        analyze_page_types = analyze_page_types or {"product", "brand", "review"}
+        analyze_page_types = analyze_page_types or {"ProductPage", "BrandStoryPage", "ReviewPage"}
 
         result = SiteCrawlResult(seed_url=seed_url)
         queue: deque[tuple[str, int]] = deque([(normalize_url(seed_url), 0)])
@@ -75,107 +90,302 @@ class SiteCrawler:
         source = self.repository.get_source(project.id, source_name)
 
         while queue and result.visited_count < max_pages:
+            if should_stop and should_stop():
+                result.queued_count = len(queue)
+                break
+
             url, depth = queue.popleft()
             if url in visited:
                 continue
             visited.add(url)
 
-            job = self._create_job(project.id, source.id, url, depth)
+            job = self._create_job(project.id, source.id, url, depth, parent_job_id)
             if depth > max_depth:
                 self._finish_job(job, "skipped", "max depth exceeded")
                 result.skipped_count += 1
-                result.pages.append(SiteCrawlPageResult(url=url, depth=depth, status="skipped", page_type="unknown"))
+                self._record_page_result(
+                    result,
+                    SiteCrawlPageResult(url=url, depth=depth, status="skipped", page_type="unknown"),
+                    len(queue),
+                    progress_callback,
+                )
                 continue
             if same_domain_only and normalized_host(url) != seed_host:
                 self._finish_job(job, "skipped", "outside same-domain filter")
                 result.skipped_count += 1
-                result.pages.append(SiteCrawlPageResult(url=url, depth=depth, status="skipped", page_type="external"))
+                self._record_page_result(
+                    result,
+                    SiteCrawlPageResult(url=url, depth=depth, status="skipped", page_type="external"),
+                    len(queue),
+                    progress_callback,
+                )
                 continue
             if not self.robots_policy.allowed(url, source_config.respect_robots_txt):
                 error = f"robots.txt does not allow crawling: {url}"
                 self._finish_job(job, "blocked", error)
                 result.skipped_count += 1
                 result.errors.append(error)
-                result.pages.append(SiteCrawlPageResult(url=url, depth=depth, status="blocked", page_type="unknown", error=error))
+                self._record_page_result(
+                    result,
+                    SiteCrawlPageResult(url=url, depth=depth, status="blocked", page_type="unknown", error=error),
+                    len(queue),
+                    progress_callback,
+                )
                 continue
 
             try:
-                fetch_result = fetcher.fetch(url)
-                parsed = parser.parse(fetch_result.html, fetch_result.final_url or url)
-                page_type = classify_page(fetch_result.final_url or url, parsed.title, parsed.text)
-                links = discover_links(fetch_result.html, fetch_result.final_url or url, limit=200)
-                discovered_count = 0
-                if depth < max_depth:
-                    for link in links:
-                        next_url = normalize_url(link.url)
-                        if next_url in queued or next_url in visited:
-                            continue
-                        if same_domain_only and normalized_host(next_url) != seed_host:
-                            continue
-                        queue.append((next_url, depth + 1))
-                        queued.add(next_url)
-                        discovered_count += 1
-
-                if page_type in analyze_page_types:
-                    bundle = self.extractor.extract(parsed.text, project_config)
-                    page = self.repository.upsert_page(
-                        project_id=project.id,
-                        source_id=source.id,
-                        url=url,
-                        title=parsed.title,
-                        status_code=fetch_result.status_code,
-                        cleaned_text=parsed.text,
-                        metadata={
-                            **parsed.metadata,
-                            "final_url": fetch_result.final_url,
-                            "page_type": page_type,
-                            "depth": depth,
-                        },
-                    )
-                    claims = self.repository.save_extraction_bundle(project.id, source, page, bundle)
-                    self._finish_job(job, "completed")
-                    result.analyzed_count += 1
-                    result.pages.append(
-                        SiteCrawlPageResult(
-                            url=url,
-                            depth=depth,
-                            status="completed",
-                            page_type=page_type,
-                            page_id=page.id,
-                            claim_count=len(claims),
-                            entity_count=len(bundle.entities),
-                            discovered_count=discovered_count,
-                        )
-                    )
-                else:
-                    self._finish_job(job, "discovered")
-                    result.skipped_count += 1
-                    result.pages.append(
-                        SiteCrawlPageResult(
-                            url=url,
-                            depth=depth,
-                            status="discovered",
-                            page_type=page_type,
-                            discovered_count=discovered_count,
-                        )
-                    )
-                result.visited_count += 1
-                result.queued_count = len(queue)
+                self._crawl_one_page(
+                    project_config=project_config,
+                    source=source,
+                    parser=parser,
+                    fetcher=fetcher,
+                    url=url,
+                    depth=depth,
+                    max_depth=max_depth,
+                    same_domain_only=same_domain_only,
+                    seed_host=seed_host,
+                    queue=queue,
+                    queued=queued,
+                    visited=visited,
+                    analyze_page_types=analyze_page_types,
+                    result=result,
+                    job=job,
+                    progress_callback=progress_callback,
+                )
             except Exception as exc:
                 error = str(exc)
                 self._finish_job(job, "failed", error)
+                result.visited_count += 1
+                result.skipped_count += 1
                 result.errors.append(error)
-                result.pages.append(SiteCrawlPageResult(url=url, depth=depth, status="failed", page_type="unknown", error=error))
+                self._record_page_result(
+                    result,
+                    SiteCrawlPageResult(url=url, depth=depth, status="failed", page_type="unknown", error=error),
+                    len(queue),
+                    progress_callback,
+                )
 
         return result
 
-    def _create_job(self, project_id: int, source_id: int, url: str, depth: int) -> models.CrawlJob:
+    def _crawl_one_page(
+        self,
+        project_config: ProjectConfig,
+        source: models.Source,
+        parser: ParserRegistry,
+        fetcher,
+        url: str,
+        depth: int,
+        max_depth: int,
+        same_domain_only: bool,
+        seed_host: str,
+        queue: deque[tuple[str, int]],
+        queued: set[str],
+        visited: set[str],
+        analyze_page_types: set[str],
+        result: SiteCrawlResult,
+        job: models.CrawlJob,
+        progress_callback: Callable[[SiteCrawlResult, SiteCrawlPageResult], None] | None,
+    ) -> None:
+        fetch_result = fetcher.fetch(url)
+        if is_failed_fetch_status(fetch_result.status_code) or fetch_result.crawl_status != "success":
+            error = f"fetch failed with status {fetch_result.status_code}; crawl_status={fetch_result.crawl_status}"
+            page = self.repository.upsert_page(
+                project_id=source.project_id,
+                source_id=source.id,
+                url=url,
+                title=fetch_result.title,
+                status_code=fetch_result.status_code,
+                cleaned_text="",
+                metadata={
+                    "final_url": fetch_result.final_url,
+                    "crawl_status": fetch_result.crawl_status,
+                    "crawl_warnings": fetch_result.warnings,
+                },
+            )
+            self._finish_job(job, "failed", error)
+            result.visited_count += 1
+            result.skipped_count += 1
+            result.errors.append(f"{error}: {url}")
+            self._record_page_result(
+                result,
+                SiteCrawlPageResult(
+                    url=url,
+                    depth=depth,
+                    status=fetch_result.crawl_status,
+                    page_type="unknown",
+                    page_id=page.id,
+                    crawl_status=fetch_result.crawl_status,
+                    warnings=fetch_result.warnings,
+                    error=error,
+                ),
+                len(queue),
+                progress_callback,
+            )
+            return
+
+        parsed = parser.parse(fetch_result.analysis_html, fetch_result.final_url or url)
+        page_type = classify_page(
+            fetch_result.final_url or url,
+            parsed.title or fetch_result.title,
+            parsed.raw_text or parsed.text,
+            fetch_result.analysis_html,
+            parsed.source_zones or [],
+        )
+        discovered_count = self._enqueue_links(
+            html=fetch_result.analysis_html,
+            base_url=fetch_result.final_url or url,
+            depth=depth,
+            max_depth=max_depth,
+            same_domain_only=same_domain_only,
+            seed_host=seed_host,
+            queue=queue,
+            queued=queued,
+            visited=visited,
+        )
+
+        warnings = [*fetch_result.warnings, *(parsed.extraction_warnings or [])]
+        metadata = {
+            **parsed.metadata,
+            "final_url": fetch_result.final_url,
+            "crawl_status": fetch_result.crawl_status,
+            "crawl_warnings": fetch_result.warnings,
+            "page_type": page_type,
+            "depth": depth,
+            "raw_text_length": len(parsed.raw_text or ""),
+            "clean_text_length": len(parsed.text or ""),
+            "main_content_preview": (parsed.main_content or parsed.text)[:800],
+        }
+        page = self.repository.upsert_page(
+            project_id=source.project_id,
+            source_id=source.id,
+            url=url,
+            title=parsed.title or fetch_result.title,
+            status_code=fetch_result.status_code,
+            cleaned_text=parsed.text,
+            metadata=metadata,
+        )
+
+        common_page_result = {
+            "url": url,
+            "depth": depth,
+            "page_type": page_type,
+            "page_id": page.id,
+            "discovered_count": discovered_count,
+            "crawl_status": fetch_result.crawl_status,
+            "extraction_status": parsed.extraction_status,
+            "raw_text_length": len(parsed.raw_text or ""),
+            "clean_text_length": len(parsed.text or ""),
+            "removed_noise_zones_count": int(parsed.metadata.get("removed_noise_zones_count") or 0),
+            "warnings": warnings,
+        }
+
+        if parsed.extraction_status == "failed":
+            self._finish_job(job, "failed", "main content extraction failed")
+            result.visited_count += 1
+            result.skipped_count += 1
+            self._record_page_result(
+                result,
+                SiteCrawlPageResult(
+                    **common_page_result,
+                    status="extraction_failed",
+                    error="main content extraction failed",
+                ),
+                len(queue),
+                progress_callback,
+            )
+            return
+
+        if should_analyze_page(page_type, analyze_page_types):
+            context = ExtractionPageContext(
+                url=url,
+                final_url=fetch_result.final_url,
+                title=parsed.title or fetch_result.title,
+                page_type=page_type,
+                clean_text=parsed.text,
+                raw_text=parsed.raw_text,
+                main_content=parsed.main_content,
+                clean_markdown=parsed.clean_markdown,
+                source_zones=parsed.source_zones or [],
+                crawl_status=fetch_result.crawl_status,
+                extraction_status=parsed.extraction_status,
+                warnings=warnings,
+                metadata=metadata,
+            )
+            bundle = self.extractor.extract_from_context(context, project_config)
+            bundle = attach_page_context(bundle, context)
+            claims = self.repository.save_extraction_bundle(source.project_id, source, page, bundle, project_config)
+            self._finish_job(job, "completed")
+            result.analyzed_count += 1
+            page_result = SiteCrawlPageResult(
+                **common_page_result,
+                status="completed",
+                claim_count=len(claims),
+                entity_count=len(bundle.entities),
+            )
+        else:
+            self._finish_job(job, "discovered")
+            result.skipped_count += 1
+            page_result = SiteCrawlPageResult(**common_page_result, status="discovered")
+
+        result.visited_count += 1
+        self._record_page_result(result, page_result, len(queue), progress_callback)
+
+    def _enqueue_links(
+        self,
+        html: str,
+        base_url: str,
+        depth: int,
+        max_depth: int,
+        same_domain_only: bool,
+        seed_host: str,
+        queue: deque[tuple[str, int]],
+        queued: set[str],
+        visited: set[str],
+    ) -> int:
+        if depth >= max_depth:
+            return 0
+        discovered_count = 0
+        links = discover_links(html, base_url, limit=200)
+        for link in links:
+            next_url = normalize_url(link.url)
+            if next_url in queued or next_url in visited:
+                continue
+            if same_domain_only and normalized_host(next_url) != seed_host:
+                continue
+            queue.append((next_url, depth + 1))
+            queued.add(next_url)
+            discovered_count += 1
+        return discovered_count
+
+    def _record_page_result(
+        self,
+        result: SiteCrawlResult,
+        page_result: SiteCrawlPageResult,
+        queued_count: int,
+        progress_callback: Callable[[SiteCrawlResult, SiteCrawlPageResult], None] | None,
+    ) -> None:
+        result.queued_count = queued_count
+        result.pages.append(page_result)
+        if progress_callback:
+            progress_callback(result, page_result)
+
+    def _create_job(
+        self,
+        project_id: int,
+        source_id: int,
+        url: str,
+        depth: int,
+        parent_job_id: int | None = None,
+    ) -> models.CrawlJob:
         job = models.CrawlJob(
             project_id=project_id,
             source_id=source_id,
             url=url,
             status="running",
-            metadata_json={"depth": depth},
+            metadata_json={
+                "depth": depth,
+                **({"parent_job_id": parent_job_id} if parent_job_id is not None else {}),
+            },
             started_at=models.utcnow(),
         )
         self.repository.session.add(job)
@@ -198,27 +408,19 @@ def normalized_host(url: str) -> str:
     return parsed.netloc.lower()
 
 
-def classify_page(url: str, title: str | None, text: str) -> str:
-    combined = f"{url}\n{title or ''}\n{text[:3000]}".lower()
-    product_tokens = [
-        "add to cart",
-        "buy now",
-        "price",
-        "top notes",
-        "middle notes",
-        "base notes",
-        "장바구니",
-        "구매",
-        "가격",
-        "탑 노트",
-        "베이스 노트",
-    ]
-    review_tokens = ["review", "reviews", "rating", "stars", "후기", "리뷰", "평점"]
-    brand_tokens = ["about us", "brand story", "official", "브랜드", "소개"]
-    if any(token in combined for token in product_tokens):
-        return "product"
-    if any(token in combined for token in review_tokens):
-        return "review"
-    if any(token in combined for token in brand_tokens):
-        return "brand"
-    return "listing"
+def is_failed_fetch_status(status_code: int | None) -> bool:
+    return status_code is None or status_code >= 400
+
+
+def should_analyze_page(page_type: str, analyze_page_types: set[str]) -> bool:
+    return should_analyze_page_type(page_type, analyze_page_types)
+
+
+def classify_page(
+    url: str,
+    title: str | None,
+    text: str,
+    html: str | None = None,
+    source_zones: list[dict[str, object]] | None = None,
+) -> str:
+    return classify_page_type(url, title, text, html=html, source_zones=source_zones)