[페이지 분류 강화 작업]

ontology_platform/crawler_platform/app/api/routes.py

@@ -29,7 +29,10 @@ from crawler_platform.app.core.database.repository import (
     make_claim_hash,
 )
 from crawler_platform.app.core.database.session import session_scope
-from crawler_platform.app.core.extractor.ai_provider import list_openai_compatible_models
+from crawler_platform.app.core.extractor.ai_provider import (
+    list_lmstudio_loaded_models,
+    list_openai_compatible_models,
+)
 from crawler_platform.app.core.extractor.factory import extractor_for_domain
 from crawler_platform.app.core.ontology.definitions import DOMAIN_ONTOLOGIES, Ontology, ontology_for_domain
 from crawler_platform.app.core.ontology.domain_discovery import DomainDiscoveryService
@@ -1686,7 +1689,7 @@ def register_routes(app, database_url: str) -> None:
     def extractor_models(request: ExtractorModelsRequest):
         try:
             if request.provider == "lm_studio":
-                models = list_openai_compatible_models(request.base_url or "http://localhost:1234/v1")
+                models = list_lmstudio_loaded_models(request.base_url or "http://localhost:1234/v1")
                 return {"ok": True, "models": models}
             if request.provider == "openai":
                 import os

ontology_platform/crawler_platform/app/core/crawler/page_analysis_policy.py

@@ -0,0 +1,82 @@
+from __future__ import annotations
+
+from crawler_platform.app.core.crawler.page_type_scorer import PAGE_TYPE_METADATA
+from crawler_platform.app.core.crawler.page_type_taxonomy import (
+    AnalyzeStrategy,
+    LLMPolicy,
+    PageClassificationResult,
+    PageType,
+    normalize_page_type,
+    normalize_semantic_page_type,
+)
+
+
+DEFAULT_ANALYZE_STRATEGY = AnalyzeStrategy.ANALYZE_METADATA_ONLY.value
+DEFAULT_LLM_POLICY = LLMPolicy.NO_LLM.value
+
+
+def decide_analyze_strategy(result_or_page_type: object | None) -> str:
+    semantic_page_type = normalize_semantic_page_type(result_or_page_type)
+    if (
+        isinstance(result_or_page_type, PageClassificationResult)
+        and result_or_page_type.analyze_strategy
+        and result_or_page_type.analyze_strategy != DEFAULT_ANALYZE_STRATEGY
+    ):
+        return result_or_page_type.analyze_strategy
+    profile = PAGE_TYPE_METADATA.get(semantic_page_type)
+    if profile:
+        return str(profile.get("analyze_strategy") or DEFAULT_ANALYZE_STRATEGY)
+    if semantic_page_type == PageType.UNKNOWN_PAGE.value:
+        return AnalyzeStrategy.ANALYZE_METADATA_ONLY.value
+    return DEFAULT_ANALYZE_STRATEGY
+
+
+def decide_llm_policy(result_or_page_type: object | None) -> str:
+    semantic_page_type = normalize_semantic_page_type(result_or_page_type)
+    if (
+        isinstance(result_or_page_type, PageClassificationResult)
+        and result_or_page_type.llm_policy
+        and result_or_page_type.llm_policy != DEFAULT_LLM_POLICY
+    ):
+        return result_or_page_type.llm_policy
+    profile = PAGE_TYPE_METADATA.get(semantic_page_type)
+    if profile:
+        return str(profile.get("llm_policy") or DEFAULT_LLM_POLICY)
+    if semantic_page_type == PageType.UNKNOWN_PAGE.value:
+        return LLMPolicy.NO_LLM.value
+    return DEFAULT_LLM_POLICY
+
+
+def is_protected_strategy(strategy: str | AnalyzeStrategy | None) -> bool:
+    return str(strategy or "") == AnalyzeStrategy.SKIP_PROTECTED.value
+
+
+def is_noise_strategy(strategy: str | AnalyzeStrategy | None) -> bool:
+    return str(strategy or "") == AnalyzeStrategy.SKIP_NOISE.value
+
+
+def should_analyze_page(result_or_page_type: object | None, analyze_page_types: set[str] | None = None) -> bool:
+    strategy = decide_analyze_strategy(result_or_page_type)
+    if is_protected_strategy(strategy) or is_noise_strategy(strategy):
+        return False
+
+    if isinstance(result_or_page_type, PageClassificationResult):
+        return True
+
+    if analyze_page_types is None:
+        return True
+
+    normalized_page_type = normalize_page_type(result_or_page_type)
+    normalized_allowlist = {normalize_page_type(item) for item in analyze_page_types}
+    return normalized_page_type in normalized_allowlist
+
+
+def apply_analysis_policy(result: PageClassificationResult) -> PageClassificationResult:
+    strategy = decide_analyze_strategy(result)
+    llm_policy = decide_llm_policy(result)
+    result.analyze_strategy = strategy
+    result.llm_policy = llm_policy
+    result.is_protected = is_protected_strategy(strategy)
+    result.is_noise = is_noise_strategy(strategy)
+    result.should_analyze = should_analyze_page(result)
+    return result

ontology_platform/crawler_platform/app/core/crawler/page_classifier.py

@@ -11,6 +11,13 @@ from crawler_platform.app.core.crawler.page_type_taxonomy import (
 )
 from crawler_platform.app.core.crawler.page_signal_extractor import extract_page_signals_from_page
 from crawler_platform.app.core.crawler.page_type_scorer import score_page_type
+from crawler_platform.app.core.crawler.page_unknown_patterns import (
+    build_unknown_pattern_payload,
+    should_store_unknown_pattern,
+)
+from crawler_platform.app.core.crawler.page_analysis_policy import (
+    should_analyze_page as should_analyze_page_by_policy,
+)
 
 
 PRODUCT_DETAIL_PREDICATES = {
@@ -109,9 +116,15 @@ def classify_page_semantic(
     text: str = "",
     html: str | None = None,
     source_zones: list[dict[str, object]] | None = None,
+    final_url: str | None = None,
+    status_code: int | None = None,
+    content_type: str | None = None,
 ) -> PageClassificationResult:
     signals = extract_page_signals_from_page(
         url=url,
+        final_url=final_url,
+        status_code=status_code,
+        content_type=content_type,
         title=title,
         text=text,
         html=html,
@@ -135,6 +148,35 @@ def classify_page_semantic(
     return result
 
 
+def classification_metadata(
+    result: PageClassificationResult,
+    *,
+    title: str | None = None,
+    text: str | None = None,
+    html: str | None = None,
+    source_zones: list[str | dict[str, object]] | None = None,
+) -> dict[str, object]:
+    """Return metadata that preserves legacy page_type while carrying semantic evidence."""
+
+    payload = result.to_dict()
+    if should_store_unknown_pattern(result):
+        payload["unknown_pattern"] = build_unknown_pattern_payload(
+            result=result,
+            url=result.url,
+            title=title,
+            text=text,
+            html=html,
+            source_zones=source_zones,
+        )
+    return {
+        "page_type": get_legacy_page_type(result),
+        "semantic_page_type": result.primary_page_type,
+        "analyze_strategy": result.analyze_strategy,
+        "llm_policy": result.llm_policy,
+        "page_classification": payload,
+    }
+
+
 def relation_allowed_for_page_type(page_type: str | None, predicate: str) -> bool:
     if page_type in PRODUCT_DETAIL_PAGE_TYPES:
         return True
@@ -150,9 +192,7 @@ def claim_allowed_for_context(page_type: str | None, predicate: str, zone_type:
 
 
 def should_analyze_page(page_type: object, analyze_page_types: set[str] | None) -> bool:
-    normalized_page_type = normalize_page_type(page_type)
+    return should_analyze_page_by_policy(page_type, analyze_page_types)
-    normalized = {normalize_page_type(item) for item in (analyze_page_types or set())}
-    return normalized_page_type in normalized
 
 
 def _zone_type(zone: dict[str, object]) -> str:

ontology_platform/crawler_platform/app/core/crawler/page_cleaner.py

@@ -71,12 +71,27 @@ ZONE_SELECTORS: dict[str, list[str]] = {
 
 ZONE_PRIORITY_BY_PAGE_TYPE = {
     "ProductPage": ["product_title", "product_summary", "product_description", "product_detail"],
+    "ProductDetailPage": ["product_title", "product_summary", "product_description", "product_detail"],
     "BrandStoryPage": ["brand_story_body"],
+    "AboutPage": ["brand_story_body"],
+    "ContactPage": ["brand_story_body"],
     "NoticePage": ["notice_body"],
+    "PublicNoticePage": ["notice_body"],
+    "ArticlePage": ["notice_body"],
+    "NewsArticlePage": ["notice_body"],
+    "BlogPostPage": ["notice_body"],
+    "FAQPage": ["notice_body"],
+    "QAPage": ["notice_body"],
     "BoardPage": ["notice_body"],
+    "ForumBoardPage": ["notice_body"],
+    "ForumThreadPage": ["notice_body"],
     "EventPage": ["event_body"],
     "PromotionPage": ["event_body"],
+    "CampaignLandingPage": ["event_body"],
     "CategoryPage": ["product_title", "product_summary"],
+    "CategoryListingPage": ["product_title", "product_summary"],
+    "SearchPage": ["product_title", "product_summary"],
+    "SearchResultsPage": ["product_title", "product_summary"],
 }
 
 STRUCTURAL_NOISE_TOKENS = {

ontology_platform/crawler_platform/app/core/crawler/page_signal_extractor.py

@@ -0,0 +1,843 @@
+from __future__ import annotations
+
+from collections import Counter
+import json
+import re
+from typing import Any
+from urllib.parse import urljoin, urlparse
+
+from crawler_platform.app.core.crawler.page_signals import PageSignals, RawPageSnapshot
+
+
+KEYWORD_GROUPS: dict[str, tuple[str, ...]] = {
+    "commerce": (
+        "product",
+        "price",
+        "sale",
+        "cart",
+        "basket",
+        "buy",
+        "checkout",
+        "sku",
+        "상품",
+        "가격",
+        "장바구니",
+        "구매",
+        "주문",
+    ),
+    "listing": (
+        "filter",
+        "sort",
+        "category",
+        "pagination",
+        "items",
+        "results",
+        "필터",
+        "정렬",
+        "카테고리",
+        "상품수",
+        "결과",
+    ),
+    "editorial": (
+        "article",
+        "author",
+        "published",
+        "updated",
+        "headline",
+        "news",
+        "blog",
+        "기사",
+        "작성자",
+        "게시일",
+    ),
+    "community": (
+        "question",
+        "answer",
+        "comment",
+        "reply",
+        "thread",
+        "vote",
+        "faq",
+        "q&a",
+        "질문",
+        "답변",
+        "댓글",
+        "문의",
+    ),
+    "knowledge": (
+        "documentation",
+        "api",
+        "endpoint",
+        "parameter",
+        "version",
+        "reference",
+        "guide",
+        "문서",
+        "가이드",
+        "버전",
+    ),
+    "corporate": (
+        "about",
+        "company",
+        "contact",
+        "address",
+        "team",
+        "careers",
+        "privacy",
+        "terms",
+        "회사",
+        "소개",
+        "문의",
+        "주소",
+        "채용",
+        "개인정보",
+        "약관",
+    ),
+    "protected": (
+        "login",
+        "password",
+        "captcha",
+        "access denied",
+        "forbidden",
+        "payment",
+        "billing",
+        "로그인",
+        "비밀번호",
+        "보안문자",
+        "접근 제한",
+        "결제",
+    ),
+}
+
+
+PRICE_PATTERN = re.compile(
+    r"(?:[$€£¥₩]\s?\d[\d,]*(?:\.\d+)?)|(?:\d[\d,]*(?:\.\d+)?\s?(?:KRW|USD|EUR|JPY|원|달러))",
+    re.IGNORECASE,
+)
+DATE_PATTERN = re.compile(r"\b(?:20\d{2}|19\d{2})[-./년]\s?\d{1,2}[-./월]\s?\d{1,2}", re.IGNORECASE)
+API_ENDPOINT_PATTERN = re.compile(r"\b(?:GET|POST|PUT|PATCH|DELETE)\s+/(?:[A-Za-z0-9_./{}:-]+)")
+VERSION_PATTERN = re.compile(r"\bv?\d+\.\d+(?:\.\d+)?\b", re.IGNORECASE)
+
+
+def build_raw_page_snapshot(
+    *,
+    url: str,
+    final_url: str | None = None,
+    status_code: int | None = None,
+    content_type: str | None = None,
+    title: str | None = None,
+    text: str | None = None,
+    html: str | None = None,
+    rendered_html: str | None = None,
+    metadata: dict[str, Any] | None = None,
+    source_zones: list[str | dict[str, Any]] | None = None,
+    collector_payload: dict[str, Any] | None = None,
+) -> RawPageSnapshot:
+    snapshot = RawPageSnapshot(
+        url=url,
+        final_url=final_url,
+        status_code=status_code,
+        content_type=content_type,
+        title=title,
+        text=text,
+        html=html,
+        rendered_html=rendered_html,
+        metadata=dict(metadata or {}),
+        source_zones=list(source_zones or []),
+        collector_payload=dict(collector_payload or {}),
+    )
+    return merge_collector_payload(snapshot)
+
+
+def snapshot_from_collector_payload(payload: dict[str, Any], *, url: str | None = None) -> RawPageSnapshot:
+    return merge_collector_payload(
+        RawPageSnapshot(
+            url=str(url or payload.get("url") or payload.get("source_url") or ""),
+            final_url=payload.get("final_url") or payload.get("resolved_url"),
+            status_code=_optional_int(payload.get("status_code")),
+            content_type=payload.get("content_type"),
+            title=payload.get("title"),
+            text=payload.get("text") or payload.get("markdown") or payload.get("clean_text"),
+            html=payload.get("html") or payload.get("raw_html"),
+            rendered_html=payload.get("rendered_html"),
+            metadata=dict(payload.get("metadata") or {}),
+            collector_payload=dict(payload),
+        )
+    )
+
+
+def merge_collector_payload(snapshot: RawPageSnapshot) -> RawPageSnapshot:
+    payload = snapshot.collector_payload
+    if not payload:
+        return snapshot
+    snapshot.open_graph.update(payload.get("open_graph") or payload.get("og") or {})
+    snapshot.twitter_card.update(payload.get("twitter_card") or payload.get("twitter") or {})
+    snapshot.json_ld.extend(_ensure_dict_list(payload.get("json_ld") or payload.get("jsonld")))
+    snapshot.microdata.extend(_ensure_dict_list(payload.get("microdata")))
+    snapshot.rdfa.extend(_ensure_dict_list(payload.get("rdfa")))
+    snapshot.headings.extend(_string_list(payload.get("headings")))
+    snapshot.links.extend(_dict_list(payload.get("links")))
+    snapshot.images.extend(_dict_list(payload.get("images")))
+    snapshot.forms.extend(_dict_list(payload.get("forms")))
+    snapshot.buttons.extend(_string_list(payload.get("buttons")))
+    snapshot.inputs.extend(_dict_list(payload.get("inputs")))
+    snapshot.tables.extend(_dict_list(payload.get("tables")))
+    snapshot.breadcrumbs.extend(_string_list(payload.get("breadcrumbs")))
+    if payload.get("screenshot_path") and not snapshot.screenshot_path:
+        snapshot.screenshot_path = str(payload["screenshot_path"])
+    return snapshot
+
+
+def extract_page_signals_from_page(
+    *,
+    url: str,
+    final_url: str | None = None,
+    status_code: int | None = None,
+    content_type: str | None = None,
+    title: str | None = None,
+    text: str | None = None,
+    html: str | None = None,
+    rendered_html: str | None = None,
+    metadata: dict[str, Any] | None = None,
+    source_zones: list[str | dict[str, Any]] | None = None,
+    collector_payload: dict[str, Any] | None = None,
+) -> PageSignals:
+    return extract_page_signals(
+        build_raw_page_snapshot(
+            url=url,
+            final_url=final_url,
+            status_code=status_code,
+            content_type=content_type,
+            title=title,
+            text=text,
+            html=html,
+            rendered_html=rendered_html,
+            metadata=metadata,
+            source_zones=source_zones,
+            collector_payload=collector_payload,
+        )
+    )
+
+
+def extract_page_signals(snapshot: RawPageSnapshot) -> PageSignals:
+    html = snapshot.rendered_html or snapshot.html or ""
+    soup = _soup_from_html(html)
+    title = snapshot.title or _title_from_soup(soup)
+    text = snapshot.text or _text_from_soup(soup) or _text_from_html(html)
+    combined = "\n".join([snapshot.url, title or "", text or "", html[:12000]]).lower()
+    open_graph = {**_extract_meta_prefix(soup, "property", "og:"), **snapshot.open_graph}
+    twitter_card = {**_extract_meta_prefix(soup, "name", "twitter:"), **snapshot.twitter_card}
+    json_ld = [*_extract_json_ld(soup), *snapshot.json_ld]
+    links = snapshot.links or _extract_links(soup, snapshot.final_url or snapshot.url)
+    images = snapshot.images or _extract_images(soup, snapshot.final_url or snapshot.url)
+    forms = snapshot.forms or _extract_forms(soup)
+    inputs = snapshot.inputs or _extract_inputs(soup)
+    buttons = snapshot.buttons or _extract_buttons(soup)
+    headings = snapshot.headings or _extract_headings(soup)
+    tables = snapshot.tables or _extract_tables(soup)
+    breadcrumbs = snapshot.breadcrumbs or _extract_breadcrumbs(soup)
+    schema_types = _schema_types(json_ld, snapshot.microdata, snapshot.rdfa, soup)
+    keyword_hits = _keyword_hits(combined)
+    link_counts = _link_counts(links, snapshot.final_url or snapshot.url)
+    layout = _layout_signals(soup, combined, links, images, tables)
+    repeated_card_count = max(
+        layout["card_count"],
+        link_counts["product_link_count"],
+        _count_selector_matches(soup, CARD_SELECTORS),
+    )
+    button_text = " ".join(buttons).lower()
+    input_text = " ".join(_input_blob(item) for item in inputs).lower()
+    form_text = " ".join(_form_blob(item) for item in forms).lower()
+    table_text = " ".join(str(table.get("text") or "") for table in tables).lower()
+    content_type = str(snapshot.content_type or snapshot.metadata.get("content_type") or "").lower()
+    path = urlparse(snapshot.final_url or snapshot.url).path.lower()
+
+    signals = PageSignals(
+        schema_types=schema_types,
+        og_type=_string_or_none(open_graph.get("type") or open_graph.get("og:type")),
+        twitter_card_type=_string_or_none(twitter_card.get("card") or twitter_card.get("twitter:card")),
+        has_price=bool(PRICE_PATTERN.search(combined)),
+        has_currency=bool(re.search(r"[$€£¥₩]|(?:\b(?:KRW|USD|EUR|JPY)\b)|원", combined, re.IGNORECASE)),
+        has_cart_button=_contains_any(button_text + " " + combined, ("cart", "basket", "장바구니", "bag")),
+        has_buy_button=_contains_any(button_text + " " + combined, ("buy now", "purchase", "구매", "주문", "결제")),
+        has_variant_selector=_has_variant_selector(soup, input_text + " " + combined),
+        has_sku=bool(re.search(r"\bsku\b|상품\s*코드|product\s*code", combined, re.IGNORECASE)),
+        has_rating=("AggregateRating" in schema_types)
+        or _contains_any(combined, ("rating", "stars", "별점", "평점")),
+        has_review_section=("Review" in schema_types) or _contains_any(combined, ("review", "reviews", "후기", "리뷰")),
+        has_product_gallery=(len(images) >= 3 and _contains_any(combined, ("gallery", "product", "상품"))),
+        has_repeated_cards=repeated_card_count >= 3,
+        repeated_card_count=repeated_card_count,
+        has_filter_panel=layout["has_filter_sidebar"]
+        or _contains_any(combined, ("filter", "facets", "refine", "필터", "조건")),
+        has_sort_control=_contains_any(combined, ("sort", "order by", "low price", "high price", "정렬", "낮은가격", "높은가격")),
+        has_pagination=_has_pagination(soup, links, combined),
+        has_author=_contains_any(combined, ("author", "byline", "작성자", "기자")),
+        has_published_date=("datePublished" in _json_keys(json_ld))
+        or bool(DATE_PATTERN.search(combined))
+        and _contains_any(combined, ("published", "posted", "게시", "등록")),
+        has_modified_date=("dateModified" in _json_keys(json_ld))
+        or _contains_any(combined, ("modified", "updated", "수정")),
+        has_article_body=("Article" in schema_types)
+        or ("NewsArticle" in schema_types)
+        or _count_selector_matches(soup, ("article", "[itemprop='articleBody']", ".article-body", ".post-content")) > 0
+        or _contains_any(combined, ("articlebody", "article body")),
+        has_tags=_has_tags(soup, links, combined),
+        has_question=("QAPage" in schema_types) or _contains_any(combined, ("question", "q:", "질문", "문의")),
+        has_answer=("Answer" in schema_types) or _contains_any(combined, ("answer", "a:", "답변")),
+        has_comments=_contains_any(combined, ("comment", "comments", "reply", "댓글", "답글")),
+        has_votes=_contains_any(combined, ("vote", "votes", "upvote", "downvote", "추천", "투표")),
+        has_thread_structure=_contains_any(combined, ("thread", "discussion", "게시글", "토론")),
+        has_faq_structure=("FAQPage" in schema_types) or _contains_any(combined, ("faq", "frequently asked", "자주 묻는")),
+        has_code_blocks=_count_selector_matches(soup, ("pre", "code", ".highlight", ".code")) > 0,
+        has_toc=_count_selector_matches(soup, ("#toc", ".toc", "[class*='table-of-contents']", "nav[aria-label*='contents']")) > 0,
+        has_api_endpoint=bool(API_ENDPOINT_PATTERN.search(f"{text or ''}\n{html or ''}")),
+        has_parameter_table=_has_parameter_table(tables, table_text),
+        has_version_info=bool(VERSION_PATTERN.search(combined)) and _contains_any(combined, ("version", "버전", "release")),
+        has_contact_info=_contains_any(combined, ("contact", "email", "tel:", "문의", "연락처")),
+        has_address=_contains_any(combined, ("address", "주소", "road", "street")),
+        has_policy_terms=_contains_any(combined, ("terms", "policy", "agreement", "약관", "정책")),
+        has_privacy_terms=_contains_any(combined, ("privacy", "personal information", "개인정보")),
+        has_career_terms=_contains_any(combined, ("career", "jobs", "recruit", "채용", "지원")),
+        has_login_form=_contains_any(form_text + " " + combined, ("login", "sign in", "로그인")) and (
+            "password" in input_text or "비밀번호" in combined
+        ),
+        has_password_field="password" in input_text,
+        has_payment_fields=_contains_any(input_text + " " + combined, ("card number", "payment", "billing", "결제", "카드")),
+        has_captcha=_contains_any(combined, ("captcha", "recaptcha", "hcaptcha", "보안문자")),
+        has_access_denied=_contains_any(combined, ("access denied", "forbidden", "permission denied", "접근 제한", "권한이 없습니다")),
+        status_code=snapshot.status_code,
+        has_error_status=bool(snapshot.status_code and snapshot.status_code >= 400)
+        or _contains_any(combined, ("404", "not found", "error page")),
+        has_not_found=snapshot.status_code == 404 or _contains_any(combined, ("404", "not found", "page not found")),
+        has_sitemap_resource=("sitemap" in path) or ("sitemap" in content_type and "xml" in content_type),
+        has_feed_resource=("rss" in content_type) or ("atom" in content_type) or path.endswith((".rss", ".atom")),
+        has_json_resource=("json" in content_type) or path.endswith(".json"),
+        has_xml_resource=("xml" in content_type) or path.endswith(".xml"),
+        has_file_resource=path.endswith((".pdf", ".csv", ".xlsx", ".xls", ".doc", ".docx", ".zip")),
+        internal_link_count=link_counts["internal_link_count"],
+        external_link_count=link_counts["external_link_count"],
+        product_link_count=link_counts["product_link_count"],
+        category_link_count=link_counts["category_link_count"],
+        profile_link_count=link_counts["profile_link_count"],
+        article_link_count=link_counts["article_link_count"],
+        layout_blocks=layout["layout_blocks"],
+        has_hero_block=layout["has_hero_block"],
+        has_card_grid=layout["has_card_grid"],
+        has_filter_sidebar=layout["has_filter_sidebar"],
+        has_sticky_action_box=layout["has_sticky_action_box"],
+        has_media_player_area=layout["has_media_player_area"],
+        has_map_area=layout["has_map_area"],
+        has_calendar_grid=layout["has_calendar_grid"],
+        has_pricing_table=layout["has_pricing_table"],
+        has_comparison_table=layout["has_comparison_table"],
+        dominant_language=_dominant_language(combined),
+        keyword_hits=keyword_hits,
+        url_hints=_url_hints(snapshot.final_url or snapshot.url),
+        title=title,
+        text_sample=(text or "")[:500],
+        external_collector_signals=_external_collector_signals(snapshot),
+    )
+    return signals
+
+
+def _soup_from_html(html: str):
+    if not html:
+        return None
+    try:
+        from bs4 import BeautifulSoup
+    except ImportError:
+        return None
+    try:
+        return BeautifulSoup(html, "html.parser")
+    except Exception:
+        return None
+
+
+def _title_from_soup(soup) -> str | None:
+    if soup is None or not soup.title:
+        return None
+    return soup.title.get_text(" ", strip=True) or None
+
+
+def _text_from_soup(soup) -> str:
+    if soup is None:
+        return ""
+    return soup.get_text("\n", strip=True)
+
+
+def _text_from_html(html: str) -> str:
+    return re.sub(r"\s+", " ", re.sub(r"<[^>]+>", " ", html or "")).strip()
+
+
+def _extract_meta_prefix(soup, attr_name: str, prefix: str) -> dict[str, str]:
+    if soup is None:
+        return {}
+    result: dict[str, str] = {}
+    for tag in soup.find_all("meta"):
+        name = str(tag.get(attr_name) or "").strip()
+        if not name.lower().startswith(prefix):
+            continue
+        content = str(tag.get("content") or "").strip()
+        if content:
+            result[name.removeprefix(prefix)] = content
+            result[name] = content
+    return result
+
+
+def _extract_json_ld(soup) -> list[dict[str, Any]]:
+    if soup is None:
+        return []
+    payloads: list[dict[str, Any]] = []
+    for tag in soup.find_all("script"):
+        script_type = str(tag.get("type") or "").lower()
+        if "ld+json" not in script_type:
+            continue
+        raw = tag.string or tag.get_text(" ", strip=True)
+        try:
+            value = json.loads(raw)
+        except Exception:
+            continue
+        payloads.extend(_ensure_dict_list(value))
+    return payloads
+
+
+def _extract_headings(soup) -> list[str]:
+    if soup is None:
+        return []
+    return [node.get_text(" ", strip=True) for node in soup.select("h1,h2,h3") if node.get_text(" ", strip=True)]
+
+
+def _extract_links(soup, base_url: str) -> list[dict[str, Any]]:
+    if soup is None:
+        return []
+    links = []
+    for tag in soup.find_all("a"):
+        href = str(tag.get("href") or "").strip()
+        if not href:
+            continue
+        links.append(
+            {
+                "href": urljoin(base_url, href),
+                "text": tag.get_text(" ", strip=True),
+                "rel": " ".join(str(item) for item in tag.get("rel", [])),
+                "class": " ".join(str(item) for item in tag.get("class", [])),
+            }
+        )
+    return links
+
+
+def _extract_images(soup, base_url: str) -> list[dict[str, Any]]:
+    if soup is None:
+        return []
+    images = []
+    for tag in soup.find_all("img"):
+        src = str(tag.get("src") or tag.get("data-src") or "").strip()
+        if not src:
+            continue
+        images.append(
+            {
+                "src": urljoin(base_url, src),
+                "alt": str(tag.get("alt") or ""),
+                "class": " ".join(str(item) for item in tag.get("class", [])),
+            }
+        )
+    return images
+
+
+def _extract_forms(soup) -> list[dict[str, Any]]:
+    if soup is None:
+        return []
+    forms = []
+    for form in soup.find_all("form"):
+        forms.append(
+            {
+                "action": str(form.get("action") or ""),
+                "method": str(form.get("method") or ""),
+                "id": str(form.get("id") or ""),
+                "class": " ".join(str(item) for item in form.get("class", [])),
+                "text": form.get_text(" ", strip=True)[:500],
+            }
+        )
+    return forms
+
+
+def _extract_inputs(soup) -> list[dict[str, Any]]:
+    if soup is None:
+        return []
+    inputs = []
+    for tag in soup.select("input,select,textarea"):
+        inputs.append(
+            {
+                "type": str(tag.get("type") or tag.name or ""),
+                "name": str(tag.get("name") or ""),
+                "id": str(tag.get("id") or ""),
+                "placeholder": str(tag.get("placeholder") or ""),
+                "autocomplete": str(tag.get("autocomplete") or ""),
+                "aria_label": str(tag.get("aria-label") or ""),
+                "class": " ".join(str(item) for item in tag.get("class", [])),
+                "text": tag.get_text(" ", strip=True)[:240],
+            }
+        )
+    return inputs
+
+
+def _extract_buttons(soup) -> list[str]:
+    if soup is None:
+        return []
+    values = []
+    for tag in soup.select("button,input[type='submit'],input[type='button'],[role='button']"):
+        text = tag.get_text(" ", strip=True) or str(tag.get("value") or tag.get("aria-label") or "")
+        if text.strip():
+            values.append(text.strip())
+    return values
+
+
+def _extract_tables(soup) -> list[dict[str, Any]]:
+    if soup is None:
+        return []
+    tables = []
+    for table in soup.find_all("table"):
+        headers = [cell.get_text(" ", strip=True) for cell in table.select("th") if cell.get_text(" ", strip=True)]
+        text = table.get_text(" ", strip=True)
+        tables.append({"headers": headers, "text": text[:1000], "row_count": len(table.select("tr"))})
+    return tables
+
+
+def _extract_breadcrumbs(soup) -> list[str]:
+    if soup is None:
+        return []
+    crumbs = []
+    selectors = [
+        "[class*='breadcrumb']",
+        "[id*='breadcrumb']",
+        "nav[aria-label*='breadcrumb' i]",
+        "[itemtype*='BreadcrumbList']",
+    ]
+    for selector in selectors:
+        for node in soup.select(selector):
+            text = node.get_text(" > ", strip=True)
+            if text:
+                crumbs.append(text)
+    return _dedupe_strings(crumbs)
+
+
+def _schema_types(
+    json_ld: list[dict[str, Any]],
+    microdata: list[dict[str, Any]],
+    rdfa: list[dict[str, Any]],
+    soup,
+) -> set[str]:
+    types: set[str] = set()
+    for payload in [*json_ld, *microdata, *rdfa]:
+        _visit_schema_types(payload, types)
+    if soup is not None:
+        for node in soup.select("[itemscope][itemtype]"):
+            raw = str(node.get("itemtype") or "")
+            if raw:
+                types.add(raw.rstrip("/").split("/")[-1])
+        for node in soup.select("[typeof]"):
+            for item in str(node.get("typeof") or "").split():
+                types.add(item.split(":")[-1])
+    return {item for item in types if item}
+
+
+def _visit_schema_types(value: Any, types: set[str]) -> None:
+    if isinstance(value, list):
+        for item in value:
+            _visit_schema_types(item, types)
+        return
+    if not isinstance(value, dict):
+        return
+    raw_type = value.get("@type") or value.get("type")
+    for item in _ensure_list(raw_type):
+        if isinstance(item, str):
+            types.add(item.rstrip("/").split("/")[-1])
+    for nested in value.values():
+        if isinstance(nested, (dict, list)):
+            _visit_schema_types(nested, types)
+
+
+def _json_keys(json_ld: list[dict[str, Any]]) -> set[str]:
+    keys: set[str] = set()
+
+    def visit(value: Any) -> None:
+        if isinstance(value, list):
+            for item in value:
+                visit(item)
+            return
+        if not isinstance(value, dict):
+            return
+        keys.update(str(key) for key in value)
+        for nested in value.values():
+            visit(nested)
+
+    visit(json_ld)
+    return keys
+
+
+def _keyword_hits(text: str) -> dict[str, int]:
+    hits = {}
+    for group, terms in KEYWORD_GROUPS.items():
+        count = sum(text.count(term.lower()) for term in terms)
+        if count:
+            hits[group] = count
+    return hits
+
+
+def _link_counts(links: list[dict[str, Any]], base_url: str) -> dict[str, int]:
+    base = urlparse(base_url)
+    base_host = base.netloc.lower()
+    base_path = base.path.lower()
+    counts = Counter()
+    for link in links:
+        href = str(link.get("href") or "")
+        parsed = urlparse(href)
+        host = parsed.netloc.lower()
+        path = parsed.path.lower()
+        if not host or host == base_host:
+            counts["internal_link_count"] += 1
+        else:
+            counts["external_link_count"] += 1
+        same_document_query_link = path == base_path and bool(parsed.query)
+        if not same_document_query_link and any(token in path for token in ("/product", "/products", "/goods", "/item", "/p/")):
+            counts["product_link_count"] += 1
+        if any(token in path for token in ("category", "collection", "/shop", "/list", "catalog")):
+            counts["category_link_count"] += 1
+        if any(token in path for token in ("profile", "user", "author", "member", "creator")):
+            counts["profile_link_count"] += 1
+        if any(token in path for token in ("article", "blog", "news", "post", "story")):
+            counts["article_link_count"] += 1
+    return {
+        "internal_link_count": counts["internal_link_count"],
+        "external_link_count": counts["external_link_count"],
+        "product_link_count": counts["product_link_count"],
+        "category_link_count": counts["category_link_count"],
+        "profile_link_count": counts["profile_link_count"],
+        "article_link_count": counts["article_link_count"],
+    }
+
+
+CARD_SELECTORS = (
+    ".product-card",
+    ".card",
+    ".item",
+    ".product",
+    ".prdList > li",
+    "[class*='product-card']",
+    "[class*='grid-item']",
+)
+
+
+def _layout_signals(soup, text: str, links: list[dict[str, Any]], images: list[dict[str, Any]], tables: list[dict[str, Any]]) -> dict[str, Any]:
+    blocks: list[str] = []
+    hero = _count_selector_matches(soup, (".hero", ".visual", ".main-visual", "[class*='hero']", "section[aria-label*='hero']")) > 0
+    card_count = _count_selector_matches(soup, CARD_SELECTORS)
+    card_grid = card_count >= 3 or _count_selector_matches(soup, (".grid", "[class*='grid']", "[class*='cards']")) > 0 and len(links) >= 3
+    filter_sidebar = _count_selector_matches(soup, (".filter", ".filters", ".facet", "aside", "[class*='filter']", "[class*='facet']")) > 0
+    sticky_action = _count_selector_matches(soup, (".sticky", "[class*='sticky']", "[class*='fixed']", "[class*='buy-box']")) > 0
+    media_player = _count_selector_matches(soup, ("video", "audio", "iframe[src*='youtube']", "[class*='player']")) > 0
+    map_area = _count_selector_matches(soup, ("[class*='map']", "#map", "iframe[src*='maps']")) > 0 or "google map" in text
+    calendar_grid = _count_selector_matches(soup, ("[class*='calendar']", "[class*='datepicker']", "table.calendar")) > 0
+    pricing_table = "pricing" in text and (bool(tables) or _count_selector_matches(soup, ("[class*='pricing']", ".price-table")) > 0)
+    comparison_table = _has_comparison_table(tables, text)
+    for label, present in [
+        ("hero_block", hero),
+        ("card_grid", card_grid),
+        ("filter_sidebar", filter_sidebar),
+        ("sticky_action_box", sticky_action),
+        ("media_player_area", media_player),
+        ("map_area", map_area),
+        ("calendar_grid", calendar_grid),
+        ("pricing_table", pricing_table),
+        ("comparison_table", comparison_table),
+    ]:
+        if present:
+            blocks.append(label)
+    return {
+        "layout_blocks": blocks,
+        "card_count": card_count,
+        "has_hero_block": hero,
+        "has_card_grid": card_grid,
+        "has_filter_sidebar": filter_sidebar,
+        "has_sticky_action_box": sticky_action,
+        "has_media_player_area": media_player,
+        "has_map_area": map_area,
+        "has_calendar_grid": calendar_grid,
+        "has_pricing_table": pricing_table,
+        "has_comparison_table": comparison_table,
+    }
+
+
+def _has_variant_selector(soup, text: str) -> bool:
+    if _contains_any(text, ("variant", "option", "size", "color", "옵션", "사이즈", "색상")):
+        return True
+    if soup is None:
+        return False
+    for select in soup.find_all("select"):
+        blob = " ".join(
+            [
+                str(select.get("name") or ""),
+                str(select.get("id") or ""),
+                select.get_text(" ", strip=True),
+            ]
+        ).lower()
+        if _contains_any(blob, ("variant", "option", "size", "color", "옵션", "사이즈", "색상")):
+            return True
+    return False
+
+
+def _has_pagination(soup, links: list[dict[str, Any]], text: str) -> bool:
+    if _count_selector_matches(soup, (".pagination", ".paging", "[class*='paginate']", "nav[aria-label*='pagination']")) > 0:
+        return True
+    if any(str(link.get("rel") or "").lower() in {"next", "prev", "previous"} for link in links):
+        return True
+    return _contains_any(text, ("next page", "previous page", "페이지", "다음", "이전"))
+
+
+def _has_tags(soup, links: list[dict[str, Any]], text: str) -> bool:
+    if _count_selector_matches(soup, (".tag", ".tags", "[rel='tag']", "[class*='tag']")) > 0:
+        return True
+    return any(str(link.get("rel") or "").lower() == "tag" for link in links) or _contains_any(text, ("tags:", "태그"))
+
+
+def _has_parameter_table(tables: list[dict[str, Any]], table_text: str) -> bool:
+    if _contains_any(table_text, ("parameter", "required", "type", "description", "파라미터", "필수")):
+        return True
+    for table in tables:
+        headers = " ".join(str(item) for item in table.get("headers") or []).lower()
+        if _contains_any(headers, ("parameter", "required", "type", "description", "파라미터", "필수")):
+            return True
+    return False
+
+
+def _has_comparison_table(tables: list[dict[str, Any]], text: str) -> bool:
+    if not tables:
+        return False
+    return _contains_any(text, ("compare", "comparison", "vs", "비교")) or any(
+        int(table.get("row_count") or 0) >= 3 and len(table.get("headers") or []) >= 3 for table in tables
+    )
+
+
+def _url_hints(url: str) -> set[str]:
+    parsed = urlparse(url)
+    value = f"{parsed.path} {parsed.query}".lower()
+    hints = set()
+    for hint, terms in {
+        "product": ("/product", "/products", "/goods", "/item", "/p/"),
+        "category": ("category", "collection", "/shop", "/list", "catalog"),
+        "search": ("search", "find", "keyword=", "query=", "q="),
+        "article": ("article", "blog", "news", "post", "story"),
+        "board": ("board", "forum", "thread", "qna"),
+        "protected": ("login", "checkout", "payment", "account", "cart"),
+        "system": ("sitemap", "robots.txt", ".json", ".xml", ".rss"),
+    }.items():
+        if any(term in value for term in terms):
+            hints.add(hint)
+    return hints
+
+
+def _dominant_language(text: str) -> str | None:
+    if not text:
+        return None
+    korean = len(re.findall(r"[가-힣]", text))
+    latin = len(re.findall(r"[A-Za-z]", text))
+    if korean == 0 and latin == 0:
+        return None
+    if korean > latin * 0.25:
+        return "ko"
+    return "en"
+
+
+def _external_collector_signals(snapshot: RawPageSnapshot) -> dict[str, Any]:
+    payload = dict(snapshot.collector_payload or {})
+    for key in {
+        "html",
+        "raw_html",
+        "rendered_html",
+        "text",
+        "markdown",
+        "clean_text",
+        "metadata",
+        "links",
+        "images",
+        "forms",
+        "inputs",
+        "buttons",
+        "tables",
+        "json_ld",
+        "jsonld",
+    }:
+        payload.pop(key, None)
+    return payload
+
+
+def _count_selector_matches(soup, selectors: tuple[str, ...]) -> int:
+    if soup is None:
+        return 0
+    count = 0
+    for selector in selectors:
+        try:
+            count += len(soup.select(selector))
+        except Exception:
+            continue
+    return count
+
+
+def _input_blob(item: dict[str, Any]) -> str:
+    return " ".join(str(item.get(key) or "") for key in ("type", "name", "id", "placeholder", "autocomplete", "aria_label", "class", "text"))
+
+
+def _form_blob(item: dict[str, Any]) -> str:
+    return " ".join(str(item.get(key) or "") for key in ("action", "method", "id", "class", "text"))
+
+
+def _contains_any(value: str, terms: tuple[str, ...]) -> bool:
+    lowered = value.lower()
+    return any(term.lower() in lowered for term in terms)
+
+
+def _string_or_none(value: Any) -> str | None:
+    clean = str(value or "").strip()
+    return clean or None
+
+
+def _optional_int(value: Any) -> int | None:
+    try:
+        return int(value)
+    except (TypeError, ValueError):
+        return None
+
+
+def _ensure_list(value: Any) -> list[Any]:
+    if value is None:
+        return []
+    if isinstance(value, list):
+        return value
+    return [value]
+
+
+def _ensure_dict_list(value: Any) -> list[dict[str, Any]]:
+    values = _ensure_list(value)
+    return [item for item in values if isinstance(item, dict)]
+
+
+def _dict_list(value: Any) -> list[dict[str, Any]]:
+    return [dict(item) for item in _ensure_list(value) if isinstance(item, dict)]
+
+
+def _string_list(value: Any) -> list[str]:
+    return [str(item) for item in _ensure_list(value) if str(item or "").strip()]
+
+
+def _dedupe_strings(values: list[str]) -> list[str]:
+    seen = set()
+    result = []
+    for value in values:
+        clean = " ".join(value.split())
+        key = clean.lower()
+        if not clean or key in seen:
+            continue
+        seen.add(key)
+        result.append(clean)
+    return result

ontology_platform/crawler_platform/app/core/crawler/page_signals.py

@@ -0,0 +1,144 @@
+from __future__ import annotations
+
+from dataclasses import asdict, dataclass, field
+from typing import Any
+
+
+@dataclass(slots=True)
+class RawPageSnapshot:
+    url: str
+    final_url: str | None = None
+    status_code: int | None = None
+    content_type: str | None = None
+    title: str | None = None
+    text: str | None = None
+    html: str | None = None
+    rendered_html: str | None = None
+    metadata: dict[str, Any] = field(default_factory=dict)
+    open_graph: dict[str, Any] = field(default_factory=dict)
+    twitter_card: dict[str, Any] = field(default_factory=dict)
+    json_ld: list[dict[str, Any]] = field(default_factory=list)
+    microdata: list[dict[str, Any]] = field(default_factory=list)
+    rdfa: list[dict[str, Any]] = field(default_factory=list)
+    headings: list[str] = field(default_factory=list)
+    links: list[dict[str, Any]] = field(default_factory=list)
+    images: list[dict[str, Any]] = field(default_factory=list)
+    forms: list[dict[str, Any]] = field(default_factory=list)
+    buttons: list[str] = field(default_factory=list)
+    inputs: list[dict[str, Any]] = field(default_factory=list)
+    tables: list[dict[str, Any]] = field(default_factory=list)
+    breadcrumbs: list[str] = field(default_factory=list)
+    source_zones: list[str | dict[str, Any]] = field(default_factory=list)
+    screenshot_path: str | None = None
+    collector_payload: dict[str, Any] = field(default_factory=dict)
+
+    def to_dict(self) -> dict[str, Any]:
+        return asdict(self)
+
+
+@dataclass(slots=True)
+class PageSignals:
+    # structured data
+    schema_types: set[str] = field(default_factory=set)
+    og_type: str | None = None
+    twitter_card_type: str | None = None
+
+    # commerce
+    has_price: bool = False
+    has_currency: bool = False
+    has_cart_button: bool = False
+    has_buy_button: bool = False
+    has_variant_selector: bool = False
+    has_sku: bool = False
+    has_rating: bool = False
+    has_review_section: bool = False
+    has_product_gallery: bool = False
+
+    # listing
+    has_repeated_cards: bool = False
+    repeated_card_count: int = 0
+    has_filter_panel: bool = False
+    has_sort_control: bool = False
+    has_pagination: bool = False
+
+    # editorial
+    has_author: bool = False
+    has_published_date: bool = False
+    has_modified_date: bool = False
+    has_article_body: bool = False
+    has_tags: bool = False
+
+    # community
+    has_question: bool = False
+    has_answer: bool = False
+    has_comments: bool = False
+    has_votes: bool = False
+    has_thread_structure: bool = False
+    has_faq_structure: bool = False
+
+    # knowledge/docs
+    has_code_blocks: bool = False
+    has_toc: bool = False
+    has_api_endpoint: bool = False
+    has_parameter_table: bool = False
+    has_version_info: bool = False
+
+    # corporate/legal
+    has_contact_info: bool = False
+    has_address: bool = False
+    has_policy_terms: bool = False
+    has_privacy_terms: bool = False
+    has_career_terms: bool = False
+
+    # transaction/protected
+    has_login_form: bool = False
+    has_password_field: bool = False
+    has_payment_fields: bool = False
+    has_captcha: bool = False
+    has_access_denied: bool = False
+
+    # system/resource
+    status_code: int | None = None
+    has_error_status: bool = False
+    has_not_found: bool = False
+    has_sitemap_resource: bool = False
+    has_feed_resource: bool = False
+    has_json_resource: bool = False
+    has_xml_resource: bool = False
+    has_file_resource: bool = False
+
+    # graph
+    internal_link_count: int = 0
+    external_link_count: int = 0
+    product_link_count: int = 0
+    category_link_count: int = 0
+    profile_link_count: int = 0
+    article_link_count: int = 0
+
+    # visual/layout candidates from DOM structure
+    layout_blocks: list[str] = field(default_factory=list)
+    has_hero_block: bool = False
+    has_card_grid: bool = False
+    has_filter_sidebar: bool = False
+    has_sticky_action_box: bool = False
+    has_media_player_area: bool = False
+    has_map_area: bool = False
+    has_calendar_grid: bool = False
+    has_pricing_table: bool = False
+    has_comparison_table: bool = False
+
+    # text/layout
+    dominant_language: str | None = None
+    keyword_hits: dict[str, int] = field(default_factory=dict)
+    url_hints: set[str] = field(default_factory=set)
+    title: str | None = None
+    text_sample: str = ""
+
+    # external collector hook
+    external_collector_signals: dict[str, Any] = field(default_factory=dict)
+
+    def to_dict(self) -> dict[str, Any]:
+        payload = asdict(self)
+        payload["schema_types"] = sorted(self.schema_types)
+        payload["url_hints"] = sorted(self.url_hints)
+        return payload

ontology_platform/crawler_platform/app/core/crawler/page_type_scorer.py

@@ -0,0 +1,747 @@
+from __future__ import annotations
+
+from collections.abc import Callable
+from dataclasses import dataclass
+from typing import Any
+
+from crawler_platform.app.core.crawler.page_signals import PageSignals
+from crawler_platform.app.core.crawler.page_type_taxonomy import (
+    ActionIntent,
+    AnalyzeStrategy,
+    EntityType,
+    EvidenceItem,
+    GraphRole,
+    LLMPolicy,
+    PAGE_TYPE_PROFILES,
+    PageArchetype,
+    PageClassificationResult,
+    PageDomain,
+    PageType,
+)
+
+
+UNKNOWN_THRESHOLD = 0.22
+
+
+@dataclass(frozen=True, slots=True)
+class SignalRule:
+    key: str
+    weight: float
+    source: str
+    message: str
+    predicate: Callable[[PageSignals], bool]
+    value: Callable[[PageSignals], str | int | float | bool | None] | None = None
+
+
+@dataclass(slots=True)
+class ScoreAccumulator:
+    scores: dict[str, float]
+    evidence_by_type: dict[str, list[EvidenceItem]]
+
+    def add(self, page_type: str, rule: SignalRule, signals: PageSignals) -> None:
+        if not rule.predicate(signals):
+            return
+        value = rule.value(signals) if rule.value else True
+        self.scores[page_type] = self.scores.get(page_type, 0.0) + rule.weight
+        self.evidence_by_type.setdefault(page_type, []).append(
+            EvidenceItem(
+                key=rule.key,
+                value=value,
+                weight=rule.weight,
+                source=rule.source,
+                message=rule.message,
+            )
+        )
+
+
+PAGE_TYPE_METADATA: dict[str, dict[str, Any]] = {
+    **PAGE_TYPE_PROFILES,
+    PageType.ARTICLE_PAGE.value: {
+        "domain": PageDomain.EDITORIAL.value,
+        "archetype": PageArchetype.ARTICLE.value,
+        "main_entity_type": EntityType.ARTICLE.value,
+        "action_intents": [ActionIntent.READ.value],
+        "graph_roles": [GraphRole.CLAIM_SOURCE.value, GraphRole.ENTITY_ANCHOR.value],
+        "analyze_strategy": AnalyzeStrategy.ANALYZE_FULL.value,
+        "llm_policy": LLMPolicy.LLM_FULL.value,
+        "should_analyze": True,
+    },
+    PageType.BLOG_POST_PAGE.value: {
+        "domain": PageDomain.EDITORIAL.value,
+        "archetype": PageArchetype.ARTICLE.value,
+        "main_entity_type": EntityType.ARTICLE.value,
+        "action_intents": [ActionIntent.READ.value],
+        "graph_roles": [GraphRole.CLAIM_SOURCE.value],
+        "analyze_strategy": AnalyzeStrategy.ANALYZE_FULL.value,
+        "llm_policy": LLMPolicy.LLM_FULL.value,
+        "should_analyze": True,
+    },
+    PageType.FAQ_PAGE.value: {
+        "domain": PageDomain.COMMUNITY.value,
+        "archetype": PageArchetype.DOCUMENT.value,
+        "main_entity_type": EntityType.QUESTION.value,
+        "action_intents": [ActionIntent.READ.value, ActionIntent.ANSWER.value],
+        "graph_roles": [GraphRole.REFERENCE_SOURCE.value],
+        "analyze_strategy": AnalyzeStrategy.ANALYZE_FULL.value,
+        "llm_policy": LLMPolicy.LLM_LIGHT.value,
+        "should_analyze": True,
+    },
+    PageType.QA_PAGE.value: {
+        "domain": PageDomain.COMMUNITY.value,
+        "archetype": PageArchetype.THREAD.value,
+        "main_entity_type": EntityType.QUESTION.value,
+        "action_intents": [ActionIntent.ASK.value, ActionIntent.ANSWER.value, ActionIntent.READ.value],
+        "graph_roles": [GraphRole.CLAIM_SOURCE.value, GraphRole.RELATION_HUB.value],
+        "analyze_strategy": AnalyzeStrategy.ANALYZE_FULL.value,
+        "llm_policy": LLMPolicy.LLM_LIGHT.value,
+        "should_analyze": True,
+    },
+    PageType.FORUM_THREAD_PAGE.value: {
+        "domain": PageDomain.COMMUNITY.value,
+        "archetype": PageArchetype.THREAD.value,
+        "main_entity_type": EntityType.ARTICLE.value,
+        "action_intents": [ActionIntent.READ.value, ActionIntent.COMMENT.value],
+        "graph_roles": [GraphRole.CLAIM_SOURCE.value, GraphRole.RELATION_HUB.value],
+        "analyze_strategy": AnalyzeStrategy.ANALYZE_FULL.value,
+        "llm_policy": LLMPolicy.LLM_LIGHT.value,
+        "should_analyze": True,
+    },
+    PageType.ABOUT_PAGE.value: {
+        "domain": PageDomain.CORPORATE.value,
+        "archetype": PageArchetype.ARTICLE.value,
+        "main_entity_type": EntityType.ORGANIZATION.value,
+        "action_intents": [ActionIntent.READ.value, ActionIntent.LEARN.value],
+        "graph_roles": [GraphRole.ENTITY_ANCHOR.value, GraphRole.CLAIM_SOURCE.value],
+        "analyze_strategy": AnalyzeStrategy.ANALYZE_FULL.value,
+        "llm_policy": LLMPolicy.LLM_FULL.value,
+        "should_analyze": True,
+    },
+    PageType.CONTACT_PAGE.value: {
+        "domain": PageDomain.CORPORATE.value,
+        "archetype": PageArchetype.DOCUMENT.value,
+        "main_entity_type": EntityType.ORGANIZATION.value,
+        "action_intents": [ActionIntent.CONTACT.value, ActionIntent.NAVIGATE.value],
+        "graph_roles": [GraphRole.ENTITY_ANCHOR.value],
+        "analyze_strategy": AnalyzeStrategy.ANALYZE_METADATA_ONLY.value,
+        "llm_policy": LLMPolicy.RULE_ONLY.value,
+        "should_analyze": True,
+    },
+    PageType.DOCUMENTATION_PAGE.value: {
+        "domain": PageDomain.KNOWLEDGE.value,
+        "archetype": PageArchetype.DOCUMENT.value,
+        "main_entity_type": EntityType.SOFTWARE_APPLICATION.value,
+        "action_intents": [ActionIntent.READ.value, ActionIntent.LEARN.value],
+        "graph_roles": [GraphRole.REFERENCE_SOURCE.value],
+        "analyze_strategy": AnalyzeStrategy.ANALYZE_FULL.value,
+        "llm_policy": LLMPolicy.LLM_LIGHT.value,
+        "should_analyze": True,
+    },
+    PageType.API_REFERENCE_PAGE.value: {
+        "domain": PageDomain.SOFTWARE.value,
+        "archetype": PageArchetype.DOCUMENT.value,
+        "main_entity_type": EntityType.SOFTWARE_APPLICATION.value,
+        "action_intents": [ActionIntent.LEARN.value, ActionIntent.CONFIGURE.value],
+        "graph_roles": [GraphRole.REFERENCE_SOURCE.value],
+        "analyze_strategy": AnalyzeStrategy.ANALYZE_FULL.value,
+        "llm_policy": LLMPolicy.LLM_LIGHT.value,
+        "should_analyze": True,
+    },
+    PageType.WIKI_PAGE.value: {
+        "domain": PageDomain.KNOWLEDGE.value,
+        "archetype": PageArchetype.DOCUMENT.value,
+        "main_entity_type": EntityType.UNKNOWN_ENTITY.value,
+        "action_intents": [ActionIntent.READ.value, ActionIntent.LEARN.value],
+        "graph_roles": [GraphRole.REFERENCE_SOURCE.value],
+        "analyze_strategy": AnalyzeStrategy.ANALYZE_FULL.value,
+        "llm_policy": LLMPolicy.LLM_LIGHT.value,
+        "should_analyze": True,
+    },
+    PageType.DATASET_PAGE.value: {
+        "domain": PageDomain.KNOWLEDGE.value,
+        "archetype": PageArchetype.DOCUMENT.value,
+        "main_entity_type": EntityType.DATASET.value,
+        "action_intents": [ActionIntent.READ.value, ActionIntent.DOWNLOAD.value],
+        "graph_roles": [GraphRole.REFERENCE_SOURCE.value],
+        "analyze_strategy": AnalyzeStrategy.ANALYZE_DOCUMENT_ONLY.value,
+        "llm_policy": LLMPolicy.LLM_LIGHT.value,
+        "should_analyze": True,
+    },
+    PageType.RESEARCH_PAPER_PAGE.value: {
+        "domain": PageDomain.KNOWLEDGE.value,
+        "archetype": PageArchetype.ARTICLE.value,
+        "main_entity_type": EntityType.ARTICLE.value,
+        "action_intents": [ActionIntent.READ.value, ActionIntent.LEARN.value],
+        "graph_roles": [GraphRole.CLAIM_SOURCE.value, GraphRole.REFERENCE_SOURCE.value],
+        "analyze_strategy": AnalyzeStrategy.ANALYZE_FULL.value,
+        "llm_policy": LLMPolicy.LLM_FULL.value,
+        "should_analyze": True,
+    },
+    PageType.JOB_POSTING_PAGE.value: {
+        "domain": PageDomain.JOBS.value,
+        "archetype": PageArchetype.DETAIL.value,
+        "main_entity_type": EntityType.JOB_POSTING.value,
+        "action_intents": [ActionIntent.APPLY.value, ActionIntent.READ.value],
+        "graph_roles": [GraphRole.ENTITY_ANCHOR.value],
+        "analyze_strategy": AnalyzeStrategy.ANALYZE_FULL.value,
+        "llm_policy": LLMPolicy.LLM_FULL.value,
+        "should_analyze": True,
+    },
+    PageType.COURSE_DETAIL_PAGE.value: {
+        "domain": PageDomain.EDUCATION.value,
+        "archetype": PageArchetype.DETAIL.value,
+        "main_entity_type": EntityType.COURSE.value,
+        "action_intents": [ActionIntent.LEARN.value, ActionIntent.SUBSCRIBE.value],
+        "graph_roles": [GraphRole.ENTITY_ANCHOR.value, GraphRole.REFERENCE_SOURCE.value],
+        "analyze_strategy": AnalyzeStrategy.ANALYZE_FULL.value,
+        "llm_policy": LLMPolicy.LLM_LIGHT.value,
+        "should_analyze": True,
+    },
+    PageType.VIDEO_PAGE.value: {
+        "domain": PageDomain.MEDIA.value,
+        "archetype": PageArchetype.MEDIA.value,
+        "main_entity_type": EntityType.MEDIA_OBJECT.value,
+        "action_intents": [ActionIntent.WATCH.value],
+        "graph_roles": [GraphRole.MEDIA_ANCHOR.value],
+        "analyze_strategy": AnalyzeStrategy.ANALYZE_METADATA_ONLY.value,
+        "llm_policy": LLMPolicy.RULE_ONLY.value,
+        "should_analyze": True,
+    },
+    PageType.LOCAL_BUSINESS_PAGE.value: {
+        "domain": PageDomain.LOCAL.value,
+        "archetype": PageArchetype.DETAIL.value,
+        "main_entity_type": EntityType.PLACE.value,
+        "action_intents": [ActionIntent.CONTACT.value, ActionIntent.NAVIGATE.value],
+        "graph_roles": [GraphRole.ENTITY_ANCHOR.value],
+        "analyze_strategy": AnalyzeStrategy.ANALYZE_FULL.value,
+        "llm_policy": LLMPolicy.LLM_LIGHT.value,
+        "should_analyze": True,
+    },
+    PageType.REAL_ESTATE_LISTING_PAGE.value: {
+        "domain": PageDomain.LOCAL.value,
+        "archetype": PageArchetype.DETAIL.value,
+        "main_entity_type": EntityType.REAL_ESTATE_PROPERTY.value,
+        "action_intents": [ActionIntent.READ.value, ActionIntent.CONTACT.value],
+        "graph_roles": [GraphRole.ENTITY_ANCHOR.value],
+        "analyze_strategy": AnalyzeStrategy.ANALYZE_FULL.value,
+        "llm_policy": LLMPolicy.LLM_LIGHT.value,
+        "should_analyze": True,
+    },
+    PageType.PROFILE_PAGE.value: {
+        "domain": PageDomain.COMMUNITY.value,
+        "archetype": PageArchetype.PROFILE.value,
+        "main_entity_type": EntityType.PERSON.value,
+        "action_intents": [ActionIntent.READ.value, ActionIntent.CONTACT.value],
+        "graph_roles": [GraphRole.PROFILE_ANCHOR.value],
+        "analyze_strategy": AnalyzeStrategy.ANALYZE_ENTITY_ONLY.value,
+        "llm_policy": LLMPolicy.LLM_LIGHT.value,
+        "should_analyze": True,
+    },
+    PageType.PRICING_PAGE.value: {
+        "domain": PageDomain.COMMERCE.value,
+        "archetype": PageArchetype.LANDING.value,
+        "main_entity_type": EntityType.SERVICE.value,
+        "action_intents": [ActionIntent.BUY.value, ActionIntent.COMPARE.value],
+        "graph_roles": [GraphRole.CLAIM_SOURCE.value],
+        "analyze_strategy": AnalyzeStrategy.ANALYZE_FULL.value,
+        "llm_policy": LLMPolicy.LLM_LIGHT.value,
+        "should_analyze": True,
+    },
+    PageType.LOGIN_PAGE.value: {
+        "domain": PageDomain.TRANSACTION.value,
+        "archetype": PageArchetype.FORM.value,
+        "main_entity_type": None,
+        "action_intents": [ActionIntent.LOGIN.value],
+        "graph_roles": [GraphRole.TRANSACTION_ONLY.value],
+        "analyze_strategy": AnalyzeStrategy.SKIP_PROTECTED.value,
+        "llm_policy": LLMPolicy.SKIP.value,
+        "should_analyze": False,
+    },
+    PageType.CHECKOUT_PAGE.value: {
+        "domain": PageDomain.TRANSACTION.value,
+        "archetype": PageArchetype.TRANSACTION.value,
+        "main_entity_type": None,
+        "action_intents": [ActionIntent.BUY.value, ActionIntent.PAY.value],
+        "graph_roles": [GraphRole.TRANSACTION_ONLY.value],
+        "analyze_strategy": AnalyzeStrategy.SKIP_PROTECTED.value,
+        "llm_policy": LLMPolicy.SKIP.value,
+        "should_analyze": False,
+    },
+    PageType.PAYMENT_PAGE.value: {
+        "domain": PageDomain.TRANSACTION.value,
+        "archetype": PageArchetype.TRANSACTION.value,
+        "main_entity_type": None,
+        "action_intents": [ActionIntent.PAY.value],
+        "graph_roles": [GraphRole.TRANSACTION_ONLY.value],
+        "analyze_strategy": AnalyzeStrategy.SKIP_PROTECTED.value,
+        "llm_policy": LLMPolicy.SKIP.value,
+        "should_analyze": False,
+    },
+    PageType.TERMS_PAGE.value: {
+        "domain": PageDomain.CORPORATE.value,
+        "archetype": PageArchetype.DOCUMENT.value,
+        "main_entity_type": EntityType.LEGAL_DOCUMENT.value,
+        "action_intents": [ActionIntent.READ.value],
+        "graph_roles": [GraphRole.POLICY_SOURCE.value],
+        "analyze_strategy": AnalyzeStrategy.ANALYZE_DOCUMENT_ONLY.value,
+        "llm_policy": LLMPolicy.LLM_LIGHT.value,
+        "should_analyze": True,
+    },
+    PageType.PRIVACY_POLICY_PAGE.value: {
+        "domain": PageDomain.CORPORATE.value,
+        "archetype": PageArchetype.DOCUMENT.value,
+        "main_entity_type": EntityType.LEGAL_DOCUMENT.value,
+        "action_intents": [ActionIntent.READ.value],
+        "graph_roles": [GraphRole.POLICY_SOURCE.value],
+        "analyze_strategy": AnalyzeStrategy.ANALYZE_DOCUMENT_ONLY.value,
+        "llm_policy": LLMPolicy.LLM_LIGHT.value,
+        "should_analyze": True,
+    },
+    PageType.SITEMAP_PAGE.value: {
+        "domain": PageDomain.SYSTEM.value,
+        "archetype": PageArchetype.SYSTEM_RESOURCE.value,
+        "main_entity_type": None,
+        "action_intents": [ActionIntent.NAVIGATE.value],
+        "graph_roles": [GraphRole.SYSTEM_RESOURCE.value, GraphRole.NAVIGATION_HUB.value],
+        "analyze_strategy": AnalyzeStrategy.ANALYZE_DISCOVERY_ONLY.value,
+        "llm_policy": LLMPolicy.RULE_ONLY.value,
+        "should_analyze": True,
+    },
+    PageType.RSS_FEED_PAGE.value: {
+        "domain": PageDomain.SYSTEM.value,
+        "archetype": PageArchetype.SYSTEM_RESOURCE.value,
+        "main_entity_type": None,
+        "action_intents": [ActionIntent.READ.value],
+        "graph_roles": [GraphRole.SYSTEM_RESOURCE.value],
+        "analyze_strategy": AnalyzeStrategy.ANALYZE_METADATA_ONLY.value,
+        "llm_policy": LLMPolicy.RULE_ONLY.value,
+        "should_analyze": True,
+    },
+    PageType.ERROR_PAGE.value: {
+        "domain": PageDomain.SYSTEM.value,
+        "archetype": PageArchetype.ERROR.value,
+        "main_entity_type": None,
+        "action_intents": [],
+        "graph_roles": [GraphRole.NOISE_PAGE.value],
+        "analyze_strategy": AnalyzeStrategy.SKIP_NOISE.value,
+        "llm_policy": LLMPolicy.SKIP.value,
+        "should_analyze": False,
+    },
+    PageType.NOT_FOUND_PAGE.value: {
+        "domain": PageDomain.SYSTEM.value,
+        "archetype": PageArchetype.ERROR.value,
+        "main_entity_type": None,
+        "action_intents": [],
+        "graph_roles": [GraphRole.NOISE_PAGE.value],
+        "analyze_strategy": AnalyzeStrategy.SKIP_NOISE.value,
+        "llm_policy": LLMPolicy.SKIP.value,
+        "should_analyze": False,
+    },
+    PageType.ACCESS_DENIED_PAGE.value: {
+        "domain": PageDomain.TRANSACTION.value,
+        "archetype": PageArchetype.ERROR.value,
+        "main_entity_type": None,
+        "action_intents": [ActionIntent.VERIFY.value],
+        "graph_roles": [GraphRole.TRANSACTION_ONLY.value],
+        "analyze_strategy": AnalyzeStrategy.SKIP_PROTECTED.value,
+        "llm_policy": LLMPolicy.SKIP.value,
+        "should_analyze": False,
+    },
+    PageType.CAPTCHA_PAGE.value: {
+        "domain": PageDomain.TRANSACTION.value,
+        "archetype": PageArchetype.FORM.value,
+        "main_entity_type": None,
+        "action_intents": [ActionIntent.VERIFY.value],
+        "graph_roles": [GraphRole.TRANSACTION_ONLY.value],
+        "analyze_strategy": AnalyzeStrategy.SKIP_PROTECTED.value,
+        "llm_policy": LLMPolicy.SKIP.value,
+        "should_analyze": False,
+    },
+}
+
+
+def score_page_type(url: str, signals: PageSignals, *, unknown_threshold: float = UNKNOWN_THRESHOLD) -> PageClassificationResult:
+    accumulator = ScoreAccumulator(scores={}, evidence_by_type={})
+    for page_type, rules in SCORING_RULES.items():
+        for rule in rules:
+            accumulator.add(page_type, rule, signals)
+
+    _apply_interactions(accumulator, signals)
+
+    if not accumulator.scores:
+        return _unknown_result(
+            url,
+            alternatives=[],
+            evidence=[
+                EvidenceItem(
+                    key="insufficient_evidence",
+                    value=True,
+                    weight=0.0,
+                    source="page_type_scorer",
+                    message="No page-type evidence was detected.",
+                )
+            ],
+        )
+
+    alternatives = sorted(
+        ((page_type, round(min(score, 1.0), 4)) for page_type, score in accumulator.scores.items()),
+        key=lambda item: item[1],
+        reverse=True,
+    )
+    primary_page_type, confidence = alternatives[0]
+    if confidence < unknown_threshold:
+        evidence = [item for page_type, _score in alternatives[:3] for item in accumulator.evidence_by_type.get(page_type, [])]
+        evidence.append(
+            EvidenceItem(
+                key="low_confidence",
+                value=confidence,
+                weight=0.0,
+                source="page_type_scorer",
+                message=f"Top score {confidence:.2f} is below UnknownPage threshold {unknown_threshold:.2f}.",
+            )
+        )
+        return _unknown_result(url, alternatives=alternatives[:5], evidence=evidence)
+
+    profile = _profile(primary_page_type)
+    evidence = accumulator.evidence_by_type.get(primary_page_type, [])
+    if not evidence:
+        evidence = [
+            EvidenceItem(
+                key="score",
+                value=confidence,
+                weight=confidence,
+                source="page_type_scorer",
+                message=f"{primary_page_type} selected from accumulated score.",
+            )
+        ]
+    secondary = [page_type for page_type, score in alternatives[1:4] if score >= 0.18]
+    return PageClassificationResult(
+        url=url,
+        primary_page_type=primary_page_type,
+        secondary_page_types=secondary,
+        domain=str(profile["domain"]),
+        archetype=str(profile["archetype"]),
+        main_entity_type=profile.get("main_entity_type"),
+        action_intents=list(profile.get("action_intents") or []),
+        graph_roles=list(profile.get("graph_roles") or []),
+        confidence=confidence,
+        alternatives=alternatives[:8],
+        evidence=evidence,
+        should_analyze=bool(profile.get("should_analyze")),
+        analyze_strategy=str(profile["analyze_strategy"]),
+        llm_policy=str(profile["llm_policy"]),
+        is_protected=str(profile["analyze_strategy"]) == AnalyzeStrategy.SKIP_PROTECTED.value,
+        is_noise=str(profile["analyze_strategy"]) == AnalyzeStrategy.SKIP_NOISE.value,
+    )
+
+
+def _profile(page_type: str) -> dict[str, Any]:
+    return PAGE_TYPE_METADATA.get(page_type, PAGE_TYPE_METADATA[PageType.UNKNOWN_PAGE.value])
+
+
+def _unknown_result(
+    url: str,
+    *,
+    alternatives: list[tuple[str, float]],
+    evidence: list[EvidenceItem],
+) -> PageClassificationResult:
+    profile = _profile(PageType.UNKNOWN_PAGE.value)
+    return PageClassificationResult(
+        url=url,
+        primary_page_type=PageType.UNKNOWN_PAGE.value,
+        secondary_page_types=[page_type for page_type, _score in alternatives[:3]],
+        domain=str(profile["domain"]),
+        archetype=str(profile["archetype"]),
+        main_entity_type=profile.get("main_entity_type"),
+        action_intents=list(profile.get("action_intents") or []),
+        graph_roles=list(profile.get("graph_roles") or []),
+        confidence=0.0,
+        alternatives=alternatives[:8],
+        evidence=evidence
+        or [
+            EvidenceItem(
+                key="unknown",
+                value=True,
+                weight=0.0,
+                source="page_type_scorer",
+                message="Page did not match known semantic page type evidence.",
+            )
+        ],
+        should_analyze=bool(profile.get("should_analyze")),
+        analyze_strategy=str(profile["analyze_strategy"]),
+        llm_policy=str(profile["llm_policy"]),
+        is_protected=False,
+        is_noise=False,
+    )
+
+
+def _apply_interactions(accumulator: ScoreAccumulator, signals: PageSignals) -> None:
+    # Protected signals must dominate commerce signals when login/payment fields are present.
+    if signals.has_password_field and signals.has_login_form:
+        _manual_add(accumulator, PageType.LOGIN_PAGE.value, "protected_priority", True, 0.3, "protected", "Login form and password field override commerce signals.")
+    if signals.has_payment_fields:
+        _manual_add(accumulator, PageType.PAYMENT_PAGE.value, "payment_priority", True, 0.35, "protected", "Payment fields override product or checkout content.")
+    if signals.has_access_denied:
+        _manual_add(accumulator, PageType.ACCESS_DENIED_PAGE.value, "access_denied_priority", True, 0.35, "protected", "Access denied signal is a protected page indicator.")
+    if signals.has_captcha:
+        _manual_add(accumulator, PageType.CAPTCHA_PAGE.value, "captcha_priority", True, 0.35, "protected", "Captcha signal is a protected page indicator.")
+    if signals.has_repeated_cards and signals.product_link_count >= 3:
+        _manual_add(accumulator, PageType.CATEGORY_LISTING_PAGE.value, "listing_product_links", signals.product_link_count, 0.18, "link_graph", "Repeated cards with multiple product links indicate a product listing.")
+    if signals.has_question and signals.has_answer and signals.has_faq_structure:
+        _manual_add(accumulator, PageType.FAQ_PAGE.value, "faq_question_answer", True, 0.18, "community", "FAQ structure with question/answer content.")
+
+
+def _manual_add(
+    accumulator: ScoreAccumulator,
+    page_type: str,
+    key: str,
+    value: str | int | float | bool | None,
+    weight: float,
+    source: str,
+    message: str,
+) -> None:
+    accumulator.scores[page_type] = accumulator.scores.get(page_type, 0.0) + weight
+    accumulator.evidence_by_type.setdefault(page_type, []).append(
+        EvidenceItem(key=key, value=value, weight=weight, source=source, message=message)
+    )
+
+
+def _has_schema(*schema_types: str) -> Callable[[PageSignals], bool]:
+    return lambda signals: any(schema_type in signals.schema_types for schema_type in schema_types)
+
+
+def _keyword(group: str, minimum: int = 1) -> Callable[[PageSignals], bool]:
+    return lambda signals: int(signals.keyword_hits.get(group) or 0) >= minimum
+
+
+def _url_hint(hint: str) -> Callable[[PageSignals], bool]:
+    return lambda signals: hint in signals.url_hints
+
+
+def _contains_terms(signals: PageSignals, terms: tuple[str, ...]) -> bool:
+    text = f"{signals.title or ''}\n{signals.text_sample or ''}".lower()
+    return any(term.lower() in text for term in terms)
+
+
+def _flag(name: str) -> Callable[[PageSignals], bool]:
+    return lambda signals: bool(getattr(signals, name))
+
+
+def _count_at_least(name: str, minimum: int) -> Callable[[PageSignals], bool]:
+    return lambda signals: int(getattr(signals, name) or 0) >= minimum
+
+
+def _value(name: str) -> Callable[[PageSignals], str | int | float | bool | None]:
+    return lambda signals: getattr(signals, name)
+
+
+def _keywords_value(group: str) -> Callable[[PageSignals], str | int | float | bool | None]:
+    return lambda signals: signals.keyword_hits.get(group, 0)
+
+
+def _schema_value(signals: PageSignals) -> str:
+    return ",".join(sorted(signals.schema_types))
+
+
+def _url_hint_value(signals: PageSignals) -> str:
+    return ",".join(sorted(signals.url_hints))
+
+
+SCORING_RULES: dict[str, list[SignalRule]] = {
+    PageType.PRODUCT_DETAIL_PAGE.value: [
+        SignalRule("schema_product", 0.4, "structured_data", "schema.org Product detected.", _has_schema("Product"), _schema_value),
+        SignalRule("schema_offer", 0.15, "structured_data", "schema.org Offer detected.", _has_schema("Offer"), _schema_value),
+        SignalRule("price", 0.15, "text", "Price detected.", _flag("has_price"), _value("has_price")),
+        SignalRule("cart_button", 0.2, "dom", "Cart button detected.", _flag("has_cart_button")),
+        SignalRule("buy_button", 0.16, "dom", "Buy button detected.", _flag("has_buy_button")),
+        SignalRule("variant_selector", 0.12, "dom", "Variant selector detected.", _flag("has_variant_selector")),
+        SignalRule("sku", 0.1, "text", "SKU or product code detected.", _flag("has_sku")),
+        SignalRule("product_gallery", 0.1, "layout", "Product image gallery detected.", _flag("has_product_gallery")),
+        SignalRule("review_section", 0.05, "dom", "Review section detected.", _flag("has_review_section")),
+        SignalRule("url_product_hint", 0.05, "url", "Product URL hint detected.", _url_hint("product"), _url_hint_value),
+    ],
+    PageType.CATEGORY_LISTING_PAGE.value: [
+        SignalRule("repeated_cards", 0.32, "layout", "Repeated cards detected.", _flag("has_repeated_cards"), _value("repeated_card_count")),
+        SignalRule("filter_panel", 0.2, "dom", "Filter panel detected.", _flag("has_filter_panel")),
+        SignalRule("sort_control", 0.15, "dom", "Sort control detected.", _flag("has_sort_control")),
+        SignalRule("pagination", 0.1, "dom", "Pagination detected.", _flag("has_pagination")),
+        SignalRule("product_links", 0.18, "link_graph", "Multiple product links detected.", _count_at_least("product_link_count", 3), _value("product_link_count")),
+        SignalRule("category_url_hint", 0.06, "url", "Category/list URL hint detected.", _url_hint("category"), _url_hint_value),
+    ],
+    PageType.SEARCH_RESULTS_PAGE.value: [
+        SignalRule("search_url_hint", 0.28, "url", "Search URL hint detected.", _url_hint("search"), _url_hint_value),
+        SignalRule("listing_results", 0.18, "layout", "Repeated result cards detected.", _flag("has_repeated_cards"), _value("repeated_card_count")),
+        SignalRule("filter_panel", 0.14, "dom", "Search filter panel detected.", _flag("has_filter_panel")),
+        SignalRule("pagination", 0.12, "dom", "Search pagination detected.", _flag("has_pagination")),
+        SignalRule("search_keywords", 0.12, "text", "Search/result keywords detected.", _keyword("listing"), _keywords_value("listing")),
+    ],
+    PageType.ARTICLE_PAGE.value: [
+        SignalRule("schema_article", 0.35, "structured_data", "Article structured data detected.", _has_schema("Article", "NewsArticle"), _schema_value),
+        SignalRule("author", 0.15, "text", "Author/byline signal detected.", _flag("has_author")),
+        SignalRule("published_date", 0.15, "text", "Published date detected.", _flag("has_published_date")),
+        SignalRule("article_body", 0.2, "dom", "Article body detected.", _flag("has_article_body")),
+        SignalRule("tags", 0.05, "dom", "Article tags detected.", _flag("has_tags")),
+        SignalRule("article_url_hint", 0.05, "url", "Article URL hint detected.", _url_hint("article"), _url_hint_value),
+    ],
+    PageType.BLOG_POST_PAGE.value: [
+        SignalRule("blog_url_hint", 0.25, "url", "Blog URL hint detected.", lambda signals: "article" in signals.url_hints and "blog" in (signals.text_sample or "").lower()),
+        SignalRule("author", 0.14, "text", "Author signal detected.", _flag("has_author")),
+        SignalRule("published_date", 0.14, "text", "Published date detected.", _flag("has_published_date")),
+        SignalRule("article_body", 0.18, "dom", "Article body detected.", _flag("has_article_body")),
+        SignalRule("tags", 0.08, "dom", "Tags detected.", _flag("has_tags")),
+    ],
+    PageType.QA_PAGE.value: [
+        SignalRule("schema_qapage", 0.35, "structured_data", "QAPage structured data detected.", _has_schema("QAPage"), _schema_value),
+        SignalRule("question", 0.2, "community", "Question block detected.", _flag("has_question")),
+        SignalRule("answer", 0.2, "community", "Answer block detected.", _flag("has_answer")),
+        SignalRule("votes", 0.1, "community", "Vote signal detected.", _flag("has_votes")),
+        SignalRule("comments", 0.05, "community", "Comments detected.", _flag("has_comments")),
+        SignalRule("board_url_hint", 0.05, "url", "Board/community URL hint detected.", _url_hint("board"), _url_hint_value),
+    ],
+    PageType.FAQ_PAGE.value: [
+        SignalRule("schema_faq", 0.36, "structured_data", "FAQPage structured data detected.", _has_schema("FAQPage"), _schema_value),
+        SignalRule("faq_structure", 0.25, "community", "FAQ structure detected.", _flag("has_faq_structure")),
+        SignalRule("question", 0.15, "community", "Question content detected.", _flag("has_question")),
+        SignalRule("answer", 0.15, "community", "Answer content detected.", _flag("has_answer")),
+    ],
+    PageType.FORUM_BOARD_PAGE.value: [
+        SignalRule("board_url_hint", 0.25, "url", "Board URL hint detected.", _url_hint("board"), _url_hint_value),
+        SignalRule("thread_structure", 0.2, "community", "Thread structure detected.", _flag("has_thread_structure")),
+        SignalRule("comments", 0.1, "community", "Comments/replies detected.", _flag("has_comments")),
+        SignalRule("repeated_cards", 0.14, "layout", "Repeated post cards detected.", _flag("has_repeated_cards"), _value("repeated_card_count")),
+        SignalRule("pagination", 0.1, "dom", "Board pagination detected.", _flag("has_pagination")),
+    ],
+    PageType.FORUM_THREAD_PAGE.value: [
+        SignalRule("thread_structure", 0.3, "community", "Thread structure detected.", _flag("has_thread_structure")),
+        SignalRule("comments", 0.16, "community", "Comment thread detected.", _flag("has_comments")),
+        SignalRule("question_answer", 0.14, "community", "Question and answer content detected.", lambda signals: signals.has_question and signals.has_answer),
+        SignalRule("votes", 0.08, "community", "Vote signal detected.", _flag("has_votes")),
+        SignalRule("article_body", 0.08, "dom", "Post body detected.", _flag("has_article_body")),
+    ],
+    PageType.BRAND_STORY_PAGE.value: [
+        SignalRule("corporate_keywords", 0.2, "text", "Corporate/brand keywords detected.", _keyword("corporate"), _keywords_value("corporate")),
+        SignalRule("about_url_hint", 0.18, "url", "About/company URL hint detected.", lambda signals: "about" in (signals.text_sample or "").lower() or "brand" in (signals.title or "").lower()),
+        SignalRule("article_body", 0.14, "dom", "Brand story body detected.", _flag("has_article_body")),
+        SignalRule("hero_block", 0.08, "layout", "Hero block detected.", _flag("has_hero_block")),
+    ],
+    PageType.ABOUT_PAGE.value: [
+        SignalRule("corporate_keywords", 0.22, "text", "About/company keywords detected.", _keyword("corporate"), _keywords_value("corporate")),
+        SignalRule("contact_info", 0.08, "text", "Organization contact signal detected.", _flag("has_contact_info")),
+        SignalRule("article_body", 0.12, "dom", "About body detected.", _flag("has_article_body")),
+    ],
+    PageType.CONTACT_PAGE.value: [
+        SignalRule("contact_info", 0.32, "text", "Contact information detected.", _flag("has_contact_info")),
+        SignalRule("address", 0.18, "text", "Address detected.", _flag("has_address")),
+        SignalRule("corporate_keywords", 0.1, "text", "Corporate keywords detected.", _keyword("corporate"), _keywords_value("corporate")),
+    ],
+    PageType.DOCUMENTATION_PAGE.value: [
+        SignalRule("docs_keywords", 0.18, "text", "Documentation keywords detected.", _keyword("knowledge"), _keywords_value("knowledge")),
+        SignalRule("toc", 0.16, "dom", "Table of contents detected.", _flag("has_toc")),
+        SignalRule("code_blocks", 0.16, "dom", "Code blocks detected.", _flag("has_code_blocks")),
+        SignalRule("version_info", 0.08, "text", "Version information detected.", _flag("has_version_info")),
+    ],
+    PageType.API_REFERENCE_PAGE.value: [
+        SignalRule("api_endpoint", 0.3, "text", "API endpoint detected.", _flag("has_api_endpoint")),
+        SignalRule("parameter_table", 0.22, "dom", "Parameter table detected.", _flag("has_parameter_table")),
+        SignalRule("code_blocks", 0.1, "dom", "Code blocks detected.", _flag("has_code_blocks")),
+        SignalRule("docs_keywords", 0.12, "text", "API/docs keywords detected.", _keyword("knowledge"), _keywords_value("knowledge")),
+    ],
+    PageType.WIKI_PAGE.value: [
+        SignalRule("wiki_schema", 0.28, "structured_data", "Wiki/DefinedTerm structured data detected.", _has_schema("DefinedTerm", "WebPage"), _schema_value),
+        SignalRule("toc", 0.12, "dom", "Reference table of contents detected.", _flag("has_toc")),
+        SignalRule("definition_terms", 0.14, "text", "Definition/reference keywords detected.", lambda signals: _contains_terms(signals, ("definition", "wiki", "glossary", "reference"))),
+    ],
+    PageType.DATASET_PAGE.value: [
+        SignalRule("schema_dataset", 0.42, "structured_data", "Dataset structured data detected.", _has_schema("Dataset", "DataCatalog"), _schema_value),
+        SignalRule("download_terms", 0.12, "text", "Dataset/download keywords detected.", lambda signals: _contains_terms(signals, ("dataset", "data catalog", "download", "csv"))),
+        SignalRule("parameter_table", 0.08, "dom", "Dataset metadata table detected.", _flag("has_parameter_table")),
+    ],
+    PageType.RESEARCH_PAPER_PAGE.value: [
+        SignalRule("schema_scholarly", 0.42, "structured_data", "Scholarly article structured data detected.", _has_schema("ScholarlyArticle", "TechArticle"), _schema_value),
+        SignalRule("published_date", 0.1, "text", "Publication date detected.", _flag("has_published_date")),
+        SignalRule("citation_terms", 0.16, "text", "Citation/references keywords detected.", lambda signals: _contains_terms(signals, ("abstract", "citation", "references", "doi"))),
+    ],
+    PageType.JOB_POSTING_PAGE.value: [
+        SignalRule("career_keywords", 0.26, "text", "Career/job keywords detected.", _flag("has_career_terms")),
+        SignalRule("apply_intent", 0.12, "text", "Apply intent detected.", lambda signals: "apply" in (signals.text_sample or "").lower() or "지원" in (signals.text_sample or "")),
+        SignalRule("address", 0.06, "text", "Location/address signal detected.", _flag("has_address")),
+    ],
+    PageType.COURSE_DETAIL_PAGE.value: [
+        SignalRule("schema_course", 0.42, "structured_data", "Course structured data detected.", _has_schema("Course"), _schema_value),
+        SignalRule("course_terms", 0.18, "text", "Course/curriculum keywords detected.", lambda signals: _contains_terms(signals, ("course", "lesson", "curriculum", "instructor", "syllabus"))),
+    ],
+    PageType.VIDEO_PAGE.value: [
+        SignalRule("schema_video", 0.42, "structured_data", "VideoObject structured data detected.", _has_schema("VideoObject"), _schema_value),
+        SignalRule("media_player", 0.2, "layout", "Video/audio player detected.", _flag("has_media_player_area")),
+        SignalRule("video_terms", 0.1, "text", "Video/watch keywords detected.", lambda signals: _contains_terms(signals, ("video", "watch", "episode", "duration"))),
+    ],
+    PageType.LOCAL_BUSINESS_PAGE.value: [
+        SignalRule("schema_local_business", 0.42, "structured_data", "Local business/place structured data detected.", _has_schema("LocalBusiness", "Place", "Restaurant"), _schema_value),
+        SignalRule("address", 0.18, "text", "Address detected.", _flag("has_address")),
+        SignalRule("contact_info", 0.12, "text", "Contact info detected.", _flag("has_contact_info")),
+        SignalRule("map_area", 0.1, "layout", "Map area detected.", _flag("has_map_area")),
+    ],
+    PageType.REAL_ESTATE_LISTING_PAGE.value: [
+        SignalRule("schema_real_estate", 0.38, "structured_data", "Real estate structured data detected.", _has_schema("RealEstateListing", "Residence", "Apartment"), _schema_value),
+        SignalRule("property_terms", 0.18, "text", "Property listing keywords detected.", lambda signals: _contains_terms(signals, ("bedroom", "bathroom", "sqft", "property", "real estate"))),
+        SignalRule("price", 0.12, "text", "Property price detected.", _flag("has_price")),
+        SignalRule("address", 0.08, "text", "Property address detected.", _flag("has_address")),
+    ],
+    PageType.PROFILE_PAGE.value: [
+        SignalRule("schema_person", 0.34, "structured_data", "Person/Profile structured data detected.", _has_schema("Person", "ProfilePage"), _schema_value),
+        SignalRule("profile_links", 0.12, "link_graph", "Profile link pattern detected.", _count_at_least("profile_link_count", 1), _value("profile_link_count")),
+        SignalRule("author", 0.1, "text", "Author/person signal detected.", _flag("has_author")),
+    ],
+    PageType.PRICING_PAGE.value: [
+        SignalRule("pricing_table", 0.3, "layout", "Pricing table detected.", _flag("has_pricing_table")),
+        SignalRule("price", 0.16, "text", "Price detected.", _flag("has_price")),
+        SignalRule("comparison_table", 0.12, "layout", "Comparison table detected.", _flag("has_comparison_table")),
+        SignalRule("commerce_keywords", 0.1, "text", "Commerce keywords detected.", _keyword("commerce"), _keywords_value("commerce")),
+    ],
+    PageType.LOGIN_PAGE.value: [
+        SignalRule("password_field", 0.4, "form", "Password field detected.", _flag("has_password_field")),
+        SignalRule("login_form", 0.25, "form", "Login form detected.", _flag("has_login_form")),
+        SignalRule("protected_url_hint", 0.08, "url", "Protected URL hint detected.", _url_hint("protected"), _url_hint_value),
+    ],
+    PageType.CHECKOUT_PAGE.value: [
+        SignalRule("protected_url_hint", 0.25, "url", "Checkout/cart URL hint detected.", _url_hint("protected"), _url_hint_value),
+        SignalRule("buy_button", 0.14, "dom", "Purchase button detected.", _flag("has_buy_button")),
+        SignalRule("commerce_keywords", 0.12, "text", "Checkout commerce keywords detected.", _keyword("commerce"), _keywords_value("commerce")),
+    ],
+    PageType.PAYMENT_PAGE.value: [
+        SignalRule("payment_fields", 0.42, "form", "Payment fields detected.", _flag("has_payment_fields")),
+        SignalRule("protected_keywords", 0.16, "text", "Payment/protected keywords detected.", _keyword("protected"), _keywords_value("protected")),
+        SignalRule("protected_url_hint", 0.1, "url", "Payment URL hint detected.", _url_hint("protected"), _url_hint_value),
+    ],
+    PageType.TERMS_PAGE.value: [
+        SignalRule("policy_terms", 0.32, "text", "Policy/terms terms detected.", _flag("has_policy_terms")),
+        SignalRule("corporate_keywords", 0.08, "text", "Corporate legal keywords detected.", _keyword("corporate"), _keywords_value("corporate")),
+    ],
+    PageType.PRIVACY_POLICY_PAGE.value: [
+        SignalRule("privacy_terms", 0.36, "text", "Privacy terms detected.", _flag("has_privacy_terms")),
+        SignalRule("policy_terms", 0.14, "text", "Policy terms detected.", _flag("has_policy_terms")),
+    ],
+    PageType.SITEMAP_PAGE.value: [
+        SignalRule("sitemap_resource", 0.45, "resource", "Sitemap resource detected.", _flag("has_sitemap_resource")),
+        SignalRule("xml_resource", 0.12, "resource", "XML resource detected.", _flag("has_xml_resource")),
+        SignalRule("system_url_hint", 0.12, "url", "System URL hint detected.", _url_hint("system"), _url_hint_value),
+    ],
+    PageType.RSS_FEED_PAGE.value: [
+        SignalRule("feed_resource", 0.45, "resource", "RSS/Atom feed detected.", _flag("has_feed_resource")),
+        SignalRule("xml_resource", 0.1, "resource", "XML feed resource detected.", _flag("has_xml_resource")),
+    ],
+    PageType.ERROR_PAGE.value: [
+        SignalRule("error_status", 0.38, "http", "Error status detected.", _flag("has_error_status")),
+    ],
+    PageType.NOT_FOUND_PAGE.value: [
+        SignalRule("not_found", 0.5, "http", "404/not found signal detected.", _flag("has_not_found")),
+        SignalRule("error_status", 0.12, "http", "Error status supports not found page.", _flag("has_error_status")),
+    ],
+    PageType.ACCESS_DENIED_PAGE.value: [
+        SignalRule("access_denied", 0.42, "protected", "Access denied text detected.", _flag("has_access_denied")),
+        SignalRule("error_status", 0.1, "http", "Error status supports access denied.", _flag("has_error_status")),
+    ],
+    PageType.CAPTCHA_PAGE.value: [
+        SignalRule("captcha", 0.45, "protected", "Captcha detected.", _flag("has_captcha")),
+        SignalRule("protected_keywords", 0.1, "text", "Protected keywords detected.", _keyword("protected"), _keywords_value("protected")),
+    ],
+}

ontology_platform/crawler_platform/app/core/crawler/page_type_taxonomy.py

@@ -0,0 +1,476 @@
+from __future__ import annotations
+
+from dataclasses import asdict, dataclass, field
+from enum import StrEnum
+from typing import Any
+
+
+class PageDomain(StrEnum):
+    COMMERCE = "Commerce"
+    EDITORIAL = "Editorial"
+    COMMUNITY = "Community"
+    KNOWLEDGE = "Knowledge"
+    CORPORATE = "Corporate"
+    LOCAL = "Local"
+    EDUCATION = "Education"
+    JOBS = "Jobs"
+    MEDIA = "Media"
+    SOFTWARE = "Software"
+    FINANCE = "Finance"
+    GOVERNMENT = "Government"
+    HEALTHCARE = "Healthcare"
+    TRANSACTION = "Transaction"
+    SYSTEM = "System"
+    UNKNOWN = "Unknown"
+
+
+class PageArchetype(StrEnum):
+    HOME = "Home"
+    LANDING = "Landing"
+    DETAIL = "Detail"
+    LISTING = "Listing"
+    COLLECTION = "Collection"
+    SEARCH_RESULT = "SearchResult"
+    PROFILE = "Profile"
+    ARTICLE = "Article"
+    THREAD = "Thread"
+    FORM = "Form"
+    TRANSACTION = "Transaction"
+    DASHBOARD = "Dashboard"
+    DOCUMENT = "Document"
+    MEDIA = "Media"
+    ERROR = "Error"
+    SYSTEM_RESOURCE = "SystemResource"
+    UNKNOWN = "Unknown"
+
+
+class PageType(StrEnum):
+    PRODUCT_PAGE = "ProductPage"
+    CATEGORY_PAGE = "CategoryPage"
+    SEARCH_PAGE = "SearchPage"
+    BOARD_PAGE = "BoardPage"
+    NOTICE_PAGE = "NoticePage"
+    BRAND_STORY_PAGE = "BrandStoryPage"
+    ABOUT_PAGE = "AboutPage"
+    CONTACT_PAGE = "ContactPage"
+    PROMOTION_PAGE = "PromotionPage"
+    REVIEW_PAGE = "ReviewPage"
+    UNKNOWN_PAGE = "UnknownPage"
+    PRODUCT_DETAIL_PAGE = "ProductDetailPage"
+    CATEGORY_LISTING_PAGE = "CategoryListingPage"
+    SEARCH_RESULTS_PAGE = "SearchResultsPage"
+    FORUM_BOARD_PAGE = "ForumBoardPage"
+    FORUM_THREAD_PAGE = "ForumThreadPage"
+    PUBLIC_NOTICE_PAGE = "PublicNoticePage"
+    CAMPAIGN_LANDING_PAGE = "CampaignLandingPage"
+    ARTICLE_PAGE = "ArticlePage"
+    NEWS_ARTICLE_PAGE = "NewsArticlePage"
+    BLOG_POST_PAGE = "BlogPostPage"
+    FAQ_PAGE = "FAQPage"
+    QA_PAGE = "QAPage"
+    PROFILE_PAGE = "ProfilePage"
+    DOCUMENTATION_PAGE = "DocumentationPage"
+    API_REFERENCE_PAGE = "APIReferencePage"
+    WIKI_PAGE = "WikiPage"
+    DATASET_PAGE = "DatasetPage"
+    RESEARCH_PAPER_PAGE = "ResearchPaperPage"
+    JOB_POSTING_PAGE = "JobPostingPage"
+    COURSE_DETAIL_PAGE = "CourseDetailPage"
+    VIDEO_PAGE = "VideoPage"
+    LOCAL_BUSINESS_PAGE = "LocalBusinessPage"
+    REAL_ESTATE_LISTING_PAGE = "RealEstateListingPage"
+    PRICING_PAGE = "PricingPage"
+    LOGIN_PAGE = "LoginPage"
+    CHECKOUT_PAGE = "CheckoutPage"
+    PAYMENT_PAGE = "PaymentPage"
+    TERMS_PAGE = "TermsPage"
+    PRIVACY_POLICY_PAGE = "PrivacyPolicyPage"
+    SITEMAP_PAGE = "SitemapPage"
+    RSS_FEED_PAGE = "RSSFeedPage"
+    ERROR_PAGE = "ErrorPage"
+    NOT_FOUND_PAGE = "NotFoundPage"
+    ACCESS_DENIED_PAGE = "AccessDeniedPage"
+    CAPTCHA_PAGE = "CaptchaPage"
+
+
+class EntityType(StrEnum):
+    PRODUCT = "Product"
+    SERVICE = "Service"
+    ARTICLE = "Article"
+    NEWS_ARTICLE = "NewsArticle"
+    PERSON = "Person"
+    ORGANIZATION = "Organization"
+    PLACE = "Place"
+    EVENT = "Event"
+    JOB_POSTING = "JobPosting"
+    COURSE = "Course"
+    QUESTION = "Question"
+    ANSWER = "Answer"
+    REVIEW = "Review"
+    DATASET = "Dataset"
+    SOFTWARE_APPLICATION = "SoftwareApplication"
+    MEDIA_OBJECT = "MediaObject"
+    RECIPE = "Recipe"
+    REAL_ESTATE_PROPERTY = "RealEstateProperty"
+    MEDICAL_CONDITION = "MedicalCondition"
+    LEGAL_DOCUMENT = "LegalDocument"
+    FINANCIAL_PRODUCT = "FinancialProduct"
+    UNKNOWN_ENTITY = "UnknownEntity"
+
+
+class ActionIntent(StrEnum):
+    READ = "Read"
+    BUY = "Buy"
+    SUBSCRIBE = "Subscribe"
+    RESERVE = "Reserve"
+    BOOK = "Book"
+    APPLY = "Apply"
+    DOWNLOAD = "Download"
+    WATCH = "Watch"
+    LISTEN = "Listen"
+    SEARCH = "Search"
+    COMPARE = "Compare"
+    FILTER = "Filter"
+    ASK = "Ask"
+    ANSWER = "Answer"
+    COMMENT = "Comment"
+    REVIEW = "Review"
+    LOGIN = "Login"
+    REGISTER = "Register"
+    PAY = "Pay"
+    CONTACT = "Contact"
+    NAVIGATE = "Navigate"
+    LEARN = "Learn"
+    VERIFY = "Verify"
+    CONFIGURE = "Configure"
+    MANAGE = "Manage"
+
+
+class GraphRole(StrEnum):
+    ENTITY_ANCHOR = "EntityAnchor"
+    RELATION_HUB = "RelationHub"
+    NAVIGATION_HUB = "NavigationHub"
+    COLLECTION_HUB = "CollectionHub"
+    SEARCH_HUB = "SearchHub"
+    TRANSACTION_ONLY = "TransactionOnly"
+    POLICY_SOURCE = "PolicySource"
+    CLAIM_SOURCE = "ClaimSource"
+    PROFILE_ANCHOR = "ProfileAnchor"
+    MEDIA_ANCHOR = "MediaAnchor"
+    REFERENCE_SOURCE = "ReferenceSource"
+    SYSTEM_RESOURCE = "SystemResource"
+    NOISE_PAGE = "NoisePage"
+    UNKNOWN_PATTERN = "UnknownPattern"
+
+
+class AnalyzeStrategy(StrEnum):
+    ANALYZE_FULL = "AnalyzeFull"
+    ANALYZE_STRUCTURE_ONLY = "AnalyzeStructureOnly"
+    ANALYZE_ENTITY_ONLY = "AnalyzeEntityOnly"
+    ANALYZE_RELATIONS_ONLY = "AnalyzeRelationsOnly"
+    ANALYZE_METADATA_ONLY = "AnalyzeMetadataOnly"
+    ANALYZE_DOCUMENT_ONLY = "AnalyzeDocumentOnly"
+    ANALYZE_DISCOVERY_ONLY = "AnalyzeDiscoveryOnly"
+    SKIP_PROTECTED = "SkipProtected"
+    SKIP_NOISE = "SkipNoise"
+
+
+class LLMPolicy(StrEnum):
+    LLM_FULL = "LLMFull"
+    LLM_LIGHT = "LLMLight"
+    LLM_FOR_AMBIGUITY_ONLY = "LLMForAmbiguityOnly"
+    RULE_ONLY = "RuleOnly"
+    NO_LLM = "NoLLM"
+    SKIP = "Skip"
+
+
+@dataclass(frozen=True, slots=True)
+class EvidenceItem:
+    key: str
+    value: str | int | float | bool | None
+    weight: float
+    source: str
+    message: str
+
+    def to_dict(self) -> dict[str, Any]:
+        return asdict(self)
+
+
+@dataclass(slots=True)
+class PageClassificationResult:
+    url: str
+    primary_page_type: str
+    secondary_page_types: list[str] = field(default_factory=list)
+    domain: str = PageDomain.UNKNOWN.value
+    archetype: str = PageArchetype.UNKNOWN.value
+    main_entity_type: str | None = None
+    action_intents: list[str] = field(default_factory=list)
+    graph_roles: list[str] = field(default_factory=lambda: [GraphRole.UNKNOWN_PATTERN.value])
+    confidence: float = 0.0
+    alternatives: list[tuple[str, float]] = field(default_factory=list)
+    evidence: list[EvidenceItem] = field(default_factory=list)
+    should_analyze: bool = False
+    analyze_strategy: str = AnalyzeStrategy.ANALYZE_METADATA_ONLY.value
+    llm_policy: str = LLMPolicy.NO_LLM.value
+    is_protected: bool = False
+    is_noise: bool = False
+
+    def to_dict(self) -> dict[str, Any]:
+        payload = asdict(self)
+        payload["evidence"] = [item.to_dict() for item in self.evidence]
+        payload["legacy_page_type"] = get_legacy_page_type(self)
+        return payload
+
+
+LEGACY_PAGE_TYPES = {
+    PageType.PRODUCT_PAGE.value,
+    PageType.CATEGORY_PAGE.value,
+    PageType.SEARCH_PAGE.value,
+    PageType.BOARD_PAGE.value,
+    PageType.NOTICE_PAGE.value,
+    PageType.BRAND_STORY_PAGE.value,
+    PageType.PROMOTION_PAGE.value,
+    PageType.REVIEW_PAGE.value,
+    PageType.UNKNOWN_PAGE.value,
+    "EventPage",
+}
+
+
+LEGACY_ALIASES: dict[str, str] = {
+    "product": PageType.PRODUCT_PAGE.value,
+    "productpage": PageType.PRODUCT_PAGE.value,
+    "productdetailpage": PageType.PRODUCT_PAGE.value,
+    "brand": PageType.BRAND_STORY_PAGE.value,
+    "brandpage": PageType.BRAND_STORY_PAGE.value,
+    "brandstorypage": PageType.BRAND_STORY_PAGE.value,
+    "about": PageType.BRAND_STORY_PAGE.value,
+    "aboutpage": "AboutPage",
+    "contact": "ContactPage",
+    "contactpage": "ContactPage",
+    "review": PageType.REVIEW_PAGE.value,
+    "reviewpage": PageType.REVIEW_PAGE.value,
+    "productreviewpage": PageType.REVIEW_PAGE.value,
+    "listing": PageType.CATEGORY_PAGE.value,
+    "listingpage": PageType.CATEGORY_PAGE.value,
+    "category": PageType.CATEGORY_PAGE.value,
+    "categorypage": PageType.CATEGORY_PAGE.value,
+    "categorylistingpage": PageType.CATEGORY_PAGE.value,
+    "productlistingpage": PageType.CATEGORY_PAGE.value,
+    "community": PageType.BOARD_PAGE.value,
+    "communitypage": PageType.BOARD_PAGE.value,
+    "board": PageType.BOARD_PAGE.value,
+    "boardpage": PageType.BOARD_PAGE.value,
+    "forumboardpage": PageType.BOARD_PAGE.value,
+    "forumthreadpage": PageType.BOARD_PAGE.value,
+    "search": PageType.SEARCH_PAGE.value,
+    "searchpage": PageType.SEARCH_PAGE.value,
+    "searchresultspage": PageType.SEARCH_PAGE.value,
+    "notice": PageType.NOTICE_PAGE.value,
+    "noticepage": PageType.NOTICE_PAGE.value,
+    "publicnoticepage": PageType.NOTICE_PAGE.value,
+    "promotion": PageType.PROMOTION_PAGE.value,
+    "promotionpage": PageType.PROMOTION_PAGE.value,
+    "campaignlandingpage": PageType.PROMOTION_PAGE.value,
+    "event": "EventPage",
+    "eventpage": "EventPage",
+    "unknown": PageType.UNKNOWN_PAGE.value,
+    "unknownpage": PageType.UNKNOWN_PAGE.value,
+    "notfoundpage": PageType.NOT_FOUND_PAGE.value,
+    "404": PageType.NOT_FOUND_PAGE.value,
+}
+
+
+LEGACY_TO_SEMANTIC_PAGE_TYPE: dict[str, str] = {
+    PageType.PRODUCT_PAGE.value: PageType.PRODUCT_DETAIL_PAGE.value,
+    PageType.CATEGORY_PAGE.value: PageType.CATEGORY_LISTING_PAGE.value,
+    PageType.SEARCH_PAGE.value: PageType.SEARCH_RESULTS_PAGE.value,
+    PageType.BOARD_PAGE.value: PageType.FORUM_BOARD_PAGE.value,
+    PageType.NOTICE_PAGE.value: PageType.PUBLIC_NOTICE_PAGE.value,
+    PageType.BRAND_STORY_PAGE.value: PageType.BRAND_STORY_PAGE.value,
+    PageType.ABOUT_PAGE.value: PageType.ABOUT_PAGE.value,
+    PageType.CONTACT_PAGE.value: PageType.CONTACT_PAGE.value,
+    PageType.PROMOTION_PAGE.value: PageType.PROMOTION_PAGE.value,
+    PageType.REVIEW_PAGE.value: PageType.REVIEW_PAGE.value,
+    PageType.UNKNOWN_PAGE.value: PageType.UNKNOWN_PAGE.value,
+    "EventPage": "EventPage",
+}
+
+
+SEMANTIC_TO_LEGACY_PAGE_TYPE: dict[str, str] = {
+    semantic: legacy for legacy, semantic in LEGACY_TO_SEMANTIC_PAGE_TYPE.items()
+}
+SEMANTIC_TO_LEGACY_PAGE_TYPE.update(
+    {
+        PageType.PRODUCT_DETAIL_PAGE.value: PageType.PRODUCT_PAGE.value,
+        PageType.CATEGORY_LISTING_PAGE.value: PageType.CATEGORY_PAGE.value,
+        PageType.SEARCH_RESULTS_PAGE.value: PageType.SEARCH_PAGE.value,
+        PageType.FORUM_BOARD_PAGE.value: PageType.BOARD_PAGE.value,
+        PageType.FORUM_THREAD_PAGE.value: PageType.BOARD_PAGE.value,
+        PageType.PUBLIC_NOTICE_PAGE.value: PageType.NOTICE_PAGE.value,
+        PageType.CAMPAIGN_LANDING_PAGE.value: PageType.PROMOTION_PAGE.value,
+        PageType.ABOUT_PAGE.value: PageType.BRAND_STORY_PAGE.value,
+        PageType.CONTACT_PAGE.value: PageType.BRAND_STORY_PAGE.value,
+        "ListingPage": PageType.CATEGORY_PAGE.value,
+        "CommunityPage": PageType.BOARD_PAGE.value,
+    }
+)
+
+
+PAGE_TYPE_PROFILES: dict[str, dict[str, Any]] = {
+    PageType.PRODUCT_DETAIL_PAGE.value: {
+        "domain": PageDomain.COMMERCE.value,
+        "archetype": PageArchetype.DETAIL.value,
+        "main_entity_type": EntityType.PRODUCT.value,
+        "action_intents": [ActionIntent.BUY.value, ActionIntent.REVIEW.value],
+        "graph_roles": [GraphRole.ENTITY_ANCHOR.value],
+        "analyze_strategy": AnalyzeStrategy.ANALYZE_FULL.value,
+        "llm_policy": LLMPolicy.LLM_LIGHT.value,
+        "should_analyze": True,
+    },
+    PageType.CATEGORY_LISTING_PAGE.value: {
+        "domain": PageDomain.COMMERCE.value,
+        "archetype": PageArchetype.LISTING.value,
+        "main_entity_type": EntityType.PRODUCT.value,
+        "action_intents": [ActionIntent.FILTER.value, ActionIntent.NAVIGATE.value],
+        "graph_roles": [GraphRole.COLLECTION_HUB.value, GraphRole.RELATION_HUB.value],
+        "analyze_strategy": AnalyzeStrategy.ANALYZE_RELATIONS_ONLY.value,
+        "llm_policy": LLMPolicy.LLM_FOR_AMBIGUITY_ONLY.value,
+        "should_analyze": True,
+    },
+    PageType.SEARCH_RESULTS_PAGE.value: {
+        "domain": PageDomain.UNKNOWN.value,
+        "archetype": PageArchetype.SEARCH_RESULT.value,
+        "main_entity_type": EntityType.UNKNOWN_ENTITY.value,
+        "action_intents": [ActionIntent.SEARCH.value, ActionIntent.NAVIGATE.value],
+        "graph_roles": [GraphRole.SEARCH_HUB.value],
+        "analyze_strategy": AnalyzeStrategy.ANALYZE_DISCOVERY_ONLY.value,
+        "llm_policy": LLMPolicy.LLM_FOR_AMBIGUITY_ONLY.value,
+        "should_analyze": True,
+    },
+    PageType.FORUM_BOARD_PAGE.value: {
+        "domain": PageDomain.COMMUNITY.value,
+        "archetype": PageArchetype.LISTING.value,
+        "main_entity_type": EntityType.ARTICLE.value,
+        "action_intents": [ActionIntent.NAVIGATE.value, ActionIntent.READ.value],
+        "graph_roles": [GraphRole.RELATION_HUB.value, GraphRole.COLLECTION_HUB.value],
+        "analyze_strategy": AnalyzeStrategy.ANALYZE_RELATIONS_ONLY.value,
+        "llm_policy": LLMPolicy.LLM_FOR_AMBIGUITY_ONLY.value,
+        "should_analyze": True,
+    },
+    PageType.PUBLIC_NOTICE_PAGE.value: {
+        "domain": PageDomain.GOVERNMENT.value,
+        "archetype": PageArchetype.DOCUMENT.value,
+        "main_entity_type": EntityType.ARTICLE.value,
+        "action_intents": [ActionIntent.READ.value],
+        "graph_roles": [GraphRole.CLAIM_SOURCE.value],
+        "analyze_strategy": AnalyzeStrategy.ANALYZE_FULL.value,
+        "llm_policy": LLMPolicy.LLM_LIGHT.value,
+        "should_analyze": True,
+    },
+    PageType.BRAND_STORY_PAGE.value: {
+        "domain": PageDomain.CORPORATE.value,
+        "archetype": PageArchetype.ARTICLE.value,
+        "main_entity_type": EntityType.ORGANIZATION.value,
+        "action_intents": [ActionIntent.READ.value, ActionIntent.LEARN.value],
+        "graph_roles": [GraphRole.ENTITY_ANCHOR.value, GraphRole.CLAIM_SOURCE.value],
+        "analyze_strategy": AnalyzeStrategy.ANALYZE_FULL.value,
+        "llm_policy": LLMPolicy.LLM_FULL.value,
+        "should_analyze": True,
+    },
+    PageType.PROMOTION_PAGE.value: {
+        "domain": PageDomain.COMMERCE.value,
+        "archetype": PageArchetype.LANDING.value,
+        "main_entity_type": EntityType.EVENT.value,
+        "action_intents": [ActionIntent.BUY.value, ActionIntent.NAVIGATE.value],
+        "graph_roles": [GraphRole.CLAIM_SOURCE.value],
+        "analyze_strategy": AnalyzeStrategy.ANALYZE_FULL.value,
+        "llm_policy": LLMPolicy.LLM_LIGHT.value,
+        "should_analyze": True,
+    },
+    PageType.REVIEW_PAGE.value: {
+        "domain": PageDomain.COMMUNITY.value,
+        "archetype": PageArchetype.ARTICLE.value,
+        "main_entity_type": EntityType.REVIEW.value,
+        "action_intents": [ActionIntent.READ.value, ActionIntent.REVIEW.value],
+        "graph_roles": [GraphRole.CLAIM_SOURCE.value],
+        "analyze_strategy": AnalyzeStrategy.ANALYZE_FULL.value,
+        "llm_policy": LLMPolicy.LLM_LIGHT.value,
+        "should_analyze": True,
+    },
+    PageType.UNKNOWN_PAGE.value: {
+        "domain": PageDomain.UNKNOWN.value,
+        "archetype": PageArchetype.UNKNOWN.value,
+        "main_entity_type": EntityType.UNKNOWN_ENTITY.value,
+        "action_intents": [],
+        "graph_roles": [GraphRole.UNKNOWN_PATTERN.value],
+        "analyze_strategy": AnalyzeStrategy.ANALYZE_METADATA_ONLY.value,
+        "llm_policy": LLMPolicy.NO_LLM.value,
+        "should_analyze": False,
+    },
+}
+
+
+def normalize_page_type(value: object | None) -> str:
+    """Return the legacy-compatible page type string for existing callers."""
+
+    if isinstance(value, PageClassificationResult):
+        return get_legacy_page_type(value)
+    clean = str(value or "").strip()
+    if not clean:
+        return PageType.UNKNOWN_PAGE.value
+    alias = LEGACY_ALIASES.get(clean.lower())
+    if alias:
+        return alias
+    return SEMANTIC_TO_LEGACY_PAGE_TYPE.get(clean, clean)
+
+
+def normalize_semantic_page_type(value: object | None) -> str:
+    if isinstance(value, PageClassificationResult):
+        return value.primary_page_type
+    legacy = normalize_page_type(value)
+    return LEGACY_TO_SEMANTIC_PAGE_TYPE.get(legacy, str(value or legacy).strip() or PageType.UNKNOWN_PAGE.value)
+
+
+def get_legacy_page_type(value: object | None) -> str:
+    if isinstance(value, PageClassificationResult):
+        return SEMANTIC_TO_LEGACY_PAGE_TYPE.get(value.primary_page_type, value.primary_page_type)
+    return normalize_page_type(value)
+
+
+def build_classification_result_from_legacy(
+    *,
+    url: str,
+    legacy_page_type: str,
+    confidence: float = 0.55,
+    source: str = "legacy_classifier",
+) -> PageClassificationResult:
+    normalized_legacy = normalize_page_type(legacy_page_type)
+    semantic_page_type = LEGACY_TO_SEMANTIC_PAGE_TYPE.get(normalized_legacy, normalized_legacy)
+    profile = PAGE_TYPE_PROFILES.get(semantic_page_type, PAGE_TYPE_PROFILES[PageType.UNKNOWN_PAGE.value])
+    evidence = [
+        EvidenceItem(
+            key="legacy_page_type",
+            value=normalized_legacy,
+            weight=confidence,
+            source=source,
+            message=f"Legacy classifier returned {normalized_legacy}.",
+        )
+    ]
+    return PageClassificationResult(
+        url=url,
+        primary_page_type=semantic_page_type,
+        secondary_page_types=[] if semantic_page_type == normalized_legacy else [normalized_legacy],
+        domain=str(profile["domain"]),
+        archetype=str(profile["archetype"]),
+        main_entity_type=profile.get("main_entity_type"),
+        action_intents=list(profile.get("action_intents") or []),
+        graph_roles=list(profile.get("graph_roles") or []),
+        confidence=confidence,
+        alternatives=[(normalized_legacy, confidence)],
+        evidence=evidence,
+        should_analyze=bool(profile.get("should_analyze")),
+        analyze_strategy=str(profile["analyze_strategy"]),
+        llm_policy=str(profile["llm_policy"]),
+        is_protected=str(profile["analyze_strategy"]) == AnalyzeStrategy.SKIP_PROTECTED.value,
+        is_noise=str(profile["analyze_strategy"]) == AnalyzeStrategy.SKIP_NOISE.value,
+    )

ontology_platform/crawler_platform/app/core/crawler/page_unknown_patterns.py

@@ -0,0 +1,347 @@
+from __future__ import annotations
+
+from collections import Counter
+import hashlib
+import json
+import re
+from typing import Any
+from urllib.parse import urljoin, urlparse
+
+from crawler_platform.app.core.crawler.page_signal_extractor import (
+    build_raw_page_snapshot,
+    extract_page_signals,
+)
+from crawler_platform.app.core.crawler.page_type_taxonomy import PageClassificationResult, PageType
+
+
+UNKNOWN_PATTERN_VERSION = 1
+LOW_CONFIDENCE_PATTERN_THRESHOLD = 0.45
+STOPWORDS = {
+    "about",
+    "after",
+    "again",
+    "also",
+    "and",
+    "are",
+    "but",
+    "can",
+    "for",
+    "from",
+    "has",
+    "have",
+    "home",
+    "into",
+    "more",
+    "not",
+    "our",
+    "page",
+    "that",
+    "the",
+    "this",
+    "with",
+    "your",
+}
+
+
+def should_store_unknown_pattern(
+    result: PageClassificationResult,
+    *,
+    confidence_threshold: float = LOW_CONFIDENCE_PATTERN_THRESHOLD,
+) -> bool:
+    if result.primary_page_type == PageType.UNKNOWN_PAGE.value:
+        return True
+    if 0.0 < result.confidence <= confidence_threshold:
+        return True
+    return any(item.key == "low_confidence" for item in result.evidence)
+
+
+def build_unknown_pattern_payload(
+    *,
+    result: PageClassificationResult,
+    url: str,
+    title: str | None = None,
+    text: str | None = None,
+    html: str | None = None,
+    source_zones: list[str | dict[str, Any]] | None = None,
+    collector_payload: dict[str, Any] | None = None,
+) -> dict[str, Any]:
+    snapshot = build_raw_page_snapshot(
+        url=url,
+        title=title,
+        text=text,
+        html=html,
+        source_zones=source_zones,
+        collector_payload=collector_payload,
+    )
+    signals = extract_page_signals(snapshot)
+    soup = _soup_from_html(html or "")
+    page_text = _normalize_text(text or signals.text_sample or _text_from_soup(soup))
+    links = snapshot.links or _extract_links(soup, url)
+    buttons = snapshot.buttons or _extract_buttons(soup)
+    forms = snapshot.forms or _extract_forms(soup)
+    link_summary = summarize_link_patterns(links, url)
+    forms_summary = summarize_forms(forms)
+    payload = {
+        "version": UNKNOWN_PATTERN_VERSION,
+        "url": url,
+        "title": title or signals.title,
+        "primary_page_type": result.primary_page_type,
+        "confidence": result.confidence,
+        "reason": "unknown_page" if result.primary_page_type == PageType.UNKNOWN_PAGE.value else "low_confidence",
+        "text_sample": page_text[:1000],
+        "text_fingerprint": text_fingerprint(page_text),
+        "html_fingerprint": html_fingerprint(html or ""),
+        "dom_fingerprint": dom_fingerprint(html or ""),
+        "link_pattern_fingerprint": link_pattern_fingerprint(link_summary),
+        "schema_types": sorted(signals.schema_types),
+        "link_pattern_summary": link_summary,
+        "button_labels": _dedupe_strings(buttons)[:20],
+        "forms_summary": forms_summary,
+        "top_keywords": top_keywords(page_text),
+        "layout_blocks": list(signals.layout_blocks),
+        "keyword_hits": dict(signals.keyword_hits),
+        "alternatives": list(result.alternatives[:8]),
+        "evidence": [item.to_dict() for item in result.evidence[:12]],
+    }
+    payload["embedding_input"] = build_unknown_embedding_input(payload)
+    payload["cluster_candidate"] = build_cluster_candidate_payload(payload)
+    return payload
+
+
+def build_unknown_embedding_input(payload: dict[str, Any]) -> str:
+    parts = [
+        str(payload.get("title") or ""),
+        str(payload.get("text_sample") or ""),
+        "schema_types: " + ", ".join(str(item) for item in payload.get("schema_types") or []),
+        "layout_blocks: " + ", ".join(str(item) for item in payload.get("layout_blocks") or []),
+        "buttons: " + ", ".join(str(item) for item in payload.get("button_labels") or []),
+        "keywords: " + ", ".join(str(item.get("term")) for item in payload.get("top_keywords") or [] if isinstance(item, dict)),
+    ]
+    return "\n".join(part for part in parts if part.strip())[:4000]
+
+
+def build_cluster_candidate_payload(payload: dict[str, Any]) -> dict[str, Any]:
+    return {
+        "version": UNKNOWN_PATTERN_VERSION,
+        "candidate_key": stable_hash(
+            {
+                "dom": payload.get("dom_fingerprint"),
+                "links": payload.get("link_pattern_fingerprint"),
+                "schema_types": payload.get("schema_types") or [],
+                "buttons": payload.get("button_labels") or [],
+            }
+        ),
+        "fingerprints": {
+            "text": payload.get("text_fingerprint"),
+            "html": payload.get("html_fingerprint"),
+            "dom": payload.get("dom_fingerprint"),
+            "links": payload.get("link_pattern_fingerprint"),
+        },
+        "features": {
+            "schema_types": payload.get("schema_types") or [],
+            "layout_blocks": payload.get("layout_blocks") or [],
+            "top_keywords": payload.get("top_keywords") or [],
+            "link_patterns": (payload.get("link_pattern_summary") or {}).get("top_path_patterns") or [],
+            "form_count": (payload.get("forms_summary") or {}).get("form_count") or 0,
+        },
+    }
+
+
+def summarize_link_patterns(links: list[dict[str, Any]], base_url: str) -> dict[str, Any]:
+    base_host = urlparse(base_url).netloc.lower()
+    path_patterns = Counter()
+    text_labels: list[str] = []
+    internal_count = 0
+    external_count = 0
+    for link in links:
+        href = str(link.get("href") or "")
+        parsed = urlparse(href)
+        if not href:
+            continue
+        if not parsed.netloc or parsed.netloc.lower() == base_host:
+            internal_count += 1
+        else:
+            external_count += 1
+        path_patterns[_path_pattern(parsed.path)] += 1
+        text = str(link.get("text") or "").strip()
+        if text:
+            text_labels.append(text)
+    return {
+        "total_count": len(links),
+        "internal_count": internal_count,
+        "external_count": external_count,
+        "top_path_patterns": [
+            {"pattern": pattern, "count": count}
+            for pattern, count in path_patterns.most_common(12)
+            if pattern
+        ],
+        "sample_texts": _dedupe_strings(text_labels)[:12],
+    }
+
+
+def summarize_forms(forms: list[dict[str, Any]]) -> dict[str, Any]:
+    methods = Counter()
+    actions = Counter()
+    input_types = Counter()
+    form_texts: list[str] = []
+    for form in forms:
+        method = str(form.get("method") or "get").lower()
+        action = _path_pattern(urlparse(str(form.get("action") or "")).path)
+        methods[method] += 1
+        if action:
+            actions[action] += 1
+        if form.get("text"):
+            form_texts.append(str(form.get("text")))
+        for input_item in form.get("inputs") or []:
+            if isinstance(input_item, dict):
+                input_types[str(input_item.get("type") or "text").lower()] += 1
+    return {
+        "form_count": len(forms),
+        "methods": dict(methods),
+        "action_patterns": dict(actions.most_common(8)),
+        "input_types": dict(input_types.most_common(12)),
+        "sample_texts": _dedupe_strings(form_texts)[:8],
+    }
+
+
+def top_keywords(text: str, *, limit: int = 16) -> list[dict[str, Any]]:
+    tokens = [
+        token.lower()
+        for token in re.findall(r"[A-Za-z][A-Za-z0-9_-]{2,}|[가-힣]{2,}", text or "")
+        if token.lower() not in STOPWORDS
+    ]
+    return [{"term": term, "count": count} for term, count in Counter(tokens).most_common(limit)]
+
+
+def text_fingerprint(text: str) -> str:
+    return f"sha256:{_hash_text(_normalize_text(text))}"
+
+
+def html_fingerprint(html: str) -> str:
+    return f"sha256:{_hash_text(_normalize_html(html))}"
+
+
+def dom_fingerprint(html: str) -> str:
+    soup = _soup_from_html(html)
+    if soup is None:
+        return f"sha256:{_hash_text('')}"
+    tags = []
+    for node in soup.find_all(True):
+        classes = ".".join(str(item).lower() for item in node.get("class", [])[:3])
+        node_id = "#" + str(node.get("id")).lower() if node.get("id") else ""
+        tags.append(f"{node.name}{node_id}{('.' + classes) if classes else ''}")
+    return f"sha256:{_hash_text('>'.join(tags[:400]))}"
+
+
+def link_pattern_fingerprint(link_summary: dict[str, Any]) -> str:
+    return stable_hash(link_summary)
+
+
+def stable_hash(value: Any) -> str:
+    payload = json.dumps(value, sort_keys=True, ensure_ascii=False, separators=(",", ":"))
+    return f"sha256:{_hash_text(payload)}"
+
+
+def _hash_text(value: str) -> str:
+    return hashlib.sha256(value.encode("utf-8", errors="ignore")).hexdigest()
+
+
+def _normalize_text(value: str | None) -> str:
+    return re.sub(r"\s+", " ", str(value or "")).strip()
+
+
+def _normalize_html(value: str | None) -> str:
+    text = re.sub(r">\s+<", "><", str(value or ""))
+    text = re.sub(r"\s+", " ", text)
+    return text.strip()[:200000]
+
+
+def _path_pattern(path: str) -> str:
+    parts = [part for part in path.split("/") if part]
+    normalized = []
+    for part in parts[:8]:
+        if part.isdigit() or re.fullmatch(r"[0-9a-fA-F-]{8,}", part):
+            normalized.append("{id}")
+        else:
+            normalized.append(re.sub(r"\d+", "{n}", part.lower())[:60])
+    return "/" + "/".join(normalized) if normalized else "/"
+
+
+def _soup_from_html(html: str):
+    if not html:
+        return None
+    try:
+        from bs4 import BeautifulSoup
+    except ImportError:
+        return None
+    try:
+        return BeautifulSoup(html, "html.parser")
+    except Exception:
+        return None
+
+
+def _text_from_soup(soup) -> str:
+    if soup is None:
+        return ""
+    return soup.get_text(" ", strip=True)
+
+
+def _extract_links(soup, base_url: str) -> list[dict[str, Any]]:
+    if soup is None:
+        return []
+    links = []
+    for tag in soup.find_all("a"):
+        href = str(tag.get("href") or "").strip()
+        if not href:
+            continue
+        links.append({"href": urljoin(base_url, href), "text": tag.get_text(" ", strip=True)})
+    return links
+
+
+def _extract_buttons(soup) -> list[str]:
+    if soup is None:
+        return []
+    labels = []
+    for tag in soup.select("button,input[type='submit'],input[type='button'],[role='button']"):
+        label = tag.get_text(" ", strip=True) or str(tag.get("value") or tag.get("aria-label") or "")
+        if label.strip():
+            labels.append(label.strip())
+    return labels
+
+
+def _extract_forms(soup) -> list[dict[str, Any]]:
+    if soup is None:
+        return []
+    forms = []
+    for form in soup.find_all("form"):
+        inputs = []
+        for tag in form.select("input,select,textarea"):
+            inputs.append(
+                {
+                    "type": str(tag.get("type") or tag.name or ""),
+                    "name": str(tag.get("name") or ""),
+                    "placeholder": str(tag.get("placeholder") or ""),
+                }
+            )
+        forms.append(
+            {
+                "action": str(form.get("action") or ""),
+                "method": str(form.get("method") or ""),
+                "text": form.get_text(" ", strip=True)[:500],
+                "inputs": inputs,
+            }
+        )
+    return forms
+
+
+def _dedupe_strings(values: list[str]) -> list[str]:
+    seen = set()
+    result = []
+    for value in values:
+        clean = _normalize_text(value)
+        key = clean.lower()
+        if not clean or key in seen:
+            continue
+        seen.add(key)
+        result.append(clean)
+    return result

ontology_platform/crawler_platform/app/core/crawler/pipeline.py

@@ -3,11 +3,17 @@ from __future__ import annotations
 from dataclasses import dataclass
 
 from crawler_platform.app.config.loader import ProjectConfig
-from crawler_platform.app.core.crawler.page_classifier import classify_page
+from crawler_platform.app.core.crawler.page_classifier import (
+    classification_metadata,
+    classify_page_semantic,
+    get_legacy_page_type,
+    should_analyze_page,
+)
 from crawler_platform.app.core.crawler.fetchers import RobotsPolicy, make_fetcher
 from crawler_platform.app.core.crawler.plugins import ParserRegistry, default_parser_registry
 from crawler_platform.app.core.database.repository import KnowledgeRepository
 from crawler_platform.app.core.extractor.base import ExtractionPageContext, Extractor
+from crawler_platform.app.core.extractor.strategy import extract_with_strategy
 from crawler_platform.app.core.extractor.validation import attach_page_context
 
 
@@ -56,23 +62,33 @@ class CrawlPipeline:
         fetch_result = fetcher.fetch(url)
         parser = self.parser_registry.get(source_config.parser)
         parsed = parser.parse(fetch_result.analysis_html, fetch_result.final_url or url)
-        page_type = classify_page(
+        page_classification = classify_page_semantic(
             fetch_result.final_url or url,
             parsed.title or fetch_result.title,
             parsed.raw_text or parsed.text,
             fetch_result.analysis_html,
             parsed.source_zones or [],
+            final_url=fetch_result.final_url,
+            status_code=fetch_result.status_code,
+            content_type=fetch_result.headers.get("content-type"),
         )
+        page_type = get_legacy_page_type(page_classification)
 
         project = self.repository.upsert_project(project_config)
         source = self.repository.get_source(project.id, source_name)
         warnings = [*fetch_result.warnings, *(parsed.extraction_warnings or [])]
         metadata = {
             **parsed.metadata,
+            **classification_metadata(
+                page_classification,
+                title=parsed.title or fetch_result.title,
+                text=parsed.raw_text or parsed.text,
+                html=fetch_result.analysis_html,
+                source_zones=parsed.source_zones or [],
+            ),
             "final_url": fetch_result.final_url,
             "crawl_status": fetch_result.crawl_status,
             "crawl_warnings": fetch_result.warnings,
-            "page_type": page_type,
             "raw_text_length": len(parsed.raw_text or ""),
             "clean_text_length": len(parsed.text or ""),
             "main_content_preview": (parsed.main_content or parsed.text)[:800],
@@ -100,6 +116,20 @@ class CrawlPipeline:
                 robots_status=robots_decision.status,
                 robots_reason=robots_decision.reason,
             )
+        if not should_analyze_page(page_classification, None):
+            return CrawlResult(
+                page_id=page.id,
+                claim_count=0,
+                entity_count=0,
+                crawl_status=fetch_result.crawl_status,
+                extraction_status="skipped",
+                page_type=page_type,
+                clean_text_length=len(parsed.text or ""),
+                raw_text_length=len(parsed.raw_text or ""),
+                warnings=warnings,
+                robots_status=robots_decision.status,
+                robots_reason=robots_decision.reason,
+            )
 
         context = ExtractionPageContext(
             url=url,
@@ -116,7 +146,7 @@ class CrawlPipeline:
             warnings=warnings,
             metadata=metadata,
         )
-        bundle = self.extractor.extract_from_context(context, project_config)
+        bundle = extract_with_strategy(self.extractor, context, project_config)
         bundle = attach_page_context(bundle, context)
         claims = self.repository.save_extraction_bundle(project.id, source, page, bundle, project_config)
         extraction_summary = extraction_summary_from_raw(bundle.raw_output)

ontology_platform/crawler_platform/app/core/crawler/site_crawler.py

@@ -10,12 +10,16 @@ from crawler_platform.app.core.crawler.discovery import discover_links
 from crawler_platform.app.core.crawler.fetchers import RobotsDecision, RobotsPolicy, make_fetcher
 from crawler_platform.app.core.crawler.page_classifier import (
     classify_page as classify_page_type,
+    classification_metadata,
+    classify_page_semantic,
+    get_legacy_page_type,
     should_analyze_page as should_analyze_page_type,
 )
 from crawler_platform.app.core.crawler.plugins import ParserRegistry, default_parser_registry
 from crawler_platform.app.core.database import models
 from crawler_platform.app.core.database.repository import KnowledgeRepository
 from crawler_platform.app.core.extractor.base import ExtractionPageContext, Extractor
+from crawler_platform.app.core.extractor.strategy import extract_with_strategy
 from crawler_platform.app.core.extractor.validation import attach_page_context
 
 
@@ -209,6 +213,16 @@ class SiteCrawler:
         fetch_result = fetcher.fetch(url)
         if is_failed_fetch_status(fetch_result.status_code) or fetch_result.crawl_status != "success":
             error = f"fetch failed with status {fetch_result.status_code}; crawl_status={fetch_result.crawl_status}"
+            page_classification = classify_page_semantic(
+                fetch_result.final_url or url,
+                title=fetch_result.title,
+                text="",
+                html=fetch_result.analysis_html,
+                final_url=fetch_result.final_url,
+                status_code=fetch_result.status_code,
+                content_type=fetch_result.headers.get("content-type"),
+            )
+            page_type = get_legacy_page_type(page_classification)
             page = self.repository.upsert_page(
                 project_id=source.project_id,
                 source_id=source.id,
@@ -217,6 +231,11 @@ class SiteCrawler:
                 status_code=fetch_result.status_code,
                 cleaned_text="",
                 metadata={
+                    **classification_metadata(
+                        page_classification,
+                        title=fetch_result.title,
+                        html=fetch_result.analysis_html,
+                    ),
                     "final_url": fetch_result.final_url,
                     "crawl_status": fetch_result.crawl_status,
                     "crawl_warnings": fetch_result.warnings,
@@ -232,7 +251,7 @@ class SiteCrawler:
                     url=url,
                     depth=depth,
                     status=fetch_result.crawl_status,
-                    page_type="unknown",
+                    page_type=page_type,
                     page_id=page.id,
                     crawl_status=fetch_result.crawl_status,
                     robots_status=robots_decision.status,
@@ -246,13 +265,17 @@ class SiteCrawler:
             return
 
         parsed = parser.parse(fetch_result.analysis_html, fetch_result.final_url or url)
-        page_type = classify_page(
+        page_classification = classify_page_semantic(
             fetch_result.final_url or url,
             parsed.title or fetch_result.title,
             parsed.raw_text or parsed.text,
             fetch_result.analysis_html,
             parsed.source_zones or [],
+            final_url=fetch_result.final_url,
+            status_code=fetch_result.status_code,
+            content_type=fetch_result.headers.get("content-type"),
         )
+        page_type = get_legacy_page_type(page_classification)
         discovered_count = self._enqueue_links(
             html=fetch_result.analysis_html,
             base_url=fetch_result.final_url or url,
@@ -268,10 +291,16 @@ class SiteCrawler:
         warnings = [*fetch_result.warnings, *(parsed.extraction_warnings or [])]
         metadata = {
             **parsed.metadata,
+            **classification_metadata(
+                page_classification,
+                title=parsed.title or fetch_result.title,
+                text=parsed.raw_text or parsed.text,
+                html=fetch_result.analysis_html,
+                source_zones=parsed.source_zones or [],
+            ),
             "final_url": fetch_result.final_url,
             "crawl_status": fetch_result.crawl_status,
             "crawl_warnings": fetch_result.warnings,
-            "page_type": page_type,
             "depth": depth,
             "raw_text_length": len(parsed.raw_text or ""),
             "clean_text_length": len(parsed.text or ""),
@@ -319,7 +348,7 @@ class SiteCrawler:
             )
             return
 
-        if should_analyze_page(page_type, analyze_page_types):
+        if should_analyze_page(page_classification, analyze_page_types):
             context = ExtractionPageContext(
                 url=url,
                 final_url=fetch_result.final_url,
@@ -335,7 +364,7 @@ class SiteCrawler:
                 warnings=warnings,
                 metadata=metadata,
             )
-            bundle = self.extractor.extract_from_context(context, project_config)
+            bundle = extract_with_strategy(self.extractor, context, project_config)
             bundle = attach_page_context(bundle, context)
             claims = self.repository.save_extraction_bundle(source.project_id, source, page, bundle, project_config)
             extraction_summary = extraction_summary_from_raw(bundle.raw_output)

ontology_platform/crawler_platform/app/core/extractor/ai_provider.py

@@ -637,6 +637,38 @@ def list_openai_compatible_models(base_url: str, api_key: str | None = None) ->
     return [{"id": item.get("id", ""), "owned_by": item.get("owned_by")} for item in items if item.get("id")]
 
 
+def list_lmstudio_loaded_models(base_url: str) -> list[dict[str, Any]]:
+    """Return only the models currently loaded in LM Studio.
+
+    Uses LM Studio's native REST API (``/api/v0/models``) which exposes a
+    ``state`` field. Falls back to the OpenAI-compatible ``/v1/models`` list
+    (treated as all-loaded) if the native endpoint is unavailable.
+    """
+    clean = base_url.rstrip("/")
+    if clean.endswith("/v1"):
+        root = clean[: -len("/v1")]
+    elif clean.endswith("/v1/chat/completions"):
+        root = clean[: -len("/v1/chat/completions")]
+    else:
+        root = clean
+    native_url = f"{root}/api/v0/models"
+    try:
+        response = requests.get(native_url, timeout=5)
+        response.raise_for_status()
+        data = response.json()
+        items = data.get("data", []) if isinstance(data, dict) else []
+        loaded = [
+            {"id": item.get("id", ""), "owned_by": item.get("owned_by"), "state": item.get("state")}
+            for item in items
+            if item.get("id") and str(item.get("state", "")).lower() == "loaded"
+        ]
+        if loaded:
+            return loaded
+    except Exception:
+        pass
+    return list_openai_compatible_models(base_url)
+
+
 def parse_entities(items: list[dict[str, Any]]) -> list[ExtractedEntity]:
     entities: list[ExtractedEntity] = []
     for item in items:

ontology_platform/crawler_platform/app/core/extractor/base.py

@@ -61,6 +61,7 @@ class ExtractionPageContext:
         clean_text = self.clean_text
         if text_limit is not None and len(clean_text) > text_limit:
             clean_text = clean_text[:text_limit]
+        semantic_metadata = self.semantic_metadata_payload()
         zones = []
         for zone in self.source_zones:
             zone_text = str(zone.get("text") or "")
@@ -85,8 +86,34 @@ class ExtractionPageContext:
             "clean_text": clean_text,
             "source_zones": zones,
             "warnings": self.warnings,
+            "metadata": semantic_metadata,
         }
 
+    def semantic_metadata_payload(self) -> dict[str, Any]:
+        payload = {
+            key: self.metadata[key]
+            for key in ("semantic_page_type", "analyze_strategy", "llm_policy")
+            if key in self.metadata
+        }
+        classification = self.metadata.get("page_classification")
+        if isinstance(classification, dict):
+            payload["page_classification"] = {
+                key: classification.get(key)
+                for key in (
+                    "primary_page_type",
+                    "secondary_page_types",
+                    "confidence",
+                    "alternatives",
+                    "legacy_page_type",
+                    "unknown_pattern",
+                )
+                if key in classification
+            }
+            evidence = classification.get("evidence")
+            if isinstance(evidence, list):
+                payload["page_classification"]["evidence"] = evidence[:8]
+        return payload
+
 
 class Extractor(ABC):
     name = "base"

ontology_platform/crawler_platform/app/core/extractor/hybrid.py

@@ -5,6 +5,7 @@ from dataclasses import replace
 from typing import Any
 
 from crawler_platform.app.config.loader import ProjectConfig
+from crawler_platform.app.core.crawler.page_type_taxonomy import LLMPolicy
 from crawler_platform.app.core.extractor.ai_provider import LLMJsonExtractor, dedupe_entities
 from crawler_platform.app.core.extractor.base import (
     ExtractedClaim,
@@ -17,9 +18,40 @@ from crawler_platform.app.core.extractor.rule_based import GenericRuleBasedExtra
 from crawler_platform.app.domains.perfume.extractor import PerfumeRuleBasedExtractor
 
 
-LLM_PAGE_TYPES = {"ProductPage", "BrandStoryPage", "ReviewPage"}
+LLM_PAGE_TYPES = {
-SKIP_LLM_PAGE_TYPES = {"CategoryPage", "SearchPage", "ListingPage"}
+    "ProductPage",
-RULE_ONLY_PAGE_TYPES = {"BoardPage", "CommunityPage", "UnknownPage"}
+    "ProductDetailPage",
+    "BrandStoryPage",
+    "AboutPage",
+    "ContactPage",
+    "ReviewPage",
+    "NoticePage",
+    "PublicNoticePage",
+    "PromotionPage",
+    "CampaignLandingPage",
+    "ArticlePage",
+    "BlogPostPage",
+    "DocumentationPage",
+    "APIReferencePage",
+    "DatasetPage",
+    "ResearchPaperPage",
+    "JobPostingPage",
+    "CourseDetailPage",
+    "LocalBusinessPage",
+    "RealEstateListingPage",
+    "ProfilePage",
+}
+SKIP_LLM_PAGE_TYPES = {"CategoryPage", "CategoryListingPage", "SearchPage", "SearchResultsPage", "ListingPage"}
+RULE_ONLY_PAGE_TYPES = {
+    "BoardPage",
+    "ForumBoardPage",
+    "ForumThreadPage",
+    "CommunityPage",
+    "FAQPage",
+    "QAPage",
+    "VideoPage",
+    "UnknownPage",
+}
 MIN_CLEAN_TEXT_CHARS_FOR_LLM = 300
 MAX_CLEAN_TEXT_CHARS_FOR_LLM = 60000
 
@@ -331,6 +363,22 @@ def llm_skip_reason(
         return None
     page_type = str(context.page_type or "UnknownPage")
     clean_length = len(context.clean_text or page_text or "")
+    policy = llm_policy_from_context(context)
+    if policy:
+        if policy == LLMPolicy.SKIP.value:
+            return "LLM policy Skip prevents LLM extraction"
+        if policy == LLMPolicy.NO_LLM.value:
+            return "LLM policy NoLLM prevents LLM extraction"
+        if policy == LLMPolicy.RULE_ONLY.value:
+            return "LLM policy RuleOnly routes page to rule-only extraction"
+        if policy == LLMPolicy.LLM_FOR_AMBIGUITY_ONLY.value and not classification_is_ambiguous(context):
+            return "LLM policy LLMForAmbiguityOnly requires ambiguous classification evidence"
+        if policy in {LLMPolicy.LLM_FULL.value, LLMPolicy.LLM_LIGHT.value, LLMPolicy.LLM_FOR_AMBIGUITY_ONLY.value}:
+            if clean_length < MIN_CLEAN_TEXT_CHARS_FOR_LLM:
+                return f"clean text is too short for LLM extraction: {clean_length} chars"
+            if clean_length > MAX_CLEAN_TEXT_CHARS_FOR_LLM:
+                return f"clean text is too long for LLM extraction: {clean_length} chars"
+            return None
     if page_type in SKIP_LLM_PAGE_TYPES:
         return f"page type {page_type} is configured to skip LLM extraction"
     if page_type in RULE_ONLY_PAGE_TYPES:
@@ -344,6 +392,34 @@ def llm_skip_reason(
     return None
 
 
+def llm_policy_from_context(context: ExtractionPageContext) -> str | None:
+    raw_policy = context.metadata.get("llm_policy")
+    if not raw_policy:
+        classification = context.metadata.get("page_classification")
+        if isinstance(classification, dict):
+            raw_policy = classification.get("llm_policy")
+    policy = str(raw_policy or "").strip()
+    return policy or None
+
+
+def classification_is_ambiguous(context: ExtractionPageContext) -> bool:
+    classification = context.metadata.get("page_classification")
+    if not isinstance(classification, dict):
+        return False
+    confidence = float(classification.get("confidence") or 0.0)
+    if 0.0 < confidence < 0.55:
+        return True
+    alternatives = classification.get("alternatives") or []
+    if not isinstance(alternatives, list) or len(alternatives) < 2:
+        return False
+    try:
+        top_score = float(alternatives[0][1])
+        next_score = float(alternatives[1][1])
+    except (TypeError, ValueError, IndexError):
+        return False
+    return abs(top_score - next_score) < 0.08
+
+
 def merge_bundles(
     rule_bundle: ExtractionBundle,
     llm_bundle: ExtractionBundle,

ontology_platform/crawler_platform/app/core/extractor/strategy.py

@@ -0,0 +1,190 @@
+from __future__ import annotations
+
+from collections import Counter
+from typing import Any
+
+from crawler_platform.app.config.loader import ProjectConfig
+from crawler_platform.app.core.crawler.page_analysis_policy import decide_analyze_strategy, decide_llm_policy
+from crawler_platform.app.core.crawler.page_type_taxonomy import AnalyzeStrategy, LLMPolicy
+from crawler_platform.app.core.extractor.base import ExtractionBundle, ExtractionPageContext, Extractor
+
+
+def extract_with_strategy(
+    extractor: Extractor,
+    context: ExtractionPageContext,
+    project_config: ProjectConfig,
+) -> ExtractionBundle:
+    strategy = analysis_strategy_from_context(context)
+    policy = llm_policy_from_context(context)
+    if strategy == AnalyzeStrategy.SKIP_PROTECTED.value:
+        return strategy_only_bundle(context, strategy, policy, "protected page")
+    if strategy == AnalyzeStrategy.SKIP_NOISE.value:
+        return strategy_only_bundle(context, strategy, policy, "noise page")
+    if strategy == AnalyzeStrategy.ANALYZE_METADATA_ONLY.value:
+        return metadata_only_bundle(context, strategy, policy)
+    if strategy == AnalyzeStrategy.ANALYZE_DISCOVERY_ONLY.value:
+        return discovery_only_bundle(context, strategy, policy)
+    if strategy == AnalyzeStrategy.ANALYZE_RELATIONS_ONLY.value:
+        return rule_or_structure_bundle(extractor, context, project_config, strategy, policy)
+    if strategy == AnalyzeStrategy.ANALYZE_ENTITY_ONLY.value:
+        bundle = extractor.extract_from_context(context, project_config)
+        bundle.claims = []
+        bundle.raw_output = {
+            **bundle.raw_output,
+            **strategy_payload(context, strategy, policy),
+            "effective_extraction_mode": "entity_only",
+        }
+        return bundle
+    bundle = extractor.extract_from_context(context, project_config)
+    bundle.raw_output = {
+        **bundle.raw_output,
+        **strategy_payload(context, strategy, policy),
+    }
+    return bundle
+
+
+def analysis_strategy_from_context(context: ExtractionPageContext) -> str:
+    raw_strategy = context.metadata.get("analyze_strategy")
+    if raw_strategy:
+        return str(raw_strategy)
+    classification = context.metadata.get("page_classification")
+    if isinstance(classification, dict) and classification.get("analyze_strategy"):
+        return str(classification["analyze_strategy"])
+    return decide_analyze_strategy(context.page_type)
+
+
+def llm_policy_from_context(context: ExtractionPageContext) -> str:
+    raw_policy = context.metadata.get("llm_policy")
+    if raw_policy:
+        return str(raw_policy)
+    classification = context.metadata.get("page_classification")
+    if isinstance(classification, dict) and classification.get("llm_policy"):
+        return str(classification["llm_policy"])
+    return decide_llm_policy(context.page_type)
+
+
+def rule_or_structure_bundle(
+    extractor: Extractor,
+    context: ExtractionPageContext,
+    project_config: ProjectConfig,
+    strategy: str,
+    policy: str,
+) -> ExtractionBundle:
+    if policy == LLMPolicy.LLM_FOR_AMBIGUITY_ONLY.value and classification_is_ambiguous(context):
+        bundle = extractor.extract_from_context(context, project_config)
+        bundle.raw_output = {
+            **bundle.raw_output,
+            **strategy_payload(context, strategy, policy),
+            "effective_extraction_mode": "ambiguity_limited",
+        }
+        return bundle
+    rule_extractor = getattr(extractor, "rule_extractor", None)
+    if rule_extractor is not None:
+        bundle = rule_extractor.extract_from_context(context, project_config)
+        bundle.extractor_name = f"{bundle.extractor_name}_strategy_routed"
+        bundle.raw_output = {
+            **bundle.raw_output,
+            **strategy_payload(context, strategy, policy),
+            "effective_extraction_mode": "rule_only",
+            "llm_skipped": True,
+            "llm_skip_reason": f"analyze strategy {strategy} uses rule/structure extraction only",
+        }
+        return bundle
+    if getattr(extractor, "provider", "") == "ai":
+        return discovery_only_bundle(context, strategy, policy)
+    bundle = extractor.extract_from_context(context, project_config)
+    bundle.raw_output = {
+        **bundle.raw_output,
+        **strategy_payload(context, strategy, policy),
+        "effective_extraction_mode": "rule_or_structure",
+        "llm_skipped": policy in {LLMPolicy.RULE_ONLY.value, LLMPolicy.NO_LLM.value, LLMPolicy.SKIP.value},
+    }
+    return bundle
+
+
+def classification_is_ambiguous(context: ExtractionPageContext) -> bool:
+    classification = context.metadata.get("page_classification")
+    if not isinstance(classification, dict):
+        return False
+    confidence = float(classification.get("confidence") or 0.0)
+    if 0.0 < confidence < 0.55:
+        return True
+    alternatives = classification.get("alternatives") or []
+    if not isinstance(alternatives, list) or len(alternatives) < 2:
+        return False
+    try:
+        top_score = float(alternatives[0][1])
+        next_score = float(alternatives[1][1])
+    except (TypeError, ValueError, IndexError):
+        return False
+    return abs(top_score - next_score) < 0.08
+
+
+def metadata_only_bundle(context: ExtractionPageContext, strategy: str, policy: str) -> ExtractionBundle:
+    return ExtractionBundle(
+        extractor_name="metadata_only_extractor",
+        provider="strategy",
+        raw_output={
+            **strategy_payload(context, strategy, policy),
+            "effective_extraction_mode": "metadata_only",
+            "llm_skipped": True,
+            "llm_skip_reason": f"analyze strategy {strategy} does not require LLM extraction",
+        },
+    )
+
+
+def discovery_only_bundle(context: ExtractionPageContext, strategy: str, policy: str) -> ExtractionBundle:
+    return ExtractionBundle(
+        extractor_name="discovery_only_extractor",
+        provider="strategy",
+        raw_output={
+            **strategy_payload(context, strategy, policy),
+            "effective_extraction_mode": "discovery_only",
+            "llm_skipped": True,
+            "llm_skip_reason": f"analyze strategy {strategy} uses discovery/structure signals only",
+        },
+    )
+
+
+def strategy_only_bundle(context: ExtractionPageContext, strategy: str, policy: str, reason: str) -> ExtractionBundle:
+    return ExtractionBundle(
+        extractor_name="strategy_skipped_extractor",
+        provider="strategy",
+        raw_output={
+            **strategy_payload(context, strategy, policy),
+            "effective_extraction_mode": "skipped",
+            "llm_skipped": True,
+            "llm_skip_reason": reason,
+            "skip_reason": reason,
+        },
+    )
+
+
+def strategy_payload(context: ExtractionPageContext, strategy: str, policy: str) -> dict[str, Any]:
+    return {
+        "analyze_strategy": strategy,
+        "llm_policy": policy,
+        "page_type": context.page_type,
+        "semantic_page_type": context.metadata.get("semantic_page_type"),
+        "strategy_summary": structure_summary(context),
+    }
+
+
+def structure_summary(context: ExtractionPageContext) -> dict[str, Any]:
+    zone_types = Counter(str(zone.get("zone_type") or "unknown") for zone in context.source_zones)
+    classification = context.metadata.get("page_classification") if isinstance(context.metadata, dict) else None
+    return {
+        "title": context.title,
+        "url": context.final_url or context.url,
+        "clean_text_length": len(context.clean_text or ""),
+        "raw_text_length": len(context.raw_text or ""),
+        "source_zone_count": len(context.source_zones or []),
+        "source_zone_types": dict(zone_types),
+        "classification": {
+            "primary_page_type": classification.get("primary_page_type"),
+            "confidence": classification.get("confidence"),
+            "alternatives": classification.get("alternatives", [])[:5],
+        }
+        if isinstance(classification, dict)
+        else {},
+    }

ontology_platform/crawler_platform/app/core/ontology/domain_discovery.py

@@ -538,12 +538,31 @@ def relation_hint(label: str) -> tuple[str, str] | None:
 def entity_type_from_page_type(page_type: str) -> str | None:
     mapping = {
         "ProductPage": "Product",
+        "ProductDetailPage": "Product",
         "CategoryPage": "Category",
+        "CategoryListingPage": "Category",
+        "SearchPage": "Category",
+        "SearchResultsPage": "Category",
         "BrandStoryPage": "Brand",
+        "AboutPage": "Organization",
+        "ContactPage": "Organization",
         "NoticePage": "Notice",
+        "PublicNoticePage": "Notice",
         "PromotionPage": "Promotion",
+        "CampaignLandingPage": "Promotion",
         "ReviewPage": "Review",
         "BoardPage": "Article",
+        "ForumBoardPage": "Article",
+        "ForumThreadPage": "Article",
+        "ArticlePage": "Article",
+        "NewsArticlePage": "Article",
+        "BlogPostPage": "Article",
+        "FAQPage": "Article",
+        "QAPage": "Article",
+        "DocumentationPage": "Article",
+        "APIReferencePage": "Article",
+        "JobPostingPage": "JobPosting",
+        "PricingPage": "Product",
     }
     return mapping.get(page_type)
 

ontology_platform/crawler_platform/app/core/ontology/relation_schema.py

@@ -3,6 +3,8 @@ from __future__ import annotations
 from dataclasses import dataclass, field
 from typing import Any
 
+from crawler_platform.app.core.crawler.page_type_taxonomy import normalize_page_type
+
 
 @dataclass(frozen=True, slots=True)
 class RelationRule:
@@ -97,13 +99,21 @@ def relation_schema_compatible(
             return f"{predicate} expects a typed entity object"
         if object_type and rule.object_types and object_type not in rule.object_types:
             return f"object type {object_type} is not allowed for {predicate}"
-    if page_type and rule.page_types and page_type not in rule.page_types:
+    if page_type and rule.page_types and not page_type_matches_rule(page_type, rule.page_types):
         return f"predicate {predicate} is not allowed for page type {page_type}"
     if source_zone and rule.source_zones and source_zone not in rule.source_zones:
         return f"source zone {source_zone} is not allowed for {predicate}"
     return None
 
 
+def page_type_matches_rule(page_type: str, allowed_page_types: set[str]) -> bool:
+    if page_type in allowed_page_types:
+        return True
+    normalized_page_type = normalize_page_type(page_type)
+    normalized_allowed = {normalize_page_type(allowed_page_type) for allowed_page_type in allowed_page_types}
+    return normalized_page_type in normalized_allowed
+
+
 def confidence_breakdown(
     *,
     llm_confidence: float,

ontology_platform/crawler_platform/app/core/research/graph_research_loop.py

@@ -7,11 +7,17 @@ from urllib.parse import urldefrag, urlparse
 from crawler_platform.app.config.loader import ProjectConfig
 from crawler_platform.app.core.crawler.discovery import discover_links
 from crawler_platform.app.core.crawler.fetchers import RobotsPolicy, make_fetcher
-from crawler_platform.app.core.crawler.page_classifier import classify_page, should_analyze_page
+from crawler_platform.app.core.crawler.page_classifier import (
+    classification_metadata,
+    classify_page_semantic,
+    get_legacy_page_type,
+    should_analyze_page,
+)
 from crawler_platform.app.core.crawler.plugins import ParserRegistry, default_parser_registry
 from crawler_platform.app.core.database import models
 from crawler_platform.app.core.database.repository import KnowledgeRepository
 from crawler_platform.app.core.extractor.base import ExtractionPageContext, Extractor
+from crawler_platform.app.core.extractor.strategy import extract_with_strategy
 from crawler_platform.app.core.extractor.validation import attach_page_context
 from crawler_platform.app.core.research.entity_expansion import EntityExpansionPlanner
 from crawler_platform.app.core.research.exploration_queue import ExplorationItem, ExplorationQueue
@@ -206,13 +212,17 @@ class GraphResearchLoop:
 
         fetch_result = fetcher.fetch(url)
         parser_result = parser.parse(fetch_result.analysis_html, fetch_result.final_url or url)
-        page_type = classify_page(
+        page_classification = classify_page_semantic(
             fetch_result.final_url or url,
             parser_result.title or fetch_result.title,
             parser_result.raw_text or parser_result.text,
             fetch_result.analysis_html,
             parser_result.source_zones or [],
+            final_url=fetch_result.final_url,
+            status_code=fetch_result.status_code,
+            content_type=fetch_result.headers.get("content-type"),
         )
+        page_type = get_legacy_page_type(page_classification)
         relevance = self.relevance.score_url(
             project_id=source.project_id,
             url=fetch_result.final_url or url,
@@ -225,11 +235,17 @@ class GraphResearchLoop:
         )
         metadata = {
             **parser_result.metadata,
+            **classification_metadata(
+                page_classification,
+                title=parser_result.title or fetch_result.title,
+                text=parser_result.raw_text or parser_result.text,
+                html=fetch_result.analysis_html,
+                source_zones=parser_result.source_zones or [],
+            ),
             "research_item": item.to_dict(),
             "research_relevance": asdict(relevance),
             "final_url": fetch_result.final_url,
             "crawl_status": fetch_result.crawl_status,
-            "page_type": page_type,
             "robots_status": robots_decision.status,
             "robots_reason": robots_decision.reason,
             "raw_text_length": len(parser_result.raw_text or ""),
@@ -289,7 +305,7 @@ class GraphResearchLoop:
             fetch_result.crawl_status == "success"
             and parser_result.extraction_status != "failed"
             and relevance.score >= min_relevance
-            and should_analyze_page(page_type, analyze_page_types)
+            and should_analyze_page(page_classification, analyze_page_types)
         ):
             context = ExtractionPageContext(
                 url=url,
@@ -306,7 +322,7 @@ class GraphResearchLoop:
                 warnings=[*fetch_result.warnings, *(parser_result.extraction_warnings or [])],
                 metadata=metadata,
             )
-            bundle = self.extractor.extract_from_context(context, project_config)
+            bundle = extract_with_strategy(self.extractor, context, project_config)
             bundle = attach_page_context(bundle, context)
             claims = self.repository.save_extraction_bundle(source.project_id, source, page, bundle, project_config)
             analyzed = True

ontology_platform/crawler_platform/app/core/research/relevance_engine.py

@@ -8,20 +8,46 @@ from urllib.parse import unquote, urlparse
 from sqlalchemy import select
 from sqlalchemy.orm import Session
 
-from crawler_platform.app.core.crawler.page_classifier import classify_page
+from crawler_platform.app.core.crawler.page_classifier import classify_page_semantic, get_legacy_page_type
 from crawler_platform.app.core.database import models
 
 
 HIGH_VALUE_PAGE_TYPES = {
     "ProductPage": 0.95,
+    "ProductDetailPage": 0.95,
     "BrandStoryPage": 0.88,
+    "AboutPage": 0.82,
+    "ContactPage": 0.72,
     "ReviewPage": 0.82,
     "NoticePage": 0.45,
+    "PublicNoticePage": 0.45,
+    "ArticlePage": 0.55,
+    "NewsArticlePage": 0.55,
+    "BlogPostPage": 0.52,
+    "FAQPage": 0.48,
+    "QAPage": 0.48,
+    "DocumentationPage": 0.5,
+    "APIReferencePage": 0.5,
+    "WikiPage": 0.48,
+    "DatasetPage": 0.52,
+    "ResearchPaperPage": 0.6,
+    "JobPostingPage": 0.46,
+    "CourseDetailPage": 0.5,
+    "VideoPage": 0.36,
+    "LocalBusinessPage": 0.44,
+    "RealEstateListingPage": 0.44,
+    "ProfilePage": 0.36,
+    "PricingPage": 0.62,
     "EventPage": 0.5,
     "PromotionPage": 0.38,
+    "CampaignLandingPage": 0.38,
     "CategoryPage": 0.34,
+    "CategoryListingPage": 0.34,
     "SearchPage": 0.22,
+    "SearchResultsPage": 0.22,
     "BoardPage": 0.18,
+    "ForumBoardPage": 0.18,
+    "ForumThreadPage": 0.24,
     "UnknownPage": 0.25,
 }
 
@@ -96,8 +122,12 @@ class RelevanceEngine:
         normalized = url.strip().rstrip("/")
         parsed = urlparse(normalized)
         combined = unquote(f"{normalized} {label} {text[:3000]}").lower()
-        page_type = classify_page(normalized, label, text, html=html)
+        page_classification = classify_page_semantic(normalized, label, text, html=html)
-        page_type_score = HIGH_VALUE_PAGE_TYPES.get(page_type, 0.25)
+        page_type = page_classification.primary_page_type
+        page_type_score = HIGH_VALUE_PAGE_TYPES.get(
+            page_type,
+            HIGH_VALUE_PAGE_TYPES.get(get_legacy_page_type(page_classification), 0.25),
+        )
         graph_terms = self.graph_terms(project_id)
         tokens = tokenize(combined)
         overlap_count = len(tokens & graph_terms)

ontology_platform/docs/crawl_flowchart_llm_decision.svg

@@ -0,0 +1,61 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="2400" height="3600" viewBox="0 0 2400 3600">
+<defs>
+  <marker id="arrow" markerWidth="10" markerHeight="10" refX="9" refY="3" orient="auto" markerUnits="strokeWidth">
+    <path d="M0,0 L0,6 L9,3 z" fill="#1E40AF"/>
+  </marker>
+</defs>
+<rect width="100%" height="100%" fill="#F7F9FC"/>
+<text x="110" y="120" font-family="Malgun Gothic, Arial" font-size="58" font-weight="700" fill="#172033">크롤 진행 단계와 LLM 사용 판단 구조</text>
+<text x="112" y="176" font-family="Malgun Gothic, Arial" font-size="28" fill="#5C6B82">API 요청부터 Rule/LLM 추출, 검증, DB 저장까지의 흐름</text>
+<path d="M 1200 402 L 1200 470" fill="none" stroke="#1E40AF" stroke-width="4" marker-end="url(#arrow)"/>
+<path d="M 1200 602 L 1200 670" fill="none" stroke="#1E40AF" stroke-width="4" marker-end="url(#arrow)"/>
+<path d="M 1200 802 L 1200 870" fill="none" stroke="#1E40AF" stroke-width="4" marker-end="url(#arrow)"/>
+<path d="M 1200 1002 L 1200 1070" fill="none" stroke="#1E40AF" stroke-width="4" marker-end="url(#arrow)"/>
+<path d="M 1200 1222 L 1200 1300" fill="none" stroke="#1E40AF" stroke-width="4" marker-end="url(#arrow)"/>
+<path d="M 1200 1432 L 1200 1525" fill="none" stroke="#1E40AF" stroke-width="4" marker-end="url(#arrow)"/>
+<path d="M 1200 1897 L 1200 1965" fill="none" stroke="#1E40AF" stroke-width="4" marker-end="url(#arrow)"/>
+<path d="M 1200 2097 L 1200 2190" fill="none" stroke="#1E40AF" stroke-width="4" marker-end="url(#arrow)"/>
+<path d="M 1200 2562 L 1200 2635" fill="none" stroke="#1E40AF" stroke-width="4" marker-end="url(#arrow)"/>
+<path d="M 1200 3012 L 1200 3080" fill="none" stroke="#1E40AF" stroke-width="4" marker-end="url(#arrow)"/>
+<path d="M 1200 3240 L 1200 3330" fill="none" stroke="#1E40AF" stroke-width="4" marker-end="url(#arrow)"/>
+<path d="M 820 1610 L 818 1610 L 818 1809 L 800 1809" fill="none" stroke="#5C6B82" stroke-width="4" marker-end="url(#arrow)"/>\n<rect x="756" y="1779" width="124" height="48" rx="12" fill="#F7F9FC" stroke="#CBD5E1" stroke-width="2"/>\n<text x="818" y="1817" text-anchor="middle" font-family="Malgun Gothic, Arial" font-size="25" font-weight="700" fill="#5C6B82">아니오</text>
+<path d="M 1200 1695 L 1200 1765" fill="none" stroke="#16A34A" stroke-width="4" marker-end="url(#arrow)"/>\n<rect x="1138" y="1735" width="124" height="48" rx="12" fill="#F7F9FC" stroke="#CBD5E1" stroke-width="2"/>\n<text x="1200" y="1773" text-anchor="middle" font-family="Malgun Gothic, Arial" font-size="25" font-weight="700" fill="#16A34A">예</text>
+<path d="M 820 2275 L 758 2275 L 758 2496 L 740 2496" fill="none" stroke="#5C6B82" stroke-width="4" marker-end="url(#arrow)"/>\n<rect x="696" y="2466" width="124" height="48" rx="12" fill="#F7F9FC" stroke="#CBD5E1" stroke-width="2"/>\n<text x="758" y="2504" text-anchor="middle" font-family="Malgun Gothic, Arial" font-size="25" font-weight="700" fill="#5C6B82">skip</text>
+<path d="M 1200 2360 L 1200 2430" fill="none" stroke="#16A34A" stroke-width="4" marker-end="url(#arrow)"/>\n<rect x="1138" y="2400" width="124" height="48" rx="12" fill="#F7F9FC" stroke="#CBD5E1" stroke-width="2"/>\n<text x="1200" y="2438" text-anchor="middle" font-family="Malgun Gothic, Arial" font-size="25" font-weight="700" fill="#16A34A">use</text>
+<path d="M 430 2562 L 430 2946 L 820 2946" fill="none" stroke="#5C6B82" stroke-width="4" marker-end="url(#arrow)"/>
+<path d="M 1200 2785 L 1200 2880" fill="none" stroke="#16A34A" stroke-width="4" marker-end="url(#arrow)"/>
+<rect x="820" y="270" width="760" height="132" rx="24" fill="#FFFFFF" stroke="#2563EB" stroke-width="4"/>
+<text x="1200" y="316" text-anchor="middle" font-family="Malgun Gothic, Arial" font-size="30" font-weight="700" fill="#172033">API 요청</text>\n<text x="1200" y="356" text-anchor="middle" font-family="Malgun Gothic, Arial" font-size="30" font-weight="700" fill="#172033">/crawl-site</text>
+<rect x="820" y="470" width="760" height="132" rx="24" fill="#FFFFFF" stroke="#2563EB" stroke-width="4"/>
+<text x="1200" y="516" text-anchor="middle" font-family="Malgun Gothic, Arial" font-size="30" font-weight="700" fill="#172033">Job 생성 및</text>\n<text x="1200" y="556" text-anchor="middle" font-family="Malgun Gothic, Arial" font-size="30" font-weight="700" fill="#172033">queue 초기화</text>
+<rect x="820" y="670" width="760" height="132" rx="24" fill="#FFFFFF" stroke="#2563EB" stroke-width="4"/>
+<text x="1200" y="716" text-anchor="middle" font-family="Malgun Gothic, Arial" font-size="30" font-weight="700" fill="#172033">URL depth / domain /</text>\n<text x="1200" y="756" text-anchor="middle" font-family="Malgun Gothic, Arial" font-size="30" font-weight="700" fill="#172033">robots 검사</text>
+<rect x="820" y="870" width="760" height="132" rx="24" fill="#FFFFFF" stroke="#2563EB" stroke-width="4"/>
+<text x="1200" y="916" text-anchor="middle" font-family="Malgun Gothic, Arial" font-size="30" font-weight="700" fill="#172033">Fetch:</text>\n<text x="1200" y="956" text-anchor="middle" font-family="Malgun Gothic, Arial" font-size="30" font-weight="700" fill="#172033">HTML 수집</text>
+<rect x="820" y="1070" width="760" height="152" rx="24" fill="#FFFFFF" stroke="#2563EB" stroke-width="4"/>
+<text x="1200" y="1106" text-anchor="middle" font-family="Malgun Gothic, Arial" font-size="30" font-weight="700" fill="#172033">Parse:</text>\n<text x="1200" y="1146" text-anchor="middle" font-family="Malgun Gothic, Arial" font-size="30" font-weight="700" fill="#172033">본문 / zone /</text>\n<text x="1200" y="1186" text-anchor="middle" font-family="Malgun Gothic, Arial" font-size="30" font-weight="700" fill="#172033">clean_text 추출</text>
+<rect x="820" y="1300" width="760" height="132" rx="24" fill="#FFFFFF" stroke="#2563EB" stroke-width="4"/>
+<text x="1200" y="1346" text-anchor="middle" font-family="Malgun Gothic, Arial" font-size="30" font-weight="700" fill="#172033">Page Classifier:</text>\n<text x="1200" y="1386" text-anchor="middle" font-family="Malgun Gothic, Arial" font-size="30" font-weight="700" fill="#172033">ProductPage 등 분류</text>
+<polygon points="1200,1525 1580,1610 1200,1695 820,1610" fill="#FFF7ED" stroke="#D97706" stroke-width="4"/>
+<text x="1200" y="1590" text-anchor="middle" font-family="Malgun Gothic, Arial" font-size="30" font-weight="700" fill="#172033">분석 대상</text>\n<text x="1200" y="1630" text-anchor="middle" font-family="Malgun Gothic, Arial" font-size="30" font-weight="700" fill="#172033">page_type인가?</text>
+<rect x="120" y="1740" width="680" height="138" rx="24" fill="#F8FAFC" stroke="#5C6B82" stroke-width="4"/>
+<text x="460" y="1769" text-anchor="middle" font-family="Malgun Gothic, Arial" font-size="30" font-weight="700" fill="#172033">discovered/skipped</text>\n<text x="460" y="1809" text-anchor="middle" font-family="Malgun Gothic, Arial" font-size="30" font-weight="700" fill="#172033">저장,</text>\n<text x="460" y="1849" text-anchor="middle" font-family="Malgun Gothic, Arial" font-size="30" font-weight="700" fill="#172033">링크만 확장</text>
+<rect x="820" y="1765" width="760" height="132" rx="24" fill="#FFFFFF" stroke="#2563EB" stroke-width="4"/>
+<text x="1200" y="1831" text-anchor="middle" font-family="Malgun Gothic, Arial" font-size="30" font-weight="700" fill="#172033">Extractor 선택</text>
+<rect x="820" y="1965" width="760" height="132" rx="24" fill="#FFFFFF" stroke="#2563EB" stroke-width="4"/>
+<text x="1200" y="2031" text-anchor="middle" font-family="Malgun Gothic, Arial" font-size="30" font-weight="700" fill="#172033">Rule extraction</text>
+<polygon points="1200,2190 1580,2275 1200,2360 820,2275" fill="#FFF7ED" stroke="#D97706" stroke-width="4"/>
+<text x="1200" y="2275" text-anchor="middle" font-family="Malgun Gothic, Arial" font-size="30" font-weight="700" fill="#172033">LLM 사용 판단</text>
+<rect x="120" y="2430" width="620" height="132" rx="24" fill="#F8FAFC" stroke="#5C6B82" stroke-width="4"/>
+<text x="430" y="2496" text-anchor="middle" font-family="Malgun Gothic, Arial" font-size="30" font-weight="700" fill="#172033">rule_only routed</text>
+<rect x="820" y="2430" width="760" height="132" rx="24" fill="#FFFFFF" stroke="#2563EB" stroke-width="4"/>
+<text x="1200" y="2496" text-anchor="middle" font-family="Malgun Gothic, Arial" font-size="30" font-weight="700" fill="#172033">LLM extraction</text>
+<rect x="820" y="2635" width="760" height="150" rx="24" fill="#ECFDF5" stroke="#16A34A" stroke-width="4"/>
+<text x="1200" y="2690" text-anchor="middle" font-family="Malgun Gothic, Arial" font-size="30" font-weight="700" fill="#172033">rule/LLM merge,</text>\n<text x="1200" y="2730" text-anchor="middle" font-family="Malgun Gothic, Arial" font-size="30" font-weight="700" fill="#172033">agreement/conflict 계산</text>
+<rect x="820" y="2880" width="760" height="132" rx="24" fill="#FFFFFF" stroke="#2563EB" stroke-width="4"/>
+<text x="1200" y="2946" text-anchor="middle" font-family="Malgun Gothic, Arial" font-size="30" font-weight="700" fill="#172033">validation</text>
+<rect x="820" y="3080" width="760" height="160" rx="24" fill="#FFFFFF" stroke="#2563EB" stroke-width="4"/>
+<text x="1200" y="3120" text-anchor="middle" font-family="Malgun Gothic, Arial" font-size="30" font-weight="700" fill="#172033">DB 저장:</text>\n<text x="1200" y="3160" text-anchor="middle" font-family="Malgun Gothic, Arial" font-size="30" font-weight="700" fill="#172033">entities / claims /</text>\n<text x="1200" y="3200" text-anchor="middle" font-family="Malgun Gothic, Arial" font-size="30" font-weight="700" fill="#172033">evidence / log</text>
+<rect x="820" y="3330" width="760" height="132" rx="24" fill="#FFFFFF" stroke="#2563EB" stroke-width="4"/>
+<text x="1200" y="3396" text-anchor="middle" font-family="Malgun Gothic, Arial" font-size="30" font-weight="700" fill="#172033">progress 업데이트</text>
+</svg>

ontology_platform/docs/phases/PHASE_INDEX.md

@@ -63,39 +63,39 @@ FILE: ./26_05_22_semantic_page_classification/phase_04_001_evidence_scoring_clas
 PHASE 5. Analyze Strategy 및 LLM Policy 분리
 FILE: ./26_05_22_semantic_page_classification/phase_05_001_analysis_llm_policy.md
 
-1) PageClassificationResult 기반 `decide_analyze_strategy()` 추가 [TODO]
+1) PageClassificationResult 기반 `decide_analyze_strategy()` 추가 [완료]
-2) PageClassificationResult 기반 `decide_llm_policy()` 추가 [TODO]
+2) PageClassificationResult 기반 `decide_llm_policy()` 추가 [완료]
-3) `should_analyze_page(result_or_page_type, analyze_page_types=None)` compatibility 구현 [TODO]
+3) `should_analyze_page(result_or_page_type, analyze_page_types=None)` compatibility 구현 [완료]
-4) Category/Search/Board 계열을 무조건 skip하지 않고 strategy 기반으로 처리 [TODO]
+4) Category/Search/Board 계열을 무조건 skip하지 않고 strategy 기반으로 처리 [완료]
-5) Login/Checkout/Payment/Captcha/AccessDenied 계열은 SkipProtected/Skip 정책으로 처리 [TODO]
+5) Login/Checkout/Payment/Captcha/AccessDenied 계열은 SkipProtected/Skip 정책으로 처리 [완료]
 
 ---
 
 PHASE 6. Crawler, Cleaner, Extractor, Discovery/Relevance 통합
 FILE: ./26_05_22_semantic_page_classification/phase_06_001_pipeline_integration.md
 
-1) `site_crawler.py`와 `pipeline.py` metadata에 semantic classification payload 저장 [TODO]
+1) `site_crawler.py`와 `pipeline.py` metadata에 semantic classification payload 저장 [완료]
-2) `ExtractionPageContext` 또는 metadata를 통해 analyze_strategy/llm_policy 전달 [TODO]
+2) `ExtractionPageContext` 또는 metadata를 통해 analyze_strategy/llm_policy 전달 [완료]
-3) `HybridExtractor`가 LLMPolicy를 우선 사용하고 legacy page_type fallback을 유지하도록 수정 [TODO]
+3) `HybridExtractor`가 LLMPolicy를 우선 사용하고 legacy page_type fallback을 유지하도록 수정 [완료]
-4) `page_cleaner.py`, `domain_discovery.py`, `relevance_engine.py`의 legacy page_type 기대 경로와 신규 semantic type을 호환 [TODO]
+4) `page_cleaner.py`, `domain_discovery.py`, `relevance_engine.py`의 legacy page_type 기대 경로와 신규 semantic type을 호환 [완료]
 
 ---
 
 PHASE 7. Unknown Pattern 저장 기반 추가
 FILE: ./26_05_22_semantic_page_classification/phase_07_001_unknown_pattern_storage.md
 
-1) UnknownPage 또는 low confidence 페이지의 evidence payload 정의 [TODO]
+1) UnknownPage 또는 low confidence 페이지의 evidence payload 정의 [완료]
-2) text/html/link/schema/button/form summary와 fingerprint hook 추가 [TODO]
+2) text/html/link/schema/button/form summary와 fingerprint hook 추가 [완료]
-3) DB schema 변경 없이 metadata_json에 저장 가능한 초기 구조 구현 [TODO]
+3) DB schema 변경 없이 metadata_json에 저장 가능한 초기 구조 구현 [완료]
-4) 향후 clustering/embedding 확장을 위한 hook만 추가하고 실제 clustering은 이번 범위에서 제외 [TODO]
+4) 향후 clustering/embedding 확장을 위한 hook만 추가하고 실제 clustering은 이번 범위에서 제외 [완료]
 
 ---
 
 PHASE 8. 테스트 Fixture 및 회귀 검증
 FILE: ./26_05_22_semantic_page_classification/phase_08_001_tests_regression.md
 
-1) 최소 10개 이상의 HTML fixture 추가 [TODO]
+1) 최소 10개 이상의 HTML fixture 추가 [완료]
-2) ProductDetailPage, CategoryListingPage, SearchResultsPage, ArticlePage, QAPage, FAQPage, ForumThreadPage, DocumentationPage, JobPostingPage, LoginPage, CheckoutPage, TermsPage, SitemapPage, UnknownPage 단위 테스트 추가 [TODO]
+2) ProductDetailPage, CategoryListingPage, SearchResultsPage, ArticlePage, QAPage, FAQPage, ForumThreadPage, DocumentationPage, JobPostingPage, LoginPage, CheckoutPage, TermsPage, SitemapPage, UnknownPage 단위 테스트 추가 [완료]
-3) legacy `classify_page()`와 `should_analyze_page()` 호환성 테스트 추가 [TODO]
+3) legacy `classify_page()`와 `should_analyze_page()` 호환성 테스트 추가 [완료]
-4) HybridExtractor LLMPolicy 회귀 테스트 추가 [TODO]
+4) HybridExtractor LLMPolicy 회귀 테스트 추가 [완료]
-5) pytest 또는 현재 프로젝트 테스트 명령 실행 및 결과 기록 [TODO]
+5) pytest 또는 현재 프로젝트 테스트 명령 실행 및 결과 기록 [완료]

ontology_platform/ont_platform/api/main.py

@@ -156,7 +156,7 @@ def create_app() -> FastAPI:
 
     # ─── /health ──────────────────────────────────────────────────────
     @app.get("/health", tags=["meta"])
-    async def health() -> JSONResponse:
+    async def health(request: Request) -> JSONResponse:
         """Liveness check for the HTTP service and optional LLM readiness."""
         settings = platform_config.load_settings()
         if _startup_error:
@@ -172,7 +172,7 @@ def create_app() -> FastAPI:
                 },
             )
 
-        ctx = get_app_context()
+        ctx = _request_app_context(request)
         if ctx.tools.llm is None:
             return JSONResponse(
                 status_code=503,
@@ -192,13 +192,13 @@ def create_app() -> FastAPI:
 
     # ─── /info ────────────────────────────────────────────────────────
     @app.get("/info", tags=["meta"])
-    async def info() -> JSONResponse:
+    async def info(request: Request) -> JSONResponse:
         """Service-level capabilities (mirrors OntoCast /info semantics)."""
         settings = platform_config.load_settings()
         phase = int(settings.phase)
         storage_backend = settings.storage_backend
         if not _startup_error:
-            ctx = get_app_context()
+            ctx = _request_app_context(request)
             phase = int(ctx.settings.phase)
             storage_backend = ctx.settings.storage_backend
 
@@ -449,6 +449,13 @@ def create_app() -> FastAPI:
     return app
 
 
+def _request_app_context(request: Request) -> AppContext:
+    override = request.app.dependency_overrides.get(get_app_context)
+    if override is not None:
+        return override()
+    return get_app_context()
+
+
 # Top-level instance for `uvicorn platform.api.main:app`.
 app = create_app()
 

ontology_platform/ont_platform/api/product_backend.py

@@ -82,6 +82,6 @@ def _remove_route(app: FastAPI, path: str, methods: set[str]) -> None:
         for route in app.router.routes
         if not (
             getattr(route, "path", None) == path
-            and set(getattr(route, "methods", set())) == methods
+            and methods.issubset(set(getattr(route, "methods", set())))
         )
     ]

ontology_platform/web/frontend/src/lib/api/crawl.ts

@@ -86,6 +86,26 @@ export function isCrawlTerminal(status: string): boolean {
   return TERMINAL_CRAWL_STATUSES.has(status);
 }
 
+export const extractorModelsResponseSchema = z.object({
+  ok: z.boolean(),
+  error: z.string().optional(),
+  models: z
+    .array(
+      z
+        .object({
+          id: z.string(),
+          owned_by: z.string().optional(),
+        })
+        .passthrough(),
+    )
+    .optional()
+    .default([]),
+});
+
+export type ExtractorModelsResponse = z.infer<
+  typeof extractorModelsResponseSchema
+>;
+
 export const crawlApi = {
   startByProject: (body: StartSiteCrawlRequest) =>
     apiClient.post("/crawl-site/by-project", crawlJobSchema, body),
@@ -99,4 +119,9 @@ export const crawlApi = {
       `/crawl-site/jobs/${encodeURIComponent(jobId)}/cancel`,
       crawlJobSchema,
     ),
+  listExtractorModels: (provider: string, baseUrl?: string | null) =>
+    apiClient.post("/extractors/models", extractorModelsResponseSchema, {
+      provider,
+      base_url: baseUrl || null,
+    }),
 };

ontology_platform/web/frontend/src/pages/CrawlPage.tsx

@@ -1,4 +1,4 @@
-import { useState } from "react";
+import { useEffect, useState } from "react";
 import { useNavigate, useParams } from "react-router-dom";
 import { useTranslation } from "react-i18next";
 import { useForm } from "react-hook-form";
@@ -36,7 +36,7 @@ import {
   useCrawlJob,
   useStartSiteCrawl,
 } from "@/hooks/useCrawl";
-import { isCrawlTerminal } from "@/lib/api/crawl";
+import { crawlApi, isCrawlTerminal } from "@/lib/api/crawl";
 
 const startCrawlSchema = z.object({
   source_name: z.string().min(1, "소스를 선택하세요"),
@@ -150,7 +150,53 @@ export default function CrawlPage() {
   const sources = project?.sources ?? [];
   const sourceName = watch("source_name");
   const extractionMode = watch("extraction_mode");
+  const provider = watch("extractor_provider");
+  const baseUrl = watch("extractor_base_url");
   const usesLlm = extractionMode !== "rule_only";
+
+  const [loadedModelHint, setLoadedModelHint] = useState<string>("");
+  const [modelLookupStatus, setModelLookupStatus] = useState<
+    "idle" | "loading" | "ok" | "error"
+  >("idle");
+  const [modelLookupError, setModelLookupError] = useState<string>("");
+  useEffect(() => {
+    if (!usesLlm || provider !== "lm_studio") {
+      setLoadedModelHint("");
+      setModelLookupStatus("idle");
+      setModelLookupError("");
+      return;
+    }
+    let cancelled = false;
+    setModelLookupStatus("loading");
+    setModelLookupError("");
+    (async () => {
+      try {
+        const res = await crawlApi.listExtractorModels(provider, baseUrl);
+        if (cancelled) return;
+        if (!res.ok) {
+          setModelLookupStatus("error");
+          setModelLookupError(res.error || "모델 조회 실패");
+          return;
+        }
+        const first = res.models?.[0]?.id;
+        if (!first) {
+          setModelLookupStatus("error");
+          setModelLookupError("로드된 모델이 없습니다");
+          return;
+        }
+        setLoadedModelHint(first);
+        setValue("extractor_model", first, { shouldDirty: false });
+        setModelLookupStatus("ok");
+      } catch (e) {
+        if (cancelled) return;
+        setModelLookupStatus("error");
+        setModelLookupError((e as Error).message);
+      }
+    })();
+    return () => {
+      cancelled = true;
+    };
+  }, [provider, baseUrl, usesLlm, setValue]);
   const selectedSource = sources.find((s) => s.name === sourceName);
   const progress = job?.progress;
   const visited = progress?.visited_count ?? 0;
@@ -385,9 +431,30 @@ export default function CrawlPage() {
                       <Label htmlFor="extractor_model">Model</Label>
                       <Input
                         id="extractor_model"
-                        placeholder="deepseek-r1-distill-qwen-7b"
+                        placeholder={
+                          loadedModelHint || "deepseek-r1-distill-qwen-7b"
+                        }
                         {...register("extractor_model")}
                       />
+                      {provider === "lm_studio" &&
+                        modelLookupStatus === "loading" && (
+                          <p className="text-xs text-muted-foreground">
+                            LM Studio 로드 모델 확인 중...
+                          </p>
+                        )}
+                      {provider === "lm_studio" &&
+                        modelLookupStatus === "ok" &&
+                        loadedModelHint && (
+                          <p className="text-xs text-muted-foreground">
+                            LM Studio 로드 모델: {loadedModelHint}
+                          </p>
+                        )}
+                      {provider === "lm_studio" &&
+                        modelLookupStatus === "error" && (
+                          <p className="text-xs text-destructive">
+                            LM Studio 모델 조회 실패: {modelLookupError}
+                          </p>
+                        )}
                     </div>
                     <div className="space-y-1.5">
                       <Label htmlFor="extractor_base_url">Base URL</Label>