fds

ontology_platform/crawler_platform/app/api/routes.py

@@ -29,7 +29,10 @@ from crawler_platform.app.core.database.repository import (
     make_claim_hash,
 )
 from crawler_platform.app.core.database.session import session_scope
-from crawler_platform.app.core.extractor.ai_provider import list_openai_compatible_models
+from crawler_platform.app.core.extractor.ai_provider import (
+    list_lmstudio_loaded_models,
+    list_openai_compatible_models,
+)
 from crawler_platform.app.core.extractor.factory import extractor_for_domain
 from crawler_platform.app.core.ontology.definitions import DOMAIN_ONTOLOGIES, Ontology, ontology_for_domain
 from crawler_platform.app.core.ontology.domain_discovery import DomainDiscoveryService
@@ -44,6 +47,100 @@ from crawler_platform.app.core.research.memory_store import ResearchMemoryStore,
 
 DOMAIN_NAME_PATTERN = re.compile(r"^[a-z0-9][a-z0-9_-]{1,79}$")
 SITE_CRAWL_CANCEL_REQUESTS: set[int] = set()
+COMPARISON_KEYS = ("both_agree", "rule_only", "llm_only", "conflict", "rejected_by_validation")
+
+
+def extraction_log_summary(raw_output: dict[str, Any]) -> dict[str, Any]:
+    candidate_claims = raw_output.get("candidate_claims")
+    if not isinstance(candidate_claims, list):
+        candidate_claims = []
+
+    raw_comparison = raw_output.get("comparison")
+    if not isinstance(raw_comparison, dict):
+        raw_comparison = {}
+    comparison = {key: int(raw_comparison.get(key) or 0) for key in COMPARISON_KEYS}
+    if not any(comparison.values()):
+        comparison.update(comparison_from_candidate_claims(candidate_claims))
+    validation = raw_output.get("validation")
+    if isinstance(validation, dict) and comparison["rejected_by_validation"] == 0:
+        comparison["rejected_by_validation"] = number_or_default(validation.get("rejected_claim_count"), 0)
+
+    return {
+        "candidate_count": len(candidate_claims),
+        "comparison": comparison,
+        "rule_entity_count": number_or_none(raw_output.get("rule_entity_count")),
+        "rule_claim_count": number_or_derived(
+            raw_output.get("rule_claim_count"),
+            candidate_claims,
+            source="rule",
+        ),
+        "llm_entity_count": number_or_none(raw_output.get("llm_entity_count")),
+        "llm_claim_count": number_or_derived(
+            raw_output.get("llm_claim_count"),
+            candidate_claims,
+            source="llm",
+        ),
+        "agreement_claim_count": number_or_default(raw_output.get("agreement_claim_count"), comparison["both_agree"]),
+        "rule_only_claim_count": number_or_default(raw_output.get("rule_only_claim_count"), comparison["rule_only"]),
+        "llm_only_claim_count": number_or_default(raw_output.get("llm_only_claim_count"), comparison["llm_only"]),
+        "conflict_claim_count": number_or_default(raw_output.get("conflict_claim_count"), comparison["conflict"]),
+    }
+
+
+def comparison_from_candidate_claims(candidate_claims: list[Any]) -> dict[str, int]:
+    comparison = {key: 0 for key in COMPARISON_KEYS}
+    for claim in candidate_claims:
+        metadata = claim_metadata(claim)
+        agreement = str(metadata.get("agreement") or metadata.get("claim_kind") or "").lower()
+        if agreement == "rule_and_llm":
+            comparison["both_agree"] += 1
+        elif agreement == "rule_only":
+            comparison["rule_only"] += 1
+        elif agreement == "llm_only":
+            comparison["llm_only"] += 1
+        elif agreement == "conflict":
+            comparison["conflict"] += 1
+    return comparison
+
+
+def claim_metadata(claim: Any) -> dict[str, Any]:
+    if not isinstance(claim, dict):
+        return {}
+    metadata = claim.get("metadata")
+    return metadata if isinstance(metadata, dict) else {}
+
+
+def number_or_none(value: Any) -> int | None:
+    if isinstance(value, bool):
+        return None
+    if isinstance(value, (int, float)):
+        return int(value)
+    return None
+
+
+def number_or_default(value: Any, default: int) -> int:
+    parsed = number_or_none(value)
+    return default if parsed is None else parsed
+
+
+def number_or_derived(value: Any, candidate_claims: list[Any], *, source: str) -> int:
+    parsed = number_or_none(value)
+    if parsed is not None:
+        return parsed
+    return sum(1 for claim in candidate_claims if claim_matches_source(claim, source))
+
+
+def claim_matches_source(claim: Any, source: str) -> bool:
+    metadata = claim_metadata(claim)
+    extraction_source = str(metadata.get("extraction_source") or "").lower()
+    agreement = str(metadata.get("agreement") or "").lower()
+    if extraction_source == source:
+        return True
+    if source == "rule":
+        return agreement in {"rule_only", "rule_and_llm"}
+    if source == "llm":
+        return agreement in {"llm_only", "rule_and_llm"}
+    return False
 
 
 class CrawlRequest(BaseModel):
@@ -1592,7 +1689,7 @@ def register_routes(app, database_url: str) -> None:
     def extractor_models(request: ExtractorModelsRequest):
         try:
             if request.provider == "lm_studio":
-                models = list_openai_compatible_models(request.base_url or "http://localhost:1234/v1")
+                models = list_lmstudio_loaded_models(request.base_url or "http://localhost:1234/v1")
                 return {"ok": True, "models": models}
             if request.provider == "openai":
                 import os
@@ -2490,35 +2587,37 @@ def register_routes(app, database_url: str) -> None:
                 .order_by(models.ExtractionLog.created_at.desc())
                 .limit(limit)
             ).all()
-            return [
+            payload = []
-                {
+            for log, page in rows:
+                raw_output = log.raw_output or {}
+                summary = extraction_log_summary(raw_output)
+                payload.append({
                     "id": log.id,
                     "page_url": page.url if page else None,
                     "extractor_name": log.extractor_name,
                     "provider": log.provider,
                     "error": log.error,
                     "created_at": log.created_at.isoformat(),
-                    "validation": (log.raw_output or {}).get("validation"),
+                    "validation": raw_output.get("validation"),
-                    "page_context": (log.raw_output or {}).get("page_context"),
+                    "page_context": raw_output.get("page_context"),
-                    "candidate_count": len((log.raw_output or {}).get("candidate_claims") or []),
+                    "candidate_count": summary["candidate_count"],
-                    "extraction_mode": (log.raw_output or {}).get("extraction_mode"),
+                    "extraction_mode": raw_output.get("extraction_mode"),
-                    "effective_extraction_mode": (log.raw_output or {}).get("effective_extraction_mode"),
+                    "effective_extraction_mode": raw_output.get("effective_extraction_mode"),
-                    "comparison": (log.raw_output or {}).get("comparison"),
+                    "comparison": summary["comparison"],
-                    "rule_entity_count": (log.raw_output or {}).get("rule_entity_count"),
+                    "rule_entity_count": summary["rule_entity_count"],
-                    "rule_claim_count": (log.raw_output or {}).get("rule_claim_count"),
+                    "rule_claim_count": summary["rule_claim_count"],
-                    "llm_entity_count": (log.raw_output or {}).get("llm_entity_count"),
+                    "llm_entity_count": summary["llm_entity_count"],
-                    "llm_claim_count": (log.raw_output or {}).get("llm_claim_count"),
+                    "llm_claim_count": summary["llm_claim_count"],
-                    "agreement_claim_count": (log.raw_output or {}).get("agreement_claim_count"),
+                    "agreement_claim_count": summary["agreement_claim_count"],
-                    "rule_only_claim_count": (log.raw_output or {}).get("rule_only_claim_count"),
+                    "rule_only_claim_count": summary["rule_only_claim_count"],
-                    "llm_only_claim_count": (log.raw_output or {}).get("llm_only_claim_count"),
+                    "llm_only_claim_count": summary["llm_only_claim_count"],
-                    "conflict_claim_count": (log.raw_output or {}).get("conflict_claim_count"),
+                    "conflict_claim_count": summary["conflict_claim_count"],
-                    "llm_skipped": (log.raw_output or {}).get("llm_skipped"),
+                    "llm_skipped": raw_output.get("llm_skipped"),
-                    "llm_skip_reason": (log.raw_output or {}).get("llm_skip_reason"),
+                    "llm_skip_reason": raw_output.get("llm_skip_reason"),
-                    "fallback": (log.raw_output or {}).get("fallback"),
+                    "fallback": raw_output.get("fallback"),
                     "raw_output": log.raw_output,
-                }
+                })
-                for log, page in rows
+            return payload
-            ]
 
     @app.patch("/claims/{claim_id}/confidence")
     def update_claim_confidence(claim_id: int, request: UpdateClaimConfidenceRequest):

ontology_platform/crawler_platform/app/core/crawler/page_analysis_policy.py

@@ -0,0 +1,82 @@
+from __future__ import annotations
+
+from crawler_platform.app.core.crawler.page_type_scorer import PAGE_TYPE_METADATA
+from crawler_platform.app.core.crawler.page_type_taxonomy import (
+    AnalyzeStrategy,
+    LLMPolicy,
+    PageClassificationResult,
+    PageType,
+    normalize_page_type,
+    normalize_semantic_page_type,
+)
+
+
+DEFAULT_ANALYZE_STRATEGY = AnalyzeStrategy.ANALYZE_METADATA_ONLY.value
+DEFAULT_LLM_POLICY = LLMPolicy.NO_LLM.value
+
+
+def decide_analyze_strategy(result_or_page_type: object | None) -> str:
+    semantic_page_type = normalize_semantic_page_type(result_or_page_type)
+    if (
+        isinstance(result_or_page_type, PageClassificationResult)
+        and result_or_page_type.analyze_strategy
+        and result_or_page_type.analyze_strategy != DEFAULT_ANALYZE_STRATEGY
+    ):
+        return result_or_page_type.analyze_strategy
+    profile = PAGE_TYPE_METADATA.get(semantic_page_type)
+    if profile:
+        return str(profile.get("analyze_strategy") or DEFAULT_ANALYZE_STRATEGY)
+    if semantic_page_type == PageType.UNKNOWN_PAGE.value:
+        return AnalyzeStrategy.ANALYZE_METADATA_ONLY.value
+    return DEFAULT_ANALYZE_STRATEGY
+
+
+def decide_llm_policy(result_or_page_type: object | None) -> str:
+    semantic_page_type = normalize_semantic_page_type(result_or_page_type)
+    if (
+        isinstance(result_or_page_type, PageClassificationResult)
+        and result_or_page_type.llm_policy
+        and result_or_page_type.llm_policy != DEFAULT_LLM_POLICY
+    ):
+        return result_or_page_type.llm_policy
+    profile = PAGE_TYPE_METADATA.get(semantic_page_type)
+    if profile:
+        return str(profile.get("llm_policy") or DEFAULT_LLM_POLICY)
+    if semantic_page_type == PageType.UNKNOWN_PAGE.value:
+        return LLMPolicy.NO_LLM.value
+    return DEFAULT_LLM_POLICY
+
+
+def is_protected_strategy(strategy: str | AnalyzeStrategy | None) -> bool:
+    return str(strategy or "") == AnalyzeStrategy.SKIP_PROTECTED.value
+
+
+def is_noise_strategy(strategy: str | AnalyzeStrategy | None) -> bool:
+    return str(strategy or "") == AnalyzeStrategy.SKIP_NOISE.value
+
+
+def should_analyze_page(result_or_page_type: object | None, analyze_page_types: set[str] | None = None) -> bool:
+    strategy = decide_analyze_strategy(result_or_page_type)
+    if is_protected_strategy(strategy) or is_noise_strategy(strategy):
+        return False
+
+    if isinstance(result_or_page_type, PageClassificationResult):
+        return True
+
+    if analyze_page_types is None:
+        return True
+
+    normalized_page_type = normalize_page_type(result_or_page_type)
+    normalized_allowlist = {normalize_page_type(item) for item in analyze_page_types}
+    return normalized_page_type in normalized_allowlist
+
+
+def apply_analysis_policy(result: PageClassificationResult) -> PageClassificationResult:
+    strategy = decide_analyze_strategy(result)
+    llm_policy = decide_llm_policy(result)
+    result.analyze_strategy = strategy
+    result.llm_policy = llm_policy
+    result.is_protected = is_protected_strategy(strategy)
+    result.is_noise = is_noise_strategy(strategy)
+    result.should_analyze = should_analyze_page(result)
+    return result

ontology_platform/crawler_platform/app/core/crawler/page_classifier.py

@@ -3,6 +3,21 @@ from __future__ import annotations
 from urllib.parse import urlparse
 
 from crawler_platform.app.core.crawler.content_zone import is_claim_allowed_zone
+from crawler_platform.app.core.crawler.page_type_taxonomy import (
+    PageClassificationResult,
+    build_classification_result_from_legacy,
+    get_legacy_page_type,
+    normalize_page_type,
+)
+from crawler_platform.app.core.crawler.page_signal_extractor import extract_page_signals_from_page
+from crawler_platform.app.core.crawler.page_type_scorer import score_page_type
+from crawler_platform.app.core.crawler.page_unknown_patterns import (
+    build_unknown_pattern_payload,
+    should_store_unknown_pattern,
+)
+from crawler_platform.app.core.crawler.page_analysis_policy import (
+    should_analyze_page as should_analyze_page_by_policy,
+)
 
 
 PRODUCT_DETAIL_PREDICATES = {
@@ -95,6 +110,73 @@ def classify_page(
     return "UnknownPage"
 
 
+def classify_page_semantic(
+    url: str,
+    title: str | None = None,
+    text: str = "",
+    html: str | None = None,
+    source_zones: list[dict[str, object]] | None = None,
+    final_url: str | None = None,
+    status_code: int | None = None,
+    content_type: str | None = None,
+) -> PageClassificationResult:
+    signals = extract_page_signals_from_page(
+        url=url,
+        final_url=final_url,
+        status_code=status_code,
+        content_type=content_type,
+        title=title,
+        text=text,
+        html=html,
+        source_zones=source_zones,
+    )
+    result = score_page_type(url, signals)
+    if result.primary_page_type == "UnknownPage" and not result.alternatives:
+        legacy_page_type = classify_page(
+            url=url,
+            title=title,
+            text=text,
+            html=html,
+            source_zones=source_zones,
+        )
+        return build_classification_result_from_legacy(
+            url=url,
+            legacy_page_type=legacy_page_type,
+            confidence=0.35,
+            source="legacy_classifier_fallback",
+        )
+    return result
+
+
+def classification_metadata(
+    result: PageClassificationResult,
+    *,
+    title: str | None = None,
+    text: str | None = None,
+    html: str | None = None,
+    source_zones: list[str | dict[str, object]] | None = None,
+) -> dict[str, object]:
+    """Return metadata that preserves legacy page_type while carrying semantic evidence."""
+
+    payload = result.to_dict()
+    if should_store_unknown_pattern(result):
+        payload["unknown_pattern"] = build_unknown_pattern_payload(
+            result=result,
+            url=result.url,
+            title=title,
+            text=text,
+            html=html,
+            source_zones=source_zones,
+        )
+    return {
+        "page_type": get_legacy_page_type(result),
+        "semantic_page_type": result.primary_page_type,
+        "analyze_strategy": result.analyze_strategy,
+        "llm_policy": result.llm_policy,
+        "page_classification": payload,
+    }
+
+
 def relation_allowed_for_page_type(page_type: str | None, predicate: str) -> bool:
     if page_type in PRODUCT_DETAIL_PAGE_TYPES:
         return True
@@ -109,27 +191,8 @@ def claim_allowed_for_context(page_type: str | None, predicate: str, zone_type:
     return relation_allowed_for_page_type(page_type, predicate) and is_claim_allowed_zone(zone_type)
 
 
-def normalize_page_type(value: str | None) -> str:
+def should_analyze_page(page_type: object, analyze_page_types: set[str] | None) -> bool:
-    aliases = {
+    return should_analyze_page_by_policy(page_type, analyze_page_types)
-        "product": "ProductPage",
-        "brand": "BrandStoryPage",
-        "review": "ReviewPage",
-        "listing": "CategoryPage",
-        "category": "CategoryPage",
-        "community": "BoardPage",
-        "board": "BoardPage",
-        "communitypage": "BoardPage",
-        "listingpage": "CategoryPage",
-        "promotionpage": "PromotionPage",
-    }
-    clean = str(value or "").strip()
-    return aliases.get(clean.lower(), aliases.get(clean, clean or "UnknownPage"))
-
-
-def should_analyze_page(page_type: str, analyze_page_types: set[str]) -> bool:
-    normalized_page_type = normalize_page_type(page_type)
-    normalized = {normalize_page_type(item) for item in analyze_page_types}
-    return normalized_page_type in normalized
 
 
 def _zone_type(zone: dict[str, object]) -> str:

ontology_platform/crawler_platform/app/core/crawler/page_cleaner.py

@@ -71,12 +71,27 @@ ZONE_SELECTORS: dict[str, list[str]] = {
 
 ZONE_PRIORITY_BY_PAGE_TYPE = {
     "ProductPage": ["product_title", "product_summary", "product_description", "product_detail"],
+    "ProductDetailPage": ["product_title", "product_summary", "product_description", "product_detail"],
     "BrandStoryPage": ["brand_story_body"],
+    "AboutPage": ["brand_story_body"],
+    "ContactPage": ["brand_story_body"],
     "NoticePage": ["notice_body"],
+    "PublicNoticePage": ["notice_body"],
+    "ArticlePage": ["notice_body"],
+    "NewsArticlePage": ["notice_body"],
+    "BlogPostPage": ["notice_body"],
+    "FAQPage": ["notice_body"],
+    "QAPage": ["notice_body"],
     "BoardPage": ["notice_body"],
+    "ForumBoardPage": ["notice_body"],
+    "ForumThreadPage": ["notice_body"],
     "EventPage": ["event_body"],
     "PromotionPage": ["event_body"],
+    "CampaignLandingPage": ["event_body"],
     "CategoryPage": ["product_title", "product_summary"],
+    "CategoryListingPage": ["product_title", "product_summary"],
+    "SearchPage": ["product_title", "product_summary"],
+    "SearchResultsPage": ["product_title", "product_summary"],
 }
 
 STRUCTURAL_NOISE_TOKENS = {

ontology_platform/crawler_platform/app/core/crawler/page_signal_extractor.py

@@ -0,0 +1,843 @@
+from __future__ import annotations
+
+from collections import Counter
+import json
+import re
+from typing import Any
+from urllib.parse import urljoin, urlparse
+
+from crawler_platform.app.core.crawler.page_signals import PageSignals, RawPageSnapshot
+
+
+KEYWORD_GROUPS: dict[str, tuple[str, ...]] = {
+    "commerce": (
+        "product",
+        "price",
+        "sale",
+        "cart",
+        "basket",
+        "buy",
+        "checkout",
+        "sku",
+        "상품",
+        "가격",
+        "장바구니",
+        "구매",
+        "주문",
+    ),
+    "listing": (
+        "filter",
+        "sort",
+        "category",
+        "pagination",
+        "items",
+        "results",
+        "필터",
+        "정렬",
+        "카테고리",
+        "상품수",
+        "결과",
+    ),
+    "editorial": (
+        "article",
+        "author",
+        "published",
+        "updated",
+        "headline",
+        "news",
+        "blog",
+        "기사",
+        "작성자",
+        "게시일",
+    ),
+    "community": (
+        "question",
+        "answer",
+        "comment",
+        "reply",
+        "thread",
+        "vote",
+        "faq",
+        "q&a",
+        "질문",
+        "답변",
+        "댓글",
+        "문의",
+    ),
+    "knowledge": (
+        "documentation",
+        "api",
+        "endpoint",
+        "parameter",
+        "version",
+        "reference",
+        "guide",
+        "문서",
+        "가이드",
+        "버전",
+    ),
+    "corporate": (
+        "about",
+        "company",
+        "contact",
+        "address",
+        "team",
+        "careers",
+        "privacy",
+        "terms",
+        "회사",
+        "소개",
+        "문의",
+        "주소",
+        "채용",
+        "개인정보",
+        "약관",
+    ),
+    "protected": (
+        "login",
+        "password",
+        "captcha",
+        "access denied",
+        "forbidden",
+        "payment",
+        "billing",
+        "로그인",
+        "비밀번호",
+        "보안문자",
+        "접근 제한",
+        "결제",
+    ),
+}
+
+
+PRICE_PATTERN = re.compile(
+    r"(?:[$€£¥₩]\s?\d[\d,]*(?:\.\d+)?)|(?:\d[\d,]*(?:\.\d+)?\s?(?:KRW|USD|EUR|JPY|원|달러))",
+    re.IGNORECASE,
+)
+DATE_PATTERN = re.compile(r"\b(?:20\d{2}|19\d{2})[-./년]\s?\d{1,2}[-./월]\s?\d{1,2}", re.IGNORECASE)
+API_ENDPOINT_PATTERN = re.compile(r"\b(?:GET|POST|PUT|PATCH|DELETE)\s+/(?:[A-Za-z0-9_./{}:-]+)")
+VERSION_PATTERN = re.compile(r"\bv?\d+\.\d+(?:\.\d+)?\b", re.IGNORECASE)
+
+
+def build_raw_page_snapshot(
+    *,
+    url: str,
+    final_url: str | None = None,
+    status_code: int | None = None,
+    content_type: str | None = None,
+    title: str | None = None,
+    text: str | None = None,
+    html: str | None = None,
+    rendered_html: str | None = None,
+    metadata: dict[str, Any] | None = None,
+    source_zones: list[str | dict[str, Any]] | None = None,
+    collector_payload: dict[str, Any] | None = None,
+) -> RawPageSnapshot:
+    snapshot = RawPageSnapshot(
+        url=url,
+        final_url=final_url,
+        status_code=status_code,
+        content_type=content_type,
+        title=title,
+        text=text,
+        html=html,
+        rendered_html=rendered_html,
+        metadata=dict(metadata or {}),
+        source_zones=list(source_zones or []),
+        collector_payload=dict(collector_payload or {}),
+    )
+    return merge_collector_payload(snapshot)
+
+
+def snapshot_from_collector_payload(payload: dict[str, Any], *, url: str | None = None) -> RawPageSnapshot:
+    return merge_collector_payload(
+        RawPageSnapshot(
+            url=str(url or payload.get("url") or payload.get("source_url") or ""),
+            final_url=payload.get("final_url") or payload.get("resolved_url"),
+            status_code=_optional_int(payload.get("status_code")),
+            content_type=payload.get("content_type"),
+            title=payload.get("title"),
+            text=payload.get("text") or payload.get("markdown") or payload.get("clean_text"),
+            html=payload.get("html") or payload.get("raw_html"),
+            rendered_html=payload.get("rendered_html"),
+            metadata=dict(payload.get("metadata") or {}),
+            collector_payload=dict(payload),
+        )
+    )
+
+
+def merge_collector_payload(snapshot: RawPageSnapshot) -> RawPageSnapshot:
+    payload = snapshot.collector_payload
+    if not payload:
+        return snapshot
+    snapshot.open_graph.update(payload.get("open_graph") or payload.get("og") or {})
+    snapshot.twitter_card.update(payload.get("twitter_card") or payload.get("twitter") or {})
+    snapshot.json_ld.extend(_ensure_dict_list(payload.get("json_ld") or payload.get("jsonld")))
+    snapshot.microdata.extend(_ensure_dict_list(payload.get("microdata")))
+    snapshot.rdfa.extend(_ensure_dict_list(payload.get("rdfa")))
+    snapshot.headings.extend(_string_list(payload.get("headings")))
+    snapshot.links.extend(_dict_list(payload.get("links")))
+    snapshot.images.extend(_dict_list(payload.get("images")))
+    snapshot.forms.extend(_dict_list(payload.get("forms")))
+    snapshot.buttons.extend(_string_list(payload.get("buttons")))
+    snapshot.inputs.extend(_dict_list(payload.get("inputs")))
+    snapshot.tables.extend(_dict_list(payload.get("tables")))
+    snapshot.breadcrumbs.extend(_string_list(payload.get("breadcrumbs")))
+    if payload.get("screenshot_path") and not snapshot.screenshot_path:
+        snapshot.screenshot_path = str(payload["screenshot_path"])
+    return snapshot
+
+
+def extract_page_signals_from_page(
+    *,
+    url: str,
+    final_url: str | None = None,
+    status_code: int | None = None,
+    content_type: str | None = None,
+    title: str | None = None,
+    text: str | None = None,
+    html: str | None = None,
+    rendered_html: str | None = None,
+    metadata: dict[str, Any] | None = None,
+    source_zones: list[str | dict[str, Any]] | None = None,
+    collector_payload: dict[str, Any] | None = None,
+) -> PageSignals:
+    return extract_page_signals(
+        build_raw_page_snapshot(
+            url=url,
+            final_url=final_url,
+            status_code=status_code,
+            content_type=content_type,
+            title=title,
+            text=text,
+            html=html,
+            rendered_html=rendered_html,
+            metadata=metadata,
+            source_zones=source_zones,
+            collector_payload=collector_payload,
+        )
+    )
+
+
+def extract_page_signals(snapshot: RawPageSnapshot) -> PageSignals:
+    html = snapshot.rendered_html or snapshot.html or ""
+    soup = _soup_from_html(html)
+    title = snapshot.title or _title_from_soup(soup)
+    text = snapshot.text or _text_from_soup(soup) or _text_from_html(html)
+    combined = "\n".join([snapshot.url, title or "", text or "", html[:12000]]).lower()
+    open_graph = {**_extract_meta_prefix(soup, "property", "og:"), **snapshot.open_graph}
+    twitter_card = {**_extract_meta_prefix(soup, "name", "twitter:"), **snapshot.twitter_card}
+    json_ld = [*_extract_json_ld(soup), *snapshot.json_ld]
+    links = snapshot.links or _extract_links(soup, snapshot.final_url or snapshot.url)
+    images = snapshot.images or _extract_images(soup, snapshot.final_url or snapshot.url)
+    forms = snapshot.forms or _extract_forms(soup)
+    inputs = snapshot.inputs or _extract_inputs(soup)
+    buttons = snapshot.buttons or _extract_buttons(soup)
+    headings = snapshot.headings or _extract_headings(soup)
+    tables = snapshot.tables or _extract_tables(soup)
+    breadcrumbs = snapshot.breadcrumbs or _extract_breadcrumbs(soup)
+    schema_types = _schema_types(json_ld, snapshot.microdata, snapshot.rdfa, soup)
+    keyword_hits = _keyword_hits(combined)
+    link_counts = _link_counts(links, snapshot.final_url or snapshot.url)
+    layout = _layout_signals(soup, combined, links, images, tables)
+    repeated_card_count = max(
+        layout["card_count"],
+        link_counts["product_link_count"],
+        _count_selector_matches(soup, CARD_SELECTORS),
+    )
+    button_text = " ".join(buttons).lower()
+    input_text = " ".join(_input_blob(item) for item in inputs).lower()
+    form_text = " ".join(_form_blob(item) for item in forms).lower()
+    table_text = " ".join(str(table.get("text") or "") for table in tables).lower()
+    content_type = str(snapshot.content_type or snapshot.metadata.get("content_type") or "").lower()
+    path = urlparse(snapshot.final_url or snapshot.url).path.lower()
+
+    signals = PageSignals(
+        schema_types=schema_types,
+        og_type=_string_or_none(open_graph.get("type") or open_graph.get("og:type")),
+        twitter_card_type=_string_or_none(twitter_card.get("card") or twitter_card.get("twitter:card")),
+        has_price=bool(PRICE_PATTERN.search(combined)),
+        has_currency=bool(re.search(r"[$€£¥₩]|(?:\b(?:KRW|USD|EUR|JPY)\b)|원", combined, re.IGNORECASE)),
+        has_cart_button=_contains_any(button_text + " " + combined, ("cart", "basket", "장바구니", "bag")),
+        has_buy_button=_contains_any(button_text + " " + combined, ("buy now", "purchase", "구매", "주문", "결제")),
+        has_variant_selector=_has_variant_selector(soup, input_text + " " + combined),
+        has_sku=bool(re.search(r"\bsku\b|상품\s*코드|product\s*code", combined, re.IGNORECASE)),
+        has_rating=("AggregateRating" in schema_types)
+        or _contains_any(combined, ("rating", "stars", "별점", "평점")),
+        has_review_section=("Review" in schema_types) or _contains_any(combined, ("review", "reviews", "후기", "리뷰")),
+        has_product_gallery=(len(images) >= 3 and _contains_any(combined, ("gallery", "product", "상품"))),
+        has_repeated_cards=repeated_card_count >= 3,
+        repeated_card_count=repeated_card_count,
+        has_filter_panel=layout["has_filter_sidebar"]
+        or _contains_any(combined, ("filter", "facets", "refine", "필터", "조건")),
+        has_sort_control=_contains_any(combined, ("sort", "order by", "low price", "high price", "정렬", "낮은가격", "높은가격")),
+        has_pagination=_has_pagination(soup, links, combined),
+        has_author=_contains_any(combined, ("author", "byline", "작성자", "기자")),
+        has_published_date=("datePublished" in _json_keys(json_ld))
+        or bool(DATE_PATTERN.search(combined))
+        and _contains_any(combined, ("published", "posted", "게시", "등록")),
+        has_modified_date=("dateModified" in _json_keys(json_ld))
+        or _contains_any(combined, ("modified", "updated", "수정")),
+        has_article_body=("Article" in schema_types)
+        or ("NewsArticle" in schema_types)
+        or _count_selector_matches(soup, ("article", "[itemprop='articleBody']", ".article-body", ".post-content")) > 0
+        or _contains_any(combined, ("articlebody", "article body")),
+        has_tags=_has_tags(soup, links, combined),
+        has_question=("QAPage" in schema_types) or _contains_any(combined, ("question", "q:", "질문", "문의")),
+        has_answer=("Answer" in schema_types) or _contains_any(combined, ("answer", "a:", "답변")),
+        has_comments=_contains_any(combined, ("comment", "comments", "reply", "댓글", "답글")),
+        has_votes=_contains_any(combined, ("vote", "votes", "upvote", "downvote", "추천", "투표")),
+        has_thread_structure=_contains_any(combined, ("thread", "discussion", "게시글", "토론")),
+        has_faq_structure=("FAQPage" in schema_types) or _contains_any(combined, ("faq", "frequently asked", "자주 묻는")),
+        has_code_blocks=_count_selector_matches(soup, ("pre", "code", ".highlight", ".code")) > 0,
+        has_toc=_count_selector_matches(soup, ("#toc", ".toc", "[class*='table-of-contents']", "nav[aria-label*='contents']")) > 0,
+        has_api_endpoint=bool(API_ENDPOINT_PATTERN.search(f"{text or ''}\n{html or ''}")),
+        has_parameter_table=_has_parameter_table(tables, table_text),
+        has_version_info=bool(VERSION_PATTERN.search(combined)) and _contains_any(combined, ("version", "버전", "release")),
+        has_contact_info=_contains_any(combined, ("contact", "email", "tel:", "문의", "연락처")),
+        has_address=_contains_any(combined, ("address", "주소", "road", "street")),
+        has_policy_terms=_contains_any(combined, ("terms", "policy", "agreement", "약관", "정책")),
+        has_privacy_terms=_contains_any(combined, ("privacy", "personal information", "개인정보")),
+        has_career_terms=_contains_any(combined, ("career", "jobs", "recruit", "채용", "지원")),
+        has_login_form=_contains_any(form_text + " " + combined, ("login", "sign in", "로그인")) and (
+            "password" in input_text or "비밀번호" in combined
+        ),
+        has_password_field="password" in input_text,
+        has_payment_fields=_contains_any(input_text + " " + combined, ("card number", "payment", "billing", "결제", "카드")),
+        has_captcha=_contains_any(combined, ("captcha", "recaptcha", "hcaptcha", "보안문자")),
+        has_access_denied=_contains_any(combined, ("access denied", "forbidden", "permission denied", "접근 제한", "권한이 없습니다")),
+        status_code=snapshot.status_code,
+        has_error_status=bool(snapshot.status_code and snapshot.status_code >= 400)
+        or _contains_any(combined, ("404", "not found", "error page")),
+        has_not_found=snapshot.status_code == 404 or _contains_any(combined, ("404", "not found", "page not found")),
+        has_sitemap_resource=("sitemap" in path) or ("sitemap" in content_type and "xml" in content_type),
+        has_feed_resource=("rss" in content_type) or ("atom" in content_type) or path.endswith((".rss", ".atom")),
+        has_json_resource=("json" in content_type) or path.endswith(".json"),
+        has_xml_resource=("xml" in content_type) or path.endswith(".xml"),
+        has_file_resource=path.endswith((".pdf", ".csv", ".xlsx", ".xls", ".doc", ".docx", ".zip")),
+        internal_link_count=link_counts["internal_link_count"],
+        external_link_count=link_counts["external_link_count"],
+        product_link_count=link_counts["product_link_count"],
+        category_link_count=link_counts["category_link_count"],
+        profile_link_count=link_counts["profile_link_count"],
+        article_link_count=link_counts["article_link_count"],
+        layout_blocks=layout["layout_blocks"],
+        has_hero_block=layout["has_hero_block"],
+        has_card_grid=layout["has_card_grid"],
+        has_filter_sidebar=layout["has_filter_sidebar"],
+        has_sticky_action_box=layout["has_sticky_action_box"],
+        has_media_player_area=layout["has_media_player_area"],
+        has_map_area=layout["has_map_area"],
+        has_calendar_grid=layout["has_calendar_grid"],
+        has_pricing_table=layout["has_pricing_table"],
+        has_comparison_table=layout["has_comparison_table"],
+        dominant_language=_dominant_language(combined),
+        keyword_hits=keyword_hits,
+        url_hints=_url_hints(snapshot.final_url or snapshot.url),
+        title=title,
+        text_sample=(text or "")[:500],
+        external_collector_signals=_external_collector_signals(snapshot),
+    )
+    return signals
+
+
+def _soup_from_html(html: str):
+    if not html:
+        return None
+    try:
+        from bs4 import BeautifulSoup
+    except ImportError:
+        return None
+    try:
+        return BeautifulSoup(html, "html.parser")
+    except Exception:
+        return None
+
+
+def _title_from_soup(soup) -> str | None:
+    if soup is None or not soup.title:
+        return None
+    return soup.title.get_text(" ", strip=True) or None
+
+
+def _text_from_soup(soup) -> str:
+    if soup is None:
+        return ""
+    return soup.get_text("\n", strip=True)
+
+
+def _text_from_html(html: str) -> str:
+    return re.sub(r"\s+", " ", re.sub(r"<[^>]+>", " ", html or "")).strip()
+
+
+def _extract_meta_prefix(soup, attr_name: str, prefix: str) -> dict[str, str]:
+    if soup is None:
+        return {}
+    result: dict[str, str] = {}
+    for tag in soup.find_all("meta"):
+        name = str(tag.get(attr_name) or "").strip()
+        if not name.lower().startswith(prefix):
+            continue
+        content = str(tag.get("content") or "").strip()
+        if content:
+            result[name.removeprefix(prefix)] = content
+            result[name] = content
+    return result
+
+
+def _extract_json_ld(soup) -> list[dict[str, Any]]:
+    if soup is None:
+        return []
+    payloads: list[dict[str, Any]] = []
+    for tag in soup.find_all("script"):
+        script_type = str(tag.get("type") or "").lower()
+        if "ld+json" not in script_type:
+            continue
+        raw = tag.string or tag.get_text(" ", strip=True)
+        try:
+            value = json.loads(raw)
+        except Exception:
+            continue
+        payloads.extend(_ensure_dict_list(value))
+    return payloads
+
+
+def _extract_headings(soup) -> list[str]:
+    if soup is None:
+        return []
+    return [node.get_text(" ", strip=True) for node in soup.select("h1,h2,h3") if node.get_text(" ", strip=True)]
+
+
+def _extract_links(soup, base_url: str) -> list[dict[str, Any]]:
+    if soup is None:
+        return []
+    links = []
+    for tag in soup.find_all("a"):
+        href = str(tag.get("href") or "").strip()
+        if not href:
+            continue
+        links.append(
+            {
+                "href": urljoin(base_url, href),
+                "text": tag.get_text(" ", strip=True),
+                "rel": " ".join(str(item) for item in tag.get("rel", [])),
+                "class": " ".join(str(item) for item in tag.get("class", [])),
+            }
+        )
+    return links
+
+
+def _extract_images(soup, base_url: str) -> list[dict[str, Any]]:
+    if soup is None:
+        return []
+    images = []
+    for tag in soup.find_all("img"):
+        src = str(tag.get("src") or tag.get("data-src") or "").strip()
+        if not src:
+            continue
+        images.append(
+            {
+                "src": urljoin(base_url, src),
+                "alt": str(tag.get("alt") or ""),
+                "class": " ".join(str(item) for item in tag.get("class", [])),
+            }
+        )
+    return images
+
+
+def _extract_forms(soup) -> list[dict[str, Any]]:
+    if soup is None:
+        return []
+    forms = []
+    for form in soup.find_all("form"):
+        forms.append(
+            {
+                "action": str(form.get("action") or ""),
+                "method": str(form.get("method") or ""),
+                "id": str(form.get("id") or ""),
+                "class": " ".join(str(item) for item in form.get("class", [])),
+                "text": form.get_text(" ", strip=True)[:500],
+            }
+        )
+    return forms
+
+
+def _extract_inputs(soup) -> list[dict[str, Any]]:
+    if soup is None:
+        return []
+    inputs = []
+    for tag in soup.select("input,select,textarea"):
+        inputs.append(
+            {
+                "type": str(tag.get("type") or tag.name or ""),
+                "name": str(tag.get("name") or ""),
+                "id": str(tag.get("id") or ""),
+                "placeholder": str(tag.get("placeholder") or ""),
+                "autocomplete": str(tag.get("autocomplete") or ""),
+                "aria_label": str(tag.get("aria-label") or ""),
+                "class": " ".join(str(item) for item in tag.get("class", [])),
+                "text": tag.get_text(" ", strip=True)[:240],
+            }
+        )
+    return inputs
+
+
+def _extract_buttons(soup) -> list[str]:
+    if soup is None:
+        return []
+    values = []
+    for tag in soup.select("button,input[type='submit'],input[type='button'],[role='button']"):
+        text = tag.get_text(" ", strip=True) or str(tag.get("value") or tag.get("aria-label") or "")
+        if text.strip():
+            values.append(text.strip())
+    return values
+
+
+def _extract_tables(soup) -> list[dict[str, Any]]:
+    if soup is None:
+        return []
+    tables = []
+    for table in soup.find_all("table"):
+        headers = [cell.get_text(" ", strip=True) for cell in table.select("th") if cell.get_text(" ", strip=True)]
+        text = table.get_text(" ", strip=True)
+        tables.append({"headers": headers, "text": text[:1000], "row_count": len(table.select("tr"))})
+    return tables
+
+
+def _extract_breadcrumbs(soup) -> list[str]:
+    if soup is None:
+        return []
+    crumbs = []
+    selectors = [
+        "[class*='breadcrumb']",
+        "[id*='breadcrumb']",
+        "nav[aria-label*='breadcrumb' i]",
+        "[itemtype*='BreadcrumbList']",
+    ]
+    for selector in selectors:
+        for node in soup.select(selector):
+            text = node.get_text(" > ", strip=True)
+            if text:
+                crumbs.append(text)
+    return _dedupe_strings(crumbs)
+
+
+def _schema_types(
+    json_ld: list[dict[str, Any]],
+    microdata: list[dict[str, Any]],
+    rdfa: list[dict[str, Any]],
+    soup,
+) -> set[str]:
+    types: set[str] = set()
+    for payload in [*json_ld, *microdata, *rdfa]:
+        _visit_schema_types(payload, types)
+    if soup is not None:
+        for node in soup.select("[itemscope][itemtype]"):
+            raw = str(node.get("itemtype") or "")
+            if raw:
+                types.add(raw.rstrip("/").split("/")[-1])
+        for node in soup.select("[typeof]"):
+            for item in str(node.get("typeof") or "").split():
+                types.add(item.split(":")[-1])
+    return {item for item in types if item}
+
+
+def _visit_schema_types(value: Any, types: set[str]) -> None:
+    if isinstance(value, list):
+        for item in value:
+            _visit_schema_types(item, types)
+        return
+    if not isinstance(value, dict):
+        return
+    raw_type = value.get("@type") or value.get("type")
+    for item in _ensure_list(raw_type):
+        if isinstance(item, str):
+            types.add(item.rstrip("/").split("/")[-1])
+    for nested in value.values():
+        if isinstance(nested, (dict, list)):
+            _visit_schema_types(nested, types)
+
+
+def _json_keys(json_ld: list[dict[str, Any]]) -> set[str]:
+    keys: set[str] = set()
+
+    def visit(value: Any) -> None:
+        if isinstance(value, list):
+            for item in value:
+                visit(item)
+            return
+        if not isinstance(value, dict):
+            return
+        keys.update(str(key) for key in value)
+        for nested in value.values():
+            visit(nested)
+
+    visit(json_ld)
+    return keys
+
+
+def _keyword_hits(text: str) -> dict[str, int]:
+    hits = {}
+    for group, terms in KEYWORD_GROUPS.items():
+        count = sum(text.count(term.lower()) for term in terms)
+        if count:
+            hits[group] = count
+    return hits
+
+
+def _link_counts(links: list[dict[str, Any]], base_url: str) -> dict[str, int]:
+    base = urlparse(base_url)
+    base_host = base.netloc.lower()
+    base_path = base.path.lower()
+    counts = Counter()
+    for link in links:
+        href = str(link.get("href") or "")
+        parsed = urlparse(href)
+        host = parsed.netloc.lower()
+        path = parsed.path.lower()
+        if not host or host == base_host:
+            counts["internal_link_count"] += 1
+        else:
+            counts["external_link_count"] += 1
+        same_document_query_link = path == base_path and bool(parsed.query)
+        if not same_document_query_link and any(token in path for token in ("/product", "/products", "/goods", "/item", "/p/")):
+            counts["product_link_count"] += 1
+        if any(token in path for token in ("category", "collection", "/shop", "/list", "catalog")):
+            counts["category_link_count"] += 1
+        if any(token in path for token in ("profile", "user", "author", "member", "creator")):
+            counts["profile_link_count"] += 1
+        if any(token in path for token in ("article", "blog", "news", "post", "story")):
+            counts["article_link_count"] += 1
+    return {
+        "internal_link_count": counts["internal_link_count"],
+        "external_link_count": counts["external_link_count"],
+        "product_link_count": counts["product_link_count"],
+        "category_link_count": counts["category_link_count"],
+        "profile_link_count": counts["profile_link_count"],
+        "article_link_count": counts["article_link_count"],
+    }
+
+
+CARD_SELECTORS = (
+    ".product-card",
+    ".card",
+    ".item",
+    ".product",
+    ".prdList > li",
+    "[class*='product-card']",
+    "[class*='grid-item']",
+)
+
+
+def _layout_signals(soup, text: str, links: list[dict[str, Any]], images: list[dict[str, Any]], tables: list[dict[str, Any]]) -> dict[str, Any]:
+    blocks: list[str] = []
+    hero = _count_selector_matches(soup, (".hero", ".visual", ".main-visual", "[class*='hero']", "section[aria-label*='hero']")) > 0
+    card_count = _count_selector_matches(soup, CARD_SELECTORS)
+    card_grid = card_count >= 3 or _count_selector_matches(soup, (".grid", "[class*='grid']", "[class*='cards']")) > 0 and len(links) >= 3
+    filter_sidebar = _count_selector_matches(soup, (".filter", ".filters", ".facet", "aside", "[class*='filter']", "[class*='facet']")) > 0
+    sticky_action = _count_selector_matches(soup, (".sticky", "[class*='sticky']", "[class*='fixed']", "[class*='buy-box']")) > 0
+    media_player = _count_selector_matches(soup, ("video", "audio", "iframe[src*='youtube']", "[class*='player']")) > 0
+    map_area = _count_selector_matches(soup, ("[class*='map']", "#map", "iframe[src*='maps']")) > 0 or "google map" in text
+    calendar_grid = _count_selector_matches(soup, ("[class*='calendar']", "[class*='datepicker']", "table.calendar")) > 0
+    pricing_table = "pricing" in text and (bool(tables) or _count_selector_matches(soup, ("[class*='pricing']", ".price-table")) > 0)
+    comparison_table = _has_comparison_table(tables, text)
+    for label, present in [
+        ("hero_block", hero),
+        ("card_grid", card_grid),
+        ("filter_sidebar", filter_sidebar),
+        ("sticky_action_box", sticky_action),
+        ("media_player_area", media_player),
+        ("map_area", map_area),
+        ("calendar_grid", calendar_grid),
+        ("pricing_table", pricing_table),
+        ("comparison_table", comparison_table),
+    ]:
+        if present:
+            blocks.append(label)
+    return {
+        "layout_blocks": blocks,
+        "card_count": card_count,
+        "has_hero_block": hero,
+        "has_card_grid": card_grid,
+        "has_filter_sidebar": filter_sidebar,
+        "has_sticky_action_box": sticky_action,
+        "has_media_player_area": media_player,
+        "has_map_area": map_area,
+        "has_calendar_grid": calendar_grid,
+        "has_pricing_table": pricing_table,
+        "has_comparison_table": comparison_table,
+    }
+
+
+def _has_variant_selector(soup, text: str) -> bool:
+    if _contains_any(text, ("variant", "option", "size", "color", "옵션", "사이즈", "색상")):
+        return True
+    if soup is None:
+        return False
+    for select in soup.find_all("select"):
+        blob = " ".join(
+            [
+                str(select.get("name") or ""),
+                str(select.get("id") or ""),
+                select.get_text(" ", strip=True),
+            ]
+        ).lower()
+        if _contains_any(blob, ("variant", "option", "size", "color", "옵션", "사이즈", "색상")):
+            return True
+    return False
+
+
+def _has_pagination(soup, links: list[dict[str, Any]], text: str) -> bool:
+    if _count_selector_matches(soup, (".pagination", ".paging", "[class*='paginate']", "nav[aria-label*='pagination']")) > 0:
+        return True
+    if any(str(link.get("rel") or "").lower() in {"next", "prev", "previous"} for link in links):
+        return True
+    return _contains_any(text, ("next page", "previous page", "페이지", "다음", "이전"))
+
+
+def _has_tags(soup, links: list[dict[str, Any]], text: str) -> bool:
+    if _count_selector_matches(soup, (".tag", ".tags", "[rel='tag']", "[class*='tag']")) > 0:
+        return True
+    return any(str(link.get("rel") or "").lower() == "tag" for link in links) or _contains_any(text, ("tags:", "태그"))
+
+
+def _has_parameter_table(tables: list[dict[str, Any]], table_text: str) -> bool:
+    if _contains_any(table_text, ("parameter", "required", "type", "description", "파라미터", "필수")):
+        return True
+    for table in tables:
+        headers = " ".join(str(item) for item in table.get("headers") or []).lower()
+        if _contains_any(headers, ("parameter", "required", "type", "description", "파라미터", "필수")):
+            return True
+    return False
+
+
+def _has_comparison_table(tables: list[dict[str, Any]], text: str) -> bool:
+    if not tables:
+        return False
+    return _contains_any(text, ("compare", "comparison", "vs", "비교")) or any(
+        int(table.get("row_count") or 0) >= 3 and len(table.get("headers") or []) >= 3 for table in tables
+    )
+
+
+def _url_hints(url: str) -> set[str]:
+    parsed = urlparse(url)
+    value = f"{parsed.path} {parsed.query}".lower()
+    hints = set()
+    for hint, terms in {
+        "product": ("/product", "/products", "/goods", "/item", "/p/"),
+        "category": ("category", "collection", "/shop", "/list", "catalog"),
+        "search": ("search", "find", "keyword=", "query=", "q="),
+        "article": ("article", "blog", "news", "post", "story"),
+        "board": ("board", "forum", "thread", "qna"),
+        "protected": ("login", "checkout", "payment", "account", "cart"),
+        "system": ("sitemap", "robots.txt", ".json", ".xml", ".rss"),
+    }.items():
+        if any(term in value for term in terms):
+            hints.add(hint)
+    return hints
+
+
+def _dominant_language(text: str) -> str | None:
+    if not text:
+        return None
+    korean = len(re.findall(r"[가-힣]", text))
+    latin = len(re.findall(r"[A-Za-z]", text))
+    if korean == 0 and latin == 0:
+        return None
+    if korean > latin * 0.25:
+        return "ko"
+    return "en"
+
+
+def _external_collector_signals(snapshot: RawPageSnapshot) -> dict[str, Any]:
+    payload = dict(snapshot.collector_payload or {})
+    for key in {
+        "html",
+        "raw_html",
+        "rendered_html",
+        "text",
+        "markdown",
+        "clean_text",
+        "metadata",
+        "links",
+        "images",
+        "forms",
+        "inputs",
+        "buttons",
+        "tables",
+        "json_ld",
+        "jsonld",
+    }:
+        payload.pop(key, None)
+    return payload
+
+
+def _count_selector_matches(soup, selectors: tuple[str, ...]) -> int:
+    if soup is None:
+        return 0
+    count = 0
+    for selector in selectors:
+        try:
+            count += len(soup.select(selector))
+        except Exception:
+            continue
+    return count
+
+
+def _input_blob(item: dict[str, Any]) -> str:
+    return " ".join(str(item.get(key) or "") for key in ("type", "name", "id", "placeholder", "autocomplete", "aria_label", "class", "text"))
+
+
+def _form_blob(item: dict[str, Any]) -> str:
+    return " ".join(str(item.get(key) or "") for key in ("action", "method", "id", "class", "text"))
+
+
+def _contains_any(value: str, terms: tuple[str, ...]) -> bool:
+    lowered = value.lower()
+    return any(term.lower() in lowered for term in terms)
+
+
+def _string_or_none(value: Any) -> str | None:
+    clean = str(value or "").strip()
+    return clean or None
+
+
+def _optional_int(value: Any) -> int | None:
+    try:
+        return int(value)
+    except (TypeError, ValueError):
+        return None
+
+
+def _ensure_list(value: Any) -> list[Any]:
+    if value is None:
+        return []
+    if isinstance(value, list):
+        return value
+    return [value]
+
+
+def _ensure_dict_list(value: Any) -> list[dict[str, Any]]:
+    values = _ensure_list(value)
+    return [item for item in values if isinstance(item, dict)]
+
+
+def _dict_list(value: Any) -> list[dict[str, Any]]:
+    return [dict(item) for item in _ensure_list(value) if isinstance(item, dict)]
+
+
+def _string_list(value: Any) -> list[str]:
+    return [str(item) for item in _ensure_list(value) if str(item or "").strip()]
+
+
+def _dedupe_strings(values: list[str]) -> list[str]:
+    seen = set()
+    result = []
+    for value in values:
+        clean = " ".join(value.split())
+        key = clean.lower()
+        if not clean or key in seen:
+            continue
+        seen.add(key)
+        result.append(clean)
+    return result

ontology_platform/crawler_platform/app/core/crawler/page_signals.py

@@ -0,0 +1,144 @@
+from __future__ import annotations
+
+from dataclasses import asdict, dataclass, field
+from typing import Any
+
+
+@dataclass(slots=True)
+class RawPageSnapshot:
+    url: str
+    final_url: str | None = None
+    status_code: int | None = None
+    content_type: str | None = None
+    title: str | None = None
+    text: str | None = None
+    html: str | None = None
+    rendered_html: str | None = None
+    metadata: dict[str, Any] = field(default_factory=dict)
+    open_graph: dict[str, Any] = field(default_factory=dict)
+    twitter_card: dict[str, Any] = field(default_factory=dict)
+    json_ld: list[dict[str, Any]] = field(default_factory=list)
+    microdata: list[dict[str, Any]] = field(default_factory=list)
+    rdfa: list[dict[str, Any]] = field(default_factory=list)
+    headings: list[str] = field(default_factory=list)
+    links: list[dict[str, Any]] = field(default_factory=list)
+    images: list[dict[str, Any]] = field(default_factory=list)
+    forms: list[dict[str, Any]] = field(default_factory=list)
+    buttons: list[str] = field(default_factory=list)
+    inputs: list[dict[str, Any]] = field(default_factory=list)
+    tables: list[dict[str, Any]] = field(default_factory=list)
+    breadcrumbs: list[str] = field(default_factory=list)
+    source_zones: list[str | dict[str, Any]] = field(default_factory=list)
+    screenshot_path: str | None = None
+    collector_payload: dict[str, Any] = field(default_factory=dict)
+
+    def to_dict(self) -> dict[str, Any]:
+        return asdict(self)
+
+
+@dataclass(slots=True)
+class PageSignals:
+    # structured data
+    schema_types: set[str] = field(default_factory=set)
+    og_type: str | None = None
+    twitter_card_type: str | None = None
+
+    # commerce
+    has_price: bool = False
+    has_currency: bool = False
+    has_cart_button: bool = False
+    has_buy_button: bool = False
+    has_variant_selector: bool = False
+    has_sku: bool = False
+    has_rating: bool = False
+    has_review_section: bool = False
+    has_product_gallery: bool = False
+
+    # listing
+    has_repeated_cards: bool = False
+    repeated_card_count: int = 0
+    has_filter_panel: bool = False
+    has_sort_control: bool = False
+    has_pagination: bool = False
+
+    # editorial
+    has_author: bool = False
+    has_published_date: bool = False
+    has_modified_date: bool = False
+    has_article_body: bool = False
+    has_tags: bool = False
+
+    # community
+    has_question: bool = False
+    has_answer: bool = False
+    has_comments: bool = False
+    has_votes: bool = False
+    has_thread_structure: bool = False
+    has_faq_structure: bool = False
+
+    # knowledge/docs
+    has_code_blocks: bool = False
+    has_toc: bool = False
+    has_api_endpoint: bool = False
+    has_parameter_table: bool = False
+    has_version_info: bool = False
+
+    # corporate/legal
+    has_contact_info: bool = False
+    has_address: bool = False
+    has_policy_terms: bool = False
+    has_privacy_terms: bool = False
+    has_career_terms: bool = False
+
+    # transaction/protected
+    has_login_form: bool = False
+    has_password_field: bool = False
+    has_payment_fields: bool = False
+    has_captcha: bool = False
+    has_access_denied: bool = False
+
+    # system/resource
+    status_code: int | None = None
+    has_error_status: bool = False
+    has_not_found: bool = False
+    has_sitemap_resource: bool = False
+    has_feed_resource: bool = False
+    has_json_resource: bool = False
+    has_xml_resource: bool = False
+    has_file_resource: bool = False
+
+    # graph
+    internal_link_count: int = 0
+    external_link_count: int = 0
+    product_link_count: int = 0
+    category_link_count: int = 0
+    profile_link_count: int = 0
+    article_link_count: int = 0
+
+    # visual/layout candidates from DOM structure
+    layout_blocks: list[str] = field(default_factory=list)
+    has_hero_block: bool = False
+    has_card_grid: bool = False
+    has_filter_sidebar: bool = False
+    has_sticky_action_box: bool = False
+    has_media_player_area: bool = False
+    has_map_area: bool = False
+    has_calendar_grid: bool = False
+    has_pricing_table: bool = False
+    has_comparison_table: bool = False
+
+    # text/layout
+    dominant_language: str | None = None
+    keyword_hits: dict[str, int] = field(default_factory=dict)
+    url_hints: set[str] = field(default_factory=set)
+    title: str | None = None
+    text_sample: str = ""
+
+    # external collector hook
+    external_collector_signals: dict[str, Any] = field(default_factory=dict)
+
+    def to_dict(self) -> dict[str, Any]:
+        payload = asdict(self)
+        payload["schema_types"] = sorted(self.schema_types)
+        payload["url_hints"] = sorted(self.url_hints)
+        return payload

ontology_platform/crawler_platform/app/core/crawler/page_type_scorer.py

@@ -0,0 +1,747 @@
+from __future__ import annotations
+
+from collections.abc import Callable
+from dataclasses import dataclass
+from typing import Any
+
+from crawler_platform.app.core.crawler.page_signals import PageSignals
+from crawler_platform.app.core.crawler.page_type_taxonomy import (
+    ActionIntent,
+    AnalyzeStrategy,
+    EntityType,
+    EvidenceItem,
+    GraphRole,
+    LLMPolicy,
+    PAGE_TYPE_PROFILES,
+    PageArchetype,
+    PageClassificationResult,
+    PageDomain,
+    PageType,
+)
+
+
+UNKNOWN_THRESHOLD = 0.22
+
+
+@dataclass(frozen=True, slots=True)
+class SignalRule:
+    key: str
+    weight: float
+    source: str
+    message: str
+    predicate: Callable[[PageSignals], bool]
+    value: Callable[[PageSignals], str | int | float | bool | None] | None = None
+
+
+@dataclass(slots=True)
+class ScoreAccumulator:
+    scores: dict[str, float]
+    evidence_by_type: dict[str, list[EvidenceItem]]
+
+    def add(self, page_type: str, rule: SignalRule, signals: PageSignals) -> None:
+        if not rule.predicate(signals):
+            return
+        value = rule.value(signals) if rule.value else True
+        self.scores[page_type] = self.scores.get(page_type, 0.0) + rule.weight
+        self.evidence_by_type.setdefault(page_type, []).append(
+            EvidenceItem(
+                key=rule.key,
+                value=value,
+                weight=rule.weight,
+                source=rule.source,
+                message=rule.message,
+            )
+        )
+
+
+PAGE_TYPE_METADATA: dict[str, dict[str, Any]] = {
+    **PAGE_TYPE_PROFILES,
+    PageType.ARTICLE_PAGE.value: {
+        "domain": PageDomain.EDITORIAL.value,
+        "archetype": PageArchetype.ARTICLE.value,
+        "main_entity_type": EntityType.ARTICLE.value,
+        "action_intents": [ActionIntent.READ.value],
+        "graph_roles": [GraphRole.CLAIM_SOURCE.value, GraphRole.ENTITY_ANCHOR.value],
+        "analyze_strategy": AnalyzeStrategy.ANALYZE_FULL.value,
+        "llm_policy": LLMPolicy.LLM_FULL.value,
+        "should_analyze": True,
+    },
+    PageType.BLOG_POST_PAGE.value: {
+        "domain": PageDomain.EDITORIAL.value,
+        "archetype": PageArchetype.ARTICLE.value,
+        "main_entity_type": EntityType.ARTICLE.value,
+        "action_intents": [ActionIntent.READ.value],
+        "graph_roles": [GraphRole.CLAIM_SOURCE.value],
+        "analyze_strategy": AnalyzeStrategy.ANALYZE_FULL.value,
+        "llm_policy": LLMPolicy.LLM_FULL.value,
+        "should_analyze": True,
+    },
+    PageType.FAQ_PAGE.value: {
+        "domain": PageDomain.COMMUNITY.value,
+        "archetype": PageArchetype.DOCUMENT.value,
+        "main_entity_type": EntityType.QUESTION.value,
+        "action_intents": [ActionIntent.READ.value, ActionIntent.ANSWER.value],
+        "graph_roles": [GraphRole.REFERENCE_SOURCE.value],
+        "analyze_strategy": AnalyzeStrategy.ANALYZE_FULL.value,
+        "llm_policy": LLMPolicy.LLM_LIGHT.value,
+        "should_analyze": True,
+    },
+    PageType.QA_PAGE.value: {
+        "domain": PageDomain.COMMUNITY.value,
+        "archetype": PageArchetype.THREAD.value,
+        "main_entity_type": EntityType.QUESTION.value,
+        "action_intents": [ActionIntent.ASK.value, ActionIntent.ANSWER.value, ActionIntent.READ.value],
+        "graph_roles": [GraphRole.CLAIM_SOURCE.value, GraphRole.RELATION_HUB.value],
+        "analyze_strategy": AnalyzeStrategy.ANALYZE_FULL.value,
+        "llm_policy": LLMPolicy.LLM_LIGHT.value,
+        "should_analyze": True,
+    },
+    PageType.FORUM_THREAD_PAGE.value: {
+        "domain": PageDomain.COMMUNITY.value,
+        "archetype": PageArchetype.THREAD.value,
+        "main_entity_type": EntityType.ARTICLE.value,
+        "action_intents": [ActionIntent.READ.value, ActionIntent.COMMENT.value],
+        "graph_roles": [GraphRole.CLAIM_SOURCE.value, GraphRole.RELATION_HUB.value],
+        "analyze_strategy": AnalyzeStrategy.ANALYZE_FULL.value,
+        "llm_policy": LLMPolicy.LLM_LIGHT.value,
+        "should_analyze": True,
+    },
+    PageType.ABOUT_PAGE.value: {
+        "domain": PageDomain.CORPORATE.value,
+        "archetype": PageArchetype.ARTICLE.value,
+        "main_entity_type": EntityType.ORGANIZATION.value,
+        "action_intents": [ActionIntent.READ.value, ActionIntent.LEARN.value],
+        "graph_roles": [GraphRole.ENTITY_ANCHOR.value, GraphRole.CLAIM_SOURCE.value],
+        "analyze_strategy": AnalyzeStrategy.ANALYZE_FULL.value,
+        "llm_policy": LLMPolicy.LLM_FULL.value,
+        "should_analyze": True,
+    },
+    PageType.CONTACT_PAGE.value: {
+        "domain": PageDomain.CORPORATE.value,
+        "archetype": PageArchetype.DOCUMENT.value,
+        "main_entity_type": EntityType.ORGANIZATION.value,
+        "action_intents": [ActionIntent.CONTACT.value, ActionIntent.NAVIGATE.value],
+        "graph_roles": [GraphRole.ENTITY_ANCHOR.value],
+        "analyze_strategy": AnalyzeStrategy.ANALYZE_METADATA_ONLY.value,
+        "llm_policy": LLMPolicy.RULE_ONLY.value,
+        "should_analyze": True,
+    },
+    PageType.DOCUMENTATION_PAGE.value: {
+        "domain": PageDomain.KNOWLEDGE.value,
+        "archetype": PageArchetype.DOCUMENT.value,
+        "main_entity_type": EntityType.SOFTWARE_APPLICATION.value,
+        "action_intents": [ActionIntent.READ.value, ActionIntent.LEARN.value],
+        "graph_roles": [GraphRole.REFERENCE_SOURCE.value],
+        "analyze_strategy": AnalyzeStrategy.ANALYZE_FULL.value,
+        "llm_policy": LLMPolicy.LLM_LIGHT.value,
+        "should_analyze": True,
+    },
+    PageType.API_REFERENCE_PAGE.value: {
+        "domain": PageDomain.SOFTWARE.value,
+        "archetype": PageArchetype.DOCUMENT.value,
+        "main_entity_type": EntityType.SOFTWARE_APPLICATION.value,
+        "action_intents": [ActionIntent.LEARN.value, ActionIntent.CONFIGURE.value],
+        "graph_roles": [GraphRole.REFERENCE_SOURCE.value],
+        "analyze_strategy": AnalyzeStrategy.ANALYZE_FULL.value,
+        "llm_policy": LLMPolicy.LLM_LIGHT.value,
+        "should_analyze": True,
+    },
+    PageType.WIKI_PAGE.value: {
+        "domain": PageDomain.KNOWLEDGE.value,
+        "archetype": PageArchetype.DOCUMENT.value,
+        "main_entity_type": EntityType.UNKNOWN_ENTITY.value,
+        "action_intents": [ActionIntent.READ.value, ActionIntent.LEARN.value],
+        "graph_roles": [GraphRole.REFERENCE_SOURCE.value],
+        "analyze_strategy": AnalyzeStrategy.ANALYZE_FULL.value,
+        "llm_policy": LLMPolicy.LLM_LIGHT.value,
+        "should_analyze": True,
+    },
+    PageType.DATASET_PAGE.value: {
+        "domain": PageDomain.KNOWLEDGE.value,
+        "archetype": PageArchetype.DOCUMENT.value,
+        "main_entity_type": EntityType.DATASET.value,
+        "action_intents": [ActionIntent.READ.value, ActionIntent.DOWNLOAD.value],
+        "graph_roles": [GraphRole.REFERENCE_SOURCE.value],
+        "analyze_strategy": AnalyzeStrategy.ANALYZE_DOCUMENT_ONLY.value,
+        "llm_policy": LLMPolicy.LLM_LIGHT.value,
+        "should_analyze": True,
+    },
+    PageType.RESEARCH_PAPER_PAGE.value: {
+        "domain": PageDomain.KNOWLEDGE.value,
+        "archetype": PageArchetype.ARTICLE.value,
+        "main_entity_type": EntityType.ARTICLE.value,
+        "action_intents": [ActionIntent.READ.value, ActionIntent.LEARN.value],
+        "graph_roles": [GraphRole.CLAIM_SOURCE.value, GraphRole.REFERENCE_SOURCE.value],
+        "analyze_strategy": AnalyzeStrategy.ANALYZE_FULL.value,
+        "llm_policy": LLMPolicy.LLM_FULL.value,
+        "should_analyze": True,
+    },
+    PageType.JOB_POSTING_PAGE.value: {
+        "domain": PageDomain.JOBS.value,
+        "archetype": PageArchetype.DETAIL.value,
+        "main_entity_type": EntityType.JOB_POSTING.value,
+        "action_intents": [ActionIntent.APPLY.value, ActionIntent.READ.value],
+        "graph_roles": [GraphRole.ENTITY_ANCHOR.value],
+        "analyze_strategy": AnalyzeStrategy.ANALYZE_FULL.value,
+        "llm_policy": LLMPolicy.LLM_FULL.value,
+        "should_analyze": True,
+    },
+    PageType.COURSE_DETAIL_PAGE.value: {
+        "domain": PageDomain.EDUCATION.value,
+        "archetype": PageArchetype.DETAIL.value,
+        "main_entity_type": EntityType.COURSE.value,
+        "action_intents": [ActionIntent.LEARN.value, ActionIntent.SUBSCRIBE.value],
+        "graph_roles": [GraphRole.ENTITY_ANCHOR.value, GraphRole.REFERENCE_SOURCE.value],
+        "analyze_strategy": AnalyzeStrategy.ANALYZE_FULL.value,
+        "llm_policy": LLMPolicy.LLM_LIGHT.value,
+        "should_analyze": True,
+    },
+    PageType.VIDEO_PAGE.value: {
+        "domain": PageDomain.MEDIA.value,
+        "archetype": PageArchetype.MEDIA.value,
+        "main_entity_type": EntityType.MEDIA_OBJECT.value,
+        "action_intents": [ActionIntent.WATCH.value],
+        "graph_roles": [GraphRole.MEDIA_ANCHOR.value],
+        "analyze_strategy": AnalyzeStrategy.ANALYZE_METADATA_ONLY.value,
+        "llm_policy": LLMPolicy.RULE_ONLY.value,
+        "should_analyze": True,
+    },
+    PageType.LOCAL_BUSINESS_PAGE.value: {
+        "domain": PageDomain.LOCAL.value,
+        "archetype": PageArchetype.DETAIL.value,
+        "main_entity_type": EntityType.PLACE.value,
+        "action_intents": [ActionIntent.CONTACT.value, ActionIntent.NAVIGATE.value],
+        "graph_roles": [GraphRole.ENTITY_ANCHOR.value],
+        "analyze_strategy": AnalyzeStrategy.ANALYZE_FULL.value,
+        "llm_policy": LLMPolicy.LLM_LIGHT.value,
+        "should_analyze": True,
+    },
+    PageType.REAL_ESTATE_LISTING_PAGE.value: {
+        "domain": PageDomain.LOCAL.value,
+        "archetype": PageArchetype.DETAIL.value,
+        "main_entity_type": EntityType.REAL_ESTATE_PROPERTY.value,
+        "action_intents": [ActionIntent.READ.value, ActionIntent.CONTACT.value],
+        "graph_roles": [GraphRole.ENTITY_ANCHOR.value],
+        "analyze_strategy": AnalyzeStrategy.ANALYZE_FULL.value,
+        "llm_policy": LLMPolicy.LLM_LIGHT.value,
+        "should_analyze": True,
+    },
+    PageType.PROFILE_PAGE.value: {
+        "domain": PageDomain.COMMUNITY.value,
+        "archetype": PageArchetype.PROFILE.value,
+        "main_entity_type": EntityType.PERSON.value,
+        "action_intents": [ActionIntent.READ.value, ActionIntent.CONTACT.value],
+        "graph_roles": [GraphRole.PROFILE_ANCHOR.value],
+        "analyze_strategy": AnalyzeStrategy.ANALYZE_ENTITY_ONLY.value,
+        "llm_policy": LLMPolicy.LLM_LIGHT.value,
+        "should_analyze": True,
+    },
+    PageType.PRICING_PAGE.value: {
+        "domain": PageDomain.COMMERCE.value,
+        "archetype": PageArchetype.LANDING.value,
+        "main_entity_type": EntityType.SERVICE.value,
+        "action_intents": [ActionIntent.BUY.value, ActionIntent.COMPARE.value],
+        "graph_roles": [GraphRole.CLAIM_SOURCE.value],
+        "analyze_strategy": AnalyzeStrategy.ANALYZE_FULL.value,
+        "llm_policy": LLMPolicy.LLM_LIGHT.value,
+        "should_analyze": True,
+    },
+    PageType.LOGIN_PAGE.value: {
+        "domain": PageDomain.TRANSACTION.value,
+        "archetype": PageArchetype.FORM.value,
+        "main_entity_type": None,
+        "action_intents": [ActionIntent.LOGIN.value],
+        "graph_roles": [GraphRole.TRANSACTION_ONLY.value],
+        "analyze_strategy": AnalyzeStrategy.SKIP_PROTECTED.value,
+        "llm_policy": LLMPolicy.SKIP.value,
+        "should_analyze": False,
+    },
+    PageType.CHECKOUT_PAGE.value: {
+        "domain": PageDomain.TRANSACTION.value,
+        "archetype": PageArchetype.TRANSACTION.value,
+        "main_entity_type": None,
+        "action_intents": [ActionIntent.BUY.value, ActionIntent.PAY.value],
+        "graph_roles": [GraphRole.TRANSACTION_ONLY.value],
+        "analyze_strategy": AnalyzeStrategy.SKIP_PROTECTED.value,
+        "llm_policy": LLMPolicy.SKIP.value,
+        "should_analyze": False,
+    },
+    PageType.PAYMENT_PAGE.value: {
+        "domain": PageDomain.TRANSACTION.value,
+        "archetype": PageArchetype.TRANSACTION.value,
+        "main_entity_type": None,
+        "action_intents": [ActionIntent.PAY.value],
+        "graph_roles": [GraphRole.TRANSACTION_ONLY.value],
+        "analyze_strategy": AnalyzeStrategy.SKIP_PROTECTED.value,
+        "llm_policy": LLMPolicy.SKIP.value,
+        "should_analyze": False,
+    },
+    PageType.TERMS_PAGE.value: {
+        "domain": PageDomain.CORPORATE.value,
+        "archetype": PageArchetype.DOCUMENT.value,
+        "main_entity_type": EntityType.LEGAL_DOCUMENT.value,
+        "action_intents": [ActionIntent.READ.value],
+        "graph_roles": [GraphRole.POLICY_SOURCE.value],
+        "analyze_strategy": AnalyzeStrategy.ANALYZE_DOCUMENT_ONLY.value,
+        "llm_policy": LLMPolicy.LLM_LIGHT.value,
+        "should_analyze": True,
+    },
+    PageType.PRIVACY_POLICY_PAGE.value: {
+        "domain": PageDomain.CORPORATE.value,
+        "archetype": PageArchetype.DOCUMENT.value,
+        "main_entity_type": EntityType.LEGAL_DOCUMENT.value,
+        "action_intents": [ActionIntent.READ.value],
+        "graph_roles": [GraphRole.POLICY_SOURCE.value],
+        "analyze_strategy": AnalyzeStrategy.ANALYZE_DOCUMENT_ONLY.value,
+        "llm_policy": LLMPolicy.LLM_LIGHT.value,
+        "should_analyze": True,
+    },
+    PageType.SITEMAP_PAGE.value: {
+        "domain": PageDomain.SYSTEM.value,
+        "archetype": PageArchetype.SYSTEM_RESOURCE.value,
+        "main_entity_type": None,
+        "action_intents": [ActionIntent.NAVIGATE.value],
+        "graph_roles": [GraphRole.SYSTEM_RESOURCE.value, GraphRole.NAVIGATION_HUB.value],
+        "analyze_strategy": AnalyzeStrategy.ANALYZE_DISCOVERY_ONLY.value,
+        "llm_policy": LLMPolicy.RULE_ONLY.value,
+        "should_analyze": True,
+    },
+    PageType.RSS_FEED_PAGE.value: {
+        "domain": PageDomain.SYSTEM.value,
+        "archetype": PageArchetype.SYSTEM_RESOURCE.value,
+        "main_entity_type": None,
+        "action_intents": [ActionIntent.READ.value],
+        "graph_roles": [GraphRole.SYSTEM_RESOURCE.value],
+        "analyze_strategy": AnalyzeStrategy.ANALYZE_METADATA_ONLY.value,
+        "llm_policy": LLMPolicy.RULE_ONLY.value,
+        "should_analyze": True,
+    },
+    PageType.ERROR_PAGE.value: {
+        "domain": PageDomain.SYSTEM.value,
+        "archetype": PageArchetype.ERROR.value,
+        "main_entity_type": None,
+        "action_intents": [],
+        "graph_roles": [GraphRole.NOISE_PAGE.value],
+        "analyze_strategy": AnalyzeStrategy.SKIP_NOISE.value,
+        "llm_policy": LLMPolicy.SKIP.value,
+        "should_analyze": False,
+    },
+    PageType.NOT_FOUND_PAGE.value: {
+        "domain": PageDomain.SYSTEM.value,
+        "archetype": PageArchetype.ERROR.value,
+        "main_entity_type": None,
+        "action_intents": [],
+        "graph_roles": [GraphRole.NOISE_PAGE.value],
+        "analyze_strategy": AnalyzeStrategy.SKIP_NOISE.value,
+        "llm_policy": LLMPolicy.SKIP.value,
+        "should_analyze": False,
+    },
+    PageType.ACCESS_DENIED_PAGE.value: {
+        "domain": PageDomain.TRANSACTION.value,
+        "archetype": PageArchetype.ERROR.value,
+        "main_entity_type": None,
+        "action_intents": [ActionIntent.VERIFY.value],
+        "graph_roles": [GraphRole.TRANSACTION_ONLY.value],
+        "analyze_strategy": AnalyzeStrategy.SKIP_PROTECTED.value,
+        "llm_policy": LLMPolicy.SKIP.value,
+        "should_analyze": False,
+    },
+    PageType.CAPTCHA_PAGE.value: {
+        "domain": PageDomain.TRANSACTION.value,
+        "archetype": PageArchetype.FORM.value,
+        "main_entity_type": None,
+        "action_intents": [ActionIntent.VERIFY.value],
+        "graph_roles": [GraphRole.TRANSACTION_ONLY.value],
+        "analyze_strategy": AnalyzeStrategy.SKIP_PROTECTED.value,
+        "llm_policy": LLMPolicy.SKIP.value,
+        "should_analyze": False,
+    },
+}
+
+
+def score_page_type(url: str, signals: PageSignals, *, unknown_threshold: float = UNKNOWN_THRESHOLD) -> PageClassificationResult:
+    accumulator = ScoreAccumulator(scores={}, evidence_by_type={})
+    for page_type, rules in SCORING_RULES.items():
+        for rule in rules:
+            accumulator.add(page_type, rule, signals)
+
+    _apply_interactions(accumulator, signals)
+
+    if not accumulator.scores:
+        return _unknown_result(
+            url,
+            alternatives=[],
+            evidence=[
+                EvidenceItem(
+                    key="insufficient_evidence",
+                    value=True,
+                    weight=0.0,
+                    source="page_type_scorer",
+                    message="No page-type evidence was detected.",
+                )
+            ],
+        )
+
+    alternatives = sorted(
+        ((page_type, round(min(score, 1.0), 4)) for page_type, score in accumulator.scores.items()),
+        key=lambda item: item[1],
+        reverse=True,
+    )
+    primary_page_type, confidence = alternatives[0]
+    if confidence < unknown_threshold:
+        evidence = [item for page_type, _score in alternatives[:3] for item in accumulator.evidence_by_type.get(page_type, [])]
+        evidence.append(
+            EvidenceItem(
+                key="low_confidence",
+                value=confidence,
+                weight=0.0,
+                source="page_type_scorer",
+                message=f"Top score {confidence:.2f} is below UnknownPage threshold {unknown_threshold:.2f}.",
+            )
+        )
+        return _unknown_result(url, alternatives=alternatives[:5], evidence=evidence)
+
+    profile = _profile(primary_page_type)
+    evidence = accumulator.evidence_by_type.get(primary_page_type, [])
+    if not evidence:
+        evidence = [
+            EvidenceItem(
+                key="score",
+                value=confidence,
+                weight=confidence,
+                source="page_type_scorer",
+                message=f"{primary_page_type} selected from accumulated score.",
+            )
+        ]
+    secondary = [page_type for page_type, score in alternatives[1:4] if score >= 0.18]
+    return PageClassificationResult(
+        url=url,
+        primary_page_type=primary_page_type,
+        secondary_page_types=secondary,
+        domain=str(profile["domain"]),
+        archetype=str(profile["archetype"]),
+        main_entity_type=profile.get("main_entity_type"),
+        action_intents=list(profile.get("action_intents") or []),
+        graph_roles=list(profile.get("graph_roles") or []),
+        confidence=confidence,
+        alternatives=alternatives[:8],
+        evidence=evidence,
+        should_analyze=bool(profile.get("should_analyze")),
+        analyze_strategy=str(profile["analyze_strategy"]),
+        llm_policy=str(profile["llm_policy"]),
+        is_protected=str(profile["analyze_strategy"]) == AnalyzeStrategy.SKIP_PROTECTED.value,
+        is_noise=str(profile["analyze_strategy"]) == AnalyzeStrategy.SKIP_NOISE.value,
+    )
+
+
+def _profile(page_type: str) -> dict[str, Any]:
+    return PAGE_TYPE_METADATA.get(page_type, PAGE_TYPE_METADATA[PageType.UNKNOWN_PAGE.value])
+
+
+def _unknown_result(
+    url: str,
+    *,
+    alternatives: list[tuple[str, float]],
+    evidence: list[EvidenceItem],
+) -> PageClassificationResult:
+    profile = _profile(PageType.UNKNOWN_PAGE.value)
+    return PageClassificationResult(
+        url=url,
+        primary_page_type=PageType.UNKNOWN_PAGE.value,
+        secondary_page_types=[page_type for page_type, _score in alternatives[:3]],
+        domain=str(profile["domain"]),
+        archetype=str(profile["archetype"]),
+        main_entity_type=profile.get("main_entity_type"),
+        action_intents=list(profile.get("action_intents") or []),
+        graph_roles=list(profile.get("graph_roles") or []),
+        confidence=0.0,
+        alternatives=alternatives[:8],
+        evidence=evidence
+        or [
+            EvidenceItem(
+                key="unknown",
+                value=True,
+                weight=0.0,
+                source="page_type_scorer",
+                message="Page did not match known semantic page type evidence.",
+            )
+        ],
+        should_analyze=bool(profile.get("should_analyze")),
+        analyze_strategy=str(profile["analyze_strategy"]),
+        llm_policy=str(profile["llm_policy"]),
+        is_protected=False,
+        is_noise=False,
+    )
+
+
+def _apply_interactions(accumulator: ScoreAccumulator, signals: PageSignals) -> None:
+    # Protected signals must dominate commerce signals when login/payment fields are present.
+    if signals.has_password_field and signals.has_login_form:
+        _manual_add(accumulator, PageType.LOGIN_PAGE.value, "protected_priority", True, 0.3, "protected", "Login form and password field override commerce signals.")
+    if signals.has_payment_fields:
+        _manual_add(accumulator, PageType.PAYMENT_PAGE.value, "payment_priority", True, 0.35, "protected", "Payment fields override product or checkout content.")
+    if signals.has_access_denied:
+        _manual_add(accumulator, PageType.ACCESS_DENIED_PAGE.value, "access_denied_priority", True, 0.35, "protected", "Access denied signal is a protected page indicator.")
+    if signals.has_captcha:
+        _manual_add(accumulator, PageType.CAPTCHA_PAGE.value, "captcha_priority", True, 0.35, "protected", "Captcha signal is a protected page indicator.")
+    if signals.has_repeated_cards and signals.product_link_count >= 3:
+        _manual_add(accumulator, PageType.CATEGORY_LISTING_PAGE.value, "listing_product_links", signals.product_link_count, 0.18, "link_graph", "Repeated cards with multiple product links indicate a product listing.")
+    if signals.has_question and signals.has_answer and signals.has_faq_structure:
+        _manual_add(accumulator, PageType.FAQ_PAGE.value, "faq_question_answer", True, 0.18, "community", "FAQ structure with question/answer content.")
+
+
+def _manual_add(
+    accumulator: ScoreAccumulator,
+    page_type: str,
+    key: str,
+    value: str | int | float | bool | None,
+    weight: float,
+    source: str,
+    message: str,
+) -> None:
+    accumulator.scores[page_type] = accumulator.scores.get(page_type, 0.0) + weight
+    accumulator.evidence_by_type.setdefault(page_type, []).append(
+        EvidenceItem(key=key, value=value, weight=weight, source=source, message=message)
+    )
+
+
+def _has_schema(*schema_types: str) -> Callable[[PageSignals], bool]:
+    return lambda signals: any(schema_type in signals.schema_types for schema_type in schema_types)
+
+
+def _keyword(group: str, minimum: int = 1) -> Callable[[PageSignals], bool]:
+    return lambda signals: int(signals.keyword_hits.get(group) or 0) >= minimum
+
+
+def _url_hint(hint: str) -> Callable[[PageSignals], bool]:
+    return lambda signals: hint in signals.url_hints
+
+
+def _contains_terms(signals: PageSignals, terms: tuple[str, ...]) -> bool:
+    text = f"{signals.title or ''}\n{signals.text_sample or ''}".lower()
+    return any(term.lower() in text for term in terms)
+
+
+def _flag(name: str) -> Callable[[PageSignals], bool]:
+    return lambda signals: bool(getattr(signals, name))
+
+
+def _count_at_least(name: str, minimum: int) -> Callable[[PageSignals], bool]:
+    return lambda signals: int(getattr(signals, name) or 0) >= minimum
+
+
+def _value(name: str) -> Callable[[PageSignals], str | int | float | bool | None]:
+    return lambda signals: getattr(signals, name)
+
+
+def _keywords_value(group: str) -> Callable[[PageSignals], str | int | float | bool | None]:
+    return lambda signals: signals.keyword_hits.get(group, 0)
+
+
+def _schema_value(signals: PageSignals) -> str:
+    return ",".join(sorted(signals.schema_types))
+
+
+def _url_hint_value(signals: PageSignals) -> str:
+    return ",".join(sorted(signals.url_hints))
+
+
+SCORING_RULES: dict[str, list[SignalRule]] = {
+    PageType.PRODUCT_DETAIL_PAGE.value: [
+        SignalRule("schema_product", 0.4, "structured_data", "schema.org Product detected.", _has_schema("Product"), _schema_value),
+        SignalRule("schema_offer", 0.15, "structured_data", "schema.org Offer detected.", _has_schema("Offer"), _schema_value),
+        SignalRule("price", 0.15, "text", "Price detected.", _flag("has_price"), _value("has_price")),
+        SignalRule("cart_button", 0.2, "dom", "Cart button detected.", _flag("has_cart_button")),
+        SignalRule("buy_button", 0.16, "dom", "Buy button detected.", _flag("has_buy_button")),
+        SignalRule("variant_selector", 0.12, "dom", "Variant selector detected.", _flag("has_variant_selector")),
+        SignalRule("sku", 0.1, "text", "SKU or product code detected.", _flag("has_sku")),
+        SignalRule("product_gallery", 0.1, "layout", "Product image gallery detected.", _flag("has_product_gallery")),
+        SignalRule("review_section", 0.05, "dom", "Review section detected.", _flag("has_review_section")),
+        SignalRule("url_product_hint", 0.05, "url", "Product URL hint detected.", _url_hint("product"), _url_hint_value),
+    ],
+    PageType.CATEGORY_LISTING_PAGE.value: [
+        SignalRule("repeated_cards", 0.32, "layout", "Repeated cards detected.", _flag("has_repeated_cards"), _value("repeated_card_count")),
+        SignalRule("filter_panel", 0.2, "dom", "Filter panel detected.", _flag("has_filter_panel")),
+        SignalRule("sort_control", 0.15, "dom", "Sort control detected.", _flag("has_sort_control")),
+        SignalRule("pagination", 0.1, "dom", "Pagination detected.", _flag("has_pagination")),
+        SignalRule("product_links", 0.18, "link_graph", "Multiple product links detected.", _count_at_least("product_link_count", 3), _value("product_link_count")),
+        SignalRule("category_url_hint", 0.06, "url", "Category/list URL hint detected.", _url_hint("category"), _url_hint_value),
+    ],
+    PageType.SEARCH_RESULTS_PAGE.value: [
+        SignalRule("search_url_hint", 0.28, "url", "Search URL hint detected.", _url_hint("search"), _url_hint_value),
+        SignalRule("listing_results", 0.18, "layout", "Repeated result cards detected.", _flag("has_repeated_cards"), _value("repeated_card_count")),
+        SignalRule("filter_panel", 0.14, "dom", "Search filter panel detected.", _flag("has_filter_panel")),
+        SignalRule("pagination", 0.12, "dom", "Search pagination detected.", _flag("has_pagination")),
+        SignalRule("search_keywords", 0.12, "text", "Search/result keywords detected.", _keyword("listing"), _keywords_value("listing")),
+    ],
+    PageType.ARTICLE_PAGE.value: [
+        SignalRule("schema_article", 0.35, "structured_data", "Article structured data detected.", _has_schema("Article", "NewsArticle"), _schema_value),
+        SignalRule("author", 0.15, "text", "Author/byline signal detected.", _flag("has_author")),
+        SignalRule("published_date", 0.15, "text", "Published date detected.", _flag("has_published_date")),
+        SignalRule("article_body", 0.2, "dom", "Article body detected.", _flag("has_article_body")),
+        SignalRule("tags", 0.05, "dom", "Article tags detected.", _flag("has_tags")),
+        SignalRule("article_url_hint", 0.05, "url", "Article URL hint detected.", _url_hint("article"), _url_hint_value),
+    ],
+    PageType.BLOG_POST_PAGE.value: [
+        SignalRule("blog_url_hint", 0.25, "url", "Blog URL hint detected.", lambda signals: "article" in signals.url_hints and "blog" in (signals.text_sample or "").lower()),
+        SignalRule("author", 0.14, "text", "Author signal detected.", _flag("has_author")),
+        SignalRule("published_date", 0.14, "text", "Published date detected.", _flag("has_published_date")),
+        SignalRule("article_body", 0.18, "dom", "Article body detected.", _flag("has_article_body")),
+        SignalRule("tags", 0.08, "dom", "Tags detected.", _flag("has_tags")),
+    ],
+    PageType.QA_PAGE.value: [
+        SignalRule("schema_qapage", 0.35, "structured_data", "QAPage structured data detected.", _has_schema("QAPage"), _schema_value),
+        SignalRule("question", 0.2, "community", "Question block detected.", _flag("has_question")),
+        SignalRule("answer", 0.2, "community", "Answer block detected.", _flag("has_answer")),
+        SignalRule("votes", 0.1, "community", "Vote signal detected.", _flag("has_votes")),
+        SignalRule("comments", 0.05, "community", "Comments detected.", _flag("has_comments")),
+        SignalRule("board_url_hint", 0.05, "url", "Board/community URL hint detected.", _url_hint("board"), _url_hint_value),
+    ],
+    PageType.FAQ_PAGE.value: [
+        SignalRule("schema_faq", 0.36, "structured_data", "FAQPage structured data detected.", _has_schema("FAQPage"), _schema_value),
+        SignalRule("faq_structure", 0.25, "community", "FAQ structure detected.", _flag("has_faq_structure")),
+        SignalRule("question", 0.15, "community", "Question content detected.", _flag("has_question")),
+        SignalRule("answer", 0.15, "community", "Answer content detected.", _flag("has_answer")),
+    ],
+    PageType.FORUM_BOARD_PAGE.value: [
+        SignalRule("board_url_hint", 0.25, "url", "Board URL hint detected.", _url_hint("board"), _url_hint_value),
+        SignalRule("thread_structure", 0.2, "community", "Thread structure detected.", _flag("has_thread_structure")),
+        SignalRule("comments", 0.1, "community", "Comments/replies detected.", _flag("has_comments")),
+        SignalRule("repeated_cards", 0.14, "layout", "Repeated post cards detected.", _flag("has_repeated_cards"), _value("repeated_card_count")),
+        SignalRule("pagination", 0.1, "dom", "Board pagination detected.", _flag("has_pagination")),
+    ],
+    PageType.FORUM_THREAD_PAGE.value: [
+        SignalRule("thread_structure", 0.3, "community", "Thread structure detected.", _flag("has_thread_structure")),
+        SignalRule("comments", 0.16, "community", "Comment thread detected.", _flag("has_comments")),
+        SignalRule("question_answer", 0.14, "community", "Question and answer content detected.", lambda signals: signals.has_question and signals.has_answer),
+        SignalRule("votes", 0.08, "community", "Vote signal detected.", _flag("has_votes")),
+        SignalRule("article_body", 0.08, "dom", "Post body detected.", _flag("has_article_body")),
+    ],
+    PageType.BRAND_STORY_PAGE.value: [
+        SignalRule("corporate_keywords", 0.2, "text", "Corporate/brand keywords detected.", _keyword("corporate"), _keywords_value("corporate")),
+        SignalRule("about_url_hint", 0.18, "url", "About/company URL hint detected.", lambda signals: "about" in (signals.text_sample or "").lower() or "brand" in (signals.title or "").lower()),
+        SignalRule("article_body", 0.14, "dom", "Brand story body detected.", _flag("has_article_body")),
+        SignalRule("hero_block", 0.08, "layout", "Hero block detected.", _flag("has_hero_block")),
+    ],
+    PageType.ABOUT_PAGE.value: [
+        SignalRule("corporate_keywords", 0.22, "text", "About/company keywords detected.", _keyword("corporate"), _keywords_value("corporate")),
+        SignalRule("contact_info", 0.08, "text", "Organization contact signal detected.", _flag("has_contact_info")),
+        SignalRule("article_body", 0.12, "dom", "About body detected.", _flag("has_article_body")),
+    ],
+    PageType.CONTACT_PAGE.value: [
+        SignalRule("contact_info", 0.32, "text", "Contact information detected.", _flag("has_contact_info")),
+        SignalRule("address", 0.18, "text", "Address detected.", _flag("has_address")),
+        SignalRule("corporate_keywords", 0.1, "text", "Corporate keywords detected.", _keyword("corporate"), _keywords_value("corporate")),
+    ],
+    PageType.DOCUMENTATION_PAGE.value: [
+        SignalRule("docs_keywords", 0.18, "text", "Documentation keywords detected.", _keyword("knowledge"), _keywords_value("knowledge")),
+        SignalRule("toc", 0.16, "dom", "Table of contents detected.", _flag("has_toc")),
+        SignalRule("code_blocks", 0.16, "dom", "Code blocks detected.", _flag("has_code_blocks")),
+        SignalRule("version_info", 0.08, "text", "Version information detected.", _flag("has_version_info")),
+    ],
+    PageType.API_REFERENCE_PAGE.value: [
+        SignalRule("api_endpoint", 0.3, "text", "API endpoint detected.", _flag("has_api_endpoint")),
+        SignalRule("parameter_table", 0.22, "dom", "Parameter table detected.", _flag("has_parameter_table")),
+        SignalRule("code_blocks", 0.1, "dom", "Code blocks detected.", _flag("has_code_blocks")),
+        SignalRule("docs_keywords", 0.12, "text", "API/docs keywords detected.", _keyword("knowledge"), _keywords_value("knowledge")),
+    ],
+    PageType.WIKI_PAGE.value: [
+        SignalRule("wiki_schema", 0.28, "structured_data", "Wiki/DefinedTerm structured data detected.", _has_schema("DefinedTerm", "WebPage"), _schema_value),
+        SignalRule("toc", 0.12, "dom", "Reference table of contents detected.", _flag("has_toc")),
+        SignalRule("definition_terms", 0.14, "text", "Definition/reference keywords detected.", lambda signals: _contains_terms(signals, ("definition", "wiki", "glossary", "reference"))),
+    ],
+    PageType.DATASET_PAGE.value: [
+        SignalRule("schema_dataset", 0.42, "structured_data", "Dataset structured data detected.", _has_schema("Dataset", "DataCatalog"), _schema_value),
+        SignalRule("download_terms", 0.12, "text", "Dataset/download keywords detected.", lambda signals: _contains_terms(signals, ("dataset", "data catalog", "download", "csv"))),
+        SignalRule("parameter_table", 0.08, "dom", "Dataset metadata table detected.", _flag("has_parameter_table")),
+    ],
+    PageType.RESEARCH_PAPER_PAGE.value: [
+        SignalRule("schema_scholarly", 0.42, "structured_data", "Scholarly article structured data detected.", _has_schema("ScholarlyArticle", "TechArticle"), _schema_value),
+        SignalRule("published_date", 0.1, "text", "Publication date detected.", _flag("has_published_date")),
+        SignalRule("citation_terms", 0.16, "text", "Citation/references keywords detected.", lambda signals: _contains_terms(signals, ("abstract", "citation", "references", "doi"))),
+    ],
+    PageType.JOB_POSTING_PAGE.value: [
+        SignalRule("career_keywords", 0.26, "text", "Career/job keywords detected.", _flag("has_career_terms")),
+        SignalRule("apply_intent", 0.12, "text", "Apply intent detected.", lambda signals: "apply" in (signals.text_sample or "").lower() or "지원" in (signals.text_sample or "")),
+        SignalRule("address", 0.06, "text", "Location/address signal detected.", _flag("has_address")),
+    ],
+    PageType.COURSE_DETAIL_PAGE.value: [
+        SignalRule("schema_course", 0.42, "structured_data", "Course structured data detected.", _has_schema("Course"), _schema_value),
+        SignalRule("course_terms", 0.18, "text", "Course/curriculum keywords detected.", lambda signals: _contains_terms(signals, ("course", "lesson", "curriculum", "instructor", "syllabus"))),
+    ],
+    PageType.VIDEO_PAGE.value: [
+        SignalRule("schema_video", 0.42, "structured_data", "VideoObject structured data detected.", _has_schema("VideoObject"), _schema_value),
+        SignalRule("media_player", 0.2, "layout", "Video/audio player detected.", _flag("has_media_player_area")),
+        SignalRule("video_terms", 0.1, "text", "Video/watch keywords detected.", lambda signals: _contains_terms(signals, ("video", "watch", "episode", "duration"))),
+    ],
+    PageType.LOCAL_BUSINESS_PAGE.value: [
+        SignalRule("schema_local_business", 0.42, "structured_data", "Local business/place structured data detected.", _has_schema("LocalBusiness", "Place", "Restaurant"), _schema_value),
+        SignalRule("address", 0.18, "text", "Address detected.", _flag("has_address")),
+        SignalRule("contact_info", 0.12, "text", "Contact info detected.", _flag("has_contact_info")),
+        SignalRule("map_area", 0.1, "layout", "Map area detected.", _flag("has_map_area")),
+    ],
+    PageType.REAL_ESTATE_LISTING_PAGE.value: [
+        SignalRule("schema_real_estate", 0.38, "structured_data", "Real estate structured data detected.", _has_schema("RealEstateListing", "Residence", "Apartment"), _schema_value),
+        SignalRule("property_terms", 0.18, "text", "Property listing keywords detected.", lambda signals: _contains_terms(signals, ("bedroom", "bathroom", "sqft", "property", "real estate"))),
+        SignalRule("price", 0.12, "text", "Property price detected.", _flag("has_price")),
+        SignalRule("address", 0.08, "text", "Property address detected.", _flag("has_address")),
+    ],
+    PageType.PROFILE_PAGE.value: [
+        SignalRule("schema_person", 0.34, "structured_data", "Person/Profile structured data detected.", _has_schema("Person", "ProfilePage"), _schema_value),
+        SignalRule("profile_links", 0.12, "link_graph", "Profile link pattern detected.", _count_at_least("profile_link_count", 1), _value("profile_link_count")),
+        SignalRule("author", 0.1, "text", "Author/person signal detected.", _flag("has_author")),
+    ],
+    PageType.PRICING_PAGE.value: [
+        SignalRule("pricing_table", 0.3, "layout", "Pricing table detected.", _flag("has_pricing_table")),
+        SignalRule("price", 0.16, "text", "Price detected.", _flag("has_price")),
+        SignalRule("comparison_table", 0.12, "layout", "Comparison table detected.", _flag("has_comparison_table")),
+        SignalRule("commerce_keywords", 0.1, "text", "Commerce keywords detected.", _keyword("commerce"), _keywords_value("commerce")),
+    ],
+    PageType.LOGIN_PAGE.value: [
+        SignalRule("password_field", 0.4, "form", "Password field detected.", _flag("has_password_field")),
+        SignalRule("login_form", 0.25, "form", "Login form detected.", _flag("has_login_form")),
+        SignalRule("protected_url_hint", 0.08, "url", "Protected URL hint detected.", _url_hint("protected"), _url_hint_value),
+    ],
+    PageType.CHECKOUT_PAGE.value: [
+        SignalRule("protected_url_hint", 0.25, "url", "Checkout/cart URL hint detected.", _url_hint("protected"), _url_hint_value),
+        SignalRule("buy_button", 0.14, "dom", "Purchase button detected.", _flag("has_buy_button")),
+        SignalRule("commerce_keywords", 0.12, "text", "Checkout commerce keywords detected.", _keyword("commerce"), _keywords_value("commerce")),
+    ],
+    PageType.PAYMENT_PAGE.value: [
+        SignalRule("payment_fields", 0.42, "form", "Payment fields detected.", _flag("has_payment_fields")),
+        SignalRule("protected_keywords", 0.16, "text", "Payment/protected keywords detected.", _keyword("protected"), _keywords_value("protected")),
+        SignalRule("protected_url_hint", 0.1, "url", "Payment URL hint detected.", _url_hint("protected"), _url_hint_value),
+    ],
+    PageType.TERMS_PAGE.value: [
+        SignalRule("policy_terms", 0.32, "text", "Policy/terms terms detected.", _flag("has_policy_terms")),
+        SignalRule("corporate_keywords", 0.08, "text", "Corporate legal keywords detected.", _keyword("corporate"), _keywords_value("corporate")),
+    ],
+    PageType.PRIVACY_POLICY_PAGE.value: [
+        SignalRule("privacy_terms", 0.36, "text", "Privacy terms detected.", _flag("has_privacy_terms")),
+        SignalRule("policy_terms", 0.14, "text", "Policy terms detected.", _flag("has_policy_terms")),
+    ],
+    PageType.SITEMAP_PAGE.value: [
+        SignalRule("sitemap_resource", 0.45, "resource", "Sitemap resource detected.", _flag("has_sitemap_resource")),
+        SignalRule("xml_resource", 0.12, "resource", "XML resource detected.", _flag("has_xml_resource")),
+        SignalRule("system_url_hint", 0.12, "url", "System URL hint detected.", _url_hint("system"), _url_hint_value),
+    ],
+    PageType.RSS_FEED_PAGE.value: [
+        SignalRule("feed_resource", 0.45, "resource", "RSS/Atom feed detected.", _flag("has_feed_resource")),
+        SignalRule("xml_resource", 0.1, "resource", "XML feed resource detected.", _flag("has_xml_resource")),
+    ],
+    PageType.ERROR_PAGE.value: [
+        SignalRule("error_status", 0.38, "http", "Error status detected.", _flag("has_error_status")),
+    ],
+    PageType.NOT_FOUND_PAGE.value: [
+        SignalRule("not_found", 0.5, "http", "404/not found signal detected.", _flag("has_not_found")),
+        SignalRule("error_status", 0.12, "http", "Error status supports not found page.", _flag("has_error_status")),
+    ],
+    PageType.ACCESS_DENIED_PAGE.value: [
+        SignalRule("access_denied", 0.42, "protected", "Access denied text detected.", _flag("has_access_denied")),
+        SignalRule("error_status", 0.1, "http", "Error status supports access denied.", _flag("has_error_status")),
+    ],
+    PageType.CAPTCHA_PAGE.value: [
+        SignalRule("captcha", 0.45, "protected", "Captcha detected.", _flag("has_captcha")),
+        SignalRule("protected_keywords", 0.1, "text", "Protected keywords detected.", _keyword("protected"), _keywords_value("protected")),
+    ],
+}

ontology_platform/crawler_platform/app/core/crawler/page_type_taxonomy.py

@@ -0,0 +1,476 @@
+from __future__ import annotations
+
+from dataclasses import asdict, dataclass, field
+from enum import StrEnum
+from typing import Any
+
+
+class PageDomain(StrEnum):
+    COMMERCE = "Commerce"
+    EDITORIAL = "Editorial"
+    COMMUNITY = "Community"
+    KNOWLEDGE = "Knowledge"
+    CORPORATE = "Corporate"
+    LOCAL = "Local"
+    EDUCATION = "Education"
+    JOBS = "Jobs"
+    MEDIA = "Media"
+    SOFTWARE = "Software"
+    FINANCE = "Finance"
+    GOVERNMENT = "Government"
+    HEALTHCARE = "Healthcare"
+    TRANSACTION = "Transaction"
+    SYSTEM = "System"
+    UNKNOWN = "Unknown"
+
+
+class PageArchetype(StrEnum):
+    HOME = "Home"
+    LANDING = "Landing"
+    DETAIL = "Detail"
+    LISTING = "Listing"
+    COLLECTION = "Collection"
+    SEARCH_RESULT = "SearchResult"
+    PROFILE = "Profile"
+    ARTICLE = "Article"
+    THREAD = "Thread"
+    FORM = "Form"
+    TRANSACTION = "Transaction"
+    DASHBOARD = "Dashboard"
+    DOCUMENT = "Document"
+    MEDIA = "Media"
+    ERROR = "Error"
+    SYSTEM_RESOURCE = "SystemResource"
+    UNKNOWN = "Unknown"
+
+
+class PageType(StrEnum):
+    PRODUCT_PAGE = "ProductPage"
+    CATEGORY_PAGE = "CategoryPage"
+    SEARCH_PAGE = "SearchPage"
+    BOARD_PAGE = "BoardPage"
+    NOTICE_PAGE = "NoticePage"
+    BRAND_STORY_PAGE = "BrandStoryPage"
+    ABOUT_PAGE = "AboutPage"
+    CONTACT_PAGE = "ContactPage"
+    PROMOTION_PAGE = "PromotionPage"
+    REVIEW_PAGE = "ReviewPage"
+    UNKNOWN_PAGE = "UnknownPage"
+    PRODUCT_DETAIL_PAGE = "ProductDetailPage"
+    CATEGORY_LISTING_PAGE = "CategoryListingPage"
+    SEARCH_RESULTS_PAGE = "SearchResultsPage"
+    FORUM_BOARD_PAGE = "ForumBoardPage"
+    FORUM_THREAD_PAGE = "ForumThreadPage"
+    PUBLIC_NOTICE_PAGE = "PublicNoticePage"
+    CAMPAIGN_LANDING_PAGE = "CampaignLandingPage"
+    ARTICLE_PAGE = "ArticlePage"
+    NEWS_ARTICLE_PAGE = "NewsArticlePage"
+    BLOG_POST_PAGE = "BlogPostPage"
+    FAQ_PAGE = "FAQPage"
+    QA_PAGE = "QAPage"
+    PROFILE_PAGE = "ProfilePage"
+    DOCUMENTATION_PAGE = "DocumentationPage"
+    API_REFERENCE_PAGE = "APIReferencePage"
+    WIKI_PAGE = "WikiPage"
+    DATASET_PAGE = "DatasetPage"
+    RESEARCH_PAPER_PAGE = "ResearchPaperPage"
+    JOB_POSTING_PAGE = "JobPostingPage"
+    COURSE_DETAIL_PAGE = "CourseDetailPage"
+    VIDEO_PAGE = "VideoPage"
+    LOCAL_BUSINESS_PAGE = "LocalBusinessPage"
+    REAL_ESTATE_LISTING_PAGE = "RealEstateListingPage"
+    PRICING_PAGE = "PricingPage"
+    LOGIN_PAGE = "LoginPage"
+    CHECKOUT_PAGE = "CheckoutPage"
+    PAYMENT_PAGE = "PaymentPage"
+    TERMS_PAGE = "TermsPage"
+    PRIVACY_POLICY_PAGE = "PrivacyPolicyPage"
+    SITEMAP_PAGE = "SitemapPage"
+    RSS_FEED_PAGE = "RSSFeedPage"
+    ERROR_PAGE = "ErrorPage"
+    NOT_FOUND_PAGE = "NotFoundPage"
+    ACCESS_DENIED_PAGE = "AccessDeniedPage"
+    CAPTCHA_PAGE = "CaptchaPage"
+
+
+class EntityType(StrEnum):
+    PRODUCT = "Product"
+    SERVICE = "Service"
+    ARTICLE = "Article"
+    NEWS_ARTICLE = "NewsArticle"
+    PERSON = "Person"
+    ORGANIZATION = "Organization"
+    PLACE = "Place"
+    EVENT = "Event"
+    JOB_POSTING = "JobPosting"
+    COURSE = "Course"
+    QUESTION = "Question"
+    ANSWER = "Answer"
+    REVIEW = "Review"
+    DATASET = "Dataset"
+    SOFTWARE_APPLICATION = "SoftwareApplication"
+    MEDIA_OBJECT = "MediaObject"
+    RECIPE = "Recipe"
+    REAL_ESTATE_PROPERTY = "RealEstateProperty"
+    MEDICAL_CONDITION = "MedicalCondition"
+    LEGAL_DOCUMENT = "LegalDocument"
+    FINANCIAL_PRODUCT = "FinancialProduct"
+    UNKNOWN_ENTITY = "UnknownEntity"
+
+
+class ActionIntent(StrEnum):
+    READ = "Read"
+    BUY = "Buy"
+    SUBSCRIBE = "Subscribe"
+    RESERVE = "Reserve"
+    BOOK = "Book"
+    APPLY = "Apply"
+    DOWNLOAD = "Download"
+    WATCH = "Watch"
+    LISTEN = "Listen"
+    SEARCH = "Search"
+    COMPARE = "Compare"
+    FILTER = "Filter"
+    ASK = "Ask"
+    ANSWER = "Answer"
+    COMMENT = "Comment"
+    REVIEW = "Review"
+    LOGIN = "Login"
+    REGISTER = "Register"
+    PAY = "Pay"
+    CONTACT = "Contact"
+    NAVIGATE = "Navigate"
+    LEARN = "Learn"
+    VERIFY = "Verify"
+    CONFIGURE = "Configure"
+    MANAGE = "Manage"
+
+
+class GraphRole(StrEnum):
+    ENTITY_ANCHOR = "EntityAnchor"
+    RELATION_HUB = "RelationHub"
+    NAVIGATION_HUB = "NavigationHub"
+    COLLECTION_HUB = "CollectionHub"
+    SEARCH_HUB = "SearchHub"
+    TRANSACTION_ONLY = "TransactionOnly"
+    POLICY_SOURCE = "PolicySource"
+    CLAIM_SOURCE = "ClaimSource"
+    PROFILE_ANCHOR = "ProfileAnchor"
+    MEDIA_ANCHOR = "MediaAnchor"
+    REFERENCE_SOURCE = "ReferenceSource"
+    SYSTEM_RESOURCE = "SystemResource"
+    NOISE_PAGE = "NoisePage"
+    UNKNOWN_PATTERN = "UnknownPattern"
+
+
+class AnalyzeStrategy(StrEnum):
+    ANALYZE_FULL = "AnalyzeFull"
+    ANALYZE_STRUCTURE_ONLY = "AnalyzeStructureOnly"
+    ANALYZE_ENTITY_ONLY = "AnalyzeEntityOnly"
+    ANALYZE_RELATIONS_ONLY = "AnalyzeRelationsOnly"
+    ANALYZE_METADATA_ONLY = "AnalyzeMetadataOnly"
+    ANALYZE_DOCUMENT_ONLY = "AnalyzeDocumentOnly"
+    ANALYZE_DISCOVERY_ONLY = "AnalyzeDiscoveryOnly"
+    SKIP_PROTECTED = "SkipProtected"
+    SKIP_NOISE = "SkipNoise"
+
+
+class LLMPolicy(StrEnum):
+    LLM_FULL = "LLMFull"
+    LLM_LIGHT = "LLMLight"
+    LLM_FOR_AMBIGUITY_ONLY = "LLMForAmbiguityOnly"
+    RULE_ONLY = "RuleOnly"
+    NO_LLM = "NoLLM"
+    SKIP = "Skip"
+
+
+@dataclass(frozen=True, slots=True)
+class EvidenceItem:
+    key: str
+    value: str | int | float | bool | None
+    weight: float
+    source: str
+    message: str
+
+    def to_dict(self) -> dict[str, Any]:
+        return asdict(self)
+
+
+@dataclass(slots=True)
+class PageClassificationResult:
+    url: str
+    primary_page_type: str
+    secondary_page_types: list[str] = field(default_factory=list)
+    domain: str = PageDomain.UNKNOWN.value
+    archetype: str = PageArchetype.UNKNOWN.value
+    main_entity_type: str | None = None
+    action_intents: list[str] = field(default_factory=list)
+    graph_roles: list[str] = field(default_factory=lambda: [GraphRole.UNKNOWN_PATTERN.value])
+    confidence: float = 0.0
+    alternatives: list[tuple[str, float]] = field(default_factory=list)
+    evidence: list[EvidenceItem] = field(default_factory=list)
+    should_analyze: bool = False
+    analyze_strategy: str = AnalyzeStrategy.ANALYZE_METADATA_ONLY.value
+    llm_policy: str = LLMPolicy.NO_LLM.value
+    is_protected: bool = False
+    is_noise: bool = False
+
+    def to_dict(self) -> dict[str, Any]:
+        payload = asdict(self)
+        payload["evidence"] = [item.to_dict() for item in self.evidence]
+        payload["legacy_page_type"] = get_legacy_page_type(self)
+        return payload
+
+
+LEGACY_PAGE_TYPES = {
+    PageType.PRODUCT_PAGE.value,
+    PageType.CATEGORY_PAGE.value,
+    PageType.SEARCH_PAGE.value,
+    PageType.BOARD_PAGE.value,
+    PageType.NOTICE_PAGE.value,
+    PageType.BRAND_STORY_PAGE.value,
+    PageType.PROMOTION_PAGE.value,
+    PageType.REVIEW_PAGE.value,
+    PageType.UNKNOWN_PAGE.value,
+    "EventPage",
+}
+
+
+LEGACY_ALIASES: dict[str, str] = {
+    "product": PageType.PRODUCT_PAGE.value,
+    "productpage": PageType.PRODUCT_PAGE.value,
+    "productdetailpage": PageType.PRODUCT_PAGE.value,
+    "brand": PageType.BRAND_STORY_PAGE.value,
+    "brandpage": PageType.BRAND_STORY_PAGE.value,
+    "brandstorypage": PageType.BRAND_STORY_PAGE.value,
+    "about": PageType.BRAND_STORY_PAGE.value,
+    "aboutpage": "AboutPage",
+    "contact": "ContactPage",
+    "contactpage": "ContactPage",
+    "review": PageType.REVIEW_PAGE.value,
+    "reviewpage": PageType.REVIEW_PAGE.value,
+    "productreviewpage": PageType.REVIEW_PAGE.value,
+    "listing": PageType.CATEGORY_PAGE.value,
+    "listingpage": PageType.CATEGORY_PAGE.value,
+    "category": PageType.CATEGORY_PAGE.value,
+    "categorypage": PageType.CATEGORY_PAGE.value,
+    "categorylistingpage": PageType.CATEGORY_PAGE.value,
+    "productlistingpage": PageType.CATEGORY_PAGE.value,
+    "community": PageType.BOARD_PAGE.value,
+    "communitypage": PageType.BOARD_PAGE.value,
+    "board": PageType.BOARD_PAGE.value,
+    "boardpage": PageType.BOARD_PAGE.value,
+    "forumboardpage": PageType.BOARD_PAGE.value,
+    "forumthreadpage": PageType.BOARD_PAGE.value,
+    "search": PageType.SEARCH_PAGE.value,
+    "searchpage": PageType.SEARCH_PAGE.value,
+    "searchresultspage": PageType.SEARCH_PAGE.value,
+    "notice": PageType.NOTICE_PAGE.value,
+    "noticepage": PageType.NOTICE_PAGE.value,
+    "publicnoticepage": PageType.NOTICE_PAGE.value,
+    "promotion": PageType.PROMOTION_PAGE.value,
+    "promotionpage": PageType.PROMOTION_PAGE.value,
+    "campaignlandingpage": PageType.PROMOTION_PAGE.value,
+    "event": "EventPage",
+    "eventpage": "EventPage",
+    "unknown": PageType.UNKNOWN_PAGE.value,
+    "unknownpage": PageType.UNKNOWN_PAGE.value,
+    "notfoundpage": PageType.NOT_FOUND_PAGE.value,
+    "404": PageType.NOT_FOUND_PAGE.value,
+}
+
+
+LEGACY_TO_SEMANTIC_PAGE_TYPE: dict[str, str] = {
+    PageType.PRODUCT_PAGE.value: PageType.PRODUCT_DETAIL_PAGE.value,
+    PageType.CATEGORY_PAGE.value: PageType.CATEGORY_LISTING_PAGE.value,
+    PageType.SEARCH_PAGE.value: PageType.SEARCH_RESULTS_PAGE.value,
+    PageType.BOARD_PAGE.value: PageType.FORUM_BOARD_PAGE.value,
+    PageType.NOTICE_PAGE.value: PageType.PUBLIC_NOTICE_PAGE.value,
+    PageType.BRAND_STORY_PAGE.value: PageType.BRAND_STORY_PAGE.value,
+    PageType.ABOUT_PAGE.value: PageType.ABOUT_PAGE.value,
+    PageType.CONTACT_PAGE.value: PageType.CONTACT_PAGE.value,
+    PageType.PROMOTION_PAGE.value: PageType.PROMOTION_PAGE.value,
+    PageType.REVIEW_PAGE.value: PageType.REVIEW_PAGE.value,
+    PageType.UNKNOWN_PAGE.value: PageType.UNKNOWN_PAGE.value,
+    "EventPage": "EventPage",
+}
+
+
+SEMANTIC_TO_LEGACY_PAGE_TYPE: dict[str, str] = {
+    semantic: legacy for legacy, semantic in LEGACY_TO_SEMANTIC_PAGE_TYPE.items()
+}
+SEMANTIC_TO_LEGACY_PAGE_TYPE.update(
+    {
+        PageType.PRODUCT_DETAIL_PAGE.value: PageType.PRODUCT_PAGE.value,
+        PageType.CATEGORY_LISTING_PAGE.value: PageType.CATEGORY_PAGE.value,
+        PageType.SEARCH_RESULTS_PAGE.value: PageType.SEARCH_PAGE.value,
+        PageType.FORUM_BOARD_PAGE.value: PageType.BOARD_PAGE.value,
+        PageType.FORUM_THREAD_PAGE.value: PageType.BOARD_PAGE.value,
+        PageType.PUBLIC_NOTICE_PAGE.value: PageType.NOTICE_PAGE.value,
+        PageType.CAMPAIGN_LANDING_PAGE.value: PageType.PROMOTION_PAGE.value,
+        PageType.ABOUT_PAGE.value: PageType.BRAND_STORY_PAGE.value,
+        PageType.CONTACT_PAGE.value: PageType.BRAND_STORY_PAGE.value,
+        "ListingPage": PageType.CATEGORY_PAGE.value,
+        "CommunityPage": PageType.BOARD_PAGE.value,
+    }
+)
+
+
+PAGE_TYPE_PROFILES: dict[str, dict[str, Any]] = {
+    PageType.PRODUCT_DETAIL_PAGE.value: {
+        "domain": PageDomain.COMMERCE.value,
+        "archetype": PageArchetype.DETAIL.value,
+        "main_entity_type": EntityType.PRODUCT.value,
+        "action_intents": [ActionIntent.BUY.value, ActionIntent.REVIEW.value],
+        "graph_roles": [GraphRole.ENTITY_ANCHOR.value],
+        "analyze_strategy": AnalyzeStrategy.ANALYZE_FULL.value,
+        "llm_policy": LLMPolicy.LLM_LIGHT.value,
+        "should_analyze": True,
+    },
+    PageType.CATEGORY_LISTING_PAGE.value: {
+        "domain": PageDomain.COMMERCE.value,
+        "archetype": PageArchetype.LISTING.value,
+        "main_entity_type": EntityType.PRODUCT.value,
+        "action_intents": [ActionIntent.FILTER.value, ActionIntent.NAVIGATE.value],
+        "graph_roles": [GraphRole.COLLECTION_HUB.value, GraphRole.RELATION_HUB.value],
+        "analyze_strategy": AnalyzeStrategy.ANALYZE_RELATIONS_ONLY.value,
+        "llm_policy": LLMPolicy.LLM_FOR_AMBIGUITY_ONLY.value,
+        "should_analyze": True,
+    },
+    PageType.SEARCH_RESULTS_PAGE.value: {
+        "domain": PageDomain.UNKNOWN.value,
+        "archetype": PageArchetype.SEARCH_RESULT.value,
+        "main_entity_type": EntityType.UNKNOWN_ENTITY.value,
+        "action_intents": [ActionIntent.SEARCH.value, ActionIntent.NAVIGATE.value],
+        "graph_roles": [GraphRole.SEARCH_HUB.value],
+        "analyze_strategy": AnalyzeStrategy.ANALYZE_DISCOVERY_ONLY.value,
+        "llm_policy": LLMPolicy.LLM_FOR_AMBIGUITY_ONLY.value,
+        "should_analyze": True,
+    },
+    PageType.FORUM_BOARD_PAGE.value: {
+        "domain": PageDomain.COMMUNITY.value,
+        "archetype": PageArchetype.LISTING.value,
+        "main_entity_type": EntityType.ARTICLE.value,
+        "action_intents": [ActionIntent.NAVIGATE.value, ActionIntent.READ.value],
+        "graph_roles": [GraphRole.RELATION_HUB.value, GraphRole.COLLECTION_HUB.value],
+        "analyze_strategy": AnalyzeStrategy.ANALYZE_RELATIONS_ONLY.value,
+        "llm_policy": LLMPolicy.LLM_FOR_AMBIGUITY_ONLY.value,
+        "should_analyze": True,
+    },
+    PageType.PUBLIC_NOTICE_PAGE.value: {
+        "domain": PageDomain.GOVERNMENT.value,
+        "archetype": PageArchetype.DOCUMENT.value,
+        "main_entity_type": EntityType.ARTICLE.value,
+        "action_intents": [ActionIntent.READ.value],
+        "graph_roles": [GraphRole.CLAIM_SOURCE.value],
+        "analyze_strategy": AnalyzeStrategy.ANALYZE_FULL.value,
+        "llm_policy": LLMPolicy.LLM_LIGHT.value,
+        "should_analyze": True,
+    },
+    PageType.BRAND_STORY_PAGE.value: {
+        "domain": PageDomain.CORPORATE.value,
+        "archetype": PageArchetype.ARTICLE.value,
+        "main_entity_type": EntityType.ORGANIZATION.value,
+        "action_intents": [ActionIntent.READ.value, ActionIntent.LEARN.value],
+        "graph_roles": [GraphRole.ENTITY_ANCHOR.value, GraphRole.CLAIM_SOURCE.value],
+        "analyze_strategy": AnalyzeStrategy.ANALYZE_FULL.value,
+        "llm_policy": LLMPolicy.LLM_FULL.value,
+        "should_analyze": True,
+    },
+    PageType.PROMOTION_PAGE.value: {
+        "domain": PageDomain.COMMERCE.value,
+        "archetype": PageArchetype.LANDING.value,
+        "main_entity_type": EntityType.EVENT.value,
+        "action_intents": [ActionIntent.BUY.value, ActionIntent.NAVIGATE.value],
+        "graph_roles": [GraphRole.CLAIM_SOURCE.value],
+        "analyze_strategy": AnalyzeStrategy.ANALYZE_FULL.value,
+        "llm_policy": LLMPolicy.LLM_LIGHT.value,
+        "should_analyze": True,
+    },
+    PageType.REVIEW_PAGE.value: {
+        "domain": PageDomain.COMMUNITY.value,
+        "archetype": PageArchetype.ARTICLE.value,
+        "main_entity_type": EntityType.REVIEW.value,
+        "action_intents": [ActionIntent.READ.value, ActionIntent.REVIEW.value],
+        "graph_roles": [GraphRole.CLAIM_SOURCE.value],
+        "analyze_strategy": AnalyzeStrategy.ANALYZE_FULL.value,
+        "llm_policy": LLMPolicy.LLM_LIGHT.value,
+        "should_analyze": True,
+    },
+    PageType.UNKNOWN_PAGE.value: {
+        "domain": PageDomain.UNKNOWN.value,
+        "archetype": PageArchetype.UNKNOWN.value,
+        "main_entity_type": EntityType.UNKNOWN_ENTITY.value,
+        "action_intents": [],
+        "graph_roles": [GraphRole.UNKNOWN_PATTERN.value],
+        "analyze_strategy": AnalyzeStrategy.ANALYZE_METADATA_ONLY.value,
+        "llm_policy": LLMPolicy.NO_LLM.value,
+        "should_analyze": False,
+    },
+}
+
+
+def normalize_page_type(value: object | None) -> str:
+    """Return the legacy-compatible page type string for existing callers."""
+
+    if isinstance(value, PageClassificationResult):
+        return get_legacy_page_type(value)
+    clean = str(value or "").strip()
+    if not clean:
+        return PageType.UNKNOWN_PAGE.value
+    alias = LEGACY_ALIASES.get(clean.lower())
+    if alias:
+        return alias
+    return SEMANTIC_TO_LEGACY_PAGE_TYPE.get(clean, clean)
+
+
+def normalize_semantic_page_type(value: object | None) -> str:
+    if isinstance(value, PageClassificationResult):
+        return value.primary_page_type
+    legacy = normalize_page_type(value)
+    return LEGACY_TO_SEMANTIC_PAGE_TYPE.get(legacy, str(value or legacy).strip() or PageType.UNKNOWN_PAGE.value)
+
+
+def get_legacy_page_type(value: object | None) -> str:
+    if isinstance(value, PageClassificationResult):
+        return SEMANTIC_TO_LEGACY_PAGE_TYPE.get(value.primary_page_type, value.primary_page_type)
+    return normalize_page_type(value)
+
+
+def build_classification_result_from_legacy(
+    *,
+    url: str,
+    legacy_page_type: str,
+    confidence: float = 0.55,
+    source: str = "legacy_classifier",
+) -> PageClassificationResult:
+    normalized_legacy = normalize_page_type(legacy_page_type)
+    semantic_page_type = LEGACY_TO_SEMANTIC_PAGE_TYPE.get(normalized_legacy, normalized_legacy)
+    profile = PAGE_TYPE_PROFILES.get(semantic_page_type, PAGE_TYPE_PROFILES[PageType.UNKNOWN_PAGE.value])
+    evidence = [
+        EvidenceItem(
+            key="legacy_page_type",
+            value=normalized_legacy,
+            weight=confidence,
+            source=source,
+            message=f"Legacy classifier returned {normalized_legacy}.",
+        )
+    ]
+    return PageClassificationResult(
+        url=url,
+        primary_page_type=semantic_page_type,
+        secondary_page_types=[] if semantic_page_type == normalized_legacy else [normalized_legacy],
+        domain=str(profile["domain"]),
+        archetype=str(profile["archetype"]),
+        main_entity_type=profile.get("main_entity_type"),
+        action_intents=list(profile.get("action_intents") or []),
+        graph_roles=list(profile.get("graph_roles") or []),
+        confidence=confidence,
+        alternatives=[(normalized_legacy, confidence)],
+        evidence=evidence,
+        should_analyze=bool(profile.get("should_analyze")),
+        analyze_strategy=str(profile["analyze_strategy"]),
+        llm_policy=str(profile["llm_policy"]),
+        is_protected=str(profile["analyze_strategy"]) == AnalyzeStrategy.SKIP_PROTECTED.value,
+        is_noise=str(profile["analyze_strategy"]) == AnalyzeStrategy.SKIP_NOISE.value,
+    )

ontology_platform/crawler_platform/app/core/crawler/page_unknown_patterns.py

@@ -0,0 +1,347 @@
+from __future__ import annotations
+
+from collections import Counter
+import hashlib
+import json
+import re
+from typing import Any
+from urllib.parse import urljoin, urlparse
+
+from crawler_platform.app.core.crawler.page_signal_extractor import (
+    build_raw_page_snapshot,
+    extract_page_signals,
+)
+from crawler_platform.app.core.crawler.page_type_taxonomy import PageClassificationResult, PageType
+
+
+UNKNOWN_PATTERN_VERSION = 1
+LOW_CONFIDENCE_PATTERN_THRESHOLD = 0.45
+STOPWORDS = {
+    "about",
+    "after",
+    "again",
+    "also",
+    "and",
+    "are",
+    "but",
+    "can",
+    "for",
+    "from",
+    "has",
+    "have",
+    "home",
+    "into",
+    "more",
+    "not",
+    "our",
+    "page",
+    "that",
+    "the",
+    "this",
+    "with",
+    "your",
+}
+
+
+def should_store_unknown_pattern(
+    result: PageClassificationResult,
+    *,
+    confidence_threshold: float = LOW_CONFIDENCE_PATTERN_THRESHOLD,
+) -> bool:
+    if result.primary_page_type == PageType.UNKNOWN_PAGE.value:
+        return True
+    if 0.0 < result.confidence <= confidence_threshold:
+        return True
+    return any(item.key == "low_confidence" for item in result.evidence)
+
+
+def build_unknown_pattern_payload(
+    *,
+    result: PageClassificationResult,
+    url: str,
+    title: str | None = None,
+    text: str | None = None,
+    html: str | None = None,
+    source_zones: list[str | dict[str, Any]] | None = None,
+    collector_payload: dict[str, Any] | None = None,
+) -> dict[str, Any]:
+    snapshot = build_raw_page_snapshot(
+        url=url,
+        title=title,
+        text=text,
+        html=html,
+        source_zones=source_zones,
+        collector_payload=collector_payload,
+    )
+    signals = extract_page_signals(snapshot)
+    soup = _soup_from_html(html or "")
+    page_text = _normalize_text(text or signals.text_sample or _text_from_soup(soup))
+    links = snapshot.links or _extract_links(soup, url)
+    buttons = snapshot.buttons or _extract_buttons(soup)
+    forms = snapshot.forms or _extract_forms(soup)
+    link_summary = summarize_link_patterns(links, url)
+    forms_summary = summarize_forms(forms)
+    payload = {
+        "version": UNKNOWN_PATTERN_VERSION,
+        "url": url,
+        "title": title or signals.title,
+        "primary_page_type": result.primary_page_type,
+        "confidence": result.confidence,
+        "reason": "unknown_page" if result.primary_page_type == PageType.UNKNOWN_PAGE.value else "low_confidence",
+        "text_sample": page_text[:1000],
+        "text_fingerprint": text_fingerprint(page_text),
+        "html_fingerprint": html_fingerprint(html or ""),
+        "dom_fingerprint": dom_fingerprint(html or ""),
+        "link_pattern_fingerprint": link_pattern_fingerprint(link_summary),
+        "schema_types": sorted(signals.schema_types),
+        "link_pattern_summary": link_summary,
+        "button_labels": _dedupe_strings(buttons)[:20],
+        "forms_summary": forms_summary,
+        "top_keywords": top_keywords(page_text),
+        "layout_blocks": list(signals.layout_blocks),
+        "keyword_hits": dict(signals.keyword_hits),
+        "alternatives": list(result.alternatives[:8]),
+        "evidence": [item.to_dict() for item in result.evidence[:12]],
+    }
+    payload["embedding_input"] = build_unknown_embedding_input(payload)
+    payload["cluster_candidate"] = build_cluster_candidate_payload(payload)
+    return payload
+
+
+def build_unknown_embedding_input(payload: dict[str, Any]) -> str:
+    parts = [
+        str(payload.get("title") or ""),
+        str(payload.get("text_sample") or ""),
+        "schema_types: " + ", ".join(str(item) for item in payload.get("schema_types") or []),
+        "layout_blocks: " + ", ".join(str(item) for item in payload.get("layout_blocks") or []),
+        "buttons: " + ", ".join(str(item) for item in payload.get("button_labels") or []),
+        "keywords: " + ", ".join(str(item.get("term")) for item in payload.get("top_keywords") or [] if isinstance(item, dict)),
+    ]
+    return "\n".join(part for part in parts if part.strip())[:4000]
+
+
+def build_cluster_candidate_payload(payload: dict[str, Any]) -> dict[str, Any]:
+    return {
+        "version": UNKNOWN_PATTERN_VERSION,
+        "candidate_key": stable_hash(
+            {
+                "dom": payload.get("dom_fingerprint"),
+                "links": payload.get("link_pattern_fingerprint"),
+                "schema_types": payload.get("schema_types") or [],
+                "buttons": payload.get("button_labels") or [],
+            }
+        ),
+        "fingerprints": {
+            "text": payload.get("text_fingerprint"),
+            "html": payload.get("html_fingerprint"),
+            "dom": payload.get("dom_fingerprint"),
+            "links": payload.get("link_pattern_fingerprint"),
+        },
+        "features": {
+            "schema_types": payload.get("schema_types") or [],
+            "layout_blocks": payload.get("layout_blocks") or [],
+            "top_keywords": payload.get("top_keywords") or [],
+            "link_patterns": (payload.get("link_pattern_summary") or {}).get("top_path_patterns") or [],
+            "form_count": (payload.get("forms_summary") or {}).get("form_count") or 0,
+        },
+    }
+
+
+def summarize_link_patterns(links: list[dict[str, Any]], base_url: str) -> dict[str, Any]:
+    base_host = urlparse(base_url).netloc.lower()
+    path_patterns = Counter()
+    text_labels: list[str] = []
+    internal_count = 0
+    external_count = 0
+    for link in links:
+        href = str(link.get("href") or "")
+        parsed = urlparse(href)
+        if not href:
+            continue
+        if not parsed.netloc or parsed.netloc.lower() == base_host:
+            internal_count += 1
+        else:
+            external_count += 1
+        path_patterns[_path_pattern(parsed.path)] += 1
+        text = str(link.get("text") or "").strip()
+        if text:
+            text_labels.append(text)
+    return {
+        "total_count": len(links),
+        "internal_count": internal_count,
+        "external_count": external_count,
+        "top_path_patterns": [
+            {"pattern": pattern, "count": count}
+            for pattern, count in path_patterns.most_common(12)
+            if pattern
+        ],
+        "sample_texts": _dedupe_strings(text_labels)[:12],
+    }
+
+
+def summarize_forms(forms: list[dict[str, Any]]) -> dict[str, Any]:
+    methods = Counter()
+    actions = Counter()
+    input_types = Counter()
+    form_texts: list[str] = []
+    for form in forms:
+        method = str(form.get("method") or "get").lower()
+        action = _path_pattern(urlparse(str(form.get("action") or "")).path)
+        methods[method] += 1
+        if action:
+            actions[action] += 1
+        if form.get("text"):
+            form_texts.append(str(form.get("text")))
+        for input_item in form.get("inputs") or []:
+            if isinstance(input_item, dict):
+                input_types[str(input_item.get("type") or "text").lower()] += 1
+    return {
+        "form_count": len(forms),
+        "methods": dict(methods),
+        "action_patterns": dict(actions.most_common(8)),
+        "input_types": dict(input_types.most_common(12)),
+        "sample_texts": _dedupe_strings(form_texts)[:8],
+    }
+
+
+def top_keywords(text: str, *, limit: int = 16) -> list[dict[str, Any]]:
+    tokens = [
+        token.lower()
+        for token in re.findall(r"[A-Za-z][A-Za-z0-9_-]{2,}|[가-힣]{2,}", text or "")
+        if token.lower() not in STOPWORDS
+    ]
+    return [{"term": term, "count": count} for term, count in Counter(tokens).most_common(limit)]
+
+
+def text_fingerprint(text: str) -> str:
+    return f"sha256:{_hash_text(_normalize_text(text))}"
+
+
+def html_fingerprint(html: str) -> str:
+    return f"sha256:{_hash_text(_normalize_html(html))}"
+
+
+def dom_fingerprint(html: str) -> str:
+    soup = _soup_from_html(html)
+    if soup is None:
+        return f"sha256:{_hash_text('')}"
+    tags = []
+    for node in soup.find_all(True):
+        classes = ".".join(str(item).lower() for item in node.get("class", [])[:3])
+        node_id = "#" + str(node.get("id")).lower() if node.get("id") else ""
+        tags.append(f"{node.name}{node_id}{('.' + classes) if classes else ''}")
+    return f"sha256:{_hash_text('>'.join(tags[:400]))}"
+
+
+def link_pattern_fingerprint(link_summary: dict[str, Any]) -> str:
+    return stable_hash(link_summary)
+
+
+def stable_hash(value: Any) -> str:
+    payload = json.dumps(value, sort_keys=True, ensure_ascii=False, separators=(",", ":"))
+    return f"sha256:{_hash_text(payload)}"
+
+
+def _hash_text(value: str) -> str:
+    return hashlib.sha256(value.encode("utf-8", errors="ignore")).hexdigest()
+
+
+def _normalize_text(value: str | None) -> str:
+    return re.sub(r"\s+", " ", str(value or "")).strip()
+
+
+def _normalize_html(value: str | None) -> str:
+    text = re.sub(r">\s+<", "><", str(value or ""))
+    text = re.sub(r"\s+", " ", text)
+    return text.strip()[:200000]
+
+
+def _path_pattern(path: str) -> str:
+    parts = [part for part in path.split("/") if part]
+    normalized = []
+    for part in parts[:8]:
+        if part.isdigit() or re.fullmatch(r"[0-9a-fA-F-]{8,}", part):
+            normalized.append("{id}")
+        else:
+            normalized.append(re.sub(r"\d+", "{n}", part.lower())[:60])
+    return "/" + "/".join(normalized) if normalized else "/"
+
+
+def _soup_from_html(html: str):
+    if not html:
+        return None
+    try:
+        from bs4 import BeautifulSoup
+    except ImportError:
+        return None
+    try:
+        return BeautifulSoup(html, "html.parser")
+    except Exception:
+        return None
+
+
+def _text_from_soup(soup) -> str:
+    if soup is None:
+        return ""
+    return soup.get_text(" ", strip=True)
+
+
+def _extract_links(soup, base_url: str) -> list[dict[str, Any]]:
+    if soup is None:
+        return []
+    links = []
+    for tag in soup.find_all("a"):
+        href = str(tag.get("href") or "").strip()
+        if not href:
+            continue
+        links.append({"href": urljoin(base_url, href), "text": tag.get_text(" ", strip=True)})
+    return links
+
+
+def _extract_buttons(soup) -> list[str]:
+    if soup is None:
+        return []
+    labels = []
+    for tag in soup.select("button,input[type='submit'],input[type='button'],[role='button']"):
+        label = tag.get_text(" ", strip=True) or str(tag.get("value") or tag.get("aria-label") or "")
+        if label.strip():
+            labels.append(label.strip())
+    return labels
+
+
+def _extract_forms(soup) -> list[dict[str, Any]]:
+    if soup is None:
+        return []
+    forms = []
+    for form in soup.find_all("form"):
+        inputs = []
+        for tag in form.select("input,select,textarea"):
+            inputs.append(
+                {
+                    "type": str(tag.get("type") or tag.name or ""),
+                    "name": str(tag.get("name") or ""),
+                    "placeholder": str(tag.get("placeholder") or ""),
+                }
+            )
+        forms.append(
+            {
+                "action": str(form.get("action") or ""),
+                "method": str(form.get("method") or ""),
+                "text": form.get_text(" ", strip=True)[:500],
+                "inputs": inputs,
+            }
+        )
+    return forms
+
+
+def _dedupe_strings(values: list[str]) -> list[str]:
+    seen = set()
+    result = []
+    for value in values:
+        clean = _normalize_text(value)
+        key = clean.lower()
+        if not clean or key in seen:
+            continue
+        seen.add(key)
+        result.append(clean)
+    return result

ontology_platform/crawler_platform/app/core/crawler/pipeline.py

@@ -3,11 +3,17 @@ from __future__ import annotations
 from dataclasses import dataclass
 
 from crawler_platform.app.config.loader import ProjectConfig
-from crawler_platform.app.core.crawler.page_classifier import classify_page
+from crawler_platform.app.core.crawler.page_classifier import (
+    classification_metadata,
+    classify_page_semantic,
+    get_legacy_page_type,
+    should_analyze_page,
+)
 from crawler_platform.app.core.crawler.fetchers import RobotsPolicy, make_fetcher
 from crawler_platform.app.core.crawler.plugins import ParserRegistry, default_parser_registry
 from crawler_platform.app.core.database.repository import KnowledgeRepository
 from crawler_platform.app.core.extractor.base import ExtractionPageContext, Extractor
+from crawler_platform.app.core.extractor.strategy import extract_with_strategy
 from crawler_platform.app.core.extractor.validation import attach_page_context
 
 
@@ -56,23 +62,33 @@ class CrawlPipeline:
         fetch_result = fetcher.fetch(url)
         parser = self.parser_registry.get(source_config.parser)
         parsed = parser.parse(fetch_result.analysis_html, fetch_result.final_url or url)
-        page_type = classify_page(
+        page_classification = classify_page_semantic(
             fetch_result.final_url or url,
             parsed.title or fetch_result.title,
             parsed.raw_text or parsed.text,
             fetch_result.analysis_html,
             parsed.source_zones or [],
+            final_url=fetch_result.final_url,
+            status_code=fetch_result.status_code,
+            content_type=fetch_result.headers.get("content-type"),
         )
+        page_type = get_legacy_page_type(page_classification)
 
         project = self.repository.upsert_project(project_config)
         source = self.repository.get_source(project.id, source_name)
         warnings = [*fetch_result.warnings, *(parsed.extraction_warnings or [])]
         metadata = {
             **parsed.metadata,
+            **classification_metadata(
+                page_classification,
+                title=parsed.title or fetch_result.title,
+                text=parsed.raw_text or parsed.text,
+                html=fetch_result.analysis_html,
+                source_zones=parsed.source_zones or [],
+            ),
             "final_url": fetch_result.final_url,
             "crawl_status": fetch_result.crawl_status,
             "crawl_warnings": fetch_result.warnings,
-            "page_type": page_type,
             "raw_text_length": len(parsed.raw_text or ""),
             "clean_text_length": len(parsed.text or ""),
             "main_content_preview": (parsed.main_content or parsed.text)[:800],
@@ -100,6 +116,20 @@ class CrawlPipeline:
                 robots_status=robots_decision.status,
                 robots_reason=robots_decision.reason,
             )
+        if not should_analyze_page(page_classification, None):
+            return CrawlResult(
+                page_id=page.id,
+                claim_count=0,
+                entity_count=0,
+                crawl_status=fetch_result.crawl_status,
+                extraction_status="skipped",
+                page_type=page_type,
+                clean_text_length=len(parsed.text or ""),
+                raw_text_length=len(parsed.raw_text or ""),
+                warnings=warnings,
+                robots_status=robots_decision.status,
+                robots_reason=robots_decision.reason,
+            )
 
         context = ExtractionPageContext(
             url=url,
@@ -116,7 +146,7 @@ class CrawlPipeline:
             warnings=warnings,
             metadata=metadata,
         )
-        bundle = self.extractor.extract_from_context(context, project_config)
+        bundle = extract_with_strategy(self.extractor, context, project_config)
         bundle = attach_page_context(bundle, context)
         claims = self.repository.save_extraction_bundle(project.id, source, page, bundle, project_config)
         extraction_summary = extraction_summary_from_raw(bundle.raw_output)

ontology_platform/crawler_platform/app/core/crawler/site_crawler.py

@@ -10,12 +10,16 @@ from crawler_platform.app.core.crawler.discovery import discover_links
 from crawler_platform.app.core.crawler.fetchers import RobotsDecision, RobotsPolicy, make_fetcher
 from crawler_platform.app.core.crawler.page_classifier import (
     classify_page as classify_page_type,
+    classification_metadata,
+    classify_page_semantic,
+    get_legacy_page_type,
     should_analyze_page as should_analyze_page_type,
 )
 from crawler_platform.app.core.crawler.plugins import ParserRegistry, default_parser_registry
 from crawler_platform.app.core.database import models
 from crawler_platform.app.core.database.repository import KnowledgeRepository
 from crawler_platform.app.core.extractor.base import ExtractionPageContext, Extractor
+from crawler_platform.app.core.extractor.strategy import extract_with_strategy
 from crawler_platform.app.core.extractor.validation import attach_page_context
 
 
@@ -209,6 +213,16 @@ class SiteCrawler:
         fetch_result = fetcher.fetch(url)
         if is_failed_fetch_status(fetch_result.status_code) or fetch_result.crawl_status != "success":
             error = f"fetch failed with status {fetch_result.status_code}; crawl_status={fetch_result.crawl_status}"
+            page_classification = classify_page_semantic(
+                fetch_result.final_url or url,
+                title=fetch_result.title,
+                text="",
+                html=fetch_result.analysis_html,
+                final_url=fetch_result.final_url,
+                status_code=fetch_result.status_code,
+                content_type=fetch_result.headers.get("content-type"),
+            )
+            page_type = get_legacy_page_type(page_classification)
             page = self.repository.upsert_page(
                 project_id=source.project_id,
                 source_id=source.id,
@@ -217,6 +231,11 @@ class SiteCrawler:
                 status_code=fetch_result.status_code,
                 cleaned_text="",
                 metadata={
+                    **classification_metadata(
+                        page_classification,
+                        title=fetch_result.title,
+                        html=fetch_result.analysis_html,
+                    ),
                     "final_url": fetch_result.final_url,
                     "crawl_status": fetch_result.crawl_status,
                     "crawl_warnings": fetch_result.warnings,
@@ -232,7 +251,7 @@ class SiteCrawler:
                     url=url,
                     depth=depth,
                     status=fetch_result.crawl_status,
-                    page_type="unknown",
+                    page_type=page_type,
                     page_id=page.id,
                     crawl_status=fetch_result.crawl_status,
                     robots_status=robots_decision.status,
@@ -246,13 +265,17 @@ class SiteCrawler:
             return
 
         parsed = parser.parse(fetch_result.analysis_html, fetch_result.final_url or url)
-        page_type = classify_page(
+        page_classification = classify_page_semantic(
             fetch_result.final_url or url,
             parsed.title or fetch_result.title,
             parsed.raw_text or parsed.text,
             fetch_result.analysis_html,
             parsed.source_zones or [],
+            final_url=fetch_result.final_url,
+            status_code=fetch_result.status_code,
+            content_type=fetch_result.headers.get("content-type"),
         )
+        page_type = get_legacy_page_type(page_classification)
         discovered_count = self._enqueue_links(
             html=fetch_result.analysis_html,
             base_url=fetch_result.final_url or url,
@@ -268,10 +291,16 @@ class SiteCrawler:
         warnings = [*fetch_result.warnings, *(parsed.extraction_warnings or [])]
         metadata = {
             **parsed.metadata,
+            **classification_metadata(
+                page_classification,
+                title=parsed.title or fetch_result.title,
+                text=parsed.raw_text or parsed.text,
+                html=fetch_result.analysis_html,
+                source_zones=parsed.source_zones or [],
+            ),
             "final_url": fetch_result.final_url,
             "crawl_status": fetch_result.crawl_status,
             "crawl_warnings": fetch_result.warnings,
-            "page_type": page_type,
             "depth": depth,
             "raw_text_length": len(parsed.raw_text or ""),
             "clean_text_length": len(parsed.text or ""),
@@ -319,7 +348,7 @@ class SiteCrawler:
             )
             return
 
-        if should_analyze_page(page_type, analyze_page_types):
+        if should_analyze_page(page_classification, analyze_page_types):
             context = ExtractionPageContext(
                 url=url,
                 final_url=fetch_result.final_url,
@@ -335,7 +364,7 @@ class SiteCrawler:
                 warnings=warnings,
                 metadata=metadata,
             )
-            bundle = self.extractor.extract_from_context(context, project_config)
+            bundle = extract_with_strategy(self.extractor, context, project_config)
             bundle = attach_page_context(bundle, context)
             claims = self.repository.save_extraction_bundle(source.project_id, source, page, bundle, project_config)
             extraction_summary = extraction_summary_from_raw(bundle.raw_output)

ontology_platform/crawler_platform/app/core/extractor/ai_provider.py

@@ -637,6 +637,38 @@ def list_openai_compatible_models(base_url: str, api_key: str | None = None) ->
     return [{"id": item.get("id", ""), "owned_by": item.get("owned_by")} for item in items if item.get("id")]
 
 
+def list_lmstudio_loaded_models(base_url: str) -> list[dict[str, Any]]:
+    """Return only the models currently loaded in LM Studio.
+
+    Uses LM Studio's native REST API (``/api/v0/models``) which exposes a
+    ``state`` field. Falls back to the OpenAI-compatible ``/v1/models`` list
+    (treated as all-loaded) if the native endpoint is unavailable.
+    """
+    clean = base_url.rstrip("/")
+    if clean.endswith("/v1"):
+        root = clean[: -len("/v1")]
+    elif clean.endswith("/v1/chat/completions"):
+        root = clean[: -len("/v1/chat/completions")]
+    else:
+        root = clean
+    native_url = f"{root}/api/v0/models"
+    try:
+        response = requests.get(native_url, timeout=5)
+        response.raise_for_status()
+        data = response.json()
+        items = data.get("data", []) if isinstance(data, dict) else []
+        loaded = [
+            {"id": item.get("id", ""), "owned_by": item.get("owned_by"), "state": item.get("state")}
+            for item in items
+            if item.get("id") and str(item.get("state", "")).lower() == "loaded"
+        ]
+        if loaded:
+            return loaded
+    except Exception:
+        pass
+    return list_openai_compatible_models(base_url)
+
+
 def parse_entities(items: list[dict[str, Any]]) -> list[ExtractedEntity]:
     entities: list[ExtractedEntity] = []
     for item in items:

ontology_platform/crawler_platform/app/core/extractor/base.py

@@ -61,6 +61,7 @@ class ExtractionPageContext:
         clean_text = self.clean_text
         if text_limit is not None and len(clean_text) > text_limit:
             clean_text = clean_text[:text_limit]
+        semantic_metadata = self.semantic_metadata_payload()
         zones = []
         for zone in self.source_zones:
             zone_text = str(zone.get("text") or "")
@@ -85,8 +86,34 @@ class ExtractionPageContext:
             "clean_text": clean_text,
             "source_zones": zones,
             "warnings": self.warnings,
+            "metadata": semantic_metadata,
         }
 
+    def semantic_metadata_payload(self) -> dict[str, Any]:
+        payload = {
+            key: self.metadata[key]
+            for key in ("semantic_page_type", "analyze_strategy", "llm_policy")
+            if key in self.metadata
+        }
+        classification = self.metadata.get("page_classification")
+        if isinstance(classification, dict):
+            payload["page_classification"] = {
+                key: classification.get(key)
+                for key in (
+                    "primary_page_type",
+                    "secondary_page_types",
+                    "confidence",
+                    "alternatives",
+                    "legacy_page_type",
+                    "unknown_pattern",
+                )
+                if key in classification
+            }
+            evidence = classification.get("evidence")
+            if isinstance(evidence, list):
+                payload["page_classification"]["evidence"] = evidence[:8]
+        return payload
+
 
 class Extractor(ABC):
     name = "base"

ontology_platform/crawler_platform/app/core/extractor/hybrid.py

@@ -5,6 +5,7 @@ from dataclasses import replace
 from typing import Any
 
 from crawler_platform.app.config.loader import ProjectConfig
+from crawler_platform.app.core.crawler.page_type_taxonomy import LLMPolicy
 from crawler_platform.app.core.extractor.ai_provider import LLMJsonExtractor, dedupe_entities
 from crawler_platform.app.core.extractor.base import (
     ExtractedClaim,
@@ -17,9 +18,40 @@ from crawler_platform.app.core.extractor.rule_based import GenericRuleBasedExtra
 from crawler_platform.app.domains.perfume.extractor import PerfumeRuleBasedExtractor
 
 
-LLM_PAGE_TYPES = {"ProductPage", "BrandStoryPage", "ReviewPage"}
+LLM_PAGE_TYPES = {
-SKIP_LLM_PAGE_TYPES = {"CategoryPage", "SearchPage", "ListingPage"}
+    "ProductPage",
-RULE_ONLY_PAGE_TYPES = {"BoardPage", "CommunityPage", "UnknownPage"}
+    "ProductDetailPage",
+    "BrandStoryPage",
+    "AboutPage",
+    "ContactPage",
+    "ReviewPage",
+    "NoticePage",
+    "PublicNoticePage",
+    "PromotionPage",
+    "CampaignLandingPage",
+    "ArticlePage",
+    "BlogPostPage",
+    "DocumentationPage",
+    "APIReferencePage",
+    "DatasetPage",
+    "ResearchPaperPage",
+    "JobPostingPage",
+    "CourseDetailPage",
+    "LocalBusinessPage",
+    "RealEstateListingPage",
+    "ProfilePage",
+}
+SKIP_LLM_PAGE_TYPES = {"CategoryPage", "CategoryListingPage", "SearchPage", "SearchResultsPage", "ListingPage"}
+RULE_ONLY_PAGE_TYPES = {
+    "BoardPage",
+    "ForumBoardPage",
+    "ForumThreadPage",
+    "CommunityPage",
+    "FAQPage",
+    "QAPage",
+    "VideoPage",
+    "UnknownPage",
+}
 MIN_CLEAN_TEXT_CHARS_FOR_LLM = 300
 MAX_CLEAN_TEXT_CHARS_FOR_LLM = 60000
 
@@ -82,6 +114,13 @@ class HybridExtractor(Extractor):
                 "effective_extraction_mode": "rule_only",
                 "llm_skipped": True,
                 "llm_skip_reason": "rule_only mode",
+                **count_payload(
+                    rule_entity_count=len(rule_bundle.entities),
+                    rule_claim_count=len(rule_bundle.claims),
+                    llm_entity_count=0,
+                    llm_claim_count=0,
+                    comparison=comparison_payload(rule_only=len(rule_bundle.claims)),
+                ),
             }
             return rule_bundle
 
@@ -103,8 +142,13 @@ class HybridExtractor(Extractor):
                 **llm_bundle.raw_output,
                 "extraction_mode": self.mode,
                 "effective_extraction_mode": "llm_only",
-                "rule_entity_count": len(rule_bundle.entities),
+                **count_payload(
-                "rule_claim_count": len(rule_bundle.claims),
+                    rule_entity_count=len(rule_bundle.entities),
+                    rule_claim_count=len(rule_bundle.claims),
+                    llm_entity_count=len(llm_bundle.entities),
+                    llm_claim_count=len(llm_bundle.claims),
+                    comparison=comparison_payload(llm_only=len(llm_bundle.claims)),
+                ),
             }
             return llm_bundle
 
@@ -202,6 +246,13 @@ def fallback_bundle(rule_bundle: ExtractionBundle, extractor: HybridExtractor, e
         "ai_model": extractor.model,
         "ai_warning": reason,
         "fallback": "rule_based",
+        **count_payload(
+            rule_entity_count=len(bundle.entities),
+            rule_claim_count=len(bundle.claims),
+            llm_entity_count=0,
+            llm_claim_count=0,
+            comparison=comparison_payload(rule_only=len(bundle.claims)),
+        ),
     }
     for entity in bundle.entities:
         entity.metadata["ai_fallback_reason"] = reason
@@ -265,6 +316,44 @@ def llm_skipped_bundle(
     return bundle
 
 
+def comparison_payload(
+    *,
+    both_agree: int = 0,
+    rule_only: int = 0,
+    llm_only: int = 0,
+    conflict: int = 0,
+    rejected_by_validation: int = 0,
+) -> dict[str, int]:
+    return {
+        "both_agree": both_agree,
+        "rule_only": rule_only,
+        "llm_only": llm_only,
+        "conflict": conflict,
+        "rejected_by_validation": rejected_by_validation,
+    }
+
+
+def count_payload(
+    *,
+    rule_entity_count: int,
+    rule_claim_count: int,
+    llm_entity_count: int,
+    llm_claim_count: int,
+    comparison: dict[str, int],
+) -> dict[str, Any]:
+    return {
+        "rule_entity_count": rule_entity_count,
+        "rule_claim_count": rule_claim_count,
+        "llm_entity_count": llm_entity_count,
+        "llm_claim_count": llm_claim_count,
+        "agreement_claim_count": comparison["both_agree"],
+        "rule_only_claim_count": comparison["rule_only"],
+        "llm_only_claim_count": comparison["llm_only"],
+        "conflict_claim_count": comparison["conflict"],
+        "comparison": comparison,
+    }
+
+
 def llm_skip_reason(
     context: ExtractionPageContext | None,
     page_text: str,
@@ -274,6 +363,22 @@ def llm_skip_reason(
         return None
     page_type = str(context.page_type or "UnknownPage")
     clean_length = len(context.clean_text or page_text or "")
+    policy = llm_policy_from_context(context)
+    if policy:
+        if policy == LLMPolicy.SKIP.value:
+            return "LLM policy Skip prevents LLM extraction"
+        if policy == LLMPolicy.NO_LLM.value:
+            return "LLM policy NoLLM prevents LLM extraction"
+        if policy == LLMPolicy.RULE_ONLY.value:
+            return "LLM policy RuleOnly routes page to rule-only extraction"
+        if policy == LLMPolicy.LLM_FOR_AMBIGUITY_ONLY.value and not classification_is_ambiguous(context):
+            return "LLM policy LLMForAmbiguityOnly requires ambiguous classification evidence"
+        if policy in {LLMPolicy.LLM_FULL.value, LLMPolicy.LLM_LIGHT.value, LLMPolicy.LLM_FOR_AMBIGUITY_ONLY.value}:
+            if clean_length < MIN_CLEAN_TEXT_CHARS_FOR_LLM:
+                return f"clean text is too short for LLM extraction: {clean_length} chars"
+            if clean_length > MAX_CLEAN_TEXT_CHARS_FOR_LLM:
+                return f"clean text is too long for LLM extraction: {clean_length} chars"
+            return None
     if page_type in SKIP_LLM_PAGE_TYPES:
         return f"page type {page_type} is configured to skip LLM extraction"
     if page_type in RULE_ONLY_PAGE_TYPES:
@@ -287,6 +392,34 @@ def llm_skip_reason(
     return None
 
 
+def llm_policy_from_context(context: ExtractionPageContext) -> str | None:
+    raw_policy = context.metadata.get("llm_policy")
+    if not raw_policy:
+        classification = context.metadata.get("page_classification")
+        if isinstance(classification, dict):
+            raw_policy = classification.get("llm_policy")
+    policy = str(raw_policy or "").strip()
+    return policy or None
+
+
+def classification_is_ambiguous(context: ExtractionPageContext) -> bool:
+    classification = context.metadata.get("page_classification")
+    if not isinstance(classification, dict):
+        return False
+    confidence = float(classification.get("confidence") or 0.0)
+    if 0.0 < confidence < 0.55:
+        return True
+    alternatives = classification.get("alternatives") or []
+    if not isinstance(alternatives, list) or len(alternatives) < 2:
+        return False
+    try:
+        top_score = float(alternatives[0][1])
+        next_score = float(alternatives[1][1])
+    except (TypeError, ValueError, IndexError):
+        return False
+    return abs(top_score - next_score) < 0.08
+
+
 def merge_bundles(
     rule_bundle: ExtractionBundle,
     llm_bundle: ExtractionBundle,

ontology_platform/crawler_platform/app/core/extractor/strategy.py

@@ -0,0 +1,190 @@
+from __future__ import annotations
+
+from collections import Counter
+from typing import Any
+
+from crawler_platform.app.config.loader import ProjectConfig
+from crawler_platform.app.core.crawler.page_analysis_policy import decide_analyze_strategy, decide_llm_policy
+from crawler_platform.app.core.crawler.page_type_taxonomy import AnalyzeStrategy, LLMPolicy
+from crawler_platform.app.core.extractor.base import ExtractionBundle, ExtractionPageContext, Extractor
+
+
+def extract_with_strategy(
+    extractor: Extractor,
+    context: ExtractionPageContext,
+    project_config: ProjectConfig,
+) -> ExtractionBundle:
+    strategy = analysis_strategy_from_context(context)
+    policy = llm_policy_from_context(context)
+    if strategy == AnalyzeStrategy.SKIP_PROTECTED.value:
+        return strategy_only_bundle(context, strategy, policy, "protected page")
+    if strategy == AnalyzeStrategy.SKIP_NOISE.value:
+        return strategy_only_bundle(context, strategy, policy, "noise page")
+    if strategy == AnalyzeStrategy.ANALYZE_METADATA_ONLY.value:
+        return metadata_only_bundle(context, strategy, policy)
+    if strategy == AnalyzeStrategy.ANALYZE_DISCOVERY_ONLY.value:
+        return discovery_only_bundle(context, strategy, policy)
+    if strategy == AnalyzeStrategy.ANALYZE_RELATIONS_ONLY.value:
+        return rule_or_structure_bundle(extractor, context, project_config, strategy, policy)
+    if strategy == AnalyzeStrategy.ANALYZE_ENTITY_ONLY.value:
+        bundle = extractor.extract_from_context(context, project_config)
+        bundle.claims = []
+        bundle.raw_output = {
+            **bundle.raw_output,
+            **strategy_payload(context, strategy, policy),
+            "effective_extraction_mode": "entity_only",
+        }
+        return bundle
+    bundle = extractor.extract_from_context(context, project_config)
+    bundle.raw_output = {
+        **bundle.raw_output,
+        **strategy_payload(context, strategy, policy),
+    }
+    return bundle
+
+
+def analysis_strategy_from_context(context: ExtractionPageContext) -> str:
+    raw_strategy = context.metadata.get("analyze_strategy")
+    if raw_strategy:
+        return str(raw_strategy)
+    classification = context.metadata.get("page_classification")
+    if isinstance(classification, dict) and classification.get("analyze_strategy"):
+        return str(classification["analyze_strategy"])
+    return decide_analyze_strategy(context.page_type)
+
+
+def llm_policy_from_context(context: ExtractionPageContext) -> str:
+    raw_policy = context.metadata.get("llm_policy")
+    if raw_policy:
+        return str(raw_policy)
+    classification = context.metadata.get("page_classification")
+    if isinstance(classification, dict) and classification.get("llm_policy"):
+        return str(classification["llm_policy"])
+    return decide_llm_policy(context.page_type)
+
+
+def rule_or_structure_bundle(
+    extractor: Extractor,
+    context: ExtractionPageContext,
+    project_config: ProjectConfig,
+    strategy: str,
+    policy: str,
+) -> ExtractionBundle:
+    if policy == LLMPolicy.LLM_FOR_AMBIGUITY_ONLY.value and classification_is_ambiguous(context):
+        bundle = extractor.extract_from_context(context, project_config)
+        bundle.raw_output = {
+            **bundle.raw_output,
+            **strategy_payload(context, strategy, policy),
+            "effective_extraction_mode": "ambiguity_limited",
+        }
+        return bundle
+    rule_extractor = getattr(extractor, "rule_extractor", None)
+    if rule_extractor is not None:
+        bundle = rule_extractor.extract_from_context(context, project_config)
+        bundle.extractor_name = f"{bundle.extractor_name}_strategy_routed"
+        bundle.raw_output = {
+            **bundle.raw_output,
+            **strategy_payload(context, strategy, policy),
+            "effective_extraction_mode": "rule_only",
+            "llm_skipped": True,
+            "llm_skip_reason": f"analyze strategy {strategy} uses rule/structure extraction only",
+        }
+        return bundle
+    if getattr(extractor, "provider", "") == "ai":
+        return discovery_only_bundle(context, strategy, policy)
+    bundle = extractor.extract_from_context(context, project_config)
+    bundle.raw_output = {
+        **bundle.raw_output,
+        **strategy_payload(context, strategy, policy),
+        "effective_extraction_mode": "rule_or_structure",
+        "llm_skipped": policy in {LLMPolicy.RULE_ONLY.value, LLMPolicy.NO_LLM.value, LLMPolicy.SKIP.value},
+    }
+    return bundle
+
+
+def classification_is_ambiguous(context: ExtractionPageContext) -> bool:
+    classification = context.metadata.get("page_classification")
+    if not isinstance(classification, dict):
+        return False
+    confidence = float(classification.get("confidence") or 0.0)
+    if 0.0 < confidence < 0.55:
+        return True
+    alternatives = classification.get("alternatives") or []
+    if not isinstance(alternatives, list) or len(alternatives) < 2:
+        return False
+    try:
+        top_score = float(alternatives[0][1])
+        next_score = float(alternatives[1][1])
+    except (TypeError, ValueError, IndexError):
+        return False
+    return abs(top_score - next_score) < 0.08
+
+
+def metadata_only_bundle(context: ExtractionPageContext, strategy: str, policy: str) -> ExtractionBundle:
+    return ExtractionBundle(
+        extractor_name="metadata_only_extractor",
+        provider="strategy",
+        raw_output={
+            **strategy_payload(context, strategy, policy),
+            "effective_extraction_mode": "metadata_only",
+            "llm_skipped": True,
+            "llm_skip_reason": f"analyze strategy {strategy} does not require LLM extraction",
+        },
+    )
+
+
+def discovery_only_bundle(context: ExtractionPageContext, strategy: str, policy: str) -> ExtractionBundle:
+    return ExtractionBundle(
+        extractor_name="discovery_only_extractor",
+        provider="strategy",
+        raw_output={
+            **strategy_payload(context, strategy, policy),
+            "effective_extraction_mode": "discovery_only",
+            "llm_skipped": True,
+            "llm_skip_reason": f"analyze strategy {strategy} uses discovery/structure signals only",
+        },
+    )
+
+
+def strategy_only_bundle(context: ExtractionPageContext, strategy: str, policy: str, reason: str) -> ExtractionBundle:
+    return ExtractionBundle(
+        extractor_name="strategy_skipped_extractor",
+        provider="strategy",
+        raw_output={
+            **strategy_payload(context, strategy, policy),
+            "effective_extraction_mode": "skipped",
+            "llm_skipped": True,
+            "llm_skip_reason": reason,
+            "skip_reason": reason,
+        },
+    )
+
+
+def strategy_payload(context: ExtractionPageContext, strategy: str, policy: str) -> dict[str, Any]:
+    return {
+        "analyze_strategy": strategy,
+        "llm_policy": policy,
+        "page_type": context.page_type,
+        "semantic_page_type": context.metadata.get("semantic_page_type"),
+        "strategy_summary": structure_summary(context),
+    }
+
+
+def structure_summary(context: ExtractionPageContext) -> dict[str, Any]:
+    zone_types = Counter(str(zone.get("zone_type") or "unknown") for zone in context.source_zones)
+    classification = context.metadata.get("page_classification") if isinstance(context.metadata, dict) else None
+    return {
+        "title": context.title,
+        "url": context.final_url or context.url,
+        "clean_text_length": len(context.clean_text or ""),
+        "raw_text_length": len(context.raw_text or ""),
+        "source_zone_count": len(context.source_zones or []),
+        "source_zone_types": dict(zone_types),
+        "classification": {
+            "primary_page_type": classification.get("primary_page_type"),
+            "confidence": classification.get("confidence"),
+            "alternatives": classification.get("alternatives", [])[:5],
+        }
+        if isinstance(classification, dict)
+        else {},
+    }

ontology_platform/crawler_platform/app/core/ontology/domain_discovery.py

@@ -538,12 +538,31 @@ def relation_hint(label: str) -> tuple[str, str] | None:
 def entity_type_from_page_type(page_type: str) -> str | None:
     mapping = {
         "ProductPage": "Product",
+        "ProductDetailPage": "Product",
         "CategoryPage": "Category",
+        "CategoryListingPage": "Category",
+        "SearchPage": "Category",
+        "SearchResultsPage": "Category",
         "BrandStoryPage": "Brand",
+        "AboutPage": "Organization",
+        "ContactPage": "Organization",
         "NoticePage": "Notice",
+        "PublicNoticePage": "Notice",
         "PromotionPage": "Promotion",
+        "CampaignLandingPage": "Promotion",
         "ReviewPage": "Review",
         "BoardPage": "Article",
+        "ForumBoardPage": "Article",
+        "ForumThreadPage": "Article",
+        "ArticlePage": "Article",
+        "NewsArticlePage": "Article",
+        "BlogPostPage": "Article",
+        "FAQPage": "Article",
+        "QAPage": "Article",
+        "DocumentationPage": "Article",
+        "APIReferencePage": "Article",
+        "JobPostingPage": "JobPosting",
+        "PricingPage": "Product",
     }
     return mapping.get(page_type)
 

ontology_platform/crawler_platform/app/core/ontology/relation_schema.py

@@ -3,6 +3,8 @@ from __future__ import annotations
 from dataclasses import dataclass, field
 from typing import Any
 
+from crawler_platform.app.core.crawler.page_type_taxonomy import normalize_page_type
+
 
 @dataclass(frozen=True, slots=True)
 class RelationRule:
@@ -97,13 +99,21 @@ def relation_schema_compatible(
             return f"{predicate} expects a typed entity object"
         if object_type and rule.object_types and object_type not in rule.object_types:
             return f"object type {object_type} is not allowed for {predicate}"
-    if page_type and rule.page_types and page_type not in rule.page_types:
+    if page_type and rule.page_types and not page_type_matches_rule(page_type, rule.page_types):
         return f"predicate {predicate} is not allowed for page type {page_type}"
     if source_zone and rule.source_zones and source_zone not in rule.source_zones:
         return f"source zone {source_zone} is not allowed for {predicate}"
     return None
 
 
+def page_type_matches_rule(page_type: str, allowed_page_types: set[str]) -> bool:
+    if page_type in allowed_page_types:
+        return True
+    normalized_page_type = normalize_page_type(page_type)
+    normalized_allowed = {normalize_page_type(allowed_page_type) for allowed_page_type in allowed_page_types}
+    return normalized_page_type in normalized_allowed
+
+
 def confidence_breakdown(
     *,
     llm_confidence: float,

ontology_platform/crawler_platform/app/core/research/graph_research_loop.py

@@ -7,11 +7,17 @@ from urllib.parse import urldefrag, urlparse
 from crawler_platform.app.config.loader import ProjectConfig
 from crawler_platform.app.core.crawler.discovery import discover_links
 from crawler_platform.app.core.crawler.fetchers import RobotsPolicy, make_fetcher
-from crawler_platform.app.core.crawler.page_classifier import classify_page, should_analyze_page
+from crawler_platform.app.core.crawler.page_classifier import (
+    classification_metadata,
+    classify_page_semantic,
+    get_legacy_page_type,
+    should_analyze_page,
+)
 from crawler_platform.app.core.crawler.plugins import ParserRegistry, default_parser_registry
 from crawler_platform.app.core.database import models
 from crawler_platform.app.core.database.repository import KnowledgeRepository
 from crawler_platform.app.core.extractor.base import ExtractionPageContext, Extractor
+from crawler_platform.app.core.extractor.strategy import extract_with_strategy
 from crawler_platform.app.core.extractor.validation import attach_page_context
 from crawler_platform.app.core.research.entity_expansion import EntityExpansionPlanner
 from crawler_platform.app.core.research.exploration_queue import ExplorationItem, ExplorationQueue
@@ -206,13 +212,17 @@ class GraphResearchLoop:
 
         fetch_result = fetcher.fetch(url)
         parser_result = parser.parse(fetch_result.analysis_html, fetch_result.final_url or url)
-        page_type = classify_page(
+        page_classification = classify_page_semantic(
             fetch_result.final_url or url,
             parser_result.title or fetch_result.title,
             parser_result.raw_text or parser_result.text,
             fetch_result.analysis_html,
             parser_result.source_zones or [],
+            final_url=fetch_result.final_url,
+            status_code=fetch_result.status_code,
+            content_type=fetch_result.headers.get("content-type"),
         )
+        page_type = get_legacy_page_type(page_classification)
         relevance = self.relevance.score_url(
             project_id=source.project_id,
             url=fetch_result.final_url or url,
@@ -225,11 +235,17 @@ class GraphResearchLoop:
         )
         metadata = {
             **parser_result.metadata,
+            **classification_metadata(
+                page_classification,
+                title=parser_result.title or fetch_result.title,
+                text=parser_result.raw_text or parser_result.text,
+                html=fetch_result.analysis_html,
+                source_zones=parser_result.source_zones or [],
+            ),
             "research_item": item.to_dict(),
             "research_relevance": asdict(relevance),
             "final_url": fetch_result.final_url,
             "crawl_status": fetch_result.crawl_status,
-            "page_type": page_type,
             "robots_status": robots_decision.status,
             "robots_reason": robots_decision.reason,
             "raw_text_length": len(parser_result.raw_text or ""),
@@ -289,7 +305,7 @@ class GraphResearchLoop:
             fetch_result.crawl_status == "success"
             and parser_result.extraction_status != "failed"
             and relevance.score >= min_relevance
-            and should_analyze_page(page_type, analyze_page_types)
+            and should_analyze_page(page_classification, analyze_page_types)
         ):
             context = ExtractionPageContext(
                 url=url,
@@ -306,7 +322,7 @@ class GraphResearchLoop:
                 warnings=[*fetch_result.warnings, *(parser_result.extraction_warnings or [])],
                 metadata=metadata,
             )
-            bundle = self.extractor.extract_from_context(context, project_config)
+            bundle = extract_with_strategy(self.extractor, context, project_config)
             bundle = attach_page_context(bundle, context)
             claims = self.repository.save_extraction_bundle(source.project_id, source, page, bundle, project_config)
             analyzed = True

ontology_platform/crawler_platform/app/core/research/relevance_engine.py

@@ -8,20 +8,46 @@ from urllib.parse import unquote, urlparse
 from sqlalchemy import select
 from sqlalchemy.orm import Session
 
-from crawler_platform.app.core.crawler.page_classifier import classify_page
+from crawler_platform.app.core.crawler.page_classifier import classify_page_semantic, get_legacy_page_type
 from crawler_platform.app.core.database import models
 
 
 HIGH_VALUE_PAGE_TYPES = {
     "ProductPage": 0.95,
+    "ProductDetailPage": 0.95,
     "BrandStoryPage": 0.88,
+    "AboutPage": 0.82,
+    "ContactPage": 0.72,
     "ReviewPage": 0.82,
     "NoticePage": 0.45,
+    "PublicNoticePage": 0.45,
+    "ArticlePage": 0.55,
+    "NewsArticlePage": 0.55,
+    "BlogPostPage": 0.52,
+    "FAQPage": 0.48,
+    "QAPage": 0.48,
+    "DocumentationPage": 0.5,
+    "APIReferencePage": 0.5,
+    "WikiPage": 0.48,
+    "DatasetPage": 0.52,
+    "ResearchPaperPage": 0.6,
+    "JobPostingPage": 0.46,
+    "CourseDetailPage": 0.5,
+    "VideoPage": 0.36,
+    "LocalBusinessPage": 0.44,
+    "RealEstateListingPage": 0.44,
+    "ProfilePage": 0.36,
+    "PricingPage": 0.62,
     "EventPage": 0.5,
     "PromotionPage": 0.38,
+    "CampaignLandingPage": 0.38,
     "CategoryPage": 0.34,
+    "CategoryListingPage": 0.34,
     "SearchPage": 0.22,
+    "SearchResultsPage": 0.22,
     "BoardPage": 0.18,
+    "ForumBoardPage": 0.18,
+    "ForumThreadPage": 0.24,
     "UnknownPage": 0.25,
 }
 
@@ -96,8 +122,12 @@ class RelevanceEngine:
         normalized = url.strip().rstrip("/")
         parsed = urlparse(normalized)
         combined = unquote(f"{normalized} {label} {text[:3000]}").lower()
-        page_type = classify_page(normalized, label, text, html=html)
+        page_classification = classify_page_semantic(normalized, label, text, html=html)
-        page_type_score = HIGH_VALUE_PAGE_TYPES.get(page_type, 0.25)
+        page_type = page_classification.primary_page_type
+        page_type_score = HIGH_VALUE_PAGE_TYPES.get(
+            page_type,
+            HIGH_VALUE_PAGE_TYPES.get(get_legacy_page_type(page_classification), 0.25),
+        )
         graph_terms = self.graph_terms(project_id)
         tokens = tokenize(combined)
         overlap_count = len(tokens & graph_terms)

ontology_platform/docs/crawl_flowchart_llm_decision.svg

@@ -0,0 +1,61 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="2400" height="3600" viewBox="0 0 2400 3600">
+<defs>
+  <marker id="arrow" markerWidth="10" markerHeight="10" refX="9" refY="3" orient="auto" markerUnits="strokeWidth">
+    <path d="M0,0 L0,6 L9,3 z" fill="#1E40AF"/>
+  </marker>
+</defs>
+<rect width="100%" height="100%" fill="#F7F9FC"/>
+<text x="110" y="120" font-family="Malgun Gothic, Arial" font-size="58" font-weight="700" fill="#172033">크롤 진행 단계와 LLM 사용 판단 구조</text>
+<text x="112" y="176" font-family="Malgun Gothic, Arial" font-size="28" fill="#5C6B82">API 요청부터 Rule/LLM 추출, 검증, DB 저장까지의 흐름</text>
+<path d="M 1200 402 L 1200 470" fill="none" stroke="#1E40AF" stroke-width="4" marker-end="url(#arrow)"/>
+<path d="M 1200 602 L 1200 670" fill="none" stroke="#1E40AF" stroke-width="4" marker-end="url(#arrow)"/>
+<path d="M 1200 802 L 1200 870" fill="none" stroke="#1E40AF" stroke-width="4" marker-end="url(#arrow)"/>
+<path d="M 1200 1002 L 1200 1070" fill="none" stroke="#1E40AF" stroke-width="4" marker-end="url(#arrow)"/>
+<path d="M 1200 1222 L 1200 1300" fill="none" stroke="#1E40AF" stroke-width="4" marker-end="url(#arrow)"/>
+<path d="M 1200 1432 L 1200 1525" fill="none" stroke="#1E40AF" stroke-width="4" marker-end="url(#arrow)"/>
+<path d="M 1200 1897 L 1200 1965" fill="none" stroke="#1E40AF" stroke-width="4" marker-end="url(#arrow)"/>
+<path d="M 1200 2097 L 1200 2190" fill="none" stroke="#1E40AF" stroke-width="4" marker-end="url(#arrow)"/>
+<path d="M 1200 2562 L 1200 2635" fill="none" stroke="#1E40AF" stroke-width="4" marker-end="url(#arrow)"/>
+<path d="M 1200 3012 L 1200 3080" fill="none" stroke="#1E40AF" stroke-width="4" marker-end="url(#arrow)"/>
+<path d="M 1200 3240 L 1200 3330" fill="none" stroke="#1E40AF" stroke-width="4" marker-end="url(#arrow)"/>
+<path d="M 820 1610 L 818 1610 L 818 1809 L 800 1809" fill="none" stroke="#5C6B82" stroke-width="4" marker-end="url(#arrow)"/>\n<rect x="756" y="1779" width="124" height="48" rx="12" fill="#F7F9FC" stroke="#CBD5E1" stroke-width="2"/>\n<text x="818" y="1817" text-anchor="middle" font-family="Malgun Gothic, Arial" font-size="25" font-weight="700" fill="#5C6B82">아니오</text>
+<path d="M 1200 1695 L 1200 1765" fill="none" stroke="#16A34A" stroke-width="4" marker-end="url(#arrow)"/>\n<rect x="1138" y="1735" width="124" height="48" rx="12" fill="#F7F9FC" stroke="#CBD5E1" stroke-width="2"/>\n<text x="1200" y="1773" text-anchor="middle" font-family="Malgun Gothic, Arial" font-size="25" font-weight="700" fill="#16A34A">예</text>
+<path d="M 820 2275 L 758 2275 L 758 2496 L 740 2496" fill="none" stroke="#5C6B82" stroke-width="4" marker-end="url(#arrow)"/>\n<rect x="696" y="2466" width="124" height="48" rx="12" fill="#F7F9FC" stroke="#CBD5E1" stroke-width="2"/>\n<text x="758" y="2504" text-anchor="middle" font-family="Malgun Gothic, Arial" font-size="25" font-weight="700" fill="#5C6B82">skip</text>
+<path d="M 1200 2360 L 1200 2430" fill="none" stroke="#16A34A" stroke-width="4" marker-end="url(#arrow)"/>\n<rect x="1138" y="2400" width="124" height="48" rx="12" fill="#F7F9FC" stroke="#CBD5E1" stroke-width="2"/>\n<text x="1200" y="2438" text-anchor="middle" font-family="Malgun Gothic, Arial" font-size="25" font-weight="700" fill="#16A34A">use</text>
+<path d="M 430 2562 L 430 2946 L 820 2946" fill="none" stroke="#5C6B82" stroke-width="4" marker-end="url(#arrow)"/>
+<path d="M 1200 2785 L 1200 2880" fill="none" stroke="#16A34A" stroke-width="4" marker-end="url(#arrow)"/>
+<rect x="820" y="270" width="760" height="132" rx="24" fill="#FFFFFF" stroke="#2563EB" stroke-width="4"/>
+<text x="1200" y="316" text-anchor="middle" font-family="Malgun Gothic, Arial" font-size="30" font-weight="700" fill="#172033">API 요청</text>\n<text x="1200" y="356" text-anchor="middle" font-family="Malgun Gothic, Arial" font-size="30" font-weight="700" fill="#172033">/crawl-site</text>
+<rect x="820" y="470" width="760" height="132" rx="24" fill="#FFFFFF" stroke="#2563EB" stroke-width="4"/>
+<text x="1200" y="516" text-anchor="middle" font-family="Malgun Gothic, Arial" font-size="30" font-weight="700" fill="#172033">Job 생성 및</text>\n<text x="1200" y="556" text-anchor="middle" font-family="Malgun Gothic, Arial" font-size="30" font-weight="700" fill="#172033">queue 초기화</text>
+<rect x="820" y="670" width="760" height="132" rx="24" fill="#FFFFFF" stroke="#2563EB" stroke-width="4"/>
+<text x="1200" y="716" text-anchor="middle" font-family="Malgun Gothic, Arial" font-size="30" font-weight="700" fill="#172033">URL depth / domain /</text>\n<text x="1200" y="756" text-anchor="middle" font-family="Malgun Gothic, Arial" font-size="30" font-weight="700" fill="#172033">robots 검사</text>
+<rect x="820" y="870" width="760" height="132" rx="24" fill="#FFFFFF" stroke="#2563EB" stroke-width="4"/>
+<text x="1200" y="916" text-anchor="middle" font-family="Malgun Gothic, Arial" font-size="30" font-weight="700" fill="#172033">Fetch:</text>\n<text x="1200" y="956" text-anchor="middle" font-family="Malgun Gothic, Arial" font-size="30" font-weight="700" fill="#172033">HTML 수집</text>
+<rect x="820" y="1070" width="760" height="152" rx="24" fill="#FFFFFF" stroke="#2563EB" stroke-width="4"/>
+<text x="1200" y="1106" text-anchor="middle" font-family="Malgun Gothic, Arial" font-size="30" font-weight="700" fill="#172033">Parse:</text>\n<text x="1200" y="1146" text-anchor="middle" font-family="Malgun Gothic, Arial" font-size="30" font-weight="700" fill="#172033">본문 / zone /</text>\n<text x="1200" y="1186" text-anchor="middle" font-family="Malgun Gothic, Arial" font-size="30" font-weight="700" fill="#172033">clean_text 추출</text>
+<rect x="820" y="1300" width="760" height="132" rx="24" fill="#FFFFFF" stroke="#2563EB" stroke-width="4"/>
+<text x="1200" y="1346" text-anchor="middle" font-family="Malgun Gothic, Arial" font-size="30" font-weight="700" fill="#172033">Page Classifier:</text>\n<text x="1200" y="1386" text-anchor="middle" font-family="Malgun Gothic, Arial" font-size="30" font-weight="700" fill="#172033">ProductPage 등 분류</text>
+<polygon points="1200,1525 1580,1610 1200,1695 820,1610" fill="#FFF7ED" stroke="#D97706" stroke-width="4"/>
+<text x="1200" y="1590" text-anchor="middle" font-family="Malgun Gothic, Arial" font-size="30" font-weight="700" fill="#172033">분석 대상</text>\n<text x="1200" y="1630" text-anchor="middle" font-family="Malgun Gothic, Arial" font-size="30" font-weight="700" fill="#172033">page_type인가?</text>
+<rect x="120" y="1740" width="680" height="138" rx="24" fill="#F8FAFC" stroke="#5C6B82" stroke-width="4"/>
+<text x="460" y="1769" text-anchor="middle" font-family="Malgun Gothic, Arial" font-size="30" font-weight="700" fill="#172033">discovered/skipped</text>\n<text x="460" y="1809" text-anchor="middle" font-family="Malgun Gothic, Arial" font-size="30" font-weight="700" fill="#172033">저장,</text>\n<text x="460" y="1849" text-anchor="middle" font-family="Malgun Gothic, Arial" font-size="30" font-weight="700" fill="#172033">링크만 확장</text>
+<rect x="820" y="1765" width="760" height="132" rx="24" fill="#FFFFFF" stroke="#2563EB" stroke-width="4"/>
+<text x="1200" y="1831" text-anchor="middle" font-family="Malgun Gothic, Arial" font-size="30" font-weight="700" fill="#172033">Extractor 선택</text>
+<rect x="820" y="1965" width="760" height="132" rx="24" fill="#FFFFFF" stroke="#2563EB" stroke-width="4"/>
+<text x="1200" y="2031" text-anchor="middle" font-family="Malgun Gothic, Arial" font-size="30" font-weight="700" fill="#172033">Rule extraction</text>
+<polygon points="1200,2190 1580,2275 1200,2360 820,2275" fill="#FFF7ED" stroke="#D97706" stroke-width="4"/>
+<text x="1200" y="2275" text-anchor="middle" font-family="Malgun Gothic, Arial" font-size="30" font-weight="700" fill="#172033">LLM 사용 판단</text>
+<rect x="120" y="2430" width="620" height="132" rx="24" fill="#F8FAFC" stroke="#5C6B82" stroke-width="4"/>
+<text x="430" y="2496" text-anchor="middle" font-family="Malgun Gothic, Arial" font-size="30" font-weight="700" fill="#172033">rule_only routed</text>
+<rect x="820" y="2430" width="760" height="132" rx="24" fill="#FFFFFF" stroke="#2563EB" stroke-width="4"/>
+<text x="1200" y="2496" text-anchor="middle" font-family="Malgun Gothic, Arial" font-size="30" font-weight="700" fill="#172033">LLM extraction</text>
+<rect x="820" y="2635" width="760" height="150" rx="24" fill="#ECFDF5" stroke="#16A34A" stroke-width="4"/>
+<text x="1200" y="2690" text-anchor="middle" font-family="Malgun Gothic, Arial" font-size="30" font-weight="700" fill="#172033">rule/LLM merge,</text>\n<text x="1200" y="2730" text-anchor="middle" font-family="Malgun Gothic, Arial" font-size="30" font-weight="700" fill="#172033">agreement/conflict 계산</text>
+<rect x="820" y="2880" width="760" height="132" rx="24" fill="#FFFFFF" stroke="#2563EB" stroke-width="4"/>
+<text x="1200" y="2946" text-anchor="middle" font-family="Malgun Gothic, Arial" font-size="30" font-weight="700" fill="#172033">validation</text>
+<rect x="820" y="3080" width="760" height="160" rx="24" fill="#FFFFFF" stroke="#2563EB" stroke-width="4"/>
+<text x="1200" y="3120" text-anchor="middle" font-family="Malgun Gothic, Arial" font-size="30" font-weight="700" fill="#172033">DB 저장:</text>\n<text x="1200" y="3160" text-anchor="middle" font-family="Malgun Gothic, Arial" font-size="30" font-weight="700" fill="#172033">entities / claims /</text>\n<text x="1200" y="3200" text-anchor="middle" font-family="Malgun Gothic, Arial" font-size="30" font-weight="700" fill="#172033">evidence / log</text>
+<rect x="820" y="3330" width="760" height="132" rx="24" fill="#FFFFFF" stroke="#2563EB" stroke-width="4"/>
+<text x="1200" y="3396" text-anchor="middle" font-family="Malgun Gothic, Arial" font-size="30" font-weight="700" fill="#172033">progress 업데이트</text>
+</svg>

ontology_platform/docs/phases/26_05_22_PHASE_INDEX_engine_respect_backup.md

@@ -0,0 +1,100 @@
+# PHASE INDEX - ontology_platform engine-respect roadmap
+
+?묒꽦?? 2026-05-19
+
+踰붿쐞: `ontology_platform` ?꾩슜. `crawler_platform`?€ ?대쾲 ?묒뾽 踰붿쐞?먯꽌 ?쒖쇅?쒕떎.
+
+湲곗? 臾몄꽌:
+- `ontology_platform/docs/?듯빀?ㅺ퀎??md`
+- `ontology_platform/README.md`
+- `ontology_platform/docs/phases/PHASE0_ACCEPTANCE_GATE.md`
+- `ontology_platform/docs/phases/PHASE1_NEXT_STEPS.md`
+- `ontology_platform/docs/phases/PHASE1_ACCEPTANCE_GATE.md`
+- `ontology_platform/docs/phases/PHASE2_ACCEPTANCE_GATE.md`
+
+?듭떖 ?먯튃:
+- OntoCast??Base ?붿쭊?쇰줈 議댁쨷?쒕떎.
+- vendored OntoCast 肄붿뼱???듯빀?ㅺ퀎?쒓? ?덉슜??踰붿쐞 ?몄뿉???섏젙?섏? ?딅뒗??
+- Trafilatura, Crawl4AI, Guardrails, Neo4j GraphRAG??吏곸젒 ?ш뎄?꾪븯吏€ ?딄퀬 ?뉗? adapter/facade濡?媛먯떬??
+- Firecrawl, OpenDeepResearcher 肄붾뱶???ы븿?섏? ?딅뒗??
+- Acceptance Gate瑜??듦낵?섍린 ???ㅼ쓬 ?듯빀?쇰줈 ?섏뼱媛€吏€ ?딅뒗??
+
+---
+
+PHASE 0. ?붿쭊 寃쎄퀎 媛먯궗 諛?Phase Gate 蹂듦뎄
+FILE: ./26_05_19_engine_respect_plan/phase_00_001_engine_boundary_gate.md
+
+1) ?꾩옱 `ont_platform` 紐⑤뱢??Base/Adapter/Draft/Excluded 梨낆엫?쇰줈 遺꾨쪟 [?꾨즺]
+2) Phase 0?먯꽌 誘몃옒 Phase ?섏〈?깆씠 import?섏뼱 ???쒖옉??源⑥? ?딅룄濡?寃뚯씠???뺣━ [?꾨즺]
+3) Phase 0 unit/integration 寃€利??덉감 怨좎젙 [?꾨즺]
+4) `PHASE0_ACCEPTANCE_GATE.md` 媛깆떊 湲곗? ?뺣━ [?꾨즺]
+
+---
+
+PHASE 1. Trafilatura 湲곕컲 URL/HTML ?낅젰 ?뺣젹
+FILE: ./26_05_19_engine_respect_plan/phase_01_001_trafilatura_ingestion.md
+
+1) `web_extractor.py`瑜?Trafilatura adapter 梨낆엫?쇰줈 ?뺣━ [?꾨즺]
+2) `SourceDocument`, `EvidenceSpan`, Content metadata ?€??寃쎄퀎 ?곌껐 [?꾨즺]
+3) `/process/url` ?먮뒗 ?숇벑??URL ?낅젰 API ?ㅺ퀎 [?꾨즺]
+4) ?쒓뎅??URL/HTML fixture 湲곕컲 異붿텧 ?뚯뒪?몄? dedup 湲곗? ?묒꽦 [?꾨즺]
+
+---
+
+PHASE 2. Candidate Storage 諛?Review 梨낆엫 寃쎄퀎
+FILE: ./26_05_19_engine_respect_plan/phase_02_001_candidate_review_boundary.md
+
+1) `storage/models.py`???꾨낫 紐⑤뜽???뺤떇 Review Queue 怨꾩빟?쇰줈 ?뺤젙 [?꾨즺]
+2) OntoCast 寃곌낵?€ lightweight extraction 寃곌낵???€??寃쎈줈 遺꾨━ [?꾨즺]
+3) ?뱀씤/諛섎젮/?먮룞?뱀씤 ?곹깭 ?꾩씠 洹쒖튃 ?뺤쓽 [?꾨즺]
+4) evidence ?녿뒗 ?꾨낫媛€ ?뺤젙 graph濡??ㅼ뼱媛€吏€ 紐삵븯寃?李⑤떒 [?꾨즺]
+
+---
+
+PHASE 3. Crawl4AI ?섏쭛 怨꾩링 諛?Job Orchestration
+FILE: ./26_05_19_engine_respect_plan/phase_03_001_crawl4ai_acquisition_jobs.md
+
+1) `crawl4ai_adapter.py`瑜??숈쟻/?€???섏쭛 adapter濡??쒗븳 [?꾨즺]
+2) crawler profile, robots policy, cache policy瑜??ㅼ젙 湲곕컲?쇰줈 遺꾨━ [?꾨즺]
+3) Job ?곹깭 紐⑤뜽怨?progress API/WebSocket 寃쎄퀎 ?뺣━ [?꾨즺]
+4) Trafilatura ?꾩쿂由ъ? SourceDocument ?€?μ쑝濡??곌껐 [?꾨즺]
+
+---
+
+PHASE 4. Guardrails Validation Gate
+FILE: ./26_05_19_engine_respect_plan/phase_04_001_guardrails_validation_gate.md
+
+1) `core/validation`??Pydantic lightweight?€ Guardrails facade濡?遺꾨━ [?꾨즺]
+2) OntoCast LLM 異쒕젰 ?섑븨 吏€?먯쓣 vendored ?섏젙 ?놁씠 ?곗꽑 ?ㅺ퀎 [?꾨즺]
+3) schema violation, endpoint missing, confidence range ?뚯뒪???묒꽦 [?꾨즺]
+4) Guard ?ㅽ뙣 寃곌낵瑜?candidate/review issue濡??€??[?꾨즺]
+
+---
+
+PHASE 5. Neo4j Projection 諛?GraphRAG 寃€??FILE: ./26_05_19_engine_respect_plan/phase_05_001_neo4j_projection_graphrag.md
+
+1) RDF/Fuseki瑜?canonical store, Neo4j瑜?projection/search store濡?怨좎젙 [?꾨즺]
+2) `core/graph` 湲곗〈 紐⑤뱢??projection/search adapter 梨낆엫?쇰줈 ?щ텇瑜?[?꾨즺]
+3) read-only Text2Cypher?€ vector/hybrid retriever API ?ㅺ퀎 [?꾨즺]
+4) provenance媛€ search result源뚯? ?댁뼱吏€??寃€利?湲곗? ?묒꽦 [?꾨즺]
+
+---
+
+PHASE 6. Maintenance Loop 諛??댁쁺 湲곕뒫 ?뺣━
+FILE: ./26_05_19_engine_respect_plan/phase_06_001_maintenance_loop_operations.md
+
+1) Knowledge Agent??肄붾뱶媛€ ?꾨땲???꾨＼?꾪듃/?뚰겕?뚮줈???⑦꽩留?李⑥슜 [?꾨즺]
+2) Analyst/Researcher/Curator/Auditor/Fixer/Advisor 梨낆엫 ?뺤쓽 [?꾨즺]
+3) `auth`, `audit`, `billing`, `realtime` 珥덉븞 紐⑤뱢???댁쁺 寃쎄퀎 ?뺣━ [?꾨즺]
+4) destructive fix???щ엺 ?뱀씤 寃뚯씠?몃? 諛섎뱶???듦낵?섎룄濡??ㅺ퀎 [?꾨즺]
+
+---
+
+PHASE 7. Hybrid Rule + LLM Extraction
+FILE: ./26_05_19_engine_respect_plan/phase_07_001_hybrid_rule_llm_extraction.md
+
+1) rule baseline, LLM extraction, fallback, validation, Review UI 흐름을 기준선으로 고정 [신규]
+2) `rule_only`, `llm_only`, `hybrid`, `compare` extraction mode 계약 정의 [신규]
+3) product backend에 명시적 HybridExtractor와 rule/LLM agreement metadata 추가 [신규]
+4) confidence breakdown에 rule agreement와 conflict/review 정책 반영 [신규]
+5) Crawl/Research UI에서 mode/provider/model/base URL 선택 지원 [신규]

ontology_platform/docs/phases/26_05_22_semantic_page_classification/phase_01_001_current_flow_boundary.md

@@ -0,0 +1,51 @@
+# PHASE 1. 현재 흐름 기준선 고정 및 영향 범위 정리
+
+## 목표
+
+`Semantic Page Understanding Layer`를 얹기 전에 현재 page classification 흐름과 page_type 문자열 의존 지점을 정확히 고정한다.
+
+## 기준 문서
+
+- `README.md`
+- `ontology_platform/README.md`
+- `ontology_platform/docs/semantic_page_classification_codex_spec.md`
+
+## 작업 범위
+
+1. `crawler_platform/app/core/crawler/page_classifier.py`의 현재 public API 확인
+   - `classify_page(...)`
+   - `normalize_page_type(...)`
+   - `should_analyze_page(...)`
+   - `relation_allowed_for_page_type(...)`
+   - `claim_allowed_for_context(...)`
+
+2. crawler 호출부 확인
+   - `site_crawler.py`
+   - `pipeline.py`
+   - `page_cleaner.py`
+   - `domain_discovery.py`
+   - `relevance_engine.py`
+
+3. extractor 연결 확인
+   - `ExtractionPageContext.page_type`
+   - `HybridExtractor.llm_skip_reason(...)`
+   - `LLM_PAGE_TYPES`, `SKIP_LLM_PAGE_TYPES`, `RULE_ONLY_PAGE_TYPES`
+   - validation metadata의 `page_type`
+
+4. 기존 page_type 문자열 기대 코드 목록화
+   - config `analyze_page_types`
+   - ontology relation rule `allowed_page_types`
+   - frontend display
+   - tests
+   - adapters
+
+## 수정 금지
+
+- 이 phase에서는 구현 변경을 하지 않는다.
+- 기존 page_type 문자열 의미를 변경하지 않는다.
+- 기존 테스트 기대값을 바꾸지 않는다.
+
+## 완료 기준
+
+- 현재 흐름과 의존 지점이 다음 phase 구현의 기준선으로 정리되어 있어야 한다.
+- 신규 semantic API를 추가할 때 깨뜨리면 안 되는 legacy contract가 명확해야 한다.

ontology_platform/docs/phases/26_05_22_semantic_page_classification/phase_01_current_flow_boundary_report.md

@@ -0,0 +1,276 @@
+# Phase 1 Current Flow Boundary Report
+
+작성일: 2026-05-22
+
+범위: Semantic Page Classification Layer 구현 전, 현재 `page_type` 문자열 흐름과 의존 지점을 고정한다.
+
+## 1. page_classifier.py public API
+
+파일: `ontology_platform/crawler_platform/app/core/crawler/page_classifier.py`
+
+현재 public API와 contract:
+
+- `classify_page(url, title=None, text="", html=None, source_zones=None) -> str`
+  - 단일 문자열 page_type을 반환한다.
+  - 호출부는 반환값이 `PageClassificationResult`가 아니라 `str`이라고 가정한다.
+- `normalize_page_type(value: str | None) -> str`
+  - 짧은 alias를 legacy page_type 문자열로 변환한다.
+  - 현재 alias 예: `product -> ProductPage`, `listing/category -> CategoryPage`, `community/board -> BoardPage`.
+- `should_analyze_page(page_type: str, analyze_page_types: set[str]) -> bool`
+  - page_type과 allowlist를 각각 normalize한 뒤 포함 여부만 본다.
+- `relation_allowed_for_page_type(page_type, predicate) -> bool`
+  - `ProductPage`는 product detail predicate를 모두 허용한다.
+  - `UnknownPage`, `SearchPage`, `CategoryPage`, `BoardPage`는 non-merge page로 간주한다.
+  - `BrandStoryPage`, `NoticePage`, `EventPage`, `PromotionPage`는 content page로 간주한다.
+- `claim_allowed_for_context(page_type, predicate, zone_type) -> bool`
+  - page_type relation policy와 source zone policy를 함께 적용한다.
+
+현재 legacy page_type 문자열:
+
+- `ProductPage`
+- `CategoryPage`
+- `SearchPage`
+- `BoardPage`
+- `NoticePage`
+- `BrandStoryPage`
+- `PromotionPage`
+- `EventPage`
+- `ReviewPage`
+- `UnknownPage`
+- 보조/외부 상태 문자열: `unknown`, `external`, `entity`
+
+Phase 2 이후에도 기존 `classify_page(...) -> str` contract는 유지해야 한다. 신규 semantic result는 별도 API로 추가하거나 compatibility wrapper 뒤에 두어야 한다.
+
+## 2. crawler 호출 흐름
+
+### site_crawler.py
+
+파일: `ontology_platform/crawler_platform/app/core/crawler/site_crawler.py`
+
+흐름:
+
+1. fetcher가 HTML을 가져온다.
+2. parser가 `analysis_html`을 parse한다.
+3. `classify_page(final_url or url, title, raw_text or text, analysis_html, source_zones)`를 호출한다.
+4. 반환된 `page_type` 문자열을 page metadata와 `SiteCrawlPageResult`에 저장한다.
+5. `should_analyze_page(page_type, analyze_page_types)`가 true일 때만 `ExtractionPageContext`를 만들고 extractor를 실행한다.
+6. context의 `page_type` 문자열이 HybridExtractor, validation, repository 저장까지 이어진다.
+
+현재 기본 analyze allowlist:
+
+```txt
+ProductPage
+BrandStoryPage
+ReviewPage
+```
+
+따라서 현재 기본 흐름에서는 `CategoryPage`, `SearchPage`, `BoardPage`, `UnknownPage`가 extractor 실행에서 빠진다.
+
+### pipeline.py
+
+파일: `ontology_platform/crawler_platform/app/core/crawler/pipeline.py`
+
+흐름:
+
+1. 단일 URL fetch/parse 후 `classify_page(...)`를 호출한다.
+2. `metadata["page_type"]`에 문자열을 저장한다.
+3. fetch/extraction 실패가 아니면 `ExtractionPageContext.page_type`에 같은 문자열을 넣고 extractor를 실행한다.
+
+주의:
+
+- `pipeline.py`에는 `should_analyze_page()` gate가 없다.
+- 단일 URL pipeline은 현재 모든 정상 parse page를 extractor로 보낸다.
+
+### page_cleaner.py / html_cleaner.py / plugins.py
+
+파일:
+
+- `ontology_platform/crawler_platform/app/core/crawler/page_cleaner.py`
+- `ontology_platform/crawler_platform/app/core/crawler/html_cleaner.py`
+- `ontology_platform/crawler_platform/app/core/crawler/plugins.py`
+
+흐름:
+
+- `PageCleaner.clean()`은 page_type이 없으면 내부에서 `classify_page()`를 호출한다.
+- `ZONE_PRIORITY_BY_PAGE_TYPE`가 legacy page_type 문자열에 의존한다.
+- `ProductPage`, `BrandStoryPage`, `NoticePage`, `BoardPage`, `EventPage`, `PromotionPage`, `CategoryPage`별로 source zone 우선순위가 다르다.
+
+Phase 6에서 semantic type을 추가할 때 `ProductDetailPage -> ProductPage`, `CategoryListingPage -> CategoryPage` 같은 zone compatibility가 필요하다.
+
+## 3. research/discovery 호출 흐름
+
+### graph_research_loop.py
+
+파일: `ontology_platform/crawler_platform/app/core/research/graph_research_loop.py`
+
+흐름:
+
+1. 기본 analyze allowlist는 `ProductPage`, `BrandStoryPage`, `ReviewPage`다.
+2. explored page에서 `classify_page(...)`를 호출한다.
+3. metadata에 `page_type`을 저장한다.
+4. relevance score와 `should_analyze_page(page_type, analyze_page_types)`를 모두 통과해야 extractor를 실행한다.
+5. link 후보 metadata에도 relevance engine이 산출한 `score.page_type`을 저장한다.
+
+### relevance_engine.py
+
+파일: `ontology_platform/crawler_platform/app/core/research/relevance_engine.py`
+
+흐름:
+
+- `score_url()` 내부에서 `classify_page(...)`를 호출한다.
+- `HIGH_VALUE_PAGE_TYPES` 점수표가 legacy page_type 문자열에 의존한다.
+- 신규 semantic page type이 들어오면 점수표 또는 legacy normalization이 필요하다.
+
+현재 주요 점수:
+
+- `ProductPage`: 0.95
+- `BrandStoryPage`: 0.88
+- `ReviewPage`: 0.82
+- `CategoryPage`: 0.34
+- `SearchPage`: 0.22
+- `BoardPage`: 0.18
+- `UnknownPage`: 0.25
+
+### domain_discovery.py
+
+파일: `ontology_platform/crawler_platform/app/core/ontology/domain_discovery.py`
+
+흐름:
+
+- discovery job에서 `classify_page(...)`를 호출한다.
+- `mine_schema_candidates(..., page_type=page_type)`로 page_type을 evidence metadata에 넣는다.
+- `entity_type_from_page_type()`가 legacy mapping을 사용한다.
+
+현재 mapping:
+
+- `ProductPage -> Product`
+- `CategoryPage -> Category`
+- `BrandStoryPage -> Brand`
+- `NoticePage -> Notice`
+- `PromotionPage -> Promotion`
+- `ReviewPage -> Review`
+- `BoardPage -> Article`
+
+## 4. Extractor / HybridExtractor 연결 흐름
+
+### ExtractionPageContext
+
+파일: `ontology_platform/crawler_platform/app/core/extractor/base.py`
+
+현재 `ExtractionPageContext.page_type: str`는 필수 문자열 필드다. `to_payload()`도 `page_type`을 그대로 내보낸다.
+
+Phase 6에서 이 필드는 유지해야 하며, semantic 정보는 `metadata`에 추가하는 방식이 가장 안전하다.
+
+### HybridExtractor
+
+파일: `ontology_platform/crawler_platform/app/core/extractor/hybrid.py`
+
+현재 LLM routing 상수:
+
+- `LLM_PAGE_TYPES = {"ProductPage", "BrandStoryPage", "ReviewPage"}`
+- `SKIP_LLM_PAGE_TYPES = {"CategoryPage", "SearchPage", "ListingPage"}`
+- `RULE_ONLY_PAGE_TYPES = {"BoardPage", "CommunityPage", "UnknownPage"}`
+
+현재 `llm_skip_reason()` 판단 순서:
+
+1. mode가 `hybrid`가 아니거나 context가 없으면 skip 판단 없음.
+2. page_type이 `SKIP_LLM_PAGE_TYPES`면 LLM skip.
+3. page_type이 `RULE_ONLY_PAGE_TYPES`면 rule-only.
+4. page_type이 `LLM_PAGE_TYPES`에 없으면 LLM allowlist 밖으로 skip.
+5. clean text 길이가 너무 짧거나 길면 skip.
+
+Phase 5/6에서 `LLMPolicy`가 있으면 이를 우선하고, 없으면 이 legacy fallback을 유지해야 한다.
+
+### ai_provider.py
+
+파일: `ontology_platform/crawler_platform/app/core/extractor/ai_provider.py`
+
+LLM prompt에 page_type semantics가 직접 들어간다.
+
+현재 prompt contract:
+
+- `ProductPage`는 product detail claim 가능
+- `CategoryPage`는 detailed product claim 제한
+- `BrandStoryPage`는 product price/note claim 제한
+- `UnknownPage`는 ontology claim을 반환하지 않도록 지시
+
+신규 semantic type을 추가할 때 prompt가 `ProductDetailPage`, `CategoryListingPage`를 이해하도록 하거나, prompt에는 legacy page_type을 계속 넘기는 compatibility가 필요하다.
+
+### validation.py
+
+파일: `ontology_platform/crawler_platform/app/core/extractor/validation.py`
+
+흐름:
+
+- `attach_page_context()`가 entity/claim metadata에 `page_type`을 저장한다.
+- `claim_status_for_bundle()`은 `UnknownPage`, `SearchPage`, `CategoryPage`, `BoardPage`를 candidate claim으로 낮춘다.
+- `invalid_claim_reason()`은 relation schema compatibility에 `page_type`을 넘긴다.
+
+신규 semantic type은 validation status와 relation schema의 allowed page type 검증에 영향을 준다.
+
+## 5. 기존 page_type 문자열 기대 지점
+
+### API request/response
+
+파일: `ontology_platform/crawler_platform/app/api/routes.py`
+
+- 여러 request model의 기본 `analyze_page_types`가 `["ProductPage", "BrandStoryPage", "ReviewPage"]`다.
+- relation/quality 관련 request에 `allowed_page_types`가 있다.
+- API response 및 progress payload에서 `page_type` 문자열을 그대로 노출한다.
+
+### Config
+
+파일: `ontology_platform/configs/perfume_subscription.yaml`
+
+- `analyze_page_types` 또는 ontology rule에서 legacy page type이 쓰일 수 있다.
+- target entity/ontology에는 `ProductPage`, `BrandStoryPage`, `ListingPage`, `PromotionPage`, `ReviewPage` 같은 page entity type이 포함되어 있다.
+
+### Adapters
+
+파일: `ontology_platform/crawler_platform/app/adapters/ecommerce/perfume.py`
+
+- relation rules가 `page_types={"ProductPage"}` 또는 `{"ProductPage", "ReviewPage"}` 같은 legacy set에 의존한다.
+
+### Ontology relation schema
+
+파일: `ontology_platform/crawler_platform/app/core/ontology/relation_schema.py`
+
+- configured relation rule의 `allowed_page_types`/`page_types`를 set으로 읽는다.
+- `relation_schema_compatible()`은 `page_type not in rule.page_types`면 reject한다.
+- semantic type 도입 시 legacy alias expansion 없이는 기존 relation rules가 거부될 수 있다.
+
+### Frontend
+
+주요 파일:
+
+- `ontology_platform/web/frontend/src/pages/CrawlPage.tsx`
+- `ontology_platform/web/frontend/src/pages/ReviewPage.tsx`
+- `ontology_platform/web/frontend/src/pages/BuildPipelinePage.tsx`
+- `ontology_platform/web/frontend/src/pages/QualityInspectorPage.tsx`
+- `ontology_platform/web/frontend/src/lib/api/crawl.ts`
+- `ontology_platform/web/frontend/src/lib/api/claims.ts`
+- `ontology_platform/web/frontend/src/lib/api/platform.ts`
+- legacy JS files under `web/frontend/src/legacy`
+
+현재 frontend는 `page_type`을 optional string으로 표시하거나, allowed page types mismatch 검사에 사용한다.
+
+### Tests
+
+주요 테스트:
+
+- `ontology_platform/tests/unit/test_phase7_hybrid_extraction.py`
+  - `CategoryPage`가 LLM skip 되는 기존 behavior를 검증한다.
+- integration/e2e tests는 API response의 `page_type` 문자열 contract에 간접 의존한다.
+
+Phase 8에서 semantic classifier 전용 테스트를 추가하되, 기존 문자열 contract 회귀 테스트도 유지해야 한다.
+
+## 6. Phase 2 이후 유지해야 할 legacy contract
+
+1. `classify_page(...)`는 계속 `str`을 반환해야 한다.
+2. 신규 API는 `classify_page_semantic(...) -> PageClassificationResult`처럼 분리하는 편이 안전하다.
+3. `normalize_page_type()`은 `PageClassificationResult`도 받을 수 있게 확장하되, 기존 string 입력 결과를 바꾸면 안 된다.
+4. `should_analyze_page(str, set)` 기존 호출은 계속 동작해야 한다.
+5. API/metadata의 `page_type` 필드는 legacy string으로 유지하고, semantic 결과는 별도 `page_classification` payload로 저장한다.
+6. `ExtractionPageContext.page_type`은 legacy string으로 유지하고, `analyze_strategy`/`llm_policy`/evidence는 `metadata`로 전달한다.
+7. relation schema, adapter page_types, frontend allowed page type 검증에는 legacy/semantic alias compatibility가 필요하다.
+8. `CategoryPage`, `SearchPage`, `BoardPage`, `UnknownPage`의 기존 skip/rule-only 동작은 Phase 5에서 strategy 기반으로 확장하되, LLM 호출이 늘지 않도록 `LLMPolicy`를 우선한다.

ontology_platform/docs/phases/26_05_22_semantic_page_classification/phase_02_001_taxonomy_result_model.md

@@ -0,0 +1,53 @@
+# PHASE 2. Taxonomy와 Classification Result 모델 추가
+
+## 목표
+
+기존 단일 문자열 page_type을 대체하지 않고, 그 위에 semantic classification result 모델을 추가한다.
+
+## 작업 범위
+
+1. 신규 taxonomy 모듈 추가
+   - 권장 위치: `crawler_platform/app/core/crawler/page_type_taxonomy.py`
+   - 기존 프로젝트 구조와 import 경계를 우선한다.
+
+2. 다음 상수 또는 enum 정의
+   - `PageDomain`
+   - `PageArchetype`
+   - `PageType`
+   - `EntityType`
+   - `ActionIntent`
+   - `GraphRole`
+   - `AnalyzeStrategy`
+   - `LLMPolicy`
+
+3. classification result 모델 추가
+   - `EvidenceItem`
+   - `PageClassificationResult`
+
+4. legacy compatibility helper 추가
+   - `normalize_page_type(result_or_page_type)`
+   - `get_legacy_page_type(result_or_page_type)`
+   - `classify_page_semantic(...)`
+   - 기존 `classify_page(...) -> str` 유지
+
+5. 기존 page type alias 유지
+   - `ProductPage`
+   - `CategoryPage`
+   - `SearchPage`
+   - `BoardPage`
+   - `NoticePage`
+   - `BrandStoryPage`
+   - `PromotionPage`
+   - `UnknownPage`
+   - `ReviewPage`
+
+## 수정 금지
+
+- 기존 `classify_page(...)` 호출부를 한 번에 모두 semantic result 기반으로 바꾸지 않는다.
+- legacy 문자열을 제거하지 않는다.
+
+## 완료 기준
+
+- 기존 호출부가 string page_type을 그대로 받을 수 있어야 한다.
+- semantic result API를 신규 테스트에서 직접 호출할 수 있어야 한다.
+- legacy alias mapping이 테스트로 보호되어야 한다.

ontology_platform/docs/phases/26_05_22_semantic_page_classification/phase_03_001_signal_extraction_layer.md

@@ -0,0 +1,93 @@
+# PHASE 3. Raw Snapshot 및 Signal Extraction 레이어 추가
+
+## 목표
+
+URL substring 중심 분류를 줄이고, HTML/DOM/metadata/link/form/text 기반 signal을 별도 레이어에서 추출한다.
+
+## 작업 범위
+
+1. `RawPageSnapshot` 모델 추가
+
+   * `url`, `final_url`, `status_code`, `content_type`
+   * `title`, `text`, `html`, `rendered_html`
+   * metadata, structured data, headings, links, images, forms, buttons, inputs, tables
+   * source_zones, screenshot_path
+
+2. `PageSignals` 모델 추가
+
+   * structured data signals
+   * commerce signals
+   * listing signals
+   * editorial signals
+   * community signals
+   * knowledge/docs signals
+   * corporate/legal signals
+   * transaction/protected signals
+   * graph/link signals
+   * text/layout keyword signals
+   * visual/layout block signals
+   * external collector signals
+
+3. signal extractor 추가
+
+   * 권장 위치: `crawler_platform/app/core/crawler/page_signal_extractor.py`
+   * BeautifulSoup 사용 가능 시 DOM parsing
+   * BeautifulSoup 미설치/HTML 깨짐 시 regex/text fallback
+
+4. 다국어 확장 고려
+
+   * 한국어/영어 키워드 dictionary를 분리 가능한 구조로 둔다.
+   * 인코딩 깨짐이 있어도 예외 없이 동작한다.
+
+5. visual/layout signal 세부화 고려
+
+   * screenshot 기반 정밀 분석은 이번 phase의 필수 구현 범위가 아니지만, 향후 visual block classification을 붙일 수 있도록 signal 구조를 열어둔다.
+   * DOM class/id/role/aria/heading 구조와 반복 레이아웃을 통해 가능한 범위에서 visual/layout block 후보를 추출한다.
+   * visual/layout block signal 예시는 다음과 같다.
+
+     * hero block
+     * product card grid
+     * article body block
+     * left filter sidebar
+     * top navigation
+     * footer navigation
+     * sticky buy box
+     * review/comment block
+     * FAQ accordion
+     * media player area
+     * map area
+     * calendar/availability grid
+     * dashboard card grid
+     * form wizard / stepper
+     * pricing table
+     * comparison table
+   * 초기 구현은 실제 computer vision까지 요구하지 않는다.
+   * 다만 `PageSignals`에는 visual/layout 후보를 담을 수 있는 필드를 둔다.
+   * 예시 필드:
+
+     * `layout_blocks: list[str]`
+     * `has_hero_block: bool`
+     * `has_card_grid: bool`
+     * `has_filter_sidebar: bool`
+     * `has_sticky_action_box: bool`
+     * `has_media_player_area: bool`
+     * `has_map_area: bool`
+     * `has_calendar_grid: bool`
+     * `has_pricing_table: bool`
+     * `has_comparison_table: bool`
+   * visual/layout signal은 page type을 단독 확정하지 않고, evidence scoring의 보조 신호로 사용한다.
+
+## 수정 금지
+
+* signal extractor에서 page_type을 확정 반환하지 않는다.
+* 이 phase에서 extractor나 crawler의 분석 여부 정책을 바꾸지 않는다.
+* Crawl4AI, Firecrawl, Trafilatura 같은 외부 도구에 핵심 classifier가 강하게 종속되도록 만들지 않는다.
+* screenshot 또는 visual analysis가 없다는 이유로 기본 signal extraction이 실패하면 안 된다.
+
+## 완료 기준
+
+* HTML이 비어도 `PageSignals`가 생성되어야 한다.
+* JSON-LD/OpenGraph/form/input/button/link/text signal이 evidence scorer에서 사용할 수 있는 형태로 정리되어야 한다.
+* 외부 수집 도구 결과를 `RawPageSnapshot`에 매핑할 수 있는 구조 또는 adapter hook이 있어야 한다.
+* visual/layout block 후보를 담을 수 있는 `PageSignals` 필드가 있어야 한다.
+* visual/layout signal이 없어도 기존 signal extraction과 scoring 흐름은 정상 동작해야 한다.

ontology_platform/docs/phases/26_05_22_semantic_page_classification/phase_04_001_evidence_scoring_classifier.md

@@ -0,0 +1,99 @@
+# PHASE 4. Evidence Scoring 기반 Semantic Classification 구현
+
+## 목표
+
+단일 if-return 방식이 아니라 signal별 evidence weight를 합산해 semantic page type을 결정한다.
+
+## 작업 범위
+
+1. `page_type_scorer.py` 추가
+
+   * score accumulator
+   * evidence item 생성 helper
+   * normalize 및 confidence 계산
+   * alternatives 산출
+
+2. 최소 구현 page type
+
+   * `ProductDetailPage`
+   * `CategoryListingPage`
+   * `SearchResultsPage`
+   * `ArticlePage`
+   * `BlogPostPage`
+   * `QAPage`
+   * `FAQPage`
+   * `ForumBoardPage`
+   * `ForumThreadPage`
+   * `BrandStoryPage`
+   * `AboutPage`
+   * `ContactPage`
+   * `DocumentationPage`
+   * `APIReferencePage`
+   * `JobPostingPage`
+   * `PricingPage`
+   * `LoginPage`
+   * `CheckoutPage`
+   * `PaymentPage`
+   * `TermsPage`
+   * `PrivacyPolicyPage`
+   * `SitemapPage`
+   * `RSSFeedPage`
+   * `ErrorPage`
+   * `AccessDeniedPage`
+   * `CaptchaPage`
+   * `UnknownPage`
+
+3. 범용 taxonomy 확장 기준
+
+   * 위의 최소 구현 page type은 1차 구현 범위로 본다.
+   * 장기 목표는 인터넷에 존재하는 다양한 페이지를 포괄할 수 있는 전체 taxonomy catalog를 유지하는 것이다.
+   * 따라서 `page_type_scorer.py`와 taxonomy 정의는 아래 계열을 나중에 확장할 수 있는 구조로 작성한다.
+
+     * Site / Navigation
+     * Commerce / Marketplace
+     * Editorial / Article
+     * Community / UGC
+     * Knowledge / Documentation
+     * Corporate / Organization
+     * Local / Place / Travel / Real Estate
+     * Education / Learning
+     * Jobs / Career
+     * Media / Entertainment
+     * Software / SaaS / App
+     * Finance / Legal / Government
+     * Healthcare / Medical
+     * Transaction / Account / Protected
+     * System / Technical / Machine-readable
+   * 현재 phase에서는 위 전체 계열을 모두 scoring 구현하지 않아도 된다.
+   * 다만 enum, alias, mapping, scorer 구조는 특정 몇 개 타입에 고정하지 말고, 전체 taxonomy catalog가 추가되어도 깨지지 않도록 확장 가능해야 한다.
+   * Phase 4의 최소 구현 page type은 1차 안정화 대상이며, 전체 taxonomy catalog는 별도 taxonomy 문서 또는 후속 phase에서 보강한다.
+   * `UnknownPage`는 전체 taxonomy에 아직 포함되지 않은 신규 페이지 패턴을 발견하기 위한 후보로 유지한다.
+
+4. result enrichment
+
+   * `domain`
+   * `archetype`
+   * `main_entity_type`
+   * `action_intents`
+   * `graph_roles`
+   * `confidence`
+   * `alternatives`
+   * `evidence`
+
+5. low confidence 처리
+
+   * threshold 아래는 `UnknownPage`
+   * alternatives/evidence는 유지
+
+## 수정 금지
+
+* URL 문자열 조건만 추가해서 바로 return하지 않는다.
+* Product/Category/Search/Board만 처리하는 구조로 고정하지 않는다.
+* 1차 최소 구현 page type만을 전체 taxonomy의 전부로 간주하지 않는다.
+
+## 완료 기준
+
+* classification result가 항상 evidence를 포함해야 한다.
+* 보호 페이지가 commerce/detail page로 오분류되지 않아야 한다.
+* category/search/board 계열은 skip 여부가 아니라 semantic type과 graph role이 남아야 한다.
+* 최소 구현 page type은 동작해야 하며, 전체 taxonomy catalog를 후속 확장할 수 있는 구조여야 한다.

ontology_platform/docs/phases/26_05_22_semantic_page_classification/phase_05_001_analysis_llm_policy.md

@@ -0,0 +1,45 @@
+# PHASE 5. Analyze Strategy 및 LLM Policy 분리
+
+## 목표
+
+분석 여부를 단순 boolean allowlist에서 page type별 strategy와 LLM policy로 분리한다.
+
+## 작업 범위
+
+1. `page_analysis_policy.py` 추가
+   - `decide_analyze_strategy(result)`
+   - `decide_llm_policy(result)`
+   - `is_protected_strategy(strategy)`
+   - `is_noise_strategy(strategy)`
+
+2. strategy mapping
+   - `ProductDetailPage`, `ArticlePage`, `FAQPage`, `QAPage`, `BrandStoryPage` -> `AnalyzeFull`
+   - `CategoryListingPage`, `ForumBoardPage` -> `AnalyzeRelationsOnly`
+   - `SearchResultsPage`, `SitemapPage` -> `AnalyzeDiscoveryOnly`
+   - `TermsPage`, `PrivacyPolicyPage` -> `AnalyzeDocumentOnly`
+   - `LoginPage`, `CheckoutPage`, `PaymentPage`, `CaptchaPage`, `AccessDeniedPage` -> `SkipProtected`
+   - `ErrorPage`, `NotFoundPage` -> `SkipNoise`
+   - `UnknownPage` -> `AnalyzeMetadataOnly`
+
+3. LLM policy mapping
+   - full content page -> `LLMFull` 또는 `LLMLight`
+   - listing/search/board -> `LLMForAmbiguityOnly` 또는 `RuleOnly`
+   - sitemap/rss/robots/protected -> `RuleOnly`, `NoLLM`, 또는 `Skip`
+
+4. compatibility
+   - `should_analyze_page(result_or_page_type, analyze_page_types=None)`
+   - 기존 allowlist가 들어오면 legacy behavior를 최대한 유지하되 protected/noise는 안전하게 skip
+   - semantic result가 들어오면 strategy 우선
+
+## 수정 금지
+
+- CategoryPage/SearchPage/BoardPage를 무조건 skip하지 않는다.
+- protected page에서 extractor/LLM이 개인정보를 추출하도록 두지 않는다.
+
+## 완료 기준
+
+- `CategoryListingPage.should_analyze == True`
+- `SearchResultsPage.should_analyze == True`
+- `ForumBoardPage.should_analyze == True`
+- `LoginPage`, `CheckoutPage`, `PaymentPage`는 `should_analyze == False`
+- LLM 호출이 기존보다 불필요하게 증가하지 않도록 policy 테스트가 있어야 한다.

ontology_platform/docs/phases/26_05_22_semantic_page_classification/phase_06_001_pipeline_integration.md

@@ -0,0 +1,45 @@
+# PHASE 6. Crawler, Cleaner, Extractor, Discovery/Relevance 통합
+
+## 목표
+
+Semantic classification result를 crawler와 extractor 흐름에 연결하되, 기존 page_type 문자열 기반 contract는 유지한다.
+
+## 작업 범위
+
+1. `site_crawler.py`
+   - `classify_page_semantic(...)` 호출
+   - 기존 `page_type` 필드는 legacy string으로 유지
+   - metadata에 `page_classification` 저장
+   - `should_analyze_page(result, analyze_page_types)` 사용
+
+2. `pipeline.py`
+   - 단일 URL 처리에서도 semantic payload 저장
+   - `ExtractionPageContext.metadata`에 strategy/policy 전달
+
+3. `ExtractionPageContext`
+   - 기존 `page_type: str`는 유지
+   - metadata 기반 `analyze_strategy`, `llm_policy`, evidence payload 전달
+
+4. `HybridExtractor`
+   - metadata의 `llm_policy`를 우선 사용
+   - 없으면 기존 `LLM_PAGE_TYPES`, `SKIP_LLM_PAGE_TYPES`, `RULE_ONLY_PAGE_TYPES` fallback
+   - protected/skip policy는 LLM 호출 금지
+
+5. compatibility update
+   - `page_cleaner.py` zone priority에 semantic aliases 추가
+   - `domain_discovery.py` page_type entity mapping에 semantic aliases 추가
+   - `relevance_engine.py` high value score mapping에 semantic aliases 추가
+   - relation schema allowed page type 검증에서 legacy/semantic alias 고려
+
+## 수정 금지
+
+- extractor 전체를 새 구조로 갈아엎지 않는다.
+- DB schema 변경을 필수로 만들지 않는다.
+- frontend 표시용 기존 `page_type` 필드를 제거하지 않는다.
+
+## 완료 기준
+
+- 기존 API response의 `page_type`은 문자열로 유지된다.
+- semantic classification payload가 metadata에 남는다.
+- HybridExtractor가 LLMPolicy에 따라 LLM 호출을 줄일 수 있다.
+- 기존 page_type 문자열 기반 relation rule이 신규 semantic type 때문에 깨지지 않는다.

ontology_platform/docs/phases/26_05_22_semantic_page_classification/phase_07_001_unknown_pattern_storage.md

@@ -0,0 +1,47 @@
+# PHASE 7. Unknown Pattern 저장 기반 추가
+
+## 목표
+
+`UnknownPage`를 분석 실패나 폐기 대상으로 두지 않고, taxonomy 확장 후보로 저장 가능한 evidence payload를 만든다.
+
+## 작업 범위
+
+1. unknown payload 정의
+   - url
+   - title
+   - text sample
+   - html fingerprint
+   - dom fingerprint
+   - schema types
+   - link pattern summary
+   - button labels
+   - forms summary
+   - top keywords
+   - alternatives
+   - evidence
+
+2. fingerprint hook
+   - text fingerprint
+   - html/dom fingerprint
+   - link pattern fingerprint
+
+3. 저장 위치
+   - 초기 구현은 `metadata_json["page_classification"]["unknown_pattern"]`
+   - DB migration 없이 시작
+
+4. 향후 확장 hook
+   - embedding input 생성 함수
+   - cluster candidate payload 생성 함수
+   - 실제 clustering은 이번 phase 범위에서 제외
+
+## 수정 금지
+
+- UnknownPage를 무조건 extractor 대상에서 제외하지 않는다.
+- low confidence 결과의 evidence와 alternatives를 버리지 않는다.
+- 이 phase에서 clustering 알고리즘을 새로 도입하지 않는다.
+
+## 완료 기준
+
+- UnknownPage result가 evidence를 가진다.
+- metadata에 unknown pattern summary가 저장 가능하다.
+- 향후 clustering/관리자 검토 UI로 넘길 수 있는 구조다.

ontology_platform/docs/phases/26_05_22_semantic_page_classification/phase_08_001_tests_regression.md

@@ -0,0 +1,59 @@
+# PHASE 8. 테스트 Fixture 및 회귀 검증
+
+## 목표
+
+Semantic Page Understanding Layer가 기존 흐름을 깨지 않고, page type별 strategy와 LLM policy를 정확히 산출하는지 검증한다.
+
+## 작업 범위
+
+1. fixture 추가
+   - `tests/fixtures/pages/product_detail.html`
+   - `tests/fixtures/pages/category_listing.html`
+   - `tests/fixtures/pages/search_results.html`
+   - `tests/fixtures/pages/article.html`
+   - `tests/fixtures/pages/qapage.html`
+   - `tests/fixtures/pages/faq.html`
+   - `tests/fixtures/pages/forum_thread.html`
+   - `tests/fixtures/pages/documentation.html`
+   - `tests/fixtures/pages/job_posting.html`
+   - `tests/fixtures/pages/login.html`
+   - `tests/fixtures/pages/checkout.html`
+   - `tests/fixtures/pages/terms.html`
+   - `tests/fixtures/pages/sitemap.xml`
+   - `tests/fixtures/pages/unknown.html`
+
+2. unit test 추가
+   - primary_page_type 확인
+   - confidence 최소 기준 확인
+   - evidence non-empty 확인
+   - analyze_strategy 확인
+   - llm_policy 확인
+   - protected page skip 확인
+   - UnknownPage 예외 없는 처리 확인
+
+3. compatibility test 추가
+   - 기존 `classify_page(...) -> str`
+   - 신규 `classify_page_semantic(...) -> PageClassificationResult`
+   - `should_analyze_page(str, set)`
+   - `should_analyze_page(result, set)`
+   - legacy aliases
+
+4. extractor policy 회귀 테스트
+   - HybridExtractor가 metadata `llm_policy`를 우선 사용
+   - `Skip`, `NoLLM`, `RuleOnly`에서 LLM 호출 금지
+   - 기존 legacy page_type fallback 유지
+
+5. 회귀 테스트 실행
+   - 기본 명령: `pytest`
+   - 필요 시 변경 범위 우선: `pytest tests/unit`
+
+## 수정 금지
+
+- 기존 테스트 기대값을 불필요하게 변경하지 않는다.
+- LLM live 호출이 필요한 테스트를 기본 회귀 테스트에 포함하지 않는다.
+
+## 완료 기준
+
+- 새 semantic classifier 테스트가 통과한다.
+- 기존 unit/integration 테스트가 통과한다.
+- 테스트 결과와 미실행 사유가 작업 완료 보고에 명확히 기록된다.

ontology_platform/docs/phases/PHASE0_ACCEPTANCE_GATE.md

@@ -1,132 +0,0 @@
-# Phase 0 — Acceptance Gate 결과
-
-본 문서는 통합설계서 §5 Phase 0의 Acceptance Gate를 객관적으로 점검한 결과다. Phase 1 진입 전에 모든 체크가 통과되어야 한다.
-
-## 결과 요약
-
-| # | Acceptance Gate 항목 | 상태 | 검증 방법 |
-|---|---|---|---|
-| 1 | 단일 PDF/JSON 입력 → ontology TTL + facts TTL이 filesystem에 생성됨 | ⚠️ **e2e 검증 대기** (로컬 LLM/API 키 필요) | `tests/e2e/test_phase0_full_pipeline.py` |
-| 2 | `/health`, `/info`, `/process` (FastAPI) 정상 동작 | ✅ **통합 테스트 11/11 통과** (2026-05-19) | `tests/integration/test_api_smoke.py` |
-| 3 | BudgetTracker가 LLM call/triple count를 정확히 기록 | ⚠️ **e2e 검증 대기** (mock 검증은 통합 테스트로 통과) | e2e 테스트가 실제 검증 |
-| 4 | LangGraph 워크플로우 (CONVERT→CHUNK→...→SERIALIZE) 전 노드 traceable | ✅ **OntoCast 원본 워크플로우 무수정 채택** | `vendored/ontocast/ontocast/stategraph/` 그대로 사용 |
-
-추가로 **단위 테스트 16/16 통과** (test_convert_document 7, test_platform_config 5, test_select_ontology 4).
-자동 검증 기준으로는 **unit + integration 27/27 통과**가 현재 Phase 0 기본선이다.
-
-**현재 진척 (2026-05-19)**:
-- Python 3.14.5 `.venv` 환경에서 unit + integration 27/27 통과
-- `python-multipart`를 Phase 0 FastAPI multipart upload 필수 의존성으로 추가
-- Phase 0 production app에서 Phase 1 Trafilatura route가 기본 mount되지 않도록 lazy phase route gate 적용
-- `pip install -e ".[dev]"` 또는 동등한 의존성 설치 필요
-- `pip install -e vendored/ontocast` 로 OntoCast 의존성 설치 완료
-- 패키지 이름 충돌 수정: `platform/` → `ont_platform/` (Python 내장 `platform` 모듈과 충돌)
-- **남은 작업**: e2e 테스트 (Acceptance Gate #1, #3) 실행 — 로컬 Ollama 또는 OpenAI 키 필요
-
-## 다음 작업자가 실행할 검증 절차
-
-### 1) 환경 준비
-
-```powershell
-# Python 3.12+ 설치 (예: https://www.python.org/downloads/)
-python --version  # Python 3.12.x 이상 확인
-
-cd C:\Users\lasta\MyProject\AI\ontology_platform
-
-# 가상환경 + 의존성 설치
-python -m venv .venv
-.venv\Scripts\activate
-pip install --upgrade pip
-pip install -e ".[dev]"
-
-# .env 생성 (실제 LLM 키 채우기)
-Copy-Item .env.example .env
-# 그 다음 .env 파일을 편집하여 LLM_API_KEY 등 채움
-```
-
-### 2) 자동 검증 (Acceptance Gate #2)
-
-```powershell
-# 단위 + 통합 테스트만 (LLM 호출 없음, 빠름)
-.venv\Scripts\python.exe -m pytest tests/unit tests/integration -v
-```
-
-**기대 결과**: 모든 케이스 PASS.
-
-- `tests/unit/test_select_ontology.py` (4 케이스) — Phase 0.2 검증
-- `tests/unit/test_convert_document.py` (7 케이스) — Phase 0.3 검증
-- `tests/unit/test_platform_config.py` (5 케이스) — Phase 0.5 검증
-- `tests/integration/test_api_smoke.py` (11 케이스) — Phase 0.4 + 0.6 mock 검증, Phase 0 future dependency route gate 검증
-
-Windows에서 `%TEMP%` 권한 문제 또는 `.pytest_cache` 쓰기 문제가 발생하면 아래처럼 pytest temp/cache 위치를 workspace 내부로 고정한다.
-
-```powershell
-$env:TMP=(Join-Path (Resolve-Path '.').Path 'pytest_tmp')
-$env:TEMP=$env:TMP
-New-Item -ItemType Directory -Force -Path $env:TMP | Out-Null
-.venv\Scripts\python.exe -m pytest tests/unit tests/integration -v --basetemp "$env:TMP\basetemp" -o cache_dir="$env:TMP\cache"
-```
-
-### 3) End-to-end 검증 (Acceptance Gate #1, #3, #4)
-
-LLM 호출이 실제로 일어남. OpenAI는 비용 발생, Ollama는 로컬에서 무료.
-
-```powershell
-# (A) Ollama 로컬 사용 (권장 — 비용 무료)
-# 사전: Ollama 설치 후 `ollama pull qwen2.5`
-$env:LLM_PROVIDER = "ollama"
-$env:LLM_MODEL_NAME = "qwen2.5"
-$env:LLM_BASE_URL = "http://localhost:11434"
-pytest tests/e2e -m e2e -v
-
-# (B) OpenAI 사용
-$env:LLM_PROVIDER = "openai"
-$env:LLM_MODEL_NAME = "gpt-4o-mini"
-$env:LLM_API_KEY = "sk-..."
-pytest tests/e2e -m e2e -v
-```
-
-**기대 결과**:
-- `test_full_pipeline_writes_ontology_and_facts` PASS
-- 응답에서 ontology TTL과 facts TTL이 비어 있지 않음
-- `metadata.budget.calls_count > 0`
-- `metadata.budget.ontology_triples_generated > 0` 또는 `facts_triples_generated > 0`
-- `tmp_path / "work"` 아래 `.ttl` 또는 `.rdf` 파일 생성됨
-
-### 4) 수동 smoke (선택)
-
-```powershell
-# 서버 기동
-uvicorn ont_platform.api.main:app --reload
-
-# 다른 셸에서
-curl http://localhost:8000/health
-curl http://localhost:8000/info
-curl -X POST http://localhost:8000/process `
-  -H "Content-Type: application/json" `
-  -d '{"text":"Alice works at Acme in Berlin."}'
-```
-
-## 통과 시 처리
-
-위 모든 검증을 통과하면 **이 문서의 표 상태 컬럼을 ✅로 갱신**하고 git에 commit한다.
-
-이후 Phase 1 작업은 [PHASE1_NEXT_STEPS.md](PHASE1_NEXT_STEPS.md)를 따른다.
-
-## 실패 시 처리
-
-- **단위 테스트 실패**: 어느 케이스가 실패했는지 확인. Phase 0.2/0.3/0.5의 vendored 수정 또는 platform/ 코드에 회귀가 발생했을 가능성. PR 단위로 롤백 후 재시도.
-- **통합 테스트 실패**: FastAPI 라우팅/의존성 주입 문제. `platform/api/main.py` 또는 `platform/api/deps.py` 확인.
-- **E2E 테스트 실패**: 
-  - `LLM_API_KEY`, `LLM_PROVIDER`, `LLM_MODEL_NAME` 환경변수 확인
-  - 워크플로우가 timeout: `ServerConfig.base_recursion_limit` 조정 검토
-  - OntoCast `select_ontology.py` 또는 `convert_document.py` 수정에 회귀가 있는지 점검 (VENDORED_MODIFICATIONS.md 참조)
-
-## 검증 이력
-
-| 일자 | 검증자 | 결과 |
-|---|---|---|
-| 2026-05-13 | (코드 작성: ontology-platform agent) | 코드 준비 완료. 실 환경 검증 보류. |
-| 2026-05-14 | lasta + Claude | **unit 16/16, integration 10/10 통과** (Gate #2 ✅). 패키지 이름 충돌 수정 (`platform`→`ont_platform`). e2e는 LLM 필요로 대기. |
-| 2026-05-19 | Codex | **unit 16/16, integration 11/11, 총 27/27 통과**. Phase 0 route gate 추가로 Trafilatura route는 PHASE>=1에서만 lazy mount. e2e는 LLM 필요로 대기. |
-| ____-__-__ | ________________ | __________________________________ |

ontology_platform/docs/phases/PHASE0_NEXT_STEPS.md

@@ -1,124 +0,0 @@
-# Phase 0 — 다음 작업자 핸드오프
-
-본 문서는 이 프로젝트를 이어받는 AI 에이전트 또는 개발자가 즉시 작업을 시작하기 위한 핸드오프 노트다.
-
-## 현재 상태 (지금까지 완료된 것)
-
-- [x] **0.1 일부**: 폴더 골격, `pyproject.toml`, `README.md`, `.env.example`, `.gitignore`, `NOTICE` 생성
-- [x] **설계 문서**: [../통합설계서.md](../통합설계서.md) 배치 완료 (모든 작업의 기준)
-
-## 즉시 시작할 작업 (순서대로)
-
-### 0.1 (잔여): OntoCast vendored copy
-
-**근거**: 통합설계서 §9.1, OntoCast 분석 §13.1
-
-```powershell
-# 1. 원본을 vendored/ontocast로 복사 (.git 제외)
-Copy-Item -Path "C:\Users\lasta\MyProject\AI\참고\ontocast-main\*" `
-          -Destination "C:\Users\lasta\MyProject\AI\ontology_platform\vendored\ontocast\" `
-          -Recurse -Exclude ".git",".github",".venv","node_modules"
-
-# 2. 원본 LICENSE 및 NOTICE를 vendored/ontocast/ 안에 그대로 유지
-# 3. NOTICE 파일의 "(원본 저장소 URL 기입)" 부분을 실제 URL로 채우기
-# 4. git init (아직 안 했다면)
-cd C:\Users\lasta\MyProject\AI\ontology_platform
-git init
-git add .
-git commit -m "Initial scaffold: folder skeleton, design doc, NOTICE"
-```
-
-**확인 사항**:
-- [ ] `vendored/ontocast/` 안에 원본 LICENSE 파일이 있어야 한다
-- [ ] `vendored/ontocast/pyproject.toml`은 그대로 두되, 우리 `pyproject.toml`이 우선
-- [ ] NOTICE 파일의 OntoCast 항목에 실제 source URL 기입
-
-### 0.2: `select_ontology.py` 버그 수정
-
-**근거**: 통합설계서 §5 Phase 0, OntoCast 분석 §13.1 / §9.1 (1번)
-
-**문제**: `vendored/ontocast/ontocast/agent/select_ontology.py`에서 None 선택 인덱스 불일치.
-- 코드는 `answer_index == 0`을 None으로 처리
-- 그러나 dynamic model은 `1..num_ontologies+1` 범위 사용
-- 실제 None 선택은 `num_ontologies + 1`이어야 자연스러움
-
-**조치**:
-1. 해당 함수의 분기 로직을 `answer_index == num_ontologies + 1` 또는 동등한 표현으로 수정
-2. **수정 사실을 파일 상단 주석으로 명시** (Apache 2.0 의무): 예) `# MODIFIED 2026-MM-DD: Fixed None index inconsistency, see docs/통합설계서.md §5 Phase 0`
-3. 회귀 테스트 작성: `tests/unit/test_select_ontology.py`
-   - 케이스 1: ontology가 0개일 때 → None 반환
-   - 케이스 2: ontology가 N개, LLM이 1~N 선택 → 해당 ontology 반환
-   - 케이스 3: ontology가 N개, LLM이 N+1 선택 → None 반환
-
-### 0.3: `convert_document.py` 다중 파일 처리 확장
-
-**근거**: 통합설계서 §5 Phase 0, OntoCast 분석 §13.1 (3번) / §21.1 (4번)
-
-**문제**: `convert_document()`가 "processing only one file"로 주석 처리되어 있고, 다중 파일 처리 시 마지막 파일 기준으로만 상태가 업데이트됨.
-
-**조치**:
-1. 입력 파일 목록을 순회하며 각 파일을 독립 `ContentUnit`으로 만들어 `AgentState.content_units`에 누적
-2. 동일 corpus 내 파일들이 함께 처리되도록 보장 (각 파일이 별도 doc IRI를 가짐)
-3. 회귀 테스트: 2개 PDF를 한 번에 처리 → 둘 다 처리되어야 함
-
-### 0.4: Robyn → FastAPI 재작성
-
-**근거**: 통합설계서 §11 (기술 스택), OntoCast 분석 §13 (API 명세)
-
-**조치**:
-1. `platform/api/main.py` 생성 (FastAPI app 인스턴스)
-2. OntoCast 분석 §13.1~§13.4의 4개 endpoint를 FastAPI로 동일 시맨틱 재작성:
-   - `GET /health`
-   - `GET /info`
-   - `POST /process` (JSON + multipart)
-   - `POST /flush` (관리자 권한 + confirmation token, 분석 §21.1 #6)
-3. OntoCast의 `ToolBox` 의존성 주입은 FastAPI `Depends`로 변환
-4. `uvicorn platform.api.main:app --reload`로 기동 가능해야 함
-
-**중요**: OntoCast 코어 모듈(`stategraph/`, `agent/`, `onto/`, `tool/`)은 **건드리지 않는다**. API 레이어만 재작성.
-
-### 0.5: Pydantic Settings 정리
-
-**근거**: 통합설계서 §5 Phase 0 (5번), OntoCast 분석 §15
-
-**조치**:
-1. `platform/config.py` 생성
-2. `pydantic-settings`의 `BaseSettings`로 `.env` 로딩
-3. **Phase 0에서는 filesystem 모드만 활성화** (Fuseki/Neo4j는 Phase 4에서):
-   - `STORAGE_BACKEND=filesystem` 강제
-   - Neo4j/Fuseki 변수가 채워져 있어도 무시
-4. OntoCast의 기존 `Config` 클래스는 우리 `Settings`에서 만들어 주입
-
-### 0.6: End-to-end 통합 테스트
-
-**근거**: 통합설계서 §5 Phase 0 Acceptance Gate
-
-**조치**:
-1. `vendored/ontocast/data/`의 예제 JSON 또는 PDF 1개를 fixture로 복사 → `tests/fixtures/`
-2. `tests/integration/test_phase0_e2e.py` 작성:
-   - FastAPI `TestClient`로 `/process` 호출
-   - 응답에 `ontology` TTL과 `facts` TTL 둘 다 포함
-   - `working_directory/`에 ontology/facts 파일 생성 확인
-   - BudgetTracker가 LLM call/triple count를 0보다 큰 값으로 기록
-
-### 0.7: Acceptance Gate 0 체크
-
-통합설계서 §5 Phase 0 Acceptance Gate의 4개 체크박스를 PR에 인용하며 모두 확인:
-
-- [ ] 단일 PDF 또는 JSON 입력 → ontology TTL + facts TTL이 filesystem에 생성됨
-- [ ] `/health`, `/info`, `/process` (FastAPI 버전) 정상 동작
-- [ ] BudgetTracker가 LLM call/triple count를 정확히 기록
-- [ ] LangGraph 워크플로우(CONVERT→CHUNK→...→SERIALIZE) 전 노드가 traceable
-
-## 작업 시 준수사항
-
-1. **PR 단위**: 위의 0.1~0.7 각각을 별도 PR/커밋으로 분리. 하나의 PR에 여러 단계를 섞지 않는다.
-2. **PR 설명에 근거 인용**: 예) "통합설계서 §5 Phase 0 (3번)에 따라 다중 파일 처리 확장. OntoCast 분석 §13.1 인용."
-3. **vendored/ 수정 시 라이선스 의무**:
-   - 수정한 파일 상단에 `# MODIFIED YYYY-MM-DD: <한 줄 설명>` 주석 추가
-   - 원본 LICENSE/NOTICE 파일은 절대 삭제하지 않는다
-4. **Phase 1로 넘어가지 말 것**: Acceptance Gate 0 통과 전까지 Trafilatura/Crawl4AI/Guardrails/Neo4j GraphRAG 의존성을 활성화하거나 import하지 않는다. (`pyproject.toml`에 명시되어 있더라도 코드에서 사용 금지)
-
-## Phase 1 이후 핸드오프
-
-Phase 0 완료 후, 본 폴더에 `PHASE1_NEXT_STEPS.md`를 작성하여 다음 작업자에게 동일한 형식으로 핸드오프한다. 통합설계서 §12 Phase 1 작업 단위(1.1~1.7)를 참조.

ontology_platform/docs/phases/PHASE1_ACCEPTANCE_GATE.md

@@ -1,33 +0,0 @@
-# Phase 1 Acceptance Gate 결과
-
-작성일: 2026-05-19
-
-범위: Trafilatura 기반 URL/HTML 입력 정렬, SourceDocument/EvidenceSpan 계약, URL 입력 API, fixture 기반 dedup 검증.
-
-## 결과 요약
-
-| # | Acceptance Gate 항목 | 상태 | 검증 방법 |
-|---|---|---|---|
-| 1 | URL/HTML 입력이 정제 문서로 변환됨 | 통과 | `tests/unit/test_web_extractor.py` |
-| 2 | source URL, title, language, content hash, fingerprint 보존 | 통과 | `test_extract_from_korean_html_preserves_document_contract` |
-| 3 | `SourceDocument`, `EvidenceSpan`, Content metadata 경계 연결 | 통과 | `test_extracted_content_maps_to_source_document_and_evidence_spans`, `test_content_unit.py` |
-| 4 | `/process/url`, `/api/v1/extract/url` URL 입력 API 제공 | 통과 | `tests/integration/test_url_ingest.py` |
-| 5 | 같은 본문 중복 입력은 fingerprint 기반으로 skip | 통과 | `test_same_clean_body_gets_same_hash_and_fingerprint`, `test_process_url_skips_duplicate_payload_by_fingerprint` |
-| 6 | Phase 0 회귀 없음 | 통과 | `python -m pytest tests/unit tests/integration -q` |
-
-## 검증 이력
-
-| 일자 | 검증자 | 결과 |
-|---|---|---|
-| 2026-05-19 | Codex | Phase 1 신규 테스트 7/7 통과. 전체 unit/integration 34/34 통과. |
-
-## 구현 메모
-
-- `ont_platform/core/extractors/web_extractor.py`는 Trafilatura 2.x `bare_extraction`을 사용하되, local HTML fixture에서 Trafilatura fingerprint가 비어 있는 경우 normalized text 기반 `sha1:` fingerprint를 생성한다.
-- `ont_platform/storage/models.py`의 SQLAlchemy 예약어 충돌을 피하기 위해 DB 컬럼명은 `metadata`로 유지하고 Python attribute는 `metadata_`로 정리했다.
-- `/process/url`, `/api/v1/process/url`, `/api/v1/extract/url`은 같은 Phase 1 응답 계약을 사용한다.
-- OntoCast vendored core는 수정하지 않았다.
-
-## 다음 Gate
-
-Phase 2는 Candidate Storage 및 Review 책임 경계를 다룬다. 진행 전 `PHASE_INDEX.md`에서 Phase 2 항목만 명시적으로 선택해 작업한다.

ontology_platform/docs/phases/PHASE1_NEXT_STEPS.md

@@ -1,162 +0,0 @@
-# Phase 1 — Trafilatura 통합 (다음 작업자 핸드오프)
-
-본 문서는 Phase 0이 완료된 시점에서 Phase 1 작업을 이어받는 AI 에이전트 또는 개발자가 즉시 작업을 시작하기 위한 핸드오프 노트다.
-
-## 시작 전 확인 사항
-
-- [ ] **Phase 0 Acceptance Gate**가 모두 ✅인가? [PHASE0_ACCEPTANCE_GATE.md](PHASE0_ACCEPTANCE_GATE.md) 참조. 통과 전에는 Phase 1 진행 금지.
-- [ ] `tests/unit`와 `tests/integration` 전체가 PASS인가?
-- [ ] git log에 Phase 0 commit들이 PR 단위로 분리되어 있는가? (0.1 vendored / 0.2 bug fix / 0.3 multi-file / 0.4 FastAPI / 0.5 config / 0.6 e2e tests / 0.7 gate)
-
-## Phase 1 목표
-
-URL이 입력일 때 원본 페이지에서 본문, 제목, 저자, 날짜, 언어, canonical URL을 정확히 뽑아 OntoCast의 `ContentUnit` metadata에 채워 넣는다.
-
-**근거**: 통합설계서 §5 Phase 1, Trafilatura 분석 §11~§17.
-
-**왜 Trafilatura를 가장 먼저 통합하는가**: 가장 작은 통합 — 단일 함수 호출(`bare_extraction`)만으로 끝남. 의존성도 명확하며 라이선스 동일 (Apache 2.0).
-
-## 작업 단위 (PR 분해)
-
-### 1.1: Trafilatura 의존성 활성화
-
-`pyproject.toml`에 이미 `trafilatura[all]>=2.0.0`이 명시되어 있다. 활성화 절차:
-
-```powershell
-pip install -e ".[dev]"  # 의존성 재설치 시 trafilatura 자동 설치
-python -c "import trafilatura; print(trafilatura.__version__)"
-```
-
-**확인**: `2.0.0` 이상이 출력되어야 한다.
-
-### 1.2: `web_extractor.py` 어댑터 작성
-
-**위치**: `platform/core/extractors/web_extractor.py`
-
-**근거**: Trafilatura 분석 §17의 `extract_for_ontology` 함수를 거의 그대로 사용.
-
-**필수 동작**:
-- 입력: `html: str`, `url: str`, `lang: str | None = None`
-- 출력: `ExtractedWebDocument` (dataclass)
-  - `url`, `title`, `author`, `date`, `sitename`, `description`
-  - `text` (정제 본문)
-  - `body_xml` (Trafilatura `Document.body`)
-  - `metadata` (raw dict)
-  - `fingerprint` (SimHash)
-- 실패 시 `None` 반환
-
-**호출 옵션** (Trafilatura 분석 §12 권장값 그대로):
-```python
-Extractor(
-    output_format="python",
-    url=url,
-    with_metadata=True,
-    comments=False,
-    tables=True,
-    formatting=True,
-    links=True,
-    images=True,
-    dedup=True,
-    lang=lang,
-)
-```
-
-### 1.3: `ContentUnit` 모델 확장
-
-OntoCast의 `vendored/ontocast/ontocast/onto/content_unit.py`를 **직접 수정하지 말고**, 우리 쪽에 wrapper 모델을 만든다.
-
-**위치**: `platform/models/content_unit.py`
-
-**필드** (통합설계서 §7.1 참조):
-- 기존 OntoCast 필드 (`text`, `index`, `doc_iri`, `graph`, `type`, `iri`) 유지/위임
-- 추가: `source_url`, `title`, `author`, `publish_date`, `language`, `sitename`, `fingerprint`, `content_hash`, `metadata`, `retrieved_at`, `extracted_by`
-
-**호환성**: 기존 OntoCast 코드가 받는 `ContentUnit`과 인터페이스 호환되도록 `as_ontocast()` 메서드 제공.
-
-### 1.4: OntoCast `ConverterTool` 분기 추가 (URL/HTML 입력)
-
-**문제**: OntoCast `ConverterTool`은 PDF/DOCX/MD만 처리. URL 또는 HTML 입력은 처리 못 함.
-
-**조치 옵션**:
-- **옵션 A (권장)**: OntoCast의 `convert_document.py` 모듈에 새 분기 추가 — `.html`, `.htm` 확장자 또는 `state.source_url`이 있으면 Trafilatura로 처리. **vendored 수정이지만 매우 작음**.
-- **옵션 B**: API 레이어(`platform/api/`)에서 입력이 URL이면 미리 fetch + Trafilatura 처리한 뒤 그 결과를 JSON envelope로 ToolBox에 넘김.
-
-**권장**: 옵션 B. vendored 수정을 늘리지 않고 platform 코드로 끝낼 수 있음.
-
-새 endpoint:
-- `POST /process/url` — body: `{"url": "...", "ontology_user_instruction": "...", ...}` — 내부적으로 `web_extractor`로 본문 추출 후 OntoCast workflow 실행.
-
-### 1.5: Fingerprint 기반 dedup
-
-- `tests/fixtures/`에 같은 본문의 두 URL fixture 만들기
-- `web_extractor` 결과의 `fingerprint`가 일치하면 OntoCast 처리 skip
-- 저장 위치: 일단 in-memory set (`platform/storage/dedup_cache.py`), Phase 2에서 Redis로 이전
-
-### 1.6: 한국어 페이지 3종 추출 검증
-
-**테스트 fixture 수집**:
-- 한국어 뉴스 1개 (예: 연합뉴스/조선/한겨레)
-- 한국어 블로그 1개 (예: 네이버 블로그)
-- 한국어 쇼핑 페이지 1개 (예: 쿠팡 상품 페이지)
-
-각각 raw HTML을 `tests/fixtures/korean/`에 저장 (실제 fetch는 운영 환경에서 한 번만, 그 결과를 fixture로 박제).
-
-**테스트**: `tests/integration/test_web_extractor_korean.py`
-- 본문 길이 > 200자
-- title 추출 성공
-- language 감지: `ko`
-- author 또는 date 중 하나 이상 추출
-
-### 1.7: Acceptance Gate 1 체크
-
-통합설계서 §5 Phase 1 Acceptance Gate 4개 항목:
-
-- [ ] URL 입력 → 본문/메타데이터가 정확히 추출되어 `ContentUnit`에 저장됨
-- [ ] 한국어 뉴스/블로그/쇼핑 페이지 각각 1개씩 본문 추출 정확도 수동 검증
-- [ ] 동일 URL 재입력 시 fingerprint 기반 dedup으로 skip
-- [ ] Phase 0의 모든 기능이 여전히 정상 동작 (회귀 없음)
-
-Phase 0의 `tests/unit/`, `tests/integration/` 전체가 여전히 PASS여야 함.
-
-## Phase 1에서 만들 새 산출물
-
-```
-platform/
-  core/
-    extractors/
-      web_extractor.py         ← 1.2
-  models/
-    content_unit.py            ← 1.3
-  storage/
-    dedup_cache.py             ← 1.5
-  api/
-    routes/
-      url_ingest.py            ← 1.4 (POST /process/url)
-tests/
-  fixtures/
-    korean/                    ← 1.6
-      news_yonhap.html
-      blog_naver.html
-      shop_coupang.html
-  unit/
-    test_web_extractor.py      ← 1.2
-    test_dedup_cache.py        ← 1.5
-  integration/
-    test_url_ingest.py         ← 1.4
-    test_web_extractor_korean.py ← 1.6
-docs/
-  phases/
-    PHASE1_ACCEPTANCE_GATE.md  ← 1.7 (PHASE0과 동일 형식)
-    PHASE2_NEXT_STEPS.md       ← 다음 작업자에게 넘김
-```
-
-## 작업 시 준수사항 (PHASE0과 동일)
-
-1. **PR 단위 분리**: 1.1~1.7 각각 별도 PR/커밋.
-2. **PR 설명에 근거 인용**: 예) "통합설계서 §5 Phase 1 (1.2)에 따라 Trafilatura adapter 작성. Trafilatura 분석 §17 인용."
-3. **vendored/ontocast/** 수정 최소화. 본 Phase에서는 옵션 B 사용 시 vendored 수정 0건이 목표.
-4. **Phase 2로 넘어가지 말 것**: Acceptance Gate 1 통과 전까지 Crawl4AI 의존성을 코드에서 import하지 않는다.
-
-## Phase 2 이후 핸드오프
-
-Phase 1 완료 후 다음 작업자에게 동일한 형식의 `PHASE2_NEXT_STEPS.md`를 작성한다. 통합설계서 §12 Phase 2 작업 단위(2.1~2.8)를 참조.

ontology_platform/docs/phases/PHASE2_ACCEPTANCE_GATE.md

@@ -1,35 +0,0 @@
-# Phase 2 Acceptance Gate 결과
-
-작성일: 2026-05-19
-
-범위: Candidate Storage 및 Review 책임 경계. Lightweight/OntoCast 후보 저장 경로, review 상태 전이, audit trail, evidence 기반 promotion gate.
-
-## 결과 요약
-
-| # | Acceptance Gate 항목 | 상태 | 검증 방법 |
-|---|---|---|---|
-| 1 | extraction 결과가 candidate로 저장됨 | 통과 | `tests/unit/test_candidate_repository.py` |
-| 2 | lightweight와 OntoCast 저장 경로가 분리됨 | 통과 | `test_repository_saves_lightweight_candidates_with_evidence`, `test_repository_saves_ontocast_candidates_on_separate_source_path` |
-| 3 | 승인/반려/자동승인 상태 변경 이력이 남음 | 통과 | `tests/unit/test_review_service.py` |
-| 4 | evidence 없는 항목은 승인 및 graph commit 대상이 아님 | 통과 | `test_candidate_without_evidence_cannot_be_approved`, `test_promotion_plan_blocks_approved_candidate_without_evidence` |
-| 5 | Review API가 ingest/list/detail/approve/reject/promote 흐름을 제공함 | 통과 | `tests/integration/test_review_api.py` |
-| 6 | Phase 0-1 회귀 없음 | 통과 | `python -m pytest tests/unit tests/integration -q` |
-
-## 검증 이력
-
-| 일자 | 검증자 | 결과 |
-|---|---|---|
-| 2026-05-19 | Codex | Phase 2 신규 테스트 9/9 통과. 전체 unit/integration 43/43 통과. 변경 파일 대상 ruff 통과. |
-
-## 구현 메모
-
-- `CandidateEntity`, `CandidateRelation`에 `source_type`, `created_by`, `validation_passed`, `promoted_at`을 추가해 review queue 계약을 명확히 했다.
-- `ReviewDecision`으로 상태 변경 audit trail을 남긴다.
-- `CandidateRepository.save_lightweight_result()`와 `save_ontocast_result()`를 분리해 두 입력 경로가 같은 candidate contract로 정규화되되, 출처는 유지된다.
-- `ReviewService`는 `pending -> approved/rejected/auto_approved`, `approved/auto_approved -> rejected`만 허용한다.
-- `CandidatePromotionService`는 `approved` 또는 `auto_approved`이면서 evidence가 실제 존재하는 후보만 commit plan에 포함한다.
-- OntoCast vendored core는 수정하지 않았다.
-
-## 다음 Gate
-
-Phase 3은 Crawl4AI 수집 계층 및 Job Orchestration이다. 진행 전 `PHASE_INDEX.md`에서 Phase 3 항목만 명시적으로 선택해 작업한다.

ontology_platform/docs/phases/PHASE2_NEXT_STEPS.md

@@ -1,27 +0,0 @@
-# Phase 2 — Candidate Storage 및 Review 책임 경계
-
-본 문서는 Phase 1 완료 후 다음 작업자가 Phase 2를 시작할 때 참고할 핸드오프 노트다. 자동으로 Phase 2를 진행하지 않는다.
-
-## 시작 전 확인
-
-- `PHASE_INDEX.md`에서 Phase 2 진행 요청이 명시되어 있는지 확인한다.
-- `PHASE1_ACCEPTANCE_GATE.md`의 unit/integration 34/34 통과 상태를 기준선으로 삼는다.
-- vendored OntoCast core는 계속 직접 수정하지 않는다.
-
-## Phase 2 목표
-
-추출 결과를 바로 확정 그래프로 보내지 않고, 사람이 검토할 수 있는 candidate/review queue 계약으로 분리한다. SourceDocument와 EvidenceSpan이 없는 후보는 확정 graph로 들어가지 못하게 한다.
-
-## 작업 범위
-
-1. `storage/models.py`의 `CandidateEntity`, `CandidateRelation`을 review queue 계약으로 확정한다.
-2. OntoCast 결과와 lightweight extraction 결과의 저장 경로를 분리한다.
-3. `pending`, `approved`, `auto_approved`, `rejected` 상태 전이 규칙을 문서와 테스트로 고정한다.
-4. evidence 없는 후보가 확정 graph로 승격되지 못하도록 validation boundary를 둔다.
-
-## 권장 테스트
-
-- 후보 생성 시 `document_id`와 `evidence_ids`가 필수로 연결되는지 검증한다.
-- 승인/반려/자동승인 상태 전이가 허용된 경로로만 움직이는지 검증한다.
-- evidence 없는 entity/relation이 commit 단계에 도달하지 못하는지 검증한다.
-- Phase 1 URL/HTML ingestion 테스트가 계속 통과하는지 회귀 검증한다.

ontology_platform/docs/phases/PHASE3_NEXT_STEPS.md

@@ -1,28 +0,0 @@
-# Phase 3 — Crawl4AI 수집 계층 및 Job Orchestration
-
-본 문서는 Phase 2 완료 후 다음 작업자가 Phase 3을 시작할 때 참고할 핸드오프 노트다. 자동으로 Phase 3을 진행하지 않는다.
-
-## 시작 전 확인
-
-- `PHASE_INDEX.md`에서 Phase 3 진행 요청이 명시되어 있는지 확인한다.
-- `PHASE2_ACCEPTANCE_GATE.md`의 unit/integration 43/43 통과 상태를 기준선으로 삼는다.
-- 수집 계층은 SourceDocument 생성 전 단계까지만 책임진다. Candidate 저장과 Review Queue는 Phase 2 계약을 사용한다.
-- vendored OntoCast core는 계속 직접 수정하지 않는다.
-
-## Phase 3 목표
-
-정적 URL 1건 처리를 넘어 동적 페이지와 대량 수집을 job 단위로 관리한다. Crawl4AI는 acquisition adapter로 감싸고, 본문 정제는 Phase 1 Trafilatura adapter, 후보 저장은 Phase 2 Review Queue로 넘긴다.
-
-## 작업 범위
-
-1. `crawl4ai_adapter.py`를 동적/대량 수집 adapter로 제한한다.
-2. crawler profile, robots policy, cache policy를 설정 기반으로 분리한다.
-3. Job 상태 모델과 progress API/WebSocket 경계를 정리한다.
-4. 수집 결과를 Trafilatura 후처리와 SourceDocument 저장으로 연결한다.
-
-## 권장 테스트
-
-- 정적 HTML/동적 페이지 profile이 같은 SourceDocument 계약으로 이어지는지 검증한다.
-- robots/cache policy가 설정값에 따라 선택되는지 검증한다.
-- job 상태가 pending/running/completed/failed로 전이되는지 검증한다.
-- Phase 1 extraction 및 Phase 2 review queue 테스트가 계속 통과하는지 회귀 검증한다.

ontology_platform/docs/phases/PHASE_INDEX.md

@@ -1,100 +1,101 @@
-# PHASE INDEX - ontology_platform engine-respect roadmap
+# PHASE INDEX - Semantic Page Classification Layer
 
-?묒꽦?? 2026-05-19
+작성일: 2026-05-22
 
-踰붿쐞: `ontology_platform` ?꾩슜. `crawler_platform`?€ ?대쾲 ?묒뾽 踰붿쐞?먯꽌 ?쒖쇅?쒕떎.
+범위: `ontology_platform`의 `crawler_platform.app.core.crawler.page_classifier` 및 page classification과 직접 연결된 crawler/extractor 흐름.
 
-湲곗? 臾몄꽌:
+기준 문서:
-- `ontology_platform/docs/?듯빀?ㅺ퀎??md`
+- `README.md`
+- `docs/PHASE_PLANNING.md`
 - `ontology_platform/README.md`
-- `ontology_platform/docs/phases/PHASE0_ACCEPTANCE_GATE.md`
+- `ontology_platform/docs/semantic_page_classification_codex_spec.md`
-- `ontology_platform/docs/phases/PHASE1_NEXT_STEPS.md`
-- `ontology_platform/docs/phases/PHASE1_ACCEPTANCE_GATE.md`
-- `ontology_platform/docs/phases/PHASE2_ACCEPTANCE_GATE.md`
 
-?듭떖 ?먯튃:
+핵심 원칙:
-- OntoCast??Base ?붿쭊?쇰줈 議댁쨷?쒕떎.
+- 기존 엔진을 폐기하거나 대규모로 교체하지 않는다.
-- vendored OntoCast 肄붿뼱???듯빀?ㅺ퀎?쒓? ?덉슜??踰붿쐞 ?몄뿉???섏젙?섏? ?딅뒗??
+- 기존 `classify_page(...) -> str` 호출부가 깨지지 않도록 legacy compatibility를 유지한다.
-- Trafilatura, Crawl4AI, Guardrails, Neo4j GraphRAG??吏곸젒 ?ш뎄?꾪븯吏€ ?딄퀬 ?뉗? adapter/facade濡?媛먯떬??
+- 기존 `ProductPage`, `CategoryPage`, `SearchPage`, `BoardPage`, `BrandStoryPage`, `UnknownPage` 문자열은 alias 또는 compatibility mapping으로 유지한다.
-- Firecrawl, OpenDeepResearcher 肄붾뱶???ы븿?섏? ?딅뒗??
+- URL substring 중심 if-return 확장이 아니라 signal extraction -> evidence scoring -> classification result -> analyze strategy -> LLM policy 구조로 확장한다.
-- Acceptance Gate瑜??듦낵?섍린 ???ㅼ쓬 ?듯빀?쇰줈 ?섏뼱媛€吏€ ?딅뒗??
+- protected page는 안전하게 skip하고, UnknownPage는 evidence와 confidence를 남긴다.
+- pytest 또는 현재 프로젝트 테스트 명령으로 회귀 검증한다.
 
 ---
 
-PHASE 0. ?붿쭊 寃쎄퀎 媛먯궗 諛?Phase Gate 蹂듦뎄
+PHASE 1. 현재 흐름 기준선 고정 및 영향 범위 정리
-FILE: ./26_05_19_engine_respect_plan/phase_00_001_engine_boundary_gate.md
+FILE: ./26_05_22_semantic_page_classification/phase_01_001_current_flow_boundary.md
 
-1) ?꾩옱 `ont_platform` 紐⑤뱢??Base/Adapter/Draft/Excluded 梨낆엫?쇰줈 遺꾨쪟 [?꾨즺]
+1) `page_classifier.py`의 현재 public API와 legacy page_type 문자열 목록 고정 [완료]
-2) Phase 0?먯꽌 誘몃옒 Phase ?섏〈?깆씠 import?섏뼱 ???쒖옉??源⑥? ?딅룄濡?寃뚯씠???뺣━ [?꾨즺]
+2) `should_analyze_page()` 호출부와 crawler의 `classify_page()` 호출 위치 문서화 [완료]
-3) Phase 0 unit/integration 寃€利??덉감 怨좎젙 [?꾨즺]
+3) Extractor/HybridExtractor에서 page_type과 LLM skip 정책이 연결되는 흐름 정리 [완료]
-4) `PHASE0_ACCEPTANCE_GATE.md` 媛깆떊 湲곗? ?뺣━ [?꾨즺]
+4) 기존 page_type 문자열을 기대하는 테스트, config, adapter, ontology rule 경로 목록화 [완료]
 
 ---
 
-PHASE 1. Trafilatura 湲곕컲 URL/HTML ?낅젰 ?뺣젹
+PHASE 2. Taxonomy와 Classification Result 모델 추가
-FILE: ./26_05_19_engine_respect_plan/phase_01_001_trafilatura_ingestion.md
+FILE: ./26_05_22_semantic_page_classification/phase_02_001_taxonomy_result_model.md
 
-1) `web_extractor.py`瑜?Trafilatura adapter 梨낆엫?쇰줈 ?뺣━ [?꾨즺]
+1) PageDomain/PageArchetype/PageType/EntityType/ActionIntent/GraphRole/AnalyzeStrategy/LLMPolicy 상수 또는 enum 추가 [완료]
-2) `SourceDocument`, `EvidenceSpan`, Content metadata ?€??寃쎄퀎 ?곌껐 [?꾨즺]
+2) `EvidenceItem`, `PageClassificationResult` dataclass 추가 [완료]
-3) `/process/url` ?먮뒗 ?숇벑??URL ?낅젰 API ?ㅺ퀎 [?꾨즺]
+3) legacy alias 및 normalize helper 추가 [완료]
-4) ?쒓뎅??URL/HTML fixture 湲곕컲 異붿텧 ?뚯뒪?몄? dedup 湲곗? ?묒꽦 [?꾨즺]
+4) 기존 `classify_page()` 문자열 반환 호환을 유지하면서 semantic result API 추가 [완료]
 
 ---
 
-PHASE 2. Candidate Storage 諛?Review 梨낆엫 寃쎄퀎
+PHASE 3. Raw Snapshot 및 Signal Extraction 레이어 추가
-FILE: ./26_05_19_engine_respect_plan/phase_02_001_candidate_review_boundary.md
+FILE: ./26_05_22_semantic_page_classification/phase_03_001_signal_extraction_layer.md
 
-1) `storage/models.py`???꾨낫 紐⑤뜽???뺤떇 Review Queue 怨꾩빟?쇰줈 ?뺤젙 [?꾨즺]
+1) `RawPageSnapshot`와 `PageSignals` 모델 추가 [완료]
-2) OntoCast 寃곌낵?€ lightweight extraction 寃곌낵???€??寃쎈줈 遺꾨━ [?꾨즺]
+2) JSON-LD, OpenGraph, Twitter Card, meta, headings, links, forms, buttons, inputs 추출 [완료]
-3) ?뱀씤/諛섎젮/?먮룞?뱀씤 ?곹깭 ?꾩씠 洹쒖튃 ?뺤쓽 [?꾨즺]
+3) commerce/listing/editorial/community/docs/corporate/protected/system signal 추출 [완료]
-4) evidence ?녿뒗 ?꾨낫媛€ ?뺤젙 graph濡??ㅼ뼱媛€吏€ 紐삵븯寃?李⑤떒 [?꾨즺]
+4) HTML 일부가 깨지거나 필드가 누락되어도 예외 없이 빈 값으로 처리 [완료]
 
 ---
 
-PHASE 3. Crawl4AI ?섏쭛 怨꾩링 諛?Job Orchestration
+PHASE 4. Evidence Scoring 기반 Semantic Classification 구현
-FILE: ./26_05_19_engine_respect_plan/phase_03_001_crawl4ai_acquisition_jobs.md
+FILE: ./26_05_22_semantic_page_classification/phase_04_001_evidence_scoring_classifier.md
 
-1) `crawl4ai_adapter.py`瑜??숈쟻/?€???섏쭛 adapter濡??쒗븳 [?꾨즺]
+1) 주요 page type별 scoring function과 evidence recording 구조 추가 [완료]
-2) crawler profile, robots policy, cache policy瑜??ㅼ젙 湲곕컲?쇰줈 遺꾨━ [?꾨즺]
+2) 최소 20개 semantic page type 분류 구현 [완료]
-3) Job ?곹깭 紐⑤뜽怨?progress API/WebSocket 寃쎄퀎 ?뺣━ [?꾨즺]
+3) confidence, alternatives, secondary_page_types 산출 [완료]
-4) Trafilatura ?꾩쿂由ъ? SourceDocument ?€?μ쑝濡??곌껐 [?꾨즺]
+4) low confidence 또는 모호한 결과를 evidence 포함 UnknownPage로 처리 [완료]
 
 ---
 
-PHASE 4. Guardrails Validation Gate
+PHASE 5. Analyze Strategy 및 LLM Policy 분리
-FILE: ./26_05_19_engine_respect_plan/phase_04_001_guardrails_validation_gate.md
+FILE: ./26_05_22_semantic_page_classification/phase_05_001_analysis_llm_policy.md
 
-1) `core/validation`??Pydantic lightweight?€ Guardrails facade濡?遺꾨━ [?꾨즺]
+1) PageClassificationResult 기반 `decide_analyze_strategy()` 추가 [완료]
-2) OntoCast LLM 異쒕젰 ?섑븨 吏€?먯쓣 vendored ?섏젙 ?놁씠 ?곗꽑 ?ㅺ퀎 [?꾨즺]
+2) PageClassificationResult 기반 `decide_llm_policy()` 추가 [완료]
-3) schema violation, endpoint missing, confidence range ?뚯뒪???묒꽦 [?꾨즺]
+3) `should_analyze_page(result_or_page_type, analyze_page_types=None)` compatibility 구현 [완료]
-4) Guard ?ㅽ뙣 寃곌낵瑜?candidate/review issue濡??€??[?꾨즺]
+4) Category/Search/Board 계열을 무조건 skip하지 않고 strategy 기반으로 처리 [완료]
+5) Login/Checkout/Payment/Captcha/AccessDenied 계열은 SkipProtected/Skip 정책으로 처리 [완료]
 
 ---
 
-PHASE 5. Neo4j Projection 諛?GraphRAG 寃€??FILE: ./26_05_19_engine_respect_plan/phase_05_001_neo4j_projection_graphrag.md
+PHASE 6. Crawler, Cleaner, Extractor, Discovery/Relevance 통합
+FILE: ./26_05_22_semantic_page_classification/phase_06_001_pipeline_integration.md
 
-1) RDF/Fuseki瑜?canonical store, Neo4j瑜?projection/search store濡?怨좎젙 [?꾨즺]
+1) `site_crawler.py`와 `pipeline.py` metadata에 semantic classification payload 저장 [완료]
-2) `core/graph` 湲곗〈 紐⑤뱢??projection/search adapter 梨낆엫?쇰줈 ?щ텇瑜?[?꾨즺]
+2) `ExtractionPageContext` 또는 metadata를 통해 analyze_strategy/llm_policy 전달 [완료]
-3) read-only Text2Cypher?€ vector/hybrid retriever API ?ㅺ퀎 [?꾨즺]
+3) `HybridExtractor`가 LLMPolicy를 우선 사용하고 legacy page_type fallback을 유지하도록 수정 [완료]
-4) provenance媛€ search result源뚯? ?댁뼱吏€??寃€利?湲곗? ?묒꽦 [?꾨즺]
+4) `page_cleaner.py`, `domain_discovery.py`, `relevance_engine.py`의 legacy page_type 기대 경로와 신규 semantic type을 호환 [완료]
 
 ---
 
-PHASE 6. Maintenance Loop 諛??댁쁺 湲곕뒫 ?뺣━
+PHASE 7. Unknown Pattern 저장 기반 추가
-FILE: ./26_05_19_engine_respect_plan/phase_06_001_maintenance_loop_operations.md
+FILE: ./26_05_22_semantic_page_classification/phase_07_001_unknown_pattern_storage.md
 
-1) Knowledge Agent??肄붾뱶媛€ ?꾨땲???꾨＼?꾪듃/?뚰겕?뚮줈???⑦꽩留?李⑥슜 [?꾨즺]
+1) UnknownPage 또는 low confidence 페이지의 evidence payload 정의 [완료]
-2) Analyst/Researcher/Curator/Auditor/Fixer/Advisor 梨낆엫 ?뺤쓽 [?꾨즺]
+2) text/html/link/schema/button/form summary와 fingerprint hook 추가 [완료]
-3) `auth`, `audit`, `billing`, `realtime` 珥덉븞 紐⑤뱢???댁쁺 寃쎄퀎 ?뺣━ [?꾨즺]
+3) DB schema 변경 없이 metadata_json에 저장 가능한 초기 구조 구현 [완료]
-4) destructive fix???щ엺 ?뱀씤 寃뚯씠?몃? 諛섎뱶???듦낵?섎룄濡??ㅺ퀎 [?꾨즺]
+4) 향후 clustering/embedding 확장을 위한 hook만 추가하고 실제 clustering은 이번 범위에서 제외 [완료]
 
 ---
 
-PHASE 7. Hybrid Rule + LLM Extraction
+PHASE 8. 테스트 Fixture 및 회귀 검증
-FILE: ./26_05_19_engine_respect_plan/phase_07_001_hybrid_rule_llm_extraction.md
+FILE: ./26_05_22_semantic_page_classification/phase_08_001_tests_regression.md
 
-1) rule baseline, LLM extraction, fallback, validation, Review UI 흐름을 기준선으로 고정 [신규]
+1) 최소 10개 이상의 HTML fixture 추가 [완료]
-2) `rule_only`, `llm_only`, `hybrid`, `compare` extraction mode 계약 정의 [신규]
+2) ProductDetailPage, CategoryListingPage, SearchResultsPage, ArticlePage, QAPage, FAQPage, ForumThreadPage, DocumentationPage, JobPostingPage, LoginPage, CheckoutPage, TermsPage, SitemapPage, UnknownPage 단위 테스트 추가 [완료]
-3) product backend에 명시적 HybridExtractor와 rule/LLM agreement metadata 추가 [신규]
+3) legacy `classify_page()`와 `should_analyze_page()` 호환성 테스트 추가 [완료]
-4) confidence breakdown에 rule agreement와 conflict/review 정책 반영 [신규]
+4) HybridExtractor LLMPolicy 회귀 테스트 추가 [완료]
-5) Crawl/Research UI에서 mode/provider/model/base URL 선택 지원 [신규]
+5) pytest 또는 현재 프로젝트 테스트 명령 실행 및 결과 기록 [완료]

ontology_platform/ont_platform/api/main.py

@@ -156,7 +156,7 @@ def create_app() -> FastAPI:
 
     # ─── /health ──────────────────────────────────────────────────────
     @app.get("/health", tags=["meta"])
-    async def health() -> JSONResponse:
+    async def health(request: Request) -> JSONResponse:
         """Liveness check for the HTTP service and optional LLM readiness."""
         settings = platform_config.load_settings()
         if _startup_error:
@@ -172,7 +172,7 @@ def create_app() -> FastAPI:
                 },
             )
 
-        ctx = get_app_context()
+        ctx = _request_app_context(request)
         if ctx.tools.llm is None:
             return JSONResponse(
                 status_code=503,
@@ -192,13 +192,13 @@ def create_app() -> FastAPI:
 
     # ─── /info ────────────────────────────────────────────────────────
     @app.get("/info", tags=["meta"])
-    async def info() -> JSONResponse:
+    async def info(request: Request) -> JSONResponse:
         """Service-level capabilities (mirrors OntoCast /info semantics)."""
         settings = platform_config.load_settings()
         phase = int(settings.phase)
         storage_backend = settings.storage_backend
         if not _startup_error:
-            ctx = get_app_context()
+            ctx = _request_app_context(request)
             phase = int(ctx.settings.phase)
             storage_backend = ctx.settings.storage_backend
 
@@ -449,6 +449,13 @@ def create_app() -> FastAPI:
     return app
 
 
+def _request_app_context(request: Request) -> AppContext:
+    override = request.app.dependency_overrides.get(get_app_context)
+    if override is not None:
+        return override()
+    return get_app_context()
+
+
 # Top-level instance for `uvicorn platform.api.main:app`.
 app = create_app()
 

ontology_platform/ont_platform/api/product_backend.py

@@ -82,6 +82,6 @@ def _remove_route(app: FastAPI, path: str, methods: set[str]) -> None:
         for route in app.router.routes
         if not (
             getattr(route, "path", None) == path
-            and set(getattr(route, "methods", set())) == methods
+            and methods.issubset(set(getattr(route, "methods", set())))
         )
     ]

ontology_platform/tests/unit/test_phase7_hybrid_extraction.py

@@ -1,6 +1,7 @@
 from crawler_platform.app.config.loader import ProjectConfig
+from crawler_platform.app.api.routes import extraction_log_summary
 from crawler_platform.app.core.extractor.base import ExtractedClaim, ExtractionBundle, ExtractionPageContext
-from crawler_platform.app.core.extractor.hybrid import HybridExtractor, mark_bundle, merge_bundles
+from crawler_platform.app.core.extractor.hybrid import HybridExtractor, fallback_bundle, mark_bundle, merge_bundles
 from crawler_platform.app.core.extractor.validation import validate_extraction_bundle
 
 
@@ -142,3 +143,61 @@ def test_hybrid_smart_routing_skips_llm_for_category_page() -> None:
     assert bundle.raw_output["llm_skipped"] is True
     assert bundle.raw_output["effective_extraction_mode"] == "rule_only"
     assert "CategoryPage" in bundle.raw_output["llm_skip_reason"]
+
+
+def test_rule_only_mode_records_rule_llm_summary_counts() -> None:
+    rule_bundle = ExtractionBundle(
+        claims=[
+            ExtractedClaim("A", "Thing", "color", object_value="red"),
+            ExtractedClaim("A", "Thing", "size", object_value="large"),
+        ]
+    )
+
+    class FixedRuleExtractor:
+        def extract(self, page_text, project_config):  # noqa: ANN001
+            return rule_bundle
+
+        def extract_from_context(self, context, project_config):  # noqa: ANN001
+            return rule_bundle
+
+    extractor = HybridExtractor("generic", mode="rule_only")
+    extractor.rule_extractor = FixedRuleExtractor()
+
+    bundle = extractor.extract("Alpha color red size large", project_config())
+
+    assert bundle.raw_output["rule_claim_count"] == 2
+    assert bundle.raw_output["llm_claim_count"] == 0
+    assert bundle.raw_output["comparison"]["rule_only"] == 2
+
+
+def test_fallback_records_rule_counts_and_comparison() -> None:
+    rule_bundle = ExtractionBundle(
+        claims=[
+            ExtractedClaim("A", "Thing", "color", object_value="red"),
+            ExtractedClaim("A", "Thing", "size", object_value="large"),
+        ]
+    )
+    mark_bundle(rule_bundle, source="rule", mode="hybrid")
+
+    bundle = fallback_bundle(rule_bundle, HybridExtractor("generic", mode="hybrid"), RuntimeError("boom"))
+
+    assert bundle.raw_output["rule_claim_count"] == 2
+    assert bundle.raw_output["llm_claim_count"] == 0
+    assert bundle.raw_output["comparison"]["rule_only"] == 2
+
+
+def test_extraction_log_summary_backfills_legacy_rule_only_counts() -> None:
+    summary = extraction_log_summary(
+        {
+            "validation": {"rejected_claim_count": 4},
+            "candidate_claims": [
+                {"metadata": {"agreement": "rule_only", "extraction_source": "rule"}},
+                {"metadata": {"agreement": "rule_only", "extraction_source": "rule"}},
+            ]
+        }
+    )
+
+    assert summary["rule_claim_count"] == 2
+    assert summary["llm_claim_count"] == 0
+    assert summary["comparison"]["rule_only"] == 2
+    assert summary["comparison"]["rejected_by_validation"] == 4

ontology_platform/web/frontend/src/lib/api/crawl.ts

@@ -86,6 +86,26 @@ export function isCrawlTerminal(status: string): boolean {
   return TERMINAL_CRAWL_STATUSES.has(status);
 }
 
+export const extractorModelsResponseSchema = z.object({
+  ok: z.boolean(),
+  error: z.string().optional(),
+  models: z
+    .array(
+      z
+        .object({
+          id: z.string(),
+          owned_by: z.string().nullish(),
+        })
+        .passthrough(),
+    )
+    .optional()
+    .default([]),
+});
+
+export type ExtractorModelsResponse = z.infer<
+  typeof extractorModelsResponseSchema
+>;
+
 export const crawlApi = {
   startByProject: (body: StartSiteCrawlRequest) =>
     apiClient.post("/crawl-site/by-project", crawlJobSchema, body),
@@ -99,4 +119,9 @@ export const crawlApi = {
       `/crawl-site/jobs/${encodeURIComponent(jobId)}/cancel`,
       crawlJobSchema,
     ),
+  listExtractorModels: (provider: string, baseUrl?: string | null) =>
+    apiClient.post("/extractors/models", extractorModelsResponseSchema, {
+      provider,
+      base_url: baseUrl || null,
+    }),
 };

ontology_platform/web/frontend/src/pages/CrawlPage.tsx

@@ -1,4 +1,4 @@
-import { useState } from "react";
+import { useEffect, useState } from "react";
 import { useNavigate, useParams } from "react-router-dom";
 import { useTranslation } from "react-i18next";
 import { useForm } from "react-hook-form";
@@ -36,7 +36,7 @@ import {
   useCrawlJob,
   useStartSiteCrawl,
 } from "@/hooks/useCrawl";
-import { isCrawlTerminal } from "@/lib/api/crawl";
+import { crawlApi, isCrawlTerminal } from "@/lib/api/crawl";
 
 const startCrawlSchema = z.object({
   source_name: z.string().min(1, "소스를 선택하세요"),
@@ -150,7 +150,53 @@ export default function CrawlPage() {
   const sources = project?.sources ?? [];
   const sourceName = watch("source_name");
   const extractionMode = watch("extraction_mode");
+  const provider = watch("extractor_provider");
+  const baseUrl = watch("extractor_base_url");
   const usesLlm = extractionMode !== "rule_only";
+
+  const [loadedModelHint, setLoadedModelHint] = useState<string>("");
+  const [modelLookupStatus, setModelLookupStatus] = useState<
+    "idle" | "loading" | "ok" | "error"
+  >("idle");
+  const [modelLookupError, setModelLookupError] = useState<string>("");
+  useEffect(() => {
+    if (!usesLlm || provider !== "lm_studio") {
+      setLoadedModelHint("");
+      setModelLookupStatus("idle");
+      setModelLookupError("");
+      return;
+    }
+    let cancelled = false;
+    setModelLookupStatus("loading");
+    setModelLookupError("");
+    (async () => {
+      try {
+        const res = await crawlApi.listExtractorModels(provider, baseUrl);
+        if (cancelled) return;
+        if (!res.ok) {
+          setModelLookupStatus("error");
+          setModelLookupError(res.error || "모델 조회 실패");
+          return;
+        }
+        const first = res.models?.[0]?.id;
+        if (!first) {
+          setModelLookupStatus("error");
+          setModelLookupError("로드된 모델이 없습니다");
+          return;
+        }
+        setLoadedModelHint(first);
+        setValue("extractor_model", first, { shouldDirty: false });
+        setModelLookupStatus("ok");
+      } catch (e) {
+        if (cancelled) return;
+        setModelLookupStatus("error");
+        setModelLookupError((e as Error).message);
+      }
+    })();
+    return () => {
+      cancelled = true;
+    };
+  }, [provider, baseUrl, usesLlm, setValue]);
   const selectedSource = sources.find((s) => s.name === sourceName);
   const progress = job?.progress;
   const visited = progress?.visited_count ?? 0;
@@ -385,9 +431,30 @@ export default function CrawlPage() {
                       <Label htmlFor="extractor_model">Model</Label>
                       <Input
                         id="extractor_model"
-                        placeholder="deepseek-r1-distill-qwen-7b"
+                        placeholder={
+                          loadedModelHint || "deepseek-r1-distill-qwen-7b"
+                        }
                         {...register("extractor_model")}
                       />
+                      {provider === "lm_studio" &&
+                        modelLookupStatus === "loading" && (
+                          <p className="text-xs text-muted-foreground">
+                            LM Studio 로드 모델 확인 중...
+                          </p>
+                        )}
+                      {provider === "lm_studio" &&
+                        modelLookupStatus === "ok" &&
+                        loadedModelHint && (
+                          <p className="text-xs text-muted-foreground">
+                            LM Studio 로드 모델: {loadedModelHint}
+                          </p>
+                        )}
+                      {provider === "lm_studio" &&
+                        modelLookupStatus === "error" && (
+                          <p className="text-xs text-destructive">
+                            LM Studio 모델 조회 실패: {modelLookupError}
+                          </p>
+                        )}
                     </div>
                     <div className="space-y-1.5">
                       <Label htmlFor="extractor_base_url">Base URL</Label>

restart_all_servers.bat

@@ -288,6 +288,32 @@ function Stop-ExistingServers {
     Start-Sleep -Milliseconds 800
 }
 
+function Quote-ProcessArguments {
+    param([string[]] $Arguments)
+
+    if (-not $Arguments) {
+        return @()
+    }
+    $quoted = @()
+    foreach ($arg in $Arguments) {
+        if ($null -eq $arg) {
+            continue
+        }
+        $text = [string] $arg
+        if ($text.Length -eq 0) {
+            $quoted += '""'
+            continue
+        }
+        if ($text -match '\s' -and -not ($text.StartsWith('"') -and $text.EndsWith('"'))) {
+            $escaped = $text -replace '"', '\"'
+            $quoted += '"' + $escaped + '"'
+        } else {
+            $quoted += $text
+        }
+    }
+    return $quoted
+}
+
 function Invoke-LoggedCommand {
     param(
         [string] $FilePath,
@@ -300,10 +326,12 @@ function Invoke-LoggedCommand {
     $stderrPath = Join-Path $logDir ($LogName -replace "\.log$", ".stderr.log")
     Write-Step ("Running {0} {1}" -f (Split-Path -Leaf $FilePath), ($Arguments -join " "))
 
+    $quotedArgs = Quote-ProcessArguments -Arguments $Arguments
+
     Remove-Item -LiteralPath $logPath, $stderrPath -Force -ErrorAction SilentlyContinue
     $proc = Start-Process `
         -FilePath $FilePath `
-        -ArgumentList $Arguments `
+        -ArgumentList $quotedArgs `
         -WorkingDirectory $WorkingDirectory `
         -RedirectStandardOutput $logPath `
         -RedirectStandardError $stderrPath `
@@ -338,10 +366,12 @@ function Start-LoggedServer {
         [Environment]::SetEnvironmentVariable($key, [string] $Environment[$key], "Process")
     }
 
+    $quotedArgs = Quote-ProcessArguments -Arguments $Arguments
+
     try {
         $proc = Start-Process `
             -FilePath $FilePath `
-            -ArgumentList $Arguments `
+            -ArgumentList $quotedArgs `
             -WorkingDirectory $WorkingDirectory `
             -RedirectStandardOutput $stdout `
             -RedirectStandardError $stderr `