diff --git a/ontology_platform/crawler_platform/app/api/routes.py b/ontology_platform/crawler_platform/app/api/routes.py index 6a59e25..3499702 100644 --- a/ontology_platform/crawler_platform/app/api/routes.py +++ b/ontology_platform/crawler_platform/app/api/routes.py @@ -29,7 +29,10 @@ from crawler_platform.app.core.database.repository import ( make_claim_hash, ) from crawler_platform.app.core.database.session import session_scope -from crawler_platform.app.core.extractor.ai_provider import list_openai_compatible_models +from crawler_platform.app.core.extractor.ai_provider import ( + list_lmstudio_loaded_models, + list_openai_compatible_models, +) from crawler_platform.app.core.extractor.factory import extractor_for_domain from crawler_platform.app.core.ontology.definitions import DOMAIN_ONTOLOGIES, Ontology, ontology_for_domain from crawler_platform.app.core.ontology.domain_discovery import DomainDiscoveryService @@ -1686,7 +1689,7 @@ def register_routes(app, database_url: str) -> None: def extractor_models(request: ExtractorModelsRequest): try: if request.provider == "lm_studio": - models = list_openai_compatible_models(request.base_url or "http://localhost:1234/v1") + models = list_lmstudio_loaded_models(request.base_url or "http://localhost:1234/v1") return {"ok": True, "models": models} if request.provider == "openai": import os diff --git a/ontology_platform/crawler_platform/app/core/crawler/page_analysis_policy.py b/ontology_platform/crawler_platform/app/core/crawler/page_analysis_policy.py new file mode 100644 index 0000000..8925913 --- /dev/null +++ b/ontology_platform/crawler_platform/app/core/crawler/page_analysis_policy.py @@ -0,0 +1,82 @@ +from __future__ import annotations + +from crawler_platform.app.core.crawler.page_type_scorer import PAGE_TYPE_METADATA +from crawler_platform.app.core.crawler.page_type_taxonomy import ( + AnalyzeStrategy, + LLMPolicy, + PageClassificationResult, + PageType, + normalize_page_type, + normalize_semantic_page_type, +) + + +DEFAULT_ANALYZE_STRATEGY = AnalyzeStrategy.ANALYZE_METADATA_ONLY.value +DEFAULT_LLM_POLICY = LLMPolicy.NO_LLM.value + + +def decide_analyze_strategy(result_or_page_type: object | None) -> str: + semantic_page_type = normalize_semantic_page_type(result_or_page_type) + if ( + isinstance(result_or_page_type, PageClassificationResult) + and result_or_page_type.analyze_strategy + and result_or_page_type.analyze_strategy != DEFAULT_ANALYZE_STRATEGY + ): + return result_or_page_type.analyze_strategy + profile = PAGE_TYPE_METADATA.get(semantic_page_type) + if profile: + return str(profile.get("analyze_strategy") or DEFAULT_ANALYZE_STRATEGY) + if semantic_page_type == PageType.UNKNOWN_PAGE.value: + return AnalyzeStrategy.ANALYZE_METADATA_ONLY.value + return DEFAULT_ANALYZE_STRATEGY + + +def decide_llm_policy(result_or_page_type: object | None) -> str: + semantic_page_type = normalize_semantic_page_type(result_or_page_type) + if ( + isinstance(result_or_page_type, PageClassificationResult) + and result_or_page_type.llm_policy + and result_or_page_type.llm_policy != DEFAULT_LLM_POLICY + ): + return result_or_page_type.llm_policy + profile = PAGE_TYPE_METADATA.get(semantic_page_type) + if profile: + return str(profile.get("llm_policy") or DEFAULT_LLM_POLICY) + if semantic_page_type == PageType.UNKNOWN_PAGE.value: + return LLMPolicy.NO_LLM.value + return DEFAULT_LLM_POLICY + + +def is_protected_strategy(strategy: str | AnalyzeStrategy | None) -> bool: + return str(strategy or "") == AnalyzeStrategy.SKIP_PROTECTED.value + + +def is_noise_strategy(strategy: str | AnalyzeStrategy | None) -> bool: + return str(strategy or "") == AnalyzeStrategy.SKIP_NOISE.value + + +def should_analyze_page(result_or_page_type: object | None, analyze_page_types: set[str] | None = None) -> bool: + strategy = decide_analyze_strategy(result_or_page_type) + if is_protected_strategy(strategy) or is_noise_strategy(strategy): + return False + + if isinstance(result_or_page_type, PageClassificationResult): + return True + + if analyze_page_types is None: + return True + + normalized_page_type = normalize_page_type(result_or_page_type) + normalized_allowlist = {normalize_page_type(item) for item in analyze_page_types} + return normalized_page_type in normalized_allowlist + + +def apply_analysis_policy(result: PageClassificationResult) -> PageClassificationResult: + strategy = decide_analyze_strategy(result) + llm_policy = decide_llm_policy(result) + result.analyze_strategy = strategy + result.llm_policy = llm_policy + result.is_protected = is_protected_strategy(strategy) + result.is_noise = is_noise_strategy(strategy) + result.should_analyze = should_analyze_page(result) + return result diff --git a/ontology_platform/crawler_platform/app/core/crawler/page_classifier.py b/ontology_platform/crawler_platform/app/core/crawler/page_classifier.py index efd759c..fbb49a4 100644 --- a/ontology_platform/crawler_platform/app/core/crawler/page_classifier.py +++ b/ontology_platform/crawler_platform/app/core/crawler/page_classifier.py @@ -11,6 +11,13 @@ from crawler_platform.app.core.crawler.page_type_taxonomy import ( ) from crawler_platform.app.core.crawler.page_signal_extractor import extract_page_signals_from_page from crawler_platform.app.core.crawler.page_type_scorer import score_page_type +from crawler_platform.app.core.crawler.page_unknown_patterns import ( + build_unknown_pattern_payload, + should_store_unknown_pattern, +) +from crawler_platform.app.core.crawler.page_analysis_policy import ( + should_analyze_page as should_analyze_page_by_policy, +) PRODUCT_DETAIL_PREDICATES = { @@ -109,9 +116,15 @@ def classify_page_semantic( text: str = "", html: str | None = None, source_zones: list[dict[str, object]] | None = None, + final_url: str | None = None, + status_code: int | None = None, + content_type: str | None = None, ) -> PageClassificationResult: signals = extract_page_signals_from_page( url=url, + final_url=final_url, + status_code=status_code, + content_type=content_type, title=title, text=text, html=html, @@ -135,6 +148,35 @@ def classify_page_semantic( return result +def classification_metadata( + result: PageClassificationResult, + *, + title: str | None = None, + text: str | None = None, + html: str | None = None, + source_zones: list[str | dict[str, object]] | None = None, +) -> dict[str, object]: + """Return metadata that preserves legacy page_type while carrying semantic evidence.""" + + payload = result.to_dict() + if should_store_unknown_pattern(result): + payload["unknown_pattern"] = build_unknown_pattern_payload( + result=result, + url=result.url, + title=title, + text=text, + html=html, + source_zones=source_zones, + ) + return { + "page_type": get_legacy_page_type(result), + "semantic_page_type": result.primary_page_type, + "analyze_strategy": result.analyze_strategy, + "llm_policy": result.llm_policy, + "page_classification": payload, + } + + def relation_allowed_for_page_type(page_type: str | None, predicate: str) -> bool: if page_type in PRODUCT_DETAIL_PAGE_TYPES: return True @@ -150,9 +192,7 @@ def claim_allowed_for_context(page_type: str | None, predicate: str, zone_type: def should_analyze_page(page_type: object, analyze_page_types: set[str] | None) -> bool: - normalized_page_type = normalize_page_type(page_type) - normalized = {normalize_page_type(item) for item in (analyze_page_types or set())} - return normalized_page_type in normalized + return should_analyze_page_by_policy(page_type, analyze_page_types) def _zone_type(zone: dict[str, object]) -> str: diff --git a/ontology_platform/crawler_platform/app/core/crawler/page_cleaner.py b/ontology_platform/crawler_platform/app/core/crawler/page_cleaner.py index d044cc5..b5fc964 100644 --- a/ontology_platform/crawler_platform/app/core/crawler/page_cleaner.py +++ b/ontology_platform/crawler_platform/app/core/crawler/page_cleaner.py @@ -71,12 +71,27 @@ ZONE_SELECTORS: dict[str, list[str]] = { ZONE_PRIORITY_BY_PAGE_TYPE = { "ProductPage": ["product_title", "product_summary", "product_description", "product_detail"], + "ProductDetailPage": ["product_title", "product_summary", "product_description", "product_detail"], "BrandStoryPage": ["brand_story_body"], + "AboutPage": ["brand_story_body"], + "ContactPage": ["brand_story_body"], "NoticePage": ["notice_body"], + "PublicNoticePage": ["notice_body"], + "ArticlePage": ["notice_body"], + "NewsArticlePage": ["notice_body"], + "BlogPostPage": ["notice_body"], + "FAQPage": ["notice_body"], + "QAPage": ["notice_body"], "BoardPage": ["notice_body"], + "ForumBoardPage": ["notice_body"], + "ForumThreadPage": ["notice_body"], "EventPage": ["event_body"], "PromotionPage": ["event_body"], + "CampaignLandingPage": ["event_body"], "CategoryPage": ["product_title", "product_summary"], + "CategoryListingPage": ["product_title", "product_summary"], + "SearchPage": ["product_title", "product_summary"], + "SearchResultsPage": ["product_title", "product_summary"], } STRUCTURAL_NOISE_TOKENS = { diff --git a/ontology_platform/crawler_platform/app/core/crawler/page_signal_extractor.py b/ontology_platform/crawler_platform/app/core/crawler/page_signal_extractor.py new file mode 100644 index 0000000..5362bc4 --- /dev/null +++ b/ontology_platform/crawler_platform/app/core/crawler/page_signal_extractor.py @@ -0,0 +1,843 @@ +from __future__ import annotations + +from collections import Counter +import json +import re +from typing import Any +from urllib.parse import urljoin, urlparse + +from crawler_platform.app.core.crawler.page_signals import PageSignals, RawPageSnapshot + + +KEYWORD_GROUPS: dict[str, tuple[str, ...]] = { + "commerce": ( + "product", + "price", + "sale", + "cart", + "basket", + "buy", + "checkout", + "sku", + "상품", + "가격", + "장바구니", + "구매", + "주문", + ), + "listing": ( + "filter", + "sort", + "category", + "pagination", + "items", + "results", + "필터", + "정렬", + "카테고리", + "상품수", + "결과", + ), + "editorial": ( + "article", + "author", + "published", + "updated", + "headline", + "news", + "blog", + "기사", + "작성자", + "게시일", + ), + "community": ( + "question", + "answer", + "comment", + "reply", + "thread", + "vote", + "faq", + "q&a", + "질문", + "답변", + "댓글", + "문의", + ), + "knowledge": ( + "documentation", + "api", + "endpoint", + "parameter", + "version", + "reference", + "guide", + "문서", + "가이드", + "버전", + ), + "corporate": ( + "about", + "company", + "contact", + "address", + "team", + "careers", + "privacy", + "terms", + "회사", + "소개", + "문의", + "주소", + "채용", + "개인정보", + "약관", + ), + "protected": ( + "login", + "password", + "captcha", + "access denied", + "forbidden", + "payment", + "billing", + "로그인", + "비밀번호", + "보안문자", + "접근 제한", + "결제", + ), +} + + +PRICE_PATTERN = re.compile( + r"(?:[$€£¥₩]\s?\d[\d,]*(?:\.\d+)?)|(?:\d[\d,]*(?:\.\d+)?\s?(?:KRW|USD|EUR|JPY|원|달러))", + re.IGNORECASE, +) +DATE_PATTERN = re.compile(r"\b(?:20\d{2}|19\d{2})[-./년]\s?\d{1,2}[-./월]\s?\d{1,2}", re.IGNORECASE) +API_ENDPOINT_PATTERN = re.compile(r"\b(?:GET|POST|PUT|PATCH|DELETE)\s+/(?:[A-Za-z0-9_./{}:-]+)") +VERSION_PATTERN = re.compile(r"\bv?\d+\.\d+(?:\.\d+)?\b", re.IGNORECASE) + + +def build_raw_page_snapshot( + *, + url: str, + final_url: str | None = None, + status_code: int | None = None, + content_type: str | None = None, + title: str | None = None, + text: str | None = None, + html: str | None = None, + rendered_html: str | None = None, + metadata: dict[str, Any] | None = None, + source_zones: list[str | dict[str, Any]] | None = None, + collector_payload: dict[str, Any] | None = None, +) -> RawPageSnapshot: + snapshot = RawPageSnapshot( + url=url, + final_url=final_url, + status_code=status_code, + content_type=content_type, + title=title, + text=text, + html=html, + rendered_html=rendered_html, + metadata=dict(metadata or {}), + source_zones=list(source_zones or []), + collector_payload=dict(collector_payload or {}), + ) + return merge_collector_payload(snapshot) + + +def snapshot_from_collector_payload(payload: dict[str, Any], *, url: str | None = None) -> RawPageSnapshot: + return merge_collector_payload( + RawPageSnapshot( + url=str(url or payload.get("url") or payload.get("source_url") or ""), + final_url=payload.get("final_url") or payload.get("resolved_url"), + status_code=_optional_int(payload.get("status_code")), + content_type=payload.get("content_type"), + title=payload.get("title"), + text=payload.get("text") or payload.get("markdown") or payload.get("clean_text"), + html=payload.get("html") or payload.get("raw_html"), + rendered_html=payload.get("rendered_html"), + metadata=dict(payload.get("metadata") or {}), + collector_payload=dict(payload), + ) + ) + + +def merge_collector_payload(snapshot: RawPageSnapshot) -> RawPageSnapshot: + payload = snapshot.collector_payload + if not payload: + return snapshot + snapshot.open_graph.update(payload.get("open_graph") or payload.get("og") or {}) + snapshot.twitter_card.update(payload.get("twitter_card") or payload.get("twitter") or {}) + snapshot.json_ld.extend(_ensure_dict_list(payload.get("json_ld") or payload.get("jsonld"))) + snapshot.microdata.extend(_ensure_dict_list(payload.get("microdata"))) + snapshot.rdfa.extend(_ensure_dict_list(payload.get("rdfa"))) + snapshot.headings.extend(_string_list(payload.get("headings"))) + snapshot.links.extend(_dict_list(payload.get("links"))) + snapshot.images.extend(_dict_list(payload.get("images"))) + snapshot.forms.extend(_dict_list(payload.get("forms"))) + snapshot.buttons.extend(_string_list(payload.get("buttons"))) + snapshot.inputs.extend(_dict_list(payload.get("inputs"))) + snapshot.tables.extend(_dict_list(payload.get("tables"))) + snapshot.breadcrumbs.extend(_string_list(payload.get("breadcrumbs"))) + if payload.get("screenshot_path") and not snapshot.screenshot_path: + snapshot.screenshot_path = str(payload["screenshot_path"]) + return snapshot + + +def extract_page_signals_from_page( + *, + url: str, + final_url: str | None = None, + status_code: int | None = None, + content_type: str | None = None, + title: str | None = None, + text: str | None = None, + html: str | None = None, + rendered_html: str | None = None, + metadata: dict[str, Any] | None = None, + source_zones: list[str | dict[str, Any]] | None = None, + collector_payload: dict[str, Any] | None = None, +) -> PageSignals: + return extract_page_signals( + build_raw_page_snapshot( + url=url, + final_url=final_url, + status_code=status_code, + content_type=content_type, + title=title, + text=text, + html=html, + rendered_html=rendered_html, + metadata=metadata, + source_zones=source_zones, + collector_payload=collector_payload, + ) + ) + + +def extract_page_signals(snapshot: RawPageSnapshot) -> PageSignals: + html = snapshot.rendered_html or snapshot.html or "" + soup = _soup_from_html(html) + title = snapshot.title or _title_from_soup(soup) + text = snapshot.text or _text_from_soup(soup) or _text_from_html(html) + combined = "\n".join([snapshot.url, title or "", text or "", html[:12000]]).lower() + open_graph = {**_extract_meta_prefix(soup, "property", "og:"), **snapshot.open_graph} + twitter_card = {**_extract_meta_prefix(soup, "name", "twitter:"), **snapshot.twitter_card} + json_ld = [*_extract_json_ld(soup), *snapshot.json_ld] + links = snapshot.links or _extract_links(soup, snapshot.final_url or snapshot.url) + images = snapshot.images or _extract_images(soup, snapshot.final_url or snapshot.url) + forms = snapshot.forms or _extract_forms(soup) + inputs = snapshot.inputs or _extract_inputs(soup) + buttons = snapshot.buttons or _extract_buttons(soup) + headings = snapshot.headings or _extract_headings(soup) + tables = snapshot.tables or _extract_tables(soup) + breadcrumbs = snapshot.breadcrumbs or _extract_breadcrumbs(soup) + schema_types = _schema_types(json_ld, snapshot.microdata, snapshot.rdfa, soup) + keyword_hits = _keyword_hits(combined) + link_counts = _link_counts(links, snapshot.final_url or snapshot.url) + layout = _layout_signals(soup, combined, links, images, tables) + repeated_card_count = max( + layout["card_count"], + link_counts["product_link_count"], + _count_selector_matches(soup, CARD_SELECTORS), + ) + button_text = " ".join(buttons).lower() + input_text = " ".join(_input_blob(item) for item in inputs).lower() + form_text = " ".join(_form_blob(item) for item in forms).lower() + table_text = " ".join(str(table.get("text") or "") for table in tables).lower() + content_type = str(snapshot.content_type or snapshot.metadata.get("content_type") or "").lower() + path = urlparse(snapshot.final_url or snapshot.url).path.lower() + + signals = PageSignals( + schema_types=schema_types, + og_type=_string_or_none(open_graph.get("type") or open_graph.get("og:type")), + twitter_card_type=_string_or_none(twitter_card.get("card") or twitter_card.get("twitter:card")), + has_price=bool(PRICE_PATTERN.search(combined)), + has_currency=bool(re.search(r"[$€£¥₩]|(?:\b(?:KRW|USD|EUR|JPY)\b)|원", combined, re.IGNORECASE)), + has_cart_button=_contains_any(button_text + " " + combined, ("cart", "basket", "장바구니", "bag")), + has_buy_button=_contains_any(button_text + " " + combined, ("buy now", "purchase", "구매", "주문", "결제")), + has_variant_selector=_has_variant_selector(soup, input_text + " " + combined), + has_sku=bool(re.search(r"\bsku\b|상품\s*코드|product\s*code", combined, re.IGNORECASE)), + has_rating=("AggregateRating" in schema_types) + or _contains_any(combined, ("rating", "stars", "별점", "평점")), + has_review_section=("Review" in schema_types) or _contains_any(combined, ("review", "reviews", "후기", "리뷰")), + has_product_gallery=(len(images) >= 3 and _contains_any(combined, ("gallery", "product", "상품"))), + has_repeated_cards=repeated_card_count >= 3, + repeated_card_count=repeated_card_count, + has_filter_panel=layout["has_filter_sidebar"] + or _contains_any(combined, ("filter", "facets", "refine", "필터", "조건")), + has_sort_control=_contains_any(combined, ("sort", "order by", "low price", "high price", "정렬", "낮은가격", "높은가격")), + has_pagination=_has_pagination(soup, links, combined), + has_author=_contains_any(combined, ("author", "byline", "작성자", "기자")), + has_published_date=("datePublished" in _json_keys(json_ld)) + or bool(DATE_PATTERN.search(combined)) + and _contains_any(combined, ("published", "posted", "게시", "등록")), + has_modified_date=("dateModified" in _json_keys(json_ld)) + or _contains_any(combined, ("modified", "updated", "수정")), + has_article_body=("Article" in schema_types) + or ("NewsArticle" in schema_types) + or _count_selector_matches(soup, ("article", "[itemprop='articleBody']", ".article-body", ".post-content")) > 0 + or _contains_any(combined, ("articlebody", "article body")), + has_tags=_has_tags(soup, links, combined), + has_question=("QAPage" in schema_types) or _contains_any(combined, ("question", "q:", "질문", "문의")), + has_answer=("Answer" in schema_types) or _contains_any(combined, ("answer", "a:", "답변")), + has_comments=_contains_any(combined, ("comment", "comments", "reply", "댓글", "답글")), + has_votes=_contains_any(combined, ("vote", "votes", "upvote", "downvote", "추천", "투표")), + has_thread_structure=_contains_any(combined, ("thread", "discussion", "게시글", "토론")), + has_faq_structure=("FAQPage" in schema_types) or _contains_any(combined, ("faq", "frequently asked", "자주 묻는")), + has_code_blocks=_count_selector_matches(soup, ("pre", "code", ".highlight", ".code")) > 0, + has_toc=_count_selector_matches(soup, ("#toc", ".toc", "[class*='table-of-contents']", "nav[aria-label*='contents']")) > 0, + has_api_endpoint=bool(API_ENDPOINT_PATTERN.search(f"{text or ''}\n{html or ''}")), + has_parameter_table=_has_parameter_table(tables, table_text), + has_version_info=bool(VERSION_PATTERN.search(combined)) and _contains_any(combined, ("version", "버전", "release")), + has_contact_info=_contains_any(combined, ("contact", "email", "tel:", "문의", "연락처")), + has_address=_contains_any(combined, ("address", "주소", "road", "street")), + has_policy_terms=_contains_any(combined, ("terms", "policy", "agreement", "약관", "정책")), + has_privacy_terms=_contains_any(combined, ("privacy", "personal information", "개인정보")), + has_career_terms=_contains_any(combined, ("career", "jobs", "recruit", "채용", "지원")), + has_login_form=_contains_any(form_text + " " + combined, ("login", "sign in", "로그인")) and ( + "password" in input_text or "비밀번호" in combined + ), + has_password_field="password" in input_text, + has_payment_fields=_contains_any(input_text + " " + combined, ("card number", "payment", "billing", "결제", "카드")), + has_captcha=_contains_any(combined, ("captcha", "recaptcha", "hcaptcha", "보안문자")), + has_access_denied=_contains_any(combined, ("access denied", "forbidden", "permission denied", "접근 제한", "권한이 없습니다")), + status_code=snapshot.status_code, + has_error_status=bool(snapshot.status_code and snapshot.status_code >= 400) + or _contains_any(combined, ("404", "not found", "error page")), + has_not_found=snapshot.status_code == 404 or _contains_any(combined, ("404", "not found", "page not found")), + has_sitemap_resource=("sitemap" in path) or ("sitemap" in content_type and "xml" in content_type), + has_feed_resource=("rss" in content_type) or ("atom" in content_type) or path.endswith((".rss", ".atom")), + has_json_resource=("json" in content_type) or path.endswith(".json"), + has_xml_resource=("xml" in content_type) or path.endswith(".xml"), + has_file_resource=path.endswith((".pdf", ".csv", ".xlsx", ".xls", ".doc", ".docx", ".zip")), + internal_link_count=link_counts["internal_link_count"], + external_link_count=link_counts["external_link_count"], + product_link_count=link_counts["product_link_count"], + category_link_count=link_counts["category_link_count"], + profile_link_count=link_counts["profile_link_count"], + article_link_count=link_counts["article_link_count"], + layout_blocks=layout["layout_blocks"], + has_hero_block=layout["has_hero_block"], + has_card_grid=layout["has_card_grid"], + has_filter_sidebar=layout["has_filter_sidebar"], + has_sticky_action_box=layout["has_sticky_action_box"], + has_media_player_area=layout["has_media_player_area"], + has_map_area=layout["has_map_area"], + has_calendar_grid=layout["has_calendar_grid"], + has_pricing_table=layout["has_pricing_table"], + has_comparison_table=layout["has_comparison_table"], + dominant_language=_dominant_language(combined), + keyword_hits=keyword_hits, + url_hints=_url_hints(snapshot.final_url or snapshot.url), + title=title, + text_sample=(text or "")[:500], + external_collector_signals=_external_collector_signals(snapshot), + ) + return signals + + +def _soup_from_html(html: str): + if not html: + return None + try: + from bs4 import BeautifulSoup + except ImportError: + return None + try: + return BeautifulSoup(html, "html.parser") + except Exception: + return None + + +def _title_from_soup(soup) -> str | None: + if soup is None or not soup.title: + return None + return soup.title.get_text(" ", strip=True) or None + + +def _text_from_soup(soup) -> str: + if soup is None: + return "" + return soup.get_text("\n", strip=True) + + +def _text_from_html(html: str) -> str: + return re.sub(r"\s+", " ", re.sub(r"<[^>]+>", " ", html or "")).strip() + + +def _extract_meta_prefix(soup, attr_name: str, prefix: str) -> dict[str, str]: + if soup is None: + return {} + result: dict[str, str] = {} + for tag in soup.find_all("meta"): + name = str(tag.get(attr_name) or "").strip() + if not name.lower().startswith(prefix): + continue + content = str(tag.get("content") or "").strip() + if content: + result[name.removeprefix(prefix)] = content + result[name] = content + return result + + +def _extract_json_ld(soup) -> list[dict[str, Any]]: + if soup is None: + return [] + payloads: list[dict[str, Any]] = [] + for tag in soup.find_all("script"): + script_type = str(tag.get("type") or "").lower() + if "ld+json" not in script_type: + continue + raw = tag.string or tag.get_text(" ", strip=True) + try: + value = json.loads(raw) + except Exception: + continue + payloads.extend(_ensure_dict_list(value)) + return payloads + + +def _extract_headings(soup) -> list[str]: + if soup is None: + return [] + return [node.get_text(" ", strip=True) for node in soup.select("h1,h2,h3") if node.get_text(" ", strip=True)] + + +def _extract_links(soup, base_url: str) -> list[dict[str, Any]]: + if soup is None: + return [] + links = [] + for tag in soup.find_all("a"): + href = str(tag.get("href") or "").strip() + if not href: + continue + links.append( + { + "href": urljoin(base_url, href), + "text": tag.get_text(" ", strip=True), + "rel": " ".join(str(item) for item in tag.get("rel", [])), + "class": " ".join(str(item) for item in tag.get("class", [])), + } + ) + return links + + +def _extract_images(soup, base_url: str) -> list[dict[str, Any]]: + if soup is None: + return [] + images = [] + for tag in soup.find_all("img"): + src = str(tag.get("src") or tag.get("data-src") or "").strip() + if not src: + continue + images.append( + { + "src": urljoin(base_url, src), + "alt": str(tag.get("alt") or ""), + "class": " ".join(str(item) for item in tag.get("class", [])), + } + ) + return images + + +def _extract_forms(soup) -> list[dict[str, Any]]: + if soup is None: + return [] + forms = [] + for form in soup.find_all("form"): + forms.append( + { + "action": str(form.get("action") or ""), + "method": str(form.get("method") or ""), + "id": str(form.get("id") or ""), + "class": " ".join(str(item) for item in form.get("class", [])), + "text": form.get_text(" ", strip=True)[:500], + } + ) + return forms + + +def _extract_inputs(soup) -> list[dict[str, Any]]: + if soup is None: + return [] + inputs = [] + for tag in soup.select("input,select,textarea"): + inputs.append( + { + "type": str(tag.get("type") or tag.name or ""), + "name": str(tag.get("name") or ""), + "id": str(tag.get("id") or ""), + "placeholder": str(tag.get("placeholder") or ""), + "autocomplete": str(tag.get("autocomplete") or ""), + "aria_label": str(tag.get("aria-label") or ""), + "class": " ".join(str(item) for item in tag.get("class", [])), + "text": tag.get_text(" ", strip=True)[:240], + } + ) + return inputs + + +def _extract_buttons(soup) -> list[str]: + if soup is None: + return [] + values = [] + for tag in soup.select("button,input[type='submit'],input[type='button'],[role='button']"): + text = tag.get_text(" ", strip=True) or str(tag.get("value") or tag.get("aria-label") or "") + if text.strip(): + values.append(text.strip()) + return values + + +def _extract_tables(soup) -> list[dict[str, Any]]: + if soup is None: + return [] + tables = [] + for table in soup.find_all("table"): + headers = [cell.get_text(" ", strip=True) for cell in table.select("th") if cell.get_text(" ", strip=True)] + text = table.get_text(" ", strip=True) + tables.append({"headers": headers, "text": text[:1000], "row_count": len(table.select("tr"))}) + return tables + + +def _extract_breadcrumbs(soup) -> list[str]: + if soup is None: + return [] + crumbs = [] + selectors = [ + "[class*='breadcrumb']", + "[id*='breadcrumb']", + "nav[aria-label*='breadcrumb' i]", + "[itemtype*='BreadcrumbList']", + ] + for selector in selectors: + for node in soup.select(selector): + text = node.get_text(" > ", strip=True) + if text: + crumbs.append(text) + return _dedupe_strings(crumbs) + + +def _schema_types( + json_ld: list[dict[str, Any]], + microdata: list[dict[str, Any]], + rdfa: list[dict[str, Any]], + soup, +) -> set[str]: + types: set[str] = set() + for payload in [*json_ld, *microdata, *rdfa]: + _visit_schema_types(payload, types) + if soup is not None: + for node in soup.select("[itemscope][itemtype]"): + raw = str(node.get("itemtype") or "") + if raw: + types.add(raw.rstrip("/").split("/")[-1]) + for node in soup.select("[typeof]"): + for item in str(node.get("typeof") or "").split(): + types.add(item.split(":")[-1]) + return {item for item in types if item} + + +def _visit_schema_types(value: Any, types: set[str]) -> None: + if isinstance(value, list): + for item in value: + _visit_schema_types(item, types) + return + if not isinstance(value, dict): + return + raw_type = value.get("@type") or value.get("type") + for item in _ensure_list(raw_type): + if isinstance(item, str): + types.add(item.rstrip("/").split("/")[-1]) + for nested in value.values(): + if isinstance(nested, (dict, list)): + _visit_schema_types(nested, types) + + +def _json_keys(json_ld: list[dict[str, Any]]) -> set[str]: + keys: set[str] = set() + + def visit(value: Any) -> None: + if isinstance(value, list): + for item in value: + visit(item) + return + if not isinstance(value, dict): + return + keys.update(str(key) for key in value) + for nested in value.values(): + visit(nested) + + visit(json_ld) + return keys + + +def _keyword_hits(text: str) -> dict[str, int]: + hits = {} + for group, terms in KEYWORD_GROUPS.items(): + count = sum(text.count(term.lower()) for term in terms) + if count: + hits[group] = count + return hits + + +def _link_counts(links: list[dict[str, Any]], base_url: str) -> dict[str, int]: + base = urlparse(base_url) + base_host = base.netloc.lower() + base_path = base.path.lower() + counts = Counter() + for link in links: + href = str(link.get("href") or "") + parsed = urlparse(href) + host = parsed.netloc.lower() + path = parsed.path.lower() + if not host or host == base_host: + counts["internal_link_count"] += 1 + else: + counts["external_link_count"] += 1 + same_document_query_link = path == base_path and bool(parsed.query) + if not same_document_query_link and any(token in path for token in ("/product", "/products", "/goods", "/item", "/p/")): + counts["product_link_count"] += 1 + if any(token in path for token in ("category", "collection", "/shop", "/list", "catalog")): + counts["category_link_count"] += 1 + if any(token in path for token in ("profile", "user", "author", "member", "creator")): + counts["profile_link_count"] += 1 + if any(token in path for token in ("article", "blog", "news", "post", "story")): + counts["article_link_count"] += 1 + return { + "internal_link_count": counts["internal_link_count"], + "external_link_count": counts["external_link_count"], + "product_link_count": counts["product_link_count"], + "category_link_count": counts["category_link_count"], + "profile_link_count": counts["profile_link_count"], + "article_link_count": counts["article_link_count"], + } + + +CARD_SELECTORS = ( + ".product-card", + ".card", + ".item", + ".product", + ".prdList > li", + "[class*='product-card']", + "[class*='grid-item']", +) + + +def _layout_signals(soup, text: str, links: list[dict[str, Any]], images: list[dict[str, Any]], tables: list[dict[str, Any]]) -> dict[str, Any]: + blocks: list[str] = [] + hero = _count_selector_matches(soup, (".hero", ".visual", ".main-visual", "[class*='hero']", "section[aria-label*='hero']")) > 0 + card_count = _count_selector_matches(soup, CARD_SELECTORS) + card_grid = card_count >= 3 or _count_selector_matches(soup, (".grid", "[class*='grid']", "[class*='cards']")) > 0 and len(links) >= 3 + filter_sidebar = _count_selector_matches(soup, (".filter", ".filters", ".facet", "aside", "[class*='filter']", "[class*='facet']")) > 0 + sticky_action = _count_selector_matches(soup, (".sticky", "[class*='sticky']", "[class*='fixed']", "[class*='buy-box']")) > 0 + media_player = _count_selector_matches(soup, ("video", "audio", "iframe[src*='youtube']", "[class*='player']")) > 0 + map_area = _count_selector_matches(soup, ("[class*='map']", "#map", "iframe[src*='maps']")) > 0 or "google map" in text + calendar_grid = _count_selector_matches(soup, ("[class*='calendar']", "[class*='datepicker']", "table.calendar")) > 0 + pricing_table = "pricing" in text and (bool(tables) or _count_selector_matches(soup, ("[class*='pricing']", ".price-table")) > 0) + comparison_table = _has_comparison_table(tables, text) + for label, present in [ + ("hero_block", hero), + ("card_grid", card_grid), + ("filter_sidebar", filter_sidebar), + ("sticky_action_box", sticky_action), + ("media_player_area", media_player), + ("map_area", map_area), + ("calendar_grid", calendar_grid), + ("pricing_table", pricing_table), + ("comparison_table", comparison_table), + ]: + if present: + blocks.append(label) + return { + "layout_blocks": blocks, + "card_count": card_count, + "has_hero_block": hero, + "has_card_grid": card_grid, + "has_filter_sidebar": filter_sidebar, + "has_sticky_action_box": sticky_action, + "has_media_player_area": media_player, + "has_map_area": map_area, + "has_calendar_grid": calendar_grid, + "has_pricing_table": pricing_table, + "has_comparison_table": comparison_table, + } + + +def _has_variant_selector(soup, text: str) -> bool: + if _contains_any(text, ("variant", "option", "size", "color", "옵션", "사이즈", "색상")): + return True + if soup is None: + return False + for select in soup.find_all("select"): + blob = " ".join( + [ + str(select.get("name") or ""), + str(select.get("id") or ""), + select.get_text(" ", strip=True), + ] + ).lower() + if _contains_any(blob, ("variant", "option", "size", "color", "옵션", "사이즈", "색상")): + return True + return False + + +def _has_pagination(soup, links: list[dict[str, Any]], text: str) -> bool: + if _count_selector_matches(soup, (".pagination", ".paging", "[class*='paginate']", "nav[aria-label*='pagination']")) > 0: + return True + if any(str(link.get("rel") or "").lower() in {"next", "prev", "previous"} for link in links): + return True + return _contains_any(text, ("next page", "previous page", "페이지", "다음", "이전")) + + +def _has_tags(soup, links: list[dict[str, Any]], text: str) -> bool: + if _count_selector_matches(soup, (".tag", ".tags", "[rel='tag']", "[class*='tag']")) > 0: + return True + return any(str(link.get("rel") or "").lower() == "tag" for link in links) or _contains_any(text, ("tags:", "태그")) + + +def _has_parameter_table(tables: list[dict[str, Any]], table_text: str) -> bool: + if _contains_any(table_text, ("parameter", "required", "type", "description", "파라미터", "필수")): + return True + for table in tables: + headers = " ".join(str(item) for item in table.get("headers") or []).lower() + if _contains_any(headers, ("parameter", "required", "type", "description", "파라미터", "필수")): + return True + return False + + +def _has_comparison_table(tables: list[dict[str, Any]], text: str) -> bool: + if not tables: + return False + return _contains_any(text, ("compare", "comparison", "vs", "비교")) or any( + int(table.get("row_count") or 0) >= 3 and len(table.get("headers") or []) >= 3 for table in tables + ) + + +def _url_hints(url: str) -> set[str]: + parsed = urlparse(url) + value = f"{parsed.path} {parsed.query}".lower() + hints = set() + for hint, terms in { + "product": ("/product", "/products", "/goods", "/item", "/p/"), + "category": ("category", "collection", "/shop", "/list", "catalog"), + "search": ("search", "find", "keyword=", "query=", "q="), + "article": ("article", "blog", "news", "post", "story"), + "board": ("board", "forum", "thread", "qna"), + "protected": ("login", "checkout", "payment", "account", "cart"), + "system": ("sitemap", "robots.txt", ".json", ".xml", ".rss"), + }.items(): + if any(term in value for term in terms): + hints.add(hint) + return hints + + +def _dominant_language(text: str) -> str | None: + if not text: + return None + korean = len(re.findall(r"[가-힣]", text)) + latin = len(re.findall(r"[A-Za-z]", text)) + if korean == 0 and latin == 0: + return None + if korean > latin * 0.25: + return "ko" + return "en" + + +def _external_collector_signals(snapshot: RawPageSnapshot) -> dict[str, Any]: + payload = dict(snapshot.collector_payload or {}) + for key in { + "html", + "raw_html", + "rendered_html", + "text", + "markdown", + "clean_text", + "metadata", + "links", + "images", + "forms", + "inputs", + "buttons", + "tables", + "json_ld", + "jsonld", + }: + payload.pop(key, None) + return payload + + +def _count_selector_matches(soup, selectors: tuple[str, ...]) -> int: + if soup is None: + return 0 + count = 0 + for selector in selectors: + try: + count += len(soup.select(selector)) + except Exception: + continue + return count + + +def _input_blob(item: dict[str, Any]) -> str: + return " ".join(str(item.get(key) or "") for key in ("type", "name", "id", "placeholder", "autocomplete", "aria_label", "class", "text")) + + +def _form_blob(item: dict[str, Any]) -> str: + return " ".join(str(item.get(key) or "") for key in ("action", "method", "id", "class", "text")) + + +def _contains_any(value: str, terms: tuple[str, ...]) -> bool: + lowered = value.lower() + return any(term.lower() in lowered for term in terms) + + +def _string_or_none(value: Any) -> str | None: + clean = str(value or "").strip() + return clean or None + + +def _optional_int(value: Any) -> int | None: + try: + return int(value) + except (TypeError, ValueError): + return None + + +def _ensure_list(value: Any) -> list[Any]: + if value is None: + return [] + if isinstance(value, list): + return value + return [value] + + +def _ensure_dict_list(value: Any) -> list[dict[str, Any]]: + values = _ensure_list(value) + return [item for item in values if isinstance(item, dict)] + + +def _dict_list(value: Any) -> list[dict[str, Any]]: + return [dict(item) for item in _ensure_list(value) if isinstance(item, dict)] + + +def _string_list(value: Any) -> list[str]: + return [str(item) for item in _ensure_list(value) if str(item or "").strip()] + + +def _dedupe_strings(values: list[str]) -> list[str]: + seen = set() + result = [] + for value in values: + clean = " ".join(value.split()) + key = clean.lower() + if not clean or key in seen: + continue + seen.add(key) + result.append(clean) + return result diff --git a/ontology_platform/crawler_platform/app/core/crawler/page_signals.py b/ontology_platform/crawler_platform/app/core/crawler/page_signals.py new file mode 100644 index 0000000..e555832 --- /dev/null +++ b/ontology_platform/crawler_platform/app/core/crawler/page_signals.py @@ -0,0 +1,144 @@ +from __future__ import annotations + +from dataclasses import asdict, dataclass, field +from typing import Any + + +@dataclass(slots=True) +class RawPageSnapshot: + url: str + final_url: str | None = None + status_code: int | None = None + content_type: str | None = None + title: str | None = None + text: str | None = None + html: str | None = None + rendered_html: str | None = None + metadata: dict[str, Any] = field(default_factory=dict) + open_graph: dict[str, Any] = field(default_factory=dict) + twitter_card: dict[str, Any] = field(default_factory=dict) + json_ld: list[dict[str, Any]] = field(default_factory=list) + microdata: list[dict[str, Any]] = field(default_factory=list) + rdfa: list[dict[str, Any]] = field(default_factory=list) + headings: list[str] = field(default_factory=list) + links: list[dict[str, Any]] = field(default_factory=list) + images: list[dict[str, Any]] = field(default_factory=list) + forms: list[dict[str, Any]] = field(default_factory=list) + buttons: list[str] = field(default_factory=list) + inputs: list[dict[str, Any]] = field(default_factory=list) + tables: list[dict[str, Any]] = field(default_factory=list) + breadcrumbs: list[str] = field(default_factory=list) + source_zones: list[str | dict[str, Any]] = field(default_factory=list) + screenshot_path: str | None = None + collector_payload: dict[str, Any] = field(default_factory=dict) + + def to_dict(self) -> dict[str, Any]: + return asdict(self) + + +@dataclass(slots=True) +class PageSignals: + # structured data + schema_types: set[str] = field(default_factory=set) + og_type: str | None = None + twitter_card_type: str | None = None + + # commerce + has_price: bool = False + has_currency: bool = False + has_cart_button: bool = False + has_buy_button: bool = False + has_variant_selector: bool = False + has_sku: bool = False + has_rating: bool = False + has_review_section: bool = False + has_product_gallery: bool = False + + # listing + has_repeated_cards: bool = False + repeated_card_count: int = 0 + has_filter_panel: bool = False + has_sort_control: bool = False + has_pagination: bool = False + + # editorial + has_author: bool = False + has_published_date: bool = False + has_modified_date: bool = False + has_article_body: bool = False + has_tags: bool = False + + # community + has_question: bool = False + has_answer: bool = False + has_comments: bool = False + has_votes: bool = False + has_thread_structure: bool = False + has_faq_structure: bool = False + + # knowledge/docs + has_code_blocks: bool = False + has_toc: bool = False + has_api_endpoint: bool = False + has_parameter_table: bool = False + has_version_info: bool = False + + # corporate/legal + has_contact_info: bool = False + has_address: bool = False + has_policy_terms: bool = False + has_privacy_terms: bool = False + has_career_terms: bool = False + + # transaction/protected + has_login_form: bool = False + has_password_field: bool = False + has_payment_fields: bool = False + has_captcha: bool = False + has_access_denied: bool = False + + # system/resource + status_code: int | None = None + has_error_status: bool = False + has_not_found: bool = False + has_sitemap_resource: bool = False + has_feed_resource: bool = False + has_json_resource: bool = False + has_xml_resource: bool = False + has_file_resource: bool = False + + # graph + internal_link_count: int = 0 + external_link_count: int = 0 + product_link_count: int = 0 + category_link_count: int = 0 + profile_link_count: int = 0 + article_link_count: int = 0 + + # visual/layout candidates from DOM structure + layout_blocks: list[str] = field(default_factory=list) + has_hero_block: bool = False + has_card_grid: bool = False + has_filter_sidebar: bool = False + has_sticky_action_box: bool = False + has_media_player_area: bool = False + has_map_area: bool = False + has_calendar_grid: bool = False + has_pricing_table: bool = False + has_comparison_table: bool = False + + # text/layout + dominant_language: str | None = None + keyword_hits: dict[str, int] = field(default_factory=dict) + url_hints: set[str] = field(default_factory=set) + title: str | None = None + text_sample: str = "" + + # external collector hook + external_collector_signals: dict[str, Any] = field(default_factory=dict) + + def to_dict(self) -> dict[str, Any]: + payload = asdict(self) + payload["schema_types"] = sorted(self.schema_types) + payload["url_hints"] = sorted(self.url_hints) + return payload diff --git a/ontology_platform/crawler_platform/app/core/crawler/page_type_scorer.py b/ontology_platform/crawler_platform/app/core/crawler/page_type_scorer.py new file mode 100644 index 0000000..b7538e8 --- /dev/null +++ b/ontology_platform/crawler_platform/app/core/crawler/page_type_scorer.py @@ -0,0 +1,747 @@ +from __future__ import annotations + +from collections.abc import Callable +from dataclasses import dataclass +from typing import Any + +from crawler_platform.app.core.crawler.page_signals import PageSignals +from crawler_platform.app.core.crawler.page_type_taxonomy import ( + ActionIntent, + AnalyzeStrategy, + EntityType, + EvidenceItem, + GraphRole, + LLMPolicy, + PAGE_TYPE_PROFILES, + PageArchetype, + PageClassificationResult, + PageDomain, + PageType, +) + + +UNKNOWN_THRESHOLD = 0.22 + + +@dataclass(frozen=True, slots=True) +class SignalRule: + key: str + weight: float + source: str + message: str + predicate: Callable[[PageSignals], bool] + value: Callable[[PageSignals], str | int | float | bool | None] | None = None + + +@dataclass(slots=True) +class ScoreAccumulator: + scores: dict[str, float] + evidence_by_type: dict[str, list[EvidenceItem]] + + def add(self, page_type: str, rule: SignalRule, signals: PageSignals) -> None: + if not rule.predicate(signals): + return + value = rule.value(signals) if rule.value else True + self.scores[page_type] = self.scores.get(page_type, 0.0) + rule.weight + self.evidence_by_type.setdefault(page_type, []).append( + EvidenceItem( + key=rule.key, + value=value, + weight=rule.weight, + source=rule.source, + message=rule.message, + ) + ) + + +PAGE_TYPE_METADATA: dict[str, dict[str, Any]] = { + **PAGE_TYPE_PROFILES, + PageType.ARTICLE_PAGE.value: { + "domain": PageDomain.EDITORIAL.value, + "archetype": PageArchetype.ARTICLE.value, + "main_entity_type": EntityType.ARTICLE.value, + "action_intents": [ActionIntent.READ.value], + "graph_roles": [GraphRole.CLAIM_SOURCE.value, GraphRole.ENTITY_ANCHOR.value], + "analyze_strategy": AnalyzeStrategy.ANALYZE_FULL.value, + "llm_policy": LLMPolicy.LLM_FULL.value, + "should_analyze": True, + }, + PageType.BLOG_POST_PAGE.value: { + "domain": PageDomain.EDITORIAL.value, + "archetype": PageArchetype.ARTICLE.value, + "main_entity_type": EntityType.ARTICLE.value, + "action_intents": [ActionIntent.READ.value], + "graph_roles": [GraphRole.CLAIM_SOURCE.value], + "analyze_strategy": AnalyzeStrategy.ANALYZE_FULL.value, + "llm_policy": LLMPolicy.LLM_FULL.value, + "should_analyze": True, + }, + PageType.FAQ_PAGE.value: { + "domain": PageDomain.COMMUNITY.value, + "archetype": PageArchetype.DOCUMENT.value, + "main_entity_type": EntityType.QUESTION.value, + "action_intents": [ActionIntent.READ.value, ActionIntent.ANSWER.value], + "graph_roles": [GraphRole.REFERENCE_SOURCE.value], + "analyze_strategy": AnalyzeStrategy.ANALYZE_FULL.value, + "llm_policy": LLMPolicy.LLM_LIGHT.value, + "should_analyze": True, + }, + PageType.QA_PAGE.value: { + "domain": PageDomain.COMMUNITY.value, + "archetype": PageArchetype.THREAD.value, + "main_entity_type": EntityType.QUESTION.value, + "action_intents": [ActionIntent.ASK.value, ActionIntent.ANSWER.value, ActionIntent.READ.value], + "graph_roles": [GraphRole.CLAIM_SOURCE.value, GraphRole.RELATION_HUB.value], + "analyze_strategy": AnalyzeStrategy.ANALYZE_FULL.value, + "llm_policy": LLMPolicy.LLM_LIGHT.value, + "should_analyze": True, + }, + PageType.FORUM_THREAD_PAGE.value: { + "domain": PageDomain.COMMUNITY.value, + "archetype": PageArchetype.THREAD.value, + "main_entity_type": EntityType.ARTICLE.value, + "action_intents": [ActionIntent.READ.value, ActionIntent.COMMENT.value], + "graph_roles": [GraphRole.CLAIM_SOURCE.value, GraphRole.RELATION_HUB.value], + "analyze_strategy": AnalyzeStrategy.ANALYZE_FULL.value, + "llm_policy": LLMPolicy.LLM_LIGHT.value, + "should_analyze": True, + }, + PageType.ABOUT_PAGE.value: { + "domain": PageDomain.CORPORATE.value, + "archetype": PageArchetype.ARTICLE.value, + "main_entity_type": EntityType.ORGANIZATION.value, + "action_intents": [ActionIntent.READ.value, ActionIntent.LEARN.value], + "graph_roles": [GraphRole.ENTITY_ANCHOR.value, GraphRole.CLAIM_SOURCE.value], + "analyze_strategy": AnalyzeStrategy.ANALYZE_FULL.value, + "llm_policy": LLMPolicy.LLM_FULL.value, + "should_analyze": True, + }, + PageType.CONTACT_PAGE.value: { + "domain": PageDomain.CORPORATE.value, + "archetype": PageArchetype.DOCUMENT.value, + "main_entity_type": EntityType.ORGANIZATION.value, + "action_intents": [ActionIntent.CONTACT.value, ActionIntent.NAVIGATE.value], + "graph_roles": [GraphRole.ENTITY_ANCHOR.value], + "analyze_strategy": AnalyzeStrategy.ANALYZE_METADATA_ONLY.value, + "llm_policy": LLMPolicy.RULE_ONLY.value, + "should_analyze": True, + }, + PageType.DOCUMENTATION_PAGE.value: { + "domain": PageDomain.KNOWLEDGE.value, + "archetype": PageArchetype.DOCUMENT.value, + "main_entity_type": EntityType.SOFTWARE_APPLICATION.value, + "action_intents": [ActionIntent.READ.value, ActionIntent.LEARN.value], + "graph_roles": [GraphRole.REFERENCE_SOURCE.value], + "analyze_strategy": AnalyzeStrategy.ANALYZE_FULL.value, + "llm_policy": LLMPolicy.LLM_LIGHT.value, + "should_analyze": True, + }, + PageType.API_REFERENCE_PAGE.value: { + "domain": PageDomain.SOFTWARE.value, + "archetype": PageArchetype.DOCUMENT.value, + "main_entity_type": EntityType.SOFTWARE_APPLICATION.value, + "action_intents": [ActionIntent.LEARN.value, ActionIntent.CONFIGURE.value], + "graph_roles": [GraphRole.REFERENCE_SOURCE.value], + "analyze_strategy": AnalyzeStrategy.ANALYZE_FULL.value, + "llm_policy": LLMPolicy.LLM_LIGHT.value, + "should_analyze": True, + }, + PageType.WIKI_PAGE.value: { + "domain": PageDomain.KNOWLEDGE.value, + "archetype": PageArchetype.DOCUMENT.value, + "main_entity_type": EntityType.UNKNOWN_ENTITY.value, + "action_intents": [ActionIntent.READ.value, ActionIntent.LEARN.value], + "graph_roles": [GraphRole.REFERENCE_SOURCE.value], + "analyze_strategy": AnalyzeStrategy.ANALYZE_FULL.value, + "llm_policy": LLMPolicy.LLM_LIGHT.value, + "should_analyze": True, + }, + PageType.DATASET_PAGE.value: { + "domain": PageDomain.KNOWLEDGE.value, + "archetype": PageArchetype.DOCUMENT.value, + "main_entity_type": EntityType.DATASET.value, + "action_intents": [ActionIntent.READ.value, ActionIntent.DOWNLOAD.value], + "graph_roles": [GraphRole.REFERENCE_SOURCE.value], + "analyze_strategy": AnalyzeStrategy.ANALYZE_DOCUMENT_ONLY.value, + "llm_policy": LLMPolicy.LLM_LIGHT.value, + "should_analyze": True, + }, + PageType.RESEARCH_PAPER_PAGE.value: { + "domain": PageDomain.KNOWLEDGE.value, + "archetype": PageArchetype.ARTICLE.value, + "main_entity_type": EntityType.ARTICLE.value, + "action_intents": [ActionIntent.READ.value, ActionIntent.LEARN.value], + "graph_roles": [GraphRole.CLAIM_SOURCE.value, GraphRole.REFERENCE_SOURCE.value], + "analyze_strategy": AnalyzeStrategy.ANALYZE_FULL.value, + "llm_policy": LLMPolicy.LLM_FULL.value, + "should_analyze": True, + }, + PageType.JOB_POSTING_PAGE.value: { + "domain": PageDomain.JOBS.value, + "archetype": PageArchetype.DETAIL.value, + "main_entity_type": EntityType.JOB_POSTING.value, + "action_intents": [ActionIntent.APPLY.value, ActionIntent.READ.value], + "graph_roles": [GraphRole.ENTITY_ANCHOR.value], + "analyze_strategy": AnalyzeStrategy.ANALYZE_FULL.value, + "llm_policy": LLMPolicy.LLM_FULL.value, + "should_analyze": True, + }, + PageType.COURSE_DETAIL_PAGE.value: { + "domain": PageDomain.EDUCATION.value, + "archetype": PageArchetype.DETAIL.value, + "main_entity_type": EntityType.COURSE.value, + "action_intents": [ActionIntent.LEARN.value, ActionIntent.SUBSCRIBE.value], + "graph_roles": [GraphRole.ENTITY_ANCHOR.value, GraphRole.REFERENCE_SOURCE.value], + "analyze_strategy": AnalyzeStrategy.ANALYZE_FULL.value, + "llm_policy": LLMPolicy.LLM_LIGHT.value, + "should_analyze": True, + }, + PageType.VIDEO_PAGE.value: { + "domain": PageDomain.MEDIA.value, + "archetype": PageArchetype.MEDIA.value, + "main_entity_type": EntityType.MEDIA_OBJECT.value, + "action_intents": [ActionIntent.WATCH.value], + "graph_roles": [GraphRole.MEDIA_ANCHOR.value], + "analyze_strategy": AnalyzeStrategy.ANALYZE_METADATA_ONLY.value, + "llm_policy": LLMPolicy.RULE_ONLY.value, + "should_analyze": True, + }, + PageType.LOCAL_BUSINESS_PAGE.value: { + "domain": PageDomain.LOCAL.value, + "archetype": PageArchetype.DETAIL.value, + "main_entity_type": EntityType.PLACE.value, + "action_intents": [ActionIntent.CONTACT.value, ActionIntent.NAVIGATE.value], + "graph_roles": [GraphRole.ENTITY_ANCHOR.value], + "analyze_strategy": AnalyzeStrategy.ANALYZE_FULL.value, + "llm_policy": LLMPolicy.LLM_LIGHT.value, + "should_analyze": True, + }, + PageType.REAL_ESTATE_LISTING_PAGE.value: { + "domain": PageDomain.LOCAL.value, + "archetype": PageArchetype.DETAIL.value, + "main_entity_type": EntityType.REAL_ESTATE_PROPERTY.value, + "action_intents": [ActionIntent.READ.value, ActionIntent.CONTACT.value], + "graph_roles": [GraphRole.ENTITY_ANCHOR.value], + "analyze_strategy": AnalyzeStrategy.ANALYZE_FULL.value, + "llm_policy": LLMPolicy.LLM_LIGHT.value, + "should_analyze": True, + }, + PageType.PROFILE_PAGE.value: { + "domain": PageDomain.COMMUNITY.value, + "archetype": PageArchetype.PROFILE.value, + "main_entity_type": EntityType.PERSON.value, + "action_intents": [ActionIntent.READ.value, ActionIntent.CONTACT.value], + "graph_roles": [GraphRole.PROFILE_ANCHOR.value], + "analyze_strategy": AnalyzeStrategy.ANALYZE_ENTITY_ONLY.value, + "llm_policy": LLMPolicy.LLM_LIGHT.value, + "should_analyze": True, + }, + PageType.PRICING_PAGE.value: { + "domain": PageDomain.COMMERCE.value, + "archetype": PageArchetype.LANDING.value, + "main_entity_type": EntityType.SERVICE.value, + "action_intents": [ActionIntent.BUY.value, ActionIntent.COMPARE.value], + "graph_roles": [GraphRole.CLAIM_SOURCE.value], + "analyze_strategy": AnalyzeStrategy.ANALYZE_FULL.value, + "llm_policy": LLMPolicy.LLM_LIGHT.value, + "should_analyze": True, + }, + PageType.LOGIN_PAGE.value: { + "domain": PageDomain.TRANSACTION.value, + "archetype": PageArchetype.FORM.value, + "main_entity_type": None, + "action_intents": [ActionIntent.LOGIN.value], + "graph_roles": [GraphRole.TRANSACTION_ONLY.value], + "analyze_strategy": AnalyzeStrategy.SKIP_PROTECTED.value, + "llm_policy": LLMPolicy.SKIP.value, + "should_analyze": False, + }, + PageType.CHECKOUT_PAGE.value: { + "domain": PageDomain.TRANSACTION.value, + "archetype": PageArchetype.TRANSACTION.value, + "main_entity_type": None, + "action_intents": [ActionIntent.BUY.value, ActionIntent.PAY.value], + "graph_roles": [GraphRole.TRANSACTION_ONLY.value], + "analyze_strategy": AnalyzeStrategy.SKIP_PROTECTED.value, + "llm_policy": LLMPolicy.SKIP.value, + "should_analyze": False, + }, + PageType.PAYMENT_PAGE.value: { + "domain": PageDomain.TRANSACTION.value, + "archetype": PageArchetype.TRANSACTION.value, + "main_entity_type": None, + "action_intents": [ActionIntent.PAY.value], + "graph_roles": [GraphRole.TRANSACTION_ONLY.value], + "analyze_strategy": AnalyzeStrategy.SKIP_PROTECTED.value, + "llm_policy": LLMPolicy.SKIP.value, + "should_analyze": False, + }, + PageType.TERMS_PAGE.value: { + "domain": PageDomain.CORPORATE.value, + "archetype": PageArchetype.DOCUMENT.value, + "main_entity_type": EntityType.LEGAL_DOCUMENT.value, + "action_intents": [ActionIntent.READ.value], + "graph_roles": [GraphRole.POLICY_SOURCE.value], + "analyze_strategy": AnalyzeStrategy.ANALYZE_DOCUMENT_ONLY.value, + "llm_policy": LLMPolicy.LLM_LIGHT.value, + "should_analyze": True, + }, + PageType.PRIVACY_POLICY_PAGE.value: { + "domain": PageDomain.CORPORATE.value, + "archetype": PageArchetype.DOCUMENT.value, + "main_entity_type": EntityType.LEGAL_DOCUMENT.value, + "action_intents": [ActionIntent.READ.value], + "graph_roles": [GraphRole.POLICY_SOURCE.value], + "analyze_strategy": AnalyzeStrategy.ANALYZE_DOCUMENT_ONLY.value, + "llm_policy": LLMPolicy.LLM_LIGHT.value, + "should_analyze": True, + }, + PageType.SITEMAP_PAGE.value: { + "domain": PageDomain.SYSTEM.value, + "archetype": PageArchetype.SYSTEM_RESOURCE.value, + "main_entity_type": None, + "action_intents": [ActionIntent.NAVIGATE.value], + "graph_roles": [GraphRole.SYSTEM_RESOURCE.value, GraphRole.NAVIGATION_HUB.value], + "analyze_strategy": AnalyzeStrategy.ANALYZE_DISCOVERY_ONLY.value, + "llm_policy": LLMPolicy.RULE_ONLY.value, + "should_analyze": True, + }, + PageType.RSS_FEED_PAGE.value: { + "domain": PageDomain.SYSTEM.value, + "archetype": PageArchetype.SYSTEM_RESOURCE.value, + "main_entity_type": None, + "action_intents": [ActionIntent.READ.value], + "graph_roles": [GraphRole.SYSTEM_RESOURCE.value], + "analyze_strategy": AnalyzeStrategy.ANALYZE_METADATA_ONLY.value, + "llm_policy": LLMPolicy.RULE_ONLY.value, + "should_analyze": True, + }, + PageType.ERROR_PAGE.value: { + "domain": PageDomain.SYSTEM.value, + "archetype": PageArchetype.ERROR.value, + "main_entity_type": None, + "action_intents": [], + "graph_roles": [GraphRole.NOISE_PAGE.value], + "analyze_strategy": AnalyzeStrategy.SKIP_NOISE.value, + "llm_policy": LLMPolicy.SKIP.value, + "should_analyze": False, + }, + PageType.NOT_FOUND_PAGE.value: { + "domain": PageDomain.SYSTEM.value, + "archetype": PageArchetype.ERROR.value, + "main_entity_type": None, + "action_intents": [], + "graph_roles": [GraphRole.NOISE_PAGE.value], + "analyze_strategy": AnalyzeStrategy.SKIP_NOISE.value, + "llm_policy": LLMPolicy.SKIP.value, + "should_analyze": False, + }, + PageType.ACCESS_DENIED_PAGE.value: { + "domain": PageDomain.TRANSACTION.value, + "archetype": PageArchetype.ERROR.value, + "main_entity_type": None, + "action_intents": [ActionIntent.VERIFY.value], + "graph_roles": [GraphRole.TRANSACTION_ONLY.value], + "analyze_strategy": AnalyzeStrategy.SKIP_PROTECTED.value, + "llm_policy": LLMPolicy.SKIP.value, + "should_analyze": False, + }, + PageType.CAPTCHA_PAGE.value: { + "domain": PageDomain.TRANSACTION.value, + "archetype": PageArchetype.FORM.value, + "main_entity_type": None, + "action_intents": [ActionIntent.VERIFY.value], + "graph_roles": [GraphRole.TRANSACTION_ONLY.value], + "analyze_strategy": AnalyzeStrategy.SKIP_PROTECTED.value, + "llm_policy": LLMPolicy.SKIP.value, + "should_analyze": False, + }, +} + + +def score_page_type(url: str, signals: PageSignals, *, unknown_threshold: float = UNKNOWN_THRESHOLD) -> PageClassificationResult: + accumulator = ScoreAccumulator(scores={}, evidence_by_type={}) + for page_type, rules in SCORING_RULES.items(): + for rule in rules: + accumulator.add(page_type, rule, signals) + + _apply_interactions(accumulator, signals) + + if not accumulator.scores: + return _unknown_result( + url, + alternatives=[], + evidence=[ + EvidenceItem( + key="insufficient_evidence", + value=True, + weight=0.0, + source="page_type_scorer", + message="No page-type evidence was detected.", + ) + ], + ) + + alternatives = sorted( + ((page_type, round(min(score, 1.0), 4)) for page_type, score in accumulator.scores.items()), + key=lambda item: item[1], + reverse=True, + ) + primary_page_type, confidence = alternatives[0] + if confidence < unknown_threshold: + evidence = [item for page_type, _score in alternatives[:3] for item in accumulator.evidence_by_type.get(page_type, [])] + evidence.append( + EvidenceItem( + key="low_confidence", + value=confidence, + weight=0.0, + source="page_type_scorer", + message=f"Top score {confidence:.2f} is below UnknownPage threshold {unknown_threshold:.2f}.", + ) + ) + return _unknown_result(url, alternatives=alternatives[:5], evidence=evidence) + + profile = _profile(primary_page_type) + evidence = accumulator.evidence_by_type.get(primary_page_type, []) + if not evidence: + evidence = [ + EvidenceItem( + key="score", + value=confidence, + weight=confidence, + source="page_type_scorer", + message=f"{primary_page_type} selected from accumulated score.", + ) + ] + secondary = [page_type for page_type, score in alternatives[1:4] if score >= 0.18] + return PageClassificationResult( + url=url, + primary_page_type=primary_page_type, + secondary_page_types=secondary, + domain=str(profile["domain"]), + archetype=str(profile["archetype"]), + main_entity_type=profile.get("main_entity_type"), + action_intents=list(profile.get("action_intents") or []), + graph_roles=list(profile.get("graph_roles") or []), + confidence=confidence, + alternatives=alternatives[:8], + evidence=evidence, + should_analyze=bool(profile.get("should_analyze")), + analyze_strategy=str(profile["analyze_strategy"]), + llm_policy=str(profile["llm_policy"]), + is_protected=str(profile["analyze_strategy"]) == AnalyzeStrategy.SKIP_PROTECTED.value, + is_noise=str(profile["analyze_strategy"]) == AnalyzeStrategy.SKIP_NOISE.value, + ) + + +def _profile(page_type: str) -> dict[str, Any]: + return PAGE_TYPE_METADATA.get(page_type, PAGE_TYPE_METADATA[PageType.UNKNOWN_PAGE.value]) + + +def _unknown_result( + url: str, + *, + alternatives: list[tuple[str, float]], + evidence: list[EvidenceItem], +) -> PageClassificationResult: + profile = _profile(PageType.UNKNOWN_PAGE.value) + return PageClassificationResult( + url=url, + primary_page_type=PageType.UNKNOWN_PAGE.value, + secondary_page_types=[page_type for page_type, _score in alternatives[:3]], + domain=str(profile["domain"]), + archetype=str(profile["archetype"]), + main_entity_type=profile.get("main_entity_type"), + action_intents=list(profile.get("action_intents") or []), + graph_roles=list(profile.get("graph_roles") or []), + confidence=0.0, + alternatives=alternatives[:8], + evidence=evidence + or [ + EvidenceItem( + key="unknown", + value=True, + weight=0.0, + source="page_type_scorer", + message="Page did not match known semantic page type evidence.", + ) + ], + should_analyze=bool(profile.get("should_analyze")), + analyze_strategy=str(profile["analyze_strategy"]), + llm_policy=str(profile["llm_policy"]), + is_protected=False, + is_noise=False, + ) + + +def _apply_interactions(accumulator: ScoreAccumulator, signals: PageSignals) -> None: + # Protected signals must dominate commerce signals when login/payment fields are present. + if signals.has_password_field and signals.has_login_form: + _manual_add(accumulator, PageType.LOGIN_PAGE.value, "protected_priority", True, 0.3, "protected", "Login form and password field override commerce signals.") + if signals.has_payment_fields: + _manual_add(accumulator, PageType.PAYMENT_PAGE.value, "payment_priority", True, 0.35, "protected", "Payment fields override product or checkout content.") + if signals.has_access_denied: + _manual_add(accumulator, PageType.ACCESS_DENIED_PAGE.value, "access_denied_priority", True, 0.35, "protected", "Access denied signal is a protected page indicator.") + if signals.has_captcha: + _manual_add(accumulator, PageType.CAPTCHA_PAGE.value, "captcha_priority", True, 0.35, "protected", "Captcha signal is a protected page indicator.") + if signals.has_repeated_cards and signals.product_link_count >= 3: + _manual_add(accumulator, PageType.CATEGORY_LISTING_PAGE.value, "listing_product_links", signals.product_link_count, 0.18, "link_graph", "Repeated cards with multiple product links indicate a product listing.") + if signals.has_question and signals.has_answer and signals.has_faq_structure: + _manual_add(accumulator, PageType.FAQ_PAGE.value, "faq_question_answer", True, 0.18, "community", "FAQ structure with question/answer content.") + + +def _manual_add( + accumulator: ScoreAccumulator, + page_type: str, + key: str, + value: str | int | float | bool | None, + weight: float, + source: str, + message: str, +) -> None: + accumulator.scores[page_type] = accumulator.scores.get(page_type, 0.0) + weight + accumulator.evidence_by_type.setdefault(page_type, []).append( + EvidenceItem(key=key, value=value, weight=weight, source=source, message=message) + ) + + +def _has_schema(*schema_types: str) -> Callable[[PageSignals], bool]: + return lambda signals: any(schema_type in signals.schema_types for schema_type in schema_types) + + +def _keyword(group: str, minimum: int = 1) -> Callable[[PageSignals], bool]: + return lambda signals: int(signals.keyword_hits.get(group) or 0) >= minimum + + +def _url_hint(hint: str) -> Callable[[PageSignals], bool]: + return lambda signals: hint in signals.url_hints + + +def _contains_terms(signals: PageSignals, terms: tuple[str, ...]) -> bool: + text = f"{signals.title or ''}\n{signals.text_sample or ''}".lower() + return any(term.lower() in text for term in terms) + + +def _flag(name: str) -> Callable[[PageSignals], bool]: + return lambda signals: bool(getattr(signals, name)) + + +def _count_at_least(name: str, minimum: int) -> Callable[[PageSignals], bool]: + return lambda signals: int(getattr(signals, name) or 0) >= minimum + + +def _value(name: str) -> Callable[[PageSignals], str | int | float | bool | None]: + return lambda signals: getattr(signals, name) + + +def _keywords_value(group: str) -> Callable[[PageSignals], str | int | float | bool | None]: + return lambda signals: signals.keyword_hits.get(group, 0) + + +def _schema_value(signals: PageSignals) -> str: + return ",".join(sorted(signals.schema_types)) + + +def _url_hint_value(signals: PageSignals) -> str: + return ",".join(sorted(signals.url_hints)) + + +SCORING_RULES: dict[str, list[SignalRule]] = { + PageType.PRODUCT_DETAIL_PAGE.value: [ + SignalRule("schema_product", 0.4, "structured_data", "schema.org Product detected.", _has_schema("Product"), _schema_value), + SignalRule("schema_offer", 0.15, "structured_data", "schema.org Offer detected.", _has_schema("Offer"), _schema_value), + SignalRule("price", 0.15, "text", "Price detected.", _flag("has_price"), _value("has_price")), + SignalRule("cart_button", 0.2, "dom", "Cart button detected.", _flag("has_cart_button")), + SignalRule("buy_button", 0.16, "dom", "Buy button detected.", _flag("has_buy_button")), + SignalRule("variant_selector", 0.12, "dom", "Variant selector detected.", _flag("has_variant_selector")), + SignalRule("sku", 0.1, "text", "SKU or product code detected.", _flag("has_sku")), + SignalRule("product_gallery", 0.1, "layout", "Product image gallery detected.", _flag("has_product_gallery")), + SignalRule("review_section", 0.05, "dom", "Review section detected.", _flag("has_review_section")), + SignalRule("url_product_hint", 0.05, "url", "Product URL hint detected.", _url_hint("product"), _url_hint_value), + ], + PageType.CATEGORY_LISTING_PAGE.value: [ + SignalRule("repeated_cards", 0.32, "layout", "Repeated cards detected.", _flag("has_repeated_cards"), _value("repeated_card_count")), + SignalRule("filter_panel", 0.2, "dom", "Filter panel detected.", _flag("has_filter_panel")), + SignalRule("sort_control", 0.15, "dom", "Sort control detected.", _flag("has_sort_control")), + SignalRule("pagination", 0.1, "dom", "Pagination detected.", _flag("has_pagination")), + SignalRule("product_links", 0.18, "link_graph", "Multiple product links detected.", _count_at_least("product_link_count", 3), _value("product_link_count")), + SignalRule("category_url_hint", 0.06, "url", "Category/list URL hint detected.", _url_hint("category"), _url_hint_value), + ], + PageType.SEARCH_RESULTS_PAGE.value: [ + SignalRule("search_url_hint", 0.28, "url", "Search URL hint detected.", _url_hint("search"), _url_hint_value), + SignalRule("listing_results", 0.18, "layout", "Repeated result cards detected.", _flag("has_repeated_cards"), _value("repeated_card_count")), + SignalRule("filter_panel", 0.14, "dom", "Search filter panel detected.", _flag("has_filter_panel")), + SignalRule("pagination", 0.12, "dom", "Search pagination detected.", _flag("has_pagination")), + SignalRule("search_keywords", 0.12, "text", "Search/result keywords detected.", _keyword("listing"), _keywords_value("listing")), + ], + PageType.ARTICLE_PAGE.value: [ + SignalRule("schema_article", 0.35, "structured_data", "Article structured data detected.", _has_schema("Article", "NewsArticle"), _schema_value), + SignalRule("author", 0.15, "text", "Author/byline signal detected.", _flag("has_author")), + SignalRule("published_date", 0.15, "text", "Published date detected.", _flag("has_published_date")), + SignalRule("article_body", 0.2, "dom", "Article body detected.", _flag("has_article_body")), + SignalRule("tags", 0.05, "dom", "Article tags detected.", _flag("has_tags")), + SignalRule("article_url_hint", 0.05, "url", "Article URL hint detected.", _url_hint("article"), _url_hint_value), + ], + PageType.BLOG_POST_PAGE.value: [ + SignalRule("blog_url_hint", 0.25, "url", "Blog URL hint detected.", lambda signals: "article" in signals.url_hints and "blog" in (signals.text_sample or "").lower()), + SignalRule("author", 0.14, "text", "Author signal detected.", _flag("has_author")), + SignalRule("published_date", 0.14, "text", "Published date detected.", _flag("has_published_date")), + SignalRule("article_body", 0.18, "dom", "Article body detected.", _flag("has_article_body")), + SignalRule("tags", 0.08, "dom", "Tags detected.", _flag("has_tags")), + ], + PageType.QA_PAGE.value: [ + SignalRule("schema_qapage", 0.35, "structured_data", "QAPage structured data detected.", _has_schema("QAPage"), _schema_value), + SignalRule("question", 0.2, "community", "Question block detected.", _flag("has_question")), + SignalRule("answer", 0.2, "community", "Answer block detected.", _flag("has_answer")), + SignalRule("votes", 0.1, "community", "Vote signal detected.", _flag("has_votes")), + SignalRule("comments", 0.05, "community", "Comments detected.", _flag("has_comments")), + SignalRule("board_url_hint", 0.05, "url", "Board/community URL hint detected.", _url_hint("board"), _url_hint_value), + ], + PageType.FAQ_PAGE.value: [ + SignalRule("schema_faq", 0.36, "structured_data", "FAQPage structured data detected.", _has_schema("FAQPage"), _schema_value), + SignalRule("faq_structure", 0.25, "community", "FAQ structure detected.", _flag("has_faq_structure")), + SignalRule("question", 0.15, "community", "Question content detected.", _flag("has_question")), + SignalRule("answer", 0.15, "community", "Answer content detected.", _flag("has_answer")), + ], + PageType.FORUM_BOARD_PAGE.value: [ + SignalRule("board_url_hint", 0.25, "url", "Board URL hint detected.", _url_hint("board"), _url_hint_value), + SignalRule("thread_structure", 0.2, "community", "Thread structure detected.", _flag("has_thread_structure")), + SignalRule("comments", 0.1, "community", "Comments/replies detected.", _flag("has_comments")), + SignalRule("repeated_cards", 0.14, "layout", "Repeated post cards detected.", _flag("has_repeated_cards"), _value("repeated_card_count")), + SignalRule("pagination", 0.1, "dom", "Board pagination detected.", _flag("has_pagination")), + ], + PageType.FORUM_THREAD_PAGE.value: [ + SignalRule("thread_structure", 0.3, "community", "Thread structure detected.", _flag("has_thread_structure")), + SignalRule("comments", 0.16, "community", "Comment thread detected.", _flag("has_comments")), + SignalRule("question_answer", 0.14, "community", "Question and answer content detected.", lambda signals: signals.has_question and signals.has_answer), + SignalRule("votes", 0.08, "community", "Vote signal detected.", _flag("has_votes")), + SignalRule("article_body", 0.08, "dom", "Post body detected.", _flag("has_article_body")), + ], + PageType.BRAND_STORY_PAGE.value: [ + SignalRule("corporate_keywords", 0.2, "text", "Corporate/brand keywords detected.", _keyword("corporate"), _keywords_value("corporate")), + SignalRule("about_url_hint", 0.18, "url", "About/company URL hint detected.", lambda signals: "about" in (signals.text_sample or "").lower() or "brand" in (signals.title or "").lower()), + SignalRule("article_body", 0.14, "dom", "Brand story body detected.", _flag("has_article_body")), + SignalRule("hero_block", 0.08, "layout", "Hero block detected.", _flag("has_hero_block")), + ], + PageType.ABOUT_PAGE.value: [ + SignalRule("corporate_keywords", 0.22, "text", "About/company keywords detected.", _keyword("corporate"), _keywords_value("corporate")), + SignalRule("contact_info", 0.08, "text", "Organization contact signal detected.", _flag("has_contact_info")), + SignalRule("article_body", 0.12, "dom", "About body detected.", _flag("has_article_body")), + ], + PageType.CONTACT_PAGE.value: [ + SignalRule("contact_info", 0.32, "text", "Contact information detected.", _flag("has_contact_info")), + SignalRule("address", 0.18, "text", "Address detected.", _flag("has_address")), + SignalRule("corporate_keywords", 0.1, "text", "Corporate keywords detected.", _keyword("corporate"), _keywords_value("corporate")), + ], + PageType.DOCUMENTATION_PAGE.value: [ + SignalRule("docs_keywords", 0.18, "text", "Documentation keywords detected.", _keyword("knowledge"), _keywords_value("knowledge")), + SignalRule("toc", 0.16, "dom", "Table of contents detected.", _flag("has_toc")), + SignalRule("code_blocks", 0.16, "dom", "Code blocks detected.", _flag("has_code_blocks")), + SignalRule("version_info", 0.08, "text", "Version information detected.", _flag("has_version_info")), + ], + PageType.API_REFERENCE_PAGE.value: [ + SignalRule("api_endpoint", 0.3, "text", "API endpoint detected.", _flag("has_api_endpoint")), + SignalRule("parameter_table", 0.22, "dom", "Parameter table detected.", _flag("has_parameter_table")), + SignalRule("code_blocks", 0.1, "dom", "Code blocks detected.", _flag("has_code_blocks")), + SignalRule("docs_keywords", 0.12, "text", "API/docs keywords detected.", _keyword("knowledge"), _keywords_value("knowledge")), + ], + PageType.WIKI_PAGE.value: [ + SignalRule("wiki_schema", 0.28, "structured_data", "Wiki/DefinedTerm structured data detected.", _has_schema("DefinedTerm", "WebPage"), _schema_value), + SignalRule("toc", 0.12, "dom", "Reference table of contents detected.", _flag("has_toc")), + SignalRule("definition_terms", 0.14, "text", "Definition/reference keywords detected.", lambda signals: _contains_terms(signals, ("definition", "wiki", "glossary", "reference"))), + ], + PageType.DATASET_PAGE.value: [ + SignalRule("schema_dataset", 0.42, "structured_data", "Dataset structured data detected.", _has_schema("Dataset", "DataCatalog"), _schema_value), + SignalRule("download_terms", 0.12, "text", "Dataset/download keywords detected.", lambda signals: _contains_terms(signals, ("dataset", "data catalog", "download", "csv"))), + SignalRule("parameter_table", 0.08, "dom", "Dataset metadata table detected.", _flag("has_parameter_table")), + ], + PageType.RESEARCH_PAPER_PAGE.value: [ + SignalRule("schema_scholarly", 0.42, "structured_data", "Scholarly article structured data detected.", _has_schema("ScholarlyArticle", "TechArticle"), _schema_value), + SignalRule("published_date", 0.1, "text", "Publication date detected.", _flag("has_published_date")), + SignalRule("citation_terms", 0.16, "text", "Citation/references keywords detected.", lambda signals: _contains_terms(signals, ("abstract", "citation", "references", "doi"))), + ], + PageType.JOB_POSTING_PAGE.value: [ + SignalRule("career_keywords", 0.26, "text", "Career/job keywords detected.", _flag("has_career_terms")), + SignalRule("apply_intent", 0.12, "text", "Apply intent detected.", lambda signals: "apply" in (signals.text_sample or "").lower() or "지원" in (signals.text_sample or "")), + SignalRule("address", 0.06, "text", "Location/address signal detected.", _flag("has_address")), + ], + PageType.COURSE_DETAIL_PAGE.value: [ + SignalRule("schema_course", 0.42, "structured_data", "Course structured data detected.", _has_schema("Course"), _schema_value), + SignalRule("course_terms", 0.18, "text", "Course/curriculum keywords detected.", lambda signals: _contains_terms(signals, ("course", "lesson", "curriculum", "instructor", "syllabus"))), + ], + PageType.VIDEO_PAGE.value: [ + SignalRule("schema_video", 0.42, "structured_data", "VideoObject structured data detected.", _has_schema("VideoObject"), _schema_value), + SignalRule("media_player", 0.2, "layout", "Video/audio player detected.", _flag("has_media_player_area")), + SignalRule("video_terms", 0.1, "text", "Video/watch keywords detected.", lambda signals: _contains_terms(signals, ("video", "watch", "episode", "duration"))), + ], + PageType.LOCAL_BUSINESS_PAGE.value: [ + SignalRule("schema_local_business", 0.42, "structured_data", "Local business/place structured data detected.", _has_schema("LocalBusiness", "Place", "Restaurant"), _schema_value), + SignalRule("address", 0.18, "text", "Address detected.", _flag("has_address")), + SignalRule("contact_info", 0.12, "text", "Contact info detected.", _flag("has_contact_info")), + SignalRule("map_area", 0.1, "layout", "Map area detected.", _flag("has_map_area")), + ], + PageType.REAL_ESTATE_LISTING_PAGE.value: [ + SignalRule("schema_real_estate", 0.38, "structured_data", "Real estate structured data detected.", _has_schema("RealEstateListing", "Residence", "Apartment"), _schema_value), + SignalRule("property_terms", 0.18, "text", "Property listing keywords detected.", lambda signals: _contains_terms(signals, ("bedroom", "bathroom", "sqft", "property", "real estate"))), + SignalRule("price", 0.12, "text", "Property price detected.", _flag("has_price")), + SignalRule("address", 0.08, "text", "Property address detected.", _flag("has_address")), + ], + PageType.PROFILE_PAGE.value: [ + SignalRule("schema_person", 0.34, "structured_data", "Person/Profile structured data detected.", _has_schema("Person", "ProfilePage"), _schema_value), + SignalRule("profile_links", 0.12, "link_graph", "Profile link pattern detected.", _count_at_least("profile_link_count", 1), _value("profile_link_count")), + SignalRule("author", 0.1, "text", "Author/person signal detected.", _flag("has_author")), + ], + PageType.PRICING_PAGE.value: [ + SignalRule("pricing_table", 0.3, "layout", "Pricing table detected.", _flag("has_pricing_table")), + SignalRule("price", 0.16, "text", "Price detected.", _flag("has_price")), + SignalRule("comparison_table", 0.12, "layout", "Comparison table detected.", _flag("has_comparison_table")), + SignalRule("commerce_keywords", 0.1, "text", "Commerce keywords detected.", _keyword("commerce"), _keywords_value("commerce")), + ], + PageType.LOGIN_PAGE.value: [ + SignalRule("password_field", 0.4, "form", "Password field detected.", _flag("has_password_field")), + SignalRule("login_form", 0.25, "form", "Login form detected.", _flag("has_login_form")), + SignalRule("protected_url_hint", 0.08, "url", "Protected URL hint detected.", _url_hint("protected"), _url_hint_value), + ], + PageType.CHECKOUT_PAGE.value: [ + SignalRule("protected_url_hint", 0.25, "url", "Checkout/cart URL hint detected.", _url_hint("protected"), _url_hint_value), + SignalRule("buy_button", 0.14, "dom", "Purchase button detected.", _flag("has_buy_button")), + SignalRule("commerce_keywords", 0.12, "text", "Checkout commerce keywords detected.", _keyword("commerce"), _keywords_value("commerce")), + ], + PageType.PAYMENT_PAGE.value: [ + SignalRule("payment_fields", 0.42, "form", "Payment fields detected.", _flag("has_payment_fields")), + SignalRule("protected_keywords", 0.16, "text", "Payment/protected keywords detected.", _keyword("protected"), _keywords_value("protected")), + SignalRule("protected_url_hint", 0.1, "url", "Payment URL hint detected.", _url_hint("protected"), _url_hint_value), + ], + PageType.TERMS_PAGE.value: [ + SignalRule("policy_terms", 0.32, "text", "Policy/terms terms detected.", _flag("has_policy_terms")), + SignalRule("corporate_keywords", 0.08, "text", "Corporate legal keywords detected.", _keyword("corporate"), _keywords_value("corporate")), + ], + PageType.PRIVACY_POLICY_PAGE.value: [ + SignalRule("privacy_terms", 0.36, "text", "Privacy terms detected.", _flag("has_privacy_terms")), + SignalRule("policy_terms", 0.14, "text", "Policy terms detected.", _flag("has_policy_terms")), + ], + PageType.SITEMAP_PAGE.value: [ + SignalRule("sitemap_resource", 0.45, "resource", "Sitemap resource detected.", _flag("has_sitemap_resource")), + SignalRule("xml_resource", 0.12, "resource", "XML resource detected.", _flag("has_xml_resource")), + SignalRule("system_url_hint", 0.12, "url", "System URL hint detected.", _url_hint("system"), _url_hint_value), + ], + PageType.RSS_FEED_PAGE.value: [ + SignalRule("feed_resource", 0.45, "resource", "RSS/Atom feed detected.", _flag("has_feed_resource")), + SignalRule("xml_resource", 0.1, "resource", "XML feed resource detected.", _flag("has_xml_resource")), + ], + PageType.ERROR_PAGE.value: [ + SignalRule("error_status", 0.38, "http", "Error status detected.", _flag("has_error_status")), + ], + PageType.NOT_FOUND_PAGE.value: [ + SignalRule("not_found", 0.5, "http", "404/not found signal detected.", _flag("has_not_found")), + SignalRule("error_status", 0.12, "http", "Error status supports not found page.", _flag("has_error_status")), + ], + PageType.ACCESS_DENIED_PAGE.value: [ + SignalRule("access_denied", 0.42, "protected", "Access denied text detected.", _flag("has_access_denied")), + SignalRule("error_status", 0.1, "http", "Error status supports access denied.", _flag("has_error_status")), + ], + PageType.CAPTCHA_PAGE.value: [ + SignalRule("captcha", 0.45, "protected", "Captcha detected.", _flag("has_captcha")), + SignalRule("protected_keywords", 0.1, "text", "Protected keywords detected.", _keyword("protected"), _keywords_value("protected")), + ], +} diff --git a/ontology_platform/crawler_platform/app/core/crawler/page_type_taxonomy.py b/ontology_platform/crawler_platform/app/core/crawler/page_type_taxonomy.py new file mode 100644 index 0000000..29fe84d --- /dev/null +++ b/ontology_platform/crawler_platform/app/core/crawler/page_type_taxonomy.py @@ -0,0 +1,476 @@ +from __future__ import annotations + +from dataclasses import asdict, dataclass, field +from enum import StrEnum +from typing import Any + + +class PageDomain(StrEnum): + COMMERCE = "Commerce" + EDITORIAL = "Editorial" + COMMUNITY = "Community" + KNOWLEDGE = "Knowledge" + CORPORATE = "Corporate" + LOCAL = "Local" + EDUCATION = "Education" + JOBS = "Jobs" + MEDIA = "Media" + SOFTWARE = "Software" + FINANCE = "Finance" + GOVERNMENT = "Government" + HEALTHCARE = "Healthcare" + TRANSACTION = "Transaction" + SYSTEM = "System" + UNKNOWN = "Unknown" + + +class PageArchetype(StrEnum): + HOME = "Home" + LANDING = "Landing" + DETAIL = "Detail" + LISTING = "Listing" + COLLECTION = "Collection" + SEARCH_RESULT = "SearchResult" + PROFILE = "Profile" + ARTICLE = "Article" + THREAD = "Thread" + FORM = "Form" + TRANSACTION = "Transaction" + DASHBOARD = "Dashboard" + DOCUMENT = "Document" + MEDIA = "Media" + ERROR = "Error" + SYSTEM_RESOURCE = "SystemResource" + UNKNOWN = "Unknown" + + +class PageType(StrEnum): + PRODUCT_PAGE = "ProductPage" + CATEGORY_PAGE = "CategoryPage" + SEARCH_PAGE = "SearchPage" + BOARD_PAGE = "BoardPage" + NOTICE_PAGE = "NoticePage" + BRAND_STORY_PAGE = "BrandStoryPage" + ABOUT_PAGE = "AboutPage" + CONTACT_PAGE = "ContactPage" + PROMOTION_PAGE = "PromotionPage" + REVIEW_PAGE = "ReviewPage" + UNKNOWN_PAGE = "UnknownPage" + PRODUCT_DETAIL_PAGE = "ProductDetailPage" + CATEGORY_LISTING_PAGE = "CategoryListingPage" + SEARCH_RESULTS_PAGE = "SearchResultsPage" + FORUM_BOARD_PAGE = "ForumBoardPage" + FORUM_THREAD_PAGE = "ForumThreadPage" + PUBLIC_NOTICE_PAGE = "PublicNoticePage" + CAMPAIGN_LANDING_PAGE = "CampaignLandingPage" + ARTICLE_PAGE = "ArticlePage" + NEWS_ARTICLE_PAGE = "NewsArticlePage" + BLOG_POST_PAGE = "BlogPostPage" + FAQ_PAGE = "FAQPage" + QA_PAGE = "QAPage" + PROFILE_PAGE = "ProfilePage" + DOCUMENTATION_PAGE = "DocumentationPage" + API_REFERENCE_PAGE = "APIReferencePage" + WIKI_PAGE = "WikiPage" + DATASET_PAGE = "DatasetPage" + RESEARCH_PAPER_PAGE = "ResearchPaperPage" + JOB_POSTING_PAGE = "JobPostingPage" + COURSE_DETAIL_PAGE = "CourseDetailPage" + VIDEO_PAGE = "VideoPage" + LOCAL_BUSINESS_PAGE = "LocalBusinessPage" + REAL_ESTATE_LISTING_PAGE = "RealEstateListingPage" + PRICING_PAGE = "PricingPage" + LOGIN_PAGE = "LoginPage" + CHECKOUT_PAGE = "CheckoutPage" + PAYMENT_PAGE = "PaymentPage" + TERMS_PAGE = "TermsPage" + PRIVACY_POLICY_PAGE = "PrivacyPolicyPage" + SITEMAP_PAGE = "SitemapPage" + RSS_FEED_PAGE = "RSSFeedPage" + ERROR_PAGE = "ErrorPage" + NOT_FOUND_PAGE = "NotFoundPage" + ACCESS_DENIED_PAGE = "AccessDeniedPage" + CAPTCHA_PAGE = "CaptchaPage" + + +class EntityType(StrEnum): + PRODUCT = "Product" + SERVICE = "Service" + ARTICLE = "Article" + NEWS_ARTICLE = "NewsArticle" + PERSON = "Person" + ORGANIZATION = "Organization" + PLACE = "Place" + EVENT = "Event" + JOB_POSTING = "JobPosting" + COURSE = "Course" + QUESTION = "Question" + ANSWER = "Answer" + REVIEW = "Review" + DATASET = "Dataset" + SOFTWARE_APPLICATION = "SoftwareApplication" + MEDIA_OBJECT = "MediaObject" + RECIPE = "Recipe" + REAL_ESTATE_PROPERTY = "RealEstateProperty" + MEDICAL_CONDITION = "MedicalCondition" + LEGAL_DOCUMENT = "LegalDocument" + FINANCIAL_PRODUCT = "FinancialProduct" + UNKNOWN_ENTITY = "UnknownEntity" + + +class ActionIntent(StrEnum): + READ = "Read" + BUY = "Buy" + SUBSCRIBE = "Subscribe" + RESERVE = "Reserve" + BOOK = "Book" + APPLY = "Apply" + DOWNLOAD = "Download" + WATCH = "Watch" + LISTEN = "Listen" + SEARCH = "Search" + COMPARE = "Compare" + FILTER = "Filter" + ASK = "Ask" + ANSWER = "Answer" + COMMENT = "Comment" + REVIEW = "Review" + LOGIN = "Login" + REGISTER = "Register" + PAY = "Pay" + CONTACT = "Contact" + NAVIGATE = "Navigate" + LEARN = "Learn" + VERIFY = "Verify" + CONFIGURE = "Configure" + MANAGE = "Manage" + + +class GraphRole(StrEnum): + ENTITY_ANCHOR = "EntityAnchor" + RELATION_HUB = "RelationHub" + NAVIGATION_HUB = "NavigationHub" + COLLECTION_HUB = "CollectionHub" + SEARCH_HUB = "SearchHub" + TRANSACTION_ONLY = "TransactionOnly" + POLICY_SOURCE = "PolicySource" + CLAIM_SOURCE = "ClaimSource" + PROFILE_ANCHOR = "ProfileAnchor" + MEDIA_ANCHOR = "MediaAnchor" + REFERENCE_SOURCE = "ReferenceSource" + SYSTEM_RESOURCE = "SystemResource" + NOISE_PAGE = "NoisePage" + UNKNOWN_PATTERN = "UnknownPattern" + + +class AnalyzeStrategy(StrEnum): + ANALYZE_FULL = "AnalyzeFull" + ANALYZE_STRUCTURE_ONLY = "AnalyzeStructureOnly" + ANALYZE_ENTITY_ONLY = "AnalyzeEntityOnly" + ANALYZE_RELATIONS_ONLY = "AnalyzeRelationsOnly" + ANALYZE_METADATA_ONLY = "AnalyzeMetadataOnly" + ANALYZE_DOCUMENT_ONLY = "AnalyzeDocumentOnly" + ANALYZE_DISCOVERY_ONLY = "AnalyzeDiscoveryOnly" + SKIP_PROTECTED = "SkipProtected" + SKIP_NOISE = "SkipNoise" + + +class LLMPolicy(StrEnum): + LLM_FULL = "LLMFull" + LLM_LIGHT = "LLMLight" + LLM_FOR_AMBIGUITY_ONLY = "LLMForAmbiguityOnly" + RULE_ONLY = "RuleOnly" + NO_LLM = "NoLLM" + SKIP = "Skip" + + +@dataclass(frozen=True, slots=True) +class EvidenceItem: + key: str + value: str | int | float | bool | None + weight: float + source: str + message: str + + def to_dict(self) -> dict[str, Any]: + return asdict(self) + + +@dataclass(slots=True) +class PageClassificationResult: + url: str + primary_page_type: str + secondary_page_types: list[str] = field(default_factory=list) + domain: str = PageDomain.UNKNOWN.value + archetype: str = PageArchetype.UNKNOWN.value + main_entity_type: str | None = None + action_intents: list[str] = field(default_factory=list) + graph_roles: list[str] = field(default_factory=lambda: [GraphRole.UNKNOWN_PATTERN.value]) + confidence: float = 0.0 + alternatives: list[tuple[str, float]] = field(default_factory=list) + evidence: list[EvidenceItem] = field(default_factory=list) + should_analyze: bool = False + analyze_strategy: str = AnalyzeStrategy.ANALYZE_METADATA_ONLY.value + llm_policy: str = LLMPolicy.NO_LLM.value + is_protected: bool = False + is_noise: bool = False + + def to_dict(self) -> dict[str, Any]: + payload = asdict(self) + payload["evidence"] = [item.to_dict() for item in self.evidence] + payload["legacy_page_type"] = get_legacy_page_type(self) + return payload + + +LEGACY_PAGE_TYPES = { + PageType.PRODUCT_PAGE.value, + PageType.CATEGORY_PAGE.value, + PageType.SEARCH_PAGE.value, + PageType.BOARD_PAGE.value, + PageType.NOTICE_PAGE.value, + PageType.BRAND_STORY_PAGE.value, + PageType.PROMOTION_PAGE.value, + PageType.REVIEW_PAGE.value, + PageType.UNKNOWN_PAGE.value, + "EventPage", +} + + +LEGACY_ALIASES: dict[str, str] = { + "product": PageType.PRODUCT_PAGE.value, + "productpage": PageType.PRODUCT_PAGE.value, + "productdetailpage": PageType.PRODUCT_PAGE.value, + "brand": PageType.BRAND_STORY_PAGE.value, + "brandpage": PageType.BRAND_STORY_PAGE.value, + "brandstorypage": PageType.BRAND_STORY_PAGE.value, + "about": PageType.BRAND_STORY_PAGE.value, + "aboutpage": "AboutPage", + "contact": "ContactPage", + "contactpage": "ContactPage", + "review": PageType.REVIEW_PAGE.value, + "reviewpage": PageType.REVIEW_PAGE.value, + "productreviewpage": PageType.REVIEW_PAGE.value, + "listing": PageType.CATEGORY_PAGE.value, + "listingpage": PageType.CATEGORY_PAGE.value, + "category": PageType.CATEGORY_PAGE.value, + "categorypage": PageType.CATEGORY_PAGE.value, + "categorylistingpage": PageType.CATEGORY_PAGE.value, + "productlistingpage": PageType.CATEGORY_PAGE.value, + "community": PageType.BOARD_PAGE.value, + "communitypage": PageType.BOARD_PAGE.value, + "board": PageType.BOARD_PAGE.value, + "boardpage": PageType.BOARD_PAGE.value, + "forumboardpage": PageType.BOARD_PAGE.value, + "forumthreadpage": PageType.BOARD_PAGE.value, + "search": PageType.SEARCH_PAGE.value, + "searchpage": PageType.SEARCH_PAGE.value, + "searchresultspage": PageType.SEARCH_PAGE.value, + "notice": PageType.NOTICE_PAGE.value, + "noticepage": PageType.NOTICE_PAGE.value, + "publicnoticepage": PageType.NOTICE_PAGE.value, + "promotion": PageType.PROMOTION_PAGE.value, + "promotionpage": PageType.PROMOTION_PAGE.value, + "campaignlandingpage": PageType.PROMOTION_PAGE.value, + "event": "EventPage", + "eventpage": "EventPage", + "unknown": PageType.UNKNOWN_PAGE.value, + "unknownpage": PageType.UNKNOWN_PAGE.value, + "notfoundpage": PageType.NOT_FOUND_PAGE.value, + "404": PageType.NOT_FOUND_PAGE.value, +} + + +LEGACY_TO_SEMANTIC_PAGE_TYPE: dict[str, str] = { + PageType.PRODUCT_PAGE.value: PageType.PRODUCT_DETAIL_PAGE.value, + PageType.CATEGORY_PAGE.value: PageType.CATEGORY_LISTING_PAGE.value, + PageType.SEARCH_PAGE.value: PageType.SEARCH_RESULTS_PAGE.value, + PageType.BOARD_PAGE.value: PageType.FORUM_BOARD_PAGE.value, + PageType.NOTICE_PAGE.value: PageType.PUBLIC_NOTICE_PAGE.value, + PageType.BRAND_STORY_PAGE.value: PageType.BRAND_STORY_PAGE.value, + PageType.ABOUT_PAGE.value: PageType.ABOUT_PAGE.value, + PageType.CONTACT_PAGE.value: PageType.CONTACT_PAGE.value, + PageType.PROMOTION_PAGE.value: PageType.PROMOTION_PAGE.value, + PageType.REVIEW_PAGE.value: PageType.REVIEW_PAGE.value, + PageType.UNKNOWN_PAGE.value: PageType.UNKNOWN_PAGE.value, + "EventPage": "EventPage", +} + + +SEMANTIC_TO_LEGACY_PAGE_TYPE: dict[str, str] = { + semantic: legacy for legacy, semantic in LEGACY_TO_SEMANTIC_PAGE_TYPE.items() +} +SEMANTIC_TO_LEGACY_PAGE_TYPE.update( + { + PageType.PRODUCT_DETAIL_PAGE.value: PageType.PRODUCT_PAGE.value, + PageType.CATEGORY_LISTING_PAGE.value: PageType.CATEGORY_PAGE.value, + PageType.SEARCH_RESULTS_PAGE.value: PageType.SEARCH_PAGE.value, + PageType.FORUM_BOARD_PAGE.value: PageType.BOARD_PAGE.value, + PageType.FORUM_THREAD_PAGE.value: PageType.BOARD_PAGE.value, + PageType.PUBLIC_NOTICE_PAGE.value: PageType.NOTICE_PAGE.value, + PageType.CAMPAIGN_LANDING_PAGE.value: PageType.PROMOTION_PAGE.value, + PageType.ABOUT_PAGE.value: PageType.BRAND_STORY_PAGE.value, + PageType.CONTACT_PAGE.value: PageType.BRAND_STORY_PAGE.value, + "ListingPage": PageType.CATEGORY_PAGE.value, + "CommunityPage": PageType.BOARD_PAGE.value, + } +) + + +PAGE_TYPE_PROFILES: dict[str, dict[str, Any]] = { + PageType.PRODUCT_DETAIL_PAGE.value: { + "domain": PageDomain.COMMERCE.value, + "archetype": PageArchetype.DETAIL.value, + "main_entity_type": EntityType.PRODUCT.value, + "action_intents": [ActionIntent.BUY.value, ActionIntent.REVIEW.value], + "graph_roles": [GraphRole.ENTITY_ANCHOR.value], + "analyze_strategy": AnalyzeStrategy.ANALYZE_FULL.value, + "llm_policy": LLMPolicy.LLM_LIGHT.value, + "should_analyze": True, + }, + PageType.CATEGORY_LISTING_PAGE.value: { + "domain": PageDomain.COMMERCE.value, + "archetype": PageArchetype.LISTING.value, + "main_entity_type": EntityType.PRODUCT.value, + "action_intents": [ActionIntent.FILTER.value, ActionIntent.NAVIGATE.value], + "graph_roles": [GraphRole.COLLECTION_HUB.value, GraphRole.RELATION_HUB.value], + "analyze_strategy": AnalyzeStrategy.ANALYZE_RELATIONS_ONLY.value, + "llm_policy": LLMPolicy.LLM_FOR_AMBIGUITY_ONLY.value, + "should_analyze": True, + }, + PageType.SEARCH_RESULTS_PAGE.value: { + "domain": PageDomain.UNKNOWN.value, + "archetype": PageArchetype.SEARCH_RESULT.value, + "main_entity_type": EntityType.UNKNOWN_ENTITY.value, + "action_intents": [ActionIntent.SEARCH.value, ActionIntent.NAVIGATE.value], + "graph_roles": [GraphRole.SEARCH_HUB.value], + "analyze_strategy": AnalyzeStrategy.ANALYZE_DISCOVERY_ONLY.value, + "llm_policy": LLMPolicy.LLM_FOR_AMBIGUITY_ONLY.value, + "should_analyze": True, + }, + PageType.FORUM_BOARD_PAGE.value: { + "domain": PageDomain.COMMUNITY.value, + "archetype": PageArchetype.LISTING.value, + "main_entity_type": EntityType.ARTICLE.value, + "action_intents": [ActionIntent.NAVIGATE.value, ActionIntent.READ.value], + "graph_roles": [GraphRole.RELATION_HUB.value, GraphRole.COLLECTION_HUB.value], + "analyze_strategy": AnalyzeStrategy.ANALYZE_RELATIONS_ONLY.value, + "llm_policy": LLMPolicy.LLM_FOR_AMBIGUITY_ONLY.value, + "should_analyze": True, + }, + PageType.PUBLIC_NOTICE_PAGE.value: { + "domain": PageDomain.GOVERNMENT.value, + "archetype": PageArchetype.DOCUMENT.value, + "main_entity_type": EntityType.ARTICLE.value, + "action_intents": [ActionIntent.READ.value], + "graph_roles": [GraphRole.CLAIM_SOURCE.value], + "analyze_strategy": AnalyzeStrategy.ANALYZE_FULL.value, + "llm_policy": LLMPolicy.LLM_LIGHT.value, + "should_analyze": True, + }, + PageType.BRAND_STORY_PAGE.value: { + "domain": PageDomain.CORPORATE.value, + "archetype": PageArchetype.ARTICLE.value, + "main_entity_type": EntityType.ORGANIZATION.value, + "action_intents": [ActionIntent.READ.value, ActionIntent.LEARN.value], + "graph_roles": [GraphRole.ENTITY_ANCHOR.value, GraphRole.CLAIM_SOURCE.value], + "analyze_strategy": AnalyzeStrategy.ANALYZE_FULL.value, + "llm_policy": LLMPolicy.LLM_FULL.value, + "should_analyze": True, + }, + PageType.PROMOTION_PAGE.value: { + "domain": PageDomain.COMMERCE.value, + "archetype": PageArchetype.LANDING.value, + "main_entity_type": EntityType.EVENT.value, + "action_intents": [ActionIntent.BUY.value, ActionIntent.NAVIGATE.value], + "graph_roles": [GraphRole.CLAIM_SOURCE.value], + "analyze_strategy": AnalyzeStrategy.ANALYZE_FULL.value, + "llm_policy": LLMPolicy.LLM_LIGHT.value, + "should_analyze": True, + }, + PageType.REVIEW_PAGE.value: { + "domain": PageDomain.COMMUNITY.value, + "archetype": PageArchetype.ARTICLE.value, + "main_entity_type": EntityType.REVIEW.value, + "action_intents": [ActionIntent.READ.value, ActionIntent.REVIEW.value], + "graph_roles": [GraphRole.CLAIM_SOURCE.value], + "analyze_strategy": AnalyzeStrategy.ANALYZE_FULL.value, + "llm_policy": LLMPolicy.LLM_LIGHT.value, + "should_analyze": True, + }, + PageType.UNKNOWN_PAGE.value: { + "domain": PageDomain.UNKNOWN.value, + "archetype": PageArchetype.UNKNOWN.value, + "main_entity_type": EntityType.UNKNOWN_ENTITY.value, + "action_intents": [], + "graph_roles": [GraphRole.UNKNOWN_PATTERN.value], + "analyze_strategy": AnalyzeStrategy.ANALYZE_METADATA_ONLY.value, + "llm_policy": LLMPolicy.NO_LLM.value, + "should_analyze": False, + }, +} + + +def normalize_page_type(value: object | None) -> str: + """Return the legacy-compatible page type string for existing callers.""" + + if isinstance(value, PageClassificationResult): + return get_legacy_page_type(value) + clean = str(value or "").strip() + if not clean: + return PageType.UNKNOWN_PAGE.value + alias = LEGACY_ALIASES.get(clean.lower()) + if alias: + return alias + return SEMANTIC_TO_LEGACY_PAGE_TYPE.get(clean, clean) + + +def normalize_semantic_page_type(value: object | None) -> str: + if isinstance(value, PageClassificationResult): + return value.primary_page_type + legacy = normalize_page_type(value) + return LEGACY_TO_SEMANTIC_PAGE_TYPE.get(legacy, str(value or legacy).strip() or PageType.UNKNOWN_PAGE.value) + + +def get_legacy_page_type(value: object | None) -> str: + if isinstance(value, PageClassificationResult): + return SEMANTIC_TO_LEGACY_PAGE_TYPE.get(value.primary_page_type, value.primary_page_type) + return normalize_page_type(value) + + +def build_classification_result_from_legacy( + *, + url: str, + legacy_page_type: str, + confidence: float = 0.55, + source: str = "legacy_classifier", +) -> PageClassificationResult: + normalized_legacy = normalize_page_type(legacy_page_type) + semantic_page_type = LEGACY_TO_SEMANTIC_PAGE_TYPE.get(normalized_legacy, normalized_legacy) + profile = PAGE_TYPE_PROFILES.get(semantic_page_type, PAGE_TYPE_PROFILES[PageType.UNKNOWN_PAGE.value]) + evidence = [ + EvidenceItem( + key="legacy_page_type", + value=normalized_legacy, + weight=confidence, + source=source, + message=f"Legacy classifier returned {normalized_legacy}.", + ) + ] + return PageClassificationResult( + url=url, + primary_page_type=semantic_page_type, + secondary_page_types=[] if semantic_page_type == normalized_legacy else [normalized_legacy], + domain=str(profile["domain"]), + archetype=str(profile["archetype"]), + main_entity_type=profile.get("main_entity_type"), + action_intents=list(profile.get("action_intents") or []), + graph_roles=list(profile.get("graph_roles") or []), + confidence=confidence, + alternatives=[(normalized_legacy, confidence)], + evidence=evidence, + should_analyze=bool(profile.get("should_analyze")), + analyze_strategy=str(profile["analyze_strategy"]), + llm_policy=str(profile["llm_policy"]), + is_protected=str(profile["analyze_strategy"]) == AnalyzeStrategy.SKIP_PROTECTED.value, + is_noise=str(profile["analyze_strategy"]) == AnalyzeStrategy.SKIP_NOISE.value, + ) diff --git a/ontology_platform/crawler_platform/app/core/crawler/page_unknown_patterns.py b/ontology_platform/crawler_platform/app/core/crawler/page_unknown_patterns.py new file mode 100644 index 0000000..64bbd35 --- /dev/null +++ b/ontology_platform/crawler_platform/app/core/crawler/page_unknown_patterns.py @@ -0,0 +1,347 @@ +from __future__ import annotations + +from collections import Counter +import hashlib +import json +import re +from typing import Any +from urllib.parse import urljoin, urlparse + +from crawler_platform.app.core.crawler.page_signal_extractor import ( + build_raw_page_snapshot, + extract_page_signals, +) +from crawler_platform.app.core.crawler.page_type_taxonomy import PageClassificationResult, PageType + + +UNKNOWN_PATTERN_VERSION = 1 +LOW_CONFIDENCE_PATTERN_THRESHOLD = 0.45 +STOPWORDS = { + "about", + "after", + "again", + "also", + "and", + "are", + "but", + "can", + "for", + "from", + "has", + "have", + "home", + "into", + "more", + "not", + "our", + "page", + "that", + "the", + "this", + "with", + "your", +} + + +def should_store_unknown_pattern( + result: PageClassificationResult, + *, + confidence_threshold: float = LOW_CONFIDENCE_PATTERN_THRESHOLD, +) -> bool: + if result.primary_page_type == PageType.UNKNOWN_PAGE.value: + return True + if 0.0 < result.confidence <= confidence_threshold: + return True + return any(item.key == "low_confidence" for item in result.evidence) + + +def build_unknown_pattern_payload( + *, + result: PageClassificationResult, + url: str, + title: str | None = None, + text: str | None = None, + html: str | None = None, + source_zones: list[str | dict[str, Any]] | None = None, + collector_payload: dict[str, Any] | None = None, +) -> dict[str, Any]: + snapshot = build_raw_page_snapshot( + url=url, + title=title, + text=text, + html=html, + source_zones=source_zones, + collector_payload=collector_payload, + ) + signals = extract_page_signals(snapshot) + soup = _soup_from_html(html or "") + page_text = _normalize_text(text or signals.text_sample or _text_from_soup(soup)) + links = snapshot.links or _extract_links(soup, url) + buttons = snapshot.buttons or _extract_buttons(soup) + forms = snapshot.forms or _extract_forms(soup) + link_summary = summarize_link_patterns(links, url) + forms_summary = summarize_forms(forms) + payload = { + "version": UNKNOWN_PATTERN_VERSION, + "url": url, + "title": title or signals.title, + "primary_page_type": result.primary_page_type, + "confidence": result.confidence, + "reason": "unknown_page" if result.primary_page_type == PageType.UNKNOWN_PAGE.value else "low_confidence", + "text_sample": page_text[:1000], + "text_fingerprint": text_fingerprint(page_text), + "html_fingerprint": html_fingerprint(html or ""), + "dom_fingerprint": dom_fingerprint(html or ""), + "link_pattern_fingerprint": link_pattern_fingerprint(link_summary), + "schema_types": sorted(signals.schema_types), + "link_pattern_summary": link_summary, + "button_labels": _dedupe_strings(buttons)[:20], + "forms_summary": forms_summary, + "top_keywords": top_keywords(page_text), + "layout_blocks": list(signals.layout_blocks), + "keyword_hits": dict(signals.keyword_hits), + "alternatives": list(result.alternatives[:8]), + "evidence": [item.to_dict() for item in result.evidence[:12]], + } + payload["embedding_input"] = build_unknown_embedding_input(payload) + payload["cluster_candidate"] = build_cluster_candidate_payload(payload) + return payload + + +def build_unknown_embedding_input(payload: dict[str, Any]) -> str: + parts = [ + str(payload.get("title") or ""), + str(payload.get("text_sample") or ""), + "schema_types: " + ", ".join(str(item) for item in payload.get("schema_types") or []), + "layout_blocks: " + ", ".join(str(item) for item in payload.get("layout_blocks") or []), + "buttons: " + ", ".join(str(item) for item in payload.get("button_labels") or []), + "keywords: " + ", ".join(str(item.get("term")) for item in payload.get("top_keywords") or [] if isinstance(item, dict)), + ] + return "\n".join(part for part in parts if part.strip())[:4000] + + +def build_cluster_candidate_payload(payload: dict[str, Any]) -> dict[str, Any]: + return { + "version": UNKNOWN_PATTERN_VERSION, + "candidate_key": stable_hash( + { + "dom": payload.get("dom_fingerprint"), + "links": payload.get("link_pattern_fingerprint"), + "schema_types": payload.get("schema_types") or [], + "buttons": payload.get("button_labels") or [], + } + ), + "fingerprints": { + "text": payload.get("text_fingerprint"), + "html": payload.get("html_fingerprint"), + "dom": payload.get("dom_fingerprint"), + "links": payload.get("link_pattern_fingerprint"), + }, + "features": { + "schema_types": payload.get("schema_types") or [], + "layout_blocks": payload.get("layout_blocks") or [], + "top_keywords": payload.get("top_keywords") or [], + "link_patterns": (payload.get("link_pattern_summary") or {}).get("top_path_patterns") or [], + "form_count": (payload.get("forms_summary") or {}).get("form_count") or 0, + }, + } + + +def summarize_link_patterns(links: list[dict[str, Any]], base_url: str) -> dict[str, Any]: + base_host = urlparse(base_url).netloc.lower() + path_patterns = Counter() + text_labels: list[str] = [] + internal_count = 0 + external_count = 0 + for link in links: + href = str(link.get("href") or "") + parsed = urlparse(href) + if not href: + continue + if not parsed.netloc or parsed.netloc.lower() == base_host: + internal_count += 1 + else: + external_count += 1 + path_patterns[_path_pattern(parsed.path)] += 1 + text = str(link.get("text") or "").strip() + if text: + text_labels.append(text) + return { + "total_count": len(links), + "internal_count": internal_count, + "external_count": external_count, + "top_path_patterns": [ + {"pattern": pattern, "count": count} + for pattern, count in path_patterns.most_common(12) + if pattern + ], + "sample_texts": _dedupe_strings(text_labels)[:12], + } + + +def summarize_forms(forms: list[dict[str, Any]]) -> dict[str, Any]: + methods = Counter() + actions = Counter() + input_types = Counter() + form_texts: list[str] = [] + for form in forms: + method = str(form.get("method") or "get").lower() + action = _path_pattern(urlparse(str(form.get("action") or "")).path) + methods[method] += 1 + if action: + actions[action] += 1 + if form.get("text"): + form_texts.append(str(form.get("text"))) + for input_item in form.get("inputs") or []: + if isinstance(input_item, dict): + input_types[str(input_item.get("type") or "text").lower()] += 1 + return { + "form_count": len(forms), + "methods": dict(methods), + "action_patterns": dict(actions.most_common(8)), + "input_types": dict(input_types.most_common(12)), + "sample_texts": _dedupe_strings(form_texts)[:8], + } + + +def top_keywords(text: str, *, limit: int = 16) -> list[dict[str, Any]]: + tokens = [ + token.lower() + for token in re.findall(r"[A-Za-z][A-Za-z0-9_-]{2,}|[가-힣]{2,}", text or "") + if token.lower() not in STOPWORDS + ] + return [{"term": term, "count": count} for term, count in Counter(tokens).most_common(limit)] + + +def text_fingerprint(text: str) -> str: + return f"sha256:{_hash_text(_normalize_text(text))}" + + +def html_fingerprint(html: str) -> str: + return f"sha256:{_hash_text(_normalize_html(html))}" + + +def dom_fingerprint(html: str) -> str: + soup = _soup_from_html(html) + if soup is None: + return f"sha256:{_hash_text('')}" + tags = [] + for node in soup.find_all(True): + classes = ".".join(str(item).lower() for item in node.get("class", [])[:3]) + node_id = "#" + str(node.get("id")).lower() if node.get("id") else "" + tags.append(f"{node.name}{node_id}{('.' + classes) if classes else ''}") + return f"sha256:{_hash_text('>'.join(tags[:400]))}" + + +def link_pattern_fingerprint(link_summary: dict[str, Any]) -> str: + return stable_hash(link_summary) + + +def stable_hash(value: Any) -> str: + payload = json.dumps(value, sort_keys=True, ensure_ascii=False, separators=(",", ":")) + return f"sha256:{_hash_text(payload)}" + + +def _hash_text(value: str) -> str: + return hashlib.sha256(value.encode("utf-8", errors="ignore")).hexdigest() + + +def _normalize_text(value: str | None) -> str: + return re.sub(r"\s+", " ", str(value or "")).strip() + + +def _normalize_html(value: str | None) -> str: + text = re.sub(r">\s+<", "><", str(value or "")) + text = re.sub(r"\s+", " ", text) + return text.strip()[:200000] + + +def _path_pattern(path: str) -> str: + parts = [part for part in path.split("/") if part] + normalized = [] + for part in parts[:8]: + if part.isdigit() or re.fullmatch(r"[0-9a-fA-F-]{8,}", part): + normalized.append("{id}") + else: + normalized.append(re.sub(r"\d+", "{n}", part.lower())[:60]) + return "/" + "/".join(normalized) if normalized else "/" + + +def _soup_from_html(html: str): + if not html: + return None + try: + from bs4 import BeautifulSoup + except ImportError: + return None + try: + return BeautifulSoup(html, "html.parser") + except Exception: + return None + + +def _text_from_soup(soup) -> str: + if soup is None: + return "" + return soup.get_text(" ", strip=True) + + +def _extract_links(soup, base_url: str) -> list[dict[str, Any]]: + if soup is None: + return [] + links = [] + for tag in soup.find_all("a"): + href = str(tag.get("href") or "").strip() + if not href: + continue + links.append({"href": urljoin(base_url, href), "text": tag.get_text(" ", strip=True)}) + return links + + +def _extract_buttons(soup) -> list[str]: + if soup is None: + return [] + labels = [] + for tag in soup.select("button,input[type='submit'],input[type='button'],[role='button']"): + label = tag.get_text(" ", strip=True) or str(tag.get("value") or tag.get("aria-label") or "") + if label.strip(): + labels.append(label.strip()) + return labels + + +def _extract_forms(soup) -> list[dict[str, Any]]: + if soup is None: + return [] + forms = [] + for form in soup.find_all("form"): + inputs = [] + for tag in form.select("input,select,textarea"): + inputs.append( + { + "type": str(tag.get("type") or tag.name or ""), + "name": str(tag.get("name") or ""), + "placeholder": str(tag.get("placeholder") or ""), + } + ) + forms.append( + { + "action": str(form.get("action") or ""), + "method": str(form.get("method") or ""), + "text": form.get_text(" ", strip=True)[:500], + "inputs": inputs, + } + ) + return forms + + +def _dedupe_strings(values: list[str]) -> list[str]: + seen = set() + result = [] + for value in values: + clean = _normalize_text(value) + key = clean.lower() + if not clean or key in seen: + continue + seen.add(key) + result.append(clean) + return result diff --git a/ontology_platform/crawler_platform/app/core/crawler/pipeline.py b/ontology_platform/crawler_platform/app/core/crawler/pipeline.py index 596d2b3..d8caadc 100644 --- a/ontology_platform/crawler_platform/app/core/crawler/pipeline.py +++ b/ontology_platform/crawler_platform/app/core/crawler/pipeline.py @@ -3,11 +3,17 @@ from __future__ import annotations from dataclasses import dataclass from crawler_platform.app.config.loader import ProjectConfig -from crawler_platform.app.core.crawler.page_classifier import classify_page +from crawler_platform.app.core.crawler.page_classifier import ( + classification_metadata, + classify_page_semantic, + get_legacy_page_type, + should_analyze_page, +) from crawler_platform.app.core.crawler.fetchers import RobotsPolicy, make_fetcher from crawler_platform.app.core.crawler.plugins import ParserRegistry, default_parser_registry from crawler_platform.app.core.database.repository import KnowledgeRepository from crawler_platform.app.core.extractor.base import ExtractionPageContext, Extractor +from crawler_platform.app.core.extractor.strategy import extract_with_strategy from crawler_platform.app.core.extractor.validation import attach_page_context @@ -56,23 +62,33 @@ class CrawlPipeline: fetch_result = fetcher.fetch(url) parser = self.parser_registry.get(source_config.parser) parsed = parser.parse(fetch_result.analysis_html, fetch_result.final_url or url) - page_type = classify_page( + page_classification = classify_page_semantic( fetch_result.final_url or url, parsed.title or fetch_result.title, parsed.raw_text or parsed.text, fetch_result.analysis_html, parsed.source_zones or [], + final_url=fetch_result.final_url, + status_code=fetch_result.status_code, + content_type=fetch_result.headers.get("content-type"), ) + page_type = get_legacy_page_type(page_classification) project = self.repository.upsert_project(project_config) source = self.repository.get_source(project.id, source_name) warnings = [*fetch_result.warnings, *(parsed.extraction_warnings or [])] metadata = { **parsed.metadata, + **classification_metadata( + page_classification, + title=parsed.title or fetch_result.title, + text=parsed.raw_text or parsed.text, + html=fetch_result.analysis_html, + source_zones=parsed.source_zones or [], + ), "final_url": fetch_result.final_url, "crawl_status": fetch_result.crawl_status, "crawl_warnings": fetch_result.warnings, - "page_type": page_type, "raw_text_length": len(parsed.raw_text or ""), "clean_text_length": len(parsed.text or ""), "main_content_preview": (parsed.main_content or parsed.text)[:800], @@ -100,6 +116,20 @@ class CrawlPipeline: robots_status=robots_decision.status, robots_reason=robots_decision.reason, ) + if not should_analyze_page(page_classification, None): + return CrawlResult( + page_id=page.id, + claim_count=0, + entity_count=0, + crawl_status=fetch_result.crawl_status, + extraction_status="skipped", + page_type=page_type, + clean_text_length=len(parsed.text or ""), + raw_text_length=len(parsed.raw_text or ""), + warnings=warnings, + robots_status=robots_decision.status, + robots_reason=robots_decision.reason, + ) context = ExtractionPageContext( url=url, @@ -116,7 +146,7 @@ class CrawlPipeline: warnings=warnings, metadata=metadata, ) - bundle = self.extractor.extract_from_context(context, project_config) + bundle = extract_with_strategy(self.extractor, context, project_config) bundle = attach_page_context(bundle, context) claims = self.repository.save_extraction_bundle(project.id, source, page, bundle, project_config) extraction_summary = extraction_summary_from_raw(bundle.raw_output) diff --git a/ontology_platform/crawler_platform/app/core/crawler/site_crawler.py b/ontology_platform/crawler_platform/app/core/crawler/site_crawler.py index 281e2e6..05587d9 100644 --- a/ontology_platform/crawler_platform/app/core/crawler/site_crawler.py +++ b/ontology_platform/crawler_platform/app/core/crawler/site_crawler.py @@ -10,12 +10,16 @@ from crawler_platform.app.core.crawler.discovery import discover_links from crawler_platform.app.core.crawler.fetchers import RobotsDecision, RobotsPolicy, make_fetcher from crawler_platform.app.core.crawler.page_classifier import ( classify_page as classify_page_type, + classification_metadata, + classify_page_semantic, + get_legacy_page_type, should_analyze_page as should_analyze_page_type, ) from crawler_platform.app.core.crawler.plugins import ParserRegistry, default_parser_registry from crawler_platform.app.core.database import models from crawler_platform.app.core.database.repository import KnowledgeRepository from crawler_platform.app.core.extractor.base import ExtractionPageContext, Extractor +from crawler_platform.app.core.extractor.strategy import extract_with_strategy from crawler_platform.app.core.extractor.validation import attach_page_context @@ -209,6 +213,16 @@ class SiteCrawler: fetch_result = fetcher.fetch(url) if is_failed_fetch_status(fetch_result.status_code) or fetch_result.crawl_status != "success": error = f"fetch failed with status {fetch_result.status_code}; crawl_status={fetch_result.crawl_status}" + page_classification = classify_page_semantic( + fetch_result.final_url or url, + title=fetch_result.title, + text="", + html=fetch_result.analysis_html, + final_url=fetch_result.final_url, + status_code=fetch_result.status_code, + content_type=fetch_result.headers.get("content-type"), + ) + page_type = get_legacy_page_type(page_classification) page = self.repository.upsert_page( project_id=source.project_id, source_id=source.id, @@ -217,6 +231,11 @@ class SiteCrawler: status_code=fetch_result.status_code, cleaned_text="", metadata={ + **classification_metadata( + page_classification, + title=fetch_result.title, + html=fetch_result.analysis_html, + ), "final_url": fetch_result.final_url, "crawl_status": fetch_result.crawl_status, "crawl_warnings": fetch_result.warnings, @@ -232,7 +251,7 @@ class SiteCrawler: url=url, depth=depth, status=fetch_result.crawl_status, - page_type="unknown", + page_type=page_type, page_id=page.id, crawl_status=fetch_result.crawl_status, robots_status=robots_decision.status, @@ -246,13 +265,17 @@ class SiteCrawler: return parsed = parser.parse(fetch_result.analysis_html, fetch_result.final_url or url) - page_type = classify_page( + page_classification = classify_page_semantic( fetch_result.final_url or url, parsed.title or fetch_result.title, parsed.raw_text or parsed.text, fetch_result.analysis_html, parsed.source_zones or [], + final_url=fetch_result.final_url, + status_code=fetch_result.status_code, + content_type=fetch_result.headers.get("content-type"), ) + page_type = get_legacy_page_type(page_classification) discovered_count = self._enqueue_links( html=fetch_result.analysis_html, base_url=fetch_result.final_url or url, @@ -268,10 +291,16 @@ class SiteCrawler: warnings = [*fetch_result.warnings, *(parsed.extraction_warnings or [])] metadata = { **parsed.metadata, + **classification_metadata( + page_classification, + title=parsed.title or fetch_result.title, + text=parsed.raw_text or parsed.text, + html=fetch_result.analysis_html, + source_zones=parsed.source_zones or [], + ), "final_url": fetch_result.final_url, "crawl_status": fetch_result.crawl_status, "crawl_warnings": fetch_result.warnings, - "page_type": page_type, "depth": depth, "raw_text_length": len(parsed.raw_text or ""), "clean_text_length": len(parsed.text or ""), @@ -319,7 +348,7 @@ class SiteCrawler: ) return - if should_analyze_page(page_type, analyze_page_types): + if should_analyze_page(page_classification, analyze_page_types): context = ExtractionPageContext( url=url, final_url=fetch_result.final_url, @@ -335,7 +364,7 @@ class SiteCrawler: warnings=warnings, metadata=metadata, ) - bundle = self.extractor.extract_from_context(context, project_config) + bundle = extract_with_strategy(self.extractor, context, project_config) bundle = attach_page_context(bundle, context) claims = self.repository.save_extraction_bundle(source.project_id, source, page, bundle, project_config) extraction_summary = extraction_summary_from_raw(bundle.raw_output) diff --git a/ontology_platform/crawler_platform/app/core/extractor/ai_provider.py b/ontology_platform/crawler_platform/app/core/extractor/ai_provider.py index bf71fe3..2f6c422 100644 --- a/ontology_platform/crawler_platform/app/core/extractor/ai_provider.py +++ b/ontology_platform/crawler_platform/app/core/extractor/ai_provider.py @@ -637,6 +637,38 @@ def list_openai_compatible_models(base_url: str, api_key: str | None = None) -> return [{"id": item.get("id", ""), "owned_by": item.get("owned_by")} for item in items if item.get("id")] +def list_lmstudio_loaded_models(base_url: str) -> list[dict[str, Any]]: + """Return only the models currently loaded in LM Studio. + + Uses LM Studio's native REST API (``/api/v0/models``) which exposes a + ``state`` field. Falls back to the OpenAI-compatible ``/v1/models`` list + (treated as all-loaded) if the native endpoint is unavailable. + """ + clean = base_url.rstrip("/") + if clean.endswith("/v1"): + root = clean[: -len("/v1")] + elif clean.endswith("/v1/chat/completions"): + root = clean[: -len("/v1/chat/completions")] + else: + root = clean + native_url = f"{root}/api/v0/models" + try: + response = requests.get(native_url, timeout=5) + response.raise_for_status() + data = response.json() + items = data.get("data", []) if isinstance(data, dict) else [] + loaded = [ + {"id": item.get("id", ""), "owned_by": item.get("owned_by"), "state": item.get("state")} + for item in items + if item.get("id") and str(item.get("state", "")).lower() == "loaded" + ] + if loaded: + return loaded + except Exception: + pass + return list_openai_compatible_models(base_url) + + def parse_entities(items: list[dict[str, Any]]) -> list[ExtractedEntity]: entities: list[ExtractedEntity] = [] for item in items: diff --git a/ontology_platform/crawler_platform/app/core/extractor/base.py b/ontology_platform/crawler_platform/app/core/extractor/base.py index 8be1558..c749313 100644 --- a/ontology_platform/crawler_platform/app/core/extractor/base.py +++ b/ontology_platform/crawler_platform/app/core/extractor/base.py @@ -61,6 +61,7 @@ class ExtractionPageContext: clean_text = self.clean_text if text_limit is not None and len(clean_text) > text_limit: clean_text = clean_text[:text_limit] + semantic_metadata = self.semantic_metadata_payload() zones = [] for zone in self.source_zones: zone_text = str(zone.get("text") or "") @@ -85,8 +86,34 @@ class ExtractionPageContext: "clean_text": clean_text, "source_zones": zones, "warnings": self.warnings, + "metadata": semantic_metadata, } + def semantic_metadata_payload(self) -> dict[str, Any]: + payload = { + key: self.metadata[key] + for key in ("semantic_page_type", "analyze_strategy", "llm_policy") + if key in self.metadata + } + classification = self.metadata.get("page_classification") + if isinstance(classification, dict): + payload["page_classification"] = { + key: classification.get(key) + for key in ( + "primary_page_type", + "secondary_page_types", + "confidence", + "alternatives", + "legacy_page_type", + "unknown_pattern", + ) + if key in classification + } + evidence = classification.get("evidence") + if isinstance(evidence, list): + payload["page_classification"]["evidence"] = evidence[:8] + return payload + class Extractor(ABC): name = "base" diff --git a/ontology_platform/crawler_platform/app/core/extractor/hybrid.py b/ontology_platform/crawler_platform/app/core/extractor/hybrid.py index 17e03e0..26b5426 100644 --- a/ontology_platform/crawler_platform/app/core/extractor/hybrid.py +++ b/ontology_platform/crawler_platform/app/core/extractor/hybrid.py @@ -5,6 +5,7 @@ from dataclasses import replace from typing import Any from crawler_platform.app.config.loader import ProjectConfig +from crawler_platform.app.core.crawler.page_type_taxonomy import LLMPolicy from crawler_platform.app.core.extractor.ai_provider import LLMJsonExtractor, dedupe_entities from crawler_platform.app.core.extractor.base import ( ExtractedClaim, @@ -17,9 +18,40 @@ from crawler_platform.app.core.extractor.rule_based import GenericRuleBasedExtra from crawler_platform.app.domains.perfume.extractor import PerfumeRuleBasedExtractor -LLM_PAGE_TYPES = {"ProductPage", "BrandStoryPage", "ReviewPage"} -SKIP_LLM_PAGE_TYPES = {"CategoryPage", "SearchPage", "ListingPage"} -RULE_ONLY_PAGE_TYPES = {"BoardPage", "CommunityPage", "UnknownPage"} +LLM_PAGE_TYPES = { + "ProductPage", + "ProductDetailPage", + "BrandStoryPage", + "AboutPage", + "ContactPage", + "ReviewPage", + "NoticePage", + "PublicNoticePage", + "PromotionPage", + "CampaignLandingPage", + "ArticlePage", + "BlogPostPage", + "DocumentationPage", + "APIReferencePage", + "DatasetPage", + "ResearchPaperPage", + "JobPostingPage", + "CourseDetailPage", + "LocalBusinessPage", + "RealEstateListingPage", + "ProfilePage", +} +SKIP_LLM_PAGE_TYPES = {"CategoryPage", "CategoryListingPage", "SearchPage", "SearchResultsPage", "ListingPage"} +RULE_ONLY_PAGE_TYPES = { + "BoardPage", + "ForumBoardPage", + "ForumThreadPage", + "CommunityPage", + "FAQPage", + "QAPage", + "VideoPage", + "UnknownPage", +} MIN_CLEAN_TEXT_CHARS_FOR_LLM = 300 MAX_CLEAN_TEXT_CHARS_FOR_LLM = 60000 @@ -331,6 +363,22 @@ def llm_skip_reason( return None page_type = str(context.page_type or "UnknownPage") clean_length = len(context.clean_text or page_text or "") + policy = llm_policy_from_context(context) + if policy: + if policy == LLMPolicy.SKIP.value: + return "LLM policy Skip prevents LLM extraction" + if policy == LLMPolicy.NO_LLM.value: + return "LLM policy NoLLM prevents LLM extraction" + if policy == LLMPolicy.RULE_ONLY.value: + return "LLM policy RuleOnly routes page to rule-only extraction" + if policy == LLMPolicy.LLM_FOR_AMBIGUITY_ONLY.value and not classification_is_ambiguous(context): + return "LLM policy LLMForAmbiguityOnly requires ambiguous classification evidence" + if policy in {LLMPolicy.LLM_FULL.value, LLMPolicy.LLM_LIGHT.value, LLMPolicy.LLM_FOR_AMBIGUITY_ONLY.value}: + if clean_length < MIN_CLEAN_TEXT_CHARS_FOR_LLM: + return f"clean text is too short for LLM extraction: {clean_length} chars" + if clean_length > MAX_CLEAN_TEXT_CHARS_FOR_LLM: + return f"clean text is too long for LLM extraction: {clean_length} chars" + return None if page_type in SKIP_LLM_PAGE_TYPES: return f"page type {page_type} is configured to skip LLM extraction" if page_type in RULE_ONLY_PAGE_TYPES: @@ -344,6 +392,34 @@ def llm_skip_reason( return None +def llm_policy_from_context(context: ExtractionPageContext) -> str | None: + raw_policy = context.metadata.get("llm_policy") + if not raw_policy: + classification = context.metadata.get("page_classification") + if isinstance(classification, dict): + raw_policy = classification.get("llm_policy") + policy = str(raw_policy or "").strip() + return policy or None + + +def classification_is_ambiguous(context: ExtractionPageContext) -> bool: + classification = context.metadata.get("page_classification") + if not isinstance(classification, dict): + return False + confidence = float(classification.get("confidence") or 0.0) + if 0.0 < confidence < 0.55: + return True + alternatives = classification.get("alternatives") or [] + if not isinstance(alternatives, list) or len(alternatives) < 2: + return False + try: + top_score = float(alternatives[0][1]) + next_score = float(alternatives[1][1]) + except (TypeError, ValueError, IndexError): + return False + return abs(top_score - next_score) < 0.08 + + def merge_bundles( rule_bundle: ExtractionBundle, llm_bundle: ExtractionBundle, diff --git a/ontology_platform/crawler_platform/app/core/extractor/strategy.py b/ontology_platform/crawler_platform/app/core/extractor/strategy.py new file mode 100644 index 0000000..4413131 --- /dev/null +++ b/ontology_platform/crawler_platform/app/core/extractor/strategy.py @@ -0,0 +1,190 @@ +from __future__ import annotations + +from collections import Counter +from typing import Any + +from crawler_platform.app.config.loader import ProjectConfig +from crawler_platform.app.core.crawler.page_analysis_policy import decide_analyze_strategy, decide_llm_policy +from crawler_platform.app.core.crawler.page_type_taxonomy import AnalyzeStrategy, LLMPolicy +from crawler_platform.app.core.extractor.base import ExtractionBundle, ExtractionPageContext, Extractor + + +def extract_with_strategy( + extractor: Extractor, + context: ExtractionPageContext, + project_config: ProjectConfig, +) -> ExtractionBundle: + strategy = analysis_strategy_from_context(context) + policy = llm_policy_from_context(context) + if strategy == AnalyzeStrategy.SKIP_PROTECTED.value: + return strategy_only_bundle(context, strategy, policy, "protected page") + if strategy == AnalyzeStrategy.SKIP_NOISE.value: + return strategy_only_bundle(context, strategy, policy, "noise page") + if strategy == AnalyzeStrategy.ANALYZE_METADATA_ONLY.value: + return metadata_only_bundle(context, strategy, policy) + if strategy == AnalyzeStrategy.ANALYZE_DISCOVERY_ONLY.value: + return discovery_only_bundle(context, strategy, policy) + if strategy == AnalyzeStrategy.ANALYZE_RELATIONS_ONLY.value: + return rule_or_structure_bundle(extractor, context, project_config, strategy, policy) + if strategy == AnalyzeStrategy.ANALYZE_ENTITY_ONLY.value: + bundle = extractor.extract_from_context(context, project_config) + bundle.claims = [] + bundle.raw_output = { + **bundle.raw_output, + **strategy_payload(context, strategy, policy), + "effective_extraction_mode": "entity_only", + } + return bundle + bundle = extractor.extract_from_context(context, project_config) + bundle.raw_output = { + **bundle.raw_output, + **strategy_payload(context, strategy, policy), + } + return bundle + + +def analysis_strategy_from_context(context: ExtractionPageContext) -> str: + raw_strategy = context.metadata.get("analyze_strategy") + if raw_strategy: + return str(raw_strategy) + classification = context.metadata.get("page_classification") + if isinstance(classification, dict) and classification.get("analyze_strategy"): + return str(classification["analyze_strategy"]) + return decide_analyze_strategy(context.page_type) + + +def llm_policy_from_context(context: ExtractionPageContext) -> str: + raw_policy = context.metadata.get("llm_policy") + if raw_policy: + return str(raw_policy) + classification = context.metadata.get("page_classification") + if isinstance(classification, dict) and classification.get("llm_policy"): + return str(classification["llm_policy"]) + return decide_llm_policy(context.page_type) + + +def rule_or_structure_bundle( + extractor: Extractor, + context: ExtractionPageContext, + project_config: ProjectConfig, + strategy: str, + policy: str, +) -> ExtractionBundle: + if policy == LLMPolicy.LLM_FOR_AMBIGUITY_ONLY.value and classification_is_ambiguous(context): + bundle = extractor.extract_from_context(context, project_config) + bundle.raw_output = { + **bundle.raw_output, + **strategy_payload(context, strategy, policy), + "effective_extraction_mode": "ambiguity_limited", + } + return bundle + rule_extractor = getattr(extractor, "rule_extractor", None) + if rule_extractor is not None: + bundle = rule_extractor.extract_from_context(context, project_config) + bundle.extractor_name = f"{bundle.extractor_name}_strategy_routed" + bundle.raw_output = { + **bundle.raw_output, + **strategy_payload(context, strategy, policy), + "effective_extraction_mode": "rule_only", + "llm_skipped": True, + "llm_skip_reason": f"analyze strategy {strategy} uses rule/structure extraction only", + } + return bundle + if getattr(extractor, "provider", "") == "ai": + return discovery_only_bundle(context, strategy, policy) + bundle = extractor.extract_from_context(context, project_config) + bundle.raw_output = { + **bundle.raw_output, + **strategy_payload(context, strategy, policy), + "effective_extraction_mode": "rule_or_structure", + "llm_skipped": policy in {LLMPolicy.RULE_ONLY.value, LLMPolicy.NO_LLM.value, LLMPolicy.SKIP.value}, + } + return bundle + + +def classification_is_ambiguous(context: ExtractionPageContext) -> bool: + classification = context.metadata.get("page_classification") + if not isinstance(classification, dict): + return False + confidence = float(classification.get("confidence") or 0.0) + if 0.0 < confidence < 0.55: + return True + alternatives = classification.get("alternatives") or [] + if not isinstance(alternatives, list) or len(alternatives) < 2: + return False + try: + top_score = float(alternatives[0][1]) + next_score = float(alternatives[1][1]) + except (TypeError, ValueError, IndexError): + return False + return abs(top_score - next_score) < 0.08 + + +def metadata_only_bundle(context: ExtractionPageContext, strategy: str, policy: str) -> ExtractionBundle: + return ExtractionBundle( + extractor_name="metadata_only_extractor", + provider="strategy", + raw_output={ + **strategy_payload(context, strategy, policy), + "effective_extraction_mode": "metadata_only", + "llm_skipped": True, + "llm_skip_reason": f"analyze strategy {strategy} does not require LLM extraction", + }, + ) + + +def discovery_only_bundle(context: ExtractionPageContext, strategy: str, policy: str) -> ExtractionBundle: + return ExtractionBundle( + extractor_name="discovery_only_extractor", + provider="strategy", + raw_output={ + **strategy_payload(context, strategy, policy), + "effective_extraction_mode": "discovery_only", + "llm_skipped": True, + "llm_skip_reason": f"analyze strategy {strategy} uses discovery/structure signals only", + }, + ) + + +def strategy_only_bundle(context: ExtractionPageContext, strategy: str, policy: str, reason: str) -> ExtractionBundle: + return ExtractionBundle( + extractor_name="strategy_skipped_extractor", + provider="strategy", + raw_output={ + **strategy_payload(context, strategy, policy), + "effective_extraction_mode": "skipped", + "llm_skipped": True, + "llm_skip_reason": reason, + "skip_reason": reason, + }, + ) + + +def strategy_payload(context: ExtractionPageContext, strategy: str, policy: str) -> dict[str, Any]: + return { + "analyze_strategy": strategy, + "llm_policy": policy, + "page_type": context.page_type, + "semantic_page_type": context.metadata.get("semantic_page_type"), + "strategy_summary": structure_summary(context), + } + + +def structure_summary(context: ExtractionPageContext) -> dict[str, Any]: + zone_types = Counter(str(zone.get("zone_type") or "unknown") for zone in context.source_zones) + classification = context.metadata.get("page_classification") if isinstance(context.metadata, dict) else None + return { + "title": context.title, + "url": context.final_url or context.url, + "clean_text_length": len(context.clean_text or ""), + "raw_text_length": len(context.raw_text or ""), + "source_zone_count": len(context.source_zones or []), + "source_zone_types": dict(zone_types), + "classification": { + "primary_page_type": classification.get("primary_page_type"), + "confidence": classification.get("confidence"), + "alternatives": classification.get("alternatives", [])[:5], + } + if isinstance(classification, dict) + else {}, + } diff --git a/ontology_platform/crawler_platform/app/core/ontology/domain_discovery.py b/ontology_platform/crawler_platform/app/core/ontology/domain_discovery.py index dfd96a8..dd078e3 100644 --- a/ontology_platform/crawler_platform/app/core/ontology/domain_discovery.py +++ b/ontology_platform/crawler_platform/app/core/ontology/domain_discovery.py @@ -538,12 +538,31 @@ def relation_hint(label: str) -> tuple[str, str] | None: def entity_type_from_page_type(page_type: str) -> str | None: mapping = { "ProductPage": "Product", + "ProductDetailPage": "Product", "CategoryPage": "Category", + "CategoryListingPage": "Category", + "SearchPage": "Category", + "SearchResultsPage": "Category", "BrandStoryPage": "Brand", + "AboutPage": "Organization", + "ContactPage": "Organization", "NoticePage": "Notice", + "PublicNoticePage": "Notice", "PromotionPage": "Promotion", + "CampaignLandingPage": "Promotion", "ReviewPage": "Review", "BoardPage": "Article", + "ForumBoardPage": "Article", + "ForumThreadPage": "Article", + "ArticlePage": "Article", + "NewsArticlePage": "Article", + "BlogPostPage": "Article", + "FAQPage": "Article", + "QAPage": "Article", + "DocumentationPage": "Article", + "APIReferencePage": "Article", + "JobPostingPage": "JobPosting", + "PricingPage": "Product", } return mapping.get(page_type) diff --git a/ontology_platform/crawler_platform/app/core/ontology/relation_schema.py b/ontology_platform/crawler_platform/app/core/ontology/relation_schema.py index ee492d4..6483e88 100644 --- a/ontology_platform/crawler_platform/app/core/ontology/relation_schema.py +++ b/ontology_platform/crawler_platform/app/core/ontology/relation_schema.py @@ -3,6 +3,8 @@ from __future__ import annotations from dataclasses import dataclass, field from typing import Any +from crawler_platform.app.core.crawler.page_type_taxonomy import normalize_page_type + @dataclass(frozen=True, slots=True) class RelationRule: @@ -97,13 +99,21 @@ def relation_schema_compatible( return f"{predicate} expects a typed entity object" if object_type and rule.object_types and object_type not in rule.object_types: return f"object type {object_type} is not allowed for {predicate}" - if page_type and rule.page_types and page_type not in rule.page_types: + if page_type and rule.page_types and not page_type_matches_rule(page_type, rule.page_types): return f"predicate {predicate} is not allowed for page type {page_type}" if source_zone and rule.source_zones and source_zone not in rule.source_zones: return f"source zone {source_zone} is not allowed for {predicate}" return None +def page_type_matches_rule(page_type: str, allowed_page_types: set[str]) -> bool: + if page_type in allowed_page_types: + return True + normalized_page_type = normalize_page_type(page_type) + normalized_allowed = {normalize_page_type(allowed_page_type) for allowed_page_type in allowed_page_types} + return normalized_page_type in normalized_allowed + + def confidence_breakdown( *, llm_confidence: float, diff --git a/ontology_platform/crawler_platform/app/core/research/graph_research_loop.py b/ontology_platform/crawler_platform/app/core/research/graph_research_loop.py index 10ac097..e42297e 100644 --- a/ontology_platform/crawler_platform/app/core/research/graph_research_loop.py +++ b/ontology_platform/crawler_platform/app/core/research/graph_research_loop.py @@ -7,11 +7,17 @@ from urllib.parse import urldefrag, urlparse from crawler_platform.app.config.loader import ProjectConfig from crawler_platform.app.core.crawler.discovery import discover_links from crawler_platform.app.core.crawler.fetchers import RobotsPolicy, make_fetcher -from crawler_platform.app.core.crawler.page_classifier import classify_page, should_analyze_page +from crawler_platform.app.core.crawler.page_classifier import ( + classification_metadata, + classify_page_semantic, + get_legacy_page_type, + should_analyze_page, +) from crawler_platform.app.core.crawler.plugins import ParserRegistry, default_parser_registry from crawler_platform.app.core.database import models from crawler_platform.app.core.database.repository import KnowledgeRepository from crawler_platform.app.core.extractor.base import ExtractionPageContext, Extractor +from crawler_platform.app.core.extractor.strategy import extract_with_strategy from crawler_platform.app.core.extractor.validation import attach_page_context from crawler_platform.app.core.research.entity_expansion import EntityExpansionPlanner from crawler_platform.app.core.research.exploration_queue import ExplorationItem, ExplorationQueue @@ -206,13 +212,17 @@ class GraphResearchLoop: fetch_result = fetcher.fetch(url) parser_result = parser.parse(fetch_result.analysis_html, fetch_result.final_url or url) - page_type = classify_page( + page_classification = classify_page_semantic( fetch_result.final_url or url, parser_result.title or fetch_result.title, parser_result.raw_text or parser_result.text, fetch_result.analysis_html, parser_result.source_zones or [], + final_url=fetch_result.final_url, + status_code=fetch_result.status_code, + content_type=fetch_result.headers.get("content-type"), ) + page_type = get_legacy_page_type(page_classification) relevance = self.relevance.score_url( project_id=source.project_id, url=fetch_result.final_url or url, @@ -225,11 +235,17 @@ class GraphResearchLoop: ) metadata = { **parser_result.metadata, + **classification_metadata( + page_classification, + title=parser_result.title or fetch_result.title, + text=parser_result.raw_text or parser_result.text, + html=fetch_result.analysis_html, + source_zones=parser_result.source_zones or [], + ), "research_item": item.to_dict(), "research_relevance": asdict(relevance), "final_url": fetch_result.final_url, "crawl_status": fetch_result.crawl_status, - "page_type": page_type, "robots_status": robots_decision.status, "robots_reason": robots_decision.reason, "raw_text_length": len(parser_result.raw_text or ""), @@ -289,7 +305,7 @@ class GraphResearchLoop: fetch_result.crawl_status == "success" and parser_result.extraction_status != "failed" and relevance.score >= min_relevance - and should_analyze_page(page_type, analyze_page_types) + and should_analyze_page(page_classification, analyze_page_types) ): context = ExtractionPageContext( url=url, @@ -306,7 +322,7 @@ class GraphResearchLoop: warnings=[*fetch_result.warnings, *(parser_result.extraction_warnings or [])], metadata=metadata, ) - bundle = self.extractor.extract_from_context(context, project_config) + bundle = extract_with_strategy(self.extractor, context, project_config) bundle = attach_page_context(bundle, context) claims = self.repository.save_extraction_bundle(source.project_id, source, page, bundle, project_config) analyzed = True diff --git a/ontology_platform/crawler_platform/app/core/research/relevance_engine.py b/ontology_platform/crawler_platform/app/core/research/relevance_engine.py index d45ad60..30fb684 100644 --- a/ontology_platform/crawler_platform/app/core/research/relevance_engine.py +++ b/ontology_platform/crawler_platform/app/core/research/relevance_engine.py @@ -8,20 +8,46 @@ from urllib.parse import unquote, urlparse from sqlalchemy import select from sqlalchemy.orm import Session -from crawler_platform.app.core.crawler.page_classifier import classify_page +from crawler_platform.app.core.crawler.page_classifier import classify_page_semantic, get_legacy_page_type from crawler_platform.app.core.database import models HIGH_VALUE_PAGE_TYPES = { "ProductPage": 0.95, + "ProductDetailPage": 0.95, "BrandStoryPage": 0.88, + "AboutPage": 0.82, + "ContactPage": 0.72, "ReviewPage": 0.82, "NoticePage": 0.45, + "PublicNoticePage": 0.45, + "ArticlePage": 0.55, + "NewsArticlePage": 0.55, + "BlogPostPage": 0.52, + "FAQPage": 0.48, + "QAPage": 0.48, + "DocumentationPage": 0.5, + "APIReferencePage": 0.5, + "WikiPage": 0.48, + "DatasetPage": 0.52, + "ResearchPaperPage": 0.6, + "JobPostingPage": 0.46, + "CourseDetailPage": 0.5, + "VideoPage": 0.36, + "LocalBusinessPage": 0.44, + "RealEstateListingPage": 0.44, + "ProfilePage": 0.36, + "PricingPage": 0.62, "EventPage": 0.5, "PromotionPage": 0.38, + "CampaignLandingPage": 0.38, "CategoryPage": 0.34, + "CategoryListingPage": 0.34, "SearchPage": 0.22, + "SearchResultsPage": 0.22, "BoardPage": 0.18, + "ForumBoardPage": 0.18, + "ForumThreadPage": 0.24, "UnknownPage": 0.25, } @@ -96,8 +122,12 @@ class RelevanceEngine: normalized = url.strip().rstrip("/") parsed = urlparse(normalized) combined = unquote(f"{normalized} {label} {text[:3000]}").lower() - page_type = classify_page(normalized, label, text, html=html) - page_type_score = HIGH_VALUE_PAGE_TYPES.get(page_type, 0.25) + page_classification = classify_page_semantic(normalized, label, text, html=html) + page_type = page_classification.primary_page_type + page_type_score = HIGH_VALUE_PAGE_TYPES.get( + page_type, + HIGH_VALUE_PAGE_TYPES.get(get_legacy_page_type(page_classification), 0.25), + ) graph_terms = self.graph_terms(project_id) tokens = tokenize(combined) overlap_count = len(tokens & graph_terms) diff --git a/ontology_platform/data/crawler_platform.db b/ontology_platform/data/crawler_platform.db index 5e445ee..eb4fa8a 100644 Binary files a/ontology_platform/data/crawler_platform.db and b/ontology_platform/data/crawler_platform.db differ diff --git a/ontology_platform/docs/crawl_flowchart_llm_decision.png b/ontology_platform/docs/crawl_flowchart_llm_decision.png new file mode 100644 index 0000000..7632991 Binary files /dev/null and b/ontology_platform/docs/crawl_flowchart_llm_decision.png differ diff --git a/ontology_platform/docs/crawl_flowchart_llm_decision.svg b/ontology_platform/docs/crawl_flowchart_llm_decision.svg new file mode 100644 index 0000000..2fbc290 --- /dev/null +++ b/ontology_platform/docs/crawl_flowchart_llm_decision.svg @@ -0,0 +1,61 @@ + + + + + + + +크롤 진행 단계와 LLM 사용 판단 구조 +API 요청부터 Rule/LLM 추출, 검증, DB 저장까지의 흐름 + + + + + + + + + + + +\n\n아니오 +\n\n예 +\n\nskip +\n\nuse + + + +API 요청\n/crawl-site + +Job 생성 및\nqueue 초기화 + +URL depth / domain /\nrobots 검사 + +Fetch:\nHTML 수집 + +Parse:\n본문 / zone /\nclean_text 추출 + +Page Classifier:\nProductPage 등 분류 + +분석 대상\npage_type인가? + +discovered/skipped\n저장,\n링크만 확장 + +Extractor 선택 + +Rule extraction + +LLM 사용 판단 + +rule_only routed + +LLM extraction + +rule/LLM merge,\nagreement/conflict 계산 + +validation + +DB 저장:\nentities / claims /\nevidence / log + +progress 업데이트 + diff --git a/ontology_platform/docs/crawl_layers_llm_decision.docx b/ontology_platform/docs/crawl_layers_llm_decision.docx new file mode 100644 index 0000000..47c4569 Binary files /dev/null and b/ontology_platform/docs/crawl_layers_llm_decision.docx differ diff --git a/ontology_platform/docs/phases/PHASE_INDEX.md b/ontology_platform/docs/phases/PHASE_INDEX.md index c5ba6f5..8ef51a2 100644 --- a/ontology_platform/docs/phases/PHASE_INDEX.md +++ b/ontology_platform/docs/phases/PHASE_INDEX.md @@ -63,39 +63,39 @@ FILE: ./26_05_22_semantic_page_classification/phase_04_001_evidence_scoring_clas PHASE 5. Analyze Strategy 및 LLM Policy 분리 FILE: ./26_05_22_semantic_page_classification/phase_05_001_analysis_llm_policy.md -1) PageClassificationResult 기반 `decide_analyze_strategy()` 추가 [TODO] -2) PageClassificationResult 기반 `decide_llm_policy()` 추가 [TODO] -3) `should_analyze_page(result_or_page_type, analyze_page_types=None)` compatibility 구현 [TODO] -4) Category/Search/Board 계열을 무조건 skip하지 않고 strategy 기반으로 처리 [TODO] -5) Login/Checkout/Payment/Captcha/AccessDenied 계열은 SkipProtected/Skip 정책으로 처리 [TODO] +1) PageClassificationResult 기반 `decide_analyze_strategy()` 추가 [완료] +2) PageClassificationResult 기반 `decide_llm_policy()` 추가 [완료] +3) `should_analyze_page(result_or_page_type, analyze_page_types=None)` compatibility 구현 [완료] +4) Category/Search/Board 계열을 무조건 skip하지 않고 strategy 기반으로 처리 [완료] +5) Login/Checkout/Payment/Captcha/AccessDenied 계열은 SkipProtected/Skip 정책으로 처리 [완료] --- PHASE 6. Crawler, Cleaner, Extractor, Discovery/Relevance 통합 FILE: ./26_05_22_semantic_page_classification/phase_06_001_pipeline_integration.md -1) `site_crawler.py`와 `pipeline.py` metadata에 semantic classification payload 저장 [TODO] -2) `ExtractionPageContext` 또는 metadata를 통해 analyze_strategy/llm_policy 전달 [TODO] -3) `HybridExtractor`가 LLMPolicy를 우선 사용하고 legacy page_type fallback을 유지하도록 수정 [TODO] -4) `page_cleaner.py`, `domain_discovery.py`, `relevance_engine.py`의 legacy page_type 기대 경로와 신규 semantic type을 호환 [TODO] +1) `site_crawler.py`와 `pipeline.py` metadata에 semantic classification payload 저장 [완료] +2) `ExtractionPageContext` 또는 metadata를 통해 analyze_strategy/llm_policy 전달 [완료] +3) `HybridExtractor`가 LLMPolicy를 우선 사용하고 legacy page_type fallback을 유지하도록 수정 [완료] +4) `page_cleaner.py`, `domain_discovery.py`, `relevance_engine.py`의 legacy page_type 기대 경로와 신규 semantic type을 호환 [완료] --- PHASE 7. Unknown Pattern 저장 기반 추가 FILE: ./26_05_22_semantic_page_classification/phase_07_001_unknown_pattern_storage.md -1) UnknownPage 또는 low confidence 페이지의 evidence payload 정의 [TODO] -2) text/html/link/schema/button/form summary와 fingerprint hook 추가 [TODO] -3) DB schema 변경 없이 metadata_json에 저장 가능한 초기 구조 구현 [TODO] -4) 향후 clustering/embedding 확장을 위한 hook만 추가하고 실제 clustering은 이번 범위에서 제외 [TODO] +1) UnknownPage 또는 low confidence 페이지의 evidence payload 정의 [완료] +2) text/html/link/schema/button/form summary와 fingerprint hook 추가 [완료] +3) DB schema 변경 없이 metadata_json에 저장 가능한 초기 구조 구현 [완료] +4) 향후 clustering/embedding 확장을 위한 hook만 추가하고 실제 clustering은 이번 범위에서 제외 [완료] --- PHASE 8. 테스트 Fixture 및 회귀 검증 FILE: ./26_05_22_semantic_page_classification/phase_08_001_tests_regression.md -1) 최소 10개 이상의 HTML fixture 추가 [TODO] -2) ProductDetailPage, CategoryListingPage, SearchResultsPage, ArticlePage, QAPage, FAQPage, ForumThreadPage, DocumentationPage, JobPostingPage, LoginPage, CheckoutPage, TermsPage, SitemapPage, UnknownPage 단위 테스트 추가 [TODO] -3) legacy `classify_page()`와 `should_analyze_page()` 호환성 테스트 추가 [TODO] -4) HybridExtractor LLMPolicy 회귀 테스트 추가 [TODO] -5) pytest 또는 현재 프로젝트 테스트 명령 실행 및 결과 기록 [TODO] +1) 최소 10개 이상의 HTML fixture 추가 [완료] +2) ProductDetailPage, CategoryListingPage, SearchResultsPage, ArticlePage, QAPage, FAQPage, ForumThreadPage, DocumentationPage, JobPostingPage, LoginPage, CheckoutPage, TermsPage, SitemapPage, UnknownPage 단위 테스트 추가 [완료] +3) legacy `classify_page()`와 `should_analyze_page()` 호환성 테스트 추가 [완료] +4) HybridExtractor LLMPolicy 회귀 테스트 추가 [완료] +5) pytest 또는 현재 프로젝트 테스트 명령 실행 및 결과 기록 [완료] diff --git a/ontology_platform/docs/semantic_page_classification_codex_spec.md b/ontology_platform/docs/semantic_page_classification_codex_spec.md new file mode 100644 index 0000000..124c388 --- /dev/null +++ b/ontology_platform/docs/semantic_page_classification_codex_spec.md @@ -0,0 +1,2265 @@ +# Semantic Page Classification Layer 설계 및 구현 지시서 + +## 0. 문서 목적 + +본 문서는 온톨로지 플랫폼의 `page_classifier.py`를 단순 URL/텍스트 기반 페이지 분류기에서 **범용 Semantic Page Understanding Layer**로 확장하기 위한 작업 지시서이다. + +현재 시스템은 다음과 같은 제한적인 page type만 다룬다. + +```txt +ProductPage +CategoryPage +SearchPage +BoardPage +NoticePage +BrandStoryPage +PromotionPage +UnknownPage +``` + +이 구조는 쇼핑몰 일부 페이지를 분류하는 데는 사용할 수 있으나, 범용 온톨로지 구축 플랫폼에는 부족하다. + +범용 플랫폼은 인터넷에 존재하는 다양한 페이지를 다음 관점으로 분류할 수 있어야 한다. + +```txt +Page Domain +Page Archetype +Semantic Page Type +Main Entity Type +Action Intent +Graph Role +Analyze Strategy +LLM Policy +Confidence +Evidence +``` + +즉, 목표는 단순히 `ProductPage` 같은 enum 하나를 맞히는 것이 아니라, 페이지의 의미적 역할을 여러 축으로 해석하고, 온톨로지 그래프 구축에 필요한 분석 전략까지 결정하는 것이다. + +--- + +## 1. 현재 구조의 문제점 + +### 1.1 Page Type이 너무 적다 + +현재 분류 항목은 대부분 commerce 또는 게시판 중심이다. + +```txt +ProductPage +CategoryPage +SearchPage +BoardPage +BrandStoryPage +``` + +하지만 인터넷에는 다음과 같은 페이지가 존재한다. + +```txt +ArticlePage +NewsPage +BlogPostPage +FAQPage +QAPage +WikiPage +DocumentationPage +JobPostingPage +CoursePage +VideoPage +ProfilePage +LocalBusinessPage +RealEstateListingPage +PricingPage +CheckoutPage +LoginPage +TermsPage +PrivacyPolicyPage +APIReferencePage +DatasetPage +ResearchPaperPage +``` + +현 구조에서는 이들을 대부분 `UnknownPage` 또는 부정확한 기존 타입으로 분류하게 된다. + +--- + +### 1.2 URL 휴리스틱 의존도가 높다 + +현재 방식은 대체로 다음과 같은 구조다. + +```python +if "/product/" in url: + return "ProductPage" + +if "/board/" in url: + return "BoardPage" +``` + +이 방식은 다음 환경에서 쉽게 깨진다. + +```txt +SPA +Headless Commerce +Dynamic Route +Query 기반 페이지 +다국어 URL +짧은 URL +해시 라우팅 +CMS 기반 자동 생성 페이지 +AI 생성 페이지 +``` + +예: + +```txt +/p/12345 +/x/abc +/node/987 +/view?id=123 +/ko/contents/123 +``` + +이 URL만 보고는 페이지 의미를 알 수 없다. + +--- + +### 1.3 CategoryPage, SearchPage, BoardPage를 너무 쉽게 skip한다 + +현재 구조는 보통 다음 흐름이다. + +```txt +classify_page() + -> should_analyze_page() + -> analyze_page_types에 없으면 Extractor 실행 안 함 +``` + +이 때문에 `CategoryPage`, `SearchPage`, `BoardPage`는 기본적으로 분석 대상에서 빠질 가능성이 높다. + +그러나 범용 온톨로지 플랫폼에서는 이 페이지들이 중요하다. + +```txt +CategoryPage -> 카테고리 계층, 상품 목록, taxonomy 관계 +SearchPage -> 검색 의도, 결과 후보, query-result 관계 +BoardPage -> 게시글 목록, 질문/답변/토론 구조 +ListingPage -> entity collection, relation hub +ArchivePage -> 시간 축 기반 콘텐츠 구조 +TagPage -> topic-entity 관계 +``` + +따라서 “분석 여부”는 단순 boolean이 아니라, 페이지 타입별 분석 전략으로 분리해야 한다. + +--- + +### 1.4 UnknownPage를 버리면 안 된다 + +현재는 분류 실패 시 `UnknownPage`로 두고 사실상 분석에서 제외될 가능성이 크다. + +하지만 범용 플랫폼에서 `UnknownPage`는 새로운 페이지 패턴을 발견하는 출발점이다. + +예: + +```txt +UnknownPage Cluster A: +- 향수 노트 비교표가 있음 +- 여러 상품을 향 계열별로 비교 +- 일반 ProductPage도 CategoryPage도 아님 + +새 후보: +PerfumeNoteComparisonPage +``` + +따라서 Unknown은 폐기 대상이 아니라, clustering 및 taxonomy 확장 후보로 저장해야 한다. + +--- + +## 2. 목표 아키텍처 + +기존 구조: + +```txt +URL / title / text / html + -> page_classifier.py + -> page_type + -> should_analyze_page() +``` + +개선 구조: + +```txt +Raw Page Snapshot + -> Signal Extraction + -> Evidence Scoring + -> Domain Classification + -> Archetype Classification + -> Entity Type Classification + -> Action Intent Classification + -> Graph Role Assignment + -> Semantic Page Type Decision + -> Analyze Strategy Decision + -> LLM Policy Decision + -> Unknown Pattern Storage / Clustering +``` + +--- + +## 3. 핵심 개념 + +### 3.1 Page Domain + +페이지가 속한 큰 의미 영역이다. + +```txt +Commerce +Editorial +Community +Knowledge +Corporate +Local +Education +Jobs +Media +Software +Finance +Government +Healthcare +Transaction +System +Unknown +``` + +--- + +### 3.2 Page Archetype + +페이지의 구조적 역할이다. + +```txt +Home +Landing +Detail +Listing +Collection +SearchResult +Profile +Article +Thread +Form +Transaction +Dashboard +Document +Media +Error +SystemResource +Unknown +``` + +--- + +### 3.3 Semantic Page Type + +구체적인 페이지 타입이다. + +예: + +```txt +ProductDetailPage +CategoryListingPage +ArticlePage +ForumThreadPage +FAQPage +JobPostingPage +LocalBusinessPage +CheckoutPage +LoginPage +DocumentationPage +``` + +--- + +### 3.4 Main Entity Type + +페이지가 중심으로 삼는 엔티티이다. + +```txt +Product +Service +Article +NewsArticle +Person +Organization +Place +Event +JobPosting +Course +Question +Answer +Review +Dataset +SoftwareApplication +MediaObject +Recipe +RealEstateProperty +MedicalCondition +LegalDocument +FinancialProduct +UnknownEntity +``` + +--- + +### 3.5 Action Intent + +페이지가 사용자를 유도하는 행동이다. + +```txt +Read +Buy +Subscribe +Reserve +Book +Apply +Download +Watch +Listen +Search +Compare +Filter +Ask +Answer +Comment +Review +Login +Register +Pay +Contact +Navigate +Learn +Verify +Configure +Manage +``` + +--- + +### 3.6 Graph Role + +온톨로지 그래프 안에서 이 페이지가 수행하는 역할이다. + +```txt +EntityAnchor +RelationHub +NavigationHub +CollectionHub +SearchHub +TransactionOnly +PolicySource +ClaimSource +ProfileAnchor +MediaAnchor +ReferenceSource +SystemResource +NoisePage +UnknownPattern +``` + +--- + +### 3.7 Analyze Strategy + +페이지를 어떻게 분석할지 결정하는 전략이다. + +```txt +AnalyzeFull +AnalyzeStructureOnly +AnalyzeEntityOnly +AnalyzeRelationsOnly +AnalyzeMetadataOnly +AnalyzeDocumentOnly +AnalyzeDiscoveryOnly +SkipProtected +SkipNoise +``` + +--- + +### 3.8 LLM Policy + +LLM 사용 여부 및 사용 범위다. + +```txt +LLMFull +LLMLight +LLMForAmbiguityOnly +RuleOnly +NoLLM +Skip +``` + +--- + +## 4. Page Taxonomy v1 + +아래 taxonomy는 초기 버전이다. 구현 시 enum 또는 문자열 상수로 관리한다. + +--- + +## 4.1 Site / Navigation 계열 + +```txt +HomePage +LandingPage +PortalPage +SectionHomePage +CategoryPage +SubcategoryPage +TagPage +TopicPage +CollectionPage +ArchivePage +SitemapPage +DirectoryPage +IndexPage +SearchPage +SearchResultsPage +FilteredResultsPage +PaginationPage +LocaleSelectorPage +LanguageRedirectPage +RedirectPage +NotFoundPage +ErrorPage +MaintenancePage +ComingSoonPage +RobotsBlockedPage +``` + +### 주요 신호 + +```txt +many internal links +breadcrumb +category tree +pagination +tag links +archive dates +search form +sitemap XML or sitemap-like links +locale links +error status text +``` + +### 온톨로지 역할 + +```txt +사이트 구조 파악 +카테고리 계층 파악 +내부 링크 그래프 구축 +탐색 우선순위 결정 +``` + +--- + +## 4.2 Commerce / Marketplace 계열 + +```txt +ProductDetailPage +ProductVariantPage +ProductBundlePage +ProductComparisonPage +ProductReviewPage +ProductQnAPage +ProductManualPage +ProductSpecPage +CategoryListingPage +ProductListingPage +BrandCatalogPage +SellerStorePage +MarketplaceListingPage +SearchProductResultsPage +DealPage +SalePage +CouponPage +PromotionPage +CampaignLandingPage +SubscriptionPlanPage +PricingPage +CartPage +CheckoutPage +PaymentPage +OrderPage +OrderConfirmationPage +OrderTrackingPage +WishlistPage +GiftCardPage +StoreLocatorPage +InventoryAvailabilityPage +AuctionPage +RentalProductPage +BookingProductPage +ServiceProductPage +``` + +### 주요 신호 + +```txt +schema.org Product +schema.org Offer +schema.org AggregateRating +price +currency +availability +add to cart +buy now +variant selector +quantity selector +SKU +brand +product image gallery +reviews +rating +shipping +return policy +repeated product cards +filters +sort control +pagination +``` + +### 온톨로지 역할 + +```txt +Product 엔티티 생성 +Brand 관계 생성 +Category 관계 생성 +Offer / Price / Availability 추출 +Review / Rating 관계 추출 +RelatedProduct 관계 추출 +``` + +--- + +## 4.3 Editorial / Article / Publishing 계열 + +```txt +ArticlePage +NewsArticlePage +BlogPostPage +OpinionPage +EditorialPage +InterviewPage +ReportPage +ColumnPage +PressArticlePage +MagazinePage +GuidePage +TutorialPage +HowToPage +RecipePage +CaseStudyPage +WhitePaperPage +ResearchSummaryPage +StoryPage +ChapterPage +SeriesPage +AuthorArticleListPage +PaywalledArticlePage +SponsoredContentPage +``` + +### 주요 신호 + +```txt +schema.org Article +schema.org NewsArticle +schema.org BlogPosting +headline +author +publisher +datePublished +dateModified +articleBody +byline +section +tags +hero image +related articles +paywall marker +``` + +### 온톨로지 역할 + +```txt +Article / Topic / Author / Publisher 엔티티 생성 +about 관계 생성 +citation / source 관계 생성 +temporal coverage 추출 +claim 후보 추출 +``` + +--- + +## 4.4 Community / UGC 계열 + +```txt +ForumHomePage +ForumBoardPage +ForumThreadPage +DiscussionPage +CommentThreadPage +QAPage +FAQPage +ReviewPage +UserReviewPage +CommunityPostPage +SocialPostPage +TimelinePage +FeedPage +UserProfilePage +CreatorProfilePage +GroupPage +CommunityPage +PollPage +PetitionPage +RankingPage +LeaderboardPage +ReputationPage +BadgePage +``` + +### 주요 신호 + +```txt +question +answer +accepted answer +comments +reply +thread +votes +likes +author profile +user avatar +posted date +edited date +review rating +FAQ accordion +Q&A structured data +``` + +### 온톨로지 역할 + +```txt +Question / Answer 엔티티 생성 +User / Author 관계 생성 +Thread 관계 생성 +Claim / Opinion 분리 +Reputation / Vote / AcceptedAnswer 추출 +``` + +--- + +## 4.5 Knowledge / Reference / Documentation 계열 + +```txt +WikiPage +EncyclopediaPage +GlossaryPage +DefinitionPage +ReferencePage +DocumentationPage +DeveloperDocsPage +APIDocumentationPage +APIReferencePage +SDKDocumentationPage +ManualPage +SpecificationPage +StandardPage +ProtocolPage +ChangelogPage +ReleaseNotesPage +ErrorCodePage +TroubleshootingPage +KnowledgeBaseArticlePage +DatasetPage +DataCatalogPage +ResearchPaperPage +PatentPage +CitationPage +BibliographyPage +``` + +### 주요 신호 + +```txt +definition +table of contents +code block +API method +parameter table +version +endpoint +changelog +release notes +specification +standard +citation +references +dataset metadata +``` + +### 온톨로지 역할 + +```txt +Concept / Term / Definition 추출 +API / Method / Parameter 관계 추출 +Version 관계 추출 +Dataset metadata 추출 +Reference graph 생성 +``` + +--- + +## 4.6 Corporate / Organization 계열 + +```txt +AboutPage +CompanyPage +BrandStoryPage +MissionPage +VisionPage +HistoryPage +TeamPage +FounderPage +LeadershipPage +ContactPage +LocationPage +BranchPage +InvestorRelationsPage +IRPage +FinancialReportPage +PressReleasePage +MediaKitPage +PartnershipPage +FranchisePage +CareersHomePage +JobPostingPage +RecruitPage +CulturePage +LegalPage +TermsPage +PrivacyPolicyPage +CookiePolicyPage +AccessibilityPage +CompliancePage +SecurityPage +TrustCenterPage +``` + +### 주요 신호 + +```txt +about us +company +mission +vision +history +team +founder +leadership +contact +address +investor relations +press release +careers +privacy policy +terms of service +cookie policy +security +compliance +``` + +### 온톨로지 역할 + +```txt +Organization 엔티티 생성 +Founder / Location / Contact 관계 생성 +Policy 문서 분류 +법적/계약적 문장 추출 +채용 정보 추출 +``` + +--- + +## 4.7 Local / Place / Travel / Real Estate 계열 + +```txt +PlaceDetailPage +LocalBusinessPage +RestaurantPage +MenuPage +HotelPage +RoomPage +VacationRentalPage +TravelDestinationPage +AttractionPage +ItineraryPage +MapPage +MapSearchResultsPage +RealEstateListingPage +PropertyDetailPage +PropertySearchResultsPage +AgentProfilePage +OpenHousePage +ReservationPage +BookingPage +AvailabilityCalendarPage +TransportRoutePage +FlightPage +TrainPage +BusRoutePage +``` + +### 주요 신호 + +```txt +address +geo coordinates +map +opening hours +menu +reservation +booking +room availability +travel dates +property price +bedrooms +bathrooms +area +agent +route +schedule +``` + +### 온톨로지 역할 + +```txt +Place 엔티티 생성 +Address / Geo / OpeningHours 추출 +Reservation 가능성 판단 +Nearby 관계 생성 +Availability 추출 +``` + +--- + +## 4.8 Education / Learning 계열 + +```txt +CourseDetailPage +CourseListPage +CurriculumPage +LessonPage +LecturePage +TutorialPage +AssignmentPage +QuizPage +ExamPage +FlashcardPage +EducationQAPage +MathSolverPage +SchoolPage +UniversityPage +ProgramPage +DegreePage +CertificationPage +InstructorProfilePage +LearningPathPage +``` + +### 주요 신호 + +```txt +course +lesson +curriculum +instructor +learning objective +assignment +quiz +exam +certificate +degree +program +tuition +syllabus +``` + +### 온톨로지 역할 + +```txt +Course / Lesson / Instructor 엔티티 생성 +Prerequisite 관계 생성 +LearningObjective 추출 +Question / Answer / Solution 구조화 +``` + +--- + +## 4.9 Jobs / Career 계열 + +```txt +JobPostingPage +JobSearchResultsPage +CompanyJobsPage +CareerCategoryPage +ApplicationFormPage +RecruitmentLandingPage +EmployerProfilePage +EmployerReviewPage +SalaryPage +InterviewReviewPage +BenefitsPage +InternshipPage +FreelanceGigPage +``` + +### 주요 신호 + +```txt +job title +employment type +salary +location +remote +apply +requirements +responsibilities +benefits +company +recruiter +deadline +``` + +### 온톨로지 역할 + +```txt +Job 엔티티 생성 +Employer 관계 생성 +Location / Salary / EmploymentType 추출 +Skill requirement 추출 +``` + +--- + +## 4.10 Media / Entertainment 계열 + +```txt +VideoPage +VideoWatchPage +LiveStreamPage +PodcastPage +EpisodePage +MusicTrackPage +AlbumPage +ArtistPage +MoviePage +TVSeriesPage +TVEpisodePage +GameDetailPage +GameGuidePage +ImagePage +ImageGalleryPage +PhotoStoryPage +MediaGalleryPage +DownloadMediaPage +StreamingChannelPage +``` + +### 주요 신호 + +```txt +video player +audio player +duration +episode +season +album +artist +track +movie +trailer +live +stream +gallery +image grid +download +``` + +### 온톨로지 역할 + +```txt +MediaObject 엔티티 생성 +Creator / Performer / Publisher 관계 생성 +Duration / Episode / Series 관계 추출 +License / UsageInfo 추출 +``` + +--- + +## 4.11 Software / SaaS / App 계열 + +```txt +SoftwareProductPage +SaaSProductPage +FeaturePage +PricingPage +IntegrationPage +PluginPage +ExtensionPage +AppStoreListingPage +PackagePage +RepositoryPage +ReleasePage +ChangelogPage +IssuePage +PullRequestPage +DocumentationPage +APIReferencePage +StatusPage +DashboardPage +SettingsPage +AdminPage +LoginPage +SignupPage +OnboardingPage +BillingPage +UsageReportPage +``` + +### 주요 신호 + +```txt +software +app +SaaS +feature +pricing +integration +plugin +extension +repository +release +changelog +issue +pull request +status +dashboard +settings +billing +API +SDK +``` + +### 온톨로지 역할 + +```txt +Software / Version / Feature 엔티티 생성 +Dependency 관계 생성 +Release 관계 생성 +Issue / PR / Commit 관계 추출 +``` + +--- + +## 4.12 Finance / Legal / Government 계열 + +```txt +BankProductPage +LoanPage +CreditCardPage +InsuranceProductPage +InvestmentProductPage +StockQuotePage +CryptoAssetPage +FinancialReportPage +TaxInfoPage +GovernmentServicePage +PublicNoticePage +RegulationPage +LawPage +CourtCasePage +LegalArticlePage +PolicyPage +FormPage +ApplicationPage +PermitPage +LicensePage +PublicDataPage +ProcurementPage +TenderPage +``` + +### 주요 신호 + +```txt +interest rate +APR +loan +credit card +insurance +investment +stock quote +financial statement +tax +government +regulation +law +court +policy +permit +license +tender +procurement +``` + +### 온톨로지 역할 + +```txt +Regulation / Policy / Law 엔티티 생성 +Obligation / Prohibition / Permission 추출 +Institution 관계 생성 +Form requirement 추출 +``` + +--- + +## 4.13 Healthcare / Medical 계열 + +```txt +MedicalArticlePage +ConditionPage +SymptomPage +TreatmentPage +DrugPage +SupplementPage +DoctorProfilePage +HospitalPage +ClinicPage +AppointmentPage +InsuranceCoveragePage +ClinicalTrialPage +MedicalFAQPage +HealthCalculatorPage +EmergencyInfoPage +``` + +### 주요 신호 + +```txt +condition +symptom +treatment +drug +dosage +side effect +doctor +hospital +clinic +appointment +clinical trial +insurance coverage +emergency +``` + +### 온톨로지 역할 + +```txt +Condition / Treatment / Drug 엔티티 생성 +Symptom 관계 생성 +Medical claim 추출 +Source reliability 분리 +``` + +--- + +## 4.14 Transaction / Account / Protected 계열 + +```txt +LoginPage +SignupPage +PasswordResetPage +AccountPage +ProfileSettingsPage +NotificationPage +MessageInboxPage +CartPage +CheckoutPage +PaymentPage +SubscriptionManagementPage +BillingPage +InvoicePage +OrderHistoryPage +UploadPage +DownloadPage +FormPage +SurveyPage +ConsentPage +AgeGatePage +CaptchaPage +PaywallPage +AccessDeniedPage +SessionExpiredPage +``` + +### 주요 신호 + +```txt +login form +password field +signup +reset password +account settings +payment fields +checkout +billing +invoice +consent +captcha +age gate +access denied +session expired +``` + +### 온톨로지 역할 + +```txt +대부분 분석 제외 +개인정보 보호 +크롤링 중단 또는 제한 +거래 흐름만 메타 수준으로 기록 +``` + +--- + +## 4.15 System / Technical / Machine-readable 계열 + +```txt +RSSFeedPage +AtomFeedPage +XMLSitemapPage +RobotsTxtPage +ManifestPage +OpenSearchDescriptionPage +JSONEndpointPage +APIEndpointPage +GraphQLEndpointPage +WebhookEndpointPage +FileDownloadPage +PDFDocumentPage +CSVDocumentPage +XMLDocumentPage +ImageAssetPage +VideoAssetPage +FontAssetPage +ScriptAssetPage +StylesheetAssetPage +``` + +### 주요 신호 + +```txt +content-type +xml +json +rss +atom +sitemap +robots.txt +manifest +API response +file extension +download headers +``` + +### 온톨로지 역할 + +```txt +크롤링 정책 파악 +사이트 구조 파악 +데이터 소스 발견 +문서형 리소스 별도 파서로 전달 +``` + +--- + +## 5. Signal Extraction 설계 + +`page_classifier.py`가 직접 모든 것을 처리하지 말고, signal extractor를 분리한다. + +권장 파일 구조: + +```txt +ontology_platform/ + classifier/ + page_classifier.py + page_type_taxonomy.py + page_signals.py + page_signal_extractor.py + page_type_scorer.py + page_analysis_policy.py + adaptive_page_classifier.py +``` + +기존 프로젝트 구조에 맞춰 경로는 조정해도 된다. + +--- + +## 5.1 Raw Page Snapshot + +분류 함수 입력은 다음 정보를 받을 수 있어야 한다. + +```python +@dataclass +class RawPageSnapshot: + url: str + final_url: str | None + status_code: int | None + content_type: str | None + + title: str | None + text: str | None + html: str | None + rendered_html: str | None + + metadata: dict + open_graph: dict + twitter_card: dict + json_ld: list[dict] + microdata: list[dict] + rdfa: list[dict] + + headings: list[str] + links: list[dict] + images: list[dict] + forms: list[dict] + buttons: list[str] + inputs: list[dict] + tables: list[dict] + + breadcrumbs: list[str] + source_zones: list[str] + screenshot_path: str | None +``` + +초기 구현에서는 모든 필드가 없어도 된다. +없는 값은 `None` 또는 빈 리스트로 처리한다. + +--- + +## 5.2 Page Signals + +추출 결과는 다음 형태로 관리한다. + +```python +@dataclass +class PageSignals: + # structured data + schema_types: set[str] + og_type: str | None + twitter_card_type: str | None + + # commerce + has_price: bool + has_currency: bool + has_cart_button: bool + has_buy_button: bool + has_variant_selector: bool + has_sku: bool + has_rating: bool + has_review_section: bool + has_product_gallery: bool + + # listing + has_repeated_cards: bool + repeated_card_count: int + has_filter_panel: bool + has_sort_control: bool + has_pagination: bool + + # editorial + has_author: bool + has_published_date: bool + has_modified_date: bool + has_article_body: bool + has_tags: bool + + # community + has_question: bool + has_answer: bool + has_comments: bool + has_votes: bool + has_thread_structure: bool + has_faq_structure: bool + + # knowledge/docs + has_code_blocks: bool + has_toc: bool + has_api_endpoint: bool + has_parameter_table: bool + has_version_info: bool + + # corporate/legal + has_contact_info: bool + has_address: bool + has_policy_terms: bool + has_privacy_terms: bool + has_career_terms: bool + + # transaction/protected + has_login_form: bool + has_password_field: bool + has_payment_fields: bool + has_captcha: bool + has_access_denied: bool + + # graph + internal_link_count: int + external_link_count: int + product_link_count: int + category_link_count: int + profile_link_count: int + article_link_count: int + + # text/layout + dominant_language: str | None + keyword_hits: dict[str, int] +``` + +--- + +## 6. Scoring 방식 + +단일 if-else가 아니라 evidence scoring으로 분류한다. + +예: + +```python +scores = { + "ProductDetailPage": 0.0, + "CategoryListingPage": 0.0, + "ArticlePage": 0.0, + "QAPage": 0.0, + "LoginPage": 0.0, +} +``` + +--- + +### 6.1 ProductDetailPage scoring 예시 + +```txt +schema.org Product +0.40 +schema.org Offer +0.15 +price detected +0.15 +cart button +0.20 +variant selector +0.15 +SKU +0.10 +product image gallery +0.10 +review section +0.05 +URL product hint +0.05 +``` + +--- + +### 6.2 CategoryListingPage scoring 예시 + +```txt +repeated product cards +0.35 +filter panel +0.20 +sort control +0.15 +pagination +0.10 +many product links +0.20 +breadcrumb category +0.10 +URL category/list hint +0.05 +``` + +--- + +### 6.3 ArticlePage scoring 예시 + +```txt +schema.org Article +0.35 +schema.org NewsArticle +0.35 +author +0.15 +published date +0.15 +article body +0.20 +headline +0.10 +tags +0.05 +URL blog/news/article hint +0.05 +``` + +--- + +### 6.4 QAPage scoring 예시 + +```txt +schema.org QAPage +0.35 +question block +0.20 +answer block +0.20 +accepted answer +0.15 +votes +0.10 +comments +0.05 +URL question/qna hint +0.05 +``` + +--- + +### 6.5 LoginPage scoring 예시 + +```txt +password input +0.40 +login keyword +0.20 +email/user id input +0.15 +submit button +0.10 +signup/reset password links +0.10 +``` + +--- + +## 7. Classification Result 모델 + +분류 결과는 단일 문자열이 아니라 아래 구조로 반환한다. + +```python +@dataclass +class EvidenceItem: + key: str + value: str | int | float | bool | None + weight: float + source: str + message: str + + +@dataclass +class PageClassificationResult: + url: str + + primary_page_type: str + secondary_page_types: list[str] + + domain: str + archetype: str + main_entity_type: str | None + + action_intents: list[str] + graph_roles: list[str] + + confidence: float + alternatives: list[tuple[str, float]] + evidence: list[EvidenceItem] + + should_analyze: bool + analyze_strategy: str + llm_policy: str + + is_protected: bool + is_noise: bool +``` + +예: + +```json +{ + "primary_page_type": "CategoryListingPage", + "secondary_page_types": ["FilteredResultsPage"], + "domain": "Commerce", + "archetype": "Listing", + "main_entity_type": "Product", + "action_intents": ["Filter", "Compare", "Navigate"], + "graph_roles": ["CollectionHub", "RelationHub"], + "confidence": 0.88, + "should_analyze": true, + "analyze_strategy": "AnalyzeRelationsOnly", + "llm_policy": "RuleOnly", + "is_protected": false, + "is_noise": false +} +``` + +--- + +## 8. Analyze Strategy 정책 + +기존 `should_analyze_page(page_type, analyze_page_types)`는 유지하되, 내부를 확장한다. + +기존 방식: + +```txt +ProductPage -> analyze +CategoryPage -> skip +SearchPage -> skip +BoardPage -> skip +``` + +개선 방식: + +```txt +ProductDetailPage -> AnalyzeFull +ArticlePage -> AnalyzeFull +BrandStoryPage -> AnalyzeFull +CategoryListingPage -> AnalyzeRelationsOnly +SearchResultsPage -> AnalyzeDiscoveryOnly +ForumThreadPage -> AnalyzeFull +ForumBoardPage -> AnalyzeRelationsOnly +FAQPage -> AnalyzeFull +QAPage -> AnalyzeFull +TermsPage -> AnalyzeDocumentOnly +PrivacyPolicyPage -> AnalyzeDocumentOnly +SitemapPage -> AnalyzeDiscoveryOnly +RobotsTxtPage -> AnalyzeMetadataOnly +LoginPage -> SkipProtected +CheckoutPage -> SkipProtected +PaymentPage -> SkipProtected +ErrorPage -> SkipNoise +NotFoundPage -> SkipNoise +``` + +--- + +## 9. LLM Policy 정책 + +LLM은 모든 페이지에 쓰지 않는다. + +```txt +LLMFull +- ArticlePage +- ProductDetailPage +- BrandStoryPage +- ResearchPaperPage +- LegalArticlePage + +LLMLight +- FAQPage +- QAPage +- DocumentationPage +- TutorialPage + +LLMForAmbiguityOnly +- CategoryListingPage +- SearchResultsPage +- ForumBoardPage +- ArchivePage +- TagPage + +RuleOnly +- SitemapPage +- RSSFeedPage +- RobotsTxtPage +- LoginPage +- CheckoutPage +- PaymentPage + +Skip +- ErrorPage +- NotFoundPage +- AccessDeniedPage +- CaptchaPage +``` + +--- + +## 10. UnknownPage 처리 + +UnknownPage는 버리지 않는다. + +분류 confidence가 낮은 경우: + +```txt +primary_page_type = "UnknownPage" +graph_roles = ["UnknownPattern"] +analyze_strategy = "AnalyzeMetadataOnly" +llm_policy = "LLMForAmbiguityOnly" 또는 "NoLLM" +``` + +저장해야 할 정보: + +```txt +url +title +text sample +html fingerprint +dom fingerprint +schema types +link pattern +button labels +forms +top keywords +embedding +classification alternatives +``` + +향후 clustering 대상: + +```txt +UnknownPatternCluster +``` + +새 page type 후보 생성 예: + +```txt +Unknown cluster 12 + -> repeated comparison tables + -> product attributes + -> no cart button + -> many product links + => ProductComparisonPage 후보 +``` + +--- + +## 11. 구현 단계 + +## Phase 1. Taxonomy와 Result 모델 추가 + +### 작업 + +1. `page_type_taxonomy.py` 추가 +2. PageDomain enum 추가 +3. PageArchetype enum 추가 +4. PageType enum 또는 문자열 상수 추가 +5. EntityType enum 추가 +6. ActionIntent enum 추가 +7. GraphRole enum 추가 +8. AnalyzeStrategy enum 추가 +9. LLMPolicy enum 추가 +10. `PageClassificationResult`, `EvidenceItem` dataclass 추가 + +### 완료 기준 + +- 기존 `ProductPage`, `CategoryPage` 등과 호환되어야 한다. +- 기존 코드에서 string page_type만 기대하는 부분은 깨지지 않도록 compatibility helper를 제공한다. + +예: + +```python +def get_legacy_page_type(result: PageClassificationResult) -> str: + return result.primary_page_type +``` + +--- + +## Phase 2. Signal Extractor 추가 + +### 작업 + +1. `page_signals.py` 추가 +2. `page_signal_extractor.py` 추가 +3. HTML에서 JSON-LD 추출 +4. OpenGraph 추출 +5. Twitter Card 추출 +6. meta 태그 추출 +7. button text 추출 +8. form/input 추출 +9. link pattern 추출 +10. 반복 카드 후보 탐지 +11. breadcrumb 후보 탐지 +12. price/currency 후보 탐지 +13. article author/date 후보 탐지 +14. login/password/payment/captcha 후보 탐지 + +### 완료 기준 + +- 입력 HTML이 일부 깨져도 예외 없이 동작해야 한다. +- BeautifulSoup 또는 현재 프로젝트에서 사용하는 parser에 맞춰 구현한다. +- 없는 값은 빈 리스트/빈 dict/False로 처리한다. + +--- + +## Phase 3. Scoring 기반 Page Type 분류 + +### 작업 + +1. `page_type_scorer.py` 추가 +2. 주요 page type별 scoring function 작성 +3. score normalize +4. top score와 alternatives 산출 +5. confidence 계산 +6. evidence 기록 + +최소 구현 대상: + +```txt +ProductDetailPage +CategoryListingPage +SearchResultsPage +ArticlePage +BlogPostPage +QAPage +FAQPage +ForumBoardPage +ForumThreadPage +BrandStoryPage +AboutPage +ContactPage +DocumentationPage +APIReferencePage +JobPostingPage +PricingPage +LoginPage +CheckoutPage +TermsPage +PrivacyPolicyPage +SitemapPage +RSSFeedPage +ErrorPage +UnknownPage +``` + +### 완료 기준 + +- 단일 if-else return 금지 +- 반드시 evidence list를 남긴다 +- confidence가 낮으면 UnknownPage로 보낼 수 있어야 한다 + +--- + +## Phase 4. Analyze Strategy / LLM Policy 분리 + +### 작업 + +1. `page_analysis_policy.py` 추가 +2. PageType -> AnalyzeStrategy mapping 작성 +3. PageType -> LLMPolicy mapping 작성 +4. 기존 `should_analyze_page()`를 compatibility 형태로 유지 +5. 신규 함수 추가 + +예: + +```python +def decide_analyze_strategy(result: PageClassificationResult) -> AnalyzeStrategy: + ... + +def decide_llm_policy(result: PageClassificationResult) -> LLMPolicy: + ... + +def should_analyze_page(result_or_page_type, analyze_page_types=None) -> bool: + ... +``` + +### 완료 기준 + +- 기존 호출부가 바로 깨지지 않아야 한다. +- CategoryListingPage는 기본 skip이 아니라 `AnalyzeRelationsOnly`가 되어야 한다. +- SearchResultsPage는 `AnalyzeDiscoveryOnly`가 되어야 한다. +- Login/Checkout/Payment는 `SkipProtected`가 되어야 한다. + +--- + +## Phase 5. 기존 Extractor 연결 수정 + +### 작업 + +기존 흐름: + +```txt +classify_page(...) + -> page_type + -> should_analyze_page(page_type, analyze_page_types) + -> Extractor 실행 + -> HybridExtractor 내부에서 LLM skip 판단 +``` + +개선 흐름: + +```txt +classify_page(...) + -> PageClassificationResult + -> decide_analyze_strategy(result) + -> decide_llm_policy(result) + -> strategy에 따라 Extractor 또는 relation/link extractor 실행 +``` + +### 전략별 처리 + +```txt +AnalyzeFull + -> 기존 Extractor + HybridExtractor + LLM policy 적용 + +AnalyzeRelationsOnly + -> 링크, breadcrumb, category, repeated card 중심 추출 + -> full LLM 금지 + +AnalyzeDiscoveryOnly + -> crawl candidate, result link, pagination만 추출 + -> content claim 추출 금지 + +AnalyzeMetadataOnly + -> title, metadata, schema, canonical, link relation만 저장 + +AnalyzeDocumentOnly + -> 법률/정책/문서 구조 추출 + -> 필요 시 LLM 사용 + +SkipProtected + -> 개인정보/계정/결제 페이지 분석 금지 + +SkipNoise + -> 저장 최소화 또는 제외 +``` + +--- + +## Phase 6. Unknown Pattern 저장 기반 추가 + +### 작업 + +1. UnknownPage 또는 confidence 낮은 페이지를 별도 저장 +2. DOM fingerprint 생성 +3. text fingerprint 생성 +4. link pattern summary 생성 +5. 향후 clustering을 위한 embedding hook 추가 + +초기에는 clustering까지 구현하지 않아도 된다. +다만 데이터 구조는 남겨야 한다. + +--- + +## Phase 7. 테스트 추가 + +### 단위 테스트 대상 + +```txt +ProductDetailPage +CategoryListingPage +SearchResultsPage +ArticlePage +QAPage +FAQPage +ForumThreadPage +DocumentationPage +JobPostingPage +LoginPage +CheckoutPage +TermsPage +SitemapPage +UnknownPage +``` + +### 테스트 샘플 + +각 page type에 대해 최소 HTML fixture를 만든다. + +예: + +```txt +tests/fixtures/pages/product_detail.html +tests/fixtures/pages/category_listing.html +tests/fixtures/pages/article.html +tests/fixtures/pages/qapage.html +tests/fixtures/pages/login.html +``` + +### 테스트 기준 + +```txt +primary_page_type이 기대값과 일치 +confidence가 최소 기준 이상 +evidence가 비어 있지 않음 +analyze_strategy가 기대값과 일치 +llm_policy가 기대값과 일치 +protected page가 분석되지 않음 +UnknownPage가 예외 없이 처리됨 +``` + +--- + +## 12. 분류 예시 + +### 12.1 ProductDetailPage + +입력 신호: + +```txt +JSON-LD @type Product +price +add to cart +variant selector +product images +reviews +``` + +결과: + +```json +{ + "primary_page_type": "ProductDetailPage", + "domain": "Commerce", + "archetype": "Detail", + "main_entity_type": "Product", + "action_intents": ["Buy", "Review"], + "graph_roles": ["EntityAnchor"], + "confidence": 0.93, + "analyze_strategy": "AnalyzeFull", + "llm_policy": "LLMLight" +} +``` + +--- + +### 12.2 CategoryListingPage + +입력 신호: + +```txt +repeated product cards +filter panel +sort control +pagination +many product links +breadcrumb +``` + +결과: + +```json +{ + "primary_page_type": "CategoryListingPage", + "domain": "Commerce", + "archetype": "Listing", + "main_entity_type": "Product", + "action_intents": ["Filter", "Navigate", "Compare"], + "graph_roles": ["CollectionHub", "RelationHub"], + "confidence": 0.88, + "analyze_strategy": "AnalyzeRelationsOnly", + "llm_policy": "RuleOnly" +} +``` + +--- + +### 12.3 ArticlePage + +입력 신호: + +```txt +schema.org Article +headline +author +published date +article body +tags +``` + +결과: + +```json +{ + "primary_page_type": "ArticlePage", + "domain": "Editorial", + "archetype": "Article", + "main_entity_type": "Article", + "action_intents": ["Read"], + "graph_roles": ["ClaimSource", "EntityAnchor"], + "confidence": 0.91, + "analyze_strategy": "AnalyzeFull", + "llm_policy": "LLMFull" +} +``` + +--- + +### 12.4 LoginPage + +입력 신호: + +```txt +password input +email input +login button +reset password link +``` + +결과: + +```json +{ + "primary_page_type": "LoginPage", + "domain": "Transaction", + "archetype": "Form", + "main_entity_type": null, + "action_intents": ["Login"], + "graph_roles": ["TransactionOnly"], + "confidence": 0.96, + "analyze_strategy": "SkipProtected", + "llm_policy": "Skip" +} +``` + +--- + +## 13. 하위 호환성 요구 + +기존 코드가 아래처럼 page_type 문자열을 기대할 수 있다. + +```python +page_type = classify_page(...) +should_analyze_page(page_type, analyze_page_types) +``` + +따라서 처음부터 모든 호출부를 바꾸지 말고, compatibility layer를 둔다. + +권장: + +```python +def classify_page_legacy(*args, **kwargs) -> str: + result = classify_page(*args, **kwargs) + return result.primary_page_type +``` + +또는: + +```python +def normalize_page_type(page_type_or_result) -> str: + if isinstance(page_type_or_result, PageClassificationResult): + return page_type_or_result.primary_page_type + return str(page_type_or_result) +``` + +--- + +## 14. 기존 page type과 신규 page type 매핑 + +```txt +ProductPage -> ProductDetailPage +CategoryPage -> CategoryListingPage 또는 CategoryPage +SearchPage -> SearchResultsPage +BoardPage -> ForumBoardPage +NoticePage -> PublicNoticePage 또는 NoticePage +BrandStoryPage -> BrandStoryPage +PromotionPage -> PromotionPage 또는 CampaignLandingPage +UnknownPage -> UnknownPage +ReviewPage -> ReviewPage 또는 ProductReviewPage +``` + +기존 이름은 당분간 alias로 유지한다. + +--- + +## 15. 코딩 원칙 + +1. 단일 if-return 방식으로 확장하지 말 것 +2. evidence를 반드시 남길 것 +3. confidence를 반드시 계산할 것 +4. PageType 하나만 반환하지 말 것 +5. Category/Search/Board 계열을 무조건 skip하지 말 것 +6. Login/Checkout/Payment는 보호 페이지로 처리할 것 +7. UnknownPage는 폐기하지 말고 저장 가능한 구조로 만들 것 +8. JSON-LD, OpenGraph, meta, DOM, link graph를 모두 signal로 사용할 것 +9. 다국어 키워드 확장을 고려할 것 +10. 기존 호출부가 깨지지 않도록 compatibility helper를 제공할 것 + +--- + +## 16. 최소 완료 기준 + +이번 작업의 최소 완료 기준은 다음과 같다. + +```txt +1. PageClassificationResult dataclass 추가 +2. PageDomain / PageArchetype / PageType / EntityType / ActionIntent / GraphRole / AnalyzeStrategy / LLMPolicy 정의 +3. JSON-LD / OpenGraph / URL / DOM / text 기반 signal 추출 +4. 최소 20개 page type scoring 구현 +5. CategoryListingPage가 AnalyzeRelationsOnly로 처리됨 +6. SearchResultsPage가 AnalyzeDiscoveryOnly로 처리됨 +7. LoginPage / CheckoutPage / PaymentPage가 SkipProtected로 처리됨 +8. ProductDetailPage / ArticlePage / FAQPage / QAPage는 AnalyzeFull 가능 +9. UnknownPage가 evidence와 함께 반환됨 +10. 기존 should_analyze_page 호환성 유지 +11. 테스트 fixture 10개 이상 추가 +12. pytest 통과 +``` + +--- + +## 17. 최종 목표 + +이 작업의 최종 목표는 `page_classifier.py`를 다음 수준으로 확장하는 것이다. + +기존: + +```txt +URL/text 기반 page type 분류기 +``` + +개선: + +```txt +Semantic Page Understanding Layer +``` + +최종 파이프라인: + +```txt +Raw Page + -> Signal Extraction + -> Evidence Scoring + -> Domain Classification + -> Archetype Classification + -> Main Entity Classification + -> Action Intent Classification + -> Graph Role Assignment + -> Analyze Strategy Decision + -> LLM Policy Decision + -> Unknown Pattern Storage +``` + +이렇게 해야 온톨로지 플랫폼이 특정 쇼핑몰이나 특정 사이트에 종속되지 않고, 인터넷 전체의 다양한 페이지를 의미 단위로 해석할 수 있다. + +--- + +## 18. Codex 작업 요청 요약 + +Codex는 이 문서를 기준으로 다음 작업을 수행한다. + +```txt +1. 현재 page_classifier.py를 확인한다. +2. 기존 호출부와 테스트를 확인한다. +3. taxonomy / signal / scoring / policy 레이어를 분리한다. +4. 기존 단순 page_type 문자열 반환 구조를 PageClassificationResult 중심으로 확장한다. +5. 기존 코드가 깨지지 않도록 legacy compatibility를 유지한다. +6. Category/Search/Board를 단순 skip하지 않고 strategy 기반으로 처리한다. +7. protected/transaction page는 안전하게 skip한다. +8. UnknownPage는 evidence와 함께 저장 가능하게 만든다. +9. 최소 fixture 테스트를 추가한다. +10. pytest로 회귀 테스트를 확인한다. +``` + +--- + +## 19. 주의 사항 + +- 기존 엔진을 대규모로 폐기하지 말 것. +- 현재 parser, crawler, extractor 흐름을 먼저 파악한 뒤 최소 침습 방식으로 확장할 것. +- page type enum 확장은 허용하되, extractor 전체를 한 번에 갈아엎지 말 것. +- LLM 사용량이 늘어나지 않도록 `LLMPolicy`를 반드시 적용할 것. +- protected page에서 개인정보나 계정 정보를 추출하지 말 것. +- 분류가 애매할 경우 억지로 하나의 타입에 넣지 말고 alternatives와 confidence를 남길 것. +- Evidence 기반 디버깅이 가능해야 한다. + +--- + +## 20. 향후 확장 방향 + +이번 작업 이후 다음 단계로 확장할 수 있다. + +```txt +1. UnknownPage clustering +2. DOM fingerprint 기반 template detection +3. site-specific learned page archetype +4. screenshot 기반 visual block classification +5. multilingual keyword dictionary +6. schema.org type mapping 강화 +7. page type별 extraction schema 자동 선택 +8. crawl priority와 page type 연동 +9. graph relation confidence와 page evidence 연동 +10. admin UI에서 page classification 결과 검토 +``` diff --git a/ontology_platform/ont_platform/api/main.py b/ontology_platform/ont_platform/api/main.py index 4627c13..429b3d6 100644 --- a/ontology_platform/ont_platform/api/main.py +++ b/ontology_platform/ont_platform/api/main.py @@ -156,7 +156,7 @@ def create_app() -> FastAPI: # ─── /health ────────────────────────────────────────────────────── @app.get("/health", tags=["meta"]) - async def health() -> JSONResponse: + async def health(request: Request) -> JSONResponse: """Liveness check for the HTTP service and optional LLM readiness.""" settings = platform_config.load_settings() if _startup_error: @@ -172,7 +172,7 @@ def create_app() -> FastAPI: }, ) - ctx = get_app_context() + ctx = _request_app_context(request) if ctx.tools.llm is None: return JSONResponse( status_code=503, @@ -192,13 +192,13 @@ def create_app() -> FastAPI: # ─── /info ──────────────────────────────────────────────────────── @app.get("/info", tags=["meta"]) - async def info() -> JSONResponse: + async def info(request: Request) -> JSONResponse: """Service-level capabilities (mirrors OntoCast /info semantics).""" settings = platform_config.load_settings() phase = int(settings.phase) storage_backend = settings.storage_backend if not _startup_error: - ctx = get_app_context() + ctx = _request_app_context(request) phase = int(ctx.settings.phase) storage_backend = ctx.settings.storage_backend @@ -449,6 +449,13 @@ def create_app() -> FastAPI: return app +def _request_app_context(request: Request) -> AppContext: + override = request.app.dependency_overrides.get(get_app_context) + if override is not None: + return override() + return get_app_context() + + # Top-level instance for `uvicorn platform.api.main:app`. app = create_app() diff --git a/ontology_platform/ont_platform/api/product_backend.py b/ontology_platform/ont_platform/api/product_backend.py index 274a3d5..922f085 100644 --- a/ontology_platform/ont_platform/api/product_backend.py +++ b/ontology_platform/ont_platform/api/product_backend.py @@ -82,6 +82,6 @@ def _remove_route(app: FastAPI, path: str, methods: set[str]) -> None: for route in app.router.routes if not ( getattr(route, "path", None) == path - and set(getattr(route, "methods", set())) == methods + and methods.issubset(set(getattr(route, "methods", set()))) ) ] diff --git a/ontology_platform/web/frontend/src/lib/api/crawl.ts b/ontology_platform/web/frontend/src/lib/api/crawl.ts index 00c9ca0..9130870 100644 --- a/ontology_platform/web/frontend/src/lib/api/crawl.ts +++ b/ontology_platform/web/frontend/src/lib/api/crawl.ts @@ -86,6 +86,26 @@ export function isCrawlTerminal(status: string): boolean { return TERMINAL_CRAWL_STATUSES.has(status); } +export const extractorModelsResponseSchema = z.object({ + ok: z.boolean(), + error: z.string().optional(), + models: z + .array( + z + .object({ + id: z.string(), + owned_by: z.string().optional(), + }) + .passthrough(), + ) + .optional() + .default([]), +}); + +export type ExtractorModelsResponse = z.infer< + typeof extractorModelsResponseSchema +>; + export const crawlApi = { startByProject: (body: StartSiteCrawlRequest) => apiClient.post("/crawl-site/by-project", crawlJobSchema, body), @@ -99,4 +119,9 @@ export const crawlApi = { `/crawl-site/jobs/${encodeURIComponent(jobId)}/cancel`, crawlJobSchema, ), + listExtractorModels: (provider: string, baseUrl?: string | null) => + apiClient.post("/extractors/models", extractorModelsResponseSchema, { + provider, + base_url: baseUrl || null, + }), }; diff --git a/ontology_platform/web/frontend/src/pages/CrawlPage.tsx b/ontology_platform/web/frontend/src/pages/CrawlPage.tsx index e7b60fe..5091da4 100644 --- a/ontology_platform/web/frontend/src/pages/CrawlPage.tsx +++ b/ontology_platform/web/frontend/src/pages/CrawlPage.tsx @@ -1,4 +1,4 @@ -import { useState } from "react"; +import { useEffect, useState } from "react"; import { useNavigate, useParams } from "react-router-dom"; import { useTranslation } from "react-i18next"; import { useForm } from "react-hook-form"; @@ -36,7 +36,7 @@ import { useCrawlJob, useStartSiteCrawl, } from "@/hooks/useCrawl"; -import { isCrawlTerminal } from "@/lib/api/crawl"; +import { crawlApi, isCrawlTerminal } from "@/lib/api/crawl"; const startCrawlSchema = z.object({ source_name: z.string().min(1, "소스를 선택하세요"), @@ -150,7 +150,53 @@ export default function CrawlPage() { const sources = project?.sources ?? []; const sourceName = watch("source_name"); const extractionMode = watch("extraction_mode"); + const provider = watch("extractor_provider"); + const baseUrl = watch("extractor_base_url"); const usesLlm = extractionMode !== "rule_only"; + + const [loadedModelHint, setLoadedModelHint] = useState(""); + const [modelLookupStatus, setModelLookupStatus] = useState< + "idle" | "loading" | "ok" | "error" + >("idle"); + const [modelLookupError, setModelLookupError] = useState(""); + useEffect(() => { + if (!usesLlm || provider !== "lm_studio") { + setLoadedModelHint(""); + setModelLookupStatus("idle"); + setModelLookupError(""); + return; + } + let cancelled = false; + setModelLookupStatus("loading"); + setModelLookupError(""); + (async () => { + try { + const res = await crawlApi.listExtractorModels(provider, baseUrl); + if (cancelled) return; + if (!res.ok) { + setModelLookupStatus("error"); + setModelLookupError(res.error || "모델 조회 실패"); + return; + } + const first = res.models?.[0]?.id; + if (!first) { + setModelLookupStatus("error"); + setModelLookupError("로드된 모델이 없습니다"); + return; + } + setLoadedModelHint(first); + setValue("extractor_model", first, { shouldDirty: false }); + setModelLookupStatus("ok"); + } catch (e) { + if (cancelled) return; + setModelLookupStatus("error"); + setModelLookupError((e as Error).message); + } + })(); + return () => { + cancelled = true; + }; + }, [provider, baseUrl, usesLlm, setValue]); const selectedSource = sources.find((s) => s.name === sourceName); const progress = job?.progress; const visited = progress?.visited_count ?? 0; @@ -385,9 +431,30 @@ export default function CrawlPage() { + {provider === "lm_studio" && + modelLookupStatus === "loading" && ( +

+ LM Studio 로드 모델 확인 중... +

+ )} + {provider === "lm_studio" && + modelLookupStatus === "ok" && + loadedModelHint && ( +

+ LM Studio 로드 모델: {loadedModelHint} +

+ )} + {provider === "lm_studio" && + modelLookupStatus === "error" && ( +

+ LM Studio 모델 조회 실패: {modelLookupError} +

+ )}