[페이지 분류 강화 작업]
This commit is contained in:
LASTA_DEV01\lasta
@@ -29,7 +29,10 @@ from crawler_platform.app.core.database.repository import (
|
||||
make_claim_hash,
|
||||
)
|
||||
from crawler_platform.app.core.database.session import session_scope
|
||||
from crawler_platform.app.core.extractor.ai_provider import list_openai_compatible_models
|
||||
from crawler_platform.app.core.extractor.ai_provider import (
|
||||
list_lmstudio_loaded_models,
|
||||
list_openai_compatible_models,
|
||||
)
|
||||
from crawler_platform.app.core.extractor.factory import extractor_for_domain
|
||||
from crawler_platform.app.core.ontology.definitions import DOMAIN_ONTOLOGIES, Ontology, ontology_for_domain
|
||||
from crawler_platform.app.core.ontology.domain_discovery import DomainDiscoveryService
|
||||
@@ -1686,7 +1689,7 @@ def register_routes(app, database_url: str) -> None:
|
||||
def extractor_models(request: ExtractorModelsRequest):
|
||||
try:
|
||||
if request.provider == "lm_studio":
|
||||
models = list_openai_compatible_models(request.base_url or "http://localhost:1234/v1")
|
||||
models = list_lmstudio_loaded_models(request.base_url or "http://localhost:1234/v1")
|
||||
return {"ok": True, "models": models}
|
||||
if request.provider == "openai":
|
||||
import os
|
||||
|
||||
@@ -0,0 +1,82 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from crawler_platform.app.core.crawler.page_type_scorer import PAGE_TYPE_METADATA
|
||||
from crawler_platform.app.core.crawler.page_type_taxonomy import (
|
||||
AnalyzeStrategy,
|
||||
LLMPolicy,
|
||||
PageClassificationResult,
|
||||
PageType,
|
||||
normalize_page_type,
|
||||
normalize_semantic_page_type,
|
||||
)
|
||||
|
||||
|
||||
DEFAULT_ANALYZE_STRATEGY = AnalyzeStrategy.ANALYZE_METADATA_ONLY.value
|
||||
DEFAULT_LLM_POLICY = LLMPolicy.NO_LLM.value
|
||||
|
||||
|
||||
def decide_analyze_strategy(result_or_page_type: object | None) -> str:
|
||||
semantic_page_type = normalize_semantic_page_type(result_or_page_type)
|
||||
if (
|
||||
isinstance(result_or_page_type, PageClassificationResult)
|
||||
and result_or_page_type.analyze_strategy
|
||||
and result_or_page_type.analyze_strategy != DEFAULT_ANALYZE_STRATEGY
|
||||
):
|
||||
return result_or_page_type.analyze_strategy
|
||||
profile = PAGE_TYPE_METADATA.get(semantic_page_type)
|
||||
if profile:
|
||||
return str(profile.get("analyze_strategy") or DEFAULT_ANALYZE_STRATEGY)
|
||||
if semantic_page_type == PageType.UNKNOWN_PAGE.value:
|
||||
return AnalyzeStrategy.ANALYZE_METADATA_ONLY.value
|
||||
return DEFAULT_ANALYZE_STRATEGY
|
||||
|
||||
|
||||
def decide_llm_policy(result_or_page_type: object | None) -> str:
|
||||
semantic_page_type = normalize_semantic_page_type(result_or_page_type)
|
||||
if (
|
||||
isinstance(result_or_page_type, PageClassificationResult)
|
||||
and result_or_page_type.llm_policy
|
||||
and result_or_page_type.llm_policy != DEFAULT_LLM_POLICY
|
||||
):
|
||||
return result_or_page_type.llm_policy
|
||||
profile = PAGE_TYPE_METADATA.get(semantic_page_type)
|
||||
if profile:
|
||||
return str(profile.get("llm_policy") or DEFAULT_LLM_POLICY)
|
||||
if semantic_page_type == PageType.UNKNOWN_PAGE.value:
|
||||
return LLMPolicy.NO_LLM.value
|
||||
return DEFAULT_LLM_POLICY
|
||||
|
||||
|
||||
def is_protected_strategy(strategy: str | AnalyzeStrategy | None) -> bool:
|
||||
return str(strategy or "") == AnalyzeStrategy.SKIP_PROTECTED.value
|
||||
|
||||
|
||||
def is_noise_strategy(strategy: str | AnalyzeStrategy | None) -> bool:
|
||||
return str(strategy or "") == AnalyzeStrategy.SKIP_NOISE.value
|
||||
|
||||
|
||||
def should_analyze_page(result_or_page_type: object | None, analyze_page_types: set[str] | None = None) -> bool:
|
||||
strategy = decide_analyze_strategy(result_or_page_type)
|
||||
if is_protected_strategy(strategy) or is_noise_strategy(strategy):
|
||||
return False
|
||||
|
||||
if isinstance(result_or_page_type, PageClassificationResult):
|
||||
return True
|
||||
|
||||
if analyze_page_types is None:
|
||||
return True
|
||||
|
||||
normalized_page_type = normalize_page_type(result_or_page_type)
|
||||
normalized_allowlist = {normalize_page_type(item) for item in analyze_page_types}
|
||||
return normalized_page_type in normalized_allowlist
|
||||
|
||||
|
||||
def apply_analysis_policy(result: PageClassificationResult) -> PageClassificationResult:
|
||||
strategy = decide_analyze_strategy(result)
|
||||
llm_policy = decide_llm_policy(result)
|
||||
result.analyze_strategy = strategy
|
||||
result.llm_policy = llm_policy
|
||||
result.is_protected = is_protected_strategy(strategy)
|
||||
result.is_noise = is_noise_strategy(strategy)
|
||||
result.should_analyze = should_analyze_page(result)
|
||||
return result
|
||||
@@ -11,6 +11,13 @@ from crawler_platform.app.core.crawler.page_type_taxonomy import (
|
||||
)
|
||||
from crawler_platform.app.core.crawler.page_signal_extractor import extract_page_signals_from_page
|
||||
from crawler_platform.app.core.crawler.page_type_scorer import score_page_type
|
||||
from crawler_platform.app.core.crawler.page_unknown_patterns import (
|
||||
build_unknown_pattern_payload,
|
||||
should_store_unknown_pattern,
|
||||
)
|
||||
from crawler_platform.app.core.crawler.page_analysis_policy import (
|
||||
should_analyze_page as should_analyze_page_by_policy,
|
||||
)
|
||||
|
||||
|
||||
PRODUCT_DETAIL_PREDICATES = {
|
||||
@@ -109,9 +116,15 @@ def classify_page_semantic(
|
||||
text: str = "",
|
||||
html: str | None = None,
|
||||
source_zones: list[dict[str, object]] | None = None,
|
||||
final_url: str | None = None,
|
||||
status_code: int | None = None,
|
||||
content_type: str | None = None,
|
||||
) -> PageClassificationResult:
|
||||
signals = extract_page_signals_from_page(
|
||||
url=url,
|
||||
final_url=final_url,
|
||||
status_code=status_code,
|
||||
content_type=content_type,
|
||||
title=title,
|
||||
text=text,
|
||||
html=html,
|
||||
@@ -135,6 +148,35 @@ def classify_page_semantic(
|
||||
return result
|
||||
|
||||
|
||||
def classification_metadata(
|
||||
result: PageClassificationResult,
|
||||
*,
|
||||
title: str | None = None,
|
||||
text: str | None = None,
|
||||
html: str | None = None,
|
||||
source_zones: list[str | dict[str, object]] | None = None,
|
||||
) -> dict[str, object]:
|
||||
"""Return metadata that preserves legacy page_type while carrying semantic evidence."""
|
||||
|
||||
payload = result.to_dict()
|
||||
if should_store_unknown_pattern(result):
|
||||
payload["unknown_pattern"] = build_unknown_pattern_payload(
|
||||
result=result,
|
||||
url=result.url,
|
||||
title=title,
|
||||
text=text,
|
||||
html=html,
|
||||
source_zones=source_zones,
|
||||
)
|
||||
return {
|
||||
"page_type": get_legacy_page_type(result),
|
||||
"semantic_page_type": result.primary_page_type,
|
||||
"analyze_strategy": result.analyze_strategy,
|
||||
"llm_policy": result.llm_policy,
|
||||
"page_classification": payload,
|
||||
}
|
||||
|
||||
|
||||
def relation_allowed_for_page_type(page_type: str | None, predicate: str) -> bool:
|
||||
if page_type in PRODUCT_DETAIL_PAGE_TYPES:
|
||||
return True
|
||||
@@ -150,9 +192,7 @@ def claim_allowed_for_context(page_type: str | None, predicate: str, zone_type:
|
||||
|
||||
|
||||
def should_analyze_page(page_type: object, analyze_page_types: set[str] | None) -> bool:
|
||||
normalized_page_type = normalize_page_type(page_type)
|
||||
normalized = {normalize_page_type(item) for item in (analyze_page_types or set())}
|
||||
return normalized_page_type in normalized
|
||||
return should_analyze_page_by_policy(page_type, analyze_page_types)
|
||||
|
||||
|
||||
def _zone_type(zone: dict[str, object]) -> str:
|
||||
|
||||
@@ -71,12 +71,27 @@ ZONE_SELECTORS: dict[str, list[str]] = {
|
||||
|
||||
ZONE_PRIORITY_BY_PAGE_TYPE = {
|
||||
"ProductPage": ["product_title", "product_summary", "product_description", "product_detail"],
|
||||
"ProductDetailPage": ["product_title", "product_summary", "product_description", "product_detail"],
|
||||
"BrandStoryPage": ["brand_story_body"],
|
||||
"AboutPage": ["brand_story_body"],
|
||||
"ContactPage": ["brand_story_body"],
|
||||
"NoticePage": ["notice_body"],
|
||||
"PublicNoticePage": ["notice_body"],
|
||||
"ArticlePage": ["notice_body"],
|
||||
"NewsArticlePage": ["notice_body"],
|
||||
"BlogPostPage": ["notice_body"],
|
||||
"FAQPage": ["notice_body"],
|
||||
"QAPage": ["notice_body"],
|
||||
"BoardPage": ["notice_body"],
|
||||
"ForumBoardPage": ["notice_body"],
|
||||
"ForumThreadPage": ["notice_body"],
|
||||
"EventPage": ["event_body"],
|
||||
"PromotionPage": ["event_body"],
|
||||
"CampaignLandingPage": ["event_body"],
|
||||
"CategoryPage": ["product_title", "product_summary"],
|
||||
"CategoryListingPage": ["product_title", "product_summary"],
|
||||
"SearchPage": ["product_title", "product_summary"],
|
||||
"SearchResultsPage": ["product_title", "product_summary"],
|
||||
}
|
||||
|
||||
STRUCTURAL_NOISE_TOKENS = {
|
||||
|
||||
@@ -0,0 +1,843 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from collections import Counter
|
||||
import json
|
||||
import re
|
||||
from typing import Any
|
||||
from urllib.parse import urljoin, urlparse
|
||||
|
||||
from crawler_platform.app.core.crawler.page_signals import PageSignals, RawPageSnapshot
|
||||
|
||||
|
||||
KEYWORD_GROUPS: dict[str, tuple[str, ...]] = {
|
||||
"commerce": (
|
||||
"product",
|
||||
"price",
|
||||
"sale",
|
||||
"cart",
|
||||
"basket",
|
||||
"buy",
|
||||
"checkout",
|
||||
"sku",
|
||||
"상품",
|
||||
"가격",
|
||||
"장바구니",
|
||||
"구매",
|
||||
"주문",
|
||||
),
|
||||
"listing": (
|
||||
"filter",
|
||||
"sort",
|
||||
"category",
|
||||
"pagination",
|
||||
"items",
|
||||
"results",
|
||||
"필터",
|
||||
"정렬",
|
||||
"카테고리",
|
||||
"상품수",
|
||||
"결과",
|
||||
),
|
||||
"editorial": (
|
||||
"article",
|
||||
"author",
|
||||
"published",
|
||||
"updated",
|
||||
"headline",
|
||||
"news",
|
||||
"blog",
|
||||
"기사",
|
||||
"작성자",
|
||||
"게시일",
|
||||
),
|
||||
"community": (
|
||||
"question",
|
||||
"answer",
|
||||
"comment",
|
||||
"reply",
|
||||
"thread",
|
||||
"vote",
|
||||
"faq",
|
||||
"q&a",
|
||||
"질문",
|
||||
"답변",
|
||||
"댓글",
|
||||
"문의",
|
||||
),
|
||||
"knowledge": (
|
||||
"documentation",
|
||||
"api",
|
||||
"endpoint",
|
||||
"parameter",
|
||||
"version",
|
||||
"reference",
|
||||
"guide",
|
||||
"문서",
|
||||
"가이드",
|
||||
"버전",
|
||||
),
|
||||
"corporate": (
|
||||
"about",
|
||||
"company",
|
||||
"contact",
|
||||
"address",
|
||||
"team",
|
||||
"careers",
|
||||
"privacy",
|
||||
"terms",
|
||||
"회사",
|
||||
"소개",
|
||||
"문의",
|
||||
"주소",
|
||||
"채용",
|
||||
"개인정보",
|
||||
"약관",
|
||||
),
|
||||
"protected": (
|
||||
"login",
|
||||
"password",
|
||||
"captcha",
|
||||
"access denied",
|
||||
"forbidden",
|
||||
"payment",
|
||||
"billing",
|
||||
"로그인",
|
||||
"비밀번호",
|
||||
"보안문자",
|
||||
"접근 제한",
|
||||
"결제",
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
PRICE_PATTERN = re.compile(
|
||||
r"(?:[$€£¥₩]\s?\d[\d,]*(?:\.\d+)?)|(?:\d[\d,]*(?:\.\d+)?\s?(?:KRW|USD|EUR|JPY|원|달러))",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
DATE_PATTERN = re.compile(r"\b(?:20\d{2}|19\d{2})[-./년]\s?\d{1,2}[-./월]\s?\d{1,2}", re.IGNORECASE)
|
||||
API_ENDPOINT_PATTERN = re.compile(r"\b(?:GET|POST|PUT|PATCH|DELETE)\s+/(?:[A-Za-z0-9_./{}:-]+)")
|
||||
VERSION_PATTERN = re.compile(r"\bv?\d+\.\d+(?:\.\d+)?\b", re.IGNORECASE)
|
||||
|
||||
|
||||
def build_raw_page_snapshot(
|
||||
*,
|
||||
url: str,
|
||||
final_url: str | None = None,
|
||||
status_code: int | None = None,
|
||||
content_type: str | None = None,
|
||||
title: str | None = None,
|
||||
text: str | None = None,
|
||||
html: str | None = None,
|
||||
rendered_html: str | None = None,
|
||||
metadata: dict[str, Any] | None = None,
|
||||
source_zones: list[str | dict[str, Any]] | None = None,
|
||||
collector_payload: dict[str, Any] | None = None,
|
||||
) -> RawPageSnapshot:
|
||||
snapshot = RawPageSnapshot(
|
||||
url=url,
|
||||
final_url=final_url,
|
||||
status_code=status_code,
|
||||
content_type=content_type,
|
||||
title=title,
|
||||
text=text,
|
||||
html=html,
|
||||
rendered_html=rendered_html,
|
||||
metadata=dict(metadata or {}),
|
||||
source_zones=list(source_zones or []),
|
||||
collector_payload=dict(collector_payload or {}),
|
||||
)
|
||||
return merge_collector_payload(snapshot)
|
||||
|
||||
|
||||
def snapshot_from_collector_payload(payload: dict[str, Any], *, url: str | None = None) -> RawPageSnapshot:
|
||||
return merge_collector_payload(
|
||||
RawPageSnapshot(
|
||||
url=str(url or payload.get("url") or payload.get("source_url") or ""),
|
||||
final_url=payload.get("final_url") or payload.get("resolved_url"),
|
||||
status_code=_optional_int(payload.get("status_code")),
|
||||
content_type=payload.get("content_type"),
|
||||
title=payload.get("title"),
|
||||
text=payload.get("text") or payload.get("markdown") or payload.get("clean_text"),
|
||||
html=payload.get("html") or payload.get("raw_html"),
|
||||
rendered_html=payload.get("rendered_html"),
|
||||
metadata=dict(payload.get("metadata") or {}),
|
||||
collector_payload=dict(payload),
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def merge_collector_payload(snapshot: RawPageSnapshot) -> RawPageSnapshot:
|
||||
payload = snapshot.collector_payload
|
||||
if not payload:
|
||||
return snapshot
|
||||
snapshot.open_graph.update(payload.get("open_graph") or payload.get("og") or {})
|
||||
snapshot.twitter_card.update(payload.get("twitter_card") or payload.get("twitter") or {})
|
||||
snapshot.json_ld.extend(_ensure_dict_list(payload.get("json_ld") or payload.get("jsonld")))
|
||||
snapshot.microdata.extend(_ensure_dict_list(payload.get("microdata")))
|
||||
snapshot.rdfa.extend(_ensure_dict_list(payload.get("rdfa")))
|
||||
snapshot.headings.extend(_string_list(payload.get("headings")))
|
||||
snapshot.links.extend(_dict_list(payload.get("links")))
|
||||
snapshot.images.extend(_dict_list(payload.get("images")))
|
||||
snapshot.forms.extend(_dict_list(payload.get("forms")))
|
||||
snapshot.buttons.extend(_string_list(payload.get("buttons")))
|
||||
snapshot.inputs.extend(_dict_list(payload.get("inputs")))
|
||||
snapshot.tables.extend(_dict_list(payload.get("tables")))
|
||||
snapshot.breadcrumbs.extend(_string_list(payload.get("breadcrumbs")))
|
||||
if payload.get("screenshot_path") and not snapshot.screenshot_path:
|
||||
snapshot.screenshot_path = str(payload["screenshot_path"])
|
||||
return snapshot
|
||||
|
||||
|
||||
def extract_page_signals_from_page(
|
||||
*,
|
||||
url: str,
|
||||
final_url: str | None = None,
|
||||
status_code: int | None = None,
|
||||
content_type: str | None = None,
|
||||
title: str | None = None,
|
||||
text: str | None = None,
|
||||
html: str | None = None,
|
||||
rendered_html: str | None = None,
|
||||
metadata: dict[str, Any] | None = None,
|
||||
source_zones: list[str | dict[str, Any]] | None = None,
|
||||
collector_payload: dict[str, Any] | None = None,
|
||||
) -> PageSignals:
|
||||
return extract_page_signals(
|
||||
build_raw_page_snapshot(
|
||||
url=url,
|
||||
final_url=final_url,
|
||||
status_code=status_code,
|
||||
content_type=content_type,
|
||||
title=title,
|
||||
text=text,
|
||||
html=html,
|
||||
rendered_html=rendered_html,
|
||||
metadata=metadata,
|
||||
source_zones=source_zones,
|
||||
collector_payload=collector_payload,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def extract_page_signals(snapshot: RawPageSnapshot) -> PageSignals:
|
||||
html = snapshot.rendered_html or snapshot.html or ""
|
||||
soup = _soup_from_html(html)
|
||||
title = snapshot.title or _title_from_soup(soup)
|
||||
text = snapshot.text or _text_from_soup(soup) or _text_from_html(html)
|
||||
combined = "\n".join([snapshot.url, title or "", text or "", html[:12000]]).lower()
|
||||
open_graph = {**_extract_meta_prefix(soup, "property", "og:"), **snapshot.open_graph}
|
||||
twitter_card = {**_extract_meta_prefix(soup, "name", "twitter:"), **snapshot.twitter_card}
|
||||
json_ld = [*_extract_json_ld(soup), *snapshot.json_ld]
|
||||
links = snapshot.links or _extract_links(soup, snapshot.final_url or snapshot.url)
|
||||
images = snapshot.images or _extract_images(soup, snapshot.final_url or snapshot.url)
|
||||
forms = snapshot.forms or _extract_forms(soup)
|
||||
inputs = snapshot.inputs or _extract_inputs(soup)
|
||||
buttons = snapshot.buttons or _extract_buttons(soup)
|
||||
headings = snapshot.headings or _extract_headings(soup)
|
||||
tables = snapshot.tables or _extract_tables(soup)
|
||||
breadcrumbs = snapshot.breadcrumbs or _extract_breadcrumbs(soup)
|
||||
schema_types = _schema_types(json_ld, snapshot.microdata, snapshot.rdfa, soup)
|
||||
keyword_hits = _keyword_hits(combined)
|
||||
link_counts = _link_counts(links, snapshot.final_url or snapshot.url)
|
||||
layout = _layout_signals(soup, combined, links, images, tables)
|
||||
repeated_card_count = max(
|
||||
layout["card_count"],
|
||||
link_counts["product_link_count"],
|
||||
_count_selector_matches(soup, CARD_SELECTORS),
|
||||
)
|
||||
button_text = " ".join(buttons).lower()
|
||||
input_text = " ".join(_input_blob(item) for item in inputs).lower()
|
||||
form_text = " ".join(_form_blob(item) for item in forms).lower()
|
||||
table_text = " ".join(str(table.get("text") or "") for table in tables).lower()
|
||||
content_type = str(snapshot.content_type or snapshot.metadata.get("content_type") or "").lower()
|
||||
path = urlparse(snapshot.final_url or snapshot.url).path.lower()
|
||||
|
||||
signals = PageSignals(
|
||||
schema_types=schema_types,
|
||||
og_type=_string_or_none(open_graph.get("type") or open_graph.get("og:type")),
|
||||
twitter_card_type=_string_or_none(twitter_card.get("card") or twitter_card.get("twitter:card")),
|
||||
has_price=bool(PRICE_PATTERN.search(combined)),
|
||||
has_currency=bool(re.search(r"[$€£¥₩]|(?:\b(?:KRW|USD|EUR|JPY)\b)|원", combined, re.IGNORECASE)),
|
||||
has_cart_button=_contains_any(button_text + " " + combined, ("cart", "basket", "장바구니", "bag")),
|
||||
has_buy_button=_contains_any(button_text + " " + combined, ("buy now", "purchase", "구매", "주문", "결제")),
|
||||
has_variant_selector=_has_variant_selector(soup, input_text + " " + combined),
|
||||
has_sku=bool(re.search(r"\bsku\b|상품\s*코드|product\s*code", combined, re.IGNORECASE)),
|
||||
has_rating=("AggregateRating" in schema_types)
|
||||
or _contains_any(combined, ("rating", "stars", "별점", "평점")),
|
||||
has_review_section=("Review" in schema_types) or _contains_any(combined, ("review", "reviews", "후기", "리뷰")),
|
||||
has_product_gallery=(len(images) >= 3 and _contains_any(combined, ("gallery", "product", "상품"))),
|
||||
has_repeated_cards=repeated_card_count >= 3,
|
||||
repeated_card_count=repeated_card_count,
|
||||
has_filter_panel=layout["has_filter_sidebar"]
|
||||
or _contains_any(combined, ("filter", "facets", "refine", "필터", "조건")),
|
||||
has_sort_control=_contains_any(combined, ("sort", "order by", "low price", "high price", "정렬", "낮은가격", "높은가격")),
|
||||
has_pagination=_has_pagination(soup, links, combined),
|
||||
has_author=_contains_any(combined, ("author", "byline", "작성자", "기자")),
|
||||
has_published_date=("datePublished" in _json_keys(json_ld))
|
||||
or bool(DATE_PATTERN.search(combined))
|
||||
and _contains_any(combined, ("published", "posted", "게시", "등록")),
|
||||
has_modified_date=("dateModified" in _json_keys(json_ld))
|
||||
or _contains_any(combined, ("modified", "updated", "수정")),
|
||||
has_article_body=("Article" in schema_types)
|
||||
or ("NewsArticle" in schema_types)
|
||||
or _count_selector_matches(soup, ("article", "[itemprop='articleBody']", ".article-body", ".post-content")) > 0
|
||||
or _contains_any(combined, ("articlebody", "article body")),
|
||||
has_tags=_has_tags(soup, links, combined),
|
||||
has_question=("QAPage" in schema_types) or _contains_any(combined, ("question", "q:", "질문", "문의")),
|
||||
has_answer=("Answer" in schema_types) or _contains_any(combined, ("answer", "a:", "답변")),
|
||||
has_comments=_contains_any(combined, ("comment", "comments", "reply", "댓글", "답글")),
|
||||
has_votes=_contains_any(combined, ("vote", "votes", "upvote", "downvote", "추천", "투표")),
|
||||
has_thread_structure=_contains_any(combined, ("thread", "discussion", "게시글", "토론")),
|
||||
has_faq_structure=("FAQPage" in schema_types) or _contains_any(combined, ("faq", "frequently asked", "자주 묻는")),
|
||||
has_code_blocks=_count_selector_matches(soup, ("pre", "code", ".highlight", ".code")) > 0,
|
||||
has_toc=_count_selector_matches(soup, ("#toc", ".toc", "[class*='table-of-contents']", "nav[aria-label*='contents']")) > 0,
|
||||
has_api_endpoint=bool(API_ENDPOINT_PATTERN.search(f"{text or ''}\n{html or ''}")),
|
||||
has_parameter_table=_has_parameter_table(tables, table_text),
|
||||
has_version_info=bool(VERSION_PATTERN.search(combined)) and _contains_any(combined, ("version", "버전", "release")),
|
||||
has_contact_info=_contains_any(combined, ("contact", "email", "tel:", "문의", "연락처")),
|
||||
has_address=_contains_any(combined, ("address", "주소", "road", "street")),
|
||||
has_policy_terms=_contains_any(combined, ("terms", "policy", "agreement", "약관", "정책")),
|
||||
has_privacy_terms=_contains_any(combined, ("privacy", "personal information", "개인정보")),
|
||||
has_career_terms=_contains_any(combined, ("career", "jobs", "recruit", "채용", "지원")),
|
||||
has_login_form=_contains_any(form_text + " " + combined, ("login", "sign in", "로그인")) and (
|
||||
"password" in input_text or "비밀번호" in combined
|
||||
),
|
||||
has_password_field="password" in input_text,
|
||||
has_payment_fields=_contains_any(input_text + " " + combined, ("card number", "payment", "billing", "결제", "카드")),
|
||||
has_captcha=_contains_any(combined, ("captcha", "recaptcha", "hcaptcha", "보안문자")),
|
||||
has_access_denied=_contains_any(combined, ("access denied", "forbidden", "permission denied", "접근 제한", "권한이 없습니다")),
|
||||
status_code=snapshot.status_code,
|
||||
has_error_status=bool(snapshot.status_code and snapshot.status_code >= 400)
|
||||
or _contains_any(combined, ("404", "not found", "error page")),
|
||||
has_not_found=snapshot.status_code == 404 or _contains_any(combined, ("404", "not found", "page not found")),
|
||||
has_sitemap_resource=("sitemap" in path) or ("sitemap" in content_type and "xml" in content_type),
|
||||
has_feed_resource=("rss" in content_type) or ("atom" in content_type) or path.endswith((".rss", ".atom")),
|
||||
has_json_resource=("json" in content_type) or path.endswith(".json"),
|
||||
has_xml_resource=("xml" in content_type) or path.endswith(".xml"),
|
||||
has_file_resource=path.endswith((".pdf", ".csv", ".xlsx", ".xls", ".doc", ".docx", ".zip")),
|
||||
internal_link_count=link_counts["internal_link_count"],
|
||||
external_link_count=link_counts["external_link_count"],
|
||||
product_link_count=link_counts["product_link_count"],
|
||||
category_link_count=link_counts["category_link_count"],
|
||||
profile_link_count=link_counts["profile_link_count"],
|
||||
article_link_count=link_counts["article_link_count"],
|
||||
layout_blocks=layout["layout_blocks"],
|
||||
has_hero_block=layout["has_hero_block"],
|
||||
has_card_grid=layout["has_card_grid"],
|
||||
has_filter_sidebar=layout["has_filter_sidebar"],
|
||||
has_sticky_action_box=layout["has_sticky_action_box"],
|
||||
has_media_player_area=layout["has_media_player_area"],
|
||||
has_map_area=layout["has_map_area"],
|
||||
has_calendar_grid=layout["has_calendar_grid"],
|
||||
has_pricing_table=layout["has_pricing_table"],
|
||||
has_comparison_table=layout["has_comparison_table"],
|
||||
dominant_language=_dominant_language(combined),
|
||||
keyword_hits=keyword_hits,
|
||||
url_hints=_url_hints(snapshot.final_url or snapshot.url),
|
||||
title=title,
|
||||
text_sample=(text or "")[:500],
|
||||
external_collector_signals=_external_collector_signals(snapshot),
|
||||
)
|
||||
return signals
|
||||
|
||||
|
||||
def _soup_from_html(html: str):
|
||||
if not html:
|
||||
return None
|
||||
try:
|
||||
from bs4 import BeautifulSoup
|
||||
except ImportError:
|
||||
return None
|
||||
try:
|
||||
return BeautifulSoup(html, "html.parser")
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def _title_from_soup(soup) -> str | None:
|
||||
if soup is None or not soup.title:
|
||||
return None
|
||||
return soup.title.get_text(" ", strip=True) or None
|
||||
|
||||
|
||||
def _text_from_soup(soup) -> str:
|
||||
if soup is None:
|
||||
return ""
|
||||
return soup.get_text("\n", strip=True)
|
||||
|
||||
|
||||
def _text_from_html(html: str) -> str:
|
||||
return re.sub(r"\s+", " ", re.sub(r"<[^>]+>", " ", html or "")).strip()
|
||||
|
||||
|
||||
def _extract_meta_prefix(soup, attr_name: str, prefix: str) -> dict[str, str]:
|
||||
if soup is None:
|
||||
return {}
|
||||
result: dict[str, str] = {}
|
||||
for tag in soup.find_all("meta"):
|
||||
name = str(tag.get(attr_name) or "").strip()
|
||||
if not name.lower().startswith(prefix):
|
||||
continue
|
||||
content = str(tag.get("content") or "").strip()
|
||||
if content:
|
||||
result[name.removeprefix(prefix)] = content
|
||||
result[name] = content
|
||||
return result
|
||||
|
||||
|
||||
def _extract_json_ld(soup) -> list[dict[str, Any]]:
|
||||
if soup is None:
|
||||
return []
|
||||
payloads: list[dict[str, Any]] = []
|
||||
for tag in soup.find_all("script"):
|
||||
script_type = str(tag.get("type") or "").lower()
|
||||
if "ld+json" not in script_type:
|
||||
continue
|
||||
raw = tag.string or tag.get_text(" ", strip=True)
|
||||
try:
|
||||
value = json.loads(raw)
|
||||
except Exception:
|
||||
continue
|
||||
payloads.extend(_ensure_dict_list(value))
|
||||
return payloads
|
||||
|
||||
|
||||
def _extract_headings(soup) -> list[str]:
|
||||
if soup is None:
|
||||
return []
|
||||
return [node.get_text(" ", strip=True) for node in soup.select("h1,h2,h3") if node.get_text(" ", strip=True)]
|
||||
|
||||
|
||||
def _extract_links(soup, base_url: str) -> list[dict[str, Any]]:
|
||||
if soup is None:
|
||||
return []
|
||||
links = []
|
||||
for tag in soup.find_all("a"):
|
||||
href = str(tag.get("href") or "").strip()
|
||||
if not href:
|
||||
continue
|
||||
links.append(
|
||||
{
|
||||
"href": urljoin(base_url, href),
|
||||
"text": tag.get_text(" ", strip=True),
|
||||
"rel": " ".join(str(item) for item in tag.get("rel", [])),
|
||||
"class": " ".join(str(item) for item in tag.get("class", [])),
|
||||
}
|
||||
)
|
||||
return links
|
||||
|
||||
|
||||
def _extract_images(soup, base_url: str) -> list[dict[str, Any]]:
|
||||
if soup is None:
|
||||
return []
|
||||
images = []
|
||||
for tag in soup.find_all("img"):
|
||||
src = str(tag.get("src") or tag.get("data-src") or "").strip()
|
||||
if not src:
|
||||
continue
|
||||
images.append(
|
||||
{
|
||||
"src": urljoin(base_url, src),
|
||||
"alt": str(tag.get("alt") or ""),
|
||||
"class": " ".join(str(item) for item in tag.get("class", [])),
|
||||
}
|
||||
)
|
||||
return images
|
||||
|
||||
|
||||
def _extract_forms(soup) -> list[dict[str, Any]]:
|
||||
if soup is None:
|
||||
return []
|
||||
forms = []
|
||||
for form in soup.find_all("form"):
|
||||
forms.append(
|
||||
{
|
||||
"action": str(form.get("action") or ""),
|
||||
"method": str(form.get("method") or ""),
|
||||
"id": str(form.get("id") or ""),
|
||||
"class": " ".join(str(item) for item in form.get("class", [])),
|
||||
"text": form.get_text(" ", strip=True)[:500],
|
||||
}
|
||||
)
|
||||
return forms
|
||||
|
||||
|
||||
def _extract_inputs(soup) -> list[dict[str, Any]]:
|
||||
if soup is None:
|
||||
return []
|
||||
inputs = []
|
||||
for tag in soup.select("input,select,textarea"):
|
||||
inputs.append(
|
||||
{
|
||||
"type": str(tag.get("type") or tag.name or ""),
|
||||
"name": str(tag.get("name") or ""),
|
||||
"id": str(tag.get("id") or ""),
|
||||
"placeholder": str(tag.get("placeholder") or ""),
|
||||
"autocomplete": str(tag.get("autocomplete") or ""),
|
||||
"aria_label": str(tag.get("aria-label") or ""),
|
||||
"class": " ".join(str(item) for item in tag.get("class", [])),
|
||||
"text": tag.get_text(" ", strip=True)[:240],
|
||||
}
|
||||
)
|
||||
return inputs
|
||||
|
||||
|
||||
def _extract_buttons(soup) -> list[str]:
|
||||
if soup is None:
|
||||
return []
|
||||
values = []
|
||||
for tag in soup.select("button,input[type='submit'],input[type='button'],[role='button']"):
|
||||
text = tag.get_text(" ", strip=True) or str(tag.get("value") or tag.get("aria-label") or "")
|
||||
if text.strip():
|
||||
values.append(text.strip())
|
||||
return values
|
||||
|
||||
|
||||
def _extract_tables(soup) -> list[dict[str, Any]]:
|
||||
if soup is None:
|
||||
return []
|
||||
tables = []
|
||||
for table in soup.find_all("table"):
|
||||
headers = [cell.get_text(" ", strip=True) for cell in table.select("th") if cell.get_text(" ", strip=True)]
|
||||
text = table.get_text(" ", strip=True)
|
||||
tables.append({"headers": headers, "text": text[:1000], "row_count": len(table.select("tr"))})
|
||||
return tables
|
||||
|
||||
|
||||
def _extract_breadcrumbs(soup) -> list[str]:
|
||||
if soup is None:
|
||||
return []
|
||||
crumbs = []
|
||||
selectors = [
|
||||
"[class*='breadcrumb']",
|
||||
"[id*='breadcrumb']",
|
||||
"nav[aria-label*='breadcrumb' i]",
|
||||
"[itemtype*='BreadcrumbList']",
|
||||
]
|
||||
for selector in selectors:
|
||||
for node in soup.select(selector):
|
||||
text = node.get_text(" > ", strip=True)
|
||||
if text:
|
||||
crumbs.append(text)
|
||||
return _dedupe_strings(crumbs)
|
||||
|
||||
|
||||
def _schema_types(
|
||||
json_ld: list[dict[str, Any]],
|
||||
microdata: list[dict[str, Any]],
|
||||
rdfa: list[dict[str, Any]],
|
||||
soup,
|
||||
) -> set[str]:
|
||||
types: set[str] = set()
|
||||
for payload in [*json_ld, *microdata, *rdfa]:
|
||||
_visit_schema_types(payload, types)
|
||||
if soup is not None:
|
||||
for node in soup.select("[itemscope][itemtype]"):
|
||||
raw = str(node.get("itemtype") or "")
|
||||
if raw:
|
||||
types.add(raw.rstrip("/").split("/")[-1])
|
||||
for node in soup.select("[typeof]"):
|
||||
for item in str(node.get("typeof") or "").split():
|
||||
types.add(item.split(":")[-1])
|
||||
return {item for item in types if item}
|
||||
|
||||
|
||||
def _visit_schema_types(value: Any, types: set[str]) -> None:
|
||||
if isinstance(value, list):
|
||||
for item in value:
|
||||
_visit_schema_types(item, types)
|
||||
return
|
||||
if not isinstance(value, dict):
|
||||
return
|
||||
raw_type = value.get("@type") or value.get("type")
|
||||
for item in _ensure_list(raw_type):
|
||||
if isinstance(item, str):
|
||||
types.add(item.rstrip("/").split("/")[-1])
|
||||
for nested in value.values():
|
||||
if isinstance(nested, (dict, list)):
|
||||
_visit_schema_types(nested, types)
|
||||
|
||||
|
||||
def _json_keys(json_ld: list[dict[str, Any]]) -> set[str]:
|
||||
keys: set[str] = set()
|
||||
|
||||
def visit(value: Any) -> None:
|
||||
if isinstance(value, list):
|
||||
for item in value:
|
||||
visit(item)
|
||||
return
|
||||
if not isinstance(value, dict):
|
||||
return
|
||||
keys.update(str(key) for key in value)
|
||||
for nested in value.values():
|
||||
visit(nested)
|
||||
|
||||
visit(json_ld)
|
||||
return keys
|
||||
|
||||
|
||||
def _keyword_hits(text: str) -> dict[str, int]:
|
||||
hits = {}
|
||||
for group, terms in KEYWORD_GROUPS.items():
|
||||
count = sum(text.count(term.lower()) for term in terms)
|
||||
if count:
|
||||
hits[group] = count
|
||||
return hits
|
||||
|
||||
|
||||
def _link_counts(links: list[dict[str, Any]], base_url: str) -> dict[str, int]:
|
||||
base = urlparse(base_url)
|
||||
base_host = base.netloc.lower()
|
||||
base_path = base.path.lower()
|
||||
counts = Counter()
|
||||
for link in links:
|
||||
href = str(link.get("href") or "")
|
||||
parsed = urlparse(href)
|
||||
host = parsed.netloc.lower()
|
||||
path = parsed.path.lower()
|
||||
if not host or host == base_host:
|
||||
counts["internal_link_count"] += 1
|
||||
else:
|
||||
counts["external_link_count"] += 1
|
||||
same_document_query_link = path == base_path and bool(parsed.query)
|
||||
if not same_document_query_link and any(token in path for token in ("/product", "/products", "/goods", "/item", "/p/")):
|
||||
counts["product_link_count"] += 1
|
||||
if any(token in path for token in ("category", "collection", "/shop", "/list", "catalog")):
|
||||
counts["category_link_count"] += 1
|
||||
if any(token in path for token in ("profile", "user", "author", "member", "creator")):
|
||||
counts["profile_link_count"] += 1
|
||||
if any(token in path for token in ("article", "blog", "news", "post", "story")):
|
||||
counts["article_link_count"] += 1
|
||||
return {
|
||||
"internal_link_count": counts["internal_link_count"],
|
||||
"external_link_count": counts["external_link_count"],
|
||||
"product_link_count": counts["product_link_count"],
|
||||
"category_link_count": counts["category_link_count"],
|
||||
"profile_link_count": counts["profile_link_count"],
|
||||
"article_link_count": counts["article_link_count"],
|
||||
}
|
||||
|
||||
|
||||
CARD_SELECTORS = (
|
||||
".product-card",
|
||||
".card",
|
||||
".item",
|
||||
".product",
|
||||
".prdList > li",
|
||||
"[class*='product-card']",
|
||||
"[class*='grid-item']",
|
||||
)
|
||||
|
||||
|
||||
def _layout_signals(soup, text: str, links: list[dict[str, Any]], images: list[dict[str, Any]], tables: list[dict[str, Any]]) -> dict[str, Any]:
|
||||
blocks: list[str] = []
|
||||
hero = _count_selector_matches(soup, (".hero", ".visual", ".main-visual", "[class*='hero']", "section[aria-label*='hero']")) > 0
|
||||
card_count = _count_selector_matches(soup, CARD_SELECTORS)
|
||||
card_grid = card_count >= 3 or _count_selector_matches(soup, (".grid", "[class*='grid']", "[class*='cards']")) > 0 and len(links) >= 3
|
||||
filter_sidebar = _count_selector_matches(soup, (".filter", ".filters", ".facet", "aside", "[class*='filter']", "[class*='facet']")) > 0
|
||||
sticky_action = _count_selector_matches(soup, (".sticky", "[class*='sticky']", "[class*='fixed']", "[class*='buy-box']")) > 0
|
||||
media_player = _count_selector_matches(soup, ("video", "audio", "iframe[src*='youtube']", "[class*='player']")) > 0
|
||||
map_area = _count_selector_matches(soup, ("[class*='map']", "#map", "iframe[src*='maps']")) > 0 or "google map" in text
|
||||
calendar_grid = _count_selector_matches(soup, ("[class*='calendar']", "[class*='datepicker']", "table.calendar")) > 0
|
||||
pricing_table = "pricing" in text and (bool(tables) or _count_selector_matches(soup, ("[class*='pricing']", ".price-table")) > 0)
|
||||
comparison_table = _has_comparison_table(tables, text)
|
||||
for label, present in [
|
||||
("hero_block", hero),
|
||||
("card_grid", card_grid),
|
||||
("filter_sidebar", filter_sidebar),
|
||||
("sticky_action_box", sticky_action),
|
||||
("media_player_area", media_player),
|
||||
("map_area", map_area),
|
||||
("calendar_grid", calendar_grid),
|
||||
("pricing_table", pricing_table),
|
||||
("comparison_table", comparison_table),
|
||||
]:
|
||||
if present:
|
||||
blocks.append(label)
|
||||
return {
|
||||
"layout_blocks": blocks,
|
||||
"card_count": card_count,
|
||||
"has_hero_block": hero,
|
||||
"has_card_grid": card_grid,
|
||||
"has_filter_sidebar": filter_sidebar,
|
||||
"has_sticky_action_box": sticky_action,
|
||||
"has_media_player_area": media_player,
|
||||
"has_map_area": map_area,
|
||||
"has_calendar_grid": calendar_grid,
|
||||
"has_pricing_table": pricing_table,
|
||||
"has_comparison_table": comparison_table,
|
||||
}
|
||||
|
||||
|
||||
def _has_variant_selector(soup, text: str) -> bool:
|
||||
if _contains_any(text, ("variant", "option", "size", "color", "옵션", "사이즈", "색상")):
|
||||
return True
|
||||
if soup is None:
|
||||
return False
|
||||
for select in soup.find_all("select"):
|
||||
blob = " ".join(
|
||||
[
|
||||
str(select.get("name") or ""),
|
||||
str(select.get("id") or ""),
|
||||
select.get_text(" ", strip=True),
|
||||
]
|
||||
).lower()
|
||||
if _contains_any(blob, ("variant", "option", "size", "color", "옵션", "사이즈", "색상")):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _has_pagination(soup, links: list[dict[str, Any]], text: str) -> bool:
|
||||
if _count_selector_matches(soup, (".pagination", ".paging", "[class*='paginate']", "nav[aria-label*='pagination']")) > 0:
|
||||
return True
|
||||
if any(str(link.get("rel") or "").lower() in {"next", "prev", "previous"} for link in links):
|
||||
return True
|
||||
return _contains_any(text, ("next page", "previous page", "페이지", "다음", "이전"))
|
||||
|
||||
|
||||
def _has_tags(soup, links: list[dict[str, Any]], text: str) -> bool:
|
||||
if _count_selector_matches(soup, (".tag", ".tags", "[rel='tag']", "[class*='tag']")) > 0:
|
||||
return True
|
||||
return any(str(link.get("rel") or "").lower() == "tag" for link in links) or _contains_any(text, ("tags:", "태그"))
|
||||
|
||||
|
||||
def _has_parameter_table(tables: list[dict[str, Any]], table_text: str) -> bool:
|
||||
if _contains_any(table_text, ("parameter", "required", "type", "description", "파라미터", "필수")):
|
||||
return True
|
||||
for table in tables:
|
||||
headers = " ".join(str(item) for item in table.get("headers") or []).lower()
|
||||
if _contains_any(headers, ("parameter", "required", "type", "description", "파라미터", "필수")):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _has_comparison_table(tables: list[dict[str, Any]], text: str) -> bool:
|
||||
if not tables:
|
||||
return False
|
||||
return _contains_any(text, ("compare", "comparison", "vs", "비교")) or any(
|
||||
int(table.get("row_count") or 0) >= 3 and len(table.get("headers") or []) >= 3 for table in tables
|
||||
)
|
||||
|
||||
|
||||
def _url_hints(url: str) -> set[str]:
|
||||
parsed = urlparse(url)
|
||||
value = f"{parsed.path} {parsed.query}".lower()
|
||||
hints = set()
|
||||
for hint, terms in {
|
||||
"product": ("/product", "/products", "/goods", "/item", "/p/"),
|
||||
"category": ("category", "collection", "/shop", "/list", "catalog"),
|
||||
"search": ("search", "find", "keyword=", "query=", "q="),
|
||||
"article": ("article", "blog", "news", "post", "story"),
|
||||
"board": ("board", "forum", "thread", "qna"),
|
||||
"protected": ("login", "checkout", "payment", "account", "cart"),
|
||||
"system": ("sitemap", "robots.txt", ".json", ".xml", ".rss"),
|
||||
}.items():
|
||||
if any(term in value for term in terms):
|
||||
hints.add(hint)
|
||||
return hints
|
||||
|
||||
|
||||
def _dominant_language(text: str) -> str | None:
|
||||
if not text:
|
||||
return None
|
||||
korean = len(re.findall(r"[가-힣]", text))
|
||||
latin = len(re.findall(r"[A-Za-z]", text))
|
||||
if korean == 0 and latin == 0:
|
||||
return None
|
||||
if korean > latin * 0.25:
|
||||
return "ko"
|
||||
return "en"
|
||||
|
||||
|
||||
def _external_collector_signals(snapshot: RawPageSnapshot) -> dict[str, Any]:
|
||||
payload = dict(snapshot.collector_payload or {})
|
||||
for key in {
|
||||
"html",
|
||||
"raw_html",
|
||||
"rendered_html",
|
||||
"text",
|
||||
"markdown",
|
||||
"clean_text",
|
||||
"metadata",
|
||||
"links",
|
||||
"images",
|
||||
"forms",
|
||||
"inputs",
|
||||
"buttons",
|
||||
"tables",
|
||||
"json_ld",
|
||||
"jsonld",
|
||||
}:
|
||||
payload.pop(key, None)
|
||||
return payload
|
||||
|
||||
|
||||
def _count_selector_matches(soup, selectors: tuple[str, ...]) -> int:
|
||||
if soup is None:
|
||||
return 0
|
||||
count = 0
|
||||
for selector in selectors:
|
||||
try:
|
||||
count += len(soup.select(selector))
|
||||
except Exception:
|
||||
continue
|
||||
return count
|
||||
|
||||
|
||||
def _input_blob(item: dict[str, Any]) -> str:
|
||||
return " ".join(str(item.get(key) or "") for key in ("type", "name", "id", "placeholder", "autocomplete", "aria_label", "class", "text"))
|
||||
|
||||
|
||||
def _form_blob(item: dict[str, Any]) -> str:
|
||||
return " ".join(str(item.get(key) or "") for key in ("action", "method", "id", "class", "text"))
|
||||
|
||||
|
||||
def _contains_any(value: str, terms: tuple[str, ...]) -> bool:
|
||||
lowered = value.lower()
|
||||
return any(term.lower() in lowered for term in terms)
|
||||
|
||||
|
||||
def _string_or_none(value: Any) -> str | None:
|
||||
clean = str(value or "").strip()
|
||||
return clean or None
|
||||
|
||||
|
||||
def _optional_int(value: Any) -> int | None:
|
||||
try:
|
||||
return int(value)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def _ensure_list(value: Any) -> list[Any]:
|
||||
if value is None:
|
||||
return []
|
||||
if isinstance(value, list):
|
||||
return value
|
||||
return [value]
|
||||
|
||||
|
||||
def _ensure_dict_list(value: Any) -> list[dict[str, Any]]:
|
||||
values = _ensure_list(value)
|
||||
return [item for item in values if isinstance(item, dict)]
|
||||
|
||||
|
||||
def _dict_list(value: Any) -> list[dict[str, Any]]:
|
||||
return [dict(item) for item in _ensure_list(value) if isinstance(item, dict)]
|
||||
|
||||
|
||||
def _string_list(value: Any) -> list[str]:
|
||||
return [str(item) for item in _ensure_list(value) if str(item or "").strip()]
|
||||
|
||||
|
||||
def _dedupe_strings(values: list[str]) -> list[str]:
|
||||
seen = set()
|
||||
result = []
|
||||
for value in values:
|
||||
clean = " ".join(value.split())
|
||||
key = clean.lower()
|
||||
if not clean or key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
result.append(clean)
|
||||
return result
|
||||
@@ -0,0 +1,144 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import asdict, dataclass, field
|
||||
from typing import Any
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class RawPageSnapshot:
|
||||
url: str
|
||||
final_url: str | None = None
|
||||
status_code: int | None = None
|
||||
content_type: str | None = None
|
||||
title: str | None = None
|
||||
text: str | None = None
|
||||
html: str | None = None
|
||||
rendered_html: str | None = None
|
||||
metadata: dict[str, Any] = field(default_factory=dict)
|
||||
open_graph: dict[str, Any] = field(default_factory=dict)
|
||||
twitter_card: dict[str, Any] = field(default_factory=dict)
|
||||
json_ld: list[dict[str, Any]] = field(default_factory=list)
|
||||
microdata: list[dict[str, Any]] = field(default_factory=list)
|
||||
rdfa: list[dict[str, Any]] = field(default_factory=list)
|
||||
headings: list[str] = field(default_factory=list)
|
||||
links: list[dict[str, Any]] = field(default_factory=list)
|
||||
images: list[dict[str, Any]] = field(default_factory=list)
|
||||
forms: list[dict[str, Any]] = field(default_factory=list)
|
||||
buttons: list[str] = field(default_factory=list)
|
||||
inputs: list[dict[str, Any]] = field(default_factory=list)
|
||||
tables: list[dict[str, Any]] = field(default_factory=list)
|
||||
breadcrumbs: list[str] = field(default_factory=list)
|
||||
source_zones: list[str | dict[str, Any]] = field(default_factory=list)
|
||||
screenshot_path: str | None = None
|
||||
collector_payload: dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
return asdict(self)
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class PageSignals:
|
||||
# structured data
|
||||
schema_types: set[str] = field(default_factory=set)
|
||||
og_type: str | None = None
|
||||
twitter_card_type: str | None = None
|
||||
|
||||
# commerce
|
||||
has_price: bool = False
|
||||
has_currency: bool = False
|
||||
has_cart_button: bool = False
|
||||
has_buy_button: bool = False
|
||||
has_variant_selector: bool = False
|
||||
has_sku: bool = False
|
||||
has_rating: bool = False
|
||||
has_review_section: bool = False
|
||||
has_product_gallery: bool = False
|
||||
|
||||
# listing
|
||||
has_repeated_cards: bool = False
|
||||
repeated_card_count: int = 0
|
||||
has_filter_panel: bool = False
|
||||
has_sort_control: bool = False
|
||||
has_pagination: bool = False
|
||||
|
||||
# editorial
|
||||
has_author: bool = False
|
||||
has_published_date: bool = False
|
||||
has_modified_date: bool = False
|
||||
has_article_body: bool = False
|
||||
has_tags: bool = False
|
||||
|
||||
# community
|
||||
has_question: bool = False
|
||||
has_answer: bool = False
|
||||
has_comments: bool = False
|
||||
has_votes: bool = False
|
||||
has_thread_structure: bool = False
|
||||
has_faq_structure: bool = False
|
||||
|
||||
# knowledge/docs
|
||||
has_code_blocks: bool = False
|
||||
has_toc: bool = False
|
||||
has_api_endpoint: bool = False
|
||||
has_parameter_table: bool = False
|
||||
has_version_info: bool = False
|
||||
|
||||
# corporate/legal
|
||||
has_contact_info: bool = False
|
||||
has_address: bool = False
|
||||
has_policy_terms: bool = False
|
||||
has_privacy_terms: bool = False
|
||||
has_career_terms: bool = False
|
||||
|
||||
# transaction/protected
|
||||
has_login_form: bool = False
|
||||
has_password_field: bool = False
|
||||
has_payment_fields: bool = False
|
||||
has_captcha: bool = False
|
||||
has_access_denied: bool = False
|
||||
|
||||
# system/resource
|
||||
status_code: int | None = None
|
||||
has_error_status: bool = False
|
||||
has_not_found: bool = False
|
||||
has_sitemap_resource: bool = False
|
||||
has_feed_resource: bool = False
|
||||
has_json_resource: bool = False
|
||||
has_xml_resource: bool = False
|
||||
has_file_resource: bool = False
|
||||
|
||||
# graph
|
||||
internal_link_count: int = 0
|
||||
external_link_count: int = 0
|
||||
product_link_count: int = 0
|
||||
category_link_count: int = 0
|
||||
profile_link_count: int = 0
|
||||
article_link_count: int = 0
|
||||
|
||||
# visual/layout candidates from DOM structure
|
||||
layout_blocks: list[str] = field(default_factory=list)
|
||||
has_hero_block: bool = False
|
||||
has_card_grid: bool = False
|
||||
has_filter_sidebar: bool = False
|
||||
has_sticky_action_box: bool = False
|
||||
has_media_player_area: bool = False
|
||||
has_map_area: bool = False
|
||||
has_calendar_grid: bool = False
|
||||
has_pricing_table: bool = False
|
||||
has_comparison_table: bool = False
|
||||
|
||||
# text/layout
|
||||
dominant_language: str | None = None
|
||||
keyword_hits: dict[str, int] = field(default_factory=dict)
|
||||
url_hints: set[str] = field(default_factory=set)
|
||||
title: str | None = None
|
||||
text_sample: str = ""
|
||||
|
||||
# external collector hook
|
||||
external_collector_signals: dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
payload = asdict(self)
|
||||
payload["schema_types"] = sorted(self.schema_types)
|
||||
payload["url_hints"] = sorted(self.url_hints)
|
||||
return payload
|
||||
@@ -0,0 +1,747 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
from crawler_platform.app.core.crawler.page_signals import PageSignals
|
||||
from crawler_platform.app.core.crawler.page_type_taxonomy import (
|
||||
ActionIntent,
|
||||
AnalyzeStrategy,
|
||||
EntityType,
|
||||
EvidenceItem,
|
||||
GraphRole,
|
||||
LLMPolicy,
|
||||
PAGE_TYPE_PROFILES,
|
||||
PageArchetype,
|
||||
PageClassificationResult,
|
||||
PageDomain,
|
||||
PageType,
|
||||
)
|
||||
|
||||
|
||||
UNKNOWN_THRESHOLD = 0.22
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class SignalRule:
|
||||
key: str
|
||||
weight: float
|
||||
source: str
|
||||
message: str
|
||||
predicate: Callable[[PageSignals], bool]
|
||||
value: Callable[[PageSignals], str | int | float | bool | None] | None = None
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class ScoreAccumulator:
|
||||
scores: dict[str, float]
|
||||
evidence_by_type: dict[str, list[EvidenceItem]]
|
||||
|
||||
def add(self, page_type: str, rule: SignalRule, signals: PageSignals) -> None:
|
||||
if not rule.predicate(signals):
|
||||
return
|
||||
value = rule.value(signals) if rule.value else True
|
||||
self.scores[page_type] = self.scores.get(page_type, 0.0) + rule.weight
|
||||
self.evidence_by_type.setdefault(page_type, []).append(
|
||||
EvidenceItem(
|
||||
key=rule.key,
|
||||
value=value,
|
||||
weight=rule.weight,
|
||||
source=rule.source,
|
||||
message=rule.message,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
PAGE_TYPE_METADATA: dict[str, dict[str, Any]] = {
|
||||
**PAGE_TYPE_PROFILES,
|
||||
PageType.ARTICLE_PAGE.value: {
|
||||
"domain": PageDomain.EDITORIAL.value,
|
||||
"archetype": PageArchetype.ARTICLE.value,
|
||||
"main_entity_type": EntityType.ARTICLE.value,
|
||||
"action_intents": [ActionIntent.READ.value],
|
||||
"graph_roles": [GraphRole.CLAIM_SOURCE.value, GraphRole.ENTITY_ANCHOR.value],
|
||||
"analyze_strategy": AnalyzeStrategy.ANALYZE_FULL.value,
|
||||
"llm_policy": LLMPolicy.LLM_FULL.value,
|
||||
"should_analyze": True,
|
||||
},
|
||||
PageType.BLOG_POST_PAGE.value: {
|
||||
"domain": PageDomain.EDITORIAL.value,
|
||||
"archetype": PageArchetype.ARTICLE.value,
|
||||
"main_entity_type": EntityType.ARTICLE.value,
|
||||
"action_intents": [ActionIntent.READ.value],
|
||||
"graph_roles": [GraphRole.CLAIM_SOURCE.value],
|
||||
"analyze_strategy": AnalyzeStrategy.ANALYZE_FULL.value,
|
||||
"llm_policy": LLMPolicy.LLM_FULL.value,
|
||||
"should_analyze": True,
|
||||
},
|
||||
PageType.FAQ_PAGE.value: {
|
||||
"domain": PageDomain.COMMUNITY.value,
|
||||
"archetype": PageArchetype.DOCUMENT.value,
|
||||
"main_entity_type": EntityType.QUESTION.value,
|
||||
"action_intents": [ActionIntent.READ.value, ActionIntent.ANSWER.value],
|
||||
"graph_roles": [GraphRole.REFERENCE_SOURCE.value],
|
||||
"analyze_strategy": AnalyzeStrategy.ANALYZE_FULL.value,
|
||||
"llm_policy": LLMPolicy.LLM_LIGHT.value,
|
||||
"should_analyze": True,
|
||||
},
|
||||
PageType.QA_PAGE.value: {
|
||||
"domain": PageDomain.COMMUNITY.value,
|
||||
"archetype": PageArchetype.THREAD.value,
|
||||
"main_entity_type": EntityType.QUESTION.value,
|
||||
"action_intents": [ActionIntent.ASK.value, ActionIntent.ANSWER.value, ActionIntent.READ.value],
|
||||
"graph_roles": [GraphRole.CLAIM_SOURCE.value, GraphRole.RELATION_HUB.value],
|
||||
"analyze_strategy": AnalyzeStrategy.ANALYZE_FULL.value,
|
||||
"llm_policy": LLMPolicy.LLM_LIGHT.value,
|
||||
"should_analyze": True,
|
||||
},
|
||||
PageType.FORUM_THREAD_PAGE.value: {
|
||||
"domain": PageDomain.COMMUNITY.value,
|
||||
"archetype": PageArchetype.THREAD.value,
|
||||
"main_entity_type": EntityType.ARTICLE.value,
|
||||
"action_intents": [ActionIntent.READ.value, ActionIntent.COMMENT.value],
|
||||
"graph_roles": [GraphRole.CLAIM_SOURCE.value, GraphRole.RELATION_HUB.value],
|
||||
"analyze_strategy": AnalyzeStrategy.ANALYZE_FULL.value,
|
||||
"llm_policy": LLMPolicy.LLM_LIGHT.value,
|
||||
"should_analyze": True,
|
||||
},
|
||||
PageType.ABOUT_PAGE.value: {
|
||||
"domain": PageDomain.CORPORATE.value,
|
||||
"archetype": PageArchetype.ARTICLE.value,
|
||||
"main_entity_type": EntityType.ORGANIZATION.value,
|
||||
"action_intents": [ActionIntent.READ.value, ActionIntent.LEARN.value],
|
||||
"graph_roles": [GraphRole.ENTITY_ANCHOR.value, GraphRole.CLAIM_SOURCE.value],
|
||||
"analyze_strategy": AnalyzeStrategy.ANALYZE_FULL.value,
|
||||
"llm_policy": LLMPolicy.LLM_FULL.value,
|
||||
"should_analyze": True,
|
||||
},
|
||||
PageType.CONTACT_PAGE.value: {
|
||||
"domain": PageDomain.CORPORATE.value,
|
||||
"archetype": PageArchetype.DOCUMENT.value,
|
||||
"main_entity_type": EntityType.ORGANIZATION.value,
|
||||
"action_intents": [ActionIntent.CONTACT.value, ActionIntent.NAVIGATE.value],
|
||||
"graph_roles": [GraphRole.ENTITY_ANCHOR.value],
|
||||
"analyze_strategy": AnalyzeStrategy.ANALYZE_METADATA_ONLY.value,
|
||||
"llm_policy": LLMPolicy.RULE_ONLY.value,
|
||||
"should_analyze": True,
|
||||
},
|
||||
PageType.DOCUMENTATION_PAGE.value: {
|
||||
"domain": PageDomain.KNOWLEDGE.value,
|
||||
"archetype": PageArchetype.DOCUMENT.value,
|
||||
"main_entity_type": EntityType.SOFTWARE_APPLICATION.value,
|
||||
"action_intents": [ActionIntent.READ.value, ActionIntent.LEARN.value],
|
||||
"graph_roles": [GraphRole.REFERENCE_SOURCE.value],
|
||||
"analyze_strategy": AnalyzeStrategy.ANALYZE_FULL.value,
|
||||
"llm_policy": LLMPolicy.LLM_LIGHT.value,
|
||||
"should_analyze": True,
|
||||
},
|
||||
PageType.API_REFERENCE_PAGE.value: {
|
||||
"domain": PageDomain.SOFTWARE.value,
|
||||
"archetype": PageArchetype.DOCUMENT.value,
|
||||
"main_entity_type": EntityType.SOFTWARE_APPLICATION.value,
|
||||
"action_intents": [ActionIntent.LEARN.value, ActionIntent.CONFIGURE.value],
|
||||
"graph_roles": [GraphRole.REFERENCE_SOURCE.value],
|
||||
"analyze_strategy": AnalyzeStrategy.ANALYZE_FULL.value,
|
||||
"llm_policy": LLMPolicy.LLM_LIGHT.value,
|
||||
"should_analyze": True,
|
||||
},
|
||||
PageType.WIKI_PAGE.value: {
|
||||
"domain": PageDomain.KNOWLEDGE.value,
|
||||
"archetype": PageArchetype.DOCUMENT.value,
|
||||
"main_entity_type": EntityType.UNKNOWN_ENTITY.value,
|
||||
"action_intents": [ActionIntent.READ.value, ActionIntent.LEARN.value],
|
||||
"graph_roles": [GraphRole.REFERENCE_SOURCE.value],
|
||||
"analyze_strategy": AnalyzeStrategy.ANALYZE_FULL.value,
|
||||
"llm_policy": LLMPolicy.LLM_LIGHT.value,
|
||||
"should_analyze": True,
|
||||
},
|
||||
PageType.DATASET_PAGE.value: {
|
||||
"domain": PageDomain.KNOWLEDGE.value,
|
||||
"archetype": PageArchetype.DOCUMENT.value,
|
||||
"main_entity_type": EntityType.DATASET.value,
|
||||
"action_intents": [ActionIntent.READ.value, ActionIntent.DOWNLOAD.value],
|
||||
"graph_roles": [GraphRole.REFERENCE_SOURCE.value],
|
||||
"analyze_strategy": AnalyzeStrategy.ANALYZE_DOCUMENT_ONLY.value,
|
||||
"llm_policy": LLMPolicy.LLM_LIGHT.value,
|
||||
"should_analyze": True,
|
||||
},
|
||||
PageType.RESEARCH_PAPER_PAGE.value: {
|
||||
"domain": PageDomain.KNOWLEDGE.value,
|
||||
"archetype": PageArchetype.ARTICLE.value,
|
||||
"main_entity_type": EntityType.ARTICLE.value,
|
||||
"action_intents": [ActionIntent.READ.value, ActionIntent.LEARN.value],
|
||||
"graph_roles": [GraphRole.CLAIM_SOURCE.value, GraphRole.REFERENCE_SOURCE.value],
|
||||
"analyze_strategy": AnalyzeStrategy.ANALYZE_FULL.value,
|
||||
"llm_policy": LLMPolicy.LLM_FULL.value,
|
||||
"should_analyze": True,
|
||||
},
|
||||
PageType.JOB_POSTING_PAGE.value: {
|
||||
"domain": PageDomain.JOBS.value,
|
||||
"archetype": PageArchetype.DETAIL.value,
|
||||
"main_entity_type": EntityType.JOB_POSTING.value,
|
||||
"action_intents": [ActionIntent.APPLY.value, ActionIntent.READ.value],
|
||||
"graph_roles": [GraphRole.ENTITY_ANCHOR.value],
|
||||
"analyze_strategy": AnalyzeStrategy.ANALYZE_FULL.value,
|
||||
"llm_policy": LLMPolicy.LLM_FULL.value,
|
||||
"should_analyze": True,
|
||||
},
|
||||
PageType.COURSE_DETAIL_PAGE.value: {
|
||||
"domain": PageDomain.EDUCATION.value,
|
||||
"archetype": PageArchetype.DETAIL.value,
|
||||
"main_entity_type": EntityType.COURSE.value,
|
||||
"action_intents": [ActionIntent.LEARN.value, ActionIntent.SUBSCRIBE.value],
|
||||
"graph_roles": [GraphRole.ENTITY_ANCHOR.value, GraphRole.REFERENCE_SOURCE.value],
|
||||
"analyze_strategy": AnalyzeStrategy.ANALYZE_FULL.value,
|
||||
"llm_policy": LLMPolicy.LLM_LIGHT.value,
|
||||
"should_analyze": True,
|
||||
},
|
||||
PageType.VIDEO_PAGE.value: {
|
||||
"domain": PageDomain.MEDIA.value,
|
||||
"archetype": PageArchetype.MEDIA.value,
|
||||
"main_entity_type": EntityType.MEDIA_OBJECT.value,
|
||||
"action_intents": [ActionIntent.WATCH.value],
|
||||
"graph_roles": [GraphRole.MEDIA_ANCHOR.value],
|
||||
"analyze_strategy": AnalyzeStrategy.ANALYZE_METADATA_ONLY.value,
|
||||
"llm_policy": LLMPolicy.RULE_ONLY.value,
|
||||
"should_analyze": True,
|
||||
},
|
||||
PageType.LOCAL_BUSINESS_PAGE.value: {
|
||||
"domain": PageDomain.LOCAL.value,
|
||||
"archetype": PageArchetype.DETAIL.value,
|
||||
"main_entity_type": EntityType.PLACE.value,
|
||||
"action_intents": [ActionIntent.CONTACT.value, ActionIntent.NAVIGATE.value],
|
||||
"graph_roles": [GraphRole.ENTITY_ANCHOR.value],
|
||||
"analyze_strategy": AnalyzeStrategy.ANALYZE_FULL.value,
|
||||
"llm_policy": LLMPolicy.LLM_LIGHT.value,
|
||||
"should_analyze": True,
|
||||
},
|
||||
PageType.REAL_ESTATE_LISTING_PAGE.value: {
|
||||
"domain": PageDomain.LOCAL.value,
|
||||
"archetype": PageArchetype.DETAIL.value,
|
||||
"main_entity_type": EntityType.REAL_ESTATE_PROPERTY.value,
|
||||
"action_intents": [ActionIntent.READ.value, ActionIntent.CONTACT.value],
|
||||
"graph_roles": [GraphRole.ENTITY_ANCHOR.value],
|
||||
"analyze_strategy": AnalyzeStrategy.ANALYZE_FULL.value,
|
||||
"llm_policy": LLMPolicy.LLM_LIGHT.value,
|
||||
"should_analyze": True,
|
||||
},
|
||||
PageType.PROFILE_PAGE.value: {
|
||||
"domain": PageDomain.COMMUNITY.value,
|
||||
"archetype": PageArchetype.PROFILE.value,
|
||||
"main_entity_type": EntityType.PERSON.value,
|
||||
"action_intents": [ActionIntent.READ.value, ActionIntent.CONTACT.value],
|
||||
"graph_roles": [GraphRole.PROFILE_ANCHOR.value],
|
||||
"analyze_strategy": AnalyzeStrategy.ANALYZE_ENTITY_ONLY.value,
|
||||
"llm_policy": LLMPolicy.LLM_LIGHT.value,
|
||||
"should_analyze": True,
|
||||
},
|
||||
PageType.PRICING_PAGE.value: {
|
||||
"domain": PageDomain.COMMERCE.value,
|
||||
"archetype": PageArchetype.LANDING.value,
|
||||
"main_entity_type": EntityType.SERVICE.value,
|
||||
"action_intents": [ActionIntent.BUY.value, ActionIntent.COMPARE.value],
|
||||
"graph_roles": [GraphRole.CLAIM_SOURCE.value],
|
||||
"analyze_strategy": AnalyzeStrategy.ANALYZE_FULL.value,
|
||||
"llm_policy": LLMPolicy.LLM_LIGHT.value,
|
||||
"should_analyze": True,
|
||||
},
|
||||
PageType.LOGIN_PAGE.value: {
|
||||
"domain": PageDomain.TRANSACTION.value,
|
||||
"archetype": PageArchetype.FORM.value,
|
||||
"main_entity_type": None,
|
||||
"action_intents": [ActionIntent.LOGIN.value],
|
||||
"graph_roles": [GraphRole.TRANSACTION_ONLY.value],
|
||||
"analyze_strategy": AnalyzeStrategy.SKIP_PROTECTED.value,
|
||||
"llm_policy": LLMPolicy.SKIP.value,
|
||||
"should_analyze": False,
|
||||
},
|
||||
PageType.CHECKOUT_PAGE.value: {
|
||||
"domain": PageDomain.TRANSACTION.value,
|
||||
"archetype": PageArchetype.TRANSACTION.value,
|
||||
"main_entity_type": None,
|
||||
"action_intents": [ActionIntent.BUY.value, ActionIntent.PAY.value],
|
||||
"graph_roles": [GraphRole.TRANSACTION_ONLY.value],
|
||||
"analyze_strategy": AnalyzeStrategy.SKIP_PROTECTED.value,
|
||||
"llm_policy": LLMPolicy.SKIP.value,
|
||||
"should_analyze": False,
|
||||
},
|
||||
PageType.PAYMENT_PAGE.value: {
|
||||
"domain": PageDomain.TRANSACTION.value,
|
||||
"archetype": PageArchetype.TRANSACTION.value,
|
||||
"main_entity_type": None,
|
||||
"action_intents": [ActionIntent.PAY.value],
|
||||
"graph_roles": [GraphRole.TRANSACTION_ONLY.value],
|
||||
"analyze_strategy": AnalyzeStrategy.SKIP_PROTECTED.value,
|
||||
"llm_policy": LLMPolicy.SKIP.value,
|
||||
"should_analyze": False,
|
||||
},
|
||||
PageType.TERMS_PAGE.value: {
|
||||
"domain": PageDomain.CORPORATE.value,
|
||||
"archetype": PageArchetype.DOCUMENT.value,
|
||||
"main_entity_type": EntityType.LEGAL_DOCUMENT.value,
|
||||
"action_intents": [ActionIntent.READ.value],
|
||||
"graph_roles": [GraphRole.POLICY_SOURCE.value],
|
||||
"analyze_strategy": AnalyzeStrategy.ANALYZE_DOCUMENT_ONLY.value,
|
||||
"llm_policy": LLMPolicy.LLM_LIGHT.value,
|
||||
"should_analyze": True,
|
||||
},
|
||||
PageType.PRIVACY_POLICY_PAGE.value: {
|
||||
"domain": PageDomain.CORPORATE.value,
|
||||
"archetype": PageArchetype.DOCUMENT.value,
|
||||
"main_entity_type": EntityType.LEGAL_DOCUMENT.value,
|
||||
"action_intents": [ActionIntent.READ.value],
|
||||
"graph_roles": [GraphRole.POLICY_SOURCE.value],
|
||||
"analyze_strategy": AnalyzeStrategy.ANALYZE_DOCUMENT_ONLY.value,
|
||||
"llm_policy": LLMPolicy.LLM_LIGHT.value,
|
||||
"should_analyze": True,
|
||||
},
|
||||
PageType.SITEMAP_PAGE.value: {
|
||||
"domain": PageDomain.SYSTEM.value,
|
||||
"archetype": PageArchetype.SYSTEM_RESOURCE.value,
|
||||
"main_entity_type": None,
|
||||
"action_intents": [ActionIntent.NAVIGATE.value],
|
||||
"graph_roles": [GraphRole.SYSTEM_RESOURCE.value, GraphRole.NAVIGATION_HUB.value],
|
||||
"analyze_strategy": AnalyzeStrategy.ANALYZE_DISCOVERY_ONLY.value,
|
||||
"llm_policy": LLMPolicy.RULE_ONLY.value,
|
||||
"should_analyze": True,
|
||||
},
|
||||
PageType.RSS_FEED_PAGE.value: {
|
||||
"domain": PageDomain.SYSTEM.value,
|
||||
"archetype": PageArchetype.SYSTEM_RESOURCE.value,
|
||||
"main_entity_type": None,
|
||||
"action_intents": [ActionIntent.READ.value],
|
||||
"graph_roles": [GraphRole.SYSTEM_RESOURCE.value],
|
||||
"analyze_strategy": AnalyzeStrategy.ANALYZE_METADATA_ONLY.value,
|
||||
"llm_policy": LLMPolicy.RULE_ONLY.value,
|
||||
"should_analyze": True,
|
||||
},
|
||||
PageType.ERROR_PAGE.value: {
|
||||
"domain": PageDomain.SYSTEM.value,
|
||||
"archetype": PageArchetype.ERROR.value,
|
||||
"main_entity_type": None,
|
||||
"action_intents": [],
|
||||
"graph_roles": [GraphRole.NOISE_PAGE.value],
|
||||
"analyze_strategy": AnalyzeStrategy.SKIP_NOISE.value,
|
||||
"llm_policy": LLMPolicy.SKIP.value,
|
||||
"should_analyze": False,
|
||||
},
|
||||
PageType.NOT_FOUND_PAGE.value: {
|
||||
"domain": PageDomain.SYSTEM.value,
|
||||
"archetype": PageArchetype.ERROR.value,
|
||||
"main_entity_type": None,
|
||||
"action_intents": [],
|
||||
"graph_roles": [GraphRole.NOISE_PAGE.value],
|
||||
"analyze_strategy": AnalyzeStrategy.SKIP_NOISE.value,
|
||||
"llm_policy": LLMPolicy.SKIP.value,
|
||||
"should_analyze": False,
|
||||
},
|
||||
PageType.ACCESS_DENIED_PAGE.value: {
|
||||
"domain": PageDomain.TRANSACTION.value,
|
||||
"archetype": PageArchetype.ERROR.value,
|
||||
"main_entity_type": None,
|
||||
"action_intents": [ActionIntent.VERIFY.value],
|
||||
"graph_roles": [GraphRole.TRANSACTION_ONLY.value],
|
||||
"analyze_strategy": AnalyzeStrategy.SKIP_PROTECTED.value,
|
||||
"llm_policy": LLMPolicy.SKIP.value,
|
||||
"should_analyze": False,
|
||||
},
|
||||
PageType.CAPTCHA_PAGE.value: {
|
||||
"domain": PageDomain.TRANSACTION.value,
|
||||
"archetype": PageArchetype.FORM.value,
|
||||
"main_entity_type": None,
|
||||
"action_intents": [ActionIntent.VERIFY.value],
|
||||
"graph_roles": [GraphRole.TRANSACTION_ONLY.value],
|
||||
"analyze_strategy": AnalyzeStrategy.SKIP_PROTECTED.value,
|
||||
"llm_policy": LLMPolicy.SKIP.value,
|
||||
"should_analyze": False,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def score_page_type(url: str, signals: PageSignals, *, unknown_threshold: float = UNKNOWN_THRESHOLD) -> PageClassificationResult:
|
||||
accumulator = ScoreAccumulator(scores={}, evidence_by_type={})
|
||||
for page_type, rules in SCORING_RULES.items():
|
||||
for rule in rules:
|
||||
accumulator.add(page_type, rule, signals)
|
||||
|
||||
_apply_interactions(accumulator, signals)
|
||||
|
||||
if not accumulator.scores:
|
||||
return _unknown_result(
|
||||
url,
|
||||
alternatives=[],
|
||||
evidence=[
|
||||
EvidenceItem(
|
||||
key="insufficient_evidence",
|
||||
value=True,
|
||||
weight=0.0,
|
||||
source="page_type_scorer",
|
||||
message="No page-type evidence was detected.",
|
||||
)
|
||||
],
|
||||
)
|
||||
|
||||
alternatives = sorted(
|
||||
((page_type, round(min(score, 1.0), 4)) for page_type, score in accumulator.scores.items()),
|
||||
key=lambda item: item[1],
|
||||
reverse=True,
|
||||
)
|
||||
primary_page_type, confidence = alternatives[0]
|
||||
if confidence < unknown_threshold:
|
||||
evidence = [item for page_type, _score in alternatives[:3] for item in accumulator.evidence_by_type.get(page_type, [])]
|
||||
evidence.append(
|
||||
EvidenceItem(
|
||||
key="low_confidence",
|
||||
value=confidence,
|
||||
weight=0.0,
|
||||
source="page_type_scorer",
|
||||
message=f"Top score {confidence:.2f} is below UnknownPage threshold {unknown_threshold:.2f}.",
|
||||
)
|
||||
)
|
||||
return _unknown_result(url, alternatives=alternatives[:5], evidence=evidence)
|
||||
|
||||
profile = _profile(primary_page_type)
|
||||
evidence = accumulator.evidence_by_type.get(primary_page_type, [])
|
||||
if not evidence:
|
||||
evidence = [
|
||||
EvidenceItem(
|
||||
key="score",
|
||||
value=confidence,
|
||||
weight=confidence,
|
||||
source="page_type_scorer",
|
||||
message=f"{primary_page_type} selected from accumulated score.",
|
||||
)
|
||||
]
|
||||
secondary = [page_type for page_type, score in alternatives[1:4] if score >= 0.18]
|
||||
return PageClassificationResult(
|
||||
url=url,
|
||||
primary_page_type=primary_page_type,
|
||||
secondary_page_types=secondary,
|
||||
domain=str(profile["domain"]),
|
||||
archetype=str(profile["archetype"]),
|
||||
main_entity_type=profile.get("main_entity_type"),
|
||||
action_intents=list(profile.get("action_intents") or []),
|
||||
graph_roles=list(profile.get("graph_roles") or []),
|
||||
confidence=confidence,
|
||||
alternatives=alternatives[:8],
|
||||
evidence=evidence,
|
||||
should_analyze=bool(profile.get("should_analyze")),
|
||||
analyze_strategy=str(profile["analyze_strategy"]),
|
||||
llm_policy=str(profile["llm_policy"]),
|
||||
is_protected=str(profile["analyze_strategy"]) == AnalyzeStrategy.SKIP_PROTECTED.value,
|
||||
is_noise=str(profile["analyze_strategy"]) == AnalyzeStrategy.SKIP_NOISE.value,
|
||||
)
|
||||
|
||||
|
||||
def _profile(page_type: str) -> dict[str, Any]:
|
||||
return PAGE_TYPE_METADATA.get(page_type, PAGE_TYPE_METADATA[PageType.UNKNOWN_PAGE.value])
|
||||
|
||||
|
||||
def _unknown_result(
|
||||
url: str,
|
||||
*,
|
||||
alternatives: list[tuple[str, float]],
|
||||
evidence: list[EvidenceItem],
|
||||
) -> PageClassificationResult:
|
||||
profile = _profile(PageType.UNKNOWN_PAGE.value)
|
||||
return PageClassificationResult(
|
||||
url=url,
|
||||
primary_page_type=PageType.UNKNOWN_PAGE.value,
|
||||
secondary_page_types=[page_type for page_type, _score in alternatives[:3]],
|
||||
domain=str(profile["domain"]),
|
||||
archetype=str(profile["archetype"]),
|
||||
main_entity_type=profile.get("main_entity_type"),
|
||||
action_intents=list(profile.get("action_intents") or []),
|
||||
graph_roles=list(profile.get("graph_roles") or []),
|
||||
confidence=0.0,
|
||||
alternatives=alternatives[:8],
|
||||
evidence=evidence
|
||||
or [
|
||||
EvidenceItem(
|
||||
key="unknown",
|
||||
value=True,
|
||||
weight=0.0,
|
||||
source="page_type_scorer",
|
||||
message="Page did not match known semantic page type evidence.",
|
||||
)
|
||||
],
|
||||
should_analyze=bool(profile.get("should_analyze")),
|
||||
analyze_strategy=str(profile["analyze_strategy"]),
|
||||
llm_policy=str(profile["llm_policy"]),
|
||||
is_protected=False,
|
||||
is_noise=False,
|
||||
)
|
||||
|
||||
|
||||
def _apply_interactions(accumulator: ScoreAccumulator, signals: PageSignals) -> None:
|
||||
# Protected signals must dominate commerce signals when login/payment fields are present.
|
||||
if signals.has_password_field and signals.has_login_form:
|
||||
_manual_add(accumulator, PageType.LOGIN_PAGE.value, "protected_priority", True, 0.3, "protected", "Login form and password field override commerce signals.")
|
||||
if signals.has_payment_fields:
|
||||
_manual_add(accumulator, PageType.PAYMENT_PAGE.value, "payment_priority", True, 0.35, "protected", "Payment fields override product or checkout content.")
|
||||
if signals.has_access_denied:
|
||||
_manual_add(accumulator, PageType.ACCESS_DENIED_PAGE.value, "access_denied_priority", True, 0.35, "protected", "Access denied signal is a protected page indicator.")
|
||||
if signals.has_captcha:
|
||||
_manual_add(accumulator, PageType.CAPTCHA_PAGE.value, "captcha_priority", True, 0.35, "protected", "Captcha signal is a protected page indicator.")
|
||||
if signals.has_repeated_cards and signals.product_link_count >= 3:
|
||||
_manual_add(accumulator, PageType.CATEGORY_LISTING_PAGE.value, "listing_product_links", signals.product_link_count, 0.18, "link_graph", "Repeated cards with multiple product links indicate a product listing.")
|
||||
if signals.has_question and signals.has_answer and signals.has_faq_structure:
|
||||
_manual_add(accumulator, PageType.FAQ_PAGE.value, "faq_question_answer", True, 0.18, "community", "FAQ structure with question/answer content.")
|
||||
|
||||
|
||||
def _manual_add(
|
||||
accumulator: ScoreAccumulator,
|
||||
page_type: str,
|
||||
key: str,
|
||||
value: str | int | float | bool | None,
|
||||
weight: float,
|
||||
source: str,
|
||||
message: str,
|
||||
) -> None:
|
||||
accumulator.scores[page_type] = accumulator.scores.get(page_type, 0.0) + weight
|
||||
accumulator.evidence_by_type.setdefault(page_type, []).append(
|
||||
EvidenceItem(key=key, value=value, weight=weight, source=source, message=message)
|
||||
)
|
||||
|
||||
|
||||
def _has_schema(*schema_types: str) -> Callable[[PageSignals], bool]:
|
||||
return lambda signals: any(schema_type in signals.schema_types for schema_type in schema_types)
|
||||
|
||||
|
||||
def _keyword(group: str, minimum: int = 1) -> Callable[[PageSignals], bool]:
|
||||
return lambda signals: int(signals.keyword_hits.get(group) or 0) >= minimum
|
||||
|
||||
|
||||
def _url_hint(hint: str) -> Callable[[PageSignals], bool]:
|
||||
return lambda signals: hint in signals.url_hints
|
||||
|
||||
|
||||
def _contains_terms(signals: PageSignals, terms: tuple[str, ...]) -> bool:
|
||||
text = f"{signals.title or ''}\n{signals.text_sample or ''}".lower()
|
||||
return any(term.lower() in text for term in terms)
|
||||
|
||||
|
||||
def _flag(name: str) -> Callable[[PageSignals], bool]:
|
||||
return lambda signals: bool(getattr(signals, name))
|
||||
|
||||
|
||||
def _count_at_least(name: str, minimum: int) -> Callable[[PageSignals], bool]:
|
||||
return lambda signals: int(getattr(signals, name) or 0) >= minimum
|
||||
|
||||
|
||||
def _value(name: str) -> Callable[[PageSignals], str | int | float | bool | None]:
|
||||
return lambda signals: getattr(signals, name)
|
||||
|
||||
|
||||
def _keywords_value(group: str) -> Callable[[PageSignals], str | int | float | bool | None]:
|
||||
return lambda signals: signals.keyword_hits.get(group, 0)
|
||||
|
||||
|
||||
def _schema_value(signals: PageSignals) -> str:
|
||||
return ",".join(sorted(signals.schema_types))
|
||||
|
||||
|
||||
def _url_hint_value(signals: PageSignals) -> str:
|
||||
return ",".join(sorted(signals.url_hints))
|
||||
|
||||
|
||||
SCORING_RULES: dict[str, list[SignalRule]] = {
|
||||
PageType.PRODUCT_DETAIL_PAGE.value: [
|
||||
SignalRule("schema_product", 0.4, "structured_data", "schema.org Product detected.", _has_schema("Product"), _schema_value),
|
||||
SignalRule("schema_offer", 0.15, "structured_data", "schema.org Offer detected.", _has_schema("Offer"), _schema_value),
|
||||
SignalRule("price", 0.15, "text", "Price detected.", _flag("has_price"), _value("has_price")),
|
||||
SignalRule("cart_button", 0.2, "dom", "Cart button detected.", _flag("has_cart_button")),
|
||||
SignalRule("buy_button", 0.16, "dom", "Buy button detected.", _flag("has_buy_button")),
|
||||
SignalRule("variant_selector", 0.12, "dom", "Variant selector detected.", _flag("has_variant_selector")),
|
||||
SignalRule("sku", 0.1, "text", "SKU or product code detected.", _flag("has_sku")),
|
||||
SignalRule("product_gallery", 0.1, "layout", "Product image gallery detected.", _flag("has_product_gallery")),
|
||||
SignalRule("review_section", 0.05, "dom", "Review section detected.", _flag("has_review_section")),
|
||||
SignalRule("url_product_hint", 0.05, "url", "Product URL hint detected.", _url_hint("product"), _url_hint_value),
|
||||
],
|
||||
PageType.CATEGORY_LISTING_PAGE.value: [
|
||||
SignalRule("repeated_cards", 0.32, "layout", "Repeated cards detected.", _flag("has_repeated_cards"), _value("repeated_card_count")),
|
||||
SignalRule("filter_panel", 0.2, "dom", "Filter panel detected.", _flag("has_filter_panel")),
|
||||
SignalRule("sort_control", 0.15, "dom", "Sort control detected.", _flag("has_sort_control")),
|
||||
SignalRule("pagination", 0.1, "dom", "Pagination detected.", _flag("has_pagination")),
|
||||
SignalRule("product_links", 0.18, "link_graph", "Multiple product links detected.", _count_at_least("product_link_count", 3), _value("product_link_count")),
|
||||
SignalRule("category_url_hint", 0.06, "url", "Category/list URL hint detected.", _url_hint("category"), _url_hint_value),
|
||||
],
|
||||
PageType.SEARCH_RESULTS_PAGE.value: [
|
||||
SignalRule("search_url_hint", 0.28, "url", "Search URL hint detected.", _url_hint("search"), _url_hint_value),
|
||||
SignalRule("listing_results", 0.18, "layout", "Repeated result cards detected.", _flag("has_repeated_cards"), _value("repeated_card_count")),
|
||||
SignalRule("filter_panel", 0.14, "dom", "Search filter panel detected.", _flag("has_filter_panel")),
|
||||
SignalRule("pagination", 0.12, "dom", "Search pagination detected.", _flag("has_pagination")),
|
||||
SignalRule("search_keywords", 0.12, "text", "Search/result keywords detected.", _keyword("listing"), _keywords_value("listing")),
|
||||
],
|
||||
PageType.ARTICLE_PAGE.value: [
|
||||
SignalRule("schema_article", 0.35, "structured_data", "Article structured data detected.", _has_schema("Article", "NewsArticle"), _schema_value),
|
||||
SignalRule("author", 0.15, "text", "Author/byline signal detected.", _flag("has_author")),
|
||||
SignalRule("published_date", 0.15, "text", "Published date detected.", _flag("has_published_date")),
|
||||
SignalRule("article_body", 0.2, "dom", "Article body detected.", _flag("has_article_body")),
|
||||
SignalRule("tags", 0.05, "dom", "Article tags detected.", _flag("has_tags")),
|
||||
SignalRule("article_url_hint", 0.05, "url", "Article URL hint detected.", _url_hint("article"), _url_hint_value),
|
||||
],
|
||||
PageType.BLOG_POST_PAGE.value: [
|
||||
SignalRule("blog_url_hint", 0.25, "url", "Blog URL hint detected.", lambda signals: "article" in signals.url_hints and "blog" in (signals.text_sample or "").lower()),
|
||||
SignalRule("author", 0.14, "text", "Author signal detected.", _flag("has_author")),
|
||||
SignalRule("published_date", 0.14, "text", "Published date detected.", _flag("has_published_date")),
|
||||
SignalRule("article_body", 0.18, "dom", "Article body detected.", _flag("has_article_body")),
|
||||
SignalRule("tags", 0.08, "dom", "Tags detected.", _flag("has_tags")),
|
||||
],
|
||||
PageType.QA_PAGE.value: [
|
||||
SignalRule("schema_qapage", 0.35, "structured_data", "QAPage structured data detected.", _has_schema("QAPage"), _schema_value),
|
||||
SignalRule("question", 0.2, "community", "Question block detected.", _flag("has_question")),
|
||||
SignalRule("answer", 0.2, "community", "Answer block detected.", _flag("has_answer")),
|
||||
SignalRule("votes", 0.1, "community", "Vote signal detected.", _flag("has_votes")),
|
||||
SignalRule("comments", 0.05, "community", "Comments detected.", _flag("has_comments")),
|
||||
SignalRule("board_url_hint", 0.05, "url", "Board/community URL hint detected.", _url_hint("board"), _url_hint_value),
|
||||
],
|
||||
PageType.FAQ_PAGE.value: [
|
||||
SignalRule("schema_faq", 0.36, "structured_data", "FAQPage structured data detected.", _has_schema("FAQPage"), _schema_value),
|
||||
SignalRule("faq_structure", 0.25, "community", "FAQ structure detected.", _flag("has_faq_structure")),
|
||||
SignalRule("question", 0.15, "community", "Question content detected.", _flag("has_question")),
|
||||
SignalRule("answer", 0.15, "community", "Answer content detected.", _flag("has_answer")),
|
||||
],
|
||||
PageType.FORUM_BOARD_PAGE.value: [
|
||||
SignalRule("board_url_hint", 0.25, "url", "Board URL hint detected.", _url_hint("board"), _url_hint_value),
|
||||
SignalRule("thread_structure", 0.2, "community", "Thread structure detected.", _flag("has_thread_structure")),
|
||||
SignalRule("comments", 0.1, "community", "Comments/replies detected.", _flag("has_comments")),
|
||||
SignalRule("repeated_cards", 0.14, "layout", "Repeated post cards detected.", _flag("has_repeated_cards"), _value("repeated_card_count")),
|
||||
SignalRule("pagination", 0.1, "dom", "Board pagination detected.", _flag("has_pagination")),
|
||||
],
|
||||
PageType.FORUM_THREAD_PAGE.value: [
|
||||
SignalRule("thread_structure", 0.3, "community", "Thread structure detected.", _flag("has_thread_structure")),
|
||||
SignalRule("comments", 0.16, "community", "Comment thread detected.", _flag("has_comments")),
|
||||
SignalRule("question_answer", 0.14, "community", "Question and answer content detected.", lambda signals: signals.has_question and signals.has_answer),
|
||||
SignalRule("votes", 0.08, "community", "Vote signal detected.", _flag("has_votes")),
|
||||
SignalRule("article_body", 0.08, "dom", "Post body detected.", _flag("has_article_body")),
|
||||
],
|
||||
PageType.BRAND_STORY_PAGE.value: [
|
||||
SignalRule("corporate_keywords", 0.2, "text", "Corporate/brand keywords detected.", _keyword("corporate"), _keywords_value("corporate")),
|
||||
SignalRule("about_url_hint", 0.18, "url", "About/company URL hint detected.", lambda signals: "about" in (signals.text_sample or "").lower() or "brand" in (signals.title or "").lower()),
|
||||
SignalRule("article_body", 0.14, "dom", "Brand story body detected.", _flag("has_article_body")),
|
||||
SignalRule("hero_block", 0.08, "layout", "Hero block detected.", _flag("has_hero_block")),
|
||||
],
|
||||
PageType.ABOUT_PAGE.value: [
|
||||
SignalRule("corporate_keywords", 0.22, "text", "About/company keywords detected.", _keyword("corporate"), _keywords_value("corporate")),
|
||||
SignalRule("contact_info", 0.08, "text", "Organization contact signal detected.", _flag("has_contact_info")),
|
||||
SignalRule("article_body", 0.12, "dom", "About body detected.", _flag("has_article_body")),
|
||||
],
|
||||
PageType.CONTACT_PAGE.value: [
|
||||
SignalRule("contact_info", 0.32, "text", "Contact information detected.", _flag("has_contact_info")),
|
||||
SignalRule("address", 0.18, "text", "Address detected.", _flag("has_address")),
|
||||
SignalRule("corporate_keywords", 0.1, "text", "Corporate keywords detected.", _keyword("corporate"), _keywords_value("corporate")),
|
||||
],
|
||||
PageType.DOCUMENTATION_PAGE.value: [
|
||||
SignalRule("docs_keywords", 0.18, "text", "Documentation keywords detected.", _keyword("knowledge"), _keywords_value("knowledge")),
|
||||
SignalRule("toc", 0.16, "dom", "Table of contents detected.", _flag("has_toc")),
|
||||
SignalRule("code_blocks", 0.16, "dom", "Code blocks detected.", _flag("has_code_blocks")),
|
||||
SignalRule("version_info", 0.08, "text", "Version information detected.", _flag("has_version_info")),
|
||||
],
|
||||
PageType.API_REFERENCE_PAGE.value: [
|
||||
SignalRule("api_endpoint", 0.3, "text", "API endpoint detected.", _flag("has_api_endpoint")),
|
||||
SignalRule("parameter_table", 0.22, "dom", "Parameter table detected.", _flag("has_parameter_table")),
|
||||
SignalRule("code_blocks", 0.1, "dom", "Code blocks detected.", _flag("has_code_blocks")),
|
||||
SignalRule("docs_keywords", 0.12, "text", "API/docs keywords detected.", _keyword("knowledge"), _keywords_value("knowledge")),
|
||||
],
|
||||
PageType.WIKI_PAGE.value: [
|
||||
SignalRule("wiki_schema", 0.28, "structured_data", "Wiki/DefinedTerm structured data detected.", _has_schema("DefinedTerm", "WebPage"), _schema_value),
|
||||
SignalRule("toc", 0.12, "dom", "Reference table of contents detected.", _flag("has_toc")),
|
||||
SignalRule("definition_terms", 0.14, "text", "Definition/reference keywords detected.", lambda signals: _contains_terms(signals, ("definition", "wiki", "glossary", "reference"))),
|
||||
],
|
||||
PageType.DATASET_PAGE.value: [
|
||||
SignalRule("schema_dataset", 0.42, "structured_data", "Dataset structured data detected.", _has_schema("Dataset", "DataCatalog"), _schema_value),
|
||||
SignalRule("download_terms", 0.12, "text", "Dataset/download keywords detected.", lambda signals: _contains_terms(signals, ("dataset", "data catalog", "download", "csv"))),
|
||||
SignalRule("parameter_table", 0.08, "dom", "Dataset metadata table detected.", _flag("has_parameter_table")),
|
||||
],
|
||||
PageType.RESEARCH_PAPER_PAGE.value: [
|
||||
SignalRule("schema_scholarly", 0.42, "structured_data", "Scholarly article structured data detected.", _has_schema("ScholarlyArticle", "TechArticle"), _schema_value),
|
||||
SignalRule("published_date", 0.1, "text", "Publication date detected.", _flag("has_published_date")),
|
||||
SignalRule("citation_terms", 0.16, "text", "Citation/references keywords detected.", lambda signals: _contains_terms(signals, ("abstract", "citation", "references", "doi"))),
|
||||
],
|
||||
PageType.JOB_POSTING_PAGE.value: [
|
||||
SignalRule("career_keywords", 0.26, "text", "Career/job keywords detected.", _flag("has_career_terms")),
|
||||
SignalRule("apply_intent", 0.12, "text", "Apply intent detected.", lambda signals: "apply" in (signals.text_sample or "").lower() or "지원" in (signals.text_sample or "")),
|
||||
SignalRule("address", 0.06, "text", "Location/address signal detected.", _flag("has_address")),
|
||||
],
|
||||
PageType.COURSE_DETAIL_PAGE.value: [
|
||||
SignalRule("schema_course", 0.42, "structured_data", "Course structured data detected.", _has_schema("Course"), _schema_value),
|
||||
SignalRule("course_terms", 0.18, "text", "Course/curriculum keywords detected.", lambda signals: _contains_terms(signals, ("course", "lesson", "curriculum", "instructor", "syllabus"))),
|
||||
],
|
||||
PageType.VIDEO_PAGE.value: [
|
||||
SignalRule("schema_video", 0.42, "structured_data", "VideoObject structured data detected.", _has_schema("VideoObject"), _schema_value),
|
||||
SignalRule("media_player", 0.2, "layout", "Video/audio player detected.", _flag("has_media_player_area")),
|
||||
SignalRule("video_terms", 0.1, "text", "Video/watch keywords detected.", lambda signals: _contains_terms(signals, ("video", "watch", "episode", "duration"))),
|
||||
],
|
||||
PageType.LOCAL_BUSINESS_PAGE.value: [
|
||||
SignalRule("schema_local_business", 0.42, "structured_data", "Local business/place structured data detected.", _has_schema("LocalBusiness", "Place", "Restaurant"), _schema_value),
|
||||
SignalRule("address", 0.18, "text", "Address detected.", _flag("has_address")),
|
||||
SignalRule("contact_info", 0.12, "text", "Contact info detected.", _flag("has_contact_info")),
|
||||
SignalRule("map_area", 0.1, "layout", "Map area detected.", _flag("has_map_area")),
|
||||
],
|
||||
PageType.REAL_ESTATE_LISTING_PAGE.value: [
|
||||
SignalRule("schema_real_estate", 0.38, "structured_data", "Real estate structured data detected.", _has_schema("RealEstateListing", "Residence", "Apartment"), _schema_value),
|
||||
SignalRule("property_terms", 0.18, "text", "Property listing keywords detected.", lambda signals: _contains_terms(signals, ("bedroom", "bathroom", "sqft", "property", "real estate"))),
|
||||
SignalRule("price", 0.12, "text", "Property price detected.", _flag("has_price")),
|
||||
SignalRule("address", 0.08, "text", "Property address detected.", _flag("has_address")),
|
||||
],
|
||||
PageType.PROFILE_PAGE.value: [
|
||||
SignalRule("schema_person", 0.34, "structured_data", "Person/Profile structured data detected.", _has_schema("Person", "ProfilePage"), _schema_value),
|
||||
SignalRule("profile_links", 0.12, "link_graph", "Profile link pattern detected.", _count_at_least("profile_link_count", 1), _value("profile_link_count")),
|
||||
SignalRule("author", 0.1, "text", "Author/person signal detected.", _flag("has_author")),
|
||||
],
|
||||
PageType.PRICING_PAGE.value: [
|
||||
SignalRule("pricing_table", 0.3, "layout", "Pricing table detected.", _flag("has_pricing_table")),
|
||||
SignalRule("price", 0.16, "text", "Price detected.", _flag("has_price")),
|
||||
SignalRule("comparison_table", 0.12, "layout", "Comparison table detected.", _flag("has_comparison_table")),
|
||||
SignalRule("commerce_keywords", 0.1, "text", "Commerce keywords detected.", _keyword("commerce"), _keywords_value("commerce")),
|
||||
],
|
||||
PageType.LOGIN_PAGE.value: [
|
||||
SignalRule("password_field", 0.4, "form", "Password field detected.", _flag("has_password_field")),
|
||||
SignalRule("login_form", 0.25, "form", "Login form detected.", _flag("has_login_form")),
|
||||
SignalRule("protected_url_hint", 0.08, "url", "Protected URL hint detected.", _url_hint("protected"), _url_hint_value),
|
||||
],
|
||||
PageType.CHECKOUT_PAGE.value: [
|
||||
SignalRule("protected_url_hint", 0.25, "url", "Checkout/cart URL hint detected.", _url_hint("protected"), _url_hint_value),
|
||||
SignalRule("buy_button", 0.14, "dom", "Purchase button detected.", _flag("has_buy_button")),
|
||||
SignalRule("commerce_keywords", 0.12, "text", "Checkout commerce keywords detected.", _keyword("commerce"), _keywords_value("commerce")),
|
||||
],
|
||||
PageType.PAYMENT_PAGE.value: [
|
||||
SignalRule("payment_fields", 0.42, "form", "Payment fields detected.", _flag("has_payment_fields")),
|
||||
SignalRule("protected_keywords", 0.16, "text", "Payment/protected keywords detected.", _keyword("protected"), _keywords_value("protected")),
|
||||
SignalRule("protected_url_hint", 0.1, "url", "Payment URL hint detected.", _url_hint("protected"), _url_hint_value),
|
||||
],
|
||||
PageType.TERMS_PAGE.value: [
|
||||
SignalRule("policy_terms", 0.32, "text", "Policy/terms terms detected.", _flag("has_policy_terms")),
|
||||
SignalRule("corporate_keywords", 0.08, "text", "Corporate legal keywords detected.", _keyword("corporate"), _keywords_value("corporate")),
|
||||
],
|
||||
PageType.PRIVACY_POLICY_PAGE.value: [
|
||||
SignalRule("privacy_terms", 0.36, "text", "Privacy terms detected.", _flag("has_privacy_terms")),
|
||||
SignalRule("policy_terms", 0.14, "text", "Policy terms detected.", _flag("has_policy_terms")),
|
||||
],
|
||||
PageType.SITEMAP_PAGE.value: [
|
||||
SignalRule("sitemap_resource", 0.45, "resource", "Sitemap resource detected.", _flag("has_sitemap_resource")),
|
||||
SignalRule("xml_resource", 0.12, "resource", "XML resource detected.", _flag("has_xml_resource")),
|
||||
SignalRule("system_url_hint", 0.12, "url", "System URL hint detected.", _url_hint("system"), _url_hint_value),
|
||||
],
|
||||
PageType.RSS_FEED_PAGE.value: [
|
||||
SignalRule("feed_resource", 0.45, "resource", "RSS/Atom feed detected.", _flag("has_feed_resource")),
|
||||
SignalRule("xml_resource", 0.1, "resource", "XML feed resource detected.", _flag("has_xml_resource")),
|
||||
],
|
||||
PageType.ERROR_PAGE.value: [
|
||||
SignalRule("error_status", 0.38, "http", "Error status detected.", _flag("has_error_status")),
|
||||
],
|
||||
PageType.NOT_FOUND_PAGE.value: [
|
||||
SignalRule("not_found", 0.5, "http", "404/not found signal detected.", _flag("has_not_found")),
|
||||
SignalRule("error_status", 0.12, "http", "Error status supports not found page.", _flag("has_error_status")),
|
||||
],
|
||||
PageType.ACCESS_DENIED_PAGE.value: [
|
||||
SignalRule("access_denied", 0.42, "protected", "Access denied text detected.", _flag("has_access_denied")),
|
||||
SignalRule("error_status", 0.1, "http", "Error status supports access denied.", _flag("has_error_status")),
|
||||
],
|
||||
PageType.CAPTCHA_PAGE.value: [
|
||||
SignalRule("captcha", 0.45, "protected", "Captcha detected.", _flag("has_captcha")),
|
||||
SignalRule("protected_keywords", 0.1, "text", "Protected keywords detected.", _keyword("protected"), _keywords_value("protected")),
|
||||
],
|
||||
}
|
||||
@@ -0,0 +1,476 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import asdict, dataclass, field
|
||||
from enum import StrEnum
|
||||
from typing import Any
|
||||
|
||||
|
||||
class PageDomain(StrEnum):
|
||||
COMMERCE = "Commerce"
|
||||
EDITORIAL = "Editorial"
|
||||
COMMUNITY = "Community"
|
||||
KNOWLEDGE = "Knowledge"
|
||||
CORPORATE = "Corporate"
|
||||
LOCAL = "Local"
|
||||
EDUCATION = "Education"
|
||||
JOBS = "Jobs"
|
||||
MEDIA = "Media"
|
||||
SOFTWARE = "Software"
|
||||
FINANCE = "Finance"
|
||||
GOVERNMENT = "Government"
|
||||
HEALTHCARE = "Healthcare"
|
||||
TRANSACTION = "Transaction"
|
||||
SYSTEM = "System"
|
||||
UNKNOWN = "Unknown"
|
||||
|
||||
|
||||
class PageArchetype(StrEnum):
|
||||
HOME = "Home"
|
||||
LANDING = "Landing"
|
||||
DETAIL = "Detail"
|
||||
LISTING = "Listing"
|
||||
COLLECTION = "Collection"
|
||||
SEARCH_RESULT = "SearchResult"
|
||||
PROFILE = "Profile"
|
||||
ARTICLE = "Article"
|
||||
THREAD = "Thread"
|
||||
FORM = "Form"
|
||||
TRANSACTION = "Transaction"
|
||||
DASHBOARD = "Dashboard"
|
||||
DOCUMENT = "Document"
|
||||
MEDIA = "Media"
|
||||
ERROR = "Error"
|
||||
SYSTEM_RESOURCE = "SystemResource"
|
||||
UNKNOWN = "Unknown"
|
||||
|
||||
|
||||
class PageType(StrEnum):
|
||||
PRODUCT_PAGE = "ProductPage"
|
||||
CATEGORY_PAGE = "CategoryPage"
|
||||
SEARCH_PAGE = "SearchPage"
|
||||
BOARD_PAGE = "BoardPage"
|
||||
NOTICE_PAGE = "NoticePage"
|
||||
BRAND_STORY_PAGE = "BrandStoryPage"
|
||||
ABOUT_PAGE = "AboutPage"
|
||||
CONTACT_PAGE = "ContactPage"
|
||||
PROMOTION_PAGE = "PromotionPage"
|
||||
REVIEW_PAGE = "ReviewPage"
|
||||
UNKNOWN_PAGE = "UnknownPage"
|
||||
PRODUCT_DETAIL_PAGE = "ProductDetailPage"
|
||||
CATEGORY_LISTING_PAGE = "CategoryListingPage"
|
||||
SEARCH_RESULTS_PAGE = "SearchResultsPage"
|
||||
FORUM_BOARD_PAGE = "ForumBoardPage"
|
||||
FORUM_THREAD_PAGE = "ForumThreadPage"
|
||||
PUBLIC_NOTICE_PAGE = "PublicNoticePage"
|
||||
CAMPAIGN_LANDING_PAGE = "CampaignLandingPage"
|
||||
ARTICLE_PAGE = "ArticlePage"
|
||||
NEWS_ARTICLE_PAGE = "NewsArticlePage"
|
||||
BLOG_POST_PAGE = "BlogPostPage"
|
||||
FAQ_PAGE = "FAQPage"
|
||||
QA_PAGE = "QAPage"
|
||||
PROFILE_PAGE = "ProfilePage"
|
||||
DOCUMENTATION_PAGE = "DocumentationPage"
|
||||
API_REFERENCE_PAGE = "APIReferencePage"
|
||||
WIKI_PAGE = "WikiPage"
|
||||
DATASET_PAGE = "DatasetPage"
|
||||
RESEARCH_PAPER_PAGE = "ResearchPaperPage"
|
||||
JOB_POSTING_PAGE = "JobPostingPage"
|
||||
COURSE_DETAIL_PAGE = "CourseDetailPage"
|
||||
VIDEO_PAGE = "VideoPage"
|
||||
LOCAL_BUSINESS_PAGE = "LocalBusinessPage"
|
||||
REAL_ESTATE_LISTING_PAGE = "RealEstateListingPage"
|
||||
PRICING_PAGE = "PricingPage"
|
||||
LOGIN_PAGE = "LoginPage"
|
||||
CHECKOUT_PAGE = "CheckoutPage"
|
||||
PAYMENT_PAGE = "PaymentPage"
|
||||
TERMS_PAGE = "TermsPage"
|
||||
PRIVACY_POLICY_PAGE = "PrivacyPolicyPage"
|
||||
SITEMAP_PAGE = "SitemapPage"
|
||||
RSS_FEED_PAGE = "RSSFeedPage"
|
||||
ERROR_PAGE = "ErrorPage"
|
||||
NOT_FOUND_PAGE = "NotFoundPage"
|
||||
ACCESS_DENIED_PAGE = "AccessDeniedPage"
|
||||
CAPTCHA_PAGE = "CaptchaPage"
|
||||
|
||||
|
||||
class EntityType(StrEnum):
|
||||
PRODUCT = "Product"
|
||||
SERVICE = "Service"
|
||||
ARTICLE = "Article"
|
||||
NEWS_ARTICLE = "NewsArticle"
|
||||
PERSON = "Person"
|
||||
ORGANIZATION = "Organization"
|
||||
PLACE = "Place"
|
||||
EVENT = "Event"
|
||||
JOB_POSTING = "JobPosting"
|
||||
COURSE = "Course"
|
||||
QUESTION = "Question"
|
||||
ANSWER = "Answer"
|
||||
REVIEW = "Review"
|
||||
DATASET = "Dataset"
|
||||
SOFTWARE_APPLICATION = "SoftwareApplication"
|
||||
MEDIA_OBJECT = "MediaObject"
|
||||
RECIPE = "Recipe"
|
||||
REAL_ESTATE_PROPERTY = "RealEstateProperty"
|
||||
MEDICAL_CONDITION = "MedicalCondition"
|
||||
LEGAL_DOCUMENT = "LegalDocument"
|
||||
FINANCIAL_PRODUCT = "FinancialProduct"
|
||||
UNKNOWN_ENTITY = "UnknownEntity"
|
||||
|
||||
|
||||
class ActionIntent(StrEnum):
|
||||
READ = "Read"
|
||||
BUY = "Buy"
|
||||
SUBSCRIBE = "Subscribe"
|
||||
RESERVE = "Reserve"
|
||||
BOOK = "Book"
|
||||
APPLY = "Apply"
|
||||
DOWNLOAD = "Download"
|
||||
WATCH = "Watch"
|
||||
LISTEN = "Listen"
|
||||
SEARCH = "Search"
|
||||
COMPARE = "Compare"
|
||||
FILTER = "Filter"
|
||||
ASK = "Ask"
|
||||
ANSWER = "Answer"
|
||||
COMMENT = "Comment"
|
||||
REVIEW = "Review"
|
||||
LOGIN = "Login"
|
||||
REGISTER = "Register"
|
||||
PAY = "Pay"
|
||||
CONTACT = "Contact"
|
||||
NAVIGATE = "Navigate"
|
||||
LEARN = "Learn"
|
||||
VERIFY = "Verify"
|
||||
CONFIGURE = "Configure"
|
||||
MANAGE = "Manage"
|
||||
|
||||
|
||||
class GraphRole(StrEnum):
|
||||
ENTITY_ANCHOR = "EntityAnchor"
|
||||
RELATION_HUB = "RelationHub"
|
||||
NAVIGATION_HUB = "NavigationHub"
|
||||
COLLECTION_HUB = "CollectionHub"
|
||||
SEARCH_HUB = "SearchHub"
|
||||
TRANSACTION_ONLY = "TransactionOnly"
|
||||
POLICY_SOURCE = "PolicySource"
|
||||
CLAIM_SOURCE = "ClaimSource"
|
||||
PROFILE_ANCHOR = "ProfileAnchor"
|
||||
MEDIA_ANCHOR = "MediaAnchor"
|
||||
REFERENCE_SOURCE = "ReferenceSource"
|
||||
SYSTEM_RESOURCE = "SystemResource"
|
||||
NOISE_PAGE = "NoisePage"
|
||||
UNKNOWN_PATTERN = "UnknownPattern"
|
||||
|
||||
|
||||
class AnalyzeStrategy(StrEnum):
|
||||
ANALYZE_FULL = "AnalyzeFull"
|
||||
ANALYZE_STRUCTURE_ONLY = "AnalyzeStructureOnly"
|
||||
ANALYZE_ENTITY_ONLY = "AnalyzeEntityOnly"
|
||||
ANALYZE_RELATIONS_ONLY = "AnalyzeRelationsOnly"
|
||||
ANALYZE_METADATA_ONLY = "AnalyzeMetadataOnly"
|
||||
ANALYZE_DOCUMENT_ONLY = "AnalyzeDocumentOnly"
|
||||
ANALYZE_DISCOVERY_ONLY = "AnalyzeDiscoveryOnly"
|
||||
SKIP_PROTECTED = "SkipProtected"
|
||||
SKIP_NOISE = "SkipNoise"
|
||||
|
||||
|
||||
class LLMPolicy(StrEnum):
|
||||
LLM_FULL = "LLMFull"
|
||||
LLM_LIGHT = "LLMLight"
|
||||
LLM_FOR_AMBIGUITY_ONLY = "LLMForAmbiguityOnly"
|
||||
RULE_ONLY = "RuleOnly"
|
||||
NO_LLM = "NoLLM"
|
||||
SKIP = "Skip"
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class EvidenceItem:
|
||||
key: str
|
||||
value: str | int | float | bool | None
|
||||
weight: float
|
||||
source: str
|
||||
message: str
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
return asdict(self)
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class PageClassificationResult:
|
||||
url: str
|
||||
primary_page_type: str
|
||||
secondary_page_types: list[str] = field(default_factory=list)
|
||||
domain: str = PageDomain.UNKNOWN.value
|
||||
archetype: str = PageArchetype.UNKNOWN.value
|
||||
main_entity_type: str | None = None
|
||||
action_intents: list[str] = field(default_factory=list)
|
||||
graph_roles: list[str] = field(default_factory=lambda: [GraphRole.UNKNOWN_PATTERN.value])
|
||||
confidence: float = 0.0
|
||||
alternatives: list[tuple[str, float]] = field(default_factory=list)
|
||||
evidence: list[EvidenceItem] = field(default_factory=list)
|
||||
should_analyze: bool = False
|
||||
analyze_strategy: str = AnalyzeStrategy.ANALYZE_METADATA_ONLY.value
|
||||
llm_policy: str = LLMPolicy.NO_LLM.value
|
||||
is_protected: bool = False
|
||||
is_noise: bool = False
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
payload = asdict(self)
|
||||
payload["evidence"] = [item.to_dict() for item in self.evidence]
|
||||
payload["legacy_page_type"] = get_legacy_page_type(self)
|
||||
return payload
|
||||
|
||||
|
||||
LEGACY_PAGE_TYPES = {
|
||||
PageType.PRODUCT_PAGE.value,
|
||||
PageType.CATEGORY_PAGE.value,
|
||||
PageType.SEARCH_PAGE.value,
|
||||
PageType.BOARD_PAGE.value,
|
||||
PageType.NOTICE_PAGE.value,
|
||||
PageType.BRAND_STORY_PAGE.value,
|
||||
PageType.PROMOTION_PAGE.value,
|
||||
PageType.REVIEW_PAGE.value,
|
||||
PageType.UNKNOWN_PAGE.value,
|
||||
"EventPage",
|
||||
}
|
||||
|
||||
|
||||
LEGACY_ALIASES: dict[str, str] = {
|
||||
"product": PageType.PRODUCT_PAGE.value,
|
||||
"productpage": PageType.PRODUCT_PAGE.value,
|
||||
"productdetailpage": PageType.PRODUCT_PAGE.value,
|
||||
"brand": PageType.BRAND_STORY_PAGE.value,
|
||||
"brandpage": PageType.BRAND_STORY_PAGE.value,
|
||||
"brandstorypage": PageType.BRAND_STORY_PAGE.value,
|
||||
"about": PageType.BRAND_STORY_PAGE.value,
|
||||
"aboutpage": "AboutPage",
|
||||
"contact": "ContactPage",
|
||||
"contactpage": "ContactPage",
|
||||
"review": PageType.REVIEW_PAGE.value,
|
||||
"reviewpage": PageType.REVIEW_PAGE.value,
|
||||
"productreviewpage": PageType.REVIEW_PAGE.value,
|
||||
"listing": PageType.CATEGORY_PAGE.value,
|
||||
"listingpage": PageType.CATEGORY_PAGE.value,
|
||||
"category": PageType.CATEGORY_PAGE.value,
|
||||
"categorypage": PageType.CATEGORY_PAGE.value,
|
||||
"categorylistingpage": PageType.CATEGORY_PAGE.value,
|
||||
"productlistingpage": PageType.CATEGORY_PAGE.value,
|
||||
"community": PageType.BOARD_PAGE.value,
|
||||
"communitypage": PageType.BOARD_PAGE.value,
|
||||
"board": PageType.BOARD_PAGE.value,
|
||||
"boardpage": PageType.BOARD_PAGE.value,
|
||||
"forumboardpage": PageType.BOARD_PAGE.value,
|
||||
"forumthreadpage": PageType.BOARD_PAGE.value,
|
||||
"search": PageType.SEARCH_PAGE.value,
|
||||
"searchpage": PageType.SEARCH_PAGE.value,
|
||||
"searchresultspage": PageType.SEARCH_PAGE.value,
|
||||
"notice": PageType.NOTICE_PAGE.value,
|
||||
"noticepage": PageType.NOTICE_PAGE.value,
|
||||
"publicnoticepage": PageType.NOTICE_PAGE.value,
|
||||
"promotion": PageType.PROMOTION_PAGE.value,
|
||||
"promotionpage": PageType.PROMOTION_PAGE.value,
|
||||
"campaignlandingpage": PageType.PROMOTION_PAGE.value,
|
||||
"event": "EventPage",
|
||||
"eventpage": "EventPage",
|
||||
"unknown": PageType.UNKNOWN_PAGE.value,
|
||||
"unknownpage": PageType.UNKNOWN_PAGE.value,
|
||||
"notfoundpage": PageType.NOT_FOUND_PAGE.value,
|
||||
"404": PageType.NOT_FOUND_PAGE.value,
|
||||
}
|
||||
|
||||
|
||||
LEGACY_TO_SEMANTIC_PAGE_TYPE: dict[str, str] = {
|
||||
PageType.PRODUCT_PAGE.value: PageType.PRODUCT_DETAIL_PAGE.value,
|
||||
PageType.CATEGORY_PAGE.value: PageType.CATEGORY_LISTING_PAGE.value,
|
||||
PageType.SEARCH_PAGE.value: PageType.SEARCH_RESULTS_PAGE.value,
|
||||
PageType.BOARD_PAGE.value: PageType.FORUM_BOARD_PAGE.value,
|
||||
PageType.NOTICE_PAGE.value: PageType.PUBLIC_NOTICE_PAGE.value,
|
||||
PageType.BRAND_STORY_PAGE.value: PageType.BRAND_STORY_PAGE.value,
|
||||
PageType.ABOUT_PAGE.value: PageType.ABOUT_PAGE.value,
|
||||
PageType.CONTACT_PAGE.value: PageType.CONTACT_PAGE.value,
|
||||
PageType.PROMOTION_PAGE.value: PageType.PROMOTION_PAGE.value,
|
||||
PageType.REVIEW_PAGE.value: PageType.REVIEW_PAGE.value,
|
||||
PageType.UNKNOWN_PAGE.value: PageType.UNKNOWN_PAGE.value,
|
||||
"EventPage": "EventPage",
|
||||
}
|
||||
|
||||
|
||||
SEMANTIC_TO_LEGACY_PAGE_TYPE: dict[str, str] = {
|
||||
semantic: legacy for legacy, semantic in LEGACY_TO_SEMANTIC_PAGE_TYPE.items()
|
||||
}
|
||||
SEMANTIC_TO_LEGACY_PAGE_TYPE.update(
|
||||
{
|
||||
PageType.PRODUCT_DETAIL_PAGE.value: PageType.PRODUCT_PAGE.value,
|
||||
PageType.CATEGORY_LISTING_PAGE.value: PageType.CATEGORY_PAGE.value,
|
||||
PageType.SEARCH_RESULTS_PAGE.value: PageType.SEARCH_PAGE.value,
|
||||
PageType.FORUM_BOARD_PAGE.value: PageType.BOARD_PAGE.value,
|
||||
PageType.FORUM_THREAD_PAGE.value: PageType.BOARD_PAGE.value,
|
||||
PageType.PUBLIC_NOTICE_PAGE.value: PageType.NOTICE_PAGE.value,
|
||||
PageType.CAMPAIGN_LANDING_PAGE.value: PageType.PROMOTION_PAGE.value,
|
||||
PageType.ABOUT_PAGE.value: PageType.BRAND_STORY_PAGE.value,
|
||||
PageType.CONTACT_PAGE.value: PageType.BRAND_STORY_PAGE.value,
|
||||
"ListingPage": PageType.CATEGORY_PAGE.value,
|
||||
"CommunityPage": PageType.BOARD_PAGE.value,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
PAGE_TYPE_PROFILES: dict[str, dict[str, Any]] = {
|
||||
PageType.PRODUCT_DETAIL_PAGE.value: {
|
||||
"domain": PageDomain.COMMERCE.value,
|
||||
"archetype": PageArchetype.DETAIL.value,
|
||||
"main_entity_type": EntityType.PRODUCT.value,
|
||||
"action_intents": [ActionIntent.BUY.value, ActionIntent.REVIEW.value],
|
||||
"graph_roles": [GraphRole.ENTITY_ANCHOR.value],
|
||||
"analyze_strategy": AnalyzeStrategy.ANALYZE_FULL.value,
|
||||
"llm_policy": LLMPolicy.LLM_LIGHT.value,
|
||||
"should_analyze": True,
|
||||
},
|
||||
PageType.CATEGORY_LISTING_PAGE.value: {
|
||||
"domain": PageDomain.COMMERCE.value,
|
||||
"archetype": PageArchetype.LISTING.value,
|
||||
"main_entity_type": EntityType.PRODUCT.value,
|
||||
"action_intents": [ActionIntent.FILTER.value, ActionIntent.NAVIGATE.value],
|
||||
"graph_roles": [GraphRole.COLLECTION_HUB.value, GraphRole.RELATION_HUB.value],
|
||||
"analyze_strategy": AnalyzeStrategy.ANALYZE_RELATIONS_ONLY.value,
|
||||
"llm_policy": LLMPolicy.LLM_FOR_AMBIGUITY_ONLY.value,
|
||||
"should_analyze": True,
|
||||
},
|
||||
PageType.SEARCH_RESULTS_PAGE.value: {
|
||||
"domain": PageDomain.UNKNOWN.value,
|
||||
"archetype": PageArchetype.SEARCH_RESULT.value,
|
||||
"main_entity_type": EntityType.UNKNOWN_ENTITY.value,
|
||||
"action_intents": [ActionIntent.SEARCH.value, ActionIntent.NAVIGATE.value],
|
||||
"graph_roles": [GraphRole.SEARCH_HUB.value],
|
||||
"analyze_strategy": AnalyzeStrategy.ANALYZE_DISCOVERY_ONLY.value,
|
||||
"llm_policy": LLMPolicy.LLM_FOR_AMBIGUITY_ONLY.value,
|
||||
"should_analyze": True,
|
||||
},
|
||||
PageType.FORUM_BOARD_PAGE.value: {
|
||||
"domain": PageDomain.COMMUNITY.value,
|
||||
"archetype": PageArchetype.LISTING.value,
|
||||
"main_entity_type": EntityType.ARTICLE.value,
|
||||
"action_intents": [ActionIntent.NAVIGATE.value, ActionIntent.READ.value],
|
||||
"graph_roles": [GraphRole.RELATION_HUB.value, GraphRole.COLLECTION_HUB.value],
|
||||
"analyze_strategy": AnalyzeStrategy.ANALYZE_RELATIONS_ONLY.value,
|
||||
"llm_policy": LLMPolicy.LLM_FOR_AMBIGUITY_ONLY.value,
|
||||
"should_analyze": True,
|
||||
},
|
||||
PageType.PUBLIC_NOTICE_PAGE.value: {
|
||||
"domain": PageDomain.GOVERNMENT.value,
|
||||
"archetype": PageArchetype.DOCUMENT.value,
|
||||
"main_entity_type": EntityType.ARTICLE.value,
|
||||
"action_intents": [ActionIntent.READ.value],
|
||||
"graph_roles": [GraphRole.CLAIM_SOURCE.value],
|
||||
"analyze_strategy": AnalyzeStrategy.ANALYZE_FULL.value,
|
||||
"llm_policy": LLMPolicy.LLM_LIGHT.value,
|
||||
"should_analyze": True,
|
||||
},
|
||||
PageType.BRAND_STORY_PAGE.value: {
|
||||
"domain": PageDomain.CORPORATE.value,
|
||||
"archetype": PageArchetype.ARTICLE.value,
|
||||
"main_entity_type": EntityType.ORGANIZATION.value,
|
||||
"action_intents": [ActionIntent.READ.value, ActionIntent.LEARN.value],
|
||||
"graph_roles": [GraphRole.ENTITY_ANCHOR.value, GraphRole.CLAIM_SOURCE.value],
|
||||
"analyze_strategy": AnalyzeStrategy.ANALYZE_FULL.value,
|
||||
"llm_policy": LLMPolicy.LLM_FULL.value,
|
||||
"should_analyze": True,
|
||||
},
|
||||
PageType.PROMOTION_PAGE.value: {
|
||||
"domain": PageDomain.COMMERCE.value,
|
||||
"archetype": PageArchetype.LANDING.value,
|
||||
"main_entity_type": EntityType.EVENT.value,
|
||||
"action_intents": [ActionIntent.BUY.value, ActionIntent.NAVIGATE.value],
|
||||
"graph_roles": [GraphRole.CLAIM_SOURCE.value],
|
||||
"analyze_strategy": AnalyzeStrategy.ANALYZE_FULL.value,
|
||||
"llm_policy": LLMPolicy.LLM_LIGHT.value,
|
||||
"should_analyze": True,
|
||||
},
|
||||
PageType.REVIEW_PAGE.value: {
|
||||
"domain": PageDomain.COMMUNITY.value,
|
||||
"archetype": PageArchetype.ARTICLE.value,
|
||||
"main_entity_type": EntityType.REVIEW.value,
|
||||
"action_intents": [ActionIntent.READ.value, ActionIntent.REVIEW.value],
|
||||
"graph_roles": [GraphRole.CLAIM_SOURCE.value],
|
||||
"analyze_strategy": AnalyzeStrategy.ANALYZE_FULL.value,
|
||||
"llm_policy": LLMPolicy.LLM_LIGHT.value,
|
||||
"should_analyze": True,
|
||||
},
|
||||
PageType.UNKNOWN_PAGE.value: {
|
||||
"domain": PageDomain.UNKNOWN.value,
|
||||
"archetype": PageArchetype.UNKNOWN.value,
|
||||
"main_entity_type": EntityType.UNKNOWN_ENTITY.value,
|
||||
"action_intents": [],
|
||||
"graph_roles": [GraphRole.UNKNOWN_PATTERN.value],
|
||||
"analyze_strategy": AnalyzeStrategy.ANALYZE_METADATA_ONLY.value,
|
||||
"llm_policy": LLMPolicy.NO_LLM.value,
|
||||
"should_analyze": False,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def normalize_page_type(value: object | None) -> str:
|
||||
"""Return the legacy-compatible page type string for existing callers."""
|
||||
|
||||
if isinstance(value, PageClassificationResult):
|
||||
return get_legacy_page_type(value)
|
||||
clean = str(value or "").strip()
|
||||
if not clean:
|
||||
return PageType.UNKNOWN_PAGE.value
|
||||
alias = LEGACY_ALIASES.get(clean.lower())
|
||||
if alias:
|
||||
return alias
|
||||
return SEMANTIC_TO_LEGACY_PAGE_TYPE.get(clean, clean)
|
||||
|
||||
|
||||
def normalize_semantic_page_type(value: object | None) -> str:
|
||||
if isinstance(value, PageClassificationResult):
|
||||
return value.primary_page_type
|
||||
legacy = normalize_page_type(value)
|
||||
return LEGACY_TO_SEMANTIC_PAGE_TYPE.get(legacy, str(value or legacy).strip() or PageType.UNKNOWN_PAGE.value)
|
||||
|
||||
|
||||
def get_legacy_page_type(value: object | None) -> str:
|
||||
if isinstance(value, PageClassificationResult):
|
||||
return SEMANTIC_TO_LEGACY_PAGE_TYPE.get(value.primary_page_type, value.primary_page_type)
|
||||
return normalize_page_type(value)
|
||||
|
||||
|
||||
def build_classification_result_from_legacy(
|
||||
*,
|
||||
url: str,
|
||||
legacy_page_type: str,
|
||||
confidence: float = 0.55,
|
||||
source: str = "legacy_classifier",
|
||||
) -> PageClassificationResult:
|
||||
normalized_legacy = normalize_page_type(legacy_page_type)
|
||||
semantic_page_type = LEGACY_TO_SEMANTIC_PAGE_TYPE.get(normalized_legacy, normalized_legacy)
|
||||
profile = PAGE_TYPE_PROFILES.get(semantic_page_type, PAGE_TYPE_PROFILES[PageType.UNKNOWN_PAGE.value])
|
||||
evidence = [
|
||||
EvidenceItem(
|
||||
key="legacy_page_type",
|
||||
value=normalized_legacy,
|
||||
weight=confidence,
|
||||
source=source,
|
||||
message=f"Legacy classifier returned {normalized_legacy}.",
|
||||
)
|
||||
]
|
||||
return PageClassificationResult(
|
||||
url=url,
|
||||
primary_page_type=semantic_page_type,
|
||||
secondary_page_types=[] if semantic_page_type == normalized_legacy else [normalized_legacy],
|
||||
domain=str(profile["domain"]),
|
||||
archetype=str(profile["archetype"]),
|
||||
main_entity_type=profile.get("main_entity_type"),
|
||||
action_intents=list(profile.get("action_intents") or []),
|
||||
graph_roles=list(profile.get("graph_roles") or []),
|
||||
confidence=confidence,
|
||||
alternatives=[(normalized_legacy, confidence)],
|
||||
evidence=evidence,
|
||||
should_analyze=bool(profile.get("should_analyze")),
|
||||
analyze_strategy=str(profile["analyze_strategy"]),
|
||||
llm_policy=str(profile["llm_policy"]),
|
||||
is_protected=str(profile["analyze_strategy"]) == AnalyzeStrategy.SKIP_PROTECTED.value,
|
||||
is_noise=str(profile["analyze_strategy"]) == AnalyzeStrategy.SKIP_NOISE.value,
|
||||
)
|
||||
@@ -0,0 +1,347 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from collections import Counter
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
from typing import Any
|
||||
from urllib.parse import urljoin, urlparse
|
||||
|
||||
from crawler_platform.app.core.crawler.page_signal_extractor import (
|
||||
build_raw_page_snapshot,
|
||||
extract_page_signals,
|
||||
)
|
||||
from crawler_platform.app.core.crawler.page_type_taxonomy import PageClassificationResult, PageType
|
||||
|
||||
|
||||
UNKNOWN_PATTERN_VERSION = 1
|
||||
LOW_CONFIDENCE_PATTERN_THRESHOLD = 0.45
|
||||
STOPWORDS = {
|
||||
"about",
|
||||
"after",
|
||||
"again",
|
||||
"also",
|
||||
"and",
|
||||
"are",
|
||||
"but",
|
||||
"can",
|
||||
"for",
|
||||
"from",
|
||||
"has",
|
||||
"have",
|
||||
"home",
|
||||
"into",
|
||||
"more",
|
||||
"not",
|
||||
"our",
|
||||
"page",
|
||||
"that",
|
||||
"the",
|
||||
"this",
|
||||
"with",
|
||||
"your",
|
||||
}
|
||||
|
||||
|
||||
def should_store_unknown_pattern(
|
||||
result: PageClassificationResult,
|
||||
*,
|
||||
confidence_threshold: float = LOW_CONFIDENCE_PATTERN_THRESHOLD,
|
||||
) -> bool:
|
||||
if result.primary_page_type == PageType.UNKNOWN_PAGE.value:
|
||||
return True
|
||||
if 0.0 < result.confidence <= confidence_threshold:
|
||||
return True
|
||||
return any(item.key == "low_confidence" for item in result.evidence)
|
||||
|
||||
|
||||
def build_unknown_pattern_payload(
|
||||
*,
|
||||
result: PageClassificationResult,
|
||||
url: str,
|
||||
title: str | None = None,
|
||||
text: str | None = None,
|
||||
html: str | None = None,
|
||||
source_zones: list[str | dict[str, Any]] | None = None,
|
||||
collector_payload: dict[str, Any] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
snapshot = build_raw_page_snapshot(
|
||||
url=url,
|
||||
title=title,
|
||||
text=text,
|
||||
html=html,
|
||||
source_zones=source_zones,
|
||||
collector_payload=collector_payload,
|
||||
)
|
||||
signals = extract_page_signals(snapshot)
|
||||
soup = _soup_from_html(html or "")
|
||||
page_text = _normalize_text(text or signals.text_sample or _text_from_soup(soup))
|
||||
links = snapshot.links or _extract_links(soup, url)
|
||||
buttons = snapshot.buttons or _extract_buttons(soup)
|
||||
forms = snapshot.forms or _extract_forms(soup)
|
||||
link_summary = summarize_link_patterns(links, url)
|
||||
forms_summary = summarize_forms(forms)
|
||||
payload = {
|
||||
"version": UNKNOWN_PATTERN_VERSION,
|
||||
"url": url,
|
||||
"title": title or signals.title,
|
||||
"primary_page_type": result.primary_page_type,
|
||||
"confidence": result.confidence,
|
||||
"reason": "unknown_page" if result.primary_page_type == PageType.UNKNOWN_PAGE.value else "low_confidence",
|
||||
"text_sample": page_text[:1000],
|
||||
"text_fingerprint": text_fingerprint(page_text),
|
||||
"html_fingerprint": html_fingerprint(html or ""),
|
||||
"dom_fingerprint": dom_fingerprint(html or ""),
|
||||
"link_pattern_fingerprint": link_pattern_fingerprint(link_summary),
|
||||
"schema_types": sorted(signals.schema_types),
|
||||
"link_pattern_summary": link_summary,
|
||||
"button_labels": _dedupe_strings(buttons)[:20],
|
||||
"forms_summary": forms_summary,
|
||||
"top_keywords": top_keywords(page_text),
|
||||
"layout_blocks": list(signals.layout_blocks),
|
||||
"keyword_hits": dict(signals.keyword_hits),
|
||||
"alternatives": list(result.alternatives[:8]),
|
||||
"evidence": [item.to_dict() for item in result.evidence[:12]],
|
||||
}
|
||||
payload["embedding_input"] = build_unknown_embedding_input(payload)
|
||||
payload["cluster_candidate"] = build_cluster_candidate_payload(payload)
|
||||
return payload
|
||||
|
||||
|
||||
def build_unknown_embedding_input(payload: dict[str, Any]) -> str:
|
||||
parts = [
|
||||
str(payload.get("title") or ""),
|
||||
str(payload.get("text_sample") or ""),
|
||||
"schema_types: " + ", ".join(str(item) for item in payload.get("schema_types") or []),
|
||||
"layout_blocks: " + ", ".join(str(item) for item in payload.get("layout_blocks") or []),
|
||||
"buttons: " + ", ".join(str(item) for item in payload.get("button_labels") or []),
|
||||
"keywords: " + ", ".join(str(item.get("term")) for item in payload.get("top_keywords") or [] if isinstance(item, dict)),
|
||||
]
|
||||
return "\n".join(part for part in parts if part.strip())[:4000]
|
||||
|
||||
|
||||
def build_cluster_candidate_payload(payload: dict[str, Any]) -> dict[str, Any]:
|
||||
return {
|
||||
"version": UNKNOWN_PATTERN_VERSION,
|
||||
"candidate_key": stable_hash(
|
||||
{
|
||||
"dom": payload.get("dom_fingerprint"),
|
||||
"links": payload.get("link_pattern_fingerprint"),
|
||||
"schema_types": payload.get("schema_types") or [],
|
||||
"buttons": payload.get("button_labels") or [],
|
||||
}
|
||||
),
|
||||
"fingerprints": {
|
||||
"text": payload.get("text_fingerprint"),
|
||||
"html": payload.get("html_fingerprint"),
|
||||
"dom": payload.get("dom_fingerprint"),
|
||||
"links": payload.get("link_pattern_fingerprint"),
|
||||
},
|
||||
"features": {
|
||||
"schema_types": payload.get("schema_types") or [],
|
||||
"layout_blocks": payload.get("layout_blocks") or [],
|
||||
"top_keywords": payload.get("top_keywords") or [],
|
||||
"link_patterns": (payload.get("link_pattern_summary") or {}).get("top_path_patterns") or [],
|
||||
"form_count": (payload.get("forms_summary") or {}).get("form_count") or 0,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def summarize_link_patterns(links: list[dict[str, Any]], base_url: str) -> dict[str, Any]:
|
||||
base_host = urlparse(base_url).netloc.lower()
|
||||
path_patterns = Counter()
|
||||
text_labels: list[str] = []
|
||||
internal_count = 0
|
||||
external_count = 0
|
||||
for link in links:
|
||||
href = str(link.get("href") or "")
|
||||
parsed = urlparse(href)
|
||||
if not href:
|
||||
continue
|
||||
if not parsed.netloc or parsed.netloc.lower() == base_host:
|
||||
internal_count += 1
|
||||
else:
|
||||
external_count += 1
|
||||
path_patterns[_path_pattern(parsed.path)] += 1
|
||||
text = str(link.get("text") or "").strip()
|
||||
if text:
|
||||
text_labels.append(text)
|
||||
return {
|
||||
"total_count": len(links),
|
||||
"internal_count": internal_count,
|
||||
"external_count": external_count,
|
||||
"top_path_patterns": [
|
||||
{"pattern": pattern, "count": count}
|
||||
for pattern, count in path_patterns.most_common(12)
|
||||
if pattern
|
||||
],
|
||||
"sample_texts": _dedupe_strings(text_labels)[:12],
|
||||
}
|
||||
|
||||
|
||||
def summarize_forms(forms: list[dict[str, Any]]) -> dict[str, Any]:
|
||||
methods = Counter()
|
||||
actions = Counter()
|
||||
input_types = Counter()
|
||||
form_texts: list[str] = []
|
||||
for form in forms:
|
||||
method = str(form.get("method") or "get").lower()
|
||||
action = _path_pattern(urlparse(str(form.get("action") or "")).path)
|
||||
methods[method] += 1
|
||||
if action:
|
||||
actions[action] += 1
|
||||
if form.get("text"):
|
||||
form_texts.append(str(form.get("text")))
|
||||
for input_item in form.get("inputs") or []:
|
||||
if isinstance(input_item, dict):
|
||||
input_types[str(input_item.get("type") or "text").lower()] += 1
|
||||
return {
|
||||
"form_count": len(forms),
|
||||
"methods": dict(methods),
|
||||
"action_patterns": dict(actions.most_common(8)),
|
||||
"input_types": dict(input_types.most_common(12)),
|
||||
"sample_texts": _dedupe_strings(form_texts)[:8],
|
||||
}
|
||||
|
||||
|
||||
def top_keywords(text: str, *, limit: int = 16) -> list[dict[str, Any]]:
|
||||
tokens = [
|
||||
token.lower()
|
||||
for token in re.findall(r"[A-Za-z][A-Za-z0-9_-]{2,}|[가-힣]{2,}", text or "")
|
||||
if token.lower() not in STOPWORDS
|
||||
]
|
||||
return [{"term": term, "count": count} for term, count in Counter(tokens).most_common(limit)]
|
||||
|
||||
|
||||
def text_fingerprint(text: str) -> str:
|
||||
return f"sha256:{_hash_text(_normalize_text(text))}"
|
||||
|
||||
|
||||
def html_fingerprint(html: str) -> str:
|
||||
return f"sha256:{_hash_text(_normalize_html(html))}"
|
||||
|
||||
|
||||
def dom_fingerprint(html: str) -> str:
|
||||
soup = _soup_from_html(html)
|
||||
if soup is None:
|
||||
return f"sha256:{_hash_text('')}"
|
||||
tags = []
|
||||
for node in soup.find_all(True):
|
||||
classes = ".".join(str(item).lower() for item in node.get("class", [])[:3])
|
||||
node_id = "#" + str(node.get("id")).lower() if node.get("id") else ""
|
||||
tags.append(f"{node.name}{node_id}{('.' + classes) if classes else ''}")
|
||||
return f"sha256:{_hash_text('>'.join(tags[:400]))}"
|
||||
|
||||
|
||||
def link_pattern_fingerprint(link_summary: dict[str, Any]) -> str:
|
||||
return stable_hash(link_summary)
|
||||
|
||||
|
||||
def stable_hash(value: Any) -> str:
|
||||
payload = json.dumps(value, sort_keys=True, ensure_ascii=False, separators=(",", ":"))
|
||||
return f"sha256:{_hash_text(payload)}"
|
||||
|
||||
|
||||
def _hash_text(value: str) -> str:
|
||||
return hashlib.sha256(value.encode("utf-8", errors="ignore")).hexdigest()
|
||||
|
||||
|
||||
def _normalize_text(value: str | None) -> str:
|
||||
return re.sub(r"\s+", " ", str(value or "")).strip()
|
||||
|
||||
|
||||
def _normalize_html(value: str | None) -> str:
|
||||
text = re.sub(r">\s+<", "><", str(value or ""))
|
||||
text = re.sub(r"\s+", " ", text)
|
||||
return text.strip()[:200000]
|
||||
|
||||
|
||||
def _path_pattern(path: str) -> str:
|
||||
parts = [part for part in path.split("/") if part]
|
||||
normalized = []
|
||||
for part in parts[:8]:
|
||||
if part.isdigit() or re.fullmatch(r"[0-9a-fA-F-]{8,}", part):
|
||||
normalized.append("{id}")
|
||||
else:
|
||||
normalized.append(re.sub(r"\d+", "{n}", part.lower())[:60])
|
||||
return "/" + "/".join(normalized) if normalized else "/"
|
||||
|
||||
|
||||
def _soup_from_html(html: str):
|
||||
if not html:
|
||||
return None
|
||||
try:
|
||||
from bs4 import BeautifulSoup
|
||||
except ImportError:
|
||||
return None
|
||||
try:
|
||||
return BeautifulSoup(html, "html.parser")
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def _text_from_soup(soup) -> str:
|
||||
if soup is None:
|
||||
return ""
|
||||
return soup.get_text(" ", strip=True)
|
||||
|
||||
|
||||
def _extract_links(soup, base_url: str) -> list[dict[str, Any]]:
|
||||
if soup is None:
|
||||
return []
|
||||
links = []
|
||||
for tag in soup.find_all("a"):
|
||||
href = str(tag.get("href") or "").strip()
|
||||
if not href:
|
||||
continue
|
||||
links.append({"href": urljoin(base_url, href), "text": tag.get_text(" ", strip=True)})
|
||||
return links
|
||||
|
||||
|
||||
def _extract_buttons(soup) -> list[str]:
|
||||
if soup is None:
|
||||
return []
|
||||
labels = []
|
||||
for tag in soup.select("button,input[type='submit'],input[type='button'],[role='button']"):
|
||||
label = tag.get_text(" ", strip=True) or str(tag.get("value") or tag.get("aria-label") or "")
|
||||
if label.strip():
|
||||
labels.append(label.strip())
|
||||
return labels
|
||||
|
||||
|
||||
def _extract_forms(soup) -> list[dict[str, Any]]:
|
||||
if soup is None:
|
||||
return []
|
||||
forms = []
|
||||
for form in soup.find_all("form"):
|
||||
inputs = []
|
||||
for tag in form.select("input,select,textarea"):
|
||||
inputs.append(
|
||||
{
|
||||
"type": str(tag.get("type") or tag.name or ""),
|
||||
"name": str(tag.get("name") or ""),
|
||||
"placeholder": str(tag.get("placeholder") or ""),
|
||||
}
|
||||
)
|
||||
forms.append(
|
||||
{
|
||||
"action": str(form.get("action") or ""),
|
||||
"method": str(form.get("method") or ""),
|
||||
"text": form.get_text(" ", strip=True)[:500],
|
||||
"inputs": inputs,
|
||||
}
|
||||
)
|
||||
return forms
|
||||
|
||||
|
||||
def _dedupe_strings(values: list[str]) -> list[str]:
|
||||
seen = set()
|
||||
result = []
|
||||
for value in values:
|
||||
clean = _normalize_text(value)
|
||||
key = clean.lower()
|
||||
if not clean or key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
result.append(clean)
|
||||
return result
|
||||
@@ -3,11 +3,17 @@ from __future__ import annotations
|
||||
from dataclasses import dataclass
|
||||
|
||||
from crawler_platform.app.config.loader import ProjectConfig
|
||||
from crawler_platform.app.core.crawler.page_classifier import classify_page
|
||||
from crawler_platform.app.core.crawler.page_classifier import (
|
||||
classification_metadata,
|
||||
classify_page_semantic,
|
||||
get_legacy_page_type,
|
||||
should_analyze_page,
|
||||
)
|
||||
from crawler_platform.app.core.crawler.fetchers import RobotsPolicy, make_fetcher
|
||||
from crawler_platform.app.core.crawler.plugins import ParserRegistry, default_parser_registry
|
||||
from crawler_platform.app.core.database.repository import KnowledgeRepository
|
||||
from crawler_platform.app.core.extractor.base import ExtractionPageContext, Extractor
|
||||
from crawler_platform.app.core.extractor.strategy import extract_with_strategy
|
||||
from crawler_platform.app.core.extractor.validation import attach_page_context
|
||||
|
||||
|
||||
@@ -56,23 +62,33 @@ class CrawlPipeline:
|
||||
fetch_result = fetcher.fetch(url)
|
||||
parser = self.parser_registry.get(source_config.parser)
|
||||
parsed = parser.parse(fetch_result.analysis_html, fetch_result.final_url or url)
|
||||
page_type = classify_page(
|
||||
page_classification = classify_page_semantic(
|
||||
fetch_result.final_url or url,
|
||||
parsed.title or fetch_result.title,
|
||||
parsed.raw_text or parsed.text,
|
||||
fetch_result.analysis_html,
|
||||
parsed.source_zones or [],
|
||||
final_url=fetch_result.final_url,
|
||||
status_code=fetch_result.status_code,
|
||||
content_type=fetch_result.headers.get("content-type"),
|
||||
)
|
||||
page_type = get_legacy_page_type(page_classification)
|
||||
|
||||
project = self.repository.upsert_project(project_config)
|
||||
source = self.repository.get_source(project.id, source_name)
|
||||
warnings = [*fetch_result.warnings, *(parsed.extraction_warnings or [])]
|
||||
metadata = {
|
||||
**parsed.metadata,
|
||||
**classification_metadata(
|
||||
page_classification,
|
||||
title=parsed.title or fetch_result.title,
|
||||
text=parsed.raw_text or parsed.text,
|
||||
html=fetch_result.analysis_html,
|
||||
source_zones=parsed.source_zones or [],
|
||||
),
|
||||
"final_url": fetch_result.final_url,
|
||||
"crawl_status": fetch_result.crawl_status,
|
||||
"crawl_warnings": fetch_result.warnings,
|
||||
"page_type": page_type,
|
||||
"raw_text_length": len(parsed.raw_text or ""),
|
||||
"clean_text_length": len(parsed.text or ""),
|
||||
"main_content_preview": (parsed.main_content or parsed.text)[:800],
|
||||
@@ -100,6 +116,20 @@ class CrawlPipeline:
|
||||
robots_status=robots_decision.status,
|
||||
robots_reason=robots_decision.reason,
|
||||
)
|
||||
if not should_analyze_page(page_classification, None):
|
||||
return CrawlResult(
|
||||
page_id=page.id,
|
||||
claim_count=0,
|
||||
entity_count=0,
|
||||
crawl_status=fetch_result.crawl_status,
|
||||
extraction_status="skipped",
|
||||
page_type=page_type,
|
||||
clean_text_length=len(parsed.text or ""),
|
||||
raw_text_length=len(parsed.raw_text or ""),
|
||||
warnings=warnings,
|
||||
robots_status=robots_decision.status,
|
||||
robots_reason=robots_decision.reason,
|
||||
)
|
||||
|
||||
context = ExtractionPageContext(
|
||||
url=url,
|
||||
@@ -116,7 +146,7 @@ class CrawlPipeline:
|
||||
warnings=warnings,
|
||||
metadata=metadata,
|
||||
)
|
||||
bundle = self.extractor.extract_from_context(context, project_config)
|
||||
bundle = extract_with_strategy(self.extractor, context, project_config)
|
||||
bundle = attach_page_context(bundle, context)
|
||||
claims = self.repository.save_extraction_bundle(project.id, source, page, bundle, project_config)
|
||||
extraction_summary = extraction_summary_from_raw(bundle.raw_output)
|
||||
|
||||
@@ -10,12 +10,16 @@ from crawler_platform.app.core.crawler.discovery import discover_links
|
||||
from crawler_platform.app.core.crawler.fetchers import RobotsDecision, RobotsPolicy, make_fetcher
|
||||
from crawler_platform.app.core.crawler.page_classifier import (
|
||||
classify_page as classify_page_type,
|
||||
classification_metadata,
|
||||
classify_page_semantic,
|
||||
get_legacy_page_type,
|
||||
should_analyze_page as should_analyze_page_type,
|
||||
)
|
||||
from crawler_platform.app.core.crawler.plugins import ParserRegistry, default_parser_registry
|
||||
from crawler_platform.app.core.database import models
|
||||
from crawler_platform.app.core.database.repository import KnowledgeRepository
|
||||
from crawler_platform.app.core.extractor.base import ExtractionPageContext, Extractor
|
||||
from crawler_platform.app.core.extractor.strategy import extract_with_strategy
|
||||
from crawler_platform.app.core.extractor.validation import attach_page_context
|
||||
|
||||
|
||||
@@ -209,6 +213,16 @@ class SiteCrawler:
|
||||
fetch_result = fetcher.fetch(url)
|
||||
if is_failed_fetch_status(fetch_result.status_code) or fetch_result.crawl_status != "success":
|
||||
error = f"fetch failed with status {fetch_result.status_code}; crawl_status={fetch_result.crawl_status}"
|
||||
page_classification = classify_page_semantic(
|
||||
fetch_result.final_url or url,
|
||||
title=fetch_result.title,
|
||||
text="",
|
||||
html=fetch_result.analysis_html,
|
||||
final_url=fetch_result.final_url,
|
||||
status_code=fetch_result.status_code,
|
||||
content_type=fetch_result.headers.get("content-type"),
|
||||
)
|
||||
page_type = get_legacy_page_type(page_classification)
|
||||
page = self.repository.upsert_page(
|
||||
project_id=source.project_id,
|
||||
source_id=source.id,
|
||||
@@ -217,6 +231,11 @@ class SiteCrawler:
|
||||
status_code=fetch_result.status_code,
|
||||
cleaned_text="",
|
||||
metadata={
|
||||
**classification_metadata(
|
||||
page_classification,
|
||||
title=fetch_result.title,
|
||||
html=fetch_result.analysis_html,
|
||||
),
|
||||
"final_url": fetch_result.final_url,
|
||||
"crawl_status": fetch_result.crawl_status,
|
||||
"crawl_warnings": fetch_result.warnings,
|
||||
@@ -232,7 +251,7 @@ class SiteCrawler:
|
||||
url=url,
|
||||
depth=depth,
|
||||
status=fetch_result.crawl_status,
|
||||
page_type="unknown",
|
||||
page_type=page_type,
|
||||
page_id=page.id,
|
||||
crawl_status=fetch_result.crawl_status,
|
||||
robots_status=robots_decision.status,
|
||||
@@ -246,13 +265,17 @@ class SiteCrawler:
|
||||
return
|
||||
|
||||
parsed = parser.parse(fetch_result.analysis_html, fetch_result.final_url or url)
|
||||
page_type = classify_page(
|
||||
page_classification = classify_page_semantic(
|
||||
fetch_result.final_url or url,
|
||||
parsed.title or fetch_result.title,
|
||||
parsed.raw_text or parsed.text,
|
||||
fetch_result.analysis_html,
|
||||
parsed.source_zones or [],
|
||||
final_url=fetch_result.final_url,
|
||||
status_code=fetch_result.status_code,
|
||||
content_type=fetch_result.headers.get("content-type"),
|
||||
)
|
||||
page_type = get_legacy_page_type(page_classification)
|
||||
discovered_count = self._enqueue_links(
|
||||
html=fetch_result.analysis_html,
|
||||
base_url=fetch_result.final_url or url,
|
||||
@@ -268,10 +291,16 @@ class SiteCrawler:
|
||||
warnings = [*fetch_result.warnings, *(parsed.extraction_warnings or [])]
|
||||
metadata = {
|
||||
**parsed.metadata,
|
||||
**classification_metadata(
|
||||
page_classification,
|
||||
title=parsed.title or fetch_result.title,
|
||||
text=parsed.raw_text or parsed.text,
|
||||
html=fetch_result.analysis_html,
|
||||
source_zones=parsed.source_zones or [],
|
||||
),
|
||||
"final_url": fetch_result.final_url,
|
||||
"crawl_status": fetch_result.crawl_status,
|
||||
"crawl_warnings": fetch_result.warnings,
|
||||
"page_type": page_type,
|
||||
"depth": depth,
|
||||
"raw_text_length": len(parsed.raw_text or ""),
|
||||
"clean_text_length": len(parsed.text or ""),
|
||||
@@ -319,7 +348,7 @@ class SiteCrawler:
|
||||
)
|
||||
return
|
||||
|
||||
if should_analyze_page(page_type, analyze_page_types):
|
||||
if should_analyze_page(page_classification, analyze_page_types):
|
||||
context = ExtractionPageContext(
|
||||
url=url,
|
||||
final_url=fetch_result.final_url,
|
||||
@@ -335,7 +364,7 @@ class SiteCrawler:
|
||||
warnings=warnings,
|
||||
metadata=metadata,
|
||||
)
|
||||
bundle = self.extractor.extract_from_context(context, project_config)
|
||||
bundle = extract_with_strategy(self.extractor, context, project_config)
|
||||
bundle = attach_page_context(bundle, context)
|
||||
claims = self.repository.save_extraction_bundle(source.project_id, source, page, bundle, project_config)
|
||||
extraction_summary = extraction_summary_from_raw(bundle.raw_output)
|
||||
|
||||
@@ -637,6 +637,38 @@ def list_openai_compatible_models(base_url: str, api_key: str | None = None) ->
|
||||
return [{"id": item.get("id", ""), "owned_by": item.get("owned_by")} for item in items if item.get("id")]
|
||||
|
||||
|
||||
def list_lmstudio_loaded_models(base_url: str) -> list[dict[str, Any]]:
|
||||
"""Return only the models currently loaded in LM Studio.
|
||||
|
||||
Uses LM Studio's native REST API (``/api/v0/models``) which exposes a
|
||||
``state`` field. Falls back to the OpenAI-compatible ``/v1/models`` list
|
||||
(treated as all-loaded) if the native endpoint is unavailable.
|
||||
"""
|
||||
clean = base_url.rstrip("/")
|
||||
if clean.endswith("/v1"):
|
||||
root = clean[: -len("/v1")]
|
||||
elif clean.endswith("/v1/chat/completions"):
|
||||
root = clean[: -len("/v1/chat/completions")]
|
||||
else:
|
||||
root = clean
|
||||
native_url = f"{root}/api/v0/models"
|
||||
try:
|
||||
response = requests.get(native_url, timeout=5)
|
||||
response.raise_for_status()
|
||||
data = response.json()
|
||||
items = data.get("data", []) if isinstance(data, dict) else []
|
||||
loaded = [
|
||||
{"id": item.get("id", ""), "owned_by": item.get("owned_by"), "state": item.get("state")}
|
||||
for item in items
|
||||
if item.get("id") and str(item.get("state", "")).lower() == "loaded"
|
||||
]
|
||||
if loaded:
|
||||
return loaded
|
||||
except Exception:
|
||||
pass
|
||||
return list_openai_compatible_models(base_url)
|
||||
|
||||
|
||||
def parse_entities(items: list[dict[str, Any]]) -> list[ExtractedEntity]:
|
||||
entities: list[ExtractedEntity] = []
|
||||
for item in items:
|
||||
|
||||
@@ -61,6 +61,7 @@ class ExtractionPageContext:
|
||||
clean_text = self.clean_text
|
||||
if text_limit is not None and len(clean_text) > text_limit:
|
||||
clean_text = clean_text[:text_limit]
|
||||
semantic_metadata = self.semantic_metadata_payload()
|
||||
zones = []
|
||||
for zone in self.source_zones:
|
||||
zone_text = str(zone.get("text") or "")
|
||||
@@ -85,8 +86,34 @@ class ExtractionPageContext:
|
||||
"clean_text": clean_text,
|
||||
"source_zones": zones,
|
||||
"warnings": self.warnings,
|
||||
"metadata": semantic_metadata,
|
||||
}
|
||||
|
||||
def semantic_metadata_payload(self) -> dict[str, Any]:
|
||||
payload = {
|
||||
key: self.metadata[key]
|
||||
for key in ("semantic_page_type", "analyze_strategy", "llm_policy")
|
||||
if key in self.metadata
|
||||
}
|
||||
classification = self.metadata.get("page_classification")
|
||||
if isinstance(classification, dict):
|
||||
payload["page_classification"] = {
|
||||
key: classification.get(key)
|
||||
for key in (
|
||||
"primary_page_type",
|
||||
"secondary_page_types",
|
||||
"confidence",
|
||||
"alternatives",
|
||||
"legacy_page_type",
|
||||
"unknown_pattern",
|
||||
)
|
||||
if key in classification
|
||||
}
|
||||
evidence = classification.get("evidence")
|
||||
if isinstance(evidence, list):
|
||||
payload["page_classification"]["evidence"] = evidence[:8]
|
||||
return payload
|
||||
|
||||
|
||||
class Extractor(ABC):
|
||||
name = "base"
|
||||
|
||||
@@ -5,6 +5,7 @@ from dataclasses import replace
|
||||
from typing import Any
|
||||
|
||||
from crawler_platform.app.config.loader import ProjectConfig
|
||||
from crawler_platform.app.core.crawler.page_type_taxonomy import LLMPolicy
|
||||
from crawler_platform.app.core.extractor.ai_provider import LLMJsonExtractor, dedupe_entities
|
||||
from crawler_platform.app.core.extractor.base import (
|
||||
ExtractedClaim,
|
||||
@@ -17,9 +18,40 @@ from crawler_platform.app.core.extractor.rule_based import GenericRuleBasedExtra
|
||||
from crawler_platform.app.domains.perfume.extractor import PerfumeRuleBasedExtractor
|
||||
|
||||
|
||||
LLM_PAGE_TYPES = {"ProductPage", "BrandStoryPage", "ReviewPage"}
|
||||
SKIP_LLM_PAGE_TYPES = {"CategoryPage", "SearchPage", "ListingPage"}
|
||||
RULE_ONLY_PAGE_TYPES = {"BoardPage", "CommunityPage", "UnknownPage"}
|
||||
LLM_PAGE_TYPES = {
|
||||
"ProductPage",
|
||||
"ProductDetailPage",
|
||||
"BrandStoryPage",
|
||||
"AboutPage",
|
||||
"ContactPage",
|
||||
"ReviewPage",
|
||||
"NoticePage",
|
||||
"PublicNoticePage",
|
||||
"PromotionPage",
|
||||
"CampaignLandingPage",
|
||||
"ArticlePage",
|
||||
"BlogPostPage",
|
||||
"DocumentationPage",
|
||||
"APIReferencePage",
|
||||
"DatasetPage",
|
||||
"ResearchPaperPage",
|
||||
"JobPostingPage",
|
||||
"CourseDetailPage",
|
||||
"LocalBusinessPage",
|
||||
"RealEstateListingPage",
|
||||
"ProfilePage",
|
||||
}
|
||||
SKIP_LLM_PAGE_TYPES = {"CategoryPage", "CategoryListingPage", "SearchPage", "SearchResultsPage", "ListingPage"}
|
||||
RULE_ONLY_PAGE_TYPES = {
|
||||
"BoardPage",
|
||||
"ForumBoardPage",
|
||||
"ForumThreadPage",
|
||||
"CommunityPage",
|
||||
"FAQPage",
|
||||
"QAPage",
|
||||
"VideoPage",
|
||||
"UnknownPage",
|
||||
}
|
||||
MIN_CLEAN_TEXT_CHARS_FOR_LLM = 300
|
||||
MAX_CLEAN_TEXT_CHARS_FOR_LLM = 60000
|
||||
|
||||
@@ -331,6 +363,22 @@ def llm_skip_reason(
|
||||
return None
|
||||
page_type = str(context.page_type or "UnknownPage")
|
||||
clean_length = len(context.clean_text or page_text or "")
|
||||
policy = llm_policy_from_context(context)
|
||||
if policy:
|
||||
if policy == LLMPolicy.SKIP.value:
|
||||
return "LLM policy Skip prevents LLM extraction"
|
||||
if policy == LLMPolicy.NO_LLM.value:
|
||||
return "LLM policy NoLLM prevents LLM extraction"
|
||||
if policy == LLMPolicy.RULE_ONLY.value:
|
||||
return "LLM policy RuleOnly routes page to rule-only extraction"
|
||||
if policy == LLMPolicy.LLM_FOR_AMBIGUITY_ONLY.value and not classification_is_ambiguous(context):
|
||||
return "LLM policy LLMForAmbiguityOnly requires ambiguous classification evidence"
|
||||
if policy in {LLMPolicy.LLM_FULL.value, LLMPolicy.LLM_LIGHT.value, LLMPolicy.LLM_FOR_AMBIGUITY_ONLY.value}:
|
||||
if clean_length < MIN_CLEAN_TEXT_CHARS_FOR_LLM:
|
||||
return f"clean text is too short for LLM extraction: {clean_length} chars"
|
||||
if clean_length > MAX_CLEAN_TEXT_CHARS_FOR_LLM:
|
||||
return f"clean text is too long for LLM extraction: {clean_length} chars"
|
||||
return None
|
||||
if page_type in SKIP_LLM_PAGE_TYPES:
|
||||
return f"page type {page_type} is configured to skip LLM extraction"
|
||||
if page_type in RULE_ONLY_PAGE_TYPES:
|
||||
@@ -344,6 +392,34 @@ def llm_skip_reason(
|
||||
return None
|
||||
|
||||
|
||||
def llm_policy_from_context(context: ExtractionPageContext) -> str | None:
|
||||
raw_policy = context.metadata.get("llm_policy")
|
||||
if not raw_policy:
|
||||
classification = context.metadata.get("page_classification")
|
||||
if isinstance(classification, dict):
|
||||
raw_policy = classification.get("llm_policy")
|
||||
policy = str(raw_policy or "").strip()
|
||||
return policy or None
|
||||
|
||||
|
||||
def classification_is_ambiguous(context: ExtractionPageContext) -> bool:
|
||||
classification = context.metadata.get("page_classification")
|
||||
if not isinstance(classification, dict):
|
||||
return False
|
||||
confidence = float(classification.get("confidence") or 0.0)
|
||||
if 0.0 < confidence < 0.55:
|
||||
return True
|
||||
alternatives = classification.get("alternatives") or []
|
||||
if not isinstance(alternatives, list) or len(alternatives) < 2:
|
||||
return False
|
||||
try:
|
||||
top_score = float(alternatives[0][1])
|
||||
next_score = float(alternatives[1][1])
|
||||
except (TypeError, ValueError, IndexError):
|
||||
return False
|
||||
return abs(top_score - next_score) < 0.08
|
||||
|
||||
|
||||
def merge_bundles(
|
||||
rule_bundle: ExtractionBundle,
|
||||
llm_bundle: ExtractionBundle,
|
||||
|
||||
@@ -0,0 +1,190 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from collections import Counter
|
||||
from typing import Any
|
||||
|
||||
from crawler_platform.app.config.loader import ProjectConfig
|
||||
from crawler_platform.app.core.crawler.page_analysis_policy import decide_analyze_strategy, decide_llm_policy
|
||||
from crawler_platform.app.core.crawler.page_type_taxonomy import AnalyzeStrategy, LLMPolicy
|
||||
from crawler_platform.app.core.extractor.base import ExtractionBundle, ExtractionPageContext, Extractor
|
||||
|
||||
|
||||
def extract_with_strategy(
|
||||
extractor: Extractor,
|
||||
context: ExtractionPageContext,
|
||||
project_config: ProjectConfig,
|
||||
) -> ExtractionBundle:
|
||||
strategy = analysis_strategy_from_context(context)
|
||||
policy = llm_policy_from_context(context)
|
||||
if strategy == AnalyzeStrategy.SKIP_PROTECTED.value:
|
||||
return strategy_only_bundle(context, strategy, policy, "protected page")
|
||||
if strategy == AnalyzeStrategy.SKIP_NOISE.value:
|
||||
return strategy_only_bundle(context, strategy, policy, "noise page")
|
||||
if strategy == AnalyzeStrategy.ANALYZE_METADATA_ONLY.value:
|
||||
return metadata_only_bundle(context, strategy, policy)
|
||||
if strategy == AnalyzeStrategy.ANALYZE_DISCOVERY_ONLY.value:
|
||||
return discovery_only_bundle(context, strategy, policy)
|
||||
if strategy == AnalyzeStrategy.ANALYZE_RELATIONS_ONLY.value:
|
||||
return rule_or_structure_bundle(extractor, context, project_config, strategy, policy)
|
||||
if strategy == AnalyzeStrategy.ANALYZE_ENTITY_ONLY.value:
|
||||
bundle = extractor.extract_from_context(context, project_config)
|
||||
bundle.claims = []
|
||||
bundle.raw_output = {
|
||||
**bundle.raw_output,
|
||||
**strategy_payload(context, strategy, policy),
|
||||
"effective_extraction_mode": "entity_only",
|
||||
}
|
||||
return bundle
|
||||
bundle = extractor.extract_from_context(context, project_config)
|
||||
bundle.raw_output = {
|
||||
**bundle.raw_output,
|
||||
**strategy_payload(context, strategy, policy),
|
||||
}
|
||||
return bundle
|
||||
|
||||
|
||||
def analysis_strategy_from_context(context: ExtractionPageContext) -> str:
|
||||
raw_strategy = context.metadata.get("analyze_strategy")
|
||||
if raw_strategy:
|
||||
return str(raw_strategy)
|
||||
classification = context.metadata.get("page_classification")
|
||||
if isinstance(classification, dict) and classification.get("analyze_strategy"):
|
||||
return str(classification["analyze_strategy"])
|
||||
return decide_analyze_strategy(context.page_type)
|
||||
|
||||
|
||||
def llm_policy_from_context(context: ExtractionPageContext) -> str:
|
||||
raw_policy = context.metadata.get("llm_policy")
|
||||
if raw_policy:
|
||||
return str(raw_policy)
|
||||
classification = context.metadata.get("page_classification")
|
||||
if isinstance(classification, dict) and classification.get("llm_policy"):
|
||||
return str(classification["llm_policy"])
|
||||
return decide_llm_policy(context.page_type)
|
||||
|
||||
|
||||
def rule_or_structure_bundle(
|
||||
extractor: Extractor,
|
||||
context: ExtractionPageContext,
|
||||
project_config: ProjectConfig,
|
||||
strategy: str,
|
||||
policy: str,
|
||||
) -> ExtractionBundle:
|
||||
if policy == LLMPolicy.LLM_FOR_AMBIGUITY_ONLY.value and classification_is_ambiguous(context):
|
||||
bundle = extractor.extract_from_context(context, project_config)
|
||||
bundle.raw_output = {
|
||||
**bundle.raw_output,
|
||||
**strategy_payload(context, strategy, policy),
|
||||
"effective_extraction_mode": "ambiguity_limited",
|
||||
}
|
||||
return bundle
|
||||
rule_extractor = getattr(extractor, "rule_extractor", None)
|
||||
if rule_extractor is not None:
|
||||
bundle = rule_extractor.extract_from_context(context, project_config)
|
||||
bundle.extractor_name = f"{bundle.extractor_name}_strategy_routed"
|
||||
bundle.raw_output = {
|
||||
**bundle.raw_output,
|
||||
**strategy_payload(context, strategy, policy),
|
||||
"effective_extraction_mode": "rule_only",
|
||||
"llm_skipped": True,
|
||||
"llm_skip_reason": f"analyze strategy {strategy} uses rule/structure extraction only",
|
||||
}
|
||||
return bundle
|
||||
if getattr(extractor, "provider", "") == "ai":
|
||||
return discovery_only_bundle(context, strategy, policy)
|
||||
bundle = extractor.extract_from_context(context, project_config)
|
||||
bundle.raw_output = {
|
||||
**bundle.raw_output,
|
||||
**strategy_payload(context, strategy, policy),
|
||||
"effective_extraction_mode": "rule_or_structure",
|
||||
"llm_skipped": policy in {LLMPolicy.RULE_ONLY.value, LLMPolicy.NO_LLM.value, LLMPolicy.SKIP.value},
|
||||
}
|
||||
return bundle
|
||||
|
||||
|
||||
def classification_is_ambiguous(context: ExtractionPageContext) -> bool:
|
||||
classification = context.metadata.get("page_classification")
|
||||
if not isinstance(classification, dict):
|
||||
return False
|
||||
confidence = float(classification.get("confidence") or 0.0)
|
||||
if 0.0 < confidence < 0.55:
|
||||
return True
|
||||
alternatives = classification.get("alternatives") or []
|
||||
if not isinstance(alternatives, list) or len(alternatives) < 2:
|
||||
return False
|
||||
try:
|
||||
top_score = float(alternatives[0][1])
|
||||
next_score = float(alternatives[1][1])
|
||||
except (TypeError, ValueError, IndexError):
|
||||
return False
|
||||
return abs(top_score - next_score) < 0.08
|
||||
|
||||
|
||||
def metadata_only_bundle(context: ExtractionPageContext, strategy: str, policy: str) -> ExtractionBundle:
|
||||
return ExtractionBundle(
|
||||
extractor_name="metadata_only_extractor",
|
||||
provider="strategy",
|
||||
raw_output={
|
||||
**strategy_payload(context, strategy, policy),
|
||||
"effective_extraction_mode": "metadata_only",
|
||||
"llm_skipped": True,
|
||||
"llm_skip_reason": f"analyze strategy {strategy} does not require LLM extraction",
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def discovery_only_bundle(context: ExtractionPageContext, strategy: str, policy: str) -> ExtractionBundle:
|
||||
return ExtractionBundle(
|
||||
extractor_name="discovery_only_extractor",
|
||||
provider="strategy",
|
||||
raw_output={
|
||||
**strategy_payload(context, strategy, policy),
|
||||
"effective_extraction_mode": "discovery_only",
|
||||
"llm_skipped": True,
|
||||
"llm_skip_reason": f"analyze strategy {strategy} uses discovery/structure signals only",
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def strategy_only_bundle(context: ExtractionPageContext, strategy: str, policy: str, reason: str) -> ExtractionBundle:
|
||||
return ExtractionBundle(
|
||||
extractor_name="strategy_skipped_extractor",
|
||||
provider="strategy",
|
||||
raw_output={
|
||||
**strategy_payload(context, strategy, policy),
|
||||
"effective_extraction_mode": "skipped",
|
||||
"llm_skipped": True,
|
||||
"llm_skip_reason": reason,
|
||||
"skip_reason": reason,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def strategy_payload(context: ExtractionPageContext, strategy: str, policy: str) -> dict[str, Any]:
|
||||
return {
|
||||
"analyze_strategy": strategy,
|
||||
"llm_policy": policy,
|
||||
"page_type": context.page_type,
|
||||
"semantic_page_type": context.metadata.get("semantic_page_type"),
|
||||
"strategy_summary": structure_summary(context),
|
||||
}
|
||||
|
||||
|
||||
def structure_summary(context: ExtractionPageContext) -> dict[str, Any]:
|
||||
zone_types = Counter(str(zone.get("zone_type") or "unknown") for zone in context.source_zones)
|
||||
classification = context.metadata.get("page_classification") if isinstance(context.metadata, dict) else None
|
||||
return {
|
||||
"title": context.title,
|
||||
"url": context.final_url or context.url,
|
||||
"clean_text_length": len(context.clean_text or ""),
|
||||
"raw_text_length": len(context.raw_text or ""),
|
||||
"source_zone_count": len(context.source_zones or []),
|
||||
"source_zone_types": dict(zone_types),
|
||||
"classification": {
|
||||
"primary_page_type": classification.get("primary_page_type"),
|
||||
"confidence": classification.get("confidence"),
|
||||
"alternatives": classification.get("alternatives", [])[:5],
|
||||
}
|
||||
if isinstance(classification, dict)
|
||||
else {},
|
||||
}
|
||||
@@ -538,12 +538,31 @@ def relation_hint(label: str) -> tuple[str, str] | None:
|
||||
def entity_type_from_page_type(page_type: str) -> str | None:
|
||||
mapping = {
|
||||
"ProductPage": "Product",
|
||||
"ProductDetailPage": "Product",
|
||||
"CategoryPage": "Category",
|
||||
"CategoryListingPage": "Category",
|
||||
"SearchPage": "Category",
|
||||
"SearchResultsPage": "Category",
|
||||
"BrandStoryPage": "Brand",
|
||||
"AboutPage": "Organization",
|
||||
"ContactPage": "Organization",
|
||||
"NoticePage": "Notice",
|
||||
"PublicNoticePage": "Notice",
|
||||
"PromotionPage": "Promotion",
|
||||
"CampaignLandingPage": "Promotion",
|
||||
"ReviewPage": "Review",
|
||||
"BoardPage": "Article",
|
||||
"ForumBoardPage": "Article",
|
||||
"ForumThreadPage": "Article",
|
||||
"ArticlePage": "Article",
|
||||
"NewsArticlePage": "Article",
|
||||
"BlogPostPage": "Article",
|
||||
"FAQPage": "Article",
|
||||
"QAPage": "Article",
|
||||
"DocumentationPage": "Article",
|
||||
"APIReferencePage": "Article",
|
||||
"JobPostingPage": "JobPosting",
|
||||
"PricingPage": "Product",
|
||||
}
|
||||
return mapping.get(page_type)
|
||||
|
||||
|
||||
@@ -3,6 +3,8 @@ from __future__ import annotations
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
from crawler_platform.app.core.crawler.page_type_taxonomy import normalize_page_type
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class RelationRule:
|
||||
@@ -97,13 +99,21 @@ def relation_schema_compatible(
|
||||
return f"{predicate} expects a typed entity object"
|
||||
if object_type and rule.object_types and object_type not in rule.object_types:
|
||||
return f"object type {object_type} is not allowed for {predicate}"
|
||||
if page_type and rule.page_types and page_type not in rule.page_types:
|
||||
if page_type and rule.page_types and not page_type_matches_rule(page_type, rule.page_types):
|
||||
return f"predicate {predicate} is not allowed for page type {page_type}"
|
||||
if source_zone and rule.source_zones and source_zone not in rule.source_zones:
|
||||
return f"source zone {source_zone} is not allowed for {predicate}"
|
||||
return None
|
||||
|
||||
|
||||
def page_type_matches_rule(page_type: str, allowed_page_types: set[str]) -> bool:
|
||||
if page_type in allowed_page_types:
|
||||
return True
|
||||
normalized_page_type = normalize_page_type(page_type)
|
||||
normalized_allowed = {normalize_page_type(allowed_page_type) for allowed_page_type in allowed_page_types}
|
||||
return normalized_page_type in normalized_allowed
|
||||
|
||||
|
||||
def confidence_breakdown(
|
||||
*,
|
||||
llm_confidence: float,
|
||||
|
||||
@@ -7,11 +7,17 @@ from urllib.parse import urldefrag, urlparse
|
||||
from crawler_platform.app.config.loader import ProjectConfig
|
||||
from crawler_platform.app.core.crawler.discovery import discover_links
|
||||
from crawler_platform.app.core.crawler.fetchers import RobotsPolicy, make_fetcher
|
||||
from crawler_platform.app.core.crawler.page_classifier import classify_page, should_analyze_page
|
||||
from crawler_platform.app.core.crawler.page_classifier import (
|
||||
classification_metadata,
|
||||
classify_page_semantic,
|
||||
get_legacy_page_type,
|
||||
should_analyze_page,
|
||||
)
|
||||
from crawler_platform.app.core.crawler.plugins import ParserRegistry, default_parser_registry
|
||||
from crawler_platform.app.core.database import models
|
||||
from crawler_platform.app.core.database.repository import KnowledgeRepository
|
||||
from crawler_platform.app.core.extractor.base import ExtractionPageContext, Extractor
|
||||
from crawler_platform.app.core.extractor.strategy import extract_with_strategy
|
||||
from crawler_platform.app.core.extractor.validation import attach_page_context
|
||||
from crawler_platform.app.core.research.entity_expansion import EntityExpansionPlanner
|
||||
from crawler_platform.app.core.research.exploration_queue import ExplorationItem, ExplorationQueue
|
||||
@@ -206,13 +212,17 @@ class GraphResearchLoop:
|
||||
|
||||
fetch_result = fetcher.fetch(url)
|
||||
parser_result = parser.parse(fetch_result.analysis_html, fetch_result.final_url or url)
|
||||
page_type = classify_page(
|
||||
page_classification = classify_page_semantic(
|
||||
fetch_result.final_url or url,
|
||||
parser_result.title or fetch_result.title,
|
||||
parser_result.raw_text or parser_result.text,
|
||||
fetch_result.analysis_html,
|
||||
parser_result.source_zones or [],
|
||||
final_url=fetch_result.final_url,
|
||||
status_code=fetch_result.status_code,
|
||||
content_type=fetch_result.headers.get("content-type"),
|
||||
)
|
||||
page_type = get_legacy_page_type(page_classification)
|
||||
relevance = self.relevance.score_url(
|
||||
project_id=source.project_id,
|
||||
url=fetch_result.final_url or url,
|
||||
@@ -225,11 +235,17 @@ class GraphResearchLoop:
|
||||
)
|
||||
metadata = {
|
||||
**parser_result.metadata,
|
||||
**classification_metadata(
|
||||
page_classification,
|
||||
title=parser_result.title or fetch_result.title,
|
||||
text=parser_result.raw_text or parser_result.text,
|
||||
html=fetch_result.analysis_html,
|
||||
source_zones=parser_result.source_zones or [],
|
||||
),
|
||||
"research_item": item.to_dict(),
|
||||
"research_relevance": asdict(relevance),
|
||||
"final_url": fetch_result.final_url,
|
||||
"crawl_status": fetch_result.crawl_status,
|
||||
"page_type": page_type,
|
||||
"robots_status": robots_decision.status,
|
||||
"robots_reason": robots_decision.reason,
|
||||
"raw_text_length": len(parser_result.raw_text or ""),
|
||||
@@ -289,7 +305,7 @@ class GraphResearchLoop:
|
||||
fetch_result.crawl_status == "success"
|
||||
and parser_result.extraction_status != "failed"
|
||||
and relevance.score >= min_relevance
|
||||
and should_analyze_page(page_type, analyze_page_types)
|
||||
and should_analyze_page(page_classification, analyze_page_types)
|
||||
):
|
||||
context = ExtractionPageContext(
|
||||
url=url,
|
||||
@@ -306,7 +322,7 @@ class GraphResearchLoop:
|
||||
warnings=[*fetch_result.warnings, *(parser_result.extraction_warnings or [])],
|
||||
metadata=metadata,
|
||||
)
|
||||
bundle = self.extractor.extract_from_context(context, project_config)
|
||||
bundle = extract_with_strategy(self.extractor, context, project_config)
|
||||
bundle = attach_page_context(bundle, context)
|
||||
claims = self.repository.save_extraction_bundle(source.project_id, source, page, bundle, project_config)
|
||||
analyzed = True
|
||||
|
||||
@@ -8,20 +8,46 @@ from urllib.parse import unquote, urlparse
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from crawler_platform.app.core.crawler.page_classifier import classify_page
|
||||
from crawler_platform.app.core.crawler.page_classifier import classify_page_semantic, get_legacy_page_type
|
||||
from crawler_platform.app.core.database import models
|
||||
|
||||
|
||||
HIGH_VALUE_PAGE_TYPES = {
|
||||
"ProductPage": 0.95,
|
||||
"ProductDetailPage": 0.95,
|
||||
"BrandStoryPage": 0.88,
|
||||
"AboutPage": 0.82,
|
||||
"ContactPage": 0.72,
|
||||
"ReviewPage": 0.82,
|
||||
"NoticePage": 0.45,
|
||||
"PublicNoticePage": 0.45,
|
||||
"ArticlePage": 0.55,
|
||||
"NewsArticlePage": 0.55,
|
||||
"BlogPostPage": 0.52,
|
||||
"FAQPage": 0.48,
|
||||
"QAPage": 0.48,
|
||||
"DocumentationPage": 0.5,
|
||||
"APIReferencePage": 0.5,
|
||||
"WikiPage": 0.48,
|
||||
"DatasetPage": 0.52,
|
||||
"ResearchPaperPage": 0.6,
|
||||
"JobPostingPage": 0.46,
|
||||
"CourseDetailPage": 0.5,
|
||||
"VideoPage": 0.36,
|
||||
"LocalBusinessPage": 0.44,
|
||||
"RealEstateListingPage": 0.44,
|
||||
"ProfilePage": 0.36,
|
||||
"PricingPage": 0.62,
|
||||
"EventPage": 0.5,
|
||||
"PromotionPage": 0.38,
|
||||
"CampaignLandingPage": 0.38,
|
||||
"CategoryPage": 0.34,
|
||||
"CategoryListingPage": 0.34,
|
||||
"SearchPage": 0.22,
|
||||
"SearchResultsPage": 0.22,
|
||||
"BoardPage": 0.18,
|
||||
"ForumBoardPage": 0.18,
|
||||
"ForumThreadPage": 0.24,
|
||||
"UnknownPage": 0.25,
|
||||
}
|
||||
|
||||
@@ -96,8 +122,12 @@ class RelevanceEngine:
|
||||
normalized = url.strip().rstrip("/")
|
||||
parsed = urlparse(normalized)
|
||||
combined = unquote(f"{normalized} {label} {text[:3000]}").lower()
|
||||
page_type = classify_page(normalized, label, text, html=html)
|
||||
page_type_score = HIGH_VALUE_PAGE_TYPES.get(page_type, 0.25)
|
||||
page_classification = classify_page_semantic(normalized, label, text, html=html)
|
||||
page_type = page_classification.primary_page_type
|
||||
page_type_score = HIGH_VALUE_PAGE_TYPES.get(
|
||||
page_type,
|
||||
HIGH_VALUE_PAGE_TYPES.get(get_legacy_page_type(page_classification), 0.25),
|
||||
)
|
||||
graph_terms = self.graph_terms(project_id)
|
||||
tokens = tokenize(combined)
|
||||
overlap_count = len(tokens & graph_terms)
|
||||
|
||||
Reference in New Issue
Block a user