ontology

crawler_platform/app/core/crawler/discovery.py

@@ -1,7 +1,7 @@
 from __future__ import annotations
 
 from dataclasses import dataclass
-from urllib.parse import parse_qs, unquote, urljoin, urlparse
+from urllib.parse import parse_qs, urlencode, unquote, urljoin, urlparse, urlunparse
 
 from bs4 import BeautifulSoup
 
@@ -21,7 +21,7 @@ def discover_links(html: str, base_url: str, limit: int = 30) -> list[Discovered
         raw_href = anchor.get("href", "")
         if should_skip_raw_href(raw_href):
             continue
-        url = normalize_search_redirect(urljoin(base_url, raw_href))
+        url = normalize_cafe24_product_url(normalize_search_redirect(urljoin(base_url, raw_href)))
         if not url or url in seen or not url.startswith(("http://", "https://")) or should_skip_url(url):
             continue
         seen.add(url)
@@ -58,12 +58,18 @@ def should_skip_url(url: str) -> bool:
     skip_path_tokens = [
         "/member/",
         "/order/",
+        "/myshop/",
+        "/event/list",
         "/exec/front/newcoupon/",
+        "/board/free/list",
+        "/board/faq/list",
         "/board/free/modify",
         "/board/free/reply",
     ]
     if any(token in path for token in skip_path_tokens):
         return True
+    if path.endswith("/product/search.html"):
+        return True
     if "facebook.com/" in path or "instagram.com/" in path:
         return True
     if "coupon_no=" in query:
@@ -82,6 +88,33 @@ def normalize_search_redirect(url: str) -> str:
     return url
 
 
+def normalize_cafe24_product_url(url: str) -> str:
+    parsed = urlparse(url)
+    path = unquote(parsed.path)
+    parts = [part for part in path.strip("/").split("/") if part]
+    query = parse_qs(parsed.query)
+
+    if path.endswith("/product/detail.html") and query.get("product_no"):
+        return urlunparse(
+            parsed._replace(
+                query=urlencode({"product_no": query["product_no"][0]}),
+                fragment="",
+            )
+        )
+
+    if parts and parts[0] == "product":
+        product_no = next((part for part in parts[1:] if part.isdigit()), None)
+        if product_no:
+            return urlunparse(
+                parsed._replace(
+                    path="/product/detail.html",
+                    query=urlencode({"product_no": product_no}),
+                    fragment="",
+                )
+            )
+    return url
+
+
 def classify_url(url: str) -> str:
     host = urlparse(url).netloc.lower()
     if "smartstore.naver.com" in host or "brand.naver.com" in host:

crawler_platform/app/core/crawler/fetchers.py

@@ -42,6 +42,16 @@ class FetchResult:
         return self.rendered_html or self.raw_html or self.html
 
 
+@dataclass(slots=True)
+class RobotsDecision:
+    allowed: bool
+    checked: bool
+    status: str
+    reason: str
+    robots_url: str | None = None
+    user_agent: str = DEFAULT_USER_AGENT
+
+
 class RateLimiter:
     def __init__(self, per_minute: int = 30):
         self.delay = 60 / max(per_minute, 1)
@@ -60,11 +70,26 @@ class RobotsPolicy:
         self._cache: dict[str, RobotFileParser] = {}
 
     def allowed(self, url: str, respect_robots_txt: bool = True) -> bool:
+        return self.check(url, respect_robots_txt).allowed
+
+    def check(self, url: str, respect_robots_txt: bool = True) -> RobotsDecision:
         if not respect_robots_txt:
-            return True
+            return RobotsDecision(
+                allowed=True,
+                checked=False,
+                status="disabled",
+                reason="robots.txt check disabled by source/request config",
+                user_agent=self.user_agent,
+            )
         parsed = urlparse(url)
         if parsed.scheme in {"", "file"}:
-            return True
+            return RobotsDecision(
+                allowed=True,
+                checked=False,
+                status="local",
+                reason="robots.txt is not applicable to local or scheme-less URLs",
+                user_agent=self.user_agent,
+            )
         robots_url = f"{parsed.scheme}://{parsed.netloc}/robots.txt"
         parser = self._cache.get(robots_url)
         if parser is None:
@@ -72,10 +97,29 @@ class RobotsPolicy:
             parser.set_url(robots_url)
             try:
                 parser.read()
-            except Exception:
-                return False
+            except Exception as exc:
+                return RobotsDecision(
+                    allowed=True,
+                    checked=True,
+                    status="unavailable",
+                    reason=f"robots.txt unavailable ({exc.__class__.__name__}); allowing crawl",
+                    robots_url=robots_url,
+                    user_agent=self.user_agent,
+                )
             self._cache[robots_url] = parser
-        return parser.can_fetch(self.user_agent, url)
+        allowed = parser.can_fetch(self.user_agent, url)
+        return RobotsDecision(
+            allowed=allowed,
+            checked=True,
+            status="allowed" if allowed else "blocked",
+            reason=(
+                "robots.txt allows crawling"
+                if allowed
+                else f"robots.txt blocks crawling for user-agent {self.user_agent}"
+            ),
+            robots_url=robots_url,
+            user_agent=self.user_agent,
+        )
 
 
 class BaseFetcher:
@@ -83,6 +127,23 @@ class BaseFetcher:
         raise NotImplementedError
 
 
+class FallbackFetcher(BaseFetcher):
+    def __init__(self, primary: BaseFetcher, fallback: BaseFetcher, fallback_label: str = "fallback"):
+        self.primary = primary
+        self.fallback = fallback
+        self.fallback_label = fallback_label
+
+    def fetch(self, url: str) -> FetchResult:
+        try:
+            return self.primary.fetch(url)
+        except Exception as exc:
+            result = self.fallback.fetch(url)
+            result.warnings.append(
+                f"primary fetcher failed ({exc.__class__.__name__}: {exc}); used {self.fallback_label}"
+            )
+            return result
+
+
 class RequestsFetcher(BaseFetcher):
     def __init__(
         self,
@@ -170,7 +231,11 @@ class PlaywrightFetcher(BaseFetcher):
 
 def make_fetcher(kind: str, rate_limit_per_minute: int = 30) -> BaseFetcher:
     if kind in {"playwright", "browser"}:
-        return PlaywrightFetcher()
+        return FallbackFetcher(
+            PlaywrightFetcher(),
+            RequestsFetcher(rate_limit_per_minute=rate_limit_per_minute),
+            fallback_label="requests",
+        )
     return RequestsFetcher(rate_limit_per_minute=rate_limit_per_minute)
 
 

crawler_platform/app/core/crawler/page_classifier.py

@@ -49,7 +49,7 @@ def classify_page(
     if any(token in path for token in ["shopinfo", "company", "about", "brand-story", "brand_story"]):
         return "BrandStoryPage"
     if any(token in path for token in ["product/detail", "/product/", "/products/", "/goods/", "/item/"]):
-        if _looks_like_category_path(path, combined):
+        if _looks_like_category_path(path, combined) and not _looks_like_product_detail_path(path):
             return "CategoryPage"
         return "ProductPage"
     if any(token in path for token in ["category", "/collections", "/collection", "/shop/", "/list"]):
@@ -142,6 +142,14 @@ def _has_product_detail_signal(text: str) -> bool:
     return note_count >= 1 or commerce_count >= 2
 
 
+def _looks_like_product_detail_path(path: str) -> bool:
+    parts = [part for part in path.strip("/").split("/") if part]
+    if "product" not in parts:
+        return False
+    product_index = parts.index("product")
+    return any(part.isdigit() for part in parts[product_index + 1 :])
+
+
 def _looks_like_category_path(path: str, text: str) -> bool:
     category_tokens = ["category", "cate_no", "display_group", "sort_method"]
     if any(token in path or token in text for token in category_tokens):

crawler_platform/app/core/crawler/pipeline.py

@@ -22,6 +22,8 @@ class CrawlResult:
     clean_text_length: int = 0
     raw_text_length: int = 0
     warnings: list[str] | None = None
+    robots_status: str = "unchecked"
+    robots_reason: str | None = None
 
 
 class CrawlPipeline:
@@ -39,8 +41,9 @@ class CrawlPipeline:
 
     def crawl_url(self, project_config: ProjectConfig, source_name: str, url: str) -> CrawlResult:
         source_config = project_config.source_by_name(source_name)
-        if not self.robots_policy.allowed(url, source_config.respect_robots_txt):
-            raise PermissionError(f"robots.txt does not allow crawling: {url}")
+        robots_decision = self.robots_policy.check(url, source_config.respect_robots_txt)
+        if not robots_decision.allowed:
+            raise PermissionError(f"{robots_decision.reason}: {url}")
 
         fetcher = make_fetcher(source_config.fetcher, source_config.rate_limit_per_minute)
         fetch_result = fetcher.fetch(url)
@@ -87,6 +90,8 @@ class CrawlPipeline:
                 clean_text_length=len(parsed.text or ""),
                 raw_text_length=len(parsed.raw_text or ""),
                 warnings=warnings,
+                robots_status=robots_decision.status,
+                robots_reason=robots_decision.reason,
             )
 
         context = ExtractionPageContext(
@@ -117,4 +122,6 @@ class CrawlPipeline:
             clean_text_length=len(parsed.text or ""),
             raw_text_length=len(parsed.raw_text or ""),
             warnings=warnings,
+            robots_status=robots_decision.status,
+            robots_reason=robots_decision.reason,
         )

crawler_platform/app/core/crawler/site_crawler.py

@@ -7,7 +7,7 @@ from urllib.parse import urldefrag, urlparse
 
 from crawler_platform.app.config.loader import ProjectConfig
 from crawler_platform.app.core.crawler.discovery import discover_links
-from crawler_platform.app.core.crawler.fetchers import RobotsPolicy, make_fetcher
+from crawler_platform.app.core.crawler.fetchers import RobotsDecision, RobotsPolicy, make_fetcher
 from crawler_platform.app.core.crawler.page_classifier import (
     classify_page as classify_page_type,
     should_analyze_page as should_analyze_page_type,
@@ -35,6 +35,8 @@ class SiteCrawlPageResult:
     clean_text_length: int = 0
     removed_noise_zones_count: int = 0
     warnings: list[str] = field(default_factory=list)
+    robots_status: str = "unchecked"
+    robots_reason: str | None = None
     error: str | None = None
 
 
@@ -120,14 +122,23 @@ class SiteCrawler:
                     progress_callback,
                 )
                 continue
-            if not self.robots_policy.allowed(url, source_config.respect_robots_txt):
-                error = f"robots.txt does not allow crawling: {url}"
+            robots_decision = self.robots_policy.check(url, source_config.respect_robots_txt)
+            if not robots_decision.allowed:
+                error = f"{robots_decision.reason}: {url}"
                 self._finish_job(job, "blocked", error)
                 result.skipped_count += 1
                 result.errors.append(error)
                 self._record_page_result(
                     result,
-                    SiteCrawlPageResult(url=url, depth=depth, status="blocked", page_type="unknown", error=error),
+                    SiteCrawlPageResult(
+                        url=url,
+                        depth=depth,
+                        status="blocked",
+                        page_type="unknown",
+                        robots_status=robots_decision.status,
+                        robots_reason=robots_decision.reason,
+                        error=error,
+                    ),
                     len(queue),
                     progress_callback,
                 )
@@ -150,6 +161,7 @@ class SiteCrawler:
                     analyze_page_types=analyze_page_types,
                     result=result,
                     job=job,
+                    robots_decision=robots_decision,
                     progress_callback=progress_callback,
                 )
             except Exception as exc:
@@ -184,6 +196,7 @@ class SiteCrawler:
         analyze_page_types: set[str],
         result: SiteCrawlResult,
         job: models.CrawlJob,
+        robots_decision: RobotsDecision,
         progress_callback: Callable[[SiteCrawlResult, SiteCrawlPageResult], None] | None,
     ) -> None:
         fetch_result = fetcher.fetch(url)
@@ -215,6 +228,8 @@ class SiteCrawler:
                     page_type="unknown",
                     page_id=page.id,
                     crawl_status=fetch_result.crawl_status,
+                    robots_status=robots_decision.status,
+                    robots_reason=robots_decision.reason,
                     warnings=fetch_result.warnings,
                     error=error,
                 ),
@@ -277,6 +292,8 @@ class SiteCrawler:
             "clean_text_length": len(parsed.text or ""),
             "removed_noise_zones_count": int(parsed.metadata.get("removed_noise_zones_count") or 0),
             "warnings": warnings,
+            "robots_status": robots_decision.status,
+            "robots_reason": robots_decision.reason,
         }
 
         if parsed.extraction_status == "failed":

crawler_platform/app/core/database/repository.py

@@ -185,7 +185,7 @@ class KnowledgeRepository:
         else:
             claim_status = "active"
 
-        if claim_status not in {"active", "validated_claim"}:
+        if claim_status not in {"active", "validated_claim", "candidate_claim", "rule_candidate"}:
             self._log_extraction(project_id, page, bundle)
             return []
 

crawler_platform/app/core/extractor/ai_provider.py

@@ -64,6 +64,9 @@ class LLMJsonExtractor(AIExtractor):
             try:
                 raw = self.complete_json(page_text, project_config, compact=compact_mode, context=context)
                 bundle = self._bundle_from_raw(raw, mode_name)
+                enriched = self._merge_rule_fallback_claims(bundle, page_text, project_config, mode_name)
+                if enriched.entities and enriched.claims:
+                    return self.normalize_to_ontology(enriched, project_config.ontology)
                 if bundle.entities and bundle.claims:
                     return self.normalize_to_ontology(bundle, project_config.ontology)
                 errors.append(f"{mode_name}: AI returned no usable entities or claims")
@@ -103,12 +106,18 @@ class LLMJsonExtractor(AIExtractor):
         context: ExtractionPageContext | None = None,
     ) -> dict[str, Any]:
         if self.provider == "lm_studio":
-            char_limit = 1200 if compact else 2200
-            max_tokens = 220 if compact else 420
+            char_limit = 700 if compact else 1100
+            max_tokens = 260 if compact else 360
         else:
             char_limit = 2200 if compact else 4000
             max_tokens = 400 if compact else 800
-        prompt = build_extraction_prompt(page_text, project_config, char_limit=char_limit, context=context)
+        prompt = build_extraction_prompt(
+            page_text,
+            project_config,
+            char_limit=char_limit,
+            context=context,
+            compact=self.provider == "lm_studio",
+        )
         if self.provider == "openai":
             return self._complete_openai_compatible(
                 prompt,
@@ -154,6 +163,45 @@ class LLMJsonExtractor(AIExtractor):
             claim.confidence_reason = f"{claim.confidence_reason}; AI fallback: {error}" if claim.confidence_reason else error
         return bundle
 
+    def _merge_rule_fallback_claims(
+        self,
+        bundle: ExtractionBundle,
+        page_text: str,
+        project_config: ProjectConfig,
+        mode: str,
+    ) -> ExtractionBundle:
+        rule_bundle = self._rule_bundle(page_text, project_config)
+        if not rule_bundle.claims:
+            return bundle
+        merged = ExtractionBundle(
+            entities=dedupe_entities([*bundle.entities, *rule_bundle.entities]),
+            claims=dedupe_claims([*bundle.claims, *rule_bundle.claims]),
+            extractor_name=f"{self.name}_with_rule_claims",
+            provider=self.provider,
+            raw_output={
+                **bundle.raw_output,
+                "extraction_mode": mode,
+                "rule_claim_merge": True,
+                "rule_entity_count": len(rule_bundle.entities),
+                "rule_claim_count": len(rule_bundle.claims),
+            },
+        )
+        for claim in merged.claims:
+            claim.metadata.setdefault("rule_claim_merge", True)
+            claim.confidence_reason = (
+                f"{claim.confidence_reason}; AI entity output enriched with rule claims"
+                if claim.confidence_reason
+                else "AI entity output enriched with rule claims"
+            )
+        return merged
+
+    def _rule_bundle(self, page_text: str, project_config: ProjectConfig) -> ExtractionBundle:
+        if project_config.domain == "perfume":
+            return PerfumeRuleBasedExtractor().extract(page_text, project_config)
+        from crawler_platform.app.core.extractor.rule_based import GenericRuleBasedExtractor
+
+        return GenericRuleBasedExtractor().extract(page_text, project_config)
+
     def _bundle_from_raw(self, raw: dict[str, Any], mode: str) -> ExtractionBundle:
         return ExtractionBundle(
             entities=parse_entities(raw.get("entities", [])),
@@ -275,7 +323,10 @@ def build_extraction_prompt(
     project_config: ProjectConfig,
     char_limit: int = 4000,
     context: ExtractionPageContext | None = None,
+    compact: bool = False,
 ) -> str:
+    if compact:
+        return build_compact_extraction_prompt(page_text, project_config, char_limit=char_limit, context=context)
     if context is not None:
         prompt_input = json.dumps(context.to_payload(text_limit=char_limit), ensure_ascii=False, indent=2)
     else:
@@ -338,6 +389,35 @@ Input payload:
 """.strip()
 
 
+def build_compact_extraction_prompt(
+    page_text: str,
+    project_config: ProjectConfig,
+    char_limit: int = 1000,
+    context: ExtractionPageContext | None = None,
+) -> str:
+    if context is not None:
+        text = prepare_page_text_for_prompt(context.clean_text, char_limit)
+        title = context.title or ""
+        page_type = context.page_type
+        url = context.final_url or context.url
+    else:
+        text = prepare_page_text_for_prompt(page_text, char_limit)
+        title = ""
+        page_type = "UnknownPage"
+        url = ""
+    predicates = ", ".join((project_config.ontology or {}).get("predicates", [])[:12])
+    return (
+        "Return minified JSON only: {\"entities\":[],\"claims\":[]}.\n"
+        f"Domain perfume. PageType={page_type}. URL={url}. Title={title}\n"
+        "Entity types: Perfume, Brand, Note, Accord, Mood, Season, Occasion, Price.\n"
+        f"Predicates: {predicates}.\n"
+        "Extract only explicit product facts. Max 6 entities, 8 claims. "
+        "Every claim needs short evidence_text from text. "
+        "Use null for missing object_name/object_type/object_value.\n"
+        f"Text:\n{text}"
+    )
+
+
 def prepare_page_text_for_prompt(page_text: str, char_limit: int) -> str:
     noisy_terms = {
         "first page",
@@ -567,6 +647,36 @@ def parse_entities(items: list[dict[str, Any]]) -> list[ExtractedEntity]:
     return entities
 
 
+def dedupe_entities(entities: list[ExtractedEntity]) -> list[ExtractedEntity]:
+    seen: set[tuple[str, str]] = set()
+    result: list[ExtractedEntity] = []
+    for entity in entities:
+        key = (entity.entity_type.strip().lower(), entity.name.strip().lower())
+        if key in seen:
+            continue
+        seen.add(key)
+        result.append(entity)
+    return result
+
+
+def dedupe_claims(claims: list[ExtractedClaim]) -> list[ExtractedClaim]:
+    seen: set[tuple[str, str, str, str]] = set()
+    result: list[ExtractedClaim] = []
+    for claim in claims:
+        object_key = claim.object_name or json.dumps(claim.object_value, ensure_ascii=False, sort_keys=True, default=str)
+        key = (
+            claim.subject_name.strip().lower(),
+            claim.subject_type.strip().lower(),
+            claim.predicate.strip(),
+            str(object_key).strip().lower(),
+        )
+        if key in seen:
+            continue
+        seen.add(key)
+        result.append(claim)
+    return result
+
+
 def parse_claims(items: list[dict[str, Any]]) -> list[ExtractedClaim]:
     claims: list[ExtractedClaim] = []
     for item in items:

crawler_platform/app/core/research/graph_research_loop.py

@@ -200,8 +200,9 @@ class GraphResearchLoop:
         url = item.target
         if same_domain_only and seed_host and normalized_host(url) != seed_host:
             return {"status": "skipped", "reason": "outside same-domain research boundary"}
-        if not self.robots_policy.allowed(url, source_config.respect_robots_txt):
-            return {"status": "skipped", "reason": f"robots.txt blocked {url}"}
+        robots_decision = self.robots_policy.check(url, source_config.respect_robots_txt)
+        if not robots_decision.allowed:
+            return {"status": "skipped", "reason": f"{robots_decision.reason}: {url}"}
 
         fetch_result = fetcher.fetch(url)
         parser_result = parser.parse(fetch_result.analysis_html, fetch_result.final_url or url)
@@ -229,6 +230,8 @@ class GraphResearchLoop:
             "final_url": fetch_result.final_url,
             "crawl_status": fetch_result.crawl_status,
             "page_type": page_type,
+            "robots_status": robots_decision.status,
+            "robots_reason": robots_decision.reason,
             "raw_text_length": len(parser_result.raw_text or ""),
             "clean_text_length": len(parser_result.text or ""),
             "main_content_preview": (parser_result.main_content or parser_result.text)[:800],