ontology

crawler_platform/app/api/routes.py

@@ -34,6 +34,8 @@ class CrawlRequest(BaseModel):
     extractor_provider: str = "lm_studio"
     extractor_model: str | None = None
     extractor_base_url: str | None = "http://localhost:1234/v1"
+    check_robots_txt: bool = False
+    respect_robots_txt: bool | None = None
 
 
 class SiteCrawlRequest(CrawlRequest):
@@ -48,6 +50,8 @@ class DiscoverRequest(BaseModel):
     source_name: str
     url: str
     limit: int = 30
+    check_robots_txt: bool = False
+    respect_robots_txt: bool | None = None
 
 
 class RecommendRequest(BaseModel):
@@ -145,6 +149,13 @@ def _apply_claim_review(claim: models.Claim, status: str, reason: str | None) ->
     claim.last_seen_at = models.utcnow()
 
 
+def apply_crawl_request_overrides(config, request: CrawlRequest | DiscoverRequest) -> None:
+    check_robots_txt = request.respect_robots_txt
+    if check_robots_txt is None:
+        check_robots_txt = request.check_robots_txt
+    config.source_by_name(request.source_name).respect_robots_txt = check_robots_txt
+
+
 def crawl_job_response(job: models.CrawlJob) -> dict[str, Any]:
     metadata = job.metadata_json or {}
     return {
@@ -176,6 +187,7 @@ def run_site_crawl_job(database_url: str, job_id: int, request_data: dict[str, A
     request = SiteCrawlRequest(**request_data)
     try:
         config = load_project_config(request.config_path)
+        apply_crawl_request_overrides(config, request)
         with session_scope(database_url) as session:
             job = session.get(models.CrawlJob, job_id)
             if job is None:
@@ -444,6 +456,7 @@ def register_routes(app, database_url: str) -> None:
     @app.post("/crawl")
     def crawl(request: CrawlRequest):
         config = load_project_config(request.config_path)
+        apply_crawl_request_overrides(config, request)
         with session_scope(database_url) as session:
             repo = KnowledgeRepository(session)
             pipeline = CrawlPipeline(
@@ -474,6 +487,7 @@ def register_routes(app, database_url: str) -> None:
     @app.post("/crawl-site")
     def crawl_site(request: SiteCrawlRequest, background_tasks: BackgroundTasks):
         config = load_project_config(request.config_path)
+        apply_crawl_request_overrides(config, request)
         with session_scope(database_url) as session:
             repo = KnowledgeRepository(session)
             project = repo.upsert_project(config)
@@ -526,10 +540,18 @@ def register_routes(app, database_url: str) -> None:
     @app.post("/discover")
     def discover(request: DiscoverRequest):
         config = load_project_config(request.config_path)
+        apply_crawl_request_overrides(config, request)
         source_config = config.source_by_name(request.source_name)
         robots = RobotsPolicy()
-        if not robots.allowed(request.url, source_config.respect_robots_txt):
-            return {"ok": False, "error": "robots.txt does not allow discovery for this URL", "links": []}
+        robots_decision = robots.check(request.url, source_config.respect_robots_txt)
+        if not robots_decision.allowed:
+            return {
+                "ok": False,
+                "error": f"{robots_decision.reason}: {request.url}",
+                "robots_status": robots_decision.status,
+                "robots_reason": robots_decision.reason,
+                "links": [],
+            }
         fetcher = make_fetcher(source_config.fetcher, source_config.rate_limit_per_minute)
         result = fetcher.fetch(request.url)
         links = discover_links(result.analysis_html, result.final_url or request.url, request.limit)
@@ -538,6 +560,8 @@ def register_routes(app, database_url: str) -> None:
             "status_code": result.status_code,
             "final_url": result.final_url,
             "crawl_status": result.crawl_status,
+            "robots_status": robots_decision.status,
+            "robots_reason": robots_decision.reason,
             "warnings": result.warnings,
             "links": [asdict(link) for link in links],
         }
@@ -545,6 +569,7 @@ def register_routes(app, database_url: str) -> None:
     @app.post("/research/run")
     def run_research(request: ResearchRunRequest):
         config = load_project_config(request.config_path)
+        apply_crawl_request_overrides(config, request)
         if request.project_name and request.project_name != config.project_name:
             raise HTTPException(
                 status_code=400,

crawler_platform/app/config/loader.py

@@ -16,7 +16,7 @@ class SourceConfig:
     parser: str = "generic"
     fetcher: str = "requests"
     rate_limit_per_minute: int = 30
-    respect_robots_txt: bool = True
+    respect_robots_txt: bool = False
 
 
 @dataclass(slots=True)

crawler_platform/app/core/crawler/discovery.py

@@ -1,7 +1,7 @@
 from __future__ import annotations
 
 from dataclasses import dataclass
-from urllib.parse import parse_qs, unquote, urljoin, urlparse
+from urllib.parse import parse_qs, urlencode, unquote, urljoin, urlparse, urlunparse
 
 from bs4 import BeautifulSoup
 
@@ -21,7 +21,7 @@ def discover_links(html: str, base_url: str, limit: int = 30) -> list[Discovered
         raw_href = anchor.get("href", "")
         if should_skip_raw_href(raw_href):
             continue
-        url = normalize_search_redirect(urljoin(base_url, raw_href))
+        url = normalize_cafe24_product_url(normalize_search_redirect(urljoin(base_url, raw_href)))
         if not url or url in seen or not url.startswith(("http://", "https://")) or should_skip_url(url):
             continue
         seen.add(url)
@@ -58,12 +58,18 @@ def should_skip_url(url: str) -> bool:
     skip_path_tokens = [
         "/member/",
         "/order/",
+        "/myshop/",
+        "/event/list",
         "/exec/front/newcoupon/",
+        "/board/free/list",
+        "/board/faq/list",
         "/board/free/modify",
         "/board/free/reply",
     ]
     if any(token in path for token in skip_path_tokens):
         return True
+    if path.endswith("/product/search.html"):
+        return True
     if "facebook.com/" in path or "instagram.com/" in path:
         return True
     if "coupon_no=" in query:
@@ -82,6 +88,33 @@ def normalize_search_redirect(url: str) -> str:
     return url
 
 
+def normalize_cafe24_product_url(url: str) -> str:
+    parsed = urlparse(url)
+    path = unquote(parsed.path)
+    parts = [part for part in path.strip("/").split("/") if part]
+    query = parse_qs(parsed.query)
+
+    if path.endswith("/product/detail.html") and query.get("product_no"):
+        return urlunparse(
+            parsed._replace(
+                query=urlencode({"product_no": query["product_no"][0]}),
+                fragment="",
+            )
+        )
+
+    if parts and parts[0] == "product":
+        product_no = next((part for part in parts[1:] if part.isdigit()), None)
+        if product_no:
+            return urlunparse(
+                parsed._replace(
+                    path="/product/detail.html",
+                    query=urlencode({"product_no": product_no}),
+                    fragment="",
+                )
+            )
+    return url
+
+
 def classify_url(url: str) -> str:
     host = urlparse(url).netloc.lower()
     if "smartstore.naver.com" in host or "brand.naver.com" in host:

crawler_platform/app/core/crawler/fetchers.py

@@ -42,6 +42,16 @@ class FetchResult:
         return self.rendered_html or self.raw_html or self.html
 
 
+@dataclass(slots=True)
+class RobotsDecision:
+    allowed: bool
+    checked: bool
+    status: str
+    reason: str
+    robots_url: str | None = None
+    user_agent: str = DEFAULT_USER_AGENT
+
+
 class RateLimiter:
     def __init__(self, per_minute: int = 30):
         self.delay = 60 / max(per_minute, 1)
@@ -60,11 +70,26 @@ class RobotsPolicy:
         self._cache: dict[str, RobotFileParser] = {}
 
     def allowed(self, url: str, respect_robots_txt: bool = True) -> bool:
+        return self.check(url, respect_robots_txt).allowed
+
+    def check(self, url: str, respect_robots_txt: bool = True) -> RobotsDecision:
         if not respect_robots_txt:
-            return True
+            return RobotsDecision(
+                allowed=True,
+                checked=False,
+                status="disabled",
+                reason="robots.txt check disabled by source/request config",
+                user_agent=self.user_agent,
+            )
         parsed = urlparse(url)
         if parsed.scheme in {"", "file"}:
-            return True
+            return RobotsDecision(
+                allowed=True,
+                checked=False,
+                status="local",
+                reason="robots.txt is not applicable to local or scheme-less URLs",
+                user_agent=self.user_agent,
+            )
         robots_url = f"{parsed.scheme}://{parsed.netloc}/robots.txt"
         parser = self._cache.get(robots_url)
         if parser is None:
@@ -72,10 +97,29 @@ class RobotsPolicy:
             parser.set_url(robots_url)
             try:
                 parser.read()
-            except Exception:
-                return False
+            except Exception as exc:
+                return RobotsDecision(
+                    allowed=True,
+                    checked=True,
+                    status="unavailable",
+                    reason=f"robots.txt unavailable ({exc.__class__.__name__}); allowing crawl",
+                    robots_url=robots_url,
+                    user_agent=self.user_agent,
+                )
             self._cache[robots_url] = parser
-        return parser.can_fetch(self.user_agent, url)
+        allowed = parser.can_fetch(self.user_agent, url)
+        return RobotsDecision(
+            allowed=allowed,
+            checked=True,
+            status="allowed" if allowed else "blocked",
+            reason=(
+                "robots.txt allows crawling"
+                if allowed
+                else f"robots.txt blocks crawling for user-agent {self.user_agent}"
+            ),
+            robots_url=robots_url,
+            user_agent=self.user_agent,
+        )
 
 
 class BaseFetcher:
@@ -83,6 +127,23 @@ class BaseFetcher:
         raise NotImplementedError
 
 
+class FallbackFetcher(BaseFetcher):
+    def __init__(self, primary: BaseFetcher, fallback: BaseFetcher, fallback_label: str = "fallback"):
+        self.primary = primary
+        self.fallback = fallback
+        self.fallback_label = fallback_label
+
+    def fetch(self, url: str) -> FetchResult:
+        try:
+            return self.primary.fetch(url)
+        except Exception as exc:
+            result = self.fallback.fetch(url)
+            result.warnings.append(
+                f"primary fetcher failed ({exc.__class__.__name__}: {exc}); used {self.fallback_label}"
+            )
+            return result
+
+
 class RequestsFetcher(BaseFetcher):
     def __init__(
         self,
@@ -170,7 +231,11 @@ class PlaywrightFetcher(BaseFetcher):
 
 def make_fetcher(kind: str, rate_limit_per_minute: int = 30) -> BaseFetcher:
     if kind in {"playwright", "browser"}:
-        return PlaywrightFetcher()
+        return FallbackFetcher(
+            PlaywrightFetcher(),
+            RequestsFetcher(rate_limit_per_minute=rate_limit_per_minute),
+            fallback_label="requests",
+        )
     return RequestsFetcher(rate_limit_per_minute=rate_limit_per_minute)
 
 

crawler_platform/app/core/crawler/page_classifier.py

@@ -49,7 +49,7 @@ def classify_page(
     if any(token in path for token in ["shopinfo", "company", "about", "brand-story", "brand_story"]):
         return "BrandStoryPage"
     if any(token in path for token in ["product/detail", "/product/", "/products/", "/goods/", "/item/"]):
-        if _looks_like_category_path(path, combined):
+        if _looks_like_category_path(path, combined) and not _looks_like_product_detail_path(path):
             return "CategoryPage"
         return "ProductPage"
     if any(token in path for token in ["category", "/collections", "/collection", "/shop/", "/list"]):
@@ -142,6 +142,14 @@ def _has_product_detail_signal(text: str) -> bool:
     return note_count >= 1 or commerce_count >= 2
 
 
+def _looks_like_product_detail_path(path: str) -> bool:
+    parts = [part for part in path.strip("/").split("/") if part]
+    if "product" not in parts:
+        return False
+    product_index = parts.index("product")
+    return any(part.isdigit() for part in parts[product_index + 1 :])
+
+
 def _looks_like_category_path(path: str, text: str) -> bool:
     category_tokens = ["category", "cate_no", "display_group", "sort_method"]
     if any(token in path or token in text for token in category_tokens):

crawler_platform/app/core/crawler/pipeline.py

@@ -22,6 +22,8 @@ class CrawlResult:
     clean_text_length: int = 0
     raw_text_length: int = 0
     warnings: list[str] | None = None
+    robots_status: str = "unchecked"
+    robots_reason: str | None = None
 
 
 class CrawlPipeline:
@@ -39,8 +41,9 @@ class CrawlPipeline:
 
     def crawl_url(self, project_config: ProjectConfig, source_name: str, url: str) -> CrawlResult:
         source_config = project_config.source_by_name(source_name)
-        if not self.robots_policy.allowed(url, source_config.respect_robots_txt):
-            raise PermissionError(f"robots.txt does not allow crawling: {url}")
+        robots_decision = self.robots_policy.check(url, source_config.respect_robots_txt)
+        if not robots_decision.allowed:
+            raise PermissionError(f"{robots_decision.reason}: {url}")
 
         fetcher = make_fetcher(source_config.fetcher, source_config.rate_limit_per_minute)
         fetch_result = fetcher.fetch(url)
@@ -87,6 +90,8 @@ class CrawlPipeline:
                 clean_text_length=len(parsed.text or ""),
                 raw_text_length=len(parsed.raw_text or ""),
                 warnings=warnings,
+                robots_status=robots_decision.status,
+                robots_reason=robots_decision.reason,
             )
 
         context = ExtractionPageContext(
@@ -117,4 +122,6 @@ class CrawlPipeline:
             clean_text_length=len(parsed.text or ""),
             raw_text_length=len(parsed.raw_text or ""),
             warnings=warnings,
+            robots_status=robots_decision.status,
+            robots_reason=robots_decision.reason,
         )

crawler_platform/app/core/crawler/site_crawler.py

@@ -7,7 +7,7 @@ from urllib.parse import urldefrag, urlparse
 
 from crawler_platform.app.config.loader import ProjectConfig
 from crawler_platform.app.core.crawler.discovery import discover_links
-from crawler_platform.app.core.crawler.fetchers import RobotsPolicy, make_fetcher
+from crawler_platform.app.core.crawler.fetchers import RobotsDecision, RobotsPolicy, make_fetcher
 from crawler_platform.app.core.crawler.page_classifier import (
     classify_page as classify_page_type,
     should_analyze_page as should_analyze_page_type,
@@ -35,6 +35,8 @@ class SiteCrawlPageResult:
     clean_text_length: int = 0
     removed_noise_zones_count: int = 0
     warnings: list[str] = field(default_factory=list)
+    robots_status: str = "unchecked"
+    robots_reason: str | None = None
     error: str | None = None
 
 
@@ -120,14 +122,23 @@ class SiteCrawler:
                     progress_callback,
                 )
                 continue
-            if not self.robots_policy.allowed(url, source_config.respect_robots_txt):
-                error = f"robots.txt does not allow crawling: {url}"
+            robots_decision = self.robots_policy.check(url, source_config.respect_robots_txt)
+            if not robots_decision.allowed:
+                error = f"{robots_decision.reason}: {url}"
                 self._finish_job(job, "blocked", error)
                 result.skipped_count += 1
                 result.errors.append(error)
                 self._record_page_result(
                     result,
-                    SiteCrawlPageResult(url=url, depth=depth, status="blocked", page_type="unknown", error=error),
+                    SiteCrawlPageResult(
+                        url=url,
+                        depth=depth,
+                        status="blocked",
+                        page_type="unknown",
+                        robots_status=robots_decision.status,
+                        robots_reason=robots_decision.reason,
+                        error=error,
+                    ),
                     len(queue),
                     progress_callback,
                 )
@@ -150,6 +161,7 @@ class SiteCrawler:
                     analyze_page_types=analyze_page_types,
                     result=result,
                     job=job,
+                    robots_decision=robots_decision,
                     progress_callback=progress_callback,
                 )
             except Exception as exc:
@@ -184,6 +196,7 @@ class SiteCrawler:
         analyze_page_types: set[str],
         result: SiteCrawlResult,
         job: models.CrawlJob,
+        robots_decision: RobotsDecision,
         progress_callback: Callable[[SiteCrawlResult, SiteCrawlPageResult], None] | None,
     ) -> None:
         fetch_result = fetcher.fetch(url)
@@ -215,6 +228,8 @@ class SiteCrawler:
                     page_type="unknown",
                     page_id=page.id,
                     crawl_status=fetch_result.crawl_status,
+                    robots_status=robots_decision.status,
+                    robots_reason=robots_decision.reason,
                     warnings=fetch_result.warnings,
                     error=error,
                 ),
@@ -277,6 +292,8 @@ class SiteCrawler:
             "clean_text_length": len(parsed.text or ""),
             "removed_noise_zones_count": int(parsed.metadata.get("removed_noise_zones_count") or 0),
             "warnings": warnings,
+            "robots_status": robots_decision.status,
+            "robots_reason": robots_decision.reason,
         }
 
         if parsed.extraction_status == "failed":

crawler_platform/app/core/database/repository.py

@@ -185,7 +185,7 @@ class KnowledgeRepository:
         else:
             claim_status = "active"
 
-        if claim_status not in {"active", "validated_claim"}:
+        if claim_status not in {"active", "validated_claim", "candidate_claim", "rule_candidate"}:
             self._log_extraction(project_id, page, bundle)
             return []
 

crawler_platform/app/core/extractor/ai_provider.py

@@ -64,6 +64,9 @@ class LLMJsonExtractor(AIExtractor):
             try:
                 raw = self.complete_json(page_text, project_config, compact=compact_mode, context=context)
                 bundle = self._bundle_from_raw(raw, mode_name)
+                enriched = self._merge_rule_fallback_claims(bundle, page_text, project_config, mode_name)
+                if enriched.entities and enriched.claims:
+                    return self.normalize_to_ontology(enriched, project_config.ontology)
                 if bundle.entities and bundle.claims:
                     return self.normalize_to_ontology(bundle, project_config.ontology)
                 errors.append(f"{mode_name}: AI returned no usable entities or claims")
@@ -103,12 +106,18 @@ class LLMJsonExtractor(AIExtractor):
         context: ExtractionPageContext | None = None,
     ) -> dict[str, Any]:
         if self.provider == "lm_studio":
-            char_limit = 1200 if compact else 2200
-            max_tokens = 220 if compact else 420
+            char_limit = 700 if compact else 1100
+            max_tokens = 260 if compact else 360
         else:
             char_limit = 2200 if compact else 4000
             max_tokens = 400 if compact else 800
-        prompt = build_extraction_prompt(page_text, project_config, char_limit=char_limit, context=context)
+        prompt = build_extraction_prompt(
+            page_text,
+            project_config,
+            char_limit=char_limit,
+            context=context,
+            compact=self.provider == "lm_studio",
+        )
         if self.provider == "openai":
             return self._complete_openai_compatible(
                 prompt,
@@ -154,6 +163,45 @@ class LLMJsonExtractor(AIExtractor):
             claim.confidence_reason = f"{claim.confidence_reason}; AI fallback: {error}" if claim.confidence_reason else error
         return bundle
 
+    def _merge_rule_fallback_claims(
+        self,
+        bundle: ExtractionBundle,
+        page_text: str,
+        project_config: ProjectConfig,
+        mode: str,
+    ) -> ExtractionBundle:
+        rule_bundle = self._rule_bundle(page_text, project_config)
+        if not rule_bundle.claims:
+            return bundle
+        merged = ExtractionBundle(
+            entities=dedupe_entities([*bundle.entities, *rule_bundle.entities]),
+            claims=dedupe_claims([*bundle.claims, *rule_bundle.claims]),
+            extractor_name=f"{self.name}_with_rule_claims",
+            provider=self.provider,
+            raw_output={
+                **bundle.raw_output,
+                "extraction_mode": mode,
+                "rule_claim_merge": True,
+                "rule_entity_count": len(rule_bundle.entities),
+                "rule_claim_count": len(rule_bundle.claims),
+            },
+        )
+        for claim in merged.claims:
+            claim.metadata.setdefault("rule_claim_merge", True)
+            claim.confidence_reason = (
+                f"{claim.confidence_reason}; AI entity output enriched with rule claims"
+                if claim.confidence_reason
+                else "AI entity output enriched with rule claims"
+            )
+        return merged
+
+    def _rule_bundle(self, page_text: str, project_config: ProjectConfig) -> ExtractionBundle:
+        if project_config.domain == "perfume":
+            return PerfumeRuleBasedExtractor().extract(page_text, project_config)
+        from crawler_platform.app.core.extractor.rule_based import GenericRuleBasedExtractor
+
+        return GenericRuleBasedExtractor().extract(page_text, project_config)
+
     def _bundle_from_raw(self, raw: dict[str, Any], mode: str) -> ExtractionBundle:
         return ExtractionBundle(
             entities=parse_entities(raw.get("entities", [])),
@@ -275,7 +323,10 @@ def build_extraction_prompt(
     project_config: ProjectConfig,
     char_limit: int = 4000,
     context: ExtractionPageContext | None = None,
+    compact: bool = False,
 ) -> str:
+    if compact:
+        return build_compact_extraction_prompt(page_text, project_config, char_limit=char_limit, context=context)
     if context is not None:
         prompt_input = json.dumps(context.to_payload(text_limit=char_limit), ensure_ascii=False, indent=2)
     else:
@@ -338,6 +389,35 @@ Input payload:
 """.strip()
 
 
+def build_compact_extraction_prompt(
+    page_text: str,
+    project_config: ProjectConfig,
+    char_limit: int = 1000,
+    context: ExtractionPageContext | None = None,
+) -> str:
+    if context is not None:
+        text = prepare_page_text_for_prompt(context.clean_text, char_limit)
+        title = context.title or ""
+        page_type = context.page_type
+        url = context.final_url or context.url
+    else:
+        text = prepare_page_text_for_prompt(page_text, char_limit)
+        title = ""
+        page_type = "UnknownPage"
+        url = ""
+    predicates = ", ".join((project_config.ontology or {}).get("predicates", [])[:12])
+    return (
+        "Return minified JSON only: {\"entities\":[],\"claims\":[]}.\n"
+        f"Domain perfume. PageType={page_type}. URL={url}. Title={title}\n"
+        "Entity types: Perfume, Brand, Note, Accord, Mood, Season, Occasion, Price.\n"
+        f"Predicates: {predicates}.\n"
+        "Extract only explicit product facts. Max 6 entities, 8 claims. "
+        "Every claim needs short evidence_text from text. "
+        "Use null for missing object_name/object_type/object_value.\n"
+        f"Text:\n{text}"
+    )
+
+
 def prepare_page_text_for_prompt(page_text: str, char_limit: int) -> str:
     noisy_terms = {
         "first page",
@@ -567,6 +647,36 @@ def parse_entities(items: list[dict[str, Any]]) -> list[ExtractedEntity]:
     return entities
 
 
+def dedupe_entities(entities: list[ExtractedEntity]) -> list[ExtractedEntity]:
+    seen: set[tuple[str, str]] = set()
+    result: list[ExtractedEntity] = []
+    for entity in entities:
+        key = (entity.entity_type.strip().lower(), entity.name.strip().lower())
+        if key in seen:
+            continue
+        seen.add(key)
+        result.append(entity)
+    return result
+
+
+def dedupe_claims(claims: list[ExtractedClaim]) -> list[ExtractedClaim]:
+    seen: set[tuple[str, str, str, str]] = set()
+    result: list[ExtractedClaim] = []
+    for claim in claims:
+        object_key = claim.object_name or json.dumps(claim.object_value, ensure_ascii=False, sort_keys=True, default=str)
+        key = (
+            claim.subject_name.strip().lower(),
+            claim.subject_type.strip().lower(),
+            claim.predicate.strip(),
+            str(object_key).strip().lower(),
+        )
+        if key in seen:
+            continue
+        seen.add(key)
+        result.append(claim)
+    return result
+
+
 def parse_claims(items: list[dict[str, Any]]) -> list[ExtractedClaim]:
     claims: list[ExtractedClaim] = []
     for item in items:

crawler_platform/app/core/research/graph_research_loop.py

@@ -200,8 +200,9 @@ class GraphResearchLoop:
         url = item.target
         if same_domain_only and seed_host and normalized_host(url) != seed_host:
             return {"status": "skipped", "reason": "outside same-domain research boundary"}
-        if not self.robots_policy.allowed(url, source_config.respect_robots_txt):
-            return {"status": "skipped", "reason": f"robots.txt blocked {url}"}
+        robots_decision = self.robots_policy.check(url, source_config.respect_robots_txt)
+        if not robots_decision.allowed:
+            return {"status": "skipped", "reason": f"{robots_decision.reason}: {url}"}
 
         fetch_result = fetcher.fetch(url)
         parser_result = parser.parse(fetch_result.analysis_html, fetch_result.final_url or url)
@@ -229,6 +230,8 @@ class GraphResearchLoop:
             "final_url": fetch_result.final_url,
             "crawl_status": fetch_result.crawl_status,
             "page_type": page_type,
+            "robots_status": robots_decision.status,
+            "robots_reason": robots_decision.reason,
             "raw_text_length": len(parser_result.raw_text or ""),
             "clean_text_length": len(parser_result.text or ""),
             "main_content_preview": (parser_result.main_content or parser_result.text)[:800],

crawler_platform/app/domains/perfume/extractor.py

@@ -118,7 +118,7 @@ class PerfumeRuleBasedExtractor(GenericRuleBasedExtractor):
         price = find_price(page_text)
         if price:
             attrs["price"] = {k: v for k, v in price.items() if k != "evidence"}
-        entities = [ExtractedEntity("Perfume", product_name, attrs, confidence=0.68)]
+        entities = [ExtractedEntity("Perfume", product_name, attrs, confidence=0.74)]
         if brand:
             entities.append(ExtractedEntity("Brand", brand, confidence=0.62))
         for field, entity_type in [
@@ -187,7 +187,7 @@ class PerfumeRuleBasedExtractor(GenericRuleBasedExtractor):
                             "hasPrice",
                             object_value=card["price"],
                             evidence_text=str(card.get("evidence") or card["name"]),
-                            confidence=0.76,
+                            confidence=0.84,
                             confidence_reason="Korean product listing price pattern matched",
                         )
                     )
@@ -236,7 +236,7 @@ class PerfumeRuleBasedExtractor(GenericRuleBasedExtractor):
                     "hasPrice",
                     object_value={k: v for k, v in price.items() if k != "evidence"},
                     evidence_text=price["evidence"],
-                    confidence=0.7,
+                    confidence=0.86,
                     confidence_reason="price pattern matched",
                 )
             )
@@ -249,10 +249,19 @@ class PerfumeRuleBasedExtractor(GenericRuleBasedExtractor):
 
 
 def extract_product_name(page_text: str) -> str:
-    for line in page_text.splitlines()[:8]:
+    candidates: list[tuple[int, str]] = []
+    for line in page_text.splitlines()[:12]:
         clean = line.strip()
-        if clean and not looks_like_navigation(clean) and not is_template_placeholder(clean):
-            return clean[:240]
+        if not clean or looks_like_navigation(clean) or is_template_placeholder(clean):
+            continue
+        if looks_like_metric_or_price(clean):
+            continue
+        candidates.append((product_line_score(clean), clean[:240]))
+    strong = [candidate for candidate in candidates if candidate[0] > 0]
+    if strong:
+        return max(strong, key=lambda item: item[0])[1]
+    if candidates:
+        return candidates[0][1]
     return first_non_empty_line(page_text) or "Unknown Perfume"
 
 
@@ -265,14 +274,40 @@ def extract_brand(page_text: str, product_name: str) -> str | None:
         match = re.search(pattern, page_text, flags=re.IGNORECASE)
         if match:
             return cleanup_value(match.group("brand"))
+    inferred = infer_site_brand(page_text)
+    if inferred:
+        return inferred
     lines = [line.strip() for line in page_text.splitlines() if line.strip()]
     if len(lines) >= 2 and lines[1].lower() not in product_name.lower():
         candidate = cleanup_value(lines[1])
-        if len(candidate) <= 80 and not looks_like_navigation(candidate):
+        if len(candidate) <= 80 and not looks_like_navigation(candidate) and product_line_score(candidate) <= 0:
             return candidate
     return None
 
 
+def product_line_score(value: str) -> int:
+    lower = value.lower()
+    score = 0
+    if re.search(r"\d+\s*(?:ml|g|개입)", lower):
+        score += 4
+    if any(keyword in value for keyword in ["향수", "디퓨저", "스프레이", "핸드크림", "미스트", "샤쉐", "퍼퓸"]):
+        score += 3
+    if value.startswith("[") or any(keyword in value for keyword in ["기획", "추가할인", "모음"]):
+        score += 2
+    if "912" in value:
+        score += 2
+    if "시작" in value and not re.search(r"\d+\s*(?:ml|g|개입)", lower):
+        score -= 4
+    return score
+
+
+def looks_like_metric_or_price(value: str) -> bool:
+    clean = value.replace(",", "").strip()
+    if re.fullmatch(r"\d+(?:\.\d+)?", clean):
+        return True
+    return bool(re.fullmatch(r"\d+(?:\.\d+)?\s*(?:원|krw|usd)?", clean, flags=re.IGNORECASE))
+
+
 def extract_product_cards(page_text: str) -> list[dict[str, object]]:
     lines = [line.strip() for line in page_text.splitlines() if line.strip()]
     cards: list[dict[str, object]] = []

crawler_platform/app/web/frontend/src/i18n.js

@@ -26,6 +26,7 @@ const dict = {
     "sidebar.max_depth": "최대 깊이",
     "sidebar.max_pages": "최대 페이지",
     "sidebar.same_domain": "같은 도메인만",
+    "sidebar.respect_robots": "robots.txt 준수",
     "sidebar.crawl_site": "시드부터 사이트 크롤",
     "sidebar.stop_crawl": "현재 크롤 중지",
 
@@ -262,6 +263,7 @@ const dict = {
     "sidebar.max_depth": "Max depth",
     "sidebar.max_pages": "Max pages",
     "sidebar.same_domain": "Same domain",
+    "sidebar.respect_robots": "Respect robots.txt",
     "sidebar.crawl_site": "Crawl site from seed",
     "sidebar.stop_crawl": "Stop current crawl",
 

crawler_platform/app/web/frontend/src/sidebar.js

@@ -85,6 +85,10 @@ function sidebarHtml() {
           <input id="sameDomainOnly" type="checkbox" checked />
           <span data-i18n="sidebar.same_domain"></span>
         </label>
+        <label class="check-row">
+          <input id="respectRobotsTxt" type="checkbox" />
+          <span data-i18n="sidebar.respect_robots"></span>
+        </label>
       </div>
       <button id="siteCrawlBtn" class="primary full" data-i18n="sidebar.crawl_site"></button>
       <button id="stopSiteCrawlBtn" class="full" disabled data-i18n="sidebar.stop_crawl"></button>
@@ -102,6 +106,7 @@ export function requestBase() {
     extractor_provider: $("extractorProvider").value,
     extractor_model: $("extractorModel").value.trim() || null,
     extractor_base_url: $("extractorBaseUrl").value.trim() || null,
+    check_robots_txt: $("respectRobotsTxt")?.checked ?? false,
   };
 }
 
@@ -331,6 +336,7 @@ async function discover() {
         source_name: $("sourceSelect").value,
         url: $("crawlUrl").value.trim(),
         limit: 30,
+        check_robots_txt: $("respectRobotsTxt")?.checked ?? false,
       }),
     });
     if (!result.ok) {

crawler_platform/app/web/static/index.html

@@ -4,7 +4,7 @@
     <meta charset="utf-8" />
     <meta name="viewport" content="width=device-width, initial-scale=1" />
     <title>Ontology Crawler Platform</title>
-    <script type="module" crossorigin src="/static/assets/index-fQcYiMpd.js"></script>
+    <script type="module" crossorigin src="/static/assets/index-BsoXaTIL.js"></script>
     <link rel="stylesheet" crossorigin href="/static/assets/index-BkBg_FAU.css">
   </head>
   <body>