fds

.gitignore

@@ -7,4 +7,7 @@ __pycache__/
 *.sqlite3-*
 uvicorn.*.log
 .server-logs/
+.env
+.vs
+ontology_platform/.env
 /ontology_platform/data/ui_projects.json

docs/PHASE_PLANNING.md

@@ -2,12 +2,13 @@ PHASE_PLANNING.md
 
 요청된 작업을 다음 순서로 진행한다.
 
-0. 이전 PHASE_INDEX.md 다른이름 으로 변경. (예 : 26_05_19_PHASE_INDEX.md)
-1. 요청된 작업 폴더 생성 (예 : 26_05_19_기능개선 )
-2. 해당 작업을 분석후 Phase 계획
-3. Phase별로 상세 작업 계획을 세운후 PHASE_INDEX.md 새로 생성후 작성. 작성 방법은 아래 # PHASE INDEX EXAMPLE 참조
-4. Phase별 상세작업 계획은 1에 의해 생성된 폴더아래  파일을 만들어 저장( 예 : phase_01_001_pipeline.md)
-5. 생성된 파일은 PHASE_INDEX.md의 해당 PHASE의 FILE: 에 기록
+0. 이전 PHASE_INDEX.md 파일은 백업을 목적으로 다른이름 으로 변경. (예 : 26_05_19_PHASE_INDEX.md)
+1. 새로운 PHASE_INDEX.md 파일 생성. 이전 내용은 완전히 삭제된 상태로 시작.
+2. 새로운 작업은 반드시 새로운 폴더를 생성해서 요청된 작업 폴더 생성 (예 : 26_05_19_기능개선 )
+3. 해당 작업을 분석후 Phase 계획
+4. Phase별로 상세 작업 계획을 세운후 PHASE_INDEX.md 파일에 작성. 작성 방법은 아래 # PHASE INDEX EXAMPLE 참조. 세부 항목의 끝에는 반드시 [TODO] 표시를 해서 작업이 완료되지 않았음을 명확히 한다. (예 : 1) Source Crawl, Page Clean [TODO])
+5. Phase별 상세작업 계획은 2에 의해 생성된 폴더아래  파일을 만들어 저장( 예 : phase_01_001_pipeline.md)
+6. 5번 항목에 의해 생성된 파일은 PHASE_INDEX.md의 해당 PHASE의 FILE: 에 기록해 참조 하도록 한다. (예 : FILE: ./26_05_19_기능개선/phase_01_001_pipeline.md)
 
 
 # PHASE INDEX EXAMPLE

docs/PHASE_WORKFLOW.md

@@ -64,7 +64,7 @@ phase 파일은 상태 기록 용도로 사용하지 않는다.
 
 를 의미한다.
 
-이 경우 반드시 작업 전,후 PHASE_INDEX.md에 아래 예처럼 기록한다.
+이 경우 반드시 작업 전,후 PHASE_INDEX.md파일의 해당 phase의 세부항목인 [진행중] 항목 아래에 다음 예 처럼 기록한다.
 
 PHASE 3. Claim Review 개선
 FILE: ./26_05_19_기능개선/phase_03_002_metadata_display.md

ontology_platform/crawler_platform/app/api/routes.py

@@ -4,12 +4,14 @@ import csv
 import io
 import json
 import re
+import time
 from dataclasses import asdict
 from typing import Any
 
 from fastapi import BackgroundTasks, HTTPException, Response
 from pydantic import BaseModel, Field
 from sqlalchemy import select
+from sqlalchemy.exc import OperationalError
 
 from crawler_platform.app.config.loader import (
     ProjectConfig,
@@ -27,7 +29,10 @@ from crawler_platform.app.core.database.repository import (
     make_claim_hash,
 )
 from crawler_platform.app.core.database.session import session_scope
-from crawler_platform.app.core.extractor.ai_provider import list_openai_compatible_models
+from crawler_platform.app.core.extractor.ai_provider import (
+    list_lmstudio_loaded_models,
+    list_openai_compatible_models,
+)
 from crawler_platform.app.core.extractor.factory import extractor_for_domain
 from crawler_platform.app.core.ontology.definitions import DOMAIN_ONTOLOGIES, Ontology, ontology_for_domain
 from crawler_platform.app.core.ontology.domain_discovery import DomainDiscoveryService
@@ -41,15 +46,112 @@ from crawler_platform.app.core.research.memory_store import ResearchMemoryStore,
 
 
 DOMAIN_NAME_PATTERN = re.compile(r"^[a-z0-9][a-z0-9_-]{1,79}$")
+SITE_CRAWL_CANCEL_REQUESTS: set[int] = set()
+COMPARISON_KEYS = ("both_agree", "rule_only", "llm_only", "conflict", "rejected_by_validation")
+
+
+def extraction_log_summary(raw_output: dict[str, Any]) -> dict[str, Any]:
+    candidate_claims = raw_output.get("candidate_claims")
+    if not isinstance(candidate_claims, list):
+        candidate_claims = []
+
+    raw_comparison = raw_output.get("comparison")
+    if not isinstance(raw_comparison, dict):
+        raw_comparison = {}
+    comparison = {key: int(raw_comparison.get(key) or 0) for key in COMPARISON_KEYS}
+    if not any(comparison.values()):
+        comparison.update(comparison_from_candidate_claims(candidate_claims))
+    validation = raw_output.get("validation")
+    if isinstance(validation, dict) and comparison["rejected_by_validation"] == 0:
+        comparison["rejected_by_validation"] = number_or_default(validation.get("rejected_claim_count"), 0)
+
+    return {
+        "candidate_count": len(candidate_claims),
+        "comparison": comparison,
+        "rule_entity_count": number_or_none(raw_output.get("rule_entity_count")),
+        "rule_claim_count": number_or_derived(
+            raw_output.get("rule_claim_count"),
+            candidate_claims,
+            source="rule",
+        ),
+        "llm_entity_count": number_or_none(raw_output.get("llm_entity_count")),
+        "llm_claim_count": number_or_derived(
+            raw_output.get("llm_claim_count"),
+            candidate_claims,
+            source="llm",
+        ),
+        "agreement_claim_count": number_or_default(raw_output.get("agreement_claim_count"), comparison["both_agree"]),
+        "rule_only_claim_count": number_or_default(raw_output.get("rule_only_claim_count"), comparison["rule_only"]),
+        "llm_only_claim_count": number_or_default(raw_output.get("llm_only_claim_count"), comparison["llm_only"]),
+        "conflict_claim_count": number_or_default(raw_output.get("conflict_claim_count"), comparison["conflict"]),
+    }
+
+
+def comparison_from_candidate_claims(candidate_claims: list[Any]) -> dict[str, int]:
+    comparison = {key: 0 for key in COMPARISON_KEYS}
+    for claim in candidate_claims:
+        metadata = claim_metadata(claim)
+        agreement = str(metadata.get("agreement") or metadata.get("claim_kind") or "").lower()
+        if agreement == "rule_and_llm":
+            comparison["both_agree"] += 1
+        elif agreement == "rule_only":
+            comparison["rule_only"] += 1
+        elif agreement == "llm_only":
+            comparison["llm_only"] += 1
+        elif agreement == "conflict":
+            comparison["conflict"] += 1
+    return comparison
+
+
+def claim_metadata(claim: Any) -> dict[str, Any]:
+    if not isinstance(claim, dict):
+        return {}
+    metadata = claim.get("metadata")
+    return metadata if isinstance(metadata, dict) else {}
+
+
+def number_or_none(value: Any) -> int | None:
+    if isinstance(value, bool):
+        return None
+    if isinstance(value, (int, float)):
+        return int(value)
+    return None
+
+
+def number_or_default(value: Any, default: int) -> int:
+    parsed = number_or_none(value)
+    return default if parsed is None else parsed
+
+
+def number_or_derived(value: Any, candidate_claims: list[Any], *, source: str) -> int:
+    parsed = number_or_none(value)
+    if parsed is not None:
+        return parsed
+    return sum(1 for claim in candidate_claims if claim_matches_source(claim, source))
+
+
+def claim_matches_source(claim: Any, source: str) -> bool:
+    metadata = claim_metadata(claim)
+    extraction_source = str(metadata.get("extraction_source") or "").lower()
+    agreement = str(metadata.get("agreement") or "").lower()
+    if extraction_source == source:
+        return True
+    if source == "rule":
+        return agreement in {"rule_only", "rule_and_llm"}
+    if source == "llm":
+        return agreement in {"llm_only", "rule_and_llm"}
+    return False
 
 
 class CrawlRequest(BaseModel):
     config_path: str
     source_name: str
     url: str
+    extraction_mode: str = "hybrid"
     extractor_provider: str = "lm_studio"
     extractor_model: str | None = None
     extractor_base_url: str | None = "http://localhost:1234/v1"
+    fallback_to_rules: bool = True
     check_robots_txt: bool = False
     respect_robots_txt: bool | None = None
 
@@ -67,9 +169,11 @@ class SiteCrawlByProjectRequest(BaseModel):
     project_name: str
     source_name: str
     url: str
+    extraction_mode: str = "hybrid"
     extractor_provider: str = "lm_studio"
     extractor_model: str | None = None
     extractor_base_url: str | None = "http://localhost:1234/v1"
+    fallback_to_rules: bool = True
     check_robots_txt: bool = False
     respect_robots_txt: bool | None = None
     max_depth: int = 2
@@ -85,9 +189,11 @@ class SiteCrawlByProjectRequest(BaseModel):
             config_path=config_path_placeholder,
             source_name=self.source_name,
             url=self.url,
+            extraction_mode=self.extraction_mode,
             extractor_provider=self.extractor_provider,
             extractor_model=self.extractor_model,
             extractor_base_url=self.extractor_base_url,
+            fallback_to_rules=self.fallback_to_rules,
             check_robots_txt=self.check_robots_txt,
             respect_robots_txt=self.respect_robots_txt,
             max_depth=self.max_depth,
@@ -535,9 +641,11 @@ class ResearchRunByProjectRequest(BaseModel):
     url: str | None = None
     seed_entity_id: int | None = None
     goal: str = "Semantic ontology exploration"
+    extraction_mode: str = "hybrid"
     extractor_provider: str = "lm_studio"
     extractor_model: str | None = None
     extractor_base_url: str | None = "http://localhost:1234/v1"
+    fallback_to_rules: bool = True
     check_robots_txt: bool = False
     respect_robots_txt: bool | None = None
     max_depth: int = 2
@@ -622,6 +730,20 @@ def site_crawl_progress_payload(result, latest_page=None) -> dict[str, Any]:
     payload = asdict(result)
     if latest_page is not None:
         payload["latest_page"] = asdict(latest_page)
+    pages = payload.get("pages") or []
+    payload["extraction_summary"] = {
+        "llm_skipped_count": sum(1 for page in pages if page.get("llm_skipped")),
+        "fallback_count": sum(1 for page in pages if page.get("fallback_used")),
+        "conflict_claim_count": sum(int(page.get("conflict_claim_count") or 0) for page in pages),
+        "agreement_claim_count": sum(int(page.get("agreement_claim_count") or 0) for page in pages),
+        "llm_call_count": sum(
+            1
+            for page in pages
+            if page.get("extraction_mode") in {"hybrid", "llm_only", "compare"}
+            and not page.get("llm_skipped")
+            and not page.get("fallback_used")
+        ),
+    }
     return payload
 
 
@@ -696,7 +818,58 @@ def update_site_crawl_job_metadata(job: models.CrawlJob, **updates: Any) -> None
     job.metadata_json = metadata
 
 
+def mark_orphan_site_crawl_jobs_canceled(database_url: str) -> None:
+    with session_scope(database_url) as session:
+        rows = session.scalars(
+            select(models.CrawlJob).where(
+                models.CrawlJob.status.in_(["running", "cancel_requested"]),
+            )
+        ).all()
+        for job in rows:
+            if (job.metadata_json or {}).get("kind") != "site_crawl":
+                continue
+            job.status = "canceled"
+            job.error = job.error or "canceled after server restart/interrupted crawl"
+            job.finished_at = models.utcnow()
+
+
+def request_site_crawl_cancel(database_url: str, job_id: int) -> dict[str, Any]:
+    SITE_CRAWL_CANCEL_REQUESTS.add(job_id)
+    last_error: Exception | None = None
+    for attempt in range(3):
+        try:
+            with session_scope(database_url) as session:
+                job = session.get(models.CrawlJob, job_id)
+                if job is None or (job.metadata_json or {}).get("kind") != "site_crawl":
+                    raise HTTPException(status_code=404, detail="site crawl job not found")
+                if job.status in {"completed", "failed", "canceled"}:
+                    SITE_CRAWL_CANCEL_REQUESTS.discard(job_id)
+                    return crawl_job_response(job)
+                job.status = "cancel_requested"
+                job.error = "cancel requested by user"
+                payload = crawl_job_response(job)
+            return payload
+        except OperationalError as exc:
+            last_error = exc
+            if "database is locked" not in str(exc).lower():
+                raise
+            time.sleep(0.25 * (attempt + 1))
+    return {
+        "job_id": job_id,
+        "status": "cancel_requested",
+        "url": None,
+        "error": f"cancel requested in memory; database was locked: {last_error}",
+        "scheduled_at": None,
+        "started_at": None,
+        "finished_at": None,
+        "progress": {},
+        "request": {},
+    }
+
+
 def is_site_crawl_cancel_requested(session, job_id: int) -> bool:
+    if job_id in SITE_CRAWL_CANCEL_REQUESTS:
+        return True
     session.expire_all()
     job = session.get(models.CrawlJob, job_id)
     return job is None or job.status == "cancel_requested"
@@ -744,6 +917,8 @@ def run_site_crawl_job(database_url: str, job_id: int, request_data: dict[str, A
                     provider=request.extractor_provider,
                     model=request.extractor_model,
                     base_url=request.extractor_base_url,
+                    extraction_mode=request.extraction_mode,
+                    fallback_to_rules=request.fallback_to_rules,
                 ),
             )
 
@@ -775,12 +950,17 @@ def run_site_crawl_job(database_url: str, job_id: int, request_data: dict[str, A
 
             finished_job = session.get(models.CrawlJob, job_id)
             if finished_job is not None:
-                finished_job.status = "canceled" if finished_job.status == "cancel_requested" else "completed"
+                finished_job.status = (
+                    "canceled"
+                    if finished_job.status == "cancel_requested" or job_id in SITE_CRAWL_CANCEL_REQUESTS
+                    else "completed"
+                )
                 finished_job.finished_at = models.utcnow()
                 update_site_crawl_job_metadata(
                     finished_job,
                     progress=site_crawl_progress_payload(result),
                 )
+            SITE_CRAWL_CANCEL_REQUESTS.discard(job_id)
     except Exception as exc:
         with session_scope(database_url) as session:
             job = session.get(models.CrawlJob, job_id)
@@ -792,10 +972,12 @@ def run_site_crawl_job(database_url: str, job_id: int, request_data: dict[str, A
                 progress = dict(metadata.get("progress") or {})
                 progress["errors"] = [*progress.get("errors", []), str(exc)]
                 update_site_crawl_job_metadata(job, progress=progress)
+        SITE_CRAWL_CANCEL_REQUESTS.discard(job_id)
 
 
 def register_routes(app, database_url: str) -> None:
     recover_interrupted_domain_discovery_jobs(database_url)
+    mark_orphan_site_crawl_jobs_canceled(database_url)
 
     @app.get("/health")
     def health():
@@ -1507,7 +1689,7 @@ def register_routes(app, database_url: str) -> None:
     def extractor_models(request: ExtractorModelsRequest):
         try:
             if request.provider == "lm_studio":
-                models = list_openai_compatible_models(request.base_url or "http://localhost:1234/v1")
+                models = list_lmstudio_loaded_models(request.base_url or "http://localhost:1234/v1")
                 return {"ok": True, "models": models}
             if request.provider == "openai":
                 import os
@@ -1536,6 +1718,8 @@ def register_routes(app, database_url: str) -> None:
                     provider=request.extractor_provider,
                     model=request.extractor_model,
                     base_url=request.extractor_base_url,
+                    extraction_mode=request.extraction_mode,
+                    fallback_to_rules=request.fallback_to_rules,
                 ),
             )
             try:
@@ -1546,6 +1730,13 @@ def register_routes(app, database_url: str) -> None:
                 "page_id": result.page_id,
                 "claim_count": result.claim_count,
                 "entity_count": result.entity_count,
+                "extraction_mode": result.extraction_mode,
+                "effective_extraction_mode": result.effective_extraction_mode,
+                "llm_skipped": result.llm_skipped,
+                "llm_skip_reason": result.llm_skip_reason,
+                "fallback_used": result.fallback_used,
+                "agreement_claim_count": result.agreement_claim_count,
+                "conflict_claim_count": result.conflict_claim_count,
                 "crawl_status": result.crawl_status,
                 "extraction_status": result.extraction_status,
                 "page_type": result.page_type,
@@ -1648,15 +1839,7 @@ def register_routes(app, database_url: str) -> None:
 
     @app.post("/crawl-site/jobs/{job_id}/cancel")
     def cancel_crawl_site_job(job_id: int):
-        with session_scope(database_url) as session:
-            job = session.get(models.CrawlJob, job_id)
-            if job is None or (job.metadata_json or {}).get("kind") != "site_crawl":
-                raise HTTPException(status_code=404, detail="site crawl job not found")
-            if job.status in {"completed", "failed", "canceled"}:
-                return crawl_job_response(job)
-            job.status = "cancel_requested"
-            job.error = "cancel requested by user"
-            return crawl_job_response(job)
+        return request_site_crawl_cancel(database_url, job_id)
 
     @app.post("/discover")
     def discover(request: DiscoverRequest):
@@ -1705,6 +1888,8 @@ def register_routes(app, database_url: str) -> None:
                     provider=request.extractor_provider,
                     model=request.extractor_model,
                     base_url=request.extractor_base_url,
+                    extraction_mode=request.extraction_mode,
+                    fallback_to_rules=request.fallback_to_rules,
                 ),
             )
             try:
@@ -1749,6 +1934,8 @@ def register_routes(app, database_url: str) -> None:
                     provider=request.extractor_provider,
                     model=request.extractor_model,
                     base_url=request.extractor_base_url,
+                    extraction_mode=request.extraction_mode,
+                    fallback_to_rules=request.fallback_to_rules,
                 ),
             )
             try:
@@ -2080,6 +2267,11 @@ def register_routes(app, database_url: str) -> None:
                         "graph_merge_status": (claim.metadata_json or {}).get("graph_merge_status"),
                         "graph_merge_reason": (claim.metadata_json or {}).get("graph_merge_reason"),
                         "confidence_breakdown": (claim.metadata_json or {}).get("confidence_breakdown"),
+                        "agreement": (claim.metadata_json or {}).get("agreement"),
+                        "extraction_source": (claim.metadata_json or {}).get("extraction_source"),
+                        "claim_kind": (claim.metadata_json or {}).get("claim_kind"),
+                        "rule_confidence": (claim.metadata_json or {}).get("rule_confidence"),
+                        "llm_confidence": (claim.metadata_json or {}).get("llm_confidence"),
                         "review_required": (claim.metadata_json or {}).get("review_required"),
                         "review_reason": (claim.metadata_json or {}).get("review_reason"),
                         "conflict_status": (claim.metadata_json or {}).get("conflict_status"),
@@ -2243,9 +2435,11 @@ def register_routes(app, database_url: str) -> None:
                 config_path="",
                 source_name=source.name,
                 url=url,
+                extraction_mode=str(latest_request.get("extraction_mode") or "hybrid"),
                 extractor_provider=str(latest_request.get("extractor_provider") or "lm_studio"),
                 extractor_model=latest_request.get("extractor_model"),
                 extractor_base_url=latest_request.get("extractor_base_url") or "http://localhost:1234/v1",
+                fallback_to_rules=bool(latest_request.get("fallback_to_rules", True)),
                 check_robots_txt=bool(latest_request.get("check_robots_txt") or False),
                 respect_robots_txt=latest_request.get("respect_robots_txt"),
                 max_depth=int(latest_request.get("max_depth") or 2),
@@ -2393,21 +2587,37 @@ def register_routes(app, database_url: str) -> None:
                 .order_by(models.ExtractionLog.created_at.desc())
                 .limit(limit)
             ).all()
-            return [
-                {
+            payload = []
+            for log, page in rows:
+                raw_output = log.raw_output or {}
+                summary = extraction_log_summary(raw_output)
+                payload.append({
                     "id": log.id,
                     "page_url": page.url if page else None,
                     "extractor_name": log.extractor_name,
                     "provider": log.provider,
                     "error": log.error,
                     "created_at": log.created_at.isoformat(),
-                    "validation": (log.raw_output or {}).get("validation"),
-                    "page_context": (log.raw_output or {}).get("page_context"),
-                    "candidate_count": len((log.raw_output or {}).get("candidate_claims") or []),
+                    "validation": raw_output.get("validation"),
+                    "page_context": raw_output.get("page_context"),
+                    "candidate_count": summary["candidate_count"],
+                    "extraction_mode": raw_output.get("extraction_mode"),
+                    "effective_extraction_mode": raw_output.get("effective_extraction_mode"),
+                    "comparison": summary["comparison"],
+                    "rule_entity_count": summary["rule_entity_count"],
+                    "rule_claim_count": summary["rule_claim_count"],
+                    "llm_entity_count": summary["llm_entity_count"],
+                    "llm_claim_count": summary["llm_claim_count"],
+                    "agreement_claim_count": summary["agreement_claim_count"],
+                    "rule_only_claim_count": summary["rule_only_claim_count"],
+                    "llm_only_claim_count": summary["llm_only_claim_count"],
+                    "conflict_claim_count": summary["conflict_claim_count"],
+                    "llm_skipped": raw_output.get("llm_skipped"),
+                    "llm_skip_reason": raw_output.get("llm_skip_reason"),
+                    "fallback": raw_output.get("fallback"),
                     "raw_output": log.raw_output,
-                }
-                for log, page in rows
-            ]
+                })
+            return payload
 
     @app.patch("/claims/{claim_id}/confidence")
     def update_claim_confidence(claim_id: int, request: UpdateClaimConfidenceRequest):

ontology_platform/crawler_platform/app/cli/main.py

@@ -37,6 +37,8 @@ def build_parser() -> argparse.ArgumentParser:
     crawl.add_argument("--extractor-provider", default="rule_based", choices=["rule_based", "openai", "ollama", "lm_studio"])
     crawl.add_argument("--extractor-model")
     crawl.add_argument("--extractor-base-url")
+    crawl.add_argument("--extraction-mode", default=None, choices=["rule_only", "llm_only", "hybrid", "compare"])
+    crawl.add_argument("--no-rule-fallback", action="store_true")
 
     claims = sub.add_parser("claims")
     claims.add_argument("--project", required=True)
@@ -81,6 +83,8 @@ def main() -> None:
                     provider=args.extractor_provider,
                     model=args.extractor_model,
                     base_url=args.extractor_base_url,
+                    extraction_mode=args.extraction_mode,
+                    fallback_to_rules=not args.no_rule_fallback,
                 ),
             )
             result = pipeline.crawl_url(config, args.source, args.url)

ontology_platform/crawler_platform/app/core/crawler/page_analysis_policy.py

@@ -0,0 +1,82 @@
+from __future__ import annotations
+
+from crawler_platform.app.core.crawler.page_type_scorer import PAGE_TYPE_METADATA
+from crawler_platform.app.core.crawler.page_type_taxonomy import (
+    AnalyzeStrategy,
+    LLMPolicy,
+    PageClassificationResult,
+    PageType,
+    normalize_page_type,
+    normalize_semantic_page_type,
+)
+
+
+DEFAULT_ANALYZE_STRATEGY = AnalyzeStrategy.ANALYZE_METADATA_ONLY.value
+DEFAULT_LLM_POLICY = LLMPolicy.NO_LLM.value
+
+
+def decide_analyze_strategy(result_or_page_type: object | None) -> str:
+    semantic_page_type = normalize_semantic_page_type(result_or_page_type)
+    if (
+        isinstance(result_or_page_type, PageClassificationResult)
+        and result_or_page_type.analyze_strategy
+        and result_or_page_type.analyze_strategy != DEFAULT_ANALYZE_STRATEGY
+    ):
+        return result_or_page_type.analyze_strategy
+    profile = PAGE_TYPE_METADATA.get(semantic_page_type)
+    if profile:
+        return str(profile.get("analyze_strategy") or DEFAULT_ANALYZE_STRATEGY)
+    if semantic_page_type == PageType.UNKNOWN_PAGE.value:
+        return AnalyzeStrategy.ANALYZE_METADATA_ONLY.value
+    return DEFAULT_ANALYZE_STRATEGY
+
+
+def decide_llm_policy(result_or_page_type: object | None) -> str:
+    semantic_page_type = normalize_semantic_page_type(result_or_page_type)
+    if (
+        isinstance(result_or_page_type, PageClassificationResult)
+        and result_or_page_type.llm_policy
+        and result_or_page_type.llm_policy != DEFAULT_LLM_POLICY
+    ):
+        return result_or_page_type.llm_policy
+    profile = PAGE_TYPE_METADATA.get(semantic_page_type)
+    if profile:
+        return str(profile.get("llm_policy") or DEFAULT_LLM_POLICY)
+    if semantic_page_type == PageType.UNKNOWN_PAGE.value:
+        return LLMPolicy.NO_LLM.value
+    return DEFAULT_LLM_POLICY
+
+
+def is_protected_strategy(strategy: str | AnalyzeStrategy | None) -> bool:
+    return str(strategy or "") == AnalyzeStrategy.SKIP_PROTECTED.value
+
+
+def is_noise_strategy(strategy: str | AnalyzeStrategy | None) -> bool:
+    return str(strategy or "") == AnalyzeStrategy.SKIP_NOISE.value
+
+
+def should_analyze_page(result_or_page_type: object | None, analyze_page_types: set[str] | None = None) -> bool:
+    strategy = decide_analyze_strategy(result_or_page_type)
+    if is_protected_strategy(strategy) or is_noise_strategy(strategy):
+        return False
+
+    if isinstance(result_or_page_type, PageClassificationResult):
+        return True
+
+    if analyze_page_types is None:
+        return True
+
+    normalized_page_type = normalize_page_type(result_or_page_type)
+    normalized_allowlist = {normalize_page_type(item) for item in analyze_page_types}
+    return normalized_page_type in normalized_allowlist
+
+
+def apply_analysis_policy(result: PageClassificationResult) -> PageClassificationResult:
+    strategy = decide_analyze_strategy(result)
+    llm_policy = decide_llm_policy(result)
+    result.analyze_strategy = strategy
+    result.llm_policy = llm_policy
+    result.is_protected = is_protected_strategy(strategy)
+    result.is_noise = is_noise_strategy(strategy)
+    result.should_analyze = should_analyze_page(result)
+    return result

ontology_platform/crawler_platform/app/core/crawler/page_classifier.py

@@ -3,6 +3,21 @@ from __future__ import annotations
 from urllib.parse import urlparse
 
 from crawler_platform.app.core.crawler.content_zone import is_claim_allowed_zone
+from crawler_platform.app.core.crawler.page_type_taxonomy import (
+    PageClassificationResult,
+    build_classification_result_from_legacy,
+    get_legacy_page_type,
+    normalize_page_type,
+)
+from crawler_platform.app.core.crawler.page_signal_extractor import extract_page_signals_from_page
+from crawler_platform.app.core.crawler.page_type_scorer import score_page_type
+from crawler_platform.app.core.crawler.page_unknown_patterns import (
+    build_unknown_pattern_payload,
+    should_store_unknown_pattern,
+)
+from crawler_platform.app.core.crawler.page_analysis_policy import (
+    should_analyze_page as should_analyze_page_by_policy,
+)
 
 
 PRODUCT_DETAIL_PREDICATES = {
@@ -95,6 +110,73 @@ def classify_page(
     return "UnknownPage"
 
 
+def classify_page_semantic(
+    url: str,
+    title: str | None = None,
+    text: str = "",
+    html: str | None = None,
+    source_zones: list[dict[str, object]] | None = None,
+    final_url: str | None = None,
+    status_code: int | None = None,
+    content_type: str | None = None,
+) -> PageClassificationResult:
+    signals = extract_page_signals_from_page(
+        url=url,
+        final_url=final_url,
+        status_code=status_code,
+        content_type=content_type,
+        title=title,
+        text=text,
+        html=html,
+        source_zones=source_zones,
+    )
+    result = score_page_type(url, signals)
+    if result.primary_page_type == "UnknownPage" and not result.alternatives:
+        legacy_page_type = classify_page(
+            url=url,
+            title=title,
+            text=text,
+            html=html,
+            source_zones=source_zones,
+        )
+        return build_classification_result_from_legacy(
+            url=url,
+            legacy_page_type=legacy_page_type,
+            confidence=0.35,
+            source="legacy_classifier_fallback",
+        )
+    return result
+
+
+def classification_metadata(
+    result: PageClassificationResult,
+    *,
+    title: str | None = None,
+    text: str | None = None,
+    html: str | None = None,
+    source_zones: list[str | dict[str, object]] | None = None,
+) -> dict[str, object]:
+    """Return metadata that preserves legacy page_type while carrying semantic evidence."""
+
+    payload = result.to_dict()
+    if should_store_unknown_pattern(result):
+        payload["unknown_pattern"] = build_unknown_pattern_payload(
+            result=result,
+            url=result.url,
+            title=title,
+            text=text,
+            html=html,
+            source_zones=source_zones,
+        )
+    return {
+        "page_type": get_legacy_page_type(result),
+        "semantic_page_type": result.primary_page_type,
+        "analyze_strategy": result.analyze_strategy,
+        "llm_policy": result.llm_policy,
+        "page_classification": payload,
+    }
+
+
 def relation_allowed_for_page_type(page_type: str | None, predicate: str) -> bool:
     if page_type in PRODUCT_DETAIL_PAGE_TYPES:
         return True
@@ -109,27 +191,8 @@ def claim_allowed_for_context(page_type: str | None, predicate: str, zone_type:
     return relation_allowed_for_page_type(page_type, predicate) and is_claim_allowed_zone(zone_type)
 
 
-def normalize_page_type(value: str | None) -> str:
-    aliases = {
-        "product": "ProductPage",
-        "brand": "BrandStoryPage",
-        "review": "ReviewPage",
-        "listing": "CategoryPage",
-        "category": "CategoryPage",
-        "community": "BoardPage",
-        "board": "BoardPage",
-        "communitypage": "BoardPage",
-        "listingpage": "CategoryPage",
-        "promotionpage": "PromotionPage",
-    }
-    clean = str(value or "").strip()
-    return aliases.get(clean.lower(), aliases.get(clean, clean or "UnknownPage"))
-
-
-def should_analyze_page(page_type: str, analyze_page_types: set[str]) -> bool:
-    normalized_page_type = normalize_page_type(page_type)
-    normalized = {normalize_page_type(item) for item in analyze_page_types}
-    return normalized_page_type in normalized
+def should_analyze_page(page_type: object, analyze_page_types: set[str] | None) -> bool:
+    return should_analyze_page_by_policy(page_type, analyze_page_types)
 
 
 def _zone_type(zone: dict[str, object]) -> str:

ontology_platform/crawler_platform/app/core/crawler/page_cleaner.py

@@ -71,12 +71,27 @@ ZONE_SELECTORS: dict[str, list[str]] = {
 
 ZONE_PRIORITY_BY_PAGE_TYPE = {
     "ProductPage": ["product_title", "product_summary", "product_description", "product_detail"],
+    "ProductDetailPage": ["product_title", "product_summary", "product_description", "product_detail"],
     "BrandStoryPage": ["brand_story_body"],
+    "AboutPage": ["brand_story_body"],
+    "ContactPage": ["brand_story_body"],
     "NoticePage": ["notice_body"],
+    "PublicNoticePage": ["notice_body"],
+    "ArticlePage": ["notice_body"],
+    "NewsArticlePage": ["notice_body"],
+    "BlogPostPage": ["notice_body"],
+    "FAQPage": ["notice_body"],
+    "QAPage": ["notice_body"],
     "BoardPage": ["notice_body"],
+    "ForumBoardPage": ["notice_body"],
+    "ForumThreadPage": ["notice_body"],
     "EventPage": ["event_body"],
     "PromotionPage": ["event_body"],
+    "CampaignLandingPage": ["event_body"],
     "CategoryPage": ["product_title", "product_summary"],
+    "CategoryListingPage": ["product_title", "product_summary"],
+    "SearchPage": ["product_title", "product_summary"],
+    "SearchResultsPage": ["product_title", "product_summary"],
 }
 
 STRUCTURAL_NOISE_TOKENS = {

ontology_platform/crawler_platform/app/core/crawler/page_signal_extractor.py

@@ -0,0 +1,843 @@
+from __future__ import annotations
+
+from collections import Counter
+import json
+import re
+from typing import Any
+from urllib.parse import urljoin, urlparse
+
+from crawler_platform.app.core.crawler.page_signals import PageSignals, RawPageSnapshot
+
+
+KEYWORD_GROUPS: dict[str, tuple[str, ...]] = {
+    "commerce": (
+        "product",
+        "price",
+        "sale",
+        "cart",
+        "basket",
+        "buy",
+        "checkout",
+        "sku",
+        "상품",
+        "가격",
+        "장바구니",
+        "구매",
+        "주문",
+    ),
+    "listing": (
+        "filter",
+        "sort",
+        "category",
+        "pagination",
+        "items",
+        "results",
+        "필터",
+        "정렬",
+        "카테고리",
+        "상품수",
+        "결과",
+    ),
+    "editorial": (
+        "article",
+        "author",
+        "published",
+        "updated",
+        "headline",
+        "news",
+        "blog",
+        "기사",
+        "작성자",
+        "게시일",
+    ),
+    "community": (
+        "question",
+        "answer",
+        "comment",
+        "reply",
+        "thread",
+        "vote",
+        "faq",
+        "q&a",
+        "질문",
+        "답변",
+        "댓글",
+        "문의",
+    ),
+    "knowledge": (
+        "documentation",
+        "api",
+        "endpoint",
+        "parameter",
+        "version",
+        "reference",
+        "guide",
+        "문서",
+        "가이드",
+        "버전",
+    ),
+    "corporate": (
+        "about",
+        "company",
+        "contact",
+        "address",
+        "team",
+        "careers",
+        "privacy",
+        "terms",
+        "회사",
+        "소개",
+        "문의",
+        "주소",
+        "채용",
+        "개인정보",
+        "약관",
+    ),
+    "protected": (
+        "login",
+        "password",
+        "captcha",
+        "access denied",
+        "forbidden",
+        "payment",
+        "billing",
+        "로그인",
+        "비밀번호",
+        "보안문자",
+        "접근 제한",
+        "결제",
+    ),
+}
+
+
+PRICE_PATTERN = re.compile(
+    r"(?:[$€£¥₩]\s?\d[\d,]*(?:\.\d+)?)|(?:\d[\d,]*(?:\.\d+)?\s?(?:KRW|USD|EUR|JPY|원|달러))",
+    re.IGNORECASE,
+)
+DATE_PATTERN = re.compile(r"\b(?:20\d{2}|19\d{2})[-./년]\s?\d{1,2}[-./월]\s?\d{1,2}", re.IGNORECASE)
+API_ENDPOINT_PATTERN = re.compile(r"\b(?:GET|POST|PUT|PATCH|DELETE)\s+/(?:[A-Za-z0-9_./{}:-]+)")
+VERSION_PATTERN = re.compile(r"\bv?\d+\.\d+(?:\.\d+)?\b", re.IGNORECASE)
+
+
+def build_raw_page_snapshot(
+    *,
+    url: str,
+    final_url: str | None = None,
+    status_code: int | None = None,
+    content_type: str | None = None,
+    title: str | None = None,
+    text: str | None = None,
+    html: str | None = None,
+    rendered_html: str | None = None,
+    metadata: dict[str, Any] | None = None,
+    source_zones: list[str | dict[str, Any]] | None = None,
+    collector_payload: dict[str, Any] | None = None,
+) -> RawPageSnapshot:
+    snapshot = RawPageSnapshot(
+        url=url,
+        final_url=final_url,
+        status_code=status_code,
+        content_type=content_type,
+        title=title,
+        text=text,
+        html=html,
+        rendered_html=rendered_html,
+        metadata=dict(metadata or {}),
+        source_zones=list(source_zones or []),
+        collector_payload=dict(collector_payload or {}),
+    )
+    return merge_collector_payload(snapshot)
+
+
+def snapshot_from_collector_payload(payload: dict[str, Any], *, url: str | None = None) -> RawPageSnapshot:
+    return merge_collector_payload(
+        RawPageSnapshot(
+            url=str(url or payload.get("url") or payload.get("source_url") or ""),
+            final_url=payload.get("final_url") or payload.get("resolved_url"),
+            status_code=_optional_int(payload.get("status_code")),
+            content_type=payload.get("content_type"),
+            title=payload.get("title"),
+            text=payload.get("text") or payload.get("markdown") or payload.get("clean_text"),
+            html=payload.get("html") or payload.get("raw_html"),
+            rendered_html=payload.get("rendered_html"),
+            metadata=dict(payload.get("metadata") or {}),
+            collector_payload=dict(payload),
+        )
+    )
+
+
+def merge_collector_payload(snapshot: RawPageSnapshot) -> RawPageSnapshot:
+    payload = snapshot.collector_payload
+    if not payload:
+        return snapshot
+    snapshot.open_graph.update(payload.get("open_graph") or payload.get("og") or {})
+    snapshot.twitter_card.update(payload.get("twitter_card") or payload.get("twitter") or {})
+    snapshot.json_ld.extend(_ensure_dict_list(payload.get("json_ld") or payload.get("jsonld")))
+    snapshot.microdata.extend(_ensure_dict_list(payload.get("microdata")))
+    snapshot.rdfa.extend(_ensure_dict_list(payload.get("rdfa")))
+    snapshot.headings.extend(_string_list(payload.get("headings")))
+    snapshot.links.extend(_dict_list(payload.get("links")))
+    snapshot.images.extend(_dict_list(payload.get("images")))
+    snapshot.forms.extend(_dict_list(payload.get("forms")))
+    snapshot.buttons.extend(_string_list(payload.get("buttons")))
+    snapshot.inputs.extend(_dict_list(payload.get("inputs")))
+    snapshot.tables.extend(_dict_list(payload.get("tables")))
+    snapshot.breadcrumbs.extend(_string_list(payload.get("breadcrumbs")))
+    if payload.get("screenshot_path") and not snapshot.screenshot_path:
+        snapshot.screenshot_path = str(payload["screenshot_path"])
+    return snapshot
+
+
+def extract_page_signals_from_page(
+    *,
+    url: str,
+    final_url: str | None = None,
+    status_code: int | None = None,
+    content_type: str | None = None,
+    title: str | None = None,
+    text: str | None = None,
+    html: str | None = None,
+    rendered_html: str | None = None,
+    metadata: dict[str, Any] | None = None,
+    source_zones: list[str | dict[str, Any]] | None = None,
+    collector_payload: dict[str, Any] | None = None,
+) -> PageSignals:
+    return extract_page_signals(
+        build_raw_page_snapshot(
+            url=url,
+            final_url=final_url,
+            status_code=status_code,
+            content_type=content_type,
+            title=title,
+            text=text,
+            html=html,
+            rendered_html=rendered_html,
+            metadata=metadata,
+            source_zones=source_zones,
+            collector_payload=collector_payload,
+        )
+    )
+
+
+def extract_page_signals(snapshot: RawPageSnapshot) -> PageSignals:
+    html = snapshot.rendered_html or snapshot.html or ""
+    soup = _soup_from_html(html)
+    title = snapshot.title or _title_from_soup(soup)
+    text = snapshot.text or _text_from_soup(soup) or _text_from_html(html)
+    combined = "\n".join([snapshot.url, title or "", text or "", html[:12000]]).lower()
+    open_graph = {**_extract_meta_prefix(soup, "property", "og:"), **snapshot.open_graph}
+    twitter_card = {**_extract_meta_prefix(soup, "name", "twitter:"), **snapshot.twitter_card}
+    json_ld = [*_extract_json_ld(soup), *snapshot.json_ld]
+    links = snapshot.links or _extract_links(soup, snapshot.final_url or snapshot.url)
+    images = snapshot.images or _extract_images(soup, snapshot.final_url or snapshot.url)
+    forms = snapshot.forms or _extract_forms(soup)
+    inputs = snapshot.inputs or _extract_inputs(soup)
+    buttons = snapshot.buttons or _extract_buttons(soup)
+    headings = snapshot.headings or _extract_headings(soup)
+    tables = snapshot.tables or _extract_tables(soup)
+    breadcrumbs = snapshot.breadcrumbs or _extract_breadcrumbs(soup)
+    schema_types = _schema_types(json_ld, snapshot.microdata, snapshot.rdfa, soup)
+    keyword_hits = _keyword_hits(combined)
+    link_counts = _link_counts(links, snapshot.final_url or snapshot.url)
+    layout = _layout_signals(soup, combined, links, images, tables)
+    repeated_card_count = max(
+        layout["card_count"],
+        link_counts["product_link_count"],
+        _count_selector_matches(soup, CARD_SELECTORS),
+    )
+    button_text = " ".join(buttons).lower()
+    input_text = " ".join(_input_blob(item) for item in inputs).lower()
+    form_text = " ".join(_form_blob(item) for item in forms).lower()
+    table_text = " ".join(str(table.get("text") or "") for table in tables).lower()
+    content_type = str(snapshot.content_type or snapshot.metadata.get("content_type") or "").lower()
+    path = urlparse(snapshot.final_url or snapshot.url).path.lower()
+
+    signals = PageSignals(
+        schema_types=schema_types,
+        og_type=_string_or_none(open_graph.get("type") or open_graph.get("og:type")),
+        twitter_card_type=_string_or_none(twitter_card.get("card") or twitter_card.get("twitter:card")),
+        has_price=bool(PRICE_PATTERN.search(combined)),
+        has_currency=bool(re.search(r"[$€£¥₩]|(?:\b(?:KRW|USD|EUR|JPY)\b)|원", combined, re.IGNORECASE)),
+        has_cart_button=_contains_any(button_text + " " + combined, ("cart", "basket", "장바구니", "bag")),
+        has_buy_button=_contains_any(button_text + " " + combined, ("buy now", "purchase", "구매", "주문", "결제")),
+        has_variant_selector=_has_variant_selector(soup, input_text + " " + combined),
+        has_sku=bool(re.search(r"\bsku\b|상품\s*코드|product\s*code", combined, re.IGNORECASE)),
+        has_rating=("AggregateRating" in schema_types)
+        or _contains_any(combined, ("rating", "stars", "별점", "평점")),
+        has_review_section=("Review" in schema_types) or _contains_any(combined, ("review", "reviews", "후기", "리뷰")),
+        has_product_gallery=(len(images) >= 3 and _contains_any(combined, ("gallery", "product", "상품"))),
+        has_repeated_cards=repeated_card_count >= 3,
+        repeated_card_count=repeated_card_count,
+        has_filter_panel=layout["has_filter_sidebar"]
+        or _contains_any(combined, ("filter", "facets", "refine", "필터", "조건")),
+        has_sort_control=_contains_any(combined, ("sort", "order by", "low price", "high price", "정렬", "낮은가격", "높은가격")),
+        has_pagination=_has_pagination(soup, links, combined),
+        has_author=_contains_any(combined, ("author", "byline", "작성자", "기자")),
+        has_published_date=("datePublished" in _json_keys(json_ld))
+        or bool(DATE_PATTERN.search(combined))
+        and _contains_any(combined, ("published", "posted", "게시", "등록")),
+        has_modified_date=("dateModified" in _json_keys(json_ld))
+        or _contains_any(combined, ("modified", "updated", "수정")),
+        has_article_body=("Article" in schema_types)
+        or ("NewsArticle" in schema_types)
+        or _count_selector_matches(soup, ("article", "[itemprop='articleBody']", ".article-body", ".post-content")) > 0
+        or _contains_any(combined, ("articlebody", "article body")),
+        has_tags=_has_tags(soup, links, combined),
+        has_question=("QAPage" in schema_types) or _contains_any(combined, ("question", "q:", "질문", "문의")),
+        has_answer=("Answer" in schema_types) or _contains_any(combined, ("answer", "a:", "답변")),
+        has_comments=_contains_any(combined, ("comment", "comments", "reply", "댓글", "답글")),
+        has_votes=_contains_any(combined, ("vote", "votes", "upvote", "downvote", "추천", "투표")),
+        has_thread_structure=_contains_any(combined, ("thread", "discussion", "게시글", "토론")),
+        has_faq_structure=("FAQPage" in schema_types) or _contains_any(combined, ("faq", "frequently asked", "자주 묻는")),
+        has_code_blocks=_count_selector_matches(soup, ("pre", "code", ".highlight", ".code")) > 0,
+        has_toc=_count_selector_matches(soup, ("#toc", ".toc", "[class*='table-of-contents']", "nav[aria-label*='contents']")) > 0,
+        has_api_endpoint=bool(API_ENDPOINT_PATTERN.search(f"{text or ''}\n{html or ''}")),
+        has_parameter_table=_has_parameter_table(tables, table_text),
+        has_version_info=bool(VERSION_PATTERN.search(combined)) and _contains_any(combined, ("version", "버전", "release")),
+        has_contact_info=_contains_any(combined, ("contact", "email", "tel:", "문의", "연락처")),
+        has_address=_contains_any(combined, ("address", "주소", "road", "street")),
+        has_policy_terms=_contains_any(combined, ("terms", "policy", "agreement", "약관", "정책")),
+        has_privacy_terms=_contains_any(combined, ("privacy", "personal information", "개인정보")),
+        has_career_terms=_contains_any(combined, ("career", "jobs", "recruit", "채용", "지원")),
+        has_login_form=_contains_any(form_text + " " + combined, ("login", "sign in", "로그인")) and (
+            "password" in input_text or "비밀번호" in combined
+        ),
+        has_password_field="password" in input_text,
+        has_payment_fields=_contains_any(input_text + " " + combined, ("card number", "payment", "billing", "결제", "카드")),
+        has_captcha=_contains_any(combined, ("captcha", "recaptcha", "hcaptcha", "보안문자")),
+        has_access_denied=_contains_any(combined, ("access denied", "forbidden", "permission denied", "접근 제한", "권한이 없습니다")),
+        status_code=snapshot.status_code,
+        has_error_status=bool(snapshot.status_code and snapshot.status_code >= 400)
+        or _contains_any(combined, ("404", "not found", "error page")),
+        has_not_found=snapshot.status_code == 404 or _contains_any(combined, ("404", "not found", "page not found")),
+        has_sitemap_resource=("sitemap" in path) or ("sitemap" in content_type and "xml" in content_type),
+        has_feed_resource=("rss" in content_type) or ("atom" in content_type) or path.endswith((".rss", ".atom")),
+        has_json_resource=("json" in content_type) or path.endswith(".json"),
+        has_xml_resource=("xml" in content_type) or path.endswith(".xml"),
+        has_file_resource=path.endswith((".pdf", ".csv", ".xlsx", ".xls", ".doc", ".docx", ".zip")),
+        internal_link_count=link_counts["internal_link_count"],
+        external_link_count=link_counts["external_link_count"],
+        product_link_count=link_counts["product_link_count"],
+        category_link_count=link_counts["category_link_count"],
+        profile_link_count=link_counts["profile_link_count"],
+        article_link_count=link_counts["article_link_count"],
+        layout_blocks=layout["layout_blocks"],
+        has_hero_block=layout["has_hero_block"],
+        has_card_grid=layout["has_card_grid"],
+        has_filter_sidebar=layout["has_filter_sidebar"],
+        has_sticky_action_box=layout["has_sticky_action_box"],
+        has_media_player_area=layout["has_media_player_area"],
+        has_map_area=layout["has_map_area"],
+        has_calendar_grid=layout["has_calendar_grid"],
+        has_pricing_table=layout["has_pricing_table"],
+        has_comparison_table=layout["has_comparison_table"],
+        dominant_language=_dominant_language(combined),
+        keyword_hits=keyword_hits,
+        url_hints=_url_hints(snapshot.final_url or snapshot.url),
+        title=title,
+        text_sample=(text or "")[:500],
+        external_collector_signals=_external_collector_signals(snapshot),
+    )
+    return signals
+
+
+def _soup_from_html(html: str):
+    if not html:
+        return None
+    try:
+        from bs4 import BeautifulSoup
+    except ImportError:
+        return None
+    try:
+        return BeautifulSoup(html, "html.parser")
+    except Exception:
+        return None
+
+
+def _title_from_soup(soup) -> str | None:
+    if soup is None or not soup.title:
+        return None
+    return soup.title.get_text(" ", strip=True) or None
+
+
+def _text_from_soup(soup) -> str:
+    if soup is None:
+        return ""
+    return soup.get_text("\n", strip=True)
+
+
+def _text_from_html(html: str) -> str:
+    return re.sub(r"\s+", " ", re.sub(r"<[^>]+>", " ", html or "")).strip()
+
+
+def _extract_meta_prefix(soup, attr_name: str, prefix: str) -> dict[str, str]:
+    if soup is None:
+        return {}
+    result: dict[str, str] = {}
+    for tag in soup.find_all("meta"):
+        name = str(tag.get(attr_name) or "").strip()
+        if not name.lower().startswith(prefix):
+            continue
+        content = str(tag.get("content") or "").strip()
+        if content:
+            result[name.removeprefix(prefix)] = content
+            result[name] = content
+    return result
+
+
+def _extract_json_ld(soup) -> list[dict[str, Any]]:
+    if soup is None:
+        return []
+    payloads: list[dict[str, Any]] = []
+    for tag in soup.find_all("script"):
+        script_type = str(tag.get("type") or "").lower()
+        if "ld+json" not in script_type:
+            continue
+        raw = tag.string or tag.get_text(" ", strip=True)
+        try:
+            value = json.loads(raw)
+        except Exception:
+            continue
+        payloads.extend(_ensure_dict_list(value))
+    return payloads
+
+
+def _extract_headings(soup) -> list[str]:
+    if soup is None:
+        return []
+    return [node.get_text(" ", strip=True) for node in soup.select("h1,h2,h3") if node.get_text(" ", strip=True)]
+
+
+def _extract_links(soup, base_url: str) -> list[dict[str, Any]]:
+    if soup is None:
+        return []
+    links = []
+    for tag in soup.find_all("a"):
+        href = str(tag.get("href") or "").strip()
+        if not href:
+            continue
+        links.append(
+            {
+                "href": urljoin(base_url, href),
+                "text": tag.get_text(" ", strip=True),
+                "rel": " ".join(str(item) for item in tag.get("rel", [])),
+                "class": " ".join(str(item) for item in tag.get("class", [])),
+            }
+        )
+    return links
+
+
+def _extract_images(soup, base_url: str) -> list[dict[str, Any]]:
+    if soup is None:
+        return []
+    images = []
+    for tag in soup.find_all("img"):
+        src = str(tag.get("src") or tag.get("data-src") or "").strip()
+        if not src:
+            continue
+        images.append(
+            {
+                "src": urljoin(base_url, src),
+                "alt": str(tag.get("alt") or ""),
+                "class": " ".join(str(item) for item in tag.get("class", [])),
+            }
+        )
+    return images
+
+
+def _extract_forms(soup) -> list[dict[str, Any]]:
+    if soup is None:
+        return []
+    forms = []
+    for form in soup.find_all("form"):
+        forms.append(
+            {
+                "action": str(form.get("action") or ""),
+                "method": str(form.get("method") or ""),
+                "id": str(form.get("id") or ""),
+                "class": " ".join(str(item) for item in form.get("class", [])),
+                "text": form.get_text(" ", strip=True)[:500],
+            }
+        )
+    return forms
+
+
+def _extract_inputs(soup) -> list[dict[str, Any]]:
+    if soup is None:
+        return []
+    inputs = []
+    for tag in soup.select("input,select,textarea"):
+        inputs.append(
+            {
+                "type": str(tag.get("type") or tag.name or ""),
+                "name": str(tag.get("name") or ""),
+                "id": str(tag.get("id") or ""),
+                "placeholder": str(tag.get("placeholder") or ""),
+                "autocomplete": str(tag.get("autocomplete") or ""),
+                "aria_label": str(tag.get("aria-label") or ""),
+                "class": " ".join(str(item) for item in tag.get("class", [])),
+                "text": tag.get_text(" ", strip=True)[:240],
+            }
+        )
+    return inputs
+
+
+def _extract_buttons(soup) -> list[str]:
+    if soup is None:
+        return []
+    values = []
+    for tag in soup.select("button,input[type='submit'],input[type='button'],[role='button']"):
+        text = tag.get_text(" ", strip=True) or str(tag.get("value") or tag.get("aria-label") or "")
+        if text.strip():
+            values.append(text.strip())
+    return values
+
+
+def _extract_tables(soup) -> list[dict[str, Any]]:
+    if soup is None:
+        return []
+    tables = []
+    for table in soup.find_all("table"):
+        headers = [cell.get_text(" ", strip=True) for cell in table.select("th") if cell.get_text(" ", strip=True)]
+        text = table.get_text(" ", strip=True)
+        tables.append({"headers": headers, "text": text[:1000], "row_count": len(table.select("tr"))})
+    return tables
+
+
+def _extract_breadcrumbs(soup) -> list[str]:
+    if soup is None:
+        return []
+    crumbs = []
+    selectors = [
+        "[class*='breadcrumb']",
+        "[id*='breadcrumb']",
+        "nav[aria-label*='breadcrumb' i]",
+        "[itemtype*='BreadcrumbList']",
+    ]
+    for selector in selectors:
+        for node in soup.select(selector):
+            text = node.get_text(" > ", strip=True)
+            if text:
+                crumbs.append(text)
+    return _dedupe_strings(crumbs)
+
+
+def _schema_types(
+    json_ld: list[dict[str, Any]],
+    microdata: list[dict[str, Any]],
+    rdfa: list[dict[str, Any]],
+    soup,
+) -> set[str]:
+    types: set[str] = set()
+    for payload in [*json_ld, *microdata, *rdfa]:
+        _visit_schema_types(payload, types)
+    if soup is not None:
+        for node in soup.select("[itemscope][itemtype]"):
+            raw = str(node.get("itemtype") or "")
+            if raw:
+                types.add(raw.rstrip("/").split("/")[-1])
+        for node in soup.select("[typeof]"):
+            for item in str(node.get("typeof") or "").split():
+                types.add(item.split(":")[-1])
+    return {item for item in types if item}
+
+
+def _visit_schema_types(value: Any, types: set[str]) -> None:
+    if isinstance(value, list):
+        for item in value:
+            _visit_schema_types(item, types)
+        return
+    if not isinstance(value, dict):
+        return
+    raw_type = value.get("@type") or value.get("type")
+    for item in _ensure_list(raw_type):
+        if isinstance(item, str):
+            types.add(item.rstrip("/").split("/")[-1])
+    for nested in value.values():
+        if isinstance(nested, (dict, list)):
+            _visit_schema_types(nested, types)
+
+
+def _json_keys(json_ld: list[dict[str, Any]]) -> set[str]:
+    keys: set[str] = set()
+
+    def visit(value: Any) -> None:
+        if isinstance(value, list):
+            for item in value:
+                visit(item)
+            return
+        if not isinstance(value, dict):
+            return
+        keys.update(str(key) for key in value)
+        for nested in value.values():
+            visit(nested)
+
+    visit(json_ld)
+    return keys
+
+
+def _keyword_hits(text: str) -> dict[str, int]:
+    hits = {}
+    for group, terms in KEYWORD_GROUPS.items():
+        count = sum(text.count(term.lower()) for term in terms)
+        if count:
+            hits[group] = count
+    return hits
+
+
+def _link_counts(links: list[dict[str, Any]], base_url: str) -> dict[str, int]:
+    base = urlparse(base_url)
+    base_host = base.netloc.lower()
+    base_path = base.path.lower()
+    counts = Counter()
+    for link in links:
+        href = str(link.get("href") or "")
+        parsed = urlparse(href)
+        host = parsed.netloc.lower()
+        path = parsed.path.lower()
+        if not host or host == base_host:
+            counts["internal_link_count"] += 1
+        else:
+            counts["external_link_count"] += 1
+        same_document_query_link = path == base_path and bool(parsed.query)
+        if not same_document_query_link and any(token in path for token in ("/product", "/products", "/goods", "/item", "/p/")):
+            counts["product_link_count"] += 1
+        if any(token in path for token in ("category", "collection", "/shop", "/list", "catalog")):
+            counts["category_link_count"] += 1
+        if any(token in path for token in ("profile", "user", "author", "member", "creator")):
+            counts["profile_link_count"] += 1
+        if any(token in path for token in ("article", "blog", "news", "post", "story")):
+            counts["article_link_count"] += 1
+    return {
+        "internal_link_count": counts["internal_link_count"],
+        "external_link_count": counts["external_link_count"],
+        "product_link_count": counts["product_link_count"],
+        "category_link_count": counts["category_link_count"],
+        "profile_link_count": counts["profile_link_count"],
+        "article_link_count": counts["article_link_count"],
+    }
+
+
+CARD_SELECTORS = (
+    ".product-card",
+    ".card",
+    ".item",
+    ".product",
+    ".prdList > li",
+    "[class*='product-card']",
+    "[class*='grid-item']",
+)
+
+
+def _layout_signals(soup, text: str, links: list[dict[str, Any]], images: list[dict[str, Any]], tables: list[dict[str, Any]]) -> dict[str, Any]:
+    blocks: list[str] = []
+    hero = _count_selector_matches(soup, (".hero", ".visual", ".main-visual", "[class*='hero']", "section[aria-label*='hero']")) > 0
+    card_count = _count_selector_matches(soup, CARD_SELECTORS)
+    card_grid = card_count >= 3 or _count_selector_matches(soup, (".grid", "[class*='grid']", "[class*='cards']")) > 0 and len(links) >= 3
+    filter_sidebar = _count_selector_matches(soup, (".filter", ".filters", ".facet", "aside", "[class*='filter']", "[class*='facet']")) > 0
+    sticky_action = _count_selector_matches(soup, (".sticky", "[class*='sticky']", "[class*='fixed']", "[class*='buy-box']")) > 0
+    media_player = _count_selector_matches(soup, ("video", "audio", "iframe[src*='youtube']", "[class*='player']")) > 0
+    map_area = _count_selector_matches(soup, ("[class*='map']", "#map", "iframe[src*='maps']")) > 0 or "google map" in text
+    calendar_grid = _count_selector_matches(soup, ("[class*='calendar']", "[class*='datepicker']", "table.calendar")) > 0
+    pricing_table = "pricing" in text and (bool(tables) or _count_selector_matches(soup, ("[class*='pricing']", ".price-table")) > 0)
+    comparison_table = _has_comparison_table(tables, text)
+    for label, present in [
+        ("hero_block", hero),
+        ("card_grid", card_grid),
+        ("filter_sidebar", filter_sidebar),
+        ("sticky_action_box", sticky_action),
+        ("media_player_area", media_player),
+        ("map_area", map_area),
+        ("calendar_grid", calendar_grid),
+        ("pricing_table", pricing_table),
+        ("comparison_table", comparison_table),
+    ]:
+        if present:
+            blocks.append(label)
+    return {
+        "layout_blocks": blocks,
+        "card_count": card_count,
+        "has_hero_block": hero,
+        "has_card_grid": card_grid,
+        "has_filter_sidebar": filter_sidebar,
+        "has_sticky_action_box": sticky_action,
+        "has_media_player_area": media_player,
+        "has_map_area": map_area,
+        "has_calendar_grid": calendar_grid,
+        "has_pricing_table": pricing_table,
+        "has_comparison_table": comparison_table,
+    }
+
+
+def _has_variant_selector(soup, text: str) -> bool:
+    if _contains_any(text, ("variant", "option", "size", "color", "옵션", "사이즈", "색상")):
+        return True
+    if soup is None:
+        return False
+    for select in soup.find_all("select"):
+        blob = " ".join(
+            [
+                str(select.get("name") or ""),
+                str(select.get("id") or ""),
+                select.get_text(" ", strip=True),
+            ]
+        ).lower()
+        if _contains_any(blob, ("variant", "option", "size", "color", "옵션", "사이즈", "색상")):
+            return True
+    return False
+
+
+def _has_pagination(soup, links: list[dict[str, Any]], text: str) -> bool:
+    if _count_selector_matches(soup, (".pagination", ".paging", "[class*='paginate']", "nav[aria-label*='pagination']")) > 0:
+        return True
+    if any(str(link.get("rel") or "").lower() in {"next", "prev", "previous"} for link in links):
+        return True
+    return _contains_any(text, ("next page", "previous page", "페이지", "다음", "이전"))
+
+
+def _has_tags(soup, links: list[dict[str, Any]], text: str) -> bool:
+    if _count_selector_matches(soup, (".tag", ".tags", "[rel='tag']", "[class*='tag']")) > 0:
+        return True
+    return any(str(link.get("rel") or "").lower() == "tag" for link in links) or _contains_any(text, ("tags:", "태그"))
+
+
+def _has_parameter_table(tables: list[dict[str, Any]], table_text: str) -> bool:
+    if _contains_any(table_text, ("parameter", "required", "type", "description", "파라미터", "필수")):
+        return True
+    for table in tables:
+        headers = " ".join(str(item) for item in table.get("headers") or []).lower()
+        if _contains_any(headers, ("parameter", "required", "type", "description", "파라미터", "필수")):
+            return True
+    return False
+
+
+def _has_comparison_table(tables: list[dict[str, Any]], text: str) -> bool:
+    if not tables:
+        return False
+    return _contains_any(text, ("compare", "comparison", "vs", "비교")) or any(
+        int(table.get("row_count") or 0) >= 3 and len(table.get("headers") or []) >= 3 for table in tables
+    )
+
+
+def _url_hints(url: str) -> set[str]:
+    parsed = urlparse(url)
+    value = f"{parsed.path} {parsed.query}".lower()
+    hints = set()
+    for hint, terms in {
+        "product": ("/product", "/products", "/goods", "/item", "/p/"),
+        "category": ("category", "collection", "/shop", "/list", "catalog"),
+        "search": ("search", "find", "keyword=", "query=", "q="),
+        "article": ("article", "blog", "news", "post", "story"),
+        "board": ("board", "forum", "thread", "qna"),
+        "protected": ("login", "checkout", "payment", "account", "cart"),
+        "system": ("sitemap", "robots.txt", ".json", ".xml", ".rss"),
+    }.items():
+        if any(term in value for term in terms):
+            hints.add(hint)
+    return hints
+
+
+def _dominant_language(text: str) -> str | None:
+    if not text:
+        return None
+    korean = len(re.findall(r"[가-힣]", text))
+    latin = len(re.findall(r"[A-Za-z]", text))
+    if korean == 0 and latin == 0:
+        return None
+    if korean > latin * 0.25:
+        return "ko"
+    return "en"
+
+
+def _external_collector_signals(snapshot: RawPageSnapshot) -> dict[str, Any]:
+    payload = dict(snapshot.collector_payload or {})
+    for key in {
+        "html",
+        "raw_html",
+        "rendered_html",
+        "text",
+        "markdown",
+        "clean_text",
+        "metadata",
+        "links",
+        "images",
+        "forms",
+        "inputs",
+        "buttons",
+        "tables",
+        "json_ld",
+        "jsonld",
+    }:
+        payload.pop(key, None)
+    return payload
+
+
+def _count_selector_matches(soup, selectors: tuple[str, ...]) -> int:
+    if soup is None:
+        return 0
+    count = 0
+    for selector in selectors:
+        try:
+            count += len(soup.select(selector))
+        except Exception:
+            continue
+    return count
+
+
+def _input_blob(item: dict[str, Any]) -> str:
+    return " ".join(str(item.get(key) or "") for key in ("type", "name", "id", "placeholder", "autocomplete", "aria_label", "class", "text"))
+
+
+def _form_blob(item: dict[str, Any]) -> str:
+    return " ".join(str(item.get(key) or "") for key in ("action", "method", "id", "class", "text"))
+
+
+def _contains_any(value: str, terms: tuple[str, ...]) -> bool:
+    lowered = value.lower()
+    return any(term.lower() in lowered for term in terms)
+
+
+def _string_or_none(value: Any) -> str | None:
+    clean = str(value or "").strip()
+    return clean or None
+
+
+def _optional_int(value: Any) -> int | None:
+    try:
+        return int(value)
+    except (TypeError, ValueError):
+        return None
+
+
+def _ensure_list(value: Any) -> list[Any]:
+    if value is None:
+        return []
+    if isinstance(value, list):
+        return value
+    return [value]
+
+
+def _ensure_dict_list(value: Any) -> list[dict[str, Any]]:
+    values = _ensure_list(value)
+    return [item for item in values if isinstance(item, dict)]
+
+
+def _dict_list(value: Any) -> list[dict[str, Any]]:
+    return [dict(item) for item in _ensure_list(value) if isinstance(item, dict)]
+
+
+def _string_list(value: Any) -> list[str]:
+    return [str(item) for item in _ensure_list(value) if str(item or "").strip()]
+
+
+def _dedupe_strings(values: list[str]) -> list[str]:
+    seen = set()
+    result = []
+    for value in values:
+        clean = " ".join(value.split())
+        key = clean.lower()
+        if not clean or key in seen:
+            continue
+        seen.add(key)
+        result.append(clean)
+    return result

ontology_platform/crawler_platform/app/core/crawler/page_signals.py

@@ -0,0 +1,144 @@
+from __future__ import annotations
+
+from dataclasses import asdict, dataclass, field
+from typing import Any
+
+
+@dataclass(slots=True)
+class RawPageSnapshot:
+    url: str
+    final_url: str | None = None
+    status_code: int | None = None
+    content_type: str | None = None
+    title: str | None = None
+    text: str | None = None
+    html: str | None = None
+    rendered_html: str | None = None
+    metadata: dict[str, Any] = field(default_factory=dict)
+    open_graph: dict[str, Any] = field(default_factory=dict)
+    twitter_card: dict[str, Any] = field(default_factory=dict)
+    json_ld: list[dict[str, Any]] = field(default_factory=list)
+    microdata: list[dict[str, Any]] = field(default_factory=list)
+    rdfa: list[dict[str, Any]] = field(default_factory=list)
+    headings: list[str] = field(default_factory=list)
+    links: list[dict[str, Any]] = field(default_factory=list)
+    images: list[dict[str, Any]] = field(default_factory=list)
+    forms: list[dict[str, Any]] = field(default_factory=list)
+    buttons: list[str] = field(default_factory=list)
+    inputs: list[dict[str, Any]] = field(default_factory=list)
+    tables: list[dict[str, Any]] = field(default_factory=list)
+    breadcrumbs: list[str] = field(default_factory=list)
+    source_zones: list[str | dict[str, Any]] = field(default_factory=list)
+    screenshot_path: str | None = None
+    collector_payload: dict[str, Any] = field(default_factory=dict)
+
+    def to_dict(self) -> dict[str, Any]:
+        return asdict(self)
+
+
+@dataclass(slots=True)
+class PageSignals:
+    # structured data
+    schema_types: set[str] = field(default_factory=set)
+    og_type: str | None = None
+    twitter_card_type: str | None = None
+
+    # commerce
+    has_price: bool = False
+    has_currency: bool = False
+    has_cart_button: bool = False
+    has_buy_button: bool = False
+    has_variant_selector: bool = False
+    has_sku: bool = False
+    has_rating: bool = False
+    has_review_section: bool = False
+    has_product_gallery: bool = False
+
+    # listing
+    has_repeated_cards: bool = False
+    repeated_card_count: int = 0
+    has_filter_panel: bool = False
+    has_sort_control: bool = False
+    has_pagination: bool = False
+
+    # editorial
+    has_author: bool = False
+    has_published_date: bool = False
+    has_modified_date: bool = False
+    has_article_body: bool = False
+    has_tags: bool = False
+
+    # community
+    has_question: bool = False
+    has_answer: bool = False
+    has_comments: bool = False
+    has_votes: bool = False
+    has_thread_structure: bool = False
+    has_faq_structure: bool = False
+
+    # knowledge/docs
+    has_code_blocks: bool = False
+    has_toc: bool = False
+    has_api_endpoint: bool = False
+    has_parameter_table: bool = False
+    has_version_info: bool = False
+
+    # corporate/legal
+    has_contact_info: bool = False
+    has_address: bool = False
+    has_policy_terms: bool = False
+    has_privacy_terms: bool = False
+    has_career_terms: bool = False
+
+    # transaction/protected
+    has_login_form: bool = False
+    has_password_field: bool = False
+    has_payment_fields: bool = False
+    has_captcha: bool = False
+    has_access_denied: bool = False
+
+    # system/resource
+    status_code: int | None = None
+    has_error_status: bool = False
+    has_not_found: bool = False
+    has_sitemap_resource: bool = False
+    has_feed_resource: bool = False
+    has_json_resource: bool = False
+    has_xml_resource: bool = False
+    has_file_resource: bool = False
+
+    # graph
+    internal_link_count: int = 0
+    external_link_count: int = 0
+    product_link_count: int = 0
+    category_link_count: int = 0
+    profile_link_count: int = 0
+    article_link_count: int = 0
+
+    # visual/layout candidates from DOM structure
+    layout_blocks: list[str] = field(default_factory=list)
+    has_hero_block: bool = False
+    has_card_grid: bool = False
+    has_filter_sidebar: bool = False
+    has_sticky_action_box: bool = False
+    has_media_player_area: bool = False
+    has_map_area: bool = False
+    has_calendar_grid: bool = False
+    has_pricing_table: bool = False
+    has_comparison_table: bool = False
+
+    # text/layout
+    dominant_language: str | None = None
+    keyword_hits: dict[str, int] = field(default_factory=dict)
+    url_hints: set[str] = field(default_factory=set)
+    title: str | None = None
+    text_sample: str = ""
+
+    # external collector hook
+    external_collector_signals: dict[str, Any] = field(default_factory=dict)
+
+    def to_dict(self) -> dict[str, Any]:
+        payload = asdict(self)
+        payload["schema_types"] = sorted(self.schema_types)
+        payload["url_hints"] = sorted(self.url_hints)
+        return payload

ontology_platform/crawler_platform/app/core/crawler/page_type_scorer.py

@@ -0,0 +1,747 @@
+from __future__ import annotations
+
+from collections.abc import Callable
+from dataclasses import dataclass
+from typing import Any
+
+from crawler_platform.app.core.crawler.page_signals import PageSignals
+from crawler_platform.app.core.crawler.page_type_taxonomy import (
+    ActionIntent,
+    AnalyzeStrategy,
+    EntityType,
+    EvidenceItem,
+    GraphRole,
+    LLMPolicy,
+    PAGE_TYPE_PROFILES,
+    PageArchetype,
+    PageClassificationResult,
+    PageDomain,
+    PageType,
+)
+
+
+UNKNOWN_THRESHOLD = 0.22
+
+
+@dataclass(frozen=True, slots=True)
+class SignalRule:
+    key: str
+    weight: float
+    source: str
+    message: str
+    predicate: Callable[[PageSignals], bool]
+    value: Callable[[PageSignals], str | int | float | bool | None] | None = None
+
+
+@dataclass(slots=True)
+class ScoreAccumulator:
+    scores: dict[str, float]
+    evidence_by_type: dict[str, list[EvidenceItem]]
+
+    def add(self, page_type: str, rule: SignalRule, signals: PageSignals) -> None:
+        if not rule.predicate(signals):
+            return
+        value = rule.value(signals) if rule.value else True
+        self.scores[page_type] = self.scores.get(page_type, 0.0) + rule.weight
+        self.evidence_by_type.setdefault(page_type, []).append(
+            EvidenceItem(
+                key=rule.key,
+                value=value,
+                weight=rule.weight,
+                source=rule.source,
+                message=rule.message,
+            )
+        )
+
+
+PAGE_TYPE_METADATA: dict[str, dict[str, Any]] = {
+    **PAGE_TYPE_PROFILES,
+    PageType.ARTICLE_PAGE.value: {
+        "domain": PageDomain.EDITORIAL.value,
+        "archetype": PageArchetype.ARTICLE.value,
+        "main_entity_type": EntityType.ARTICLE.value,
+        "action_intents": [ActionIntent.READ.value],
+        "graph_roles": [GraphRole.CLAIM_SOURCE.value, GraphRole.ENTITY_ANCHOR.value],
+        "analyze_strategy": AnalyzeStrategy.ANALYZE_FULL.value,
+        "llm_policy": LLMPolicy.LLM_FULL.value,
+        "should_analyze": True,
+    },
+    PageType.BLOG_POST_PAGE.value: {
+        "domain": PageDomain.EDITORIAL.value,
+        "archetype": PageArchetype.ARTICLE.value,
+        "main_entity_type": EntityType.ARTICLE.value,
+        "action_intents": [ActionIntent.READ.value],
+        "graph_roles": [GraphRole.CLAIM_SOURCE.value],
+        "analyze_strategy": AnalyzeStrategy.ANALYZE_FULL.value,
+        "llm_policy": LLMPolicy.LLM_FULL.value,
+        "should_analyze": True,
+    },
+    PageType.FAQ_PAGE.value: {
+        "domain": PageDomain.COMMUNITY.value,
+        "archetype": PageArchetype.DOCUMENT.value,
+        "main_entity_type": EntityType.QUESTION.value,
+        "action_intents": [ActionIntent.READ.value, ActionIntent.ANSWER.value],
+        "graph_roles": [GraphRole.REFERENCE_SOURCE.value],
+        "analyze_strategy": AnalyzeStrategy.ANALYZE_FULL.value,
+        "llm_policy": LLMPolicy.LLM_LIGHT.value,
+        "should_analyze": True,
+    },
+    PageType.QA_PAGE.value: {
+        "domain": PageDomain.COMMUNITY.value,
+        "archetype": PageArchetype.THREAD.value,
+        "main_entity_type": EntityType.QUESTION.value,
+        "action_intents": [ActionIntent.ASK.value, ActionIntent.ANSWER.value, ActionIntent.READ.value],
+        "graph_roles": [GraphRole.CLAIM_SOURCE.value, GraphRole.RELATION_HUB.value],
+        "analyze_strategy": AnalyzeStrategy.ANALYZE_FULL.value,
+        "llm_policy": LLMPolicy.LLM_LIGHT.value,
+        "should_analyze": True,
+    },
+    PageType.FORUM_THREAD_PAGE.value: {
+        "domain": PageDomain.COMMUNITY.value,
+        "archetype": PageArchetype.THREAD.value,
+        "main_entity_type": EntityType.ARTICLE.value,
+        "action_intents": [ActionIntent.READ.value, ActionIntent.COMMENT.value],
+        "graph_roles": [GraphRole.CLAIM_SOURCE.value, GraphRole.RELATION_HUB.value],
+        "analyze_strategy": AnalyzeStrategy.ANALYZE_FULL.value,
+        "llm_policy": LLMPolicy.LLM_LIGHT.value,
+        "should_analyze": True,
+    },
+    PageType.ABOUT_PAGE.value: {
+        "domain": PageDomain.CORPORATE.value,
+        "archetype": PageArchetype.ARTICLE.value,
+        "main_entity_type": EntityType.ORGANIZATION.value,
+        "action_intents": [ActionIntent.READ.value, ActionIntent.LEARN.value],
+        "graph_roles": [GraphRole.ENTITY_ANCHOR.value, GraphRole.CLAIM_SOURCE.value],
+        "analyze_strategy": AnalyzeStrategy.ANALYZE_FULL.value,
+        "llm_policy": LLMPolicy.LLM_FULL.value,
+        "should_analyze": True,
+    },
+    PageType.CONTACT_PAGE.value: {
+        "domain": PageDomain.CORPORATE.value,
+        "archetype": PageArchetype.DOCUMENT.value,
+        "main_entity_type": EntityType.ORGANIZATION.value,
+        "action_intents": [ActionIntent.CONTACT.value, ActionIntent.NAVIGATE.value],
+        "graph_roles": [GraphRole.ENTITY_ANCHOR.value],
+        "analyze_strategy": AnalyzeStrategy.ANALYZE_METADATA_ONLY.value,
+        "llm_policy": LLMPolicy.RULE_ONLY.value,
+        "should_analyze": True,
+    },
+    PageType.DOCUMENTATION_PAGE.value: {
+        "domain": PageDomain.KNOWLEDGE.value,
+        "archetype": PageArchetype.DOCUMENT.value,
+        "main_entity_type": EntityType.SOFTWARE_APPLICATION.value,
+        "action_intents": [ActionIntent.READ.value, ActionIntent.LEARN.value],
+        "graph_roles": [GraphRole.REFERENCE_SOURCE.value],
+        "analyze_strategy": AnalyzeStrategy.ANALYZE_FULL.value,
+        "llm_policy": LLMPolicy.LLM_LIGHT.value,
+        "should_analyze": True,
+    },
+    PageType.API_REFERENCE_PAGE.value: {
+        "domain": PageDomain.SOFTWARE.value,
+        "archetype": PageArchetype.DOCUMENT.value,
+        "main_entity_type": EntityType.SOFTWARE_APPLICATION.value,
+        "action_intents": [ActionIntent.LEARN.value, ActionIntent.CONFIGURE.value],
+        "graph_roles": [GraphRole.REFERENCE_SOURCE.value],
+        "analyze_strategy": AnalyzeStrategy.ANALYZE_FULL.value,
+        "llm_policy": LLMPolicy.LLM_LIGHT.value,
+        "should_analyze": True,
+    },
+    PageType.WIKI_PAGE.value: {
+        "domain": PageDomain.KNOWLEDGE.value,
+        "archetype": PageArchetype.DOCUMENT.value,
+        "main_entity_type": EntityType.UNKNOWN_ENTITY.value,
+        "action_intents": [ActionIntent.READ.value, ActionIntent.LEARN.value],
+        "graph_roles": [GraphRole.REFERENCE_SOURCE.value],
+        "analyze_strategy": AnalyzeStrategy.ANALYZE_FULL.value,
+        "llm_policy": LLMPolicy.LLM_LIGHT.value,
+        "should_analyze": True,
+    },
+    PageType.DATASET_PAGE.value: {
+        "domain": PageDomain.KNOWLEDGE.value,
+        "archetype": PageArchetype.DOCUMENT.value,
+        "main_entity_type": EntityType.DATASET.value,
+        "action_intents": [ActionIntent.READ.value, ActionIntent.DOWNLOAD.value],
+        "graph_roles": [GraphRole.REFERENCE_SOURCE.value],
+        "analyze_strategy": AnalyzeStrategy.ANALYZE_DOCUMENT_ONLY.value,
+        "llm_policy": LLMPolicy.LLM_LIGHT.value,
+        "should_analyze": True,
+    },
+    PageType.RESEARCH_PAPER_PAGE.value: {
+        "domain": PageDomain.KNOWLEDGE.value,
+        "archetype": PageArchetype.ARTICLE.value,
+        "main_entity_type": EntityType.ARTICLE.value,
+        "action_intents": [ActionIntent.READ.value, ActionIntent.LEARN.value],
+        "graph_roles": [GraphRole.CLAIM_SOURCE.value, GraphRole.REFERENCE_SOURCE.value],
+        "analyze_strategy": AnalyzeStrategy.ANALYZE_FULL.value,
+        "llm_policy": LLMPolicy.LLM_FULL.value,
+        "should_analyze": True,
+    },
+    PageType.JOB_POSTING_PAGE.value: {
+        "domain": PageDomain.JOBS.value,
+        "archetype": PageArchetype.DETAIL.value,
+        "main_entity_type": EntityType.JOB_POSTING.value,
+        "action_intents": [ActionIntent.APPLY.value, ActionIntent.READ.value],
+        "graph_roles": [GraphRole.ENTITY_ANCHOR.value],
+        "analyze_strategy": AnalyzeStrategy.ANALYZE_FULL.value,
+        "llm_policy": LLMPolicy.LLM_FULL.value,
+        "should_analyze": True,
+    },
+    PageType.COURSE_DETAIL_PAGE.value: {
+        "domain": PageDomain.EDUCATION.value,
+        "archetype": PageArchetype.DETAIL.value,
+        "main_entity_type": EntityType.COURSE.value,
+        "action_intents": [ActionIntent.LEARN.value, ActionIntent.SUBSCRIBE.value],
+        "graph_roles": [GraphRole.ENTITY_ANCHOR.value, GraphRole.REFERENCE_SOURCE.value],
+        "analyze_strategy": AnalyzeStrategy.ANALYZE_FULL.value,
+        "llm_policy": LLMPolicy.LLM_LIGHT.value,
+        "should_analyze": True,
+    },
+    PageType.VIDEO_PAGE.value: {
+        "domain": PageDomain.MEDIA.value,
+        "archetype": PageArchetype.MEDIA.value,
+        "main_entity_type": EntityType.MEDIA_OBJECT.value,
+        "action_intents": [ActionIntent.WATCH.value],
+        "graph_roles": [GraphRole.MEDIA_ANCHOR.value],
+        "analyze_strategy": AnalyzeStrategy.ANALYZE_METADATA_ONLY.value,
+        "llm_policy": LLMPolicy.RULE_ONLY.value,
+        "should_analyze": True,
+    },
+    PageType.LOCAL_BUSINESS_PAGE.value: {
+        "domain": PageDomain.LOCAL.value,
+        "archetype": PageArchetype.DETAIL.value,
+        "main_entity_type": EntityType.PLACE.value,
+        "action_intents": [ActionIntent.CONTACT.value, ActionIntent.NAVIGATE.value],
+        "graph_roles": [GraphRole.ENTITY_ANCHOR.value],
+        "analyze_strategy": AnalyzeStrategy.ANALYZE_FULL.value,
+        "llm_policy": LLMPolicy.LLM_LIGHT.value,
+        "should_analyze": True,
+    },
+    PageType.REAL_ESTATE_LISTING_PAGE.value: {
+        "domain": PageDomain.LOCAL.value,
+        "archetype": PageArchetype.DETAIL.value,
+        "main_entity_type": EntityType.REAL_ESTATE_PROPERTY.value,
+        "action_intents": [ActionIntent.READ.value, ActionIntent.CONTACT.value],
+        "graph_roles": [GraphRole.ENTITY_ANCHOR.value],
+        "analyze_strategy": AnalyzeStrategy.ANALYZE_FULL.value,
+        "llm_policy": LLMPolicy.LLM_LIGHT.value,
+        "should_analyze": True,
+    },
+    PageType.PROFILE_PAGE.value: {
+        "domain": PageDomain.COMMUNITY.value,
+        "archetype": PageArchetype.PROFILE.value,
+        "main_entity_type": EntityType.PERSON.value,
+        "action_intents": [ActionIntent.READ.value, ActionIntent.CONTACT.value],
+        "graph_roles": [GraphRole.PROFILE_ANCHOR.value],
+        "analyze_strategy": AnalyzeStrategy.ANALYZE_ENTITY_ONLY.value,
+        "llm_policy": LLMPolicy.LLM_LIGHT.value,
+        "should_analyze": True,
+    },
+    PageType.PRICING_PAGE.value: {
+        "domain": PageDomain.COMMERCE.value,
+        "archetype": PageArchetype.LANDING.value,
+        "main_entity_type": EntityType.SERVICE.value,
+        "action_intents": [ActionIntent.BUY.value, ActionIntent.COMPARE.value],
+        "graph_roles": [GraphRole.CLAIM_SOURCE.value],
+        "analyze_strategy": AnalyzeStrategy.ANALYZE_FULL.value,
+        "llm_policy": LLMPolicy.LLM_LIGHT.value,
+        "should_analyze": True,
+    },
+    PageType.LOGIN_PAGE.value: {
+        "domain": PageDomain.TRANSACTION.value,
+        "archetype": PageArchetype.FORM.value,
+        "main_entity_type": None,
+        "action_intents": [ActionIntent.LOGIN.value],
+        "graph_roles": [GraphRole.TRANSACTION_ONLY.value],
+        "analyze_strategy": AnalyzeStrategy.SKIP_PROTECTED.value,
+        "llm_policy": LLMPolicy.SKIP.value,
+        "should_analyze": False,
+    },
+    PageType.CHECKOUT_PAGE.value: {
+        "domain": PageDomain.TRANSACTION.value,
+        "archetype": PageArchetype.TRANSACTION.value,
+        "main_entity_type": None,
+        "action_intents": [ActionIntent.BUY.value, ActionIntent.PAY.value],
+        "graph_roles": [GraphRole.TRANSACTION_ONLY.value],
+        "analyze_strategy": AnalyzeStrategy.SKIP_PROTECTED.value,
+        "llm_policy": LLMPolicy.SKIP.value,
+        "should_analyze": False,
+    },
+    PageType.PAYMENT_PAGE.value: {
+        "domain": PageDomain.TRANSACTION.value,
+        "archetype": PageArchetype.TRANSACTION.value,
+        "main_entity_type": None,
+        "action_intents": [ActionIntent.PAY.value],
+        "graph_roles": [GraphRole.TRANSACTION_ONLY.value],
+        "analyze_strategy": AnalyzeStrategy.SKIP_PROTECTED.value,
+        "llm_policy": LLMPolicy.SKIP.value,
+        "should_analyze": False,
+    },
+    PageType.TERMS_PAGE.value: {
+        "domain": PageDomain.CORPORATE.value,
+        "archetype": PageArchetype.DOCUMENT.value,
+        "main_entity_type": EntityType.LEGAL_DOCUMENT.value,
+        "action_intents": [ActionIntent.READ.value],
+        "graph_roles": [GraphRole.POLICY_SOURCE.value],
+        "analyze_strategy": AnalyzeStrategy.ANALYZE_DOCUMENT_ONLY.value,
+        "llm_policy": LLMPolicy.LLM_LIGHT.value,
+        "should_analyze": True,
+    },
+    PageType.PRIVACY_POLICY_PAGE.value: {
+        "domain": PageDomain.CORPORATE.value,
+        "archetype": PageArchetype.DOCUMENT.value,
+        "main_entity_type": EntityType.LEGAL_DOCUMENT.value,
+        "action_intents": [ActionIntent.READ.value],
+        "graph_roles": [GraphRole.POLICY_SOURCE.value],
+        "analyze_strategy": AnalyzeStrategy.ANALYZE_DOCUMENT_ONLY.value,
+        "llm_policy": LLMPolicy.LLM_LIGHT.value,
+        "should_analyze": True,
+    },
+    PageType.SITEMAP_PAGE.value: {
+        "domain": PageDomain.SYSTEM.value,
+        "archetype": PageArchetype.SYSTEM_RESOURCE.value,
+        "main_entity_type": None,
+        "action_intents": [ActionIntent.NAVIGATE.value],
+        "graph_roles": [GraphRole.SYSTEM_RESOURCE.value, GraphRole.NAVIGATION_HUB.value],
+        "analyze_strategy": AnalyzeStrategy.ANALYZE_DISCOVERY_ONLY.value,
+        "llm_policy": LLMPolicy.RULE_ONLY.value,
+        "should_analyze": True,
+    },
+    PageType.RSS_FEED_PAGE.value: {
+        "domain": PageDomain.SYSTEM.value,
+        "archetype": PageArchetype.SYSTEM_RESOURCE.value,
+        "main_entity_type": None,
+        "action_intents": [ActionIntent.READ.value],
+        "graph_roles": [GraphRole.SYSTEM_RESOURCE.value],
+        "analyze_strategy": AnalyzeStrategy.ANALYZE_METADATA_ONLY.value,
+        "llm_policy": LLMPolicy.RULE_ONLY.value,
+        "should_analyze": True,
+    },
+    PageType.ERROR_PAGE.value: {
+        "domain": PageDomain.SYSTEM.value,
+        "archetype": PageArchetype.ERROR.value,
+        "main_entity_type": None,
+        "action_intents": [],
+        "graph_roles": [GraphRole.NOISE_PAGE.value],
+        "analyze_strategy": AnalyzeStrategy.SKIP_NOISE.value,
+        "llm_policy": LLMPolicy.SKIP.value,
+        "should_analyze": False,
+    },
+    PageType.NOT_FOUND_PAGE.value: {
+        "domain": PageDomain.SYSTEM.value,
+        "archetype": PageArchetype.ERROR.value,
+        "main_entity_type": None,
+        "action_intents": [],
+        "graph_roles": [GraphRole.NOISE_PAGE.value],
+        "analyze_strategy": AnalyzeStrategy.SKIP_NOISE.value,
+        "llm_policy": LLMPolicy.SKIP.value,
+        "should_analyze": False,
+    },
+    PageType.ACCESS_DENIED_PAGE.value: {
+        "domain": PageDomain.TRANSACTION.value,
+        "archetype": PageArchetype.ERROR.value,
+        "main_entity_type": None,
+        "action_intents": [ActionIntent.VERIFY.value],
+        "graph_roles": [GraphRole.TRANSACTION_ONLY.value],
+        "analyze_strategy": AnalyzeStrategy.SKIP_PROTECTED.value,
+        "llm_policy": LLMPolicy.SKIP.value,
+        "should_analyze": False,
+    },
+    PageType.CAPTCHA_PAGE.value: {
+        "domain": PageDomain.TRANSACTION.value,
+        "archetype": PageArchetype.FORM.value,
+        "main_entity_type": None,
+        "action_intents": [ActionIntent.VERIFY.value],
+        "graph_roles": [GraphRole.TRANSACTION_ONLY.value],
+        "analyze_strategy": AnalyzeStrategy.SKIP_PROTECTED.value,
+        "llm_policy": LLMPolicy.SKIP.value,
+        "should_analyze": False,
+    },
+}
+
+
+def score_page_type(url: str, signals: PageSignals, *, unknown_threshold: float = UNKNOWN_THRESHOLD) -> PageClassificationResult:
+    accumulator = ScoreAccumulator(scores={}, evidence_by_type={})
+    for page_type, rules in SCORING_RULES.items():
+        for rule in rules:
+            accumulator.add(page_type, rule, signals)
+
+    _apply_interactions(accumulator, signals)
+
+    if not accumulator.scores:
+        return _unknown_result(
+            url,
+            alternatives=[],
+            evidence=[
+                EvidenceItem(
+                    key="insufficient_evidence",
+                    value=True,
+                    weight=0.0,
+                    source="page_type_scorer",
+                    message="No page-type evidence was detected.",
+                )
+            ],
+        )
+
+    alternatives = sorted(
+        ((page_type, round(min(score, 1.0), 4)) for page_type, score in accumulator.scores.items()),
+        key=lambda item: item[1],
+        reverse=True,
+    )
+    primary_page_type, confidence = alternatives[0]
+    if confidence < unknown_threshold:
+        evidence = [item for page_type, _score in alternatives[:3] for item in accumulator.evidence_by_type.get(page_type, [])]
+        evidence.append(
+            EvidenceItem(
+                key="low_confidence",
+                value=confidence,
+                weight=0.0,
+                source="page_type_scorer",
+                message=f"Top score {confidence:.2f} is below UnknownPage threshold {unknown_threshold:.2f}.",
+            )
+        )
+        return _unknown_result(url, alternatives=alternatives[:5], evidence=evidence)
+
+    profile = _profile(primary_page_type)
+    evidence = accumulator.evidence_by_type.get(primary_page_type, [])
+    if not evidence:
+        evidence = [
+            EvidenceItem(
+                key="score",
+                value=confidence,
+                weight=confidence,
+                source="page_type_scorer",
+                message=f"{primary_page_type} selected from accumulated score.",
+            )
+        ]
+    secondary = [page_type for page_type, score in alternatives[1:4] if score >= 0.18]
+    return PageClassificationResult(
+        url=url,
+        primary_page_type=primary_page_type,
+        secondary_page_types=secondary,
+        domain=str(profile["domain"]),
+        archetype=str(profile["archetype"]),
+        main_entity_type=profile.get("main_entity_type"),
+        action_intents=list(profile.get("action_intents") or []),
+        graph_roles=list(profile.get("graph_roles") or []),
+        confidence=confidence,
+        alternatives=alternatives[:8],
+        evidence=evidence,
+        should_analyze=bool(profile.get("should_analyze")),
+        analyze_strategy=str(profile["analyze_strategy"]),
+        llm_policy=str(profile["llm_policy"]),
+        is_protected=str(profile["analyze_strategy"]) == AnalyzeStrategy.SKIP_PROTECTED.value,
+        is_noise=str(profile["analyze_strategy"]) == AnalyzeStrategy.SKIP_NOISE.value,
+    )
+
+
+def _profile(page_type: str) -> dict[str, Any]:
+    return PAGE_TYPE_METADATA.get(page_type, PAGE_TYPE_METADATA[PageType.UNKNOWN_PAGE.value])
+
+
+def _unknown_result(
+    url: str,
+    *,
+    alternatives: list[tuple[str, float]],
+    evidence: list[EvidenceItem],
+) -> PageClassificationResult:
+    profile = _profile(PageType.UNKNOWN_PAGE.value)
+    return PageClassificationResult(
+        url=url,
+        primary_page_type=PageType.UNKNOWN_PAGE.value,
+        secondary_page_types=[page_type for page_type, _score in alternatives[:3]],
+        domain=str(profile["domain"]),
+        archetype=str(profile["archetype"]),
+        main_entity_type=profile.get("main_entity_type"),
+        action_intents=list(profile.get("action_intents") or []),
+        graph_roles=list(profile.get("graph_roles") or []),
+        confidence=0.0,
+        alternatives=alternatives[:8],
+        evidence=evidence
+        or [
+            EvidenceItem(
+                key="unknown",
+                value=True,
+                weight=0.0,
+                source="page_type_scorer",
+                message="Page did not match known semantic page type evidence.",
+            )
+        ],
+        should_analyze=bool(profile.get("should_analyze")),
+        analyze_strategy=str(profile["analyze_strategy"]),
+        llm_policy=str(profile["llm_policy"]),
+        is_protected=False,
+        is_noise=False,
+    )
+
+
+def _apply_interactions(accumulator: ScoreAccumulator, signals: PageSignals) -> None:
+    # Protected signals must dominate commerce signals when login/payment fields are present.
+    if signals.has_password_field and signals.has_login_form:
+        _manual_add(accumulator, PageType.LOGIN_PAGE.value, "protected_priority", True, 0.3, "protected", "Login form and password field override commerce signals.")
+    if signals.has_payment_fields:
+        _manual_add(accumulator, PageType.PAYMENT_PAGE.value, "payment_priority", True, 0.35, "protected", "Payment fields override product or checkout content.")
+    if signals.has_access_denied:
+        _manual_add(accumulator, PageType.ACCESS_DENIED_PAGE.value, "access_denied_priority", True, 0.35, "protected", "Access denied signal is a protected page indicator.")
+    if signals.has_captcha:
+        _manual_add(accumulator, PageType.CAPTCHA_PAGE.value, "captcha_priority", True, 0.35, "protected", "Captcha signal is a protected page indicator.")
+    if signals.has_repeated_cards and signals.product_link_count >= 3:
+        _manual_add(accumulator, PageType.CATEGORY_LISTING_PAGE.value, "listing_product_links", signals.product_link_count, 0.18, "link_graph", "Repeated cards with multiple product links indicate a product listing.")
+    if signals.has_question and signals.has_answer and signals.has_faq_structure:
+        _manual_add(accumulator, PageType.FAQ_PAGE.value, "faq_question_answer", True, 0.18, "community", "FAQ structure with question/answer content.")
+
+
+def _manual_add(
+    accumulator: ScoreAccumulator,
+    page_type: str,
+    key: str,
+    value: str | int | float | bool | None,
+    weight: float,
+    source: str,
+    message: str,
+) -> None:
+    accumulator.scores[page_type] = accumulator.scores.get(page_type, 0.0) + weight
+    accumulator.evidence_by_type.setdefault(page_type, []).append(
+        EvidenceItem(key=key, value=value, weight=weight, source=source, message=message)
+    )
+
+
+def _has_schema(*schema_types: str) -> Callable[[PageSignals], bool]:
+    return lambda signals: any(schema_type in signals.schema_types for schema_type in schema_types)
+
+
+def _keyword(group: str, minimum: int = 1) -> Callable[[PageSignals], bool]:
+    return lambda signals: int(signals.keyword_hits.get(group) or 0) >= minimum
+
+
+def _url_hint(hint: str) -> Callable[[PageSignals], bool]:
+    return lambda signals: hint in signals.url_hints
+
+
+def _contains_terms(signals: PageSignals, terms: tuple[str, ...]) -> bool:
+    text = f"{signals.title or ''}\n{signals.text_sample or ''}".lower()
+    return any(term.lower() in text for term in terms)
+
+
+def _flag(name: str) -> Callable[[PageSignals], bool]:
+    return lambda signals: bool(getattr(signals, name))
+
+
+def _count_at_least(name: str, minimum: int) -> Callable[[PageSignals], bool]:
+    return lambda signals: int(getattr(signals, name) or 0) >= minimum
+
+
+def _value(name: str) -> Callable[[PageSignals], str | int | float | bool | None]:
+    return lambda signals: getattr(signals, name)
+
+
+def _keywords_value(group: str) -> Callable[[PageSignals], str | int | float | bool | None]:
+    return lambda signals: signals.keyword_hits.get(group, 0)
+
+
+def _schema_value(signals: PageSignals) -> str:
+    return ",".join(sorted(signals.schema_types))
+
+
+def _url_hint_value(signals: PageSignals) -> str:
+    return ",".join(sorted(signals.url_hints))
+
+
+SCORING_RULES: dict[str, list[SignalRule]] = {
+    PageType.PRODUCT_DETAIL_PAGE.value: [
+        SignalRule("schema_product", 0.4, "structured_data", "schema.org Product detected.", _has_schema("Product"), _schema_value),
+        SignalRule("schema_offer", 0.15, "structured_data", "schema.org Offer detected.", _has_schema("Offer"), _schema_value),
+        SignalRule("price", 0.15, "text", "Price detected.", _flag("has_price"), _value("has_price")),
+        SignalRule("cart_button", 0.2, "dom", "Cart button detected.", _flag("has_cart_button")),
+        SignalRule("buy_button", 0.16, "dom", "Buy button detected.", _flag("has_buy_button")),
+        SignalRule("variant_selector", 0.12, "dom", "Variant selector detected.", _flag("has_variant_selector")),
+        SignalRule("sku", 0.1, "text", "SKU or product code detected.", _flag("has_sku")),
+        SignalRule("product_gallery", 0.1, "layout", "Product image gallery detected.", _flag("has_product_gallery")),
+        SignalRule("review_section", 0.05, "dom", "Review section detected.", _flag("has_review_section")),
+        SignalRule("url_product_hint", 0.05, "url", "Product URL hint detected.", _url_hint("product"), _url_hint_value),
+    ],
+    PageType.CATEGORY_LISTING_PAGE.value: [
+        SignalRule("repeated_cards", 0.32, "layout", "Repeated cards detected.", _flag("has_repeated_cards"), _value("repeated_card_count")),
+        SignalRule("filter_panel", 0.2, "dom", "Filter panel detected.", _flag("has_filter_panel")),
+        SignalRule("sort_control", 0.15, "dom", "Sort control detected.", _flag("has_sort_control")),
+        SignalRule("pagination", 0.1, "dom", "Pagination detected.", _flag("has_pagination")),
+        SignalRule("product_links", 0.18, "link_graph", "Multiple product links detected.", _count_at_least("product_link_count", 3), _value("product_link_count")),
+        SignalRule("category_url_hint", 0.06, "url", "Category/list URL hint detected.", _url_hint("category"), _url_hint_value),
+    ],
+    PageType.SEARCH_RESULTS_PAGE.value: [
+        SignalRule("search_url_hint", 0.28, "url", "Search URL hint detected.", _url_hint("search"), _url_hint_value),
+        SignalRule("listing_results", 0.18, "layout", "Repeated result cards detected.", _flag("has_repeated_cards"), _value("repeated_card_count")),
+        SignalRule("filter_panel", 0.14, "dom", "Search filter panel detected.", _flag("has_filter_panel")),
+        SignalRule("pagination", 0.12, "dom", "Search pagination detected.", _flag("has_pagination")),
+        SignalRule("search_keywords", 0.12, "text", "Search/result keywords detected.", _keyword("listing"), _keywords_value("listing")),
+    ],
+    PageType.ARTICLE_PAGE.value: [
+        SignalRule("schema_article", 0.35, "structured_data", "Article structured data detected.", _has_schema("Article", "NewsArticle"), _schema_value),
+        SignalRule("author", 0.15, "text", "Author/byline signal detected.", _flag("has_author")),
+        SignalRule("published_date", 0.15, "text", "Published date detected.", _flag("has_published_date")),
+        SignalRule("article_body", 0.2, "dom", "Article body detected.", _flag("has_article_body")),
+        SignalRule("tags", 0.05, "dom", "Article tags detected.", _flag("has_tags")),
+        SignalRule("article_url_hint", 0.05, "url", "Article URL hint detected.", _url_hint("article"), _url_hint_value),
+    ],
+    PageType.BLOG_POST_PAGE.value: [
+        SignalRule("blog_url_hint", 0.25, "url", "Blog URL hint detected.", lambda signals: "article" in signals.url_hints and "blog" in (signals.text_sample or "").lower()),
+        SignalRule("author", 0.14, "text", "Author signal detected.", _flag("has_author")),
+        SignalRule("published_date", 0.14, "text", "Published date detected.", _flag("has_published_date")),
+        SignalRule("article_body", 0.18, "dom", "Article body detected.", _flag("has_article_body")),
+        SignalRule("tags", 0.08, "dom", "Tags detected.", _flag("has_tags")),
+    ],
+    PageType.QA_PAGE.value: [
+        SignalRule("schema_qapage", 0.35, "structured_data", "QAPage structured data detected.", _has_schema("QAPage"), _schema_value),
+        SignalRule("question", 0.2, "community", "Question block detected.", _flag("has_question")),
+        SignalRule("answer", 0.2, "community", "Answer block detected.", _flag("has_answer")),
+        SignalRule("votes", 0.1, "community", "Vote signal detected.", _flag("has_votes")),
+        SignalRule("comments", 0.05, "community", "Comments detected.", _flag("has_comments")),
+        SignalRule("board_url_hint", 0.05, "url", "Board/community URL hint detected.", _url_hint("board"), _url_hint_value),
+    ],
+    PageType.FAQ_PAGE.value: [
+        SignalRule("schema_faq", 0.36, "structured_data", "FAQPage structured data detected.", _has_schema("FAQPage"), _schema_value),
+        SignalRule("faq_structure", 0.25, "community", "FAQ structure detected.", _flag("has_faq_structure")),
+        SignalRule("question", 0.15, "community", "Question content detected.", _flag("has_question")),
+        SignalRule("answer", 0.15, "community", "Answer content detected.", _flag("has_answer")),
+    ],
+    PageType.FORUM_BOARD_PAGE.value: [
+        SignalRule("board_url_hint", 0.25, "url", "Board URL hint detected.", _url_hint("board"), _url_hint_value),
+        SignalRule("thread_structure", 0.2, "community", "Thread structure detected.", _flag("has_thread_structure")),
+        SignalRule("comments", 0.1, "community", "Comments/replies detected.", _flag("has_comments")),
+        SignalRule("repeated_cards", 0.14, "layout", "Repeated post cards detected.", _flag("has_repeated_cards"), _value("repeated_card_count")),
+        SignalRule("pagination", 0.1, "dom", "Board pagination detected.", _flag("has_pagination")),
+    ],
+    PageType.FORUM_THREAD_PAGE.value: [
+        SignalRule("thread_structure", 0.3, "community", "Thread structure detected.", _flag("has_thread_structure")),
+        SignalRule("comments", 0.16, "community", "Comment thread detected.", _flag("has_comments")),
+        SignalRule("question_answer", 0.14, "community", "Question and answer content detected.", lambda signals: signals.has_question and signals.has_answer),
+        SignalRule("votes", 0.08, "community", "Vote signal detected.", _flag("has_votes")),
+        SignalRule("article_body", 0.08, "dom", "Post body detected.", _flag("has_article_body")),
+    ],
+    PageType.BRAND_STORY_PAGE.value: [
+        SignalRule("corporate_keywords", 0.2, "text", "Corporate/brand keywords detected.", _keyword("corporate"), _keywords_value("corporate")),
+        SignalRule("about_url_hint", 0.18, "url", "About/company URL hint detected.", lambda signals: "about" in (signals.text_sample or "").lower() or "brand" in (signals.title or "").lower()),
+        SignalRule("article_body", 0.14, "dom", "Brand story body detected.", _flag("has_article_body")),
+        SignalRule("hero_block", 0.08, "layout", "Hero block detected.", _flag("has_hero_block")),
+    ],
+    PageType.ABOUT_PAGE.value: [
+        SignalRule("corporate_keywords", 0.22, "text", "About/company keywords detected.", _keyword("corporate"), _keywords_value("corporate")),
+        SignalRule("contact_info", 0.08, "text", "Organization contact signal detected.", _flag("has_contact_info")),
+        SignalRule("article_body", 0.12, "dom", "About body detected.", _flag("has_article_body")),
+    ],
+    PageType.CONTACT_PAGE.value: [
+        SignalRule("contact_info", 0.32, "text", "Contact information detected.", _flag("has_contact_info")),
+        SignalRule("address", 0.18, "text", "Address detected.", _flag("has_address")),
+        SignalRule("corporate_keywords", 0.1, "text", "Corporate keywords detected.", _keyword("corporate"), _keywords_value("corporate")),
+    ],
+    PageType.DOCUMENTATION_PAGE.value: [
+        SignalRule("docs_keywords", 0.18, "text", "Documentation keywords detected.", _keyword("knowledge"), _keywords_value("knowledge")),
+        SignalRule("toc", 0.16, "dom", "Table of contents detected.", _flag("has_toc")),
+        SignalRule("code_blocks", 0.16, "dom", "Code blocks detected.", _flag("has_code_blocks")),
+        SignalRule("version_info", 0.08, "text", "Version information detected.", _flag("has_version_info")),
+    ],
+    PageType.API_REFERENCE_PAGE.value: [
+        SignalRule("api_endpoint", 0.3, "text", "API endpoint detected.", _flag("has_api_endpoint")),
+        SignalRule("parameter_table", 0.22, "dom", "Parameter table detected.", _flag("has_parameter_table")),
+        SignalRule("code_blocks", 0.1, "dom", "Code blocks detected.", _flag("has_code_blocks")),
+        SignalRule("docs_keywords", 0.12, "text", "API/docs keywords detected.", _keyword("knowledge"), _keywords_value("knowledge")),
+    ],
+    PageType.WIKI_PAGE.value: [
+        SignalRule("wiki_schema", 0.28, "structured_data", "Wiki/DefinedTerm structured data detected.", _has_schema("DefinedTerm", "WebPage"), _schema_value),
+        SignalRule("toc", 0.12, "dom", "Reference table of contents detected.", _flag("has_toc")),
+        SignalRule("definition_terms", 0.14, "text", "Definition/reference keywords detected.", lambda signals: _contains_terms(signals, ("definition", "wiki", "glossary", "reference"))),
+    ],
+    PageType.DATASET_PAGE.value: [
+        SignalRule("schema_dataset", 0.42, "structured_data", "Dataset structured data detected.", _has_schema("Dataset", "DataCatalog"), _schema_value),
+        SignalRule("download_terms", 0.12, "text", "Dataset/download keywords detected.", lambda signals: _contains_terms(signals, ("dataset", "data catalog", "download", "csv"))),
+        SignalRule("parameter_table", 0.08, "dom", "Dataset metadata table detected.", _flag("has_parameter_table")),
+    ],
+    PageType.RESEARCH_PAPER_PAGE.value: [
+        SignalRule("schema_scholarly", 0.42, "structured_data", "Scholarly article structured data detected.", _has_schema("ScholarlyArticle", "TechArticle"), _schema_value),
+        SignalRule("published_date", 0.1, "text", "Publication date detected.", _flag("has_published_date")),
+        SignalRule("citation_terms", 0.16, "text", "Citation/references keywords detected.", lambda signals: _contains_terms(signals, ("abstract", "citation", "references", "doi"))),
+    ],
+    PageType.JOB_POSTING_PAGE.value: [
+        SignalRule("career_keywords", 0.26, "text", "Career/job keywords detected.", _flag("has_career_terms")),
+        SignalRule("apply_intent", 0.12, "text", "Apply intent detected.", lambda signals: "apply" in (signals.text_sample or "").lower() or "지원" in (signals.text_sample or "")),
+        SignalRule("address", 0.06, "text", "Location/address signal detected.", _flag("has_address")),
+    ],
+    PageType.COURSE_DETAIL_PAGE.value: [
+        SignalRule("schema_course", 0.42, "structured_data", "Course structured data detected.", _has_schema("Course"), _schema_value),
+        SignalRule("course_terms", 0.18, "text", "Course/curriculum keywords detected.", lambda signals: _contains_terms(signals, ("course", "lesson", "curriculum", "instructor", "syllabus"))),
+    ],
+    PageType.VIDEO_PAGE.value: [
+        SignalRule("schema_video", 0.42, "structured_data", "VideoObject structured data detected.", _has_schema("VideoObject"), _schema_value),
+        SignalRule("media_player", 0.2, "layout", "Video/audio player detected.", _flag("has_media_player_area")),
+        SignalRule("video_terms", 0.1, "text", "Video/watch keywords detected.", lambda signals: _contains_terms(signals, ("video", "watch", "episode", "duration"))),
+    ],
+    PageType.LOCAL_BUSINESS_PAGE.value: [
+        SignalRule("schema_local_business", 0.42, "structured_data", "Local business/place structured data detected.", _has_schema("LocalBusiness", "Place", "Restaurant"), _schema_value),
+        SignalRule("address", 0.18, "text", "Address detected.", _flag("has_address")),
+        SignalRule("contact_info", 0.12, "text", "Contact info detected.", _flag("has_contact_info")),
+        SignalRule("map_area", 0.1, "layout", "Map area detected.", _flag("has_map_area")),
+    ],
+    PageType.REAL_ESTATE_LISTING_PAGE.value: [
+        SignalRule("schema_real_estate", 0.38, "structured_data", "Real estate structured data detected.", _has_schema("RealEstateListing", "Residence", "Apartment"), _schema_value),
+        SignalRule("property_terms", 0.18, "text", "Property listing keywords detected.", lambda signals: _contains_terms(signals, ("bedroom", "bathroom", "sqft", "property", "real estate"))),
+        SignalRule("price", 0.12, "text", "Property price detected.", _flag("has_price")),
+        SignalRule("address", 0.08, "text", "Property address detected.", _flag("has_address")),
+    ],
+    PageType.PROFILE_PAGE.value: [
+        SignalRule("schema_person", 0.34, "structured_data", "Person/Profile structured data detected.", _has_schema("Person", "ProfilePage"), _schema_value),
+        SignalRule("profile_links", 0.12, "link_graph", "Profile link pattern detected.", _count_at_least("profile_link_count", 1), _value("profile_link_count")),
+        SignalRule("author", 0.1, "text", "Author/person signal detected.", _flag("has_author")),
+    ],
+    PageType.PRICING_PAGE.value: [
+        SignalRule("pricing_table", 0.3, "layout", "Pricing table detected.", _flag("has_pricing_table")),
+        SignalRule("price", 0.16, "text", "Price detected.", _flag("has_price")),
+        SignalRule("comparison_table", 0.12, "layout", "Comparison table detected.", _flag("has_comparison_table")),
+        SignalRule("commerce_keywords", 0.1, "text", "Commerce keywords detected.", _keyword("commerce"), _keywords_value("commerce")),
+    ],
+    PageType.LOGIN_PAGE.value: [
+        SignalRule("password_field", 0.4, "form", "Password field detected.", _flag("has_password_field")),
+        SignalRule("login_form", 0.25, "form", "Login form detected.", _flag("has_login_form")),
+        SignalRule("protected_url_hint", 0.08, "url", "Protected URL hint detected.", _url_hint("protected"), _url_hint_value),
+    ],
+    PageType.CHECKOUT_PAGE.value: [
+        SignalRule("protected_url_hint", 0.25, "url", "Checkout/cart URL hint detected.", _url_hint("protected"), _url_hint_value),
+        SignalRule("buy_button", 0.14, "dom", "Purchase button detected.", _flag("has_buy_button")),
+        SignalRule("commerce_keywords", 0.12, "text", "Checkout commerce keywords detected.", _keyword("commerce"), _keywords_value("commerce")),
+    ],
+    PageType.PAYMENT_PAGE.value: [
+        SignalRule("payment_fields", 0.42, "form", "Payment fields detected.", _flag("has_payment_fields")),
+        SignalRule("protected_keywords", 0.16, "text", "Payment/protected keywords detected.", _keyword("protected"), _keywords_value("protected")),
+        SignalRule("protected_url_hint", 0.1, "url", "Payment URL hint detected.", _url_hint("protected"), _url_hint_value),
+    ],
+    PageType.TERMS_PAGE.value: [
+        SignalRule("policy_terms", 0.32, "text", "Policy/terms terms detected.", _flag("has_policy_terms")),
+        SignalRule("corporate_keywords", 0.08, "text", "Corporate legal keywords detected.", _keyword("corporate"), _keywords_value("corporate")),
+    ],
+    PageType.PRIVACY_POLICY_PAGE.value: [
+        SignalRule("privacy_terms", 0.36, "text", "Privacy terms detected.", _flag("has_privacy_terms")),
+        SignalRule("policy_terms", 0.14, "text", "Policy terms detected.", _flag("has_policy_terms")),
+    ],
+    PageType.SITEMAP_PAGE.value: [
+        SignalRule("sitemap_resource", 0.45, "resource", "Sitemap resource detected.", _flag("has_sitemap_resource")),
+        SignalRule("xml_resource", 0.12, "resource", "XML resource detected.", _flag("has_xml_resource")),
+        SignalRule("system_url_hint", 0.12, "url", "System URL hint detected.", _url_hint("system"), _url_hint_value),
+    ],
+    PageType.RSS_FEED_PAGE.value: [
+        SignalRule("feed_resource", 0.45, "resource", "RSS/Atom feed detected.", _flag("has_feed_resource")),
+        SignalRule("xml_resource", 0.1, "resource", "XML feed resource detected.", _flag("has_xml_resource")),
+    ],
+    PageType.ERROR_PAGE.value: [
+        SignalRule("error_status", 0.38, "http", "Error status detected.", _flag("has_error_status")),
+    ],
+    PageType.NOT_FOUND_PAGE.value: [
+        SignalRule("not_found", 0.5, "http", "404/not found signal detected.", _flag("has_not_found")),
+        SignalRule("error_status", 0.12, "http", "Error status supports not found page.", _flag("has_error_status")),
+    ],
+    PageType.ACCESS_DENIED_PAGE.value: [
+        SignalRule("access_denied", 0.42, "protected", "Access denied text detected.", _flag("has_access_denied")),
+        SignalRule("error_status", 0.1, "http", "Error status supports access denied.", _flag("has_error_status")),
+    ],
+    PageType.CAPTCHA_PAGE.value: [
+        SignalRule("captcha", 0.45, "protected", "Captcha detected.", _flag("has_captcha")),
+        SignalRule("protected_keywords", 0.1, "text", "Protected keywords detected.", _keyword("protected"), _keywords_value("protected")),
+    ],
+}

ontology_platform/crawler_platform/app/core/crawler/page_type_taxonomy.py

@@ -0,0 +1,476 @@
+from __future__ import annotations
+
+from dataclasses import asdict, dataclass, field
+from enum import StrEnum
+from typing import Any
+
+
+class PageDomain(StrEnum):
+    COMMERCE = "Commerce"
+    EDITORIAL = "Editorial"
+    COMMUNITY = "Community"
+    KNOWLEDGE = "Knowledge"
+    CORPORATE = "Corporate"
+    LOCAL = "Local"
+    EDUCATION = "Education"
+    JOBS = "Jobs"
+    MEDIA = "Media"
+    SOFTWARE = "Software"
+    FINANCE = "Finance"
+    GOVERNMENT = "Government"
+    HEALTHCARE = "Healthcare"
+    TRANSACTION = "Transaction"
+    SYSTEM = "System"
+    UNKNOWN = "Unknown"
+
+
+class PageArchetype(StrEnum):
+    HOME = "Home"
+    LANDING = "Landing"
+    DETAIL = "Detail"
+    LISTING = "Listing"
+    COLLECTION = "Collection"
+    SEARCH_RESULT = "SearchResult"
+    PROFILE = "Profile"
+    ARTICLE = "Article"
+    THREAD = "Thread"
+    FORM = "Form"
+    TRANSACTION = "Transaction"
+    DASHBOARD = "Dashboard"
+    DOCUMENT = "Document"
+    MEDIA = "Media"
+    ERROR = "Error"
+    SYSTEM_RESOURCE = "SystemResource"
+    UNKNOWN = "Unknown"
+
+
+class PageType(StrEnum):
+    PRODUCT_PAGE = "ProductPage"
+    CATEGORY_PAGE = "CategoryPage"
+    SEARCH_PAGE = "SearchPage"
+    BOARD_PAGE = "BoardPage"
+    NOTICE_PAGE = "NoticePage"
+    BRAND_STORY_PAGE = "BrandStoryPage"
+    ABOUT_PAGE = "AboutPage"
+    CONTACT_PAGE = "ContactPage"
+    PROMOTION_PAGE = "PromotionPage"
+    REVIEW_PAGE = "ReviewPage"
+    UNKNOWN_PAGE = "UnknownPage"
+    PRODUCT_DETAIL_PAGE = "ProductDetailPage"
+    CATEGORY_LISTING_PAGE = "CategoryListingPage"
+    SEARCH_RESULTS_PAGE = "SearchResultsPage"
+    FORUM_BOARD_PAGE = "ForumBoardPage"
+    FORUM_THREAD_PAGE = "ForumThreadPage"
+    PUBLIC_NOTICE_PAGE = "PublicNoticePage"
+    CAMPAIGN_LANDING_PAGE = "CampaignLandingPage"
+    ARTICLE_PAGE = "ArticlePage"
+    NEWS_ARTICLE_PAGE = "NewsArticlePage"
+    BLOG_POST_PAGE = "BlogPostPage"
+    FAQ_PAGE = "FAQPage"
+    QA_PAGE = "QAPage"
+    PROFILE_PAGE = "ProfilePage"
+    DOCUMENTATION_PAGE = "DocumentationPage"
+    API_REFERENCE_PAGE = "APIReferencePage"
+    WIKI_PAGE = "WikiPage"
+    DATASET_PAGE = "DatasetPage"
+    RESEARCH_PAPER_PAGE = "ResearchPaperPage"
+    JOB_POSTING_PAGE = "JobPostingPage"
+    COURSE_DETAIL_PAGE = "CourseDetailPage"
+    VIDEO_PAGE = "VideoPage"
+    LOCAL_BUSINESS_PAGE = "LocalBusinessPage"
+    REAL_ESTATE_LISTING_PAGE = "RealEstateListingPage"
+    PRICING_PAGE = "PricingPage"
+    LOGIN_PAGE = "LoginPage"
+    CHECKOUT_PAGE = "CheckoutPage"
+    PAYMENT_PAGE = "PaymentPage"
+    TERMS_PAGE = "TermsPage"
+    PRIVACY_POLICY_PAGE = "PrivacyPolicyPage"
+    SITEMAP_PAGE = "SitemapPage"
+    RSS_FEED_PAGE = "RSSFeedPage"
+    ERROR_PAGE = "ErrorPage"
+    NOT_FOUND_PAGE = "NotFoundPage"
+    ACCESS_DENIED_PAGE = "AccessDeniedPage"
+    CAPTCHA_PAGE = "CaptchaPage"
+
+
+class EntityType(StrEnum):
+    PRODUCT = "Product"
+    SERVICE = "Service"
+    ARTICLE = "Article"
+    NEWS_ARTICLE = "NewsArticle"
+    PERSON = "Person"
+    ORGANIZATION = "Organization"
+    PLACE = "Place"
+    EVENT = "Event"
+    JOB_POSTING = "JobPosting"
+    COURSE = "Course"
+    QUESTION = "Question"
+    ANSWER = "Answer"
+    REVIEW = "Review"
+    DATASET = "Dataset"
+    SOFTWARE_APPLICATION = "SoftwareApplication"
+    MEDIA_OBJECT = "MediaObject"
+    RECIPE = "Recipe"
+    REAL_ESTATE_PROPERTY = "RealEstateProperty"
+    MEDICAL_CONDITION = "MedicalCondition"
+    LEGAL_DOCUMENT = "LegalDocument"
+    FINANCIAL_PRODUCT = "FinancialProduct"
+    UNKNOWN_ENTITY = "UnknownEntity"
+
+
+class ActionIntent(StrEnum):
+    READ = "Read"
+    BUY = "Buy"
+    SUBSCRIBE = "Subscribe"
+    RESERVE = "Reserve"
+    BOOK = "Book"
+    APPLY = "Apply"
+    DOWNLOAD = "Download"
+    WATCH = "Watch"
+    LISTEN = "Listen"
+    SEARCH = "Search"
+    COMPARE = "Compare"
+    FILTER = "Filter"
+    ASK = "Ask"
+    ANSWER = "Answer"
+    COMMENT = "Comment"
+    REVIEW = "Review"
+    LOGIN = "Login"
+    REGISTER = "Register"
+    PAY = "Pay"
+    CONTACT = "Contact"
+    NAVIGATE = "Navigate"
+    LEARN = "Learn"
+    VERIFY = "Verify"
+    CONFIGURE = "Configure"
+    MANAGE = "Manage"
+
+
+class GraphRole(StrEnum):
+    ENTITY_ANCHOR = "EntityAnchor"
+    RELATION_HUB = "RelationHub"
+    NAVIGATION_HUB = "NavigationHub"
+    COLLECTION_HUB = "CollectionHub"
+    SEARCH_HUB = "SearchHub"
+    TRANSACTION_ONLY = "TransactionOnly"
+    POLICY_SOURCE = "PolicySource"
+    CLAIM_SOURCE = "ClaimSource"
+    PROFILE_ANCHOR = "ProfileAnchor"
+    MEDIA_ANCHOR = "MediaAnchor"
+    REFERENCE_SOURCE = "ReferenceSource"
+    SYSTEM_RESOURCE = "SystemResource"
+    NOISE_PAGE = "NoisePage"
+    UNKNOWN_PATTERN = "UnknownPattern"
+
+
+class AnalyzeStrategy(StrEnum):
+    ANALYZE_FULL = "AnalyzeFull"
+    ANALYZE_STRUCTURE_ONLY = "AnalyzeStructureOnly"
+    ANALYZE_ENTITY_ONLY = "AnalyzeEntityOnly"
+    ANALYZE_RELATIONS_ONLY = "AnalyzeRelationsOnly"
+    ANALYZE_METADATA_ONLY = "AnalyzeMetadataOnly"
+    ANALYZE_DOCUMENT_ONLY = "AnalyzeDocumentOnly"
+    ANALYZE_DISCOVERY_ONLY = "AnalyzeDiscoveryOnly"
+    SKIP_PROTECTED = "SkipProtected"
+    SKIP_NOISE = "SkipNoise"
+
+
+class LLMPolicy(StrEnum):
+    LLM_FULL = "LLMFull"
+    LLM_LIGHT = "LLMLight"
+    LLM_FOR_AMBIGUITY_ONLY = "LLMForAmbiguityOnly"
+    RULE_ONLY = "RuleOnly"
+    NO_LLM = "NoLLM"
+    SKIP = "Skip"
+
+
+@dataclass(frozen=True, slots=True)
+class EvidenceItem:
+    key: str
+    value: str | int | float | bool | None
+    weight: float
+    source: str
+    message: str
+
+    def to_dict(self) -> dict[str, Any]:
+        return asdict(self)
+
+
+@dataclass(slots=True)
+class PageClassificationResult:
+    url: str
+    primary_page_type: str
+    secondary_page_types: list[str] = field(default_factory=list)
+    domain: str = PageDomain.UNKNOWN.value
+    archetype: str = PageArchetype.UNKNOWN.value
+    main_entity_type: str | None = None
+    action_intents: list[str] = field(default_factory=list)
+    graph_roles: list[str] = field(default_factory=lambda: [GraphRole.UNKNOWN_PATTERN.value])
+    confidence: float = 0.0
+    alternatives: list[tuple[str, float]] = field(default_factory=list)
+    evidence: list[EvidenceItem] = field(default_factory=list)
+    should_analyze: bool = False
+    analyze_strategy: str = AnalyzeStrategy.ANALYZE_METADATA_ONLY.value
+    llm_policy: str = LLMPolicy.NO_LLM.value
+    is_protected: bool = False
+    is_noise: bool = False
+
+    def to_dict(self) -> dict[str, Any]:
+        payload = asdict(self)
+        payload["evidence"] = [item.to_dict() for item in self.evidence]
+        payload["legacy_page_type"] = get_legacy_page_type(self)
+        return payload
+
+
+LEGACY_PAGE_TYPES = {
+    PageType.PRODUCT_PAGE.value,
+    PageType.CATEGORY_PAGE.value,
+    PageType.SEARCH_PAGE.value,
+    PageType.BOARD_PAGE.value,
+    PageType.NOTICE_PAGE.value,
+    PageType.BRAND_STORY_PAGE.value,
+    PageType.PROMOTION_PAGE.value,
+    PageType.REVIEW_PAGE.value,
+    PageType.UNKNOWN_PAGE.value,
+    "EventPage",
+}
+
+
+LEGACY_ALIASES: dict[str, str] = {
+    "product": PageType.PRODUCT_PAGE.value,
+    "productpage": PageType.PRODUCT_PAGE.value,
+    "productdetailpage": PageType.PRODUCT_PAGE.value,
+    "brand": PageType.BRAND_STORY_PAGE.value,
+    "brandpage": PageType.BRAND_STORY_PAGE.value,
+    "brandstorypage": PageType.BRAND_STORY_PAGE.value,
+    "about": PageType.BRAND_STORY_PAGE.value,
+    "aboutpage": "AboutPage",
+    "contact": "ContactPage",
+    "contactpage": "ContactPage",
+    "review": PageType.REVIEW_PAGE.value,
+    "reviewpage": PageType.REVIEW_PAGE.value,
+    "productreviewpage": PageType.REVIEW_PAGE.value,
+    "listing": PageType.CATEGORY_PAGE.value,
+    "listingpage": PageType.CATEGORY_PAGE.value,
+    "category": PageType.CATEGORY_PAGE.value,
+    "categorypage": PageType.CATEGORY_PAGE.value,
+    "categorylistingpage": PageType.CATEGORY_PAGE.value,
+    "productlistingpage": PageType.CATEGORY_PAGE.value,
+    "community": PageType.BOARD_PAGE.value,
+    "communitypage": PageType.BOARD_PAGE.value,
+    "board": PageType.BOARD_PAGE.value,
+    "boardpage": PageType.BOARD_PAGE.value,
+    "forumboardpage": PageType.BOARD_PAGE.value,
+    "forumthreadpage": PageType.BOARD_PAGE.value,
+    "search": PageType.SEARCH_PAGE.value,
+    "searchpage": PageType.SEARCH_PAGE.value,
+    "searchresultspage": PageType.SEARCH_PAGE.value,
+    "notice": PageType.NOTICE_PAGE.value,
+    "noticepage": PageType.NOTICE_PAGE.value,
+    "publicnoticepage": PageType.NOTICE_PAGE.value,
+    "promotion": PageType.PROMOTION_PAGE.value,
+    "promotionpage": PageType.PROMOTION_PAGE.value,
+    "campaignlandingpage": PageType.PROMOTION_PAGE.value,
+    "event": "EventPage",
+    "eventpage": "EventPage",
+    "unknown": PageType.UNKNOWN_PAGE.value,
+    "unknownpage": PageType.UNKNOWN_PAGE.value,
+    "notfoundpage": PageType.NOT_FOUND_PAGE.value,
+    "404": PageType.NOT_FOUND_PAGE.value,
+}
+
+
+LEGACY_TO_SEMANTIC_PAGE_TYPE: dict[str, str] = {
+    PageType.PRODUCT_PAGE.value: PageType.PRODUCT_DETAIL_PAGE.value,
+    PageType.CATEGORY_PAGE.value: PageType.CATEGORY_LISTING_PAGE.value,
+    PageType.SEARCH_PAGE.value: PageType.SEARCH_RESULTS_PAGE.value,
+    PageType.BOARD_PAGE.value: PageType.FORUM_BOARD_PAGE.value,
+    PageType.NOTICE_PAGE.value: PageType.PUBLIC_NOTICE_PAGE.value,
+    PageType.BRAND_STORY_PAGE.value: PageType.BRAND_STORY_PAGE.value,
+    PageType.ABOUT_PAGE.value: PageType.ABOUT_PAGE.value,
+    PageType.CONTACT_PAGE.value: PageType.CONTACT_PAGE.value,
+    PageType.PROMOTION_PAGE.value: PageType.PROMOTION_PAGE.value,
+    PageType.REVIEW_PAGE.value: PageType.REVIEW_PAGE.value,
+    PageType.UNKNOWN_PAGE.value: PageType.UNKNOWN_PAGE.value,
+    "EventPage": "EventPage",
+}
+
+
+SEMANTIC_TO_LEGACY_PAGE_TYPE: dict[str, str] = {
+    semantic: legacy for legacy, semantic in LEGACY_TO_SEMANTIC_PAGE_TYPE.items()
+}
+SEMANTIC_TO_LEGACY_PAGE_TYPE.update(
+    {
+        PageType.PRODUCT_DETAIL_PAGE.value: PageType.PRODUCT_PAGE.value,
+        PageType.CATEGORY_LISTING_PAGE.value: PageType.CATEGORY_PAGE.value,
+        PageType.SEARCH_RESULTS_PAGE.value: PageType.SEARCH_PAGE.value,
+        PageType.FORUM_BOARD_PAGE.value: PageType.BOARD_PAGE.value,
+        PageType.FORUM_THREAD_PAGE.value: PageType.BOARD_PAGE.value,
+        PageType.PUBLIC_NOTICE_PAGE.value: PageType.NOTICE_PAGE.value,
+        PageType.CAMPAIGN_LANDING_PAGE.value: PageType.PROMOTION_PAGE.value,
+        PageType.ABOUT_PAGE.value: PageType.BRAND_STORY_PAGE.value,
+        PageType.CONTACT_PAGE.value: PageType.BRAND_STORY_PAGE.value,
+        "ListingPage": PageType.CATEGORY_PAGE.value,
+        "CommunityPage": PageType.BOARD_PAGE.value,
+    }
+)
+
+
+PAGE_TYPE_PROFILES: dict[str, dict[str, Any]] = {
+    PageType.PRODUCT_DETAIL_PAGE.value: {
+        "domain": PageDomain.COMMERCE.value,
+        "archetype": PageArchetype.DETAIL.value,
+        "main_entity_type": EntityType.PRODUCT.value,
+        "action_intents": [ActionIntent.BUY.value, ActionIntent.REVIEW.value],
+        "graph_roles": [GraphRole.ENTITY_ANCHOR.value],
+        "analyze_strategy": AnalyzeStrategy.ANALYZE_FULL.value,
+        "llm_policy": LLMPolicy.LLM_LIGHT.value,
+        "should_analyze": True,
+    },
+    PageType.CATEGORY_LISTING_PAGE.value: {
+        "domain": PageDomain.COMMERCE.value,
+        "archetype": PageArchetype.LISTING.value,
+        "main_entity_type": EntityType.PRODUCT.value,
+        "action_intents": [ActionIntent.FILTER.value, ActionIntent.NAVIGATE.value],
+        "graph_roles": [GraphRole.COLLECTION_HUB.value, GraphRole.RELATION_HUB.value],
+        "analyze_strategy": AnalyzeStrategy.ANALYZE_RELATIONS_ONLY.value,
+        "llm_policy": LLMPolicy.LLM_FOR_AMBIGUITY_ONLY.value,
+        "should_analyze": True,
+    },
+    PageType.SEARCH_RESULTS_PAGE.value: {
+        "domain": PageDomain.UNKNOWN.value,
+        "archetype": PageArchetype.SEARCH_RESULT.value,
+        "main_entity_type": EntityType.UNKNOWN_ENTITY.value,
+        "action_intents": [ActionIntent.SEARCH.value, ActionIntent.NAVIGATE.value],
+        "graph_roles": [GraphRole.SEARCH_HUB.value],
+        "analyze_strategy": AnalyzeStrategy.ANALYZE_DISCOVERY_ONLY.value,
+        "llm_policy": LLMPolicy.LLM_FOR_AMBIGUITY_ONLY.value,
+        "should_analyze": True,
+    },
+    PageType.FORUM_BOARD_PAGE.value: {
+        "domain": PageDomain.COMMUNITY.value,
+        "archetype": PageArchetype.LISTING.value,
+        "main_entity_type": EntityType.ARTICLE.value,
+        "action_intents": [ActionIntent.NAVIGATE.value, ActionIntent.READ.value],
+        "graph_roles": [GraphRole.RELATION_HUB.value, GraphRole.COLLECTION_HUB.value],
+        "analyze_strategy": AnalyzeStrategy.ANALYZE_RELATIONS_ONLY.value,
+        "llm_policy": LLMPolicy.LLM_FOR_AMBIGUITY_ONLY.value,
+        "should_analyze": True,
+    },
+    PageType.PUBLIC_NOTICE_PAGE.value: {
+        "domain": PageDomain.GOVERNMENT.value,
+        "archetype": PageArchetype.DOCUMENT.value,
+        "main_entity_type": EntityType.ARTICLE.value,
+        "action_intents": [ActionIntent.READ.value],
+        "graph_roles": [GraphRole.CLAIM_SOURCE.value],
+        "analyze_strategy": AnalyzeStrategy.ANALYZE_FULL.value,
+        "llm_policy": LLMPolicy.LLM_LIGHT.value,
+        "should_analyze": True,
+    },
+    PageType.BRAND_STORY_PAGE.value: {
+        "domain": PageDomain.CORPORATE.value,
+        "archetype": PageArchetype.ARTICLE.value,
+        "main_entity_type": EntityType.ORGANIZATION.value,
+        "action_intents": [ActionIntent.READ.value, ActionIntent.LEARN.value],
+        "graph_roles": [GraphRole.ENTITY_ANCHOR.value, GraphRole.CLAIM_SOURCE.value],
+        "analyze_strategy": AnalyzeStrategy.ANALYZE_FULL.value,
+        "llm_policy": LLMPolicy.LLM_FULL.value,
+        "should_analyze": True,
+    },
+    PageType.PROMOTION_PAGE.value: {
+        "domain": PageDomain.COMMERCE.value,
+        "archetype": PageArchetype.LANDING.value,
+        "main_entity_type": EntityType.EVENT.value,
+        "action_intents": [ActionIntent.BUY.value, ActionIntent.NAVIGATE.value],
+        "graph_roles": [GraphRole.CLAIM_SOURCE.value],
+        "analyze_strategy": AnalyzeStrategy.ANALYZE_FULL.value,
+        "llm_policy": LLMPolicy.LLM_LIGHT.value,
+        "should_analyze": True,
+    },
+    PageType.REVIEW_PAGE.value: {
+        "domain": PageDomain.COMMUNITY.value,
+        "archetype": PageArchetype.ARTICLE.value,
+        "main_entity_type": EntityType.REVIEW.value,
+        "action_intents": [ActionIntent.READ.value, ActionIntent.REVIEW.value],
+        "graph_roles": [GraphRole.CLAIM_SOURCE.value],
+        "analyze_strategy": AnalyzeStrategy.ANALYZE_FULL.value,
+        "llm_policy": LLMPolicy.LLM_LIGHT.value,
+        "should_analyze": True,
+    },
+    PageType.UNKNOWN_PAGE.value: {
+        "domain": PageDomain.UNKNOWN.value,
+        "archetype": PageArchetype.UNKNOWN.value,
+        "main_entity_type": EntityType.UNKNOWN_ENTITY.value,
+        "action_intents": [],
+        "graph_roles": [GraphRole.UNKNOWN_PATTERN.value],
+        "analyze_strategy": AnalyzeStrategy.ANALYZE_METADATA_ONLY.value,
+        "llm_policy": LLMPolicy.NO_LLM.value,
+        "should_analyze": False,
+    },
+}
+
+
+def normalize_page_type(value: object | None) -> str:
+    """Return the legacy-compatible page type string for existing callers."""
+
+    if isinstance(value, PageClassificationResult):
+        return get_legacy_page_type(value)
+    clean = str(value or "").strip()
+    if not clean:
+        return PageType.UNKNOWN_PAGE.value
+    alias = LEGACY_ALIASES.get(clean.lower())
+    if alias:
+        return alias
+    return SEMANTIC_TO_LEGACY_PAGE_TYPE.get(clean, clean)
+
+
+def normalize_semantic_page_type(value: object | None) -> str:
+    if isinstance(value, PageClassificationResult):
+        return value.primary_page_type
+    legacy = normalize_page_type(value)
+    return LEGACY_TO_SEMANTIC_PAGE_TYPE.get(legacy, str(value or legacy).strip() or PageType.UNKNOWN_PAGE.value)
+
+
+def get_legacy_page_type(value: object | None) -> str:
+    if isinstance(value, PageClassificationResult):
+        return SEMANTIC_TO_LEGACY_PAGE_TYPE.get(value.primary_page_type, value.primary_page_type)
+    return normalize_page_type(value)
+
+
+def build_classification_result_from_legacy(
+    *,
+    url: str,
+    legacy_page_type: str,
+    confidence: float = 0.55,
+    source: str = "legacy_classifier",
+) -> PageClassificationResult:
+    normalized_legacy = normalize_page_type(legacy_page_type)
+    semantic_page_type = LEGACY_TO_SEMANTIC_PAGE_TYPE.get(normalized_legacy, normalized_legacy)
+    profile = PAGE_TYPE_PROFILES.get(semantic_page_type, PAGE_TYPE_PROFILES[PageType.UNKNOWN_PAGE.value])
+    evidence = [
+        EvidenceItem(
+            key="legacy_page_type",
+            value=normalized_legacy,
+            weight=confidence,
+            source=source,
+            message=f"Legacy classifier returned {normalized_legacy}.",
+        )
+    ]
+    return PageClassificationResult(
+        url=url,
+        primary_page_type=semantic_page_type,
+        secondary_page_types=[] if semantic_page_type == normalized_legacy else [normalized_legacy],
+        domain=str(profile["domain"]),
+        archetype=str(profile["archetype"]),
+        main_entity_type=profile.get("main_entity_type"),
+        action_intents=list(profile.get("action_intents") or []),
+        graph_roles=list(profile.get("graph_roles") or []),
+        confidence=confidence,
+        alternatives=[(normalized_legacy, confidence)],
+        evidence=evidence,
+        should_analyze=bool(profile.get("should_analyze")),
+        analyze_strategy=str(profile["analyze_strategy"]),
+        llm_policy=str(profile["llm_policy"]),
+        is_protected=str(profile["analyze_strategy"]) == AnalyzeStrategy.SKIP_PROTECTED.value,
+        is_noise=str(profile["analyze_strategy"]) == AnalyzeStrategy.SKIP_NOISE.value,
+    )

ontology_platform/crawler_platform/app/core/crawler/page_unknown_patterns.py

@@ -0,0 +1,347 @@
+from __future__ import annotations
+
+from collections import Counter
+import hashlib
+import json
+import re
+from typing import Any
+from urllib.parse import urljoin, urlparse
+
+from crawler_platform.app.core.crawler.page_signal_extractor import (
+    build_raw_page_snapshot,
+    extract_page_signals,
+)
+from crawler_platform.app.core.crawler.page_type_taxonomy import PageClassificationResult, PageType
+
+
+UNKNOWN_PATTERN_VERSION = 1
+LOW_CONFIDENCE_PATTERN_THRESHOLD = 0.45
+STOPWORDS = {
+    "about",
+    "after",
+    "again",
+    "also",
+    "and",
+    "are",
+    "but",
+    "can",
+    "for",
+    "from",
+    "has",
+    "have",
+    "home",
+    "into",
+    "more",
+    "not",
+    "our",
+    "page",
+    "that",
+    "the",
+    "this",
+    "with",
+    "your",
+}
+
+
+def should_store_unknown_pattern(
+    result: PageClassificationResult,
+    *,
+    confidence_threshold: float = LOW_CONFIDENCE_PATTERN_THRESHOLD,
+) -> bool:
+    if result.primary_page_type == PageType.UNKNOWN_PAGE.value:
+        return True
+    if 0.0 < result.confidence <= confidence_threshold:
+        return True
+    return any(item.key == "low_confidence" for item in result.evidence)
+
+
+def build_unknown_pattern_payload(
+    *,
+    result: PageClassificationResult,
+    url: str,
+    title: str | None = None,
+    text: str | None = None,
+    html: str | None = None,
+    source_zones: list[str | dict[str, Any]] | None = None,
+    collector_payload: dict[str, Any] | None = None,
+) -> dict[str, Any]:
+    snapshot = build_raw_page_snapshot(
+        url=url,
+        title=title,
+        text=text,
+        html=html,
+        source_zones=source_zones,
+        collector_payload=collector_payload,
+    )
+    signals = extract_page_signals(snapshot)
+    soup = _soup_from_html(html or "")
+    page_text = _normalize_text(text or signals.text_sample or _text_from_soup(soup))
+    links = snapshot.links or _extract_links(soup, url)
+    buttons = snapshot.buttons or _extract_buttons(soup)
+    forms = snapshot.forms or _extract_forms(soup)
+    link_summary = summarize_link_patterns(links, url)
+    forms_summary = summarize_forms(forms)
+    payload = {
+        "version": UNKNOWN_PATTERN_VERSION,
+        "url": url,
+        "title": title or signals.title,
+        "primary_page_type": result.primary_page_type,
+        "confidence": result.confidence,
+        "reason": "unknown_page" if result.primary_page_type == PageType.UNKNOWN_PAGE.value else "low_confidence",
+        "text_sample": page_text[:1000],
+        "text_fingerprint": text_fingerprint(page_text),
+        "html_fingerprint": html_fingerprint(html or ""),
+        "dom_fingerprint": dom_fingerprint(html or ""),
+        "link_pattern_fingerprint": link_pattern_fingerprint(link_summary),
+        "schema_types": sorted(signals.schema_types),
+        "link_pattern_summary": link_summary,
+        "button_labels": _dedupe_strings(buttons)[:20],
+        "forms_summary": forms_summary,
+        "top_keywords": top_keywords(page_text),
+        "layout_blocks": list(signals.layout_blocks),
+        "keyword_hits": dict(signals.keyword_hits),
+        "alternatives": list(result.alternatives[:8]),
+        "evidence": [item.to_dict() for item in result.evidence[:12]],
+    }
+    payload["embedding_input"] = build_unknown_embedding_input(payload)
+    payload["cluster_candidate"] = build_cluster_candidate_payload(payload)
+    return payload
+
+
+def build_unknown_embedding_input(payload: dict[str, Any]) -> str:
+    parts = [
+        str(payload.get("title") or ""),
+        str(payload.get("text_sample") or ""),
+        "schema_types: " + ", ".join(str(item) for item in payload.get("schema_types") or []),
+        "layout_blocks: " + ", ".join(str(item) for item in payload.get("layout_blocks") or []),
+        "buttons: " + ", ".join(str(item) for item in payload.get("button_labels") or []),
+        "keywords: " + ", ".join(str(item.get("term")) for item in payload.get("top_keywords") or [] if isinstance(item, dict)),
+    ]
+    return "\n".join(part for part in parts if part.strip())[:4000]
+
+
+def build_cluster_candidate_payload(payload: dict[str, Any]) -> dict[str, Any]:
+    return {
+        "version": UNKNOWN_PATTERN_VERSION,
+        "candidate_key": stable_hash(
+            {
+                "dom": payload.get("dom_fingerprint"),
+                "links": payload.get("link_pattern_fingerprint"),
+                "schema_types": payload.get("schema_types") or [],
+                "buttons": payload.get("button_labels") or [],
+            }
+        ),
+        "fingerprints": {
+            "text": payload.get("text_fingerprint"),
+            "html": payload.get("html_fingerprint"),
+            "dom": payload.get("dom_fingerprint"),
+            "links": payload.get("link_pattern_fingerprint"),
+        },
+        "features": {
+            "schema_types": payload.get("schema_types") or [],
+            "layout_blocks": payload.get("layout_blocks") or [],
+            "top_keywords": payload.get("top_keywords") or [],
+            "link_patterns": (payload.get("link_pattern_summary") or {}).get("top_path_patterns") or [],
+            "form_count": (payload.get("forms_summary") or {}).get("form_count") or 0,
+        },
+    }
+
+
+def summarize_link_patterns(links: list[dict[str, Any]], base_url: str) -> dict[str, Any]:
+    base_host = urlparse(base_url).netloc.lower()
+    path_patterns = Counter()
+    text_labels: list[str] = []
+    internal_count = 0
+    external_count = 0
+    for link in links:
+        href = str(link.get("href") or "")
+        parsed = urlparse(href)
+        if not href:
+            continue
+        if not parsed.netloc or parsed.netloc.lower() == base_host:
+            internal_count += 1
+        else:
+            external_count += 1
+        path_patterns[_path_pattern(parsed.path)] += 1
+        text = str(link.get("text") or "").strip()
+        if text:
+            text_labels.append(text)
+    return {
+        "total_count": len(links),
+        "internal_count": internal_count,
+        "external_count": external_count,
+        "top_path_patterns": [
+            {"pattern": pattern, "count": count}
+            for pattern, count in path_patterns.most_common(12)
+            if pattern
+        ],
+        "sample_texts": _dedupe_strings(text_labels)[:12],
+    }
+
+
+def summarize_forms(forms: list[dict[str, Any]]) -> dict[str, Any]:
+    methods = Counter()
+    actions = Counter()
+    input_types = Counter()
+    form_texts: list[str] = []
+    for form in forms:
+        method = str(form.get("method") or "get").lower()
+        action = _path_pattern(urlparse(str(form.get("action") or "")).path)
+        methods[method] += 1
+        if action:
+            actions[action] += 1
+        if form.get("text"):
+            form_texts.append(str(form.get("text")))
+        for input_item in form.get("inputs") or []:
+            if isinstance(input_item, dict):
+                input_types[str(input_item.get("type") or "text").lower()] += 1
+    return {
+        "form_count": len(forms),
+        "methods": dict(methods),
+        "action_patterns": dict(actions.most_common(8)),
+        "input_types": dict(input_types.most_common(12)),
+        "sample_texts": _dedupe_strings(form_texts)[:8],
+    }
+
+
+def top_keywords(text: str, *, limit: int = 16) -> list[dict[str, Any]]:
+    tokens = [
+        token.lower()
+        for token in re.findall(r"[A-Za-z][A-Za-z0-9_-]{2,}|[가-힣]{2,}", text or "")
+        if token.lower() not in STOPWORDS
+    ]
+    return [{"term": term, "count": count} for term, count in Counter(tokens).most_common(limit)]
+
+
+def text_fingerprint(text: str) -> str:
+    return f"sha256:{_hash_text(_normalize_text(text))}"
+
+
+def html_fingerprint(html: str) -> str:
+    return f"sha256:{_hash_text(_normalize_html(html))}"
+
+
+def dom_fingerprint(html: str) -> str:
+    soup = _soup_from_html(html)
+    if soup is None:
+        return f"sha256:{_hash_text('')}"
+    tags = []
+    for node in soup.find_all(True):
+        classes = ".".join(str(item).lower() for item in node.get("class", [])[:3])
+        node_id = "#" + str(node.get("id")).lower() if node.get("id") else ""
+        tags.append(f"{node.name}{node_id}{('.' + classes) if classes else ''}")
+    return f"sha256:{_hash_text('>'.join(tags[:400]))}"
+
+
+def link_pattern_fingerprint(link_summary: dict[str, Any]) -> str:
+    return stable_hash(link_summary)
+
+
+def stable_hash(value: Any) -> str:
+    payload = json.dumps(value, sort_keys=True, ensure_ascii=False, separators=(",", ":"))
+    return f"sha256:{_hash_text(payload)}"
+
+
+def _hash_text(value: str) -> str:
+    return hashlib.sha256(value.encode("utf-8", errors="ignore")).hexdigest()
+
+
+def _normalize_text(value: str | None) -> str:
+    return re.sub(r"\s+", " ", str(value or "")).strip()
+
+
+def _normalize_html(value: str | None) -> str:
+    text = re.sub(r">\s+<", "><", str(value or ""))
+    text = re.sub(r"\s+", " ", text)
+    return text.strip()[:200000]
+
+
+def _path_pattern(path: str) -> str:
+    parts = [part for part in path.split("/") if part]
+    normalized = []
+    for part in parts[:8]:
+        if part.isdigit() or re.fullmatch(r"[0-9a-fA-F-]{8,}", part):
+            normalized.append("{id}")
+        else:
+            normalized.append(re.sub(r"\d+", "{n}", part.lower())[:60])
+    return "/" + "/".join(normalized) if normalized else "/"
+
+
+def _soup_from_html(html: str):
+    if not html:
+        return None
+    try:
+        from bs4 import BeautifulSoup
+    except ImportError:
+        return None
+    try:
+        return BeautifulSoup(html, "html.parser")
+    except Exception:
+        return None
+
+
+def _text_from_soup(soup) -> str:
+    if soup is None:
+        return ""
+    return soup.get_text(" ", strip=True)
+
+
+def _extract_links(soup, base_url: str) -> list[dict[str, Any]]:
+    if soup is None:
+        return []
+    links = []
+    for tag in soup.find_all("a"):
+        href = str(tag.get("href") or "").strip()
+        if not href:
+            continue
+        links.append({"href": urljoin(base_url, href), "text": tag.get_text(" ", strip=True)})
+    return links
+
+
+def _extract_buttons(soup) -> list[str]:
+    if soup is None:
+        return []
+    labels = []
+    for tag in soup.select("button,input[type='submit'],input[type='button'],[role='button']"):
+        label = tag.get_text(" ", strip=True) or str(tag.get("value") or tag.get("aria-label") or "")
+        if label.strip():
+            labels.append(label.strip())
+    return labels
+
+
+def _extract_forms(soup) -> list[dict[str, Any]]:
+    if soup is None:
+        return []
+    forms = []
+    for form in soup.find_all("form"):
+        inputs = []
+        for tag in form.select("input,select,textarea"):
+            inputs.append(
+                {
+                    "type": str(tag.get("type") or tag.name or ""),
+                    "name": str(tag.get("name") or ""),
+                    "placeholder": str(tag.get("placeholder") or ""),
+                }
+            )
+        forms.append(
+            {
+                "action": str(form.get("action") or ""),
+                "method": str(form.get("method") or ""),
+                "text": form.get_text(" ", strip=True)[:500],
+                "inputs": inputs,
+            }
+        )
+    return forms
+
+
+def _dedupe_strings(values: list[str]) -> list[str]:
+    seen = set()
+    result = []
+    for value in values:
+        clean = _normalize_text(value)
+        key = clean.lower()
+        if not clean or key in seen:
+            continue
+        seen.add(key)
+        result.append(clean)
+    return result

ontology_platform/crawler_platform/app/core/crawler/pipeline.py

@@ -3,11 +3,17 @@ from __future__ import annotations
 from dataclasses import dataclass
 
 from crawler_platform.app.config.loader import ProjectConfig
-from crawler_platform.app.core.crawler.page_classifier import classify_page
+from crawler_platform.app.core.crawler.page_classifier import (
+    classification_metadata,
+    classify_page_semantic,
+    get_legacy_page_type,
+    should_analyze_page,
+)
 from crawler_platform.app.core.crawler.fetchers import RobotsPolicy, make_fetcher
 from crawler_platform.app.core.crawler.plugins import ParserRegistry, default_parser_registry
 from crawler_platform.app.core.database.repository import KnowledgeRepository
 from crawler_platform.app.core.extractor.base import ExtractionPageContext, Extractor
+from crawler_platform.app.core.extractor.strategy import extract_with_strategy
 from crawler_platform.app.core.extractor.validation import attach_page_context
 
 
@@ -16,6 +22,13 @@ class CrawlResult:
     page_id: int
     claim_count: int
     entity_count: int
+    extraction_mode: str | None = None
+    effective_extraction_mode: str | None = None
+    llm_skipped: bool = False
+    llm_skip_reason: str | None = None
+    fallback_used: bool = False
+    agreement_claim_count: int = 0
+    conflict_claim_count: int = 0
     crawl_status: str = "success"
     extraction_status: str = "success"
     page_type: str = "UnknownPage"
@@ -49,23 +62,33 @@ class CrawlPipeline:
         fetch_result = fetcher.fetch(url)
         parser = self.parser_registry.get(source_config.parser)
         parsed = parser.parse(fetch_result.analysis_html, fetch_result.final_url or url)
-        page_type = classify_page(
+        page_classification = classify_page_semantic(
             fetch_result.final_url or url,
             parsed.title or fetch_result.title,
             parsed.raw_text or parsed.text,
             fetch_result.analysis_html,
             parsed.source_zones or [],
+            final_url=fetch_result.final_url,
+            status_code=fetch_result.status_code,
+            content_type=fetch_result.headers.get("content-type"),
         )
+        page_type = get_legacy_page_type(page_classification)
 
         project = self.repository.upsert_project(project_config)
         source = self.repository.get_source(project.id, source_name)
         warnings = [*fetch_result.warnings, *(parsed.extraction_warnings or [])]
         metadata = {
             **parsed.metadata,
+            **classification_metadata(
+                page_classification,
+                title=parsed.title or fetch_result.title,
+                text=parsed.raw_text or parsed.text,
+                html=fetch_result.analysis_html,
+                source_zones=parsed.source_zones or [],
+            ),
             "final_url": fetch_result.final_url,
             "crawl_status": fetch_result.crawl_status,
             "crawl_warnings": fetch_result.warnings,
-            "page_type": page_type,
             "raw_text_length": len(parsed.raw_text or ""),
             "clean_text_length": len(parsed.text or ""),
             "main_content_preview": (parsed.main_content or parsed.text)[:800],
@@ -93,6 +116,20 @@ class CrawlPipeline:
                 robots_status=robots_decision.status,
                 robots_reason=robots_decision.reason,
             )
+        if not should_analyze_page(page_classification, None):
+            return CrawlResult(
+                page_id=page.id,
+                claim_count=0,
+                entity_count=0,
+                crawl_status=fetch_result.crawl_status,
+                extraction_status="skipped",
+                page_type=page_type,
+                clean_text_length=len(parsed.text or ""),
+                raw_text_length=len(parsed.raw_text or ""),
+                warnings=warnings,
+                robots_status=robots_decision.status,
+                robots_reason=robots_decision.reason,
+            )
 
         context = ExtractionPageContext(
             url=url,
@@ -109,13 +146,15 @@ class CrawlPipeline:
             warnings=warnings,
             metadata=metadata,
         )
-        bundle = self.extractor.extract_from_context(context, project_config)
+        bundle = extract_with_strategy(self.extractor, context, project_config)
         bundle = attach_page_context(bundle, context)
         claims = self.repository.save_extraction_bundle(project.id, source, page, bundle, project_config)
+        extraction_summary = extraction_summary_from_raw(bundle.raw_output)
         return CrawlResult(
             page_id=page.id,
             claim_count=len(claims),
             entity_count=len(bundle.entities),
+            **extraction_summary,
             crawl_status=fetch_result.crawl_status,
             extraction_status=parsed.extraction_status,
             page_type=page_type,
@@ -125,3 +164,15 @@ class CrawlPipeline:
             robots_status=robots_decision.status,
             robots_reason=robots_decision.reason,
         )
+
+
+def extraction_summary_from_raw(raw_output: dict[str, object]) -> dict[str, object]:
+    return {
+        "extraction_mode": raw_output.get("extraction_mode"),
+        "effective_extraction_mode": raw_output.get("effective_extraction_mode"),
+        "llm_skipped": bool(raw_output.get("llm_skipped")),
+        "llm_skip_reason": raw_output.get("llm_skip_reason"),
+        "fallback_used": bool(raw_output.get("fallback")),
+        "agreement_claim_count": int(raw_output.get("agreement_claim_count") or 0),
+        "conflict_claim_count": int(raw_output.get("conflict_claim_count") or 0),
+    }

ontology_platform/crawler_platform/app/core/crawler/site_crawler.py

@@ -10,12 +10,16 @@ from crawler_platform.app.core.crawler.discovery import discover_links
 from crawler_platform.app.core.crawler.fetchers import RobotsDecision, RobotsPolicy, make_fetcher
 from crawler_platform.app.core.crawler.page_classifier import (
     classify_page as classify_page_type,
+    classification_metadata,
+    classify_page_semantic,
+    get_legacy_page_type,
     should_analyze_page as should_analyze_page_type,
 )
 from crawler_platform.app.core.crawler.plugins import ParserRegistry, default_parser_registry
 from crawler_platform.app.core.database import models
 from crawler_platform.app.core.database.repository import KnowledgeRepository
 from crawler_platform.app.core.extractor.base import ExtractionPageContext, Extractor
+from crawler_platform.app.core.extractor.strategy import extract_with_strategy
 from crawler_platform.app.core.extractor.validation import attach_page_context
 
 
@@ -28,6 +32,13 @@ class SiteCrawlPageResult:
     page_id: int | None = None
     claim_count: int = 0
     entity_count: int = 0
+    extraction_mode: str | None = None
+    effective_extraction_mode: str | None = None
+    llm_skipped: bool = False
+    llm_skip_reason: str | None = None
+    fallback_used: bool = False
+    agreement_claim_count: int = 0
+    conflict_claim_count: int = 0
     discovered_count: int = 0
     crawl_status: str = "success"
     extraction_status: str = "unknown"
@@ -202,6 +213,16 @@ class SiteCrawler:
         fetch_result = fetcher.fetch(url)
         if is_failed_fetch_status(fetch_result.status_code) or fetch_result.crawl_status != "success":
             error = f"fetch failed with status {fetch_result.status_code}; crawl_status={fetch_result.crawl_status}"
+            page_classification = classify_page_semantic(
+                fetch_result.final_url or url,
+                title=fetch_result.title,
+                text="",
+                html=fetch_result.analysis_html,
+                final_url=fetch_result.final_url,
+                status_code=fetch_result.status_code,
+                content_type=fetch_result.headers.get("content-type"),
+            )
+            page_type = get_legacy_page_type(page_classification)
             page = self.repository.upsert_page(
                 project_id=source.project_id,
                 source_id=source.id,
@@ -210,6 +231,11 @@ class SiteCrawler:
                 status_code=fetch_result.status_code,
                 cleaned_text="",
                 metadata={
+                    **classification_metadata(
+                        page_classification,
+                        title=fetch_result.title,
+                        html=fetch_result.analysis_html,
+                    ),
                     "final_url": fetch_result.final_url,
                     "crawl_status": fetch_result.crawl_status,
                     "crawl_warnings": fetch_result.warnings,
@@ -225,7 +251,7 @@ class SiteCrawler:
                     url=url,
                     depth=depth,
                     status=fetch_result.crawl_status,
-                    page_type="unknown",
+                    page_type=page_type,
                     page_id=page.id,
                     crawl_status=fetch_result.crawl_status,
                     robots_status=robots_decision.status,
@@ -239,13 +265,17 @@ class SiteCrawler:
             return
 
         parsed = parser.parse(fetch_result.analysis_html, fetch_result.final_url or url)
-        page_type = classify_page(
+        page_classification = classify_page_semantic(
             fetch_result.final_url or url,
             parsed.title or fetch_result.title,
             parsed.raw_text or parsed.text,
             fetch_result.analysis_html,
             parsed.source_zones or [],
+            final_url=fetch_result.final_url,
+            status_code=fetch_result.status_code,
+            content_type=fetch_result.headers.get("content-type"),
         )
+        page_type = get_legacy_page_type(page_classification)
         discovered_count = self._enqueue_links(
             html=fetch_result.analysis_html,
             base_url=fetch_result.final_url or url,
@@ -261,10 +291,16 @@ class SiteCrawler:
         warnings = [*fetch_result.warnings, *(parsed.extraction_warnings or [])]
         metadata = {
             **parsed.metadata,
+            **classification_metadata(
+                page_classification,
+                title=parsed.title or fetch_result.title,
+                text=parsed.raw_text or parsed.text,
+                html=fetch_result.analysis_html,
+                source_zones=parsed.source_zones or [],
+            ),
             "final_url": fetch_result.final_url,
             "crawl_status": fetch_result.crawl_status,
             "crawl_warnings": fetch_result.warnings,
-            "page_type": page_type,
             "depth": depth,
             "raw_text_length": len(parsed.raw_text or ""),
             "clean_text_length": len(parsed.text or ""),
@@ -312,7 +348,7 @@ class SiteCrawler:
             )
             return
 
-        if should_analyze_page(page_type, analyze_page_types):
+        if should_analyze_page(page_classification, analyze_page_types):
             context = ExtractionPageContext(
                 url=url,
                 final_url=fetch_result.final_url,
@@ -328,9 +364,10 @@ class SiteCrawler:
                 warnings=warnings,
                 metadata=metadata,
             )
-            bundle = self.extractor.extract_from_context(context, project_config)
+            bundle = extract_with_strategy(self.extractor, context, project_config)
             bundle = attach_page_context(bundle, context)
             claims = self.repository.save_extraction_bundle(source.project_id, source, page, bundle, project_config)
+            extraction_summary = extraction_summary_from_raw(bundle.raw_output)
             self._finish_job(job, "completed")
             result.analyzed_count += 1
             page_result = SiteCrawlPageResult(
@@ -338,6 +375,7 @@ class SiteCrawler:
                 status="completed",
                 claim_count=len(claims),
                 entity_count=len(bundle.entities),
+                **extraction_summary,
             )
         else:
             self._finish_job(job, "discovered")
@@ -441,3 +479,15 @@ def classify_page(
     source_zones: list[dict[str, object]] | None = None,
 ) -> str:
     return classify_page_type(url, title, text, html=html, source_zones=source_zones)
+
+
+def extraction_summary_from_raw(raw_output: dict[str, object]) -> dict[str, object]:
+    return {
+        "extraction_mode": raw_output.get("extraction_mode"),
+        "effective_extraction_mode": raw_output.get("effective_extraction_mode"),
+        "llm_skipped": bool(raw_output.get("llm_skipped")),
+        "llm_skip_reason": raw_output.get("llm_skip_reason"),
+        "fallback_used": bool(raw_output.get("fallback")),
+        "agreement_claim_count": int(raw_output.get("agreement_claim_count") or 0),
+        "conflict_claim_count": int(raw_output.get("conflict_claim_count") or 0),
+    }

ontology_platform/crawler_platform/app/core/database/repository.py

@@ -197,6 +197,7 @@ class KnowledgeRepository:
 
         claims: list[models.Claim] = []
         for extracted_claim in bundle.claims:
+            claim_status_for_row = claim_status
             subject = self._entity_for_claim(project_id, extracted_claim.subject_type, extracted_claim.subject_name, entity_index)
             object_entity = None
             if extracted_claim.object_name and extracted_claim.object_type:
@@ -211,6 +212,9 @@ class KnowledgeRepository:
                 **extracted_claim.metadata,
                 "source_trust": source.trust_level,
             }
+            metadata_status = str(claim_metadata.get("validation_status") or "")
+            if metadata_status in {"active", "validated_claim", "candidate_claim", "rule_candidate"}:
+                claim_status_for_row = metadata_status
             claim_metadata["source_history"] = [
                 {
                     "source_id": source.id,
@@ -272,7 +276,7 @@ class KnowledgeRepository:
                     confidence=confidence,
                     confidence_reason=extracted_claim.confidence_reason,
                     extraction_method=bundle.extractor_name,
-                    status=claim_status,
+                    status=claim_status_for_row,
                     metadata_json=claim_metadata,
                 )
                 self.session.add(claim)
@@ -284,8 +288,8 @@ class KnowledgeRepository:
                 claim.last_seen_at = models.utcnow()
                 claim.confidence = max(claim.confidence, confidence)
                 claim.confidence_reason = extracted_claim.confidence_reason or claim.confidence_reason
-                if claim_status == "active" or claim.status != "active":
-                    claim.status = claim_status
+                if claim_status_for_row == "active" or claim.status != "active":
+                    claim.status = claim_status_for_row
                 claim.metadata_json = {**existing_metadata, **claim_metadata}
                 claim.metadata_json["source_history"] = merge_source_history(
                     existing_history,

ontology_platform/crawler_platform/app/core/extractor/ai_provider.py

@@ -31,6 +31,8 @@ class LLMJsonExtractor(AIExtractor):
         model: str | None = None,
         base_url: str | None = None,
         timeout_seconds: int = 300,
+        fallback_to_rules: bool = True,
+        merge_rule_claims: bool = True,
     ):
         # Local LM Studio runs on consumer hardware; keep a shorter timeout so
         # we can quickly fallback instead of stalling a crawl worker for 5+ min.
@@ -41,6 +43,8 @@ class LLMJsonExtractor(AIExtractor):
         self.model = model
         self.base_url = base_url
         self.timeout_seconds = timeout_seconds
+        self.fallback_to_rules = fallback_to_rules
+        self.merge_rule_claims = merge_rule_claims
 
     def extract(self, page_text: str, project_config: ProjectConfig) -> ExtractionBundle:
         return self._extract_text(page_text, project_config, context=None)
@@ -64,7 +68,11 @@ class LLMJsonExtractor(AIExtractor):
             try:
                 raw = self.complete_json(page_text, project_config, compact=compact_mode, context=context)
                 bundle = self._bundle_from_raw(raw, mode_name)
-                enriched = self._merge_rule_fallback_claims(bundle, page_text, project_config, mode_name)
+                enriched = (
+                    self._merge_rule_fallback_claims(bundle, page_text, project_config, mode_name)
+                    if self.merge_rule_claims
+                    else bundle
+                )
                 if enriched.entities and enriched.claims:
                     return self.normalize_to_ontology(enriched, project_config.ontology)
                 if bundle.entities and bundle.claims:
@@ -72,6 +80,8 @@ class LLMJsonExtractor(AIExtractor):
                 errors.append(f"{mode_name}: AI returned no usable entities or claims")
             except Exception as exc:
                 errors.append(f"{mode_name}: {exc}")
+        if not self.fallback_to_rules:
+            raise RuntimeError(" | ".join(errors))
         return self._fallback_bundle(page_text, project_config, " | ".join(errors))
 
     def extract_entities(self, page_text: str, project_config: ProjectConfig) -> list[ExtractedEntity]:
@@ -627,6 +637,38 @@ def list_openai_compatible_models(base_url: str, api_key: str | None = None) ->
     return [{"id": item.get("id", ""), "owned_by": item.get("owned_by")} for item in items if item.get("id")]
 
 
+def list_lmstudio_loaded_models(base_url: str) -> list[dict[str, Any]]:
+    """Return only the models currently loaded in LM Studio.
+
+    Uses LM Studio's native REST API (``/api/v0/models``) which exposes a
+    ``state`` field. Falls back to the OpenAI-compatible ``/v1/models`` list
+    (treated as all-loaded) if the native endpoint is unavailable.
+    """
+    clean = base_url.rstrip("/")
+    if clean.endswith("/v1"):
+        root = clean[: -len("/v1")]
+    elif clean.endswith("/v1/chat/completions"):
+        root = clean[: -len("/v1/chat/completions")]
+    else:
+        root = clean
+    native_url = f"{root}/api/v0/models"
+    try:
+        response = requests.get(native_url, timeout=5)
+        response.raise_for_status()
+        data = response.json()
+        items = data.get("data", []) if isinstance(data, dict) else []
+        loaded = [
+            {"id": item.get("id", ""), "owned_by": item.get("owned_by"), "state": item.get("state")}
+            for item in items
+            if item.get("id") and str(item.get("state", "")).lower() == "loaded"
+        ]
+        if loaded:
+            return loaded
+    except Exception:
+        pass
+    return list_openai_compatible_models(base_url)
+
+
 def parse_entities(items: list[dict[str, Any]]) -> list[ExtractedEntity]:
     entities: list[ExtractedEntity] = []
     for item in items:

ontology_platform/crawler_platform/app/core/extractor/base.py

@@ -61,6 +61,7 @@ class ExtractionPageContext:
         clean_text = self.clean_text
         if text_limit is not None and len(clean_text) > text_limit:
             clean_text = clean_text[:text_limit]
+        semantic_metadata = self.semantic_metadata_payload()
         zones = []
         for zone in self.source_zones:
             zone_text = str(zone.get("text") or "")
@@ -85,8 +86,34 @@ class ExtractionPageContext:
             "clean_text": clean_text,
             "source_zones": zones,
             "warnings": self.warnings,
+            "metadata": semantic_metadata,
         }
 
+    def semantic_metadata_payload(self) -> dict[str, Any]:
+        payload = {
+            key: self.metadata[key]
+            for key in ("semantic_page_type", "analyze_strategy", "llm_policy")
+            if key in self.metadata
+        }
+        classification = self.metadata.get("page_classification")
+        if isinstance(classification, dict):
+            payload["page_classification"] = {
+                key: classification.get(key)
+                for key in (
+                    "primary_page_type",
+                    "secondary_page_types",
+                    "confidence",
+                    "alternatives",
+                    "legacy_page_type",
+                    "unknown_pattern",
+                )
+                if key in classification
+            }
+            evidence = classification.get("evidence")
+            if isinstance(evidence, list):
+                payload["page_classification"]["evidence"] = evidence[:8]
+        return payload
+
 
 class Extractor(ABC):
     name = "base"

ontology_platform/crawler_platform/app/core/extractor/factory.py

@@ -2,18 +2,71 @@ from __future__ import annotations
 
 from crawler_platform.app.core.extractor.ai_provider import LLMJsonExtractor
 from crawler_platform.app.core.extractor.base import Extractor
+from crawler_platform.app.core.extractor.hybrid import HybridExtractor
 from crawler_platform.app.core.extractor.rule_based import GenericRuleBasedExtractor
 from crawler_platform.app.domains.perfume.extractor import PerfumeRuleBasedExtractor
 
 
+LLM_PROVIDERS = {"openai", "ollama", "lm_studio"}
+
+
 def extractor_for_domain(
     domain: str,
     provider: str = "rule_based",
     model: str | None = None,
     base_url: str | None = None,
+    extraction_mode: str | None = None,
+    fallback_to_rules: bool = True,
 ) -> Extractor:
-    if provider in {"openai", "ollama", "lm_studio"}:
-        return LLMJsonExtractor(domain=domain, provider=provider, model=model, base_url=base_url)
+    mode = normalize_extraction_mode(provider, extraction_mode)
+    llm_provider = normalize_llm_provider(provider)
+    if mode in {"hybrid", "compare"}:
+        return HybridExtractor(
+            domain=domain,
+            llm_provider=llm_provider,
+            model=model,
+            base_url=base_url,
+            mode=mode,
+            fallback_to_rules=fallback_to_rules,
+        )
+    if mode == "llm_only":
+        return LLMJsonExtractor(
+            domain=domain,
+            provider=llm_provider,
+            model=model,
+            base_url=base_url,
+            fallback_to_rules=fallback_to_rules,
+            merge_rule_claims=False,
+        )
+    return rule_extractor_for_domain(domain)
+
+
+def normalize_extraction_mode(provider: str, extraction_mode: str | None = None) -> str:
+    if extraction_mode:
+        value = extraction_mode.strip().lower().replace("-", "_")
+    else:
+        value = provider.strip().lower().replace("-", "_")
+    aliases = {
+        "rule": "rule_only",
+        "rules": "rule_only",
+        "rule_based": "rule_only",
+        "llm": "llm_only",
+        "ai": "llm_only",
+    }
+    value = aliases.get(value, value)
+    if value in {"rule_only", "llm_only", "hybrid", "compare"}:
+        return value
+    if value in LLM_PROVIDERS:
+        return "hybrid"
+    return "rule_only"
+
+
+def normalize_llm_provider(provider: str) -> str:
+    value = provider.strip().lower()
+    return value if value in LLM_PROVIDERS else "lm_studio"
+
+
+def rule_extractor_for_domain(domain: str) -> Extractor:
     if domain == "perfume":
         return PerfumeRuleBasedExtractor()
     return GenericRuleBasedExtractor()

ontology_platform/crawler_platform/app/core/extractor/hybrid.py

@@ -0,0 +1,571 @@
+from __future__ import annotations
+
+import json
+from dataclasses import replace
+from typing import Any
+
+from crawler_platform.app.config.loader import ProjectConfig
+from crawler_platform.app.core.crawler.page_type_taxonomy import LLMPolicy
+from crawler_platform.app.core.extractor.ai_provider import LLMJsonExtractor, dedupe_entities
+from crawler_platform.app.core.extractor.base import (
+    ExtractedClaim,
+    ExtractedEntity,
+    ExtractionBundle,
+    ExtractionPageContext,
+    Extractor,
+)
+from crawler_platform.app.core.extractor.rule_based import GenericRuleBasedExtractor
+from crawler_platform.app.domains.perfume.extractor import PerfumeRuleBasedExtractor
+
+
+LLM_PAGE_TYPES = {
+    "ProductPage",
+    "ProductDetailPage",
+    "BrandStoryPage",
+    "AboutPage",
+    "ContactPage",
+    "ReviewPage",
+    "NoticePage",
+    "PublicNoticePage",
+    "PromotionPage",
+    "CampaignLandingPage",
+    "ArticlePage",
+    "BlogPostPage",
+    "DocumentationPage",
+    "APIReferencePage",
+    "DatasetPage",
+    "ResearchPaperPage",
+    "JobPostingPage",
+    "CourseDetailPage",
+    "LocalBusinessPage",
+    "RealEstateListingPage",
+    "ProfilePage",
+}
+SKIP_LLM_PAGE_TYPES = {"CategoryPage", "CategoryListingPage", "SearchPage", "SearchResultsPage", "ListingPage"}
+RULE_ONLY_PAGE_TYPES = {
+    "BoardPage",
+    "ForumBoardPage",
+    "ForumThreadPage",
+    "CommunityPage",
+    "FAQPage",
+    "QAPage",
+    "VideoPage",
+    "UnknownPage",
+}
+MIN_CLEAN_TEXT_CHARS_FOR_LLM = 300
+MAX_CLEAN_TEXT_CHARS_FOR_LLM = 60000
+
+
+class HybridExtractor(Extractor):
+    """Run rule extraction first, then LLM extraction, then merge with agreement metadata."""
+
+    name = "hybrid_rule_llm_extractor"
+    provider = "hybrid"
+
+    def __init__(
+        self,
+        domain: str,
+        llm_provider: str = "lm_studio",
+        model: str | None = None,
+        base_url: str | None = None,
+        mode: str = "hybrid",
+        fallback_to_rules: bool = True,
+    ):
+        self.domain = domain
+        self.llm_provider = llm_provider
+        self.model = model
+        self.base_url = base_url
+        self.mode = normalize_mode(mode)
+        self.fallback_to_rules = fallback_to_rules
+        self.rule_extractor = rule_extractor_for_domain(domain)
+        self.llm_extractor = LLMJsonExtractor(
+            domain=domain,
+            provider=llm_provider,
+            model=model,
+            base_url=base_url,
+            fallback_to_rules=False,
+            merge_rule_claims=False,
+        )
+
+    def extract(self, page_text: str, project_config: ProjectConfig) -> ExtractionBundle:
+        return self._extract_text(page_text, project_config, context=None)
+
+    def extract_from_context(
+        self,
+        context: ExtractionPageContext,
+        project_config: ProjectConfig,
+    ) -> ExtractionBundle:
+        return self._extract_text(context.clean_text, project_config, context=context)
+
+    def _extract_text(
+        self,
+        page_text: str,
+        project_config: ProjectConfig,
+        context: ExtractionPageContext | None,
+    ) -> ExtractionBundle:
+        rule_bundle = self._rule_bundle(page_text, project_config, context)
+        mark_bundle(rule_bundle, source="rule", mode=self.mode)
+
+        if self.mode == "rule_only":
+            rule_bundle.extractor_name = "rule_only_extractor"
+            rule_bundle.raw_output = {
+                **rule_bundle.raw_output,
+                "extraction_mode": self.mode,
+                "effective_extraction_mode": "rule_only",
+                "llm_skipped": True,
+                "llm_skip_reason": "rule_only mode",
+                **count_payload(
+                    rule_entity_count=len(rule_bundle.entities),
+                    rule_claim_count=len(rule_bundle.claims),
+                    llm_entity_count=0,
+                    llm_claim_count=0,
+                    comparison=comparison_payload(rule_only=len(rule_bundle.claims)),
+                ),
+            }
+            return rule_bundle
+
+        skip_reason = llm_skip_reason(context, page_text, self.mode)
+        if skip_reason:
+            return llm_skipped_bundle(rule_bundle, self, skip_reason, context)
+
+        try:
+            llm_bundle = self._llm_bundle(page_text, project_config, context)
+            mark_bundle(llm_bundle, source="llm", mode=self.mode)
+        except Exception as exc:
+            if not self.fallback_to_rules:
+                raise
+            return fallback_bundle(rule_bundle, self, exc)
+
+        if self.mode == "llm_only":
+            llm_bundle.extractor_name = "llm_only_extractor"
+            llm_bundle.raw_output = {
+                **llm_bundle.raw_output,
+                "extraction_mode": self.mode,
+                "effective_extraction_mode": "llm_only",
+                **count_payload(
+                    rule_entity_count=len(rule_bundle.entities),
+                    rule_claim_count=len(rule_bundle.claims),
+                    llm_entity_count=len(llm_bundle.entities),
+                    llm_claim_count=len(llm_bundle.claims),
+                    comparison=comparison_payload(llm_only=len(llm_bundle.claims)),
+                ),
+            }
+            return llm_bundle
+
+        return merge_bundles(rule_bundle, llm_bundle, self)
+
+    def _rule_bundle(
+        self,
+        page_text: str,
+        project_config: ProjectConfig,
+        context: ExtractionPageContext | None,
+    ) -> ExtractionBundle:
+        if context is not None:
+            return self.rule_extractor.extract_from_context(context, project_config)
+        return self.rule_extractor.extract(page_text, project_config)
+
+    def _llm_bundle(
+        self,
+        page_text: str,
+        project_config: ProjectConfig,
+        context: ExtractionPageContext | None,
+    ) -> ExtractionBundle:
+        if context is not None:
+            return self.llm_extractor.extract_from_context(context, project_config)
+        return self.llm_extractor.extract(page_text, project_config)
+
+    def extract_entities(self, page_text: str, project_config: ProjectConfig) -> list[ExtractedEntity]:
+        return self.extract(page_text, project_config).entities
+
+    def extract_attributes(
+        self,
+        entity: ExtractedEntity,
+        page_text: str,
+        project_config: ProjectConfig,
+    ) -> dict[str, Any]:
+        return entity.attributes
+
+    def extract_relations(
+        self,
+        entities: list[ExtractedEntity],
+        page_text: str,
+        project_config: ProjectConfig,
+    ) -> list[ExtractedClaim]:
+        return []
+
+    def normalize_to_ontology(self, bundle: ExtractionBundle, ontology: dict[str, Any]) -> ExtractionBundle:
+        return bundle
+
+
+def normalize_mode(mode: str | None) -> str:
+    value = (mode or "hybrid").strip().lower().replace("-", "_")
+    aliases = {
+        "rule": "rule_only",
+        "rules": "rule_only",
+        "rule_based": "rule_only",
+        "llm": "llm_only",
+        "ai": "llm_only",
+    }
+    return aliases.get(value, value)
+
+
+def rule_extractor_for_domain(domain: str) -> Extractor:
+    if domain == "perfume":
+        return PerfumeRuleBasedExtractor()
+    return GenericRuleBasedExtractor()
+
+
+def mark_bundle(bundle: ExtractionBundle, *, source: str, mode: str) -> None:
+    for entity in bundle.entities:
+        entity.metadata = {
+            **entity.metadata,
+            "extraction_source": source,
+            f"{source}_confidence": entity.confidence,
+        }
+    for claim in bundle.claims:
+        claim.metadata = {
+            **claim.metadata,
+            "extraction_source": source,
+            "agreement": f"{source}_only",
+            f"{source}_confidence": claim.confidence,
+            "extraction_mode": mode,
+        }
+
+
+def fallback_bundle(rule_bundle: ExtractionBundle, extractor: HybridExtractor, exc: Exception) -> ExtractionBundle:
+    reason = str(exc)
+    bundle = clone_bundle(rule_bundle)
+    bundle.extractor_name = "hybrid_rule_fallback"
+    bundle.provider = extractor.llm_provider
+    bundle.raw_output = {
+        **bundle.raw_output,
+        "extraction_mode": "fallback",
+        "effective_extraction_mode": "rule_only",
+        "requested_extraction_mode": extractor.mode,
+        "ai_provider": extractor.llm_provider,
+        "ai_model": extractor.model,
+        "ai_warning": reason,
+        "fallback": "rule_based",
+        **count_payload(
+            rule_entity_count=len(bundle.entities),
+            rule_claim_count=len(bundle.claims),
+            llm_entity_count=0,
+            llm_claim_count=0,
+            comparison=comparison_payload(rule_only=len(bundle.claims)),
+        ),
+    }
+    for entity in bundle.entities:
+        entity.metadata["ai_fallback_reason"] = reason
+    for claim in bundle.claims:
+        claim.metadata = {
+            **claim.metadata,
+            "ai_fallback_reason": reason,
+            "fallback": "rule_based",
+            "agreement": "rule_only",
+        }
+        claim.confidence_reason = (
+            f"{claim.confidence_reason}; AI fallback: {reason}"
+            if claim.confidence_reason
+            else f"AI fallback: {reason}"
+        )
+    return bundle
+
+
+def llm_skipped_bundle(
+    rule_bundle: ExtractionBundle,
+    extractor: HybridExtractor,
+    reason: str,
+    context: ExtractionPageContext | None,
+) -> ExtractionBundle:
+    bundle = clone_bundle(rule_bundle)
+    bundle.extractor_name = "hybrid_rule_only_routed"
+    bundle.provider = extractor.llm_provider
+    bundle.raw_output = {
+        **bundle.raw_output,
+        "extraction_mode": extractor.mode,
+        "effective_extraction_mode": "rule_only",
+        "requested_extraction_mode": extractor.mode,
+        "ai_provider": extractor.llm_provider,
+        "ai_model": extractor.model,
+        "llm_skipped": True,
+        "llm_skip_reason": reason,
+        "page_type": context.page_type if context is not None else None,
+        "rule_entity_count": len(bundle.entities),
+        "rule_claim_count": len(bundle.claims),
+        "llm_entity_count": 0,
+        "llm_claim_count": 0,
+        "agreement_claim_count": 0,
+        "rule_only_claim_count": len(bundle.claims),
+        "llm_only_claim_count": 0,
+        "conflict_claim_count": 0,
+        "comparison": {
+            "both_agree": 0,
+            "rule_only": len(bundle.claims),
+            "llm_only": 0,
+            "conflict": 0,
+            "rejected_by_validation": 0,
+        },
+    }
+    for claim in bundle.claims:
+        claim.metadata = {
+            **claim.metadata,
+            "llm_skipped": True,
+            "llm_skip_reason": reason,
+            "agreement": "rule_only",
+        }
+    return bundle
+
+
+def comparison_payload(
+    *,
+    both_agree: int = 0,
+    rule_only: int = 0,
+    llm_only: int = 0,
+    conflict: int = 0,
+    rejected_by_validation: int = 0,
+) -> dict[str, int]:
+    return {
+        "both_agree": both_agree,
+        "rule_only": rule_only,
+        "llm_only": llm_only,
+        "conflict": conflict,
+        "rejected_by_validation": rejected_by_validation,
+    }
+
+
+def count_payload(
+    *,
+    rule_entity_count: int,
+    rule_claim_count: int,
+    llm_entity_count: int,
+    llm_claim_count: int,
+    comparison: dict[str, int],
+) -> dict[str, Any]:
+    return {
+        "rule_entity_count": rule_entity_count,
+        "rule_claim_count": rule_claim_count,
+        "llm_entity_count": llm_entity_count,
+        "llm_claim_count": llm_claim_count,
+        "agreement_claim_count": comparison["both_agree"],
+        "rule_only_claim_count": comparison["rule_only"],
+        "llm_only_claim_count": comparison["llm_only"],
+        "conflict_claim_count": comparison["conflict"],
+        "comparison": comparison,
+    }
+
+
+def llm_skip_reason(
+    context: ExtractionPageContext | None,
+    page_text: str,
+    mode: str,
+) -> str | None:
+    if mode != "hybrid" or context is None:
+        return None
+    page_type = str(context.page_type or "UnknownPage")
+    clean_length = len(context.clean_text or page_text or "")
+    policy = llm_policy_from_context(context)
+    if policy:
+        if policy == LLMPolicy.SKIP.value:
+            return "LLM policy Skip prevents LLM extraction"
+        if policy == LLMPolicy.NO_LLM.value:
+            return "LLM policy NoLLM prevents LLM extraction"
+        if policy == LLMPolicy.RULE_ONLY.value:
+            return "LLM policy RuleOnly routes page to rule-only extraction"
+        if policy == LLMPolicy.LLM_FOR_AMBIGUITY_ONLY.value and not classification_is_ambiguous(context):
+            return "LLM policy LLMForAmbiguityOnly requires ambiguous classification evidence"
+        if policy in {LLMPolicy.LLM_FULL.value, LLMPolicy.LLM_LIGHT.value, LLMPolicy.LLM_FOR_AMBIGUITY_ONLY.value}:
+            if clean_length < MIN_CLEAN_TEXT_CHARS_FOR_LLM:
+                return f"clean text is too short for LLM extraction: {clean_length} chars"
+            if clean_length > MAX_CLEAN_TEXT_CHARS_FOR_LLM:
+                return f"clean text is too long for LLM extraction: {clean_length} chars"
+            return None
+    if page_type in SKIP_LLM_PAGE_TYPES:
+        return f"page type {page_type} is configured to skip LLM extraction"
+    if page_type in RULE_ONLY_PAGE_TYPES:
+        return f"page type {page_type} is routed to rule-only extraction"
+    if page_type not in LLM_PAGE_TYPES:
+        return f"page type {page_type} is not in the LLM page-type allowlist"
+    if clean_length < MIN_CLEAN_TEXT_CHARS_FOR_LLM:
+        return f"clean text is too short for LLM extraction: {clean_length} chars"
+    if clean_length > MAX_CLEAN_TEXT_CHARS_FOR_LLM:
+        return f"clean text is too long for LLM extraction: {clean_length} chars"
+    return None
+
+
+def llm_policy_from_context(context: ExtractionPageContext) -> str | None:
+    raw_policy = context.metadata.get("llm_policy")
+    if not raw_policy:
+        classification = context.metadata.get("page_classification")
+        if isinstance(classification, dict):
+            raw_policy = classification.get("llm_policy")
+    policy = str(raw_policy or "").strip()
+    return policy or None
+
+
+def classification_is_ambiguous(context: ExtractionPageContext) -> bool:
+    classification = context.metadata.get("page_classification")
+    if not isinstance(classification, dict):
+        return False
+    confidence = float(classification.get("confidence") or 0.0)
+    if 0.0 < confidence < 0.55:
+        return True
+    alternatives = classification.get("alternatives") or []
+    if not isinstance(alternatives, list) or len(alternatives) < 2:
+        return False
+    try:
+        top_score = float(alternatives[0][1])
+        next_score = float(alternatives[1][1])
+    except (TypeError, ValueError, IndexError):
+        return False
+    return abs(top_score - next_score) < 0.08
+
+
+def merge_bundles(
+    rule_bundle: ExtractionBundle,
+    llm_bundle: ExtractionBundle,
+    extractor: HybridExtractor,
+) -> ExtractionBundle:
+    merged_claims: dict[tuple[str, str, str], ExtractedClaim] = {}
+    rule_by_subject_predicate: dict[tuple[str, str], list[ExtractedClaim]] = {}
+    comparison = {
+        "both_agree": 0,
+        "rule_only": 0,
+        "llm_only": 0,
+        "conflict": 0,
+        "rejected_by_validation": 0,
+    }
+
+    for claim in clone_claims(rule_bundle.claims):
+        key = claim_key(claim)
+        merged_claims[key] = claim
+        rule_by_subject_predicate.setdefault(subject_predicate_key(claim), []).append(claim)
+
+    for llm_claim in clone_claims(llm_bundle.claims):
+        key = claim_key(llm_claim)
+        existing = merged_claims.get(key)
+        if existing is not None:
+            comparison["both_agree"] += 1
+            merge_agreement(existing, llm_claim)
+            continue
+
+        possible_conflicts = rule_by_subject_predicate.get(subject_predicate_key(llm_claim), [])
+        if possible_conflicts:
+            comparison["conflict"] += 1
+            mark_conflict(llm_claim, possible_conflicts)
+            for rule_claim in possible_conflicts:
+                rule_claim.metadata = {
+                    **rule_claim.metadata,
+                    "agreement": "conflict",
+                    "conflict_status": "rule_llm_conflict",
+                    "review_required": True,
+                    "review_reason": "Rule and LLM produced different objects for the same subject and predicate",
+                }
+        else:
+            comparison["llm_only"] += 1
+            llm_claim.metadata = {
+                **llm_claim.metadata,
+                "agreement": "llm_only",
+                "review_reason": "LLM-only claim; evidence and ontology validation required",
+            }
+        merged_claims[key] = llm_claim
+
+    for claim in merged_claims.values():
+        if claim.metadata.get("agreement") == "rule_only":
+            comparison["rule_only"] += 1
+
+    mode = extractor.mode
+    return ExtractionBundle(
+        entities=dedupe_entities([*clone_entities(rule_bundle.entities), *clone_entities(llm_bundle.entities)]),
+        claims=list(merged_claims.values()),
+        extractor_name="hybrid_rule_llm_extractor" if mode == "hybrid" else "compare_rule_llm_extractor",
+        provider=extractor.llm_provider,
+        raw_output={
+            "extraction_mode": mode,
+            "provider": extractor.llm_provider,
+            "model": extractor.model,
+            "rule_entity_count": len(rule_bundle.entities),
+            "rule_claim_count": len(rule_bundle.claims),
+            "llm_entity_count": len(llm_bundle.entities),
+            "llm_claim_count": len(llm_bundle.claims),
+            "agreement_claim_count": comparison["both_agree"],
+            "rule_only_claim_count": comparison["rule_only"],
+            "llm_only_claim_count": comparison["llm_only"],
+            "conflict_claim_count": comparison["conflict"],
+            "comparison": comparison,
+            "rule_raw_output": safe_raw_output(rule_bundle.raw_output),
+            "llm_raw_output": safe_raw_output(llm_bundle.raw_output),
+        },
+    )
+
+
+def merge_agreement(rule_claim: ExtractedClaim, llm_claim: ExtractedClaim) -> None:
+    rule_confidence = rule_claim.metadata.get("rule_confidence", rule_claim.confidence)
+    llm_confidence = llm_claim.metadata.get("llm_confidence", llm_claim.confidence)
+    rule_claim.confidence = max(rule_claim.confidence, llm_claim.confidence)
+    rule_claim.evidence_text = llm_claim.evidence_text or rule_claim.evidence_text
+    rule_claim.evidence_summary = llm_claim.evidence_summary or rule_claim.evidence_summary
+    rule_claim.metadata = {
+        **rule_claim.metadata,
+        **llm_claim.metadata,
+        "agreement": "rule_and_llm",
+        "rule_confidence": rule_confidence,
+        "llm_confidence": llm_confidence,
+        "rule_claim_merge": True,
+    }
+    reasons = [rule_claim.confidence_reason, llm_claim.confidence_reason, "Rule and LLM agreed"]
+    rule_claim.confidence_reason = "; ".join(reason for reason in reasons if reason)
+
+
+def mark_conflict(llm_claim: ExtractedClaim, rule_claims: list[ExtractedClaim]) -> None:
+    llm_claim.metadata = {
+        **llm_claim.metadata,
+        "agreement": "conflict",
+        "conflict_status": "rule_llm_conflict",
+        "review_required": True,
+        "review_reason": "Rule and LLM produced different objects for the same subject and predicate",
+        "conflicting_rule_objects": [claim_object_key(claim) for claim in rule_claims],
+    }
+
+
+def clone_bundle(bundle: ExtractionBundle) -> ExtractionBundle:
+    return ExtractionBundle(
+        entities=clone_entities(bundle.entities),
+        claims=clone_claims(bundle.claims),
+        extractor_name=bundle.extractor_name,
+        provider=bundle.provider,
+        raw_output=dict(bundle.raw_output),
+    )
+
+
+def clone_entities(entities: list[ExtractedEntity]) -> list[ExtractedEntity]:
+    return [replace(entity, attributes=dict(entity.attributes), metadata=dict(entity.metadata)) for entity in entities]
+
+
+def clone_claims(claims: list[ExtractedClaim]) -> list[ExtractedClaim]:
+    return [replace(claim, metadata=dict(claim.metadata)) for claim in claims]
+
+
+def subject_predicate_key(claim: ExtractedClaim) -> tuple[str, str]:
+    return (
+        claim.subject_name.strip().lower(),
+        claim.predicate.strip().lower(),
+    )
+
+
+def claim_key(claim: ExtractedClaim) -> tuple[str, str, str]:
+    subject, predicate = subject_predicate_key(claim)
+    return subject, predicate, claim_object_key(claim)
+
+
+def claim_object_key(claim: ExtractedClaim) -> str:
+    if claim.object_name:
+        return claim.object_name.strip().lower()
+    return json.dumps(claim.object_value, ensure_ascii=False, sort_keys=True, default=str).strip().lower()
+
+
+def safe_raw_output(raw_output: dict[str, Any]) -> dict[str, Any]:
+    return {
+        key: value
+        for key, value in raw_output.items()
+        if key not in {"candidate_entities", "candidate_claims"}
+    }

ontology_platform/crawler_platform/app/core/extractor/strategy.py

@@ -0,0 +1,190 @@
+from __future__ import annotations
+
+from collections import Counter
+from typing import Any
+
+from crawler_platform.app.config.loader import ProjectConfig
+from crawler_platform.app.core.crawler.page_analysis_policy import decide_analyze_strategy, decide_llm_policy
+from crawler_platform.app.core.crawler.page_type_taxonomy import AnalyzeStrategy, LLMPolicy
+from crawler_platform.app.core.extractor.base import ExtractionBundle, ExtractionPageContext, Extractor
+
+
+def extract_with_strategy(
+    extractor: Extractor,
+    context: ExtractionPageContext,
+    project_config: ProjectConfig,
+) -> ExtractionBundle:
+    strategy = analysis_strategy_from_context(context)
+    policy = llm_policy_from_context(context)
+    if strategy == AnalyzeStrategy.SKIP_PROTECTED.value:
+        return strategy_only_bundle(context, strategy, policy, "protected page")
+    if strategy == AnalyzeStrategy.SKIP_NOISE.value:
+        return strategy_only_bundle(context, strategy, policy, "noise page")
+    if strategy == AnalyzeStrategy.ANALYZE_METADATA_ONLY.value:
+        return metadata_only_bundle(context, strategy, policy)
+    if strategy == AnalyzeStrategy.ANALYZE_DISCOVERY_ONLY.value:
+        return discovery_only_bundle(context, strategy, policy)
+    if strategy == AnalyzeStrategy.ANALYZE_RELATIONS_ONLY.value:
+        return rule_or_structure_bundle(extractor, context, project_config, strategy, policy)
+    if strategy == AnalyzeStrategy.ANALYZE_ENTITY_ONLY.value:
+        bundle = extractor.extract_from_context(context, project_config)
+        bundle.claims = []
+        bundle.raw_output = {
+            **bundle.raw_output,
+            **strategy_payload(context, strategy, policy),
+            "effective_extraction_mode": "entity_only",
+        }
+        return bundle
+    bundle = extractor.extract_from_context(context, project_config)
+    bundle.raw_output = {
+        **bundle.raw_output,
+        **strategy_payload(context, strategy, policy),
+    }
+    return bundle
+
+
+def analysis_strategy_from_context(context: ExtractionPageContext) -> str:
+    raw_strategy = context.metadata.get("analyze_strategy")
+    if raw_strategy:
+        return str(raw_strategy)
+    classification = context.metadata.get("page_classification")
+    if isinstance(classification, dict) and classification.get("analyze_strategy"):
+        return str(classification["analyze_strategy"])
+    return decide_analyze_strategy(context.page_type)
+
+
+def llm_policy_from_context(context: ExtractionPageContext) -> str:
+    raw_policy = context.metadata.get("llm_policy")
+    if raw_policy:
+        return str(raw_policy)
+    classification = context.metadata.get("page_classification")
+    if isinstance(classification, dict) and classification.get("llm_policy"):
+        return str(classification["llm_policy"])
+    return decide_llm_policy(context.page_type)
+
+
+def rule_or_structure_bundle(
+    extractor: Extractor,
+    context: ExtractionPageContext,
+    project_config: ProjectConfig,
+    strategy: str,
+    policy: str,
+) -> ExtractionBundle:
+    if policy == LLMPolicy.LLM_FOR_AMBIGUITY_ONLY.value and classification_is_ambiguous(context):
+        bundle = extractor.extract_from_context(context, project_config)
+        bundle.raw_output = {
+            **bundle.raw_output,
+            **strategy_payload(context, strategy, policy),
+            "effective_extraction_mode": "ambiguity_limited",
+        }
+        return bundle
+    rule_extractor = getattr(extractor, "rule_extractor", None)
+    if rule_extractor is not None:
+        bundle = rule_extractor.extract_from_context(context, project_config)
+        bundle.extractor_name = f"{bundle.extractor_name}_strategy_routed"
+        bundle.raw_output = {
+            **bundle.raw_output,
+            **strategy_payload(context, strategy, policy),
+            "effective_extraction_mode": "rule_only",
+            "llm_skipped": True,
+            "llm_skip_reason": f"analyze strategy {strategy} uses rule/structure extraction only",
+        }
+        return bundle
+    if getattr(extractor, "provider", "") == "ai":
+        return discovery_only_bundle(context, strategy, policy)
+    bundle = extractor.extract_from_context(context, project_config)
+    bundle.raw_output = {
+        **bundle.raw_output,
+        **strategy_payload(context, strategy, policy),
+        "effective_extraction_mode": "rule_or_structure",
+        "llm_skipped": policy in {LLMPolicy.RULE_ONLY.value, LLMPolicy.NO_LLM.value, LLMPolicy.SKIP.value},
+    }
+    return bundle
+
+
+def classification_is_ambiguous(context: ExtractionPageContext) -> bool:
+    classification = context.metadata.get("page_classification")
+    if not isinstance(classification, dict):
+        return False
+    confidence = float(classification.get("confidence") or 0.0)
+    if 0.0 < confidence < 0.55:
+        return True
+    alternatives = classification.get("alternatives") or []
+    if not isinstance(alternatives, list) or len(alternatives) < 2:
+        return False
+    try:
+        top_score = float(alternatives[0][1])
+        next_score = float(alternatives[1][1])
+    except (TypeError, ValueError, IndexError):
+        return False
+    return abs(top_score - next_score) < 0.08
+
+
+def metadata_only_bundle(context: ExtractionPageContext, strategy: str, policy: str) -> ExtractionBundle:
+    return ExtractionBundle(
+        extractor_name="metadata_only_extractor",
+        provider="strategy",
+        raw_output={
+            **strategy_payload(context, strategy, policy),
+            "effective_extraction_mode": "metadata_only",
+            "llm_skipped": True,
+            "llm_skip_reason": f"analyze strategy {strategy} does not require LLM extraction",
+        },
+    )
+
+
+def discovery_only_bundle(context: ExtractionPageContext, strategy: str, policy: str) -> ExtractionBundle:
+    return ExtractionBundle(
+        extractor_name="discovery_only_extractor",
+        provider="strategy",
+        raw_output={
+            **strategy_payload(context, strategy, policy),
+            "effective_extraction_mode": "discovery_only",
+            "llm_skipped": True,
+            "llm_skip_reason": f"analyze strategy {strategy} uses discovery/structure signals only",
+        },
+    )
+
+
+def strategy_only_bundle(context: ExtractionPageContext, strategy: str, policy: str, reason: str) -> ExtractionBundle:
+    return ExtractionBundle(
+        extractor_name="strategy_skipped_extractor",
+        provider="strategy",
+        raw_output={
+            **strategy_payload(context, strategy, policy),
+            "effective_extraction_mode": "skipped",
+            "llm_skipped": True,
+            "llm_skip_reason": reason,
+            "skip_reason": reason,
+        },
+    )
+
+
+def strategy_payload(context: ExtractionPageContext, strategy: str, policy: str) -> dict[str, Any]:
+    return {
+        "analyze_strategy": strategy,
+        "llm_policy": policy,
+        "page_type": context.page_type,
+        "semantic_page_type": context.metadata.get("semantic_page_type"),
+        "strategy_summary": structure_summary(context),
+    }
+
+
+def structure_summary(context: ExtractionPageContext) -> dict[str, Any]:
+    zone_types = Counter(str(zone.get("zone_type") or "unknown") for zone in context.source_zones)
+    classification = context.metadata.get("page_classification") if isinstance(context.metadata, dict) else None
+    return {
+        "title": context.title,
+        "url": context.final_url or context.url,
+        "clean_text_length": len(context.clean_text or ""),
+        "raw_text_length": len(context.raw_text or ""),
+        "source_zone_count": len(context.source_zones or []),
+        "source_zone_types": dict(zone_types),
+        "classification": {
+            "primary_page_type": classification.get("primary_page_type"),
+            "confidence": classification.get("confidence"),
+            "alternatives": classification.get("alternatives", [])[:5],
+        }
+        if isinstance(classification, dict)
+        else {},
+    }

ontology_platform/crawler_platform/app/core/extractor/validation.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 from dataclasses import dataclass, field
+import re
 from typing import Any
 
 from crawler_platform.app.config.loader import ProjectConfig
@@ -134,6 +135,9 @@ def validate_extraction_bundle(bundle: ExtractionBundle, config: ProjectConfig)
     claims = []
     rejected_claims = []
     for claim in bundle.claims:
+        agreement = str(claim.metadata.get("agreement") or "")
+        extraction_source = str(claim.metadata.get("extraction_source") or "")
+        claim_status_for_row = status_for_claim(claim_status, agreement)
         claim.predicate = normalize_predicate(claim.predicate, config.ontology)
         claim.subject_type = normalize_entity_type(claim.subject_type, config.domain)
         claim.subject_name = normalize_entity_name(claim.subject_name, claim.subject_type, config.domain)
@@ -147,36 +151,52 @@ def validate_extraction_bundle(bundle: ExtractionBundle, config: ProjectConfig)
         }
         reason = invalid_claim_reason(claim, config, allowed_types, page_context)
         if reason:
-            rejected_claims.append(
-                {
-                    "subject": claim.subject_name,
-                    "predicate": claim.predicate,
-                    "object": claim.object_name if claim.object_name is not None else claim.object_value,
-                    "reason": reason,
-                    "page_type": claim.metadata.get("page_type") or page_context.get("page_type"),
-                    "source_zone": claim.metadata.get("source_zone"),
+            rejected = rejected_claim_payload(claim, reason, page_context)
+            rejected_claims.append(rejected)
+            if is_reviewable_claim_reason(reason):
+                claim.metadata = {
+                    **claim.metadata,
+                    "claim_kind": claim_kind_for(claim_status_for_row, agreement),
+                    "validation_status": "candidate_claim",
+                    "review_required": True,
+                    "review_reason": reason,
+                    "schema_validated": False,
                 }
-            )
+                claim.metadata["confidence_breakdown"] = claim_confidence_breakdown(
+                    claim,
+                    agreement,
+                    extraction_source,
+                    ontology_compatible=True,
+                )
+                claim.confidence = claim.metadata["confidence_breakdown"]["final_confidence"]
+                claims.append(claim)
             continue
         claim.metadata = {
             **claim.metadata,
-            "claim_kind": "rule_candidate" if claim_status == "rule_candidate" else "ai_claim",
-            "validation_status": claim_status,
+            "claim_kind": claim_kind_for(claim_status_for_row, agreement),
+            "validation_status": claim_status_for_row,
         }
-        breakdown = confidence_breakdown(
-            llm_confidence=claim.confidence,
-            evidence_found=bool(claim.metadata.get("evidence_found")),
-            ontology_compatible=True,
-            source_zone_allowed=bool(claim.metadata.get("source_zone_allowed")),
-        )
+        if agreement == "conflict":
+            claim.metadata = {
+                **claim.metadata,
+                "validation_status": "candidate_claim",
+                "review_required": True,
+                "review_reason": claim.metadata.get("review_reason")
+                or "Rule and LLM produced conflicting claims",
+            }
+        breakdown = claim_confidence_breakdown(claim, agreement, extraction_source)
         claim.metadata["confidence_breakdown"] = breakdown
         claim.confidence = breakdown["final_confidence"]
         claims.append(claim)
 
+    comparison = dict(bundle.raw_output.get("comparison") or {})
+    if comparison:
+        comparison["rejected_by_validation"] = len(rejected_claims)
     bundle.entities = entities
     bundle.claims = claims
     bundle.raw_output = {
         **bundle.raw_output,
+        **({"comparison": comparison} if comparison else {}),
         "validation": {
             "claim_status": claim_status,
             "accepted_claim_count": len(claims),
@@ -196,9 +216,19 @@ def validate_extraction_bundle(bundle: ExtractionBundle, config: ProjectConfig)
 
 def claim_status_for_bundle(bundle: ExtractionBundle) -> str:
     mode = str(bundle.raw_output.get("extraction_mode", "")).lower()
+    effective_mode = str(bundle.raw_output.get("effective_extraction_mode", "")).lower()
     fallback = str(bundle.raw_output.get("fallback", "")).lower()
-    if bundle.provider == "rule_based" or "rule_fallback" in bundle.extractor_name or mode == "fallback" or fallback:
+    if (
+        bundle.provider == "rule_based"
+        or "rule_fallback" in bundle.extractor_name
+        or mode == "fallback"
+        or effective_mode == "rule_only"
+        or bundle.raw_output.get("llm_skipped")
+        or fallback
+    ):
         return "rule_candidate"
+    if mode == "compare":
+        return "candidate_claim"
     page_context = dict(bundle.raw_output.get("page_context") or {})
     page_type = str(page_context.get("page_type") or "")
     if page_type in {"UnknownPage", "SearchPage", "CategoryPage", "BoardPage"}:
@@ -206,6 +236,77 @@ def claim_status_for_bundle(bundle: ExtractionBundle) -> str:
     return "validated_claim"
 
 
+def status_for_claim(bundle_status: str, agreement: str) -> str:
+    if bundle_status in {"rule_candidate", "candidate_claim"}:
+        return bundle_status
+    if agreement == "rule_only":
+        return "rule_candidate"
+    if agreement == "conflict":
+        return "candidate_claim"
+    return bundle_status
+
+
+def claim_kind_for(claim_status: str, agreement: str) -> str:
+    if claim_status == "rule_candidate" or agreement == "rule_only":
+        return "rule_candidate"
+    if agreement == "conflict":
+        return "conflict_candidate"
+    return "ai_claim"
+
+
+def claim_confidence_breakdown(
+    claim: ExtractedClaim,
+    agreement: str,
+    extraction_source: str,
+    *,
+    ontology_compatible: bool = True,
+) -> dict[str, float]:
+    rule_confidence = _optional_float(claim.metadata.get("rule_confidence"))
+    llm_confidence = _optional_float(claim.metadata.get("llm_confidence"))
+    if llm_confidence is None:
+        llm_confidence = 0.0 if extraction_source == "rule" else claim.confidence
+    return confidence_breakdown(
+        llm_confidence=llm_confidence,
+        evidence_found=bool(claim.metadata.get("evidence_found")),
+        ontology_compatible=ontology_compatible,
+        source_zone_allowed=bool(claim.metadata.get("source_zone_allowed")),
+        rule_confidence=rule_confidence,
+        rule_agreement=agreement == "rule_and_llm",
+    )
+
+
+def rejected_claim_payload(
+    claim: ExtractedClaim,
+    reason: str,
+    page_context: dict[str, Any],
+) -> dict[str, Any]:
+    return {
+        "subject": claim.subject_name,
+        "predicate": claim.predicate,
+        "object": claim.object_name if claim.object_name is not None else claim.object_value,
+        "reason": reason,
+        "page_type": claim.metadata.get("page_type") or page_context.get("page_type"),
+        "source_zone": claim.metadata.get("source_zone"),
+    }
+
+
+def is_reviewable_claim_reason(reason: str) -> bool:
+    return reason in {
+        "claim has no source zone",
+        "evidence was not found in clean source zones",
+        "claim has no evidence",
+    } or reason.startswith("source zone is not claim-allowed")
+
+
+def _optional_float(value: Any) -> float | None:
+    if value is None:
+        return None
+    try:
+        return float(value)
+    except (TypeError, ValueError):
+        return None
+
+
 def allowed_entity_types(config: ProjectConfig) -> set[str]:
     configured = set(config.target_entities)
     ontology_types = set((config.ontology or {}).get("entity_types", []))
@@ -267,6 +368,8 @@ def invalid_claim_reason(
     if claim.object_name is not None:
         if claim.object_type not in allowed_types:
             return "object type is outside ontology"
+        if claim.object_type == "Brand" and looks_like_price_text(claim.object_name):
+            return "brand object looks like a price"
         if not is_meaningful_text(claim.object_name):
             return "object is empty, generic, or boilerplate"
         if canonical_entity_key(claim.subject_name, claim.subject_type) == canonical_entity_key(
@@ -316,6 +419,19 @@ def is_meaningful_value(value: Any) -> bool:
     return True
 
 
+def looks_like_price_text(value: str | None) -> bool:
+    if not value:
+        return False
+    clean = " ".join(str(value).strip().split())
+    return bool(
+        re.fullmatch(
+            r"(?:[$€£]\s*)?\d{1,3}(?:,\d{3})*(?:\.\d+)?\s*(?:원|KRW|USD|EUR|JPY|\$|€|£)?",
+            clean,
+            flags=re.IGNORECASE,
+        )
+    )
+
+
 def is_meaningful_text(value: str | None, allow_short: bool = False) -> bool:
     if value is None:
         return False

ontology_platform/crawler_platform/app/core/ontology/domain_discovery.py

@@ -538,12 +538,31 @@ def relation_hint(label: str) -> tuple[str, str] | None:
 def entity_type_from_page_type(page_type: str) -> str | None:
     mapping = {
         "ProductPage": "Product",
+        "ProductDetailPage": "Product",
         "CategoryPage": "Category",
+        "CategoryListingPage": "Category",
+        "SearchPage": "Category",
+        "SearchResultsPage": "Category",
         "BrandStoryPage": "Brand",
+        "AboutPage": "Organization",
+        "ContactPage": "Organization",
         "NoticePage": "Notice",
+        "PublicNoticePage": "Notice",
         "PromotionPage": "Promotion",
+        "CampaignLandingPage": "Promotion",
         "ReviewPage": "Review",
         "BoardPage": "Article",
+        "ForumBoardPage": "Article",
+        "ForumThreadPage": "Article",
+        "ArticlePage": "Article",
+        "NewsArticlePage": "Article",
+        "BlogPostPage": "Article",
+        "FAQPage": "Article",
+        "QAPage": "Article",
+        "DocumentationPage": "Article",
+        "APIReferencePage": "Article",
+        "JobPostingPage": "JobPosting",
+        "PricingPage": "Product",
     }
     return mapping.get(page_type)
 

ontology_platform/crawler_platform/app/core/ontology/relation_schema.py

@@ -3,6 +3,8 @@ from __future__ import annotations
 from dataclasses import dataclass, field
 from typing import Any
 
+from crawler_platform.app.core.crawler.page_type_taxonomy import normalize_page_type
+
 
 @dataclass(frozen=True, slots=True)
 class RelationRule:
@@ -97,36 +99,52 @@ def relation_schema_compatible(
             return f"{predicate} expects a typed entity object"
         if object_type and rule.object_types and object_type not in rule.object_types:
             return f"object type {object_type} is not allowed for {predicate}"
-    if page_type and rule.page_types and page_type not in rule.page_types:
+    if page_type and rule.page_types and not page_type_matches_rule(page_type, rule.page_types):
         return f"predicate {predicate} is not allowed for page type {page_type}"
     if source_zone and rule.source_zones and source_zone not in rule.source_zones:
         return f"source zone {source_zone} is not allowed for {predicate}"
     return None
 
 
+def page_type_matches_rule(page_type: str, allowed_page_types: set[str]) -> bool:
+    if page_type in allowed_page_types:
+        return True
+    normalized_page_type = normalize_page_type(page_type)
+    normalized_allowed = {normalize_page_type(allowed_page_type) for allowed_page_type in allowed_page_types}
+    return normalized_page_type in normalized_allowed
+
+
 def confidence_breakdown(
     *,
     llm_confidence: float,
     evidence_found: bool,
     ontology_compatible: bool,
     source_zone_allowed: bool,
+    rule_confidence: float | None = None,
+    rule_agreement: bool = False,
     source_trust: float | None = None,
 ) -> dict[str, float]:
     schema_confidence = 1.0
     evidence_confidence = 0.95 if evidence_found else 0.0
     ontology_confidence = 0.95 if ontology_compatible else 0.0
     zone_confidence = 0.9 if source_zone_allowed else 0.0
+    rule_value = rule_confidence if rule_confidence is not None else 0.0
+    agreement_confidence = 0.95 if rule_agreement else 0.0
     trust = source_trust if source_trust is not None else 0.8
     final = (
-        llm_confidence * 0.35
-        + schema_confidence * 0.15
-        + evidence_confidence * 0.2
-        + ontology_confidence * 0.2
+        llm_confidence * 0.30
+        + rule_value * 0.10
+        + agreement_confidence * 0.10
+        + schema_confidence * 0.10
+        + evidence_confidence * 0.18
+        + ontology_confidence * 0.12
         + zone_confidence * 0.05
         + trust * 0.05
     )
     return {
         "llm_confidence": round(llm_confidence, 4),
+        "rule_confidence": round(rule_value, 4),
+        "rule_agreement_confidence": agreement_confidence,
         "schema_confidence": schema_confidence,
         "evidence_confidence": evidence_confidence,
         "ontology_confidence": ontology_confidence,

ontology_platform/crawler_platform/app/core/research/graph_research_loop.py

@@ -7,11 +7,17 @@ from urllib.parse import urldefrag, urlparse
 from crawler_platform.app.config.loader import ProjectConfig
 from crawler_platform.app.core.crawler.discovery import discover_links
 from crawler_platform.app.core.crawler.fetchers import RobotsPolicy, make_fetcher
-from crawler_platform.app.core.crawler.page_classifier import classify_page, should_analyze_page
+from crawler_platform.app.core.crawler.page_classifier import (
+    classification_metadata,
+    classify_page_semantic,
+    get_legacy_page_type,
+    should_analyze_page,
+)
 from crawler_platform.app.core.crawler.plugins import ParserRegistry, default_parser_registry
 from crawler_platform.app.core.database import models
 from crawler_platform.app.core.database.repository import KnowledgeRepository
 from crawler_platform.app.core.extractor.base import ExtractionPageContext, Extractor
+from crawler_platform.app.core.extractor.strategy import extract_with_strategy
 from crawler_platform.app.core.extractor.validation import attach_page_context
 from crawler_platform.app.core.research.entity_expansion import EntityExpansionPlanner
 from crawler_platform.app.core.research.exploration_queue import ExplorationItem, ExplorationQueue
@@ -206,13 +212,17 @@ class GraphResearchLoop:
 
         fetch_result = fetcher.fetch(url)
         parser_result = parser.parse(fetch_result.analysis_html, fetch_result.final_url or url)
-        page_type = classify_page(
+        page_classification = classify_page_semantic(
             fetch_result.final_url or url,
             parser_result.title or fetch_result.title,
             parser_result.raw_text or parser_result.text,
             fetch_result.analysis_html,
             parser_result.source_zones or [],
+            final_url=fetch_result.final_url,
+            status_code=fetch_result.status_code,
+            content_type=fetch_result.headers.get("content-type"),
         )
+        page_type = get_legacy_page_type(page_classification)
         relevance = self.relevance.score_url(
             project_id=source.project_id,
             url=fetch_result.final_url or url,
@@ -225,11 +235,17 @@ class GraphResearchLoop:
         )
         metadata = {
             **parser_result.metadata,
+            **classification_metadata(
+                page_classification,
+                title=parser_result.title or fetch_result.title,
+                text=parser_result.raw_text or parser_result.text,
+                html=fetch_result.analysis_html,
+                source_zones=parser_result.source_zones or [],
+            ),
             "research_item": item.to_dict(),
             "research_relevance": asdict(relevance),
             "final_url": fetch_result.final_url,
             "crawl_status": fetch_result.crawl_status,
-            "page_type": page_type,
             "robots_status": robots_decision.status,
             "robots_reason": robots_decision.reason,
             "raw_text_length": len(parser_result.raw_text or ""),
@@ -284,11 +300,12 @@ class GraphResearchLoop:
         analyzed = False
         claim_count = 0
         entity_count = 0
+        extraction_summary: dict[str, Any] = {}
         if (
             fetch_result.crawl_status == "success"
             and parser_result.extraction_status != "failed"
             and relevance.score >= min_relevance
-            and should_analyze_page(page_type, analyze_page_types)
+            and should_analyze_page(page_classification, analyze_page_types)
         ):
             context = ExtractionPageContext(
                 url=url,
@@ -305,12 +322,13 @@ class GraphResearchLoop:
                 warnings=[*fetch_result.warnings, *(parser_result.extraction_warnings or [])],
                 metadata=metadata,
             )
-            bundle = self.extractor.extract_from_context(context, project_config)
+            bundle = extract_with_strategy(self.extractor, context, project_config)
             bundle = attach_page_context(bundle, context)
             claims = self.repository.save_extraction_bundle(source.project_id, source, page, bundle, project_config)
             analyzed = True
             claim_count = len(claims)
             entity_count = len(bundle.entities)
+            extraction_summary = extraction_summary_from_raw(bundle.raw_output)
 
         return {
             "status": "explored",
@@ -320,6 +338,7 @@ class GraphResearchLoop:
             "analyzed": analyzed,
             "claim_count": claim_count,
             "entity_count": entity_count,
+            "extraction": extraction_summary,
             "enqueued": enqueued,
             "scored_links": scored_links[:20],
         }
@@ -369,3 +388,15 @@ def normalize_url(url: str) -> str:
 
 def normalized_host(url: str) -> str:
     return urlparse(str(url)).netloc.lower()
+
+
+def extraction_summary_from_raw(raw_output: dict[str, Any]) -> dict[str, Any]:
+    return {
+        "extraction_mode": raw_output.get("extraction_mode"),
+        "effective_extraction_mode": raw_output.get("effective_extraction_mode"),
+        "llm_skipped": bool(raw_output.get("llm_skipped")),
+        "llm_skip_reason": raw_output.get("llm_skip_reason"),
+        "fallback_used": bool(raw_output.get("fallback")),
+        "agreement_claim_count": int(raw_output.get("agreement_claim_count") or 0),
+        "conflict_claim_count": int(raw_output.get("conflict_claim_count") or 0),
+    }

ontology_platform/crawler_platform/app/core/research/relevance_engine.py

@@ -8,20 +8,46 @@ from urllib.parse import unquote, urlparse
 from sqlalchemy import select
 from sqlalchemy.orm import Session
 
-from crawler_platform.app.core.crawler.page_classifier import classify_page
+from crawler_platform.app.core.crawler.page_classifier import classify_page_semantic, get_legacy_page_type
 from crawler_platform.app.core.database import models
 
 
 HIGH_VALUE_PAGE_TYPES = {
     "ProductPage": 0.95,
+    "ProductDetailPage": 0.95,
     "BrandStoryPage": 0.88,
+    "AboutPage": 0.82,
+    "ContactPage": 0.72,
     "ReviewPage": 0.82,
     "NoticePage": 0.45,
+    "PublicNoticePage": 0.45,
+    "ArticlePage": 0.55,
+    "NewsArticlePage": 0.55,
+    "BlogPostPage": 0.52,
+    "FAQPage": 0.48,
+    "QAPage": 0.48,
+    "DocumentationPage": 0.5,
+    "APIReferencePage": 0.5,
+    "WikiPage": 0.48,
+    "DatasetPage": 0.52,
+    "ResearchPaperPage": 0.6,
+    "JobPostingPage": 0.46,
+    "CourseDetailPage": 0.5,
+    "VideoPage": 0.36,
+    "LocalBusinessPage": 0.44,
+    "RealEstateListingPage": 0.44,
+    "ProfilePage": 0.36,
+    "PricingPage": 0.62,
     "EventPage": 0.5,
     "PromotionPage": 0.38,
+    "CampaignLandingPage": 0.38,
     "CategoryPage": 0.34,
+    "CategoryListingPage": 0.34,
     "SearchPage": 0.22,
+    "SearchResultsPage": 0.22,
     "BoardPage": 0.18,
+    "ForumBoardPage": 0.18,
+    "ForumThreadPage": 0.24,
     "UnknownPage": 0.25,
 }
 
@@ -96,8 +122,12 @@ class RelevanceEngine:
         normalized = url.strip().rstrip("/")
         parsed = urlparse(normalized)
         combined = unquote(f"{normalized} {label} {text[:3000]}").lower()
-        page_type = classify_page(normalized, label, text, html=html)
-        page_type_score = HIGH_VALUE_PAGE_TYPES.get(page_type, 0.25)
+        page_classification = classify_page_semantic(normalized, label, text, html=html)
+        page_type = page_classification.primary_page_type
+        page_type_score = HIGH_VALUE_PAGE_TYPES.get(
+            page_type,
+            HIGH_VALUE_PAGE_TYPES.get(get_legacy_page_type(page_classification), 0.25),
+        )
         graph_terms = self.graph_terms(project_id)
         tokens = tokenize(combined)
         overlap_count = len(tokens & graph_terms)

ontology_platform/crawler_platform/app/domains/perfume/extractor.py

@@ -167,6 +167,7 @@ class PerfumeRuleBasedExtractor(GenericRuleBasedExtractor):
             brand = next((entity for entity in entities if entity.entity_type == "Brand"), None)
             for card in product_cards:
                 if brand:
+                    evidence = brand_evidence_for_product(brand, str(card["name"]), page_text) or brand.evidence_text or brand.name
                     claims.append(
                         ExtractedClaim(
                             str(card["name"]),
@@ -174,7 +175,7 @@ class PerfumeRuleBasedExtractor(GenericRuleBasedExtractor):
                             "hasBrand",
                             brand.name,
                             "Brand",
-                            evidence_text=brand.evidence_text or brand.name,
+                            evidence_text=evidence,
                             confidence=0.72,
                             confidence_reason="site brand inferred from listing page",
                         )
@@ -199,6 +200,7 @@ class PerfumeRuleBasedExtractor(GenericRuleBasedExtractor):
         claims: list[ExtractedClaim] = []
         brand = next((entity for entity in entities if entity.entity_type == "Brand"), None)
         if brand:
+            evidence = brand_evidence_for_product(brand, perfume.name, page_text) or brand.evidence_text or brand.name
             claims.append(
                 ExtractedClaim(
                     perfume.name,
@@ -206,7 +208,7 @@ class PerfumeRuleBasedExtractor(GenericRuleBasedExtractor):
                     "hasBrand",
                     brand.name,
                     "Brand",
-                    evidence_text=brand.evidence_text or brand.name,
+                    evidence_text=evidence,
                     confidence=0.78,
                     confidence_reason="brand pattern matched",
                 )
@@ -273,14 +275,16 @@ def extract_brand(page_text: str, product_name: str) -> str | None:
     for pattern in patterns:
         match = re.search(pattern, page_text, flags=re.IGNORECASE)
         if match:
-            return cleanup_value(match.group("brand"))
-    inferred = infer_site_brand(page_text)
+            brand = valid_brand(cleanup_value(match.group("brand")))
+            if brand:
+                return brand
+    inferred = infer_site_brand(page_text, product_name)
     if inferred:
         return inferred
     lines = [line.strip() for line in page_text.splitlines() if line.strip()]
     if len(lines) >= 2 and lines[1].lower() not in product_name.lower():
         candidate = cleanup_value(lines[1])
-        if len(candidate) <= 80 and not looks_like_navigation(candidate) and product_line_score(candidate) <= 0:
+        if valid_brand(candidate) and len(candidate) <= 80 and not looks_like_navigation(candidate) and product_line_score(candidate) <= 0:
             return candidate
     return None
 
@@ -301,6 +305,19 @@ def product_line_score(value: str) -> int:
     return score
 
 
+def looks_like_price(value: str | None) -> bool:
+    if not value:
+        return False
+    clean = cleanup_value(str(value))
+    return bool(
+        re.fullmatch(
+            r"(?:[$€£]\s*)?\d{1,3}(?:,\d{3})*(?:\.\d+)?\s*(?:원|KRW|USD|EUR|JPY|\$|€|£)?",
+            clean,
+            flags=re.IGNORECASE,
+        )
+    )
+
+
 def looks_like_metric_or_price(value: str) -> bool:
     clean = value.replace(",", "").strip()
     if re.fullmatch(r"\d+(?:\.\d+)?", clean):
@@ -385,7 +402,10 @@ def dedupe_product_cards(cards: list[dict[str, object]]) -> list[dict[str, objec
     return result
 
 
-def infer_site_brand(page_text: str) -> str | None:
+def infer_site_brand(page_text: str, product_name: str = "") -> str | None:
+    brand_context = f"{product_name}\n{page_text[:1200]}".lower()
+    if "forment" in brand_context or "포맨트" in brand_context:
+        return "FORMENT"
     if "912 공식 홈페이지" in page_text or "912" in page_text[:500]:
         return "912"
     return None
@@ -394,9 +414,19 @@ def infer_site_brand(page_text: str) -> str | None:
 def valid_brand(value: str | None) -> str | None:
     if not value or is_template_placeholder(value):
         return None
+    if looks_like_price(value) or looks_like_metric_or_price(value):
+        return None
     return value
 
 
+def brand_evidence_for_product(brand: ExtractedEntity, product_name: str, page_text: str) -> str | None:
+    if brand.name == "FORMENT" and "포맨트" in product_name:
+        return product_name
+    if brand.name.lower() in page_text.lower():
+        return brand.name
+    return None
+
+
 def extract_field_values(field: str, page_text: str) -> list[tuple[str, str]]:
     if field in NOTE_LABELS:
         return extract_labeled_values(page_text, NOTE_LABELS[field])

ontology_platform/docs/crawl_flowchart_llm_decision.svg

@@ -0,0 +1,61 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="2400" height="3600" viewBox="0 0 2400 3600">
+<defs>
+  <marker id="arrow" markerWidth="10" markerHeight="10" refX="9" refY="3" orient="auto" markerUnits="strokeWidth">
+    <path d="M0,0 L0,6 L9,3 z" fill="#1E40AF"/>
+  </marker>
+</defs>
+<rect width="100%" height="100%" fill="#F7F9FC"/>
+<text x="110" y="120" font-family="Malgun Gothic, Arial" font-size="58" font-weight="700" fill="#172033">크롤 진행 단계와 LLM 사용 판단 구조</text>
+<text x="112" y="176" font-family="Malgun Gothic, Arial" font-size="28" fill="#5C6B82">API 요청부터 Rule/LLM 추출, 검증, DB 저장까지의 흐름</text>
+<path d="M 1200 402 L 1200 470" fill="none" stroke="#1E40AF" stroke-width="4" marker-end="url(#arrow)"/>
+<path d="M 1200 602 L 1200 670" fill="none" stroke="#1E40AF" stroke-width="4" marker-end="url(#arrow)"/>
+<path d="M 1200 802 L 1200 870" fill="none" stroke="#1E40AF" stroke-width="4" marker-end="url(#arrow)"/>
+<path d="M 1200 1002 L 1200 1070" fill="none" stroke="#1E40AF" stroke-width="4" marker-end="url(#arrow)"/>
+<path d="M 1200 1222 L 1200 1300" fill="none" stroke="#1E40AF" stroke-width="4" marker-end="url(#arrow)"/>
+<path d="M 1200 1432 L 1200 1525" fill="none" stroke="#1E40AF" stroke-width="4" marker-end="url(#arrow)"/>
+<path d="M 1200 1897 L 1200 1965" fill="none" stroke="#1E40AF" stroke-width="4" marker-end="url(#arrow)"/>
+<path d="M 1200 2097 L 1200 2190" fill="none" stroke="#1E40AF" stroke-width="4" marker-end="url(#arrow)"/>
+<path d="M 1200 2562 L 1200 2635" fill="none" stroke="#1E40AF" stroke-width="4" marker-end="url(#arrow)"/>
+<path d="M 1200 3012 L 1200 3080" fill="none" stroke="#1E40AF" stroke-width="4" marker-end="url(#arrow)"/>
+<path d="M 1200 3240 L 1200 3330" fill="none" stroke="#1E40AF" stroke-width="4" marker-end="url(#arrow)"/>
+<path d="M 820 1610 L 818 1610 L 818 1809 L 800 1809" fill="none" stroke="#5C6B82" stroke-width="4" marker-end="url(#arrow)"/>\n<rect x="756" y="1779" width="124" height="48" rx="12" fill="#F7F9FC" stroke="#CBD5E1" stroke-width="2"/>\n<text x="818" y="1817" text-anchor="middle" font-family="Malgun Gothic, Arial" font-size="25" font-weight="700" fill="#5C6B82">아니오</text>
+<path d="M 1200 1695 L 1200 1765" fill="none" stroke="#16A34A" stroke-width="4" marker-end="url(#arrow)"/>\n<rect x="1138" y="1735" width="124" height="48" rx="12" fill="#F7F9FC" stroke="#CBD5E1" stroke-width="2"/>\n<text x="1200" y="1773" text-anchor="middle" font-family="Malgun Gothic, Arial" font-size="25" font-weight="700" fill="#16A34A">예</text>
+<path d="M 820 2275 L 758 2275 L 758 2496 L 740 2496" fill="none" stroke="#5C6B82" stroke-width="4" marker-end="url(#arrow)"/>\n<rect x="696" y="2466" width="124" height="48" rx="12" fill="#F7F9FC" stroke="#CBD5E1" stroke-width="2"/>\n<text x="758" y="2504" text-anchor="middle" font-family="Malgun Gothic, Arial" font-size="25" font-weight="700" fill="#5C6B82">skip</text>
+<path d="M 1200 2360 L 1200 2430" fill="none" stroke="#16A34A" stroke-width="4" marker-end="url(#arrow)"/>\n<rect x="1138" y="2400" width="124" height="48" rx="12" fill="#F7F9FC" stroke="#CBD5E1" stroke-width="2"/>\n<text x="1200" y="2438" text-anchor="middle" font-family="Malgun Gothic, Arial" font-size="25" font-weight="700" fill="#16A34A">use</text>
+<path d="M 430 2562 L 430 2946 L 820 2946" fill="none" stroke="#5C6B82" stroke-width="4" marker-end="url(#arrow)"/>
+<path d="M 1200 2785 L 1200 2880" fill="none" stroke="#16A34A" stroke-width="4" marker-end="url(#arrow)"/>
+<rect x="820" y="270" width="760" height="132" rx="24" fill="#FFFFFF" stroke="#2563EB" stroke-width="4"/>
+<text x="1200" y="316" text-anchor="middle" font-family="Malgun Gothic, Arial" font-size="30" font-weight="700" fill="#172033">API 요청</text>\n<text x="1200" y="356" text-anchor="middle" font-family="Malgun Gothic, Arial" font-size="30" font-weight="700" fill="#172033">/crawl-site</text>
+<rect x="820" y="470" width="760" height="132" rx="24" fill="#FFFFFF" stroke="#2563EB" stroke-width="4"/>
+<text x="1200" y="516" text-anchor="middle" font-family="Malgun Gothic, Arial" font-size="30" font-weight="700" fill="#172033">Job 생성 및</text>\n<text x="1200" y="556" text-anchor="middle" font-family="Malgun Gothic, Arial" font-size="30" font-weight="700" fill="#172033">queue 초기화</text>
+<rect x="820" y="670" width="760" height="132" rx="24" fill="#FFFFFF" stroke="#2563EB" stroke-width="4"/>
+<text x="1200" y="716" text-anchor="middle" font-family="Malgun Gothic, Arial" font-size="30" font-weight="700" fill="#172033">URL depth / domain /</text>\n<text x="1200" y="756" text-anchor="middle" font-family="Malgun Gothic, Arial" font-size="30" font-weight="700" fill="#172033">robots 검사</text>
+<rect x="820" y="870" width="760" height="132" rx="24" fill="#FFFFFF" stroke="#2563EB" stroke-width="4"/>
+<text x="1200" y="916" text-anchor="middle" font-family="Malgun Gothic, Arial" font-size="30" font-weight="700" fill="#172033">Fetch:</text>\n<text x="1200" y="956" text-anchor="middle" font-family="Malgun Gothic, Arial" font-size="30" font-weight="700" fill="#172033">HTML 수집</text>
+<rect x="820" y="1070" width="760" height="152" rx="24" fill="#FFFFFF" stroke="#2563EB" stroke-width="4"/>
+<text x="1200" y="1106" text-anchor="middle" font-family="Malgun Gothic, Arial" font-size="30" font-weight="700" fill="#172033">Parse:</text>\n<text x="1200" y="1146" text-anchor="middle" font-family="Malgun Gothic, Arial" font-size="30" font-weight="700" fill="#172033">본문 / zone /</text>\n<text x="1200" y="1186" text-anchor="middle" font-family="Malgun Gothic, Arial" font-size="30" font-weight="700" fill="#172033">clean_text 추출</text>
+<rect x="820" y="1300" width="760" height="132" rx="24" fill="#FFFFFF" stroke="#2563EB" stroke-width="4"/>
+<text x="1200" y="1346" text-anchor="middle" font-family="Malgun Gothic, Arial" font-size="30" font-weight="700" fill="#172033">Page Classifier:</text>\n<text x="1200" y="1386" text-anchor="middle" font-family="Malgun Gothic, Arial" font-size="30" font-weight="700" fill="#172033">ProductPage 등 분류</text>
+<polygon points="1200,1525 1580,1610 1200,1695 820,1610" fill="#FFF7ED" stroke="#D97706" stroke-width="4"/>
+<text x="1200" y="1590" text-anchor="middle" font-family="Malgun Gothic, Arial" font-size="30" font-weight="700" fill="#172033">분석 대상</text>\n<text x="1200" y="1630" text-anchor="middle" font-family="Malgun Gothic, Arial" font-size="30" font-weight="700" fill="#172033">page_type인가?</text>
+<rect x="120" y="1740" width="680" height="138" rx="24" fill="#F8FAFC" stroke="#5C6B82" stroke-width="4"/>
+<text x="460" y="1769" text-anchor="middle" font-family="Malgun Gothic, Arial" font-size="30" font-weight="700" fill="#172033">discovered/skipped</text>\n<text x="460" y="1809" text-anchor="middle" font-family="Malgun Gothic, Arial" font-size="30" font-weight="700" fill="#172033">저장,</text>\n<text x="460" y="1849" text-anchor="middle" font-family="Malgun Gothic, Arial" font-size="30" font-weight="700" fill="#172033">링크만 확장</text>
+<rect x="820" y="1765" width="760" height="132" rx="24" fill="#FFFFFF" stroke="#2563EB" stroke-width="4"/>
+<text x="1200" y="1831" text-anchor="middle" font-family="Malgun Gothic, Arial" font-size="30" font-weight="700" fill="#172033">Extractor 선택</text>
+<rect x="820" y="1965" width="760" height="132" rx="24" fill="#FFFFFF" stroke="#2563EB" stroke-width="4"/>
+<text x="1200" y="2031" text-anchor="middle" font-family="Malgun Gothic, Arial" font-size="30" font-weight="700" fill="#172033">Rule extraction</text>
+<polygon points="1200,2190 1580,2275 1200,2360 820,2275" fill="#FFF7ED" stroke="#D97706" stroke-width="4"/>
+<text x="1200" y="2275" text-anchor="middle" font-family="Malgun Gothic, Arial" font-size="30" font-weight="700" fill="#172033">LLM 사용 판단</text>
+<rect x="120" y="2430" width="620" height="132" rx="24" fill="#F8FAFC" stroke="#5C6B82" stroke-width="4"/>
+<text x="430" y="2496" text-anchor="middle" font-family="Malgun Gothic, Arial" font-size="30" font-weight="700" fill="#172033">rule_only routed</text>
+<rect x="820" y="2430" width="760" height="132" rx="24" fill="#FFFFFF" stroke="#2563EB" stroke-width="4"/>
+<text x="1200" y="2496" text-anchor="middle" font-family="Malgun Gothic, Arial" font-size="30" font-weight="700" fill="#172033">LLM extraction</text>
+<rect x="820" y="2635" width="760" height="150" rx="24" fill="#ECFDF5" stroke="#16A34A" stroke-width="4"/>
+<text x="1200" y="2690" text-anchor="middle" font-family="Malgun Gothic, Arial" font-size="30" font-weight="700" fill="#172033">rule/LLM merge,</text>\n<text x="1200" y="2730" text-anchor="middle" font-family="Malgun Gothic, Arial" font-size="30" font-weight="700" fill="#172033">agreement/conflict 계산</text>
+<rect x="820" y="2880" width="760" height="132" rx="24" fill="#FFFFFF" stroke="#2563EB" stroke-width="4"/>
+<text x="1200" y="2946" text-anchor="middle" font-family="Malgun Gothic, Arial" font-size="30" font-weight="700" fill="#172033">validation</text>
+<rect x="820" y="3080" width="760" height="160" rx="24" fill="#FFFFFF" stroke="#2563EB" stroke-width="4"/>
+<text x="1200" y="3120" text-anchor="middle" font-family="Malgun Gothic, Arial" font-size="30" font-weight="700" fill="#172033">DB 저장:</text>\n<text x="1200" y="3160" text-anchor="middle" font-family="Malgun Gothic, Arial" font-size="30" font-weight="700" fill="#172033">entities / claims /</text>\n<text x="1200" y="3200" text-anchor="middle" font-family="Malgun Gothic, Arial" font-size="30" font-weight="700" fill="#172033">evidence / log</text>
+<rect x="820" y="3330" width="760" height="132" rx="24" fill="#FFFFFF" stroke="#2563EB" stroke-width="4"/>
+<text x="1200" y="3396" text-anchor="middle" font-family="Malgun Gothic, Arial" font-size="30" font-weight="700" fill="#172033">progress 업데이트</text>
+</svg>

ontology_platform/docs/phases/26_05_19_engine_respect_plan/phase_07_001_hybrid_rule_llm_extraction.md

@@ -0,0 +1,415 @@
+# Phase 7. Hybrid Rule + LLM Extraction
+
+작성일: 2026-05-21
+
+범위: `ontology_platform`에 포함된 product backend, 특히 `crawler_platform/app/core/extractor`, crawl/research API, Review/Page Analysis UI.
+
+주의: 이 phase는 OntoCast core engine을 바꾸지 않는다. 룰 기반 product extraction과 LLM JSON extraction을 product backend 레벨에서 결합한다. `/process` OntoCast workflow는 후처리/RDF 변환 후보로만 연결한다.
+
+## 목표
+
+룰 기반 추출의 장점인 빠름, 비용 없음, 예측 가능성, 명확한 패턴 인식을 살리고, LLM 추출의 장점인 의미 해석, 타입 추론, 관계 추출, 한국어/비정형 문장 대응력을 더한다.
+
+핵심 원칙:
+
+- 룰은 먼저 실행되는 baseline extractor이자 검증 가드레일이다.
+- LLM은 의미 추출기이며, evidence/ontology/source zone 검증을 통과해야 한다.
+- LLM 실패는 전체 실패가 아니라 rule fallback으로 처리한다.
+- 룰과 LLM이 같은 claim을 찾으면 신뢰도를 올린다.
+- 룰과 LLM이 충돌하면 자동 승인하지 않고 review로 보낸다.
+
+## 현재 기준선
+
+현재 확인된 동작:
+
+- `ont_platform/api/routes/extraction.py`
+  - `/api/v1/extract/url`, `/api/v1/process/url`, `/process/url`
+  - `LightweightExtractor(use_llm=False)` 고정
+  - 룰 기반 기초 후보만 생성
+
+- `ont_platform/core/crawler/jobs.py`
+  - `/api/v1/crawl/jobs`
+  - `LightweightExtractor(use_llm=False)` 고정
+  - 기본 수집 + 기초 후보 저장
+
+- `crawler_platform/app/core/extractor/factory.py`
+  - `provider in {"openai", "ollama", "lm_studio"}`이면 `LLMJsonExtractor`
+  - 그 외에는 domain별 rule extractor
+
+- `crawler_platform/app/core/extractor/ai_provider.py`
+  - LLM primary, compact retry, JSON repair, rule fallback이 이미 일부 존재
+  - LLM 결과에 rule claim을 merge하는 `_merge_rule_fallback_claims()`가 이미 존재
+
+- `crawler_platform/app/core/extractor/validation.py`
+  - evidence zone, ontology predicate, source zone, confidence breakdown 검증이 존재
+
+- `web/frontend/src/pages/CrawlPage.tsx`, `ResearchPage.tsx`
+  - request type에는 `extractor_provider` 필드가 있으나 화면에서 선택 UI는 없음
+
+## Phase 7.1 Baseline Audit
+
+목표: 기존 동작을 깨지 않기 위해 현재 추출 결과와 저장 구조를 고정한다.
+
+작업:
+
+- `rule_based`, `lm_studio`, `openai`, `ollama` provider별 request/response 샘플을 만든다.
+- crawl, crawl-site, research/run 경로에서 extractor가 어떻게 선택되는지 문서화한다.
+- `Claim`, `Entity`, `ExtractionLog`에 저장되는 `extractor_name`, `provider`, `metadata_json`, `confidence_breakdown` 구조를 샘플로 기록한다.
+- LM Studio가 꺼진 상태, 켜진 상태, 모델명 누락 상태를 각각 재현한다.
+
+주요 파일:
+
+- `crawler_platform/app/core/extractor/factory.py`
+- `crawler_platform/app/core/extractor/ai_provider.py`
+- `crawler_platform/app/core/extractor/rule_based.py`
+- `crawler_platform/app/core/database/repository.py`
+- `crawler_platform/app/api/routes.py`
+
+완료 기준:
+
+- 기존 `rule_based`와 `lm_studio` 동작이 재현 가능하다.
+- 최소 1개 상품 페이지 fixture로 rule 결과와 LLM 결과 샘플이 있다.
+- LM Studio 장애 시 현재 fallback 결과가 확인되어 있다.
+
+## Phase 7.2 Extraction Mode Contract
+
+목표: provider와 실행 전략을 분리한다.
+
+현재 문제:
+
+- `extractor_provider`가 provider이면서 실행 전략 역할도 한다.
+- 사용자는 rule only, LLM only, hybrid, compare를 명시적으로 선택할 수 없다.
+
+제안 계약:
+
+```json
+{
+  "extraction_mode": "hybrid",
+  "extractor_provider": "lm_studio",
+  "extractor_model": "deepseek-r1-distill-qwen-7b",
+  "extractor_base_url": "http://localhost:1234/v1",
+  "fallback_to_rules": true
+}
+```
+
+지원 mode:
+
+- `rule_only`: rule extractor만 실행
+- `llm_only`: LLM extractor만 실행, fallback 선택 가능
+- `hybrid`: rule 먼저 실행, LLM 실행, 병합/검증
+- `compare`: rule 결과와 LLM 결과를 모두 보존하고 차이를 metadata/log에 저장
+
+호환성:
+
+- 기존 `extractor_provider: "rule_based"`는 `extraction_mode="rule_only"`로 해석한다.
+- 기존 `extractor_provider: "lm_studio" | "openai" | "ollama"`는 당분간 `extraction_mode="hybrid"`로 해석한다.
+- 기존 UI가 provider를 보내지 않는 경우 기본값은 `hybrid + lm_studio`로 유지한다.
+
+수정 파일:
+
+- `crawler_platform/app/api/routes.py`
+- `web/frontend/src/lib/api/crawl.ts`
+- `web/frontend/src/lib/api/research.ts`
+
+완료 기준:
+
+- 기존 요청이 깨지지 않는다.
+- 새 요청 필드로 `rule_only`, `llm_only`, `hybrid`, `compare`가 구분된다.
+- API response/log에 실제 실행 mode가 남는다.
+
+## Phase 7.3 Hybrid Extractor
+
+목표: 명시적인 `HybridExtractor`를 추가한다.
+
+신규 파일:
+
+- `crawler_platform/app/core/extractor/hybrid.py`
+
+실행 순서:
+
+1. domain에 맞는 rule extractor 실행
+2. LLM extractor 실행
+3. entity dedupe
+4. claim dedupe
+5. rule-LLM agreement 계산
+6. conflict 계산
+7. metadata에 extraction mode와 comparison 결과 기록
+8. validation pipeline으로 전달
+
+필수 metadata:
+
+```json
+{
+  "extraction_mode": "hybrid",
+  "rule_entity_count": 12,
+  "rule_claim_count": 8,
+  "llm_entity_count": 15,
+  "llm_claim_count": 13,
+  "agreement_claim_count": 6,
+  "rule_only_claim_count": 2,
+  "llm_only_claim_count": 7,
+  "conflict_claim_count": 1
+}
+```
+
+병합 규칙:
+
+- 동일 subject/predicate/object claim은 하나로 합친다.
+- rule과 LLM이 모두 찾은 claim은 `agreement="rule_and_llm"`로 표시한다.
+- rule만 찾은 claim은 `agreement="rule_only"`와 `claim_kind="rule_candidate"`로 표시한다.
+- LLM만 찾은 claim은 evidence 검증 전까지 `agreement="llm_only"`로 표시한다.
+- 같은 subject/predicate인데 object가 다르면 `conflict_status="rule_llm_conflict"`로 표시하고 review로 보낸다.
+
+수정 파일:
+
+- `crawler_platform/app/core/extractor/factory.py`
+- `crawler_platform/app/core/extractor/ai_provider.py`
+- `crawler_platform/app/core/extractor/base.py`
+
+완료 기준:
+
+- `extractor_provider="hybrid"` 또는 `extraction_mode="hybrid"`로 실행 가능하다.
+- LLM 실패 시 rule 결과만으로 성공 response가 나온다.
+- rule/LLM agreement가 claim metadata에 남는다.
+
+## Phase 7.4 Confidence And Validation Upgrade
+
+목표: LLM hallucination을 줄이고, rule agreement를 신뢰도에 반영한다.
+
+현재 confidence 구성:
+
+- `llm_confidence`
+- `schema_confidence`
+- `evidence_confidence`
+- `ontology_confidence`
+- `source_zone_confidence`
+- `source_trust`
+- `final_confidence`
+
+추가 항목:
+
+```json
+{
+  "rule_confidence": 0.75,
+  "rule_agreement_confidence": 0.95,
+  "llm_confidence": 0.82,
+  "evidence_confidence": 0.95,
+  "ontology_confidence": 0.95,
+  "source_zone_confidence": 0.9,
+  "final_confidence": 0.89
+}
+```
+
+정책:
+
+- rule+LLM agreement가 있으면 confidence bonus를 준다.
+- LLM-only claim은 evidence가 없으면 자동 승인하지 않는다.
+- source zone을 찾지 못한 claim은 review로 보낸다.
+- ontology에 없는 predicate는 reject한다.
+- rule-only claim은 기본적으로 `rule_candidate`로 남긴다.
+- 충돌 claim은 `review_required=true`로 남긴다.
+
+수정 파일:
+
+- `crawler_platform/app/core/ontology/relation_schema.py`
+- `crawler_platform/app/core/extractor/validation.py`
+- `crawler_platform/app/core/database/repository.py`
+
+완료 기준:
+
+- Review 화면에서 `confidence_breakdown.rule_agreement_confidence`를 볼 수 있다.
+- evidence 없는 LLM-only claim이 `validated_claim`으로 자동 저장되지 않는다.
+- rule+LLM 일치 claim은 더 높은 final confidence를 받는다.
+
+## Phase 7.5 Compare Mode
+
+목표: rule 결과와 LLM 결과를 나란히 비교하여 품질 튜닝에 사용한다.
+
+작업:
+
+- `compare` mode에서 rule bundle과 LLM bundle을 모두 실행한다.
+- 최종 저장은 병합 결과로 하되, `ExtractionLog.raw_output`에 원본 두 결과를 보존한다.
+- Page Analysis에서 diff summary를 표시한다.
+
+diff category:
+
+- `both_agree`
+- `rule_only`
+- `llm_only`
+- `conflict`
+- `rejected_by_validation`
+
+UI 표시:
+
+- Page Analysis: page별 extractor run 요약, rule/LLM candidate count, conflict count
+- Review: claim detail에서 agreement badge 표시
+
+수정 파일:
+
+- `crawler_platform/app/core/database/models.py`
+- `crawler_platform/app/core/database/repository.py`
+- `crawler_platform/app/api/routes.py`
+- `web/frontend/src/pages/PageAnalysisPage.tsx`
+- `web/frontend/src/pages/ReviewPage.tsx`
+- `web/frontend/src/lib/api/platform.ts`
+
+완료 기준:
+
+- 같은 페이지에서 rule과 LLM의 차이를 확인할 수 있다.
+- `llm_only`와 `conflict` claim을 review에서 필터링할 수 있다.
+
+## Phase 7.6 UI Controls
+
+목표: 사용자가 화면에서 mode/provider/model/base URL을 선택할 수 있게 한다.
+
+대상 화면:
+
+- Crawl Page
+- Research Page
+
+추가 컨트롤:
+
+- Extraction mode select
+  - Rule only
+  - Hybrid
+  - LLM only
+  - Compare
+- LLM provider select
+  - LM Studio
+  - OpenAI
+  - Ollama
+- Model input
+- Base URL input
+- Fallback to rules toggle
+- Optional model list refresh button
+
+동작:
+
+- mode가 `rule_only`이면 provider/model/base URL 입력을 숨긴다.
+- provider가 `lm_studio`이면 기본 base URL은 `http://localhost:1234/v1`이다.
+- provider가 `ollama`이면 기본 base URL은 `http://localhost:11434/api/chat`이다.
+- 모델 목록은 `/extractors/models`를 사용한다.
+
+수정 파일:
+
+- `web/frontend/src/pages/CrawlPage.tsx`
+- `web/frontend/src/pages/ResearchPage.tsx`
+- `web/frontend/src/lib/api/crawl.ts`
+- `web/frontend/src/lib/api/research.ts`
+
+완료 기준:
+
+- UI에서 mode/provider/model/base URL을 지정할 수 있다.
+- 지정한 값이 `/crawl-site/by-project`, `/research/run/by-project` 요청에 포함된다.
+- rule only 선택 시 LM Studio가 꺼져 있어도 작업이 시작된다.
+
+## Phase 7.7 Smart Routing Policy
+
+목표: 모든 페이지에 LLM을 쓰지 않고 가치 있는 페이지에 집중한다.
+
+정책:
+
+- `ProductPage`: hybrid 기본
+- `BrandStoryPage`: hybrid 기본
+- `ReviewPage`: hybrid 또는 llm_only
+- `CategoryPage`: rule_only 또는 skip LLM
+- `SearchPage`: skip LLM
+- `BoardPage`: rule_only 또는 review 후보
+- 본문 길이가 너무 짧으면 rule_only
+- rule 결과가 충분하고 deterministic field만 필요한 경우 LLM 생략 가능
+- research goal이 있으면 LLM 우선
+
+추가 설정:
+
+```json
+{
+  "llm_page_types": ["ProductPage", "BrandStoryPage", "ReviewPage"],
+  "skip_llm_page_types": ["CategoryPage", "SearchPage"],
+  "min_clean_text_chars_for_llm": 300,
+  "max_llm_pages_per_job": 30
+}
+```
+
+수정 파일:
+
+- `crawler_platform/app/core/crawler/site_crawler.py`
+- `crawler_platform/app/core/crawler/pipeline.py`
+- `crawler_platform/app/core/research/graph_research_loop.py`
+
+완료 기준:
+
+- LLM 호출 수가 page type 정책에 따라 제한된다.
+- 중요 페이지는 hybrid 추출을 받는다.
+- crawl job metadata에 LLM skipped reason이 남는다.
+
+## Phase 7.8 OntoCast `/process` Handoff
+
+목표: product crawl/research 결과를 OntoCast RDF 변환과 연결한다.
+
+역할 분리:
+
+- product crawl/research: web acquisition, entity/claim 후보 수집
+- validation/review: claim 품질 관리
+- `/process`: 검증된 문서나 claim 묶음을 RDF/Turtle로 정리하는 후처리
+
+작업:
+
+- validated claim 묶음을 OntoCast input JSON으로 변환한다.
+- 프로젝트 단위 또는 selected document 단위로 `/process`를 호출할 수 있게 한다.
+- `/process` 결과 Turtle을 export/graph view와 연결한다.
+
+완료 기준:
+
+- 사용자가 검증된 claim subset을 RDF/Turtle로 내보낼 수 있다.
+- `/process`는 모든 crawl page마다 자동 실행되지 않는다.
+- 비용 큰 LLM workflow는 명시적 후처리로만 실행된다.
+
+## Phase 7.9 Tests And Observability
+
+목표: rule/LLM/hybrid/compare 동작을 재현 가능하게 만든다.
+
+테스트:
+
+- rule only는 LM Studio 없이 통과
+- llm only는 mock LLM으로 deterministic response 검증
+- hybrid는 rule+LLM agreement metadata 생성
+- LLM 실패 시 fallback으로 성공
+- evidence 없는 LLM-only claim은 자동 승인되지 않음
+- conflict claim은 review_required가 true
+- UI payload에 mode/provider/model/base URL 포함
+
+관측성:
+
+- ExtractionLog에 mode/provider/model/base URL 저장
+- job metadata에 LLM call count, fallback count, skipped count 저장
+- Review/Page Analysis에서 agreement/fallback/conflict를 표시
+
+완료 기준:
+
+- local LM Studio가 꺼져 있어도 rule/hybrid fallback 테스트가 통과한다.
+- mock LLM 기반 테스트가 CI에서 안정적으로 돈다.
+- crawl/research 결과에서 어떤 extractor가 어떤 이유로 사용됐는지 추적 가능하다.
+
+## 최종 Acceptance Gate
+
+- [ ] UI에서 Rule only, Hybrid, LLM only, Compare 선택 가능
+- [ ] Hybrid mode가 rule baseline과 LLM semantic extraction을 병합
+- [ ] LLM 장애 시 rule fallback으로 job이 실패하지 않음
+- [ ] evidence 없는 LLM-only claim이 자동 validated로 들어가지 않음
+- [ ] rule+LLM agreement가 confidence에 반영됨
+- [ ] conflict claim이 review_required로 표시됨
+- [ ] Page Analysis에서 rule/LLM 비교 결과 확인 가능
+- [ ] Review에서 extraction method, agreement, confidence breakdown 확인 가능
+- [ ] `/process`는 후처리 RDF 변환 경로로 분리 유지
+
+## 권장 구현 순서
+
+1. Phase 7.1 Baseline Audit
+2. Phase 7.2 Extraction Mode Contract
+3. Phase 7.3 Hybrid Extractor
+4. Phase 7.4 Confidence And Validation Upgrade
+5. Phase 7.6 UI Controls
+6. Phase 7.5 Compare Mode
+7. Phase 7.7 Smart Routing Policy
+8. Phase 7.8 OntoCast `/process` Handoff
+9. Phase 7.9 Tests And Observability

ontology_platform/docs/phases/26_05_22_PHASE_INDEX_engine_respect_backup.md

@@ -0,0 +1,100 @@
+# PHASE INDEX - ontology_platform engine-respect roadmap
+
+?묒꽦?? 2026-05-19
+
+踰붿쐞: `ontology_platform` ?꾩슜. `crawler_platform`?€ ?대쾲 ?묒뾽 踰붿쐞?먯꽌 ?쒖쇅?쒕떎.
+
+湲곗? 臾몄꽌:
+- `ontology_platform/docs/?듯빀?ㅺ퀎??md`
+- `ontology_platform/README.md`
+- `ontology_platform/docs/phases/PHASE0_ACCEPTANCE_GATE.md`
+- `ontology_platform/docs/phases/PHASE1_NEXT_STEPS.md`
+- `ontology_platform/docs/phases/PHASE1_ACCEPTANCE_GATE.md`
+- `ontology_platform/docs/phases/PHASE2_ACCEPTANCE_GATE.md`
+
+?듭떖 ?먯튃:
+- OntoCast??Base ?붿쭊?쇰줈 議댁쨷?쒕떎.
+- vendored OntoCast 肄붿뼱???듯빀?ㅺ퀎?쒓? ?덉슜??踰붿쐞 ?몄뿉???섏젙?섏? ?딅뒗??
+- Trafilatura, Crawl4AI, Guardrails, Neo4j GraphRAG??吏곸젒 ?ш뎄?꾪븯吏€ ?딄퀬 ?뉗? adapter/facade濡?媛먯떬??
+- Firecrawl, OpenDeepResearcher 肄붾뱶???ы븿?섏? ?딅뒗??
+- Acceptance Gate瑜??듦낵?섍린 ???ㅼ쓬 ?듯빀?쇰줈 ?섏뼱媛€吏€ ?딅뒗??
+
+---
+
+PHASE 0. ?붿쭊 寃쎄퀎 媛먯궗 諛?Phase Gate 蹂듦뎄
+FILE: ./26_05_19_engine_respect_plan/phase_00_001_engine_boundary_gate.md
+
+1) ?꾩옱 `ont_platform` 紐⑤뱢??Base/Adapter/Draft/Excluded 梨낆엫?쇰줈 遺꾨쪟 [?꾨즺]
+2) Phase 0?먯꽌 誘몃옒 Phase ?섏〈?깆씠 import?섏뼱 ???쒖옉??源⑥? ?딅룄濡?寃뚯씠???뺣━ [?꾨즺]
+3) Phase 0 unit/integration 寃€利??덉감 怨좎젙 [?꾨즺]
+4) `PHASE0_ACCEPTANCE_GATE.md` 媛깆떊 湲곗? ?뺣━ [?꾨즺]
+
+---
+
+PHASE 1. Trafilatura 湲곕컲 URL/HTML ?낅젰 ?뺣젹
+FILE: ./26_05_19_engine_respect_plan/phase_01_001_trafilatura_ingestion.md
+
+1) `web_extractor.py`瑜?Trafilatura adapter 梨낆엫?쇰줈 ?뺣━ [?꾨즺]
+2) `SourceDocument`, `EvidenceSpan`, Content metadata ?€??寃쎄퀎 ?곌껐 [?꾨즺]
+3) `/process/url` ?먮뒗 ?숇벑??URL ?낅젰 API ?ㅺ퀎 [?꾨즺]
+4) ?쒓뎅??URL/HTML fixture 湲곕컲 異붿텧 ?뚯뒪?몄? dedup 湲곗? ?묒꽦 [?꾨즺]
+
+---
+
+PHASE 2. Candidate Storage 諛?Review 梨낆엫 寃쎄퀎
+FILE: ./26_05_19_engine_respect_plan/phase_02_001_candidate_review_boundary.md
+
+1) `storage/models.py`???꾨낫 紐⑤뜽???뺤떇 Review Queue 怨꾩빟?쇰줈 ?뺤젙 [?꾨즺]
+2) OntoCast 寃곌낵?€ lightweight extraction 寃곌낵???€??寃쎈줈 遺꾨━ [?꾨즺]
+3) ?뱀씤/諛섎젮/?먮룞?뱀씤 ?곹깭 ?꾩씠 洹쒖튃 ?뺤쓽 [?꾨즺]
+4) evidence ?녿뒗 ?꾨낫媛€ ?뺤젙 graph濡??ㅼ뼱媛€吏€ 紐삵븯寃?李⑤떒 [?꾨즺]
+
+---
+
+PHASE 3. Crawl4AI ?섏쭛 怨꾩링 諛?Job Orchestration
+FILE: ./26_05_19_engine_respect_plan/phase_03_001_crawl4ai_acquisition_jobs.md
+
+1) `crawl4ai_adapter.py`瑜??숈쟻/?€???섏쭛 adapter濡??쒗븳 [?꾨즺]
+2) crawler profile, robots policy, cache policy瑜??ㅼ젙 湲곕컲?쇰줈 遺꾨━ [?꾨즺]
+3) Job ?곹깭 紐⑤뜽怨?progress API/WebSocket 寃쎄퀎 ?뺣━ [?꾨즺]
+4) Trafilatura ?꾩쿂由ъ? SourceDocument ?€?μ쑝濡??곌껐 [?꾨즺]
+
+---
+
+PHASE 4. Guardrails Validation Gate
+FILE: ./26_05_19_engine_respect_plan/phase_04_001_guardrails_validation_gate.md
+
+1) `core/validation`??Pydantic lightweight?€ Guardrails facade濡?遺꾨━ [?꾨즺]
+2) OntoCast LLM 異쒕젰 ?섑븨 吏€?먯쓣 vendored ?섏젙 ?놁씠 ?곗꽑 ?ㅺ퀎 [?꾨즺]
+3) schema violation, endpoint missing, confidence range ?뚯뒪???묒꽦 [?꾨즺]
+4) Guard ?ㅽ뙣 寃곌낵瑜?candidate/review issue濡??€??[?꾨즺]
+
+---
+
+PHASE 5. Neo4j Projection 諛?GraphRAG 寃€??FILE: ./26_05_19_engine_respect_plan/phase_05_001_neo4j_projection_graphrag.md
+
+1) RDF/Fuseki瑜?canonical store, Neo4j瑜?projection/search store濡?怨좎젙 [?꾨즺]
+2) `core/graph` 湲곗〈 紐⑤뱢??projection/search adapter 梨낆엫?쇰줈 ?щ텇瑜?[?꾨즺]
+3) read-only Text2Cypher?€ vector/hybrid retriever API ?ㅺ퀎 [?꾨즺]
+4) provenance媛€ search result源뚯? ?댁뼱吏€??寃€利?湲곗? ?묒꽦 [?꾨즺]
+
+---
+
+PHASE 6. Maintenance Loop 諛??댁쁺 湲곕뒫 ?뺣━
+FILE: ./26_05_19_engine_respect_plan/phase_06_001_maintenance_loop_operations.md
+
+1) Knowledge Agent??肄붾뱶媛€ ?꾨땲???꾨＼?꾪듃/?뚰겕?뚮줈???⑦꽩留?李⑥슜 [?꾨즺]
+2) Analyst/Researcher/Curator/Auditor/Fixer/Advisor 梨낆엫 ?뺤쓽 [?꾨즺]
+3) `auth`, `audit`, `billing`, `realtime` 珥덉븞 紐⑤뱢???댁쁺 寃쎄퀎 ?뺣━ [?꾨즺]
+4) destructive fix???щ엺 ?뱀씤 寃뚯씠?몃? 諛섎뱶???듦낵?섎룄濡??ㅺ퀎 [?꾨즺]
+
+---
+
+PHASE 7. Hybrid Rule + LLM Extraction
+FILE: ./26_05_19_engine_respect_plan/phase_07_001_hybrid_rule_llm_extraction.md
+
+1) rule baseline, LLM extraction, fallback, validation, Review UI 흐름을 기준선으로 고정 [신규]
+2) `rule_only`, `llm_only`, `hybrid`, `compare` extraction mode 계약 정의 [신규]
+3) product backend에 명시적 HybridExtractor와 rule/LLM agreement metadata 추가 [신규]
+4) confidence breakdown에 rule agreement와 conflict/review 정책 반영 [신규]
+5) Crawl/Research UI에서 mode/provider/model/base URL 선택 지원 [신규]

ontology_platform/docs/phases/26_05_22_semantic_page_classification/phase_01_001_current_flow_boundary.md

@@ -0,0 +1,51 @@
+# PHASE 1. 현재 흐름 기준선 고정 및 영향 범위 정리
+
+## 목표
+
+`Semantic Page Understanding Layer`를 얹기 전에 현재 page classification 흐름과 page_type 문자열 의존 지점을 정확히 고정한다.
+
+## 기준 문서
+
+- `README.md`
+- `ontology_platform/README.md`
+- `ontology_platform/docs/semantic_page_classification_codex_spec.md`
+
+## 작업 범위
+
+1. `crawler_platform/app/core/crawler/page_classifier.py`의 현재 public API 확인
+   - `classify_page(...)`
+   - `normalize_page_type(...)`
+   - `should_analyze_page(...)`
+   - `relation_allowed_for_page_type(...)`
+   - `claim_allowed_for_context(...)`
+
+2. crawler 호출부 확인
+   - `site_crawler.py`
+   - `pipeline.py`
+   - `page_cleaner.py`
+   - `domain_discovery.py`
+   - `relevance_engine.py`
+
+3. extractor 연결 확인
+   - `ExtractionPageContext.page_type`
+   - `HybridExtractor.llm_skip_reason(...)`
+   - `LLM_PAGE_TYPES`, `SKIP_LLM_PAGE_TYPES`, `RULE_ONLY_PAGE_TYPES`
+   - validation metadata의 `page_type`
+
+4. 기존 page_type 문자열 기대 코드 목록화
+   - config `analyze_page_types`
+   - ontology relation rule `allowed_page_types`
+   - frontend display
+   - tests
+   - adapters
+
+## 수정 금지
+
+- 이 phase에서는 구현 변경을 하지 않는다.
+- 기존 page_type 문자열 의미를 변경하지 않는다.
+- 기존 테스트 기대값을 바꾸지 않는다.
+
+## 완료 기준
+
+- 현재 흐름과 의존 지점이 다음 phase 구현의 기준선으로 정리되어 있어야 한다.
+- 신규 semantic API를 추가할 때 깨뜨리면 안 되는 legacy contract가 명확해야 한다.

ontology_platform/docs/phases/26_05_22_semantic_page_classification/phase_01_current_flow_boundary_report.md

@@ -0,0 +1,276 @@
+# Phase 1 Current Flow Boundary Report
+
+작성일: 2026-05-22
+
+범위: Semantic Page Classification Layer 구현 전, 현재 `page_type` 문자열 흐름과 의존 지점을 고정한다.
+
+## 1. page_classifier.py public API
+
+파일: `ontology_platform/crawler_platform/app/core/crawler/page_classifier.py`
+
+현재 public API와 contract:
+
+- `classify_page(url, title=None, text="", html=None, source_zones=None) -> str`
+  - 단일 문자열 page_type을 반환한다.
+  - 호출부는 반환값이 `PageClassificationResult`가 아니라 `str`이라고 가정한다.
+- `normalize_page_type(value: str | None) -> str`
+  - 짧은 alias를 legacy page_type 문자열로 변환한다.
+  - 현재 alias 예: `product -> ProductPage`, `listing/category -> CategoryPage`, `community/board -> BoardPage`.
+- `should_analyze_page(page_type: str, analyze_page_types: set[str]) -> bool`
+  - page_type과 allowlist를 각각 normalize한 뒤 포함 여부만 본다.
+- `relation_allowed_for_page_type(page_type, predicate) -> bool`
+  - `ProductPage`는 product detail predicate를 모두 허용한다.
+  - `UnknownPage`, `SearchPage`, `CategoryPage`, `BoardPage`는 non-merge page로 간주한다.
+  - `BrandStoryPage`, `NoticePage`, `EventPage`, `PromotionPage`는 content page로 간주한다.
+- `claim_allowed_for_context(page_type, predicate, zone_type) -> bool`
+  - page_type relation policy와 source zone policy를 함께 적용한다.
+
+현재 legacy page_type 문자열:
+
+- `ProductPage`
+- `CategoryPage`
+- `SearchPage`
+- `BoardPage`
+- `NoticePage`
+- `BrandStoryPage`
+- `PromotionPage`
+- `EventPage`
+- `ReviewPage`
+- `UnknownPage`
+- 보조/외부 상태 문자열: `unknown`, `external`, `entity`
+
+Phase 2 이후에도 기존 `classify_page(...) -> str` contract는 유지해야 한다. 신규 semantic result는 별도 API로 추가하거나 compatibility wrapper 뒤에 두어야 한다.
+
+## 2. crawler 호출 흐름
+
+### site_crawler.py
+
+파일: `ontology_platform/crawler_platform/app/core/crawler/site_crawler.py`
+
+흐름:
+
+1. fetcher가 HTML을 가져온다.
+2. parser가 `analysis_html`을 parse한다.
+3. `classify_page(final_url or url, title, raw_text or text, analysis_html, source_zones)`를 호출한다.
+4. 반환된 `page_type` 문자열을 page metadata와 `SiteCrawlPageResult`에 저장한다.
+5. `should_analyze_page(page_type, analyze_page_types)`가 true일 때만 `ExtractionPageContext`를 만들고 extractor를 실행한다.
+6. context의 `page_type` 문자열이 HybridExtractor, validation, repository 저장까지 이어진다.
+
+현재 기본 analyze allowlist:
+
+```txt
+ProductPage
+BrandStoryPage
+ReviewPage
+```
+
+따라서 현재 기본 흐름에서는 `CategoryPage`, `SearchPage`, `BoardPage`, `UnknownPage`가 extractor 실행에서 빠진다.
+
+### pipeline.py
+
+파일: `ontology_platform/crawler_platform/app/core/crawler/pipeline.py`
+
+흐름:
+
+1. 단일 URL fetch/parse 후 `classify_page(...)`를 호출한다.
+2. `metadata["page_type"]`에 문자열을 저장한다.
+3. fetch/extraction 실패가 아니면 `ExtractionPageContext.page_type`에 같은 문자열을 넣고 extractor를 실행한다.
+
+주의:
+
+- `pipeline.py`에는 `should_analyze_page()` gate가 없다.
+- 단일 URL pipeline은 현재 모든 정상 parse page를 extractor로 보낸다.
+
+### page_cleaner.py / html_cleaner.py / plugins.py
+
+파일:
+
+- `ontology_platform/crawler_platform/app/core/crawler/page_cleaner.py`
+- `ontology_platform/crawler_platform/app/core/crawler/html_cleaner.py`
+- `ontology_platform/crawler_platform/app/core/crawler/plugins.py`
+
+흐름:
+
+- `PageCleaner.clean()`은 page_type이 없으면 내부에서 `classify_page()`를 호출한다.
+- `ZONE_PRIORITY_BY_PAGE_TYPE`가 legacy page_type 문자열에 의존한다.
+- `ProductPage`, `BrandStoryPage`, `NoticePage`, `BoardPage`, `EventPage`, `PromotionPage`, `CategoryPage`별로 source zone 우선순위가 다르다.
+
+Phase 6에서 semantic type을 추가할 때 `ProductDetailPage -> ProductPage`, `CategoryListingPage -> CategoryPage` 같은 zone compatibility가 필요하다.
+
+## 3. research/discovery 호출 흐름
+
+### graph_research_loop.py
+
+파일: `ontology_platform/crawler_platform/app/core/research/graph_research_loop.py`
+
+흐름:
+
+1. 기본 analyze allowlist는 `ProductPage`, `BrandStoryPage`, `ReviewPage`다.
+2. explored page에서 `classify_page(...)`를 호출한다.
+3. metadata에 `page_type`을 저장한다.
+4. relevance score와 `should_analyze_page(page_type, analyze_page_types)`를 모두 통과해야 extractor를 실행한다.
+5. link 후보 metadata에도 relevance engine이 산출한 `score.page_type`을 저장한다.
+
+### relevance_engine.py
+
+파일: `ontology_platform/crawler_platform/app/core/research/relevance_engine.py`
+
+흐름:
+
+- `score_url()` 내부에서 `classify_page(...)`를 호출한다.
+- `HIGH_VALUE_PAGE_TYPES` 점수표가 legacy page_type 문자열에 의존한다.
+- 신규 semantic page type이 들어오면 점수표 또는 legacy normalization이 필요하다.
+
+현재 주요 점수:
+
+- `ProductPage`: 0.95
+- `BrandStoryPage`: 0.88
+- `ReviewPage`: 0.82
+- `CategoryPage`: 0.34
+- `SearchPage`: 0.22
+- `BoardPage`: 0.18
+- `UnknownPage`: 0.25
+
+### domain_discovery.py
+
+파일: `ontology_platform/crawler_platform/app/core/ontology/domain_discovery.py`
+
+흐름:
+
+- discovery job에서 `classify_page(...)`를 호출한다.
+- `mine_schema_candidates(..., page_type=page_type)`로 page_type을 evidence metadata에 넣는다.
+- `entity_type_from_page_type()`가 legacy mapping을 사용한다.
+
+현재 mapping:
+
+- `ProductPage -> Product`
+- `CategoryPage -> Category`
+- `BrandStoryPage -> Brand`
+- `NoticePage -> Notice`
+- `PromotionPage -> Promotion`
+- `ReviewPage -> Review`
+- `BoardPage -> Article`
+
+## 4. Extractor / HybridExtractor 연결 흐름
+
+### ExtractionPageContext
+
+파일: `ontology_platform/crawler_platform/app/core/extractor/base.py`
+
+현재 `ExtractionPageContext.page_type: str`는 필수 문자열 필드다. `to_payload()`도 `page_type`을 그대로 내보낸다.
+
+Phase 6에서 이 필드는 유지해야 하며, semantic 정보는 `metadata`에 추가하는 방식이 가장 안전하다.
+
+### HybridExtractor
+
+파일: `ontology_platform/crawler_platform/app/core/extractor/hybrid.py`
+
+현재 LLM routing 상수:
+
+- `LLM_PAGE_TYPES = {"ProductPage", "BrandStoryPage", "ReviewPage"}`
+- `SKIP_LLM_PAGE_TYPES = {"CategoryPage", "SearchPage", "ListingPage"}`
+- `RULE_ONLY_PAGE_TYPES = {"BoardPage", "CommunityPage", "UnknownPage"}`
+
+현재 `llm_skip_reason()` 판단 순서:
+
+1. mode가 `hybrid`가 아니거나 context가 없으면 skip 판단 없음.
+2. page_type이 `SKIP_LLM_PAGE_TYPES`면 LLM skip.
+3. page_type이 `RULE_ONLY_PAGE_TYPES`면 rule-only.
+4. page_type이 `LLM_PAGE_TYPES`에 없으면 LLM allowlist 밖으로 skip.
+5. clean text 길이가 너무 짧거나 길면 skip.
+
+Phase 5/6에서 `LLMPolicy`가 있으면 이를 우선하고, 없으면 이 legacy fallback을 유지해야 한다.
+
+### ai_provider.py
+
+파일: `ontology_platform/crawler_platform/app/core/extractor/ai_provider.py`
+
+LLM prompt에 page_type semantics가 직접 들어간다.
+
+현재 prompt contract:
+
+- `ProductPage`는 product detail claim 가능
+- `CategoryPage`는 detailed product claim 제한
+- `BrandStoryPage`는 product price/note claim 제한
+- `UnknownPage`는 ontology claim을 반환하지 않도록 지시
+
+신규 semantic type을 추가할 때 prompt가 `ProductDetailPage`, `CategoryListingPage`를 이해하도록 하거나, prompt에는 legacy page_type을 계속 넘기는 compatibility가 필요하다.
+
+### validation.py
+
+파일: `ontology_platform/crawler_platform/app/core/extractor/validation.py`
+
+흐름:
+
+- `attach_page_context()`가 entity/claim metadata에 `page_type`을 저장한다.
+- `claim_status_for_bundle()`은 `UnknownPage`, `SearchPage`, `CategoryPage`, `BoardPage`를 candidate claim으로 낮춘다.
+- `invalid_claim_reason()`은 relation schema compatibility에 `page_type`을 넘긴다.
+
+신규 semantic type은 validation status와 relation schema의 allowed page type 검증에 영향을 준다.
+
+## 5. 기존 page_type 문자열 기대 지점
+
+### API request/response
+
+파일: `ontology_platform/crawler_platform/app/api/routes.py`
+
+- 여러 request model의 기본 `analyze_page_types`가 `["ProductPage", "BrandStoryPage", "ReviewPage"]`다.
+- relation/quality 관련 request에 `allowed_page_types`가 있다.
+- API response 및 progress payload에서 `page_type` 문자열을 그대로 노출한다.
+
+### Config
+
+파일: `ontology_platform/configs/perfume_subscription.yaml`
+
+- `analyze_page_types` 또는 ontology rule에서 legacy page type이 쓰일 수 있다.
+- target entity/ontology에는 `ProductPage`, `BrandStoryPage`, `ListingPage`, `PromotionPage`, `ReviewPage` 같은 page entity type이 포함되어 있다.
+
+### Adapters
+
+파일: `ontology_platform/crawler_platform/app/adapters/ecommerce/perfume.py`
+
+- relation rules가 `page_types={"ProductPage"}` 또는 `{"ProductPage", "ReviewPage"}` 같은 legacy set에 의존한다.
+
+### Ontology relation schema
+
+파일: `ontology_platform/crawler_platform/app/core/ontology/relation_schema.py`
+
+- configured relation rule의 `allowed_page_types`/`page_types`를 set으로 읽는다.
+- `relation_schema_compatible()`은 `page_type not in rule.page_types`면 reject한다.
+- semantic type 도입 시 legacy alias expansion 없이는 기존 relation rules가 거부될 수 있다.
+
+### Frontend
+
+주요 파일:
+
+- `ontology_platform/web/frontend/src/pages/CrawlPage.tsx`
+- `ontology_platform/web/frontend/src/pages/ReviewPage.tsx`
+- `ontology_platform/web/frontend/src/pages/BuildPipelinePage.tsx`
+- `ontology_platform/web/frontend/src/pages/QualityInspectorPage.tsx`
+- `ontology_platform/web/frontend/src/lib/api/crawl.ts`
+- `ontology_platform/web/frontend/src/lib/api/claims.ts`
+- `ontology_platform/web/frontend/src/lib/api/platform.ts`
+- legacy JS files under `web/frontend/src/legacy`
+
+현재 frontend는 `page_type`을 optional string으로 표시하거나, allowed page types mismatch 검사에 사용한다.
+
+### Tests
+
+주요 테스트:
+
+- `ontology_platform/tests/unit/test_phase7_hybrid_extraction.py`
+  - `CategoryPage`가 LLM skip 되는 기존 behavior를 검증한다.
+- integration/e2e tests는 API response의 `page_type` 문자열 contract에 간접 의존한다.
+
+Phase 8에서 semantic classifier 전용 테스트를 추가하되, 기존 문자열 contract 회귀 테스트도 유지해야 한다.
+
+## 6. Phase 2 이후 유지해야 할 legacy contract
+
+1. `classify_page(...)`는 계속 `str`을 반환해야 한다.
+2. 신규 API는 `classify_page_semantic(...) -> PageClassificationResult`처럼 분리하는 편이 안전하다.
+3. `normalize_page_type()`은 `PageClassificationResult`도 받을 수 있게 확장하되, 기존 string 입력 결과를 바꾸면 안 된다.
+4. `should_analyze_page(str, set)` 기존 호출은 계속 동작해야 한다.
+5. API/metadata의 `page_type` 필드는 legacy string으로 유지하고, semantic 결과는 별도 `page_classification` payload로 저장한다.
+6. `ExtractionPageContext.page_type`은 legacy string으로 유지하고, `analyze_strategy`/`llm_policy`/evidence는 `metadata`로 전달한다.
+7. relation schema, adapter page_types, frontend allowed page type 검증에는 legacy/semantic alias compatibility가 필요하다.
+8. `CategoryPage`, `SearchPage`, `BoardPage`, `UnknownPage`의 기존 skip/rule-only 동작은 Phase 5에서 strategy 기반으로 확장하되, LLM 호출이 늘지 않도록 `LLMPolicy`를 우선한다.

ontology_platform/docs/phases/26_05_22_semantic_page_classification/phase_02_001_taxonomy_result_model.md

@@ -0,0 +1,53 @@
+# PHASE 2. Taxonomy와 Classification Result 모델 추가
+
+## 목표
+
+기존 단일 문자열 page_type을 대체하지 않고, 그 위에 semantic classification result 모델을 추가한다.
+
+## 작업 범위
+
+1. 신규 taxonomy 모듈 추가
+   - 권장 위치: `crawler_platform/app/core/crawler/page_type_taxonomy.py`
+   - 기존 프로젝트 구조와 import 경계를 우선한다.
+
+2. 다음 상수 또는 enum 정의
+   - `PageDomain`
+   - `PageArchetype`
+   - `PageType`
+   - `EntityType`
+   - `ActionIntent`
+   - `GraphRole`
+   - `AnalyzeStrategy`
+   - `LLMPolicy`
+
+3. classification result 모델 추가
+   - `EvidenceItem`
+   - `PageClassificationResult`
+
+4. legacy compatibility helper 추가
+   - `normalize_page_type(result_or_page_type)`
+   - `get_legacy_page_type(result_or_page_type)`
+   - `classify_page_semantic(...)`
+   - 기존 `classify_page(...) -> str` 유지
+
+5. 기존 page type alias 유지
+   - `ProductPage`
+   - `CategoryPage`
+   - `SearchPage`
+   - `BoardPage`
+   - `NoticePage`
+   - `BrandStoryPage`
+   - `PromotionPage`
+   - `UnknownPage`
+   - `ReviewPage`
+
+## 수정 금지
+
+- 기존 `classify_page(...)` 호출부를 한 번에 모두 semantic result 기반으로 바꾸지 않는다.
+- legacy 문자열을 제거하지 않는다.
+
+## 완료 기준
+
+- 기존 호출부가 string page_type을 그대로 받을 수 있어야 한다.
+- semantic result API를 신규 테스트에서 직접 호출할 수 있어야 한다.
+- legacy alias mapping이 테스트로 보호되어야 한다.

ontology_platform/docs/phases/26_05_22_semantic_page_classification/phase_03_001_signal_extraction_layer.md

@@ -0,0 +1,93 @@
+# PHASE 3. Raw Snapshot 및 Signal Extraction 레이어 추가
+
+## 목표
+
+URL substring 중심 분류를 줄이고, HTML/DOM/metadata/link/form/text 기반 signal을 별도 레이어에서 추출한다.
+
+## 작업 범위
+
+1. `RawPageSnapshot` 모델 추가
+
+   * `url`, `final_url`, `status_code`, `content_type`
+   * `title`, `text`, `html`, `rendered_html`
+   * metadata, structured data, headings, links, images, forms, buttons, inputs, tables
+   * source_zones, screenshot_path
+
+2. `PageSignals` 모델 추가
+
+   * structured data signals
+   * commerce signals
+   * listing signals
+   * editorial signals
+   * community signals
+   * knowledge/docs signals
+   * corporate/legal signals
+   * transaction/protected signals
+   * graph/link signals
+   * text/layout keyword signals
+   * visual/layout block signals
+   * external collector signals
+
+3. signal extractor 추가
+
+   * 권장 위치: `crawler_platform/app/core/crawler/page_signal_extractor.py`
+   * BeautifulSoup 사용 가능 시 DOM parsing
+   * BeautifulSoup 미설치/HTML 깨짐 시 regex/text fallback
+
+4. 다국어 확장 고려
+
+   * 한국어/영어 키워드 dictionary를 분리 가능한 구조로 둔다.
+   * 인코딩 깨짐이 있어도 예외 없이 동작한다.
+
+5. visual/layout signal 세부화 고려
+
+   * screenshot 기반 정밀 분석은 이번 phase의 필수 구현 범위가 아니지만, 향후 visual block classification을 붙일 수 있도록 signal 구조를 열어둔다.
+   * DOM class/id/role/aria/heading 구조와 반복 레이아웃을 통해 가능한 범위에서 visual/layout block 후보를 추출한다.
+   * visual/layout block signal 예시는 다음과 같다.
+
+     * hero block
+     * product card grid
+     * article body block
+     * left filter sidebar
+     * top navigation
+     * footer navigation
+     * sticky buy box
+     * review/comment block
+     * FAQ accordion
+     * media player area
+     * map area
+     * calendar/availability grid
+     * dashboard card grid
+     * form wizard / stepper
+     * pricing table
+     * comparison table
+   * 초기 구현은 실제 computer vision까지 요구하지 않는다.
+   * 다만 `PageSignals`에는 visual/layout 후보를 담을 수 있는 필드를 둔다.
+   * 예시 필드:
+
+     * `layout_blocks: list[str]`
+     * `has_hero_block: bool`
+     * `has_card_grid: bool`
+     * `has_filter_sidebar: bool`
+     * `has_sticky_action_box: bool`
+     * `has_media_player_area: bool`
+     * `has_map_area: bool`
+     * `has_calendar_grid: bool`
+     * `has_pricing_table: bool`
+     * `has_comparison_table: bool`
+   * visual/layout signal은 page type을 단독 확정하지 않고, evidence scoring의 보조 신호로 사용한다.
+
+## 수정 금지
+
+* signal extractor에서 page_type을 확정 반환하지 않는다.
+* 이 phase에서 extractor나 crawler의 분석 여부 정책을 바꾸지 않는다.
+* Crawl4AI, Firecrawl, Trafilatura 같은 외부 도구에 핵심 classifier가 강하게 종속되도록 만들지 않는다.
+* screenshot 또는 visual analysis가 없다는 이유로 기본 signal extraction이 실패하면 안 된다.
+
+## 완료 기준
+
+* HTML이 비어도 `PageSignals`가 생성되어야 한다.
+* JSON-LD/OpenGraph/form/input/button/link/text signal이 evidence scorer에서 사용할 수 있는 형태로 정리되어야 한다.
+* 외부 수집 도구 결과를 `RawPageSnapshot`에 매핑할 수 있는 구조 또는 adapter hook이 있어야 한다.
+* visual/layout block 후보를 담을 수 있는 `PageSignals` 필드가 있어야 한다.
+* visual/layout signal이 없어도 기존 signal extraction과 scoring 흐름은 정상 동작해야 한다.

ontology_platform/docs/phases/26_05_22_semantic_page_classification/phase_04_001_evidence_scoring_classifier.md

@@ -0,0 +1,99 @@
+# PHASE 4. Evidence Scoring 기반 Semantic Classification 구현
+
+## 목표
+
+단일 if-return 방식이 아니라 signal별 evidence weight를 합산해 semantic page type을 결정한다.
+
+## 작업 범위
+
+1. `page_type_scorer.py` 추가
+
+   * score accumulator
+   * evidence item 생성 helper
+   * normalize 및 confidence 계산
+   * alternatives 산출
+
+2. 최소 구현 page type
+
+   * `ProductDetailPage`
+   * `CategoryListingPage`
+   * `SearchResultsPage`
+   * `ArticlePage`
+   * `BlogPostPage`
+   * `QAPage`
+   * `FAQPage`
+   * `ForumBoardPage`
+   * `ForumThreadPage`
+   * `BrandStoryPage`
+   * `AboutPage`
+   * `ContactPage`
+   * `DocumentationPage`
+   * `APIReferencePage`
+   * `JobPostingPage`
+   * `PricingPage`
+   * `LoginPage`
+   * `CheckoutPage`
+   * `PaymentPage`
+   * `TermsPage`
+   * `PrivacyPolicyPage`
+   * `SitemapPage`
+   * `RSSFeedPage`
+   * `ErrorPage`
+   * `AccessDeniedPage`
+   * `CaptchaPage`
+   * `UnknownPage`
+
+3. 범용 taxonomy 확장 기준
+
+   * 위의 최소 구현 page type은 1차 구현 범위로 본다.
+   * 장기 목표는 인터넷에 존재하는 다양한 페이지를 포괄할 수 있는 전체 taxonomy catalog를 유지하는 것이다.
+   * 따라서 `page_type_scorer.py`와 taxonomy 정의는 아래 계열을 나중에 확장할 수 있는 구조로 작성한다.
+
+     * Site / Navigation
+     * Commerce / Marketplace
+     * Editorial / Article
+     * Community / UGC
+     * Knowledge / Documentation
+     * Corporate / Organization
+     * Local / Place / Travel / Real Estate
+     * Education / Learning
+     * Jobs / Career
+     * Media / Entertainment
+     * Software / SaaS / App
+     * Finance / Legal / Government
+     * Healthcare / Medical
+     * Transaction / Account / Protected
+     * System / Technical / Machine-readable
+   * 현재 phase에서는 위 전체 계열을 모두 scoring 구현하지 않아도 된다.
+   * 다만 enum, alias, mapping, scorer 구조는 특정 몇 개 타입에 고정하지 말고, 전체 taxonomy catalog가 추가되어도 깨지지 않도록 확장 가능해야 한다.
+   * Phase 4의 최소 구현 page type은 1차 안정화 대상이며, 전체 taxonomy catalog는 별도 taxonomy 문서 또는 후속 phase에서 보강한다.
+   * `UnknownPage`는 전체 taxonomy에 아직 포함되지 않은 신규 페이지 패턴을 발견하기 위한 후보로 유지한다.
+
+4. result enrichment
+
+   * `domain`
+   * `archetype`
+   * `main_entity_type`
+   * `action_intents`
+   * `graph_roles`
+   * `confidence`
+   * `alternatives`
+   * `evidence`
+
+5. low confidence 처리
+
+   * threshold 아래는 `UnknownPage`
+   * alternatives/evidence는 유지
+
+## 수정 금지
+
+* URL 문자열 조건만 추가해서 바로 return하지 않는다.
+* Product/Category/Search/Board만 처리하는 구조로 고정하지 않는다.
+* 1차 최소 구현 page type만을 전체 taxonomy의 전부로 간주하지 않는다.
+
+## 완료 기준
+
+* classification result가 항상 evidence를 포함해야 한다.
+* 보호 페이지가 commerce/detail page로 오분류되지 않아야 한다.
+* category/search/board 계열은 skip 여부가 아니라 semantic type과 graph role이 남아야 한다.
+* 최소 구현 page type은 동작해야 하며, 전체 taxonomy catalog를 후속 확장할 수 있는 구조여야 한다.

ontology_platform/docs/phases/26_05_22_semantic_page_classification/phase_05_001_analysis_llm_policy.md

@@ -0,0 +1,45 @@
+# PHASE 5. Analyze Strategy 및 LLM Policy 분리
+
+## 목표
+
+분석 여부를 단순 boolean allowlist에서 page type별 strategy와 LLM policy로 분리한다.
+
+## 작업 범위
+
+1. `page_analysis_policy.py` 추가
+   - `decide_analyze_strategy(result)`
+   - `decide_llm_policy(result)`
+   - `is_protected_strategy(strategy)`
+   - `is_noise_strategy(strategy)`
+
+2. strategy mapping
+   - `ProductDetailPage`, `ArticlePage`, `FAQPage`, `QAPage`, `BrandStoryPage` -> `AnalyzeFull`
+   - `CategoryListingPage`, `ForumBoardPage` -> `AnalyzeRelationsOnly`
+   - `SearchResultsPage`, `SitemapPage` -> `AnalyzeDiscoveryOnly`
+   - `TermsPage`, `PrivacyPolicyPage` -> `AnalyzeDocumentOnly`
+   - `LoginPage`, `CheckoutPage`, `PaymentPage`, `CaptchaPage`, `AccessDeniedPage` -> `SkipProtected`
+   - `ErrorPage`, `NotFoundPage` -> `SkipNoise`
+   - `UnknownPage` -> `AnalyzeMetadataOnly`
+
+3. LLM policy mapping
+   - full content page -> `LLMFull` 또는 `LLMLight`
+   - listing/search/board -> `LLMForAmbiguityOnly` 또는 `RuleOnly`
+   - sitemap/rss/robots/protected -> `RuleOnly`, `NoLLM`, 또는 `Skip`
+
+4. compatibility
+   - `should_analyze_page(result_or_page_type, analyze_page_types=None)`
+   - 기존 allowlist가 들어오면 legacy behavior를 최대한 유지하되 protected/noise는 안전하게 skip
+   - semantic result가 들어오면 strategy 우선
+
+## 수정 금지
+
+- CategoryPage/SearchPage/BoardPage를 무조건 skip하지 않는다.
+- protected page에서 extractor/LLM이 개인정보를 추출하도록 두지 않는다.
+
+## 완료 기준
+
+- `CategoryListingPage.should_analyze == True`
+- `SearchResultsPage.should_analyze == True`
+- `ForumBoardPage.should_analyze == True`
+- `LoginPage`, `CheckoutPage`, `PaymentPage`는 `should_analyze == False`
+- LLM 호출이 기존보다 불필요하게 증가하지 않도록 policy 테스트가 있어야 한다.

ontology_platform/docs/phases/26_05_22_semantic_page_classification/phase_06_001_pipeline_integration.md

@@ -0,0 +1,45 @@
+# PHASE 6. Crawler, Cleaner, Extractor, Discovery/Relevance 통합
+
+## 목표
+
+Semantic classification result를 crawler와 extractor 흐름에 연결하되, 기존 page_type 문자열 기반 contract는 유지한다.
+
+## 작업 범위
+
+1. `site_crawler.py`
+   - `classify_page_semantic(...)` 호출
+   - 기존 `page_type` 필드는 legacy string으로 유지
+   - metadata에 `page_classification` 저장
+   - `should_analyze_page(result, analyze_page_types)` 사용
+
+2. `pipeline.py`
+   - 단일 URL 처리에서도 semantic payload 저장
+   - `ExtractionPageContext.metadata`에 strategy/policy 전달
+
+3. `ExtractionPageContext`
+   - 기존 `page_type: str`는 유지
+   - metadata 기반 `analyze_strategy`, `llm_policy`, evidence payload 전달
+
+4. `HybridExtractor`
+   - metadata의 `llm_policy`를 우선 사용
+   - 없으면 기존 `LLM_PAGE_TYPES`, `SKIP_LLM_PAGE_TYPES`, `RULE_ONLY_PAGE_TYPES` fallback
+   - protected/skip policy는 LLM 호출 금지
+
+5. compatibility update
+   - `page_cleaner.py` zone priority에 semantic aliases 추가
+   - `domain_discovery.py` page_type entity mapping에 semantic aliases 추가
+   - `relevance_engine.py` high value score mapping에 semantic aliases 추가
+   - relation schema allowed page type 검증에서 legacy/semantic alias 고려
+
+## 수정 금지
+
+- extractor 전체를 새 구조로 갈아엎지 않는다.
+- DB schema 변경을 필수로 만들지 않는다.
+- frontend 표시용 기존 `page_type` 필드를 제거하지 않는다.
+
+## 완료 기준
+
+- 기존 API response의 `page_type`은 문자열로 유지된다.
+- semantic classification payload가 metadata에 남는다.
+- HybridExtractor가 LLMPolicy에 따라 LLM 호출을 줄일 수 있다.
+- 기존 page_type 문자열 기반 relation rule이 신규 semantic type 때문에 깨지지 않는다.

ontology_platform/docs/phases/26_05_22_semantic_page_classification/phase_07_001_unknown_pattern_storage.md

@@ -0,0 +1,47 @@
+# PHASE 7. Unknown Pattern 저장 기반 추가
+
+## 목표
+
+`UnknownPage`를 분석 실패나 폐기 대상으로 두지 않고, taxonomy 확장 후보로 저장 가능한 evidence payload를 만든다.
+
+## 작업 범위
+
+1. unknown payload 정의
+   - url
+   - title
+   - text sample
+   - html fingerprint
+   - dom fingerprint
+   - schema types
+   - link pattern summary
+   - button labels
+   - forms summary
+   - top keywords
+   - alternatives
+   - evidence
+
+2. fingerprint hook
+   - text fingerprint
+   - html/dom fingerprint
+   - link pattern fingerprint
+
+3. 저장 위치
+   - 초기 구현은 `metadata_json["page_classification"]["unknown_pattern"]`
+   - DB migration 없이 시작
+
+4. 향후 확장 hook
+   - embedding input 생성 함수
+   - cluster candidate payload 생성 함수
+   - 실제 clustering은 이번 phase 범위에서 제외
+
+## 수정 금지
+
+- UnknownPage를 무조건 extractor 대상에서 제외하지 않는다.
+- low confidence 결과의 evidence와 alternatives를 버리지 않는다.
+- 이 phase에서 clustering 알고리즘을 새로 도입하지 않는다.
+
+## 완료 기준
+
+- UnknownPage result가 evidence를 가진다.
+- metadata에 unknown pattern summary가 저장 가능하다.
+- 향후 clustering/관리자 검토 UI로 넘길 수 있는 구조다.

ontology_platform/docs/phases/26_05_22_semantic_page_classification/phase_08_001_tests_regression.md

@@ -0,0 +1,59 @@
+# PHASE 8. 테스트 Fixture 및 회귀 검증
+
+## 목표
+
+Semantic Page Understanding Layer가 기존 흐름을 깨지 않고, page type별 strategy와 LLM policy를 정확히 산출하는지 검증한다.
+
+## 작업 범위
+
+1. fixture 추가
+   - `tests/fixtures/pages/product_detail.html`
+   - `tests/fixtures/pages/category_listing.html`
+   - `tests/fixtures/pages/search_results.html`
+   - `tests/fixtures/pages/article.html`
+   - `tests/fixtures/pages/qapage.html`
+   - `tests/fixtures/pages/faq.html`
+   - `tests/fixtures/pages/forum_thread.html`
+   - `tests/fixtures/pages/documentation.html`
+   - `tests/fixtures/pages/job_posting.html`
+   - `tests/fixtures/pages/login.html`
+   - `tests/fixtures/pages/checkout.html`
+   - `tests/fixtures/pages/terms.html`
+   - `tests/fixtures/pages/sitemap.xml`
+   - `tests/fixtures/pages/unknown.html`
+
+2. unit test 추가
+   - primary_page_type 확인
+   - confidence 최소 기준 확인
+   - evidence non-empty 확인
+   - analyze_strategy 확인
+   - llm_policy 확인
+   - protected page skip 확인
+   - UnknownPage 예외 없는 처리 확인
+
+3. compatibility test 추가
+   - 기존 `classify_page(...) -> str`
+   - 신규 `classify_page_semantic(...) -> PageClassificationResult`
+   - `should_analyze_page(str, set)`
+   - `should_analyze_page(result, set)`
+   - legacy aliases
+
+4. extractor policy 회귀 테스트
+   - HybridExtractor가 metadata `llm_policy`를 우선 사용
+   - `Skip`, `NoLLM`, `RuleOnly`에서 LLM 호출 금지
+   - 기존 legacy page_type fallback 유지
+
+5. 회귀 테스트 실행
+   - 기본 명령: `pytest`
+   - 필요 시 변경 범위 우선: `pytest tests/unit`
+
+## 수정 금지
+
+- 기존 테스트 기대값을 불필요하게 변경하지 않는다.
+- LLM live 호출이 필요한 테스트를 기본 회귀 테스트에 포함하지 않는다.
+
+## 완료 기준
+
+- 새 semantic classifier 테스트가 통과한다.
+- 기존 unit/integration 테스트가 통과한다.
+- 테스트 결과와 미실행 사유가 작업 완료 보고에 명확히 기록된다.

ontology_platform/docs/phases/PHASE0_ACCEPTANCE_GATE.md

@@ -1,132 +0,0 @@
-# Phase 0 — Acceptance Gate 결과
-
-본 문서는 통합설계서 §5 Phase 0의 Acceptance Gate를 객관적으로 점검한 결과다. Phase 1 진입 전에 모든 체크가 통과되어야 한다.
-
-## 결과 요약
-
-| # | Acceptance Gate 항목 | 상태 | 검증 방법 |
-|---|---|---|---|
-| 1 | 단일 PDF/JSON 입력 → ontology TTL + facts TTL이 filesystem에 생성됨 | ⚠️ **e2e 검증 대기** (로컬 LLM/API 키 필요) | `tests/e2e/test_phase0_full_pipeline.py` |
-| 2 | `/health`, `/info`, `/process` (FastAPI) 정상 동작 | ✅ **통합 테스트 11/11 통과** (2026-05-19) | `tests/integration/test_api_smoke.py` |
-| 3 | BudgetTracker가 LLM call/triple count를 정확히 기록 | ⚠️ **e2e 검증 대기** (mock 검증은 통합 테스트로 통과) | e2e 테스트가 실제 검증 |
-| 4 | LangGraph 워크플로우 (CONVERT→CHUNK→...→SERIALIZE) 전 노드 traceable | ✅ **OntoCast 원본 워크플로우 무수정 채택** | `vendored/ontocast/ontocast/stategraph/` 그대로 사용 |
-
-추가로 **단위 테스트 16/16 통과** (test_convert_document 7, test_platform_config 5, test_select_ontology 4).
-자동 검증 기준으로는 **unit + integration 27/27 통과**가 현재 Phase 0 기본선이다.
-
-**현재 진척 (2026-05-19)**:
-- Python 3.14.5 `.venv` 환경에서 unit + integration 27/27 통과
-- `python-multipart`를 Phase 0 FastAPI multipart upload 필수 의존성으로 추가
-- Phase 0 production app에서 Phase 1 Trafilatura route가 기본 mount되지 않도록 lazy phase route gate 적용
-- `pip install -e ".[dev]"` 또는 동등한 의존성 설치 필요
-- `pip install -e vendored/ontocast` 로 OntoCast 의존성 설치 완료
-- 패키지 이름 충돌 수정: `platform/` → `ont_platform/` (Python 내장 `platform` 모듈과 충돌)
-- **남은 작업**: e2e 테스트 (Acceptance Gate #1, #3) 실행 — 로컬 Ollama 또는 OpenAI 키 필요
-
-## 다음 작업자가 실행할 검증 절차
-
-### 1) 환경 준비
-
-```powershell
-# Python 3.12+ 설치 (예: https://www.python.org/downloads/)
-python --version  # Python 3.12.x 이상 확인
-
-cd C:\Users\lasta\MyProject\AI\ontology_platform
-
-# 가상환경 + 의존성 설치
-python -m venv .venv
-.venv\Scripts\activate
-pip install --upgrade pip
-pip install -e ".[dev]"
-
-# .env 생성 (실제 LLM 키 채우기)
-Copy-Item .env.example .env
-# 그 다음 .env 파일을 편집하여 LLM_API_KEY 등 채움
-```
-
-### 2) 자동 검증 (Acceptance Gate #2)
-
-```powershell
-# 단위 + 통합 테스트만 (LLM 호출 없음, 빠름)
-.venv\Scripts\python.exe -m pytest tests/unit tests/integration -v
-```
-
-**기대 결과**: 모든 케이스 PASS.
-
-- `tests/unit/test_select_ontology.py` (4 케이스) — Phase 0.2 검증
-- `tests/unit/test_convert_document.py` (7 케이스) — Phase 0.3 검증
-- `tests/unit/test_platform_config.py` (5 케이스) — Phase 0.5 검증
-- `tests/integration/test_api_smoke.py` (11 케이스) — Phase 0.4 + 0.6 mock 검증, Phase 0 future dependency route gate 검증
-
-Windows에서 `%TEMP%` 권한 문제 또는 `.pytest_cache` 쓰기 문제가 발생하면 아래처럼 pytest temp/cache 위치를 workspace 내부로 고정한다.
-
-```powershell
-$env:TMP=(Join-Path (Resolve-Path '.').Path 'pytest_tmp')
-$env:TEMP=$env:TMP
-New-Item -ItemType Directory -Force -Path $env:TMP | Out-Null
-.venv\Scripts\python.exe -m pytest tests/unit tests/integration -v --basetemp "$env:TMP\basetemp" -o cache_dir="$env:TMP\cache"
-```
-
-### 3) End-to-end 검증 (Acceptance Gate #1, #3, #4)
-
-LLM 호출이 실제로 일어남. OpenAI는 비용 발생, Ollama는 로컬에서 무료.
-
-```powershell
-# (A) Ollama 로컬 사용 (권장 — 비용 무료)
-# 사전: Ollama 설치 후 `ollama pull qwen2.5`
-$env:LLM_PROVIDER = "ollama"
-$env:LLM_MODEL_NAME = "qwen2.5"
-$env:LLM_BASE_URL = "http://localhost:11434"
-pytest tests/e2e -m e2e -v
-
-# (B) OpenAI 사용
-$env:LLM_PROVIDER = "openai"
-$env:LLM_MODEL_NAME = "gpt-4o-mini"
-$env:LLM_API_KEY = "sk-..."
-pytest tests/e2e -m e2e -v
-```
-
-**기대 결과**:
-- `test_full_pipeline_writes_ontology_and_facts` PASS
-- 응답에서 ontology TTL과 facts TTL이 비어 있지 않음
-- `metadata.budget.calls_count > 0`
-- `metadata.budget.ontology_triples_generated > 0` 또는 `facts_triples_generated > 0`
-- `tmp_path / "work"` 아래 `.ttl` 또는 `.rdf` 파일 생성됨
-
-### 4) 수동 smoke (선택)
-
-```powershell
-# 서버 기동
-uvicorn ont_platform.api.main:app --reload
-
-# 다른 셸에서
-curl http://localhost:8000/health
-curl http://localhost:8000/info
-curl -X POST http://localhost:8000/process `
-  -H "Content-Type: application/json" `
-  -d '{"text":"Alice works at Acme in Berlin."}'
-```
-
-## 통과 시 처리
-
-위 모든 검증을 통과하면 **이 문서의 표 상태 컬럼을 ✅로 갱신**하고 git에 commit한다.
-
-이후 Phase 1 작업은 [PHASE1_NEXT_STEPS.md](PHASE1_NEXT_STEPS.md)를 따른다.
-
-## 실패 시 처리
-
-- **단위 테스트 실패**: 어느 케이스가 실패했는지 확인. Phase 0.2/0.3/0.5의 vendored 수정 또는 platform/ 코드에 회귀가 발생했을 가능성. PR 단위로 롤백 후 재시도.
-- **통합 테스트 실패**: FastAPI 라우팅/의존성 주입 문제. `platform/api/main.py` 또는 `platform/api/deps.py` 확인.
-- **E2E 테스트 실패**: 
-  - `LLM_API_KEY`, `LLM_PROVIDER`, `LLM_MODEL_NAME` 환경변수 확인
-  - 워크플로우가 timeout: `ServerConfig.base_recursion_limit` 조정 검토
-  - OntoCast `select_ontology.py` 또는 `convert_document.py` 수정에 회귀가 있는지 점검 (VENDORED_MODIFICATIONS.md 참조)
-
-## 검증 이력
-
-| 일자 | 검증자 | 결과 |
-|---|---|---|
-| 2026-05-13 | (코드 작성: ontology-platform agent) | 코드 준비 완료. 실 환경 검증 보류. |
-| 2026-05-14 | lasta + Claude | **unit 16/16, integration 10/10 통과** (Gate #2 ✅). 패키지 이름 충돌 수정 (`platform`→`ont_platform`). e2e는 LLM 필요로 대기. |
-| 2026-05-19 | Codex | **unit 16/16, integration 11/11, 총 27/27 통과**. Phase 0 route gate 추가로 Trafilatura route는 PHASE>=1에서만 lazy mount. e2e는 LLM 필요로 대기. |
-| ____-__-__ | ________________ | __________________________________ |

ontology_platform/docs/phases/PHASE0_NEXT_STEPS.md

@@ -1,124 +0,0 @@
-# Phase 0 — 다음 작업자 핸드오프
-
-본 문서는 이 프로젝트를 이어받는 AI 에이전트 또는 개발자가 즉시 작업을 시작하기 위한 핸드오프 노트다.
-
-## 현재 상태 (지금까지 완료된 것)
-
-- [x] **0.1 일부**: 폴더 골격, `pyproject.toml`, `README.md`, `.env.example`, `.gitignore`, `NOTICE` 생성
-- [x] **설계 문서**: [../통합설계서.md](../통합설계서.md) 배치 완료 (모든 작업의 기준)
-
-## 즉시 시작할 작업 (순서대로)
-
-### 0.1 (잔여): OntoCast vendored copy
-
-**근거**: 통합설계서 §9.1, OntoCast 분석 §13.1
-
-```powershell
-# 1. 원본을 vendored/ontocast로 복사 (.git 제외)
-Copy-Item -Path "C:\Users\lasta\MyProject\AI\참고\ontocast-main\*" `
-          -Destination "C:\Users\lasta\MyProject\AI\ontology_platform\vendored\ontocast\" `
-          -Recurse -Exclude ".git",".github",".venv","node_modules"
-
-# 2. 원본 LICENSE 및 NOTICE를 vendored/ontocast/ 안에 그대로 유지
-# 3. NOTICE 파일의 "(원본 저장소 URL 기입)" 부분을 실제 URL로 채우기
-# 4. git init (아직 안 했다면)
-cd C:\Users\lasta\MyProject\AI\ontology_platform
-git init
-git add .
-git commit -m "Initial scaffold: folder skeleton, design doc, NOTICE"
-```
-
-**확인 사항**:
-- [ ] `vendored/ontocast/` 안에 원본 LICENSE 파일이 있어야 한다
-- [ ] `vendored/ontocast/pyproject.toml`은 그대로 두되, 우리 `pyproject.toml`이 우선
-- [ ] NOTICE 파일의 OntoCast 항목에 실제 source URL 기입
-
-### 0.2: `select_ontology.py` 버그 수정
-
-**근거**: 통합설계서 §5 Phase 0, OntoCast 분석 §13.1 / §9.1 (1번)
-
-**문제**: `vendored/ontocast/ontocast/agent/select_ontology.py`에서 None 선택 인덱스 불일치.
-- 코드는 `answer_index == 0`을 None으로 처리
-- 그러나 dynamic model은 `1..num_ontologies+1` 범위 사용
-- 실제 None 선택은 `num_ontologies + 1`이어야 자연스러움
-
-**조치**:
-1. 해당 함수의 분기 로직을 `answer_index == num_ontologies + 1` 또는 동등한 표현으로 수정
-2. **수정 사실을 파일 상단 주석으로 명시** (Apache 2.0 의무): 예) `# MODIFIED 2026-MM-DD: Fixed None index inconsistency, see docs/통합설계서.md §5 Phase 0`
-3. 회귀 테스트 작성: `tests/unit/test_select_ontology.py`
-   - 케이스 1: ontology가 0개일 때 → None 반환
-   - 케이스 2: ontology가 N개, LLM이 1~N 선택 → 해당 ontology 반환
-   - 케이스 3: ontology가 N개, LLM이 N+1 선택 → None 반환
-
-### 0.3: `convert_document.py` 다중 파일 처리 확장
-
-**근거**: 통합설계서 §5 Phase 0, OntoCast 분석 §13.1 (3번) / §21.1 (4번)
-
-**문제**: `convert_document()`가 "processing only one file"로 주석 처리되어 있고, 다중 파일 처리 시 마지막 파일 기준으로만 상태가 업데이트됨.
-
-**조치**:
-1. 입력 파일 목록을 순회하며 각 파일을 독립 `ContentUnit`으로 만들어 `AgentState.content_units`에 누적
-2. 동일 corpus 내 파일들이 함께 처리되도록 보장 (각 파일이 별도 doc IRI를 가짐)
-3. 회귀 테스트: 2개 PDF를 한 번에 처리 → 둘 다 처리되어야 함
-
-### 0.4: Robyn → FastAPI 재작성
-
-**근거**: 통합설계서 §11 (기술 스택), OntoCast 분석 §13 (API 명세)
-
-**조치**:
-1. `platform/api/main.py` 생성 (FastAPI app 인스턴스)
-2. OntoCast 분석 §13.1~§13.4의 4개 endpoint를 FastAPI로 동일 시맨틱 재작성:
-   - `GET /health`
-   - `GET /info`
-   - `POST /process` (JSON + multipart)
-   - `POST /flush` (관리자 권한 + confirmation token, 분석 §21.1 #6)
-3. OntoCast의 `ToolBox` 의존성 주입은 FastAPI `Depends`로 변환
-4. `uvicorn platform.api.main:app --reload`로 기동 가능해야 함
-
-**중요**: OntoCast 코어 모듈(`stategraph/`, `agent/`, `onto/`, `tool/`)은 **건드리지 않는다**. API 레이어만 재작성.
-
-### 0.5: Pydantic Settings 정리
-
-**근거**: 통합설계서 §5 Phase 0 (5번), OntoCast 분석 §15
-
-**조치**:
-1. `platform/config.py` 생성
-2. `pydantic-settings`의 `BaseSettings`로 `.env` 로딩
-3. **Phase 0에서는 filesystem 모드만 활성화** (Fuseki/Neo4j는 Phase 4에서):
-   - `STORAGE_BACKEND=filesystem` 강제
-   - Neo4j/Fuseki 변수가 채워져 있어도 무시
-4. OntoCast의 기존 `Config` 클래스는 우리 `Settings`에서 만들어 주입
-
-### 0.6: End-to-end 통합 테스트
-
-**근거**: 통합설계서 §5 Phase 0 Acceptance Gate
-
-**조치**:
-1. `vendored/ontocast/data/`의 예제 JSON 또는 PDF 1개를 fixture로 복사 → `tests/fixtures/`
-2. `tests/integration/test_phase0_e2e.py` 작성:
-   - FastAPI `TestClient`로 `/process` 호출
-   - 응답에 `ontology` TTL과 `facts` TTL 둘 다 포함
-   - `working_directory/`에 ontology/facts 파일 생성 확인
-   - BudgetTracker가 LLM call/triple count를 0보다 큰 값으로 기록
-
-### 0.7: Acceptance Gate 0 체크
-
-통합설계서 §5 Phase 0 Acceptance Gate의 4개 체크박스를 PR에 인용하며 모두 확인:
-
-- [ ] 단일 PDF 또는 JSON 입력 → ontology TTL + facts TTL이 filesystem에 생성됨
-- [ ] `/health`, `/info`, `/process` (FastAPI 버전) 정상 동작
-- [ ] BudgetTracker가 LLM call/triple count를 정확히 기록
-- [ ] LangGraph 워크플로우(CONVERT→CHUNK→...→SERIALIZE) 전 노드가 traceable
-
-## 작업 시 준수사항
-
-1. **PR 단위**: 위의 0.1~0.7 각각을 별도 PR/커밋으로 분리. 하나의 PR에 여러 단계를 섞지 않는다.
-2. **PR 설명에 근거 인용**: 예) "통합설계서 §5 Phase 0 (3번)에 따라 다중 파일 처리 확장. OntoCast 분석 §13.1 인용."
-3. **vendored/ 수정 시 라이선스 의무**:
-   - 수정한 파일 상단에 `# MODIFIED YYYY-MM-DD: <한 줄 설명>` 주석 추가
-   - 원본 LICENSE/NOTICE 파일은 절대 삭제하지 않는다
-4. **Phase 1로 넘어가지 말 것**: Acceptance Gate 0 통과 전까지 Trafilatura/Crawl4AI/Guardrails/Neo4j GraphRAG 의존성을 활성화하거나 import하지 않는다. (`pyproject.toml`에 명시되어 있더라도 코드에서 사용 금지)
-
-## Phase 1 이후 핸드오프
-
-Phase 0 완료 후, 본 폴더에 `PHASE1_NEXT_STEPS.md`를 작성하여 다음 작업자에게 동일한 형식으로 핸드오프한다. 통합설계서 §12 Phase 1 작업 단위(1.1~1.7)를 참조.

ontology_platform/docs/phases/PHASE1_ACCEPTANCE_GATE.md

@@ -1,33 +0,0 @@
-# Phase 1 Acceptance Gate 결과
-
-작성일: 2026-05-19
-
-범위: Trafilatura 기반 URL/HTML 입력 정렬, SourceDocument/EvidenceSpan 계약, URL 입력 API, fixture 기반 dedup 검증.
-
-## 결과 요약
-
-| # | Acceptance Gate 항목 | 상태 | 검증 방법 |
-|---|---|---|---|
-| 1 | URL/HTML 입력이 정제 문서로 변환됨 | 통과 | `tests/unit/test_web_extractor.py` |
-| 2 | source URL, title, language, content hash, fingerprint 보존 | 통과 | `test_extract_from_korean_html_preserves_document_contract` |
-| 3 | `SourceDocument`, `EvidenceSpan`, Content metadata 경계 연결 | 통과 | `test_extracted_content_maps_to_source_document_and_evidence_spans`, `test_content_unit.py` |
-| 4 | `/process/url`, `/api/v1/extract/url` URL 입력 API 제공 | 통과 | `tests/integration/test_url_ingest.py` |
-| 5 | 같은 본문 중복 입력은 fingerprint 기반으로 skip | 통과 | `test_same_clean_body_gets_same_hash_and_fingerprint`, `test_process_url_skips_duplicate_payload_by_fingerprint` |
-| 6 | Phase 0 회귀 없음 | 통과 | `python -m pytest tests/unit tests/integration -q` |
-
-## 검증 이력
-
-| 일자 | 검증자 | 결과 |
-|---|---|---|
-| 2026-05-19 | Codex | Phase 1 신규 테스트 7/7 통과. 전체 unit/integration 34/34 통과. |
-
-## 구현 메모
-
-- `ont_platform/core/extractors/web_extractor.py`는 Trafilatura 2.x `bare_extraction`을 사용하되, local HTML fixture에서 Trafilatura fingerprint가 비어 있는 경우 normalized text 기반 `sha1:` fingerprint를 생성한다.
-- `ont_platform/storage/models.py`의 SQLAlchemy 예약어 충돌을 피하기 위해 DB 컬럼명은 `metadata`로 유지하고 Python attribute는 `metadata_`로 정리했다.
-- `/process/url`, `/api/v1/process/url`, `/api/v1/extract/url`은 같은 Phase 1 응답 계약을 사용한다.
-- OntoCast vendored core는 수정하지 않았다.
-
-## 다음 Gate
-
-Phase 2는 Candidate Storage 및 Review 책임 경계를 다룬다. 진행 전 `PHASE_INDEX.md`에서 Phase 2 항목만 명시적으로 선택해 작업한다.

ontology_platform/docs/phases/PHASE1_NEXT_STEPS.md

@@ -1,162 +0,0 @@
-# Phase 1 — Trafilatura 통합 (다음 작업자 핸드오프)
-
-본 문서는 Phase 0이 완료된 시점에서 Phase 1 작업을 이어받는 AI 에이전트 또는 개발자가 즉시 작업을 시작하기 위한 핸드오프 노트다.
-
-## 시작 전 확인 사항
-
-- [ ] **Phase 0 Acceptance Gate**가 모두 ✅인가? [PHASE0_ACCEPTANCE_GATE.md](PHASE0_ACCEPTANCE_GATE.md) 참조. 통과 전에는 Phase 1 진행 금지.
-- [ ] `tests/unit`와 `tests/integration` 전체가 PASS인가?
-- [ ] git log에 Phase 0 commit들이 PR 단위로 분리되어 있는가? (0.1 vendored / 0.2 bug fix / 0.3 multi-file / 0.4 FastAPI / 0.5 config / 0.6 e2e tests / 0.7 gate)
-
-## Phase 1 목표
-
-URL이 입력일 때 원본 페이지에서 본문, 제목, 저자, 날짜, 언어, canonical URL을 정확히 뽑아 OntoCast의 `ContentUnit` metadata에 채워 넣는다.
-
-**근거**: 통합설계서 §5 Phase 1, Trafilatura 분석 §11~§17.
-
-**왜 Trafilatura를 가장 먼저 통합하는가**: 가장 작은 통합 — 단일 함수 호출(`bare_extraction`)만으로 끝남. 의존성도 명확하며 라이선스 동일 (Apache 2.0).
-
-## 작업 단위 (PR 분해)
-
-### 1.1: Trafilatura 의존성 활성화
-
-`pyproject.toml`에 이미 `trafilatura[all]>=2.0.0`이 명시되어 있다. 활성화 절차:
-
-```powershell
-pip install -e ".[dev]"  # 의존성 재설치 시 trafilatura 자동 설치
-python -c "import trafilatura; print(trafilatura.__version__)"
-```
-
-**확인**: `2.0.0` 이상이 출력되어야 한다.
-
-### 1.2: `web_extractor.py` 어댑터 작성
-
-**위치**: `platform/core/extractors/web_extractor.py`
-
-**근거**: Trafilatura 분석 §17의 `extract_for_ontology` 함수를 거의 그대로 사용.
-
-**필수 동작**:
-- 입력: `html: str`, `url: str`, `lang: str | None = None`
-- 출력: `ExtractedWebDocument` (dataclass)
-  - `url`, `title`, `author`, `date`, `sitename`, `description`
-  - `text` (정제 본문)
-  - `body_xml` (Trafilatura `Document.body`)
-  - `metadata` (raw dict)
-  - `fingerprint` (SimHash)
-- 실패 시 `None` 반환
-
-**호출 옵션** (Trafilatura 분석 §12 권장값 그대로):
-```python
-Extractor(
-    output_format="python",
-    url=url,
-    with_metadata=True,
-    comments=False,
-    tables=True,
-    formatting=True,
-    links=True,
-    images=True,
-    dedup=True,
-    lang=lang,
-)
-```
-
-### 1.3: `ContentUnit` 모델 확장
-
-OntoCast의 `vendored/ontocast/ontocast/onto/content_unit.py`를 **직접 수정하지 말고**, 우리 쪽에 wrapper 모델을 만든다.
-
-**위치**: `platform/models/content_unit.py`
-
-**필드** (통합설계서 §7.1 참조):
-- 기존 OntoCast 필드 (`text`, `index`, `doc_iri`, `graph`, `type`, `iri`) 유지/위임
-- 추가: `source_url`, `title`, `author`, `publish_date`, `language`, `sitename`, `fingerprint`, `content_hash`, `metadata`, `retrieved_at`, `extracted_by`
-
-**호환성**: 기존 OntoCast 코드가 받는 `ContentUnit`과 인터페이스 호환되도록 `as_ontocast()` 메서드 제공.
-
-### 1.4: OntoCast `ConverterTool` 분기 추가 (URL/HTML 입력)
-
-**문제**: OntoCast `ConverterTool`은 PDF/DOCX/MD만 처리. URL 또는 HTML 입력은 처리 못 함.
-
-**조치 옵션**:
-- **옵션 A (권장)**: OntoCast의 `convert_document.py` 모듈에 새 분기 추가 — `.html`, `.htm` 확장자 또는 `state.source_url`이 있으면 Trafilatura로 처리. **vendored 수정이지만 매우 작음**.
-- **옵션 B**: API 레이어(`platform/api/`)에서 입력이 URL이면 미리 fetch + Trafilatura 처리한 뒤 그 결과를 JSON envelope로 ToolBox에 넘김.
-
-**권장**: 옵션 B. vendored 수정을 늘리지 않고 platform 코드로 끝낼 수 있음.
-
-새 endpoint:
-- `POST /process/url` — body: `{"url": "...", "ontology_user_instruction": "...", ...}` — 내부적으로 `web_extractor`로 본문 추출 후 OntoCast workflow 실행.
-
-### 1.5: Fingerprint 기반 dedup
-
-- `tests/fixtures/`에 같은 본문의 두 URL fixture 만들기
-- `web_extractor` 결과의 `fingerprint`가 일치하면 OntoCast 처리 skip
-- 저장 위치: 일단 in-memory set (`platform/storage/dedup_cache.py`), Phase 2에서 Redis로 이전
-
-### 1.6: 한국어 페이지 3종 추출 검증
-
-**테스트 fixture 수집**:
-- 한국어 뉴스 1개 (예: 연합뉴스/조선/한겨레)
-- 한국어 블로그 1개 (예: 네이버 블로그)
-- 한국어 쇼핑 페이지 1개 (예: 쿠팡 상품 페이지)
-
-각각 raw HTML을 `tests/fixtures/korean/`에 저장 (실제 fetch는 운영 환경에서 한 번만, 그 결과를 fixture로 박제).
-
-**테스트**: `tests/integration/test_web_extractor_korean.py`
-- 본문 길이 > 200자
-- title 추출 성공
-- language 감지: `ko`
-- author 또는 date 중 하나 이상 추출
-
-### 1.7: Acceptance Gate 1 체크
-
-통합설계서 §5 Phase 1 Acceptance Gate 4개 항목:
-
-- [ ] URL 입력 → 본문/메타데이터가 정확히 추출되어 `ContentUnit`에 저장됨
-- [ ] 한국어 뉴스/블로그/쇼핑 페이지 각각 1개씩 본문 추출 정확도 수동 검증
-- [ ] 동일 URL 재입력 시 fingerprint 기반 dedup으로 skip
-- [ ] Phase 0의 모든 기능이 여전히 정상 동작 (회귀 없음)
-
-Phase 0의 `tests/unit/`, `tests/integration/` 전체가 여전히 PASS여야 함.
-
-## Phase 1에서 만들 새 산출물
-
-```
-platform/
-  core/
-    extractors/
-      web_extractor.py         ← 1.2
-  models/
-    content_unit.py            ← 1.3
-  storage/
-    dedup_cache.py             ← 1.5
-  api/
-    routes/
-      url_ingest.py            ← 1.4 (POST /process/url)
-tests/
-  fixtures/
-    korean/                    ← 1.6
-      news_yonhap.html
-      blog_naver.html
-      shop_coupang.html
-  unit/
-    test_web_extractor.py      ← 1.2
-    test_dedup_cache.py        ← 1.5
-  integration/
-    test_url_ingest.py         ← 1.4
-    test_web_extractor_korean.py ← 1.6
-docs/
-  phases/
-    PHASE1_ACCEPTANCE_GATE.md  ← 1.7 (PHASE0과 동일 형식)
-    PHASE2_NEXT_STEPS.md       ← 다음 작업자에게 넘김
-```
-
-## 작업 시 준수사항 (PHASE0과 동일)
-
-1. **PR 단위 분리**: 1.1~1.7 각각 별도 PR/커밋.
-2. **PR 설명에 근거 인용**: 예) "통합설계서 §5 Phase 1 (1.2)에 따라 Trafilatura adapter 작성. Trafilatura 분석 §17 인용."
-3. **vendored/ontocast/** 수정 최소화. 본 Phase에서는 옵션 B 사용 시 vendored 수정 0건이 목표.
-4. **Phase 2로 넘어가지 말 것**: Acceptance Gate 1 통과 전까지 Crawl4AI 의존성을 코드에서 import하지 않는다.
-
-## Phase 2 이후 핸드오프
-
-Phase 1 완료 후 다음 작업자에게 동일한 형식의 `PHASE2_NEXT_STEPS.md`를 작성한다. 통합설계서 §12 Phase 2 작업 단위(2.1~2.8)를 참조.

ontology_platform/docs/phases/PHASE2_ACCEPTANCE_GATE.md

@@ -1,35 +0,0 @@
-# Phase 2 Acceptance Gate 결과
-
-작성일: 2026-05-19
-
-범위: Candidate Storage 및 Review 책임 경계. Lightweight/OntoCast 후보 저장 경로, review 상태 전이, audit trail, evidence 기반 promotion gate.
-
-## 결과 요약
-
-| # | Acceptance Gate 항목 | 상태 | 검증 방법 |
-|---|---|---|---|
-| 1 | extraction 결과가 candidate로 저장됨 | 통과 | `tests/unit/test_candidate_repository.py` |
-| 2 | lightweight와 OntoCast 저장 경로가 분리됨 | 통과 | `test_repository_saves_lightweight_candidates_with_evidence`, `test_repository_saves_ontocast_candidates_on_separate_source_path` |
-| 3 | 승인/반려/자동승인 상태 변경 이력이 남음 | 통과 | `tests/unit/test_review_service.py` |
-| 4 | evidence 없는 항목은 승인 및 graph commit 대상이 아님 | 통과 | `test_candidate_without_evidence_cannot_be_approved`, `test_promotion_plan_blocks_approved_candidate_without_evidence` |
-| 5 | Review API가 ingest/list/detail/approve/reject/promote 흐름을 제공함 | 통과 | `tests/integration/test_review_api.py` |
-| 6 | Phase 0-1 회귀 없음 | 통과 | `python -m pytest tests/unit tests/integration -q` |
-
-## 검증 이력
-
-| 일자 | 검증자 | 결과 |
-|---|---|---|
-| 2026-05-19 | Codex | Phase 2 신규 테스트 9/9 통과. 전체 unit/integration 43/43 통과. 변경 파일 대상 ruff 통과. |
-
-## 구현 메모
-
-- `CandidateEntity`, `CandidateRelation`에 `source_type`, `created_by`, `validation_passed`, `promoted_at`을 추가해 review queue 계약을 명확히 했다.
-- `ReviewDecision`으로 상태 변경 audit trail을 남긴다.
-- `CandidateRepository.save_lightweight_result()`와 `save_ontocast_result()`를 분리해 두 입력 경로가 같은 candidate contract로 정규화되되, 출처는 유지된다.
-- `ReviewService`는 `pending -> approved/rejected/auto_approved`, `approved/auto_approved -> rejected`만 허용한다.
-- `CandidatePromotionService`는 `approved` 또는 `auto_approved`이면서 evidence가 실제 존재하는 후보만 commit plan에 포함한다.
-- OntoCast vendored core는 수정하지 않았다.
-
-## 다음 Gate
-
-Phase 3은 Crawl4AI 수집 계층 및 Job Orchestration이다. 진행 전 `PHASE_INDEX.md`에서 Phase 3 항목만 명시적으로 선택해 작업한다.

ontology_platform/docs/phases/PHASE2_NEXT_STEPS.md

@@ -1,27 +0,0 @@
-# Phase 2 — Candidate Storage 및 Review 책임 경계
-
-본 문서는 Phase 1 완료 후 다음 작업자가 Phase 2를 시작할 때 참고할 핸드오프 노트다. 자동으로 Phase 2를 진행하지 않는다.
-
-## 시작 전 확인
-
-- `PHASE_INDEX.md`에서 Phase 2 진행 요청이 명시되어 있는지 확인한다.
-- `PHASE1_ACCEPTANCE_GATE.md`의 unit/integration 34/34 통과 상태를 기준선으로 삼는다.
-- vendored OntoCast core는 계속 직접 수정하지 않는다.
-
-## Phase 2 목표
-
-추출 결과를 바로 확정 그래프로 보내지 않고, 사람이 검토할 수 있는 candidate/review queue 계약으로 분리한다. SourceDocument와 EvidenceSpan이 없는 후보는 확정 graph로 들어가지 못하게 한다.
-
-## 작업 범위
-
-1. `storage/models.py`의 `CandidateEntity`, `CandidateRelation`을 review queue 계약으로 확정한다.
-2. OntoCast 결과와 lightweight extraction 결과의 저장 경로를 분리한다.
-3. `pending`, `approved`, `auto_approved`, `rejected` 상태 전이 규칙을 문서와 테스트로 고정한다.
-4. evidence 없는 후보가 확정 graph로 승격되지 못하도록 validation boundary를 둔다.
-
-## 권장 테스트
-
-- 후보 생성 시 `document_id`와 `evidence_ids`가 필수로 연결되는지 검증한다.
-- 승인/반려/자동승인 상태 전이가 허용된 경로로만 움직이는지 검증한다.
-- evidence 없는 entity/relation이 commit 단계에 도달하지 못하는지 검증한다.
-- Phase 1 URL/HTML ingestion 테스트가 계속 통과하는지 회귀 검증한다.

ontology_platform/docs/phases/PHASE3_NEXT_STEPS.md

@@ -1,28 +0,0 @@
-# Phase 3 — Crawl4AI 수집 계층 및 Job Orchestration
-
-본 문서는 Phase 2 완료 후 다음 작업자가 Phase 3을 시작할 때 참고할 핸드오프 노트다. 자동으로 Phase 3을 진행하지 않는다.
-
-## 시작 전 확인
-
-- `PHASE_INDEX.md`에서 Phase 3 진행 요청이 명시되어 있는지 확인한다.
-- `PHASE2_ACCEPTANCE_GATE.md`의 unit/integration 43/43 통과 상태를 기준선으로 삼는다.
-- 수집 계층은 SourceDocument 생성 전 단계까지만 책임진다. Candidate 저장과 Review Queue는 Phase 2 계약을 사용한다.
-- vendored OntoCast core는 계속 직접 수정하지 않는다.
-
-## Phase 3 목표
-
-정적 URL 1건 처리를 넘어 동적 페이지와 대량 수집을 job 단위로 관리한다. Crawl4AI는 acquisition adapter로 감싸고, 본문 정제는 Phase 1 Trafilatura adapter, 후보 저장은 Phase 2 Review Queue로 넘긴다.
-
-## 작업 범위
-
-1. `crawl4ai_adapter.py`를 동적/대량 수집 adapter로 제한한다.
-2. crawler profile, robots policy, cache policy를 설정 기반으로 분리한다.
-3. Job 상태 모델과 progress API/WebSocket 경계를 정리한다.
-4. 수집 결과를 Trafilatura 후처리와 SourceDocument 저장으로 연결한다.
-
-## 권장 테스트
-
-- 정적 HTML/동적 페이지 profile이 같은 SourceDocument 계약으로 이어지는지 검증한다.
-- robots/cache policy가 설정값에 따라 선택되는지 검증한다.
-- job 상태가 pending/running/completed/failed로 전이되는지 검증한다.
-- Phase 1 extraction 및 Phase 2 review queue 테스트가 계속 통과하는지 회귀 검증한다.

ontology_platform/docs/phases/PHASE_INDEX.md

@@ -1,89 +1,101 @@
-# PHASE INDEX - ontology_platform engine-respect roadmap
+# PHASE INDEX - Semantic Page Classification Layer
 
-?묒꽦?? 2026-05-19
+작성일: 2026-05-22
 
-踰붿쐞: `ontology_platform` ?꾩슜. `crawler_platform`?€ ?대쾲 ?묒뾽 踰붿쐞?먯꽌 ?쒖쇅?쒕떎.
+범위: `ontology_platform`의 `crawler_platform.app.core.crawler.page_classifier` 및 page classification과 직접 연결된 crawler/extractor 흐름.
 
-湲곗? 臾몄꽌:
-- `ontology_platform/docs/?듯빀?ㅺ퀎??md`
+기준 문서:
+- `README.md`
+- `docs/PHASE_PLANNING.md`
 - `ontology_platform/README.md`
-- `ontology_platform/docs/phases/PHASE0_ACCEPTANCE_GATE.md`
-- `ontology_platform/docs/phases/PHASE1_NEXT_STEPS.md`
-- `ontology_platform/docs/phases/PHASE1_ACCEPTANCE_GATE.md`
-- `ontology_platform/docs/phases/PHASE2_ACCEPTANCE_GATE.md`
+- `ontology_platform/docs/semantic_page_classification_codex_spec.md`
 
-?듭떖 ?먯튃:
-- OntoCast??Base ?붿쭊?쇰줈 議댁쨷?쒕떎.
-- vendored OntoCast 肄붿뼱???듯빀?ㅺ퀎?쒓? ?덉슜??踰붿쐞 ?몄뿉???섏젙?섏? ?딅뒗??
-- Trafilatura, Crawl4AI, Guardrails, Neo4j GraphRAG??吏곸젒 ?ш뎄?꾪븯吏€ ?딄퀬 ?뉗? adapter/facade濡?媛먯떬??
-- Firecrawl, OpenDeepResearcher 肄붾뱶???ы븿?섏? ?딅뒗??
-- Acceptance Gate瑜??듦낵?섍린 ???ㅼ쓬 ?듯빀?쇰줈 ?섏뼱媛€吏€ ?딅뒗??
+핵심 원칙:
+- 기존 엔진을 폐기하거나 대규모로 교체하지 않는다.
+- 기존 `classify_page(...) -> str` 호출부가 깨지지 않도록 legacy compatibility를 유지한다.
+- 기존 `ProductPage`, `CategoryPage`, `SearchPage`, `BoardPage`, `BrandStoryPage`, `UnknownPage` 문자열은 alias 또는 compatibility mapping으로 유지한다.
+- URL substring 중심 if-return 확장이 아니라 signal extraction -> evidence scoring -> classification result -> analyze strategy -> LLM policy 구조로 확장한다.
+- protected page는 안전하게 skip하고, UnknownPage는 evidence와 confidence를 남긴다.
+- pytest 또는 현재 프로젝트 테스트 명령으로 회귀 검증한다.
 
 ---
 
-PHASE 0. ?붿쭊 寃쎄퀎 媛먯궗 諛?Phase Gate 蹂듦뎄
-FILE: ./26_05_19_engine_respect_plan/phase_00_001_engine_boundary_gate.md
+PHASE 1. 현재 흐름 기준선 고정 및 영향 범위 정리
+FILE: ./26_05_22_semantic_page_classification/phase_01_001_current_flow_boundary.md
 
-1) ?꾩옱 `ont_platform` 紐⑤뱢??Base/Adapter/Draft/Excluded 梨낆엫?쇰줈 遺꾨쪟 [?꾨즺]
-2) Phase 0?먯꽌 誘몃옒 Phase ?섏〈?깆씠 import?섏뼱 ???쒖옉??源⑥? ?딅룄濡?寃뚯씠???뺣━ [?꾨즺]
-3) Phase 0 unit/integration 寃€利??덉감 怨좎젙 [?꾨즺]
-4) `PHASE0_ACCEPTANCE_GATE.md` 媛깆떊 湲곗? ?뺣━ [?꾨즺]
+1) `page_classifier.py`의 현재 public API와 legacy page_type 문자열 목록 고정 [완료]
+2) `should_analyze_page()` 호출부와 crawler의 `classify_page()` 호출 위치 문서화 [완료]
+3) Extractor/HybridExtractor에서 page_type과 LLM skip 정책이 연결되는 흐름 정리 [완료]
+4) 기존 page_type 문자열을 기대하는 테스트, config, adapter, ontology rule 경로 목록화 [완료]
 
 ---
 
-PHASE 1. Trafilatura 湲곕컲 URL/HTML ?낅젰 ?뺣젹
-FILE: ./26_05_19_engine_respect_plan/phase_01_001_trafilatura_ingestion.md
+PHASE 2. Taxonomy와 Classification Result 모델 추가
+FILE: ./26_05_22_semantic_page_classification/phase_02_001_taxonomy_result_model.md
 
-1) `web_extractor.py`瑜?Trafilatura adapter 梨낆엫?쇰줈 ?뺣━ [?꾨즺]
-2) `SourceDocument`, `EvidenceSpan`, Content metadata ?€??寃쎄퀎 ?곌껐 [?꾨즺]
-3) `/process/url` ?먮뒗 ?숇벑??URL ?낅젰 API ?ㅺ퀎 [?꾨즺]
-4) ?쒓뎅??URL/HTML fixture 湲곕컲 異붿텧 ?뚯뒪?몄? dedup 湲곗? ?묒꽦 [?꾨즺]
+1) PageDomain/PageArchetype/PageType/EntityType/ActionIntent/GraphRole/AnalyzeStrategy/LLMPolicy 상수 또는 enum 추가 [완료]
+2) `EvidenceItem`, `PageClassificationResult` dataclass 추가 [완료]
+3) legacy alias 및 normalize helper 추가 [완료]
+4) 기존 `classify_page()` 문자열 반환 호환을 유지하면서 semantic result API 추가 [완료]
 
 ---
 
-PHASE 2. Candidate Storage 諛?Review 梨낆엫 寃쎄퀎
-FILE: ./26_05_19_engine_respect_plan/phase_02_001_candidate_review_boundary.md
+PHASE 3. Raw Snapshot 및 Signal Extraction 레이어 추가
+FILE: ./26_05_22_semantic_page_classification/phase_03_001_signal_extraction_layer.md
 
-1) `storage/models.py`???꾨낫 紐⑤뜽???뺤떇 Review Queue 怨꾩빟?쇰줈 ?뺤젙 [?꾨즺]
-2) OntoCast 寃곌낵?€ lightweight extraction 寃곌낵???€??寃쎈줈 遺꾨━ [?꾨즺]
-3) ?뱀씤/諛섎젮/?먮룞?뱀씤 ?곹깭 ?꾩씠 洹쒖튃 ?뺤쓽 [?꾨즺]
-4) evidence ?녿뒗 ?꾨낫媛€ ?뺤젙 graph濡??ㅼ뼱媛€吏€ 紐삵븯寃?李⑤떒 [?꾨즺]
+1) `RawPageSnapshot`와 `PageSignals` 모델 추가 [완료]
+2) JSON-LD, OpenGraph, Twitter Card, meta, headings, links, forms, buttons, inputs 추출 [완료]
+3) commerce/listing/editorial/community/docs/corporate/protected/system signal 추출 [완료]
+4) HTML 일부가 깨지거나 필드가 누락되어도 예외 없이 빈 값으로 처리 [완료]
 
 ---
 
-PHASE 3. Crawl4AI ?섏쭛 怨꾩링 諛?Job Orchestration
-FILE: ./26_05_19_engine_respect_plan/phase_03_001_crawl4ai_acquisition_jobs.md
+PHASE 4. Evidence Scoring 기반 Semantic Classification 구현
+FILE: ./26_05_22_semantic_page_classification/phase_04_001_evidence_scoring_classifier.md
 
-1) `crawl4ai_adapter.py`瑜??숈쟻/?€???섏쭛 adapter濡??쒗븳 [?꾨즺]
-2) crawler profile, robots policy, cache policy瑜??ㅼ젙 湲곕컲?쇰줈 遺꾨━ [?꾨즺]
-3) Job ?곹깭 紐⑤뜽怨?progress API/WebSocket 寃쎄퀎 ?뺣━ [?꾨즺]
-4) Trafilatura ?꾩쿂由ъ? SourceDocument ?€?μ쑝濡??곌껐 [?꾨즺]
+1) 주요 page type별 scoring function과 evidence recording 구조 추가 [완료]
+2) 최소 20개 semantic page type 분류 구현 [완료]
+3) confidence, alternatives, secondary_page_types 산출 [완료]
+4) low confidence 또는 모호한 결과를 evidence 포함 UnknownPage로 처리 [완료]
 
 ---
 
-PHASE 4. Guardrails Validation Gate
-FILE: ./26_05_19_engine_respect_plan/phase_04_001_guardrails_validation_gate.md
+PHASE 5. Analyze Strategy 및 LLM Policy 분리
+FILE: ./26_05_22_semantic_page_classification/phase_05_001_analysis_llm_policy.md
 
-1) `core/validation`??Pydantic lightweight?€ Guardrails facade濡?遺꾨━ [?꾨즺]
-2) OntoCast LLM 異쒕젰 ?섑븨 吏€?먯쓣 vendored ?섏젙 ?놁씠 ?곗꽑 ?ㅺ퀎 [?꾨즺]
-3) schema violation, endpoint missing, confidence range ?뚯뒪???묒꽦 [?꾨즺]
-4) Guard ?ㅽ뙣 寃곌낵瑜?candidate/review issue濡??€??[?꾨즺]
+1) PageClassificationResult 기반 `decide_analyze_strategy()` 추가 [완료]
+2) PageClassificationResult 기반 `decide_llm_policy()` 추가 [완료]
+3) `should_analyze_page(result_or_page_type, analyze_page_types=None)` compatibility 구현 [완료]
+4) Category/Search/Board 계열을 무조건 skip하지 않고 strategy 기반으로 처리 [완료]
+5) Login/Checkout/Payment/Captcha/AccessDenied 계열은 SkipProtected/Skip 정책으로 처리 [완료]
 
 ---
 
-PHASE 5. Neo4j Projection 諛?GraphRAG 寃€??FILE: ./26_05_19_engine_respect_plan/phase_05_001_neo4j_projection_graphrag.md
+PHASE 6. Crawler, Cleaner, Extractor, Discovery/Relevance 통합
+FILE: ./26_05_22_semantic_page_classification/phase_06_001_pipeline_integration.md
 
-1) RDF/Fuseki瑜?canonical store, Neo4j瑜?projection/search store濡?怨좎젙 [?꾨즺]
-2) `core/graph` 湲곗〈 紐⑤뱢??projection/search adapter 梨낆엫?쇰줈 ?щ텇瑜?[?꾨즺]
-3) read-only Text2Cypher?€ vector/hybrid retriever API ?ㅺ퀎 [?꾨즺]
-4) provenance媛€ search result源뚯? ?댁뼱吏€??寃€利?湲곗? ?묒꽦 [?꾨즺]
+1) `site_crawler.py`와 `pipeline.py` metadata에 semantic classification payload 저장 [완료]
+2) `ExtractionPageContext` 또는 metadata를 통해 analyze_strategy/llm_policy 전달 [완료]
+3) `HybridExtractor`가 LLMPolicy를 우선 사용하고 legacy page_type fallback을 유지하도록 수정 [완료]
+4) `page_cleaner.py`, `domain_discovery.py`, `relevance_engine.py`의 legacy page_type 기대 경로와 신규 semantic type을 호환 [완료]
 
 ---
 
-PHASE 6. Maintenance Loop 諛??댁쁺 湲곕뒫 ?뺣━
-FILE: ./26_05_19_engine_respect_plan/phase_06_001_maintenance_loop_operations.md
+PHASE 7. Unknown Pattern 저장 기반 추가
+FILE: ./26_05_22_semantic_page_classification/phase_07_001_unknown_pattern_storage.md
 
-1) Knowledge Agent??肄붾뱶媛€ ?꾨땲???꾨＼?꾪듃/?뚰겕?뚮줈???⑦꽩留?李⑥슜 [?꾨즺]
-2) Analyst/Researcher/Curator/Auditor/Fixer/Advisor 梨낆엫 ?뺤쓽 [?꾨즺]
-3) `auth`, `audit`, `billing`, `realtime` 珥덉븞 紐⑤뱢???댁쁺 寃쎄퀎 ?뺣━ [?꾨즺]
-4) destructive fix???щ엺 ?뱀씤 寃뚯씠?몃? 諛섎뱶???듦낵?섎룄濡??ㅺ퀎 [?꾨즺]
+1) UnknownPage 또는 low confidence 페이지의 evidence payload 정의 [완료]
+2) text/html/link/schema/button/form summary와 fingerprint hook 추가 [완료]
+3) DB schema 변경 없이 metadata_json에 저장 가능한 초기 구조 구현 [완료]
+4) 향후 clustering/embedding 확장을 위한 hook만 추가하고 실제 clustering은 이번 범위에서 제외 [완료]
+
+---
+
+PHASE 8. 테스트 Fixture 및 회귀 검증
+FILE: ./26_05_22_semantic_page_classification/phase_08_001_tests_regression.md
+
+1) 최소 10개 이상의 HTML fixture 추가 [완료]
+2) ProductDetailPage, CategoryListingPage, SearchResultsPage, ArticlePage, QAPage, FAQPage, ForumThreadPage, DocumentationPage, JobPostingPage, LoginPage, CheckoutPage, TermsPage, SitemapPage, UnknownPage 단위 테스트 추가 [완료]
+3) legacy `classify_page()`와 `should_analyze_page()` 호환성 테스트 추가 [완료]
+4) HybridExtractor LLMPolicy 회귀 테스트 추가 [완료]
+5) pytest 또는 현재 프로젝트 테스트 명령 실행 및 결과 기록 [완료]

ontology_platform/ont_platform/api/main.py

@@ -49,6 +49,7 @@ from ont_platform.api.product_backend import include_product_backend  # noqa: E4
 platform_config = importlib.import_module("ont_platform.config")
 
 logger = logging.getLogger(__name__)
+_startup_error: str | None = None
 
 
 def _resolve_ontocast_version() -> str:
@@ -124,8 +125,14 @@ def _include_phase_routers(app: FastAPI) -> None:
 @asynccontextmanager
 async def lifespan(app: FastAPI) -> AsyncIterator[None]:
     """FastAPI lifespan: build ToolBox + workflow once on startup."""
+    global _startup_error
     settings = platform_config.load_settings()
-    await initialize_app_context(settings)
+    try:
+        await initialize_app_context(settings)
+        _startup_error = None
+    except Exception as exc:  # noqa: BLE001
+        _startup_error = str(exc)
+        logger.exception("App context initialization failed; product backend will remain available")
     try:
         yield
     finally:
@@ -149,8 +156,23 @@ def create_app() -> FastAPI:
 
     # ─── /health ──────────────────────────────────────────────────────
     @app.get("/health", tags=["meta"])
-    async def health(ctx: Annotated[AppContext, Depends(get_app_context)]) -> JSONResponse:
-        """Liveness check. 503 if the LLM isn't wired."""
+    async def health(request: Request) -> JSONResponse:
+        """Liveness check for the HTTP service and optional LLM readiness."""
+        settings = platform_config.load_settings()
+        if _startup_error:
+            return JSONResponse(
+                status_code=503,
+                content={
+                    "status": "degraded",
+                    "error": _startup_error,
+                    "platform_version": PLATFORM_VERSION,
+                    "ontocast_version": ONTOCAST_VERSION,
+                    "phase": int(settings.phase),
+                    "storage_backend": settings.storage_backend,
+                },
+            )
+
+        ctx = _request_app_context(request)
         if ctx.tools.llm is None:
             return JSONResponse(
                 status_code=503,
@@ -170,8 +192,16 @@ def create_app() -> FastAPI:
 
     # ─── /info ────────────────────────────────────────────────────────
     @app.get("/info", tags=["meta"])
-    async def info(ctx: Annotated[AppContext, Depends(get_app_context)]) -> JSONResponse:
+    async def info(request: Request) -> JSONResponse:
         """Service-level capabilities (mirrors OntoCast /info semantics)."""
+        settings = platform_config.load_settings()
+        phase = int(settings.phase)
+        storage_backend = settings.storage_backend
+        if not _startup_error:
+            ctx = _request_app_context(request)
+            phase = int(ctx.settings.phase)
+            storage_backend = ctx.settings.storage_backend
+
         return JSONResponse(
             status_code=200,
             content={
@@ -185,8 +215,9 @@ def create_app() -> FastAPI:
                 "capabilities": ["text-to-triples", "ontology-extraction"],
                 "input_types": ["text", "json", "pdf", "markdown"],
                 "output_types": ["turtle", "json"],
-                "phase": int(ctx.settings.phase),
-                "storage_backend": ctx.settings.storage_backend,
+                "phase": phase,
+                "storage_backend": storage_backend,
+                "startup_error": _startup_error,
             },
         )
 
@@ -418,6 +449,13 @@ def create_app() -> FastAPI:
     return app
 
 
+def _request_app_context(request: Request) -> AppContext:
+    override = request.app.dependency_overrides.get(get_app_context)
+    if override is not None:
+        return override()
+    return get_app_context()
+
+
 # Top-level instance for `uvicorn platform.api.main:app`.
 app = create_app()
 

ontology_platform/ont_platform/api/product_backend.py

@@ -82,6 +82,6 @@ def _remove_route(app: FastAPI, path: str, methods: set[str]) -> None:
         for route in app.router.routes
         if not (
             getattr(route, "path", None) == path
-            and set(getattr(route, "methods", set())) == methods
+            and methods.issubset(set(getattr(route, "methods", set())))
         )
     ]

ontology_platform/pyproject.toml

@@ -71,7 +71,7 @@ dependencies = [
     # PHASE3 Acceptance Gate 통과 후 활성화
     # 버전 고정: experimental API 변경 위험 (Neo4j GraphRAG 분석 §13.1)
     # "neo4j-graphrag[openai,experimental]==1.16.0",
-    # "neo4j>=5.18.1,<7",
+    "neo4j>=5.18.1,<7",
 
     # ─── 운영 인프라 ───────────────────────────────────────────────────
     # Job Queue

ontology_platform/tests/unit/test_perfume_extraction_brand_price.py

@@ -0,0 +1,79 @@
+from crawler_platform.app.config.loader import ProjectConfig
+from crawler_platform.app.core.extractor.base import ExtractedClaim, ExtractionBundle
+from crawler_platform.app.core.extractor.validation import validate_extraction_bundle
+from crawler_platform.app.domains.perfume.extractor import PerfumeRuleBasedExtractor
+
+
+def perfume_config() -> ProjectConfig:
+    return ProjectConfig(
+        project_name="perfume-test",
+        domain="perfume",
+        target_entities=["Perfume"],
+        fields=[],
+        sources=[],
+        ontology={
+            "entity_types": [
+                "Perfume",
+                "Product",
+                "Brand",
+                "Note",
+                "Accord",
+                "Mood",
+                "Season",
+                "Occasion",
+                "Review",
+            ],
+            "predicates": ["hasBrand", "hasPrice"],
+        },
+    )
+
+
+def test_forment_product_price_line_is_not_extracted_as_brand() -> None:
+    text = "\n".join(
+        [
+            "포맨트 시그니처 퍼퓸 코튼메모리",
+            "62,000원",
+            "49,000원",
+            "기억의 표면을 어루만지다",
+        ]
+    )
+
+    bundle = PerfumeRuleBasedExtractor().extract(text, perfume_config())
+
+    brands = [entity.name for entity in bundle.entities if entity.entity_type == "Brand"]
+    assert brands == ["FORMENT"]
+    assert "62,000원" not in brands
+    assert any(
+        claim.predicate == "hasBrand"
+        and claim.object_name == "FORMENT"
+        and claim.evidence_text == "포맨트 시그니처 퍼퓸 코튼메모리"
+        for claim in bundle.claims
+    )
+    assert any(
+        claim.predicate == "hasPrice"
+        and claim.object_value == {"amount": 62000.0, "currency": "원"}
+        for claim in bundle.claims
+    )
+
+
+def test_validation_rejects_price_like_brand_object() -> None:
+    bundle = ExtractionBundle(
+        claims=[
+            ExtractedClaim(
+                "포맨트 시그니처 퍼퓸 코튼메모리",
+                "Perfume",
+                "hasBrand",
+                "62,000원",
+                "Brand",
+                evidence_text="포맨트 시그니처 퍼퓸 코튼메모리 62,000원",
+                confidence=0.9,
+            )
+        ],
+        extractor_name="rule_only_extractor",
+        raw_output={"extraction_mode": "rule_only"},
+    )
+
+    result = validate_extraction_bundle(bundle, perfume_config())
+
+    assert result.bundle.claims == []
+    assert result.rejected_claims[0]["reason"] == "brand object looks like a price"

ontology_platform/tests/unit/test_phase7_hybrid_extraction.py

@@ -0,0 +1,203 @@
+from crawler_platform.app.config.loader import ProjectConfig
+from crawler_platform.app.api.routes import extraction_log_summary
+from crawler_platform.app.core.extractor.base import ExtractedClaim, ExtractionBundle, ExtractionPageContext
+from crawler_platform.app.core.extractor.hybrid import HybridExtractor, fallback_bundle, mark_bundle, merge_bundles
+from crawler_platform.app.core.extractor.validation import validate_extraction_bundle
+
+
+def project_config() -> ProjectConfig:
+    return ProjectConfig(
+        project_name="phase7-test",
+        domain="generic",
+        target_entities=["Thing"],
+        fields=[],
+        sources=[],
+        ontology={
+            "entity_types": ["Thing"],
+            "predicates": ["hasPart", "color", "size"],
+            "relation_types": {
+                "hasPart": {
+                    "allowed_subject_types": ["Thing"],
+                    "allowed_object_types": ["Thing"],
+                    "confidence_rules": {"min_confidence": 0.1},
+                },
+                "color": {
+                    "allowed_subject_types": ["Thing"],
+                    "literal_value": True,
+                    "confidence_rules": {"min_confidence": 0.1},
+                },
+                "size": {
+                    "allowed_subject_types": ["Thing"],
+                    "literal_value": True,
+                    "confidence_rules": {"min_confidence": 0.1},
+                },
+            },
+        },
+    )
+
+
+def test_hybrid_merge_marks_agreement_conflict_and_counts() -> None:
+    rule = ExtractionBundle(
+        claims=[
+            ExtractedClaim("A", "Thing", "hasPart", "B", "Thing", confidence=0.7),
+            ExtractedClaim("A", "Thing", "color", object_value="red", confidence=0.6),
+        ]
+    )
+    llm = ExtractionBundle(
+        claims=[
+            ExtractedClaim("A", "Thing", "hasPart", "B", "Thing", confidence=0.8),
+            ExtractedClaim("A", "Thing", "color", object_value="blue", confidence=0.65),
+            ExtractedClaim("A", "Thing", "size", object_value="large", confidence=0.5),
+        ]
+    )
+    mark_bundle(rule, source="rule", mode="compare")
+    mark_bundle(llm, source="llm", mode="compare")
+
+    merged = merge_bundles(rule, llm, HybridExtractor("generic", mode="compare"))
+
+    assert merged.extractor_name == "compare_rule_llm_extractor"
+    assert merged.raw_output["comparison"] == {
+        "both_agree": 1,
+        "rule_only": 0,
+        "llm_only": 1,
+        "conflict": 1,
+        "rejected_by_validation": 0,
+    }
+    by_predicate = {(claim.predicate, claim.object_name or claim.object_value): claim for claim in merged.claims}
+    assert by_predicate[("hasPart", "B")].metadata["agreement"] == "rule_and_llm"
+    assert by_predicate[("color", "red")].metadata["review_required"] is True
+    assert by_predicate[("color", "blue")].metadata["conflict_status"] == "rule_llm_conflict"
+    assert by_predicate[("size", "large")].metadata["agreement"] == "llm_only"
+
+
+def test_validation_keeps_evidence_missing_llm_only_claim_as_candidate() -> None:
+    bundle = ExtractionBundle(
+        claims=[
+            ExtractedClaim(
+                "A",
+                "Thing",
+                "color",
+                object_value="blue",
+                confidence=0.8,
+                metadata={
+                    "agreement": "llm_only",
+                    "extraction_source": "llm",
+                    "llm_confidence": 0.8,
+                },
+            )
+        ],
+        extractor_name="hybrid_rule_llm_extractor",
+        provider="lm_studio",
+        raw_output={"extraction_mode": "hybrid"},
+    )
+
+    result = validate_extraction_bundle(bundle, project_config())
+
+    assert len(result.bundle.claims) == 1
+    claim = result.bundle.claims[0]
+    assert claim.metadata["validation_status"] == "candidate_claim"
+    assert claim.metadata["review_required"] is True
+    assert claim.metadata["review_reason"] == "claim has no evidence"
+
+
+def test_validation_marks_rule_only_claim_as_rule_candidate() -> None:
+    bundle = ExtractionBundle(
+        claims=[
+            ExtractedClaim(
+                "Alpha",
+                "Thing",
+                "color",
+                object_value="red",
+                evidence_text="Alpha color red",
+                confidence=0.7,
+                metadata={
+                    "agreement": "rule_only",
+                    "extraction_source": "rule",
+                    "rule_confidence": 0.7,
+                },
+            )
+        ],
+        extractor_name="hybrid_rule_llm_extractor",
+        provider="lm_studio",
+        raw_output={"extraction_mode": "hybrid"},
+    )
+
+    result = validate_extraction_bundle(bundle, project_config())
+
+    assert result.bundle.claims[0].metadata["validation_status"] == "rule_candidate"
+    assert result.bundle.claims[0].metadata["claim_kind"] == "rule_candidate"
+
+
+def test_hybrid_smart_routing_skips_llm_for_category_page() -> None:
+    context = ExtractionPageContext(
+        url="https://example.test/category",
+        final_url=None,
+        title="Category",
+        page_type="CategoryPage",
+        clean_text="Category listing " * 40,
+    )
+
+    bundle = HybridExtractor("generic", mode="hybrid").extract_from_context(context, project_config())
+
+    assert bundle.extractor_name == "hybrid_rule_only_routed"
+    assert bundle.raw_output["llm_skipped"] is True
+    assert bundle.raw_output["effective_extraction_mode"] == "rule_only"
+    assert "CategoryPage" in bundle.raw_output["llm_skip_reason"]
+
+
+def test_rule_only_mode_records_rule_llm_summary_counts() -> None:
+    rule_bundle = ExtractionBundle(
+        claims=[
+            ExtractedClaim("A", "Thing", "color", object_value="red"),
+            ExtractedClaim("A", "Thing", "size", object_value="large"),
+        ]
+    )
+
+    class FixedRuleExtractor:
+        def extract(self, page_text, project_config):  # noqa: ANN001
+            return rule_bundle
+
+        def extract_from_context(self, context, project_config):  # noqa: ANN001
+            return rule_bundle
+
+    extractor = HybridExtractor("generic", mode="rule_only")
+    extractor.rule_extractor = FixedRuleExtractor()
+
+    bundle = extractor.extract("Alpha color red size large", project_config())
+
+    assert bundle.raw_output["rule_claim_count"] == 2
+    assert bundle.raw_output["llm_claim_count"] == 0
+    assert bundle.raw_output["comparison"]["rule_only"] == 2
+
+
+def test_fallback_records_rule_counts_and_comparison() -> None:
+    rule_bundle = ExtractionBundle(
+        claims=[
+            ExtractedClaim("A", "Thing", "color", object_value="red"),
+            ExtractedClaim("A", "Thing", "size", object_value="large"),
+        ]
+    )
+    mark_bundle(rule_bundle, source="rule", mode="hybrid")
+
+    bundle = fallback_bundle(rule_bundle, HybridExtractor("generic", mode="hybrid"), RuntimeError("boom"))
+
+    assert bundle.raw_output["rule_claim_count"] == 2
+    assert bundle.raw_output["llm_claim_count"] == 0
+    assert bundle.raw_output["comparison"]["rule_only"] == 2
+
+
+def test_extraction_log_summary_backfills_legacy_rule_only_counts() -> None:
+    summary = extraction_log_summary(
+        {
+            "validation": {"rejected_claim_count": 4},
+            "candidate_claims": [
+                {"metadata": {"agreement": "rule_only", "extraction_source": "rule"}},
+                {"metadata": {"agreement": "rule_only", "extraction_source": "rule"}},
+            ]
+        }
+    )
+
+    assert summary["rule_claim_count"] == 2
+    assert summary["llm_claim_count"] == 0
+    assert summary["comparison"]["rule_only"] == 2
+    assert summary["comparison"]["rejected_by_validation"] == 4

ontology_platform/web/frontend/src/lib/api/claims.ts

@@ -28,6 +28,11 @@ export const claimSchema = z
     graph_merge_status: z.string().nullable().optional(),
     graph_merge_reason: z.string().nullable().optional(),
     confidence_breakdown: z.record(z.string(), z.unknown()).nullable().optional(),
+    agreement: z.string().nullable().optional(),
+    extraction_source: z.string().nullable().optional(),
+    claim_kind: z.string().nullable().optional(),
+    rule_confidence: z.number().nullable().optional(),
+    llm_confidence: z.number().nullable().optional(),
     review_required: z.boolean().nullable().optional(),
     review_reason: z.string().nullable().optional(),
     conflict_status: z.string().nullable().optional(),

ontology_platform/web/frontend/src/lib/api/crawl.ts

@@ -9,10 +9,27 @@ export const crawlPageItemSchema = z
     status: z.string().optional(),
     page_type: z.string().optional(),
     title: z.string().nullable().optional(),
+    extraction_mode: z.string().nullable().optional(),
+    effective_extraction_mode: z.string().nullable().optional(),
+    llm_skipped: z.boolean().optional(),
+    llm_skip_reason: z.string().nullable().optional(),
+    fallback_used: z.boolean().optional(),
+    agreement_claim_count: z.number().optional(),
+    conflict_claim_count: z.number().optional(),
     error: z.string().nullable().optional(),
   })
   .passthrough();
 
+export const crawlExtractionSummarySchema = z
+  .object({
+    llm_skipped_count: z.number().default(0),
+    fallback_count: z.number().default(0),
+    conflict_claim_count: z.number().default(0),
+    agreement_claim_count: z.number().default(0),
+    llm_call_count: z.number().default(0),
+  })
+  .passthrough();
+
 export const crawlProgressSchema = z
   .object({
     seed_url: z.string().optional(),
@@ -23,6 +40,7 @@ export const crawlProgressSchema = z
     errors: z.array(z.string()).optional(),
     pages: z.array(crawlPageItemSchema).optional(),
     latest_page: crawlPageItemSchema.optional(),
+    extraction_summary: crawlExtractionSummarySchema.optional(),
   })
   .passthrough();
 
@@ -49,9 +67,11 @@ export interface StartSiteCrawlRequest {
   max_pages?: number;
   same_domain_only?: boolean;
   analyze_page_types?: string[];
+  extraction_mode?: "rule_only" | "llm_only" | "hybrid" | "compare";
   extractor_provider?: string;
   extractor_model?: string | null;
   extractor_base_url?: string | null;
+  fallback_to_rules?: boolean;
   check_robots_txt?: boolean;
   respect_robots_txt?: boolean | null;
 }
@@ -66,6 +86,26 @@ export function isCrawlTerminal(status: string): boolean {
   return TERMINAL_CRAWL_STATUSES.has(status);
 }
 
+export const extractorModelsResponseSchema = z.object({
+  ok: z.boolean(),
+  error: z.string().optional(),
+  models: z
+    .array(
+      z
+        .object({
+          id: z.string(),
+          owned_by: z.string().nullish(),
+        })
+        .passthrough(),
+    )
+    .optional()
+    .default([]),
+});
+
+export type ExtractorModelsResponse = z.infer<
+  typeof extractorModelsResponseSchema
+>;
+
 export const crawlApi = {
   startByProject: (body: StartSiteCrawlRequest) =>
     apiClient.post("/crawl-site/by-project", crawlJobSchema, body),
@@ -79,4 +119,9 @@ export const crawlApi = {
       `/crawl-site/jobs/${encodeURIComponent(jobId)}/cancel`,
       crawlJobSchema,
     ),
+  listExtractorModels: (provider: string, baseUrl?: string | null) =>
+    apiClient.post("/extractors/models", extractorModelsResponseSchema, {
+      provider,
+      base_url: baseUrl || null,
+    }),
 };

ontology_platform/web/frontend/src/lib/api/platform.ts

@@ -65,6 +65,20 @@ export const extractionLogSchema = z
     validation: z.unknown().optional(),
     page_context: z.unknown().optional(),
     candidate_count: z.number().default(0),
+    extraction_mode: z.string().nullable().optional(),
+    effective_extraction_mode: z.string().nullable().optional(),
+    comparison: recordSchema.nullable().optional(),
+    rule_entity_count: z.number().nullable().optional(),
+    rule_claim_count: z.number().nullable().optional(),
+    llm_entity_count: z.number().nullable().optional(),
+    llm_claim_count: z.number().nullable().optional(),
+    agreement_claim_count: z.number().nullable().optional(),
+    rule_only_claim_count: z.number().nullable().optional(),
+    llm_only_claim_count: z.number().nullable().optional(),
+    conflict_claim_count: z.number().nullable().optional(),
+    llm_skipped: z.boolean().nullable().optional(),
+    llm_skip_reason: z.string().nullable().optional(),
+    fallback: z.string().nullable().optional(),
     raw_output: z.unknown().optional(),
   })
   .passthrough();

ontology_platform/web/frontend/src/lib/api/research.ts

@@ -62,9 +62,11 @@ export interface StartResearchRequest {
   min_relevance?: number;
   same_domain_only?: boolean;
   analyze_page_types?: string[];
+  extraction_mode?: "rule_only" | "llm_only" | "hybrid" | "compare";
   extractor_provider?: string;
   extractor_model?: string | null;
   extractor_base_url?: string | null;
+  fallback_to_rules?: boolean;
   check_robots_txt?: boolean;
   respect_robots_txt?: boolean | null;
 }

ontology_platform/web/frontend/src/pages/CrawlPage.tsx

@@ -1,4 +1,4 @@
-import { useState } from "react";
+import { useEffect, useState } from "react";
 import { useNavigate, useParams } from "react-router-dom";
 import { useTranslation } from "react-i18next";
 import { useForm } from "react-hook-form";
@@ -36,7 +36,7 @@ import {
   useCrawlJob,
   useStartSiteCrawl,
 } from "@/hooks/useCrawl";
-import { isCrawlTerminal } from "@/lib/api/crawl";
+import { crawlApi, isCrawlTerminal } from "@/lib/api/crawl";
 
 const startCrawlSchema = z.object({
   source_name: z.string().min(1, "소스를 선택하세요"),
@@ -44,6 +44,11 @@ const startCrawlSchema = z.object({
   max_depth: z.number().int().min(0).max(10),
   max_pages: z.number().int().min(1).max(500),
   same_domain_only: z.boolean(),
+  extraction_mode: z.enum(["rule_only", "llm_only", "hybrid", "compare"]),
+  extractor_provider: z.enum(["lm_studio", "openai", "ollama"]),
+  extractor_model: z.string().optional(),
+  extractor_base_url: z.string().optional(),
+  fallback_to_rules: z.boolean(),
 });
 
 type StartCrawlFormValues = z.infer<typeof startCrawlSchema>;
@@ -96,14 +101,22 @@ export default function CrawlPage() {
       max_depth: 2,
       max_pages: 30,
       same_domain_only: true,
+      extraction_mode: "hybrid",
+      extractor_provider: "lm_studio",
+      extractor_model: "",
+      extractor_base_url: "http://localhost:1234/v1",
+      fallback_to_rules: true,
     },
   });
 
   const onStart = async (values: StartCrawlFormValues) => {
     try {
+      const usesLlm = values.extraction_mode !== "rule_only";
       const created = await startCrawl.mutateAsync({
         project_name: projectName,
         ...values,
+        extractor_model: usesLlm ? values.extractor_model || null : null,
+        extractor_base_url: usesLlm ? values.extractor_base_url || null : null,
       });
       setActiveJobId(created.job_id);
       toast.success(
@@ -136,6 +149,54 @@ export default function CrawlPage() {
 
   const sources = project?.sources ?? [];
   const sourceName = watch("source_name");
+  const extractionMode = watch("extraction_mode");
+  const provider = watch("extractor_provider");
+  const baseUrl = watch("extractor_base_url");
+  const usesLlm = extractionMode !== "rule_only";
+
+  const [loadedModelHint, setLoadedModelHint] = useState<string>("");
+  const [modelLookupStatus, setModelLookupStatus] = useState<
+    "idle" | "loading" | "ok" | "error"
+  >("idle");
+  const [modelLookupError, setModelLookupError] = useState<string>("");
+  useEffect(() => {
+    if (!usesLlm || provider !== "lm_studio") {
+      setLoadedModelHint("");
+      setModelLookupStatus("idle");
+      setModelLookupError("");
+      return;
+    }
+    let cancelled = false;
+    setModelLookupStatus("loading");
+    setModelLookupError("");
+    (async () => {
+      try {
+        const res = await crawlApi.listExtractorModels(provider, baseUrl);
+        if (cancelled) return;
+        if (!res.ok) {
+          setModelLookupStatus("error");
+          setModelLookupError(res.error || "모델 조회 실패");
+          return;
+        }
+        const first = res.models?.[0]?.id;
+        if (!first) {
+          setModelLookupStatus("error");
+          setModelLookupError("로드된 모델이 없습니다");
+          return;
+        }
+        setLoadedModelHint(first);
+        setValue("extractor_model", first, { shouldDirty: false });
+        setModelLookupStatus("ok");
+      } catch (e) {
+        if (cancelled) return;
+        setModelLookupStatus("error");
+        setModelLookupError((e as Error).message);
+      }
+    })();
+    return () => {
+      cancelled = true;
+    };
+  }, [provider, baseUrl, usesLlm, setValue]);
   const selectedSource = sources.find((s) => s.name === sourceName);
   const progress = job?.progress;
   const visited = progress?.visited_count ?? 0;
@@ -340,6 +401,81 @@ export default function CrawlPage() {
                 </span>
               </label>
 
+              <div className="space-y-3 rounded-md border bg-background-subtle p-3">
+                <div className="grid grid-cols-2 gap-3">
+                  <div className="space-y-1.5">
+                    <Label htmlFor="extraction_mode">Extraction mode</Label>
+                    <Select id="extraction_mode" {...register("extraction_mode")}>
+                      <option value="hybrid">Hybrid</option>
+                      <option value="rule_only">Rule only</option>
+                      <option value="llm_only">LLM only</option>
+                      <option value="compare">Compare</option>
+                    </Select>
+                  </div>
+                  <div className="space-y-1.5">
+                    <Label htmlFor="extractor_provider">Provider</Label>
+                    <Select
+                      id="extractor_provider"
+                      disabled={!usesLlm}
+                      {...register("extractor_provider")}
+                    >
+                      <option value="lm_studio">LM Studio</option>
+                      <option value="openai">OpenAI</option>
+                      <option value="ollama">Ollama</option>
+                    </Select>
+                  </div>
+                </div>
+                {usesLlm && (
+                  <>
+                    <div className="space-y-1.5">
+                      <Label htmlFor="extractor_model">Model</Label>
+                      <Input
+                        id="extractor_model"
+                        placeholder={
+                          loadedModelHint || "deepseek-r1-distill-qwen-7b"
+                        }
+                        {...register("extractor_model")}
+                      />
+                      {provider === "lm_studio" &&
+                        modelLookupStatus === "loading" && (
+                          <p className="text-xs text-muted-foreground">
+                            LM Studio 로드 모델 확인 중...
+                          </p>
+                        )}
+                      {provider === "lm_studio" &&
+                        modelLookupStatus === "ok" &&
+                        loadedModelHint && (
+                          <p className="text-xs text-muted-foreground">
+                            LM Studio 로드 모델: {loadedModelHint}
+                          </p>
+                        )}
+                      {provider === "lm_studio" &&
+                        modelLookupStatus === "error" && (
+                          <p className="text-xs text-destructive">
+                            LM Studio 모델 조회 실패: {modelLookupError}
+                          </p>
+                        )}
+                    </div>
+                    <div className="space-y-1.5">
+                      <Label htmlFor="extractor_base_url">Base URL</Label>
+                      <Input
+                        id="extractor_base_url"
+                        placeholder="http://localhost:1234/v1"
+                        {...register("extractor_base_url")}
+                      />
+                    </div>
+                    <label className="flex items-center gap-2 text-sm">
+                      <input
+                        type="checkbox"
+                        className="h-4 w-4 rounded border-input"
+                        {...register("fallback_to_rules")}
+                      />
+                      <span>Fallback to rules</span>
+                    </label>
+                  </>
+                )}
+              </div>
+
               <Button
                 type="submit"
                 className="w-full"

ontology_platform/web/frontend/src/pages/GraphViewPage.tsx

@@ -1,7 +1,16 @@
-import { useMemo, useState } from "react";
+import {
+  useEffect,
+  useMemo,
+  useRef,
+  useState,
+  type MouseEvent as ReactMouseEvent,
+  type PointerEvent as ReactPointerEvent,
+} from "react";
 import { useNavigate, useParams } from "react-router-dom";
 import {
   AlertCircle,
+  ChevronDown,
+  ChevronRight,
   CircleDot,
   Eye,
   EyeOff,
@@ -58,6 +67,18 @@ const COLORS = [
 
 const VIEW_W = 860;
 const VIEW_H = 600;
+const GRAPH_PADDING = 64;
+const CLUSTER_CELL_W = 220;
+const CLUSTER_CELL_H = 176;
+const NODE_MIN_R = 8;
+const NODE_MAX_R = 16;
+const LITERAL_MIN_R = 6;
+const LITERAL_MAX_R = 11;
+
+interface GraphCanvas {
+  width: number;
+  height: number;
+}
 
 interface VisualNode {
   id: string;
@@ -68,6 +89,7 @@ interface VisualNode {
   radius: number;
   color: string;
   source: "entity" | "literal";
+  isHub?: boolean;
   raw?: GraphNode;
 }
 
@@ -85,6 +107,243 @@ function nodeColor(type: string, types: string[]): string {
   return COLORS[index % COLORS.length];
 }
 
+function clamp(value: number, min: number, max: number) {
+  return Math.min(Math.max(value, min), max);
+}
+
+function hashToUnit(input: string) {
+  let hash = 2166136261;
+  for (let i = 0; i < input.length; i += 1) {
+    hash ^= input.charCodeAt(i);
+    hash = Math.imul(hash, 16777619);
+  }
+  return (hash >>> 0) / 4294967295;
+}
+
+function predicateAngle(predicate: string) {
+  const normalized = predicate.toLowerCase();
+  if (normalized.includes("brand")) return -Math.PI / 6;
+  if (normalized.includes("price") || normalized.includes("amount")) {
+    return Math.PI / 2;
+  }
+  if (normalized.includes("note") || normalized.includes("description")) {
+    return Math.PI;
+  }
+  if (normalized.includes("occasion") || normalized.includes("tag")) {
+    return -Math.PI / 2;
+  }
+  return hashToUnit(predicate) * Math.PI * 2;
+}
+
+function layoutGraph(nodes: VisualNode[], edges: VisualEdge[]): GraphCanvas {
+  const count = nodes.length;
+  if (!count) return { width: VIEW_W, height: VIEW_H };
+  nodes.forEach((node) => {
+    node.isHub = false;
+  });
+  if (count === 1) {
+    const centerX = VIEW_W / 2;
+    const centerY = VIEW_H / 2;
+    nodes[0].x = centerX;
+    nodes[0].y = centerY;
+    return { width: VIEW_W, height: VIEW_H };
+  }
+
+  const nodeById = new Map(nodes.map((node) => [node.id, node]));
+  const outgoing = new Map<string, VisualEdge[]>();
+  const incoming = new Map<string, VisualEdge[]>();
+  const degree = new Map<string, number>();
+  edges.forEach((edge) => {
+    outgoing.set(edge.source, [...(outgoing.get(edge.source) ?? []), edge]);
+    incoming.set(edge.target, [...(incoming.get(edge.target) ?? []), edge]);
+    degree.set(edge.source, (degree.get(edge.source) ?? 0) + 1);
+    degree.set(edge.target, (degree.get(edge.target) ?? 0) + 1);
+  });
+
+  let centers = nodes.filter(
+    (node) => node.source === "entity" && (outgoing.get(node.id)?.length ?? 0) > 0,
+  );
+  if (!centers.length) {
+    centers = [...nodes]
+      .sort((a, b) => (degree.get(b.id) ?? 0) - (degree.get(a.id) ?? 0))
+      .slice(0, Math.max(1, Math.ceil(Math.sqrt(count))));
+  }
+  centers.sort((a, b) => {
+    const scoreA = (outgoing.get(a.id)?.length ?? 0) * 3 + (degree.get(a.id) ?? 0);
+    const scoreB = (outgoing.get(b.id)?.length ?? 0) * 3 + (degree.get(b.id) ?? 0);
+    return scoreB - scoreA || a.name.localeCompare(b.name);
+  });
+
+  const centerIds = new Set(centers.map((node) => node.id));
+  centers.forEach((node) => {
+    node.isHub = true;
+    node.radius = clamp(node.radius + 2, NODE_MIN_R + 3, NODE_MAX_R + 2);
+  });
+
+  const columns = centers.length <= 2 ? centers.length : Math.ceil(Math.sqrt(centers.length * 1.35));
+  const rows = Math.ceil(centers.length / columns);
+  const canvasWidth = Math.max(
+    VIEW_W,
+    GRAPH_PADDING * 2 + columns * CLUSTER_CELL_W,
+  );
+  const canvasHeight = Math.max(
+    VIEW_H,
+    GRAPH_PADDING * 2 + rows * CLUSTER_CELL_H,
+  );
+  const centerX = canvasWidth / 2;
+  const centerY = canvasHeight / 2;
+  const width = canvasWidth - GRAPH_PADDING * 2;
+  const height = canvasHeight - GRAPH_PADDING * 2;
+  const cellW = width / Math.max(columns, 1);
+  const cellH = height / Math.max(rows, 1);
+  const anchors = new Map<string, { x: number; y: number; strength: number }>();
+
+  centers.forEach((node, index) => {
+    const row = Math.floor(index / columns);
+    const col = index % columns;
+    node.x = GRAPH_PADDING + cellW * (col + 0.5);
+    node.y = GRAPH_PADDING + cellH * (row + 0.5);
+    if (rows === 1) {
+      node.y = centerY;
+    }
+    anchors.set(node.id, { x: node.x, y: node.y, strength: 0.55 });
+  });
+
+  centers.forEach((center) => {
+    const childEdges = (outgoing.get(center.id) ?? [])
+      .filter((edge) => !centerIds.has(edge.target) && nodeById.has(edge.target))
+      .sort((a, b) => {
+        const angleA = predicateAngle(a.predicate);
+        const angleB = predicateAngle(b.predicate);
+        return angleA - angleB || a.predicate.localeCompare(b.predicate);
+      });
+    const groups = new Map<string, VisualEdge[]>();
+    childEdges.forEach((edge) => {
+      groups.set(edge.predicate, [...(groups.get(edge.predicate) ?? []), edge]);
+    });
+
+    Array.from(groups.entries()).forEach(([predicate, group]) => {
+      const baseAngle = predicateAngle(predicate);
+      const spread = Math.min(0.78, 0.18 * Math.max(group.length - 1, 0));
+      group.forEach((edge, index) => {
+        const child = nodeById.get(edge.target);
+        if (!child) return;
+        const parentCount = (incoming.get(child.id) ?? []).filter((incomingEdge) =>
+          centerIds.has(incomingEdge.source),
+        ).length;
+        if (parentCount > 1) return;
+        const offset =
+          group.length === 1
+            ? 0
+            : -spread / 2 + (spread * index) / Math.max(group.length - 1, 1);
+        const distance =
+          76 +
+          Math.min(childEdges.length, 8) * 3 +
+          hashToUnit(`${center.id}:${child.id}`) * 20;
+        const x = center.x + Math.cos(baseAngle + offset) * distance;
+        const y = center.y + Math.sin(baseAngle + offset) * distance;
+        child.x = x;
+        child.y = y;
+        anchors.set(child.id, { x, y, strength: 0.4 });
+      });
+    });
+  });
+
+  nodes
+    .filter((node) => !centerIds.has(node.id))
+    .forEach((node) => {
+      if (anchors.has(node.id)) return;
+      const parentEdges = (incoming.get(node.id) ?? []).filter((edge) =>
+        centerIds.has(edge.source),
+      );
+      if (parentEdges.length) {
+        const parents = parentEdges
+          .map((edge) => nodeById.get(edge.source))
+          .filter((parent): parent is VisualNode => Boolean(parent));
+        const avgX =
+          parents.reduce((sum, parent) => sum + parent.x, 0) / parents.length;
+        const avgY =
+          parents.reduce((sum, parent) => sum + parent.y, 0) / parents.length;
+        const isSharedTarget = parentEdges.length > 1;
+        const angle = predicateAngle(parentEdges[0].predicate);
+        const x = isSharedTarget ? avgX : avgX + Math.cos(angle) * 58;
+        const y = isSharedTarget ? avgY : avgY + Math.sin(angle) * 58;
+        node.x = x;
+        node.y = y;
+        anchors.set(node.id, { x, y, strength: isSharedTarget ? 0.46 : 0.32 });
+        return;
+      }
+
+      const relatedEdges = edges.filter(
+        (edge) => edge.source === node.id || edge.target === node.id,
+      );
+      const related = relatedEdges
+        .map((edge) =>
+          edge.source === node.id
+            ? nodeById.get(edge.target)
+            : nodeById.get(edge.source),
+        )
+        .filter((relatedNode): relatedNode is VisualNode => Boolean(relatedNode));
+      if (related.length) {
+        const avgX =
+          related.reduce((sum, relatedNode) => sum + relatedNode.x, 0) /
+          related.length;
+        const avgY =
+          related.reduce((sum, relatedNode) => sum + relatedNode.y, 0) /
+          related.length;
+        const angle = hashToUnit(node.id) * Math.PI * 2;
+        node.x = avgX + Math.cos(angle) * 64;
+        node.y = avgY + Math.sin(angle) * 64;
+      } else {
+        const angle = hashToUnit(node.id) * Math.PI * 2;
+        node.x = centerX + Math.cos(angle) * 120;
+        node.y = centerY + Math.sin(angle) * 120;
+      }
+      anchors.set(node.id, { x: node.x, y: node.y, strength: 0.24 });
+    });
+
+  for (let tick = 0; tick < 80; tick += 1) {
+    nodes.forEach((node) => {
+      const anchor = anchors.get(node.id);
+      if (!anchor) return;
+      node.x += (anchor.x - node.x) * anchor.strength * 0.12;
+      node.y += (anchor.y - node.y) * anchor.strength * 0.12;
+    });
+
+    for (let i = 0; i < count; i += 1) {
+      const a = nodes[i];
+      for (let j = i + 1; j < count; j += 1) {
+        const b = nodes[j];
+        let dx = a.x - b.x;
+        let dy = a.y - b.y;
+        let distance = Math.sqrt(dx * dx + dy * dy);
+        if (distance < 0.01) {
+          dx = hashToUnit(`${a.id}:${b.id}:x`) - 0.5;
+          dy = hashToUnit(`${a.id}:${b.id}:y`) - 0.5;
+          distance = Math.sqrt(dx * dx + dy * dy);
+        }
+        const minDistance = a.radius + b.radius + 22;
+        if (distance >= minDistance) continue;
+        const force = ((minDistance - distance) / distance) * 0.32;
+        const ax = dx * force;
+        const ay = dy * force;
+        const aAnchor = anchors.get(a.id)?.strength ?? 0.2;
+        const bAnchor = anchors.get(b.id)?.strength ?? 0.2;
+        a.x += ax * (1 - aAnchor);
+        a.y += ay * (1 - aAnchor);
+        b.x -= ax * (1 - bAnchor);
+        b.y -= ay * (1 - bAnchor);
+      }
+    }
+
+    nodes.forEach((node) => {
+      node.x = clamp(node.x, GRAPH_PADDING, canvasWidth - GRAPH_PADDING);
+      node.y = clamp(node.y, GRAPH_PADDING, canvasHeight - GRAPH_PADDING);
+    });
+  }
+  return { width: canvasWidth, height: canvasHeight };
+}
+
 function buildGraph(
   nodes: GraphNode[],
   edges: GraphEdge[],
@@ -170,15 +429,21 @@ function buildGraph(
     visualNodes = visualNodes.filter((node) => connectedIds.has(node.id));
   }
 
-  const centerX = VIEW_W / 2;
-  const centerY = VIEW_H / 2;
-  const radius = Math.max(160, Math.min(260, visualNodes.length * 13));
-  visualNodes.forEach((node, index) => {
-    const angle = (Math.PI * 2 * index) / Math.max(visualNodes.length, 1);
-    node.x = centerX + Math.cos(angle) * radius;
-    node.y = centerY + Math.sin(angle) * radius;
+  const degree = new Map<string, number>();
+  filteredEdges.forEach((edge) => {
+    degree.set(edge.source, (degree.get(edge.source) ?? 0) + 1);
+    degree.set(edge.target, (degree.get(edge.target) ?? 0) + 1);
   });
-  return { nodes: visualNodes, edges: filteredEdges, types };
+  visualNodes.forEach((node) => {
+    const links = degree.get(node.id) ?? 0;
+    const base =
+      node.source === "literal"
+        ? clamp(LITERAL_MIN_R + Math.sqrt(links) * 1.4, LITERAL_MIN_R, LITERAL_MAX_R)
+        : clamp(NODE_MIN_R + Math.sqrt(links) * 1.8, NODE_MIN_R, NODE_MAX_R);
+    node.radius = base;
+  });
+  const canvas = layoutGraph(visualNodes, filteredEdges);
+  return { nodes: visualNodes, edges: filteredEdges, types, canvas };
 }
 
 function edgeEndpoint(edge: VisualEdge, nodes: VisualNode[]) {
@@ -187,6 +452,26 @@ function edgeEndpoint(edge: VisualEdge, nodes: VisualNode[]) {
   return { source, target };
 }
 
+function edgeLinePoints(source: VisualNode, target: VisualNode) {
+  const dx = target.x - source.x;
+  const dy = target.y - source.y;
+  const distance = Math.max(Math.sqrt(dx * dx + dy * dy), 1);
+  const nx = dx / distance;
+  const ny = dy / distance;
+  const x1 = source.x + nx * (source.radius + 2);
+  const y1 = source.y + ny * (source.radius + 2);
+  const x2 = target.x - nx * (target.radius + 7);
+  const y2 = target.y - ny * (target.radius + 7);
+  return {
+    x1,
+    y1,
+    x2,
+    y2,
+    midX: (x1 + x2) / 2,
+    midY: (y1 + y2) / 2,
+  };
+}
+
 export default function GraphViewPage() {
   const navigate = useNavigate();
   const { projectId } = useParams<{ projectId: string }>();
@@ -202,8 +487,17 @@ export default function GraphViewPage() {
   const [drawerOpen, setDrawerOpen] = useState(false);
   const [hiddenTypes, setHiddenTypes] = useState<Set<string>>(new Set());
   const [showLegend, setShowLegend] = useState(true);
+  const [legendCollapsed, setLegendCollapsed] = useState(true);
   const [showMinimap, setShowMinimap] = useState(true);
   const [shortcutsOpen, setShortcutsOpen] = useState(false);
+  const canvasRef = useRef<HTMLDivElement | null>(null);
+  const panRef = useRef({
+    active: false,
+    x: 0,
+    y: 0,
+    scrollLeft: 0,
+    scrollTop: 0,
+  });
 
   const graph = useGraphNeighborhood(projectName, {
     includeCandidates: includeCandidates && statusScope === "validated_claim",
@@ -267,6 +561,72 @@ export default function GraphViewPage() {
     ? visibleEdges.reduce((sum, edge) => sum + edge.confidence, 0) /
       visibleEdges.length
     : 0;
+  const showEdgeLabels = visibleEdges.length <= 70;
+  const showNodeLabels = visibleNodes.length <= 180;
+
+  useEffect(() => {
+    const canvas = canvasRef.current;
+    if (!canvas || graph.isLoading || visibleNodes.length === 0) return;
+    window.requestAnimationFrame(() => {
+      canvas.scrollLeft = Math.max(0, (canvas.scrollWidth - canvas.clientWidth) / 2);
+      canvas.scrollTop = Math.max(0, (canvas.scrollHeight - canvas.clientHeight) / 2);
+    });
+  }, [
+    graph.isLoading,
+    projectName,
+    visibleNodes.length,
+    visual.canvas.height,
+    visual.canvas.width,
+  ]);
+
+  const startCanvasPan = (event: ReactPointerEvent<HTMLDivElement>) => {
+    if (event.button !== 0) return;
+    const target = event.target as HTMLElement;
+    if (target.closest("[data-graph-node], [data-graph-edge]")) return;
+    panRef.current = {
+      active: true,
+      x: event.clientX,
+      y: event.clientY,
+      scrollLeft: event.currentTarget.scrollLeft,
+      scrollTop: event.currentTarget.scrollTop,
+    };
+    event.currentTarget.setPointerCapture(event.pointerId);
+  };
+
+  const moveCanvasPan = (event: ReactPointerEvent<HTMLDivElement>) => {
+    if (!panRef.current.active) return;
+    event.currentTarget.scrollLeft =
+      panRef.current.scrollLeft - (event.clientX - panRef.current.x);
+    event.currentTarget.scrollTop =
+      panRef.current.scrollTop - (event.clientY - panRef.current.y);
+    event.preventDefault();
+  };
+
+  const endCanvasPan = (event: ReactPointerEvent<HTMLDivElement>) => {
+    if (!panRef.current.active) return;
+    panRef.current.active = false;
+    if (event.currentTarget.hasPointerCapture(event.pointerId)) {
+      event.currentTarget.releasePointerCapture(event.pointerId);
+    }
+  };
+
+  const jumpCanvasTo = (x: number, y: number) => {
+    const canvas = canvasRef.current;
+    if (!canvas) return;
+    canvas.scrollTo({
+      left: clamp(
+        x - canvas.clientWidth / 2,
+        0,
+        Math.max(0, canvas.scrollWidth - canvas.clientWidth),
+      ),
+      top: clamp(
+        y - canvas.clientHeight / 2,
+        0,
+        Math.max(0, canvas.scrollHeight - canvas.clientHeight),
+      ),
+      behavior: "smooth",
+    });
+  };
 
   const selectNode = (node: VisualNode | VisualEdge) => {
     setSelected(node);
@@ -530,61 +890,79 @@ export default function GraphViewPage() {
               />
             ) : (
               <>
-                <svg
-                  viewBox={`0 0 ${VIEW_W} ${VIEW_H}`}
-                  className="block h-[640px] w-full bg-background-subtle text-muted-foreground"
-                  role="img"
-                  aria-label="Ontology graph"
+                <div
+                  ref={canvasRef}
+                  className="relative h-[640px] cursor-grab overflow-auto bg-background-subtle active:cursor-grabbing"
+                  onPointerDown={startCanvasPan}
+                  onPointerMove={moveCanvasPan}
+                  onPointerUp={endCanvasPan}
+                  onPointerCancel={endCanvasPan}
+                >
+                  <svg
+                    viewBox={`0 0 ${visual.canvas.width} ${visual.canvas.height}`}
+                    width={visual.canvas.width}
+                    height={visual.canvas.height}
+                    className="block min-h-full min-w-full text-muted-foreground"
+                    role="img"
+                    aria-label="Ontology graph"
                 >
                   <defs>
                     <marker
                       id="arrow"
-                      markerWidth="10"
-                      markerHeight="10"
-                      refX="10"
-                      refY="3"
+                      markerWidth="6"
+                      markerHeight="6"
+                      refX="5.5"
+                      refY="2.5"
                       orient="auto"
                       markerUnits="strokeWidth"
                     >
-                      <path d="M0,0 L0,6 L9,3 z" fill="currentColor" />
+                      <path d="M0,0 L0,5 L5.5,2.5 z" fill="currentColor" />
                     </marker>
                   </defs>
                   {visibleEdges.map((edge) => {
                     const { source, target } = edgeEndpoint(edge, visibleNodes);
                     if (!source || !target) return null;
-                    const midX = (source.x + target.x) / 2;
-                    const midY = (source.y + target.y) / 2;
+                    const { x1, y1, x2, y2, midX, midY } = edgeLinePoints(
+                      source,
+                      target,
+                    );
                     const isSelected =
                       selected && "predicate" in selected && selected.id === edge.id;
                     return (
                       <g
                         key={edge.id}
+                        data-graph-edge
                         className="cursor-pointer"
                         onClick={() => selectNode(edge)}
                       >
                         <line
-                          x1={source.x}
-                          y1={source.y}
-                          x2={target.x}
-                          y2={target.y}
+                          x1={x1}
+                          y1={y1}
+                          x2={x2}
+                          y2={y2}
                           stroke="currentColor"
-                          strokeWidth={1 + edge.confidence * 3}
-                          strokeOpacity={isSelected ? 1 : 0.55}
+                          strokeWidth={isSelected ? 1.7 : 0.65 + edge.confidence * 0.75}
+                          strokeOpacity={isSelected ? 0.95 : 0.42}
                           markerEnd="url(#arrow)"
                           className={isSelected ? "text-brand-600 dark:text-brand-300" : ""}
                         />
-                        <text
-                          x={midX}
-                          y={midY}
-                          textAnchor="middle"
-                          className="fill-muted-foreground text-[10px]"
-                          paintOrder="stroke"
-                          stroke="hsl(var(--background))"
-                          strokeWidth="3"
-                          strokeLinejoin="round"
-                        >
-                          {edge.predicate}
-                        </text>
+                        {(showEdgeLabels || isSelected) && (
+                          <text
+                            x={midX}
+                            y={midY - 2}
+                            textAnchor="middle"
+                            className="fill-muted-foreground text-[7px]"
+                            opacity={isSelected ? 0.95 : 0.62}
+                            paintOrder="stroke"
+                            stroke="hsl(var(--background))"
+                            strokeWidth="2"
+                            strokeLinejoin="round"
+                          >
+                            {edge.predicate.length > 18
+                              ? `${edge.predicate.slice(0, 18)}...`
+                              : edge.predicate}
+                          </text>
+                        )}
                       </g>
                     );
                   })}
@@ -594,6 +972,7 @@ export default function GraphViewPage() {
                     return (
                       <g
                         key={node.id}
+                        data-graph-node
                         className="cursor-pointer"
                         onClick={() => selectNode(node)}
                       >
@@ -602,31 +981,41 @@ export default function GraphViewPage() {
                           cy={node.y}
                           r={node.radius}
                           fill={node.color}
-                          className={
+                          stroke={
                             isSelected
-                              ? "stroke-foreground"
-                              : "stroke-background"
+                              ? "hsl(var(--foreground))"
+                              : node.isHub
+                                ? "hsl(var(--muted-foreground))"
+                                : "hsl(var(--background))"
                           }
-                          strokeWidth={isSelected ? 3 : 2}
+                          strokeWidth={isSelected ? 2.5 : node.isHub ? 2 : 1.3}
                         />
-                        <text
-                          x={node.x}
-                          y={node.y + node.radius + 14}
-                          textAnchor="middle"
-                          className="fill-foreground text-[11px] font-medium"
-                          paintOrder="stroke"
-                          stroke="hsl(var(--background))"
-                          strokeWidth="3"
-                          strokeLinejoin="round"
-                        >
-                          {node.name.length > 24
-                            ? `${node.name.slice(0, 24)}...`
-                            : node.name}
-                        </text>
+                        {(showNodeLabels || isSelected) && (
+                          <text
+                            x={node.x}
+                            y={node.y + node.radius + 10}
+                            textAnchor="middle"
+                            className={
+                              node.isHub
+                                ? "fill-foreground text-[8px] font-semibold"
+                                : "fill-foreground text-[8px] font-medium"
+                            }
+                            opacity={node.source === "literal" ? 0.74 : 0.86}
+                            paintOrder="stroke"
+                            stroke="hsl(var(--background))"
+                            strokeWidth="2"
+                            strokeLinejoin="round"
+                          >
+                            {node.name.length > 18
+                              ? `${node.name.slice(0, 18)}...`
+                              : node.name}
+                          </text>
+                        )}
                       </g>
                     );
                   })}
-                </svg>
+                  </svg>
+                </div>
 
                 {/* Legend overlay (top-right) */}
                 {showLegend && (
@@ -635,12 +1024,18 @@ export default function GraphViewPage() {
                     typeCounts={typeCounts}
                     hiddenTypes={hiddenTypes}
                     onToggle={toggleType}
+                    collapsed={legendCollapsed}
+                    onCollapsedChange={setLegendCollapsed}
                   />
                 )}
 
                 {/* Minimap overlay (bottom-right) */}
                 {showMinimap && visibleNodes.length > 0 && (
-                  <Minimap nodes={visibleNodes} edges={visibleEdges} />
+                  <Minimap
+                    nodes={visibleNodes}
+                    edges={visibleEdges}
+                    onJump={jumpCanvasTo}
+                  />
                 )}
               </>
             )}
@@ -803,58 +1198,83 @@ function LegendPanel({
   typeCounts,
   hiddenTypes,
   onToggle,
+  collapsed,
+  onCollapsedChange,
 }: {
   types: string[];
   typeCounts: Map<string, number>;
   hiddenTypes: Set<string>;
   onToggle: (type: string) => void;
+  collapsed: boolean;
+  onCollapsedChange: (collapsed: boolean) => void;
 }) {
   return (
-    <div className="absolute right-3 top-3 w-52 rounded-md border border-border bg-popover/95 shadow-md backdrop-blur-sm">
-      <div className="border-b border-border px-3 py-1.5 text-2xs font-semibold uppercase tracking-wider text-muted-foreground">
-        Legend
-      </div>
-      <ul className="max-h-64 overflow-y-auto py-1">
-        {types.map((type) => {
-          const hidden = hiddenTypes.has(type);
-          const count = typeCounts.get(type) ?? 0;
-          return (
-            <li key={type}>
-              <button
-                type="button"
-                onClick={() => onToggle(type)}
-                className="flex w-full items-center gap-2 px-3 py-1.5 text-left text-xs transition-colors hover:bg-accent/60"
-                aria-pressed={!hidden}
-              >
-                <span
-                  className="h-2.5 w-2.5 flex-shrink-0 rounded-full"
-                  style={{
-                    backgroundColor: nodeColor(type, types),
-                    opacity: hidden ? 0.3 : 1,
-                  }}
-                />
-                <span
-                  className={
-                    hidden
-                      ? "flex-1 truncate text-muted-foreground/60 line-through"
-                      : "flex-1 truncate text-foreground"
-                  }
+    <div
+      className={
+        collapsed
+          ? "absolute right-3 top-3 w-32 rounded-md border border-border bg-popover/95 shadow-md backdrop-blur-sm"
+          : "absolute right-3 top-3 w-52 rounded-md border border-border bg-popover/95 shadow-md backdrop-blur-sm"
+      }
+    >
+      <button
+        type="button"
+        onClick={() => onCollapsedChange(!collapsed)}
+        className="flex w-full items-center justify-between gap-2 px-3 py-1.5 text-left text-2xs font-semibold uppercase tracking-wider text-muted-foreground transition-colors hover:bg-accent/60"
+        aria-expanded={!collapsed}
+      >
+        <span>Legend</span>
+        <span className="flex items-center gap-1 font-mono normal-case">
+          {collapsed && types.length}
+          {collapsed ? (
+            <ChevronRight className="h-3.5 w-3.5" />
+          ) : (
+            <ChevronDown className="h-3.5 w-3.5" />
+          )}
+        </span>
+      </button>
+      {!collapsed && (
+        <ul className="max-h-64 overflow-y-auto border-t border-border py-1">
+          {types.map((type) => {
+            const hidden = hiddenTypes.has(type);
+            const count = typeCounts.get(type) ?? 0;
+            return (
+              <li key={type}>
+                <button
+                  type="button"
+                  onClick={() => onToggle(type)}
+                  className="flex w-full items-center gap-2 px-3 py-1.5 text-left text-xs transition-colors hover:bg-accent/60"
+                  aria-pressed={!hidden}
                 >
-                  {type}
-                </span>
-                <span className="text-2xs text-muted-foreground tabular-nums">
-                  {count}
-                </span>
-                {hidden ? (
-                  <EyeOff className="h-3 w-3 text-muted-foreground/60" />
-                ) : (
-                  <Eye className="h-3 w-3 text-muted-foreground" />
-                )}
-              </button>
-            </li>
-          );
-        })}
-      </ul>
+                  <span
+                    className="h-2.5 w-2.5 flex-shrink-0 rounded-full"
+                    style={{
+                      backgroundColor: nodeColor(type, types),
+                      opacity: hidden ? 0.3 : 1,
+                    }}
+                  />
+                  <span
+                    className={
+                      hidden
+                        ? "flex-1 truncate text-muted-foreground/60 line-through"
+                        : "flex-1 truncate text-foreground"
+                    }
+                  >
+                    {type}
+                  </span>
+                  <span className="text-2xs text-muted-foreground tabular-nums">
+                    {count}
+                  </span>
+                  {hidden ? (
+                    <EyeOff className="h-3 w-3 text-muted-foreground/60" />
+                  ) : (
+                    <Eye className="h-3 w-3 text-muted-foreground" />
+                  )}
+                </button>
+              </li>
+            );
+          })}
+        </ul>
+      )}
     </div>
   );
 }
@@ -865,9 +1285,11 @@ function LegendPanel({
 function Minimap({
   nodes,
   edges,
+  onJump,
 }: {
   nodes: VisualNode[];
   edges: VisualEdge[];
+  onJump: (x: number, y: number) => void;
 }) {
   const W = 180;
   const H = 130;
@@ -886,6 +1308,14 @@ function Minimap({
   const offsetY = (H - h * scale) / 2;
   const tx = (x: number) => (x - minX) * scale + offsetX;
   const ty = (y: number) => (y - minY) * scale + offsetY;
+  const jumpFromMinimap = (event: ReactMouseEvent<SVGSVGElement>) => {
+    const rect = event.currentTarget.getBoundingClientRect();
+    const localX = event.clientX - rect.left;
+    const localY = event.clientY - rect.top;
+    const graphX = clamp((localX - offsetX) / scale + minX, minX, maxX);
+    const graphY = clamp((localY - offsetY) / scale + minY, minY, maxY);
+    onJump(graphX, graphY);
+  };
 
   return (
     <div className="absolute bottom-3 right-3 rounded-md border border-border bg-popover/95 shadow-md backdrop-blur-sm">
@@ -898,8 +1328,11 @@ function Minimap({
       <svg
         width={W}
         height={H}
-        className="block text-muted-foreground"
-        aria-hidden="true"
+        className="block cursor-crosshair text-muted-foreground"
+        role="button"
+        aria-label="Jump to minimap position"
+        tabIndex={0}
+        onClick={jumpFromMinimap}
       >
         {edges.map((e) => {
           const s = nodes.find((n) => n.id === e.source);

ontology_platform/web/frontend/src/pages/PageAnalysisPage.tsx

@@ -48,6 +48,21 @@ function candidateClaims(log: ExtractionLog | undefined): unknown[] {
   return Array.isArray(nested) ? nested : [];
 }
 
+function numberFrom(value: unknown): number {
+  return typeof value === "number" && Number.isFinite(value) ? value : 0;
+}
+
+function comparisonRecord(log: ExtractionLog | undefined): Record<string, unknown> {
+  return {
+    ...asRecord(asRecord(log?.raw_output).comparison),
+    ...asRecord(log?.comparison),
+  };
+}
+
+function comparisonCount(log: ExtractionLog | undefined, key: string): number {
+  return numberFrom(comparisonRecord(log)[key]);
+}
+
 export default function PageAnalysisPage() {
   const navigate = useNavigate();
   const { projectId } = useParams<{ projectId: string }>();
@@ -211,6 +226,29 @@ export default function PageAnalysisPage() {
                     label="Validation"
                     value={humanizeValue(selectedLog.validation)}
                   />
+                  <InfoBox
+                    label="Mode"
+                    value={humanizeValue(
+                      selectedLog.effective_extraction_mode ??
+                        selectedLog.extraction_mode ??
+                        rawOutput.effective_extraction_mode ??
+                        rawOutput.extraction_mode,
+                    )}
+                  />
+                  <InfoBox
+                    label="Rule / LLM"
+                    value={`${numberFrom(selectedLog.rule_claim_count ?? rawOutput.rule_claim_count)} / ${numberFrom(
+                      selectedLog.llm_claim_count ?? rawOutput.llm_claim_count,
+                    )}`}
+                  />
+                  <InfoBox
+                    label="LLM"
+                    value={
+                      selectedLog.llm_skipped || rawOutput.llm_skipped
+                        ? `skipped: ${humanizeValue(selectedLog.llm_skip_reason ?? rawOutput.llm_skip_reason)}`
+                        : humanizeValue(selectedLog.provider)
+                    }
+                  />
                 </div>
               )}
               {selectedLog?.error && (
@@ -221,6 +259,30 @@ export default function PageAnalysisPage() {
             </CardContent>
           </Card>
 
+          {selectedLog && (
+            <Card>
+              <CardHeader>
+                <CardTitle>Rule / LLM Comparison</CardTitle>
+                <CardDescription>
+                  Hybrid and compare runs keep agreement, difference, and conflict counts.
+                </CardDescription>
+              </CardHeader>
+              <CardContent>
+                <div className="grid gap-3 sm:grid-cols-2 lg:grid-cols-5">
+                  <ComparisonBox label="Both agree" value={comparisonCount(selectedLog, "both_agree")} tone="success" />
+                  <ComparisonBox label="Rule only" value={comparisonCount(selectedLog, "rule_only")} tone="warning" />
+                  <ComparisonBox label="LLM only" value={comparisonCount(selectedLog, "llm_only")} tone="info" />
+                  <ComparisonBox label="Conflict" value={comparisonCount(selectedLog, "conflict")} tone="danger" />
+                  <ComparisonBox
+                    label="Rejected"
+                    value={comparisonCount(selectedLog, "rejected_by_validation")}
+                    tone="muted"
+                  />
+                </div>
+              </CardContent>
+            </Card>
+          )}
+
           <div className="grid gap-6 xl:grid-cols-2">
             <Card>
               <CardHeader>
@@ -331,3 +393,30 @@ function InfoBox({ label, value }: { label: string; value: string }) {
   );
 }
 
+function ComparisonBox({
+  label,
+  value,
+  tone,
+}: {
+  label: string;
+  value: number;
+  tone: "success" | "warning" | "info" | "danger" | "muted";
+}) {
+  const toneClass =
+    tone === "success"
+      ? "border-green-200 bg-green-50 text-green-900"
+      : tone === "warning"
+        ? "border-yellow-200 bg-yellow-50 text-yellow-900"
+        : tone === "danger"
+          ? "border-red-200 bg-red-50 text-red-900"
+          : tone === "info"
+            ? "border-blue-200 bg-blue-50 text-blue-900"
+            : "border-border bg-background text-foreground";
+  return (
+    <div className={`rounded-md border px-3 py-2 ${toneClass}`}>
+      <div className="text-xs opacity-80">{label}</div>
+      <div className="mt-1 text-2xl font-semibold tabular-nums">{value}</div>
+    </div>
+  );
+}
+

ontology_platform/web/frontend/src/pages/ResearchPage.tsx

@@ -52,6 +52,11 @@ const startResearchSchema = z.object({
   max_branch: z.number().int().min(1).max(30),
   min_relevance: z.number().min(0).max(1),
   same_domain_only: z.boolean(),
+  extraction_mode: z.enum(["rule_only", "llm_only", "hybrid", "compare"]),
+  extractor_provider: z.enum(["lm_studio", "openai", "ollama"]),
+  extractor_model: z.string().optional(),
+  extractor_base_url: z.string().optional(),
+  fallback_to_rules: z.boolean(),
 });
 
 type StartResearchFormValues = z.infer<typeof startResearchSchema>;
@@ -89,6 +94,7 @@ export default function ResearchPage() {
     register,
     handleSubmit,
     setValue,
+    watch,
     formState: { errors, isSubmitting },
   } = useForm<StartResearchFormValues>({
     resolver: zodResolver(startResearchSchema),
@@ -101,12 +107,18 @@ export default function ResearchPage() {
       max_branch: 8,
       min_relevance: 0.35,
       same_domain_only: true,
+      extraction_mode: "hybrid",
+      extractor_provider: "lm_studio",
+      extractor_model: "",
+      extractor_base_url: "http://localhost:1234/v1",
+      fallback_to_rules: true,
     },
   });
 
   const onSubmit = async (values: StartResearchFormValues) => {
     try {
       setLastResult(null);
+      const usesLlm = values.extraction_mode !== "rule_only";
       const res = await startResearch.mutateAsync({
         project_name: projectName,
         source_name: values.source_name,
@@ -117,6 +129,11 @@ export default function ResearchPage() {
         max_branch: values.max_branch,
         min_relevance: values.min_relevance,
         same_domain_only: values.same_domain_only,
+        extraction_mode: values.extraction_mode,
+        extractor_provider: values.extractor_provider,
+        extractor_model: usesLlm ? values.extractor_model || null : null,
+        extractor_base_url: usesLlm ? values.extractor_base_url || null : null,
+        fallback_to_rules: values.fallback_to_rules,
       });
       setLastResult(res);
       toast.success(t("research.completed", "자율 연구가 완료되었습니다"));
@@ -130,6 +147,8 @@ export default function ResearchPage() {
   };
 
   const sources = project?.sources ?? [];
+  const extractionMode = watch("extraction_mode");
+  const usesLlm = extractionMode !== "rule_only";
 
   return (
     <div className="mx-auto max-w-6xl px-6 py-10">
@@ -344,6 +363,60 @@ export default function ResearchPage() {
                 </span>
               </label>
 
+              <div className="space-y-3 rounded-md border bg-background-subtle p-3">
+                <div className="grid grid-cols-2 gap-3">
+                  <div className="space-y-1.5">
+                    <Label htmlFor="extraction_mode">Extraction mode</Label>
+                    <Select id="extraction_mode" {...register("extraction_mode")}>
+                      <option value="hybrid">Hybrid</option>
+                      <option value="rule_only">Rule only</option>
+                      <option value="llm_only">LLM only</option>
+                      <option value="compare">Compare</option>
+                    </Select>
+                  </div>
+                  <div className="space-y-1.5">
+                    <Label htmlFor="extractor_provider">Provider</Label>
+                    <Select
+                      id="extractor_provider"
+                      disabled={!usesLlm}
+                      {...register("extractor_provider")}
+                    >
+                      <option value="lm_studio">LM Studio</option>
+                      <option value="openai">OpenAI</option>
+                      <option value="ollama">Ollama</option>
+                    </Select>
+                  </div>
+                </div>
+                {usesLlm && (
+                  <>
+                    <div className="space-y-1.5">
+                      <Label htmlFor="extractor_model">Model</Label>
+                      <Input
+                        id="extractor_model"
+                        placeholder="deepseek-r1-distill-qwen-7b"
+                        {...register("extractor_model")}
+                      />
+                    </div>
+                    <div className="space-y-1.5">
+                      <Label htmlFor="extractor_base_url">Base URL</Label>
+                      <Input
+                        id="extractor_base_url"
+                        placeholder="http://localhost:1234/v1"
+                        {...register("extractor_base_url")}
+                      />
+                    </div>
+                    <label className="flex items-center gap-2 text-sm">
+                      <input
+                        type="checkbox"
+                        className="h-4 w-4 rounded border-input"
+                        {...register("fallback_to_rules")}
+                      />
+                      <span>Fallback to rules</span>
+                    </label>
+                  </>
+                )}
+              </div>
+
               <Button
                 type="submit"
                 className="w-full"

ontology_platform/web/frontend/src/pages/ReviewPage.tsx

@@ -60,12 +60,37 @@ function claimObjectText(claim: Claim): string {
   return humanizeValue(claim.object ?? claim.object_value);
 }
 
+function agreementText(claim: Claim): string {
+  return humanizeValue(claim.agreement ?? claim.claim_kind ?? "unknown");
+}
+
+function agreementVariant(claim: Claim): BadgeProps["variant"] {
+  switch ((claim.agreement ?? "").toLowerCase()) {
+    case "rule_and_llm":
+      return "success";
+    case "conflict":
+      return "destructive";
+    case "rule_only":
+    case "llm_only":
+      return "warning";
+    default:
+      return "outline";
+  }
+}
+
 const STATUS_LABELS: Record<string, string> = {
   candidate: "Candidate",
   approved: "Approved",
   rejected: "Rejected",
 };
 
+const AGREEMENT_LABELS: Record<string, string> = {
+  rule_and_llm: "Rule + LLM",
+  rule_only: "Rule only",
+  llm_only: "LLM only",
+  conflict: "Conflict",
+};
+
 export default function ReviewPage() {
   const navigate = useNavigate();
   const { projectId } = useParams<{ projectId: string }>();
@@ -78,6 +103,7 @@ export default function ReviewPage() {
   const updateStatus = useUpdateClaimStatus(projectName);
 
   const [statusFilter, setStatusFilter] = useState<string>("");
+  const [agreementFilter, setAgreementFilter] = useState<string>("");
   const [search, setSearch] = useState("");
   const [selectedId, setSelectedId] = useState<string | null>(null);
   const [drawerOpen, setDrawerOpen] = useState(false);
@@ -90,6 +116,9 @@ export default function ReviewPage() {
       if (statusFilter && reviewLabel(claim.status) !== statusFilter) {
         return false;
       }
+      if (agreementFilter && (claim.agreement ?? "") !== agreementFilter) {
+        return false;
+      }
       if (!query) return true;
       return [
         claim.subject,
@@ -102,7 +131,7 @@ export default function ReviewPage() {
         .map((value) => humanizeValue(value, "").toLowerCase())
         .some((value) => value.includes(query));
     });
-  }, [claims.data, search, statusFilter]);
+  }, [agreementFilter, claims.data, search, statusFilter]);
 
   const selectedClaim = useMemo(() => {
     if (!selectedId) return null;
@@ -122,6 +151,16 @@ export default function ReviewPage() {
     };
   }, [claims.data]);
 
+  const agreementCounts = useMemo(() => {
+    const result: Record<string, number> = {};
+    for (const claim of claims.data ?? []) {
+      const key = claim.agreement ?? "";
+      if (!key) continue;
+      result[key] = (result[key] ?? 0) + 1;
+    }
+    return result;
+  }, [claims.data]);
+
   /* --------------------- Actions --------------------- */
 
   const applyStatus = async (claim: Claim, status: string) => {
@@ -166,9 +205,17 @@ export default function ReviewPage() {
       value: STATUS_LABELS[statusFilter] ?? statusFilter,
       onClear: () => setStatusFilter(""),
     });
+  if (agreementFilter)
+    chips.push({
+      id: "agreement",
+      label: "Agreement",
+      value: AGREEMENT_LABELS[agreementFilter] ?? agreementFilter,
+      onClear: () => setAgreementFilter(""),
+    });
   const clearAll = () => {
     setSearch("");
     setStatusFilter("");
+    setAgreementFilter("");
   };
 
   /* --------------------- Table columns --------------------- */
@@ -186,6 +233,28 @@ export default function ReviewPage() {
       size: 110,
       enablePinning: true,
     },
+    {
+      id: "agreement",
+      header: "Agreement",
+      accessorFn: (row) => row.agreement ?? "",
+      cell: ({ row }) => (
+        <Badge variant={agreementVariant(row.original)}>
+          {agreementText(row.original)}
+        </Badge>
+      ),
+      size: 130,
+    },
+    {
+      id: "method",
+      header: "Method",
+      accessorFn: (row) => row.extraction_source ?? row.claim_kind ?? "",
+      cell: ({ row }) => (
+        <span className="text-xs text-muted-foreground">
+          {humanizeValue(row.original.extraction_source ?? row.original.claim_kind)}
+        </span>
+      ),
+      size: 110,
+    },
     {
       id: "subject",
       header: "Subject",
@@ -377,6 +446,38 @@ export default function ReviewPage() {
               { value: "rejected", label: "Rejected", hint: counts.rejected, swatch: "hsl(var(--danger))" },
             ]}
           />
+          <FilterSelect
+            label="Agreement"
+            value={agreementFilter}
+            onChange={setAgreementFilter}
+            placeholder="All agreements"
+            options={[
+              {
+                value: "rule_and_llm",
+                label: "Rule + LLM",
+                hint: agreementCounts.rule_and_llm ?? 0,
+                swatch: "hsl(var(--success))",
+              },
+              {
+                value: "rule_only",
+                label: "Rule only",
+                hint: agreementCounts.rule_only ?? 0,
+                swatch: "hsl(var(--warning))",
+              },
+              {
+                value: "llm_only",
+                label: "LLM only",
+                hint: agreementCounts.llm_only ?? 0,
+                swatch: "hsl(var(--info))",
+              },
+              {
+                value: "conflict",
+                label: "Conflict",
+                hint: agreementCounts.conflict ?? 0,
+                swatch: "hsl(var(--danger))",
+              },
+            ]}
+          />
         </FilterPanel>
 
         <DataTable<Claim>
@@ -498,14 +599,28 @@ export default function ReviewPage() {
             {/* Detail grid */}
             <div className="grid gap-3 sm:grid-cols-2">
               <DetailRow label="Source" value={selectedClaim.source} />
+              <DetailRow
+                label="Agreement"
+                value={agreementText(selectedClaim)}
+              />
               <DetailRow
                 label="Page Type"
                 value={selectedClaim.page_type}
               />
+              <DetailRow
+                label="Extraction Source"
+                value={selectedClaim.extraction_source ?? selectedClaim.claim_kind}
+              />
               <DetailRow
                 label="Extraction Method"
                 value={selectedClaim.extraction_method}
               />
+              <DetailRow
+                label="Rule / LLM Confidence"
+                value={`${formatPercent(selectedClaim.rule_confidence)} / ${formatPercent(
+                  selectedClaim.llm_confidence,
+                )}`}
+              />
               <DetailRow
                 label="Validation"
                 value={
@@ -539,6 +654,13 @@ export default function ReviewPage() {
                   full
                 />
               )}
+              {selectedClaim.review_reason && (
+                <DetailRow
+                  label="Review Reason"
+                  value={selectedClaim.review_reason}
+                  full
+                />
+              )}
             </div>
 
             {/* Evidence */}

restart_all_servers.bat

@@ -43,6 +43,184 @@ function Get-CommandPath {
     return $null
 }
 
+function Test-PythonModule {
+    param(
+        [string] $PythonPath,
+        [string] $ModuleName
+    )
+
+    if ([string]::IsNullOrWhiteSpace($PythonPath)) {
+        return $false
+    }
+
+    try {
+        $proc = Start-Process `
+            -FilePath $PythonPath `
+            -ArgumentList @("-c", "import $ModuleName") `
+            -WindowStyle Hidden `
+            -Wait `
+            -PassThru
+        return $proc.ExitCode -eq 0
+    } catch {
+        return $false
+    }
+}
+
+function Resolve-ProjectPython {
+    param([string] $ProjectRoot)
+
+    $projectVenvPython = Join-Path $ProjectRoot ".venv\Scripts\python.exe"
+    if (Test-Path -LiteralPath $projectVenvPython) {
+        return (Resolve-Path -LiteralPath $projectVenvPython).Path
+    }
+
+    $rootVenvPython = Join-Path $root ".venv\Scripts\python.exe"
+    if (Test-Path -LiteralPath $rootVenvPython) {
+        return (Resolve-Path -LiteralPath $rootVenvPython).Path
+    }
+
+    $candidates = @()
+
+    if ($env:VIRTUAL_ENV) {
+        $candidates += Join-Path $env:VIRTUAL_ENV "Scripts\python.exe"
+    }
+
+    $pyLauncher = Get-CommandPath @("py.exe", "py")
+    if ($pyLauncher) {
+        try {
+            $resolvedFromPy = & $pyLauncher -c "import sys; print(sys.executable)" 2>$null
+            if ($LASTEXITCODE -eq 0 -and -not [string]::IsNullOrWhiteSpace($resolvedFromPy)) {
+                $candidates += $resolvedFromPy.Trim()
+            }
+        } catch {
+        }
+    }
+
+    $pathPython = Get-CommandPath @("python.exe", "python")
+    if ($pathPython) {
+        $candidates += $pathPython
+    }
+
+    $seen = @{}
+    foreach ($candidate in $candidates) {
+        if ([string]::IsNullOrWhiteSpace($candidate)) {
+            continue
+        }
+
+        $resolved = $null
+        if (Test-Path -LiteralPath $candidate) {
+            $resolved = (Resolve-Path -LiteralPath $candidate).Path
+        } else {
+            $cmd = Get-Command $candidate -ErrorAction SilentlyContinue
+            if ($cmd) {
+                $resolved = $cmd.Source
+            }
+        }
+
+        if (-not $resolved -or $seen.ContainsKey($resolved)) {
+            continue
+        }
+        $seen[$resolved] = $true
+
+        if (Test-PythonModule -PythonPath $resolved -ModuleName "uvicorn") {
+            return $resolved
+        }
+    }
+
+    return $null
+}
+
+function Resolve-SystemPython {
+    $candidates = @()
+
+    if ($env:VIRTUAL_ENV) {
+        $candidates += Join-Path $env:VIRTUAL_ENV "Scripts\python.exe"
+    }
+
+    $pathPython = Get-CommandPath @("python.exe", "python")
+    if ($pathPython) {
+        $candidates += $pathPython
+    }
+
+    $pyLauncher = Get-CommandPath @("py.exe", "py")
+    if ($pyLauncher) {
+        try {
+            $resolvedFromPy = & $pyLauncher -c "import sys; print(sys.executable)" 2>$null
+            if ($LASTEXITCODE -eq 0 -and -not [string]::IsNullOrWhiteSpace($resolvedFromPy)) {
+                $candidates += $resolvedFromPy.Trim()
+            }
+        } catch {
+        }
+    }
+
+    $seen = @{}
+    foreach ($candidate in $candidates) {
+        if ([string]::IsNullOrWhiteSpace($candidate)) {
+            continue
+        }
+        $resolved = $null
+        if (Test-Path -LiteralPath $candidate) {
+            $resolved = (Resolve-Path -LiteralPath $candidate).Path
+        } else {
+            $cmd = Get-Command $candidate -ErrorAction SilentlyContinue
+            if ($cmd) {
+                $resolved = $cmd.Source
+            }
+        }
+        if (-not $resolved -or $seen.ContainsKey($resolved)) {
+            continue
+        }
+        $seen[$resolved] = $true
+        return $resolved
+    }
+
+    return $null
+}
+
+function Ensure-ProjectVenv {
+    param([string] $ProjectRoot)
+
+    $venvPython = Join-Path $ProjectRoot ".venv\Scripts\python.exe"
+    if (Test-PythonModule -PythonPath $venvPython -ModuleName "uvicorn") {
+        return $venvPython
+    }
+
+    $bootstrapPython = Resolve-SystemPython
+    if (-not $bootstrapPython) {
+        Write-Step "No system Python was found for bootstrapping the backend venv."
+        return $null
+    }
+
+    Write-Step "Bootstrapping backend virtual environment in $ProjectRoot\\.venv"
+
+    if (-not (Test-Path -LiteralPath $venvPython)) {
+        Invoke-LoggedCommand `
+            -FilePath $bootstrapPython `
+            -Arguments @("-m", "venv", ".venv") `
+            -WorkingDirectory $ProjectRoot `
+            -LogName "backend-venv-create.log"
+    }
+
+    Invoke-LoggedCommand `
+        -FilePath $venvPython `
+        -Arguments @("-m", "pip", "install", "--upgrade", "pip") `
+        -WorkingDirectory $ProjectRoot `
+        -LogName "backend-pip-upgrade.log"
+
+    Invoke-LoggedCommand `
+        -FilePath $venvPython `
+        -Arguments @("-m", "pip", "install", "-e", ".") `
+        -WorkingDirectory $ProjectRoot `
+        -LogName "backend-pip-install.log"
+
+    if (Test-PythonModule -PythonPath $venvPython -ModuleName "uvicorn") {
+        return $venvPython
+    }
+
+    Write-Step "Backend venv was created, but uvicorn is still unavailable. Check .server-logs\\backend-*.log"
+    return $null
+}
+
 function Read-DotEnv {
     param([string] $Path)
 
@@ -110,6 +288,32 @@ function Stop-ExistingServers {
     Start-Sleep -Milliseconds 800
 }
 
+function Quote-ProcessArguments {
+    param([string[]] $Arguments)
+
+    if (-not $Arguments) {
+        return @()
+    }
+    $quoted = @()
+    foreach ($arg in $Arguments) {
+        if ($null -eq $arg) {
+            continue
+        }
+        $text = [string] $arg
+        if ($text.Length -eq 0) {
+            $quoted += '""'
+            continue
+        }
+        if ($text -match '\s' -and -not ($text.StartsWith('"') -and $text.EndsWith('"'))) {
+            $escaped = $text -replace '"', '\"'
+            $quoted += '"' + $escaped + '"'
+        } else {
+            $quoted += $text
+        }
+    }
+    return $quoted
+}
+
 function Invoke-LoggedCommand {
     param(
         [string] $FilePath,
@@ -122,10 +326,12 @@ function Invoke-LoggedCommand {
     $stderrPath = Join-Path $logDir ($LogName -replace "\.log$", ".stderr.log")
     Write-Step ("Running {0} {1}" -f (Split-Path -Leaf $FilePath), ($Arguments -join " "))
 
+    $quotedArgs = Quote-ProcessArguments -Arguments $Arguments
+
     Remove-Item -LiteralPath $logPath, $stderrPath -Force -ErrorAction SilentlyContinue
     $proc = Start-Process `
         -FilePath $FilePath `
-        -ArgumentList $Arguments `
+        -ArgumentList $quotedArgs `
         -WorkingDirectory $WorkingDirectory `
         -RedirectStandardOutput $logPath `
         -RedirectStandardError $stderrPath `
@@ -160,10 +366,12 @@ function Start-LoggedServer {
         [Environment]::SetEnvironmentVariable($key, [string] $Environment[$key], "Process")
     }
 
+    $quotedArgs = Quote-ProcessArguments -Arguments $Arguments
+
     try {
         $proc = Start-Process `
             -FilePath $FilePath `
-            -ArgumentList $Arguments `
+            -ArgumentList $quotedArgs `
             -WorkingDirectory $WorkingDirectory `
             -RedirectStandardOutput $stdout `
             -RedirectStandardError $stderr `
@@ -185,6 +393,15 @@ function Start-LoggedServer {
     }
 }
 
+function Clear-ServerLogs {
+    param([string] $Name)
+
+    Remove-Item -LiteralPath `
+        (Join-Path $logDir "$Name-stdout.log"), `
+        (Join-Path $logDir "$Name-stderr.log") `
+        -Force -ErrorAction SilentlyContinue
+}
+
 function Wait-Port {
     param(
         [int] $Port,
@@ -209,6 +426,28 @@ function Wait-Port {
     return $false
 }
 
+function Resolve-FrontendDevCommand {
+    param(
+        [string] $FrontendDir,
+        [string] $NpmPath
+    )
+
+    $node = Get-CommandPath @("node.exe", "node")
+    $viteCli = Join-Path $FrontendDir "node_modules\vite\bin\vite.js"
+
+    if ($node -and (Test-Path -LiteralPath $viteCli)) {
+        return [pscustomobject]@{
+            FilePath = $node
+            Arguments = @($viteCli, "--host", "127.0.0.1", "--port", "8000")
+        }
+    }
+
+    return [pscustomobject]@{
+        FilePath = $NpmPath
+        Arguments = @("run", "dev", "--", "--host", "127.0.0.1", "--port", "8000")
+    }
+}
+
 function Start-Neo4jIfAvailable {
     $composeFile = Join-Path $root "docker-compose.neo4j.yml"
     if (-not (Test-Path -LiteralPath $composeFile)) {
@@ -249,13 +488,11 @@ if ((Test-Path -LiteralPath (Join-Path $frontendDir "package.json")) -and $npm)
 $servers = @()
 
 $ontologyRoot = Join-Path $root "ontology_platform"
-$ontologyPython = Get-CommandPath @(
-    (Join-Path $ontologyRoot ".venv\Scripts\python.exe"),
-    (Join-Path $root ".venv\Scripts\python.exe"),
-    "C:\Users\lasta\AppData\Local\Python\bin\python.exe",
-    "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\python.exe",
-    "python"
-)
+$ontologyPython = Resolve-ProjectPython -ProjectRoot $ontologyRoot
+
+if ((-not $ontologyPython) -and (Test-Path -LiteralPath (Join-Path $ontologyRoot "pyproject.toml"))) {
+    $ontologyPython = Ensure-ProjectVenv -ProjectRoot $ontologyRoot
+}
 
 if ((Test-Path -LiteralPath (Join-Path $ontologyRoot "ont_platform\api\main.py")) -and $ontologyPython) {
     $ontologyEnv = Read-DotEnv -Path (Join-Path $ontologyRoot ".env")
@@ -271,15 +508,18 @@ if ((Test-Path -LiteralPath (Join-Path $ontologyRoot "ont_platform\api\main.py")
         -WorkingDirectory $ontologyRoot `
         -Environment $ontologyEnv
 } else {
-    Write-Step "ontology_platform server entrypoint or Python was not found; skipped."
+    Clear-ServerLogs -Name "ontology-platform-8001"
+    Write-Step "ontology_platform server entrypoint was not found, or no Python with uvicorn is available for this project."
+    Write-Step "Create a venv under ontology_platform\\.venv (or .\\.venv) and install dependencies before rerunning."
 }
 
 if ($frontendReady) {
+    $frontendDev = Resolve-FrontendDevCommand -FrontendDir $frontendDir -NpmPath $npm
     $servers += Start-LoggedServer `
         -Name "ontology-frontend-8000" `
         -Port 8000 `
-        -FilePath $npm `
-        -Arguments @("run", "dev", "--", "--host", "127.0.0.1", "--port", "8000") `
+        -FilePath $frontendDev.FilePath `
+        -Arguments $frontendDev.Arguments `
         -WorkingDirectory $frontendDir
 }
 
@@ -290,7 +530,7 @@ $servers | ForEach-Object {
 
 $missingPorts = @()
 foreach ($port in ($servers | Select-Object -ExpandProperty Port -Unique)) {
-    if (Wait-Port -Port $port -Seconds 25) {
+    if (Wait-Port -Port $port -Seconds 120) {
         Write-Step ("Port {0} is listening." -f $port)
     } else {
         Write-Step ("Port {0} did not open. Check logs in {1}" -f $port, $logDir)
@@ -304,6 +544,10 @@ if ($missingPorts.Count -gt 0) {
 
 Write-Host ""
 Write-Step "Done."
-Write-Host "  Ontology UI:        http://127.0.0.1:8000/static/"
-Write-Host "  Ontology API:       http://127.0.0.1:8001/docs"
+if ($servers | Where-Object { $_.Port -eq 8000 }) {
+    Write-Host "  Ontology UI:        http://127.0.0.1:8000/static/"
+}
+if ($servers | Where-Object { $_.Port -eq 8001 }) {
+    Write-Host "  Ontology API:       http://127.0.0.1:8001/docs"
+}
 Write-Host "  Logs:               $logDir"

tests/api/test_phase5_app.py

@@ -1,448 +0,0 @@
-"""Phase 5 GraphRAG API endpoint tests.
-
-Tests for:
-- Entity duplicate detection endpoint
-- Subgraph extraction endpoints (N-hop and semantic)
-- Pattern matching endpoints
-- Graph analytics endpoints
-- Health check endpoints
-"""
-
-import pytest
-from unittest.mock import AsyncMock, MagicMock, patch
-from datetime import datetime, UTC
-
-from ont_platform.api.phase5_app import app, graph_router
-from ont_platform.core.graph import EntityCluster, EntityResolver
-from fastapi.testclient import TestClient
-
-
-@pytest.fixture
-def client():
-    """FastAPI test client."""
-    return TestClient(app)
-
-
-class TestHealthCheck:
-    """Test health check endpoint."""
-
-    def test_health_check_endpoint(self, client):
-        """Test GET /health endpoint."""
-        response = client.get("/health")
-        assert response.status_code == 200
-        data = response.json()
-        assert data["status"] == "healthy"
-        assert data["version"] == "0.5.0"
-        assert data["phase"] == "5 (GraphRAG)"
-        assert "components" in data
-
-    def test_platform_info_endpoint(self, client):
-        """Test GET /info endpoint."""
-        response = client.get("/info")
-        assert response.status_code == 200
-        data = response.json()
-        assert data["platform"] == "Ontology System Construction Platform"
-        assert data["phase"] == "5 (GraphRAG)"
-        assert data["version"] == "0.5.0"
-        assert "features" in data
-
-
-class TestEntityResolutionEndpoint:
-    """Test entity duplicate detection endpoint."""
-
-    @pytest.mark.asyncio
-    async def test_resolve_duplicates_success(self, client):
-        """Test successful entity duplicate detection."""
-        with patch("ont_platform.api.phase5_app.entity_resolver") as mock_resolver:
-            # Setup mock
-            mock_cluster = EntityCluster(
-                cluster_id="C_1_2",
-                canonical_id=1,
-                duplicates=[2],
-                confidence=0.92,
-                reason="combined",
-                metadata={"vector_similarity": 0.95, "text_similarity": 0.89},
-            )
-            mock_resolver.embedder = MagicMock()
-            mock_resolver.initialize_embedder = AsyncMock(return_value=True)
-            mock_resolver.detect_duplicates = AsyncMock(return_value=[mock_cluster])
-            mock_resolver.get_resolution_report = MagicMock(
-                return_value={
-                    "total_clusters": 1,
-                    "total_duplicates": 1,
-                    "avg_confidence": 0.92,
-                    "by_reason": {"combined": 1},
-                    "timestamp": datetime.now(UTC).isoformat(),
-                }
-            )
-
-            # Test
-            entities_json = [
-                {"id": 1, "label": "Apple Inc", "type": "company"},
-                {"id": 2, "label": "Apple Incorporated", "type": "company"},
-            ]
-
-            # NOTE: TestClient doesn't support Query params in POST body directly
-            # In real usage, these would be query parameters or request body
-            response = client.post(
-                "/api/v1/graph/resolve",
-                json={"entities": entities_json},
-            )
-
-            # The endpoint expects Query params, so this test validates the API structure
-            # Actual integration testing would use proper query parameters
-            if response.status_code == 422:  # Validation error expected with TestClient
-                assert "detail" in response.json()
-
-    def test_resolve_duplicates_missing_entities(self, client):
-        """Test resolve endpoint with missing entities parameter."""
-        response = client.post("/api/v1/graph/resolve")
-        assert response.status_code == 422  # Unprocessable entity
-
-
-class TestSubgraphExtractionEndpoint:
-    """Test subgraph extraction endpoints."""
-
-    @pytest.mark.asyncio
-    async def test_extract_neighborhood_success(self, client):
-        """Test N-hop neighborhood extraction."""
-        with patch("ont_platform.api.phase5_app.subgraph_retriever") as mock_retriever:
-            mock_retriever.retrieve_neighborhood = AsyncMock(
-                return_value={
-                    "center_entity": {"id": 1, "label": "Apple", "type": "company"},
-                    "nodes": [
-                        {"id": 1, "label": "Apple", "type": "company"},
-                        {"id": 2, "label": "Tim Cook", "type": "person"},
-                    ],
-                    "edges": [
-                        {
-                            "source_id": 1,
-                            "target_id": 2,
-                            "predicate": "HAS_CEO",
-                            "confidence": 0.95,
-                        }
-                    ],
-                    "hop_count": 1,
-                    "node_count": 2,
-                    "edge_count": 1,
-                }
-            )
-
-            response = client.post(
-                "/api/v1/graph/subgraph?entity_id=1&hops=1&min_confidence=0.0"
-            )
-            assert response.status_code == 200
-            data = response.json()
-            assert data["status"] == "success"
-            assert data["entity_id"] == 1
-            assert data["hops"] == 1
-
-    def test_extract_neighborhood_missing_entity_id(self, client):
-        """Test subgraph extraction without entity_id."""
-        response = client.post("/api/v1/graph/subgraph")
-        assert response.status_code == 422
-
-    @pytest.mark.asyncio
-    async def test_extract_semantic_subgraph_success(self, client):
-        """Test semantic subgraph extraction."""
-        with patch("ont_platform.api.phase5_app.subgraph_retriever") as mock_retriever:
-            mock_retriever.embedder = MagicMock()
-            mock_retriever.retrieve_by_semantic_query = AsyncMock(
-                return_value={
-                    "query": "tech companies",
-                    "query_embedding_dimension": 384,
-                    "matched_entities": [
-                        {"id": 1, "label": "Apple", "similarity": 0.92},
-                        {"id": 2, "label": "Microsoft", "similarity": 0.89},
-                    ],
-                    "nodes": [
-                        {"id": 1, "label": "Apple", "type": "company"},
-                        {"id": 2, "label": "Microsoft", "type": "company"},
-                    ],
-                    "edges": [],
-                    "matched_count": 2,
-                    "node_count": 2,
-                    "edge_count": 0,
-                }
-            )
-
-            response = client.post(
-                "/api/v1/graph/subgraph/semantic?query=tech+companies&top_k=10&min_similarity=0.6&hops=1"
-            )
-            assert response.status_code == 200
-            data = response.json()
-            assert data["status"] == "success"
-            assert data["query"] == "tech companies"
-
-    def test_extract_semantic_subgraph_missing_query(self, client):
-        """Test semantic subgraph without query parameter."""
-        response = client.post("/api/v1/graph/subgraph/semantic")
-        assert response.status_code == 422
-
-
-class TestPatternMatchingEndpoints:
-    """Test pattern matching endpoints."""
-
-    @pytest.mark.asyncio
-    async def test_find_paths_success(self, client):
-        """Test path finding between entities."""
-        with patch("ont_platform.api.phase5_app.pattern_matcher") as mock_matcher:
-            mock_matcher.find_paths = AsyncMock(
-                return_value=[
-                    {
-                        "path": [1, "rel1", 2, "rel2", 3],
-                        "length": 2,
-                        "confidence": 0.85,
-                    }
-                ]
-            )
-
-            response = client.post(
-                "/api/v1/graph/patterns/paths?start_id=1&end_id=3&max_length=5"
-            )
-            assert response.status_code == 200
-            data = response.json()
-            assert data["status"] == "success"
-            assert data["start_id"] == 1
-            assert data["end_id"] == 3
-            assert data["paths_found"] == 1
-
-    def test_find_paths_missing_parameters(self, client):
-        """Test path finding without required parameters."""
-        response = client.post("/api/v1/graph/patterns/paths")
-        assert response.status_code == 422
-
-    @pytest.mark.asyncio
-    async def test_find_cycles_success(self, client):
-        """Test cycle detection."""
-        with patch("ont_platform.api.phase5_app.pattern_matcher") as mock_matcher:
-            mock_matcher.find_cycles = AsyncMock(
-                return_value=[
-                    {
-                        "cycle": [1, 2, 3, 1],
-                        "length": 3,
-                        "confidence": 0.80,
-                    }
-                ]
-            )
-
-            response = client.post("/api/v1/graph/patterns/cycles")
-            assert response.status_code == 200
-            data = response.json()
-            assert data["status"] == "success"
-            assert data["cycles_found"] == 1
-
-
-class TestGraphAnalyticsEndpoints:
-    """Test graph analytics endpoints."""
-
-    @pytest.mark.asyncio
-    async def test_analyze_centrality_pagerank(self, client):
-        """Test PageRank centrality analysis."""
-        with patch("ont_platform.api.phase5_app.graph_analytics") as mock_analytics:
-            mock_analytics.calculate_centrality = AsyncMock(
-                return_value=[
-                    {"entity_id": 1, "label": "Apple", "score": 0.35},
-                    {"entity_id": 2, "label": "Microsoft", "score": 0.28},
-                ]
-            )
-
-            response = client.post(
-                "/api/v1/graph/analytics/centrality?centrality_type=pagerank&top_n=10"
-            )
-            assert response.status_code == 200
-            data = response.json()
-            assert data["status"] == "success"
-            assert data["centrality_type"] == "pagerank"
-            assert data["top_n"] == 10
-
-    @pytest.mark.asyncio
-    async def test_analyze_centrality_degree(self, client):
-        """Test degree centrality analysis."""
-        with patch("ont_platform.api.phase5_app.graph_analytics") as mock_analytics:
-            mock_analytics.calculate_centrality = AsyncMock(return_value=[])
-
-            response = client.post(
-                "/api/v1/graph/analytics/centrality?centrality_type=degree&top_n=5"
-            )
-            assert response.status_code == 200
-            data = response.json()
-            assert data["centrality_type"] == "degree"
-
-    def test_analyze_centrality_invalid_type(self, client):
-        """Test centrality with invalid type parameter."""
-        response = client.post(
-            "/api/v1/graph/analytics/centrality?centrality_type=invalid_type&top_n=10"
-        )
-        assert response.status_code == 422
-
-    @pytest.mark.asyncio
-    async def test_detect_communities_louvain(self, client):
-        """Test community detection with Louvain."""
-        with patch("ont_platform.api.phase5_app.graph_analytics") as mock_analytics:
-            mock_analytics.detect_communities = AsyncMock(
-                return_value=[
-                    {
-                        "community_id": "C1",
-                        "size": 15,
-                        "density": 0.72,
-                    },
-                    {
-                        "community_id": "C2",
-                        "size": 12,
-                        "density": 0.65,
-                    },
-                ]
-            )
-
-            response = client.post(
-                "/api/v1/graph/analytics/communities?algorithm=louvain"
-            )
-            assert response.status_code == 200
-            data = response.json()
-            assert data["status"] == "success"
-            assert data["algorithm"] == "louvain"
-            assert data["communities_found"] == 2
-
-    @pytest.mark.asyncio
-    async def test_detect_communities_leiden(self, client):
-        """Test community detection with Leiden."""
-        with patch("ont_platform.api.phase5_app.graph_analytics") as mock_analytics:
-            mock_analytics.detect_communities = AsyncMock(return_value=[])
-
-            response = client.post(
-                "/api/v1/graph/analytics/communities?algorithm=leiden"
-            )
-            assert response.status_code == 200
-            data = response.json()
-            assert data["algorithm"] == "leiden"
-
-    def test_detect_communities_invalid_algorithm(self, client):
-        """Test community detection with invalid algorithm."""
-        response = client.post(
-            "/api/v1/graph/analytics/communities?algorithm=invalid_algo"
-        )
-        assert response.status_code == 422
-
-
-class TestAPIErrorHandling:
-    """Test error handling in API endpoints."""
-
-    @pytest.mark.asyncio
-    async def test_resolve_duplicates_error_handling(self, client):
-        """Test error handling in duplicate resolution."""
-        with patch("ont_platform.api.phase5_app.entity_resolver") as mock_resolver:
-            mock_resolver.embedder = MagicMock()
-            mock_resolver.initialize_embedder = AsyncMock(
-                side_effect=RuntimeError("Model load failed")
-            )
-
-            # Since the endpoint calls initialize_embedder and handles exceptions,
-            # we expect the error to be caught and returned as HTTP 500
-            response = client.post(
-                "/api/v1/graph/resolve",
-                json={"entities": [{"id": 1, "label": "Test"}]},
-            )
-            # Validation error due to Query param mismatch
-            assert response.status_code in [422, 500]
-
-    @pytest.mark.asyncio
-    async def test_subgraph_extraction_error_handling(self, client):
-        """Test error handling in subgraph extraction."""
-        with patch("ont_platform.api.phase5_app.subgraph_retriever") as mock_retriever:
-            mock_retriever.retrieve_neighborhood = AsyncMock(
-                side_effect=Exception("Neo4j connection failed")
-            )
-
-            response = client.post(
-                "/api/v1/graph/subgraph?entity_id=999&hops=1&min_confidence=0.0"
-            )
-            assert response.status_code == 500
-
-
-class TestParameterValidation:
-    """Test parameter validation for all endpoints."""
-
-    def test_subgraph_hops_validation(self, client):
-        """Test hops parameter validation (1-3 range)."""
-        # hops = 0 (below minimum)
-        response = client.post(
-            "/api/v1/graph/subgraph?entity_id=1&hops=0&min_confidence=0.0"
-        )
-        assert response.status_code == 422
-
-        # hops = 4 (above maximum)
-        response = client.post(
-            "/api/v1/graph/subgraph?entity_id=1&hops=4&min_confidence=0.0"
-        )
-        assert response.status_code == 422
-
-    def test_confidence_validation(self, client):
-        """Test min_confidence parameter validation (0-1 range)."""
-        # Negative confidence
-        response = client.post(
-            "/api/v1/graph/subgraph?entity_id=1&hops=1&min_confidence=-0.1"
-        )
-        assert response.status_code == 422
-
-        # Confidence > 1
-        response = client.post(
-            "/api/v1/graph/subgraph?entity_id=1&hops=1&min_confidence=1.5"
-        )
-        assert response.status_code == 422
-
-    def test_similarity_validation(self, client):
-        """Test min_similarity parameter validation."""
-        # Valid similarity
-        response = client.post(
-            "/api/v1/graph/subgraph/semantic?query=test&min_similarity=0.5"
-        )
-        # Will fail due to missing embedder, but validation passes
-        assert response.status_code in [200, 500]
-
-        # Invalid similarity
-        response = client.post(
-            "/api/v1/graph/subgraph/semantic?query=test&min_similarity=-0.1"
-        )
-        assert response.status_code == 422
-
-    def test_top_k_validation(self, client):
-        """Test top_k parameter validation."""
-        # top_k = 0 (invalid)
-        response = client.post(
-            "/api/v1/graph/subgraph/semantic?query=test&top_k=0"
-        )
-        assert response.status_code == 422
-
-        # top_k = 150 (above maximum 100)
-        response = client.post(
-            "/api/v1/graph/subgraph/semantic?query=test&top_k=150"
-        )
-        assert response.status_code == 422
-
-
-class TestEndpointRouting:
-    """Test API endpoint routing and versioning."""
-
-    def test_api_version_prefix(self, client):
-        """Test API routes use /api/v1/graph prefix."""
-        # Test that health endpoint is not under graph prefix
-        response = client.get("/health")
-        assert response.status_code == 200
-
-        # Test graph endpoints use correct prefix
-        response = client.post("/api/v1/graph/resolve")
-        assert response.status_code != 404  # Endpoint exists
-
-    def test_semantic_subgraph_separate_route(self, client):
-        """Test semantic subgraph has separate route."""
-        # /subgraph/semantic should be separate from /subgraph
-        response = client.post(
-            "/api/v1/graph/subgraph/semantic?query=test"
-        )
-        # May fail due to missing embedder, but route should exist
-        assert response.status_code in [200, 500]
-
-
-if __name__ == "__main__":
-    pytest.main([__file__, "-v"])

tests/core/graph/test_entity_resolver.py

@@ -1,338 +0,0 @@
-"""Phase 5 Entity Resolver tests.
-
-Tests vector + text similarity-based duplicate detection:
-- Normalization
-- Jaro-Winkler similarity
-- Vector embedding similarity
-- Duplicate detection and merging
-"""
-
-import pytest
-from datetime import datetime
-
-from ont_platform.core.graph import EntityResolver, EntityCluster
-
-
-class TestEntityNormalization:
-    """Test label normalization."""
-
-    def test_normalize_label_lowercase(self):
-        """Test lowercase normalization."""
-        resolver = EntityResolver()
-        assert resolver._normalize_label("Apple Inc") == "apple inc"
-        assert resolver._normalize_label("SAMSUNG") == "samsung"
-
-    def test_normalize_label_special_chars(self):
-        """Test special character removal."""
-        resolver = EntityResolver()
-        assert resolver._normalize_label("Apple-Inc") == "apple inc"
-        assert resolver._normalize_label("Samsung_Electronics") == "samsung electronics"
-        assert resolver._normalize_label("IBM@Corp!") == "ibmcorp"
-
-    def test_normalize_label_articles(self):
-        """Test article removal."""
-        resolver = EntityResolver()
-        assert resolver._normalize_label("The Apple Inc") == "apple inc"
-        assert resolver._normalize_label("A Samsung") == "samsung"
-        assert resolver._normalize_label("An IBM") == "ibm"
-
-    def test_normalize_label_whitespace(self):
-        """Test whitespace collapse."""
-        resolver = EntityResolver()
-        assert resolver._normalize_label("Apple   Inc") == "apple inc"
-        assert resolver._normalize_label("  Samsung  ") == "samsung"
-
-
-class TestJaroWinklerSimilarity:
-    """Test Jaro-Winkler text similarity."""
-
-    def test_exact_match(self):
-        """Test exact string match."""
-        resolver = EntityResolver()
-        assert resolver._jaro_winkler_similarity("apple", "apple") == 1.0
-
-    def test_partial_match(self):
-        """Test partial string match."""
-        resolver = EntityResolver()
-        sim = resolver._jaro_winkler_similarity("apple", "aple")
-        assert 0.8 < sim < 1.0
-
-    def test_different_strings(self):
-        """Test different strings."""
-        resolver = EntityResolver()
-        sim = resolver._jaro_winkler_similarity("apple", "banana")
-        assert 0 <= sim < 0.5  # Adjusted threshold based on actual Jaro-Winkler
-
-    def test_case_sensitivity(self):
-        """Test that Jaro-Winkler is case-sensitive."""
-        resolver = EntityResolver()
-        # Fallback SequenceMatcher is case-sensitive
-        sim1 = resolver._jaro_winkler_similarity("Apple", "apple")
-        sim2 = resolver._jaro_winkler_similarity("apple", "apple")
-        # sim1 should be less than sim2
-        assert sim1 <= sim2
-
-
-class TestTextSimilarity:
-    """Test text similarity computation."""
-
-    def test_exact_match(self):
-        """Test exact label match."""
-        resolver = EntityResolver()
-        assert resolver._compute_text_similarity("samsung", "samsung") == 1.0
-
-    def test_jaro_winkler_dominance(self):
-        """Test that Jaro-Winkler dominates (70% weight)."""
-        resolver = EntityResolver()
-        # Slightly different strings
-        sim = resolver._compute_text_similarity("samsung", "samsu")
-        # Should be close but less than 1.0
-        assert 0.65 < sim < 1.0  # Adjusted based on actual similarity
-
-    def test_token_overlap_single_word(self):
-        """Test token overlap with single word."""
-        resolver = EntityResolver()
-        # Both are single tokens
-        sim = resolver._compute_text_similarity("apple", "apple")
-        assert sim == 1.0
-
-    def test_token_overlap_multiword(self):
-        """Test token overlap with multi-word labels."""
-        resolver = EntityResolver()
-        # Partial token overlap
-        sim = resolver._compute_text_similarity("apple inc", "apple corp")
-        # Should be > 0 due to "apple" token overlap
-        assert sim > 0.5
-
-
-class TestEntityResolverInit:
-    """Test EntityResolver initialization."""
-
-    def test_init_default_thresholds(self):
-        """Test default threshold values."""
-        resolver = EntityResolver()
-        assert resolver.vector_threshold == 0.85
-        assert resolver.text_threshold == 0.88
-        assert resolver.model_name == "all-MiniLM-L6-v2"
-
-    def test_init_custom_thresholds(self):
-        """Test custom threshold values."""
-        resolver = EntityResolver(
-            vector_threshold=0.80,
-            text_threshold=0.90,
-            model_name="sentence-transformers/all-MiniLM-L6-v2"
-        )
-        assert resolver.vector_threshold == 0.80
-        assert resolver.text_threshold == 0.90
-        assert resolver.model_name == "sentence-transformers/all-MiniLM-L6-v2"
-
-    @pytest.mark.asyncio
-    async def test_initialize_embedder(self):
-        """Test embedder initialization."""
-        resolver = EntityResolver()
-        result = await resolver.initialize_embedder()
-        assert result is True
-        assert resolver.embedder is not None
-
-
-class TestDuplicateDetection:
-    """Test duplicate detection (requires embedder)."""
-
-    @pytest.mark.asyncio
-    async def test_detect_duplicates_exact_match(self):
-        """Test detection of exact duplicate labels."""
-        resolver = EntityResolver()
-        await resolver.initialize_embedder()
-
-        entities = [
-            {"id": 1, "label": "Apple Inc", "type": "company"},
-            {"id": 2, "label": "Apple Inc", "type": "company"},  # Exact duplicate
-            {"id": 3, "label": "Microsoft Corp", "type": "company"},
-        ]
-
-        clusters = await resolver.detect_duplicates(entities)
-        assert len(clusters) >= 1
-        # Should detect at least one duplicate pair
-        assert any(c.canonical_id == 1 and 2 in c.duplicates for c in clusters)
-
-    @pytest.mark.asyncio
-    async def test_detect_duplicates_similar_labels(self):
-        """Test detection of similar labels."""
-        resolver = EntityResolver()
-        await resolver.initialize_embedder()
-
-        entities = [
-            {"id": 1, "label": "Apple Inc", "type": "company"},
-            {"id": 2, "label": "Apple Incorporated", "type": "company"},
-            {"id": 3, "label": "Samsung", "type": "company"},
-        ]
-
-        clusters = await resolver.detect_duplicates(entities)
-        # May or may not detect depending on similarity thresholds
-        assert isinstance(clusters, list)
-
-    @pytest.mark.asyncio
-    async def test_detect_duplicates_empty_list(self):
-        """Test with empty entity list - should handle gracefully."""
-        resolver = EntityResolver()
-        await resolver.initialize_embedder()
-
-        # Empty list with no embedder should return empty clusters
-        # (The method checks if embedder exists before processing)
-        try:
-            clusters = await resolver.detect_duplicates([])
-            assert isinstance(clusters, list)
-        except (ValueError, Exception):
-            # May raise error due to numpy handling empty arrays
-            pass
-
-    @pytest.mark.asyncio
-    async def test_detect_duplicates_single_entity(self):
-        """Test with single entity."""
-        resolver = EntityResolver()
-        await resolver.initialize_embedder()
-
-        entities = [
-            {"id": 1, "label": "Apple Inc", "type": "company"},
-        ]
-
-        clusters = await resolver.detect_duplicates(entities)
-        assert clusters == []
-
-
-class TestClusterResolution:
-    """Test cluster merging."""
-
-    @pytest.mark.asyncio
-    async def test_resolve_cluster_basic(self):
-        """Test basic cluster resolution."""
-        resolver = EntityResolver()
-
-        cluster = EntityCluster(
-            cluster_id="C_1_2",
-            canonical_id=1,
-            duplicates=[2],
-            confidence=0.92,
-            reason="combined",
-            metadata={},
-        )
-
-        entities_map = {
-            1: {
-                "id": 1,
-                "label": "Apple Inc",
-                "type": "company",
-                "aliases": ["Apple"],
-                "evidence": [{"source": "source1"}],
-            },
-            2: {
-                "id": 2,
-                "label": "Apple Incorporated",
-                "type": "company",
-                "aliases": ["Apple Inc"],
-                "evidence": [{"source": "source2"}],
-            },
-        }
-
-        merged = await resolver.resolve_cluster(cluster, entities_map)
-
-        assert merged["id"] == 1
-        assert merged["label"] == "Apple Inc"
-        assert merged["merged_from"] == [2]
-        assert merged["merge_confidence"] == 0.92
-        assert len(merged["aliases"]) >= 3
-        assert len(merged["evidence"]) == 2
-
-    @pytest.mark.asyncio
-    async def test_resolve_cluster_missing_entity(self):
-        """Test resolution with missing entity."""
-        resolver = EntityResolver()
-
-        cluster = EntityCluster(
-            cluster_id="C_1_2",
-            canonical_id=1,
-            duplicates=[2],
-            confidence=0.92,
-            reason="combined",
-            metadata={},
-        )
-
-        entities_map = {
-            1: {"id": 1, "label": "Apple Inc"},
-            # Missing entity 2
-        }
-
-        merged = await resolver.resolve_cluster(cluster, entities_map)
-
-        assert merged["id"] == 1
-        assert "aliases" in merged
-
-
-class TestResolutionReport:
-    """Test resolution report generation."""
-
-    def test_get_resolution_report_empty(self):
-        """Test report with no clusters."""
-        resolver = EntityResolver()
-        report = resolver.get_resolution_report([])
-
-        assert report["total_clusters"] == 0
-        assert report["total_duplicates"] == 0
-        assert report["avg_confidence"] == 0
-
-    def test_get_resolution_report_single_cluster(self):
-        """Test report with one cluster."""
-        resolver = EntityResolver()
-
-        clusters = [
-            EntityCluster(
-                cluster_id="C_1_2",
-                canonical_id=1,
-                duplicates=[2],
-                confidence=0.90,
-                reason="combined",
-                metadata={},
-            ),
-        ]
-
-        report = resolver.get_resolution_report(clusters)
-
-        assert report["total_clusters"] == 1
-        assert report["total_duplicates"] == 1
-        assert report["avg_confidence"] == 0.90
-        assert report["by_reason"]["combined"] == 1
-
-    def test_get_resolution_report_multiple_clusters(self):
-        """Test report with multiple clusters."""
-        resolver = EntityResolver()
-
-        clusters = [
-            EntityCluster(
-                cluster_id="C_1_2",
-                canonical_id=1,
-                duplicates=[2],
-                confidence=0.90,
-                reason="combined",
-                metadata={},
-            ),
-            EntityCluster(
-                cluster_id="C_3_4",
-                canonical_id=3,
-                duplicates=[4],
-                confidence=0.85,
-                reason="vector_similarity",
-                metadata={},
-            ),
-        ]
-
-        report = resolver.get_resolution_report(clusters)
-
-        assert report["total_clusters"] == 2
-        assert report["total_duplicates"] == 2
-        assert abs(report["avg_confidence"] - 0.875) < 0.01
-        assert report["by_reason"]["combined"] == 1
-        assert report["by_reason"]["vector_similarity"] == 1
-
-
-if __name__ == "__main__":
-    pytest.main([__file__, "-v"])

tests/core/graph/test_rdf_converter.py

@@ -1,30 +0,0 @@
-"""Phase 5 RDF Converter tests.
-
-Tests RDF ↔ Property Graph conversion:
-- Triple to node/edge conversion
-- Graph roundtrip integrity
-"""
-
-import pytest
-from ont_platform.core.graph import RDFToPropertyGraphConverter
-
-
-class TestRDFConverter:
-    """Test RDF to Property Graph conversion."""
-
-    def test_converter_init(self):
-        """Test converter initialization."""
-        converter = RDFToPropertyGraphConverter()
-        assert converter is not None
-
-    def test_converter_has_required_methods(self):
-        """Test that converter has required methods."""
-        converter = RDFToPropertyGraphConverter()
-        assert hasattr(converter, 'convert_triples_to_graph')
-        assert hasattr(converter, 'to_rdf_triples')
-        assert callable(converter.convert_triples_to_graph)
-        assert callable(converter.to_rdf_triples)
-
-
-if __name__ == "__main__":
-    pytest.main([__file__, "-v"])

tests/core/graph/test_subgraph_retriever.py

@@ -1,458 +0,0 @@
-"""Phase 5 Subgraph Retriever tests.
-
-Tests semantic-based subgraph extraction:
-- N-hop neighborhood retrieval
-- Context retrieval between multiple entities
-- Semantic query-based entity search
-- Induced subgraph extraction
-"""
-
-import pytest
-from unittest.mock import AsyncMock, MagicMock
-import numpy as np
-
-from ont_platform.core.graph import SubgraphRetriever
-
-
-@pytest.fixture
-def mock_adapter():
-    """Mock Neo4j adapter."""
-    adapter = AsyncMock()
-    return adapter
-
-
-@pytest.fixture
-def mock_embedder():
-    """Mock sentence transformer embedder."""
-    embedder = MagicMock()
-    # Return 384-dim embeddings (all-MiniLM-L6-v2 default)
-    embedder.encode = MagicMock(
-        return_value=np.random.randn(384).astype(np.float32)
-    )
-    return embedder
-
-
-@pytest.fixture
-def subgraph_retriever(mock_adapter, mock_embedder):
-    """Create SubgraphRetriever with mocks."""
-    retriever = SubgraphRetriever(adapter=mock_adapter, embedder=mock_embedder)
-    return retriever
-
-
-class TestSubgraphRetrieverInit:
-    """Test SubgraphRetriever initialization."""
-
-    def test_init_with_adapter_only(self, mock_adapter):
-        """Test initialization with adapter only."""
-        retriever = SubgraphRetriever(adapter=mock_adapter)
-        assert retriever.adapter is mock_adapter
-        assert retriever.embedder is None
-
-    def test_init_with_adapter_and_embedder(self, mock_adapter, mock_embedder):
-        """Test initialization with adapter and embedder."""
-        retriever = SubgraphRetriever(adapter=mock_adapter, embedder=mock_embedder)
-        assert retriever.adapter is mock_adapter
-        assert retriever.embedder is mock_embedder
-
-
-class TestSemanticQuery:
-    """Test semantic query-based entity search."""
-
-    @pytest.mark.asyncio
-    async def test_retrieve_by_semantic_query_success(
-        self, subgraph_retriever, mock_adapter, mock_embedder
-    ):
-        """Test successful semantic query retrieval."""
-        # Setup mock responses
-        mock_adapter.execute_cypher = AsyncMock(
-            side_effect=[
-                # First call: fetch entities with embeddings
-                [
-                    {
-                        "entity": {
-                            "id": 1,
-                            "label": "Apple Inc",
-                            "type": "company",
-                            "confidence": 0.95,
-                            "embedding": np.random.randn(384).tolist(),
-                        }
-                    },
-                    {
-                        "entity": {
-                            "id": 2,
-                            "label": "Microsoft Corp",
-                            "type": "company",
-                            "confidence": 0.92,
-                            "embedding": np.random.randn(384).tolist(),
-                        }
-                    },
-                ],
-                # Second call: fetch neighbors
-                [{"id": 3}, {"id": 4}],
-                # Third call: fetch all nodes
-                [
-                    {
-                        "node": {
-                            "id": 1,
-                            "label": "Apple Inc",
-                            "type": "company",
-                            "confidence": 0.95,
-                        }
-                    },
-                    {
-                        "node": {
-                            "id": 2,
-                            "label": "Microsoft Corp",
-                            "type": "company",
-                            "confidence": 0.92,
-                        }
-                    },
-                ],
-                # Fourth call: fetch edges
-                [
-                    {
-                        "edge": {
-                            "source_id": 1,
-                            "target_id": 2,
-                            "predicate": "COMPETES_WITH",
-                            "confidence": 0.85,
-                        }
-                    }
-                ],
-            ]
-        )
-
-        result = await subgraph_retriever.retrieve_by_semantic_query(
-            query="tech companies",
-            top_k=10,
-            min_similarity=0.6,
-            hops=1,
-        )
-
-        assert "error" not in result
-        assert result["query"] == "tech companies"
-        assert "matched_entities" in result
-        assert "nodes" in result
-        assert "edges" in result
-
-    @pytest.mark.asyncio
-    async def test_semantic_query_without_embedder(self, mock_adapter):
-        """Test semantic query without embedder returns error."""
-        retriever = SubgraphRetriever(adapter=mock_adapter, embedder=None)
-
-        result = await retriever.retrieve_by_semantic_query(
-            query="test",
-            top_k=10,
-        )
-
-        assert result["error"] == "Embedder not initialized"
-        assert result["matched_count"] == 0
-
-    @pytest.mark.asyncio
-    async def test_semantic_query_empty_query(self, subgraph_retriever):
-        """Test semantic query with empty query string."""
-        result = await subgraph_retriever.retrieve_by_semantic_query(
-            query="",
-            top_k=10,
-        )
-
-        assert result["error"] == "Empty query"
-        assert result["matched_count"] == 0
-
-    @pytest.mark.asyncio
-    async def test_semantic_query_whitespace_only(self, subgraph_retriever):
-        """Test semantic query with whitespace-only query."""
-        result = await subgraph_retriever.retrieve_by_semantic_query(
-            query="   ",
-            top_k=10,
-        )
-
-        assert result["error"] == "Empty query"
-
-    @pytest.mark.asyncio
-    async def test_semantic_query_no_entities_with_embeddings(
-        self, subgraph_retriever, mock_adapter
-    ):
-        """Test semantic query when no entities have embeddings."""
-        mock_adapter.execute_cypher = AsyncMock(return_value=[])
-
-        result = await subgraph_retriever.retrieve_by_semantic_query(
-            query="test",
-            top_k=10,
-        )
-
-        assert "warning" in result
-        assert result["matched_count"] == 0
-
-    @pytest.mark.asyncio
-    async def test_semantic_query_similarity_filtering(
-        self, subgraph_retriever, mock_adapter, mock_embedder
-    ):
-        """Test similarity threshold filtering."""
-        # Create deterministic embeddings for testing
-        query_vec = np.ones(384, dtype=np.float32)
-        query_vec = query_vec / np.linalg.norm(query_vec)
-
-        mock_embedder.encode = MagicMock(return_value=query_vec)
-
-        # Create entity embeddings with varying similarities
-        high_sim_vec = np.ones(384, dtype=np.float32)
-        high_sim_vec = high_sim_vec / np.linalg.norm(high_sim_vec)
-        # Similarity will be 1.0
-
-        low_sim_vec = -np.ones(384, dtype=np.float32)
-        low_sim_vec = low_sim_vec / np.linalg.norm(low_sim_vec)
-        # Similarity will be -1.0
-
-        mock_adapter.execute_cypher = AsyncMock(
-            side_effect=[
-                # Entities with different similarities
-                [
-                    {
-                        "entity": {
-                            "id": 1,
-                            "label": "High Sim",
-                            "type": "test",
-                            "confidence": 0.9,
-                            "embedding": high_sim_vec.tolist(),
-                        }
-                    },
-                    {
-                        "entity": {
-                            "id": 2,
-                            "label": "Low Sim",
-                            "type": "test",
-                            "confidence": 0.9,
-                            "embedding": low_sim_vec.tolist(),
-                        }
-                    },
-                ],
-                # Neighbors for matched entities only
-                [],
-                # Nodes
-                [{"node": {"id": 1, "label": "High Sim", "type": "test"}}],
-                # Edges
-                [],
-            ]
-        )
-
-        result = await subgraph_retriever.retrieve_by_semantic_query(
-            query="test",
-            top_k=10,
-            min_similarity=0.5,
-        )
-
-        # Only high similarity entity should be matched
-        assert result["matched_count"] == 1
-
-    @pytest.mark.asyncio
-    async def test_semantic_query_top_k_limiting(
-        self, subgraph_retriever, mock_adapter
-    ):
-        """Test top_k parameter limits results."""
-        # Create 5 entities, request top_k=2
-        mock_adapter.execute_cypher = AsyncMock(
-            side_effect=[
-                # 5 entities
-                [
-                    {"entity": {"id": i, "label": f"E{i}", "embedding": np.random.randn(384).tolist()}}
-                    for i in range(1, 6)
-                ],
-                # Neighbors
-                [],
-                # Nodes
-                [{"node": {"id": i, "label": f"E{i}", "type": "test"}} for i in range(1, 3)],
-                # Edges
-                [],
-            ]
-        )
-
-        result = await subgraph_retriever.retrieve_by_semantic_query(
-            query="test",
-            top_k=2,
-            min_similarity=0.0,  # Accept all
-        )
-
-        # Should return at most top_k matches
-        assert result["matched_count"] <= 2
-
-    @pytest.mark.asyncio
-    async def test_semantic_query_with_hops(self, subgraph_retriever, mock_adapter):
-        """Test semantic query with N-hop neighborhood expansion."""
-        # Create a deterministic vector for the query
-        query_vec = np.ones(384, dtype=np.float32)
-        query_vec = query_vec / np.linalg.norm(query_vec)
-        subgraph_retriever.embedder.encode = MagicMock(return_value=query_vec)
-
-        entity_vec = np.ones(384, dtype=np.float32)
-        entity_vec = entity_vec / np.linalg.norm(entity_vec)
-
-        mock_adapter.execute_cypher = AsyncMock(
-            side_effect=[
-                # Entities with embeddings (must include 'type' field)
-                [
-                    {
-                        "entity": {
-                            "id": 1,
-                            "label": "Center",
-                            "type": "company",
-                            "confidence": 0.9,
-                            "embedding": entity_vec.tolist(),
-                        }
-                    }
-                ],
-                # Neighbors (2-hop)
-                [{"id": 2}, {"id": 3}],
-                # Nodes
-                [
-                    {"node": {"id": 1, "label": "Center", "type": "company", "confidence": 0.9}},
-                    {"node": {"id": 2, "label": "N1", "type": "person", "confidence": 0.85}},
-                    {"node": {"id": 3, "label": "N2", "type": "person", "confidence": 0.8}},
-                ],
-                # Edges
-                [],
-            ]
-        )
-
-        result = await subgraph_retriever.retrieve_by_semantic_query(
-            query="test",
-            hops=2,
-        )
-
-        # Should include center and neighbors
-        assert result["node_count"] > 0
-
-
-class TestNeighborhoodRetrieval:
-    """Test N-hop neighborhood extraction."""
-
-    @pytest.mark.asyncio
-    async def test_retrieve_neighborhood_success(self, subgraph_retriever, mock_adapter):
-        """Test successful neighborhood retrieval."""
-        mock_adapter.execute_cypher = AsyncMock(
-            side_effect=[
-                # Center entity query
-                [
-                    {
-                        "result": {
-                            "center": {
-                                "id": 1,
-                                "label": "Apple",
-                                "type": "company",
-                                "confidence": 0.95,
-                            },
-                            "neighbor_ids": [2, 3],
-                            "neighbor_count": 2,
-                        }
-                    }
-                ],
-                # Nodes fetch
-                [
-                    {"node": {"id": 1, "label": "Apple"}},
-                    {"node": {"id": 2, "label": "Tim Cook"}},
-                    {"node": {"id": 3, "label": "Steve Wozniak"}},
-                ],
-                # Edges fetch
-                [
-                    {
-                        "edge": {
-                            "source_id": 1,
-                            "target_id": 2,
-                            "predicate": "HAS_CEO",
-                            "confidence": 0.95,
-                        }
-                    }
-                ],
-            ]
-        )
-
-        result = await subgraph_retriever.retrieve_neighborhood(
-            entity_id=1,
-            hops=2,
-        )
-
-        assert result["center_entity"]["id"] == 1
-        assert result["node_count"] == 3
-        assert len(result["edges"]) > 0
-
-    @pytest.mark.asyncio
-    async def test_retrieve_neighborhood_invalid_hops(self, subgraph_retriever):
-        """Test neighborhood retrieval with invalid hops."""
-        # hops < 1
-        with pytest.raises(ValueError):
-            await subgraph_retriever.retrieve_neighborhood(
-                entity_id=1,
-                hops=0,
-            )
-
-        # hops > 3
-        with pytest.raises(ValueError):
-            await subgraph_retriever.retrieve_neighborhood(
-                entity_id=1,
-                hops=4,
-            )
-
-    @pytest.mark.asyncio
-    async def test_retrieve_neighborhood_entity_not_found(
-        self, subgraph_retriever, mock_adapter
-    ):
-        """Test neighborhood retrieval for non-existent entity."""
-        mock_adapter.execute_cypher = AsyncMock(return_value=[])
-
-        result = await subgraph_retriever.retrieve_neighborhood(entity_id=999)
-
-        assert result["center_entity"] is None
-        assert "error" in result
-
-
-class TestInducedSubgraph:
-    """Test induced subgraph extraction."""
-
-    @pytest.mark.asyncio
-    async def test_retrieve_induced_subgraph_success(
-        self, subgraph_retriever, mock_adapter
-    ):
-        """Test successful induced subgraph extraction."""
-        # Set up mock to return appropriate responses for each call
-        def side_effect_func(cypher, params):
-            if "WHERE n.id IN" in cypher and "RELATES" not in cypher:
-                # Nodes fetch
-                return [
-                    {"node": {"id": 1, "label": "Apple"}},
-                    {"node": {"id": 2, "label": "Microsoft"}},
-                ]
-            elif "RELATES" in cypher:
-                # Edges fetch
-                return [
-                    {
-                        "edge": {
-                            "source_id": 1,
-                            "target_id": 2,
-                            "predicate": "COMPETES_WITH",
-                            "confidence": 0.85,
-                        }
-                    }
-                ]
-            return []
-
-        mock_adapter.execute_cypher = AsyncMock(side_effect=side_effect_func)
-
-        result = await subgraph_retriever.retrieve_induced_subgraph(
-            entity_ids=[1, 2],
-        )
-
-        assert result["node_count"] >= 0  # May have 0 if mock doesn't match cypher
-        assert isinstance(result["edges"], list)
-
-    @pytest.mark.asyncio
-    async def test_retrieve_induced_subgraph_empty_list(self, subgraph_retriever):
-        """Test induced subgraph with empty entity list."""
-        result = await subgraph_retriever.retrieve_induced_subgraph(
-            entity_ids=[],
-        )
-
-        assert "error" in result
-
-
-if __name__ == "__main__":
-    pytest.main([__file__, "-v"])

tests/fixtures/sample_perfume.html

@@ -1,17 +0,0 @@
-<!doctype html>
-<html lang="en">
-  <head>
-    <title>Neroli Summer Eau de Parfum</title>
-  </head>
-  <body>
-    <main>
-      <h1>Neroli Summer Eau de Parfum</h1>
-      <p>Brand: Example Maison</p>
-      <p>Top notes: Bergamot, Neroli</p>
-      <p>Middle notes: Jasmine</p>
-      <p>Base notes: Musk</p>
-      <p>A fresh daily perfume for summer office wear.</p>
-      <p>Price $89</p>
-    </main>
-  </body>
-</html>

tests/integration/test_phase5_phase7_integration.py

@@ -1,221 +0,0 @@
-"""Phase 5 GraphRAG + Phase 7 LLM integration tests.
-
-Validates that Phase 5 enhances Phase 7 LLM's RAG pipeline:
-- Entity deduplication improves graph quality
-- Semantic query retrieves relevant context for LLM
-- Pattern analysis detects data inconsistencies
-"""
-
-import pytest
-from unittest.mock import AsyncMock, MagicMock
-from datetime import datetime, UTC
-
-
-class TestEntityResolutionEnhancesLLM:
-    """Test how entity resolution improves LLM context."""
-
-    @pytest.mark.asyncio
-    async def test_duplicate_entities_merged_before_rag(self):
-        """Test that duplicate entities are merged before RAG context retrieval."""
-        from ont_platform.core.graph import EntityResolver
-
-        resolver = EntityResolver(
-            vector_threshold=0.85,
-            text_threshold=0.88,
-        )
-
-        # Simulate entities that are duplicates with slight variations
-        entities = [
-            {"id": 1, "label": "Apple Inc", "type": "company"},
-            {"id": 2, "label": "Apple Incorporated", "type": "company"},
-            {"id": 3, "label": "Microsoft Corporation", "type": "company"},
-            {"id": 4, "label": "Microsoft Corp", "type": "company"},
-        ]
-
-        # Initialize embedder
-        result = await resolver.initialize_embedder()
-        assert result is True
-
-        # Detect duplicates
-        clusters = await resolver.detect_duplicates(entities)
-
-        # Should find at least 2 clusters (Apple duplicates and Microsoft duplicates)
-        assert len(clusters) >= 0  # May vary based on similarity thresholds
-        assert all(c.confidence >= 0.5 for c in clusters)
-
-
-class TestSemanticQueryForLLMContext:
-    """Test semantic query retrieval for LLM RAG."""
-
-    @pytest.mark.asyncio
-    async def test_semantic_query_finds_relevant_context(self):
-        """Test that semantic queries find relevant entities for LLM context."""
-        from ont_platform.core.graph import SubgraphRetriever
-        from unittest.mock import AsyncMock
-        import numpy as np
-
-        mock_adapter = AsyncMock()
-        embedder = MagicMock()
-
-        # Create embedder that returns consistent vectors
-        query_vec = np.ones(384, dtype=np.float32)
-        query_vec = query_vec / np.linalg.norm(query_vec)
-        embedder.encode = MagicMock(return_value=query_vec)
-
-        retriever = SubgraphRetriever(adapter=mock_adapter, embedder=embedder)
-
-        # Setup mock to return relevant entities
-        entity_vec = np.ones(384, dtype=np.float32)
-        entity_vec = entity_vec / np.linalg.norm(entity_vec)
-
-        mock_adapter.execute_cypher = AsyncMock(
-            side_effect=[
-                # Entities matching query "tech companies"
-                [
-                    {
-                        "entity": {
-                            "id": 1,
-                            "label": "Apple Inc",
-                            "type": "company",
-                            "confidence": 0.95,
-                            "embedding": entity_vec.tolist(),
-                        }
-                    },
-                    {
-                        "entity": {
-                            "id": 2,
-                            "label": "Microsoft",
-                            "type": "company",
-                            "confidence": 0.92,
-                            "embedding": entity_vec.tolist(),
-                        }
-                    },
-                ],
-                # Neighbors
-                [],
-                # Nodes
-                [
-                    {"node": {"id": 1, "label": "Apple Inc", "type": "company"}},
-                    {"node": {"id": 2, "label": "Microsoft", "type": "company"}},
-                ],
-                # Edges
-                [],
-            ]
-        )
-
-        result = await retriever.retrieve_by_semantic_query(
-            query="tech companies",
-            top_k=10,
-            min_similarity=0.6,
-        )
-
-        # Should find relevant entities
-        assert result["matched_count"] >= 0
-        assert "matched_entities" in result
-        assert len(result["nodes"]) >= 0
-
-
-class TestGraphAnalyticsForDataQuality:
-    """Test graph analytics for data quality assurance."""
-
-    @pytest.mark.asyncio
-    async def test_centrality_identifies_important_entities(self):
-        """Test that centrality analysis identifies important entities for RAG."""
-        # This test validates that graph analytics can identify
-        # which entities are most relevant for RAG context
-        assert True  # Placeholder for integration with actual GraphAnalytics
-
-    @pytest.mark.asyncio
-    async def test_pattern_matching_detects_inconsistencies(self):
-        """Test that pattern matching detects data inconsistencies."""
-        # This test validates that cycles/paths detection helps ensure
-        # graph integrity before using it for RAG
-        assert True  # Placeholder for integration with actual PatternMatcher
-
-
-class TestPhase5EnhancesPhase7RAGPipeline:
-    """Integration test: Phase 5 + Phase 7 RAG pipeline."""
-
-    @pytest.mark.asyncio
-    async def test_rag_context_quality_with_phase5(self):
-        """Test that Phase 5 improves RAG context quality."""
-        # Expected workflow:
-        # 1. Extract entities from documents (Phase 0-4)
-        # 2. Detect and merge duplicates (Phase 5 EntityResolver)
-        # 3. Retrieve semantic context (Phase 5 SubgraphRetriever)
-        # 4. Pass to LLM for generation (Phase 7)
-
-        # Verify components work together
-        from ont_platform.core.graph import EntityResolver
-
-        resolver = EntityResolver()
-        assert resolver.vector_threshold == 0.85
-        assert resolver.text_threshold == 0.88
-
-    @pytest.mark.asyncio
-    async def test_end_to_end_entity_resolution_pipeline(self):
-        """Test complete entity resolution pipeline."""
-        from ont_platform.core.graph import EntityResolver, EntityCluster
-
-        resolver = EntityResolver()
-
-        # Initialize embedder
-        await resolver.initialize_embedder()
-
-        # Test data: duplicate entities with variations
-        entities = [
-            {"id": 1, "label": "Apple", "type": "company"},
-            {"id": 2, "label": "Apple Inc", "type": "company"},  # Duplicate
-            {"id": 3, "label": "Google", "type": "company"},
-            {"id": 4, "label": "Alphabet", "type": "company"},  # Potential duplicate
-        ]
-
-        # Detect duplicates
-        clusters = await resolver.detect_duplicates(entities)
-
-        # Generate report
-        report = resolver.get_resolution_report(clusters)
-
-        # Validate report structure
-        assert "total_clusters" in report
-        assert "total_duplicates" in report
-        assert "avg_confidence" in report
-        assert "by_reason" in report
-        assert "timestamp" in report
-
-        # Validate each cluster can be resolved
-        entities_map = {e["id"]: e for e in entities}
-
-        for cluster in clusters:
-            merged = await resolver.resolve_cluster(cluster, entities_map)
-
-            # Validate merged entity has required fields
-            assert "id" in merged
-            assert "label" in merged
-            assert "merged_from" in merged or merged.get("id") in [e["id"] for e in entities]
-
-        print(f"✅ Entity resolution pipeline: {len(clusters)} clusters, "
-              f"{report['total_duplicates']} duplicates detected")
-
-
-class TestRAGContextRelevance:
-    """Test that Phase 5 improves RAG context relevance."""
-
-    @pytest.mark.asyncio
-    async def test_semantic_context_more_relevant_than_random(self):
-        """Test that semantic context selection is better than random."""
-        # This demonstrates that Phase 5 semantic queries should return
-        # more relevant entities than a random selection would
-        assert True  # Conceptual test - validates the approach
-
-    @pytest.mark.asyncio
-    async def test_deduplicated_graph_smaller_and_cleaner(self):
-        """Test that deduplication reduces graph noise."""
-        # Before deduplication: 100 entities (with duplicates)
-        # After deduplication: 80 entities (20 removed as duplicates)
-        # Result: Smaller, cleaner graph for RAG
-        assert True  # Conceptual test - validates the benefit
-
-
-if __name__ == "__main__":
-    pytest.main([__file__, "-v"])

tests/test_config_loader.py

@@ -1,12 +0,0 @@
-from pathlib import Path
-
-from crawler_platform.app.config.loader import load_project_config
-
-
-def test_load_perfume_config():
-    config = load_project_config(Path("configs/perfume_subscription.yaml"))
-
-    assert config.project_name == "perfume_subscription"
-    assert config.domain == "perfume"
-    assert config.source_by_name("official_brand_site").trust_level == 0.95
-

tests/test_content_quality.py

@@ -1,457 +0,0 @@
-import json
-
-from crawler_platform.app.config.loader import load_project_config
-from crawler_platform.app.core.database import models
-from crawler_platform.app.core.database.repository import KnowledgeRepository
-from crawler_platform.app.core.database.session import make_engine
-from crawler_platform.app.core.crawler.html_cleaner import clean_html_with_metadata
-from crawler_platform.app.core.extractor.base import (
-    ExtractedClaim,
-    ExtractedEntity,
-    ExtractionPageContext,
-    ExtractionBundle,
-)
-from crawler_platform.app.core.extractor.ai_provider import parse_json_content
-from crawler_platform.app.core.extractor.ai_provider import LLMJsonExtractor
-from crawler_platform.app.core.extractor.validation import attach_page_context, validate_extraction_bundle
-from crawler_platform.app.domains.perfume.extractor import PerfumeRuleBasedExtractor
-
-
-def test_cleaner_prefers_product_content_and_removes_boilerplate():
-    html = """
-    <html>
-      <head><title>Sample Product</title></head>
-      <body>
-        <header>CAFE24 Login Cart Low price Product count</header>
-        <nav>Home Board Event Shipping</nav>
-        <main class="product-detail">
-          <h1>Neroli Summer Eau de Parfum</h1>
-          <p>Brand: Example Maison</p>
-          <p>Top notes: Bergamot, Neroli</p>
-          <p>Base notes: Musk</p>
-          <p>Price $89</p>
-        </main>
-        <footer>Powered by CAFE24 Copyright Shipping Country List</footer>
-      </body>
-    </html>
-    """
-
-    cleaned = clean_html_with_metadata(html)
-
-    assert "Neroli Summer Eau de Parfum" in cleaned.text
-    assert "Bergamot" in cleaned.text
-    assert "CAFE24" not in cleaned.text
-    assert "Low price" not in cleaned.text
-    assert cleaned.page_type == "ProductPage"
-    assert cleaned.extraction_status == "success"
-    assert any(zone["zone_type"] == "product_detail" for zone in cleaned.source_zones)
-
-
-def test_cleaner_keeps_raw_and_clean_text_separate_with_noise_debug():
-    html = """
-    <html>
-      <head><title>Sample Product</title></head>
-      <body>
-        <header>CAFE24 Login Cart Low price Product count</header>
-        <main class="product-detail">
-          <h1>Neroli Summer Eau de Parfum</h1>
-          <p>Brand: Example Maison</p>
-          <p>Top notes: Bergamot, Neroli</p>
-          <p>Price $89</p>
-        </main>
-        <section class="shipping">Shipping delivery return exchange country list Korea USA Japan Canada</section>
-      </body>
-    </html>
-    """
-
-    cleaned = clean_html_with_metadata(html, url="https://example.com/product/detail.html")
-
-    assert "CAFE24" in cleaned.raw_text
-    assert "CAFE24" not in cleaned.text
-    assert "Shipping delivery" not in cleaned.text
-    assert cleaned.clean_markdown
-    assert cleaned.metadata["removed_noise_zones_count"] >= 1
-
-
-def test_cleaner_handles_empty_dom_attributes():
-    html = """
-    <html>
-      <body>
-        <div id class aria-label>
-          <p>Brand: Example Maison</p>
-          <p>Top notes: Bergamot, Neroli</p>
-        </div>
-      </body>
-    </html>
-    """
-
-    cleaned = clean_html_with_metadata(html)
-
-    assert "Example Maison" in cleaned.text
-
-
-def test_cleaner_skips_decomposed_nodes_during_boilerplate_scan():
-    from bs4 import BeautifulSoup
-
-    soup = BeautifulSoup("<div id='footer'><span class='child'>x</span></div>", "html.parser")
-    tag = soup.span
-    assert tag is not None
-    tag.decompose()
-
-    from crawler_platform.app.core.crawler.html_cleaner import node_attr
-
-    assert node_attr(tag, "class") == ""
-
-
-def test_validation_rejects_placeholders_and_marks_rule_candidates():
-    config = load_project_config("configs/perfume_subscription.yaml")
-    bundle = ExtractionBundle(
-        entities=[
-            ExtractedEntity("Brand", "CAFE24"),
-            ExtractedEntity("Perfume", "Neroli Summer"),
-            ExtractedEntity("Accord", "Green"),
-        ],
-        claims=[
-            ExtractedClaim("Neroli Summer", "Perfume", "hasBrand", "CAFE24", "Brand", evidence_text="Brand: CAFE24"),
-            ExtractedClaim(
-                "Neroli Summer",
-                "Perfume",
-                "hasTopNote",
-                "Bergamot",
-                "Note",
-                evidence_text="Top notes: Bergamot",
-                confidence=0.82,
-            ),
-            ExtractedClaim("Neroli Summer", "Perfume", "hasAccord", "Green", "Accord", evidence_text="Accord: Green"),
-        ],
-        extractor_name="perfume_rule_based",
-        provider="rule_based",
-        raw_output={},
-    )
-
-    result = validate_extraction_bundle(bundle, config)
-
-    assert result.claim_status == "rule_candidate"
-    assert [claim.object_name for claim in result.bundle.claims] == ["Bergamot"]
-    assert all(claim.metadata["validation_status"] == "rule_candidate" for claim in result.bundle.claims)
-    assert len(result.rejected_claims) == 2
-
-
-def test_validation_rejects_product_detail_claims_on_category_page_context():
-    config = load_project_config("configs/perfume_subscription.yaml")
-    bundle = ExtractionBundle(
-        entities=[ExtractedEntity("Perfume", "Neroli Summer", evidence_text="Neroli Summer")],
-        claims=[
-            ExtractedClaim(
-                "Neroli Summer",
-                "Perfume",
-                "hasPrice",
-                object_value={"amount": 89, "currency": "USD"},
-                evidence_text="Neroli Summer Price $89",
-            )
-        ],
-        extractor_name="llm_json_extractor",
-        provider="lm_studio",
-        raw_output={"extraction_mode": "primary"},
-    )
-    context = ExtractionPageContext(
-        url="https://example.com/category/perfume",
-        final_url="https://example.com/category/perfume",
-        title="Perfume list",
-        page_type="CategoryPage",
-        clean_text="Neroli Summer Price $89",
-        source_zones=[
-            {
-                "zone_type": "product_summary",
-                "selector": ".product-list",
-                "text": "Neroli Summer Price $89",
-                "claim_allowed": True,
-            }
-        ],
-    )
-
-    result = validate_extraction_bundle(attach_page_context(bundle, context), config)
-
-    assert result.claim_status == "candidate_claim"
-    assert result.bundle.claims == []
-    assert result.rejected_claims[0]["reason"] == "predicate hasPrice is not allowed for page type CategoryPage"
-
-
-def test_repository_persists_rule_candidates_without_merging_graph_relations():
-    config = load_project_config("configs/perfume_subscription.yaml")
-    engine = make_engine("sqlite:///:memory:")
-    models.Base.metadata.create_all(engine)
-
-    from sqlalchemy.orm import Session
-
-    session = Session(engine)
-    repo = KnowledgeRepository(session)
-    project = repo.upsert_project(config)
-    source = repo.get_source(project.id, "official_brand_site")
-    page = repo.upsert_page(project.id, source.id, "https://example.com/product", "Product", 200, "Neroli Summer")
-    bundle = ExtractionBundle(
-        entities=[ExtractedEntity("Perfume", "Neroli Summer")],
-        claims=[
-            ExtractedClaim(
-                "Neroli Summer",
-                "Perfume",
-                "hasTopNote",
-                    "Bergamot",
-                    "Note",
-                    evidence_text="Top notes: Bergamot",
-                    confidence=0.9,
-                )
-        ],
-        extractor_name="perfume_rule_based",
-        provider="rule_based",
-        raw_output={},
-    )
-    context = ExtractionPageContext(
-        url="https://example.com/product",
-        final_url="https://example.com/product",
-        title="Product",
-        page_type="ProductPage",
-        clean_text="Neroli Summer\nTop notes: Bergamot",
-        source_zones=[
-            {
-                "zone_type": "product_description",
-                "selector": ".description",
-                "text": "Top notes: Bergamot",
-                "claim_allowed": True,
-            }
-        ],
-    )
-
-    claims = repo.save_extraction_bundle(project.id, source, page, attach_page_context(bundle, context), config)
-    session.commit()
-
-    assert len(claims) == 1
-    assert claims[0].status == "rule_candidate"
-    assert session.query(models.Claim).count() == 1
-    assert session.query(models.Entity).count() == 2
-    assert session.query(models.ExtractionLog).count() == 1
-    assert session.query(models.Relation).count() == 0
-    session.close()
-
-
-def test_parse_json_content_accepts_markdown_fenced_json():
-    raw = """```json
-    {"entities": [], "claims": []}
-    ```"""
-
-    assert parse_json_content(raw) == {"entities": [], "claims": []}
-
-
-def test_lm_studio_extractor_merges_rule_claims_even_when_ai_claim_is_invalid():
-    config = load_project_config("configs/perfume_subscription.yaml")
-
-    class StubExtractor(LLMJsonExtractor):
-        def complete_json(self, page_text, project_config, compact=False, context=None):
-            return {
-                "entities": [
-                    {
-                        "entity_type": "Brand",
-                        "name": "912",
-                        "attributes": {},
-                        "confidence": 0.9,
-                        "evidence_text": "912",
-                    }
-                ],
-                "claims": [
-                    {
-                        "subject_name": "912",
-                        "subject_type": "Brand",
-                        "predicate": "hasBrand",
-                        "object_name": "912",
-                        "object_type": "Brand",
-                        "object_value": None,
-                        "evidence_text": "912",
-                        "evidence_summary": "bad self-brand claim",
-                        "confidence": 0.9,
-                        "confidence_reason": "stub",
-                        "source_zone": "product_title",
-                    }
-                ],
-            }
-
-    text = "Neroli Summer\nTop notes: Bergamot\nPrice $89"
-    bundle = StubExtractor("perfume", "lm_studio", model="stub").extract(text, config)
-
-    assert any(claim.predicate == "hasTopNote" for claim in bundle.claims)
-
-
-def test_the912_product_rule_fallback_uses_product_title_brand_and_price():
-    config = load_project_config("configs/perfume_subscription.yaml")
-    text = """
-    새로운 향수의 시작, 클론 향수
-    [2+1 기획] 912 클론 니치향수 모음 40ml
-    4.8
-    114,414
-    일반 구매가격
-    72,000원
-    할인 적용금액
-    37,000원
-    최종 구매금액
-    35,000원
-    현재 위치
-    전체 상품
-    니치향수
-    오드 퍼퓸
-    """
-    bundle = PerfumeRuleBasedExtractor().extract(text, config)
-    context = ExtractionPageContext(
-        url="https://the912.co.kr/product/detail.html?product_no=513",
-        final_url="https://the912.co.kr/product/detail.html?product_no=513",
-        title="[2+1 기획] 912 클론 니치향수 모음 40ml - 912",
-        page_type="ProductPage",
-        clean_text=text,
-        source_zones=[
-            {
-                "zone_type": "product_title",
-                "selector": "h1",
-                "text": "[2+1 기획] 912 클론 니치향수 모음 40ml",
-                "claim_allowed": True,
-            },
-            {
-                "zone_type": "product_summary",
-                "selector": ".detail",
-                "text": "일반 구매가격\n72,000원\n할인 적용금액\n37,000원\n최종 구매금액\n35,000원",
-                "claim_allowed": True,
-            },
-        ],
-    )
-
-    result = validate_extraction_bundle(attach_page_context(bundle, context), config)
-    predicates = {claim.predicate for claim in result.bundle.claims}
-    perfume_names = {entity.name for entity in result.bundle.entities if entity.entity_type == "Perfume"}
-
-    assert "[2+1 기획] 912 클론 니치향수 모음 40ml" in perfume_names
-    assert {"hasBrand", "hasPrice"} <= predicates
-
-
-def test_parse_json_content_requires_structured_claim_schema():
-    raw = {
-        "entities": [
-            {
-                "entity_type": "Perfume",
-                "name": "Neroli Summer",
-                "attributes": {},
-                "confidence": 0.9,
-                "evidence_text": "Neroli Summer",
-            }
-        ],
-        "claims": [
-            {
-                "subject_name": "Neroli Summer",
-                "subject_type": "Perfume",
-                "predicate": "hasTopNote",
-                "object_name": "Bergamot",
-                "object_type": "FragranceNote",
-                "object_value": None,
-                "evidence_text": "Top notes: Bergamot",
-                "evidence_summary": "explicit note listing",
-                "confidence": 0.9,
-                "confidence_reason": "directly stated",
-                "source_zone": "product_description",
-            }
-        ],
-    }
-
-    parsed = parse_json_content(json.dumps(raw))
-
-    assert parsed["claims"][0]["source_zone"] == "product_description"
-
-
-def test_parse_json_content_rejects_claim_without_evidence():
-    raw = {
-        "entities": [],
-        "claims": [
-            {
-                "subject_name": "Neroli Summer",
-                "subject_type": "Perfume",
-                "predicate": "hasTopNote",
-                "object_name": "Bergamot",
-                "object_type": "Note",
-                "object_value": None,
-                "evidence_text": "",
-                "evidence_summary": "missing evidence",
-                "confidence": 0.9,
-                "confidence_reason": "bad",
-                "source_zone": "product_description",
-            }
-        ],
-    }
-
-    try:
-        parse_json_content(json.dumps(raw))
-    except ValueError as exc:
-        assert "schema_validation_failed" in str(exc)
-    else:
-        raise AssertionError("expected schema validation failure")
-
-
-def test_repository_merges_only_validated_typed_claims_to_graph():
-    config = load_project_config("configs/perfume_subscription.yaml")
-    engine = make_engine("sqlite:///:memory:")
-    models.Base.metadata.create_all(engine)
-
-    from sqlalchemy.orm import Session
-
-    session = Session(engine)
-    repo = KnowledgeRepository(session)
-    project = repo.upsert_project(config)
-    source = repo.get_source(project.id, "official_brand_site")
-    page = repo.upsert_page(
-        project.id,
-        source.id,
-        "https://example.com/product/detail.html",
-        "Neroli Summer",
-        200,
-        "Neroli Summer\nTop notes: Bergamot",
-    )
-    bundle = ExtractionBundle(
-        entities=[
-            ExtractedEntity("Perfume", "Neroli Summer", evidence_text="Neroli Summer", confidence=0.9),
-            ExtractedEntity("FragranceNote", "Bergamot", evidence_text="Top notes: Bergamot", confidence=0.9),
-        ],
-        claims=[
-            ExtractedClaim(
-                "Neroli Summer",
-                "Perfume",
-                "hasTopNote",
-                "Bergamot",
-                "FragranceNote",
-                evidence_text="Top notes: Bergamot",
-                confidence=0.92,
-                confidence_reason="directly stated",
-            )
-        ],
-        extractor_name="llm_json_extractor",
-        provider="lm_studio",
-        raw_output={"extraction_mode": "primary"},
-    )
-    context = ExtractionPageContext(
-        url="https://example.com/product/detail.html",
-        final_url="https://example.com/product/detail.html",
-        title="Neroli Summer",
-        page_type="ProductPage",
-        clean_text="Neroli Summer\nTop notes: Bergamot",
-        source_zones=[
-            {
-                "zone_type": "product_description",
-                "selector": ".description",
-                "text": "Top notes: Bergamot",
-                "claim_allowed": True,
-            }
-        ],
-    )
-
-    claims = repo.save_extraction_bundle(project.id, source, page, attach_page_context(bundle, context), config)
-    session.commit()
-
-    assert len(claims) == 1
-    assert claims[0].status == "validated_claim"
-    assert claims[0].metadata_json["graph_merge_status"] == "merged"
-    assert session.query(models.Relation).count() == 1
-    assert session.query(models.Entity).filter(models.Entity.entity_type == "Note").count() == 1
-    session.close()

tests/test_generic_ontology_core.py

@@ -1,208 +0,0 @@
-from sqlalchemy.orm import Session
-
-from crawler_platform.app.config.loader import ProjectConfig, SourceConfig, load_project_config
-from crawler_platform.app.core.database import models
-from crawler_platform.app.core.database.repository import KnowledgeRepository
-from crawler_platform.app.core.database.session import make_engine
-from crawler_platform.app.core.extractor.base import ExtractedClaim, ExtractedEntity, ExtractionBundle, ExtractionPageContext
-from crawler_platform.app.core.extractor.validation import attach_page_context
-from crawler_platform.app.core.ontology.gap_detector import KnowledgeGapDetector
-from crawler_platform.app.core.ontology.registry import OntologyRegistry
-from crawler_platform.app.core.research.graph_query import SemanticGraphQuery
-
-
-def make_repo():
-    config = load_project_config("configs/perfume_subscription.yaml")
-    engine = make_engine("sqlite:///:memory:")
-    models.Base.metadata.create_all(engine)
-    session = Session(engine)
-    repo = KnowledgeRepository(session)
-    project = repo.upsert_project(config)
-    source = repo.get_source(project.id, "official_brand_site")
-    return config, session, repo, project, source
-
-
-def save_validated_note_claim(repo, project, source, config):
-    page = repo.upsert_page(
-        project.id,
-        source.id,
-        "https://example.com/product/cotton",
-        "Cotton Example",
-        200,
-        "Cotton Example\nTop notes: Bergamot",
-    )
-    bundle = ExtractionBundle(
-        entities=[
-            ExtractedEntity("Perfume", "Cotton Example", evidence_text="Cotton Example", confidence=0.95),
-            ExtractedEntity("Note", "Bergamot", evidence_text="Top notes: Bergamot", confidence=0.95),
-        ],
-        claims=[
-            ExtractedClaim(
-                "Cotton Example",
-                "Perfume",
-                "hasTopNote",
-                "Bergamot",
-                "Note",
-                evidence_text="Top notes: Bergamot",
-                confidence=0.96,
-                confidence_reason="direct evidence",
-            )
-        ],
-        extractor_name="llm_json_extractor",
-        provider="lm_studio",
-        raw_output={"extraction_mode": "primary"},
-    )
-    context = ExtractionPageContext(
-        url=page.url,
-        final_url=page.url,
-        title=page.title,
-        page_type="ProductPage",
-        clean_text=page.cleaned_text_summary,
-        source_zones=[
-            {
-                "zone_type": "product_description",
-                "selector": "main",
-                "text": "Top notes: Bergamot",
-                "claim_allowed": True,
-            }
-        ],
-    )
-    return repo.save_extraction_bundle(project.id, source, page, attach_page_context(bundle, context), config)
-
-
-def test_project_upsert_seeds_schema_registry():
-    _config, session, _repo, project, _source = make_repo()
-
-    registry = OntologyRegistry(session).registry_payload(project.id)
-
-    entity_names = {row["name"] for row in registry["entity_types"]}
-    relation_names = {row["name"] for row in registry["relation_types"]}
-    assert {"Entity", "Product", "Perfume"}.issubset(entity_names)
-    assert {"relatedTo", "hasTopNote", "hasBrand"}.issubset(relation_names)
-    session.close()
-
-
-def test_validated_claim_creates_separate_ontology_triple():
-    config, session, repo, project, source = make_repo()
-
-    claims = save_validated_note_claim(repo, project, source, config)
-
-    triples = session.query(models.OntologyTriple).all()
-    assert len(claims) == 1
-    assert len(triples) == 1
-    assert triples[0].predicate == "hasTopNote"
-    assert triples[0].status == "merged"
-    assert claims[0].metadata_json["ontology_triple_id"] == triples[0].id
-    session.close()
-
-
-def test_schema_proposal_and_knowledge_gap_are_governance_records():
-    _config, session, _repo, project, _source = make_repo()
-    registry = OntologyRegistry(session)
-
-    proposal = registry.propose_schema_change(
-        project.id,
-        proposal_type="relation_type",
-        name="influencedBy",
-        reason="New cross-domain relation observed in evidence.",
-        evidence="A research paper influenced a product strategy.",
-        confidence=0.73,
-    )
-    gaps = KnowledgeGapDetector(session).list_open(project.id)
-
-    assert proposal.status == "pending_review"
-    assert any(gap["gap_type"] == "schema_governance" and gap["target_name"] == "influencedBy" for gap in gaps)
-    session.close()
-
-
-def test_generic_graph_query_uses_ontology_triples():
-    config, session, repo, project, source = make_repo()
-    save_validated_note_claim(repo, project, source, config)
-
-    rows = SemanticGraphQuery(session).relation_summary(project.id)
-
-    assert rows[0]["predicate"] == "hasTopNote"
-    assert rows[0]["support_count"] == 1
-    session.close()
-
-
-def test_config_driven_academic_relation_without_perfume_adapter():
-    config = ProjectConfig(
-        project_name="academic_demo",
-        domain="academic",
-        target_entities=["ResearchPaper", "Person"],
-        fields=["title", "author"],
-        sources=[SourceConfig(name="research_site", type="paper_index", trust_level=0.9)],
-        ontology={
-            "entity_types": ["ResearchPaper", "Person"],
-            "predicates": ["authoredBy"],
-            "relation_types": {
-                "authoredBy": {
-                    "allowed_subject_types": ["ResearchPaper"],
-                    "allowed_object_types": ["Person"],
-                    "allowed_page_types": ["ArticlePage"],
-                    "allowed_source_zones": ["article_body"],
-                    "confidence_rules": {"min_confidence": 0.82},
-                }
-            },
-        },
-    )
-    engine = make_engine("sqlite:///:memory:")
-    models.Base.metadata.create_all(engine)
-    session = Session(engine)
-    repo = KnowledgeRepository(session)
-    project = repo.upsert_project(config)
-    source = repo.get_source(project.id, "research_site")
-    page = repo.upsert_page(
-        project.id,
-        source.id,
-        "https://example.org/papers/semantic-systems",
-        "Semantic Systems",
-        200,
-        "Semantic Systems was authored by Ada Kim.",
-    )
-    bundle = ExtractionBundle(
-        entities=[
-            ExtractedEntity("ResearchPaper", "Semantic Systems", evidence_text="Semantic Systems", confidence=0.95),
-            ExtractedEntity("Person", "Ada Kim", evidence_text="authored by Ada Kim", confidence=0.95),
-        ],
-        claims=[
-            ExtractedClaim(
-                "Semantic Systems",
-                "ResearchPaper",
-                "authoredBy",
-                "Ada Kim",
-                "Person",
-                evidence_text="Semantic Systems was authored by Ada Kim.",
-                confidence=0.97,
-                confidence_reason="directly stated",
-            )
-        ],
-        extractor_name="llm_json_extractor",
-        provider="lm_studio",
-        raw_output={"extraction_mode": "primary"},
-    )
-    context = ExtractionPageContext(
-        url=page.url,
-        final_url=page.url,
-        title=page.title,
-        page_type="ArticlePage",
-        clean_text=page.cleaned_text_summary,
-        source_zones=[
-            {
-                "zone_type": "article_body",
-                "selector": "article",
-                "text": "Semantic Systems was authored by Ada Kim.",
-                "claim_allowed": True,
-            }
-        ],
-    )
-
-    claims = repo.save_extraction_bundle(project.id, source, page, attach_page_context(bundle, context), config)
-
-    triples = session.query(models.OntologyTriple).all()
-    assert len(claims) == 1
-    assert triples[0].predicate == "authoredBy"
-    assert triples[0].status == "merged"
-    assert session.query(models.OntologyRelationType).filter_by(name="authoredBy").one().domain == "academic"
-    session.close()

tests/test_perfume_extractor.py

@@ -1,31 +0,0 @@
-from crawler_platform.app.config.loader import ProjectConfig, SourceConfig
-from crawler_platform.app.domains.perfume.extractor import PerfumeRuleBasedExtractor
-
-
-def test_perfume_rule_based_extractor_maps_claims():
-    config = ProjectConfig(
-        project_name="perfume_subscription",
-        domain="perfume",
-        target_entities=["Perfume", "Brand", "Note"],
-        fields=["name", "brand", "top_notes", "middle_notes", "base_notes"],
-        sources=[SourceConfig(name="official_brand_site", trust_level=0.95)],
-        ontology={"aliases": {"top_notes": "hasTopNote"}},
-    )
-    text = """
-    Neroli Summer Eau de Parfum
-    Brand: Example Maison
-    Top notes: Bergamot, Neroli
-    Middle notes: Jasmine
-    Base notes: Musk
-    A fresh daily perfume for summer office wear. Price $89.
-    """
-    bundle = PerfumeRuleBasedExtractor().extract(text, config)
-    predicates = {claim.predicate for claim in bundle.claims}
-    entity_names = {entity.name for entity in bundle.entities}
-
-    assert "Neroli Summer Eau de Parfum" in entity_names
-    assert "Bergamot" in entity_names
-    assert "hasTopNote" in predicates
-    assert "suitableForSeason" in predicates
-    assert "hasPrice" in predicates
-

tests/test_phase7_llm_integration.py

@@ -1,541 +0,0 @@
-"""Phase 7 LLM Integration Tests.
-
-Tests for:
-- LLM provider abstraction
-- Streaming responses
-- Caching mechanism
-- RAG + LLM pipeline
-- Error handling
-"""
-
-import asyncio
-import json
-import pytest
-from typing import AsyncGenerator
-from unittest.mock import AsyncMock, MagicMock, patch
-
-from ont_platform.llm.llm_integration import (
-    LLMProvider,
-    LLMConfig,
-    LLMManager,
-)
-
-
-# ============================================================================
-# Fixtures
-# ============================================================================
-
-
-@pytest.fixture
-def openai_config():
-    """OpenAI configuration."""
-    return LLMConfig(
-        provider=LLMProvider.OPENAI,
-        api_key="sk-test-key",
-        model="gpt-4",
-        temperature=0.7,
-        max_tokens=500,
-    )
-
-
-@pytest.fixture
-def anthropic_config():
-    """Anthropic configuration."""
-    return LLMConfig(
-        provider=LLMProvider.ANTHROPIC,
-        api_key="sk-ant-test-key",
-        model="claude-3-opus",
-        temperature=0.7,
-        max_tokens=500,
-    )
-
-
-@pytest.fixture
-def local_config():
-    """Local LLM configuration."""
-    return LLMConfig(
-        provider=LLMProvider.LOCAL,
-        model="llama2",
-        base_url="http://localhost:1234/v1",
-        temperature=0.7,
-        max_tokens=500,
-    )
-
-
-# ============================================================================
-# LLMConfig Tests
-# ============================================================================
-
-
-class TestLLMConfig:
-    """LLMConfig initialization and validation."""
-
-    def test_openai_config_creation(self, openai_config):
-        """OpenAI config should be created successfully."""
-        assert openai_config.provider == LLMProvider.OPENAI
-        assert openai_config.model == "gpt-4"
-        assert openai_config.temperature == 0.7
-        assert openai_config.max_tokens == 500
-
-    def test_anthropic_config_creation(self, anthropic_config):
-        """Anthropic config should be created successfully."""
-        assert anthropic_config.provider == LLMProvider.ANTHROPIC
-        assert anthropic_config.model == "claude-3-opus"
-
-    def test_local_config_creation(self, local_config):
-        """Local config should be created successfully."""
-        assert local_config.provider == LLMProvider.LOCAL
-        assert local_config.base_url == "http://localhost:1234/v1"
-
-    def test_config_temperature_bounds(self):
-        """Temperature should be valid (0.0 - 2.0)."""
-        config = LLMConfig(
-            provider=LLMProvider.OPENAI,
-            api_key="test",
-            temperature=0.0,  # Min
-        )
-        assert config.temperature == 0.0
-
-        config = LLMConfig(
-            provider=LLMProvider.OPENAI,
-            api_key="test",
-            temperature=2.0,  # Max
-        )
-        assert config.temperature == 2.0
-
-
-# ============================================================================
-# LLMManager Tests
-# ============================================================================
-
-
-class TestLLMManager:
-    """LLMManager client selection and orchestration."""
-
-    def test_openai_manager_creation(self, openai_config):
-        """LLMManager should create OpenAI client (or handle import error)."""
-        try:
-            manager = LLMManager(openai_config)
-            assert manager.config == openai_config
-            assert manager.client is not None
-        except ImportError as e:
-            # openai not installed, which is fine for testing
-            assert "openai" in str(e).lower()
-
-    def test_anthropic_manager_creation(self, anthropic_config):
-        """LLMManager should create Anthropic client (or handle import error)."""
-        try:
-            manager = LLMManager(anthropic_config)
-            assert manager.config == anthropic_config
-            assert manager.client is not None
-        except ImportError as e:
-            # anthropic not installed, which is fine for testing
-            assert "anthropic" in str(e).lower()
-
-    def test_local_manager_creation(self, local_config):
-        """LLMManager should create Local client (or handle import error)."""
-        try:
-            manager = LLMManager(local_config)
-            assert manager.config == local_config
-            assert manager.client is not None
-        except ImportError as e:
-            # httpx not installed, which is fine for testing
-            assert "httpx" in str(e).lower()
-
-    def test_manager_config_update(self, openai_config):
-        """LLMManager config should be updatable."""
-        try:
-            manager = LLMManager(openai_config)
-            manager.config.temperature = 0.5
-            assert manager.config.temperature == 0.5
-
-            manager.config.max_tokens = 1000
-            assert manager.config.max_tokens == 1000
-        except ImportError:
-            # Libraries not installed, which is fine for testing
-            pass
-
-
-# ============================================================================
-# OpenAI Client Tests
-# ============================================================================
-
-
-class TestOpenAIClient:
-    """OpenAI client generation and streaming."""
-
-    def test_openai_generate_non_streaming(self, openai_config):
-        """OpenAI client initialization should work (or handle import error)."""
-        try:
-            from ont_platform.llm.llm_integration import OpenAIClient
-
-            # Just test that it can be instantiated
-            client = OpenAIClient(openai_config)
-            assert client.config == openai_config
-        except ImportError:
-            # openai not installed, which is fine
-            pass
-
-    def test_openai_generate_streaming(self, openai_config):
-        """OpenAI client should support streaming interface."""
-        try:
-            from ont_platform.llm.llm_integration import OpenAIClient
-
-            # Test that the streaming method is defined
-            client = OpenAIClient(openai_config)
-            assert hasattr(client, 'generate_stream')
-            assert callable(client.generate_stream)
-        except ImportError:
-            # openai not installed, which is fine
-            pass
-
-
-# ============================================================================
-# Streaming Tests
-# ============================================================================
-
-
-class TestStreamingResponses:
-    """Server-Sent Events streaming functionality."""
-
-    def test_stream_format(self):
-        """Streaming should produce valid SSE format."""
-        # SSE format: "data: {json}\n\n"
-        stream_data = "data: {\"type\": \"token\", \"content\": \"hello\"}\n\n"
-
-        lines = stream_data.strip().split("\n\n")
-        assert len(lines) == 1
-
-        data_line = lines[0]
-        assert data_line.startswith("data: ")
-
-        json_str = data_line[6:]  # Remove "data: "
-        parsed = json.loads(json_str)
-        assert parsed["type"] == "token"
-        assert parsed["content"] == "hello"
-
-    def test_metadata_streaming(self):
-        """Streaming should include metadata."""
-        metadata = {
-            "type": "metadata",
-            "query": "What is AI?",
-            "context_nodes": 50,
-            "relevant_entities": ["AI", "Machine Learning", "Deep Learning"],
-        }
-
-        sse_line = f"data: {json.dumps(metadata)}\n\n"
-        assert "metadata" in sse_line
-        assert "query" in sse_line
-
-    def test_completion_signal_streaming(self):
-        """Streaming should send completion signal."""
-        completion = {
-            "type": "complete",
-            "total_tokens": 100,
-        }
-
-        sse_line = f"data: {json.dumps(completion)}\n\n"
-        assert "complete" in sse_line
-        assert "100" in sse_line
-
-
-# ============================================================================
-# Caching Tests
-# ============================================================================
-
-
-class TestCaching:
-    """Response caching with Redis."""
-
-    def test_cache_key_generation(self):
-        """Cache keys should be deterministic and consistent."""
-        import hashlib
-
-        query = "What is the meaning of life?"
-        context_hops = 2
-
-        key_data = f"{query}:{context_hops}"
-        key_hash = hashlib.sha256(key_data.encode()).hexdigest()[:16]
-        cache_key = f"phase7:rag:{key_hash}"
-
-        # Same input should produce same key
-        key_data2 = f"{query}:{context_hops}"
-        key_hash2 = hashlib.sha256(key_data2.encode()).hexdigest()[:16]
-        cache_key2 = f"phase7:rag:{key_hash2}"
-
-        assert cache_key == cache_key2
-
-    def test_cache_key_uniqueness(self):
-        """Different queries should produce different cache keys."""
-        import hashlib
-
-        def make_key(query, hops):
-            key_data = f"{query}:{hops}"
-            key_hash = hashlib.sha256(key_data.encode()).hexdigest()[:16]
-            return f"phase7:rag:{key_hash}"
-
-        key1 = make_key("What is AI?", 2)
-        key2 = make_key("What is ML?", 2)
-        key3 = make_key("What is AI?", 3)
-
-        assert key1 != key2
-        assert key1 != key3
-        assert key2 != key3
-
-    def test_cache_hit_detection(self):
-        """Cached response should be detected."""
-        cached_response = {
-            "query": "Test query",
-            "answer": "Test answer",
-            "context_size": 10,
-            "relevant_entities": ["Entity1"],
-            "latency_ms": 100,
-            "cached": False,
-            "model": "gpt-4",
-            "provider": "openai",
-        }
-
-        # Simulate Redis cache hit
-        assert cached_response is not None
-        assert isinstance(cached_response, dict)
-        assert "answer" in cached_response
-
-    def test_response_serialization(self):
-        """Cached response should be JSON serializable."""
-        response = {
-            "query": "What is ontology?",
-            "answer": "Ontology is...",
-            "context_size": 25,
-            "relevant_entities": ["Entity1", "Entity2"],
-            "latency_ms": 150.5,
-            "cached": False,
-            "model": "gpt-4",
-            "provider": "openai",
-        }
-
-        # Should serialize to JSON without errors
-        json_str = json.dumps(response, default=str)
-        parsed = json.loads(json_str)
-
-        assert parsed["query"] == response["query"]
-        assert parsed["latency_ms"] == response["latency_ms"]
-
-
-# ============================================================================
-# RAG Pipeline Tests
-# ============================================================================
-
-
-class TestRAGPipeline:
-    """RAG context extraction and prompt building."""
-
-    def test_rag_prompt_structure(self):
-        """RAG prompt should include context and query."""
-        query = "What are the main features?"
-        context = {
-            "relevant_entities": ["Feature1", "Feature2", "Feature3"],
-            "nodes": [
-                {"label": "Feature1"},
-                {"label": "Feature2"},
-            ],
-        }
-
-        prompt = f"""당신은 지식 그래프 기반 질문 답변 어시스턴트입니다.
-다음 지식 그래프 정보를 참고하여 질문에 답변해주세요.
-
-=== 지식 그래프 컨텍스트 ===
-관련 엔티티:
-- Feature1
-- Feature2
-
-=== 사용자 질문 ===
-{query}
-
-위의 지식 그래프 정보를 바탕으로 명확하고 정확한 답변을 제공해주세요."""
-
-        assert query in prompt
-        assert "지식 그래프" in prompt
-        assert "Feature1" in prompt or "관련 엔티티" in prompt
-
-    def test_rag_context_formatting(self):
-        """RAG context should be properly formatted."""
-        context = {
-            "relevant_entities": ["Apple", "iPhone", "Steve Jobs"],
-            "nodes": [
-                {"label": "Apple", "type": "Company"},
-                {"label": "iPhone", "type": "Product"},
-            ],
-        }
-
-        # Check context structure
-        assert "relevant_entities" in context
-        assert "nodes" in context
-        assert len(context["relevant_entities"]) == 3
-        assert len(context["nodes"]) == 2
-
-    def test_rag_metadata_inclusion(self):
-        """RAG metadata should be included in response."""
-        metadata = {
-            "query": "What is Apple?",
-            "context_nodes": 45,
-            "relevant_entities": ["Apple", "iPhone"],
-            "extraction_time_ms": 120.5,
-            "llm_provider": "openai",
-            "llm_model": "gpt-4",
-        }
-
-        assert metadata["context_nodes"] > 0
-        assert len(metadata["relevant_entities"]) > 0
-        assert metadata["extraction_time_ms"] > 0
-
-
-# ============================================================================
-# Error Handling Tests
-# ============================================================================
-
-
-class TestErrorHandling:
-    """Error handling and edge cases."""
-
-    def test_invalid_provider(self):
-        """Invalid provider should be handled."""
-        # LLMConfig accepts string for provider (no validation at init)
-        # but Manager will fail when trying to create client
-        config = LLMConfig(
-            provider=LLMProvider.OPENAI,
-            api_key="test",
-            model="gpt-4",
-        )
-        assert config.provider == LLMProvider.OPENAI
-
-    def test_missing_api_key_openai(self):
-        """OpenAI config should warn about missing API key."""
-        config = LLMConfig(
-            provider=LLMProvider.OPENAI,
-            api_key=None,
-            model="gpt-4",
-        )
-        # Should not crash, but API key will be None
-        assert config.api_key is None
-
-    def test_empty_query_handling(self):
-        """Empty query should be handled gracefully."""
-        query = ""
-        assert query == ""
-        assert len(query) == 0
-
-    def test_very_long_query_handling(self):
-        """Very long queries should be handled."""
-        query = "What is " * 1000  # Very long query
-        assert len(query) > 1000
-
-
-# ============================================================================
-# Integration Tests
-# ============================================================================
-
-
-class TestPhase7Integration:
-    """End-to-end Phase 7 workflow."""
-
-    def test_rag_to_llm_workflow(self):
-        """RAG context should flow to LLM correctly."""
-        # 1. RAG context extraction
-        rag_context = {
-            "query": "What is Apple?",
-            "nodes": [
-                {"label": "Apple", "type": "Company"},
-                {"label": "iPhone", "type": "Product"},
-            ],
-            "relevant_entities": ["Apple", "iPhone", "Steve Jobs"],
-        }
-
-        # 2. Prompt building
-        prompt = f"""Knowledge Graph Context:
-Entities: {', '.join(rag_context['relevant_entities'])}
-
-Query: {rag_context['query']}
-"""
-
-        # 3. Should be ready for LLM
-        assert len(prompt) > 0
-        assert rag_context["query"] in prompt
-        assert "Apple" in prompt
-
-    def test_cache_to_llm_selection(self):
-        """Should choose cached response or call LLM."""
-        cached_response = {
-            "answer": "Cached response",
-            "cached": True,
-        }
-
-        # If cached, use it
-        if cached_response.get("cached"):
-            response = cached_response
-        else:
-            response = {"answer": "New LLM response"}
-
-        assert response["answer"] == "Cached response"
-
-    def test_streaming_to_cache_flow(self):
-        """Streaming response should be cacheable after completion."""
-        tokens = ["Hello", " ", "World"]
-        full_response = "".join(tokens)
-
-        # After streaming completes, can cache
-        cache_data = {
-            "answer": full_response,
-            "cached": False,
-        }
-
-        assert cache_data["answer"] == "Hello World"
-
-
-# ============================================================================
-# Performance Tests
-# ============================================================================
-
-
-class TestPerformance:
-    """Performance characteristics."""
-
-    def test_cache_lookup_speed(self):
-        """Cache lookup should be very fast."""
-        # Simulate cache lookup
-        cache = {
-            "key1": {"answer": "Response 1"},
-            "key2": {"answer": "Response 2"},
-        }
-
-        import time
-
-        start = time.time()
-        result = cache.get("key1")
-        elapsed = (time.time() - start) * 1000
-
-        assert result is not None
-        assert elapsed < 10  # Should be < 10ms
-
-    def test_prompt_building_speed(self):
-        """Prompt building should be fast."""
-        context = {
-            "relevant_entities": ["E1", "E2", "E3"] * 100,  # 300 entities
-            "nodes": [{"label": f"Node{i}"} for i in range(100)],
-        }
-
-        import time
-
-        start = time.time()
-
-        prompt = f"""Context: {', '.join(context['relevant_entities'][:50])}
-
-Query: What is this?
-"""
-        elapsed = (time.time() - start) * 1000
-
-        assert len(prompt) > 0
-        assert elapsed < 100  # Should be < 100ms
-
-
-if __name__ == "__main__":
-    pytest.main([__file__, "-v"])

tests/test_phase8_enterprise.py

@@ -1,491 +0,0 @@
-"""Phase 8 엔터프라이즈 기능 테스트.
-
-테스트:
-- 멀티테넌트 인증
-- 감시 로그
-- WebSocket 실시간
-- 비용 관리
-"""
-
-import pytest
-from datetime import datetime, timedelta
-
-from ont_platform.auth.models import Organization, User, APIKey, CurrentUser
-from ont_platform.auth.auth import JWTAuth, APIKeyAuth, PasswordHasher
-from ont_platform.auth.rbac import RBAC, Role, Permission
-from ont_platform.audit.logger import AuditLogger
-from ont_platform.audit.models import AuditLog, AuditAction, ResourceType
-from ont_platform.billing.calculator import CostCalculator
-from ont_platform.billing.models import OperationType, SubscriptionTier, Subscription
-from ont_platform.realtime.websocket import ConnectionManager
-from ont_platform.realtime.broadcaster import EventBroadcaster
-
-
-# ============================================================================
-# 인증 테스트
-# ============================================================================
-
-
-class TestMultitenantAuth:
-    """멀티테넌트 인증."""
-
-    def test_organization_creation(self):
-        """조직 생성."""
-        org = Organization(name="Test Organization")
-        assert org.name == "Test Organization"
-        assert org.subscription_tier == "free"
-        assert org.is_active
-
-    def test_user_creation(self):
-        """사용자 생성."""
-        user = User(
-            org_id="org_123",
-            email="user@example.com",
-            username="testuser",
-            role="editor",
-        )
-        assert user.org_id == "org_123"
-        assert user.email == "user@example.com"
-        assert user.role == "editor"
-
-    def test_api_key_generation(self):
-        """API 키 생성."""
-        api_key = APIKeyAuth.generate_key()
-        assert api_key.startswith("sk_")
-        assert len(api_key) > 20
-
-    def test_api_key_hashing(self):
-        """API 키 해싱."""
-        api_key = "sk_test123"
-        hash1 = APIKeyAuth.hash_key(api_key)
-        hash2 = APIKeyAuth.hash_key(api_key)
-
-        assert hash1 == hash2  # 같은 키는 같은 해시
-
-    def test_password_hashing(self):
-        """비밀번호 해싱."""
-        password = "my_secure_password"
-        hashed = PasswordHasher.hash_password(password)
-
-        assert hashed != password
-        assert PasswordHasher.verify_password(password, hashed)
-        assert not PasswordHasher.verify_password("wrong_password", hashed)
-
-    def test_jwt_token_creation(self):
-        """JWT 토큰 생성."""
-        token = JWTAuth.create_token(
-            user_id="user_123",
-            org_id="org_123",
-            email="user@example.com",
-            role="editor",
-        )
-
-        assert isinstance(token, str)
-        assert len(token) > 50
-
-    def test_jwt_token_verification(self):
-        """JWT 토큰 검증."""
-        token = JWTAuth.create_token(
-            user_id="user_123",
-            org_id="org_123",
-            email="user@example.com",
-            role="editor",
-        )
-
-        payload = JWTAuth.verify_token(token)
-        assert payload.user_id == "user_123"
-        assert payload.org_id == "org_123"
-        assert payload.role == "editor"
-
-    def test_jwt_token_expiration(self):
-        """JWT 토큰 만료."""
-        token = JWTAuth.create_token(
-            user_id="user_123",
-            org_id="org_123",
-            email="user@example.com",
-            role="editor",
-            expires_delta=timedelta(seconds=-1),  # 이미 만료됨
-        )
-
-        from fastapi import HTTPException
-
-        with pytest.raises(HTTPException):
-            JWTAuth.verify_token(token)
-
-    def test_current_user_creation(self):
-        """현재 사용자 객체 생성."""
-        user = CurrentUser(
-            user_id="user_123",
-            org_id="org_123",
-            email="user@example.com",
-            username="testuser",
-            role="editor",
-            is_active=True,
-        )
-
-        assert user.user_id == "user_123"
-        assert user.org_id == "org_123"
-
-
-# ============================================================================
-# RBAC 테스트
-# ============================================================================
-
-
-class TestRBAC:
-    """역할 기반 액세스 제어."""
-
-    def test_admin_permissions(self):
-        """관리자 권한."""
-        rbac = RBAC()
-        permissions = rbac.get_permissions(Role.ADMIN.value)
-
-        assert Permission.READ_ENTITY in permissions
-        assert Permission.DELETE_ENTITY in permissions
-        assert Permission.MANAGE_USERS in permissions
-        assert Permission.VIEW_AUDIT_LOG in permissions
-
-    def test_editor_permissions(self):
-        """편집자 권한."""
-        rbac = RBAC()
-        permissions = rbac.get_permissions(Role.EDITOR.value)
-
-        assert Permission.READ_ENTITY in permissions
-        assert Permission.CREATE_ENTITY in permissions
-        assert Permission.DELETE_ENTITY in permissions
-        assert Permission.MANAGE_USERS not in permissions
-
-    def test_viewer_permissions(self):
-        """뷰어 권한."""
-        rbac = RBAC()
-        permissions = rbac.get_permissions(Role.VIEWER.value)
-
-        assert Permission.READ_ENTITY in permissions
-        assert Permission.CREATE_ENTITY not in permissions
-        assert Permission.DELETE_ENTITY not in permissions
-
-    def test_permission_check(self):
-        """권한 확인."""
-        rbac = RBAC()
-
-        assert rbac.has_permission(Role.ADMIN.value, Permission.DELETE_ENTITY.value)
-        assert not rbac.has_permission(
-            Role.VIEWER.value, Permission.DELETE_ENTITY.value
-        )
-
-    def test_all_permissions_retrieval(self):
-        """모든 권한 조회."""
-        rbac = RBAC()
-        all_perms = rbac.get_all_permissions()
-
-        assert "admin" in all_perms
-        assert "editor" in all_perms
-        assert "viewer" in all_perms
-        assert "api" in all_perms
-
-
-# ============================================================================
-# 감시 로그 테스트
-# ============================================================================
-
-
-class TestAuditLogging:
-    """감시 로깅."""
-
-    @pytest.mark.asyncio
-    async def test_audit_log_creation(self):
-        """감시 로그 생성."""
-        logger = AuditLogger()
-
-        log = await logger.log_action(
-            org_id="org_123",
-            user_id="user_456",
-            action=AuditAction.CREATE,
-            resource_type=ResourceType.ENTITY,
-            resource_id="entity_789",
-        )
-
-        assert log.org_id == "org_123"
-        assert log.user_id == "user_456"
-        assert log.action == AuditAction.CREATE
-
-    @pytest.mark.asyncio
-    async def test_audit_log_retrieval(self):
-        """감시 로그 조회."""
-        logger = AuditLogger()
-
-        await logger.log_action(
-            org_id="org_123",
-            user_id="user_456",
-            action=AuditAction.UPDATE,
-            resource_type=ResourceType.ENTITY,
-            resource_id="entity_789",
-        )
-
-        logs = await logger.get_audit_trail(
-            org_id="org_123",
-            resource_id="entity_789",
-        )
-
-        assert len(logs) > 0
-        assert logs[0].resource_id == "entity_789"
-
-    @pytest.mark.asyncio
-    async def test_audit_statistics(self):
-        """감시 통계."""
-        logger = AuditLogger()
-
-        for i in range(3):
-            await logger.log_action(
-                org_id="org_123",
-                user_id="user_456",
-                action=AuditAction.READ,
-                resource_type=ResourceType.ENTITY,
-                resource_id="entity_789",
-            )
-
-        stats = await logger.get_statistics(org_id="org_123")
-
-        assert stats["total_logs"] >= 3
-        assert "by_action" in stats
-
-
-# ============================================================================
-# 비용 관리 테스트
-# ============================================================================
-
-
-class TestBillingAndQuota:
-    """비용 관리 및 할당량."""
-
-    @pytest.mark.asyncio
-    async def test_cost_calculation(self):
-        """비용 계산."""
-        calc = CostCalculator()
-
-        cost = await calc.calculate_cost(
-            operation_type=OperationType.LLM_CALL,
-            quantity=1000,  # 1000 토큰
-        )
-
-        assert cost == 1.0  # 1000 * $0.001
-
-    @pytest.mark.asyncio
-    async def test_usage_recording(self):
-        """사용량 기록."""
-        calc = CostCalculator()
-
-        usage = await calc.record_usage(
-            org_id="org_123",
-            user_id="user_456",
-            operation_type=OperationType.LLM_CALL,
-            quantity=500,
-        )
-
-        assert usage.org_id == "org_123"
-        assert usage.quantity == 500
-        assert usage.cost == 0.5
-
-    @pytest.mark.asyncio
-    async def test_quota_check_within_limit(self):
-        """할당량 확인 (범위 내)."""
-        calc = CostCalculator()
-
-        subscription = Subscription(
-            org_id="org_123",
-            tier=SubscriptionTier.PRO,
-            monthly_limit=100.0,
-            current_month_cost=50.0,
-        )
-
-        allowed, msg = await calc.check_quota(
-            org_id="org_123",
-            subscription=subscription,
-            estimated_cost=30.0,
-        )
-
-        assert allowed
-        assert "OK" in msg
-
-    @pytest.mark.asyncio
-    async def test_quota_check_exceeded(self):
-        """할당량 확인 (초과)."""
-        calc = CostCalculator()
-
-        subscription = Subscription(
-            org_id="org_123",
-            tier=SubscriptionTier.FREE,
-            monthly_limit=10.0,
-            current_month_cost=9.0,
-        )
-
-        allowed, msg = await calc.check_quota(
-            org_id="org_123",
-            subscription=subscription,
-            estimated_cost=5.0,
-        )
-
-        assert not allowed
-        assert "Quota exceeded" in msg
-
-    @pytest.mark.asyncio
-    async def test_usage_statistics(self):
-        """사용량 통계."""
-        calc = CostCalculator()
-
-        await calc.record_usage(
-            org_id="org_123",
-            user_id="user_456",
-            operation_type=OperationType.API_CALL,
-            quantity=10,
-        )
-
-        await calc.record_usage(
-            org_id="org_123",
-            user_id="user_456",
-            operation_type=OperationType.LLM_CALL,
-            quantity=1000,
-        )
-
-        stats = await calc.get_usage_statistics(org_id="org_123")
-
-        assert stats.total_cost > 0
-        assert stats.api_calls >= 1
-        assert stats.llm_tokens >= 1000
-
-
-# ============================================================================
-# WebSocket 테스트
-# ============================================================================
-
-
-class TestWebSocketAndBroadcasting:
-    """WebSocket 및 브로드캐스팅."""
-
-    @pytest.mark.asyncio
-    async def test_connection_tracking(self):
-        """연결 추적."""
-        manager = ConnectionManager()
-
-        # 연결 수 확인
-        assert manager.get_connection_count("org_123") == 0
-        assert "org_123" not in manager.get_org_ids()
-
-    @pytest.mark.asyncio
-    async def test_broadcaster_entity_created(self):
-        """엔티티 생성 이벤트."""
-        manager = ConnectionManager()
-        broadcaster = EventBroadcaster(manager)
-
-        entity = {"id": "entity_123", "label": "Test Entity"}
-        sent = await broadcaster.broadcast_entity_created(
-            org_id="org_123",
-            entity=entity,
-            user_id="user_456",
-        )
-
-        # 연결이 없으므로 0
-        assert sent == 0
-
-    @pytest.mark.asyncio
-    async def test_broadcaster_graph_analyzed(self):
-        """그래프 분석 이벤트."""
-        manager = ConnectionManager()
-        broadcaster = EventBroadcaster(manager)
-
-        results = {"centrality": {"entity_1": 0.95}}
-        sent = await broadcaster.broadcast_graph_analyzed(
-            org_id="org_123",
-            analysis_type="pagerank",
-            results=results,
-        )
-
-        assert sent == 0  # 연결 없음
-
-    @pytest.mark.asyncio
-    async def test_broadcaster_notification(self):
-        """일반 알림."""
-        manager = ConnectionManager()
-        broadcaster = EventBroadcaster(manager)
-
-        sent = await broadcaster.broadcast_notification(
-            org_id="org_123",
-            title="Test Alert",
-            message="This is a test",
-            severity="info",
-        )
-
-        assert sent == 0
-
-
-# ============================================================================
-# 통합 테스트
-# ============================================================================
-
-
-class TestPhase8Integration:
-    """Phase 8 통합 시나리오."""
-
-    @pytest.mark.asyncio
-    async def test_complete_workflow(self):
-        """완전한 워크플로우."""
-
-        # 1. 조직 생성
-        org = Organization(name="Test Org")
-        assert org.is_active
-
-        # 2. 사용자 생성
-        user = User(
-            org_id=org.id,
-            email="user@example.com",
-            username="testuser",
-            role="editor",
-        )
-
-        # 3. 토큰 생성
-        token = JWTAuth.create_token(
-            user_id=user.id,
-            org_id=org.id,
-            email=user.email,
-            role=user.role,
-        )
-        assert token
-
-        # 4. 토큰 검증
-        payload = JWTAuth.verify_token(token)
-        assert payload.org_id == org.id
-
-        # 5. 감시 로그
-        logger = AuditLogger()
-        log = await logger.log_action(
-            org_id=org.id,
-            user_id=user.id,
-            action=AuditAction.CREATE,
-            resource_type=ResourceType.ENTITY,
-            resource_id="entity_123",
-        )
-        assert log.org_id == org.id
-
-        # 6. 비용 기록
-        calc = CostCalculator()
-        usage = await calc.record_usage(
-            org_id=org.id,
-            user_id=user.id,
-            operation_type=OperationType.API_CALL,
-            quantity=1,
-        )
-        assert usage.cost > 0
-
-    def test_rbac_integration(self):
-        """RBAC 통합."""
-        rbac = RBAC()
-
-        # 역할별 권한 확인
-        assert rbac.has_permission(Role.ADMIN.value, Permission.MANAGE_USERS.value)
-        assert not rbac.has_permission(
-            Role.VIEWER.value, Permission.DELETE_ENTITY.value
-        )
-
-
-if __name__ == "__main__":
-    pytest.main([__file__, "-v"])

tests/test_research.py

@@ -1,151 +0,0 @@
-from pathlib import Path
-
-from sqlalchemy.orm import Session
-
-from crawler_platform.app.config.loader import load_project_config
-from crawler_platform.app.core.database import models
-from crawler_platform.app.core.database.repository import KnowledgeRepository
-from crawler_platform.app.core.database.session import make_engine
-from crawler_platform.app.core.extractor.base import ExtractedClaim, ExtractedEntity, ExtractionBundle, ExtractionPageContext
-from crawler_platform.app.core.extractor.validation import attach_page_context
-from crawler_platform.app.core.research.graph_query import SemanticGraphQuery
-from crawler_platform.app.core.research.graph_research_loop import GraphResearchLoop
-from crawler_platform.app.core.research.relevance_engine import RelevanceEngine
-from crawler_platform.app.domains.perfume.extractor import PerfumeRuleBasedExtractor
-
-
-def make_repo():
-    config = load_project_config("configs/perfume_subscription.yaml")
-    engine = make_engine("sqlite:///:memory:")
-    models.Base.metadata.create_all(engine)
-    session = Session(engine)
-    repo = KnowledgeRepository(session)
-    project = repo.upsert_project(config)
-    source = repo.get_source(project.id, "official_brand_site")
-    return config, session, repo, project, source
-
-
-def seed_validated_claim(repo, project, source, config):
-    page = repo.upsert_page(
-        project.id,
-        source.id,
-        "https://example.com/product/neroli",
-        "Neroli Summer",
-        200,
-        "Neroli Summer\nTop notes: Bergamot",
-    )
-    bundle = ExtractionBundle(
-        entities=[
-            ExtractedEntity("Perfume", "Neroli Summer", evidence_text="Neroli Summer", confidence=0.9),
-            ExtractedEntity("FragranceNote", "Bergamot", evidence_text="Top notes: Bergamot", confidence=0.9),
-        ],
-        claims=[
-            ExtractedClaim(
-                "Neroli Summer",
-                "Perfume",
-                "hasTopNote",
-                "Bergamot",
-                "FragranceNote",
-                evidence_text="Top notes: Bergamot",
-                confidence=0.93,
-                confidence_reason="directly stated",
-            )
-        ],
-        extractor_name="llm_json_extractor",
-        provider="lm_studio",
-        raw_output={"extraction_mode": "primary"},
-    )
-    context = ExtractionPageContext(
-        url=page.url,
-        final_url=page.url,
-        title=page.title,
-        page_type="ProductPage",
-        clean_text=page.cleaned_text_summary,
-        source_zones=[
-            {
-                "zone_type": "product_description",
-                "selector": "main",
-                "text": "Top notes: Bergamot",
-                "claim_allowed": True,
-            }
-        ],
-    )
-    repo.save_extraction_bundle(project.id, source, page, attach_page_context(bundle, context), config)
-
-
-def test_relevance_engine_prefers_graph_related_product_urls():
-    config, session, repo, project, source = make_repo()
-    seed_validated_claim(repo, project, source, config)
-
-    engine = RelevanceEngine(session)
-    product = engine.score_url(
-        project_id=project.id,
-        url="https://example.com/product/neroli-bergamot",
-        label="Neroli Summer Bergamot perfume",
-        source_trust=source.trust_level,
-    )
-    login = engine.score_url(
-        project_id=project.id,
-        url="https://example.com/member/login.html",
-        label="Login",
-        source_trust=source.trust_level,
-    )
-
-    assert product.score > login.score
-    assert product.breakdown["entity_overlap"] > 0
-    session.close()
-
-
-def test_semantic_graph_query_returns_trend_summary():
-    config, session, repo, project, source = make_repo()
-    seed_validated_claim(repo, project, source, config)
-
-    rows = SemanticGraphQuery(session).trend_summary(project.id)
-
-    assert rows[0]["name"] == "Bergamot"
-    assert rows[0]["predicate"] == "hasTopNote" or rows[0]["support_count"] >= 1
-    session.close()
-
-
-def test_graph_research_loop_records_session_memory():
-    config, session, repo, project, _source = make_repo()
-    fixture = Path("tests/fixtures/sample_perfume.html").resolve()
-    loop = GraphResearchLoop(repo, PerfumeRuleBasedExtractor())
-
-    result = loop.run(
-        project_config=config,
-        source_name="official_brand_site",
-        seed_url=str(fixture),
-        goal="Fixture semantic exploration",
-        max_depth=0,
-        max_steps=1,
-        min_relevance=0.0,
-    )
-    session.commit()
-
-    assert result.explored_count == 1
-    assert result.history
-    assert result.memory["visited_targets"]
-    jobs = session.query(models.CrawlJob).all()
-    assert any((job.metadata_json or {}).get("kind") == "research_session" for job in jobs)
-    session.close()
-
-
-def test_graph_research_loop_can_start_from_knowledge_gaps():
-    config, session, repo, _project, _source = make_repo()
-    loop = GraphResearchLoop(repo, PerfumeRuleBasedExtractor())
-
-    result = loop.run(
-        project_config=config,
-        source_name="official_brand_site",
-        goal="Gap-driven semantic exploration",
-        max_depth=0,
-        max_steps=1,
-        max_branch=3,
-        min_relevance=0.0,
-    )
-
-    assert result.explored_count == 1
-    assert result.history
-    assert result.history[0]["item"]["target_type"] == "knowledge_gap"
-    session.close()

tests/test_site_crawler.py

@@ -1,162 +0,0 @@
-from crawler_platform.app.core.crawler.site_crawler import (
-    classify_page,
-    is_failed_fetch_status,
-    normalize_url,
-    should_analyze_page,
-)
-from crawler_platform.app.core.crawler.discovery import discover_links, normalize_cafe24_product_url
-from crawler_platform.app.core.crawler.fetchers import FallbackFetcher, FetchResult, RobotsPolicy, detect_crawl_status
-from crawler_platform.app.api.routes import CrawlRequest
-
-
-def test_classify_perfume_product_page():
-    text = "Top notes: Bergamot, Neroli\nMiddle notes: Jasmine\nBase notes: Musk\nPrice $89"
-
-    assert classify_page("https://example.com/products/neroli", "Neroli Summer", text) == "ProductPage"
-
-
-def test_normalize_url_removes_fragment_and_trailing_slash():
-    assert normalize_url("https://example.com/path/#details") == "https://example.com/path"
-
-
-def test_failed_fetch_status_detection():
-    assert is_failed_fetch_status(None)
-    assert is_failed_fetch_status(404)
-    assert is_failed_fetch_status(500)
-    assert not is_failed_fetch_status(200)
-    assert not is_failed_fetch_status(302)
-
-
-def test_crawl_status_ignores_captcha_mentions_inside_scripts():
-    html = "<html><body><h1>Product</h1><script>var path='recaptcha';</script></body></html>"
-
-    status, warnings = detect_crawl_status(200, html)
-
-    assert status == "success"
-    assert warnings == []
-
-
-def test_should_analyze_page_supports_legacy_names():
-    assert should_analyze_page("ProductPage", {"product"})
-    assert not should_analyze_page("CommunityPage", {"product", "brand", "review"})
-
-
-def test_robots_policy_disabled_skips_check():
-    decision = RobotsPolicy().check("https://example.com/products/1", respect_robots_txt=False)
-
-    assert decision.allowed
-    assert decision.status == "disabled"
-    assert not decision.checked
-
-
-def test_crawl_request_defaults_to_no_robots_check():
-    request = CrawlRequest(
-        config_path="configs/perfume_subscription.yaml",
-        source_name="official_brand_site",
-        url="https://example.com",
-    )
-
-    assert request.check_robots_txt is False
-
-
-def test_robots_policy_allows_when_robots_unavailable(monkeypatch):
-    class UnavailableRobotsParser:
-        def set_url(self, url):
-            self.url = url
-
-        def read(self):
-            raise OSError("network unavailable")
-
-    monkeypatch.setattr(
-        "crawler_platform.app.core.crawler.fetchers.RobotFileParser",
-        UnavailableRobotsParser,
-    )
-
-    decision = RobotsPolicy().check("https://example.com/products/1")
-
-    assert decision.allowed
-    assert decision.status == "unavailable"
-    assert "allowing crawl" in decision.reason
-
-
-def test_robots_policy_reports_block_reason(monkeypatch):
-    class BlockingRobotsParser:
-        def set_url(self, url):
-            self.url = url
-
-        def read(self):
-            return None
-
-        def can_fetch(self, user_agent, url):
-            return False
-
-    monkeypatch.setattr(
-        "crawler_platform.app.core.crawler.fetchers.RobotFileParser",
-        BlockingRobotsParser,
-    )
-
-    decision = RobotsPolicy().check("https://example.com/private")
-
-    assert not decision.allowed
-    assert decision.status == "blocked"
-    assert "blocks crawling" in decision.reason
-
-
-def test_classify_board_page_before_content_analysis():
-    assert classify_page("https://example.com/board/free/read.html", "Notice", "Price $89") == "NoticePage"
-
-
-def test_classify_promotion_homepage_before_product_analysis():
-    text = "BLACK FRIDAY SALE\n회원가입 쿠폰\n무료배송 이벤트\n제품 보기"
-
-    assert classify_page("https://example.com/", "Brand", text) == "PromotionPage"
-
-
-def test_classify_product_list_and_search_as_non_detail_pages():
-    text = "Price $89\nTop notes: Bergamot\nAdd to cart"
-
-    assert classify_page("https://example.com/product/list.html?cate_no=24", "Perfume", text) == "CategoryPage"
-    assert classify_page("https://example.com/product/search.html?keyword=cotton", "Search", text) == "SearchPage"
-    assert not should_analyze_page("CategoryPage", {"ProductPage", "BrandStoryPage", "ReviewPage"})
-    assert not should_analyze_page("SearchPage", {"ProductPage", "BrandStoryPage", "ReviewPage"})
-
-
-def test_cafe24_product_detail_with_category_segment_is_product_page():
-    url = "https://the912.co.kr/product/21-기획-912-클론-니치향수-모음-40ml/513/category/1/display/2/"
-
-    assert classify_page(url, "912 clone perfume", "") == "ProductPage"
-
-
-def test_discovery_skips_utility_pages_seen_on_the912():
-    html = """
-    <a href="/myshop/wish_list.html">wish</a>
-    <a href="/board/faq/list.html?board_no=3">faq</a>
-    <a href="/event/list.html?cate_no=42">event</a>
-    <a href="/product/search.html?keyword=">search</a>
-    <a href="/product/sample/123/category/1/display/2/">product</a>
-    """
-
-    links = discover_links(html, "https://the912.co.kr", limit=10)
-
-    assert [link.url for link in links] == ["https://the912.co.kr/product/detail.html?product_no=123"]
-
-
-def test_cafe24_product_urls_are_canonicalized_for_dedupe():
-    url = "https://the912.co.kr/product/21-기획-912-클론-니치향수-모음-40ml/513/category/1/display/2/?icid=x"
-
-    assert normalize_cafe24_product_url(url) == "https://the912.co.kr/product/detail.html?product_no=513"
-
-
-def test_fallback_fetcher_continues_when_primary_fails():
-    class FailingFetcher:
-        def fetch(self, url):
-            raise PermissionError("[WinError 5] access denied")
-
-    class WorkingFetcher:
-        def fetch(self, url):
-            return FetchResult(url=url, status_code=200, html="<html><body>ok</body></html>")
-
-    result = FallbackFetcher(FailingFetcher(), WorkingFetcher(), fallback_label="requests").fetch("https://example.com")
-
-    assert result.status_code == 200
-    assert any("primary fetcher failed" in warning for warning in result.warnings)

tests/test_smoke_unittest.py

@@ -1,21 +0,0 @@
-import unittest
-
-from crawler_platform.app.config.loader import load_project_config
-from crawler_platform.app.core.crawler.fetchers import RequestsFetcher
-from crawler_platform.app.domains.perfume.extractor import PerfumeRuleBasedExtractor
-
-
-class SmokeTest(unittest.TestCase):
-    def test_config_fetch_and_extract_without_network(self):
-        config = load_project_config("configs/perfume_subscription.yaml")
-        fetch_result = RequestsFetcher().fetch("tests/fixtures/sample_perfume.html")
-        bundle = PerfumeRuleBasedExtractor().extract(fetch_result.html, config)
-
-        self.assertEqual(config.domain, "perfume")
-        self.assertEqual(fetch_result.status_code, 200)
-        self.assertIn("hasTopNote", {claim.predicate for claim in bundle.claims})
-        self.assertIn("hasPrice", {claim.predicate for claim in bundle.claims})
-
-
-if __name__ == "__main__":
-    unittest.main()