[crawler_platform 삭제]

ontology_platform/crawler_platform/app/__init__.py

@@ -0,0 +1,2 @@
+"""Application package for the crawler platform."""
+

ontology_platform/crawler_platform/app/adapters/__init__.py

@@ -0,0 +1,2 @@
+"""Domain adapters keep domain-specific semantics out of ontology core."""
+

ontology_platform/crawler_platform/app/adapters/base.py

@@ -0,0 +1,35 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any
+
+
+@dataclass(frozen=True, slots=True)
+class DomainAdapter:
+    """Domain-specific semantics plugged into the generic ontology engine."""
+
+    domain: str
+    relation_rules: dict[str, Any] = field(default_factory=dict)
+    entity_type_aliases: dict[str, str] = field(default_factory=dict)
+    entity_name_aliases: dict[str, dict[str, str]] = field(default_factory=dict)
+
+    def relation_rule(self, predicate: str) -> Any | None:
+        return self.relation_rules.get(predicate)
+
+    def normalize_entity_type(self, raw_type: str) -> str | None:
+        key = compact_key(raw_type)
+        return self.entity_type_aliases.get(key)
+
+    def normalize_entity_name(self, raw_name: str, entity_type: str | None = None) -> str | None:
+        if not entity_type:
+            return None
+        aliases = self.entity_name_aliases.get(entity_type, {})
+        return aliases.get(canonical_key(raw_name))
+
+
+def compact_key(value: Any) -> str:
+    return "".join(ch for ch in str(value or "").strip().lower() if ch.isalnum())
+
+
+def canonical_key(value: Any) -> str:
+    return " ".join(str(value or "").strip().lower().split())

ontology_platform/crawler_platform/app/adapters/ecommerce/__init__.py

@@ -0,0 +1,2 @@
+"""E-commerce domain adapters."""
+

ontology_platform/crawler_platform/app/adapters/ecommerce/perfume.py

@@ -0,0 +1,158 @@
+from __future__ import annotations
+
+from crawler_platform.app.adapters.base import DomainAdapter
+from crawler_platform.app.core.ontology.relation_schema import RelationRule
+
+
+PRODUCT_TYPES = {"Perfume", "Product"}
+
+PERFUME_RELATION_RULES: dict[str, RelationRule] = {
+    "hasBrand": RelationRule(
+        "hasBrand",
+        PRODUCT_TYPES,
+        {"Brand"},
+        page_types={"ProductPage"},
+        source_zones={"product_title", "product_summary", "product_description", "product_detail"},
+        min_confidence=0.72,
+    ),
+    "hasTopNote": RelationRule(
+        "hasTopNote",
+        PRODUCT_TYPES,
+        {"Note"},
+        page_types={"ProductPage"},
+        source_zones={"product_description", "product_detail"},
+        min_confidence=0.78,
+    ),
+    "hasMiddleNote": RelationRule(
+        "hasMiddleNote",
+        PRODUCT_TYPES,
+        {"Note"},
+        page_types={"ProductPage"},
+        source_zones={"product_description", "product_detail"},
+        min_confidence=0.78,
+    ),
+    "hasBaseNote": RelationRule(
+        "hasBaseNote",
+        PRODUCT_TYPES,
+        {"Note"},
+        page_types={"ProductPage"},
+        source_zones={"product_description", "product_detail"},
+        min_confidence=0.78,
+    ),
+    "hasAccord": RelationRule(
+        "hasAccord",
+        PRODUCT_TYPES,
+        {"Accord"},
+        page_types={"ProductPage"},
+        source_zones={"product_description", "product_detail"},
+        min_confidence=0.74,
+    ),
+    "evokesMood": RelationRule(
+        "evokesMood",
+        PRODUCT_TYPES,
+        {"Mood"},
+        page_types={"ProductPage"},
+        source_zones={"product_description", "product_detail", "review_body"},
+        min_confidence=0.75,
+    ),
+    "suitableForSeason": RelationRule(
+        "suitableForSeason",
+        PRODUCT_TYPES,
+        {"Season"},
+        page_types={"ProductPage"},
+        source_zones={"product_description", "product_detail", "review_body"},
+        min_confidence=0.75,
+    ),
+    "suitableForOccasion": RelationRule(
+        "suitableForOccasion",
+        PRODUCT_TYPES,
+        {"Occasion"},
+        page_types={"ProductPage"},
+        source_zones={"product_description", "product_detail", "review_body"},
+        min_confidence=0.75,
+    ),
+    "hasReviewKeyword": RelationRule(
+        "hasReviewKeyword",
+        PRODUCT_TYPES,
+        {"Review"},
+        page_types={"ProductPage", "ReviewPage"},
+        source_zones={"review_body", "product_description", "product_detail"},
+        min_confidence=0.78,
+    ),
+    "soldBy": RelationRule(
+        "soldBy",
+        {"Perfume", "Product", "Brand"},
+        {"Brand"},
+        page_types={"ProductPage", "BrandStoryPage"},
+        source_zones={"product_detail", "product_description", "brand_story_body"},
+        min_confidence=0.82,
+    ),
+    "hasPrice": RelationRule(
+        "hasPrice",
+        PRODUCT_TYPES,
+        literal_value=True,
+        page_types={"ProductPage"},
+        source_zones={"product_summary", "product_detail"},
+        min_confidence=0.82,
+    ),
+    "similarTo": RelationRule(
+        "similarTo",
+        PRODUCT_TYPES,
+        PRODUCT_TYPES,
+        page_types={"ProductPage"},
+        source_zones={"product_description", "product_detail"},
+        min_confidence=0.86,
+    ),
+}
+
+PERFUME_TYPE_ALIASES = {
+    "perfumeproduct": "Perfume",
+    "fragrance": "Perfume",
+    "product": "Perfume",
+    "fragrancenote": "Note",
+    "ingredient": "Note",
+    "scentnote": "Note",
+    "note": "Note",
+    "brand": "Brand",
+    "accord": "Accord",
+    "mood": "Mood",
+    "season": "Season",
+    "occasion": "Occasion",
+    "reviewkeyword": "Review",
+    "review": "Review",
+    "price": "Price",
+}
+
+PERFUME_NAME_ALIASES = {
+    "Brand": {
+        "forment": "FORMENT",
+        "theforment": "FORMENT",
+        "the forment": "FORMENT",
+        "forment korea": "FORMENT",
+    },
+    "Note": {
+        "musk": "Musk",
+        "powdery": "Powdery",
+        "floral": "Floral",
+        "citrus": "Citrus",
+    },
+    "Accord": {
+        "musk": "Musk",
+        "powdery": "Powdery",
+        "floral": "Floral",
+        "citrus": "Citrus",
+    },
+    "Mood": {
+        "fresh": "Fresh",
+        "cozy": "Cozy",
+        "romantic": "Romantic",
+        "elegant": "Elegant",
+    },
+}
+
+PERFUME_ADAPTER = DomainAdapter(
+    domain="perfume",
+    relation_rules=PERFUME_RELATION_RULES,
+    entity_type_aliases=PERFUME_TYPE_ALIASES,
+    entity_name_aliases=PERFUME_NAME_ALIASES,
+)

ontology_platform/crawler_platform/app/adapters/registry.py

@@ -0,0 +1,15 @@
+from __future__ import annotations
+
+from crawler_platform.app.adapters.base import DomainAdapter
+from crawler_platform.app.adapters.ecommerce.perfume import PERFUME_ADAPTER
+
+
+ADAPTERS: dict[str, DomainAdapter] = {
+    PERFUME_ADAPTER.domain: PERFUME_ADAPTER,
+}
+
+
+def adapter_for_domain(domain: str | None) -> DomainAdapter | None:
+    if not domain:
+        return None
+    return ADAPTERS.get(str(domain).strip().lower())

ontology_platform/crawler_platform/app/api/__init__.py

@@ -0,0 +1,2 @@
+"""FastAPI routes."""
+

ontology_platform/crawler_platform/app/cli/__init__.py

@@ -0,0 +1,2 @@
+"""Command line admin tools."""
+

ontology_platform/crawler_platform/app/cli/main.py

@@ -0,0 +1,140 @@
+from __future__ import annotations
+
+import argparse
+from dataclasses import asdict
+import json
+
+from sqlalchemy import select
+
+from crawler_platform.app.config.loader import load_project_config
+from crawler_platform.app.core.crawler.pipeline import CrawlPipeline
+from crawler_platform.app.core.database import models
+from crawler_platform.app.core.database.repository import KnowledgeRepository
+from crawler_platform.app.core.database.session import init_db, session_scope
+from crawler_platform.app.core.extractor.factory import extractor_for_domain
+from crawler_platform.app.core.ontology.definitions import ontology_for_domain
+from crawler_platform.app.core.ontology.mapper import ontology_to_dict
+from crawler_platform.app.core.recommendation.scorer import PreferenceInput, RuleBasedRecommender
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(description="Ontology crawler platform admin CLI")
+    parser.add_argument("--db", default="sqlite:///crawler_platform.db", help="SQLAlchemy database URL")
+    sub = parser.add_subparsers(dest="command", required=True)
+
+    sub.add_parser("init-db")
+
+    create_project = sub.add_parser("create-project")
+    create_project.add_argument("--config", required=True)
+
+    ontology = sub.add_parser("ontology")
+    ontology.add_argument("--domain", required=True)
+
+    crawl = sub.add_parser("crawl-url")
+    crawl.add_argument("--config", required=True)
+    crawl.add_argument("--source", required=True)
+    crawl.add_argument("--url", required=True)
+    crawl.add_argument("--extractor-provider", default="rule_based", choices=["rule_based", "openai", "ollama", "lm_studio"])
+    crawl.add_argument("--extractor-model")
+    crawl.add_argument("--extractor-base-url")
+
+    claims = sub.add_parser("claims")
+    claims.add_argument("--project", required=True)
+    claims.add_argument("--limit", type=int, default=20)
+
+    entities = sub.add_parser("entities")
+    entities.add_argument("--project", required=True)
+    entities.add_argument("--type")
+    entities.add_argument("--limit", type=int, default=20)
+
+    recommend = sub.add_parser("recommend")
+    recommend.add_argument("--project", required=True)
+    recommend.add_argument("--target-type", default="Perfume")
+    recommend.add_argument("--preferences-json", default="{}")
+    recommend.add_argument("--limit", type=int, default=10)
+    return parser
+
+
+def main() -> None:
+    args = build_parser().parse_args()
+    if args.command == "init-db":
+        init_db(args.db)
+        print(f"initialized database: {args.db}")
+        return
+    if args.command == "ontology":
+        print(json.dumps(ontology_to_dict(ontology_for_domain(args.domain)), ensure_ascii=False, indent=2))
+        return
+    if args.command == "create-project":
+        config = load_project_config(args.config)
+        with session_scope(args.db) as session:
+            project = KnowledgeRepository(session).upsert_project(config)
+            print(json.dumps({"project_id": project.id, "name": project.name, "domain": project.domain}, ensure_ascii=False))
+        return
+    if args.command == "crawl-url":
+        config = load_project_config(args.config)
+        with session_scope(args.db) as session:
+            repo = KnowledgeRepository(session)
+            pipeline = CrawlPipeline(
+                repo,
+                extractor_for_domain(
+                    config.domain,
+                    provider=args.extractor_provider,
+                    model=args.extractor_model,
+                    base_url=args.extractor_base_url,
+                ),
+            )
+            result = pipeline.crawl_url(config, args.source, args.url)
+            print(json.dumps(asdict(result), ensure_ascii=False))
+        return
+    if args.command == "claims":
+        with session_scope(args.db) as session:
+            project = KnowledgeRepository(session).get_project(args.project)
+            rows = session.execute(
+                select(models.Claim, models.Entity)
+                .join(models.Entity, models.Claim.subject_entity_id == models.Entity.id)
+                .where(models.Claim.project_id == project.id)
+                .limit(args.limit)
+            ).all()
+            print(
+                json.dumps(
+                    [
+                        {
+                            "claim_id": claim.id,
+                            "subject": subject.name,
+                            "predicate": claim.predicate,
+                            "object_entity_id": claim.object_entity_id,
+                            "object_value": claim.object_value,
+                            "confidence": claim.confidence,
+                        }
+                        for claim, subject in rows
+                    ],
+                    ensure_ascii=False,
+                    indent=2,
+                )
+            )
+        return
+    if args.command == "entities":
+        with session_scope(args.db) as session:
+            project = KnowledgeRepository(session).get_project(args.project)
+            query = select(models.Entity).where(models.Entity.project_id == project.id)
+            if args.type:
+                query = query.where(models.Entity.entity_type == args.type)
+            result = session.scalars(query.limit(args.limit)).all()
+            print(
+                json.dumps(
+                    [{"id": entity.id, "type": entity.entity_type, "name": entity.name} for entity in result],
+                    ensure_ascii=False,
+                    indent=2,
+                )
+            )
+        return
+    if args.command == "recommend":
+        preference = PreferenceInput(**json.loads(args.preferences_json))
+        with session_scope(args.db) as session:
+            project = KnowledgeRepository(session).get_project(args.project)
+            result = RuleBasedRecommender(session).recommend(project.id, args.target_type, preference, args.limit)
+            print(json.dumps([asdict(item) for item in result], ensure_ascii=False, indent=2))
+
+
+if __name__ == "__main__":
+    main()

ontology_platform/crawler_platform/app/config/__init__.py

@@ -0,0 +1,2 @@
+"""Project configuration loading."""
+

ontology_platform/crawler_platform/app/config/loader.py

@@ -0,0 +1,159 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+import json
+from pathlib import Path
+from typing import Any
+
+
+@dataclass(slots=True)
+class SourceConfig:
+    name: str
+    type: str = "unknown"
+    trust_level: float = 0.5
+    base_url: str | None = None
+    allowed_paths: list[str] = field(default_factory=list)
+    parser: str = "generic"
+    fetcher: str = "requests"
+    rate_limit_per_minute: int = 30
+    respect_robots_txt: bool = False
+
+
+@dataclass(slots=True)
+class ProjectConfig:
+    project_name: str
+    domain: str
+    target_entities: list[str]
+    fields: list[str]
+    sources: list[SourceConfig]
+    ontology: dict[str, Any] = field(default_factory=dict)
+    recommendation: dict[str, Any] = field(default_factory=dict)
+    update_policy: dict[str, Any] = field(default_factory=dict)
+
+    def source_by_name(self, name: str) -> SourceConfig:
+        for source in self.sources:
+            if source.name == name:
+                return source
+        raise KeyError(f"Unknown source in project config: {name}")
+
+
+def load_project_config(path: str | Path) -> ProjectConfig:
+    config_path = Path(path)
+    data = _load_mapping(config_path)
+    return project_config_from_dict(data)
+
+
+def project_config_from_dict(data: dict[str, Any]) -> ProjectConfig:
+    """Build a ProjectConfig from an in-memory dict (DB row, JSON payload, etc.)."""
+    sources = [SourceConfig(**item) for item in data.get("sources", [])]
+    return ProjectConfig(
+        project_name=data["project_name"],
+        domain=data["domain"],
+        target_entities=list(data.get("target_entities", [])),
+        fields=list(data.get("fields", [])),
+        sources=sources,
+        ontology=dict(data.get("ontology", {})),
+        recommendation=dict(data.get("recommendation", {})),
+        update_policy=dict(data.get("update_policy", {})),
+    )
+
+
+def _load_mapping(path: Path) -> dict[str, Any]:
+    text = path.read_text(encoding="utf-8")
+    if path.suffix.lower() == ".json":
+        return json.loads(text)
+    try:
+        import yaml
+    except ImportError as exc:
+        return _load_simple_yaml(text)
+    return yaml.safe_load(text)
+
+
+def _load_simple_yaml(text: str) -> dict[str, Any]:
+    """Small YAML fallback for project configs when PyYAML is unavailable.
+
+    It intentionally supports only the subset used by the sample project files:
+    nested mappings, lists, booleans, ints, floats, and strings.
+    """
+
+    lines: list[tuple[int, str]] = []
+    for raw_line in text.splitlines():
+        if not raw_line.strip() or raw_line.lstrip().startswith("#"):
+            continue
+        indent = len(raw_line) - len(raw_line.lstrip(" "))
+        lines.append((indent, raw_line.strip()))
+    value, index = _parse_yaml_block(lines, 0, 0)
+    if index != len(lines):
+        raise RuntimeError("Could not parse full YAML config. Install PyYAML for full YAML support.")
+    if not isinstance(value, dict):
+        raise RuntimeError("Project config root must be a mapping.")
+    return value
+
+
+def _parse_yaml_block(lines: list[tuple[int, str]], index: int, indent: int) -> tuple[Any, int]:
+    if index >= len(lines):
+        return {}, index
+    current_indent, content = lines[index]
+    if current_indent < indent:
+        return {}, index
+    if content.startswith("- "):
+        result: list[Any] = []
+        while index < len(lines):
+            item_indent, item_content = lines[index]
+            if item_indent != indent or not item_content.startswith("- "):
+                break
+            item_raw = item_content[2:].strip()
+            index += 1
+            if not item_raw:
+                child, index = _parse_yaml_block(lines, index, indent + 2)
+                result.append(child)
+            elif ":" in item_raw:
+                key, raw_value = _split_key_value(item_raw)
+                item: dict[str, Any] = {key: _parse_scalar(raw_value)} if raw_value else {key: None}
+                if index < len(lines) and lines[index][0] > indent:
+                    child, index = _parse_yaml_block(lines, index, lines[index][0])
+                    if isinstance(child, dict):
+                        item.update(child)
+                result.append(item)
+            else:
+                result.append(_parse_scalar(item_raw))
+        return result, index
+
+    result: dict[str, Any] = {}
+    while index < len(lines):
+        line_indent, line_content = lines[index]
+        if line_indent != indent or line_content.startswith("- "):
+            break
+        key, raw_value = _split_key_value(line_content)
+        index += 1
+        if raw_value:
+            result[key] = _parse_scalar(raw_value)
+        elif index < len(lines) and lines[index][0] > indent:
+            child, index = _parse_yaml_block(lines, index, lines[index][0])
+            result[key] = child
+        else:
+            result[key] = None
+    return result, index
+
+
+def _split_key_value(content: str) -> tuple[str, str]:
+    key, _, raw_value = content.partition(":")
+    return key.strip(), raw_value.strip()
+
+
+def _parse_scalar(value: str) -> Any:
+    if value == "":
+        return None
+    lowered = value.lower()
+    if lowered == "true":
+        return True
+    if lowered == "false":
+        return False
+    if lowered in {"null", "none"}:
+        return None
+    try:
+        if "." in value:
+            return float(value)
+        return int(value)
+    except ValueError:
+        return value.strip("\"'")

ontology_platform/crawler_platform/app/core/__init__.py

@@ -0,0 +1,2 @@
+"""Core platform modules."""
+

ontology_platform/crawler_platform/app/core/crawler/__init__.py

@@ -0,0 +1,2 @@
+"""Crawler pipeline."""
+

ontology_platform/crawler_platform/app/core/crawler/content_zone.py

@@ -0,0 +1,60 @@
+from __future__ import annotations
+
+from dataclasses import asdict, dataclass
+
+
+CLAIM_ALLOWED_ZONE_TYPES = {
+    "main_content",
+    "document_body",
+    "article_body",
+    "research_body",
+    "product_title",
+    "product_summary",
+    "product_description",
+    "product_detail",
+    "brand_story_body",
+    "notice_body",
+    "event_body",
+}
+
+CLAIM_BLOCKED_ZONE_TYPES = {
+    "header",
+    "footer",
+    "nav",
+    "menu",
+    "category_filter",
+    "sort_control",
+    "shipping_policy",
+    "payment_policy",
+    "exchange_policy",
+    "recommendation",
+    "related_products",
+    "login_join",
+    "copyright",
+    "platform_credit",
+    "unknown",
+}
+
+
+@dataclass(slots=True)
+class ContentZone:
+    zone_type: str
+    text: str
+    selector: str | None = None
+    confidence: float = 0.0
+    claim_allowed: bool = False
+    reason: str | None = None
+
+    def to_dict(self, max_text_length: int | None = None) -> dict[str, object]:
+        data = asdict(self)
+        if max_text_length is not None and len(self.text) > max_text_length:
+            data["text"] = self.text[:max_text_length]
+        return data
+
+
+def is_claim_allowed_zone(zone_type: str | None) -> bool:
+    return bool(zone_type and zone_type in CLAIM_ALLOWED_ZONE_TYPES)
+
+
+def zone_dicts(zones: list[ContentZone], max_text_length: int | None = None) -> list[dict[str, object]]:
+    return [zone.to_dict(max_text_length=max_text_length) for zone in zones]

ontology_platform/crawler_platform/app/core/crawler/discovery.py

@@ -0,0 +1,126 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from urllib.parse import parse_qs, urlencode, unquote, urljoin, urlparse, urlunparse
+
+from bs4 import BeautifulSoup
+
+
+@dataclass(slots=True)
+class DiscoveredUrl:
+    url: str
+    label: str
+    kind: str = "link"
+
+
+def discover_links(html: str, base_url: str, limit: int = 30) -> list[DiscoveredUrl]:
+    soup = BeautifulSoup(html, "html.parser")
+    seen: set[str] = set()
+    results: list[DiscoveredUrl] = []
+    for anchor in soup.find_all("a", href=True):
+        raw_href = anchor.get("href", "")
+        if should_skip_raw_href(raw_href):
+            continue
+        url = normalize_cafe24_product_url(normalize_search_redirect(urljoin(base_url, raw_href)))
+        if not url or url in seen or not url.startswith(("http://", "https://")) or should_skip_url(url):
+            continue
+        seen.add(url)
+        label = anchor.get_text(" ", strip=True)[:160] or urlparse(url).netloc
+        results.append(DiscoveredUrl(url=url, label=label, kind=classify_url(url)))
+        if len(results) >= limit:
+            break
+    return results
+
+
+def should_skip_raw_href(raw_href: str) -> bool:
+    href = (raw_href or "").strip()
+    lowered = href.lower()
+    if not href or href in {"#", "/", "javascript:;", "javascript:void(0)"}:
+        return True
+    if "{" in href or "}" in href:
+        return True
+    if lowered.startswith(("mailto:", "tel:", "sms:", "javascript:")):
+        return True
+    if lowered.startswith(("facebook.com/", "instagram.com/", "kakao.com/", "pf.kakao.com/")):
+        return True
+    return False
+
+
+def should_skip_url(url: str) -> bool:
+    parsed = urlparse(url)
+    host = parsed.netloc.lower()
+    path = unquote(parsed.path.lower())
+    query = unquote(parsed.query.lower())
+    if any(token in host for token in ["facebook.com", "instagram.com", "kakao.com", "pf.kakao.com"]):
+        return True
+    if "{" in path or "}" in path or "%7b" in url.lower() or "%7d" in url.lower():
+        return True
+    skip_path_tokens = [
+        "/member/",
+        "/order/",
+        "/myshop/",
+        "/event/list",
+        "/exec/front/newcoupon/",
+        "/board/free/list",
+        "/board/faq/list",
+        "/board/free/modify",
+        "/board/free/reply",
+    ]
+    if any(token in path for token in skip_path_tokens):
+        return True
+    if path.endswith("/product/search.html"):
+        return True
+    if "facebook.com/" in path or "instagram.com/" in path:
+        return True
+    if "coupon_no=" in query:
+        return True
+    return False
+
+
+def normalize_search_redirect(url: str) -> str:
+    parsed = urlparse(url)
+    query = parse_qs(parsed.query)
+    for key in ("url", "u", "target"):
+        if key in query and query[key]:
+            candidate = query[key][0]
+            if candidate.startswith(("http://", "https://")):
+                return candidate
+    return url
+
+
+def normalize_cafe24_product_url(url: str) -> str:
+    parsed = urlparse(url)
+    path = unquote(parsed.path)
+    parts = [part for part in path.strip("/").split("/") if part]
+    query = parse_qs(parsed.query)
+
+    if path.endswith("/product/detail.html") and query.get("product_no"):
+        return urlunparse(
+            parsed._replace(
+                query=urlencode({"product_no": query["product_no"][0]}),
+                fragment="",
+            )
+        )
+
+    if parts and parts[0] == "product":
+        product_no = next((part for part in parts[1:] if part.isdigit()), None)
+        if product_no:
+            return urlunparse(
+                parsed._replace(
+                    path="/product/detail.html",
+                    query=urlencode({"product_no": product_no}),
+                    fragment="",
+                )
+            )
+    return url
+
+
+def classify_url(url: str) -> str:
+    host = urlparse(url).netloc.lower()
+    if "smartstore.naver.com" in host or "brand.naver.com" in host:
+        return "marketplace_product_or_store"
+    if "shopping.naver.com" in host:
+        return "shopping"
+    if any(token in host for token in ("fragrantica", "official", "perfume", "parfum")):
+        return "product_or_review"
+    return "link"

ontology_platform/crawler_platform/app/core/crawler/fetchers.py

@@ -0,0 +1,294 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from pathlib import Path
+import re
+import time
+from urllib.parse import urlparse
+from urllib.robotparser import RobotFileParser
+
+
+DEFAULT_USER_AGENT = "OntologyCrawlerBot/0.1 (+contact: admin@example.com)"
+
+
+@dataclass(slots=True)
+class FetchResult:
+    url: str
+    status_code: int | None
+    html: str
+    final_url: str | None = None
+    headers: dict[str, str] = field(default_factory=dict)
+    raw_html: str | None = None
+    rendered_html: str | None = None
+    title: str | None = None
+    crawl_status: str = "success"
+    warnings: list[str] = field(default_factory=list)
+
+    def __post_init__(self) -> None:
+        if self.raw_html is None:
+            self.raw_html = self.html
+        if self.rendered_html is None:
+            self.rendered_html = self.html
+        detected_status, detected_warnings = detect_crawl_status(
+            status_code=self.status_code,
+            html=self.rendered_html or self.raw_html or self.html,
+        )
+        if self.crawl_status == "success":
+            self.crawl_status = detected_status
+        self.warnings.extend(detected_warnings)
+
+    @property
+    def analysis_html(self) -> str:
+        return self.rendered_html or self.raw_html or self.html
+
+
+@dataclass(slots=True)
+class RobotsDecision:
+    allowed: bool
+    checked: bool
+    status: str
+    reason: str
+    robots_url: str | None = None
+    user_agent: str = DEFAULT_USER_AGENT
+
+
+class RateLimiter:
+    def __init__(self, per_minute: int = 30):
+        self.delay = 60 / max(per_minute, 1)
+        self._last_called = 0.0
+
+    def wait(self) -> None:
+        elapsed = time.monotonic() - self._last_called
+        if elapsed < self.delay:
+            time.sleep(self.delay - elapsed)
+        self._last_called = time.monotonic()
+
+
+class RobotsPolicy:
+    def __init__(self, user_agent: str = DEFAULT_USER_AGENT):
+        self.user_agent = user_agent
+        self._cache: dict[str, RobotFileParser] = {}
+
+    def allowed(self, url: str, respect_robots_txt: bool = True) -> bool:
+        return self.check(url, respect_robots_txt).allowed
+
+    def check(self, url: str, respect_robots_txt: bool = True) -> RobotsDecision:
+        if not respect_robots_txt:
+            return RobotsDecision(
+                allowed=True,
+                checked=False,
+                status="disabled",
+                reason="robots.txt check disabled by source/request config",
+                user_agent=self.user_agent,
+            )
+        parsed = urlparse(url)
+        if parsed.scheme in {"", "file"}:
+            return RobotsDecision(
+                allowed=True,
+                checked=False,
+                status="local",
+                reason="robots.txt is not applicable to local or scheme-less URLs",
+                user_agent=self.user_agent,
+            )
+        robots_url = f"{parsed.scheme}://{parsed.netloc}/robots.txt"
+        parser = self._cache.get(robots_url)
+        if parser is None:
+            parser = RobotFileParser()
+            parser.set_url(robots_url)
+            try:
+                parser.read()
+            except Exception as exc:
+                return RobotsDecision(
+                    allowed=True,
+                    checked=True,
+                    status="unavailable",
+                    reason=f"robots.txt unavailable ({exc.__class__.__name__}); allowing crawl",
+                    robots_url=robots_url,
+                    user_agent=self.user_agent,
+                )
+            self._cache[robots_url] = parser
+        allowed = parser.can_fetch(self.user_agent, url)
+        return RobotsDecision(
+            allowed=allowed,
+            checked=True,
+            status="allowed" if allowed else "blocked",
+            reason=(
+                "robots.txt allows crawling"
+                if allowed
+                else f"robots.txt blocks crawling for user-agent {self.user_agent}"
+            ),
+            robots_url=robots_url,
+            user_agent=self.user_agent,
+        )
+
+
+class BaseFetcher:
+    def fetch(self, url: str) -> FetchResult:
+        raise NotImplementedError
+
+
+class FallbackFetcher(BaseFetcher):
+    def __init__(self, primary: BaseFetcher, fallback: BaseFetcher, fallback_label: str = "fallback"):
+        self.primary = primary
+        self.fallback = fallback
+        self.fallback_label = fallback_label
+
+    def fetch(self, url: str) -> FetchResult:
+        try:
+            return self.primary.fetch(url)
+        except Exception as exc:
+            result = self.fallback.fetch(url)
+            result.warnings.append(
+                f"primary fetcher failed ({exc.__class__.__name__}: {exc}); used {self.fallback_label}"
+            )
+            return result
+
+
+class RequestsFetcher(BaseFetcher):
+    def __init__(
+        self,
+        user_agent: str = DEFAULT_USER_AGENT,
+        timeout_seconds: int = 15,
+        retries: int = 2,
+        rate_limit_per_minute: int = 30,
+    ):
+        self.user_agent = user_agent
+        self.timeout_seconds = timeout_seconds
+        self.retries = retries
+        self.rate_limiter = RateLimiter(rate_limit_per_minute)
+
+    def fetch(self, url: str) -> FetchResult:
+        local_path = _local_path_from_url(url)
+        if local_path:
+            html = local_path.read_text(encoding="utf-8")
+            return FetchResult(url=url, status_code=200, html=html, final_url=url, raw_html=html, rendered_html=html)
+
+        import requests
+
+        last_error: Exception | None = None
+        for _ in range(self.retries + 1):
+            self.rate_limiter.wait()
+            try:
+                response = requests.get(
+                    url,
+                    headers={"User-Agent": self.user_agent},
+                    timeout=self.timeout_seconds,
+                )
+                return FetchResult(
+                    url=url,
+                    status_code=response.status_code,
+                    html=response.text,
+                    final_url=response.url,
+                    headers=dict(response.headers),
+                )
+            except requests.RequestException as exc:
+                last_error = exc
+        raise RuntimeError(f"Failed to fetch {url}: {last_error}")
+
+
+class PlaywrightFetcher(BaseFetcher):
+    def __init__(self, user_agent: str = DEFAULT_USER_AGENT, timeout_ms: int = 20000):
+        self.user_agent = user_agent
+        self.timeout_ms = timeout_ms
+
+    def fetch(self, url: str) -> FetchResult:
+        local_path = _local_path_from_url(url)
+        if local_path:
+            html = local_path.read_text(encoding="utf-8")
+            return FetchResult(url=url, status_code=200, html=html, final_url=url, raw_html=html, rendered_html=html)
+
+        from playwright.sync_api import sync_playwright
+
+        warnings: list[str] = []
+        with sync_playwright() as playwright:
+            browser = playwright.chromium.launch(headless=True)
+            page = browser.new_page(user_agent=self.user_agent)
+            try:
+                response = page.goto(url, wait_until="domcontentloaded", timeout=self.timeout_ms)
+                try:
+                    page.wait_for_load_state("networkidle", timeout=min(self.timeout_ms, 8000))
+                except Exception as exc:
+                    warnings.append(f"networkidle wait timed out: {exc}")
+                for _ in range(3):
+                    page.mouse.wheel(0, 1200)
+                    page.wait_for_timeout(250)
+                title = page.title()
+                html = page.content()
+                final_url = page.url
+                status = response.status if response else None
+            finally:
+                browser.close()
+        return FetchResult(
+            url=url,
+            status_code=status,
+            html=html,
+            final_url=final_url,
+            rendered_html=html,
+            title=title,
+            warnings=warnings,
+        )
+
+
+def make_fetcher(kind: str, rate_limit_per_minute: int = 30) -> BaseFetcher:
+    if kind in {"playwright", "browser"}:
+        return FallbackFetcher(
+            PlaywrightFetcher(),
+            RequestsFetcher(rate_limit_per_minute=rate_limit_per_minute),
+            fallback_label="requests",
+        )
+    return RequestsFetcher(rate_limit_per_minute=rate_limit_per_minute)
+
+
+def _local_path_from_url(url: str) -> Path | None:
+    if url.startswith("file://"):
+        path = Path(url.removeprefix("file://"))
+        return path if path.exists() and path.is_file() else None
+    path = Path(url)
+    if path.exists() and path.is_file():
+        return path
+    return None
+
+
+def detect_crawl_status(status_code: int | None, html: str | None) -> tuple[str, list[str]]:
+    warnings: list[str] = []
+    text = visible_text_for_status_detection(html or "")
+    lowered = " ".join(text.lower().split())
+    if status_code is None:
+        warnings.append("missing status code")
+        return "crawl_failed", warnings
+    if status_code in {401, 403, 407, 429}:
+        warnings.append(f"blocked status code: {status_code}")
+        return "blocked", warnings
+    if status_code >= 400:
+        warnings.append(f"unavailable status code: {status_code}")
+        return "unavailable", warnings
+    if not lowered:
+        warnings.append("empty html")
+        return "empty", warnings
+    blocked_patterns = ["access denied", "request blocked", "bot detection", "unusual traffic"]
+    unavailable_patterns = ["page unavailable", "page not found", "temporarily unavailable", "service unavailable"]
+    if any(pattern in lowered for pattern in blocked_patterns):
+        warnings.append("blocked page pattern detected")
+        return "blocked", warnings
+    if ("captcha" in lowered or "recaptcha" in lowered) and any(
+        token in lowered for token in ["verify", "verification", "robot", "blocked", "challenge"]
+    ):
+        warnings.append("captcha challenge detected")
+        return "blocked", warnings
+    if any(pattern in lowered for pattern in unavailable_patterns):
+        warnings.append("unavailable page pattern detected")
+        return "unavailable", warnings
+    return "success", warnings
+
+
+def visible_text_for_status_detection(html: str) -> str:
+    try:
+        from bs4 import BeautifulSoup
+    except ImportError:
+        return re.sub(r"<[^>]+>", " ", html)
+
+    soup = BeautifulSoup(html, "html.parser")
+    for selector in ["script", "style", "noscript", "svg", "iframe"]:
+        for tag in soup.select(selector):
+            tag.decompose()
+    return soup.get_text(" ", strip=True)

ontology_platform/crawler_platform/app/core/crawler/html_cleaner.py

@@ -0,0 +1,288 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+import re
+from typing import Any
+
+
+BOILERPLATE_SELECTORS = [
+    "script",
+    "style",
+    "noscript",
+    "svg",
+    "iframe",
+    "header",
+    "footer",
+    "nav",
+    "aside",
+    "form",
+    "[role='navigation']",
+    "[role='banner']",
+    "[role='contentinfo']",
+    ".header",
+    ".footer",
+    ".nav",
+    ".navigation",
+    ".gnb",
+    ".lnb",
+    ".menu",
+    ".breadcrumb",
+    ".pagination",
+    ".paging",
+    ".toolbar",
+    ".sort",
+    ".search",
+    ".login",
+    ".cart",
+    ".basket",
+    ".coupon",
+    ".event",
+    ".banner",
+    ".promotion",
+    ".recommend",
+    ".related",
+    ".recent",
+    ".review-list",
+    ".board",
+    ".notice",
+    ".cs",
+    ".customer",
+    ".shipping",
+    ".delivery",
+    "#header",
+    "#footer",
+    "#nav",
+    "#gnb",
+    "#lnb",
+    "#sidebar",
+    "#aside",
+    "#event",
+    "#banner",
+    "#board",
+    "#notice",
+]
+
+MAIN_CONTENT_SELECTORS = [
+    "main",
+    "article",
+    "[role='main']",
+    "#contents",
+    "#content",
+    "#container",
+    "#main",
+    ".contents",
+    ".content",
+    ".container",
+    ".product-detail",
+    ".prd-detail",
+    ".detailArea",
+    ".xans-product-detail",
+    ".xans-product-additional",
+    ".ec-base-product",
+    ".description",
+    ".summary",
+]
+
+NOISY_LINE_TERMS = {
+    "cafe24",
+    "powered by cafe24",
+    "hosting by cafe24",
+    "home",
+    "login",
+    "logout",
+    "cart",
+    "basket",
+    "checkout",
+    "my page",
+    "search",
+    "sort",
+    "low price",
+    "high price",
+    "new item",
+    "best item",
+    "product count",
+    "privacy policy",
+    "terms",
+    "company",
+    "customer center",
+    "notice",
+    "q&a",
+    "faq",
+    "review",
+    "event",
+    "copyright",
+}
+
+NOISY_LINE_PATTERNS = [
+    re.compile(pattern, re.IGNORECASE)
+    for pattern in [
+        r"^\d+\s*/\s*\d+$",
+        r"^page\s+\d+",
+        r"^(prev|previous|next|first|last)$",
+        r"^(add to cart|buy now|wish list)$",
+        r"^(usd|krw|eur|jpy)$",
+        r"shipping|delivery|return|exchange|refund",
+        r"country|language|currency",
+        r"facebook|instagram|youtube|kakao|naver",
+        r"cafe24|copyright|all rights reserved",
+    ]
+]
+
+
+@dataclass(slots=True)
+class CleanedHtml:
+    title: str | None
+    text: str
+    metadata: dict[str, Any] = field(default_factory=dict)
+    raw_text: str = ""
+    main_content: str = ""
+    clean_markdown: str = ""
+    source_zones: list[dict[str, object]] = field(default_factory=list)
+    noise_zones: list[dict[str, object]] = field(default_factory=list)
+    page_type: str = "UnknownPage"
+    extraction_status: str = "failed"
+    extraction_warnings: list[str] = field(default_factory=list)
+
+
+def clean_html(html: str) -> tuple[str | None, str]:
+    cleaned = clean_html_with_metadata(html)
+    return cleaned.title, cleaned.text
+
+
+def clean_html_with_metadata(html: str, url: str = "", page_type: str | None = None) -> CleanedHtml:
+    from crawler_platform.app.core.crawler.page_cleaner import PageCleaner
+
+    cleaned = PageCleaner().clean(html, url=url, page_type=page_type)
+    source_zones = [zone.to_dict(max_text_length=1000) for zone in cleaned.source_zones]
+    noise_zones = [zone.to_dict(max_text_length=500) for zone in cleaned.noise_zones]
+    return CleanedHtml(
+        title=cleaned.title,
+        text=cleaned.clean_text,
+        metadata=cleaned.metadata,
+        raw_text=cleaned.raw_text,
+        main_content=cleaned.main_content,
+        clean_markdown=cleaned.clean_markdown,
+        source_zones=source_zones,
+        noise_zones=noise_zones,
+        page_type=cleaned.page_type,
+        extraction_status=cleaned.extraction_status,
+        extraction_warnings=cleaned.extraction_warnings,
+    )
+
+
+def remove_boilerplate_nodes(soup) -> None:
+    for selector in BOILERPLATE_SELECTORS:
+        for tag in soup.select(selector):
+            tag.decompose()
+    for tag in list(soup.find_all(True)):
+        if getattr(tag, "attrs", None) is None:
+            continue
+        token_text = " ".join([node_attr(tag, "id"), node_classes(tag), node_attr(tag, "aria-label")]).lower()
+        if any(token in token_text for token in ["footer", "header", "nav", "menu", "shipping", "delivery", "cafe24"]):
+            tag.decompose()
+
+
+def content_candidates(soup) -> list:
+    candidates = []
+    for selector in MAIN_CONTENT_SELECTORS:
+        candidates.extend(soup.select(selector))
+    if soup.body:
+        candidates.append(soup.body)
+    candidates.append(soup)
+    return [candidate for candidate in candidates if candidate is not None]
+
+
+def choose_main_node(candidates: list, soup) -> tuple[Any | None, str]:
+    scored = [(content_score(candidate), candidate) for candidate in candidates]
+    scored = [(score, candidate) for score, candidate in scored if score > 0]
+    if scored:
+        scored.sort(key=lambda item: item[0], reverse=True)
+        return scored[0][1], "selector_or_density"
+    return soup.body or soup, "body_fallback"
+
+
+def content_score(node) -> float:
+    text = normalize_content_lines(node.get_text("\n", strip=True) if node else "")
+    if len(text) < 40:
+        return 0
+    lower = text.lower()
+    product_signals = sum(
+        1
+        for token in [
+            "brand",
+            "price",
+            "notes",
+            "top notes",
+            "middle notes",
+            "base notes",
+            "description",
+            "ingredient",
+            "option",
+            "product",
+        ]
+        if token in lower
+    )
+    link_count = len(node.find_all("a")) if hasattr(node, "find_all") else 0
+    text_len = max(len(text), 1)
+    link_penalty = min(link_count * 30 / text_len, 0.7)
+    return len(text) * (1 + product_signals * 0.25) * (1 - link_penalty)
+
+
+def normalize_content_lines(text: str) -> str:
+    normalized = normalize_whitespace(text)
+    lines: list[str] = []
+    seen: set[str] = set()
+    for raw in normalized.splitlines():
+        line = cleanup_line(raw)
+        if not line or is_noisy_line(line):
+            continue
+        key = line.lower()
+        if key in seen:
+            continue
+        seen.add(key)
+        lines.append(line)
+    return "\n".join(lines)
+
+
+def cleanup_line(value: str) -> str:
+    return re.sub(r"\s+", " ", value.replace("\xa0", " ")).strip(" -:|")
+
+
+def is_noisy_line(line: str) -> bool:
+    lowered = line.lower().strip()
+    if not lowered or lowered in NOISY_LINE_TERMS:
+        return True
+    if len(lowered) <= 1:
+        return True
+    if len(lowered) <= 3 and not any(ch.isdigit() for ch in lowered):
+        return True
+    if any(pattern.search(lowered) for pattern in NOISY_LINE_PATTERNS):
+        return True
+    if lowered.count("|") >= 3:
+        return True
+    return False
+
+
+def normalize_whitespace(text: str) -> str:
+    lines = [" ".join(line.split()) for line in text.splitlines()]
+    return "\n".join(line for line in lines if line)
+
+
+def node_attr(node, name: str) -> str:
+    if node is None or not hasattr(node, "get"):
+        return ""
+    if getattr(node, "attrs", None) is None:
+        return ""
+    try:
+        value = node.get(name)
+    except AttributeError:
+        return ""
+    if value is None:
+        return ""
+    if isinstance(value, list):
+        return " ".join(str(item) for item in value if item)
+    return str(value)
+
+
+def node_classes(node) -> str:
+    return node_attr(node, "class")

ontology_platform/crawler_platform/app/core/crawler/page_classifier.py

@@ -0,0 +1,157 @@
+from __future__ import annotations
+
+from urllib.parse import urlparse
+
+from crawler_platform.app.core.crawler.content_zone import is_claim_allowed_zone
+
+
+PRODUCT_DETAIL_PREDICATES = {
+    "hasBrand",
+    "hasTopNote",
+    "hasMiddleNote",
+    "hasBaseNote",
+    "hasAccord",
+    "evokesMood",
+    "suitableForSeason",
+    "suitableForOccasion",
+    "soldBy",
+    "hasPrice",
+    "hasReviewKeyword",
+}
+
+PRODUCT_DETAIL_PAGE_TYPES = {"ProductPage"}
+CONTENT_PAGE_TYPES = {"BrandStoryPage", "NoticePage", "EventPage", "PromotionPage"}
+NON_MERGE_PAGE_TYPES = {"UnknownPage", "SearchPage", "CategoryPage", "BoardPage"}
+
+
+def classify_page(
+    url: str,
+    title: str | None = None,
+    text: str = "",
+    html: str | None = None,
+    source_zones: list[dict[str, object]] | None = None,
+) -> str:
+    path = urlparse(url).path.lower()
+    query = urlparse(url).query.lower()
+    combined = f"{url}\n{title or ''}\n{text[:5000]}".lower()
+    html_lower = (html or "")[:8000].lower()
+
+    if "/product/list" in path or path.endswith("/product/list.html"):
+        return "CategoryPage"
+    if "/product/search" in path or path.endswith("/product/search.html"):
+        return "SearchPage"
+    if any(token in path for token in ["/search", "/find"]) or "keyword=" in query or "search" in query:
+        return "SearchPage"
+    if any(token in path for token in ["/board/", "board/free", "board/product", "/article/"]):
+        if any(token in combined for token in ["notice", "공지"]):
+            return "NoticePage"
+        return "BoardPage"
+    if any(token in path for token in ["shopinfo", "company", "about", "brand-story", "brand_story"]):
+        return "BrandStoryPage"
+    if any(token in path for token in ["product/detail", "/product/", "/products/", "/goods/", "/item/"]):
+        if _looks_like_category_path(path, combined) and not _looks_like_product_detail_path(path):
+            return "CategoryPage"
+        return "ProductPage"
+    if any(token in path for token in ["category", "/collections", "/collection", "/shop/", "/list"]):
+        return "CategoryPage"
+    if any(token in path for token in ["event", "promotion", "promo", "sale"]):
+        return "PromotionPage"
+
+    if source_zones and any(_zone_type(zone) in {"product_detail", "product_description"} for zone in source_zones):
+        return "ProductPage"
+
+    product_tokens = [
+        "add to cart",
+        "buy now",
+        "price",
+        "top notes",
+        "middle notes",
+        "base notes",
+        "fragrance notes",
+        "장바구니",
+        "구매",
+        "가격",
+        "탑노트",
+        "베이스노트",
+    ]
+    listing_tokens = ["sort", "low price", "high price", "product count", "상품수", "낮은가격", "높은가격"]
+    brand_tokens = ["about us", "brand story", "our story", "philosophy", "official", "브랜드", "소개"]
+    promotion_tokens = ["event", "sale", "coupon", "promotion", "black friday", "회원가입", "쿠폰", "증정", "무료배송"]
+    notice_tokens = ["notice", "공지", "announcement"]
+    board_tokens = ["q&a", "faq", "review", "게시판", "문의"]
+
+    if sum(1 for token in promotion_tokens if token in combined) >= 2:
+        return "PromotionPage"
+    if any(token in combined for token in notice_tokens):
+        return "NoticePage"
+    if any(token in combined for token in board_tokens):
+        return "BoardPage"
+    if any(token in combined for token in brand_tokens):
+        return "BrandStoryPage"
+    if sum(1 for token in listing_tokens if token in combined) >= 2 and not _has_product_detail_signal(combined):
+        return "CategoryPage"
+    if _has_product_detail_signal(combined) or _has_product_detail_signal(html_lower):
+        return "ProductPage"
+    return "UnknownPage"
+
+
+def relation_allowed_for_page_type(page_type: str | None, predicate: str) -> bool:
+    if page_type in PRODUCT_DETAIL_PAGE_TYPES:
+        return True
+    if predicate in PRODUCT_DETAIL_PREDICATES:
+        return False
+    if page_type in NON_MERGE_PAGE_TYPES:
+        return False
+    return page_type in CONTENT_PAGE_TYPES
+
+
+def claim_allowed_for_context(page_type: str | None, predicate: str, zone_type: str | None) -> bool:
+    return relation_allowed_for_page_type(page_type, predicate) and is_claim_allowed_zone(zone_type)
+
+
+def normalize_page_type(value: str | None) -> str:
+    aliases = {
+        "product": "ProductPage",
+        "brand": "BrandStoryPage",
+        "review": "ReviewPage",
+        "listing": "CategoryPage",
+        "category": "CategoryPage",
+        "community": "BoardPage",
+        "board": "BoardPage",
+        "communitypage": "BoardPage",
+        "listingpage": "CategoryPage",
+        "promotionpage": "PromotionPage",
+    }
+    clean = str(value or "").strip()
+    return aliases.get(clean.lower(), aliases.get(clean, clean or "UnknownPage"))
+
+
+def should_analyze_page(page_type: str, analyze_page_types: set[str]) -> bool:
+    normalized_page_type = normalize_page_type(page_type)
+    normalized = {normalize_page_type(item) for item in analyze_page_types}
+    return normalized_page_type in normalized
+
+
+def _zone_type(zone: dict[str, object]) -> str:
+    return str(zone.get("zone_type") or "")
+
+
+def _has_product_detail_signal(text: str) -> bool:
+    note_count = sum(1 for token in ["top notes", "middle notes", "base notes", "탑노트", "미들노트", "베이스노트"] if token in text)
+    commerce_count = sum(1 for token in ["price", "add to cart", "buy now", "장바구니", "구매", "가격"] if token in text)
+    return note_count >= 1 or commerce_count >= 2
+
+
+def _looks_like_product_detail_path(path: str) -> bool:
+    parts = [part for part in path.strip("/").split("/") if part]
+    if "product" not in parts:
+        return False
+    product_index = parts.index("product")
+    return any(part.isdigit() for part in parts[product_index + 1 :])
+
+
+def _looks_like_category_path(path: str, text: str) -> bool:
+    category_tokens = ["category", "cate_no", "display_group", "sort_method"]
+    if any(token in path or token in text for token in category_tokens):
+        return True
+    return sum(1 for token in ["low price", "high price", "product count", "상품수", "낮은가격"] if token in text) >= 2

ontology_platform/crawler_platform/app/core/crawler/page_cleaner.py

@@ -0,0 +1,606 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+import re
+from typing import Any
+
+from crawler_platform.app.core.crawler.content_zone import ContentZone, zone_dicts
+from crawler_platform.app.core.crawler.page_classifier import classify_page
+
+
+SCRIPT_STYLE_SELECTORS = ["script", "style", "noscript", "svg", "iframe"]
+
+NOISE_SELECTORS: dict[str, list[str]] = {
+    "header": ["header", "[role='banner']", ".header", "#header", ".topbar"],
+    "footer": ["footer", "[role='contentinfo']", ".footer", "#footer"],
+    "nav": ["nav", "[role='navigation']", ".nav", ".navigation", ".gnb", ".lnb", "#gnb", "#lnb"],
+    "menu": [".menu", ".breadcrumb", ".pagination", ".paging", ".toolbar", ".tabs"],
+    "category_filter": [".filter", ".category", ".categories", ".xans-product-menupackage"],
+    "sort_control": [".sort", ".order", ".ec-base-paginate"],
+    "shipping_policy": [".shipping", ".delivery", ".policy", ".guide", ".return", ".exchange", ".refund"],
+    "recommendation": [".recommend", ".related", ".recent", ".best", ".new", ".also", ".relation"],
+    "login_join": [".login", ".join", ".account", ".member", ".cart", ".basket"],
+    "platform_credit": [".cafe24", ".hosting", ".powered"],
+}
+
+ZONE_SELECTORS: dict[str, list[str]] = {
+    "product_title": [
+        "h1",
+        ".product-title",
+        ".product_name",
+        ".product-name",
+        ".name",
+        ".headingArea h2",
+        ".xans-product-detail .headingArea",
+    ],
+    "product_summary": [
+        ".summary",
+        ".prd-summary",
+        ".prdSummary",
+        ".simple_desc",
+        ".xans-product-detaildesign",
+    ],
+    "product_description": [
+        ".description",
+        ".desc",
+        ".product-description",
+        ".prdDesc",
+        ".detail-description",
+    ],
+    "product_detail": [
+        "main",
+        "article",
+        "[role='main']",
+        ".product-detail",
+        ".prd-detail",
+    ],
+    "brand_story_body": [
+        "main",
+        "article",
+        ".brand-story",
+        ".brand_story",
+        ".about",
+        ".company",
+        ".story",
+        "#about",
+        "#company",
+    ],
+    "notice_body": ["main", "article", ".notice", ".boardView", ".view", ".post", ".article"],
+    "event_body": ["main", "article", ".event", ".promotion", ".promo", ".event-view", ".post", ".article"],
+}
+
+ZONE_PRIORITY_BY_PAGE_TYPE = {
+    "ProductPage": ["product_title", "product_summary", "product_description", "product_detail"],
+    "BrandStoryPage": ["brand_story_body"],
+    "NoticePage": ["notice_body"],
+    "BoardPage": ["notice_body"],
+    "EventPage": ["event_body"],
+    "PromotionPage": ["event_body"],
+    "CategoryPage": ["product_title", "product_summary"],
+}
+
+STRUCTURAL_NOISE_TOKENS = {
+    "footer",
+    "header",
+    "nav",
+    "menu",
+    "breadcrumb",
+    "pagination",
+    "shipping",
+    "delivery",
+    "exchange",
+    "refund",
+    "policy",
+    "recommend",
+    "related",
+    "recent",
+    "login",
+    "join",
+    "cart",
+    "basket",
+    "cafe24",
+    "copyright",
+}
+
+NOISY_LINE_TERMS = {
+    "cafe24",
+    "powered by cafe24",
+    "hosting by cafe24",
+    "home",
+    "login",
+    "logout",
+    "cart",
+    "basket",
+    "checkout",
+    "my page",
+    "search",
+    "sort",
+    "low price",
+    "high price",
+    "new item",
+    "best item",
+    "product count",
+    "privacy policy",
+    "terms",
+    "company",
+    "customer center",
+    "notice",
+    "q&a",
+    "faq",
+    "review",
+    "event",
+    "copyright",
+    "상품수",
+    "낮은가격",
+    "높은가격",
+}
+
+NOISY_LINE_PATTERNS = [
+    re.compile(pattern, re.IGNORECASE)
+    for pattern in [
+        r"^\d+\s*/\s*\d+$",
+        r"^page\s+\d+",
+        r"^(prev|previous|next|first|last)$",
+        r"^(add to cart|buy now|wish list)$",
+        r"^(usd|krw|eur|jpy)$",
+        r"shipping|delivery|return|exchange|refund",
+        r"country|language|currency",
+        r"facebook|instagram|youtube|kakao|naver",
+        r"cafe24|copyright|all rights reserved",
+    ]
+]
+
+POLICY_TERMS = {
+    "shipping",
+    "delivery",
+    "return",
+    "exchange",
+    "refund",
+    "privacy",
+    "terms",
+    "country",
+    "language",
+    "배송",
+    "교환",
+    "반품",
+    "환불",
+}
+
+
+@dataclass(slots=True)
+class CleanedPage:
+    title: str | None
+    raw_text: str
+    main_content: str
+    clean_text: str
+    clean_markdown: str
+    page_type: str
+    source_zones: list[ContentZone] = field(default_factory=list)
+    noise_zones: list[ContentZone] = field(default_factory=list)
+    extraction_status: str = "failed"
+    extraction_warnings: list[str] = field(default_factory=list)
+    metadata: dict[str, Any] = field(default_factory=dict)
+
+
+class PageCleaner:
+    def clean(self, html: str, url: str = "", page_type: str | None = None) -> CleanedPage:
+        try:
+            from bs4 import BeautifulSoup
+        except ImportError:
+            text = normalize_content_lines(re.sub(r"<[^>]+>", " ", html))
+            resolved_page_type = page_type or classify_page(url, None, text)
+            return CleanedPage(
+                title=None,
+                raw_text=text,
+                main_content=text,
+                clean_text=text,
+                clean_markdown=text,
+                page_type=resolved_page_type,
+                extraction_status="partial" if text else "failed",
+                extraction_warnings=["beautifulsoup unavailable; used regex fallback"],
+                metadata={"cleaner": "regex_fallback", "page_type": resolved_page_type},
+            )
+
+        soup = BeautifulSoup(html or "", "html.parser")
+        title = soup.title.get_text(" ", strip=True) if soup.title else None
+        raw_text = normalize_whitespace(soup.get_text("\n", strip=True))
+        resolved_page_type = page_type or classify_page(url, title, raw_text, html)
+
+        work = BeautifulSoup(html or "", "html.parser")
+        for selector in SCRIPT_STYLE_SELECTORS:
+            for tag in work.select(selector):
+                tag.decompose()
+
+        noise_zones = self._remove_noise_zones(work, resolved_page_type)
+        source_zones = self._source_zones(work, resolved_page_type)
+        if not source_zones:
+            fallback_zone = self._fallback_main_zone(work)
+            if fallback_zone:
+                source_zones = [fallback_zone]
+        if not source_zones and raw_text:
+            recovered_text = normalize_content_lines(raw_text)
+            if _is_contentful(recovered_text):
+                source_zones = [
+                    ContentZone(
+                        zone_type="unknown",
+                        selector="raw_text",
+                        text=recovered_text,
+                        confidence=0.2,
+                        claim_allowed=False,
+                        reason="raw text fallback after noise removal stripped content",
+                    )
+                ]
+
+        main_content = normalize_content_lines("\n".join(zone.text for zone in source_zones))
+        clean_text = normalize_content_lines(main_content)
+        clean_markdown = build_clean_markdown(source_zones)
+        warnings: list[str] = []
+        if not source_zones:
+            warnings.append("no content zones detected")
+        if len(clean_text) < 80:
+            warnings.append("clean text is short")
+        extraction_status = self._status_for(clean_text, source_zones)
+        metadata = {
+            "cleaner": "page_cleaner_v1",
+            "page_type": resolved_page_type,
+            "raw_text_length": len(raw_text),
+            "main_content_length": len(main_content),
+            "clean_text_length": len(clean_text),
+            "clean_markdown_length": len(clean_markdown),
+            "source_zone_count": len(source_zones),
+            "noise_zone_count": len(noise_zones),
+            "removed_noise_zones_count": len(noise_zones),
+            "source_zones": zone_dicts(source_zones, max_text_length=500),
+            "noise_zones": zone_dicts(noise_zones, max_text_length=240),
+            "extraction_status": extraction_status,
+            "extraction_warnings": warnings,
+        }
+        return CleanedPage(
+            title=title,
+            raw_text=raw_text,
+            main_content=main_content,
+            clean_text=clean_text,
+            clean_markdown=clean_markdown,
+            page_type=resolved_page_type,
+            source_zones=source_zones,
+            noise_zones=noise_zones,
+            extraction_status=extraction_status,
+            extraction_warnings=warnings,
+            metadata=metadata,
+        )
+
+    def _remove_noise_zones(self, soup, page_type: str) -> list[ContentZone]:
+        zones: list[ContentZone] = []
+        for zone_type, selectors in NOISE_SELECTORS.items():
+            if _preserve_noise_type_for_page(zone_type, page_type):
+                continue
+            for selector in selectors:
+                for tag in list(soup.select(selector)):
+                    text = normalize_whitespace(tag.get_text("\n", strip=True))
+                    if text:
+                        zones.append(ContentZone(zone_type=zone_type, selector=selector, text=text, confidence=0.9))
+                    tag.decompose()
+
+        total_text_len = len(normalize_whitespace(soup.get_text("\n", strip=True)))
+        for tag in list(soup.find_all(True)):
+            if not _tag_alive(tag):
+                continue
+            if tag.name in {"html", "body", "head", "[document]"}:
+                continue
+            token_text = " ".join([node_attr(tag, "id"), node_classes(tag), node_attr(tag, "aria-label")]).lower()
+            text = normalize_whitespace(tag.get_text("\n", strip=True))
+            if not text:
+                continue
+            structural_match = any(token in token_text for token in STRUCTURAL_NOISE_TOKENS)
+            policy_match = _looks_like_policy_block(text)
+            sort_match = _looks_like_sort_or_filter(text)
+            if (policy_match or sort_match) and not structural_match and _contains_content_selector(tag):
+                continue
+            if not structural_match and total_text_len > 0 and len(text) >= max(800, total_text_len * 0.5):
+                continue
+            if structural_match or policy_match or sort_match:
+                zone_type = _zone_type_for_noise(token_text, text)
+                if _preserve_noise_type_for_page(zone_type, page_type):
+                    continue
+                zones.append(
+                    ContentZone(
+                        zone_type=zone_type,
+                        selector=css_hint(tag),
+                        text=text,
+                        confidence=0.72,
+                        reason="structural or policy-like block",
+                    )
+                )
+                tag.decompose()
+        return _dedupe_zones(zones)
+
+    def _source_zones(self, soup, page_type: str) -> list[ContentZone]:
+        zones: list[ContentZone] = []
+        zone_types = ZONE_PRIORITY_BY_PAGE_TYPE.get(page_type) or ["product_detail", "brand_story_body", "notice_body"]
+        for zone_type in zone_types:
+            for selector in ZONE_SELECTORS.get(zone_type, []):
+                for tag in soup.select(selector):
+                    text = normalize_content_lines(tag.get_text("\n", strip=True))
+                    if not _is_contentful(text):
+                        continue
+                    zones.append(
+                        ContentZone(
+                            zone_type=zone_type,
+                            selector=selector,
+                            text=text,
+                            confidence=_zone_confidence(zone_type, text),
+                            claim_allowed=zone_type in {
+                                "product_title",
+                                "product_summary",
+                                "product_description",
+                                "product_detail",
+                                "brand_story_body",
+                                "notice_body",
+                                "event_body",
+                            },
+                        )
+                    )
+        return _dedupe_zones(zones)
+
+    def _fallback_main_zone(self, soup) -> ContentZone | None:
+        candidates = []
+        for selector in ["main", "article", "[role='main']", "body"]:
+            candidates.extend(soup.select(selector))
+        candidates.append(soup)
+        scored = []
+        for candidate in candidates:
+            text = normalize_content_lines(candidate.get_text("\n", strip=True))
+            if _is_contentful(text):
+                scored.append((content_score(candidate, text), candidate, text))
+        if not scored:
+            return None
+        scored.sort(key=lambda item: item[0], reverse=True)
+        _score, node, text = scored[0]
+        return ContentZone(
+            zone_type="unknown",
+            selector=css_hint(node),
+            text=text,
+            confidence=0.35,
+            claim_allowed=False,
+            reason="fallback body/main candidate",
+        )
+
+    def _status_for(self, clean_text: str, source_zones: list[ContentZone]) -> str:
+        if not clean_text or len(clean_text) < 20:
+            return "failed"
+        if not source_zones or all(zone.zone_type == "unknown" for zone in source_zones):
+            return "partial"
+        if len(clean_text) < 80:
+            return "partial"
+        return "success"
+
+
+def build_clean_markdown(source_zones: list[ContentZone]) -> str:
+    blocks = []
+    for zone in source_zones:
+        text = normalize_content_lines(zone.text)
+        if not text:
+            continue
+        heading = zone.zone_type.replace("_", " ").title()
+        blocks.append(f"## {heading}\n{text}")
+    return "\n\n".join(blocks)
+
+
+def normalize_content_lines(text: str) -> str:
+    normalized = normalize_whitespace(text)
+    raw_lines = normalized.splitlines()
+    lines: list[str] = []
+    seen: set[str] = set()
+    for idx, raw in enumerate(raw_lines):
+        line = cleanup_line(raw)
+        if not line or is_noisy_line(line) or is_contextual_noise_line(raw_lines, idx, line):
+            continue
+        key = line.lower()
+        if key in seen:
+            continue
+        seen.add(key)
+        lines.append(line)
+    return "\n".join(lines)
+
+
+def normalize_whitespace(text: str) -> str:
+    lines = [" ".join(line.split()) for line in text.splitlines()]
+    return "\n".join(line for line in lines if line)
+
+
+def cleanup_line(value: str) -> str:
+    return re.sub(r"\s+", " ", value.replace("\xa0", " ")).strip(" -:|")
+
+
+def is_noisy_line(line: str) -> bool:
+    lowered = line.lower().strip()
+    if not lowered or lowered in NOISY_LINE_TERMS:
+        return True
+    if lowered in {
+        "기본 정보",
+        "소비자가",
+        "상품정보",
+        "상품 간략설명",
+        "목록 내 상품 간단 설명",
+        "상세페이지 참고",
+        "상품 옵션",
+        "옵션 선택",
+        "사이즈 가이드",
+        "배송 예정일",
+        "상품 목록",
+        "구매하기",
+        "sold out",
+        "유의사항",
+    }:
+        return True
+    if "{#" in lowered or "{$" in lowered or "display_" in lowered or "*display" in lowered:
+        return True
+    if len(lowered) <= 1:
+        return True
+    if len(lowered) <= 3 and not any(ch.isdigit() for ch in lowered):
+        return True
+    if any(pattern.search(lowered) for pattern in NOISY_LINE_PATTERNS):
+        return True
+    if lowered.count("|") >= 3:
+        return True
+    return False
+
+
+def is_contextual_noise_line(raw_lines: list[str], idx: int, line: str) -> bool:
+    lowered_line = line.lower()
+    window = "\n".join(raw_lines[max(0, idx - 2) : min(len(raw_lines), idx + 3)]).lower()
+    if re.fullmatch(r"\d{1,3}(?:,\d{3})*\s*원", line) and any(
+        token in window for token in ["배송", "무료", "이상 구매", "shipping", "delivery", "free"]
+    ):
+        return True
+    if any(token in lowered_line for token in ["배송비", "무료배송", "이상 구매 시 무료", "회원 가입", "카카오톡 채널", "적립금"]):
+        return True
+    if lowered_line.startswith(("상품에 대해 궁금", "상품의 사용후기", "로그인 후 적립")):
+        return True
+    return False
+
+
+def content_score(node, text: str) -> float:
+    link_count = len(node.find_all("a")) if hasattr(node, "find_all") else 0
+    text_len = max(len(text), 1)
+    link_penalty = min(link_count * 30 / text_len, 0.7)
+    semantic_bonus = sum(
+        1
+        for token in ["brand", "price", "notes", "description", "ingredient", "product", "story", "notice"]
+        if token in text.lower()
+    )
+    return text_len * (1 + semantic_bonus * 0.2) * (1 - link_penalty)
+
+
+def node_attr(node, name: str) -> str:
+    if node is None or not hasattr(node, "get"):
+        return ""
+    if getattr(node, "attrs", None) is None:
+        return ""
+    try:
+        value = node.get(name)
+    except (AttributeError, TypeError):
+        return ""
+    if value is None:
+        return ""
+    if isinstance(value, list):
+        return " ".join(str(item) for item in value if item)
+    return str(value)
+
+
+def node_classes(node) -> str:
+    return node_attr(node, "class")
+
+
+def css_hint(node) -> str | None:
+    if node is None or not getattr(node, "name", None):
+        return None
+    node_id = node_attr(node, "id")
+    if node_id:
+        return f"#{node_id}"
+    classes = node_classes(node).split()
+    if classes:
+        return f"{node.name}.{classes[0]}"
+    return str(node.name)
+
+
+def _preserve_noise_type_for_page(zone_type: str, page_type: str) -> bool:
+    if page_type in {"EventPage", "PromotionPage"} and zone_type in {"recommendation"}:
+        return False
+    if page_type in {"EventPage", "PromotionPage"} and zone_type in {"category_filter", "sort_control"}:
+        return False
+    if page_type == "NoticePage" and zone_type == "menu":
+        return False
+    return False
+
+
+def _looks_like_policy_block(text: str) -> bool:
+    lowered = text.lower()
+    if len(text) < 120:
+        return False
+    term_count = sum(1 for term in POLICY_TERMS if term in lowered)
+    comma_or_line_count = text.count("\n") + text.count(",")
+    return term_count >= 2 and comma_or_line_count >= 4
+
+
+def _looks_like_sort_or_filter(text: str) -> bool:
+    lowered = text.lower()
+    sort_terms = ["sort", "low price", "high price", "product count", "상품수", "낮은가격", "높은가격"]
+    return len(text) < 500 and sum(1 for term in sort_terms if term in lowered) >= 2
+
+
+def _zone_type_for_noise(token_text: str, text: str) -> str:
+    lowered = f"{token_text}\n{text}".lower()
+    if "shipping" in lowered or "delivery" in lowered or "배송" in lowered:
+        return "shipping_policy"
+    if "exchange" in lowered or "return" in lowered or "refund" in lowered or "교환" in lowered:
+        return "exchange_policy"
+    if "sort" in lowered or "low price" in lowered or "상품수" in lowered:
+        return "sort_control"
+    if "recommend" in lowered or "related" in lowered:
+        return "recommendation"
+    if "cafe24" in lowered or "powered" in lowered:
+        return "platform_credit"
+    if "login" in lowered or "cart" in lowered or "join" in lowered:
+        return "login_join"
+    if "footer" in lowered or "copyright" in lowered:
+        return "footer"
+    if "header" in lowered:
+        return "header"
+    if "nav" in lowered or "menu" in lowered:
+        return "nav"
+    return "unknown"
+
+
+def _is_contentful(text: str) -> bool:
+    return bool(text and len(text) >= 8 and not is_noisy_line(text))
+
+
+def _zone_confidence(zone_type: str, text: str) -> float:
+    base = {
+        "product_title": 0.86,
+        "product_summary": 0.78,
+        "product_description": 0.78,
+        "product_detail": 0.74,
+        "brand_story_body": 0.72,
+        "notice_body": 0.68,
+        "event_body": 0.66,
+    }.get(zone_type, 0.45)
+    if len(text) > 160:
+        base += 0.05
+    return min(base, 0.95)
+
+
+def _dedupe_zones(zones: list[ContentZone]) -> list[ContentZone]:
+    result: list[ContentZone] = []
+    seen: set[str] = set()
+    for zone in sorted(zones, key=lambda item: (item.confidence, len(item.text)), reverse=True):
+        key_source = normalize_content_lines(zone.text) if zone.claim_allowed else normalize_whitespace(zone.text)
+        key = key_source.lower()
+        if not key or key in seen:
+            continue
+        if any(key in existing or existing in key for existing in seen if min(len(key), len(existing)) > 80):
+            continue
+        seen.add(key)
+        result.append(zone)
+    result.sort(key=lambda item: (-item.confidence, item.zone_type))
+    return result
+
+
+def _tag_alive(tag) -> bool:
+    return bool(getattr(tag, "name", None))
+
+
+def _contains_content_selector(tag) -> bool:
+    if not hasattr(tag, "select"):
+        return False
+    content_selectors = [
+        "h1",
+        ".detailArea",
+        ".infoArea",
+        "#prdDetail",
+        ".xans-product-detail",
+        ".product-detail",
+        ".product-description",
+    ]
+    return any(tag.select(selector) for selector in content_selectors)

ontology_platform/crawler_platform/app/core/crawler/pipeline.py

@@ -0,0 +1,127 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from crawler_platform.app.config.loader import ProjectConfig
+from crawler_platform.app.core.crawler.page_classifier import classify_page
+from crawler_platform.app.core.crawler.fetchers import RobotsPolicy, make_fetcher
+from crawler_platform.app.core.crawler.plugins import ParserRegistry, default_parser_registry
+from crawler_platform.app.core.database.repository import KnowledgeRepository
+from crawler_platform.app.core.extractor.base import ExtractionPageContext, Extractor
+from crawler_platform.app.core.extractor.validation import attach_page_context
+
+
+@dataclass(slots=True)
+class CrawlResult:
+    page_id: int
+    claim_count: int
+    entity_count: int
+    crawl_status: str = "success"
+    extraction_status: str = "success"
+    page_type: str = "UnknownPage"
+    clean_text_length: int = 0
+    raw_text_length: int = 0
+    warnings: list[str] | None = None
+    robots_status: str = "unchecked"
+    robots_reason: str | None = None
+
+
+class CrawlPipeline:
+    def __init__(
+        self,
+        repository: KnowledgeRepository,
+        extractor: Extractor,
+        parser_registry: ParserRegistry | None = None,
+        robots_policy: RobotsPolicy | None = None,
+    ):
+        self.repository = repository
+        self.extractor = extractor
+        self.parser_registry = parser_registry or default_parser_registry()
+        self.robots_policy = robots_policy or RobotsPolicy()
+
+    def crawl_url(self, project_config: ProjectConfig, source_name: str, url: str) -> CrawlResult:
+        source_config = project_config.source_by_name(source_name)
+        robots_decision = self.robots_policy.check(url, source_config.respect_robots_txt)
+        if not robots_decision.allowed:
+            raise PermissionError(f"{robots_decision.reason}: {url}")
+
+        fetcher = make_fetcher(source_config.fetcher, source_config.rate_limit_per_minute)
+        fetch_result = fetcher.fetch(url)
+        parser = self.parser_registry.get(source_config.parser)
+        parsed = parser.parse(fetch_result.analysis_html, fetch_result.final_url or url)
+        page_type = classify_page(
+            fetch_result.final_url or url,
+            parsed.title or fetch_result.title,
+            parsed.raw_text or parsed.text,
+            fetch_result.analysis_html,
+            parsed.source_zones or [],
+        )
+
+        project = self.repository.upsert_project(project_config)
+        source = self.repository.get_source(project.id, source_name)
+        warnings = [*fetch_result.warnings, *(parsed.extraction_warnings or [])]
+        metadata = {
+            **parsed.metadata,
+            "final_url": fetch_result.final_url,
+            "crawl_status": fetch_result.crawl_status,
+            "crawl_warnings": fetch_result.warnings,
+            "page_type": page_type,
+            "raw_text_length": len(parsed.raw_text or ""),
+            "clean_text_length": len(parsed.text or ""),
+            "main_content_preview": (parsed.main_content or parsed.text)[:800],
+        }
+        page = self.repository.upsert_page(
+            project_id=project.id,
+            source_id=source.id,
+            url=url,
+            title=parsed.title or fetch_result.title,
+            status_code=fetch_result.status_code,
+            cleaned_text=parsed.text,
+            metadata=metadata,
+        )
+        if fetch_result.crawl_status != "success" or parsed.extraction_status == "failed":
+            return CrawlResult(
+                page_id=page.id,
+                claim_count=0,
+                entity_count=0,
+                crawl_status=fetch_result.crawl_status,
+                extraction_status=parsed.extraction_status,
+                page_type=page_type,
+                clean_text_length=len(parsed.text or ""),
+                raw_text_length=len(parsed.raw_text or ""),
+                warnings=warnings,
+                robots_status=robots_decision.status,
+                robots_reason=robots_decision.reason,
+            )
+
+        context = ExtractionPageContext(
+            url=url,
+            final_url=fetch_result.final_url,
+            title=parsed.title or fetch_result.title,
+            page_type=page_type,
+            clean_text=parsed.text,
+            raw_text=parsed.raw_text,
+            main_content=parsed.main_content,
+            clean_markdown=parsed.clean_markdown,
+            source_zones=parsed.source_zones or [],
+            crawl_status=fetch_result.crawl_status,
+            extraction_status=parsed.extraction_status,
+            warnings=warnings,
+            metadata=metadata,
+        )
+        bundle = self.extractor.extract_from_context(context, project_config)
+        bundle = attach_page_context(bundle, context)
+        claims = self.repository.save_extraction_bundle(project.id, source, page, bundle, project_config)
+        return CrawlResult(
+            page_id=page.id,
+            claim_count=len(claims),
+            entity_count=len(bundle.entities),
+            crawl_status=fetch_result.crawl_status,
+            extraction_status=parsed.extraction_status,
+            page_type=page_type,
+            clean_text_length=len(parsed.text or ""),
+            raw_text_length=len(parsed.raw_text or ""),
+            warnings=warnings,
+            robots_status=robots_decision.status,
+            robots_reason=robots_decision.reason,
+        )

ontology_platform/crawler_platform/app/core/crawler/plugins.py

@@ -0,0 +1,67 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Protocol
+
+
+@dataclass(slots=True)
+class ParsedPage:
+    title: str | None
+    text: str
+    metadata: dict[str, object]
+    raw_text: str = ""
+    main_content: str = ""
+    clean_markdown: str = ""
+    source_zones: list[dict[str, object]] | None = None
+    noise_zones: list[dict[str, object]] | None = None
+    page_type: str = "UnknownPage"
+    extraction_status: str = "failed"
+    extraction_warnings: list[str] | None = None
+
+
+class SiteParser(Protocol):
+    name: str
+
+    def parse(self, html: str, url: str) -> ParsedPage:
+        raise NotImplementedError
+
+
+class ParserRegistry:
+    def __init__(self):
+        self._parsers: dict[str, SiteParser] = {}
+
+    def register(self, parser: SiteParser) -> None:
+        self._parsers[parser.name] = parser
+
+    def get(self, name: str) -> SiteParser:
+        if name not in self._parsers:
+            raise KeyError(f"Parser not registered: {name}")
+        return self._parsers[name]
+
+
+class GenericProductParser:
+    name = "generic"
+
+    def parse(self, html: str, url: str) -> ParsedPage:
+        from crawler_platform.app.core.crawler.html_cleaner import clean_html_with_metadata
+
+        cleaned = clean_html_with_metadata(html, url=url)
+        return ParsedPage(
+            title=cleaned.title,
+            text=cleaned.text,
+            metadata={"parser": self.name, "url": url, **cleaned.metadata},
+            raw_text=cleaned.raw_text,
+            main_content=cleaned.main_content,
+            clean_markdown=cleaned.clean_markdown,
+            source_zones=cleaned.source_zones,
+            noise_zones=cleaned.noise_zones,
+            page_type=cleaned.page_type,
+            extraction_status=cleaned.extraction_status,
+            extraction_warnings=cleaned.extraction_warnings,
+        )
+
+
+def default_parser_registry() -> ParserRegistry:
+    registry = ParserRegistry()
+    registry.register(GenericProductParser())
+    return registry

ontology_platform/crawler_platform/app/core/crawler/site_crawler.py

@@ -0,0 +1,443 @@
+from __future__ import annotations
+
+from collections import deque
+from dataclasses import dataclass, field
+from typing import Callable
+from urllib.parse import urldefrag, urlparse
+
+from crawler_platform.app.config.loader import ProjectConfig
+from crawler_platform.app.core.crawler.discovery import discover_links
+from crawler_platform.app.core.crawler.fetchers import RobotsDecision, RobotsPolicy, make_fetcher
+from crawler_platform.app.core.crawler.page_classifier import (
+    classify_page as classify_page_type,
+    should_analyze_page as should_analyze_page_type,
+)
+from crawler_platform.app.core.crawler.plugins import ParserRegistry, default_parser_registry
+from crawler_platform.app.core.database import models
+from crawler_platform.app.core.database.repository import KnowledgeRepository
+from crawler_platform.app.core.extractor.base import ExtractionPageContext, Extractor
+from crawler_platform.app.core.extractor.validation import attach_page_context
+
+
+@dataclass(slots=True)
+class SiteCrawlPageResult:
+    url: str
+    depth: int
+    status: str
+    page_type: str
+    page_id: int | None = None
+    claim_count: int = 0
+    entity_count: int = 0
+    discovered_count: int = 0
+    crawl_status: str = "success"
+    extraction_status: str = "unknown"
+    raw_text_length: int = 0
+    clean_text_length: int = 0
+    removed_noise_zones_count: int = 0
+    warnings: list[str] = field(default_factory=list)
+    robots_status: str = "unchecked"
+    robots_reason: str | None = None
+    error: str | None = None
+
+
+@dataclass(slots=True)
+class SiteCrawlResult:
+    seed_url: str
+    visited_count: int = 0
+    analyzed_count: int = 0
+    queued_count: int = 0
+    skipped_count: int = 0
+    errors: list[str] = field(default_factory=list)
+    pages: list[SiteCrawlPageResult] = field(default_factory=list)
+
+
+class SiteCrawler:
+    def __init__(
+        self,
+        repository: KnowledgeRepository,
+        extractor: Extractor,
+        parser_registry: ParserRegistry | None = None,
+        robots_policy: RobotsPolicy | None = None,
+    ):
+        self.repository = repository
+        self.extractor = extractor
+        self.parser_registry = parser_registry or default_parser_registry()
+        self.robots_policy = robots_policy or RobotsPolicy()
+
+    def crawl_site(
+        self,
+        project_config: ProjectConfig,
+        source_name: str,
+        seed_url: str,
+        max_depth: int = 2,
+        max_pages: int = 50,
+        same_domain_only: bool = True,
+        analyze_page_types: set[str] | None = None,
+        progress_callback: Callable[[SiteCrawlResult, SiteCrawlPageResult], None] | None = None,
+        should_stop: Callable[[], bool] | None = None,
+        parent_job_id: int | None = None,
+    ) -> SiteCrawlResult:
+        source_config = project_config.source_by_name(source_name)
+        fetcher = make_fetcher(source_config.fetcher, source_config.rate_limit_per_minute)
+        parser = self.parser_registry.get(source_config.parser)
+        seed_host = normalized_host(seed_url)
+        analyze_page_types = analyze_page_types or {"ProductPage", "BrandStoryPage", "ReviewPage"}
+
+        result = SiteCrawlResult(seed_url=seed_url)
+        queue: deque[tuple[str, int]] = deque([(normalize_url(seed_url), 0)])
+        queued: set[str] = {normalize_url(seed_url)}
+        visited: set[str] = set()
+
+        project = self.repository.upsert_project(project_config)
+        source = self.repository.get_source(project.id, source_name)
+
+        while queue and result.visited_count < max_pages:
+            if should_stop and should_stop():
+                result.queued_count = len(queue)
+                break
+
+            url, depth = queue.popleft()
+            if url in visited:
+                continue
+            visited.add(url)
+
+            job = self._create_job(project.id, source.id, url, depth, parent_job_id)
+            if depth > max_depth:
+                self._finish_job(job, "skipped", "max depth exceeded")
+                result.skipped_count += 1
+                self._record_page_result(
+                    result,
+                    SiteCrawlPageResult(url=url, depth=depth, status="skipped", page_type="unknown"),
+                    len(queue),
+                    progress_callback,
+                )
+                continue
+            if same_domain_only and normalized_host(url) != seed_host:
+                self._finish_job(job, "skipped", "outside same-domain filter")
+                result.skipped_count += 1
+                self._record_page_result(
+                    result,
+                    SiteCrawlPageResult(url=url, depth=depth, status="skipped", page_type="external"),
+                    len(queue),
+                    progress_callback,
+                )
+                continue
+            robots_decision = self.robots_policy.check(url, source_config.respect_robots_txt)
+            if not robots_decision.allowed:
+                error = f"{robots_decision.reason}: {url}"
+                self._finish_job(job, "blocked", error)
+                result.skipped_count += 1
+                result.errors.append(error)
+                self._record_page_result(
+                    result,
+                    SiteCrawlPageResult(
+                        url=url,
+                        depth=depth,
+                        status="blocked",
+                        page_type="unknown",
+                        robots_status=robots_decision.status,
+                        robots_reason=robots_decision.reason,
+                        error=error,
+                    ),
+                    len(queue),
+                    progress_callback,
+                )
+                continue
+
+            try:
+                self._crawl_one_page(
+                    project_config=project_config,
+                    source=source,
+                    parser=parser,
+                    fetcher=fetcher,
+                    url=url,
+                    depth=depth,
+                    max_depth=max_depth,
+                    same_domain_only=same_domain_only,
+                    seed_host=seed_host,
+                    queue=queue,
+                    queued=queued,
+                    visited=visited,
+                    analyze_page_types=analyze_page_types,
+                    result=result,
+                    job=job,
+                    robots_decision=robots_decision,
+                    progress_callback=progress_callback,
+                )
+            except Exception as exc:
+                error = str(exc)
+                self._finish_job(job, "failed", error)
+                result.visited_count += 1
+                result.skipped_count += 1
+                result.errors.append(error)
+                self._record_page_result(
+                    result,
+                    SiteCrawlPageResult(url=url, depth=depth, status="failed", page_type="unknown", error=error),
+                    len(queue),
+                    progress_callback,
+                )
+
+        return result
+
+    def _crawl_one_page(
+        self,
+        project_config: ProjectConfig,
+        source: models.Source,
+        parser: ParserRegistry,
+        fetcher,
+        url: str,
+        depth: int,
+        max_depth: int,
+        same_domain_only: bool,
+        seed_host: str,
+        queue: deque[tuple[str, int]],
+        queued: set[str],
+        visited: set[str],
+        analyze_page_types: set[str],
+        result: SiteCrawlResult,
+        job: models.CrawlJob,
+        robots_decision: RobotsDecision,
+        progress_callback: Callable[[SiteCrawlResult, SiteCrawlPageResult], None] | None,
+    ) -> None:
+        fetch_result = fetcher.fetch(url)
+        if is_failed_fetch_status(fetch_result.status_code) or fetch_result.crawl_status != "success":
+            error = f"fetch failed with status {fetch_result.status_code}; crawl_status={fetch_result.crawl_status}"
+            page = self.repository.upsert_page(
+                project_id=source.project_id,
+                source_id=source.id,
+                url=url,
+                title=fetch_result.title,
+                status_code=fetch_result.status_code,
+                cleaned_text="",
+                metadata={
+                    "final_url": fetch_result.final_url,
+                    "crawl_status": fetch_result.crawl_status,
+                    "crawl_warnings": fetch_result.warnings,
+                },
+            )
+            self._finish_job(job, "failed", error)
+            result.visited_count += 1
+            result.skipped_count += 1
+            result.errors.append(f"{error}: {url}")
+            self._record_page_result(
+                result,
+                SiteCrawlPageResult(
+                    url=url,
+                    depth=depth,
+                    status=fetch_result.crawl_status,
+                    page_type="unknown",
+                    page_id=page.id,
+                    crawl_status=fetch_result.crawl_status,
+                    robots_status=robots_decision.status,
+                    robots_reason=robots_decision.reason,
+                    warnings=fetch_result.warnings,
+                    error=error,
+                ),
+                len(queue),
+                progress_callback,
+            )
+            return
+
+        parsed = parser.parse(fetch_result.analysis_html, fetch_result.final_url or url)
+        page_type = classify_page(
+            fetch_result.final_url or url,
+            parsed.title or fetch_result.title,
+            parsed.raw_text or parsed.text,
+            fetch_result.analysis_html,
+            parsed.source_zones or [],
+        )
+        discovered_count = self._enqueue_links(
+            html=fetch_result.analysis_html,
+            base_url=fetch_result.final_url or url,
+            depth=depth,
+            max_depth=max_depth,
+            same_domain_only=same_domain_only,
+            seed_host=seed_host,
+            queue=queue,
+            queued=queued,
+            visited=visited,
+        )
+
+        warnings = [*fetch_result.warnings, *(parsed.extraction_warnings or [])]
+        metadata = {
+            **parsed.metadata,
+            "final_url": fetch_result.final_url,
+            "crawl_status": fetch_result.crawl_status,
+            "crawl_warnings": fetch_result.warnings,
+            "page_type": page_type,
+            "depth": depth,
+            "raw_text_length": len(parsed.raw_text or ""),
+            "clean_text_length": len(parsed.text or ""),
+            "main_content_preview": (parsed.main_content or parsed.text)[:800],
+        }
+        page = self.repository.upsert_page(
+            project_id=source.project_id,
+            source_id=source.id,
+            url=url,
+            title=parsed.title or fetch_result.title,
+            status_code=fetch_result.status_code,
+            cleaned_text=parsed.text,
+            metadata=metadata,
+        )
+
+        common_page_result = {
+            "url": url,
+            "depth": depth,
+            "page_type": page_type,
+            "page_id": page.id,
+            "discovered_count": discovered_count,
+            "crawl_status": fetch_result.crawl_status,
+            "extraction_status": parsed.extraction_status,
+            "raw_text_length": len(parsed.raw_text or ""),
+            "clean_text_length": len(parsed.text or ""),
+            "removed_noise_zones_count": int(parsed.metadata.get("removed_noise_zones_count") or 0),
+            "warnings": warnings,
+            "robots_status": robots_decision.status,
+            "robots_reason": robots_decision.reason,
+        }
+
+        if parsed.extraction_status == "failed":
+            self._finish_job(job, "failed", "main content extraction failed")
+            result.visited_count += 1
+            result.skipped_count += 1
+            self._record_page_result(
+                result,
+                SiteCrawlPageResult(
+                    **common_page_result,
+                    status="extraction_failed",
+                    error="main content extraction failed",
+                ),
+                len(queue),
+                progress_callback,
+            )
+            return
+
+        if should_analyze_page(page_type, analyze_page_types):
+            context = ExtractionPageContext(
+                url=url,
+                final_url=fetch_result.final_url,
+                title=parsed.title or fetch_result.title,
+                page_type=page_type,
+                clean_text=parsed.text,
+                raw_text=parsed.raw_text,
+                main_content=parsed.main_content,
+                clean_markdown=parsed.clean_markdown,
+                source_zones=parsed.source_zones or [],
+                crawl_status=fetch_result.crawl_status,
+                extraction_status=parsed.extraction_status,
+                warnings=warnings,
+                metadata=metadata,
+            )
+            bundle = self.extractor.extract_from_context(context, project_config)
+            bundle = attach_page_context(bundle, context)
+            claims = self.repository.save_extraction_bundle(source.project_id, source, page, bundle, project_config)
+            self._finish_job(job, "completed")
+            result.analyzed_count += 1
+            page_result = SiteCrawlPageResult(
+                **common_page_result,
+                status="completed",
+                claim_count=len(claims),
+                entity_count=len(bundle.entities),
+            )
+        else:
+            self._finish_job(job, "discovered")
+            result.skipped_count += 1
+            page_result = SiteCrawlPageResult(**common_page_result, status="discovered")
+
+        result.visited_count += 1
+        self._record_page_result(result, page_result, len(queue), progress_callback)
+
+    def _enqueue_links(
+        self,
+        html: str,
+        base_url: str,
+        depth: int,
+        max_depth: int,
+        same_domain_only: bool,
+        seed_host: str,
+        queue: deque[tuple[str, int]],
+        queued: set[str],
+        visited: set[str],
+    ) -> int:
+        if depth >= max_depth:
+            return 0
+        discovered_count = 0
+        links = discover_links(html, base_url, limit=200)
+        for link in links:
+            next_url = normalize_url(link.url)
+            if next_url in queued or next_url in visited:
+                continue
+            if same_domain_only and normalized_host(next_url) != seed_host:
+                continue
+            queue.append((next_url, depth + 1))
+            queued.add(next_url)
+            discovered_count += 1
+        return discovered_count
+
+    def _record_page_result(
+        self,
+        result: SiteCrawlResult,
+        page_result: SiteCrawlPageResult,
+        queued_count: int,
+        progress_callback: Callable[[SiteCrawlResult, SiteCrawlPageResult], None] | None,
+    ) -> None:
+        result.queued_count = queued_count
+        result.pages.append(page_result)
+        if progress_callback:
+            progress_callback(result, page_result)
+
+    def _create_job(
+        self,
+        project_id: int,
+        source_id: int,
+        url: str,
+        depth: int,
+        parent_job_id: int | None = None,
+    ) -> models.CrawlJob:
+        job = models.CrawlJob(
+            project_id=project_id,
+            source_id=source_id,
+            url=url,
+            status="running",
+            metadata_json={
+                "depth": depth,
+                **({"parent_job_id": parent_job_id} if parent_job_id is not None else {}),
+            },
+            started_at=models.utcnow(),
+        )
+        self.repository.session.add(job)
+        self.repository.session.flush()
+        return job
+
+    def _finish_job(self, job: models.CrawlJob, status: str, error: str | None = None) -> None:
+        job.status = status
+        job.error = error
+        job.finished_at = models.utcnow()
+
+
+def normalize_url(url: str) -> str:
+    clean, _fragment = urldefrag(url.strip())
+    return clean.rstrip("/")
+
+
+def normalized_host(url: str) -> str:
+    parsed = urlparse(url)
+    return parsed.netloc.lower()
+
+
+def is_failed_fetch_status(status_code: int | None) -> bool:
+    return status_code is None or status_code >= 400
+
+
+def should_analyze_page(page_type: str, analyze_page_types: set[str]) -> bool:
+    return should_analyze_page_type(page_type, analyze_page_types)
+
+
+def classify_page(
+    url: str,
+    title: str | None,
+    text: str,
+    html: str | None = None,
+    source_zones: list[dict[str, object]] | None = None,
+) -> str:
+    return classify_page_type(url, title, text, html=html, source_zones=source_zones)

ontology_platform/crawler_platform/app/core/database/__init__.py

@@ -0,0 +1,2 @@
+"""Database models and repositories."""
+

ontology_platform/crawler_platform/app/core/database/models.py

@@ -0,0 +1,349 @@
+from __future__ import annotations
+
+from datetime import datetime, timezone
+
+from sqlalchemy import (
+    JSON,
+    Boolean,
+    Column,
+    DateTime,
+    Float,
+    ForeignKey,
+    Integer,
+    String,
+    Text,
+    UniqueConstraint,
+)
+from sqlalchemy.orm import declarative_base, relationship
+
+
+Base = declarative_base()
+
+
+def utcnow() -> datetime:
+    return datetime.now(timezone.utc)
+
+
+class Project(Base):
+    __tablename__ = "projects"
+
+    id = Column(Integer, primary_key=True)
+    name = Column(String(160), unique=True, nullable=False, index=True)
+    domain = Column(String(80), nullable=False, index=True)
+    config = Column(JSON, nullable=False, default=dict)
+    created_at = Column(DateTime(timezone=True), default=utcnow, nullable=False)
+    updated_at = Column(DateTime(timezone=True), default=utcnow, nullable=False)
+
+    sources = relationship("Source", back_populates="project", cascade="all, delete-orphan")
+
+
+class Source(Base):
+    __tablename__ = "sources"
+    __table_args__ = (UniqueConstraint("project_id", "name", name="uq_source_project_name"),)
+
+    id = Column(Integer, primary_key=True)
+    project_id = Column(Integer, ForeignKey("projects.id"), nullable=False, index=True)
+    name = Column(String(160), nullable=False)
+    type = Column(String(80), nullable=False, default="unknown")
+    base_url = Column(Text)
+    trust_level = Column(Float, nullable=False, default=0.5)
+    respect_robots_txt = Column(Boolean, nullable=False, default=True)
+    rate_limit_per_minute = Column(Integer, nullable=False, default=30)
+    update_policy = Column(JSON, nullable=False, default=dict)
+    created_at = Column(DateTime(timezone=True), default=utcnow, nullable=False)
+    updated_at = Column(DateTime(timezone=True), default=utcnow, nullable=False)
+
+    project = relationship("Project", back_populates="sources")
+
+
+class Page(Base):
+    __tablename__ = "pages"
+    __table_args__ = (UniqueConstraint("project_id", "url", name="uq_page_project_url"),)
+
+    id = Column(Integer, primary_key=True)
+    project_id = Column(Integer, ForeignKey("projects.id"), nullable=False, index=True)
+    source_id = Column(Integer, ForeignKey("sources.id"), nullable=False, index=True)
+    url = Column(Text, nullable=False)
+    canonical_url = Column(Text)
+    title = Column(Text)
+    content_hash = Column(String(80), index=True)
+    status_code = Column(Integer)
+    fetched_at = Column(DateTime(timezone=True), default=utcnow, nullable=False)
+    cleaned_text_summary = Column(Text)
+    raw_storage_ref = Column(Text)
+    metadata_json = Column(JSON, nullable=False, default=dict)
+
+
+class Entity(Base):
+    __tablename__ = "entities"
+    __table_args__ = (
+        UniqueConstraint("project_id", "entity_type", "canonical_name", name="uq_entity_identity"),
+    )
+
+    id = Column(Integer, primary_key=True)
+    project_id = Column(Integer, ForeignKey("projects.id"), nullable=False, index=True)
+    entity_type = Column(String(120), nullable=False, index=True)
+    name = Column(String(240), nullable=False)
+    canonical_name = Column(String(240), nullable=False, index=True)
+    external_ids = Column(JSON, nullable=False, default=dict)
+    metadata_json = Column(JSON, nullable=False, default=dict)
+    created_at = Column(DateTime(timezone=True), default=utcnow, nullable=False)
+    updated_at = Column(DateTime(timezone=True), default=utcnow, nullable=False)
+
+
+class Attribute(Base):
+    __tablename__ = "attributes"
+    __table_args__ = (
+        UniqueConstraint("entity_id", "name", "source_id", name="uq_attribute_entity_source"),
+    )
+
+    id = Column(Integer, primary_key=True)
+    project_id = Column(Integer, ForeignKey("projects.id"), nullable=False, index=True)
+    entity_id = Column(Integer, ForeignKey("entities.id"), nullable=False, index=True)
+    source_id = Column(Integer, ForeignKey("sources.id"), nullable=True, index=True)
+    name = Column(String(120), nullable=False, index=True)
+    value = Column(JSON, nullable=False)
+    confidence = Column(Float, nullable=False, default=0.5)
+    updated_at = Column(DateTime(timezone=True), default=utcnow, nullable=False)
+
+
+class Relation(Base):
+    __tablename__ = "relations"
+    __table_args__ = (
+        UniqueConstraint("project_id", "subject_entity_id", "predicate", "object_entity_id", name="uq_relation"),
+    )
+
+    id = Column(Integer, primary_key=True)
+    project_id = Column(Integer, ForeignKey("projects.id"), nullable=False, index=True)
+    subject_entity_id = Column(Integer, ForeignKey("entities.id"), nullable=False, index=True)
+    predicate = Column(String(160), nullable=False, index=True)
+    object_entity_id = Column(Integer, ForeignKey("entities.id"), nullable=False, index=True)
+    confidence = Column(Float, nullable=False, default=0.5)
+    support_count = Column(Integer, nullable=False, default=1)
+    updated_at = Column(DateTime(timezone=True), default=utcnow, nullable=False)
+
+
+class OntologyEntityType(Base):
+    __tablename__ = "ontology_entity_types"
+    __table_args__ = (
+        UniqueConstraint("project_id", "name", name="uq_ontology_entity_type_project_name"),
+    )
+
+    id = Column(Integer, primary_key=True)
+    project_id = Column(Integer, ForeignKey("projects.id"), nullable=False, index=True)
+    name = Column(String(160), nullable=False, index=True)
+    domain = Column(String(120), nullable=False, default="generic", index=True)
+    description = Column(Text)
+    status = Column(String(40), nullable=False, default="active", index=True)
+    version = Column(String(40), nullable=False, default="1.0.0")
+    confidence = Column(Float, nullable=False, default=1.0)
+    metadata_json = Column(JSON, nullable=False, default=dict)
+    created_at = Column(DateTime(timezone=True), default=utcnow, nullable=False)
+    updated_at = Column(DateTime(timezone=True), default=utcnow, nullable=False)
+
+
+class OntologyRelationType(Base):
+    __tablename__ = "ontology_relation_types"
+    __table_args__ = (
+        UniqueConstraint("project_id", "name", name="uq_ontology_relation_type_project_name"),
+    )
+
+    id = Column(Integer, primary_key=True)
+    project_id = Column(Integer, ForeignKey("projects.id"), nullable=False, index=True)
+    name = Column(String(160), nullable=False, index=True)
+    domain = Column(String(120), nullable=False, default="generic", index=True)
+    description = Column(Text)
+    allowed_subject_types = Column(JSON, nullable=False, default=list)
+    allowed_object_types = Column(JSON, nullable=False, default=list)
+    allowed_page_types = Column(JSON, nullable=False, default=list)
+    allowed_source_zones = Column(JSON, nullable=False, default=list)
+    semantic_constraints = Column(JSON, nullable=False, default=dict)
+    confidence_rules = Column(JSON, nullable=False, default=dict)
+    status = Column(String(40), nullable=False, default="active", index=True)
+    version = Column(String(40), nullable=False, default="1.0.0")
+    confidence = Column(Float, nullable=False, default=1.0)
+    metadata_json = Column(JSON, nullable=False, default=dict)
+    created_at = Column(DateTime(timezone=True), default=utcnow, nullable=False)
+    updated_at = Column(DateTime(timezone=True), default=utcnow, nullable=False)
+
+
+class OntologyTriple(Base):
+    __tablename__ = "ontology_triples"
+    __table_args__ = (UniqueConstraint("project_id", "triple_hash", name="uq_ontology_triple_hash"),)
+
+    id = Column(Integer, primary_key=True)
+    project_id = Column(Integer, ForeignKey("projects.id"), nullable=False, index=True)
+    claim_id = Column(Integer, ForeignKey("claims.id"), nullable=True, index=True)
+    source_id = Column(Integer, ForeignKey("sources.id"), nullable=True, index=True)
+    page_id = Column(Integer, ForeignKey("pages.id"), nullable=True, index=True)
+    subject_entity_id = Column(Integer, ForeignKey("entities.id"), nullable=False, index=True)
+    subject_type = Column(String(160), nullable=False, index=True)
+    predicate = Column(String(160), nullable=False, index=True)
+    relation_type_id = Column(Integer, ForeignKey("ontology_relation_types.id"), nullable=True, index=True)
+    object_entity_id = Column(Integer, ForeignKey("entities.id"), nullable=True, index=True)
+    object_type = Column(String(160), nullable=True, index=True)
+    object_value = Column(JSON, nullable=True)
+    value_type = Column(String(80), nullable=False, default="entity")
+    triple_hash = Column(String(80), nullable=False, index=True)
+    status = Column(String(40), nullable=False, default="candidate", index=True)
+    confidence = Column(Float, nullable=False, default=0.5)
+    support_count = Column(Integer, nullable=False, default=1)
+    first_seen_at = Column(DateTime(timezone=True), default=utcnow, nullable=False)
+    last_seen_at = Column(DateTime(timezone=True), default=utcnow, nullable=False)
+    metadata_json = Column(JSON, nullable=False, default=dict)
+
+
+class OntologyProposal(Base):
+    __tablename__ = "ontology_proposals"
+    __table_args__ = (
+        UniqueConstraint("project_id", "proposal_hash", name="uq_ontology_proposal_hash"),
+    )
+
+    id = Column(Integer, primary_key=True)
+    project_id = Column(Integer, ForeignKey("projects.id"), nullable=False, index=True)
+    proposal_type = Column(String(80), nullable=False, index=True)
+    name = Column(String(160), nullable=False, index=True)
+    reason = Column(Text)
+    evidence = Column(Text)
+    status = Column(String(40), nullable=False, default="pending_review", index=True)
+    confidence = Column(Float, nullable=False, default=0.5)
+    proposal_hash = Column(String(80), nullable=False, index=True)
+    metadata_json = Column(JSON, nullable=False, default=dict)
+    created_at = Column(DateTime(timezone=True), default=utcnow, nullable=False)
+    updated_at = Column(DateTime(timezone=True), default=utcnow, nullable=False)
+
+
+class KnowledgeGap(Base):
+    __tablename__ = "knowledge_gaps"
+    __table_args__ = (UniqueConstraint("project_id", "gap_hash", name="uq_knowledge_gap_hash"),)
+
+    id = Column(Integer, primary_key=True)
+    project_id = Column(Integer, ForeignKey("projects.id"), nullable=False, index=True)
+    gap_type = Column(String(80), nullable=False, index=True)
+    target_type = Column(String(160), nullable=True, index=True)
+    target_name = Column(String(240), nullable=True, index=True)
+    description = Column(Text, nullable=False)
+    priority = Column(Float, nullable=False, default=0.5, index=True)
+    status = Column(String(40), nullable=False, default="open", index=True)
+    gap_hash = Column(String(80), nullable=False, index=True)
+    evidence = Column(JSON, nullable=False, default=dict)
+    metadata_json = Column(JSON, nullable=False, default=dict)
+    created_at = Column(DateTime(timezone=True), default=utcnow, nullable=False)
+    updated_at = Column(DateTime(timezone=True), default=utcnow, nullable=False)
+
+
+class Claim(Base):
+    __tablename__ = "claims"
+    __table_args__ = (UniqueConstraint("project_id", "claim_hash", name="uq_claim_hash"),)
+
+    id = Column(Integer, primary_key=True)
+    project_id = Column(Integer, ForeignKey("projects.id"), nullable=False, index=True)
+    source_id = Column(Integer, ForeignKey("sources.id"), nullable=False, index=True)
+    page_id = Column(Integer, ForeignKey("pages.id"), nullable=True, index=True)
+    subject_entity_id = Column(Integer, ForeignKey("entities.id"), nullable=False, index=True)
+    predicate = Column(String(160), nullable=False, index=True)
+    object_entity_id = Column(Integer, ForeignKey("entities.id"), nullable=True, index=True)
+    object_value = Column(JSON, nullable=True)
+    value_type = Column(String(80), nullable=False, default="entity")
+    claim_hash = Column(String(80), nullable=False, index=True)
+    confidence = Column(Float, nullable=False, default=0.5)
+    confidence_reason = Column(Text)
+    extraction_method = Column(String(120), nullable=False, default="rule_based")
+    status = Column(String(40), nullable=False, default="active")
+    first_seen_at = Column(DateTime(timezone=True), default=utcnow, nullable=False)
+    last_seen_at = Column(DateTime(timezone=True), default=utcnow, nullable=False)
+    valid_until = Column(DateTime(timezone=True), nullable=True)
+    metadata_json = Column(JSON, nullable=False, default=dict)
+
+    evidence_items = relationship("Evidence", back_populates="claim", cascade="all, delete-orphan")
+
+
+class Evidence(Base):
+    __tablename__ = "evidence"
+
+    id = Column(Integer, primary_key=True)
+    project_id = Column(Integer, ForeignKey("projects.id"), nullable=False, index=True)
+    claim_id = Column(Integer, ForeignKey("claims.id"), nullable=False, index=True)
+    page_id = Column(Integer, ForeignKey("pages.id"), nullable=True, index=True)
+    evidence_text = Column(Text, nullable=False)
+    evidence_summary = Column(Text)
+    selector = Column(Text)
+    start_offset = Column(Integer)
+    end_offset = Column(Integer)
+    created_at = Column(DateTime(timezone=True), default=utcnow, nullable=False)
+
+    claim = relationship("Claim", back_populates="evidence_items")
+
+
+class ExtractionLog(Base):
+    __tablename__ = "extraction_logs"
+
+    id = Column(Integer, primary_key=True)
+    project_id = Column(Integer, ForeignKey("projects.id"), nullable=False, index=True)
+    page_id = Column(Integer, ForeignKey("pages.id"), nullable=True, index=True)
+    extractor_name = Column(String(160), nullable=False)
+    provider = Column(String(120), nullable=False, default="rule_based")
+    input_hash = Column(String(80))
+    raw_output = Column(JSON, nullable=False, default=dict)
+    error = Column(Text)
+    created_at = Column(DateTime(timezone=True), default=utcnow, nullable=False)
+
+
+class CrawlJob(Base):
+    __tablename__ = "crawl_jobs"
+
+    id = Column(Integer, primary_key=True)
+    project_id = Column(Integer, ForeignKey("projects.id"), nullable=False, index=True)
+    source_id = Column(Integer, ForeignKey("sources.id"), nullable=True, index=True)
+    url = Column(Text)
+    status = Column(String(40), nullable=False, default="pending")
+    priority = Column(Integer, nullable=False, default=100)
+    scheduled_at = Column(DateTime(timezone=True), default=utcnow, nullable=False)
+    started_at = Column(DateTime(timezone=True), nullable=True)
+    finished_at = Column(DateTime(timezone=True), nullable=True)
+    error = Column(Text)
+    metadata_json = Column(JSON, nullable=False, default=dict)
+
+
+class UserProfile(Base):
+    __tablename__ = "user_profiles"
+
+    id = Column(Integer, primary_key=True)
+    project_id = Column(Integer, ForeignKey("projects.id"), nullable=False, index=True)
+    external_user_id = Column(String(160), nullable=False, index=True)
+    display_name = Column(String(160))
+    context = Column(JSON, nullable=False, default=dict)
+    created_at = Column(DateTime(timezone=True), default=utcnow, nullable=False)
+    updated_at = Column(DateTime(timezone=True), default=utcnow, nullable=False)
+
+
+class UserPreference(Base):
+    __tablename__ = "user_preferences"
+
+    id = Column(Integer, primary_key=True)
+    project_id = Column(Integer, ForeignKey("projects.id"), nullable=False, index=True)
+    user_profile_id = Column(Integer, ForeignKey("user_profiles.id"), nullable=False, index=True)
+    likes = Column(JSON, nullable=False, default=list)
+    dislikes = Column(JSON, nullable=False, default=list)
+    preferred_moods = Column(JSON, nullable=False, default=list)
+    preferred_notes = Column(JSON, nullable=False, default=list)
+    avoided_notes = Column(JSON, nullable=False, default=list)
+    price_preference = Column(JSON, nullable=False, default=dict)
+    season_context = Column(String(80))
+    occasion_context = Column(String(120))
+    feedback_history = Column(JSON, nullable=False, default=list)
+    updated_at = Column(DateTime(timezone=True), default=utcnow, nullable=False)
+
+
+class FeedbackLog(Base):
+    __tablename__ = "feedback_logs"
+
+    id = Column(Integer, primary_key=True)
+    project_id = Column(Integer, ForeignKey("projects.id"), nullable=False, index=True)
+    user_profile_id = Column(Integer, ForeignKey("user_profiles.id"), nullable=False, index=True)
+    entity_id = Column(Integer, ForeignKey("entities.id"), nullable=True, index=True)
+    action = Column(String(80), nullable=False)
+    score = Column(Float)
+    reason = Column(Text)
+    metadata_json = Column(JSON, nullable=False, default=dict)
+    created_at = Column(DateTime(timezone=True), default=utcnow, nullable=False)

ontology_platform/crawler_platform/app/core/database/repository.py

@@ -0,0 +1,504 @@
+from __future__ import annotations
+
+from dataclasses import asdict
+import hashlib
+import json
+from typing import Any
+
+from sqlalchemy import select
+from sqlalchemy.orm import Session
+
+from crawler_platform.app.config.loader import ProjectConfig, SourceConfig
+from crawler_platform.app.core.database import models
+from crawler_platform.app.core.extractor.base import ExtractedClaim, ExtractedEntity, ExtractionBundle
+from crawler_platform.app.core.extractor.validation import validate_extraction_bundle
+from crawler_platform.app.core.ontology.entity_normalizer import canonical_entity_key, normalize_entity_name
+from crawler_platform.app.core.ontology.graph_merge import should_merge_claim_to_graph
+from crawler_platform.app.core.ontology.registry import OntologyRegistry
+from crawler_platform.app.core.ontology.triple_store import OntologyTripleStore
+
+
+def canonicalize(value: str) -> str:
+    return canonical_entity_key(value)
+
+
+def short_hash(text: str) -> str:
+    return hashlib.sha256(text.encode("utf-8")).hexdigest()
+
+
+class KnowledgeRepository:
+    def __init__(self, session: Session):
+        self.session = session
+
+    def upsert_project(self, config: ProjectConfig) -> models.Project:
+        project = self.session.scalar(select(models.Project).where(models.Project.name == config.project_name))
+        config_dict = _project_config_to_dict(config)
+        if project is None:
+            project = models.Project(name=config.project_name, domain=config.domain, config=config_dict)
+            self.session.add(project)
+            self.session.flush()
+        else:
+            if project.domain != config.domain or project.config != config_dict:
+                project.domain = config.domain
+                project.config = config_dict
+                project.updated_at = models.utcnow()
+        for source_config in config.sources:
+            self.upsert_source(project, source_config)
+        OntologyRegistry(self.session).seed_from_config(project, config)
+        OntologyTripleStore(self.session).backfill_project(project.id)
+        return project
+
+    def upsert_source(self, project: models.Project, source_config: SourceConfig) -> models.Source:
+        source = self.session.scalar(
+            select(models.Source).where(
+                models.Source.project_id == project.id,
+                models.Source.name == source_config.name,
+            )
+        )
+        if source is None:
+            source = models.Source(project_id=project.id, name=source_config.name)
+            self.session.add(source)
+            self.session.flush()
+        changed = (
+            source.type != source_config.type
+            or source.base_url != source_config.base_url
+            or source.trust_level != source_config.trust_level
+            or source.respect_robots_txt != source_config.respect_robots_txt
+            or source.rate_limit_per_minute != source_config.rate_limit_per_minute
+        )
+        if changed:
+            source.type = source_config.type
+            source.base_url = source_config.base_url
+            source.trust_level = source_config.trust_level
+            source.respect_robots_txt = source_config.respect_robots_txt
+            source.rate_limit_per_minute = source_config.rate_limit_per_minute
+            source.updated_at = models.utcnow()
+        return source
+
+    def get_project(self, project_name: str) -> models.Project:
+        project = self.session.scalar(select(models.Project).where(models.Project.name == project_name))
+        if project is None:
+            raise KeyError(f"Project not found: {project_name}")
+        return project
+
+    def get_source(self, project_id: int, source_name: str) -> models.Source:
+        source = self.session.scalar(
+            select(models.Source).where(models.Source.project_id == project_id, models.Source.name == source_name)
+        )
+        if source is None:
+            raise KeyError(f"Source not found: {source_name}")
+        return source
+
+    def reset_project_runtime_data(self, project_id: int) -> dict[str, int]:
+        delete_order = [
+            models.FeedbackLog,
+            models.UserPreference,
+            models.UserProfile,
+            models.CrawlJob,
+            models.ExtractionLog,
+            models.KnowledgeGap,
+            models.OntologyTriple,
+            models.Evidence,
+            models.Relation,
+            models.Claim,
+            models.Attribute,
+            models.Page,
+            models.Entity,
+        ]
+        deleted: dict[str, int] = {}
+        for table_model in delete_order:
+            count = (
+                self.session.query(table_model)
+                .filter(table_model.project_id == project_id)
+                .delete(synchronize_session=False)
+            )
+            deleted[table_model.__tablename__] = int(count or 0)
+        return deleted
+
+    def upsert_page(
+        self,
+        project_id: int,
+        source_id: int,
+        url: str,
+        title: str | None,
+        status_code: int | None,
+        cleaned_text: str,
+        metadata: dict[str, Any] | None = None,
+    ) -> models.Page:
+        page = self.session.scalar(select(models.Page).where(models.Page.project_id == project_id, models.Page.url == url))
+        digest = short_hash(cleaned_text)
+        summary = cleaned_text[:2000]
+        if page is None:
+            page = models.Page(project_id=project_id, source_id=source_id, url=url)
+            self.session.add(page)
+            self.session.flush()
+        page.title = title
+        page.status_code = status_code
+        page.content_hash = digest
+        page.cleaned_text_summary = summary
+        page.metadata_json = metadata or {}
+        page.fetched_at = models.utcnow()
+        return page
+
+    def upsert_entity(
+        self,
+        project_id: int,
+        entity_type: str,
+        name: str,
+        metadata: dict[str, Any] | None = None,
+    ) -> models.Entity:
+        canonical_name = canonicalize(name)
+        entity = self.session.scalar(
+            select(models.Entity).where(
+                models.Entity.project_id == project_id,
+                models.Entity.entity_type == entity_type,
+                models.Entity.canonical_name == canonical_name,
+            )
+        )
+        if entity is None:
+            entity = models.Entity(
+                project_id=project_id,
+                entity_type=entity_type,
+                name=name.strip(),
+                canonical_name=canonical_name,
+                metadata_json=metadata or {},
+            )
+            self.session.add(entity)
+            self.session.flush()
+        else:
+            entity.metadata_json = {**(entity.metadata_json or {}), **(metadata or {})}
+            entity.updated_at = models.utcnow()
+        return entity
+
+    def save_extraction_bundle(
+        self,
+        project_id: int,
+        source: models.Source,
+        page: models.Page,
+        bundle: ExtractionBundle,
+        project_config: ProjectConfig | None = None,
+    ) -> list[models.Claim]:
+        if project_config is not None:
+            validation = validate_extraction_bundle(bundle, project_config)
+            bundle = validation.bundle
+            claim_status = validation.claim_status
+        else:
+            claim_status = "active"
+
+        if claim_status not in {"active", "validated_claim", "candidate_claim", "rule_candidate"}:
+            self._log_extraction(project_id, page, bundle)
+            return []
+
+        entity_index: dict[tuple[str, str], models.Entity] = {}
+        for extracted_entity in bundle.entities:
+            entity = self._save_extracted_entity(project_id, source.id, extracted_entity)
+            normalized_name = normalize_entity_name(extracted_entity.name, extracted_entity.entity_type)
+            entity_index[(extracted_entity.entity_type, canonicalize(normalized_name))] = entity
+
+        claims: list[models.Claim] = []
+        for extracted_claim in bundle.claims:
+            subject = self._entity_for_claim(project_id, extracted_claim.subject_type, extracted_claim.subject_name, entity_index)
+            object_entity = None
+            if extracted_claim.object_name and extracted_claim.object_type:
+                object_entity = self._entity_for_claim(
+                    project_id,
+                    extracted_claim.object_type,
+                    extracted_claim.object_name,
+                    entity_index,
+                )
+            confidence = combine_confidence(extracted_claim.confidence, source.trust_level)
+            claim_metadata = {
+                **extracted_claim.metadata,
+                "source_trust": source.trust_level,
+            }
+            claim_metadata["source_history"] = [
+                {
+                    "source_id": source.id,
+                    "source_name": source.name,
+                    "source_type": source.type,
+                    "source_trust": source.trust_level,
+                    "page_id": page.id,
+                    "page_url": page.url,
+                    "seen_at": models.utcnow().isoformat(),
+                    "confidence": extracted_claim.confidence,
+                }
+            ]
+            conflict = self._find_conflicting_claim(
+                project_id,
+                subject.id,
+                extracted_claim.predicate,
+                object_entity.id if object_entity else None,
+                extracted_claim.object_value,
+            )
+            if conflict is not None:
+                claim_metadata = {
+                    **claim_metadata,
+                    "conflict_status": "conflicting_claim",
+                    "conflicts_with_claim_id": conflict.id,
+                    "review_required": True,
+                    "review_reason": "conflicting validated claim exists for same subject and predicate",
+                }
+            if "confidence_breakdown" in claim_metadata:
+                claim_metadata["confidence_breakdown"] = {
+                    **claim_metadata["confidence_breakdown"],
+                    "source_trust": round(source.trust_level, 4),
+                    "stored_confidence": confidence,
+                }
+            claim_hash = make_claim_hash(
+                project_id=project_id,
+                source_id=source.id,
+                subject_entity_id=subject.id,
+                predicate=extracted_claim.predicate,
+                object_entity_id=object_entity.id if object_entity else None,
+                object_value=extracted_claim.object_value,
+            )
+            claim = self.session.scalar(
+                select(models.Claim).where(
+                    models.Claim.project_id == project_id,
+                    models.Claim.claim_hash == claim_hash,
+                )
+            )
+            if claim is None:
+                claim = models.Claim(
+                    project_id=project_id,
+                    source_id=source.id,
+                    page_id=page.id,
+                    subject_entity_id=subject.id,
+                    predicate=extracted_claim.predicate,
+                    object_entity_id=object_entity.id if object_entity else None,
+                    object_value=extracted_claim.object_value,
+                    value_type="entity" if object_entity else "literal",
+                    claim_hash=claim_hash,
+                    confidence=confidence,
+                    confidence_reason=extracted_claim.confidence_reason,
+                    extraction_method=bundle.extractor_name,
+                    status=claim_status,
+                    metadata_json=claim_metadata,
+                )
+                self.session.add(claim)
+                self.session.flush()
+            else:
+                existing_metadata = dict(claim.metadata_json or {})
+                existing_history = list(existing_metadata.get("source_history") or [])
+                claim.page_id = page.id
+                claim.last_seen_at = models.utcnow()
+                claim.confidence = max(claim.confidence, confidence)
+                claim.confidence_reason = extracted_claim.confidence_reason or claim.confidence_reason
+                if claim_status == "active" or claim.status != "active":
+                    claim.status = claim_status
+                claim.metadata_json = {**existing_metadata, **claim_metadata}
+                claim.metadata_json["source_history"] = merge_source_history(
+                    existing_history,
+                    claim_metadata["source_history"],
+                )
+            if extracted_claim.evidence_text:
+                self.session.add(
+                    models.Evidence(
+                        project_id=project_id,
+                        claim_id=claim.id,
+                        page_id=page.id,
+                        evidence_text=extracted_claim.evidence_text[:1000],
+                        evidence_summary=extracted_claim.evidence_summary,
+                    )
+                )
+            can_merge, merge_reason = should_merge_claim_to_graph(
+                claim,
+                project_config.ontology if project_config is not None else None,
+                project_config.domain if project_config is not None else None,
+            )
+            claim.metadata_json = {
+                **(claim.metadata_json or {}),
+                "graph_merge_status": "merged" if can_merge else "skipped",
+                "graph_merge_reason": merge_reason,
+            }
+            if object_entity and can_merge:
+                self._upsert_relation(project_id, subject.id, extracted_claim.predicate, object_entity.id, confidence)
+            triple = OntologyTripleStore(self.session).upsert_from_claim(claim)
+            claim.metadata_json = {
+                **(claim.metadata_json or {}),
+                "ontology_triple_id": triple.id,
+                "ontology_triple_status": triple.status,
+            }
+            claims.append(claim)
+
+        self._log_extraction(project_id, page, bundle)
+        return claims
+
+    def _save_extracted_entity(
+        self,
+        project_id: int,
+        source_id: int,
+        extracted_entity: ExtractedEntity,
+    ) -> models.Entity:
+        normalized_type = extracted_entity.entity_type
+        normalized_name = normalize_entity_name(extracted_entity.name, normalized_type)
+        entity = self.upsert_entity(
+            project_id,
+            normalized_type,
+            normalized_name,
+            {
+                **extracted_entity.metadata,
+                "raw_name": extracted_entity.name,
+                "normalization": "canonical_entity_resolver",
+            },
+        )
+        for name, value in extracted_entity.attributes.items():
+            attribute = self.session.scalar(
+                select(models.Attribute).where(
+                    models.Attribute.entity_id == entity.id,
+                    models.Attribute.name == name,
+                    models.Attribute.source_id == source_id,
+                )
+            )
+            if attribute is None:
+                attribute = models.Attribute(
+                    project_id=project_id,
+                    entity_id=entity.id,
+                    source_id=source_id,
+                    name=name,
+                    value=value,
+                    confidence=extracted_entity.confidence,
+                )
+                self.session.add(attribute)
+            else:
+                attribute.value = value
+                attribute.confidence = max(attribute.confidence, extracted_entity.confidence)
+                attribute.updated_at = models.utcnow()
+        return entity
+
+    def _entity_for_claim(
+        self,
+        project_id: int,
+        entity_type: str,
+        name: str,
+        entity_index: dict[tuple[str, str], models.Entity],
+    ) -> models.Entity:
+        normalized_name = normalize_entity_name(name, entity_type)
+        key = (entity_type, canonicalize(normalized_name))
+        if key not in entity_index:
+            entity_index[key] = self.upsert_entity(
+                project_id,
+                entity_type,
+                normalized_name,
+                {"raw_name": name, "normalization": "canonical_entity_resolver"},
+            )
+        return entity_index[key]
+
+    def _upsert_relation(
+        self,
+        project_id: int,
+        subject_id: int,
+        predicate: str,
+        object_id: int,
+        confidence: float,
+    ) -> models.Relation:
+        relation = self.session.scalar(
+            select(models.Relation).where(
+                models.Relation.project_id == project_id,
+                models.Relation.subject_entity_id == subject_id,
+                models.Relation.predicate == predicate,
+                models.Relation.object_entity_id == object_id,
+            )
+        )
+        if relation is None:
+            relation = models.Relation(
+                project_id=project_id,
+                subject_entity_id=subject_id,
+                predicate=predicate,
+                object_entity_id=object_id,
+                confidence=confidence,
+            )
+            self.session.add(relation)
+            self.session.flush()
+        else:
+            relation.support_count += 1
+            relation.confidence = max(relation.confidence, confidence)
+            relation.updated_at = models.utcnow()
+        return relation
+
+    def _find_conflicting_claim(
+        self,
+        project_id: int,
+        subject_id: int,
+        predicate: str,
+        object_entity_id: int | None,
+        object_value: Any | None,
+    ) -> models.Claim | None:
+        rows = self.session.scalars(
+            select(models.Claim).where(
+                models.Claim.project_id == project_id,
+                models.Claim.subject_entity_id == subject_id,
+                models.Claim.predicate == predicate,
+                models.Claim.status == "validated_claim",
+            )
+        ).all()
+        for row in rows:
+            if object_entity_id is not None:
+                if row.object_entity_id is not None and row.object_entity_id != object_entity_id:
+                    return row
+            elif row.object_value != object_value:
+                return row
+        return None
+
+    def _log_extraction(self, project_id: int, page: models.Page, bundle: ExtractionBundle) -> None:
+        self.session.add(
+            models.ExtractionLog(
+                project_id=project_id,
+                page_id=page.id,
+                extractor_name=bundle.extractor_name,
+                provider=bundle.provider,
+                input_hash=page.content_hash,
+                raw_output={
+                    **bundle.raw_output,
+                    "candidate_entities": [asdict(entity) for entity in bundle.entities[:50]],
+                    "candidate_claims": [asdict(claim) for claim in bundle.claims[:100]],
+                },
+            )
+        )
+
+
+def _project_config_to_dict(config: ProjectConfig) -> dict[str, Any]:
+    return {
+        "project_name": config.project_name,
+        "domain": config.domain,
+        "target_entities": config.target_entities,
+        "fields": config.fields,
+        "sources": [asdict(source) for source in config.sources],
+        "ontology": config.ontology,
+        "recommendation": config.recommendation,
+        "update_policy": config.update_policy,
+    }
+
+
+def combine_confidence(extraction_confidence: float, source_trust: float) -> float:
+    return round(min(max((extraction_confidence * 0.7) + (source_trust * 0.3), 0.0), 1.0), 4)
+
+
+def merge_source_history(existing: list[dict[str, Any]], new_items: list[dict[str, Any]]) -> list[dict[str, Any]]:
+    merged = [item for item in existing if isinstance(item, dict)]
+    seen = {(item.get("source_id"), item.get("page_id"), item.get("seen_at")) for item in merged}
+    for item in new_items:
+        key = (item.get("source_id"), item.get("page_id"), item.get("seen_at"))
+        if key not in seen:
+            merged.append(item)
+            seen.add(key)
+    return merged[-50:]
+
+
+def make_claim_hash(
+    project_id: int,
+    source_id: int,
+    subject_entity_id: int,
+    predicate: str,
+    object_entity_id: int | None,
+    object_value: Any | None,
+) -> str:
+    payload = {
+        "project_id": project_id,
+        "source_id": source_id,
+        "subject_entity_id": subject_entity_id,
+        "predicate": predicate,
+        "object_entity_id": object_entity_id,
+        "object_value": object_value,
+    }
+    return hashlib.sha256(json.dumps(payload, sort_keys=True, default=str).encode("utf-8")).hexdigest()

ontology_platform/crawler_platform/app/core/database/session.py

@@ -0,0 +1,51 @@
+from __future__ import annotations
+
+from contextlib import contextmanager
+from typing import Iterator
+
+from sqlalchemy import create_engine, event
+from sqlalchemy.orm import Session, sessionmaker
+
+from crawler_platform.app.core.database.models import Base
+
+
+def make_engine(database_url: str = "sqlite:///crawler_platform.db"):
+    connect_args = {"check_same_thread": False, "timeout": 60} if database_url.startswith("sqlite") else {}
+    engine = create_engine(database_url, future=True, connect_args=connect_args)
+    if database_url.startswith("sqlite"):
+        install_sqlite_pragmas(engine)
+    return engine
+
+
+def install_sqlite_pragmas(engine) -> None:
+    @event.listens_for(engine, "connect")
+    def _set_sqlite_pragmas(dbapi_connection, connection_record):
+        cursor = dbapi_connection.cursor()
+        cursor.execute("PRAGMA busy_timeout=60000")
+        cursor.execute("PRAGMA journal_mode=WAL")
+        cursor.execute("PRAGMA synchronous=NORMAL")
+        cursor.close()
+
+
+def init_db(database_url: str = "sqlite:///crawler_platform.db") -> None:
+    engine = make_engine(database_url)
+    Base.metadata.create_all(engine)
+
+
+def make_session_factory(database_url: str = "sqlite:///crawler_platform.db") -> sessionmaker[Session]:
+    engine = make_engine(database_url)
+    return sessionmaker(bind=engine, expire_on_commit=False, autoflush=False, class_=Session, future=True)
+
+
+@contextmanager
+def session_scope(database_url: str = "sqlite:///crawler_platform.db") -> Iterator[Session]:
+    factory = make_session_factory(database_url)
+    session = factory()
+    try:
+        yield session
+        session.commit()
+    except Exception:
+        session.rollback()
+        raise
+    finally:
+        session.close()

ontology_platform/crawler_platform/app/core/extractor/__init__.py

@@ -0,0 +1,2 @@
+"""Extractor provider interfaces and implementations."""
+

ontology_platform/crawler_platform/app/core/extractor/ai_provider.py

@@ -0,0 +1,734 @@
+from __future__ import annotations
+
+import json
+import os
+import re
+from typing import Any
+
+import requests
+
+from crawler_platform.app.config.loader import ProjectConfig
+from crawler_platform.app.core.extractor.base import (
+    AIExtractor,
+    ExtractedClaim,
+    ExtractedEntity,
+    ExtractionPageContext,
+    ExtractionBundle,
+)
+from crawler_platform.app.core.ontology.entity_normalizer import normalize_entity_type
+from crawler_platform.app.core.ontology.mapper import normalize_predicate
+from crawler_platform.app.core.ontology.structured_output import validate_structured_extraction
+from crawler_platform.app.domains.perfume.extractor import PerfumeRuleBasedExtractor
+
+
+class LLMJsonExtractor(AIExtractor):
+    name = "llm_json_extractor"
+
+    def __init__(
+        self,
+        domain: str,
+        provider: str,
+        model: str | None = None,
+        base_url: str | None = None,
+        timeout_seconds: int = 300,
+    ):
+        # Local LM Studio runs on consumer hardware; keep a shorter timeout so
+        # we can quickly fallback instead of stalling a crawl worker for 5+ min.
+        if provider == "lm_studio" and timeout_seconds == 300:
+            timeout_seconds = 120
+        self.domain = domain
+        self.provider = provider
+        self.model = model
+        self.base_url = base_url
+        self.timeout_seconds = timeout_seconds
+
+    def extract(self, page_text: str, project_config: ProjectConfig) -> ExtractionBundle:
+        return self._extract_text(page_text, project_config, context=None)
+
+    def extract_from_context(
+        self,
+        context: ExtractionPageContext,
+        project_config: ProjectConfig,
+    ) -> ExtractionBundle:
+        return self._extract_text(context.clean_text, project_config, context=context)
+
+    def _extract_text(
+        self,
+        page_text: str,
+        project_config: ProjectConfig,
+        context: ExtractionPageContext | None,
+    ) -> ExtractionBundle:
+        errors: list[str] = []
+        for compact_mode in (False, True):
+            mode_name = "compact_retry" if compact_mode else "primary"
+            try:
+                raw = self.complete_json(page_text, project_config, compact=compact_mode, context=context)
+                bundle = self._bundle_from_raw(raw, mode_name)
+                enriched = self._merge_rule_fallback_claims(bundle, page_text, project_config, mode_name)
+                if enriched.entities and enriched.claims:
+                    return self.normalize_to_ontology(enriched, project_config.ontology)
+                if bundle.entities and bundle.claims:
+                    return self.normalize_to_ontology(bundle, project_config.ontology)
+                errors.append(f"{mode_name}: AI returned no usable entities or claims")
+            except Exception as exc:
+                errors.append(f"{mode_name}: {exc}")
+        return self._fallback_bundle(page_text, project_config, " | ".join(errors))
+
+    def extract_entities(self, page_text: str, project_config: ProjectConfig) -> list[ExtractedEntity]:
+        return self.extract(page_text, project_config).entities
+
+    def extract_attributes(
+        self,
+        entity: ExtractedEntity,
+        page_text: str,
+        project_config: ProjectConfig,
+    ) -> dict[str, Any]:
+        return entity.attributes
+
+    def extract_relations(
+        self,
+        entities: list[ExtractedEntity],
+        page_text: str,
+        project_config: ProjectConfig,
+    ) -> list[ExtractedClaim]:
+        return []
+
+    def normalize_to_ontology(self, bundle: ExtractionBundle, ontology: dict[str, Any]) -> ExtractionBundle:
+        for claim in bundle.claims:
+            claim.predicate = normalize_predicate(claim.predicate, ontology)
+        return bundle
+
+    def complete_json(
+        self,
+        page_text: str,
+        project_config: ProjectConfig,
+        compact: bool = False,
+        context: ExtractionPageContext | None = None,
+    ) -> dict[str, Any]:
+        if self.provider == "lm_studio":
+            char_limit = 700 if compact else 1100
+            max_tokens = 260 if compact else 360
+        else:
+            char_limit = 2200 if compact else 4000
+            max_tokens = 400 if compact else 800
+        prompt = build_extraction_prompt(
+            page_text,
+            project_config,
+            char_limit=char_limit,
+            context=context,
+            compact=self.provider == "lm_studio",
+        )
+        if self.provider == "openai":
+            return self._complete_openai_compatible(
+                prompt,
+                "OPENAI_API_KEY",
+                "OPENAI_MODEL",
+                self.base_url,
+                max_tokens=max_tokens,
+            )
+        if self.provider == "lm_studio":
+            return self._complete_openai_compatible(
+                prompt,
+                "LM_STUDIO_API_KEY",
+                "LM_STUDIO_MODEL",
+                normalize_openai_chat_url(self.base_url or "http://localhost:1234/v1"),
+                api_key_optional=True,
+                max_tokens=max_tokens,
+            )
+        if self.provider == "ollama":
+            return self._complete_ollama(prompt, max_tokens=max_tokens)
+        raise ValueError(f"Unsupported AI extractor provider: {self.provider}")
+
+    def _fallback_bundle(self, page_text: str, project_config: ProjectConfig, error: str) -> ExtractionBundle:
+        if project_config.domain == "perfume":
+            bundle = PerfumeRuleBasedExtractor().extract(page_text, project_config)
+        else:
+            from crawler_platform.app.core.extractor.rule_based import GenericRuleBasedExtractor
+
+            bundle = GenericRuleBasedExtractor().extract(page_text, project_config)
+        bundle.extractor_name = f"{self.name}_with_rule_fallback"
+        bundle.provider = self.provider
+        bundle.raw_output = {
+            **bundle.raw_output,
+            "ai_provider": self.provider,
+            "ai_model": self.model,
+            "ai_warning": error,
+            "fallback": "rule_based",
+            "extraction_mode": "fallback",
+        }
+        for entity in bundle.entities:
+            entity.metadata["ai_fallback_reason"] = error
+        for claim in bundle.claims:
+            claim.metadata["ai_fallback_reason"] = error
+            claim.confidence_reason = f"{claim.confidence_reason}; AI fallback: {error}" if claim.confidence_reason else error
+        return bundle
+
+    def _merge_rule_fallback_claims(
+        self,
+        bundle: ExtractionBundle,
+        page_text: str,
+        project_config: ProjectConfig,
+        mode: str,
+    ) -> ExtractionBundle:
+        rule_bundle = self._rule_bundle(page_text, project_config)
+        if not rule_bundle.claims:
+            return bundle
+        merged = ExtractionBundle(
+            entities=dedupe_entities([*bundle.entities, *rule_bundle.entities]),
+            claims=dedupe_claims([*bundle.claims, *rule_bundle.claims]),
+            extractor_name=f"{self.name}_with_rule_claims",
+            provider=self.provider,
+            raw_output={
+                **bundle.raw_output,
+                "extraction_mode": mode,
+                "rule_claim_merge": True,
+                "rule_entity_count": len(rule_bundle.entities),
+                "rule_claim_count": len(rule_bundle.claims),
+            },
+        )
+        for claim in merged.claims:
+            claim.metadata.setdefault("rule_claim_merge", True)
+            claim.confidence_reason = (
+                f"{claim.confidence_reason}; AI entity output enriched with rule claims"
+                if claim.confidence_reason
+                else "AI entity output enriched with rule claims"
+            )
+        return merged
+
+    def _rule_bundle(self, page_text: str, project_config: ProjectConfig) -> ExtractionBundle:
+        if project_config.domain == "perfume":
+            return PerfumeRuleBasedExtractor().extract(page_text, project_config)
+        from crawler_platform.app.core.extractor.rule_based import GenericRuleBasedExtractor
+
+        return GenericRuleBasedExtractor().extract(page_text, project_config)
+
+    def _bundle_from_raw(self, raw: dict[str, Any], mode: str) -> ExtractionBundle:
+        return ExtractionBundle(
+            entities=parse_entities(raw.get("entities", [])),
+            claims=parse_claims(raw.get("claims", [])),
+            extractor_name=self.name,
+            provider=self.provider,
+            raw_output={
+                "provider": self.provider,
+                "model": self.model,
+                "entity_count": len(raw.get("entities", [])),
+                "claim_count": len(raw.get("claims", [])),
+                "extraction_mode": mode,
+            },
+        )
+
+    def _complete_openai_compatible(
+        self,
+        prompt: str,
+        api_key_env: str,
+        model_env: str,
+        endpoint: str | None,
+        api_key_optional: bool = False,
+        max_tokens: int | None = None,
+    ) -> dict[str, Any]:
+        api_key = os.getenv(api_key_env)
+        model = self.model or os.getenv(model_env)
+        if not model and api_key_optional and endpoint:
+            model = first_openai_compatible_model(endpoint, api_key)
+        if not model:
+            raise RuntimeError(f"AI model is required. Set UI model field or {model_env}.")
+        if not api_key and not api_key_optional:
+            raise RuntimeError(f"API key is required. Set {api_key_env}.")
+        url = endpoint or "https://api.openai.com/v1/chat/completions"
+        headers = {"Content-Type": "application/json"}
+        if api_key:
+            headers["Authorization"] = f"Bearer {api_key}"
+        response = requests.post(
+            url,
+            headers=headers,
+            json={
+                "model": model,
+                "messages": [
+                    {"role": "system", "content": "Extract ontology knowledge as strict JSON only."},
+                    {"role": "user", "content": prompt},
+                ],
+                "temperature": 0,
+                "response_format": extraction_response_format(),
+                "max_tokens": max_tokens,
+            },
+            timeout=self.timeout_seconds,
+        )
+        if response.status_code >= 400:
+            raise RuntimeError(f"LLM request failed {response.status_code}: {response.text[:1000]}")
+        data = response.json()
+        content = data["choices"][0]["message"]["content"]
+        return parse_json_content(content, retry=lambda bad: self._repair_json_with_model(bad, endpoint, headers, model))
+
+    def _complete_ollama(self, prompt: str, max_tokens: int | None = None) -> dict[str, Any]:
+        model = self.model or os.getenv("OLLAMA_MODEL")
+        if not model:
+            raise RuntimeError("Ollama model is required. Set UI model field or OLLAMA_MODEL.")
+        url = self.base_url or "http://localhost:11434/api/chat"
+        response = requests.post(
+            url,
+            json={
+                "model": model,
+                "messages": [
+                    {"role": "system", "content": "Extract ontology knowledge as strict JSON only."},
+                    {"role": "user", "content": prompt},
+                ],
+                "stream": False,
+                "format": "json",
+                "options": {"num_predict": max_tokens} if max_tokens else {},
+            },
+            timeout=self.timeout_seconds,
+        )
+        response.raise_for_status()
+        content = response.json()["message"]["content"]
+        return parse_json_content(content)
+
+    def _repair_json_with_model(
+        self,
+        bad_content: str,
+        endpoint: str,
+        headers: dict[str, str],
+        model: str,
+    ) -> dict[str, Any]:
+        response = requests.post(
+            endpoint,
+            headers=headers,
+            json={
+                "model": model,
+                "messages": [
+                    {
+                        "role": "system",
+                        "content": "You repair malformed JSON. Return valid JSON only. No markdown.",
+                    },
+                    {
+                        "role": "user",
+                        "content": (
+                            "Repair this into valid JSON with top-level keys entities and claims. "
+                            "Drop invalid fragments if needed.\n\n"
+                            f"{bad_content[:12000]}"
+                        ),
+                    },
+                ],
+                "temperature": 0,
+                "max_tokens": 300,
+            },
+            timeout=self.timeout_seconds,
+        )
+        if response.status_code >= 400:
+            raise RuntimeError(f"LLM JSON repair failed {response.status_code}: {response.text[:1000]}")
+        return parse_json_content(response.json()["choices"][0]["message"]["content"])
+
+
+def build_extraction_prompt(
+    page_text: str,
+    project_config: ProjectConfig,
+    char_limit: int = 4000,
+    context: ExtractionPageContext | None = None,
+    compact: bool = False,
+) -> str:
+    if compact:
+        return build_compact_extraction_prompt(page_text, project_config, char_limit=char_limit, context=context)
+    if context is not None:
+        prompt_input = json.dumps(context.to_payload(text_limit=char_limit), ensure_ascii=False, indent=2)
+    else:
+        clipped_text = prepare_page_text_for_prompt(page_text, char_limit)
+        prompt_input = json.dumps({"clean_text": clipped_text, "source_zones": []}, ensure_ascii=False, indent=2)
+    ontology = project_config.ontology or {}
+    return f"""
+Project domain: {project_config.domain}
+Target entity types: {project_config.target_entities}
+Fields: {project_config.fields}
+Allowed predicates: {ontology.get("predicates", [])}
+
+Return only minified strict JSON. Do not include markdown, analysis, or prose.
+Use this shape:
+{{
+  "entities": [
+    {{
+      "entity_type": "Perfume",
+      "name": "Product name",
+      "attributes": {{"name": "Product name"}},
+      "confidence": 0.0,
+      "evidence_text": "short evidence from page"
+    }}
+  ],
+  "claims": [
+    {{
+      "subject_name": "Product name",
+      "subject_type": "Perfume",
+      "predicate": "hasTopNote",
+      "object_name": "Bergamot",
+      "object_type": "Note",
+      "object_value": null,
+      "evidence_text": "short evidence from page",
+      "evidence_summary": "why this claim was extracted",
+      "confidence": 0.0,
+      "confidence_reason": "reason",
+      "source_zone": "product_description"
+    }}
+  ]
+}}
+
+Rules:
+- Store information as source claims, not absolute facts.
+- Keep evidence_text short. Do not copy long descriptions.
+- Use only ontology predicates when possible.
+- If object is a simple value like price, put it in object_value and leave object_name/object_type null.
+- If unsure, lower confidence instead of inventing.
+- Do not output placeholders such as value, accord, keyword, unknown, empty string, CAFE24, product count, low price.
+- Extract only facts explicitly supported by clean_text or source_zones.
+- Do not create claims from common UI, menus, footer, navigation, shipping, policy, recommendation, event banner, or platform text.
+- Every claim must include evidence_text copied from the provided clean_text/source_zones.
+- Every claim should include source_zone copied from the matching source_zones item when available.
+- Respect page_type: ProductPage may have product detail claims; CategoryPage may not have detailed product claims; BrandStoryPage may not have product price or note claims; UnknownPage should return no ontology claims.
+- Extract at most 10 entities and 15 claims.
+- For perfume, prioritize name, brand, top/middle/base notes, accords, mood, season, occasion, price, review keywords.
+- Skip navigation, cart, coupon, pagination, login, and policy boilerplate unless it contains product facts.
+
+Input payload:
+{prompt_input}
+""".strip()
+
+
+def build_compact_extraction_prompt(
+    page_text: str,
+    project_config: ProjectConfig,
+    char_limit: int = 1000,
+    context: ExtractionPageContext | None = None,
+) -> str:
+    if context is not None:
+        text = prepare_page_text_for_prompt(context.clean_text, char_limit)
+        title = context.title or ""
+        page_type = context.page_type
+        url = context.final_url or context.url
+    else:
+        text = prepare_page_text_for_prompt(page_text, char_limit)
+        title = ""
+        page_type = "UnknownPage"
+        url = ""
+    predicates = ", ".join((project_config.ontology or {}).get("predicates", [])[:12])
+    return (
+        "Return minified JSON only: {\"entities\":[],\"claims\":[]}.\n"
+        f"Domain perfume. PageType={page_type}. URL={url}. Title={title}\n"
+        "Entity types: Perfume, Brand, Note, Accord, Mood, Season, Occasion, Price.\n"
+        f"Predicates: {predicates}.\n"
+        "Extract only explicit product facts. Max 6 entities, 8 claims. "
+        "Every claim needs short evidence_text from text. "
+        "Use null for missing object_name/object_type/object_value.\n"
+        f"Text:\n{text}"
+    )
+
+
+def prepare_page_text_for_prompt(page_text: str, char_limit: int) -> str:
+    noisy_terms = {
+        "first page",
+        "previous page",
+        "next page",
+        "last page",
+        "add to cart",
+        "cart",
+        "checkout",
+        "coupon",
+        "login",
+        "sign in",
+        "privacy policy",
+        "terms",
+        "review write",
+        "all reviews",
+        "first",
+        "previous",
+        "next",
+        "last",
+    }
+    lines = []
+    for raw in page_text.splitlines():
+        line = raw.strip()
+        if not line:
+            continue
+        lowered = line.lower()
+        if lowered in noisy_terms:
+            continue
+        if line.isdigit():
+            continue
+        if len(line) <= 2:
+            continue
+        lines.append(line)
+
+    compact = "\n".join(lines) if lines else page_text
+    if len(compact) <= char_limit:
+        return compact
+    head_len = int(char_limit * 0.7)
+    tail_len = char_limit - head_len
+    return f"{compact[:head_len]}\n...\n{compact[-tail_len:]}"
+
+
+def extraction_response_format() -> dict[str, Any]:
+    return {
+        "type": "json_schema",
+        "json_schema": {
+            "name": "ontology_extraction",
+            "strict": True,
+            "schema": {
+                "type": "object",
+                "additionalProperties": False,
+                "properties": {
+                    "entities": {
+                        "type": "array",
+                        "items": {
+                            "type": "object",
+                            "additionalProperties": False,
+                            "properties": {
+                                "entity_type": {"type": "string"},
+                                "name": {"type": "string"},
+                                "attributes": {"type": "object"},
+                                "confidence": {"type": "number"},
+                                "evidence_text": {"type": "string"},
+                            },
+                            "required": ["entity_type", "name", "attributes", "confidence", "evidence_text"],
+                        },
+                    },
+                    "claims": {
+                        "type": "array",
+                        "items": {
+                            "type": "object",
+                            "additionalProperties": False,
+                            "properties": {
+                                "subject_name": {"type": "string"},
+                                "subject_type": {"type": "string"},
+                                "predicate": {"type": "string"},
+                                "object_name": {"type": ["string", "null"]},
+                                "object_type": {"type": ["string", "null"]},
+                                "object_value": {},
+                                "evidence_text": {"type": ["string", "null"]},
+                                "evidence_summary": {"type": ["string", "null"]},
+                                "confidence": {"type": "number"},
+                                "confidence_reason": {"type": ["string", "null"]},
+                                "source_zone": {"type": ["string", "null"]},
+                            },
+                            "required": [
+                                "subject_name",
+                                "subject_type",
+                                "predicate",
+                                "object_name",
+                                "object_type",
+                                "object_value",
+                                "evidence_text",
+                                "evidence_summary",
+                                "confidence",
+                                "confidence_reason",
+                                "source_zone",
+                            ],
+                        },
+                    },
+                },
+                "required": ["entities", "claims"],
+            },
+        },
+    }
+
+
+def parse_json_content(content: str, retry=None) -> dict[str, Any]:
+    content = strip_json_noise(content)
+    if not content:
+        if retry:
+            return validate_raw_extraction(retry(content))
+        raise ValueError("LLM returned an empty response")
+    try:
+        return validate_raw_extraction(json.loads(content))
+    except json.JSONDecodeError:
+        match = re.search(r"\{.*\}", content, flags=re.DOTALL)
+        if not match:
+            if retry:
+                return validate_raw_extraction(retry(content))
+            raise
+        try:
+            return validate_raw_extraction(json.loads(match.group(0)))
+        except json.JSONDecodeError:
+            if retry:
+                return validate_raw_extraction(retry(content))
+            repaired = heuristic_repair_json(content)
+            if repaired is not None:
+                return validate_raw_extraction(repaired)
+            raise
+
+
+def strip_json_noise(content: str | None) -> str:
+    if content is None:
+        return ""
+    clean = content.strip()
+    fence = re.search(r"```(?:json)?\s*(.*?)```", clean, flags=re.DOTALL | re.IGNORECASE)
+    if fence:
+        clean = fence.group(1).strip()
+    return clean.strip()
+
+
+def validate_raw_extraction(raw: dict[str, Any]) -> dict[str, Any]:
+    if not isinstance(raw, dict):
+        raise ValueError("LLM JSON root must be an object")
+    entities = raw.get("entities", [])
+    claims = raw.get("claims", [])
+    if not isinstance(entities, list) or not isinstance(claims, list):
+        raise ValueError("LLM JSON must contain list fields: entities and claims")
+    return validate_structured_extraction({"entities": entities, "claims": claims})
+
+
+def heuristic_repair_json(content: str) -> dict[str, Any] | None:
+    """Best-effort extraction for chatty local models that emit broken JSON."""
+
+    entities = []
+    claims = []
+    for block in re.findall(r"\{[^{}]*(?:\{[^{}]*\}[^{}]*)?\}", content, flags=re.DOTALL):
+        try:
+            item = json.loads(block)
+        except json.JSONDecodeError:
+            continue
+        if {"entity_type", "name"}.issubset(item):
+            entities.append(item)
+        if {"subject_name", "subject_type", "predicate"}.issubset(item):
+            claims.append(item)
+    if entities or claims:
+        return {"entities": entities, "claims": claims}
+    return None
+
+
+def normalize_openai_chat_url(base_url: str) -> str:
+    clean = base_url.rstrip("/")
+    if clean.endswith("/chat/completions"):
+        return clean
+    if clean.endswith("/v1"):
+        return f"{clean}/chat/completions"
+    return f"{clean}/v1/chat/completions"
+
+
+def normalize_openai_models_url(base_url: str) -> str:
+    clean = base_url.rstrip("/")
+    if clean.endswith("/chat/completions"):
+        return clean.removesuffix("/chat/completions") + "/models"
+    if clean.endswith("/models"):
+        return clean
+    if clean.endswith("/v1"):
+        return f"{clean}/models"
+    return f"{clean}/v1/models"
+
+
+def first_openai_compatible_model(base_url: str, api_key: str | None = None) -> str | None:
+    models = list_openai_compatible_models(base_url, api_key)
+    return models[0]["id"] if models else None
+
+
+def list_openai_compatible_models(base_url: str, api_key: str | None = None) -> list[dict[str, Any]]:
+    url = normalize_openai_models_url(base_url)
+    headers = {}
+    if api_key:
+        headers["Authorization"] = f"Bearer {api_key}"
+    response = requests.get(url, headers=headers, timeout=10)
+    response.raise_for_status()
+    data = response.json()
+    items = data.get("data", [])
+    return [{"id": item.get("id", ""), "owned_by": item.get("owned_by")} for item in items if item.get("id")]
+
+
+def parse_entities(items: list[dict[str, Any]]) -> list[ExtractedEntity]:
+    entities: list[ExtractedEntity] = []
+    for item in items:
+        name = item.get("name")
+        entity_type = normalize_entity_type(item.get("entity_type") or item.get("type"))
+        if not name or not entity_type:
+            continue
+        entities.append(
+            ExtractedEntity(
+                entity_type=str(entity_type),
+                name=str(name),
+                attributes=dict(item.get("attributes") or {}),
+                evidence_text=item.get("evidence_text"),
+                confidence=float(item.get("confidence") or 0.55),
+                metadata={"ai_extracted": True},
+            )
+        )
+    return entities
+
+
+def dedupe_entities(entities: list[ExtractedEntity]) -> list[ExtractedEntity]:
+    seen: set[tuple[str, str]] = set()
+    result: list[ExtractedEntity] = []
+    for entity in entities:
+        key = (entity.entity_type.strip().lower(), entity.name.strip().lower())
+        if key in seen:
+            continue
+        seen.add(key)
+        result.append(entity)
+    return result
+
+
+def dedupe_claims(claims: list[ExtractedClaim]) -> list[ExtractedClaim]:
+    seen: set[tuple[str, str, str, str]] = set()
+    result: list[ExtractedClaim] = []
+    for claim in claims:
+        object_key = claim.object_name or json.dumps(claim.object_value, ensure_ascii=False, sort_keys=True, default=str)
+        key = (
+            claim.subject_name.strip().lower(),
+            claim.subject_type.strip().lower(),
+            claim.predicate.strip(),
+            str(object_key).strip().lower(),
+        )
+        if key in seen:
+            continue
+        seen.add(key)
+        result.append(claim)
+    return result
+
+
+def parse_claims(items: list[dict[str, Any]]) -> list[ExtractedClaim]:
+    claims: list[ExtractedClaim] = []
+    for item in items:
+        subject_name = item.get("subject_name")
+        subject_type = normalize_entity_type(item.get("subject_type"))
+        predicate = item.get("predicate")
+        if not subject_name or not subject_type or not predicate:
+            continue
+        object_name = item.get("object_name")
+        object_type = normalize_entity_type(item.get("object_type")) if item.get("object_type") else None
+        object_value = item.get("object_value")
+        if not object_name and isinstance(object_value, dict) and len(object_value) == 1:
+            object_name = next(iter(object_value.keys()))
+            object_value = None
+        if object_name and not object_type:
+            object_type = infer_object_type(str(predicate))
+
+        object_names = split_object_names(object_name) if object_name and object_type else [object_name]
+        for resolved_object_name in object_names:
+            claims.append(
+                ExtractedClaim(
+                    subject_name=str(subject_name),
+                    subject_type=str(subject_type),
+                    predicate=str(predicate),
+                    object_name=resolved_object_name,
+                    object_type=object_type if resolved_object_name else None,
+                    object_value=None if resolved_object_name else object_value,
+                    evidence_text=item.get("evidence_text"),
+                    evidence_summary=item.get("evidence_summary"),
+                    confidence=float(item.get("confidence") or 0.55),
+                    confidence_reason=item.get("confidence_reason") or "AI extractor output",
+                    metadata={"ai_extracted": True, "source_zone_hint": item.get("source_zone")},
+                )
+            )
+    return claims
+
+
+def infer_object_type(predicate: str) -> str | None:
+    return {
+        "hasBrand": "Brand",
+        "hasTopNote": "Note",
+        "hasMiddleNote": "Note",
+        "hasBaseNote": "Note",
+        "hasAccord": "Accord",
+        "evokesMood": "Mood",
+        "suitableForSeason": "Season",
+        "suitableForOccasion": "Occasion",
+        "hasReviewKeyword": "Review",
+    }.get(predicate)
+
+
+def split_object_names(value: Any) -> list[str]:
+    text = str(value)
+    parts = re.split(r"[,/|·ㆍ]+|\band\b| 및 | 그리고 ", text, flags=re.IGNORECASE)
+    return [part.strip() for part in parts if part.strip()]

ontology_platform/crawler_platform/app/core/extractor/base.py

@@ -0,0 +1,154 @@
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from dataclasses import dataclass, field
+from typing import Any
+
+from crawler_platform.app.config.loader import ProjectConfig
+
+
+@dataclass(slots=True)
+class ExtractedEntity:
+    entity_type: str
+    name: str
+    attributes: dict[str, Any] = field(default_factory=dict)
+    evidence_text: str | None = None
+    confidence: float = 0.5
+    metadata: dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass(slots=True)
+class ExtractedClaim:
+    subject_name: str
+    subject_type: str
+    predicate: str
+    object_name: str | None = None
+    object_type: str | None = None
+    object_value: Any | None = None
+    evidence_text: str | None = None
+    evidence_summary: str | None = None
+    confidence: float = 0.5
+    confidence_reason: str | None = None
+    metadata: dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass(slots=True)
+class ExtractionBundle:
+    entities: list[ExtractedEntity] = field(default_factory=list)
+    claims: list[ExtractedClaim] = field(default_factory=list)
+    extractor_name: str = "unknown"
+    provider: str = "unknown"
+    raw_output: dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass(slots=True)
+class ExtractionPageContext:
+    url: str
+    final_url: str | None
+    title: str | None
+    page_type: str
+    clean_text: str
+    raw_text: str = ""
+    main_content: str = ""
+    clean_markdown: str = ""
+    source_zones: list[dict[str, Any]] = field(default_factory=list)
+    crawl_status: str = "success"
+    extraction_status: str = "success"
+    warnings: list[str] = field(default_factory=list)
+    metadata: dict[str, Any] = field(default_factory=dict)
+
+    def to_payload(self, text_limit: int | None = None, zone_text_limit: int = 700) -> dict[str, Any]:
+        clean_text = self.clean_text
+        if text_limit is not None and len(clean_text) > text_limit:
+            clean_text = clean_text[:text_limit]
+        zones = []
+        for zone in self.source_zones:
+            zone_text = str(zone.get("text") or "")
+            if len(zone_text) > zone_text_limit:
+                zone_text = zone_text[:zone_text_limit]
+            zones.append(
+                {
+                    "zone_type": zone.get("zone_type"),
+                    "selector": zone.get("selector"),
+                    "text": zone_text,
+                    "confidence": zone.get("confidence"),
+                    "claim_allowed": zone.get("claim_allowed"),
+                }
+            )
+        return {
+            "url": self.url,
+            "final_url": self.final_url,
+            "title": self.title,
+            "page_type": self.page_type,
+            "crawl_status": self.crawl_status,
+            "extraction_status": self.extraction_status,
+            "clean_text": clean_text,
+            "source_zones": zones,
+            "warnings": self.warnings,
+        }
+
+
+class Extractor(ABC):
+    name = "base"
+    provider = "base"
+
+    @abstractmethod
+    def extract_entities(self, page_text: str, project_config: ProjectConfig) -> list[ExtractedEntity]:
+        raise NotImplementedError
+
+    @abstractmethod
+    def extract_attributes(
+        self,
+        entity: ExtractedEntity,
+        page_text: str,
+        project_config: ProjectConfig,
+    ) -> dict[str, Any]:
+        raise NotImplementedError
+
+    @abstractmethod
+    def extract_relations(
+        self,
+        entities: list[ExtractedEntity],
+        page_text: str,
+        project_config: ProjectConfig,
+    ) -> list[ExtractedClaim]:
+        raise NotImplementedError
+
+    @abstractmethod
+    def normalize_to_ontology(self, bundle: ExtractionBundle, ontology: dict[str, Any]) -> ExtractionBundle:
+        raise NotImplementedError
+
+    def extract(self, page_text: str, project_config: ProjectConfig) -> ExtractionBundle:
+        entities = self.extract_entities(page_text, project_config)
+        for entity in entities:
+            entity.attributes.update(self.extract_attributes(entity, page_text, project_config))
+        claims = self.extract_relations(entities, page_text, project_config)
+        bundle = ExtractionBundle(
+            entities=entities,
+            claims=claims,
+            extractor_name=self.name,
+            provider=self.provider,
+            raw_output={"entity_count": len(entities), "claim_count": len(claims)},
+        )
+        return self.normalize_to_ontology(bundle, project_config.ontology)
+
+    def extract_from_context(
+        self,
+        context: ExtractionPageContext,
+        project_config: ProjectConfig,
+    ) -> ExtractionBundle:
+        return self.extract(context.clean_text, project_config)
+
+
+class AIExtractor(Extractor):
+    """Provider-neutral AI extractor contract.
+
+    OpenAI, local LLM, Ollama, and LM Studio adapters can subclass this and
+    implement ``complete_json`` while keeping the rest of the pipeline stable.
+    """
+
+    provider = "ai"
+
+    @abstractmethod
+    def complete_json(self, page_text: str, project_config: ProjectConfig) -> dict[str, Any]:
+        raise NotImplementedError

ontology_platform/crawler_platform/app/core/extractor/factory.py

@@ -0,0 +1,19 @@
+from __future__ import annotations
+
+from crawler_platform.app.core.extractor.ai_provider import LLMJsonExtractor
+from crawler_platform.app.core.extractor.base import Extractor
+from crawler_platform.app.core.extractor.rule_based import GenericRuleBasedExtractor
+from crawler_platform.app.domains.perfume.extractor import PerfumeRuleBasedExtractor
+
+
+def extractor_for_domain(
+    domain: str,
+    provider: str = "rule_based",
+    model: str | None = None,
+    base_url: str | None = None,
+) -> Extractor:
+    if provider in {"openai", "ollama", "lm_studio"}:
+        return LLMJsonExtractor(domain=domain, provider=provider, model=model, base_url=base_url)
+    if domain == "perfume":
+        return PerfumeRuleBasedExtractor()
+    return GenericRuleBasedExtractor()

ontology_platform/crawler_platform/app/core/extractor/rule_based.py

@@ -0,0 +1,85 @@
+from __future__ import annotations
+
+import re
+
+from crawler_platform.app.config.loader import ProjectConfig
+from crawler_platform.app.core.extractor.base import ExtractedClaim, ExtractedEntity, ExtractionBundle, Extractor
+from crawler_platform.app.core.ontology.mapper import normalize_predicate
+
+
+class GenericRuleBasedExtractor(Extractor):
+    name = "generic_rule_based"
+    provider = "rule_based"
+
+    def extract_entities(self, page_text: str, project_config: ProjectConfig) -> list[ExtractedEntity]:
+        name = first_non_empty_line(page_text) or "Unknown Product"
+        primary_type = project_config.target_entities[0] if project_config.target_entities else "Product"
+        return [ExtractedEntity(entity_type=primary_type, name=name, confidence=0.45)]
+
+    def extract_attributes(
+        self,
+        entity: ExtractedEntity,
+        page_text: str,
+        project_config: ProjectConfig,
+    ) -> dict[str, object]:
+        attrs: dict[str, object] = {"name": entity.name}
+        price = find_price(page_text)
+        if price:
+            attrs["price"] = price
+        return attrs
+
+    def extract_relations(
+        self,
+        entities: list[ExtractedEntity],
+        page_text: str,
+        project_config: ProjectConfig,
+    ) -> list[ExtractedClaim]:
+        if not entities:
+            return []
+        entity = entities[0]
+        claims: list[ExtractedClaim] = []
+        price = find_price(page_text)
+        if price:
+            claims.append(
+                ExtractedClaim(
+                    subject_name=entity.name,
+                    subject_type=entity.entity_type,
+                    predicate="hasPrice",
+                    object_value=price,
+                    evidence_text=price["evidence"],
+                    confidence=0.6,
+                    confidence_reason="price pattern matched",
+                )
+            )
+        return claims
+
+    def normalize_to_ontology(self, bundle: ExtractionBundle, ontology: dict[str, object]) -> ExtractionBundle:
+        for claim in bundle.claims:
+            claim.predicate = normalize_predicate(claim.predicate, ontology)
+        return bundle
+
+
+def first_non_empty_line(text: str) -> str | None:
+    for line in text.splitlines():
+        clean = line.strip()
+        if clean:
+            return clean[:240]
+    return None
+
+
+def find_price(text: str) -> dict[str, object] | None:
+    patterns = [
+        r"(?P<currency>[$€£])\s?(?P<amount>\d+(?:[,.]\d{2})?)",
+        r"(?P<amount>\d{1,3}(?:,\d{3})*)\s?(?P<currency>원|KRW|USD)",
+    ]
+    for pattern in patterns:
+        match = re.search(pattern, text, flags=re.IGNORECASE)
+        if match:
+            amount = match.group("amount").replace(",", "")
+            return {
+                "amount": float(amount),
+                "currency": match.group("currency"),
+                "evidence": match.group(0),
+            }
+    return None
+

ontology_platform/crawler_platform/app/core/extractor/validation.py

@@ -0,0 +1,368 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any
+
+from crawler_platform.app.config.loader import ProjectConfig
+from crawler_platform.app.core.crawler.content_zone import is_claim_allowed_zone
+from crawler_platform.app.core.extractor.base import (
+    ExtractedClaim,
+    ExtractedEntity,
+    ExtractionPageContext,
+    ExtractionBundle,
+)
+from crawler_platform.app.core.ontology.entity_normalizer import (
+    canonical_entity_key,
+    normalize_entity_name,
+    normalize_entity_type as normalize_schema_entity_type,
+)
+from crawler_platform.app.core.ontology.mapper import is_allowed_predicate, normalize_predicate
+from crawler_platform.app.core.ontology.relation_schema import (
+    confidence_breakdown,
+    minimum_confidence,
+    relation_schema_compatible,
+)
+
+
+COMMON_PAGE_ENTITY_TYPES = {
+    "Entity",
+    "Concept",
+    "Document",
+    "Source",
+    "Organization",
+    "Person",
+    "Place",
+    "Event",
+    "Page",
+    "ProductPage",
+    "CommunityPage",
+    "BrandStoryPage",
+    "ListingPage",
+    "PromotionPage",
+    "ReviewPage",
+}
+
+GENERIC_PLACEHOLDERS = {
+    "",
+    "-",
+    "--",
+    "n/a",
+    "na",
+    "none",
+    "null",
+    "unknown",
+    "unknown product",
+    "value",
+    "name",
+    "keyword",
+    "accord",
+    "green",
+    "product",
+    "item",
+    "sample",
+    "test",
+    "cafe24",
+    "powered by cafe24",
+    "hosting by cafe24",
+    "low price",
+    "high price",
+    "product count",
+    "\uc0c1\ud488\uc218",
+    "\ub0ae\uc740\uac00\uaca9",
+}
+
+
+@dataclass(slots=True)
+class BundleValidationResult:
+    bundle: ExtractionBundle
+    rejected_claims: list[dict[str, Any]] = field(default_factory=list)
+    rejected_entities: list[dict[str, Any]] = field(default_factory=list)
+    claim_status: str = "active"
+
+
+def attach_page_context(bundle: ExtractionBundle, context: ExtractionPageContext) -> ExtractionBundle:
+    context_payload = context.to_payload(text_limit=2000, zone_text_limit=500)
+    bundle.raw_output = {
+        **bundle.raw_output,
+        "page_context": {
+            **context_payload,
+            "raw_text_length": len(context.raw_text),
+            "main_content_length": len(context.main_content),
+            "clean_text_length": len(context.clean_text),
+            "clean_markdown_length": len(context.clean_markdown),
+        },
+    }
+    for entity in bundle.entities:
+        entity.metadata = {
+            **entity.metadata,
+            "page_type": context.page_type,
+            "crawl_status": context.crawl_status,
+            "extraction_status": context.extraction_status,
+        }
+    for claim in bundle.claims:
+        zone = find_evidence_zone(claim.evidence_text, context)
+        zone_type = zone.get("zone_type") if zone else claim.metadata.get("source_zone_hint")
+        claim.metadata = {
+            **claim.metadata,
+            "page_type": context.page_type,
+            "crawl_status": context.crawl_status,
+            "extraction_status": context.extraction_status,
+            "source_zone": zone_type,
+            "source_selector": zone.get("selector") if zone else None,
+            "source_zone_allowed": is_claim_allowed_zone(str(zone_type)) if zone_type else False,
+            "evidence_found": bool(zone),
+        }
+    return bundle
+
+
+def validate_extraction_bundle(bundle: ExtractionBundle, config: ProjectConfig) -> BundleValidationResult:
+    claim_status = claim_status_for_bundle(bundle)
+    page_context = dict(bundle.raw_output.get("page_context") or {})
+    allowed_types = allowed_entity_types(config)
+    entities = []
+    rejected_entities = []
+    for entity in bundle.entities:
+        normalized_type = normalize_entity_type(entity.entity_type, config.domain)
+        entity.entity_type = normalized_type
+        entity.name = normalize_entity_name(entity.name, normalized_type, config.domain)
+        reason = invalid_entity_reason(entity, allowed_types)
+        if reason:
+            rejected_entities.append({"entity": entity.name, "entity_type": entity.entity_type, "reason": reason})
+            continue
+        entities.append(entity)
+
+    claims = []
+    rejected_claims = []
+    for claim in bundle.claims:
+        claim.predicate = normalize_predicate(claim.predicate, config.ontology)
+        claim.subject_type = normalize_entity_type(claim.subject_type, config.domain)
+        claim.subject_name = normalize_entity_name(claim.subject_name, claim.subject_type, config.domain)
+        if claim.object_type:
+            claim.object_type = normalize_entity_type(claim.object_type, config.domain)
+        if claim.object_name is not None:
+            claim.object_name = normalize_entity_name(claim.object_name, claim.object_type, config.domain)
+        claim.metadata = {
+            **claim.metadata,
+            "schema_validated": True,
+        }
+        reason = invalid_claim_reason(claim, config, allowed_types, page_context)
+        if reason:
+            rejected_claims.append(
+                {
+                    "subject": claim.subject_name,
+                    "predicate": claim.predicate,
+                    "object": claim.object_name if claim.object_name is not None else claim.object_value,
+                    "reason": reason,
+                    "page_type": claim.metadata.get("page_type") or page_context.get("page_type"),
+                    "source_zone": claim.metadata.get("source_zone"),
+                }
+            )
+            continue
+        claim.metadata = {
+            **claim.metadata,
+            "claim_kind": "rule_candidate" if claim_status == "rule_candidate" else "ai_claim",
+            "validation_status": claim_status,
+        }
+        breakdown = confidence_breakdown(
+            llm_confidence=claim.confidence,
+            evidence_found=bool(claim.metadata.get("evidence_found")),
+            ontology_compatible=True,
+            source_zone_allowed=bool(claim.metadata.get("source_zone_allowed")),
+        )
+        claim.metadata["confidence_breakdown"] = breakdown
+        claim.confidence = breakdown["final_confidence"]
+        claims.append(claim)
+
+    bundle.entities = entities
+    bundle.claims = claims
+    bundle.raw_output = {
+        **bundle.raw_output,
+        "validation": {
+            "claim_status": claim_status,
+            "accepted_claim_count": len(claims),
+            "rejected_claim_count": len(rejected_claims),
+            "rejected_entity_count": len(rejected_entities),
+            "rejected_claims": rejected_claims[:25],
+            "rejected_entities": rejected_entities[:25],
+        },
+    }
+    return BundleValidationResult(
+        bundle=bundle,
+        rejected_claims=rejected_claims,
+        rejected_entities=rejected_entities,
+        claim_status=claim_status,
+    )
+
+
+def claim_status_for_bundle(bundle: ExtractionBundle) -> str:
+    mode = str(bundle.raw_output.get("extraction_mode", "")).lower()
+    fallback = str(bundle.raw_output.get("fallback", "")).lower()
+    if bundle.provider == "rule_based" or "rule_fallback" in bundle.extractor_name or mode == "fallback" or fallback:
+        return "rule_candidate"
+    page_context = dict(bundle.raw_output.get("page_context") or {})
+    page_type = str(page_context.get("page_type") or "")
+    if page_type in {"UnknownPage", "SearchPage", "CategoryPage", "BoardPage"}:
+        return "candidate_claim"
+    return "validated_claim"
+
+
+def allowed_entity_types(config: ProjectConfig) -> set[str]:
+    configured = set(config.target_entities)
+    ontology_types = set((config.ontology or {}).get("entity_types", []))
+    return configured | ontology_types | COMMON_PAGE_ENTITY_TYPES
+
+
+def normalize_entity_type(value: str, domain: str | None = None) -> str:
+    schema_type = normalize_schema_entity_type(value, domain)
+    if schema_type:
+        return schema_type
+    aliases = {
+        "ProductDetailPage": "ProductPage",
+        "BoardPage": "CommunityPage",
+        "BrandPage": "BrandStoryPage",
+    }
+    clean = str(value or "").strip()
+    return aliases.get(clean, clean)
+
+
+def invalid_entity_reason(entity: ExtractedEntity, allowed_types: set[str]) -> str | None:
+    if entity.entity_type not in allowed_types:
+        return "entity type is outside ontology"
+    if not is_meaningful_text(entity.name):
+        return "entity name is empty, generic, or boilerplate"
+    return None
+
+
+def invalid_claim_reason(
+    claim: ExtractedClaim,
+    config: ProjectConfig,
+    allowed_types: set[str],
+    page_context: dict[str, Any] | None = None,
+) -> str | None:
+    page_context = page_context or {}
+    if not is_allowed_predicate(claim.predicate, config.ontology):
+        return "predicate is outside ontology"
+    crawl_status = str(claim.metadata.get("crawl_status") or page_context.get("crawl_status") or "success")
+    extraction_status = str(
+        claim.metadata.get("extraction_status") or page_context.get("extraction_status") or "success"
+    )
+    if crawl_status != "success":
+        return f"crawl status is not analyzable: {crawl_status}"
+    if extraction_status not in {"success", "partial"}:
+        return f"main content extraction failed: {extraction_status}"
+    page_type = str(claim.metadata.get("page_type") or page_context.get("page_type") or "")
+    source_zone = claim.metadata.get("source_zone")
+    if page_context and not source_zone:
+        return "claim has no source zone"
+    if source_zone and not is_claim_allowed_zone(str(source_zone)):
+        return f"source zone is not claim-allowed: {source_zone}"
+    if page_context and not claim.metadata.get("evidence_found"):
+        return "evidence was not found in clean source zones"
+    if not claim.evidence_text:
+        return "claim has no evidence"
+    if claim.subject_type not in allowed_types:
+        return "subject type is outside ontology"
+    if not is_meaningful_text(claim.subject_name):
+        return "subject is empty, generic, or boilerplate"
+    if claim.object_name is not None:
+        if claim.object_type not in allowed_types:
+            return "object type is outside ontology"
+        if not is_meaningful_text(claim.object_name):
+            return "object is empty, generic, or boilerplate"
+        if canonical_entity_key(claim.subject_name, claim.subject_type) == canonical_entity_key(
+            claim.object_name, claim.object_type
+        ):
+            return "subject and object resolve to the same entity"
+    elif not is_meaningful_value(claim.object_value):
+        return "literal object is empty or meaningless"
+    if claim.evidence_text and not is_meaningful_text(claim.evidence_text, allow_short=True):
+        return "evidence is boilerplate"
+    schema_reason = relation_schema_compatible(
+        predicate=claim.predicate,
+        subject_type=claim.subject_type,
+        object_type=claim.object_type,
+        has_literal_value=claim.object_value is not None,
+        page_type=page_type,
+        source_zone=str(source_zone) if source_zone else None,
+        ontology=config.ontology,
+        domain=config.domain,
+    )
+    if schema_reason:
+        return schema_reason
+    if claim.confidence < minimum_confidence(claim.predicate, config.ontology, config.domain):
+        return (
+            f"confidence {claim.confidence:.2f} is below threshold "
+            f"{minimum_confidence(claim.predicate, config.ontology, config.domain):.2f}"
+        )
+    return None
+
+
+def is_meaningful_value(value: Any) -> bool:
+    if value is None:
+        return False
+    if isinstance(value, str):
+        return is_meaningful_text(value)
+    if isinstance(value, dict):
+        if not value:
+            return False
+        if "amount" in value:
+            try:
+                return float(value["amount"]) > 0
+            except (TypeError, ValueError):
+                return False
+        return any(is_meaningful_value(item) for item in value.values())
+    if isinstance(value, list):
+        return any(is_meaningful_value(item) for item in value)
+    return True
+
+
+def is_meaningful_text(value: str | None, allow_short: bool = False) -> bool:
+    if value is None:
+        return False
+    clean = " ".join(str(value).strip().split())
+    lowered = clean.lower()
+    if lowered in GENERIC_PLACEHOLDERS:
+        return False
+    if clean.startswith("{#") or clean.endswith("}"):
+        return False
+    if not allow_short and len(clean) < 3:
+        return False
+    max_length = 1000 if allow_short else 240
+    if len(clean) > max_length:
+        return False
+    return True
+
+
+def find_evidence_zone(evidence_text: str | None, context: ExtractionPageContext) -> dict[str, Any] | None:
+    if not evidence_text:
+        return None
+    evidence = normalize_for_match(evidence_text)
+    if not evidence:
+        return None
+    best_zone: dict[str, Any] | None = None
+    best_score = 0
+    for zone in context.source_zones:
+        zone_text = normalize_for_match(str(zone.get("text") or ""))
+        if not zone_text:
+            continue
+        if evidence in zone_text:
+            score = len(evidence)
+        else:
+            evidence_terms = {term for term in evidence.split() if len(term) > 2}
+            if not evidence_terms:
+                continue
+            zone_terms = set(zone_text.split())
+            score = len(evidence_terms & zone_terms)
+        if score > best_score:
+            best_zone = zone
+            best_score = score
+    if best_zone and best_score > 0:
+        return best_zone
+    clean_text = normalize_for_match(context.clean_text)
+    if evidence in clean_text:
+        return {"zone_type": "unknown", "selector": None, "text": context.clean_text, "confidence": 0.25}
+    return None
+
+
+def normalize_for_match(value: str) -> str:
+    return " ".join(str(value).lower().split())

ontology_platform/crawler_platform/app/core/ontology/__init__.py

@@ -0,0 +1,2 @@
+"""Ontology definitions and mapping helpers."""
+

ontology_platform/crawler_platform/app/core/ontology/definitions.py

@@ -0,0 +1,141 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+
+@dataclass(slots=True)
+class Ontology:
+    domain: str
+    entity_types: list[str]
+    predicates: list[str]
+    attributes: list[str]
+    aliases: dict[str, str] = field(default_factory=dict)
+
+
+COMMON_ONTOLOGY = Ontology(
+    domain="common",
+    entity_types=[
+        "Source",
+        "Page",
+        "Entity",
+        "Attribute",
+        "Relation",
+        "Claim",
+        "Evidence",
+        "Extraction",
+        "Confidence",
+        "UpdatePolicy",
+    ],
+    predicates=[
+        "mentions",
+        "hasAttribute",
+        "relatedTo",
+        "sameAs",
+        "soldBy",
+        "hasPrice",
+    ],
+    attributes=["name", "source_url", "updated_at", "confidence"],
+)
+
+
+DOMAIN_ONTOLOGIES: dict[str, Ontology] = {
+    "perfume": Ontology(
+        domain="perfume",
+        entity_types=[
+            "Perfume",
+            "Product",
+            "Brand",
+            "Event",
+            "Article",
+            "Promotion",
+            "Category",
+            "Notice",
+            "Page",
+            "Note",
+            "Accord",
+            "Mood",
+            "Season",
+            "Occasion",
+            "Review",
+            "Price",
+            "ProductPage",
+            "CommunityPage",
+            "BrandStoryPage",
+            "ListingPage",
+            "PromotionPage",
+            "ReviewPage",
+        ],
+        predicates=[
+            "hasBrand",
+            "hasTopNote",
+            "hasMiddleNote",
+            "hasBaseNote",
+            "hasAccord",
+            "evokesMood",
+            "suitableForSeason",
+            "suitableForOccasion",
+            "similarTo",
+            "soldBy",
+            "hasPrice",
+            "hasReviewKeyword",
+        ],
+        attributes=[
+            "name",
+            "brand",
+            "gender_bias",
+            "longevity",
+            "sillage",
+            "price_range",
+            "popularity_score",
+            "review_count",
+            "source_url",
+            "updated_at",
+        ],
+        aliases={
+            "top_notes": "hasTopNote",
+            "middle_notes": "hasMiddleNote",
+            "heart_notes": "hasMiddleNote",
+            "base_notes": "hasBaseNote",
+            "accords": "hasAccord",
+            "mood_tags": "evokesMood",
+            "season_tags": "suitableForSeason",
+            "occasion_tags": "suitableForOccasion",
+            "review_keywords": "hasReviewKeyword",
+            "price": "hasPrice",
+        },
+    ),
+    "tea": Ontology(
+        domain="tea",
+        entity_types=["Tea", "Ingredient", "Flavor", "Effect", "CaffeineLevel", "MoodState"],
+        predicates=["hasIngredient", "hasFlavor", "hasEffect", "suitableForCondition"],
+        attributes=["name", "origin", "caffeine_level", "price_range"],
+    ),
+    "coffee": Ontology(
+        domain="coffee",
+        entity_types=["CoffeeBean", "Origin", "RoastLevel", "FlavorNote", "BrewMethod"],
+        predicates=["hasOrigin", "hasRoastLevel", "hasFlavorNote", "recommendedForBrewMethod"],
+        attributes=["name", "origin", "roast_level", "process", "price_range"],
+    ),
+    "candle": Ontology(
+        domain="candle",
+        entity_types=["ScentProduct", "ScentNote", "SpaceType", "Mood", "Season"],
+        predicates=["suitableForSpace", "evokesMood", "hasScentNote"],
+        attributes=["name", "burn_time", "volume", "price_range"],
+    ),
+    "supplement": Ontology(
+        domain="supplement",
+        entity_types=["Supplement", "Ingredient", "HealthGoal", "Symptom", "Dosage"],
+        predicates=["hasIngredient", "supportsGoal", "recommendedForCondition"],
+        attributes=["name", "dosage", "warnings", "price_range"],
+    ),
+    "gift": Ontology(
+        domain="gift",
+        entity_types=["GiftProduct", "RecipientType", "Relationship", "Occasion", "PersonalityTag"],
+        predicates=["suitableForRecipient", "suitableForOccasion", "matchesPersonality"],
+        attributes=["name", "price_range", "availability", "gift_wrap_available"],
+    ),
+}
+
+
+def ontology_for_domain(domain: str) -> Ontology:
+    return DOMAIN_ONTOLOGIES.get(domain, COMMON_ONTOLOGY)

ontology_platform/crawler_platform/app/core/ontology/entity_normalizer.py

@@ -0,0 +1,76 @@
+from __future__ import annotations
+
+import re
+import unicodedata
+from typing import Any
+
+
+GENERIC_TYPE_ALIASES = {
+    "entity": "Entity",
+    "concept": "Concept",
+    "organization": "Organization",
+    "organisation": "Organization",
+    "org": "Organization",
+    "company": "Organization",
+    "person": "Person",
+    "people": "Person",
+    "place": "Place",
+    "location": "Place",
+    "event": "Event",
+    "document": "Document",
+    "article": "Document",
+    "paper": "Document",
+    "source": "Source",
+    "page": "Page",
+}
+
+
+def normalize_entity_type(value: Any, domain: str | None = None) -> str | None:
+    if not value:
+        return None
+    if domain:
+        from crawler_platform.app.adapters.registry import adapter_for_domain
+
+        adapter = adapter_for_domain(domain)
+        if adapter is not None:
+            adapter_type = adapter.normalize_entity_type(str(value))
+            if adapter_type:
+                return adapter_type
+    clean = compact_key(value)
+    return GENERIC_TYPE_ALIASES.get(clean, str(value).strip())
+
+
+def normalize_entity_name(value: Any, entity_type: str | None = None, domain: str | None = None) -> str:
+    text = normalize_text(value)
+    if domain:
+        from crawler_platform.app.adapters.registry import adapter_for_domain
+
+        adapter = adapter_for_domain(domain)
+        if adapter is not None:
+            adapter_name = adapter.normalize_entity_name(text, entity_type)
+            if adapter_name:
+                return adapter_name
+    return text
+
+
+def canonical_entity_key(value: Any, entity_type: str | None = None, domain: str | None = None) -> str:
+    normalized = normalize_entity_name(value, entity_type, domain)
+    return canonical_key(normalized)
+
+
+def normalize_text(value: Any) -> str:
+    text = unicodedata.normalize("NFKC", str(value or "")).strip()
+    text = re.sub(r"\s+", " ", text)
+    text = text.strip(" \t\r\n|/,:;")
+    return text
+
+
+def canonical_key(value: Any) -> str:
+    text = normalize_text(value).lower()
+    text = re.sub(r"[\[\]{}()<>\"'`]", "", text)
+    text = re.sub(r"\s+", " ", text)
+    return text.strip()
+
+
+def compact_key(value: Any) -> str:
+    return re.sub(r"[^0-9a-zA-Z]+", "", str(value or "")).strip().lower()

ontology_platform/crawler_platform/app/core/ontology/gap_detector.py

@@ -0,0 +1,209 @@
+from __future__ import annotations
+
+from typing import Any
+
+from sqlalchemy import func, select
+from sqlalchemy.orm import Session
+
+from crawler_platform.app.core.database import models
+from crawler_platform.app.core.ontology.registry import stable_hash
+
+
+class KnowledgeGapDetector:
+    """Detects ontology coverage gaps that can drive future research loops."""
+
+    def __init__(self, session: Session):
+        self.session = session
+
+    def detect(self, project_id: int, limit: int = 100) -> list[models.KnowledgeGap]:
+        gaps: list[models.KnowledgeGap] = []
+        gaps.extend(self._entity_type_coverage_gaps(project_id))
+        gaps.extend(self._relation_type_usage_gaps(project_id))
+        gaps.extend(self._underconnected_entity_gaps(project_id))
+        gaps.extend(self._schema_proposal_gaps(project_id))
+        return sorted(gaps, key=lambda row: (-row.priority, row.updated_at), reverse=False)[:limit]
+
+    def list_open(self, project_id: int, limit: int = 100) -> list[dict[str, Any]]:
+        self.detect(project_id, limit=limit)
+        rows = self.session.scalars(
+            select(models.KnowledgeGap)
+            .where(models.KnowledgeGap.project_id == project_id, models.KnowledgeGap.status == "open")
+            .order_by(models.KnowledgeGap.priority.desc(), models.KnowledgeGap.updated_at.desc())
+            .limit(limit)
+        ).all()
+        return [gap_payload(row) for row in rows]
+
+    def _entity_type_coverage_gaps(self, project_id: int) -> list[models.KnowledgeGap]:
+        counts = dict(
+            self.session.execute(
+                select(models.Entity.entity_type, func.count(models.Entity.id))
+                .where(models.Entity.project_id == project_id)
+                .group_by(models.Entity.entity_type)
+            ).all()
+        )
+        rows = self.session.scalars(
+            select(models.OntologyEntityType).where(
+                models.OntologyEntityType.project_id == project_id,
+                models.OntologyEntityType.status == "active",
+                models.OntologyEntityType.domain != "core",
+            )
+        ).all()
+        gaps = []
+        for row in rows:
+            if int(counts.get(row.name, 0)) == 0:
+                gaps.append(
+                    self._upsert_gap(
+                        project_id,
+                        gap_type="entity_type_coverage",
+                        target_type="EntityType",
+                        target_name=row.name,
+                        description=f"Entity type '{row.name}' is registered but has no extracted entities.",
+                        priority=0.42,
+                        evidence={"entity_type_id": row.id, "count": 0},
+                    )
+                )
+        return gaps
+
+    def _relation_type_usage_gaps(self, project_id: int) -> list[models.KnowledgeGap]:
+        counts = dict(
+            self.session.execute(
+                select(models.OntologyTriple.predicate, func.count(models.OntologyTriple.id))
+                .where(models.OntologyTriple.project_id == project_id)
+                .group_by(models.OntologyTriple.predicate)
+            ).all()
+        )
+        rows = self.session.scalars(
+            select(models.OntologyRelationType).where(
+                models.OntologyRelationType.project_id == project_id,
+                models.OntologyRelationType.status == "active",
+                models.OntologyRelationType.domain != "core",
+            )
+        ).all()
+        gaps = []
+        for row in rows:
+            if int(counts.get(row.name, 0)) == 0:
+                gaps.append(
+                    self._upsert_gap(
+                        project_id,
+                        gap_type="relation_type_usage",
+                        target_type="RelationType",
+                        target_name=row.name,
+                        description=f"Relation type '{row.name}' is registered but has no graph triples.",
+                        priority=0.55,
+                        evidence={"relation_type_id": row.id, "count": 0},
+                    )
+                )
+        return gaps
+
+    def _underconnected_entity_gaps(self, project_id: int) -> list[models.KnowledgeGap]:
+        rows = self.session.scalars(
+            select(models.Entity)
+            .where(models.Entity.project_id == project_id, models.Entity.entity_type.notin_(["Page", "ProductPage"]))
+            .order_by(models.Entity.updated_at.desc())
+            .limit(50)
+        ).all()
+        gaps = []
+        for entity in rows:
+            count = self.session.scalar(
+                select(func.count(models.OntologyTriple.id)).where(
+                    models.OntologyTriple.project_id == project_id,
+                    (
+                        (models.OntologyTriple.subject_entity_id == entity.id)
+                        | (models.OntologyTriple.object_entity_id == entity.id)
+                    ),
+                    models.OntologyTriple.status.in_(["merged", "validated", "validated_literal"]),
+                )
+            )
+            if int(count or 0) == 0:
+                gaps.append(
+                    self._upsert_gap(
+                        project_id,
+                        gap_type="entity_connectivity",
+                        target_type=entity.entity_type,
+                        target_name=entity.name,
+                        description=f"Entity '{entity.name}' has no validated ontology triples.",
+                        priority=0.48,
+                        evidence={"entity_id": entity.id, "triple_count": 0},
+                    )
+                )
+        return gaps
+
+    def _schema_proposal_gaps(self, project_id: int) -> list[models.KnowledgeGap]:
+        rows = self.session.scalars(
+            select(models.OntologyProposal).where(
+                models.OntologyProposal.project_id == project_id,
+                models.OntologyProposal.status == "pending_review",
+            )
+        ).all()
+        return [
+            self._upsert_gap(
+                project_id,
+                gap_type="schema_governance",
+                target_type=row.proposal_type,
+                target_name=row.name,
+                description=f"Ontology schema proposal '{row.name}' is waiting for review.",
+                priority=0.86,
+                evidence={"proposal_id": row.id, "reason": row.reason},
+            )
+            for row in rows
+        ]
+
+    def _upsert_gap(
+        self,
+        project_id: int,
+        *,
+        gap_type: str,
+        target_type: str,
+        target_name: str,
+        description: str,
+        priority: float,
+        evidence: dict[str, Any],
+    ) -> models.KnowledgeGap:
+        gap_hash = stable_hash(
+            {
+                "project_id": project_id,
+                "gap_type": gap_type,
+                "target_type": target_type,
+                "target_name": target_name,
+            }
+        )
+        row = self.session.scalar(
+            select(models.KnowledgeGap).where(
+                models.KnowledgeGap.project_id == project_id,
+                models.KnowledgeGap.gap_hash == gap_hash,
+            )
+        )
+        if row is None:
+            row = models.KnowledgeGap(
+                project_id=project_id,
+                gap_type=gap_type,
+                target_type=target_type,
+                target_name=target_name,
+                description=description,
+                priority=min(max(priority, 0.0), 1.0),
+                gap_hash=gap_hash,
+                evidence=evidence,
+            )
+            self.session.add(row)
+            self.session.flush()
+            return row
+        row.description = description
+        row.priority = min(max(priority, 0.0), 1.0)
+        row.evidence = evidence
+        row.updated_at = models.utcnow()
+        return row
+
+
+def gap_payload(row: models.KnowledgeGap) -> dict[str, Any]:
+    return {
+        "id": row.id,
+        "gap_type": row.gap_type,
+        "target_type": row.target_type,
+        "target_name": row.target_name,
+        "description": row.description,
+        "priority": row.priority,
+        "status": row.status,
+        "evidence": row.evidence or {},
+        "metadata": row.metadata_json or {},
+        "updated_at": row.updated_at.isoformat() if row.updated_at else None,
+    }

ontology_platform/crawler_platform/app/core/ontology/graph_merge.py

@@ -0,0 +1,30 @@
+from __future__ import annotations
+
+from typing import Any
+
+from crawler_platform.app.core.database import models
+from crawler_platform.app.core.ontology.relation_schema import minimum_confidence
+
+
+def should_merge_claim_to_graph(
+    claim: models.Claim,
+    ontology: dict[str, Any] | None = None,
+    domain: str | None = None,
+) -> tuple[bool, str | None]:
+    if claim.status != "validated_claim":
+        return False, f"claim status is not validated_claim: {claim.status}"
+    if claim.object_entity_id is None:
+        return False, "literal claims are stored as evidence claims, not graph relations"
+    threshold = minimum_confidence(claim.predicate, ontology, domain)
+    if claim.confidence < threshold:
+        return False, f"claim confidence {claim.confidence:.2f} is below graph threshold {threshold:.2f}"
+    metadata = claim.metadata_json or {}
+    if metadata.get("validation_status") != "validated_claim":
+        return False, "claim metadata validation_status is not validated_claim"
+    if metadata.get("review_required"):
+        return False, metadata.get("review_reason") or "claim requires human review"
+    if not metadata.get("evidence_found"):
+        return False, "claim has no matched evidence span"
+    if not metadata.get("source_zone_allowed"):
+        return False, "claim source zone is not graph-mergeable"
+    return True, None

ontology_platform/crawler_platform/app/core/ontology/mapper.py

@@ -0,0 +1,27 @@
+from __future__ import annotations
+
+from typing import Any
+
+
+def normalize_predicate(predicate: str, ontology: dict[str, Any] | None) -> str:
+    if not ontology:
+        return predicate
+    aliases = ontology.get("aliases", {})
+    return aliases.get(predicate, predicate)
+
+
+def is_allowed_predicate(predicate: str, ontology: dict[str, Any] | None) -> bool:
+    if not ontology or not ontology.get("predicates"):
+        return True
+    return predicate in ontology["predicates"]
+
+
+def ontology_to_dict(domain_ontology) -> dict[str, Any]:
+    return {
+        "domain": domain_ontology.domain,
+        "entity_types": domain_ontology.entity_types,
+        "predicates": domain_ontology.predicates,
+        "attributes": domain_ontology.attributes,
+        "aliases": domain_ontology.aliases,
+    }
+

ontology_platform/crawler_platform/app/core/ontology/registry.py

@@ -0,0 +1,339 @@
+from __future__ import annotations
+
+import hashlib
+import json
+from typing import Any
+
+from sqlalchemy import select
+from sqlalchemy.orm import Session
+
+from crawler_platform.app.config.loader import ProjectConfig
+from crawler_platform.app.core.database import models
+from crawler_platform.app.core.ontology.relation_schema import relation_rule
+
+
+CORE_ENTITY_TYPES = {
+    "Entity",
+    "Concept",
+    "Event",
+    "Organization",
+    "Person",
+    "Place",
+    "Document",
+    "Source",
+    "Page",
+}
+
+CORE_RELATION_TYPES = {
+    "relatedTo": {
+        "allowed_subject_types": [],
+        "allowed_object_types": [],
+        "semantic_constraints": {"domain_agnostic": True},
+        "confidence_rules": {"min_confidence": 0.7},
+    },
+    "mentions": {
+        "allowed_subject_types": ["Document", "Page", "Source"],
+        "allowed_object_types": [],
+        "semantic_constraints": {"provenance_relation": True},
+        "confidence_rules": {"min_confidence": 0.6},
+    },
+    "sameAs": {
+        "allowed_subject_types": [],
+        "allowed_object_types": [],
+        "semantic_constraints": {"entity_resolution": True},
+        "confidence_rules": {"min_confidence": 0.9},
+    },
+    "partOf": {
+        "allowed_subject_types": [],
+        "allowed_object_types": [],
+        "semantic_constraints": {"transitive_candidate": True},
+        "confidence_rules": {"min_confidence": 0.75},
+    },
+}
+
+
+class OntologyRegistry:
+    """Project-scoped ontology schema registry.
+
+    The registry keeps schema-level objects in the database so entity and
+    relation types can evolve through proposals instead of remaining hidden in
+    code-only constants.
+    """
+
+    def __init__(self, session: Session):
+        self.session = session
+
+    def seed_from_config(self, project: models.Project, config: ProjectConfig) -> None:
+        metadata = {"origin": "project_config", "config_domain": config.domain}
+        for name in sorted(CORE_ENTITY_TYPES | set(config.target_entities) | set(config.ontology.get("entity_types") or [])):
+            self.upsert_entity_type(
+                project.id,
+                name=name,
+                domain=config.domain if name not in CORE_ENTITY_TYPES else "core",
+                metadata=metadata,
+            )
+        for name, spec in CORE_RELATION_TYPES.items():
+            self.upsert_relation_type(project.id, name=name, domain="core", metadata={"origin": "core"}, **spec)
+        configured_relation_types = config.ontology.get("relation_types") or {}
+        for predicate in sorted(set(config.ontology.get("predicates") or [])):
+            spec = configured_relation_types.get(predicate) if isinstance(configured_relation_types, dict) else None
+            self.upsert_relation_type(
+                project.id,
+                name=predicate,
+                domain=config.domain,
+                metadata=metadata,
+                **self._relation_metadata_from_config(predicate, spec, config.ontology, config.domain),
+            )
+
+    def upsert_entity_type(
+        self,
+        project_id: int,
+        *,
+        name: str,
+        domain: str = "generic",
+        description: str | None = None,
+        status: str = "active",
+        confidence: float = 1.0,
+        metadata: dict[str, Any] | None = None,
+    ) -> models.OntologyEntityType:
+        clean_name = normalize_schema_name(name)
+        row = self.session.scalar(
+            select(models.OntologyEntityType).where(
+                models.OntologyEntityType.project_id == project_id,
+                models.OntologyEntityType.name == clean_name,
+            )
+        )
+        if row is None:
+            row = models.OntologyEntityType(
+                project_id=project_id,
+                name=clean_name,
+                domain=domain,
+                description=description,
+                status=status,
+                confidence=confidence,
+                metadata_json=metadata or {},
+            )
+            self.session.add(row)
+            self.session.flush()
+            return row
+        row.domain = domain or row.domain
+        row.description = description or row.description
+        row.status = status or row.status
+        row.confidence = max(row.confidence or 0.0, confidence)
+        row.metadata_json = {**(row.metadata_json or {}), **(metadata or {})}
+        row.updated_at = models.utcnow()
+        return row
+
+    def upsert_relation_type(
+        self,
+        project_id: int,
+        *,
+        name: str,
+        domain: str = "generic",
+        description: str | None = None,
+        allowed_subject_types: list[str] | None = None,
+        allowed_object_types: list[str] | None = None,
+        allowed_page_types: list[str] | None = None,
+        allowed_source_zones: list[str] | None = None,
+        semantic_constraints: dict[str, Any] | None = None,
+        confidence_rules: dict[str, Any] | None = None,
+        status: str = "active",
+        confidence: float = 1.0,
+        metadata: dict[str, Any] | None = None,
+    ) -> models.OntologyRelationType:
+        clean_name = normalize_schema_name(name)
+        row = self.session.scalar(
+            select(models.OntologyRelationType).where(
+                models.OntologyRelationType.project_id == project_id,
+                models.OntologyRelationType.name == clean_name,
+            )
+        )
+        values = {
+            "allowed_subject_types": sorted(set(allowed_subject_types or [])),
+            "allowed_object_types": sorted(set(allowed_object_types or [])),
+            "allowed_page_types": sorted(set(allowed_page_types or [])),
+            "allowed_source_zones": sorted(set(allowed_source_zones or [])),
+            "semantic_constraints": semantic_constraints or {},
+            "confidence_rules": confidence_rules or {},
+        }
+        if row is None:
+            row = models.OntologyRelationType(
+                project_id=project_id,
+                name=clean_name,
+                domain=domain,
+                description=description,
+                status=status,
+                confidence=confidence,
+                metadata_json=metadata or {},
+                **values,
+            )
+            self.session.add(row)
+            self.session.flush()
+            return row
+        row.domain = domain or row.domain
+        row.description = description or row.description
+        row.status = status or row.status
+        row.confidence = max(row.confidence or 0.0, confidence)
+        row.allowed_subject_types = merge_unique(row.allowed_subject_types, values["allowed_subject_types"])
+        row.allowed_object_types = merge_unique(row.allowed_object_types, values["allowed_object_types"])
+        row.allowed_page_types = merge_unique(row.allowed_page_types, values["allowed_page_types"])
+        row.allowed_source_zones = merge_unique(row.allowed_source_zones, values["allowed_source_zones"])
+        row.semantic_constraints = {**(row.semantic_constraints or {}), **values["semantic_constraints"]}
+        row.confidence_rules = {**(row.confidence_rules or {}), **values["confidence_rules"]}
+        row.metadata_json = {**(row.metadata_json or {}), **(metadata or {})}
+        row.updated_at = models.utcnow()
+        return row
+
+    def relation_type(self, project_id: int, name: str) -> models.OntologyRelationType | None:
+        return self.session.scalar(
+            select(models.OntologyRelationType).where(
+                models.OntologyRelationType.project_id == project_id,
+                models.OntologyRelationType.name == normalize_schema_name(name),
+            )
+        )
+
+    def entity_type(self, project_id: int, name: str) -> models.OntologyEntityType | None:
+        return self.session.scalar(
+            select(models.OntologyEntityType).where(
+                models.OntologyEntityType.project_id == project_id,
+                models.OntologyEntityType.name == normalize_schema_name(name),
+            )
+        )
+
+    def propose_schema_change(
+        self,
+        project_id: int,
+        *,
+        proposal_type: str,
+        name: str,
+        reason: str,
+        evidence: str | None = None,
+        confidence: float = 0.5,
+        metadata: dict[str, Any] | None = None,
+    ) -> models.OntologyProposal:
+        proposal_hash = stable_hash(
+            {
+                "project_id": project_id,
+                "proposal_type": proposal_type,
+                "name": normalize_schema_name(name),
+                "metadata": metadata or {},
+            }
+        )
+        row = self.session.scalar(
+            select(models.OntologyProposal).where(
+                models.OntologyProposal.project_id == project_id,
+                models.OntologyProposal.proposal_hash == proposal_hash,
+            )
+        )
+        if row is None:
+            row = models.OntologyProposal(
+                project_id=project_id,
+                proposal_type=proposal_type,
+                name=normalize_schema_name(name),
+                reason=reason,
+                evidence=evidence,
+                confidence=min(max(confidence, 0.0), 1.0),
+                proposal_hash=proposal_hash,
+                metadata_json=metadata or {},
+            )
+            self.session.add(row)
+            self.session.flush()
+            return row
+        row.confidence = max(row.confidence, min(max(confidence, 0.0), 1.0))
+        row.reason = reason or row.reason
+        row.evidence = evidence or row.evidence
+        row.metadata_json = {**(row.metadata_json or {}), **(metadata or {})}
+        row.updated_at = models.utcnow()
+        return row
+
+    def registry_payload(self, project_id: int) -> dict[str, Any]:
+        entity_types = self.session.scalars(
+            select(models.OntologyEntityType)
+            .where(models.OntologyEntityType.project_id == project_id)
+            .order_by(models.OntologyEntityType.domain, models.OntologyEntityType.name)
+        ).all()
+        relation_types = self.session.scalars(
+            select(models.OntologyRelationType)
+            .where(models.OntologyRelationType.project_id == project_id)
+            .order_by(models.OntologyRelationType.domain, models.OntologyRelationType.name)
+        ).all()
+        return {
+            "entity_types": [entity_type_payload(row) for row in entity_types],
+            "relation_types": [relation_type_payload(row) for row in relation_types],
+        }
+
+    def _relation_metadata_from_config(
+        self,
+        predicate: str,
+        spec: dict[str, Any] | None,
+        ontology: dict[str, Any],
+        domain: str | None,
+    ) -> dict[str, Any]:
+        if spec:
+            return {
+                "description": spec.get("description"),
+                "allowed_subject_types": list(spec.get("allowed_subject_types") or spec.get("subject_types") or []),
+                "allowed_object_types": list(spec.get("allowed_object_types") or spec.get("object_types") or []),
+                "allowed_page_types": list(spec.get("allowed_page_types") or spec.get("page_types") or []),
+                "allowed_source_zones": list(spec.get("allowed_source_zones") or spec.get("source_zones") or []),
+                "semantic_constraints": dict(spec.get("semantic_constraints") or {}),
+                "confidence_rules": dict(spec.get("confidence_rules") or {}),
+            }
+        rule = relation_rule(predicate, ontology, domain)
+        if rule is None:
+            return {
+                "semantic_constraints": {"domain_defined": True, "requires_governance_for_new_constraints": True},
+                "confidence_rules": {"min_confidence": 0.8},
+            }
+        return {
+            "allowed_subject_types": sorted(rule.subject_types),
+            "allowed_object_types": sorted(rule.object_types),
+            "allowed_page_types": sorted(rule.page_types),
+            "allowed_source_zones": sorted(rule.source_zones),
+            "semantic_constraints": {"literal_value": rule.literal_value},
+            "confidence_rules": {"min_confidence": rule.min_confidence},
+        }
+
+
+def normalize_schema_name(value: str) -> str:
+    return " ".join(str(value or "").strip().split())
+
+
+def merge_unique(existing: Any, new_values: list[str]) -> list[str]:
+    return sorted({str(item) for item in (existing or []) if str(item)} | {str(item) for item in new_values if str(item)})
+
+
+def stable_hash(payload: dict[str, Any]) -> str:
+    return hashlib.sha256(json.dumps(payload, sort_keys=True, default=str).encode("utf-8")).hexdigest()
+
+
+def entity_type_payload(row: models.OntologyEntityType) -> dict[str, Any]:
+    return {
+        "id": row.id,
+        "name": row.name,
+        "domain": row.domain,
+        "description": row.description,
+        "status": row.status,
+        "version": row.version,
+        "confidence": row.confidence,
+        "metadata": row.metadata_json or {},
+    }
+
+
+def relation_type_payload(row: models.OntologyRelationType) -> dict[str, Any]:
+    return {
+        "id": row.id,
+        "name": row.name,
+        "domain": row.domain,
+        "description": row.description,
+        "allowed_subject_types": row.allowed_subject_types or [],
+        "allowed_object_types": row.allowed_object_types or [],
+        "allowed_page_types": row.allowed_page_types or [],
+        "allowed_source_zones": row.allowed_source_zones or [],
+        "semantic_constraints": row.semantic_constraints or {},
+        "confidence_rules": row.confidence_rules or {},
+        "status": row.status,
+        "version": row.version,
+        "confidence": row.confidence,
+        "metadata": row.metadata_json or {},
+    }

ontology_platform/crawler_platform/app/core/ontology/relation_schema.py

@@ -0,0 +1,145 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any
+
+
+@dataclass(frozen=True, slots=True)
+class RelationRule:
+    predicate: str
+    subject_types: set[str]
+    object_types: set[str] = field(default_factory=set)
+    literal_value: bool = False
+    page_types: set[str] = field(default_factory=set)
+    source_zones: set[str] = field(default_factory=set)
+    min_confidence: float = 0.7
+
+
+CORE_RELATION_SCHEMA: dict[str, RelationRule] = {
+    "relatedTo": RelationRule("relatedTo", subject_types=set(), object_types=set(), page_types=set(), min_confidence=0.7),
+    "mentions": RelationRule("mentions", subject_types=set(), object_types=set(), page_types=set(), min_confidence=0.6),
+    "sameAs": RelationRule("sameAs", subject_types=set(), object_types=set(), page_types=set(), min_confidence=0.9),
+    "partOf": RelationRule("partOf", subject_types=set(), object_types=set(), page_types=set(), min_confidence=0.75),
+}
+
+
+def relation_rule(
+    predicate: str,
+    ontology: dict[str, Any] | None = None,
+    domain: str | None = None,
+) -> RelationRule | None:
+    configured_rule = configured_relation_rule(predicate, ontology)
+    if configured_rule is not None:
+        return configured_rule
+    if domain:
+        from crawler_platform.app.adapters.registry import adapter_for_domain
+
+        adapter = adapter_for_domain(domain)
+        if adapter is not None:
+            adapter_rule = adapter.relation_rule(predicate)
+            if adapter_rule is not None:
+                return adapter_rule
+    if predicate in CORE_RELATION_SCHEMA:
+        return CORE_RELATION_SCHEMA[predicate]
+    predicates = set((ontology or {}).get("predicates") or [])
+    if predicate in predicates:
+        return RelationRule(predicate=predicate, subject_types=set(), object_types=set(), page_types=set(), min_confidence=0.8)
+    return None
+
+
+def configured_relation_rule(predicate: str, ontology: dict[str, Any] | None = None) -> RelationRule | None:
+    relation_types = (ontology or {}).get("relation_types") or {}
+    if not isinstance(relation_types, dict) or predicate not in relation_types:
+        return None
+    spec = relation_types.get(predicate) or {}
+    constraints = dict(spec.get("semantic_constraints") or {})
+    confidence_rules = dict(spec.get("confidence_rules") or {})
+    literal_value = bool(
+        spec.get("literal_value")
+        or constraints.get("literal_value")
+        or spec.get("value_type") == "literal"
+        or spec.get("object_kind") == "literal"
+    )
+    return RelationRule(
+        predicate=predicate,
+        subject_types=set(spec.get("allowed_subject_types") or spec.get("subject_types") or []),
+        object_types=set(spec.get("allowed_object_types") or spec.get("object_types") or []),
+        literal_value=literal_value,
+        page_types=set(spec.get("allowed_page_types") or spec.get("page_types") or []),
+        source_zones=set(spec.get("allowed_source_zones") or spec.get("source_zones") or []),
+        min_confidence=float(confidence_rules.get("min_confidence") or spec.get("min_confidence") or 0.8),
+    )
+
+
+def relation_schema_compatible(
+    *,
+    predicate: str,
+    subject_type: str,
+    object_type: str | None,
+    has_literal_value: bool,
+    page_type: str | None,
+    source_zone: str | None,
+    ontology: dict[str, Any] | None = None,
+    domain: str | None = None,
+) -> str | None:
+    rule = relation_rule(predicate, ontology, domain)
+    if rule is None:
+        return "predicate has no relation schema"
+    if rule.subject_types and subject_type not in rule.subject_types:
+        return f"subject type {subject_type} is not allowed for {predicate}"
+    if rule.literal_value:
+        if not has_literal_value:
+            return f"{predicate} expects a literal object"
+    else:
+        if has_literal_value and rule.object_types:
+            return f"{predicate} expects a typed entity object"
+        if not has_literal_value and not object_type:
+            return f"{predicate} expects a typed entity object"
+        if object_type and rule.object_types and object_type not in rule.object_types:
+            return f"object type {object_type} is not allowed for {predicate}"
+    if page_type and rule.page_types and page_type not in rule.page_types:
+        return f"predicate {predicate} is not allowed for page type {page_type}"
+    if source_zone and rule.source_zones and source_zone not in rule.source_zones:
+        return f"source zone {source_zone} is not allowed for {predicate}"
+    return None
+
+
+def confidence_breakdown(
+    *,
+    llm_confidence: float,
+    evidence_found: bool,
+    ontology_compatible: bool,
+    source_zone_allowed: bool,
+    source_trust: float | None = None,
+) -> dict[str, float]:
+    schema_confidence = 1.0
+    evidence_confidence = 0.95 if evidence_found else 0.0
+    ontology_confidence = 0.95 if ontology_compatible else 0.0
+    zone_confidence = 0.9 if source_zone_allowed else 0.0
+    trust = source_trust if source_trust is not None else 0.8
+    final = (
+        llm_confidence * 0.35
+        + schema_confidence * 0.15
+        + evidence_confidence * 0.2
+        + ontology_confidence * 0.2
+        + zone_confidence * 0.05
+        + trust * 0.05
+    )
+    return {
+        "llm_confidence": round(llm_confidence, 4),
+        "schema_confidence": schema_confidence,
+        "evidence_confidence": evidence_confidence,
+        "ontology_confidence": ontology_confidence,
+        "source_zone_confidence": zone_confidence,
+        "source_trust": round(trust, 4),
+        "final_confidence": round(min(max(final, 0.0), 1.0), 4),
+    }
+
+
+def minimum_confidence(
+    predicate: str,
+    ontology: dict[str, Any] | None = None,
+    domain: str | None = None,
+) -> float:
+    rule = relation_rule(predicate, ontology, domain)
+    return rule.min_confidence if rule else 0.8

ontology_platform/crawler_platform/app/core/ontology/structured_output.py

@@ -0,0 +1,111 @@
+from __future__ import annotations
+
+from typing import Any
+
+from pydantic import BaseModel, ConfigDict, Field, ValidationError, field_validator, model_validator
+
+
+class StructuredEntity(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    entity_type: str = Field(min_length=1)
+    name: str = Field(min_length=1)
+    attributes: dict[str, Any] = Field(default_factory=dict)
+    confidence: float = Field(ge=0.0, le=1.0)
+    evidence_text: str = Field(min_length=1)
+
+    @field_validator("entity_type", "name", "evidence_text")
+    @classmethod
+    def _strip_required_text(cls, value: str) -> str:
+        clean = value.strip()
+        if not clean:
+            raise ValueError("field cannot be blank")
+        return clean
+
+
+class StructuredClaim(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    subject_name: str = Field(min_length=1)
+    subject_type: str = Field(min_length=1)
+    predicate: str = Field(min_length=1)
+    object_name: str | None = None
+    object_type: str | None = None
+    object_value: Any | None = None
+    evidence_text: str = Field(min_length=1)
+    evidence_summary: str | None = None
+    confidence: float = Field(ge=0.0, le=1.0)
+    confidence_reason: str | None = None
+    source_zone: str | None = None
+
+    @field_validator("subject_name", "subject_type", "predicate", "evidence_text")
+    @classmethod
+    def _strip_required_text(cls, value: str) -> str:
+        clean = value.strip()
+        if not clean:
+            raise ValueError("field cannot be blank")
+        return clean
+
+    @field_validator("object_name", "object_type", "evidence_summary", "confidence_reason", "source_zone")
+    @classmethod
+    def _strip_optional_text(cls, value: str | None) -> str | None:
+        if value is None:
+            return None
+        clean = value.strip()
+        return clean or None
+
+    @model_validator(mode="after")
+    def _has_object(self) -> "StructuredClaim":
+        if self.object_name is None and self.object_value is None:
+            raise ValueError("claim must have object_name or object_value")
+        return self
+
+
+class StructuredExtraction(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    entities: list[StructuredEntity] = Field(default_factory=list)
+    claims: list[StructuredClaim] = Field(default_factory=list)
+
+
+def validate_structured_extraction(raw: dict[str, Any]) -> dict[str, Any]:
+    try:
+        parsed = StructuredExtraction.model_validate(raw)
+    except ValidationError as exc:
+        raise ValueError(f"schema_validation_failed: {exc.errors(include_url=False)}") from exc
+    return parsed.model_dump()
+
+
+def extraction_json_schema() -> dict[str, Any]:
+    schema = StructuredExtraction.model_json_schema()
+    _inline_defs(schema)
+    _disallow_additional_properties(schema)
+    return schema
+
+
+def _inline_defs(schema: dict[str, Any]) -> None:
+    defs = schema.pop("$defs", {})
+
+    def resolve(node: Any) -> Any:
+        if isinstance(node, dict):
+            ref = node.get("$ref")
+            if ref and ref.startswith("#/$defs/"):
+                name = ref.rsplit("/", 1)[-1]
+                return resolve(dict(defs[name]))
+            return {key: resolve(value) for key, value in node.items()}
+        if isinstance(node, list):
+            return [resolve(item) for item in node]
+        return node
+
+    schema.update(resolve(schema))
+
+
+def _disallow_additional_properties(node: Any) -> None:
+    if isinstance(node, dict):
+        if node.get("type") == "object":
+            node.setdefault("additionalProperties", False)
+        for value in node.values():
+            _disallow_additional_properties(value)
+    elif isinstance(node, list):
+        for item in node:
+            _disallow_additional_properties(item)

ontology_platform/crawler_platform/app/core/ontology/triple_store.py

@@ -0,0 +1,183 @@
+from __future__ import annotations
+
+from typing import Any
+
+from sqlalchemy import select
+from sqlalchemy.orm import Session
+
+from crawler_platform.app.core.database import models
+from crawler_platform.app.core.ontology.registry import OntologyRegistry, stable_hash
+
+
+class OntologyTripleStore:
+    """Stores graph triples separately from extraction claims.
+
+    Claims remain source/evidence records. Triples are the project-level graph
+    facts that can accumulate support from many claims and sources.
+    """
+
+    def __init__(self, session: Session):
+        self.session = session
+        self.registry = OntologyRegistry(session)
+
+    def backfill_project(self, project_id: int, limit: int = 2000) -> int:
+        claims = self.session.scalars(
+            select(models.Claim)
+            .where(models.Claim.project_id == project_id, models.Claim.status == "validated_claim")
+            .order_by(models.Claim.last_seen_at.desc())
+            .limit(limit)
+        ).all()
+        count = 0
+        for claim in claims:
+            self.upsert_from_claim(claim)
+            count += 1
+        return count
+
+    def upsert_from_claim(self, claim: models.Claim) -> models.OntologyTriple:
+        subject = self.session.get(models.Entity, claim.subject_entity_id)
+        object_entity = self.session.get(models.Entity, claim.object_entity_id) if claim.object_entity_id else None
+        relation_type = self.registry.relation_type(claim.project_id, claim.predicate)
+        metadata = dict(claim.metadata_json or {})
+        if subject is None:
+            raise ValueError(f"Claim {claim.id} has no subject entity")
+        if self.registry.entity_type(claim.project_id, subject.entity_type) is None:
+            self.registry.propose_schema_change(
+                claim.project_id,
+                proposal_type="entity_type",
+                name=subject.entity_type,
+                reason="Validated claim references an entity type not present in ontology registry.",
+                evidence=subject.name,
+                confidence=claim.confidence,
+                metadata={"claim_id": claim.id},
+            )
+        if object_entity is not None and self.registry.entity_type(claim.project_id, object_entity.entity_type) is None:
+            self.registry.propose_schema_change(
+                claim.project_id,
+                proposal_type="entity_type",
+                name=object_entity.entity_type,
+                reason="Validated claim references an object entity type not present in ontology registry.",
+                evidence=object_entity.name,
+                confidence=claim.confidence,
+                metadata={"claim_id": claim.id},
+            )
+        if relation_type is None:
+            self.registry.propose_schema_change(
+                claim.project_id,
+                proposal_type="relation_type",
+                name=claim.predicate,
+                reason="Validated claim references a relation type not present in ontology registry.",
+                evidence=metadata.get("evidence_text") or claim.confidence_reason,
+                confidence=claim.confidence,
+                metadata={"claim_id": claim.id},
+            )
+
+        triple_hash = make_triple_hash(claim)
+        row = self.session.scalar(
+            select(models.OntologyTriple).where(
+                models.OntologyTriple.project_id == claim.project_id,
+                models.OntologyTriple.triple_hash == triple_hash,
+            )
+        )
+        source_item = {
+            "claim_id": claim.id,
+            "source_id": claim.source_id,
+            "page_id": claim.page_id,
+            "seen_at": models.utcnow().isoformat(),
+            "confidence": claim.confidence,
+            "status": claim.status,
+        }
+        triple_metadata = {
+            "source_history": [source_item],
+            "claim_metadata": {
+                "page_type": metadata.get("page_type"),
+                "source_zone": metadata.get("source_zone"),
+                "graph_merge_reason": metadata.get("graph_merge_reason"),
+                "review_reason": metadata.get("review_reason"),
+            },
+        }
+        status = triple_status_from_claim(claim)
+        if row is None:
+            row = models.OntologyTriple(
+                project_id=claim.project_id,
+                claim_id=claim.id,
+                source_id=claim.source_id,
+                page_id=claim.page_id,
+                subject_entity_id=claim.subject_entity_id,
+                subject_type=subject.entity_type,
+                predicate=claim.predicate,
+                relation_type_id=relation_type.id if relation_type else None,
+                object_entity_id=claim.object_entity_id,
+                object_type=object_entity.entity_type if object_entity else None,
+                object_value=claim.object_value,
+                value_type=claim.value_type,
+                triple_hash=triple_hash,
+                status=status,
+                confidence=claim.confidence,
+                metadata_json=triple_metadata,
+            )
+            self.session.add(row)
+            self.session.flush()
+            return row
+        existing_metadata = dict(row.metadata_json or {})
+        row.claim_id = claim.id
+        row.source_id = claim.source_id
+        row.page_id = claim.page_id
+        row.relation_type_id = relation_type.id if relation_type else row.relation_type_id
+        row.status = merge_status(row.status, status)
+        row.confidence = max(row.confidence, claim.confidence)
+        row.support_count += 1
+        row.last_seen_at = models.utcnow()
+        row.metadata_json = {
+            **existing_metadata,
+            "source_history": merge_source_history(existing_metadata.get("source_history") or [], [source_item]),
+            "claim_metadata": triple_metadata["claim_metadata"],
+        }
+        return row
+
+
+def make_triple_hash(claim: models.Claim) -> str:
+    return stable_hash(
+        {
+            "project_id": claim.project_id,
+            "subject_entity_id": claim.subject_entity_id,
+            "predicate": claim.predicate,
+            "object_entity_id": claim.object_entity_id,
+            "object_value": claim.object_value,
+            "value_type": claim.value_type,
+        }
+    )
+
+
+def triple_status_from_claim(claim: models.Claim) -> str:
+    metadata = claim.metadata_json or {}
+    if metadata.get("review_required") or metadata.get("conflict_status"):
+        return "review_required"
+    if claim.status != "validated_claim":
+        return "candidate"
+    if metadata.get("graph_merge_status") == "merged":
+        return "merged"
+    if claim.value_type == "literal":
+        return "validated_literal"
+    return "validated"
+
+
+def merge_status(existing: str, new_status: str) -> str:
+    rank = {
+        "candidate": 0,
+        "validated": 1,
+        "validated_literal": 1,
+        "review_required": 2,
+        "merged": 3,
+    }
+    return new_status if rank.get(new_status, 0) > rank.get(existing, 0) else existing
+
+
+def merge_source_history(existing: list[dict[str, Any]], new_items: list[dict[str, Any]]) -> list[dict[str, Any]]:
+    merged = [item for item in existing if isinstance(item, dict)]
+    seen = {(item.get("claim_id"), item.get("source_id"), item.get("page_id")) for item in merged}
+    for item in new_items:
+        key = (item.get("claim_id"), item.get("source_id"), item.get("page_id"))
+        if key not in seen:
+            merged.append(item)
+            seen.add(key)
+    return merged[-100:]

ontology_platform/crawler_platform/app/core/recommendation/__init__.py

@@ -0,0 +1,2 @@
+"""Recommendation integration helpers."""
+

ontology_platform/crawler_platform/app/core/recommendation/scorer.py

@@ -0,0 +1,103 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any
+
+from sqlalchemy import select
+from sqlalchemy.orm import Session
+
+from crawler_platform.app.core.database import models
+from crawler_platform.app.core.database.repository import canonicalize
+
+
+@dataclass(slots=True)
+class PreferenceInput:
+    likes: list[str] = field(default_factory=list)
+    dislikes: list[str] = field(default_factory=list)
+    preferred_moods: list[str] = field(default_factory=list)
+    preferred_notes: list[str] = field(default_factory=list)
+    avoided_notes: list[str] = field(default_factory=list)
+    price_preference: dict[str, Any] = field(default_factory=dict)
+    season_context: str | None = None
+    occasion_context: str | None = None
+
+
+@dataclass(slots=True)
+class Recommendation:
+    entity_id: int
+    name: str
+    entity_type: str
+    score: float
+    reasons: list[str]
+
+
+class RuleBasedRecommender:
+    def __init__(self, session: Session):
+        self.session = session
+
+    def recommend(
+        self,
+        project_id: int,
+        target_entity_type: str,
+        preference: PreferenceInput,
+        limit: int = 10,
+    ) -> list[Recommendation]:
+        entities = self.session.scalars(
+            select(models.Entity).where(
+                models.Entity.project_id == project_id,
+                models.Entity.entity_type == target_entity_type,
+            )
+        ).all()
+        scored = [self._score_entity(project_id, entity, preference) for entity in entities]
+        scored = [item for item in scored if item.score > 0]
+        return sorted(scored, key=lambda item: item.score, reverse=True)[:limit]
+
+    def _score_entity(self, project_id: int, entity: models.Entity, preference: PreferenceInput) -> Recommendation:
+        claims = self.session.execute(
+            select(models.Claim, models.Entity)
+            .join(models.Entity, models.Claim.object_entity_id == models.Entity.id, isouter=True)
+            .where(
+                models.Claim.project_id == project_id,
+                models.Claim.subject_entity_id == entity.id,
+                models.Claim.status == "validated_claim",
+            )
+        ).all()
+        score = 0.0
+        reasons: list[str] = []
+        preferred_notes = {canonicalize(item) for item in preference.preferred_notes}
+        avoided_notes = {canonicalize(item) for item in preference.avoided_notes}
+        preferred_moods = {canonicalize(item) for item in preference.preferred_moods}
+        for claim, object_entity in claims:
+            object_name = canonicalize(object_entity.name) if object_entity else ""
+            weight = claim.confidence
+            if claim.predicate in {"hasTopNote", "hasMiddleNote", "hasBaseNote", "hasScentNote", "hasFlavorNote"}:
+                if object_name in preferred_notes:
+                    score += 2.0 * weight
+                    reasons.append(f"preferred note matched: {object_entity.name}")
+                if object_name in avoided_notes:
+                    score -= 3.0 * weight
+                    reasons.append(f"avoided note matched: {object_entity.name}")
+            if claim.predicate == "evokesMood" and object_name in preferred_moods:
+                score += 1.5 * weight
+                reasons.append(f"preferred mood matched: {object_entity.name}")
+            if preference.season_context and claim.predicate == "suitableForSeason":
+                if object_name == canonicalize(preference.season_context):
+                    score += 1.2 * weight
+                    reasons.append(f"season context matched: {preference.season_context}")
+            if preference.occasion_context and claim.predicate == "suitableForOccasion":
+                if object_name == canonicalize(preference.occasion_context):
+                    score += 1.0 * weight
+                    reasons.append(f"occasion context matched: {preference.occasion_context}")
+        if canonicalize(entity.name) in {canonicalize(item) for item in preference.dislikes}:
+            score -= 10
+            reasons.append("explicit dislike")
+        if canonicalize(entity.name) in {canonicalize(item) for item in preference.likes}:
+            score += 5
+            reasons.append("explicit like")
+        return Recommendation(
+            entity_id=entity.id,
+            name=entity.name,
+            entity_type=entity.entity_type,
+            score=round(score, 4),
+            reasons=reasons[:5],
+        )

ontology_platform/crawler_platform/app/core/research/__init__.py

@@ -0,0 +1 @@
+"""Graph-driven semantic research utilities."""

ontology_platform/crawler_platform/app/core/research/entity_expansion.py

@@ -0,0 +1,106 @@
+from __future__ import annotations
+
+from sqlalchemy import select
+from sqlalchemy.orm import Session
+
+from crawler_platform.app.core.database import models
+from crawler_platform.app.core.research.exploration_queue import ExplorationItem
+from crawler_platform.app.core.research.relevance_engine import RelevanceEngine
+
+
+class EntityExpansionPlanner:
+    def __init__(self, session: Session):
+        self.session = session
+        self.relevance = RelevanceEngine(session)
+
+    def expand_entity(self, project_id: int, entity_id: int, max_items: int = 8) -> list[ExplorationItem]:
+        entity = self.session.get(models.Entity, entity_id)
+        if entity is None or entity.project_id != project_id:
+            return []
+        candidates: list[ExplorationItem] = []
+        for page in self._matching_pages(project_id, entity.name):
+            score = self.relevance.score_url(
+                project_id=project_id,
+                url=page.url,
+                label=page.title or entity.name,
+                source_trust=0.8,
+                text=page.cleaned_text_summary or "",
+            )
+            candidates.append(
+                ExplorationItem(
+                    target_type="url",
+                    target=page.url,
+                    reason=f"Existing page mentions {entity.name}: {score.reason}",
+                    priority=score.score,
+                    source_entity=entity.name,
+                    relevance_breakdown=score.breakdown,
+                    metadata={"page_id": page.id, "page_type": score.page_type},
+                )
+            )
+        for related in self._related_entities(project_id, entity.id):
+            score = self.relevance.score_entity(project_id, related)
+            candidates.append(
+                ExplorationItem(
+                    target_type="entity",
+                    target=str(related.id),
+                    reason=f"Related via validated graph: {related.name}",
+                    priority=score.score,
+                    source_entity=entity.name,
+                    relevance_breakdown=score.breakdown,
+                    metadata={"entity_name": related.name, "entity_type": related.entity_type},
+                )
+            )
+        if entity.entity_type == "Brand":
+            candidates.extend(self._brand_topics(entity))
+        return sorted(candidates, key=lambda item: item.priority, reverse=True)[:max_items]
+
+    def _matching_pages(self, project_id: int, name: str) -> list[models.Page]:
+        needle = name.lower()
+        pages = self.session.scalars(
+            select(models.Page).where(models.Page.project_id == project_id).limit(250)
+        ).all()
+        return [
+            page
+            for page in pages
+            if needle in ((page.title or "") + " " + (page.cleaned_text_summary or "") + " " + page.url).lower()
+        ][:20]
+
+    def _related_entities(self, project_id: int, entity_id: int) -> list[models.Entity]:
+        claims = self.session.scalars(
+            select(models.Claim)
+            .where(
+                models.Claim.project_id == project_id,
+                models.Claim.status == "validated_claim",
+                (models.Claim.subject_entity_id == entity_id) | (models.Claim.object_entity_id == entity_id),
+            )
+            .limit(50)
+        ).all()
+        ids: set[int] = set()
+        for claim in claims:
+            if claim.subject_entity_id != entity_id:
+                ids.add(claim.subject_entity_id)
+            if claim.object_entity_id and claim.object_entity_id != entity_id:
+                ids.add(claim.object_entity_id)
+        if not ids:
+            return []
+        return self.session.scalars(
+            select(models.Entity).where(models.Entity.project_id == project_id, models.Entity.id.in_(ids)).limit(50)
+        ).all()
+
+    def _brand_topics(self, entity: models.Entity) -> list[ExplorationItem]:
+        topics = [
+            ("topic", f"{entity.name} product ecosystem", "Brand to product expansion"),
+            ("topic", f"{entity.name} fragrance notes", "Brand to note trend expansion"),
+            ("topic", f"{entity.name} review sentiment", "Brand to review expansion"),
+        ]
+        return [
+            ExplorationItem(
+                target_type=target_type,
+                target=target,
+                reason=reason,
+                priority=0.58,
+                source_entity=entity.name,
+                metadata={"entity_id": entity.id, "entity_type": entity.entity_type},
+            )
+            for target_type, target, reason in topics
+        ]

ontology_platform/crawler_platform/app/core/research/exploration_queue.py

@@ -0,0 +1,54 @@
+from __future__ import annotations
+
+from dataclasses import asdict, dataclass, field
+import heapq
+from itertools import count
+from typing import Any
+
+
+@dataclass(slots=True)
+class ExplorationItem:
+    target_type: str
+    target: str
+    reason: str
+    priority: float
+    source_entity: str | None = None
+    exploration_depth: int = 0
+    parent_target: str | None = None
+    status: str = "pending"
+    relevance_breakdown: dict[str, float] = field(default_factory=dict)
+    metadata: dict[str, Any] = field(default_factory=dict)
+
+    def key(self) -> tuple[str, str]:
+        return (self.target_type, self.target.strip().lower())
+
+    def to_dict(self) -> dict[str, Any]:
+        return asdict(self)
+
+
+class ExplorationQueue:
+    def __init__(self):
+        self._heap: list[tuple[float, int, ExplorationItem]] = []
+        self._counter = count()
+        self._queued: set[tuple[str, str]] = set()
+
+    def push(self, item: ExplorationItem) -> bool:
+        key = item.key()
+        if key in self._queued:
+            return False
+        self._queued.add(key)
+        heapq.heappush(self._heap, (-item.priority, next(self._counter), item))
+        return True
+
+    def pop(self) -> ExplorationItem | None:
+        if not self._heap:
+            return None
+        _priority, _idx, item = heapq.heappop(self._heap)
+        return item
+
+    def __len__(self) -> int:
+        return len(self._heap)
+
+    def pending_items(self, limit: int = 100) -> list[dict[str, Any]]:
+        ordered = sorted(self._heap, key=lambda entry: (entry[0], entry[1]))
+        return [entry[2].to_dict() for entry in ordered[:limit]]

ontology_platform/crawler_platform/app/core/research/gap_task_planner.py

@@ -0,0 +1,47 @@
+from __future__ import annotations
+
+from sqlalchemy.orm import Session
+
+from crawler_platform.app.core.ontology.gap_detector import KnowledgeGapDetector
+from crawler_platform.app.core.research.exploration_queue import ExplorationItem
+
+
+class GapTaskPlanner:
+    """Turns ontology knowledge gaps into exploration queue items."""
+
+    def __init__(self, session: Session):
+        self.session = session
+
+    def plan(self, project_id: int, max_items: int = 10) -> list[ExplorationItem]:
+        gaps = KnowledgeGapDetector(self.session).list_open(project_id, limit=max_items)
+        items: list[ExplorationItem] = []
+        for gap in gaps:
+            evidence = gap.get("evidence") or {}
+            entity_id = evidence.get("entity_id")
+            if gap.get("gap_type") == "entity_connectivity" and entity_id:
+                items.append(
+                    ExplorationItem(
+                        target_type="entity",
+                        target=str(entity_id),
+                        reason=f"Knowledge gap: {gap['description']}",
+                        priority=float(gap.get("priority") or 0.5),
+                        metadata={"gap_id": gap.get("id"), "gap_type": gap.get("gap_type")},
+                    )
+                )
+                continue
+            target = f"{gap.get('target_type') or 'ontology'}:{gap.get('target_name') or gap.get('gap_type')}"
+            items.append(
+                ExplorationItem(
+                    target_type="knowledge_gap",
+                    target=target,
+                    reason=f"Knowledge gap: {gap['description']}",
+                    priority=float(gap.get("priority") or 0.5),
+                    metadata={
+                        "gap_id": gap.get("id"),
+                        "gap_type": gap.get("gap_type"),
+                        "target_type": gap.get("target_type"),
+                        "target_name": gap.get("target_name"),
+                    },
+                )
+            )
+        return items

ontology_platform/crawler_platform/app/core/research/graph_query.py

@@ -0,0 +1,205 @@
+from __future__ import annotations
+
+from typing import Any
+
+from sqlalchemy import func, select
+from sqlalchemy.orm import Session
+
+from crawler_platform.app.core.database import models
+
+
+class SemanticGraphQuery:
+    def __init__(self, session: Session):
+        self.session = session
+
+    def neighborhood(
+        self,
+        project_id: int,
+        entity_id: int | None = None,
+        limit: int = 120,
+        statuses: list[str] | None = None,
+    ) -> dict[str, Any]:
+        query = (
+            select(models.Claim, models.Entity)
+            .join(models.Entity, models.Claim.subject_entity_id == models.Entity.id)
+            .where(models.Claim.project_id == project_id)
+        )
+        if statuses is not None:
+            query = query.where(models.Claim.status.in_(statuses))
+        if entity_id is not None:
+            query = query.where(
+                (models.Claim.subject_entity_id == entity_id) | (models.Claim.object_entity_id == entity_id)
+            )
+        rows = self.session.execute(query.order_by(models.Claim.confidence.desc()).limit(limit)).all()
+        nodes: dict[int, dict[str, Any]] = {}
+        edges: list[dict[str, Any]] = []
+        for claim, subject in rows:
+            nodes[subject.id] = node_payload(subject)
+            object_payload = None
+            if claim.object_entity_id:
+                object_entity = self.session.get(models.Entity, claim.object_entity_id)
+                if object_entity is not None:
+                    nodes[object_entity.id] = node_payload(object_entity)
+                    object_payload = {"id": object_entity.id, "name": object_entity.name}
+            edges.append(
+                {
+                    "claim_id": claim.id,
+                    "source": subject.id,
+                    "target": claim.object_entity_id,
+                    "target_value": claim.object_value if not claim.object_entity_id else None,
+                    "predicate": claim.predicate,
+                    "status": claim.status,
+                    "confidence": claim.confidence,
+                    "last_seen_at": claim.last_seen_at.isoformat() if claim.last_seen_at else None,
+                    "metadata": claim.metadata_json or {},
+                    "object": object_payload,
+                }
+            )
+        return {"nodes": list(nodes.values()), "edges": edges}
+
+    def brand_products(self, project_id: int, brand_name: str | None = None, limit: int = 100) -> list[dict[str, Any]]:
+        query = (
+            select(models.Claim, models.Entity)
+            .join(models.Entity, models.Claim.subject_entity_id == models.Entity.id)
+            .where(
+                models.Claim.project_id == project_id,
+                models.Claim.status == "validated_claim",
+                models.Claim.predicate == "hasBrand",
+            )
+            .limit(limit)
+        )
+        rows = self.session.execute(query).all()
+        results: list[dict[str, Any]] = []
+        for claim, product in rows:
+            brand = self.session.get(models.Entity, claim.object_entity_id) if claim.object_entity_id else None
+            if brand_name and brand and brand_name.lower() not in brand.name.lower():
+                continue
+            results.append(
+                {
+                    "product": product.name,
+                    "product_type": product.entity_type,
+                    "brand": brand.name if brand else None,
+                    "confidence": claim.confidence,
+                    "claim_id": claim.id,
+                }
+            )
+        return results
+
+    def products_by_tag(
+        self,
+        project_id: int,
+        predicate: str,
+        tag: str | None = None,
+        limit: int = 100,
+    ) -> list[dict[str, Any]]:
+        rows = self.session.execute(
+            select(models.Claim, models.Entity)
+            .join(models.Entity, models.Claim.subject_entity_id == models.Entity.id)
+            .where(
+                models.Claim.project_id == project_id,
+                models.Claim.status == "validated_claim",
+                models.Claim.predicate == predicate,
+            )
+            .order_by(models.Claim.confidence.desc())
+            .limit(limit)
+        ).all()
+        results: list[dict[str, Any]] = []
+        for claim, product in rows:
+            object_entity = self.session.get(models.Entity, claim.object_entity_id) if claim.object_entity_id else None
+            object_name = object_entity.name if object_entity else str(claim.object_value)
+            if tag and tag.lower() not in object_name.lower():
+                continue
+            results.append(
+                {
+                    "product": product.name,
+                    "predicate": predicate,
+                    "tag": object_name,
+                    "confidence": claim.confidence,
+                    "claim_id": claim.id,
+                }
+            )
+        return results
+
+    def trend_summary(self, project_id: int, limit: int = 50) -> list[dict[str, Any]]:
+        rows = self.session.execute(
+            select(models.Claim, models.Entity)
+            .join(models.Entity, models.Claim.object_entity_id == models.Entity.id)
+            .where(
+                models.Claim.project_id == project_id,
+                models.Claim.status == "validated_claim",
+                models.Claim.predicate.in_(
+                    [
+                        "hasTopNote",
+                        "hasMiddleNote",
+                        "hasBaseNote",
+                        "hasAccord",
+                        "evokesMood",
+                        "suitableForSeason",
+                        "suitableForOccasion",
+                        "hasReviewKeyword",
+                    ]
+                ),
+            )
+        ).all()
+        grouped: dict[tuple[str, str], dict[str, Any]] = {}
+        for claim, entity in rows:
+            key = (claim.predicate, entity.canonical_name)
+            item = grouped.setdefault(
+                key,
+                {
+                    "predicate": claim.predicate,
+                    "name": entity.name,
+                    "entity_type": entity.entity_type,
+                    "support_count": 0,
+                    "max_confidence": 0.0,
+                    "last_seen_at": None,
+                },
+            )
+            item["support_count"] += 1
+            item["max_confidence"] = max(item["max_confidence"], claim.confidence)
+            last = claim.last_seen_at.isoformat() if claim.last_seen_at else None
+            item["last_seen_at"] = max(filter(None, [item["last_seen_at"], last]), default=last)
+        return sorted(grouped.values(), key=lambda item: (-item["support_count"], -item["max_confidence"]))[:limit]
+
+    def relation_summary(self, project_id: int, limit: int = 50) -> list[dict[str, Any]]:
+        rows = self.session.execute(
+            select(
+                models.OntologyTriple.predicate,
+                models.OntologyTriple.status,
+                func.count(models.OntologyTriple.id),
+                func.max(models.OntologyTriple.confidence),
+            )
+            .where(models.OntologyTriple.project_id == project_id)
+            .group_by(models.OntologyTriple.predicate, models.OntologyTriple.status)
+            .order_by(func.count(models.OntologyTriple.id).desc())
+            .limit(limit)
+        ).all()
+        return [
+            {
+                "predicate": predicate,
+                "status": status,
+                "support_count": count,
+                "max_confidence": max_confidence or 0.0,
+            }
+            for predicate, status, count, max_confidence in rows
+        ]
+
+    def entity_type_summary(self, project_id: int, limit: int = 50) -> list[dict[str, Any]]:
+        rows = self.session.execute(
+            select(models.Entity.entity_type, func.count(models.Entity.id))
+            .where(models.Entity.project_id == project_id)
+            .group_by(models.Entity.entity_type)
+            .order_by(func.count(models.Entity.id).desc())
+            .limit(limit)
+        ).all()
+        return [{"entity_type": entity_type, "count": count} for entity_type, count in rows]
+
+
+def node_payload(entity: models.Entity) -> dict[str, Any]:
+    return {
+        "id": entity.id,
+        "name": entity.name,
+        "type": entity.entity_type,
+        "canonical_name": entity.canonical_name,
+        "metadata": entity.metadata_json or {},
+    }

ontology_platform/crawler_platform/app/core/research/graph_research_loop.py

@@ -0,0 +1,371 @@
+from __future__ import annotations
+
+from dataclasses import asdict, dataclass, field
+from typing import Any
+from urllib.parse import urldefrag, urlparse
+
+from crawler_platform.app.config.loader import ProjectConfig
+from crawler_platform.app.core.crawler.discovery import discover_links
+from crawler_platform.app.core.crawler.fetchers import RobotsPolicy, make_fetcher
+from crawler_platform.app.core.crawler.page_classifier import classify_page, should_analyze_page
+from crawler_platform.app.core.crawler.plugins import ParserRegistry, default_parser_registry
+from crawler_platform.app.core.database import models
+from crawler_platform.app.core.database.repository import KnowledgeRepository
+from crawler_platform.app.core.extractor.base import ExtractionPageContext, Extractor
+from crawler_platform.app.core.extractor.validation import attach_page_context
+from crawler_platform.app.core.research.entity_expansion import EntityExpansionPlanner
+from crawler_platform.app.core.research.exploration_queue import ExplorationItem, ExplorationQueue
+from crawler_platform.app.core.research.gap_task_planner import GapTaskPlanner
+from crawler_platform.app.core.research.memory_store import ResearchMemoryStore, research_session_payload
+from crawler_platform.app.core.research.relevance_engine import RelevanceEngine
+
+
+@dataclass(slots=True)
+class ResearchLoopResult:
+    session_id: int
+    status: str
+    explored_count: int = 0
+    analyzed_count: int = 0
+    queued_count: int = 0
+    skipped_count: int = 0
+    errors: list[str] = field(default_factory=list)
+    history: list[dict[str, Any]] = field(default_factory=list)
+    queue: list[dict[str, Any]] = field(default_factory=list)
+    memory: dict[str, Any] = field(default_factory=dict)
+
+
+class GraphResearchLoop:
+    def __init__(
+        self,
+        repository: KnowledgeRepository,
+        extractor: Extractor,
+        parser_registry: ParserRegistry | None = None,
+        robots_policy: RobotsPolicy | None = None,
+    ):
+        self.repository = repository
+        self.extractor = extractor
+        self.parser_registry = parser_registry or default_parser_registry()
+        self.robots_policy = robots_policy or RobotsPolicy()
+        self.relevance = RelevanceEngine(repository.session)
+        self.memory = ResearchMemoryStore(repository.session)
+        self.entity_expansion = EntityExpansionPlanner(repository.session)
+        self.gap_task_planner = GapTaskPlanner(repository.session)
+
+    def run(
+        self,
+        *,
+        project_config: ProjectConfig,
+        source_name: str,
+        seed_url: str | None = None,
+        seed_entity_id: int | None = None,
+        goal: str = "Semantic ontology exploration",
+        max_depth: int = 2,
+        max_steps: int = 12,
+        max_branch: int = 8,
+        min_relevance: float = 0.35,
+        same_domain_only: bool = True,
+        analyze_page_types: set[str] | None = None,
+    ) -> ResearchLoopResult:
+        analyze_page_types = analyze_page_types or {"ProductPage", "BrandStoryPage", "ReviewPage"}
+        project = self.repository.upsert_project(project_config)
+        source_config = project_config.source_by_name(source_name)
+        source = self.repository.get_source(project.id, source_name)
+        seed = seed_url or (f"entity:{seed_entity_id}" if seed_entity_id is not None else "")
+        session = self.memory.create_session(
+            project_id=project.id,
+            source_id=source.id,
+            name=goal[:120] or "Semantic research session",
+            goal=goal,
+            seed=seed,
+            config={
+                "max_depth": max_depth,
+                "max_steps": max_steps,
+                "max_branch": max_branch,
+                "min_relevance": min_relevance,
+                "same_domain_only": same_domain_only,
+                "analyze_page_types": sorted(analyze_page_types),
+            },
+        )
+        queue = ExplorationQueue()
+        visited: set[str] = set()
+        seed_host = normalized_host(seed_url or "")
+
+        if seed_url:
+            queue.push(
+                ExplorationItem(
+                    target_type="url",
+                    target=normalize_url(seed_url),
+                    reason="Research seed URL",
+                    priority=1.0,
+                    exploration_depth=0,
+                )
+            )
+        if seed_entity_id is not None:
+            entity = self.repository.session.get(models.Entity, seed_entity_id)
+            if entity is not None and entity.project_id == project.id:
+                entity_score = self.relevance.score_entity(project.id, entity)
+                queue.push(
+                    ExplorationItem(
+                        target_type="entity",
+                        target=str(entity.id),
+                        reason=entity_score.reason,
+                        priority=entity_score.score,
+                        source_entity=entity.name,
+                        relevance_breakdown=entity_score.breakdown,
+                        metadata={"entity_name": entity.name, "entity_type": entity.entity_type},
+                    )
+                )
+        if not seed_url and seed_entity_id is None:
+            for gap_item in self.gap_task_planner.plan(project.id, max_items=max_branch):
+                queue.push(gap_item)
+
+        result = ResearchLoopResult(session_id=session.id, status="running")
+        fetcher = make_fetcher(source_config.fetcher, source_config.rate_limit_per_minute)
+        parser = self.parser_registry.get(source_config.parser)
+
+        for _step in range(max_steps):
+            item = queue.pop()
+            if item is None:
+                break
+            if item.key()[1] in visited:
+                result.skipped_count += 1
+                continue
+            visited.add(item.key()[1])
+            if item.exploration_depth > max_depth:
+                self._record(session, queue, item, {"status": "skipped", "reason": "max exploration depth exceeded"})
+                result.skipped_count += 1
+                continue
+
+            try:
+                if item.target_type == "entity":
+                    outcome = self._expand_entity(project.id, item, queue, max_branch=max_branch)
+                elif item.target_type == "url":
+                    outcome = self._explore_url(
+                        project_config=project_config,
+                        source=source,
+                        source_config=source_config,
+                        parser=parser,
+                        fetcher=fetcher,
+                        item=item,
+                        queue=queue,
+                        visited=visited,
+                        seed_host=seed_host,
+                        same_domain_only=same_domain_only,
+                        analyze_page_types=analyze_page_types,
+                        max_depth=max_depth,
+                        max_branch=max_branch,
+                        min_relevance=min_relevance,
+                    )
+                    if outcome.get("analyzed"):
+                        result.analyzed_count += 1
+                else:
+                    outcome = self._record_gap_task(item)
+                result.explored_count += 1
+                if outcome.get("status") == "skipped":
+                    result.skipped_count += 1
+                self._record(session, queue, item, outcome)
+            except Exception as exc:
+                error = str(exc)
+                result.errors.append(error)
+                result.skipped_count += 1
+                self._record(session, queue, item, {"status": "failed", "error": error})
+
+        result.queued_count = len(queue)
+        payload = research_session_payload(session)
+        result.history = payload["history"]
+        result.queue = payload["queue"]
+        result.memory = payload["memory"]
+        self.memory.finish_session(session, "completed" if not result.errors else "partial", None)
+        result.status = session.status
+        return result
+
+    def _explore_url(
+        self,
+        *,
+        project_config: ProjectConfig,
+        source: models.Source,
+        source_config,
+        parser,
+        fetcher,
+        item: ExplorationItem,
+        queue: ExplorationQueue,
+        visited: set[str],
+        seed_host: str,
+        same_domain_only: bool,
+        analyze_page_types: set[str],
+        max_depth: int,
+        max_branch: int,
+        min_relevance: float,
+    ) -> dict[str, Any]:
+        url = item.target
+        if same_domain_only and seed_host and normalized_host(url) != seed_host:
+            return {"status": "skipped", "reason": "outside same-domain research boundary"}
+        robots_decision = self.robots_policy.check(url, source_config.respect_robots_txt)
+        if not robots_decision.allowed:
+            return {"status": "skipped", "reason": f"{robots_decision.reason}: {url}"}
+
+        fetch_result = fetcher.fetch(url)
+        parser_result = parser.parse(fetch_result.analysis_html, fetch_result.final_url or url)
+        page_type = classify_page(
+            fetch_result.final_url or url,
+            parser_result.title or fetch_result.title,
+            parser_result.raw_text or parser_result.text,
+            fetch_result.analysis_html,
+            parser_result.source_zones or [],
+        )
+        relevance = self.relevance.score_url(
+            project_id=source.project_id,
+            url=fetch_result.final_url or url,
+            label=parser_result.title or fetch_result.title or "",
+            source_trust=source.trust_level,
+            depth=item.exploration_depth,
+            visited=visited,
+            html=fetch_result.analysis_html,
+            text=parser_result.text or parser_result.raw_text,
+        )
+        metadata = {
+            **parser_result.metadata,
+            "research_item": item.to_dict(),
+            "research_relevance": asdict(relevance),
+            "final_url": fetch_result.final_url,
+            "crawl_status": fetch_result.crawl_status,
+            "page_type": page_type,
+            "robots_status": robots_decision.status,
+            "robots_reason": robots_decision.reason,
+            "raw_text_length": len(parser_result.raw_text or ""),
+            "clean_text_length": len(parser_result.text or ""),
+            "main_content_preview": (parser_result.main_content or parser_result.text)[:800],
+        }
+        page = self.repository.upsert_page(
+            project_id=source.project_id,
+            source_id=source.id,
+            url=url,
+            title=parser_result.title or fetch_result.title,
+            status_code=fetch_result.status_code,
+            cleaned_text=parser_result.text,
+            metadata=metadata,
+        )
+        links = discover_links(fetch_result.analysis_html, fetch_result.final_url or url, limit=200)
+        enqueued = 0
+        scored_links = []
+        for link in links:
+            if item.exploration_depth + 1 > max_depth:
+                break
+            candidate_url = normalize_url(link.url)
+            if same_domain_only and seed_host and normalized_host(candidate_url) != seed_host:
+                continue
+            score = self.relevance.score_url(
+                project_id=source.project_id,
+                url=candidate_url,
+                label=link.label,
+                source_trust=source.trust_level,
+                depth=item.exploration_depth + 1,
+                visited=visited,
+            )
+            scored_links.append({"url": candidate_url, "score": score.score, "reason": score.reason})
+            if score.score < min_relevance:
+                continue
+            if queue.push(
+                ExplorationItem(
+                    target_type="url",
+                    target=candidate_url,
+                    reason=f"Discovered from {url}: {score.reason}",
+                    priority=score.score,
+                    exploration_depth=item.exploration_depth + 1,
+                    parent_target=url,
+                    relevance_breakdown=score.breakdown,
+                    metadata={"label": link.label, "kind": link.kind, "page_type": score.page_type},
+                )
+            ):
+                enqueued += 1
+            if enqueued >= max_branch:
+                break
+
+        analyzed = False
+        claim_count = 0
+        entity_count = 0
+        if (
+            fetch_result.crawl_status == "success"
+            and parser_result.extraction_status != "failed"
+            and relevance.score >= min_relevance
+            and should_analyze_page(page_type, analyze_page_types)
+        ):
+            context = ExtractionPageContext(
+                url=url,
+                final_url=fetch_result.final_url,
+                title=parser_result.title or fetch_result.title,
+                page_type=page_type,
+                clean_text=parser_result.text,
+                raw_text=parser_result.raw_text,
+                main_content=parser_result.main_content,
+                clean_markdown=parser_result.clean_markdown,
+                source_zones=parser_result.source_zones or [],
+                crawl_status=fetch_result.crawl_status,
+                extraction_status=parser_result.extraction_status,
+                warnings=[*fetch_result.warnings, *(parser_result.extraction_warnings or [])],
+                metadata=metadata,
+            )
+            bundle = self.extractor.extract_from_context(context, project_config)
+            bundle = attach_page_context(bundle, context)
+            claims = self.repository.save_extraction_bundle(source.project_id, source, page, bundle, project_config)
+            analyzed = True
+            claim_count = len(claims)
+            entity_count = len(bundle.entities)
+
+        return {
+            "status": "explored",
+            "page_id": page.id,
+            "page_type": page_type,
+            "relevance": asdict(relevance),
+            "analyzed": analyzed,
+            "claim_count": claim_count,
+            "entity_count": entity_count,
+            "enqueued": enqueued,
+            "scored_links": scored_links[:20],
+        }
+
+    def _expand_entity(
+        self,
+        project_id: int,
+        item: ExplorationItem,
+        queue: ExplorationQueue,
+        max_branch: int,
+    ) -> dict[str, Any]:
+        candidates = self.entity_expansion.expand_entity(project_id, int(item.target), max_items=max_branch)
+        enqueued = 0
+        for candidate in candidates:
+            candidate.exploration_depth = item.exploration_depth + 1
+            candidate.parent_target = item.target
+            if queue.push(candidate):
+                enqueued += 1
+        entity = self.repository.session.get(models.Entity, int(item.target))
+        if entity is not None:
+            entity.metadata_json = {**(entity.metadata_json or {}), "research_expanded": True}
+        return {"status": "expanded", "enqueued": enqueued, "candidates": [candidate.to_dict() for candidate in candidates]}
+
+    def _record_gap_task(self, item: ExplorationItem) -> dict[str, Any]:
+        return {
+            "status": "candidate",
+            "reason": "knowledge gap converted to research task; external search planner not configured",
+            "gap": item.metadata,
+        }
+
+    def _record(
+        self,
+        session: models.CrawlJob,
+        queue: ExplorationQueue,
+        item: ExplorationItem,
+        outcome: dict[str, Any],
+    ) -> None:
+        self.memory.append_history(session, item, outcome)
+        self.memory.update_queue(session, queue.pending_items(limit=100))
+        self.repository.session.flush()
+
+
+def normalize_url(url: str) -> str:
+    clean, _fragment = urldefrag(str(url).strip())
+    return clean.rstrip("/")
+
+
+def normalized_host(url: str) -> str:
+    return urlparse(str(url)).netloc.lower()

ontology_platform/crawler_platform/app/core/research/memory_store.py

@@ -0,0 +1,163 @@
+from __future__ import annotations
+
+from dataclasses import asdict
+from typing import Any
+
+from sqlalchemy import select
+from sqlalchemy.orm import Session
+
+from crawler_platform.app.core.database import models
+from crawler_platform.app.core.research.exploration_queue import ExplorationItem
+
+
+class ResearchMemoryStore:
+    def __init__(self, session: Session):
+        self.session = session
+
+    def create_session(
+        self,
+        *,
+        project_id: int,
+        source_id: int | None,
+        name: str,
+        goal: str,
+        seed: str,
+        config: dict[str, Any],
+    ) -> models.CrawlJob:
+        job = models.CrawlJob(
+            project_id=project_id,
+            source_id=source_id,
+            url=seed,
+            status="running",
+            started_at=models.utcnow(),
+            metadata_json={
+                "kind": "research_session",
+                "name": name,
+                "goal": goal,
+                "seed": seed,
+                "config": config,
+                "queue": [],
+                "history": [],
+                "memory": {
+                    "visited_targets": [],
+                    "claim_evolution": {},
+                    "conflicts": [],
+                },
+            },
+        )
+        self.session.add(job)
+        self.session.flush()
+        return job
+
+    def append_history(self, job: models.CrawlJob, item: ExplorationItem, outcome: dict[str, Any]) -> None:
+        metadata = dict(job.metadata_json or {})
+        history = list(metadata.get("history") or [])
+        history.append(
+            {
+                "item": item.to_dict(),
+                "outcome": outcome,
+                "created_at": models.utcnow().isoformat(),
+            }
+        )
+        metadata["history"] = history[-200:]
+        memory = dict(metadata.get("memory") or {})
+        visited = list(memory.get("visited_targets") or [])
+        visited.append(f"{item.target_type}:{item.target}")
+        memory["visited_targets"] = sorted(set(visited))
+        memory["claim_evolution"] = self.claim_evolution(job.project_id)
+        memory["conflicts"] = self.conflicts(job.project_id)
+        metadata["memory"] = memory
+        job.metadata_json = metadata
+
+    def update_queue(self, job: models.CrawlJob, queue_items: list[dict[str, Any]]) -> None:
+        metadata = dict(job.metadata_json or {})
+        metadata["queue"] = queue_items
+        job.metadata_json = metadata
+
+    def finish_session(self, job: models.CrawlJob, status: str = "completed", error: str | None = None) -> None:
+        job.status = status
+        job.error = error
+        job.finished_at = models.utcnow()
+
+    def list_sessions(self, project_id: int, limit: int = 25) -> list[dict[str, Any]]:
+        rows = self.session.scalars(
+            select(models.CrawlJob)
+            .where(models.CrawlJob.project_id == project_id)
+            .order_by(models.CrawlJob.scheduled_at.desc())
+            .limit(limit * 4)
+        ).all()
+        return [
+            research_session_payload(row)
+            for row in rows
+            if (row.metadata_json or {}).get("kind") == "research_session"
+        ][:limit]
+
+    def claim_evolution(self, project_id: int) -> dict[str, Any]:
+        rows = self.session.scalars(
+            select(models.Claim).where(
+                models.Claim.project_id == project_id,
+                models.Claim.status == "validated_claim",
+            )
+        ).all()
+        by_predicate: dict[str, dict[str, Any]] = {}
+        for claim in rows:
+            bucket = by_predicate.setdefault(
+                claim.predicate,
+                {"count": 0, "max_confidence": 0.0, "first_seen": None, "last_seen": None},
+            )
+            bucket["count"] += 1
+            bucket["max_confidence"] = max(bucket["max_confidence"], claim.confidence)
+            first = claim.first_seen_at.isoformat() if claim.first_seen_at else None
+            last = claim.last_seen_at.isoformat() if claim.last_seen_at else None
+            bucket["first_seen"] = min(filter(None, [bucket["first_seen"], first]), default=first)
+            bucket["last_seen"] = max(filter(None, [bucket["last_seen"], last]), default=last)
+        return by_predicate
+
+    def conflicts(self, project_id: int) -> list[dict[str, Any]]:
+        rows = self.session.execute(
+            select(models.Claim, models.Entity)
+            .join(models.Entity, models.Claim.subject_entity_id == models.Entity.id)
+            .where(models.Claim.project_id == project_id, models.Claim.status == "validated_claim")
+        ).all()
+        grouped: dict[tuple[int, str], list[models.Claim]] = {}
+        subjects: dict[int, models.Entity] = {}
+        for claim, subject in rows:
+            grouped.setdefault((claim.subject_entity_id, claim.predicate), []).append(claim)
+            subjects[subject.id] = subject
+        conflicts: list[dict[str, Any]] = []
+        for (subject_id, predicate), claims in grouped.items():
+            values = {claim.object_entity_id or repr(claim.object_value) for claim in claims}
+            if len(values) <= 1:
+                continue
+            subject = subjects.get(subject_id)
+            conflicts.append(
+                {
+                    "subject": subject.name if subject else str(subject_id),
+                    "predicate": predicate,
+                    "claim_ids": [claim.id for claim in claims],
+                    "values": list(values),
+                }
+            )
+        return conflicts[:50]
+
+
+def research_session_payload(job: models.CrawlJob) -> dict[str, Any]:
+    metadata = job.metadata_json or {}
+    return {
+        "job_id": job.id,
+        "status": job.status,
+        "name": metadata.get("name"),
+        "goal": metadata.get("goal"),
+        "seed": metadata.get("seed") or job.url,
+        "error": job.error,
+        "started_at": job.started_at.isoformat() if job.started_at else None,
+        "finished_at": job.finished_at.isoformat() if job.finished_at else None,
+        "queue": metadata.get("queue") or [],
+        "history": metadata.get("history") or [],
+        "memory": metadata.get("memory") or {},
+        "config": metadata.get("config") or {},
+    }
+
+
+def item_dict(item: ExplorationItem) -> dict[str, Any]:
+    return asdict(item)

ontology_platform/crawler_platform/app/core/research/relevance_engine.py

@@ -0,0 +1,185 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+import re
+from typing import Any
+from urllib.parse import unquote, urlparse
+
+from sqlalchemy import select
+from sqlalchemy.orm import Session
+
+from crawler_platform.app.core.crawler.page_classifier import classify_page
+from crawler_platform.app.core.database import models
+
+
+HIGH_VALUE_PAGE_TYPES = {
+    "ProductPage": 0.95,
+    "BrandStoryPage": 0.88,
+    "ReviewPage": 0.82,
+    "NoticePage": 0.45,
+    "EventPage": 0.5,
+    "PromotionPage": 0.38,
+    "CategoryPage": 0.34,
+    "SearchPage": 0.22,
+    "BoardPage": 0.18,
+    "UnknownPage": 0.25,
+}
+
+LOW_VALUE_URL_TOKENS = {
+    "/member/",
+    "/order/",
+    "/basket",
+    "/cart",
+    "/login",
+    "/join",
+    "/privacy",
+    "/agreement",
+    "/coupon",
+    "/board/free/modify",
+    "/board/free/reply",
+}
+
+SEMANTIC_TERMS = {
+    "perfume",
+    "parfum",
+    "fragrance",
+    "scent",
+    "note",
+    "accord",
+    "musk",
+    "floral",
+    "citrus",
+    "powdery",
+    "향",
+    "퍼퓸",
+    "노트",
+    "머스크",
+    "플로럴",
+    "시트러스",
+}
+
+
+@dataclass(slots=True)
+class RelevanceScore:
+    score: float
+    reason: str
+    page_type: str
+    breakdown: dict[str, float] = field(default_factory=dict)
+
+
+class RelevanceEngine:
+    def __init__(self, session: Session):
+        self.session = session
+
+    def graph_terms(self, project_id: int, limit: int = 250) -> set[str]:
+        rows = self.session.execute(
+            select(models.Entity.name).where(models.Entity.project_id == project_id).limit(limit)
+        ).all()
+        terms: set[str] = set()
+        for (name,) in rows:
+            terms.update(tokenize(name))
+        return {term for term in terms if len(term) >= 2}
+
+    def score_url(
+        self,
+        *,
+        project_id: int,
+        url: str,
+        label: str = "",
+        source_trust: float = 0.8,
+        depth: int = 0,
+        visited: set[str] | None = None,
+        html: str | None = None,
+        text: str = "",
+    ) -> RelevanceScore:
+        visited = visited or set()
+        normalized = url.strip().rstrip("/")
+        parsed = urlparse(normalized)
+        combined = unquote(f"{normalized} {label} {text[:3000]}").lower()
+        page_type = classify_page(normalized, label, text, html=html)
+        page_type_score = HIGH_VALUE_PAGE_TYPES.get(page_type, 0.25)
+        graph_terms = self.graph_terms(project_id)
+        tokens = tokenize(combined)
+        overlap_count = len(tokens & graph_terms)
+        entity_overlap = min(overlap_count / 6, 1.0)
+        semantic_overlap = min(len(tokens & SEMANTIC_TERMS) / 4, 1.0)
+        novelty = self._novelty_score(project_id, normalized, visited)
+        duplicate_penalty = 1.0 - novelty
+        trust = min(max(source_trust, 0.0), 1.0)
+        depth_penalty = min(depth * 0.08, 0.35)
+        low_value_penalty = 0.45 if any(token in parsed.path.lower() for token in LOW_VALUE_URL_TOKENS) else 0.0
+        evidence_density = min(count_semantic_phrases(combined) / 8, 1.0)
+        score = (
+            page_type_score * 0.28
+            + entity_overlap * 0.22
+            + semantic_overlap * 0.16
+            + novelty * 0.16
+            + trust * 0.08
+            + evidence_density * 0.1
+            - duplicate_penalty * 0.16
+            - depth_penalty
+            - low_value_penalty
+        )
+        score = round(min(max(score, 0.0), 1.0), 4)
+        reason = (
+            f"{page_type}; entity_overlap={overlap_count}; novelty={novelty:.2f}; "
+            f"semantic={semantic_overlap:.2f}; evidence_density={evidence_density:.2f}"
+        )
+        return RelevanceScore(
+            score=score,
+            reason=reason,
+            page_type=page_type,
+            breakdown={
+                "page_type": round(page_type_score, 4),
+                "entity_overlap": round(entity_overlap, 4),
+                "semantic_similarity": round(semantic_overlap, 4),
+                "novelty": round(novelty, 4),
+                "duplicate_penalty": round(duplicate_penalty, 4),
+                "source_trust": round(trust, 4),
+                "evidence_density": round(evidence_density, 4),
+                "depth_penalty": round(depth_penalty, 4),
+                "low_value_penalty": low_value_penalty,
+            },
+        )
+
+    def score_entity(self, project_id: int, entity: models.Entity) -> RelevanceScore:
+        relation_count = self.session.scalar(
+            select(models.Claim.id)
+            .where(
+                models.Claim.project_id == project_id,
+                models.Claim.status == "validated_claim",
+                (models.Claim.subject_entity_id == entity.id) | (models.Claim.object_entity_id == entity.id),
+            )
+            .limit(1)
+        )
+        metadata = entity.metadata_json or {}
+        known = 0.7 if relation_count else 0.35
+        novelty = 0.25 if metadata.get("research_expanded") else 0.95
+        score = round(min(max((known * 0.45) + (novelty * 0.4) + 0.15, 0.0), 1.0), 4)
+        return RelevanceScore(
+            score=score,
+            reason=f"{entity.entity_type} expansion; novelty={novelty:.2f}",
+            page_type="entity",
+            breakdown={"graph_connectivity": known, "novelty": novelty, "type_relevance": 0.15},
+        )
+
+    def _novelty_score(self, project_id: int, url: str, visited: set[str]) -> float:
+        if url in visited:
+            return 0.0
+        existing = self.session.scalar(
+            select(models.Page.id).where(models.Page.project_id == project_id, models.Page.url == url)
+        )
+        return 0.35 if existing else 1.0
+
+
+def tokenize(value: str) -> set[str]:
+    return {
+        token.lower()
+        for token in re.findall(r"[0-9A-Za-z가-힣]{2,}", value or "")
+        if token.strip()
+    }
+
+
+def count_semantic_phrases(value: str) -> int:
+    lowered = (value or "").lower()
+    return sum(1 for term in SEMANTIC_TERMS if term in lowered)

ontology_platform/crawler_platform/app/core/scheduler/__init__.py

@@ -0,0 +1,2 @@
+"""Scheduling primitives for recrawls."""
+

ontology_platform/crawler_platform/app/core/scheduler/update_policy.py

@@ -0,0 +1,11 @@
+from __future__ import annotations
+
+from datetime import datetime, timedelta, timezone
+
+
+def next_crawl_at(policy: dict[str, object] | None, now: datetime | None = None) -> datetime:
+    now = now or datetime.now(timezone.utc)
+    policy = policy or {}
+    interval_days = int(policy.get("interval_days", 7))
+    return now + timedelta(days=interval_days)
+

ontology_platform/crawler_platform/app/domains/__init__.py

@@ -0,0 +1,2 @@
+"""Domain plugins."""
+

ontology_platform/crawler_platform/app/domains/candle/__init__.py

@@ -0,0 +1,2 @@
+"""Candle and diffuser domain extension point."""
+

ontology_platform/crawler_platform/app/domains/coffee/__init__.py

@@ -0,0 +1,2 @@
+"""Coffee domain extension point."""
+

ontology_platform/crawler_platform/app/domains/gift/__init__.py

@@ -0,0 +1,2 @@
+"""Gift recommendation domain extension point."""
+

ontology_platform/crawler_platform/app/domains/perfume/__init__.py

@@ -0,0 +1,2 @@
+"""Perfume domain plugin."""
+

ontology_platform/crawler_platform/app/domains/perfume/extractor.py

@@ -0,0 +1,538 @@
+from __future__ import annotations
+
+import re
+
+from crawler_platform.app.config.loader import ProjectConfig
+from crawler_platform.app.core.extractor.base import ExtractedClaim, ExtractedEntity, ExtractionBundle
+from crawler_platform.app.core.extractor.rule_based import GenericRuleBasedExtractor, find_price, first_non_empty_line
+from crawler_platform.app.core.ontology.mapper import normalize_predicate
+
+
+NOTE_LABELS = {
+    "top_notes": ["top notes", "top note", "opening notes", "탑 노트", "탑노트", "상단 노트"],
+    "middle_notes": ["middle notes", "heart notes", "미들 노트", "하트 노트", "미들노트"],
+    "base_notes": ["base notes", "base note", "베이스 노트", "베이스노트", "잔향"],
+}
+
+FIELD_TO_PREDICATE = {
+    "top_notes": "hasTopNote",
+    "middle_notes": "hasMiddleNote",
+    "base_notes": "hasBaseNote",
+    "accords": "hasAccord",
+    "mood_tags": "evokesMood",
+    "season_tags": "suitableForSeason",
+    "occasion_tags": "suitableForOccasion",
+    "review_keywords": "hasReviewKeyword",
+}
+
+MOOD_KEYWORDS = {
+    "Fresh": ["fresh", "clean", "상쾌", "깨끗", "청량"],
+    "Romantic": ["romantic", "soft", "로맨틱", "부드러운"],
+    "Elegant": ["elegant", "luxury", "우아", "고급"],
+    "Cozy": ["cozy", "warm", "포근", "따뜻"],
+    "Energetic": ["bright", "sparkling", "활기", "발랄"],
+}
+
+SEASON_KEYWORDS = {
+    "Spring": ["spring", "봄"],
+    "Summer": ["summer", "여름"],
+    "Autumn": ["autumn", "fall", "가을"],
+    "Winter": ["winter", "겨울"],
+}
+
+OCCASION_KEYWORDS = {
+    "Daily": ["daily", "everyday", "데일리", "매일"],
+    "Date": ["date", "데이트"],
+    "Office": ["office", "work", "오피스", "출근"],
+    "Evening": ["evening", "night", "저녁", "밤"],
+}
+
+ACCORD_KEYWORDS = [
+    "citrus",
+    "floral",
+    "woody",
+    "musky",
+    "amber",
+    "powdery",
+    "green",
+    "spicy",
+    "sweet",
+    "fresh",
+    "시트러스",
+    "플로럴",
+    "우디",
+    "머스크",
+    "앰버",
+    "파우더리",
+]
+
+REVIEW_KEYWORDS = [
+    "long lasting",
+    "compliment",
+    "too strong",
+    "soft",
+    "fresh",
+    "powdery",
+    "지속력",
+    "잔향",
+    "호불호",
+    "칭찬",
+    "은은",
+    "강한",
+]
+
+
+class PerfumeRuleBasedExtractor(GenericRuleBasedExtractor):
+    name = "perfume_rule_based"
+
+    def extract_entities(self, page_text: str, project_config: ProjectConfig) -> list[ExtractedEntity]:
+        product_cards = extract_product_cards(page_text)
+        if product_cards:
+            brand = valid_brand(extract_brand(page_text, "")) or infer_site_brand(page_text)
+            entities: list[ExtractedEntity] = []
+            if brand:
+                entities.append(ExtractedEntity("Brand", brand, confidence=0.62))
+            for card in product_cards:
+                attrs: dict[str, object] = {"name": card["name"]}
+                if brand:
+                    attrs["brand"] = brand
+                if card.get("price"):
+                    attrs["price"] = card["price"]
+                entities.append(
+                    ExtractedEntity(
+                        "Perfume",
+                        str(card["name"]),
+                        attrs,
+                        evidence_text=str(card.get("evidence") or card["name"]),
+                        confidence=0.74,
+                        metadata={"page_pattern": "product_listing"},
+                    )
+                )
+            return dedupe_entities(entities)
+
+        product_name = extract_product_name(page_text)
+        attrs = {"name": product_name}
+        brand = extract_brand(page_text, product_name)
+        if brand:
+            attrs["brand"] = brand
+        price = find_price(page_text)
+        if price:
+            attrs["price"] = {k: v for k, v in price.items() if k != "evidence"}
+        entities = [ExtractedEntity("Perfume", product_name, attrs, confidence=0.74)]
+        if brand:
+            entities.append(ExtractedEntity("Brand", brand, confidence=0.62))
+        for field, entity_type in [
+            ("top_notes", "Note"),
+            ("middle_notes", "Note"),
+            ("base_notes", "Note"),
+            ("accords", "Accord"),
+            ("mood_tags", "Mood"),
+            ("season_tags", "Season"),
+            ("occasion_tags", "Occasion"),
+            ("review_keywords", "Review"),
+        ]:
+            for value, evidence in extract_field_values(field, page_text):
+                entities.append(ExtractedEntity(entity_type, value, evidence_text=evidence, confidence=0.6))
+        return dedupe_entities(entities)
+
+    def extract_attributes(
+        self,
+        entity: ExtractedEntity,
+        page_text: str,
+        project_config: ProjectConfig,
+    ) -> dict[str, object]:
+        if entity.entity_type != "Perfume":
+            return {}
+        attrs: dict[str, object] = {}
+        longevity = find_metric(page_text, ["longevity", "lasting", "지속력"])
+        sillage = find_metric(page_text, ["sillage", "projection", "확산력", "발향"])
+        if longevity:
+            attrs["longevity"] = longevity
+        if sillage:
+            attrs["sillage"] = sillage
+        gender_bias = find_gender_bias(page_text)
+        if gender_bias:
+            attrs["gender_bias"] = gender_bias
+        return attrs
+
+    def extract_relations(
+        self,
+        entities: list[ExtractedEntity],
+        page_text: str,
+        project_config: ProjectConfig,
+    ) -> list[ExtractedClaim]:
+        product_cards = extract_product_cards(page_text)
+        if product_cards:
+            claims: list[ExtractedClaim] = []
+            brand = next((entity for entity in entities if entity.entity_type == "Brand"), None)
+            for card in product_cards:
+                if brand:
+                    claims.append(
+                        ExtractedClaim(
+                            str(card["name"]),
+                            "Perfume",
+                            "hasBrand",
+                            brand.name,
+                            "Brand",
+                            evidence_text=brand.evidence_text or brand.name,
+                            confidence=0.72,
+                            confidence_reason="site brand inferred from listing page",
+                        )
+                    )
+                if card.get("price"):
+                    claims.append(
+                        ExtractedClaim(
+                            str(card["name"]),
+                            "Perfume",
+                            "hasPrice",
+                            object_value=card["price"],
+                            evidence_text=str(card.get("evidence") or card["name"]),
+                            confidence=0.84,
+                            confidence_reason="Korean product listing price pattern matched",
+                        )
+                    )
+            return claims
+
+        perfume = next((entity for entity in entities if entity.entity_type == "Perfume"), None)
+        if perfume is None:
+            return []
+        claims: list[ExtractedClaim] = []
+        brand = next((entity for entity in entities if entity.entity_type == "Brand"), None)
+        if brand:
+            claims.append(
+                ExtractedClaim(
+                    perfume.name,
+                    "Perfume",
+                    "hasBrand",
+                    brand.name,
+                    "Brand",
+                    evidence_text=brand.evidence_text or brand.name,
+                    confidence=0.78,
+                    confidence_reason="brand pattern matched",
+                )
+            )
+        for field, predicate in FIELD_TO_PREDICATE.items():
+            entity_type = field_entity_type(field)
+            for value, evidence in extract_field_values(field, page_text):
+                claims.append(
+                    ExtractedClaim(
+                        perfume.name,
+                        "Perfume",
+                        predicate,
+                        value,
+                        entity_type,
+                        evidence_text=evidence,
+                        evidence_summary=f"{field} includes {value}",
+                        confidence=field_confidence(field),
+                        confidence_reason=f"{field} rule matched",
+                    )
+                )
+        price = find_price(page_text)
+        if price:
+            claims.append(
+                ExtractedClaim(
+                    perfume.name,
+                    "Perfume",
+                    "hasPrice",
+                    object_value={k: v for k, v in price.items() if k != "evidence"},
+                    evidence_text=price["evidence"],
+                    confidence=0.86,
+                    confidence_reason="price pattern matched",
+                )
+            )
+        return claims
+
+    def normalize_to_ontology(self, bundle: ExtractionBundle, ontology: dict[str, object]) -> ExtractionBundle:
+        for claim in bundle.claims:
+            claim.predicate = normalize_predicate(claim.predicate, ontology)
+        return bundle
+
+
+def extract_product_name(page_text: str) -> str:
+    candidates: list[tuple[int, str]] = []
+    for line in page_text.splitlines()[:12]:
+        clean = line.strip()
+        if not clean or looks_like_navigation(clean) or is_template_placeholder(clean):
+            continue
+        if looks_like_metric_or_price(clean):
+            continue
+        candidates.append((product_line_score(clean), clean[:240]))
+    strong = [candidate for candidate in candidates if candidate[0] > 0]
+    if strong:
+        return max(strong, key=lambda item: item[0])[1]
+    if candidates:
+        return candidates[0][1]
+    return first_non_empty_line(page_text) or "Unknown Perfume"
+
+
+def extract_brand(page_text: str, product_name: str) -> str | None:
+    patterns = [
+        r"(?:brand|브랜드)\s*[:：]\s*(?P<brand>[A-Za-z0-9가-힣 '&.-]{2,80})",
+        r"by\s+(?P<brand>[A-Z][A-Za-z0-9 '&.-]{2,80})",
+    ]
+    for pattern in patterns:
+        match = re.search(pattern, page_text, flags=re.IGNORECASE)
+        if match:
+            return cleanup_value(match.group("brand"))
+    inferred = infer_site_brand(page_text)
+    if inferred:
+        return inferred
+    lines = [line.strip() for line in page_text.splitlines() if line.strip()]
+    if len(lines) >= 2 and lines[1].lower() not in product_name.lower():
+        candidate = cleanup_value(lines[1])
+        if len(candidate) <= 80 and not looks_like_navigation(candidate) and product_line_score(candidate) <= 0:
+            return candidate
+    return None
+
+
+def product_line_score(value: str) -> int:
+    lower = value.lower()
+    score = 0
+    if re.search(r"\d+\s*(?:ml|g|개입)", lower):
+        score += 4
+    if any(keyword in value for keyword in ["향수", "디퓨저", "스프레이", "핸드크림", "미스트", "샤쉐", "퍼퓸"]):
+        score += 3
+    if value.startswith("[") or any(keyword in value for keyword in ["기획", "추가할인", "모음"]):
+        score += 2
+    if "912" in value:
+        score += 2
+    if "시작" in value and not re.search(r"\d+\s*(?:ml|g|개입)", lower):
+        score -= 4
+    return score
+
+
+def looks_like_metric_or_price(value: str) -> bool:
+    clean = value.replace(",", "").strip()
+    if re.fullmatch(r"\d+(?:\.\d+)?", clean):
+        return True
+    return bool(re.fullmatch(r"\d+(?:\.\d+)?\s*(?:원|krw|usd)?", clean, flags=re.IGNORECASE))
+
+
+def extract_product_cards(page_text: str) -> list[dict[str, object]]:
+    lines = [line.strip() for line in page_text.splitlines() if line.strip()]
+    cards: list[dict[str, object]] = []
+    idx = 0
+    while idx < len(lines):
+        if lines[idx] != "상품명":
+            idx += 1
+            continue
+        name, name_idx = next_value_after_label(lines, idx)
+        if not name or is_template_placeholder(name) or name in {":", "상품명"}:
+            idx += 1
+            continue
+        card: dict[str, object] = {"name": cleanup_value(name), "evidence": f"상품명: {name}"}
+        scan_end = next_label_index(lines, "상품명", name_idx + 1) or min(len(lines), name_idx + 12)
+        for price_label in ("할인판매가", "판매가", "price", "Price"):
+            label_idx = find_label_index(lines, price_label, name_idx + 1, scan_end)
+            if label_idx is None:
+                continue
+            raw_price, _price_idx = next_value_after_label(lines, label_idx)
+            parsed = parse_price_value(raw_price)
+            if parsed:
+                card["price"] = parsed
+                card["evidence"] = f"{card['evidence']} / {price_label}: {raw_price}"
+                break
+        cards.append(card)
+        idx = scan_end
+    return dedupe_product_cards(cards)
+
+
+def next_value_after_label(lines: list[str], label_idx: int) -> tuple[str | None, int]:
+    for idx in range(label_idx + 1, min(len(lines), label_idx + 5)):
+        value = cleanup_value(lines[idx])
+        if not value or value == ":":
+            continue
+        return value, idx
+    return None, label_idx
+
+
+def next_label_index(lines: list[str], label: str, start: int) -> int | None:
+    for idx in range(start, len(lines)):
+        if lines[idx] == label:
+            return idx
+    return None
+
+
+def find_label_index(lines: list[str], label: str, start: int, end: int) -> int | None:
+    lower_label = label.lower()
+    for idx in range(start, min(end, len(lines))):
+        if lines[idx].lower() == lower_label:
+            return idx
+    return None
+
+
+def parse_price_value(raw_price: str | None) -> dict[str, object] | None:
+    if not raw_price:
+        return None
+    match = re.search(r"(?P<amount>\d{1,3}(?:,\d{3})*|\d+)\s*(?P<currency>원|KRW|₩|USD|\$)?", raw_price)
+    if not match:
+        return None
+    currency = match.group("currency") or "KRW"
+    if currency in {"원", "₩"}:
+        currency = "KRW"
+    return {"amount": float(match.group("amount").replace(",", "")), "currency": currency}
+
+
+def dedupe_product_cards(cards: list[dict[str, object]]) -> list[dict[str, object]]:
+    seen: set[str] = set()
+    result: list[dict[str, object]] = []
+    for card in cards:
+        key = str(card["name"]).strip().lower()
+        if key in seen:
+            continue
+        seen.add(key)
+        result.append(card)
+    return result
+
+
+def infer_site_brand(page_text: str) -> str | None:
+    if "912 공식 홈페이지" in page_text or "912" in page_text[:500]:
+        return "912"
+    return None
+
+
+def valid_brand(value: str | None) -> str | None:
+    if not value or is_template_placeholder(value):
+        return None
+    return value
+
+
+def extract_field_values(field: str, page_text: str) -> list[tuple[str, str]]:
+    if field in NOTE_LABELS:
+        return extract_labeled_values(page_text, NOTE_LABELS[field])
+    if field == "accords":
+        return keyword_values(page_text, ACCORD_KEYWORDS)
+    if field == "mood_tags":
+        return taxonomy_keyword_values(page_text, MOOD_KEYWORDS)
+    if field == "season_tags":
+        return taxonomy_keyword_values(page_text, SEASON_KEYWORDS)
+    if field == "occasion_tags":
+        return taxonomy_keyword_values(page_text, OCCASION_KEYWORDS)
+    if field == "review_keywords":
+        return keyword_values(page_text, REVIEW_KEYWORDS)
+    return []
+
+
+def extract_labeled_values(page_text: str, labels: list[str]) -> list[tuple[str, str]]:
+    values: list[tuple[str, str]] = []
+    lines = page_text.splitlines()
+    for idx, line in enumerate(lines):
+        lower = line.lower()
+        if any(label.lower() in lower for label in labels):
+            evidence = line
+            raw = line.split(":", 1)[-1] if ":" in line else ""
+            if not raw and idx + 1 < len(lines):
+                raw = lines[idx + 1]
+                evidence = f"{line} {raw}"
+            for value in split_values(raw):
+                values.append((value, evidence[:1000]))
+    return values
+
+
+def split_values(raw: str) -> list[str]:
+    raw = re.sub(r"\([^)]*\)", "", raw)
+    parts = re.split(r"[,/|·ㆍ]+|\band\b| 및 | 그리고 ", raw, flags=re.IGNORECASE)
+    return [cleanup_value(part) for part in parts if 1 < len(cleanup_value(part)) <= 80]
+
+
+def keyword_values(page_text: str, keywords: list[str]) -> list[tuple[str, str]]:
+    lower = page_text.lower()
+    found: list[tuple[str, str]] = []
+    for keyword in keywords:
+        if keyword.lower() in lower:
+            found.append((keyword.title() if keyword.isascii() else keyword, snippet_for(page_text, keyword)))
+    return found
+
+
+def taxonomy_keyword_values(page_text: str, taxonomy: dict[str, list[str]]) -> list[tuple[str, str]]:
+    lower = page_text.lower()
+    found: list[tuple[str, str]] = []
+    for label, keywords in taxonomy.items():
+        for keyword in keywords:
+            if keyword.lower() in lower:
+                found.append((label, snippet_for(page_text, keyword)))
+                break
+    return found
+
+
+def snippet_for(text: str, keyword: str, window: int = 160) -> str:
+    index = text.lower().find(keyword.lower())
+    if index < 0:
+        return keyword
+    start = max(index - window // 2, 0)
+    end = min(index + len(keyword) + window // 2, len(text))
+    return text[start:end].replace("\n", " ")
+
+
+def cleanup_value(value: str) -> str:
+    return re.sub(r"\s+", " ", value.strip(" -:：[]()")).strip()
+
+
+def looks_like_navigation(value: str) -> bool:
+    return value.lower() in {"home", "shop", "menu", "cart", "login", "검색", "장바구니", "홈"}
+
+
+def is_template_placeholder(value: str) -> bool:
+    clean = value.strip()
+    return clean.startswith("{#") or clean.endswith("}") or clean in {
+        "CLONE FRAGRANCE",
+        "NICHE FRAGRANCE",
+        "HOME FRAGRANCE",
+    }
+
+
+def field_entity_type(field: str) -> str:
+    return {
+        "top_notes": "Note",
+        "middle_notes": "Note",
+        "base_notes": "Note",
+        "accords": "Accord",
+        "mood_tags": "Mood",
+        "season_tags": "Season",
+        "occasion_tags": "Occasion",
+        "review_keywords": "Review",
+    }[field]
+
+
+def field_confidence(field: str) -> float:
+    return {
+        "top_notes": 0.82,
+        "middle_notes": 0.82,
+        "base_notes": 0.82,
+        "accords": 0.66,
+        "mood_tags": 0.62,
+        "season_tags": 0.62,
+        "occasion_tags": 0.6,
+        "review_keywords": 0.58,
+    }[field]
+
+
+def find_metric(page_text: str, labels: list[str]) -> str | None:
+    for label in labels:
+        match = re.search(rf"{label}\s*[:：]?\s*(?P<value>\d(?:\.\d)?/5|moderate|strong|weak|long|short|좋음|강함|약함)", page_text, re.IGNORECASE)
+        if match:
+            return cleanup_value(match.group("value"))
+    return None
+
+
+def find_gender_bias(page_text: str) -> str | None:
+    lower = page_text.lower()
+    if "unisex" in lower or "공용" in lower:
+        return "unisex"
+    if "for women" in lower or "여성" in lower:
+        return "feminine"
+    if "for men" in lower or "남성" in lower:
+        return "masculine"
+    return None
+
+
+def dedupe_entities(entities: list[ExtractedEntity]) -> list[ExtractedEntity]:
+    seen: set[tuple[str, str]] = set()
+    result: list[ExtractedEntity] = []
+    for entity in entities:
+        key = (entity.entity_type, entity.name.strip().lower())
+        if key in seen:
+            continue
+        seen.add(key)
+        result.append(entity)
+    return result

ontology_platform/crawler_platform/app/domains/supplement/__init__.py

@@ -0,0 +1,2 @@
+"""Supplement domain extension point."""
+

ontology_platform/crawler_platform/app/domains/tea/__init__.py

@@ -0,0 +1,2 @@
+"""Tea domain extension point."""
+

ontology_platform/crawler_platform/app/main.py

@@ -0,0 +1,45 @@
+from __future__ import annotations
+
+import os
+from pathlib import Path
+
+from fastapi import FastAPI, Request
+from fastapi.responses import FileResponse, JSONResponse
+
+from crawler_platform.app.api.routes import register_routes
+from crawler_platform.app.core.database.session import init_db
+
+
+ONTOLOGY_ROOT = Path(__file__).resolve().parents[2]
+DATABASE_URL = os.getenv(
+    "CRAWLER_DATABASE_URL",
+    f"sqlite:///{(ONTOLOGY_ROOT / 'data' / 'crawler_platform.db').as_posix()}",
+)
+STATIC_DIR = Path(os.getenv("ONTOLOGY_PRODUCT_STATIC_DIR", str(ONTOLOGY_ROOT / "web" / "static")))
+
+app = FastAPI(title="Ontology Crawler Platform", version="0.1.0")
+init_db(DATABASE_URL)
+register_routes(app, DATABASE_URL)
+
+
+@app.exception_handler(KeyError)
+def _handle_key_error(_request: Request, exc: KeyError) -> JSONResponse:
+    return JSONResponse(status_code=404, content={"detail": str(exc.args[0]) if exc.args else "not found"})
+
+@app.get("/static/{asset_path:path}", include_in_schema=False)
+def static_or_spa(asset_path: str):
+    static_root = STATIC_DIR.resolve()
+    target = (static_root / asset_path).resolve()
+    try:
+        target.relative_to(static_root)
+    except ValueError:
+        return JSONResponse(status_code=404, content={"detail": "not found"})
+
+    if target.is_file():
+        return FileResponse(target)
+    return FileResponse(STATIC_DIR / "index.html")
+
+
+@app.get("/", include_in_schema=False)
+def admin_ui():
+    return FileResponse(STATIC_DIR / "index.html")