[crawler]

crawler_platform/__init__.py

@@ -0,0 +1,2 @@
+"""Reusable ontology-centered crawler platform."""
+

crawler_platform/app/__init__.py

@@ -0,0 +1,2 @@
+"""Application package for the crawler platform."""
+

crawler_platform/app/api/__init__.py

@@ -0,0 +1,2 @@
+"""FastAPI routes."""
+

crawler_platform/app/api/routes.py

@@ -0,0 +1,317 @@
+from __future__ import annotations
+
+from dataclasses import asdict
+from typing import Any
+
+from fastapi import HTTPException
+from pydantic import BaseModel, Field
+from sqlalchemy import select
+
+from crawler_platform.app.config.loader import load_project_config
+from crawler_platform.app.core.crawler.discovery import discover_links
+from crawler_platform.app.core.crawler.fetchers import RobotsPolicy, make_fetcher
+from crawler_platform.app.core.crawler.pipeline import CrawlPipeline
+from crawler_platform.app.core.database import models
+from crawler_platform.app.core.database.repository import KnowledgeRepository
+from crawler_platform.app.core.database.session import session_scope
+from crawler_platform.app.core.extractor.ai_provider import list_openai_compatible_models
+from crawler_platform.app.core.extractor.factory import extractor_for_domain
+from crawler_platform.app.core.ontology.definitions import ontology_for_domain
+from crawler_platform.app.core.ontology.mapper import ontology_to_dict
+from crawler_platform.app.core.recommendation.scorer import PreferenceInput, RuleBasedRecommender
+
+
+class CrawlRequest(BaseModel):
+    config_path: str
+    source_name: str
+    url: str
+    extractor_provider: str = "rule_based"
+    extractor_model: str | None = None
+    extractor_base_url: str | None = None
+
+
+class DiscoverRequest(BaseModel):
+    config_path: str
+    source_name: str
+    url: str
+    limit: int = 30
+
+
+class RecommendRequest(BaseModel):
+    project_name: str
+    target_entity_type: str = "Perfume"
+    preferences: dict = Field(default_factory=dict)
+    limit: int = 10
+
+
+class CreateProjectRequest(BaseModel):
+    config_path: str
+
+
+class UpdateClaimConfidenceRequest(BaseModel):
+    confidence: float
+    reason: str | None = None
+
+
+class MergeEntitiesRequest(BaseModel):
+    project_name: str
+    source_entity_id: int
+    target_entity_id: int
+
+
+class ExtractorModelsRequest(BaseModel):
+    provider: str
+    base_url: str | None = None
+
+
+def register_routes(app, database_url: str) -> None:
+    @app.get("/health")
+    def health():
+        return {"ok": True}
+
+    @app.get("/projects")
+    def projects():
+        with session_scope(database_url) as session:
+            rows = session.scalars(select(models.Project).order_by(models.Project.created_at.desc())).all()
+            return [
+                {
+                    "id": project.id,
+                    "name": project.name,
+                    "domain": project.domain,
+                    "created_at": project.created_at.isoformat(),
+                    "updated_at": project.updated_at.isoformat(),
+                }
+                for project in rows
+            ]
+
+    @app.post("/projects")
+    def create_project(request: CreateProjectRequest):
+        config = load_project_config(request.config_path)
+        with session_scope(database_url) as session:
+            project = KnowledgeRepository(session).upsert_project(config)
+            return {"id": project.id, "name": project.name, "domain": project.domain}
+
+    @app.get("/projects/{project_name}")
+    def project_detail(project_name: str):
+        with session_scope(database_url) as session:
+            project = KnowledgeRepository(session).get_project(project_name)
+            sources = session.scalars(select(models.Source).where(models.Source.project_id == project.id)).all()
+            return {
+                "id": project.id,
+                "name": project.name,
+                "domain": project.domain,
+                "config": project.config,
+                "sources": [
+                    {
+                        "id": source.id,
+                        "name": source.name,
+                        "type": source.type,
+                        "trust_level": source.trust_level,
+                        "respect_robots_txt": source.respect_robots_txt,
+                        "rate_limit_per_minute": source.rate_limit_per_minute,
+                    }
+                    for source in sources
+                ],
+            }
+
+    @app.get("/ontology/{domain}")
+    def ontology(domain: str):
+        return ontology_to_dict(ontology_for_domain(domain))
+
+    @app.post("/extractors/models")
+    def extractor_models(request: ExtractorModelsRequest):
+        try:
+            if request.provider == "lm_studio":
+                models = list_openai_compatible_models(request.base_url or "http://localhost:1234/v1")
+                return {"ok": True, "models": models}
+            if request.provider == "openai":
+                import os
+
+                models = list_openai_compatible_models(
+                    request.base_url or "https://api.openai.com/v1",
+                    os.getenv("OPENAI_API_KEY"),
+                )
+                return {"ok": True, "models": models}
+            if request.provider == "ollama":
+                return {"ok": False, "error": "Ollama model listing is not implemented yet. Enter the model manually."}
+            return {"ok": True, "models": [{"id": "rule_based", "owned_by": "local"}]}
+        except Exception as exc:
+            return {"ok": False, "error": str(exc), "models": []}
+
+    @app.post("/crawl")
+    def crawl(request: CrawlRequest):
+        config = load_project_config(request.config_path)
+        with session_scope(database_url) as session:
+            repo = KnowledgeRepository(session)
+            pipeline = CrawlPipeline(
+                repo,
+                extractor_for_domain(
+                    config.domain,
+                    provider=request.extractor_provider,
+                    model=request.extractor_model,
+                    base_url=request.extractor_base_url,
+                ),
+            )
+            try:
+                result = pipeline.crawl_url(config, request.source_name, request.url)
+            except Exception as exc:
+                raise HTTPException(status_code=400, detail=str(exc)) from exc
+            return {"page_id": result.page_id, "claim_count": result.claim_count, "entity_count": result.entity_count}
+
+    @app.post("/discover")
+    def discover(request: DiscoverRequest):
+        config = load_project_config(request.config_path)
+        source_config = config.source_by_name(request.source_name)
+        robots = RobotsPolicy()
+        if not robots.allowed(request.url, source_config.respect_robots_txt):
+            return {"ok": False, "error": "robots.txt does not allow discovery for this URL", "links": []}
+        fetcher = make_fetcher(source_config.fetcher, source_config.rate_limit_per_minute)
+        result = fetcher.fetch(request.url)
+        links = discover_links(result.html, result.final_url or request.url, request.limit)
+        return {
+            "ok": True,
+            "status_code": result.status_code,
+            "final_url": result.final_url,
+            "links": [asdict(link) for link in links],
+        }
+
+    @app.get("/projects/{project_name}/entities")
+    def project_entities(project_name: str, entity_type: str | None = None, limit: int = 50):
+        with session_scope(database_url) as session:
+            project = KnowledgeRepository(session).get_project(project_name)
+            query = select(models.Entity).where(models.Entity.project_id == project.id)
+            if entity_type:
+                query = query.where(models.Entity.entity_type == entity_type)
+            entities = session.scalars(query.limit(limit)).all()
+            return [
+                {
+                    "id": entity.id,
+                    "type": entity.entity_type,
+                    "name": entity.name,
+                    "metadata": entity.metadata_json,
+                }
+                for entity in entities
+            ]
+
+    @app.get("/projects/{project_name}/claims")
+    def project_claims(project_name: str, limit: int = 100):
+        with session_scope(database_url) as session:
+            project = KnowledgeRepository(session).get_project(project_name)
+            rows = session.execute(
+                select(models.Claim, models.Source, models.Page, models.Entity)
+                .join(models.Source, models.Claim.source_id == models.Source.id)
+                .join(models.Page, models.Claim.page_id == models.Page.id, isouter=True)
+                .join(models.Entity, models.Claim.subject_entity_id == models.Entity.id)
+                .where(models.Claim.project_id == project.id)
+                .order_by(models.Claim.last_seen_at.desc())
+                .limit(limit)
+            ).all()
+            results: list[dict[str, Any]] = []
+            for claim, source, page, subject in rows:
+                object_name = None
+                if claim.object_entity_id:
+                    object_entity = session.get(models.Entity, claim.object_entity_id)
+                    object_name = object_entity.name if object_entity else None
+                evidence = session.scalar(
+                    select(models.Evidence)
+                    .where(models.Evidence.claim_id == claim.id)
+                    .order_by(models.Evidence.created_at.desc())
+                )
+                results.append(
+                    {
+                        "id": claim.id,
+                        "subject": subject.name,
+                        "subject_type": subject.entity_type,
+                        "predicate": claim.predicate,
+                        "object": object_name,
+                        "object_value": claim.object_value,
+                        "source": source.name,
+                        "page_url": page.url if page else None,
+                        "confidence": claim.confidence,
+                        "confidence_reason": claim.confidence_reason,
+                        "evidence_text": evidence.evidence_text if evidence else None,
+                        "last_seen_at": claim.last_seen_at.isoformat(),
+                    }
+                )
+            return results
+
+    @app.patch("/claims/{claim_id}/confidence")
+    def update_claim_confidence(claim_id: int, request: UpdateClaimConfidenceRequest):
+        confidence = min(max(request.confidence, 0.0), 1.0)
+        with session_scope(database_url) as session:
+            claim = session.get(models.Claim, claim_id)
+            if claim is None:
+                return {"ok": False, "error": "claim not found"}
+            claim.confidence = confidence
+            claim.confidence_reason = request.reason or "manual admin update"
+            claim.last_seen_at = models.utcnow()
+            return {"ok": True, "claim_id": claim.id, "confidence": claim.confidence}
+
+    @app.post("/entities/merge")
+    def merge_entities(request: MergeEntitiesRequest):
+        with session_scope(database_url) as session:
+            project = KnowledgeRepository(session).get_project(request.project_name)
+            source = session.get(models.Entity, request.source_entity_id)
+            target = session.get(models.Entity, request.target_entity_id)
+            if source is None or target is None or source.project_id != project.id or target.project_id != project.id:
+                return {"ok": False, "error": "entity not found in project"}
+            session.query(models.Claim).filter(models.Claim.subject_entity_id == source.id).update(
+                {models.Claim.subject_entity_id: target.id}
+            )
+            session.query(models.Claim).filter(models.Claim.object_entity_id == source.id).update(
+                {models.Claim.object_entity_id: target.id}
+            )
+            session.query(models.Relation).filter(models.Relation.subject_entity_id == source.id).update(
+                {models.Relation.subject_entity_id: target.id}
+            )
+            session.query(models.Relation).filter(models.Relation.object_entity_id == source.id).update(
+                {models.Relation.object_entity_id: target.id}
+            )
+            source.metadata_json = {**(source.metadata_json or {}), "merged_into": target.id}
+            source.updated_at = models.utcnow()
+            return {"ok": True, "source_entity_id": source.id, "target_entity_id": target.id}
+
+    @app.get("/projects/{project_name}/recommendation-tags")
+    def recommendation_tags(project_name: str):
+        with session_scope(database_url) as session:
+            project = KnowledgeRepository(session).get_project(project_name)
+            tag_predicates = {
+                "hasTopNote",
+                "hasMiddleNote",
+                "hasBaseNote",
+                "hasScentNote",
+                "hasFlavorNote",
+                "evokesMood",
+                "suitableForSeason",
+                "suitableForOccasion",
+                "hasReviewKeyword",
+            }
+            rows = session.execute(
+                select(models.Claim, models.Entity)
+                .join(models.Entity, models.Claim.object_entity_id == models.Entity.id)
+                .where(models.Claim.project_id == project.id, models.Claim.predicate.in_(tag_predicates))
+            ).all()
+            grouped: dict[str, dict[str, Any]] = {}
+            for claim, entity in rows:
+                key = f"{claim.predicate}:{entity.canonical_name}"
+                if key not in grouped:
+                    grouped[key] = {
+                        "predicate": claim.predicate,
+                        "name": entity.name,
+                        "type": entity.entity_type,
+                        "support_count": 0,
+                        "max_confidence": 0.0,
+                    }
+                grouped[key]["support_count"] += 1
+                grouped[key]["max_confidence"] = max(grouped[key]["max_confidence"], claim.confidence)
+            return sorted(grouped.values(), key=lambda item: (item["predicate"], -item["support_count"], item["name"]))
+
+    @app.post("/recommend")
+    def recommend(request: RecommendRequest):
+        with session_scope(database_url) as session:
+            repo = KnowledgeRepository(session)
+            project = repo.get_project(request.project_name)
+            recommender = RuleBasedRecommender(session)
+            pref = PreferenceInput(**request.preferences)
+            items = recommender.recommend(project.id, request.target_entity_type, pref, request.limit)
+            return [asdict(item) for item in items]

crawler_platform/app/cli/__init__.py

@@ -0,0 +1,2 @@
+"""Command line admin tools."""
+

crawler_platform/app/cli/main.py

@@ -0,0 +1,140 @@
+from __future__ import annotations
+
+import argparse
+from dataclasses import asdict
+import json
+
+from sqlalchemy import select
+
+from crawler_platform.app.config.loader import load_project_config
+from crawler_platform.app.core.crawler.pipeline import CrawlPipeline
+from crawler_platform.app.core.database import models
+from crawler_platform.app.core.database.repository import KnowledgeRepository
+from crawler_platform.app.core.database.session import init_db, session_scope
+from crawler_platform.app.core.extractor.factory import extractor_for_domain
+from crawler_platform.app.core.ontology.definitions import ontology_for_domain
+from crawler_platform.app.core.ontology.mapper import ontology_to_dict
+from crawler_platform.app.core.recommendation.scorer import PreferenceInput, RuleBasedRecommender
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(description="Ontology crawler platform admin CLI")
+    parser.add_argument("--db", default="sqlite:///crawler_platform.db", help="SQLAlchemy database URL")
+    sub = parser.add_subparsers(dest="command", required=True)
+
+    sub.add_parser("init-db")
+
+    create_project = sub.add_parser("create-project")
+    create_project.add_argument("--config", required=True)
+
+    ontology = sub.add_parser("ontology")
+    ontology.add_argument("--domain", required=True)
+
+    crawl = sub.add_parser("crawl-url")
+    crawl.add_argument("--config", required=True)
+    crawl.add_argument("--source", required=True)
+    crawl.add_argument("--url", required=True)
+    crawl.add_argument("--extractor-provider", default="rule_based", choices=["rule_based", "openai", "ollama", "lm_studio"])
+    crawl.add_argument("--extractor-model")
+    crawl.add_argument("--extractor-base-url")
+
+    claims = sub.add_parser("claims")
+    claims.add_argument("--project", required=True)
+    claims.add_argument("--limit", type=int, default=20)
+
+    entities = sub.add_parser("entities")
+    entities.add_argument("--project", required=True)
+    entities.add_argument("--type")
+    entities.add_argument("--limit", type=int, default=20)
+
+    recommend = sub.add_parser("recommend")
+    recommend.add_argument("--project", required=True)
+    recommend.add_argument("--target-type", default="Perfume")
+    recommend.add_argument("--preferences-json", default="{}")
+    recommend.add_argument("--limit", type=int, default=10)
+    return parser
+
+
+def main() -> None:
+    args = build_parser().parse_args()
+    if args.command == "init-db":
+        init_db(args.db)
+        print(f"initialized database: {args.db}")
+        return
+    if args.command == "ontology":
+        print(json.dumps(ontology_to_dict(ontology_for_domain(args.domain)), ensure_ascii=False, indent=2))
+        return
+    if args.command == "create-project":
+        config = load_project_config(args.config)
+        with session_scope(args.db) as session:
+            project = KnowledgeRepository(session).upsert_project(config)
+            print(json.dumps({"project_id": project.id, "name": project.name, "domain": project.domain}, ensure_ascii=False))
+        return
+    if args.command == "crawl-url":
+        config = load_project_config(args.config)
+        with session_scope(args.db) as session:
+            repo = KnowledgeRepository(session)
+            pipeline = CrawlPipeline(
+                repo,
+                extractor_for_domain(
+                    config.domain,
+                    provider=args.extractor_provider,
+                    model=args.extractor_model,
+                    base_url=args.extractor_base_url,
+                ),
+            )
+            result = pipeline.crawl_url(config, args.source, args.url)
+            print(json.dumps(asdict(result), ensure_ascii=False))
+        return
+    if args.command == "claims":
+        with session_scope(args.db) as session:
+            project = KnowledgeRepository(session).get_project(args.project)
+            rows = session.execute(
+                select(models.Claim, models.Entity)
+                .join(models.Entity, models.Claim.subject_entity_id == models.Entity.id)
+                .where(models.Claim.project_id == project.id)
+                .limit(args.limit)
+            ).all()
+            print(
+                json.dumps(
+                    [
+                        {
+                            "claim_id": claim.id,
+                            "subject": subject.name,
+                            "predicate": claim.predicate,
+                            "object_entity_id": claim.object_entity_id,
+                            "object_value": claim.object_value,
+                            "confidence": claim.confidence,
+                        }
+                        for claim, subject in rows
+                    ],
+                    ensure_ascii=False,
+                    indent=2,
+                )
+            )
+        return
+    if args.command == "entities":
+        with session_scope(args.db) as session:
+            project = KnowledgeRepository(session).get_project(args.project)
+            query = select(models.Entity).where(models.Entity.project_id == project.id)
+            if args.type:
+                query = query.where(models.Entity.entity_type == args.type)
+            result = session.scalars(query.limit(args.limit)).all()
+            print(
+                json.dumps(
+                    [{"id": entity.id, "type": entity.entity_type, "name": entity.name} for entity in result],
+                    ensure_ascii=False,
+                    indent=2,
+                )
+            )
+        return
+    if args.command == "recommend":
+        preference = PreferenceInput(**json.loads(args.preferences_json))
+        with session_scope(args.db) as session:
+            project = KnowledgeRepository(session).get_project(args.project)
+            result = RuleBasedRecommender(session).recommend(project.id, args.target_type, preference, args.limit)
+            print(json.dumps([asdict(item) for item in result], ensure_ascii=False, indent=2))
+
+
+if __name__ == "__main__":
+    main()

crawler_platform/app/config/__init__.py

@@ -0,0 +1,2 @@
+"""Project configuration loading."""
+

crawler_platform/app/config/loader.py

@@ -0,0 +1,154 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+import json
+from pathlib import Path
+from typing import Any
+
+
+@dataclass(slots=True)
+class SourceConfig:
+    name: str
+    type: str = "unknown"
+    trust_level: float = 0.5
+    base_url: str | None = None
+    allowed_paths: list[str] = field(default_factory=list)
+    parser: str = "generic"
+    fetcher: str = "requests"
+    rate_limit_per_minute: int = 30
+    respect_robots_txt: bool = True
+
+
+@dataclass(slots=True)
+class ProjectConfig:
+    project_name: str
+    domain: str
+    target_entities: list[str]
+    fields: list[str]
+    sources: list[SourceConfig]
+    ontology: dict[str, Any] = field(default_factory=dict)
+    recommendation: dict[str, Any] = field(default_factory=dict)
+    update_policy: dict[str, Any] = field(default_factory=dict)
+
+    def source_by_name(self, name: str) -> SourceConfig:
+        for source in self.sources:
+            if source.name == name:
+                return source
+        raise KeyError(f"Unknown source in project config: {name}")
+
+
+def load_project_config(path: str | Path) -> ProjectConfig:
+    config_path = Path(path)
+    data = _load_mapping(config_path)
+    sources = [SourceConfig(**item) for item in data.get("sources", [])]
+    return ProjectConfig(
+        project_name=data["project_name"],
+        domain=data["domain"],
+        target_entities=list(data.get("target_entities", [])),
+        fields=list(data.get("fields", [])),
+        sources=sources,
+        ontology=dict(data.get("ontology", {})),
+        recommendation=dict(data.get("recommendation", {})),
+        update_policy=dict(data.get("update_policy", {})),
+    )
+
+
+def _load_mapping(path: Path) -> dict[str, Any]:
+    text = path.read_text(encoding="utf-8")
+    if path.suffix.lower() == ".json":
+        return json.loads(text)
+    try:
+        import yaml
+    except ImportError as exc:
+        return _load_simple_yaml(text)
+    return yaml.safe_load(text)
+
+
+def _load_simple_yaml(text: str) -> dict[str, Any]:
+    """Small YAML fallback for project configs when PyYAML is unavailable.
+
+    It intentionally supports only the subset used by the sample project files:
+    nested mappings, lists, booleans, ints, floats, and strings.
+    """
+
+    lines: list[tuple[int, str]] = []
+    for raw_line in text.splitlines():
+        if not raw_line.strip() or raw_line.lstrip().startswith("#"):
+            continue
+        indent = len(raw_line) - len(raw_line.lstrip(" "))
+        lines.append((indent, raw_line.strip()))
+    value, index = _parse_yaml_block(lines, 0, 0)
+    if index != len(lines):
+        raise RuntimeError("Could not parse full YAML config. Install PyYAML for full YAML support.")
+    if not isinstance(value, dict):
+        raise RuntimeError("Project config root must be a mapping.")
+    return value
+
+
+def _parse_yaml_block(lines: list[tuple[int, str]], index: int, indent: int) -> tuple[Any, int]:
+    if index >= len(lines):
+        return {}, index
+    current_indent, content = lines[index]
+    if current_indent < indent:
+        return {}, index
+    if content.startswith("- "):
+        result: list[Any] = []
+        while index < len(lines):
+            item_indent, item_content = lines[index]
+            if item_indent != indent or not item_content.startswith("- "):
+                break
+            item_raw = item_content[2:].strip()
+            index += 1
+            if not item_raw:
+                child, index = _parse_yaml_block(lines, index, indent + 2)
+                result.append(child)
+            elif ":" in item_raw:
+                key, raw_value = _split_key_value(item_raw)
+                item: dict[str, Any] = {key: _parse_scalar(raw_value)} if raw_value else {key: None}
+                if index < len(lines) and lines[index][0] > indent:
+                    child, index = _parse_yaml_block(lines, index, lines[index][0])
+                    if isinstance(child, dict):
+                        item.update(child)
+                result.append(item)
+            else:
+                result.append(_parse_scalar(item_raw))
+        return result, index
+
+    result: dict[str, Any] = {}
+    while index < len(lines):
+        line_indent, line_content = lines[index]
+        if line_indent != indent or line_content.startswith("- "):
+            break
+        key, raw_value = _split_key_value(line_content)
+        index += 1
+        if raw_value:
+            result[key] = _parse_scalar(raw_value)
+        elif index < len(lines) and lines[index][0] > indent:
+            child, index = _parse_yaml_block(lines, index, lines[index][0])
+            result[key] = child
+        else:
+            result[key] = None
+    return result, index
+
+
+def _split_key_value(content: str) -> tuple[str, str]:
+    key, _, raw_value = content.partition(":")
+    return key.strip(), raw_value.strip()
+
+
+def _parse_scalar(value: str) -> Any:
+    if value == "":
+        return None
+    lowered = value.lower()
+    if lowered == "true":
+        return True
+    if lowered == "false":
+        return False
+    if lowered in {"null", "none"}:
+        return None
+    try:
+        if "." in value:
+            return float(value)
+        return int(value)
+    except ValueError:
+        return value.strip("\"'")

crawler_platform/app/core/__init__.py

@@ -0,0 +1,2 @@
+"""Core platform modules."""
+

crawler_platform/app/core/crawler/__init__.py

@@ -0,0 +1,2 @@
+"""Crawler pipeline."""
+

crawler_platform/app/core/crawler/discovery.py

@@ -0,0 +1,52 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from urllib.parse import parse_qs, urljoin, urlparse
+
+from bs4 import BeautifulSoup
+
+
+@dataclass(slots=True)
+class DiscoveredUrl:
+    url: str
+    label: str
+    kind: str = "link"
+
+
+def discover_links(html: str, base_url: str, limit: int = 30) -> list[DiscoveredUrl]:
+    soup = BeautifulSoup(html, "html.parser")
+    seen: set[str] = set()
+    results: list[DiscoveredUrl] = []
+    for anchor in soup.find_all("a", href=True):
+        raw_href = anchor.get("href", "")
+        url = normalize_search_redirect(urljoin(base_url, raw_href))
+        if not url or url in seen or not url.startswith(("http://", "https://")):
+            continue
+        seen.add(url)
+        label = anchor.get_text(" ", strip=True)[:160] or urlparse(url).netloc
+        results.append(DiscoveredUrl(url=url, label=label, kind=classify_url(url)))
+        if len(results) >= limit:
+            break
+    return results
+
+
+def normalize_search_redirect(url: str) -> str:
+    parsed = urlparse(url)
+    query = parse_qs(parsed.query)
+    for key in ("url", "u", "target"):
+        if key in query and query[key]:
+            candidate = query[key][0]
+            if candidate.startswith(("http://", "https://")):
+                return candidate
+    return url
+
+
+def classify_url(url: str) -> str:
+    host = urlparse(url).netloc.lower()
+    if "smartstore.naver.com" in host or "brand.naver.com" in host:
+        return "marketplace_product_or_store"
+    if "shopping.naver.com" in host:
+        return "shopping"
+    if any(token in host for token in ("fragrantica", "official", "perfume", "parfum")):
+        return "product_or_review"
+    return "link"

crawler_platform/app/core/crawler/fetchers.py

@@ -0,0 +1,136 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from pathlib import Path
+import time
+from urllib.parse import urlparse
+from urllib.robotparser import RobotFileParser
+
+
+DEFAULT_USER_AGENT = "OntologyCrawlerBot/0.1 (+contact: admin@example.com)"
+
+
+@dataclass(slots=True)
+class FetchResult:
+    url: str
+    status_code: int | None
+    html: str
+    final_url: str | None = None
+    headers: dict[str, str] = field(default_factory=dict)
+
+
+class RateLimiter:
+    def __init__(self, per_minute: int = 30):
+        self.delay = 60 / max(per_minute, 1)
+        self._last_called = 0.0
+
+    def wait(self) -> None:
+        elapsed = time.monotonic() - self._last_called
+        if elapsed < self.delay:
+            time.sleep(self.delay - elapsed)
+        self._last_called = time.monotonic()
+
+
+class RobotsPolicy:
+    def __init__(self, user_agent: str = DEFAULT_USER_AGENT):
+        self.user_agent = user_agent
+        self._cache: dict[str, RobotFileParser] = {}
+
+    def allowed(self, url: str, respect_robots_txt: bool = True) -> bool:
+        if not respect_robots_txt:
+            return True
+        parsed = urlparse(url)
+        if parsed.scheme in {"", "file"}:
+            return True
+        robots_url = f"{parsed.scheme}://{parsed.netloc}/robots.txt"
+        parser = self._cache.get(robots_url)
+        if parser is None:
+            parser = RobotFileParser()
+            parser.set_url(robots_url)
+            try:
+                parser.read()
+            except Exception:
+                return False
+            self._cache[robots_url] = parser
+        return parser.can_fetch(self.user_agent, url)
+
+
+class BaseFetcher:
+    def fetch(self, url: str) -> FetchResult:
+        raise NotImplementedError
+
+
+class RequestsFetcher(BaseFetcher):
+    def __init__(
+        self,
+        user_agent: str = DEFAULT_USER_AGENT,
+        timeout_seconds: int = 15,
+        retries: int = 2,
+        rate_limit_per_minute: int = 30,
+    ):
+        self.user_agent = user_agent
+        self.timeout_seconds = timeout_seconds
+        self.retries = retries
+        self.rate_limiter = RateLimiter(rate_limit_per_minute)
+
+    def fetch(self, url: str) -> FetchResult:
+        local_path = _local_path_from_url(url)
+        if local_path:
+            return FetchResult(url=url, status_code=200, html=local_path.read_text(encoding="utf-8"), final_url=url)
+
+        import requests
+
+        last_error: Exception | None = None
+        for _ in range(self.retries + 1):
+            self.rate_limiter.wait()
+            try:
+                response = requests.get(
+                    url,
+                    headers={"User-Agent": self.user_agent},
+                    timeout=self.timeout_seconds,
+                )
+                return FetchResult(
+                    url=url,
+                    status_code=response.status_code,
+                    html=response.text,
+                    final_url=response.url,
+                    headers=dict(response.headers),
+                )
+            except requests.RequestException as exc:
+                last_error = exc
+        raise RuntimeError(f"Failed to fetch {url}: {last_error}")
+
+
+class PlaywrightFetcher(BaseFetcher):
+    def __init__(self, user_agent: str = DEFAULT_USER_AGENT, timeout_ms: int = 20000):
+        self.user_agent = user_agent
+        self.timeout_ms = timeout_ms
+
+    def fetch(self, url: str) -> FetchResult:
+        from playwright.sync_api import sync_playwright
+
+        with sync_playwright() as playwright:
+            browser = playwright.chromium.launch(headless=True)
+            page = browser.new_page(user_agent=self.user_agent)
+            response = page.goto(url, wait_until="networkidle", timeout=self.timeout_ms)
+            html = page.content()
+            final_url = page.url
+            status = response.status if response else None
+            browser.close()
+        return FetchResult(url=url, status_code=status, html=html, final_url=final_url)
+
+
+def make_fetcher(kind: str, rate_limit_per_minute: int = 30) -> BaseFetcher:
+    if kind == "playwright":
+        return PlaywrightFetcher()
+    return RequestsFetcher(rate_limit_per_minute=rate_limit_per_minute)
+
+
+def _local_path_from_url(url: str) -> Path | None:
+    if url.startswith("file://"):
+        path = Path(url.removeprefix("file://"))
+        return path if path.exists() and path.is_file() else None
+    path = Path(url)
+    if path.exists() and path.is_file():
+        return path
+    return None

crawler_platform/app/core/crawler/html_cleaner.py

@@ -0,0 +1,25 @@
+from __future__ import annotations
+
+import re
+
+
+def clean_html(html: str) -> tuple[str | None, str]:
+    try:
+        from bs4 import BeautifulSoup
+    except ImportError:
+        text = re.sub(r"<[^>]+>", " ", html)
+        return None, normalize_whitespace(text)
+
+    soup = BeautifulSoup(html, "html.parser")
+    for tag in soup(["script", "style", "noscript", "svg", "iframe"]):
+        tag.decompose()
+    title = soup.title.get_text(" ", strip=True) if soup.title else None
+    main = soup.find("main") or soup.body or soup
+    text = main.get_text("\n", strip=True)
+    return title, normalize_whitespace(text)
+
+
+def normalize_whitespace(text: str) -> str:
+    lines = [" ".join(line.split()) for line in text.splitlines()]
+    return "\n".join(line for line in lines if line)
+

crawler_platform/app/core/crawler/pipeline.py

@@ -0,0 +1,55 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from crawler_platform.app.config.loader import ProjectConfig
+from crawler_platform.app.core.crawler.fetchers import RobotsPolicy, make_fetcher
+from crawler_platform.app.core.crawler.plugins import ParserRegistry, default_parser_registry
+from crawler_platform.app.core.database.repository import KnowledgeRepository
+from crawler_platform.app.core.extractor.base import Extractor
+
+
+@dataclass(slots=True)
+class CrawlResult:
+    page_id: int
+    claim_count: int
+    entity_count: int
+
+
+class CrawlPipeline:
+    def __init__(
+        self,
+        repository: KnowledgeRepository,
+        extractor: Extractor,
+        parser_registry: ParserRegistry | None = None,
+        robots_policy: RobotsPolicy | None = None,
+    ):
+        self.repository = repository
+        self.extractor = extractor
+        self.parser_registry = parser_registry or default_parser_registry()
+        self.robots_policy = robots_policy or RobotsPolicy()
+
+    def crawl_url(self, project_config: ProjectConfig, source_name: str, url: str) -> CrawlResult:
+        project = self.repository.upsert_project(project_config)
+        source_config = project_config.source_by_name(source_name)
+        source = self.repository.get_source(project.id, source_name)
+        if not self.robots_policy.allowed(url, source_config.respect_robots_txt):
+            raise PermissionError(f"robots.txt does not allow crawling: {url}")
+
+        fetcher = make_fetcher(source_config.fetcher, source_config.rate_limit_per_minute)
+        fetch_result = fetcher.fetch(url)
+        parser = self.parser_registry.get(source_config.parser)
+        parsed = parser.parse(fetch_result.html, fetch_result.final_url or url)
+        page = self.repository.upsert_page(
+            project_id=project.id,
+            source_id=source.id,
+            url=url,
+            title=parsed.title,
+            status_code=fetch_result.status_code,
+            cleaned_text=parsed.text,
+            metadata={**parsed.metadata, "final_url": fetch_result.final_url},
+        )
+        bundle = self.extractor.extract(parsed.text, project_config)
+        claims = self.repository.save_extraction_bundle(project.id, source, page, bundle)
+        return CrawlResult(page_id=page.id, claim_count=len(claims), entity_count=len(bundle.entities))
+

crawler_platform/app/core/crawler/plugins.py

@@ -0,0 +1,48 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Protocol
+
+
+@dataclass(slots=True)
+class ParsedPage:
+    title: str | None
+    text: str
+    metadata: dict[str, object]
+
+
+class SiteParser(Protocol):
+    name: str
+
+    def parse(self, html: str, url: str) -> ParsedPage:
+        raise NotImplementedError
+
+
+class ParserRegistry:
+    def __init__(self):
+        self._parsers: dict[str, SiteParser] = {}
+
+    def register(self, parser: SiteParser) -> None:
+        self._parsers[parser.name] = parser
+
+    def get(self, name: str) -> SiteParser:
+        if name not in self._parsers:
+            raise KeyError(f"Parser not registered: {name}")
+        return self._parsers[name]
+
+
+class GenericProductParser:
+    name = "generic"
+
+    def parse(self, html: str, url: str) -> ParsedPage:
+        from crawler_platform.app.core.crawler.html_cleaner import clean_html
+
+        title, text = clean_html(html)
+        return ParsedPage(title=title, text=text, metadata={"parser": self.name, "url": url})
+
+
+def default_parser_registry() -> ParserRegistry:
+    registry = ParserRegistry()
+    registry.register(GenericProductParser())
+    return registry
+

crawler_platform/app/core/database/__init__.py

@@ -0,0 +1,2 @@
+"""Database models and repositories."""
+

crawler_platform/app/core/database/models.py

@@ -0,0 +1,240 @@
+from __future__ import annotations
+
+from datetime import datetime, timezone
+
+from sqlalchemy import (
+    JSON,
+    Boolean,
+    Column,
+    DateTime,
+    Float,
+    ForeignKey,
+    Integer,
+    String,
+    Text,
+    UniqueConstraint,
+)
+from sqlalchemy.orm import declarative_base, relationship
+
+
+Base = declarative_base()
+
+
+def utcnow() -> datetime:
+    return datetime.now(timezone.utc)
+
+
+class Project(Base):
+    __tablename__ = "projects"
+
+    id = Column(Integer, primary_key=True)
+    name = Column(String(160), unique=True, nullable=False, index=True)
+    domain = Column(String(80), nullable=False, index=True)
+    config = Column(JSON, nullable=False, default=dict)
+    created_at = Column(DateTime(timezone=True), default=utcnow, nullable=False)
+    updated_at = Column(DateTime(timezone=True), default=utcnow, nullable=False)
+
+    sources = relationship("Source", back_populates="project", cascade="all, delete-orphan")
+
+
+class Source(Base):
+    __tablename__ = "sources"
+    __table_args__ = (UniqueConstraint("project_id", "name", name="uq_source_project_name"),)
+
+    id = Column(Integer, primary_key=True)
+    project_id = Column(Integer, ForeignKey("projects.id"), nullable=False, index=True)
+    name = Column(String(160), nullable=False)
+    type = Column(String(80), nullable=False, default="unknown")
+    base_url = Column(Text)
+    trust_level = Column(Float, nullable=False, default=0.5)
+    respect_robots_txt = Column(Boolean, nullable=False, default=True)
+    rate_limit_per_minute = Column(Integer, nullable=False, default=30)
+    update_policy = Column(JSON, nullable=False, default=dict)
+    created_at = Column(DateTime(timezone=True), default=utcnow, nullable=False)
+    updated_at = Column(DateTime(timezone=True), default=utcnow, nullable=False)
+
+    project = relationship("Project", back_populates="sources")
+
+
+class Page(Base):
+    __tablename__ = "pages"
+    __table_args__ = (UniqueConstraint("project_id", "url", name="uq_page_project_url"),)
+
+    id = Column(Integer, primary_key=True)
+    project_id = Column(Integer, ForeignKey("projects.id"), nullable=False, index=True)
+    source_id = Column(Integer, ForeignKey("sources.id"), nullable=False, index=True)
+    url = Column(Text, nullable=False)
+    canonical_url = Column(Text)
+    title = Column(Text)
+    content_hash = Column(String(80), index=True)
+    status_code = Column(Integer)
+    fetched_at = Column(DateTime(timezone=True), default=utcnow, nullable=False)
+    cleaned_text_summary = Column(Text)
+    raw_storage_ref = Column(Text)
+    metadata_json = Column(JSON, nullable=False, default=dict)
+
+
+class Entity(Base):
+    __tablename__ = "entities"
+    __table_args__ = (
+        UniqueConstraint("project_id", "entity_type", "canonical_name", name="uq_entity_identity"),
+    )
+
+    id = Column(Integer, primary_key=True)
+    project_id = Column(Integer, ForeignKey("projects.id"), nullable=False, index=True)
+    entity_type = Column(String(120), nullable=False, index=True)
+    name = Column(String(240), nullable=False)
+    canonical_name = Column(String(240), nullable=False, index=True)
+    external_ids = Column(JSON, nullable=False, default=dict)
+    metadata_json = Column(JSON, nullable=False, default=dict)
+    created_at = Column(DateTime(timezone=True), default=utcnow, nullable=False)
+    updated_at = Column(DateTime(timezone=True), default=utcnow, nullable=False)
+
+
+class Attribute(Base):
+    __tablename__ = "attributes"
+    __table_args__ = (
+        UniqueConstraint("entity_id", "name", "source_id", name="uq_attribute_entity_source"),
+    )
+
+    id = Column(Integer, primary_key=True)
+    project_id = Column(Integer, ForeignKey("projects.id"), nullable=False, index=True)
+    entity_id = Column(Integer, ForeignKey("entities.id"), nullable=False, index=True)
+    source_id = Column(Integer, ForeignKey("sources.id"), nullable=True, index=True)
+    name = Column(String(120), nullable=False, index=True)
+    value = Column(JSON, nullable=False)
+    confidence = Column(Float, nullable=False, default=0.5)
+    updated_at = Column(DateTime(timezone=True), default=utcnow, nullable=False)
+
+
+class Relation(Base):
+    __tablename__ = "relations"
+    __table_args__ = (
+        UniqueConstraint("project_id", "subject_entity_id", "predicate", "object_entity_id", name="uq_relation"),
+    )
+
+    id = Column(Integer, primary_key=True)
+    project_id = Column(Integer, ForeignKey("projects.id"), nullable=False, index=True)
+    subject_entity_id = Column(Integer, ForeignKey("entities.id"), nullable=False, index=True)
+    predicate = Column(String(160), nullable=False, index=True)
+    object_entity_id = Column(Integer, ForeignKey("entities.id"), nullable=False, index=True)
+    confidence = Column(Float, nullable=False, default=0.5)
+    support_count = Column(Integer, nullable=False, default=1)
+    updated_at = Column(DateTime(timezone=True), default=utcnow, nullable=False)
+
+
+class Claim(Base):
+    __tablename__ = "claims"
+    __table_args__ = (UniqueConstraint("project_id", "claim_hash", name="uq_claim_hash"),)
+
+    id = Column(Integer, primary_key=True)
+    project_id = Column(Integer, ForeignKey("projects.id"), nullable=False, index=True)
+    source_id = Column(Integer, ForeignKey("sources.id"), nullable=False, index=True)
+    page_id = Column(Integer, ForeignKey("pages.id"), nullable=True, index=True)
+    subject_entity_id = Column(Integer, ForeignKey("entities.id"), nullable=False, index=True)
+    predicate = Column(String(160), nullable=False, index=True)
+    object_entity_id = Column(Integer, ForeignKey("entities.id"), nullable=True, index=True)
+    object_value = Column(JSON, nullable=True)
+    value_type = Column(String(80), nullable=False, default="entity")
+    claim_hash = Column(String(80), nullable=False, index=True)
+    confidence = Column(Float, nullable=False, default=0.5)
+    confidence_reason = Column(Text)
+    extraction_method = Column(String(120), nullable=False, default="rule_based")
+    status = Column(String(40), nullable=False, default="active")
+    first_seen_at = Column(DateTime(timezone=True), default=utcnow, nullable=False)
+    last_seen_at = Column(DateTime(timezone=True), default=utcnow, nullable=False)
+    valid_until = Column(DateTime(timezone=True), nullable=True)
+    metadata_json = Column(JSON, nullable=False, default=dict)
+
+    evidence_items = relationship("Evidence", back_populates="claim", cascade="all, delete-orphan")
+
+
+class Evidence(Base):
+    __tablename__ = "evidence"
+
+    id = Column(Integer, primary_key=True)
+    project_id = Column(Integer, ForeignKey("projects.id"), nullable=False, index=True)
+    claim_id = Column(Integer, ForeignKey("claims.id"), nullable=False, index=True)
+    page_id = Column(Integer, ForeignKey("pages.id"), nullable=True, index=True)
+    evidence_text = Column(Text, nullable=False)
+    evidence_summary = Column(Text)
+    selector = Column(Text)
+    start_offset = Column(Integer)
+    end_offset = Column(Integer)
+    created_at = Column(DateTime(timezone=True), default=utcnow, nullable=False)
+
+    claim = relationship("Claim", back_populates="evidence_items")
+
+
+class ExtractionLog(Base):
+    __tablename__ = "extraction_logs"
+
+    id = Column(Integer, primary_key=True)
+    project_id = Column(Integer, ForeignKey("projects.id"), nullable=False, index=True)
+    page_id = Column(Integer, ForeignKey("pages.id"), nullable=True, index=True)
+    extractor_name = Column(String(160), nullable=False)
+    provider = Column(String(120), nullable=False, default="rule_based")
+    input_hash = Column(String(80))
+    raw_output = Column(JSON, nullable=False, default=dict)
+    error = Column(Text)
+    created_at = Column(DateTime(timezone=True), default=utcnow, nullable=False)
+
+
+class CrawlJob(Base):
+    __tablename__ = "crawl_jobs"
+
+    id = Column(Integer, primary_key=True)
+    project_id = Column(Integer, ForeignKey("projects.id"), nullable=False, index=True)
+    source_id = Column(Integer, ForeignKey("sources.id"), nullable=True, index=True)
+    url = Column(Text)
+    status = Column(String(40), nullable=False, default="pending")
+    priority = Column(Integer, nullable=False, default=100)
+    scheduled_at = Column(DateTime(timezone=True), default=utcnow, nullable=False)
+    started_at = Column(DateTime(timezone=True), nullable=True)
+    finished_at = Column(DateTime(timezone=True), nullable=True)
+    error = Column(Text)
+    metadata_json = Column(JSON, nullable=False, default=dict)
+
+
+class UserProfile(Base):
+    __tablename__ = "user_profiles"
+
+    id = Column(Integer, primary_key=True)
+    project_id = Column(Integer, ForeignKey("projects.id"), nullable=False, index=True)
+    external_user_id = Column(String(160), nullable=False, index=True)
+    display_name = Column(String(160))
+    context = Column(JSON, nullable=False, default=dict)
+    created_at = Column(DateTime(timezone=True), default=utcnow, nullable=False)
+    updated_at = Column(DateTime(timezone=True), default=utcnow, nullable=False)
+
+
+class UserPreference(Base):
+    __tablename__ = "user_preferences"
+
+    id = Column(Integer, primary_key=True)
+    project_id = Column(Integer, ForeignKey("projects.id"), nullable=False, index=True)
+    user_profile_id = Column(Integer, ForeignKey("user_profiles.id"), nullable=False, index=True)
+    likes = Column(JSON, nullable=False, default=list)
+    dislikes = Column(JSON, nullable=False, default=list)
+    preferred_moods = Column(JSON, nullable=False, default=list)
+    preferred_notes = Column(JSON, nullable=False, default=list)
+    avoided_notes = Column(JSON, nullable=False, default=list)
+    price_preference = Column(JSON, nullable=False, default=dict)
+    season_context = Column(String(80))
+    occasion_context = Column(String(120))
+    feedback_history = Column(JSON, nullable=False, default=list)
+    updated_at = Column(DateTime(timezone=True), default=utcnow, nullable=False)
+
+
+class FeedbackLog(Base):
+    __tablename__ = "feedback_logs"
+
+    id = Column(Integer, primary_key=True)
+    project_id = Column(Integer, ForeignKey("projects.id"), nullable=False, index=True)
+    user_profile_id = Column(Integer, ForeignKey("user_profiles.id"), nullable=False, index=True)
+    entity_id = Column(Integer, ForeignKey("entities.id"), nullable=True, index=True)
+    action = Column(String(80), nullable=False)
+    score = Column(Float)
+    reason = Column(Text)
+    metadata_json = Column(JSON, nullable=False, default=dict)
+    created_at = Column(DateTime(timezone=True), default=utcnow, nullable=False)

crawler_platform/app/core/database/repository.py

@@ -0,0 +1,333 @@
+from __future__ import annotations
+
+from dataclasses import asdict
+import hashlib
+import json
+from typing import Any
+
+from sqlalchemy import select
+from sqlalchemy.orm import Session
+
+from crawler_platform.app.config.loader import ProjectConfig, SourceConfig
+from crawler_platform.app.core.database import models
+from crawler_platform.app.core.extractor.base import ExtractedClaim, ExtractedEntity, ExtractionBundle
+
+
+def canonicalize(value: str) -> str:
+    return " ".join(value.strip().lower().split())
+
+
+def short_hash(text: str) -> str:
+    return hashlib.sha256(text.encode("utf-8")).hexdigest()
+
+
+class KnowledgeRepository:
+    def __init__(self, session: Session):
+        self.session = session
+
+    def upsert_project(self, config: ProjectConfig) -> models.Project:
+        project = self.session.scalar(select(models.Project).where(models.Project.name == config.project_name))
+        config_dict = _project_config_to_dict(config)
+        if project is None:
+            project = models.Project(name=config.project_name, domain=config.domain, config=config_dict)
+            self.session.add(project)
+            self.session.flush()
+        else:
+            project.domain = config.domain
+            project.config = config_dict
+            project.updated_at = models.utcnow()
+        for source_config in config.sources:
+            self.upsert_source(project, source_config)
+        return project
+
+    def upsert_source(self, project: models.Project, source_config: SourceConfig) -> models.Source:
+        source = self.session.scalar(
+            select(models.Source).where(
+                models.Source.project_id == project.id,
+                models.Source.name == source_config.name,
+            )
+        )
+        if source is None:
+            source = models.Source(project_id=project.id, name=source_config.name)
+            self.session.add(source)
+            self.session.flush()
+        source.type = source_config.type
+        source.base_url = source_config.base_url
+        source.trust_level = source_config.trust_level
+        source.respect_robots_txt = source_config.respect_robots_txt
+        source.rate_limit_per_minute = source_config.rate_limit_per_minute
+        source.updated_at = models.utcnow()
+        return source
+
+    def get_project(self, project_name: str) -> models.Project:
+        project = self.session.scalar(select(models.Project).where(models.Project.name == project_name))
+        if project is None:
+            raise KeyError(f"Project not found: {project_name}")
+        return project
+
+    def get_source(self, project_id: int, source_name: str) -> models.Source:
+        source = self.session.scalar(
+            select(models.Source).where(models.Source.project_id == project_id, models.Source.name == source_name)
+        )
+        if source is None:
+            raise KeyError(f"Source not found: {source_name}")
+        return source
+
+    def upsert_page(
+        self,
+        project_id: int,
+        source_id: int,
+        url: str,
+        title: str | None,
+        status_code: int | None,
+        cleaned_text: str,
+        metadata: dict[str, Any] | None = None,
+    ) -> models.Page:
+        page = self.session.scalar(select(models.Page).where(models.Page.project_id == project_id, models.Page.url == url))
+        digest = short_hash(cleaned_text)
+        summary = cleaned_text[:2000]
+        if page is None:
+            page = models.Page(project_id=project_id, source_id=source_id, url=url)
+            self.session.add(page)
+            self.session.flush()
+        page.title = title
+        page.status_code = status_code
+        page.content_hash = digest
+        page.cleaned_text_summary = summary
+        page.metadata_json = metadata or {}
+        page.fetched_at = models.utcnow()
+        return page
+
+    def upsert_entity(
+        self,
+        project_id: int,
+        entity_type: str,
+        name: str,
+        metadata: dict[str, Any] | None = None,
+    ) -> models.Entity:
+        canonical_name = canonicalize(name)
+        entity = self.session.scalar(
+            select(models.Entity).where(
+                models.Entity.project_id == project_id,
+                models.Entity.entity_type == entity_type,
+                models.Entity.canonical_name == canonical_name,
+            )
+        )
+        if entity is None:
+            entity = models.Entity(
+                project_id=project_id,
+                entity_type=entity_type,
+                name=name.strip(),
+                canonical_name=canonical_name,
+                metadata_json=metadata or {},
+            )
+            self.session.add(entity)
+            self.session.flush()
+        else:
+            entity.metadata_json = {**(entity.metadata_json or {}), **(metadata or {})}
+            entity.updated_at = models.utcnow()
+        return entity
+
+    def save_extraction_bundle(
+        self,
+        project_id: int,
+        source: models.Source,
+        page: models.Page,
+        bundle: ExtractionBundle,
+    ) -> list[models.Claim]:
+        entity_index: dict[tuple[str, str], models.Entity] = {}
+        for extracted_entity in bundle.entities:
+            entity = self._save_extracted_entity(project_id, source.id, extracted_entity)
+            entity_index[(extracted_entity.entity_type, canonicalize(extracted_entity.name))] = entity
+
+        claims: list[models.Claim] = []
+        for extracted_claim in bundle.claims:
+            subject = self._entity_for_claim(project_id, extracted_claim.subject_type, extracted_claim.subject_name, entity_index)
+            object_entity = None
+            if extracted_claim.object_name and extracted_claim.object_type:
+                object_entity = self._entity_for_claim(
+                    project_id,
+                    extracted_claim.object_type,
+                    extracted_claim.object_name,
+                    entity_index,
+                )
+            confidence = combine_confidence(extracted_claim.confidence, source.trust_level)
+            claim_hash = make_claim_hash(
+                project_id=project_id,
+                source_id=source.id,
+                subject_entity_id=subject.id,
+                predicate=extracted_claim.predicate,
+                object_entity_id=object_entity.id if object_entity else None,
+                object_value=extracted_claim.object_value,
+            )
+            claim = self.session.scalar(
+                select(models.Claim).where(
+                    models.Claim.project_id == project_id,
+                    models.Claim.claim_hash == claim_hash,
+                )
+            )
+            if claim is None:
+                claim = models.Claim(
+                    project_id=project_id,
+                    source_id=source.id,
+                    page_id=page.id,
+                    subject_entity_id=subject.id,
+                    predicate=extracted_claim.predicate,
+                    object_entity_id=object_entity.id if object_entity else None,
+                    object_value=extracted_claim.object_value,
+                    value_type="entity" if object_entity else "literal",
+                    claim_hash=claim_hash,
+                    confidence=confidence,
+                    confidence_reason=extracted_claim.confidence_reason,
+                    extraction_method=bundle.extractor_name,
+                    metadata_json=extracted_claim.metadata,
+                )
+                self.session.add(claim)
+                self.session.flush()
+            else:
+                claim.page_id = page.id
+                claim.last_seen_at = models.utcnow()
+                claim.confidence = max(claim.confidence, confidence)
+                claim.confidence_reason = extracted_claim.confidence_reason or claim.confidence_reason
+                claim.metadata_json = {**(claim.metadata_json or {}), **extracted_claim.metadata}
+            if extracted_claim.evidence_text:
+                self.session.add(
+                    models.Evidence(
+                        project_id=project_id,
+                        claim_id=claim.id,
+                        page_id=page.id,
+                        evidence_text=extracted_claim.evidence_text[:1000],
+                        evidence_summary=extracted_claim.evidence_summary,
+                    )
+                )
+            if object_entity:
+                self._upsert_relation(project_id, subject.id, extracted_claim.predicate, object_entity.id, confidence)
+            claims.append(claim)
+
+        self.session.add(
+            models.ExtractionLog(
+                project_id=project_id,
+                page_id=page.id,
+                extractor_name=bundle.extractor_name,
+                provider=bundle.provider,
+                input_hash=page.content_hash,
+                raw_output=bundle.raw_output,
+            )
+        )
+        return claims
+
+    def _save_extracted_entity(
+        self,
+        project_id: int,
+        source_id: int,
+        extracted_entity: ExtractedEntity,
+    ) -> models.Entity:
+        entity = self.upsert_entity(
+            project_id,
+            extracted_entity.entity_type,
+            extracted_entity.name,
+            extracted_entity.metadata,
+        )
+        for name, value in extracted_entity.attributes.items():
+            attribute = self.session.scalar(
+                select(models.Attribute).where(
+                    models.Attribute.entity_id == entity.id,
+                    models.Attribute.name == name,
+                    models.Attribute.source_id == source_id,
+                )
+            )
+            if attribute is None:
+                attribute = models.Attribute(
+                    project_id=project_id,
+                    entity_id=entity.id,
+                    source_id=source_id,
+                    name=name,
+                    value=value,
+                    confidence=extracted_entity.confidence,
+                )
+                self.session.add(attribute)
+            else:
+                attribute.value = value
+                attribute.confidence = max(attribute.confidence, extracted_entity.confidence)
+                attribute.updated_at = models.utcnow()
+        return entity
+
+    def _entity_for_claim(
+        self,
+        project_id: int,
+        entity_type: str,
+        name: str,
+        entity_index: dict[tuple[str, str], models.Entity],
+    ) -> models.Entity:
+        key = (entity_type, canonicalize(name))
+        if key not in entity_index:
+            entity_index[key] = self.upsert_entity(project_id, entity_type, name)
+        return entity_index[key]
+
+    def _upsert_relation(
+        self,
+        project_id: int,
+        subject_id: int,
+        predicate: str,
+        object_id: int,
+        confidence: float,
+    ) -> models.Relation:
+        relation = self.session.scalar(
+            select(models.Relation).where(
+                models.Relation.project_id == project_id,
+                models.Relation.subject_entity_id == subject_id,
+                models.Relation.predicate == predicate,
+                models.Relation.object_entity_id == object_id,
+            )
+        )
+        if relation is None:
+            relation = models.Relation(
+                project_id=project_id,
+                subject_entity_id=subject_id,
+                predicate=predicate,
+                object_entity_id=object_id,
+                confidence=confidence,
+            )
+            self.session.add(relation)
+            self.session.flush()
+        else:
+            relation.support_count += 1
+            relation.confidence = max(relation.confidence, confidence)
+            relation.updated_at = models.utcnow()
+        return relation
+
+
+def _project_config_to_dict(config: ProjectConfig) -> dict[str, Any]:
+    return {
+        "project_name": config.project_name,
+        "domain": config.domain,
+        "target_entities": config.target_entities,
+        "fields": config.fields,
+        "sources": [asdict(source) for source in config.sources],
+        "ontology": config.ontology,
+        "recommendation": config.recommendation,
+        "update_policy": config.update_policy,
+    }
+
+
+def combine_confidence(extraction_confidence: float, source_trust: float) -> float:
+    return round(min(max((extraction_confidence * 0.7) + (source_trust * 0.3), 0.0), 1.0), 4)
+
+
+def make_claim_hash(
+    project_id: int,
+    source_id: int,
+    subject_entity_id: int,
+    predicate: str,
+    object_entity_id: int | None,
+    object_value: Any | None,
+) -> str:
+    payload = {
+        "project_id": project_id,
+        "source_id": source_id,
+        "subject_entity_id": subject_entity_id,
+        "predicate": predicate,
+        "object_entity_id": object_entity_id,
+        "object_value": object_value,
+    }
+    return hashlib.sha256(json.dumps(payload, sort_keys=True, default=str).encode("utf-8")).hexdigest()

crawler_platform/app/core/database/session.py

@@ -0,0 +1,39 @@
+from __future__ import annotations
+
+from contextlib import contextmanager
+from typing import Iterator
+
+from sqlalchemy import create_engine
+from sqlalchemy.orm import Session, sessionmaker
+
+from crawler_platform.app.core.database.models import Base
+
+
+def make_engine(database_url: str = "sqlite:///crawler_platform.db"):
+    connect_args = {"check_same_thread": False} if database_url.startswith("sqlite") else {}
+    return create_engine(database_url, future=True, connect_args=connect_args)
+
+
+def init_db(database_url: str = "sqlite:///crawler_platform.db") -> None:
+    engine = make_engine(database_url)
+    Base.metadata.create_all(engine)
+
+
+def make_session_factory(database_url: str = "sqlite:///crawler_platform.db") -> sessionmaker[Session]:
+    engine = make_engine(database_url)
+    return sessionmaker(bind=engine, expire_on_commit=False, class_=Session, future=True)
+
+
+@contextmanager
+def session_scope(database_url: str = "sqlite:///crawler_platform.db") -> Iterator[Session]:
+    factory = make_session_factory(database_url)
+    session = factory()
+    try:
+        yield session
+        session.commit()
+    except Exception:
+        session.rollback()
+        raise
+    finally:
+        session.close()
+

crawler_platform/app/core/extractor/__init__.py

@@ -0,0 +1,2 @@
+"""Extractor provider interfaces and implementations."""
+

crawler_platform/app/core/extractor/ai_provider.py

@@ -0,0 +1,388 @@
+from __future__ import annotations
+
+import json
+import os
+import re
+from typing import Any
+
+import requests
+
+from crawler_platform.app.config.loader import ProjectConfig
+from crawler_platform.app.core.extractor.base import (
+    AIExtractor,
+    ExtractedClaim,
+    ExtractedEntity,
+    ExtractionBundle,
+)
+from crawler_platform.app.core.ontology.mapper import normalize_predicate
+from crawler_platform.app.domains.perfume.extractor import PerfumeRuleBasedExtractor
+
+
+class LLMJsonExtractor(AIExtractor):
+    name = "llm_json_extractor"
+
+    def __init__(
+        self,
+        domain: str,
+        provider: str,
+        model: str | None = None,
+        base_url: str | None = None,
+        timeout_seconds: int = 300,
+    ):
+        self.domain = domain
+        self.provider = provider
+        self.model = model
+        self.base_url = base_url
+        self.timeout_seconds = timeout_seconds
+
+    def extract(self, page_text: str, project_config: ProjectConfig) -> ExtractionBundle:
+        try:
+            raw = self.complete_json(page_text, project_config)
+        except Exception as exc:
+            return self._fallback_bundle(page_text, project_config, str(exc))
+        bundle = ExtractionBundle(
+            entities=parse_entities(raw.get("entities", [])),
+            claims=parse_claims(raw.get("claims", [])),
+            extractor_name=self.name,
+            provider=self.provider,
+            raw_output={
+                "provider": self.provider,
+                "model": self.model,
+                "entity_count": len(raw.get("entities", [])),
+                "claim_count": len(raw.get("claims", [])),
+            },
+        )
+        if not bundle.entities or not bundle.claims:
+            return self._fallback_bundle(page_text, project_config, "AI returned no usable entities or claims")
+        return self.normalize_to_ontology(bundle, project_config.ontology)
+
+    def extract_entities(self, page_text: str, project_config: ProjectConfig) -> list[ExtractedEntity]:
+        return self.extract(page_text, project_config).entities
+
+    def extract_attributes(
+        self,
+        entity: ExtractedEntity,
+        page_text: str,
+        project_config: ProjectConfig,
+    ) -> dict[str, Any]:
+        return entity.attributes
+
+    def extract_relations(
+        self,
+        entities: list[ExtractedEntity],
+        page_text: str,
+        project_config: ProjectConfig,
+    ) -> list[ExtractedClaim]:
+        return []
+
+    def normalize_to_ontology(self, bundle: ExtractionBundle, ontology: dict[str, Any]) -> ExtractionBundle:
+        for claim in bundle.claims:
+            claim.predicate = normalize_predicate(claim.predicate, ontology)
+        return bundle
+
+    def complete_json(self, page_text: str, project_config: ProjectConfig) -> dict[str, Any]:
+        prompt = build_extraction_prompt(page_text, project_config)
+        if self.provider == "openai":
+            return self._complete_openai_compatible(prompt, "OPENAI_API_KEY", "OPENAI_MODEL", self.base_url)
+        if self.provider == "lm_studio":
+            return self._complete_openai_compatible(
+                prompt,
+                "LM_STUDIO_API_KEY",
+                "LM_STUDIO_MODEL",
+                normalize_openai_chat_url(self.base_url or "http://localhost:1234/v1"),
+                api_key_optional=True,
+            )
+        if self.provider == "ollama":
+            return self._complete_ollama(prompt)
+        raise ValueError(f"Unsupported AI extractor provider: {self.provider}")
+
+    def _fallback_bundle(self, page_text: str, project_config: ProjectConfig, error: str) -> ExtractionBundle:
+        if project_config.domain == "perfume":
+            bundle = PerfumeRuleBasedExtractor().extract(page_text, project_config)
+        else:
+            from crawler_platform.app.core.extractor.rule_based import GenericRuleBasedExtractor
+
+            bundle = GenericRuleBasedExtractor().extract(page_text, project_config)
+        bundle.extractor_name = f"{self.name}_with_rule_fallback"
+        bundle.provider = f"{self.provider}_fallback"
+        bundle.raw_output = {
+            **bundle.raw_output,
+            "ai_provider": self.provider,
+            "ai_model": self.model,
+            "ai_error": error,
+            "fallback": "rule_based",
+        }
+        for entity in bundle.entities:
+            entity.metadata["ai_fallback_reason"] = error
+        for claim in bundle.claims:
+            claim.metadata["ai_fallback_reason"] = error
+            claim.confidence_reason = f"{claim.confidence_reason}; AI fallback: {error}" if claim.confidence_reason else error
+        return bundle
+
+    def _complete_openai_compatible(
+        self,
+        prompt: str,
+        api_key_env: str,
+        model_env: str,
+        endpoint: str | None,
+        api_key_optional: bool = False,
+    ) -> dict[str, Any]:
+        api_key = os.getenv(api_key_env)
+        model = self.model or os.getenv(model_env)
+        if not model and api_key_optional and endpoint:
+            model = first_openai_compatible_model(endpoint, api_key)
+        if not model:
+            raise RuntimeError(f"AI model is required. Set UI model field or {model_env}.")
+        if not api_key and not api_key_optional:
+            raise RuntimeError(f"API key is required. Set {api_key_env}.")
+        url = endpoint or "https://api.openai.com/v1/chat/completions"
+        headers = {"Content-Type": "application/json"}
+        if api_key:
+            headers["Authorization"] = f"Bearer {api_key}"
+        response = requests.post(
+            url,
+            headers=headers,
+            json={
+                "model": model,
+                "messages": [
+                    {"role": "system", "content": "Extract ontology knowledge as strict JSON only."},
+                    {"role": "user", "content": prompt},
+                ],
+                "temperature": 0,
+            },
+            timeout=self.timeout_seconds,
+        )
+        if response.status_code >= 400:
+            raise RuntimeError(f"LLM request failed {response.status_code}: {response.text[:1000]}")
+        data = response.json()
+        content = data["choices"][0]["message"]["content"]
+        return parse_json_content(content, retry=lambda bad: self._repair_json_with_model(bad, endpoint, headers, model))
+
+    def _complete_ollama(self, prompt: str) -> dict[str, Any]:
+        model = self.model or os.getenv("OLLAMA_MODEL")
+        if not model:
+            raise RuntimeError("Ollama model is required. Set UI model field or OLLAMA_MODEL.")
+        url = self.base_url or "http://localhost:11434/api/chat"
+        response = requests.post(
+            url,
+            json={
+                "model": model,
+                "messages": [
+                    {"role": "system", "content": "Extract ontology knowledge as strict JSON only."},
+                    {"role": "user", "content": prompt},
+                ],
+                "stream": False,
+                "format": "json",
+            },
+            timeout=self.timeout_seconds,
+        )
+        response.raise_for_status()
+        content = response.json()["message"]["content"]
+        return parse_json_content(content)
+
+    def _repair_json_with_model(
+        self,
+        bad_content: str,
+        endpoint: str,
+        headers: dict[str, str],
+        model: str,
+    ) -> dict[str, Any]:
+        response = requests.post(
+            endpoint,
+            headers=headers,
+            json={
+                "model": model,
+                "messages": [
+                    {
+                        "role": "system",
+                        "content": "You repair malformed JSON. Return valid JSON only. No markdown.",
+                    },
+                    {
+                        "role": "user",
+                        "content": (
+                            "Repair this into valid JSON with top-level keys entities and claims. "
+                            "Drop invalid fragments if needed.\n\n"
+                            f"{bad_content[:12000]}"
+                        ),
+                    },
+                ],
+                "temperature": 0,
+            },
+            timeout=self.timeout_seconds,
+        )
+        if response.status_code >= 400:
+            raise RuntimeError(f"LLM JSON repair failed {response.status_code}: {response.text[:1000]}")
+        return parse_json_content(response.json()["choices"][0]["message"]["content"])
+
+
+def build_extraction_prompt(page_text: str, project_config: ProjectConfig) -> str:
+    clipped_text = page_text[:6000]
+    ontology = project_config.ontology or {}
+    return f"""
+Project domain: {project_config.domain}
+Target entity types: {project_config.target_entities}
+Fields: {project_config.fields}
+Allowed predicates: {ontology.get("predicates", [])}
+
+Return only minified strict JSON. Do not include markdown, analysis, or prose.
+Use this shape:
+{{
+  "entities": [
+    {{
+      "entity_type": "Perfume",
+      "name": "Product name",
+      "attributes": {{"name": "Product name"}},
+      "confidence": 0.0,
+      "evidence_text": "short evidence from page"
+    }}
+  ],
+  "claims": [
+    {{
+      "subject_name": "Product name",
+      "subject_type": "Perfume",
+      "predicate": "hasTopNote",
+      "object_name": "Bergamot",
+      "object_type": "Note",
+      "object_value": null,
+      "evidence_text": "short evidence from page",
+      "evidence_summary": "why this claim was extracted",
+      "confidence": 0.0,
+      "confidence_reason": "reason"
+    }}
+  ]
+}}
+
+Rules:
+- Store information as source claims, not absolute facts.
+- Keep evidence_text short. Do not copy long descriptions.
+- Use only ontology predicates when possible.
+- If object is a simple value like price, put it in object_value and leave object_name/object_type null.
+- If unsure, lower confidence instead of inventing.
+- Extract at most 20 entities and 30 claims.
+- For perfume, prioritize name, brand, top/middle/base notes, accords, mood, season, occasion, price, review keywords.
+
+Page text:
+{clipped_text}
+""".strip()
+
+
+def parse_json_content(content: str, retry=None) -> dict[str, Any]:
+    try:
+        return json.loads(content)
+    except json.JSONDecodeError:
+        match = re.search(r"\{.*\}", content, flags=re.DOTALL)
+        if not match:
+            if retry:
+                return retry(content)
+            raise
+        try:
+            return json.loads(match.group(0))
+        except json.JSONDecodeError:
+            if retry:
+                return retry(content)
+            repaired = heuristic_repair_json(content)
+            if repaired is not None:
+                return repaired
+            raise
+
+
+def heuristic_repair_json(content: str) -> dict[str, Any] | None:
+    """Best-effort extraction for chatty local models that emit broken JSON."""
+
+    entities = []
+    claims = []
+    for block in re.findall(r"\{[^{}]*(?:\{[^{}]*\}[^{}]*)?\}", content, flags=re.DOTALL):
+        try:
+            item = json.loads(block)
+        except json.JSONDecodeError:
+            continue
+        if {"entity_type", "name"}.issubset(item):
+            entities.append(item)
+        if {"subject_name", "subject_type", "predicate"}.issubset(item):
+            claims.append(item)
+    if entities or claims:
+        return {"entities": entities, "claims": claims}
+    return None
+
+
+def normalize_openai_chat_url(base_url: str) -> str:
+    clean = base_url.rstrip("/")
+    if clean.endswith("/chat/completions"):
+        return clean
+    if clean.endswith("/v1"):
+        return f"{clean}/chat/completions"
+    return f"{clean}/v1/chat/completions"
+
+
+def normalize_openai_models_url(base_url: str) -> str:
+    clean = base_url.rstrip("/")
+    if clean.endswith("/chat/completions"):
+        return clean.removesuffix("/chat/completions") + "/models"
+    if clean.endswith("/models"):
+        return clean
+    if clean.endswith("/v1"):
+        return f"{clean}/models"
+    return f"{clean}/v1/models"
+
+
+def first_openai_compatible_model(base_url: str, api_key: str | None = None) -> str | None:
+    models = list_openai_compatible_models(base_url, api_key)
+    return models[0]["id"] if models else None
+
+
+def list_openai_compatible_models(base_url: str, api_key: str | None = None) -> list[dict[str, Any]]:
+    url = normalize_openai_models_url(base_url)
+    headers = {}
+    if api_key:
+        headers["Authorization"] = f"Bearer {api_key}"
+    response = requests.get(url, headers=headers, timeout=10)
+    response.raise_for_status()
+    data = response.json()
+    items = data.get("data", [])
+    return [{"id": item.get("id", ""), "owned_by": item.get("owned_by")} for item in items if item.get("id")]
+
+
+def parse_entities(items: list[dict[str, Any]]) -> list[ExtractedEntity]:
+    entities: list[ExtractedEntity] = []
+    for item in items:
+        name = item.get("name")
+        entity_type = item.get("entity_type") or item.get("type")
+        if not name or not entity_type:
+            continue
+        entities.append(
+            ExtractedEntity(
+                entity_type=str(entity_type),
+                name=str(name),
+                attributes=dict(item.get("attributes") or {}),
+                evidence_text=item.get("evidence_text"),
+                confidence=float(item.get("confidence") or 0.55),
+                metadata={"ai_extracted": True},
+            )
+        )
+    return entities
+
+
+def parse_claims(items: list[dict[str, Any]]) -> list[ExtractedClaim]:
+    claims: list[ExtractedClaim] = []
+    for item in items:
+        subject_name = item.get("subject_name")
+        subject_type = item.get("subject_type")
+        predicate = item.get("predicate")
+        if not subject_name or not subject_type or not predicate:
+            continue
+        claims.append(
+            ExtractedClaim(
+                subject_name=str(subject_name),
+                subject_type=str(subject_type),
+                predicate=str(predicate),
+                object_name=item.get("object_name"),
+                object_type=item.get("object_type"),
+                object_value=item.get("object_value"),
+                evidence_text=item.get("evidence_text"),
+                evidence_summary=item.get("evidence_summary"),
+                confidence=float(item.get("confidence") or 0.55),
+                confidence_reason=item.get("confidence_reason") or "AI extractor output",
+                metadata={"ai_extracted": True},
+            )
+        )
+    return claims

crawler_platform/app/core/extractor/base.py

@@ -0,0 +1,101 @@
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from dataclasses import dataclass, field
+from typing import Any
+
+from crawler_platform.app.config.loader import ProjectConfig
+
+
+@dataclass(slots=True)
+class ExtractedEntity:
+    entity_type: str
+    name: str
+    attributes: dict[str, Any] = field(default_factory=dict)
+    evidence_text: str | None = None
+    confidence: float = 0.5
+    metadata: dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass(slots=True)
+class ExtractedClaim:
+    subject_name: str
+    subject_type: str
+    predicate: str
+    object_name: str | None = None
+    object_type: str | None = None
+    object_value: Any | None = None
+    evidence_text: str | None = None
+    evidence_summary: str | None = None
+    confidence: float = 0.5
+    confidence_reason: str | None = None
+    metadata: dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass(slots=True)
+class ExtractionBundle:
+    entities: list[ExtractedEntity] = field(default_factory=list)
+    claims: list[ExtractedClaim] = field(default_factory=list)
+    extractor_name: str = "unknown"
+    provider: str = "unknown"
+    raw_output: dict[str, Any] = field(default_factory=dict)
+
+
+class Extractor(ABC):
+    name = "base"
+    provider = "base"
+
+    @abstractmethod
+    def extract_entities(self, page_text: str, project_config: ProjectConfig) -> list[ExtractedEntity]:
+        raise NotImplementedError
+
+    @abstractmethod
+    def extract_attributes(
+        self,
+        entity: ExtractedEntity,
+        page_text: str,
+        project_config: ProjectConfig,
+    ) -> dict[str, Any]:
+        raise NotImplementedError
+
+    @abstractmethod
+    def extract_relations(
+        self,
+        entities: list[ExtractedEntity],
+        page_text: str,
+        project_config: ProjectConfig,
+    ) -> list[ExtractedClaim]:
+        raise NotImplementedError
+
+    @abstractmethod
+    def normalize_to_ontology(self, bundle: ExtractionBundle, ontology: dict[str, Any]) -> ExtractionBundle:
+        raise NotImplementedError
+
+    def extract(self, page_text: str, project_config: ProjectConfig) -> ExtractionBundle:
+        entities = self.extract_entities(page_text, project_config)
+        for entity in entities:
+            entity.attributes.update(self.extract_attributes(entity, page_text, project_config))
+        claims = self.extract_relations(entities, page_text, project_config)
+        bundle = ExtractionBundle(
+            entities=entities,
+            claims=claims,
+            extractor_name=self.name,
+            provider=self.provider,
+            raw_output={"entity_count": len(entities), "claim_count": len(claims)},
+        )
+        return self.normalize_to_ontology(bundle, project_config.ontology)
+
+
+class AIExtractor(Extractor):
+    """Provider-neutral AI extractor contract.
+
+    OpenAI, local LLM, Ollama, and LM Studio adapters can subclass this and
+    implement ``complete_json`` while keeping the rest of the pipeline stable.
+    """
+
+    provider = "ai"
+
+    @abstractmethod
+    def complete_json(self, page_text: str, project_config: ProjectConfig) -> dict[str, Any]:
+        raise NotImplementedError
+

crawler_platform/app/core/extractor/factory.py

@@ -0,0 +1,19 @@
+from __future__ import annotations
+
+from crawler_platform.app.core.extractor.ai_provider import LLMJsonExtractor
+from crawler_platform.app.core.extractor.base import Extractor
+from crawler_platform.app.core.extractor.rule_based import GenericRuleBasedExtractor
+from crawler_platform.app.domains.perfume.extractor import PerfumeRuleBasedExtractor
+
+
+def extractor_for_domain(
+    domain: str,
+    provider: str = "rule_based",
+    model: str | None = None,
+    base_url: str | None = None,
+) -> Extractor:
+    if provider in {"openai", "ollama", "lm_studio"}:
+        return LLMJsonExtractor(domain=domain, provider=provider, model=model, base_url=base_url)
+    if domain == "perfume":
+        return PerfumeRuleBasedExtractor()
+    return GenericRuleBasedExtractor()

crawler_platform/app/core/extractor/rule_based.py

@@ -0,0 +1,85 @@
+from __future__ import annotations
+
+import re
+
+from crawler_platform.app.config.loader import ProjectConfig
+from crawler_platform.app.core.extractor.base import ExtractedClaim, ExtractedEntity, ExtractionBundle, Extractor
+from crawler_platform.app.core.ontology.mapper import normalize_predicate
+
+
+class GenericRuleBasedExtractor(Extractor):
+    name = "generic_rule_based"
+    provider = "rule_based"
+
+    def extract_entities(self, page_text: str, project_config: ProjectConfig) -> list[ExtractedEntity]:
+        name = first_non_empty_line(page_text) or "Unknown Product"
+        primary_type = project_config.target_entities[0] if project_config.target_entities else "Product"
+        return [ExtractedEntity(entity_type=primary_type, name=name, confidence=0.45)]
+
+    def extract_attributes(
+        self,
+        entity: ExtractedEntity,
+        page_text: str,
+        project_config: ProjectConfig,
+    ) -> dict[str, object]:
+        attrs: dict[str, object] = {"name": entity.name}
+        price = find_price(page_text)
+        if price:
+            attrs["price"] = price
+        return attrs
+
+    def extract_relations(
+        self,
+        entities: list[ExtractedEntity],
+        page_text: str,
+        project_config: ProjectConfig,
+    ) -> list[ExtractedClaim]:
+        if not entities:
+            return []
+        entity = entities[0]
+        claims: list[ExtractedClaim] = []
+        price = find_price(page_text)
+        if price:
+            claims.append(
+                ExtractedClaim(
+                    subject_name=entity.name,
+                    subject_type=entity.entity_type,
+                    predicate="hasPrice",
+                    object_value=price,
+                    evidence_text=price["evidence"],
+                    confidence=0.6,
+                    confidence_reason="price pattern matched",
+                )
+            )
+        return claims
+
+    def normalize_to_ontology(self, bundle: ExtractionBundle, ontology: dict[str, object]) -> ExtractionBundle:
+        for claim in bundle.claims:
+            claim.predicate = normalize_predicate(claim.predicate, ontology)
+        return bundle
+
+
+def first_non_empty_line(text: str) -> str | None:
+    for line in text.splitlines():
+        clean = line.strip()
+        if clean:
+            return clean[:240]
+    return None
+
+
+def find_price(text: str) -> dict[str, object] | None:
+    patterns = [
+        r"(?P<currency>[$€£])\s?(?P<amount>\d+(?:[,.]\d{2})?)",
+        r"(?P<amount>\d{1,3}(?:,\d{3})*)\s?(?P<currency>원|KRW|USD)",
+    ]
+    for pattern in patterns:
+        match = re.search(pattern, text, flags=re.IGNORECASE)
+        if match:
+            amount = match.group("amount").replace(",", "")
+            return {
+                "amount": float(amount),
+                "currency": match.group("currency"),
+                "evidence": match.group(0),
+            }
+    return None
+

crawler_platform/app/core/ontology/__init__.py

@@ -0,0 +1,2 @@
+"""Ontology definitions and mapping helpers."""
+

crawler_platform/app/core/ontology/definitions.py

@@ -0,0 +1,130 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+
+@dataclass(slots=True)
+class Ontology:
+    domain: str
+    entity_types: list[str]
+    predicates: list[str]
+    attributes: list[str]
+    aliases: dict[str, str] = field(default_factory=dict)
+
+
+COMMON_ONTOLOGY = Ontology(
+    domain="common",
+    entity_types=[
+        "Source",
+        "Page",
+        "Entity",
+        "Attribute",
+        "Relation",
+        "Claim",
+        "Evidence",
+        "Extraction",
+        "Confidence",
+        "UpdatePolicy",
+    ],
+    predicates=[
+        "mentions",
+        "hasAttribute",
+        "relatedTo",
+        "sameAs",
+        "soldBy",
+        "hasPrice",
+    ],
+    attributes=["name", "source_url", "updated_at", "confidence"],
+)
+
+
+DOMAIN_ONTOLOGIES: dict[str, Ontology] = {
+    "perfume": Ontology(
+        domain="perfume",
+        entity_types=[
+            "Perfume",
+            "Brand",
+            "Note",
+            "Accord",
+            "Mood",
+            "Season",
+            "Occasion",
+            "Review",
+            "Price",
+            "ProductPage",
+        ],
+        predicates=[
+            "hasBrand",
+            "hasTopNote",
+            "hasMiddleNote",
+            "hasBaseNote",
+            "hasAccord",
+            "evokesMood",
+            "suitableForSeason",
+            "suitableForOccasion",
+            "similarTo",
+            "soldBy",
+            "hasPrice",
+            "hasReviewKeyword",
+        ],
+        attributes=[
+            "name",
+            "brand",
+            "gender_bias",
+            "longevity",
+            "sillage",
+            "price_range",
+            "popularity_score",
+            "review_count",
+            "source_url",
+            "updated_at",
+        ],
+        aliases={
+            "top_notes": "hasTopNote",
+            "middle_notes": "hasMiddleNote",
+            "heart_notes": "hasMiddleNote",
+            "base_notes": "hasBaseNote",
+            "accords": "hasAccord",
+            "mood_tags": "evokesMood",
+            "season_tags": "suitableForSeason",
+            "occasion_tags": "suitableForOccasion",
+            "review_keywords": "hasReviewKeyword",
+            "price": "hasPrice",
+        },
+    ),
+    "tea": Ontology(
+        domain="tea",
+        entity_types=["Tea", "Ingredient", "Flavor", "Effect", "CaffeineLevel", "MoodState"],
+        predicates=["hasIngredient", "hasFlavor", "hasEffect", "suitableForCondition"],
+        attributes=["name", "origin", "caffeine_level", "price_range"],
+    ),
+    "coffee": Ontology(
+        domain="coffee",
+        entity_types=["CoffeeBean", "Origin", "RoastLevel", "FlavorNote", "BrewMethod"],
+        predicates=["hasOrigin", "hasRoastLevel", "hasFlavorNote", "recommendedForBrewMethod"],
+        attributes=["name", "origin", "roast_level", "process", "price_range"],
+    ),
+    "candle": Ontology(
+        domain="candle",
+        entity_types=["ScentProduct", "ScentNote", "SpaceType", "Mood", "Season"],
+        predicates=["suitableForSpace", "evokesMood", "hasScentNote"],
+        attributes=["name", "burn_time", "volume", "price_range"],
+    ),
+    "supplement": Ontology(
+        domain="supplement",
+        entity_types=["Supplement", "Ingredient", "HealthGoal", "Symptom", "Dosage"],
+        predicates=["hasIngredient", "supportsGoal", "recommendedForCondition"],
+        attributes=["name", "dosage", "warnings", "price_range"],
+    ),
+    "gift": Ontology(
+        domain="gift",
+        entity_types=["GiftProduct", "RecipientType", "Relationship", "Occasion", "PersonalityTag"],
+        predicates=["suitableForRecipient", "suitableForOccasion", "matchesPersonality"],
+        attributes=["name", "price_range", "availability", "gift_wrap_available"],
+    ),
+}
+
+
+def ontology_for_domain(domain: str) -> Ontology:
+    return DOMAIN_ONTOLOGIES.get(domain, COMMON_ONTOLOGY)
+

crawler_platform/app/core/ontology/mapper.py

@@ -0,0 +1,27 @@
+from __future__ import annotations
+
+from typing import Any
+
+
+def normalize_predicate(predicate: str, ontology: dict[str, Any] | None) -> str:
+    if not ontology:
+        return predicate
+    aliases = ontology.get("aliases", {})
+    return aliases.get(predicate, predicate)
+
+
+def is_allowed_predicate(predicate: str, ontology: dict[str, Any] | None) -> bool:
+    if not ontology or not ontology.get("predicates"):
+        return True
+    return predicate in ontology["predicates"]
+
+
+def ontology_to_dict(domain_ontology) -> dict[str, Any]:
+    return {
+        "domain": domain_ontology.domain,
+        "entity_types": domain_ontology.entity_types,
+        "predicates": domain_ontology.predicates,
+        "attributes": domain_ontology.attributes,
+        "aliases": domain_ontology.aliases,
+    }
+

crawler_platform/app/core/recommendation/__init__.py

@@ -0,0 +1,2 @@
+"""Recommendation integration helpers."""
+

crawler_platform/app/core/recommendation/scorer.py

@@ -0,0 +1,100 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any
+
+from sqlalchemy import select
+from sqlalchemy.orm import Session
+
+from crawler_platform.app.core.database import models
+from crawler_platform.app.core.database.repository import canonicalize
+
+
+@dataclass(slots=True)
+class PreferenceInput:
+    likes: list[str] = field(default_factory=list)
+    dislikes: list[str] = field(default_factory=list)
+    preferred_moods: list[str] = field(default_factory=list)
+    preferred_notes: list[str] = field(default_factory=list)
+    avoided_notes: list[str] = field(default_factory=list)
+    price_preference: dict[str, Any] = field(default_factory=dict)
+    season_context: str | None = None
+    occasion_context: str | None = None
+
+
+@dataclass(slots=True)
+class Recommendation:
+    entity_id: int
+    name: str
+    entity_type: str
+    score: float
+    reasons: list[str]
+
+
+class RuleBasedRecommender:
+    def __init__(self, session: Session):
+        self.session = session
+
+    def recommend(
+        self,
+        project_id: int,
+        target_entity_type: str,
+        preference: PreferenceInput,
+        limit: int = 10,
+    ) -> list[Recommendation]:
+        entities = self.session.scalars(
+            select(models.Entity).where(
+                models.Entity.project_id == project_id,
+                models.Entity.entity_type == target_entity_type,
+            )
+        ).all()
+        scored = [self._score_entity(project_id, entity, preference) for entity in entities]
+        scored = [item for item in scored if item.score > 0]
+        return sorted(scored, key=lambda item: item.score, reverse=True)[:limit]
+
+    def _score_entity(self, project_id: int, entity: models.Entity, preference: PreferenceInput) -> Recommendation:
+        claims = self.session.execute(
+            select(models.Claim, models.Entity)
+            .join(models.Entity, models.Claim.object_entity_id == models.Entity.id, isouter=True)
+            .where(models.Claim.project_id == project_id, models.Claim.subject_entity_id == entity.id)
+        ).all()
+        score = 0.0
+        reasons: list[str] = []
+        preferred_notes = {canonicalize(item) for item in preference.preferred_notes}
+        avoided_notes = {canonicalize(item) for item in preference.avoided_notes}
+        preferred_moods = {canonicalize(item) for item in preference.preferred_moods}
+        for claim, object_entity in claims:
+            object_name = canonicalize(object_entity.name) if object_entity else ""
+            weight = claim.confidence
+            if claim.predicate in {"hasTopNote", "hasMiddleNote", "hasBaseNote", "hasScentNote", "hasFlavorNote"}:
+                if object_name in preferred_notes:
+                    score += 2.0 * weight
+                    reasons.append(f"preferred note matched: {object_entity.name}")
+                if object_name in avoided_notes:
+                    score -= 3.0 * weight
+                    reasons.append(f"avoided note matched: {object_entity.name}")
+            if claim.predicate == "evokesMood" and object_name in preferred_moods:
+                score += 1.5 * weight
+                reasons.append(f"preferred mood matched: {object_entity.name}")
+            if preference.season_context and claim.predicate == "suitableForSeason":
+                if object_name == canonicalize(preference.season_context):
+                    score += 1.2 * weight
+                    reasons.append(f"season context matched: {preference.season_context}")
+            if preference.occasion_context and claim.predicate == "suitableForOccasion":
+                if object_name == canonicalize(preference.occasion_context):
+                    score += 1.0 * weight
+                    reasons.append(f"occasion context matched: {preference.occasion_context}")
+        if canonicalize(entity.name) in {canonicalize(item) for item in preference.dislikes}:
+            score -= 10
+            reasons.append("explicit dislike")
+        if canonicalize(entity.name) in {canonicalize(item) for item in preference.likes}:
+            score += 5
+            reasons.append("explicit like")
+        return Recommendation(
+            entity_id=entity.id,
+            name=entity.name,
+            entity_type=entity.entity_type,
+            score=round(score, 4),
+            reasons=reasons[:5],
+        )
+

crawler_platform/app/core/scheduler/__init__.py

@@ -0,0 +1,2 @@
+"""Scheduling primitives for recrawls."""
+

crawler_platform/app/core/scheduler/update_policy.py

@@ -0,0 +1,11 @@
+from __future__ import annotations
+
+from datetime import datetime, timedelta, timezone
+
+
+def next_crawl_at(policy: dict[str, object] | None, now: datetime | None = None) -> datetime:
+    now = now or datetime.now(timezone.utc)
+    policy = policy or {}
+    interval_days = int(policy.get("interval_days", 7))
+    return now + timedelta(days=interval_days)
+

crawler_platform/app/domains/__init__.py

@@ -0,0 +1,2 @@
+"""Domain plugins."""
+

crawler_platform/app/domains/candle/__init__.py

@@ -0,0 +1,2 @@
+"""Candle and diffuser domain extension point."""
+

crawler_platform/app/domains/coffee/__init__.py

@@ -0,0 +1,2 @@
+"""Coffee domain extension point."""
+

crawler_platform/app/domains/gift/__init__.py

@@ -0,0 +1,2 @@
+"""Gift recommendation domain extension point."""
+

crawler_platform/app/domains/perfume/__init__.py

@@ -0,0 +1,2 @@
+"""Perfume domain plugin."""
+

crawler_platform/app/domains/perfume/extractor.py

@@ -0,0 +1,350 @@
+from __future__ import annotations
+
+import re
+
+from crawler_platform.app.config.loader import ProjectConfig
+from crawler_platform.app.core.extractor.base import ExtractedClaim, ExtractedEntity, ExtractionBundle
+from crawler_platform.app.core.extractor.rule_based import GenericRuleBasedExtractor, find_price, first_non_empty_line
+from crawler_platform.app.core.ontology.mapper import normalize_predicate
+
+
+NOTE_LABELS = {
+    "top_notes": ["top notes", "top note", "opening notes", "탑 노트", "탑노트", "상단 노트"],
+    "middle_notes": ["middle notes", "heart notes", "미들 노트", "하트 노트", "미들노트"],
+    "base_notes": ["base notes", "base note", "베이스 노트", "베이스노트", "잔향"],
+}
+
+FIELD_TO_PREDICATE = {
+    "top_notes": "hasTopNote",
+    "middle_notes": "hasMiddleNote",
+    "base_notes": "hasBaseNote",
+    "accords": "hasAccord",
+    "mood_tags": "evokesMood",
+    "season_tags": "suitableForSeason",
+    "occasion_tags": "suitableForOccasion",
+    "review_keywords": "hasReviewKeyword",
+}
+
+MOOD_KEYWORDS = {
+    "Fresh": ["fresh", "clean", "상쾌", "깨끗", "청량"],
+    "Romantic": ["romantic", "soft", "로맨틱", "부드러운"],
+    "Elegant": ["elegant", "luxury", "우아", "고급"],
+    "Cozy": ["cozy", "warm", "포근", "따뜻"],
+    "Energetic": ["bright", "sparkling", "활기", "발랄"],
+}
+
+SEASON_KEYWORDS = {
+    "Spring": ["spring", "봄"],
+    "Summer": ["summer", "여름"],
+    "Autumn": ["autumn", "fall", "가을"],
+    "Winter": ["winter", "겨울"],
+}
+
+OCCASION_KEYWORDS = {
+    "Daily": ["daily", "everyday", "데일리", "매일"],
+    "Date": ["date", "데이트"],
+    "Office": ["office", "work", "오피스", "출근"],
+    "Evening": ["evening", "night", "저녁", "밤"],
+}
+
+ACCORD_KEYWORDS = [
+    "citrus",
+    "floral",
+    "woody",
+    "musky",
+    "amber",
+    "powdery",
+    "green",
+    "spicy",
+    "sweet",
+    "fresh",
+    "시트러스",
+    "플로럴",
+    "우디",
+    "머스크",
+    "앰버",
+    "파우더리",
+]
+
+REVIEW_KEYWORDS = [
+    "long lasting",
+    "compliment",
+    "too strong",
+    "soft",
+    "fresh",
+    "powdery",
+    "지속력",
+    "잔향",
+    "호불호",
+    "칭찬",
+    "은은",
+    "강한",
+]
+
+
+class PerfumeRuleBasedExtractor(GenericRuleBasedExtractor):
+    name = "perfume_rule_based"
+
+    def extract_entities(self, page_text: str, project_config: ProjectConfig) -> list[ExtractedEntity]:
+        product_name = extract_product_name(page_text)
+        attrs = {"name": product_name}
+        brand = extract_brand(page_text, product_name)
+        if brand:
+            attrs["brand"] = brand
+        price = find_price(page_text)
+        if price:
+            attrs["price"] = {k: v for k, v in price.items() if k != "evidence"}
+        entities = [ExtractedEntity("Perfume", product_name, attrs, confidence=0.68)]
+        if brand:
+            entities.append(ExtractedEntity("Brand", brand, confidence=0.62))
+        for field, entity_type in [
+            ("top_notes", "Note"),
+            ("middle_notes", "Note"),
+            ("base_notes", "Note"),
+            ("accords", "Accord"),
+            ("mood_tags", "Mood"),
+            ("season_tags", "Season"),
+            ("occasion_tags", "Occasion"),
+            ("review_keywords", "Review"),
+        ]:
+            for value, evidence in extract_field_values(field, page_text):
+                entities.append(ExtractedEntity(entity_type, value, evidence_text=evidence, confidence=0.6))
+        return dedupe_entities(entities)
+
+    def extract_attributes(
+        self,
+        entity: ExtractedEntity,
+        page_text: str,
+        project_config: ProjectConfig,
+    ) -> dict[str, object]:
+        if entity.entity_type != "Perfume":
+            return {}
+        attrs: dict[str, object] = {}
+        longevity = find_metric(page_text, ["longevity", "lasting", "지속력"])
+        sillage = find_metric(page_text, ["sillage", "projection", "확산력", "발향"])
+        if longevity:
+            attrs["longevity"] = longevity
+        if sillage:
+            attrs["sillage"] = sillage
+        gender_bias = find_gender_bias(page_text)
+        if gender_bias:
+            attrs["gender_bias"] = gender_bias
+        return attrs
+
+    def extract_relations(
+        self,
+        entities: list[ExtractedEntity],
+        page_text: str,
+        project_config: ProjectConfig,
+    ) -> list[ExtractedClaim]:
+        perfume = next((entity for entity in entities if entity.entity_type == "Perfume"), None)
+        if perfume is None:
+            return []
+        claims: list[ExtractedClaim] = []
+        brand = next((entity for entity in entities if entity.entity_type == "Brand"), None)
+        if brand:
+            claims.append(
+                ExtractedClaim(
+                    perfume.name,
+                    "Perfume",
+                    "hasBrand",
+                    brand.name,
+                    "Brand",
+                    evidence_text=brand.evidence_text or brand.name,
+                    confidence=0.78,
+                    confidence_reason="brand pattern matched",
+                )
+            )
+        for field, predicate in FIELD_TO_PREDICATE.items():
+            entity_type = field_entity_type(field)
+            for value, evidence in extract_field_values(field, page_text):
+                claims.append(
+                    ExtractedClaim(
+                        perfume.name,
+                        "Perfume",
+                        predicate,
+                        value,
+                        entity_type,
+                        evidence_text=evidence,
+                        evidence_summary=f"{field} includes {value}",
+                        confidence=field_confidence(field),
+                        confidence_reason=f"{field} rule matched",
+                    )
+                )
+        price = find_price(page_text)
+        if price:
+            claims.append(
+                ExtractedClaim(
+                    perfume.name,
+                    "Perfume",
+                    "hasPrice",
+                    object_value={k: v for k, v in price.items() if k != "evidence"},
+                    evidence_text=price["evidence"],
+                    confidence=0.7,
+                    confidence_reason="price pattern matched",
+                )
+            )
+        return claims
+
+    def normalize_to_ontology(self, bundle: ExtractionBundle, ontology: dict[str, object]) -> ExtractionBundle:
+        for claim in bundle.claims:
+            claim.predicate = normalize_predicate(claim.predicate, ontology)
+        return bundle
+
+
+def extract_product_name(page_text: str) -> str:
+    for line in page_text.splitlines()[:8]:
+        clean = line.strip()
+        if clean and not looks_like_navigation(clean):
+            return clean[:240]
+    return first_non_empty_line(page_text) or "Unknown Perfume"
+
+
+def extract_brand(page_text: str, product_name: str) -> str | None:
+    patterns = [
+        r"(?:brand|브랜드)\s*[:：]\s*(?P<brand>[A-Za-z0-9가-힣 '&.-]{2,80})",
+        r"by\s+(?P<brand>[A-Z][A-Za-z0-9 '&.-]{2,80})",
+    ]
+    for pattern in patterns:
+        match = re.search(pattern, page_text, flags=re.IGNORECASE)
+        if match:
+            return cleanup_value(match.group("brand"))
+    lines = [line.strip() for line in page_text.splitlines() if line.strip()]
+    if len(lines) >= 2 and lines[1].lower() not in product_name.lower():
+        candidate = cleanup_value(lines[1])
+        if len(candidate) <= 80 and not looks_like_navigation(candidate):
+            return candidate
+    return None
+
+
+def extract_field_values(field: str, page_text: str) -> list[tuple[str, str]]:
+    if field in NOTE_LABELS:
+        return extract_labeled_values(page_text, NOTE_LABELS[field])
+    if field == "accords":
+        return keyword_values(page_text, ACCORD_KEYWORDS)
+    if field == "mood_tags":
+        return taxonomy_keyword_values(page_text, MOOD_KEYWORDS)
+    if field == "season_tags":
+        return taxonomy_keyword_values(page_text, SEASON_KEYWORDS)
+    if field == "occasion_tags":
+        return taxonomy_keyword_values(page_text, OCCASION_KEYWORDS)
+    if field == "review_keywords":
+        return keyword_values(page_text, REVIEW_KEYWORDS)
+    return []
+
+
+def extract_labeled_values(page_text: str, labels: list[str]) -> list[tuple[str, str]]:
+    values: list[tuple[str, str]] = []
+    lines = page_text.splitlines()
+    for idx, line in enumerate(lines):
+        lower = line.lower()
+        if any(label.lower() in lower for label in labels):
+            evidence = line
+            raw = line.split(":", 1)[-1] if ":" in line else ""
+            if not raw and idx + 1 < len(lines):
+                raw = lines[idx + 1]
+                evidence = f"{line} {raw}"
+            for value in split_values(raw):
+                values.append((value, evidence[:1000]))
+    return values
+
+
+def split_values(raw: str) -> list[str]:
+    raw = re.sub(r"\([^)]*\)", "", raw)
+    parts = re.split(r"[,/|·ㆍ]+|\band\b| 및 | 그리고 ", raw, flags=re.IGNORECASE)
+    return [cleanup_value(part) for part in parts if 1 < len(cleanup_value(part)) <= 80]
+
+
+def keyword_values(page_text: str, keywords: list[str]) -> list[tuple[str, str]]:
+    lower = page_text.lower()
+    found: list[tuple[str, str]] = []
+    for keyword in keywords:
+        if keyword.lower() in lower:
+            found.append((keyword.title() if keyword.isascii() else keyword, snippet_for(page_text, keyword)))
+    return found
+
+
+def taxonomy_keyword_values(page_text: str, taxonomy: dict[str, list[str]]) -> list[tuple[str, str]]:
+    lower = page_text.lower()
+    found: list[tuple[str, str]] = []
+    for label, keywords in taxonomy.items():
+        for keyword in keywords:
+            if keyword.lower() in lower:
+                found.append((label, snippet_for(page_text, keyword)))
+                break
+    return found
+
+
+def snippet_for(text: str, keyword: str, window: int = 160) -> str:
+    index = text.lower().find(keyword.lower())
+    if index < 0:
+        return keyword
+    start = max(index - window // 2, 0)
+    end = min(index + len(keyword) + window // 2, len(text))
+    return text[start:end].replace("\n", " ")
+
+
+def cleanup_value(value: str) -> str:
+    return re.sub(r"\s+", " ", value.strip(" -:：[]()")).strip()
+
+
+def looks_like_navigation(value: str) -> bool:
+    return value.lower() in {"home", "shop", "menu", "cart", "login", "검색", "장바구니", "홈"}
+
+
+def field_entity_type(field: str) -> str:
+    return {
+        "top_notes": "Note",
+        "middle_notes": "Note",
+        "base_notes": "Note",
+        "accords": "Accord",
+        "mood_tags": "Mood",
+        "season_tags": "Season",
+        "occasion_tags": "Occasion",
+        "review_keywords": "Review",
+    }[field]
+
+
+def field_confidence(field: str) -> float:
+    return {
+        "top_notes": 0.82,
+        "middle_notes": 0.82,
+        "base_notes": 0.82,
+        "accords": 0.66,
+        "mood_tags": 0.62,
+        "season_tags": 0.62,
+        "occasion_tags": 0.6,
+        "review_keywords": 0.58,
+    }[field]
+
+
+def find_metric(page_text: str, labels: list[str]) -> str | None:
+    for label in labels:
+        match = re.search(rf"{label}\s*[:：]?\s*(?P<value>\d(?:\.\d)?/5|moderate|strong|weak|long|short|좋음|강함|약함)", page_text, re.IGNORECASE)
+        if match:
+            return cleanup_value(match.group("value"))
+    return None
+
+
+def find_gender_bias(page_text: str) -> str | None:
+    lower = page_text.lower()
+    if "unisex" in lower or "공용" in lower:
+        return "unisex"
+    if "for women" in lower or "여성" in lower:
+        return "feminine"
+    if "for men" in lower or "남성" in lower:
+        return "masculine"
+    return None
+
+
+def dedupe_entities(entities: list[ExtractedEntity]) -> list[ExtractedEntity]:
+    seen: set[tuple[str, str]] = set()
+    result: list[ExtractedEntity] = []
+    for entity in entities:
+        key = (entity.entity_type, entity.name.strip().lower())
+        if key in seen:
+            continue
+        seen.add(key)
+        result.append(entity)
+    return result
+

crawler_platform/app/domains/supplement/__init__.py

@@ -0,0 +1,2 @@
+"""Supplement domain extension point."""
+

crawler_platform/app/domains/tea/__init__.py

@@ -0,0 +1,2 @@
+"""Tea domain extension point."""
+

crawler_platform/app/main.py

@@ -0,0 +1,27 @@
+from __future__ import annotations
+
+import os
+from pathlib import Path
+
+from fastapi import FastAPI
+from fastapi.responses import FileResponse
+from fastapi.staticfiles import StaticFiles
+
+from crawler_platform.app.api.routes import register_routes
+from crawler_platform.app.core.database.session import init_db
+
+
+DATABASE_URL = os.getenv("CRAWLER_DATABASE_URL", "sqlite:///crawler_platform.db")
+STATIC_DIR = Path(__file__).parent / "web" / "static"
+
+app = FastAPI(title="Ontology Crawler Platform", version="0.1.0")
+init_db(DATABASE_URL)
+register_routes(app, DATABASE_URL)
+
+if STATIC_DIR.exists():
+    app.mount("/static", StaticFiles(directory=STATIC_DIR), name="static")
+
+
+@app.get("/", include_in_schema=False)
+def admin_ui():
+    return FileResponse(STATIC_DIR / "index.html")

crawler_platform/app/web/__init__.py

@@ -0,0 +1,2 @@
+"""Admin web UI."""
+

crawler_platform/app/web/static/app.js

@@ -0,0 +1,405 @@
+const state = {
+  projects: [],
+  selectedProject: null,
+  projectDetail: null,
+  ontology: null,
+};
+
+const $ = (id) => document.getElementById(id);
+
+function csv(value) {
+  return value
+    .split(",")
+    .map((item) => item.trim())
+    .filter(Boolean);
+}
+
+async function api(path, options = {}) {
+  const response = await fetch(path, {
+    headers: { "Content-Type": "application/json" },
+    ...options,
+  });
+  if (!response.ok) {
+    let detail = `${response.status} ${response.statusText}`;
+    try {
+      const body = await response.json();
+      detail = body.detail || body.error || detail;
+    } catch {
+      // Keep the HTTP status text.
+    }
+    throw new Error(detail);
+  }
+  return response.json();
+}
+
+function toast(message) {
+  const node = $("toast");
+  node.textContent = message;
+  node.classList.add("show");
+  window.setTimeout(() => node.classList.remove("show"), 2400);
+}
+
+function renderProjects() {
+  const list = $("projectList");
+  list.innerHTML = "";
+  state.projects.forEach((project) => {
+    const button = document.createElement("button");
+    button.className = `project-item ${state.selectedProject === project.name ? "active" : ""}`;
+    button.innerHTML = `<strong>${project.name}</strong><span>${project.domain}</span>`;
+    button.addEventListener("click", () => selectProject(project.name));
+    list.appendChild(button);
+  });
+}
+
+async function loadProjects() {
+  state.projects = await api("/projects");
+  if (!state.selectedProject && state.projects.length) {
+    state.selectedProject = state.projects[0].name;
+  }
+  renderProjects();
+  if (state.selectedProject) {
+    await selectProject(state.selectedProject);
+  }
+}
+
+async function selectProject(projectName) {
+  state.selectedProject = projectName;
+  state.projectDetail = await api(`/projects/${encodeURIComponent(projectName)}`);
+  state.ontology = await api(`/ontology/${encodeURIComponent(state.projectDetail.domain)}`);
+  renderProjects();
+  renderOverview();
+  renderOntology();
+  await Promise.all([loadEntities(), loadClaims(), loadTags()]);
+}
+
+function renderOverview() {
+  const detail = state.projectDetail;
+  $("metricProject").textContent = detail?.name ?? "-";
+  $("metricDomain").textContent = detail?.domain ?? "-";
+  $("metricSources").textContent = detail?.sources?.length ?? 0;
+  const sourceSelect = $("sourceSelect");
+  sourceSelect.innerHTML = "";
+  (detail?.sources ?? []).forEach((source) => {
+    const option = document.createElement("option");
+    option.value = source.name;
+    option.textContent = `${source.name} (${source.type})`;
+    sourceSelect.appendChild(option);
+  });
+  $("sourceTable").innerHTML = table(
+    ["Name", "Type", "Trust", "Robots", "Rate"],
+    (detail?.sources ?? []).map((source) => [
+      source.name,
+      source.type,
+      source.trust_level,
+      source.respect_robots_txt ? "on" : "off",
+      `${source.rate_limit_per_minute}/min`,
+    ])
+  );
+}
+
+function renderOntology() {
+  const entityTypes = state.ontology?.entity_types ?? [];
+  const predicates = state.ontology?.predicates ?? [];
+  $("ontologyEntities").innerHTML = entityTypes.map(chip).join("");
+  $("ontologyPredicates").innerHTML = predicates.map(chip).join("");
+  const filter = $("entityTypeFilter");
+  filter.innerHTML = `<option value="">All types</option>${entityTypes
+    .map((type) => `<option value="${escapeHtml(type)}">${escapeHtml(type)}</option>`)
+    .join("")}`;
+}
+
+async function createProject() {
+  const configPath = $("configPath").value.trim();
+  if (!configPath) return;
+  const result = await api("/projects", {
+    method: "POST",
+    body: JSON.stringify({ config_path: configPath }),
+  });
+  toast(`프로젝트 생성: ${result.name}`);
+  state.selectedProject = result.name;
+  await loadProjects();
+}
+
+async function crawl() {
+  if (!state.selectedProject) return;
+  const sourceName = $("sourceSelect").value;
+  const url = $("crawlUrl").value.trim();
+  const provider = $("extractorProvider").value;
+  const model = $("extractorModel").value.trim();
+  const baseUrl = $("extractorBaseUrl").value.trim();
+  $("crawlResult").textContent = "수집 중...";
+  try {
+    const result = await api("/crawl", {
+      method: "POST",
+      body: JSON.stringify({
+        config_path: $("configPath").value.trim(),
+        source_name: sourceName,
+        url,
+        extractor_provider: provider,
+        extractor_model: model || null,
+        extractor_base_url: baseUrl || null,
+      }),
+    });
+    $("crawlResult").textContent = `analyzer ${provider}, page ${result.page_id}, claims ${result.claim_count}, entities ${result.entity_count}`;
+    toast("수집 완료");
+    await Promise.all([loadEntities(), loadClaims(), loadTags()]);
+  } catch (error) {
+    $("crawlResult").textContent = `수집 실패: ${error.message}`;
+    toast("수집 실패");
+  }
+}
+
+function updateExtractorOptions() {
+  const provider = $("extractorProvider").value;
+  $("extractorOptions").classList.toggle("active", provider !== "rule_based");
+  if (provider === "ollama" && !$("extractorBaseUrl").value.trim()) {
+    $("extractorBaseUrl").placeholder = "http://localhost:11434/api/chat";
+  } else if (provider === "lm_studio" && !$("extractorBaseUrl").value.trim()) {
+    $("extractorBaseUrl").placeholder = "http://localhost:1234/v1";
+  } else {
+    $("extractorBaseUrl").placeholder = "optional provider endpoint";
+  }
+}
+
+async function testExtractor() {
+  const provider = $("extractorProvider").value;
+  const baseUrl = $("extractorBaseUrl").value.trim();
+  $("crawlResult").textContent = "분석기 연결 확인 중...";
+  const result = await api("/extractors/models", {
+    method: "POST",
+    body: JSON.stringify({
+      provider,
+      base_url: baseUrl || null,
+    }),
+  });
+  if (!result.ok) {
+    $("crawlResult").textContent = `분석기 연결 실패: ${result.error}`;
+    toast("분석기 연결 실패");
+    return;
+  }
+  const models = result.models ?? [];
+  if (models.length && !$("extractorModel").value.trim()) {
+    $("extractorModel").value = models[0].id;
+  }
+  $("crawlResult").textContent = models.length
+    ? `연결됨. 모델 ${models.length}개: ${models.map((model) => model.id).join(", ")}`
+    : "연결됨. 모델 목록은 비어 있습니다.";
+  toast("분석기 연결 확인 완료");
+}
+
+async function discover() {
+  const sourceName = $("sourceSelect").value;
+  const url = $("crawlUrl").value.trim();
+  $("crawlResult").textContent = "주소 발견 중...";
+  $("discoveredLinks").innerHTML = "";
+  try {
+    const result = await api("/discover", {
+      method: "POST",
+      body: JSON.stringify({
+        config_path: $("configPath").value.trim(),
+        source_name: sourceName,
+        url,
+        limit: 30,
+      }),
+    });
+    if (!result.ok) {
+      $("crawlResult").textContent = result.error ?? "주소 발견 실패";
+      return;
+    }
+    $("crawlResult").textContent = `발견된 주소 ${result.links.length}개`;
+    $("discoveredLinks").innerHTML = result.links.map(renderDiscoveredLink).join("");
+    document.querySelectorAll("[data-discovered-url]").forEach((button) => {
+      button.addEventListener("click", () => {
+        $("crawlUrl").value = button.dataset.discoveredUrl;
+        toast("URL 입력칸에 넣었습니다.");
+      });
+    });
+  } catch (error) {
+    $("crawlResult").textContent = `주소 발견 실패: ${error.message}`;
+    toast("주소 발견 실패");
+  }
+}
+
+function renderDiscoveredLink(link) {
+  return `
+    <button class="discovered-link" data-discovered-url="${escapeHtml(link.url)}">
+      <strong>${escapeHtml(link.label)}</strong>
+      <span>${escapeHtml(link.kind)} · ${escapeHtml(link.url)}</span>
+    </button>
+  `;
+}
+
+async function loadEntities() {
+  if (!state.selectedProject) return;
+  const type = $("entityTypeFilter").value;
+  const query = type ? `?entity_type=${encodeURIComponent(type)}&limit=100` : "?limit=100";
+  const entities = await api(`/projects/${encodeURIComponent(state.selectedProject)}/entities${query}`);
+  $("metricEntities").textContent = entities.length;
+  $("entityTable").innerHTML = table(
+    ["ID", "Type", "Name", "Metadata"],
+    entities.map((entity) => [
+      entity.id,
+      entity.type,
+      entity.name,
+      `<code>${escapeHtml(JSON.stringify(entity.metadata ?? {}))}</code>`,
+    ])
+  );
+}
+
+async function mergeEntities() {
+  if (!state.selectedProject) return;
+  const sourceId = Number($("mergeSourceId").value);
+  const targetId = Number($("mergeTargetId").value);
+  if (!sourceId || !targetId || sourceId === targetId) {
+    toast("병합할 ID와 남길 ID를 확인하세요.");
+    return;
+  }
+  const result = await api("/entities/merge", {
+    method: "POST",
+    body: JSON.stringify({
+      project_name: state.selectedProject,
+      source_entity_id: sourceId,
+      target_entity_id: targetId,
+    }),
+  });
+  if (!result.ok) {
+    toast(result.error ?? "병합 실패");
+    return;
+  }
+  toast("Entity 병합 완료");
+  $("mergeSourceId").value = "";
+  $("mergeTargetId").value = "";
+  await Promise.all([loadEntities(), loadClaims(), loadTags()]);
+}
+
+async function loadClaims() {
+  if (!state.selectedProject) return;
+  const claims = await api(`/projects/${encodeURIComponent(state.selectedProject)}/claims?limit=100`);
+  $("claimTable").innerHTML = claims.map(renderClaim).join("");
+  document.querySelectorAll("[data-save-claim]").forEach((button) => {
+    button.addEventListener("click", () => updateClaimConfidence(button.dataset.saveClaim));
+  });
+}
+
+function renderClaim(claim) {
+  const object = claim.object ?? JSON.stringify(claim.object_value ?? "");
+  return `
+    <article class="claim-card">
+      <div class="claim-main">
+        <strong>${escapeHtml(claim.subject)}</strong>
+        <span class="predicate">${escapeHtml(claim.predicate)}</span>
+        <span>${escapeHtml(object)}</span>
+        <span class="confidence">${Math.round(claim.confidence * 100)}%</span>
+      </div>
+      <div class="evidence">${escapeHtml(claim.evidence_text ?? "")}</div>
+      <div class="evidence">${escapeHtml(claim.source)} · ${escapeHtml(claim.page_url ?? "")}</div>
+      <div class="claim-actions">
+        <input id="confidence-${claim.id}" type="number" min="0" max="1" step="0.01" value="${claim.confidence}" aria-label="confidence" />
+        <input id="reason-${claim.id}" value="${escapeHtml(claim.confidence_reason ?? "")}" aria-label="reason" />
+        <button data-save-claim="${claim.id}">저장</button>
+      </div>
+    </article>
+  `;
+}
+
+async function updateClaimConfidence(claimId) {
+  const confidence = Number($(`confidence-${claimId}`).value);
+  const reason = $(`reason-${claimId}`).value.trim();
+  await api(`/claims/${claimId}/confidence`, {
+    method: "PATCH",
+    body: JSON.stringify({ confidence, reason }),
+  });
+  toast("신뢰도 수정 완료");
+  await loadClaims();
+}
+
+async function loadTags() {
+  if (!state.selectedProject) return;
+  const tags = await api(`/projects/${encodeURIComponent(state.selectedProject)}/recommendation-tags`);
+  $("tagTable").innerHTML = table(
+    ["Predicate", "Type", "Name", "Support", "Confidence"],
+    tags.map((tag) => [
+      tag.predicate,
+      tag.type,
+      tag.name,
+      tag.support_count,
+      Math.round(tag.max_confidence * 100) + "%",
+    ])
+  );
+}
+
+async function recommend() {
+  if (!state.selectedProject) return;
+  const result = await api("/recommend", {
+    method: "POST",
+    body: JSON.stringify({
+      project_name: state.selectedProject,
+      target_entity_type: state.projectDetail?.config?.recommendation?.target_entity_type ?? "Perfume",
+      preferences: {
+        preferred_notes: csv($("preferredNotes").value),
+        avoided_notes: csv($("avoidedNotes").value),
+        preferred_moods: csv($("preferredMoods").value),
+        season_context: $("seasonContext").value.trim() || null,
+        occasion_context: $("occasionContext").value.trim() || null,
+      },
+      limit: 10,
+    }),
+  });
+  $("recommendTable").innerHTML = table(
+    ["Name", "Type", "Score", "Reasons"],
+    result.map((item) => [item.name, item.entity_type, item.score, item.reasons.join(", ")])
+  );
+}
+
+function chip(value) {
+  return `<span class="chip">${escapeHtml(value)}</span>`;
+}
+
+function table(headers, rows) {
+  if (!rows.length) {
+    return `<table><tbody><tr><td>데이터가 없습니다.</td></tr></tbody></table>`;
+  }
+  return `
+    <table>
+      <thead><tr>${headers.map((header) => `<th>${escapeHtml(header)}</th>`).join("")}</tr></thead>
+      <tbody>
+        ${rows
+          .map((row) => `<tr>${row.map((cell) => `<td>${String(cell)}</td>`).join("")}</tr>`)
+          .join("")}
+      </tbody>
+    </table>
+  `;
+}
+
+function escapeHtml(value) {
+  return String(value)
+    .replaceAll("&", "&amp;")
+    .replaceAll("<", "&lt;")
+    .replaceAll(">", "&gt;")
+    .replaceAll('"', "&quot;")
+    .replaceAll("'", "&#039;");
+}
+
+document.querySelectorAll(".tab").forEach((tab) => {
+  tab.addEventListener("click", () => {
+    document.querySelectorAll(".tab").forEach((node) => node.classList.remove("active"));
+    document.querySelectorAll(".tab-panel").forEach((node) => node.classList.remove("active"));
+    tab.classList.add("active");
+    $(tab.dataset.tab).classList.add("active");
+  });
+});
+
+$("refreshBtn").addEventListener("click", loadProjects);
+$("createProjectBtn").addEventListener("click", createProject);
+$("discoverBtn").addEventListener("click", discover);
+$("crawlBtn").addEventListener("click", crawl);
+$("extractorProvider").addEventListener("change", updateExtractorOptions);
+$("testExtractorBtn").addEventListener("click", testExtractor);
+$("loadEntitiesBtn").addEventListener("click", loadEntities);
+$("mergeEntitiesBtn").addEventListener("click", mergeEntities);
+$("loadClaimsBtn").addEventListener("click", loadClaims);
+$("loadTagsBtn").addEventListener("click", loadTags);
+$("recommendBtn").addEventListener("click", recommend);
+
+updateExtractorOptions();
+loadProjects().catch((error) => toast(error.message));

crawler_platform/app/web/static/index.html

@@ -0,0 +1,179 @@
+<!doctype html>
+<html lang="ko">
+  <head>
+    <meta charset="utf-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1" />
+    <title>Ontology Crawler Platform</title>
+    <link rel="stylesheet" href="/static/styles.css" />
+  </head>
+  <body>
+    <header class="topbar">
+      <div>
+        <h1>Ontology Crawler</h1>
+        <p>프로젝트별 수집, Claim 검수, 온톨로지 매핑, 추천 태그 확인</p>
+      </div>
+      <button id="refreshBtn" class="icon-button" title="새로고침" aria-label="새로고침">↻</button>
+    </header>
+
+    <main class="layout">
+      <aside class="sidebar">
+        <section class="panel">
+          <div class="panel-head">
+            <h2>Projects</h2>
+          </div>
+          <div class="field-row">
+            <input id="configPath" value="configs/perfume_subscription.yaml" aria-label="Config path" />
+            <button id="createProjectBtn" title="프로젝트 생성">＋</button>
+          </div>
+          <div id="projectList" class="list"></div>
+        </section>
+
+        <section class="panel">
+          <div class="panel-head">
+            <h2>Crawl</h2>
+          </div>
+          <label>
+            Source
+            <select id="sourceSelect"></select>
+          </label>
+          <label>
+            URL
+            <input id="crawlUrl" value="tests/fixtures/sample_perfume.html" />
+          </label>
+          <label>
+            Analyzer
+            <select id="extractorProvider">
+              <option value="rule_based">Rule-based</option>
+              <option value="openai">OpenAI API</option>
+              <option value="ollama">Ollama</option>
+              <option value="lm_studio">LM Studio</option>
+            </select>
+          </label>
+          <div class="extractor-options" id="extractorOptions">
+            <label>
+              Model
+              <input id="extractorModel" placeholder="예: local model or API model" />
+            </label>
+            <label>
+              Base URL
+              <input id="extractorBaseUrl" placeholder="optional provider endpoint" />
+            </label>
+            <button id="testExtractorBtn">연결 테스트</button>
+          </div>
+          <div class="button-grid">
+            <button id="discoverBtn">주소 발견</button>
+            <button id="crawlBtn" class="primary">수집 실행</button>
+          </div>
+          <div id="crawlResult" class="mini-log"></div>
+          <div id="discoveredLinks" class="discovered-links"></div>
+        </section>
+      </aside>
+
+      <section class="workspace">
+        <nav class="tabs" aria-label="Admin sections">
+          <button class="tab active" data-tab="overview">Overview</button>
+          <button class="tab" data-tab="ontology">Ontology</button>
+          <button class="tab" data-tab="entities">Entities</button>
+          <button class="tab" data-tab="claims">Claims</button>
+          <button class="tab" data-tab="tags">Tags</button>
+          <button class="tab" data-tab="recommend">Recommend</button>
+        </nav>
+
+        <section id="overview" class="tab-panel active">
+          <div class="summary-grid">
+            <div class="metric">
+              <span>Project</span>
+              <strong id="metricProject">-</strong>
+            </div>
+            <div class="metric">
+              <span>Domain</span>
+              <strong id="metricDomain">-</strong>
+            </div>
+            <div class="metric">
+              <span>Sources</span>
+              <strong id="metricSources">0</strong>
+            </div>
+            <div class="metric">
+              <span>Entities</span>
+              <strong id="metricEntities">0</strong>
+            </div>
+          </div>
+          <div class="wide-panel">
+            <h2>Sources</h2>
+            <div id="sourceTable" class="table"></div>
+          </div>
+        </section>
+
+        <section id="ontology" class="tab-panel">
+          <div class="split">
+            <div class="wide-panel">
+              <h2>Entity Types</h2>
+              <div id="ontologyEntities" class="chips"></div>
+            </div>
+            <div class="wide-panel">
+              <h2>Predicates</h2>
+              <div id="ontologyPredicates" class="chips"></div>
+            </div>
+          </div>
+        </section>
+
+        <section id="entities" class="tab-panel">
+          <div class="toolbar wrap">
+            <select id="entityTypeFilter"></select>
+            <button id="loadEntitiesBtn">조회</button>
+          </div>
+          <div class="merge-bar">
+            <input id="mergeSourceId" placeholder="병합할 Entity ID" aria-label="source entity id" />
+            <input id="mergeTargetId" placeholder="남길 Entity ID" aria-label="target entity id" />
+            <button id="mergeEntitiesBtn">병합</button>
+          </div>
+          <div id="entityTable" class="table"></div>
+        </section>
+
+        <section id="claims" class="tab-panel">
+          <div class="toolbar">
+            <button id="loadClaimsBtn">Claim 새로고침</button>
+          </div>
+          <div id="claimTable" class="claim-list"></div>
+        </section>
+
+        <section id="tags" class="tab-panel">
+          <div class="toolbar">
+            <button id="loadTagsBtn">태그 조회</button>
+          </div>
+          <div id="tagTable" class="table"></div>
+        </section>
+
+        <section id="recommend" class="tab-panel">
+          <div class="recommend-grid">
+            <label>
+              Preferred notes
+              <input id="preferredNotes" value="Bergamot, Musk" />
+            </label>
+            <label>
+              Avoided notes
+              <input id="avoidedNotes" value="" />
+            </label>
+            <label>
+              Preferred moods
+              <input id="preferredMoods" value="Fresh" />
+            </label>
+            <label>
+              Season
+              <input id="seasonContext" value="Summer" />
+            </label>
+            <label>
+              Occasion
+              <input id="occasionContext" value="Daily" />
+            </label>
+          </div>
+          <button id="recommendBtn" class="primary">추천 테스트</button>
+          <div id="recommendTable" class="table"></div>
+        </section>
+      </section>
+    </main>
+
+    <div id="toast" role="status" aria-live="polite"></div>
+    <script src="/static/app.js"></script>
+  </body>
+</html>

crawler_platform/app/web/static/styles.css

@@ -0,0 +1,452 @@
+:root {
+  color-scheme: light;
+  --bg: #f6f7f4;
+  --surface: #ffffff;
+  --surface-2: #eef3ee;
+  --text: #202420;
+  --muted: #667063;
+  --line: #d8ded6;
+  --accent: #236b5b;
+  --accent-2: #9c4f30;
+  --danger: #a33434;
+  --shadow: 0 16px 40px rgba(24, 35, 28, 0.08);
+}
+
+* {
+  box-sizing: border-box;
+}
+
+body {
+  margin: 0;
+  min-width: 320px;
+  background: var(--bg);
+  color: var(--text);
+  font-family: Inter, "Segoe UI", Arial, sans-serif;
+  letter-spacing: 0;
+}
+
+button,
+input,
+select {
+  font: inherit;
+}
+
+button {
+  border: 1px solid var(--line);
+  background: var(--surface);
+  color: var(--text);
+  border-radius: 6px;
+  min-height: 36px;
+  padding: 0 12px;
+  cursor: pointer;
+}
+
+button:hover {
+  border-color: var(--accent);
+}
+
+button.primary {
+  background: var(--accent);
+  color: white;
+  border-color: var(--accent);
+}
+
+input,
+select {
+  width: 100%;
+  min-height: 36px;
+  border: 1px solid var(--line);
+  border-radius: 6px;
+  background: white;
+  padding: 0 10px;
+  color: var(--text);
+}
+
+label {
+  display: grid;
+  gap: 6px;
+  color: var(--muted);
+  font-size: 12px;
+  font-weight: 700;
+}
+
+.topbar {
+  display: flex;
+  align-items: center;
+  justify-content: space-between;
+  gap: 16px;
+  padding: 18px 24px;
+  border-bottom: 1px solid var(--line);
+  background: rgba(255, 255, 255, 0.86);
+  position: sticky;
+  top: 0;
+  z-index: 5;
+  backdrop-filter: blur(12px);
+}
+
+.topbar h1 {
+  margin: 0;
+  font-size: 22px;
+}
+
+.topbar p {
+  margin: 4px 0 0;
+  color: var(--muted);
+  font-size: 13px;
+}
+
+.icon-button {
+  width: 40px;
+  padding: 0;
+  font-size: 20px;
+}
+
+.layout {
+  display: grid;
+  grid-template-columns: 320px minmax(0, 1fr);
+  gap: 18px;
+  padding: 18px;
+}
+
+.sidebar,
+.workspace {
+  min-width: 0;
+}
+
+.sidebar {
+  display: grid;
+  align-content: start;
+  gap: 14px;
+}
+
+.panel,
+.wide-panel,
+.metric {
+  background: var(--surface);
+  border: 1px solid var(--line);
+  border-radius: 8px;
+  box-shadow: var(--shadow);
+}
+
+.panel,
+.wide-panel {
+  padding: 14px;
+}
+
+.panel-head {
+  display: flex;
+  align-items: center;
+  justify-content: space-between;
+  margin-bottom: 10px;
+}
+
+h2 {
+  margin: 0;
+  font-size: 15px;
+}
+
+.field-row {
+  display: grid;
+  grid-template-columns: minmax(0, 1fr) 42px;
+  gap: 8px;
+}
+
+.list {
+  display: grid;
+  gap: 8px;
+  margin-top: 12px;
+}
+
+.project-item {
+  display: grid;
+  gap: 2px;
+  text-align: left;
+  height: auto;
+  min-height: 52px;
+  padding: 8px 10px;
+}
+
+.project-item.active {
+  border-color: var(--accent);
+  background: var(--surface-2);
+}
+
+.project-item strong {
+  font-size: 14px;
+}
+
+.project-item span {
+  color: var(--muted);
+  font-size: 12px;
+}
+
+.mini-log {
+  min-height: 38px;
+  margin-top: 10px;
+  color: var(--muted);
+  font-size: 12px;
+  line-height: 1.45;
+}
+
+.button-grid {
+  display: grid;
+  grid-template-columns: 1fr 1fr;
+  gap: 8px;
+}
+
+.extractor-options {
+  display: none;
+  gap: 8px;
+}
+
+.extractor-options.active {
+  display: grid;
+}
+
+.extractor-options button {
+  width: 100%;
+}
+
+.discovered-links {
+  display: grid;
+  gap: 6px;
+  margin-top: 10px;
+  max-height: 280px;
+  overflow: auto;
+}
+
+.discovered-link {
+  display: grid;
+  gap: 2px;
+  min-height: 44px;
+  padding: 7px 8px;
+  text-align: left;
+  overflow-wrap: anywhere;
+}
+
+.discovered-link span {
+  color: var(--muted);
+  font-size: 11px;
+}
+
+.tabs {
+  display: flex;
+  gap: 6px;
+  overflow-x: auto;
+  padding-bottom: 10px;
+}
+
+.tab {
+  white-space: nowrap;
+}
+
+.tab.active {
+  background: var(--text);
+  color: white;
+  border-color: var(--text);
+}
+
+.tab-panel {
+  display: none;
+}
+
+.tab-panel.active {
+  display: grid;
+  gap: 14px;
+}
+
+.summary-grid {
+  display: grid;
+  grid-template-columns: repeat(4, minmax(120px, 1fr));
+  gap: 12px;
+}
+
+.metric {
+  padding: 14px;
+}
+
+.metric span {
+  display: block;
+  color: var(--muted);
+  font-size: 12px;
+  font-weight: 700;
+}
+
+.metric strong {
+  display: block;
+  margin-top: 8px;
+  font-size: 20px;
+  overflow-wrap: anywhere;
+}
+
+.split {
+  display: grid;
+  grid-template-columns: repeat(2, minmax(0, 1fr));
+  gap: 14px;
+}
+
+.chips {
+  display: flex;
+  flex-wrap: wrap;
+  gap: 8px;
+  margin-top: 12px;
+}
+
+.chip {
+  display: inline-flex;
+  align-items: center;
+  min-height: 28px;
+  padding: 0 10px;
+  border: 1px solid var(--line);
+  border-radius: 999px;
+  background: var(--surface-2);
+  font-size: 12px;
+}
+
+.toolbar {
+  display: flex;
+  gap: 8px;
+  align-items: center;
+}
+
+.toolbar.wrap {
+  flex-wrap: wrap;
+}
+
+.toolbar select {
+  max-width: 240px;
+}
+
+.merge-bar {
+  display: grid;
+  grid-template-columns: minmax(120px, 180px) minmax(120px, 180px) 80px;
+  gap: 8px;
+}
+
+.table {
+  overflow-x: auto;
+  background: var(--surface);
+  border: 1px solid var(--line);
+  border-radius: 8px;
+}
+
+table {
+  width: 100%;
+  border-collapse: collapse;
+  font-size: 13px;
+}
+
+th,
+td {
+  padding: 10px;
+  border-bottom: 1px solid var(--line);
+  text-align: left;
+  vertical-align: top;
+}
+
+th {
+  color: var(--muted);
+  background: #fafbf8;
+  font-size: 12px;
+}
+
+.claim-list {
+  display: grid;
+  gap: 10px;
+}
+
+.claim-card {
+  display: grid;
+  gap: 8px;
+  padding: 12px;
+  background: var(--surface);
+  border: 1px solid var(--line);
+  border-radius: 8px;
+}
+
+.claim-main {
+  display: flex;
+  flex-wrap: wrap;
+  gap: 8px;
+  align-items: center;
+}
+
+.predicate {
+  color: var(--accent);
+  font-weight: 800;
+}
+
+.confidence {
+  margin-left: auto;
+  color: var(--accent-2);
+  font-weight: 800;
+}
+
+.evidence {
+  color: var(--muted);
+  font-size: 12px;
+  line-height: 1.45;
+}
+
+.claim-actions {
+  display: grid;
+  grid-template-columns: 120px minmax(0, 1fr) 90px;
+  gap: 8px;
+}
+
+.recommend-grid {
+  display: grid;
+  grid-template-columns: repeat(5, minmax(120px, 1fr));
+  gap: 10px;
+}
+
+#toast {
+  position: fixed;
+  right: 18px;
+  bottom: 18px;
+  max-width: min(420px, calc(100vw - 36px));
+  padding: 12px 14px;
+  border-radius: 8px;
+  background: var(--text);
+  color: white;
+  opacity: 0;
+  transform: translateY(10px);
+  transition: 180ms ease;
+  pointer-events: none;
+  font-size: 13px;
+}
+
+#toast.show {
+  opacity: 1;
+  transform: translateY(0);
+}
+
+@media (max-width: 920px) {
+  .layout {
+    grid-template-columns: 1fr;
+  }
+
+  .summary-grid,
+  .split,
+  .recommend-grid {
+    grid-template-columns: 1fr 1fr;
+  }
+}
+
+@media (max-width: 560px) {
+  .topbar {
+    padding: 14px;
+  }
+
+  .layout {
+    padding: 12px;
+  }
+
+  .summary-grid,
+  .split,
+  .recommend-grid,
+  .claim-actions,
+  .merge-bar {
+    grid-template-columns: 1fr;
+  }
+}