[crawler]

crawler_platform/app/core/__init__.py

@@ -0,0 +1,2 @@
+"""Core platform modules."""
+

crawler_platform/app/core/crawler/__init__.py

@@ -0,0 +1,2 @@
+"""Crawler pipeline."""
+

crawler_platform/app/core/crawler/discovery.py

@@ -0,0 +1,52 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from urllib.parse import parse_qs, urljoin, urlparse
+
+from bs4 import BeautifulSoup
+
+
+@dataclass(slots=True)
+class DiscoveredUrl:
+    url: str
+    label: str
+    kind: str = "link"
+
+
+def discover_links(html: str, base_url: str, limit: int = 30) -> list[DiscoveredUrl]:
+    soup = BeautifulSoup(html, "html.parser")
+    seen: set[str] = set()
+    results: list[DiscoveredUrl] = []
+    for anchor in soup.find_all("a", href=True):
+        raw_href = anchor.get("href", "")
+        url = normalize_search_redirect(urljoin(base_url, raw_href))
+        if not url or url in seen or not url.startswith(("http://", "https://")):
+            continue
+        seen.add(url)
+        label = anchor.get_text(" ", strip=True)[:160] or urlparse(url).netloc
+        results.append(DiscoveredUrl(url=url, label=label, kind=classify_url(url)))
+        if len(results) >= limit:
+            break
+    return results
+
+
+def normalize_search_redirect(url: str) -> str:
+    parsed = urlparse(url)
+    query = parse_qs(parsed.query)
+    for key in ("url", "u", "target"):
+        if key in query and query[key]:
+            candidate = query[key][0]
+            if candidate.startswith(("http://", "https://")):
+                return candidate
+    return url
+
+
+def classify_url(url: str) -> str:
+    host = urlparse(url).netloc.lower()
+    if "smartstore.naver.com" in host or "brand.naver.com" in host:
+        return "marketplace_product_or_store"
+    if "shopping.naver.com" in host:
+        return "shopping"
+    if any(token in host for token in ("fragrantica", "official", "perfume", "parfum")):
+        return "product_or_review"
+    return "link"

crawler_platform/app/core/crawler/fetchers.py

@@ -0,0 +1,136 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from pathlib import Path
+import time
+from urllib.parse import urlparse
+from urllib.robotparser import RobotFileParser
+
+
+DEFAULT_USER_AGENT = "OntologyCrawlerBot/0.1 (+contact: admin@example.com)"
+
+
+@dataclass(slots=True)
+class FetchResult:
+    url: str
+    status_code: int | None
+    html: str
+    final_url: str | None = None
+    headers: dict[str, str] = field(default_factory=dict)
+
+
+class RateLimiter:
+    def __init__(self, per_minute: int = 30):
+        self.delay = 60 / max(per_minute, 1)
+        self._last_called = 0.0
+
+    def wait(self) -> None:
+        elapsed = time.monotonic() - self._last_called
+        if elapsed < self.delay:
+            time.sleep(self.delay - elapsed)
+        self._last_called = time.monotonic()
+
+
+class RobotsPolicy:
+    def __init__(self, user_agent: str = DEFAULT_USER_AGENT):
+        self.user_agent = user_agent
+        self._cache: dict[str, RobotFileParser] = {}
+
+    def allowed(self, url: str, respect_robots_txt: bool = True) -> bool:
+        if not respect_robots_txt:
+            return True
+        parsed = urlparse(url)
+        if parsed.scheme in {"", "file"}:
+            return True
+        robots_url = f"{parsed.scheme}://{parsed.netloc}/robots.txt"
+        parser = self._cache.get(robots_url)
+        if parser is None:
+            parser = RobotFileParser()
+            parser.set_url(robots_url)
+            try:
+                parser.read()
+            except Exception:
+                return False
+            self._cache[robots_url] = parser
+        return parser.can_fetch(self.user_agent, url)
+
+
+class BaseFetcher:
+    def fetch(self, url: str) -> FetchResult:
+        raise NotImplementedError
+
+
+class RequestsFetcher(BaseFetcher):
+    def __init__(
+        self,
+        user_agent: str = DEFAULT_USER_AGENT,
+        timeout_seconds: int = 15,
+        retries: int = 2,
+        rate_limit_per_minute: int = 30,
+    ):
+        self.user_agent = user_agent
+        self.timeout_seconds = timeout_seconds
+        self.retries = retries
+        self.rate_limiter = RateLimiter(rate_limit_per_minute)
+
+    def fetch(self, url: str) -> FetchResult:
+        local_path = _local_path_from_url(url)
+        if local_path:
+            return FetchResult(url=url, status_code=200, html=local_path.read_text(encoding="utf-8"), final_url=url)
+
+        import requests
+
+        last_error: Exception | None = None
+        for _ in range(self.retries + 1):
+            self.rate_limiter.wait()
+            try:
+                response = requests.get(
+                    url,
+                    headers={"User-Agent": self.user_agent},
+                    timeout=self.timeout_seconds,
+                )
+                return FetchResult(
+                    url=url,
+                    status_code=response.status_code,
+                    html=response.text,
+                    final_url=response.url,
+                    headers=dict(response.headers),
+                )
+            except requests.RequestException as exc:
+                last_error = exc
+        raise RuntimeError(f"Failed to fetch {url}: {last_error}")
+
+
+class PlaywrightFetcher(BaseFetcher):
+    def __init__(self, user_agent: str = DEFAULT_USER_AGENT, timeout_ms: int = 20000):
+        self.user_agent = user_agent
+        self.timeout_ms = timeout_ms
+
+    def fetch(self, url: str) -> FetchResult:
+        from playwright.sync_api import sync_playwright
+
+        with sync_playwright() as playwright:
+            browser = playwright.chromium.launch(headless=True)
+            page = browser.new_page(user_agent=self.user_agent)
+            response = page.goto(url, wait_until="networkidle", timeout=self.timeout_ms)
+            html = page.content()
+            final_url = page.url
+            status = response.status if response else None
+            browser.close()
+        return FetchResult(url=url, status_code=status, html=html, final_url=final_url)
+
+
+def make_fetcher(kind: str, rate_limit_per_minute: int = 30) -> BaseFetcher:
+    if kind == "playwright":
+        return PlaywrightFetcher()
+    return RequestsFetcher(rate_limit_per_minute=rate_limit_per_minute)
+
+
+def _local_path_from_url(url: str) -> Path | None:
+    if url.startswith("file://"):
+        path = Path(url.removeprefix("file://"))
+        return path if path.exists() and path.is_file() else None
+    path = Path(url)
+    if path.exists() and path.is_file():
+        return path
+    return None

crawler_platform/app/core/crawler/html_cleaner.py

@@ -0,0 +1,25 @@
+from __future__ import annotations
+
+import re
+
+
+def clean_html(html: str) -> tuple[str | None, str]:
+    try:
+        from bs4 import BeautifulSoup
+    except ImportError:
+        text = re.sub(r"<[^>]+>", " ", html)
+        return None, normalize_whitespace(text)
+
+    soup = BeautifulSoup(html, "html.parser")
+    for tag in soup(["script", "style", "noscript", "svg", "iframe"]):
+        tag.decompose()
+    title = soup.title.get_text(" ", strip=True) if soup.title else None
+    main = soup.find("main") or soup.body or soup
+    text = main.get_text("\n", strip=True)
+    return title, normalize_whitespace(text)
+
+
+def normalize_whitespace(text: str) -> str:
+    lines = [" ".join(line.split()) for line in text.splitlines()]
+    return "\n".join(line for line in lines if line)
+

crawler_platform/app/core/crawler/pipeline.py

@@ -0,0 +1,55 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from crawler_platform.app.config.loader import ProjectConfig
+from crawler_platform.app.core.crawler.fetchers import RobotsPolicy, make_fetcher
+from crawler_platform.app.core.crawler.plugins import ParserRegistry, default_parser_registry
+from crawler_platform.app.core.database.repository import KnowledgeRepository
+from crawler_platform.app.core.extractor.base import Extractor
+
+
+@dataclass(slots=True)
+class CrawlResult:
+    page_id: int
+    claim_count: int
+    entity_count: int
+
+
+class CrawlPipeline:
+    def __init__(
+        self,
+        repository: KnowledgeRepository,
+        extractor: Extractor,
+        parser_registry: ParserRegistry | None = None,
+        robots_policy: RobotsPolicy | None = None,
+    ):
+        self.repository = repository
+        self.extractor = extractor
+        self.parser_registry = parser_registry or default_parser_registry()
+        self.robots_policy = robots_policy or RobotsPolicy()
+
+    def crawl_url(self, project_config: ProjectConfig, source_name: str, url: str) -> CrawlResult:
+        project = self.repository.upsert_project(project_config)
+        source_config = project_config.source_by_name(source_name)
+        source = self.repository.get_source(project.id, source_name)
+        if not self.robots_policy.allowed(url, source_config.respect_robots_txt):
+            raise PermissionError(f"robots.txt does not allow crawling: {url}")
+
+        fetcher = make_fetcher(source_config.fetcher, source_config.rate_limit_per_minute)
+        fetch_result = fetcher.fetch(url)
+        parser = self.parser_registry.get(source_config.parser)
+        parsed = parser.parse(fetch_result.html, fetch_result.final_url or url)
+        page = self.repository.upsert_page(
+            project_id=project.id,
+            source_id=source.id,
+            url=url,
+            title=parsed.title,
+            status_code=fetch_result.status_code,
+            cleaned_text=parsed.text,
+            metadata={**parsed.metadata, "final_url": fetch_result.final_url},
+        )
+        bundle = self.extractor.extract(parsed.text, project_config)
+        claims = self.repository.save_extraction_bundle(project.id, source, page, bundle)
+        return CrawlResult(page_id=page.id, claim_count=len(claims), entity_count=len(bundle.entities))
+

crawler_platform/app/core/crawler/plugins.py

@@ -0,0 +1,48 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Protocol
+
+
+@dataclass(slots=True)
+class ParsedPage:
+    title: str | None
+    text: str
+    metadata: dict[str, object]
+
+
+class SiteParser(Protocol):
+    name: str
+
+    def parse(self, html: str, url: str) -> ParsedPage:
+        raise NotImplementedError
+
+
+class ParserRegistry:
+    def __init__(self):
+        self._parsers: dict[str, SiteParser] = {}
+
+    def register(self, parser: SiteParser) -> None:
+        self._parsers[parser.name] = parser
+
+    def get(self, name: str) -> SiteParser:
+        if name not in self._parsers:
+            raise KeyError(f"Parser not registered: {name}")
+        return self._parsers[name]
+
+
+class GenericProductParser:
+    name = "generic"
+
+    def parse(self, html: str, url: str) -> ParsedPage:
+        from crawler_platform.app.core.crawler.html_cleaner import clean_html
+
+        title, text = clean_html(html)
+        return ParsedPage(title=title, text=text, metadata={"parser": self.name, "url": url})
+
+
+def default_parser_registry() -> ParserRegistry:
+    registry = ParserRegistry()
+    registry.register(GenericProductParser())
+    return registry
+

crawler_platform/app/core/database/__init__.py

@@ -0,0 +1,2 @@
+"""Database models and repositories."""
+

crawler_platform/app/core/database/models.py

@@ -0,0 +1,240 @@
+from __future__ import annotations
+
+from datetime import datetime, timezone
+
+from sqlalchemy import (
+    JSON,
+    Boolean,
+    Column,
+    DateTime,
+    Float,
+    ForeignKey,
+    Integer,
+    String,
+    Text,
+    UniqueConstraint,
+)
+from sqlalchemy.orm import declarative_base, relationship
+
+
+Base = declarative_base()
+
+
+def utcnow() -> datetime:
+    return datetime.now(timezone.utc)
+
+
+class Project(Base):
+    __tablename__ = "projects"
+
+    id = Column(Integer, primary_key=True)
+    name = Column(String(160), unique=True, nullable=False, index=True)
+    domain = Column(String(80), nullable=False, index=True)
+    config = Column(JSON, nullable=False, default=dict)
+    created_at = Column(DateTime(timezone=True), default=utcnow, nullable=False)
+    updated_at = Column(DateTime(timezone=True), default=utcnow, nullable=False)
+
+    sources = relationship("Source", back_populates="project", cascade="all, delete-orphan")
+
+
+class Source(Base):
+    __tablename__ = "sources"
+    __table_args__ = (UniqueConstraint("project_id", "name", name="uq_source_project_name"),)
+
+    id = Column(Integer, primary_key=True)
+    project_id = Column(Integer, ForeignKey("projects.id"), nullable=False, index=True)
+    name = Column(String(160), nullable=False)
+    type = Column(String(80), nullable=False, default="unknown")
+    base_url = Column(Text)
+    trust_level = Column(Float, nullable=False, default=0.5)
+    respect_robots_txt = Column(Boolean, nullable=False, default=True)
+    rate_limit_per_minute = Column(Integer, nullable=False, default=30)
+    update_policy = Column(JSON, nullable=False, default=dict)
+    created_at = Column(DateTime(timezone=True), default=utcnow, nullable=False)
+    updated_at = Column(DateTime(timezone=True), default=utcnow, nullable=False)
+
+    project = relationship("Project", back_populates="sources")
+
+
+class Page(Base):
+    __tablename__ = "pages"
+    __table_args__ = (UniqueConstraint("project_id", "url", name="uq_page_project_url"),)
+
+    id = Column(Integer, primary_key=True)
+    project_id = Column(Integer, ForeignKey("projects.id"), nullable=False, index=True)
+    source_id = Column(Integer, ForeignKey("sources.id"), nullable=False, index=True)
+    url = Column(Text, nullable=False)
+    canonical_url = Column(Text)
+    title = Column(Text)
+    content_hash = Column(String(80), index=True)
+    status_code = Column(Integer)
+    fetched_at = Column(DateTime(timezone=True), default=utcnow, nullable=False)
+    cleaned_text_summary = Column(Text)
+    raw_storage_ref = Column(Text)
+    metadata_json = Column(JSON, nullable=False, default=dict)
+
+
+class Entity(Base):
+    __tablename__ = "entities"
+    __table_args__ = (
+        UniqueConstraint("project_id", "entity_type", "canonical_name", name="uq_entity_identity"),
+    )
+
+    id = Column(Integer, primary_key=True)
+    project_id = Column(Integer, ForeignKey("projects.id"), nullable=False, index=True)
+    entity_type = Column(String(120), nullable=False, index=True)
+    name = Column(String(240), nullable=False)
+    canonical_name = Column(String(240), nullable=False, index=True)
+    external_ids = Column(JSON, nullable=False, default=dict)
+    metadata_json = Column(JSON, nullable=False, default=dict)
+    created_at = Column(DateTime(timezone=True), default=utcnow, nullable=False)
+    updated_at = Column(DateTime(timezone=True), default=utcnow, nullable=False)
+
+
+class Attribute(Base):
+    __tablename__ = "attributes"
+    __table_args__ = (
+        UniqueConstraint("entity_id", "name", "source_id", name="uq_attribute_entity_source"),
+    )
+
+    id = Column(Integer, primary_key=True)
+    project_id = Column(Integer, ForeignKey("projects.id"), nullable=False, index=True)
+    entity_id = Column(Integer, ForeignKey("entities.id"), nullable=False, index=True)
+    source_id = Column(Integer, ForeignKey("sources.id"), nullable=True, index=True)
+    name = Column(String(120), nullable=False, index=True)
+    value = Column(JSON, nullable=False)
+    confidence = Column(Float, nullable=False, default=0.5)
+    updated_at = Column(DateTime(timezone=True), default=utcnow, nullable=False)
+
+
+class Relation(Base):
+    __tablename__ = "relations"
+    __table_args__ = (
+        UniqueConstraint("project_id", "subject_entity_id", "predicate", "object_entity_id", name="uq_relation"),
+    )
+
+    id = Column(Integer, primary_key=True)
+    project_id = Column(Integer, ForeignKey("projects.id"), nullable=False, index=True)
+    subject_entity_id = Column(Integer, ForeignKey("entities.id"), nullable=False, index=True)
+    predicate = Column(String(160), nullable=False, index=True)
+    object_entity_id = Column(Integer, ForeignKey("entities.id"), nullable=False, index=True)
+    confidence = Column(Float, nullable=False, default=0.5)
+    support_count = Column(Integer, nullable=False, default=1)
+    updated_at = Column(DateTime(timezone=True), default=utcnow, nullable=False)
+
+
+class Claim(Base):
+    __tablename__ = "claims"
+    __table_args__ = (UniqueConstraint("project_id", "claim_hash", name="uq_claim_hash"),)
+
+    id = Column(Integer, primary_key=True)
+    project_id = Column(Integer, ForeignKey("projects.id"), nullable=False, index=True)
+    source_id = Column(Integer, ForeignKey("sources.id"), nullable=False, index=True)
+    page_id = Column(Integer, ForeignKey("pages.id"), nullable=True, index=True)
+    subject_entity_id = Column(Integer, ForeignKey("entities.id"), nullable=False, index=True)
+    predicate = Column(String(160), nullable=False, index=True)
+    object_entity_id = Column(Integer, ForeignKey("entities.id"), nullable=True, index=True)
+    object_value = Column(JSON, nullable=True)
+    value_type = Column(String(80), nullable=False, default="entity")
+    claim_hash = Column(String(80), nullable=False, index=True)
+    confidence = Column(Float, nullable=False, default=0.5)
+    confidence_reason = Column(Text)
+    extraction_method = Column(String(120), nullable=False, default="rule_based")
+    status = Column(String(40), nullable=False, default="active")
+    first_seen_at = Column(DateTime(timezone=True), default=utcnow, nullable=False)
+    last_seen_at = Column(DateTime(timezone=True), default=utcnow, nullable=False)
+    valid_until = Column(DateTime(timezone=True), nullable=True)
+    metadata_json = Column(JSON, nullable=False, default=dict)
+
+    evidence_items = relationship("Evidence", back_populates="claim", cascade="all, delete-orphan")
+
+
+class Evidence(Base):
+    __tablename__ = "evidence"
+
+    id = Column(Integer, primary_key=True)
+    project_id = Column(Integer, ForeignKey("projects.id"), nullable=False, index=True)
+    claim_id = Column(Integer, ForeignKey("claims.id"), nullable=False, index=True)
+    page_id = Column(Integer, ForeignKey("pages.id"), nullable=True, index=True)
+    evidence_text = Column(Text, nullable=False)
+    evidence_summary = Column(Text)
+    selector = Column(Text)
+    start_offset = Column(Integer)
+    end_offset = Column(Integer)
+    created_at = Column(DateTime(timezone=True), default=utcnow, nullable=False)
+
+    claim = relationship("Claim", back_populates="evidence_items")
+
+
+class ExtractionLog(Base):
+    __tablename__ = "extraction_logs"
+
+    id = Column(Integer, primary_key=True)
+    project_id = Column(Integer, ForeignKey("projects.id"), nullable=False, index=True)
+    page_id = Column(Integer, ForeignKey("pages.id"), nullable=True, index=True)
+    extractor_name = Column(String(160), nullable=False)
+    provider = Column(String(120), nullable=False, default="rule_based")
+    input_hash = Column(String(80))
+    raw_output = Column(JSON, nullable=False, default=dict)
+    error = Column(Text)
+    created_at = Column(DateTime(timezone=True), default=utcnow, nullable=False)
+
+
+class CrawlJob(Base):
+    __tablename__ = "crawl_jobs"
+
+    id = Column(Integer, primary_key=True)
+    project_id = Column(Integer, ForeignKey("projects.id"), nullable=False, index=True)
+    source_id = Column(Integer, ForeignKey("sources.id"), nullable=True, index=True)
+    url = Column(Text)
+    status = Column(String(40), nullable=False, default="pending")
+    priority = Column(Integer, nullable=False, default=100)
+    scheduled_at = Column(DateTime(timezone=True), default=utcnow, nullable=False)
+    started_at = Column(DateTime(timezone=True), nullable=True)
+    finished_at = Column(DateTime(timezone=True), nullable=True)
+    error = Column(Text)
+    metadata_json = Column(JSON, nullable=False, default=dict)
+
+
+class UserProfile(Base):
+    __tablename__ = "user_profiles"
+
+    id = Column(Integer, primary_key=True)
+    project_id = Column(Integer, ForeignKey("projects.id"), nullable=False, index=True)
+    external_user_id = Column(String(160), nullable=False, index=True)
+    display_name = Column(String(160))
+    context = Column(JSON, nullable=False, default=dict)
+    created_at = Column(DateTime(timezone=True), default=utcnow, nullable=False)
+    updated_at = Column(DateTime(timezone=True), default=utcnow, nullable=False)
+
+
+class UserPreference(Base):
+    __tablename__ = "user_preferences"
+
+    id = Column(Integer, primary_key=True)
+    project_id = Column(Integer, ForeignKey("projects.id"), nullable=False, index=True)
+    user_profile_id = Column(Integer, ForeignKey("user_profiles.id"), nullable=False, index=True)
+    likes = Column(JSON, nullable=False, default=list)
+    dislikes = Column(JSON, nullable=False, default=list)
+    preferred_moods = Column(JSON, nullable=False, default=list)
+    preferred_notes = Column(JSON, nullable=False, default=list)
+    avoided_notes = Column(JSON, nullable=False, default=list)
+    price_preference = Column(JSON, nullable=False, default=dict)
+    season_context = Column(String(80))
+    occasion_context = Column(String(120))
+    feedback_history = Column(JSON, nullable=False, default=list)
+    updated_at = Column(DateTime(timezone=True), default=utcnow, nullable=False)
+
+
+class FeedbackLog(Base):
+    __tablename__ = "feedback_logs"
+
+    id = Column(Integer, primary_key=True)
+    project_id = Column(Integer, ForeignKey("projects.id"), nullable=False, index=True)
+    user_profile_id = Column(Integer, ForeignKey("user_profiles.id"), nullable=False, index=True)
+    entity_id = Column(Integer, ForeignKey("entities.id"), nullable=True, index=True)
+    action = Column(String(80), nullable=False)
+    score = Column(Float)
+    reason = Column(Text)
+    metadata_json = Column(JSON, nullable=False, default=dict)
+    created_at = Column(DateTime(timezone=True), default=utcnow, nullable=False)

crawler_platform/app/core/database/repository.py

@@ -0,0 +1,333 @@
+from __future__ import annotations
+
+from dataclasses import asdict
+import hashlib
+import json
+from typing import Any
+
+from sqlalchemy import select
+from sqlalchemy.orm import Session
+
+from crawler_platform.app.config.loader import ProjectConfig, SourceConfig
+from crawler_platform.app.core.database import models
+from crawler_platform.app.core.extractor.base import ExtractedClaim, ExtractedEntity, ExtractionBundle
+
+
+def canonicalize(value: str) -> str:
+    return " ".join(value.strip().lower().split())
+
+
+def short_hash(text: str) -> str:
+    return hashlib.sha256(text.encode("utf-8")).hexdigest()
+
+
+class KnowledgeRepository:
+    def __init__(self, session: Session):
+        self.session = session
+
+    def upsert_project(self, config: ProjectConfig) -> models.Project:
+        project = self.session.scalar(select(models.Project).where(models.Project.name == config.project_name))
+        config_dict = _project_config_to_dict(config)
+        if project is None:
+            project = models.Project(name=config.project_name, domain=config.domain, config=config_dict)
+            self.session.add(project)
+            self.session.flush()
+        else:
+            project.domain = config.domain
+            project.config = config_dict
+            project.updated_at = models.utcnow()
+        for source_config in config.sources:
+            self.upsert_source(project, source_config)
+        return project
+
+    def upsert_source(self, project: models.Project, source_config: SourceConfig) -> models.Source:
+        source = self.session.scalar(
+            select(models.Source).where(
+                models.Source.project_id == project.id,
+                models.Source.name == source_config.name,
+            )
+        )
+        if source is None:
+            source = models.Source(project_id=project.id, name=source_config.name)
+            self.session.add(source)
+            self.session.flush()
+        source.type = source_config.type
+        source.base_url = source_config.base_url
+        source.trust_level = source_config.trust_level
+        source.respect_robots_txt = source_config.respect_robots_txt
+        source.rate_limit_per_minute = source_config.rate_limit_per_minute
+        source.updated_at = models.utcnow()
+        return source
+
+    def get_project(self, project_name: str) -> models.Project:
+        project = self.session.scalar(select(models.Project).where(models.Project.name == project_name))
+        if project is None:
+            raise KeyError(f"Project not found: {project_name}")
+        return project
+
+    def get_source(self, project_id: int, source_name: str) -> models.Source:
+        source = self.session.scalar(
+            select(models.Source).where(models.Source.project_id == project_id, models.Source.name == source_name)
+        )
+        if source is None:
+            raise KeyError(f"Source not found: {source_name}")
+        return source
+
+    def upsert_page(
+        self,
+        project_id: int,
+        source_id: int,
+        url: str,
+        title: str | None,
+        status_code: int | None,
+        cleaned_text: str,
+        metadata: dict[str, Any] | None = None,
+    ) -> models.Page:
+        page = self.session.scalar(select(models.Page).where(models.Page.project_id == project_id, models.Page.url == url))
+        digest = short_hash(cleaned_text)
+        summary = cleaned_text[:2000]
+        if page is None:
+            page = models.Page(project_id=project_id, source_id=source_id, url=url)
+            self.session.add(page)
+            self.session.flush()
+        page.title = title
+        page.status_code = status_code
+        page.content_hash = digest
+        page.cleaned_text_summary = summary
+        page.metadata_json = metadata or {}
+        page.fetched_at = models.utcnow()
+        return page
+
+    def upsert_entity(
+        self,
+        project_id: int,
+        entity_type: str,
+        name: str,
+        metadata: dict[str, Any] | None = None,
+    ) -> models.Entity:
+        canonical_name = canonicalize(name)
+        entity = self.session.scalar(
+            select(models.Entity).where(
+                models.Entity.project_id == project_id,
+                models.Entity.entity_type == entity_type,
+                models.Entity.canonical_name == canonical_name,
+            )
+        )
+        if entity is None:
+            entity = models.Entity(
+                project_id=project_id,
+                entity_type=entity_type,
+                name=name.strip(),
+                canonical_name=canonical_name,
+                metadata_json=metadata or {},
+            )
+            self.session.add(entity)
+            self.session.flush()
+        else:
+            entity.metadata_json = {**(entity.metadata_json or {}), **(metadata or {})}
+            entity.updated_at = models.utcnow()
+        return entity
+
+    def save_extraction_bundle(
+        self,
+        project_id: int,
+        source: models.Source,
+        page: models.Page,
+        bundle: ExtractionBundle,
+    ) -> list[models.Claim]:
+        entity_index: dict[tuple[str, str], models.Entity] = {}
+        for extracted_entity in bundle.entities:
+            entity = self._save_extracted_entity(project_id, source.id, extracted_entity)
+            entity_index[(extracted_entity.entity_type, canonicalize(extracted_entity.name))] = entity
+
+        claims: list[models.Claim] = []
+        for extracted_claim in bundle.claims:
+            subject = self._entity_for_claim(project_id, extracted_claim.subject_type, extracted_claim.subject_name, entity_index)
+            object_entity = None
+            if extracted_claim.object_name and extracted_claim.object_type:
+                object_entity = self._entity_for_claim(
+                    project_id,
+                    extracted_claim.object_type,
+                    extracted_claim.object_name,
+                    entity_index,
+                )
+            confidence = combine_confidence(extracted_claim.confidence, source.trust_level)
+            claim_hash = make_claim_hash(
+                project_id=project_id,
+                source_id=source.id,
+                subject_entity_id=subject.id,
+                predicate=extracted_claim.predicate,
+                object_entity_id=object_entity.id if object_entity else None,
+                object_value=extracted_claim.object_value,
+            )
+            claim = self.session.scalar(
+                select(models.Claim).where(
+                    models.Claim.project_id == project_id,
+                    models.Claim.claim_hash == claim_hash,
+                )
+            )
+            if claim is None:
+                claim = models.Claim(
+                    project_id=project_id,
+                    source_id=source.id,
+                    page_id=page.id,
+                    subject_entity_id=subject.id,
+                    predicate=extracted_claim.predicate,
+                    object_entity_id=object_entity.id if object_entity else None,
+                    object_value=extracted_claim.object_value,
+                    value_type="entity" if object_entity else "literal",
+                    claim_hash=claim_hash,
+                    confidence=confidence,
+                    confidence_reason=extracted_claim.confidence_reason,
+                    extraction_method=bundle.extractor_name,
+                    metadata_json=extracted_claim.metadata,
+                )
+                self.session.add(claim)
+                self.session.flush()
+            else:
+                claim.page_id = page.id
+                claim.last_seen_at = models.utcnow()
+                claim.confidence = max(claim.confidence, confidence)
+                claim.confidence_reason = extracted_claim.confidence_reason or claim.confidence_reason
+                claim.metadata_json = {**(claim.metadata_json or {}), **extracted_claim.metadata}
+            if extracted_claim.evidence_text:
+                self.session.add(
+                    models.Evidence(
+                        project_id=project_id,
+                        claim_id=claim.id,
+                        page_id=page.id,
+                        evidence_text=extracted_claim.evidence_text[:1000],
+                        evidence_summary=extracted_claim.evidence_summary,
+                    )
+                )
+            if object_entity:
+                self._upsert_relation(project_id, subject.id, extracted_claim.predicate, object_entity.id, confidence)
+            claims.append(claim)
+
+        self.session.add(
+            models.ExtractionLog(
+                project_id=project_id,
+                page_id=page.id,
+                extractor_name=bundle.extractor_name,
+                provider=bundle.provider,
+                input_hash=page.content_hash,
+                raw_output=bundle.raw_output,
+            )
+        )
+        return claims
+
+    def _save_extracted_entity(
+        self,
+        project_id: int,
+        source_id: int,
+        extracted_entity: ExtractedEntity,
+    ) -> models.Entity:
+        entity = self.upsert_entity(
+            project_id,
+            extracted_entity.entity_type,
+            extracted_entity.name,
+            extracted_entity.metadata,
+        )
+        for name, value in extracted_entity.attributes.items():
+            attribute = self.session.scalar(
+                select(models.Attribute).where(
+                    models.Attribute.entity_id == entity.id,
+                    models.Attribute.name == name,
+                    models.Attribute.source_id == source_id,
+                )
+            )
+            if attribute is None:
+                attribute = models.Attribute(
+                    project_id=project_id,
+                    entity_id=entity.id,
+                    source_id=source_id,
+                    name=name,
+                    value=value,
+                    confidence=extracted_entity.confidence,
+                )
+                self.session.add(attribute)
+            else:
+                attribute.value = value
+                attribute.confidence = max(attribute.confidence, extracted_entity.confidence)
+                attribute.updated_at = models.utcnow()
+        return entity
+
+    def _entity_for_claim(
+        self,
+        project_id: int,
+        entity_type: str,
+        name: str,
+        entity_index: dict[tuple[str, str], models.Entity],
+    ) -> models.Entity:
+        key = (entity_type, canonicalize(name))
+        if key not in entity_index:
+            entity_index[key] = self.upsert_entity(project_id, entity_type, name)
+        return entity_index[key]
+
+    def _upsert_relation(
+        self,
+        project_id: int,
+        subject_id: int,
+        predicate: str,
+        object_id: int,
+        confidence: float,
+    ) -> models.Relation:
+        relation = self.session.scalar(
+            select(models.Relation).where(
+                models.Relation.project_id == project_id,
+                models.Relation.subject_entity_id == subject_id,
+                models.Relation.predicate == predicate,
+                models.Relation.object_entity_id == object_id,
+            )
+        )
+        if relation is None:
+            relation = models.Relation(
+                project_id=project_id,
+                subject_entity_id=subject_id,
+                predicate=predicate,
+                object_entity_id=object_id,
+                confidence=confidence,
+            )
+            self.session.add(relation)
+            self.session.flush()
+        else:
+            relation.support_count += 1
+            relation.confidence = max(relation.confidence, confidence)
+            relation.updated_at = models.utcnow()
+        return relation
+
+
+def _project_config_to_dict(config: ProjectConfig) -> dict[str, Any]:
+    return {
+        "project_name": config.project_name,
+        "domain": config.domain,
+        "target_entities": config.target_entities,
+        "fields": config.fields,
+        "sources": [asdict(source) for source in config.sources],
+        "ontology": config.ontology,
+        "recommendation": config.recommendation,
+        "update_policy": config.update_policy,
+    }
+
+
+def combine_confidence(extraction_confidence: float, source_trust: float) -> float:
+    return round(min(max((extraction_confidence * 0.7) + (source_trust * 0.3), 0.0), 1.0), 4)
+
+
+def make_claim_hash(
+    project_id: int,
+    source_id: int,
+    subject_entity_id: int,
+    predicate: str,
+    object_entity_id: int | None,
+    object_value: Any | None,
+) -> str:
+    payload = {
+        "project_id": project_id,
+        "source_id": source_id,
+        "subject_entity_id": subject_entity_id,
+        "predicate": predicate,
+        "object_entity_id": object_entity_id,
+        "object_value": object_value,
+    }
+    return hashlib.sha256(json.dumps(payload, sort_keys=True, default=str).encode("utf-8")).hexdigest()

crawler_platform/app/core/database/session.py

@@ -0,0 +1,39 @@
+from __future__ import annotations
+
+from contextlib import contextmanager
+from typing import Iterator
+
+from sqlalchemy import create_engine
+from sqlalchemy.orm import Session, sessionmaker
+
+from crawler_platform.app.core.database.models import Base
+
+
+def make_engine(database_url: str = "sqlite:///crawler_platform.db"):
+    connect_args = {"check_same_thread": False} if database_url.startswith("sqlite") else {}
+    return create_engine(database_url, future=True, connect_args=connect_args)
+
+
+def init_db(database_url: str = "sqlite:///crawler_platform.db") -> None:
+    engine = make_engine(database_url)
+    Base.metadata.create_all(engine)
+
+
+def make_session_factory(database_url: str = "sqlite:///crawler_platform.db") -> sessionmaker[Session]:
+    engine = make_engine(database_url)
+    return sessionmaker(bind=engine, expire_on_commit=False, class_=Session, future=True)
+
+
+@contextmanager
+def session_scope(database_url: str = "sqlite:///crawler_platform.db") -> Iterator[Session]:
+    factory = make_session_factory(database_url)
+    session = factory()
+    try:
+        yield session
+        session.commit()
+    except Exception:
+        session.rollback()
+        raise
+    finally:
+        session.close()
+

crawler_platform/app/core/extractor/__init__.py

@@ -0,0 +1,2 @@
+"""Extractor provider interfaces and implementations."""
+

crawler_platform/app/core/extractor/ai_provider.py

@@ -0,0 +1,388 @@
+from __future__ import annotations
+
+import json
+import os
+import re
+from typing import Any
+
+import requests
+
+from crawler_platform.app.config.loader import ProjectConfig
+from crawler_platform.app.core.extractor.base import (
+    AIExtractor,
+    ExtractedClaim,
+    ExtractedEntity,
+    ExtractionBundle,
+)
+from crawler_platform.app.core.ontology.mapper import normalize_predicate
+from crawler_platform.app.domains.perfume.extractor import PerfumeRuleBasedExtractor
+
+
+class LLMJsonExtractor(AIExtractor):
+    name = "llm_json_extractor"
+
+    def __init__(
+        self,
+        domain: str,
+        provider: str,
+        model: str | None = None,
+        base_url: str | None = None,
+        timeout_seconds: int = 300,
+    ):
+        self.domain = domain
+        self.provider = provider
+        self.model = model
+        self.base_url = base_url
+        self.timeout_seconds = timeout_seconds
+
+    def extract(self, page_text: str, project_config: ProjectConfig) -> ExtractionBundle:
+        try:
+            raw = self.complete_json(page_text, project_config)
+        except Exception as exc:
+            return self._fallback_bundle(page_text, project_config, str(exc))
+        bundle = ExtractionBundle(
+            entities=parse_entities(raw.get("entities", [])),
+            claims=parse_claims(raw.get("claims", [])),
+            extractor_name=self.name,
+            provider=self.provider,
+            raw_output={
+                "provider": self.provider,
+                "model": self.model,
+                "entity_count": len(raw.get("entities", [])),
+                "claim_count": len(raw.get("claims", [])),
+            },
+        )
+        if not bundle.entities or not bundle.claims:
+            return self._fallback_bundle(page_text, project_config, "AI returned no usable entities or claims")
+        return self.normalize_to_ontology(bundle, project_config.ontology)
+
+    def extract_entities(self, page_text: str, project_config: ProjectConfig) -> list[ExtractedEntity]:
+        return self.extract(page_text, project_config).entities
+
+    def extract_attributes(
+        self,
+        entity: ExtractedEntity,
+        page_text: str,
+        project_config: ProjectConfig,
+    ) -> dict[str, Any]:
+        return entity.attributes
+
+    def extract_relations(
+        self,
+        entities: list[ExtractedEntity],
+        page_text: str,
+        project_config: ProjectConfig,
+    ) -> list[ExtractedClaim]:
+        return []
+
+    def normalize_to_ontology(self, bundle: ExtractionBundle, ontology: dict[str, Any]) -> ExtractionBundle:
+        for claim in bundle.claims:
+            claim.predicate = normalize_predicate(claim.predicate, ontology)
+        return bundle
+
+    def complete_json(self, page_text: str, project_config: ProjectConfig) -> dict[str, Any]:
+        prompt = build_extraction_prompt(page_text, project_config)
+        if self.provider == "openai":
+            return self._complete_openai_compatible(prompt, "OPENAI_API_KEY", "OPENAI_MODEL", self.base_url)
+        if self.provider == "lm_studio":
+            return self._complete_openai_compatible(
+                prompt,
+                "LM_STUDIO_API_KEY",
+                "LM_STUDIO_MODEL",
+                normalize_openai_chat_url(self.base_url or "http://localhost:1234/v1"),
+                api_key_optional=True,
+            )
+        if self.provider == "ollama":
+            return self._complete_ollama(prompt)
+        raise ValueError(f"Unsupported AI extractor provider: {self.provider}")
+
+    def _fallback_bundle(self, page_text: str, project_config: ProjectConfig, error: str) -> ExtractionBundle:
+        if project_config.domain == "perfume":
+            bundle = PerfumeRuleBasedExtractor().extract(page_text, project_config)
+        else:
+            from crawler_platform.app.core.extractor.rule_based import GenericRuleBasedExtractor
+
+            bundle = GenericRuleBasedExtractor().extract(page_text, project_config)
+        bundle.extractor_name = f"{self.name}_with_rule_fallback"
+        bundle.provider = f"{self.provider}_fallback"
+        bundle.raw_output = {
+            **bundle.raw_output,
+            "ai_provider": self.provider,
+            "ai_model": self.model,
+            "ai_error": error,
+            "fallback": "rule_based",
+        }
+        for entity in bundle.entities:
+            entity.metadata["ai_fallback_reason"] = error
+        for claim in bundle.claims:
+            claim.metadata["ai_fallback_reason"] = error
+            claim.confidence_reason = f"{claim.confidence_reason}; AI fallback: {error}" if claim.confidence_reason else error
+        return bundle
+
+    def _complete_openai_compatible(
+        self,
+        prompt: str,
+        api_key_env: str,
+        model_env: str,
+        endpoint: str | None,
+        api_key_optional: bool = False,
+    ) -> dict[str, Any]:
+        api_key = os.getenv(api_key_env)
+        model = self.model or os.getenv(model_env)
+        if not model and api_key_optional and endpoint:
+            model = first_openai_compatible_model(endpoint, api_key)
+        if not model:
+            raise RuntimeError(f"AI model is required. Set UI model field or {model_env}.")
+        if not api_key and not api_key_optional:
+            raise RuntimeError(f"API key is required. Set {api_key_env}.")
+        url = endpoint or "https://api.openai.com/v1/chat/completions"
+        headers = {"Content-Type": "application/json"}
+        if api_key:
+            headers["Authorization"] = f"Bearer {api_key}"
+        response = requests.post(
+            url,
+            headers=headers,
+            json={
+                "model": model,
+                "messages": [
+                    {"role": "system", "content": "Extract ontology knowledge as strict JSON only."},
+                    {"role": "user", "content": prompt},
+                ],
+                "temperature": 0,
+            },
+            timeout=self.timeout_seconds,
+        )
+        if response.status_code >= 400:
+            raise RuntimeError(f"LLM request failed {response.status_code}: {response.text[:1000]}")
+        data = response.json()
+        content = data["choices"][0]["message"]["content"]
+        return parse_json_content(content, retry=lambda bad: self._repair_json_with_model(bad, endpoint, headers, model))
+
+    def _complete_ollama(self, prompt: str) -> dict[str, Any]:
+        model = self.model or os.getenv("OLLAMA_MODEL")
+        if not model:
+            raise RuntimeError("Ollama model is required. Set UI model field or OLLAMA_MODEL.")
+        url = self.base_url or "http://localhost:11434/api/chat"
+        response = requests.post(
+            url,
+            json={
+                "model": model,
+                "messages": [
+                    {"role": "system", "content": "Extract ontology knowledge as strict JSON only."},
+                    {"role": "user", "content": prompt},
+                ],
+                "stream": False,
+                "format": "json",
+            },
+            timeout=self.timeout_seconds,
+        )
+        response.raise_for_status()
+        content = response.json()["message"]["content"]
+        return parse_json_content(content)
+
+    def _repair_json_with_model(
+        self,
+        bad_content: str,
+        endpoint: str,
+        headers: dict[str, str],
+        model: str,
+    ) -> dict[str, Any]:
+        response = requests.post(
+            endpoint,
+            headers=headers,
+            json={
+                "model": model,
+                "messages": [
+                    {
+                        "role": "system",
+                        "content": "You repair malformed JSON. Return valid JSON only. No markdown.",
+                    },
+                    {
+                        "role": "user",
+                        "content": (
+                            "Repair this into valid JSON with top-level keys entities and claims. "
+                            "Drop invalid fragments if needed.\n\n"
+                            f"{bad_content[:12000]}"
+                        ),
+                    },
+                ],
+                "temperature": 0,
+            },
+            timeout=self.timeout_seconds,
+        )
+        if response.status_code >= 400:
+            raise RuntimeError(f"LLM JSON repair failed {response.status_code}: {response.text[:1000]}")
+        return parse_json_content(response.json()["choices"][0]["message"]["content"])
+
+
+def build_extraction_prompt(page_text: str, project_config: ProjectConfig) -> str:
+    clipped_text = page_text[:6000]
+    ontology = project_config.ontology or {}
+    return f"""
+Project domain: {project_config.domain}
+Target entity types: {project_config.target_entities}
+Fields: {project_config.fields}
+Allowed predicates: {ontology.get("predicates", [])}
+
+Return only minified strict JSON. Do not include markdown, analysis, or prose.
+Use this shape:
+{{
+  "entities": [
+    {{
+      "entity_type": "Perfume",
+      "name": "Product name",
+      "attributes": {{"name": "Product name"}},
+      "confidence": 0.0,
+      "evidence_text": "short evidence from page"
+    }}
+  ],
+  "claims": [
+    {{
+      "subject_name": "Product name",
+      "subject_type": "Perfume",
+      "predicate": "hasTopNote",
+      "object_name": "Bergamot",
+      "object_type": "Note",
+      "object_value": null,
+      "evidence_text": "short evidence from page",
+      "evidence_summary": "why this claim was extracted",
+      "confidence": 0.0,
+      "confidence_reason": "reason"
+    }}
+  ]
+}}
+
+Rules:
+- Store information as source claims, not absolute facts.
+- Keep evidence_text short. Do not copy long descriptions.
+- Use only ontology predicates when possible.
+- If object is a simple value like price, put it in object_value and leave object_name/object_type null.
+- If unsure, lower confidence instead of inventing.
+- Extract at most 20 entities and 30 claims.
+- For perfume, prioritize name, brand, top/middle/base notes, accords, mood, season, occasion, price, review keywords.
+
+Page text:
+{clipped_text}
+""".strip()
+
+
+def parse_json_content(content: str, retry=None) -> dict[str, Any]:
+    try:
+        return json.loads(content)
+    except json.JSONDecodeError:
+        match = re.search(r"\{.*\}", content, flags=re.DOTALL)
+        if not match:
+            if retry:
+                return retry(content)
+            raise
+        try:
+            return json.loads(match.group(0))
+        except json.JSONDecodeError:
+            if retry:
+                return retry(content)
+            repaired = heuristic_repair_json(content)
+            if repaired is not None:
+                return repaired
+            raise
+
+
+def heuristic_repair_json(content: str) -> dict[str, Any] | None:
+    """Best-effort extraction for chatty local models that emit broken JSON."""
+
+    entities = []
+    claims = []
+    for block in re.findall(r"\{[^{}]*(?:\{[^{}]*\}[^{}]*)?\}", content, flags=re.DOTALL):
+        try:
+            item = json.loads(block)
+        except json.JSONDecodeError:
+            continue
+        if {"entity_type", "name"}.issubset(item):
+            entities.append(item)
+        if {"subject_name", "subject_type", "predicate"}.issubset(item):
+            claims.append(item)
+    if entities or claims:
+        return {"entities": entities, "claims": claims}
+    return None
+
+
+def normalize_openai_chat_url(base_url: str) -> str:
+    clean = base_url.rstrip("/")
+    if clean.endswith("/chat/completions"):
+        return clean
+    if clean.endswith("/v1"):
+        return f"{clean}/chat/completions"
+    return f"{clean}/v1/chat/completions"
+
+
+def normalize_openai_models_url(base_url: str) -> str:
+    clean = base_url.rstrip("/")
+    if clean.endswith("/chat/completions"):
+        return clean.removesuffix("/chat/completions") + "/models"
+    if clean.endswith("/models"):
+        return clean
+    if clean.endswith("/v1"):
+        return f"{clean}/models"
+    return f"{clean}/v1/models"
+
+
+def first_openai_compatible_model(base_url: str, api_key: str | None = None) -> str | None:
+    models = list_openai_compatible_models(base_url, api_key)
+    return models[0]["id"] if models else None
+
+
+def list_openai_compatible_models(base_url: str, api_key: str | None = None) -> list[dict[str, Any]]:
+    url = normalize_openai_models_url(base_url)
+    headers = {}
+    if api_key:
+        headers["Authorization"] = f"Bearer {api_key}"
+    response = requests.get(url, headers=headers, timeout=10)
+    response.raise_for_status()
+    data = response.json()
+    items = data.get("data", [])
+    return [{"id": item.get("id", ""), "owned_by": item.get("owned_by")} for item in items if item.get("id")]
+
+
+def parse_entities(items: list[dict[str, Any]]) -> list[ExtractedEntity]:
+    entities: list[ExtractedEntity] = []
+    for item in items:
+        name = item.get("name")
+        entity_type = item.get("entity_type") or item.get("type")
+        if not name or not entity_type:
+            continue
+        entities.append(
+            ExtractedEntity(
+                entity_type=str(entity_type),
+                name=str(name),
+                attributes=dict(item.get("attributes") or {}),
+                evidence_text=item.get("evidence_text"),
+                confidence=float(item.get("confidence") or 0.55),
+                metadata={"ai_extracted": True},
+            )
+        )
+    return entities
+
+
+def parse_claims(items: list[dict[str, Any]]) -> list[ExtractedClaim]:
+    claims: list[ExtractedClaim] = []
+    for item in items:
+        subject_name = item.get("subject_name")
+        subject_type = item.get("subject_type")
+        predicate = item.get("predicate")
+        if not subject_name or not subject_type or not predicate:
+            continue
+        claims.append(
+            ExtractedClaim(
+                subject_name=str(subject_name),
+                subject_type=str(subject_type),
+                predicate=str(predicate),
+                object_name=item.get("object_name"),
+                object_type=item.get("object_type"),
+                object_value=item.get("object_value"),
+                evidence_text=item.get("evidence_text"),
+                evidence_summary=item.get("evidence_summary"),
+                confidence=float(item.get("confidence") or 0.55),
+                confidence_reason=item.get("confidence_reason") or "AI extractor output",
+                metadata={"ai_extracted": True},
+            )
+        )
+    return claims

crawler_platform/app/core/extractor/base.py

@@ -0,0 +1,101 @@
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from dataclasses import dataclass, field
+from typing import Any
+
+from crawler_platform.app.config.loader import ProjectConfig
+
+
+@dataclass(slots=True)
+class ExtractedEntity:
+    entity_type: str
+    name: str
+    attributes: dict[str, Any] = field(default_factory=dict)
+    evidence_text: str | None = None
+    confidence: float = 0.5
+    metadata: dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass(slots=True)
+class ExtractedClaim:
+    subject_name: str
+    subject_type: str
+    predicate: str
+    object_name: str | None = None
+    object_type: str | None = None
+    object_value: Any | None = None
+    evidence_text: str | None = None
+    evidence_summary: str | None = None
+    confidence: float = 0.5
+    confidence_reason: str | None = None
+    metadata: dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass(slots=True)
+class ExtractionBundle:
+    entities: list[ExtractedEntity] = field(default_factory=list)
+    claims: list[ExtractedClaim] = field(default_factory=list)
+    extractor_name: str = "unknown"
+    provider: str = "unknown"
+    raw_output: dict[str, Any] = field(default_factory=dict)
+
+
+class Extractor(ABC):
+    name = "base"
+    provider = "base"
+
+    @abstractmethod
+    def extract_entities(self, page_text: str, project_config: ProjectConfig) -> list[ExtractedEntity]:
+        raise NotImplementedError
+
+    @abstractmethod
+    def extract_attributes(
+        self,
+        entity: ExtractedEntity,
+        page_text: str,
+        project_config: ProjectConfig,
+    ) -> dict[str, Any]:
+        raise NotImplementedError
+
+    @abstractmethod
+    def extract_relations(
+        self,
+        entities: list[ExtractedEntity],
+        page_text: str,
+        project_config: ProjectConfig,
+    ) -> list[ExtractedClaim]:
+        raise NotImplementedError
+
+    @abstractmethod
+    def normalize_to_ontology(self, bundle: ExtractionBundle, ontology: dict[str, Any]) -> ExtractionBundle:
+        raise NotImplementedError
+
+    def extract(self, page_text: str, project_config: ProjectConfig) -> ExtractionBundle:
+        entities = self.extract_entities(page_text, project_config)
+        for entity in entities:
+            entity.attributes.update(self.extract_attributes(entity, page_text, project_config))
+        claims = self.extract_relations(entities, page_text, project_config)
+        bundle = ExtractionBundle(
+            entities=entities,
+            claims=claims,
+            extractor_name=self.name,
+            provider=self.provider,
+            raw_output={"entity_count": len(entities), "claim_count": len(claims)},
+        )
+        return self.normalize_to_ontology(bundle, project_config.ontology)
+
+
+class AIExtractor(Extractor):
+    """Provider-neutral AI extractor contract.
+
+    OpenAI, local LLM, Ollama, and LM Studio adapters can subclass this and
+    implement ``complete_json`` while keeping the rest of the pipeline stable.
+    """
+
+    provider = "ai"
+
+    @abstractmethod
+    def complete_json(self, page_text: str, project_config: ProjectConfig) -> dict[str, Any]:
+        raise NotImplementedError
+

crawler_platform/app/core/extractor/factory.py

@@ -0,0 +1,19 @@
+from __future__ import annotations
+
+from crawler_platform.app.core.extractor.ai_provider import LLMJsonExtractor
+from crawler_platform.app.core.extractor.base import Extractor
+from crawler_platform.app.core.extractor.rule_based import GenericRuleBasedExtractor
+from crawler_platform.app.domains.perfume.extractor import PerfumeRuleBasedExtractor
+
+
+def extractor_for_domain(
+    domain: str,
+    provider: str = "rule_based",
+    model: str | None = None,
+    base_url: str | None = None,
+) -> Extractor:
+    if provider in {"openai", "ollama", "lm_studio"}:
+        return LLMJsonExtractor(domain=domain, provider=provider, model=model, base_url=base_url)
+    if domain == "perfume":
+        return PerfumeRuleBasedExtractor()
+    return GenericRuleBasedExtractor()

crawler_platform/app/core/extractor/rule_based.py

@@ -0,0 +1,85 @@
+from __future__ import annotations
+
+import re
+
+from crawler_platform.app.config.loader import ProjectConfig
+from crawler_platform.app.core.extractor.base import ExtractedClaim, ExtractedEntity, ExtractionBundle, Extractor
+from crawler_platform.app.core.ontology.mapper import normalize_predicate
+
+
+class GenericRuleBasedExtractor(Extractor):
+    name = "generic_rule_based"
+    provider = "rule_based"
+
+    def extract_entities(self, page_text: str, project_config: ProjectConfig) -> list[ExtractedEntity]:
+        name = first_non_empty_line(page_text) or "Unknown Product"
+        primary_type = project_config.target_entities[0] if project_config.target_entities else "Product"
+        return [ExtractedEntity(entity_type=primary_type, name=name, confidence=0.45)]
+
+    def extract_attributes(
+        self,
+        entity: ExtractedEntity,
+        page_text: str,
+        project_config: ProjectConfig,
+    ) -> dict[str, object]:
+        attrs: dict[str, object] = {"name": entity.name}
+        price = find_price(page_text)
+        if price:
+            attrs["price"] = price
+        return attrs
+
+    def extract_relations(
+        self,
+        entities: list[ExtractedEntity],
+        page_text: str,
+        project_config: ProjectConfig,
+    ) -> list[ExtractedClaim]:
+        if not entities:
+            return []
+        entity = entities[0]
+        claims: list[ExtractedClaim] = []
+        price = find_price(page_text)
+        if price:
+            claims.append(
+                ExtractedClaim(
+                    subject_name=entity.name,
+                    subject_type=entity.entity_type,
+                    predicate="hasPrice",
+                    object_value=price,
+                    evidence_text=price["evidence"],
+                    confidence=0.6,
+                    confidence_reason="price pattern matched",
+                )
+            )
+        return claims
+
+    def normalize_to_ontology(self, bundle: ExtractionBundle, ontology: dict[str, object]) -> ExtractionBundle:
+        for claim in bundle.claims:
+            claim.predicate = normalize_predicate(claim.predicate, ontology)
+        return bundle
+
+
+def first_non_empty_line(text: str) -> str | None:
+    for line in text.splitlines():
+        clean = line.strip()
+        if clean:
+            return clean[:240]
+    return None
+
+
+def find_price(text: str) -> dict[str, object] | None:
+    patterns = [
+        r"(?P<currency>[$€£])\s?(?P<amount>\d+(?:[,.]\d{2})?)",
+        r"(?P<amount>\d{1,3}(?:,\d{3})*)\s?(?P<currency>원|KRW|USD)",
+    ]
+    for pattern in patterns:
+        match = re.search(pattern, text, flags=re.IGNORECASE)
+        if match:
+            amount = match.group("amount").replace(",", "")
+            return {
+                "amount": float(amount),
+                "currency": match.group("currency"),
+                "evidence": match.group(0),
+            }
+    return None
+

crawler_platform/app/core/ontology/__init__.py

@@ -0,0 +1,2 @@
+"""Ontology definitions and mapping helpers."""
+

crawler_platform/app/core/ontology/definitions.py

@@ -0,0 +1,130 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+
+@dataclass(slots=True)
+class Ontology:
+    domain: str
+    entity_types: list[str]
+    predicates: list[str]
+    attributes: list[str]
+    aliases: dict[str, str] = field(default_factory=dict)
+
+
+COMMON_ONTOLOGY = Ontology(
+    domain="common",
+    entity_types=[
+        "Source",
+        "Page",
+        "Entity",
+        "Attribute",
+        "Relation",
+        "Claim",
+        "Evidence",
+        "Extraction",
+        "Confidence",
+        "UpdatePolicy",
+    ],
+    predicates=[
+        "mentions",
+        "hasAttribute",
+        "relatedTo",
+        "sameAs",
+        "soldBy",
+        "hasPrice",
+    ],
+    attributes=["name", "source_url", "updated_at", "confidence"],
+)
+
+
+DOMAIN_ONTOLOGIES: dict[str, Ontology] = {
+    "perfume": Ontology(
+        domain="perfume",
+        entity_types=[
+            "Perfume",
+            "Brand",
+            "Note",
+            "Accord",
+            "Mood",
+            "Season",
+            "Occasion",
+            "Review",
+            "Price",
+            "ProductPage",
+        ],
+        predicates=[
+            "hasBrand",
+            "hasTopNote",
+            "hasMiddleNote",
+            "hasBaseNote",
+            "hasAccord",
+            "evokesMood",
+            "suitableForSeason",
+            "suitableForOccasion",
+            "similarTo",
+            "soldBy",
+            "hasPrice",
+            "hasReviewKeyword",
+        ],
+        attributes=[
+            "name",
+            "brand",
+            "gender_bias",
+            "longevity",
+            "sillage",
+            "price_range",
+            "popularity_score",
+            "review_count",
+            "source_url",
+            "updated_at",
+        ],
+        aliases={
+            "top_notes": "hasTopNote",
+            "middle_notes": "hasMiddleNote",
+            "heart_notes": "hasMiddleNote",
+            "base_notes": "hasBaseNote",
+            "accords": "hasAccord",
+            "mood_tags": "evokesMood",
+            "season_tags": "suitableForSeason",
+            "occasion_tags": "suitableForOccasion",
+            "review_keywords": "hasReviewKeyword",
+            "price": "hasPrice",
+        },
+    ),
+    "tea": Ontology(
+        domain="tea",
+        entity_types=["Tea", "Ingredient", "Flavor", "Effect", "CaffeineLevel", "MoodState"],
+        predicates=["hasIngredient", "hasFlavor", "hasEffect", "suitableForCondition"],
+        attributes=["name", "origin", "caffeine_level", "price_range"],
+    ),
+    "coffee": Ontology(
+        domain="coffee",
+        entity_types=["CoffeeBean", "Origin", "RoastLevel", "FlavorNote", "BrewMethod"],
+        predicates=["hasOrigin", "hasRoastLevel", "hasFlavorNote", "recommendedForBrewMethod"],
+        attributes=["name", "origin", "roast_level", "process", "price_range"],
+    ),
+    "candle": Ontology(
+        domain="candle",
+        entity_types=["ScentProduct", "ScentNote", "SpaceType", "Mood", "Season"],
+        predicates=["suitableForSpace", "evokesMood", "hasScentNote"],
+        attributes=["name", "burn_time", "volume", "price_range"],
+    ),
+    "supplement": Ontology(
+        domain="supplement",
+        entity_types=["Supplement", "Ingredient", "HealthGoal", "Symptom", "Dosage"],
+        predicates=["hasIngredient", "supportsGoal", "recommendedForCondition"],
+        attributes=["name", "dosage", "warnings", "price_range"],
+    ),
+    "gift": Ontology(
+        domain="gift",
+        entity_types=["GiftProduct", "RecipientType", "Relationship", "Occasion", "PersonalityTag"],
+        predicates=["suitableForRecipient", "suitableForOccasion", "matchesPersonality"],
+        attributes=["name", "price_range", "availability", "gift_wrap_available"],
+    ),
+}
+
+
+def ontology_for_domain(domain: str) -> Ontology:
+    return DOMAIN_ONTOLOGIES.get(domain, COMMON_ONTOLOGY)
+

crawler_platform/app/core/ontology/mapper.py

@@ -0,0 +1,27 @@
+from __future__ import annotations
+
+from typing import Any
+
+
+def normalize_predicate(predicate: str, ontology: dict[str, Any] | None) -> str:
+    if not ontology:
+        return predicate
+    aliases = ontology.get("aliases", {})
+    return aliases.get(predicate, predicate)
+
+
+def is_allowed_predicate(predicate: str, ontology: dict[str, Any] | None) -> bool:
+    if not ontology or not ontology.get("predicates"):
+        return True
+    return predicate in ontology["predicates"]
+
+
+def ontology_to_dict(domain_ontology) -> dict[str, Any]:
+    return {
+        "domain": domain_ontology.domain,
+        "entity_types": domain_ontology.entity_types,
+        "predicates": domain_ontology.predicates,
+        "attributes": domain_ontology.attributes,
+        "aliases": domain_ontology.aliases,
+    }
+

crawler_platform/app/core/recommendation/__init__.py

@@ -0,0 +1,2 @@
+"""Recommendation integration helpers."""
+

crawler_platform/app/core/recommendation/scorer.py

@@ -0,0 +1,100 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any
+
+from sqlalchemy import select
+from sqlalchemy.orm import Session
+
+from crawler_platform.app.core.database import models
+from crawler_platform.app.core.database.repository import canonicalize
+
+
+@dataclass(slots=True)
+class PreferenceInput:
+    likes: list[str] = field(default_factory=list)
+    dislikes: list[str] = field(default_factory=list)
+    preferred_moods: list[str] = field(default_factory=list)
+    preferred_notes: list[str] = field(default_factory=list)
+    avoided_notes: list[str] = field(default_factory=list)
+    price_preference: dict[str, Any] = field(default_factory=dict)
+    season_context: str | None = None
+    occasion_context: str | None = None
+
+
+@dataclass(slots=True)
+class Recommendation:
+    entity_id: int
+    name: str
+    entity_type: str
+    score: float
+    reasons: list[str]
+
+
+class RuleBasedRecommender:
+    def __init__(self, session: Session):
+        self.session = session
+
+    def recommend(
+        self,
+        project_id: int,
+        target_entity_type: str,
+        preference: PreferenceInput,
+        limit: int = 10,
+    ) -> list[Recommendation]:
+        entities = self.session.scalars(
+            select(models.Entity).where(
+                models.Entity.project_id == project_id,
+                models.Entity.entity_type == target_entity_type,
+            )
+        ).all()
+        scored = [self._score_entity(project_id, entity, preference) for entity in entities]
+        scored = [item for item in scored if item.score > 0]
+        return sorted(scored, key=lambda item: item.score, reverse=True)[:limit]
+
+    def _score_entity(self, project_id: int, entity: models.Entity, preference: PreferenceInput) -> Recommendation:
+        claims = self.session.execute(
+            select(models.Claim, models.Entity)
+            .join(models.Entity, models.Claim.object_entity_id == models.Entity.id, isouter=True)
+            .where(models.Claim.project_id == project_id, models.Claim.subject_entity_id == entity.id)
+        ).all()
+        score = 0.0
+        reasons: list[str] = []
+        preferred_notes = {canonicalize(item) for item in preference.preferred_notes}
+        avoided_notes = {canonicalize(item) for item in preference.avoided_notes}
+        preferred_moods = {canonicalize(item) for item in preference.preferred_moods}
+        for claim, object_entity in claims:
+            object_name = canonicalize(object_entity.name) if object_entity else ""
+            weight = claim.confidence
+            if claim.predicate in {"hasTopNote", "hasMiddleNote", "hasBaseNote", "hasScentNote", "hasFlavorNote"}:
+                if object_name in preferred_notes:
+                    score += 2.0 * weight
+                    reasons.append(f"preferred note matched: {object_entity.name}")
+                if object_name in avoided_notes:
+                    score -= 3.0 * weight
+                    reasons.append(f"avoided note matched: {object_entity.name}")
+            if claim.predicate == "evokesMood" and object_name in preferred_moods:
+                score += 1.5 * weight
+                reasons.append(f"preferred mood matched: {object_entity.name}")
+            if preference.season_context and claim.predicate == "suitableForSeason":
+                if object_name == canonicalize(preference.season_context):
+                    score += 1.2 * weight
+                    reasons.append(f"season context matched: {preference.season_context}")
+            if preference.occasion_context and claim.predicate == "suitableForOccasion":
+                if object_name == canonicalize(preference.occasion_context):
+                    score += 1.0 * weight
+                    reasons.append(f"occasion context matched: {preference.occasion_context}")
+        if canonicalize(entity.name) in {canonicalize(item) for item in preference.dislikes}:
+            score -= 10
+            reasons.append("explicit dislike")
+        if canonicalize(entity.name) in {canonicalize(item) for item in preference.likes}:
+            score += 5
+            reasons.append("explicit like")
+        return Recommendation(
+            entity_id=entity.id,
+            name=entity.name,
+            entity_type=entity.entity_type,
+            score=round(score, 4),
+            reasons=reasons[:5],
+        )
+

crawler_platform/app/core/scheduler/__init__.py

@@ -0,0 +1,2 @@
+"""Scheduling primitives for recrawls."""
+

crawler_platform/app/core/scheduler/update_policy.py

@@ -0,0 +1,11 @@
+from __future__ import annotations
+
+from datetime import datetime, timedelta, timezone
+
+
+def next_crawl_at(policy: dict[str, object] | None, now: datetime | None = None) -> datetime:
+    now = now or datetime.now(timezone.utc)
+    policy = policy or {}
+    interval_days = int(policy.get("interval_days", 7))
+    return now + timedelta(days=interval_days)
+