[crawler]

crawler_platform/app/core/crawler/__init__.py

@@ -0,0 +1,2 @@
+"""Crawler pipeline."""
+

crawler_platform/app/core/crawler/discovery.py

@@ -0,0 +1,52 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from urllib.parse import parse_qs, urljoin, urlparse
+
+from bs4 import BeautifulSoup
+
+
+@dataclass(slots=True)
+class DiscoveredUrl:
+    url: str
+    label: str
+    kind: str = "link"
+
+
+def discover_links(html: str, base_url: str, limit: int = 30) -> list[DiscoveredUrl]:
+    soup = BeautifulSoup(html, "html.parser")
+    seen: set[str] = set()
+    results: list[DiscoveredUrl] = []
+    for anchor in soup.find_all("a", href=True):
+        raw_href = anchor.get("href", "")
+        url = normalize_search_redirect(urljoin(base_url, raw_href))
+        if not url or url in seen or not url.startswith(("http://", "https://")):
+            continue
+        seen.add(url)
+        label = anchor.get_text(" ", strip=True)[:160] or urlparse(url).netloc
+        results.append(DiscoveredUrl(url=url, label=label, kind=classify_url(url)))
+        if len(results) >= limit:
+            break
+    return results
+
+
+def normalize_search_redirect(url: str) -> str:
+    parsed = urlparse(url)
+    query = parse_qs(parsed.query)
+    for key in ("url", "u", "target"):
+        if key in query and query[key]:
+            candidate = query[key][0]
+            if candidate.startswith(("http://", "https://")):
+                return candidate
+    return url
+
+
+def classify_url(url: str) -> str:
+    host = urlparse(url).netloc.lower()
+    if "smartstore.naver.com" in host or "brand.naver.com" in host:
+        return "marketplace_product_or_store"
+    if "shopping.naver.com" in host:
+        return "shopping"
+    if any(token in host for token in ("fragrantica", "official", "perfume", "parfum")):
+        return "product_or_review"
+    return "link"

crawler_platform/app/core/crawler/fetchers.py

@@ -0,0 +1,136 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from pathlib import Path
+import time
+from urllib.parse import urlparse
+from urllib.robotparser import RobotFileParser
+
+
+DEFAULT_USER_AGENT = "OntologyCrawlerBot/0.1 (+contact: admin@example.com)"
+
+
+@dataclass(slots=True)
+class FetchResult:
+    url: str
+    status_code: int | None
+    html: str
+    final_url: str | None = None
+    headers: dict[str, str] = field(default_factory=dict)
+
+
+class RateLimiter:
+    def __init__(self, per_minute: int = 30):
+        self.delay = 60 / max(per_minute, 1)
+        self._last_called = 0.0
+
+    def wait(self) -> None:
+        elapsed = time.monotonic() - self._last_called
+        if elapsed < self.delay:
+            time.sleep(self.delay - elapsed)
+        self._last_called = time.monotonic()
+
+
+class RobotsPolicy:
+    def __init__(self, user_agent: str = DEFAULT_USER_AGENT):
+        self.user_agent = user_agent
+        self._cache: dict[str, RobotFileParser] = {}
+
+    def allowed(self, url: str, respect_robots_txt: bool = True) -> bool:
+        if not respect_robots_txt:
+            return True
+        parsed = urlparse(url)
+        if parsed.scheme in {"", "file"}:
+            return True
+        robots_url = f"{parsed.scheme}://{parsed.netloc}/robots.txt"
+        parser = self._cache.get(robots_url)
+        if parser is None:
+            parser = RobotFileParser()
+            parser.set_url(robots_url)
+            try:
+                parser.read()
+            except Exception:
+                return False
+            self._cache[robots_url] = parser
+        return parser.can_fetch(self.user_agent, url)
+
+
+class BaseFetcher:
+    def fetch(self, url: str) -> FetchResult:
+        raise NotImplementedError
+
+
+class RequestsFetcher(BaseFetcher):
+    def __init__(
+        self,
+        user_agent: str = DEFAULT_USER_AGENT,
+        timeout_seconds: int = 15,
+        retries: int = 2,
+        rate_limit_per_minute: int = 30,
+    ):
+        self.user_agent = user_agent
+        self.timeout_seconds = timeout_seconds
+        self.retries = retries
+        self.rate_limiter = RateLimiter(rate_limit_per_minute)
+
+    def fetch(self, url: str) -> FetchResult:
+        local_path = _local_path_from_url(url)
+        if local_path:
+            return FetchResult(url=url, status_code=200, html=local_path.read_text(encoding="utf-8"), final_url=url)
+
+        import requests
+
+        last_error: Exception | None = None
+        for _ in range(self.retries + 1):
+            self.rate_limiter.wait()
+            try:
+                response = requests.get(
+                    url,
+                    headers={"User-Agent": self.user_agent},
+                    timeout=self.timeout_seconds,
+                )
+                return FetchResult(
+                    url=url,
+                    status_code=response.status_code,
+                    html=response.text,
+                    final_url=response.url,
+                    headers=dict(response.headers),
+                )
+            except requests.RequestException as exc:
+                last_error = exc
+        raise RuntimeError(f"Failed to fetch {url}: {last_error}")
+
+
+class PlaywrightFetcher(BaseFetcher):
+    def __init__(self, user_agent: str = DEFAULT_USER_AGENT, timeout_ms: int = 20000):
+        self.user_agent = user_agent
+        self.timeout_ms = timeout_ms
+
+    def fetch(self, url: str) -> FetchResult:
+        from playwright.sync_api import sync_playwright
+
+        with sync_playwright() as playwright:
+            browser = playwright.chromium.launch(headless=True)
+            page = browser.new_page(user_agent=self.user_agent)
+            response = page.goto(url, wait_until="networkidle", timeout=self.timeout_ms)
+            html = page.content()
+            final_url = page.url
+            status = response.status if response else None
+            browser.close()
+        return FetchResult(url=url, status_code=status, html=html, final_url=final_url)
+
+
+def make_fetcher(kind: str, rate_limit_per_minute: int = 30) -> BaseFetcher:
+    if kind == "playwright":
+        return PlaywrightFetcher()
+    return RequestsFetcher(rate_limit_per_minute=rate_limit_per_minute)
+
+
+def _local_path_from_url(url: str) -> Path | None:
+    if url.startswith("file://"):
+        path = Path(url.removeprefix("file://"))
+        return path if path.exists() and path.is_file() else None
+    path = Path(url)
+    if path.exists() and path.is_file():
+        return path
+    return None

crawler_platform/app/core/crawler/html_cleaner.py

@@ -0,0 +1,25 @@
+from __future__ import annotations
+
+import re
+
+
+def clean_html(html: str) -> tuple[str | None, str]:
+    try:
+        from bs4 import BeautifulSoup
+    except ImportError:
+        text = re.sub(r"<[^>]+>", " ", html)
+        return None, normalize_whitespace(text)
+
+    soup = BeautifulSoup(html, "html.parser")
+    for tag in soup(["script", "style", "noscript", "svg", "iframe"]):
+        tag.decompose()
+    title = soup.title.get_text(" ", strip=True) if soup.title else None
+    main = soup.find("main") or soup.body or soup
+    text = main.get_text("\n", strip=True)
+    return title, normalize_whitespace(text)
+
+
+def normalize_whitespace(text: str) -> str:
+    lines = [" ".join(line.split()) for line in text.splitlines()]
+    return "\n".join(line for line in lines if line)
+

crawler_platform/app/core/crawler/pipeline.py

@@ -0,0 +1,55 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from crawler_platform.app.config.loader import ProjectConfig
+from crawler_platform.app.core.crawler.fetchers import RobotsPolicy, make_fetcher
+from crawler_platform.app.core.crawler.plugins import ParserRegistry, default_parser_registry
+from crawler_platform.app.core.database.repository import KnowledgeRepository
+from crawler_platform.app.core.extractor.base import Extractor
+
+
+@dataclass(slots=True)
+class CrawlResult:
+    page_id: int
+    claim_count: int
+    entity_count: int
+
+
+class CrawlPipeline:
+    def __init__(
+        self,
+        repository: KnowledgeRepository,
+        extractor: Extractor,
+        parser_registry: ParserRegistry | None = None,
+        robots_policy: RobotsPolicy | None = None,
+    ):
+        self.repository = repository
+        self.extractor = extractor
+        self.parser_registry = parser_registry or default_parser_registry()
+        self.robots_policy = robots_policy or RobotsPolicy()
+
+    def crawl_url(self, project_config: ProjectConfig, source_name: str, url: str) -> CrawlResult:
+        project = self.repository.upsert_project(project_config)
+        source_config = project_config.source_by_name(source_name)
+        source = self.repository.get_source(project.id, source_name)
+        if not self.robots_policy.allowed(url, source_config.respect_robots_txt):
+            raise PermissionError(f"robots.txt does not allow crawling: {url}")
+
+        fetcher = make_fetcher(source_config.fetcher, source_config.rate_limit_per_minute)
+        fetch_result = fetcher.fetch(url)
+        parser = self.parser_registry.get(source_config.parser)
+        parsed = parser.parse(fetch_result.html, fetch_result.final_url or url)
+        page = self.repository.upsert_page(
+            project_id=project.id,
+            source_id=source.id,
+            url=url,
+            title=parsed.title,
+            status_code=fetch_result.status_code,
+            cleaned_text=parsed.text,
+            metadata={**parsed.metadata, "final_url": fetch_result.final_url},
+        )
+        bundle = self.extractor.extract(parsed.text, project_config)
+        claims = self.repository.save_extraction_bundle(project.id, source, page, bundle)
+        return CrawlResult(page_id=page.id, claim_count=len(claims), entity_count=len(bundle.entities))
+

crawler_platform/app/core/crawler/plugins.py

@@ -0,0 +1,48 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Protocol
+
+
+@dataclass(slots=True)
+class ParsedPage:
+    title: str | None
+    text: str
+    metadata: dict[str, object]
+
+
+class SiteParser(Protocol):
+    name: str
+
+    def parse(self, html: str, url: str) -> ParsedPage:
+        raise NotImplementedError
+
+
+class ParserRegistry:
+    def __init__(self):
+        self._parsers: dict[str, SiteParser] = {}
+
+    def register(self, parser: SiteParser) -> None:
+        self._parsers[parser.name] = parser
+
+    def get(self, name: str) -> SiteParser:
+        if name not in self._parsers:
+            raise KeyError(f"Parser not registered: {name}")
+        return self._parsers[name]
+
+
+class GenericProductParser:
+    name = "generic"
+
+    def parse(self, html: str, url: str) -> ParsedPage:
+        from crawler_platform.app.core.crawler.html_cleaner import clean_html
+
+        title, text = clean_html(html)
+        return ParsedPage(title=title, text=text, metadata={"parser": self.name, "url": url})
+
+
+def default_parser_registry() -> ParserRegistry:
+    registry = ParserRegistry()
+    registry.register(GenericProductParser())
+    return registry
+