[버그수정]

crawler_platform/app/core/crawler/pipeline.py

@@ -30,9 +30,7 @@ class CrawlPipeline:
         self.robots_policy = robots_policy or RobotsPolicy()
 
     def crawl_url(self, project_config: ProjectConfig, source_name: str, url: str) -> CrawlResult:
-        project = self.repository.upsert_project(project_config)
         source_config = project_config.source_by_name(source_name)
-        source = self.repository.get_source(project.id, source_name)
         if not self.robots_policy.allowed(url, source_config.respect_robots_txt):
             raise PermissionError(f"robots.txt does not allow crawling: {url}")
 
@@ -40,6 +38,10 @@ class CrawlPipeline:
         fetch_result = fetcher.fetch(url)
         parser = self.parser_registry.get(source_config.parser)
         parsed = parser.parse(fetch_result.html, fetch_result.final_url or url)
+        bundle = self.extractor.extract(parsed.text, project_config)
+
+        project = self.repository.upsert_project(project_config)
+        source = self.repository.get_source(project.id, source_name)
         page = self.repository.upsert_page(
             project_id=project.id,
             source_id=source.id,
@@ -49,7 +51,5 @@ class CrawlPipeline:
             cleaned_text=parsed.text,
             metadata={**parsed.metadata, "final_url": fetch_result.final_url},
         )
-        bundle = self.extractor.extract(parsed.text, project_config)
         claims = self.repository.save_extraction_bundle(project.id, source, page, bundle)
         return CrawlResult(page_id=page.id, claim_count=len(claims), entity_count=len(bundle.entities))
-

crawler_platform/app/core/crawler/site_crawler.py

@@ -0,0 +1,224 @@
+from __future__ import annotations
+
+from collections import deque
+from dataclasses import dataclass, field
+from urllib.parse import urldefrag, urlparse
+
+from crawler_platform.app.config.loader import ProjectConfig
+from crawler_platform.app.core.crawler.discovery import discover_links
+from crawler_platform.app.core.crawler.fetchers import RobotsPolicy, make_fetcher
+from crawler_platform.app.core.crawler.plugins import ParserRegistry, default_parser_registry
+from crawler_platform.app.core.database import models
+from crawler_platform.app.core.database.repository import KnowledgeRepository
+from crawler_platform.app.core.extractor.base import Extractor
+
+
+@dataclass(slots=True)
+class SiteCrawlPageResult:
+    url: str
+    depth: int
+    status: str
+    page_type: str
+    page_id: int | None = None
+    claim_count: int = 0
+    entity_count: int = 0
+    discovered_count: int = 0
+    error: str | None = None
+
+
+@dataclass(slots=True)
+class SiteCrawlResult:
+    seed_url: str
+    visited_count: int = 0
+    analyzed_count: int = 0
+    queued_count: int = 0
+    skipped_count: int = 0
+    errors: list[str] = field(default_factory=list)
+    pages: list[SiteCrawlPageResult] = field(default_factory=list)
+
+
+class SiteCrawler:
+    def __init__(
+        self,
+        repository: KnowledgeRepository,
+        extractor: Extractor,
+        parser_registry: ParserRegistry | None = None,
+        robots_policy: RobotsPolicy | None = None,
+    ):
+        self.repository = repository
+        self.extractor = extractor
+        self.parser_registry = parser_registry or default_parser_registry()
+        self.robots_policy = robots_policy or RobotsPolicy()
+
+    def crawl_site(
+        self,
+        project_config: ProjectConfig,
+        source_name: str,
+        seed_url: str,
+        max_depth: int = 2,
+        max_pages: int = 50,
+        same_domain_only: bool = True,
+        analyze_page_types: set[str] | None = None,
+    ) -> SiteCrawlResult:
+        source_config = project_config.source_by_name(source_name)
+        fetcher = make_fetcher(source_config.fetcher, source_config.rate_limit_per_minute)
+        parser = self.parser_registry.get(source_config.parser)
+        seed_host = normalized_host(seed_url)
+        analyze_page_types = analyze_page_types or {"product", "brand", "review"}
+
+        result = SiteCrawlResult(seed_url=seed_url)
+        queue: deque[tuple[str, int]] = deque([(normalize_url(seed_url), 0)])
+        queued: set[str] = {normalize_url(seed_url)}
+        visited: set[str] = set()
+
+        project = self.repository.upsert_project(project_config)
+        source = self.repository.get_source(project.id, source_name)
+
+        while queue and result.visited_count < max_pages:
+            url, depth = queue.popleft()
+            if url in visited:
+                continue
+            visited.add(url)
+
+            job = self._create_job(project.id, source.id, url, depth)
+            if depth > max_depth:
+                self._finish_job(job, "skipped", "max depth exceeded")
+                result.skipped_count += 1
+                result.pages.append(SiteCrawlPageResult(url=url, depth=depth, status="skipped", page_type="unknown"))
+                continue
+            if same_domain_only and normalized_host(url) != seed_host:
+                self._finish_job(job, "skipped", "outside same-domain filter")
+                result.skipped_count += 1
+                result.pages.append(SiteCrawlPageResult(url=url, depth=depth, status="skipped", page_type="external"))
+                continue
+            if not self.robots_policy.allowed(url, source_config.respect_robots_txt):
+                error = f"robots.txt does not allow crawling: {url}"
+                self._finish_job(job, "blocked", error)
+                result.skipped_count += 1
+                result.errors.append(error)
+                result.pages.append(SiteCrawlPageResult(url=url, depth=depth, status="blocked", page_type="unknown", error=error))
+                continue
+
+            try:
+                fetch_result = fetcher.fetch(url)
+                parsed = parser.parse(fetch_result.html, fetch_result.final_url or url)
+                page_type = classify_page(fetch_result.final_url or url, parsed.title, parsed.text)
+                links = discover_links(fetch_result.html, fetch_result.final_url or url, limit=200)
+                discovered_count = 0
+                if depth < max_depth:
+                    for link in links:
+                        next_url = normalize_url(link.url)
+                        if next_url in queued or next_url in visited:
+                            continue
+                        if same_domain_only and normalized_host(next_url) != seed_host:
+                            continue
+                        queue.append((next_url, depth + 1))
+                        queued.add(next_url)
+                        discovered_count += 1
+
+                if page_type in analyze_page_types:
+                    bundle = self.extractor.extract(parsed.text, project_config)
+                    page = self.repository.upsert_page(
+                        project_id=project.id,
+                        source_id=source.id,
+                        url=url,
+                        title=parsed.title,
+                        status_code=fetch_result.status_code,
+                        cleaned_text=parsed.text,
+                        metadata={
+                            **parsed.metadata,
+                            "final_url": fetch_result.final_url,
+                            "page_type": page_type,
+                            "depth": depth,
+                        },
+                    )
+                    claims = self.repository.save_extraction_bundle(project.id, source, page, bundle)
+                    self._finish_job(job, "completed")
+                    result.analyzed_count += 1
+                    result.pages.append(
+                        SiteCrawlPageResult(
+                            url=url,
+                            depth=depth,
+                            status="completed",
+                            page_type=page_type,
+                            page_id=page.id,
+                            claim_count=len(claims),
+                            entity_count=len(bundle.entities),
+                            discovered_count=discovered_count,
+                        )
+                    )
+                else:
+                    self._finish_job(job, "discovered")
+                    result.skipped_count += 1
+                    result.pages.append(
+                        SiteCrawlPageResult(
+                            url=url,
+                            depth=depth,
+                            status="discovered",
+                            page_type=page_type,
+                            discovered_count=discovered_count,
+                        )
+                    )
+                result.visited_count += 1
+                result.queued_count = len(queue)
+            except Exception as exc:
+                error = str(exc)
+                self._finish_job(job, "failed", error)
+                result.errors.append(error)
+                result.pages.append(SiteCrawlPageResult(url=url, depth=depth, status="failed", page_type="unknown", error=error))
+
+        return result
+
+    def _create_job(self, project_id: int, source_id: int, url: str, depth: int) -> models.CrawlJob:
+        job = models.CrawlJob(
+            project_id=project_id,
+            source_id=source_id,
+            url=url,
+            status="running",
+            metadata_json={"depth": depth},
+            started_at=models.utcnow(),
+        )
+        self.repository.session.add(job)
+        self.repository.session.flush()
+        return job
+
+    def _finish_job(self, job: models.CrawlJob, status: str, error: str | None = None) -> None:
+        job.status = status
+        job.error = error
+        job.finished_at = models.utcnow()
+
+
+def normalize_url(url: str) -> str:
+    clean, _fragment = urldefrag(url.strip())
+    return clean.rstrip("/")
+
+
+def normalized_host(url: str) -> str:
+    parsed = urlparse(url)
+    return parsed.netloc.lower()
+
+
+def classify_page(url: str, title: str | None, text: str) -> str:
+    combined = f"{url}\n{title or ''}\n{text[:3000]}".lower()
+    product_tokens = [
+        "add to cart",
+        "buy now",
+        "price",
+        "top notes",
+        "middle notes",
+        "base notes",
+        "장바구니",
+        "구매",
+        "가격",
+        "탑 노트",
+        "베이스 노트",
+    ]
+    review_tokens = ["review", "reviews", "rating", "stars", "후기", "리뷰", "평점"]
+    brand_tokens = ["about us", "brand story", "official", "브랜드", "소개"]
+    if any(token in combined for token in product_tokens):
+        return "product"
+    if any(token in combined for token in review_tokens):
+        return "review"
+    if any(token in combined for token in brand_tokens):
+        return "brand"
+    return "listing"