[버그수정]

2026-05-11 13:02:11 +09:00
parent 5935c6cbe8
commit 3663734781
13 changed files with 686 additions and 135 deletions
--- a/.gitignore
+++ b/.gitignore
@@ -2,6 +2,9 @@ __pycache__/
 *.py[cod]
 .pytest_cache/
 crawler_platform.db
+crawler_platform.db-*
 *.sqlite
 *.sqlite3
+*.sqlite-*
+*.sqlite3-*
 uvicorn.*.log
--- a/README.md
+++ b/README.md
@@ -214,6 +214,7 @@ http://127.0.0.1:8000/
 - `POST /projects`
 - `GET /ontology/{domain}`
 - `POST /crawl`
+- `POST /crawl-site`
 - `GET /projects/{project_name}/entities`
 - `GET /projects/{project_name}/claims`
 - `PATCH /claims/{claim_id}/confidence`
@@ -221,6 +222,45 @@ http://127.0.0.1:8000/
 - `GET /projects/{project_name}/recommendation-tags`
 - `POST /recommend`

+## 사이트 순회 수집
+
+단일 상품 URL뿐 아니라 Seed URL에서 시작해 같은 도메인의 링크를 따라가는 수집도 지원합니다.
+
+```text
+Seed URL
+→ robots 확인
+→ 링크 추출
+→ same-domain 필터
+→ URL queue 저장
+→ depth / max pages 제한
+→ 각 페이지 fetch
+→ 상품/브랜드/리뷰 페이지 판별
+→ 분석
+→ DB 저장
+→ 다음 링크 반복
+```
+
+웹 UI에서는 `Crawl site from seed`를 사용합니다.
+
+API 예시:
+
+```bash
+curl -X POST http://127.0.0.1:8000/crawl-site \
+  -H "Content-Type: application/json" \
+  -d '{
+    "config_path": "configs/perfume_subscription.yaml",
+    "source_name": "official_brand_site",
+    "url": "https://example-brand.com",
+    "extractor_provider": "rule_based",
+    "max_depth": 2,
+    "max_pages": 50,
+    "same_domain_only": true,
+    "analyze_page_types": ["product", "brand", "review"]
+  }'
+```
+
+주의: 검색 결과 페이지나 robots가 막는 페이지는 수집하지 않습니다. 그런 데이터는 공식 API Provider로 붙이는 방식이 맞습니다.
+
 ## 향수 도메인 MVP

 기본 엔티티:
--- a/crawler_platform.db
+++ b/crawler_platform.db
--- a/crawler_platform/app/api/routes.py
+++ b/crawler_platform/app/api/routes.py
@@ -11,6 +11,7 @@ from crawler_platform.app.config.loader import load_project_config
 from crawler_platform.app.core.crawler.discovery import discover_links
 from crawler_platform.app.core.crawler.fetchers import RobotsPolicy, make_fetcher
 from crawler_platform.app.core.crawler.pipeline import CrawlPipeline
+from crawler_platform.app.core.crawler.site_crawler import SiteCrawler
 from crawler_platform.app.core.database import models
 from crawler_platform.app.core.database.repository import KnowledgeRepository
 from crawler_platform.app.core.database.session import session_scope
@@ -30,6 +31,13 @@ class CrawlRequest(BaseModel):
    extractor_base_url: str | None = None


+class SiteCrawlRequest(CrawlRequest):
+    max_depth: int = 2
+    max_pages: int = 50
+    same_domain_only: bool = True
+    analyze_page_types: list[str] = Field(default_factory=lambda: ["product", "brand", "review"])
+
+
 class DiscoverRequest(BaseModel):
    config_path: str
    source_name: str
@@ -158,6 +166,34 @@ def register_routes(app, database_url: str) -> None:
                raise HTTPException(status_code=400, detail=str(exc)) from exc
            return {"page_id": result.page_id, "claim_count": result.claim_count, "entity_count": result.entity_count}

+    @app.post("/crawl-site")
+    def crawl_site(request: SiteCrawlRequest):
+        config = load_project_config(request.config_path)
+        with session_scope(database_url) as session:
+            repo = KnowledgeRepository(session)
+            crawler = SiteCrawler(
+                repo,
+                extractor_for_domain(
+                    config.domain,
+                    provider=request.extractor_provider,
+                    model=request.extractor_model,
+                    base_url=request.extractor_base_url,
+                ),
+            )
+            try:
+                result = crawler.crawl_site(
+                    config,
+                    request.source_name,
+                    request.url,
+                    max_depth=max(request.max_depth, 0),
+                    max_pages=max(min(request.max_pages, 500), 1),
+                    same_domain_only=request.same_domain_only,
+                    analyze_page_types=set(request.analyze_page_types),
+                )
+            except Exception as exc:
+                raise HTTPException(status_code=400, detail=str(exc)) from exc
+            return asdict(result)
+
    @app.post("/discover")
    def discover(request: DiscoverRequest):
        config = load_project_config(request.config_path)
--- a/crawler_platform/app/core/crawler/pipeline.py
+++ b/crawler_platform/app/core/crawler/pipeline.py
@@ -30,9 +30,7 @@ class CrawlPipeline:
        self.robots_policy = robots_policy or RobotsPolicy()

    def crawl_url(self, project_config: ProjectConfig, source_name: str, url: str) -> CrawlResult:
-        project = self.repository.upsert_project(project_config)
        source_config = project_config.source_by_name(source_name)
-        source = self.repository.get_source(project.id, source_name)
        if not self.robots_policy.allowed(url, source_config.respect_robots_txt):
            raise PermissionError(f"robots.txt does not allow crawling: {url}")

@@ -40,6 +38,10 @@ class CrawlPipeline:
        fetch_result = fetcher.fetch(url)
        parser = self.parser_registry.get(source_config.parser)
        parsed = parser.parse(fetch_result.html, fetch_result.final_url or url)
+        bundle = self.extractor.extract(parsed.text, project_config)
+
+        project = self.repository.upsert_project(project_config)
+        source = self.repository.get_source(project.id, source_name)
        page = self.repository.upsert_page(
            project_id=project.id,
            source_id=source.id,
@@ -49,7 +51,5 @@ class CrawlPipeline:
            cleaned_text=parsed.text,
            metadata={**parsed.metadata, "final_url": fetch_result.final_url},
        )
-        bundle = self.extractor.extract(parsed.text, project_config)
        claims = self.repository.save_extraction_bundle(project.id, source, page, bundle)
        return CrawlResult(page_id=page.id, claim_count=len(claims), entity_count=len(bundle.entities))
-
--- a/crawler_platform/app/core/crawler/site_crawler.py
+++ b/crawler_platform/app/core/crawler/site_crawler.py
@@ -0,0 +1,224 @@
+from __future__ import annotations
+
+from collections import deque
+from dataclasses import dataclass, field
+from urllib.parse import urldefrag, urlparse
+
+from crawler_platform.app.config.loader import ProjectConfig
+from crawler_platform.app.core.crawler.discovery import discover_links
+from crawler_platform.app.core.crawler.fetchers import RobotsPolicy, make_fetcher
+from crawler_platform.app.core.crawler.plugins import ParserRegistry, default_parser_registry
+from crawler_platform.app.core.database import models
+from crawler_platform.app.core.database.repository import KnowledgeRepository
+from crawler_platform.app.core.extractor.base import Extractor
+
+
+@dataclass(slots=True)
+class SiteCrawlPageResult:
+    url: str
+    depth: int
+    status: str
+    page_type: str
+    page_id: int | None = None
+    claim_count: int = 0
+    entity_count: int = 0
+    discovered_count: int = 0
+    error: str | None = None
+
+
+@dataclass(slots=True)
+class SiteCrawlResult:
+    seed_url: str
+    visited_count: int = 0
+    analyzed_count: int = 0
+    queued_count: int = 0
+    skipped_count: int = 0
+    errors: list[str] = field(default_factory=list)
+    pages: list[SiteCrawlPageResult] = field(default_factory=list)
+
+
+class SiteCrawler:
+    def __init__(
+        self,
+        repository: KnowledgeRepository,
+        extractor: Extractor,
+        parser_registry: ParserRegistry | None = None,
+        robots_policy: RobotsPolicy | None = None,
+    ):
+        self.repository = repository
+        self.extractor = extractor
+        self.parser_registry = parser_registry or default_parser_registry()
+        self.robots_policy = robots_policy or RobotsPolicy()
+
+    def crawl_site(
+        self,
+        project_config: ProjectConfig,
+        source_name: str,
+        seed_url: str,
+        max_depth: int = 2,
+        max_pages: int = 50,
+        same_domain_only: bool = True,
+        analyze_page_types: set[str] | None = None,
+    ) -> SiteCrawlResult:
+        source_config = project_config.source_by_name(source_name)
+        fetcher = make_fetcher(source_config.fetcher, source_config.rate_limit_per_minute)
+        parser = self.parser_registry.get(source_config.parser)
+        seed_host = normalized_host(seed_url)
+        analyze_page_types = analyze_page_types or {"product", "brand", "review"}
+
+        result = SiteCrawlResult(seed_url=seed_url)
+        queue: deque[tuple[str, int]] = deque([(normalize_url(seed_url), 0)])
+        queued: set[str] = {normalize_url(seed_url)}
+        visited: set[str] = set()
+
+        project = self.repository.upsert_project(project_config)
+        source = self.repository.get_source(project.id, source_name)
+
+        while queue and result.visited_count < max_pages:
+            url, depth = queue.popleft()
+            if url in visited:
+                continue
+            visited.add(url)
+
+            job = self._create_job(project.id, source.id, url, depth)
+            if depth > max_depth:
+                self._finish_job(job, "skipped", "max depth exceeded")
+                result.skipped_count += 1
+                result.pages.append(SiteCrawlPageResult(url=url, depth=depth, status="skipped", page_type="unknown"))
+                continue
+            if same_domain_only and normalized_host(url) != seed_host:
+                self._finish_job(job, "skipped", "outside same-domain filter")
+                result.skipped_count += 1
+                result.pages.append(SiteCrawlPageResult(url=url, depth=depth, status="skipped", page_type="external"))
+                continue
+            if not self.robots_policy.allowed(url, source_config.respect_robots_txt):
+                error = f"robots.txt does not allow crawling: {url}"
+                self._finish_job(job, "blocked", error)
+                result.skipped_count += 1
+                result.errors.append(error)
+                result.pages.append(SiteCrawlPageResult(url=url, depth=depth, status="blocked", page_type="unknown", error=error))
+                continue
+
+            try:
+                fetch_result = fetcher.fetch(url)
+                parsed = parser.parse(fetch_result.html, fetch_result.final_url or url)
+                page_type = classify_page(fetch_result.final_url or url, parsed.title, parsed.text)
+                links = discover_links(fetch_result.html, fetch_result.final_url or url, limit=200)
+                discovered_count = 0
+                if depth < max_depth:
+                    for link in links:
+                        next_url = normalize_url(link.url)
+                        if next_url in queued or next_url in visited:
+                            continue
+                        if same_domain_only and normalized_host(next_url) != seed_host:
+                            continue
+                        queue.append((next_url, depth + 1))
+                        queued.add(next_url)
+                        discovered_count += 1
+
+                if page_type in analyze_page_types:
+                    bundle = self.extractor.extract(parsed.text, project_config)
+                    page = self.repository.upsert_page(
+                        project_id=project.id,
+                        source_id=source.id,
+                        url=url,
+                        title=parsed.title,
+                        status_code=fetch_result.status_code,
+                        cleaned_text=parsed.text,
+                        metadata={
+                            **parsed.metadata,
+                            "final_url": fetch_result.final_url,
+                            "page_type": page_type,
+                            "depth": depth,
+                        },
+                    )
+                    claims = self.repository.save_extraction_bundle(project.id, source, page, bundle)
+                    self._finish_job(job, "completed")
+                    result.analyzed_count += 1
+                    result.pages.append(
+                        SiteCrawlPageResult(
+                            url=url,
+                            depth=depth,
+                            status="completed",
+                            page_type=page_type,
+                            page_id=page.id,
+                            claim_count=len(claims),
+                            entity_count=len(bundle.entities),
+                            discovered_count=discovered_count,
+                        )
+                    )
+                else:
+                    self._finish_job(job, "discovered")
+                    result.skipped_count += 1
+                    result.pages.append(
+                        SiteCrawlPageResult(
+                            url=url,
+                            depth=depth,
+                            status="discovered",
+                            page_type=page_type,
+                            discovered_count=discovered_count,
+                        )
+                    )
+                result.visited_count += 1
+                result.queued_count = len(queue)
+            except Exception as exc:
+                error = str(exc)
+                self._finish_job(job, "failed", error)
+                result.errors.append(error)
+                result.pages.append(SiteCrawlPageResult(url=url, depth=depth, status="failed", page_type="unknown", error=error))
+
+        return result
+
+    def _create_job(self, project_id: int, source_id: int, url: str, depth: int) -> models.CrawlJob:
+        job = models.CrawlJob(
+            project_id=project_id,
+            source_id=source_id,
+            url=url,
+            status="running",
+            metadata_json={"depth": depth},
+            started_at=models.utcnow(),
+        )
+        self.repository.session.add(job)
+        self.repository.session.flush()
+        return job
+
+    def _finish_job(self, job: models.CrawlJob, status: str, error: str | None = None) -> None:
+        job.status = status
+        job.error = error
+        job.finished_at = models.utcnow()
+
+
+def normalize_url(url: str) -> str:
+    clean, _fragment = urldefrag(url.strip())
+    return clean.rstrip("/")
+
+
+def normalized_host(url: str) -> str:
+    parsed = urlparse(url)
+    return parsed.netloc.lower()
+
+
+def classify_page(url: str, title: str | None, text: str) -> str:
+    combined = f"{url}\n{title or ''}\n{text[:3000]}".lower()
+    product_tokens = [
+        "add to cart",
+        "buy now",
+        "price",
+        "top notes",
+        "middle notes",
+        "base notes",
+        "장바구니",
+        "구매",
+        "가격",
+        "탑 노트",
+        "베이스 노트",
+    ]
+    review_tokens = ["review", "reviews", "rating", "stars", "후기", "리뷰", "평점"]
+    brand_tokens = ["about us", "brand story", "official", "브랜드", "소개"]
+    if any(token in combined for token in product_tokens):
+        return "product"
+    if any(token in combined for token in review_tokens):
+        return "review"
+    if any(token in combined for token in brand_tokens):
+        return "brand"
+    return "listing"
--- a/crawler_platform/app/core/database/repository.py
+++ b/crawler_platform/app/core/database/repository.py
@@ -33,9 +33,10 @@ class KnowledgeRepository:
            self.session.add(project)
            self.session.flush()
        else:
-            project.domain = config.domain
-            project.config = config_dict
-            project.updated_at = models.utcnow()
+            if project.domain != config.domain or project.config != config_dict:
+                project.domain = config.domain
+                project.config = config_dict
+                project.updated_at = models.utcnow()
        for source_config in config.sources:
            self.upsert_source(project, source_config)
        return project
@@ -51,12 +52,20 @@ class KnowledgeRepository:
            source = models.Source(project_id=project.id, name=source_config.name)
            self.session.add(source)
            self.session.flush()
-        source.type = source_config.type
-        source.base_url = source_config.base_url
-        source.trust_level = source_config.trust_level
-        source.respect_robots_txt = source_config.respect_robots_txt
-        source.rate_limit_per_minute = source_config.rate_limit_per_minute
-        source.updated_at = models.utcnow()
+        changed = (
+            source.type != source_config.type
+            or source.base_url != source_config.base_url
+            or source.trust_level != source_config.trust_level
+            or source.respect_robots_txt != source_config.respect_robots_txt
+            or source.rate_limit_per_minute != source_config.rate_limit_per_minute
+        )
+        if changed:
+            source.type = source_config.type
+            source.base_url = source_config.base_url
+            source.trust_level = source_config.trust_level
+            source.respect_robots_txt = source_config.respect_robots_txt
+            source.rate_limit_per_minute = source_config.rate_limit_per_minute
+            source.updated_at = models.utcnow()
        return source

    def get_project(self, project_name: str) -> models.Project:
--- a/crawler_platform/app/core/database/session.py
+++ b/crawler_platform/app/core/database/session.py
@@ -3,15 +3,28 @@ from __future__ import annotations
 from contextlib import contextmanager
 from typing import Iterator

-from sqlalchemy import create_engine
+from sqlalchemy import create_engine, event
 from sqlalchemy.orm import Session, sessionmaker

 from crawler_platform.app.core.database.models import Base


 def make_engine(database_url: str = "sqlite:///crawler_platform.db"):
-    connect_args = {"check_same_thread": False} if database_url.startswith("sqlite") else {}
-    return create_engine(database_url, future=True, connect_args=connect_args)
+    connect_args = {"check_same_thread": False, "timeout": 60} if database_url.startswith("sqlite") else {}
+    engine = create_engine(database_url, future=True, connect_args=connect_args)
+    if database_url.startswith("sqlite"):
+        install_sqlite_pragmas(engine)
+    return engine
+
+
+def install_sqlite_pragmas(engine) -> None:
+    @event.listens_for(engine, "connect")
+    def _set_sqlite_pragmas(dbapi_connection, connection_record):
+        cursor = dbapi_connection.cursor()
+        cursor.execute("PRAGMA busy_timeout=60000")
+        cursor.execute("PRAGMA journal_mode=WAL")
+        cursor.execute("PRAGMA synchronous=NORMAL")
+        cursor.close()


 def init_db(database_url: str = "sqlite:///crawler_platform.db") -> None:
@@ -21,7 +34,7 @@ def init_db(database_url: str = "sqlite:///crawler_platform.db") -> None:

 def make_session_factory(database_url: str = "sqlite:///crawler_platform.db") -> sessionmaker[Session]:
    engine = make_engine(database_url)
-    return sessionmaker(bind=engine, expire_on_commit=False, class_=Session, future=True)
+    return sessionmaker(bind=engine, expire_on_commit=False, autoflush=False, class_=Session, future=True)


@contextmanager
@@ -36,4 +49,3 @@ def session_scope(database_url: str = "sqlite:///crawler_platform.db") -> Iterat
        raise
    finally:
        session.close()
-
--- a/crawler_platform/app/domains/perfume/extractor.py
+++ b/crawler_platform/app/domains/perfume/extractor.py
@@ -86,6 +86,30 @@ class PerfumeRuleBasedExtractor(GenericRuleBasedExtractor):
    name = "perfume_rule_based"

    def extract_entities(self, page_text: str, project_config: ProjectConfig) -> list[ExtractedEntity]:
+        product_cards = extract_product_cards(page_text)
+        if product_cards:
+            brand = valid_brand(extract_brand(page_text, "")) or infer_site_brand(page_text)
+            entities: list[ExtractedEntity] = []
+            if brand:
+                entities.append(ExtractedEntity("Brand", brand, confidence=0.62))
+            for card in product_cards:
+                attrs: dict[str, object] = {"name": card["name"]}
+                if brand:
+                    attrs["brand"] = brand
+                if card.get("price"):
+                    attrs["price"] = card["price"]
+                entities.append(
+                    ExtractedEntity(
+                        "Perfume",
+                        str(card["name"]),
+                        attrs,
+                        evidence_text=str(card.get("evidence") or card["name"]),
+                        confidence=0.74,
+                        metadata={"page_pattern": "product_listing"},
+                    )
+                )
+            return dedupe_entities(entities)
+
        product_name = extract_product_name(page_text)
        attrs = {"name": product_name}
        brand = extract_brand(page_text, product_name)
@@ -137,6 +161,38 @@ class PerfumeRuleBasedExtractor(GenericRuleBasedExtractor):
        page_text: str,
        project_config: ProjectConfig,
    ) -> list[ExtractedClaim]:
+        product_cards = extract_product_cards(page_text)
+        if product_cards:
+            claims: list[ExtractedClaim] = []
+            brand = next((entity for entity in entities if entity.entity_type == "Brand"), None)
+            for card in product_cards:
+                if brand:
+                    claims.append(
+                        ExtractedClaim(
+                            str(card["name"]),
+                            "Perfume",
+                            "hasBrand",
+                            brand.name,
+                            "Brand",
+                            evidence_text=brand.evidence_text or brand.name,
+                            confidence=0.72,
+                            confidence_reason="site brand inferred from listing page",
+                        )
+                    )
+                if card.get("price"):
+                    claims.append(
+                        ExtractedClaim(
+                            str(card["name"]),
+                            "Perfume",
+                            "hasPrice",
+                            object_value=card["price"],
+                            evidence_text=str(card.get("evidence") or card["name"]),
+                            confidence=0.76,
+                            confidence_reason="Korean product listing price pattern matched",
+                        )
+                    )
+            return claims
+
        perfume = next((entity for entity in entities if entity.entity_type == "Perfume"), None)
        if perfume is None:
            return []
@@ -195,7 +251,7 @@ class PerfumeRuleBasedExtractor(GenericRuleBasedExtractor):
 def extract_product_name(page_text: str) -> str:
    for line in page_text.splitlines()[:8]:
        clean = line.strip()
-        if clean and not looks_like_navigation(clean):
+        if clean and not looks_like_navigation(clean) and not is_template_placeholder(clean):
            return clean[:240]
    return first_non_empty_line(page_text) or "Unknown Perfume"

@@ -217,6 +273,95 @@ def extract_brand(page_text: str, product_name: str) -> str | None:
    return None


+def extract_product_cards(page_text: str) -> list[dict[str, object]]:
+    lines = [line.strip() for line in page_text.splitlines() if line.strip()]
+    cards: list[dict[str, object]] = []
+    idx = 0
+    while idx < len(lines):
+        if lines[idx] != "상품명":
+            idx += 1
+            continue
+        name, name_idx = next_value_after_label(lines, idx)
+        if not name or is_template_placeholder(name) or name in {":", "상품명"}:
+            idx += 1
+            continue
+        card: dict[str, object] = {"name": cleanup_value(name), "evidence": f"상품명: {name}"}
+        scan_end = next_label_index(lines, "상품명", name_idx + 1) or min(len(lines), name_idx + 12)
+        for price_label in ("할인판매가", "판매가", "price", "Price"):
+            label_idx = find_label_index(lines, price_label, name_idx + 1, scan_end)
+            if label_idx is None:
+                continue
+            raw_price, _price_idx = next_value_after_label(lines, label_idx)
+            parsed = parse_price_value(raw_price)
+            if parsed:
+                card["price"] = parsed
+                card["evidence"] = f"{card['evidence']} / {price_label}: {raw_price}"
+                break
+        cards.append(card)
+        idx = scan_end
+    return dedupe_product_cards(cards)
+
+
+def next_value_after_label(lines: list[str], label_idx: int) -> tuple[str | None, int]:
+    for idx in range(label_idx + 1, min(len(lines), label_idx + 5)):
+        value = cleanup_value(lines[idx])
+        if not value or value == ":":
+            continue
+        return value, idx
+    return None, label_idx
+
+
+def next_label_index(lines: list[str], label: str, start: int) -> int | None:
+    for idx in range(start, len(lines)):
+        if lines[idx] == label:
+            return idx
+    return None
+
+
+def find_label_index(lines: list[str], label: str, start: int, end: int) -> int | None:
+    lower_label = label.lower()
+    for idx in range(start, min(end, len(lines))):
+        if lines[idx].lower() == lower_label:
+            return idx
+    return None
+
+
+def parse_price_value(raw_price: str | None) -> dict[str, object] | None:
+    if not raw_price:
+        return None
+    match = re.search(r"(?P<amount>\d{1,3}(?:,\d{3})*|\d+)\s*(?P<currency>원|KRW|₩|USD|\$)?", raw_price)
+    if not match:
+        return None
+    currency = match.group("currency") or "KRW"
+    if currency in {"원", "₩"}:
+        currency = "KRW"
+    return {"amount": float(match.group("amount").replace(",", "")), "currency": currency}
+
+
+def dedupe_product_cards(cards: list[dict[str, object]]) -> list[dict[str, object]]:
+    seen: set[str] = set()
+    result: list[dict[str, object]] = []
+    for card in cards:
+        key = str(card["name"]).strip().lower()
+        if key in seen:
+            continue
+        seen.add(key)
+        result.append(card)
+    return result
+
+
+def infer_site_brand(page_text: str) -> str | None:
+    if "912 공식 홈페이지" in page_text or "912" in page_text[:500]:
+        return "912"
+    return None
+
+
+def valid_brand(value: str | None) -> str | None:
+    if not value or is_template_placeholder(value):
+        return None
+    return value
+
+
 def extract_field_values(field: str, page_text: str) -> list[tuple[str, str]]:
    if field in NOTE_LABELS:
        return extract_labeled_values(page_text, NOTE_LABELS[field])
@@ -292,6 +437,15 @@ def looks_like_navigation(value: str) -> bool:
    return value.lower() in {"home", "shop", "menu", "cart", "login", "검색", "장바구니", "홈"}


+def is_template_placeholder(value: str) -> bool:
+    clean = value.strip()
+    return clean.startswith("{#") or clean.endswith("}") or clean in {
+        "CLONE FRAGRANCE",
+        "NICHE FRAGRANCE",
+        "HOME FRAGRANCE",
+    }
+
+
 def field_entity_type(field: str) -> str:
    return {
        "top_notes": "Note",
@@ -347,4 +501,3 @@ def dedupe_entities(entities: list[ExtractedEntity]) -> list[ExtractedEntity]:
        seen.add(key)
        result.append(entity)
    return result
-
--- a/crawler_platform/app/web/static/app.js
+++ b/crawler_platform/app/web/static/app.js
@@ -8,10 +8,7 @@ const state = {
 const $ = (id) => document.getElementById(id);

 function csv(value) {
-  return value
-    .split(",")
-    .map((item) => item.trim())
-    .filter(Boolean);
+  return value.split(",").map((item) => item.trim()).filter(Boolean);
 }

 async function api(path, options = {}) {
@@ -25,7 +22,7 @@ async function api(path, options = {}) {
      const body = await response.json();
      detail = body.detail || body.error || detail;
    } catch {
-      // Keep the HTTP status text.
+      // Keep status text.
    }
    throw new Error(detail);
  }
@@ -39,16 +36,15 @@ function toast(message) {
  window.setTimeout(() => node.classList.remove("show"), 2400);
 }

-function renderProjects() {
-  const list = $("projectList");
-  list.innerHTML = "";
-  state.projects.forEach((project) => {
-    const button = document.createElement("button");
-    button.className = `project-item ${state.selectedProject === project.name ? "active" : ""}`;
-    button.innerHTML = `<strong>${project.name}</strong><span>${project.domain}</span>`;
-    button.addEventListener("click", () => selectProject(project.name));
-    list.appendChild(button);
-  });
+function requestBase() {
+  return {
+    config_path: $("configPath").value.trim(),
+    source_name: $("sourceSelect").value,
+    url: $("crawlUrl").value.trim(),
+    extractor_provider: $("extractorProvider").value,
+    extractor_model: $("extractorModel").value.trim() || null,
+    extractor_base_url: $("extractorBaseUrl").value.trim() || null,
+  };
 }

 async function loadProjects() {
@@ -62,6 +58,18 @@ async function loadProjects() {
  }
 }

+function renderProjects() {
+  const list = $("projectList");
+  list.innerHTML = "";
+  state.projects.forEach((project) => {
+    const button = document.createElement("button");
+    button.className = `project-item ${state.selectedProject === project.name ? "active" : ""}`;
+    button.innerHTML = `<strong>${escapeHtml(project.name)}</strong><span>${escapeHtml(project.domain)}</span>`;
+    button.addEventListener("click", () => selectProject(project.name));
+    list.appendChild(button);
+  });
+}
+
 async function selectProject(projectName) {
  state.selectedProject = projectName;
  state.projectDetail = await api(`/projects/${encodeURIComponent(projectName)}`);
@@ -77,6 +85,7 @@ function renderOverview() {
  $("metricProject").textContent = detail?.name ?? "-";
  $("metricDomain").textContent = detail?.domain ?? "-";
  $("metricSources").textContent = detail?.sources?.length ?? 0;
+
  const sourceSelect = $("sourceSelect");
  sourceSelect.innerHTML = "";
  (detail?.sources ?? []).forEach((source) => {
@@ -102,8 +111,7 @@ function renderOntology() {
  const predicates = state.ontology?.predicates ?? [];
  $("ontologyEntities").innerHTML = entityTypes.map(chip).join("");
  $("ontologyPredicates").innerHTML = predicates.map(chip).join("");
-  const filter = $("entityTypeFilter");
-  filter.innerHTML = `<option value="">All types</option>${entityTypes
+  $("entityTypeFilter").innerHTML = `<option value="">All types</option>${entityTypes
    .map((type) => `<option value="${escapeHtml(type)}">${escapeHtml(type)}</option>`)
    .join("")}`;
 }
@@ -115,40 +123,104 @@ async function createProject() {
    method: "POST",
    body: JSON.stringify({ config_path: configPath }),
  });
-  toast(`프로젝트 생성: ${result.name}`);
+  toast(`Project created: ${result.name}`);
  state.selectedProject = result.name;
  await loadProjects();
 }

 async function crawl() {
  if (!state.selectedProject) return;
-  const sourceName = $("sourceSelect").value;
-  const url = $("crawlUrl").value.trim();
-  const provider = $("extractorProvider").value;
-  const model = $("extractorModel").value.trim();
-  const baseUrl = $("extractorBaseUrl").value.trim();
-  $("crawlResult").textContent = "수집 중...";
+  $("crawlResult").textContent = "Crawling one URL...";
  try {
    const result = await api("/crawl", {
      method: "POST",
-      body: JSON.stringify({
-        config_path: $("configPath").value.trim(),
-        source_name: sourceName,
-        url,
-        extractor_provider: provider,
-        extractor_model: model || null,
-        extractor_base_url: baseUrl || null,
-      }),
+      body: JSON.stringify(requestBase()),
    });
-    $("crawlResult").textContent = `analyzer ${provider}, page ${result.page_id}, claims ${result.claim_count}, entities ${result.entity_count}`;
-    toast("수집 완료");
+    $("crawlResult").textContent = `URL done: page ${result.page_id}, claims ${result.claim_count}, entities ${result.entity_count}`;
+    toast("URL crawl completed");
    await Promise.all([loadEntities(), loadClaims(), loadTags()]);
  } catch (error) {
-    $("crawlResult").textContent = `수집 실패: ${error.message}`;
-    toast("수집 실패");
+    $("crawlResult").textContent = `Crawl failed: ${error.message}`;
+    toast("Crawl failed");
  }
 }

+async function crawlSite() {
+  if (!state.selectedProject) return;
+  $("crawlResult").textContent = "Crawling site from seed...";
+  $("discoveredLinks").innerHTML = "";
+  try {
+    const result = await api("/crawl-site", {
+      method: "POST",
+      body: JSON.stringify({
+        ...requestBase(),
+        max_depth: Number($("siteMaxDepth").value || 0),
+        max_pages: Number($("siteMaxPages").value || 1),
+        same_domain_only: $("sameDomainOnly").checked,
+        analyze_page_types: ["product", "brand", "review"],
+      }),
+    });
+    $("crawlResult").textContent =
+      `Site done: visited ${result.visited_count}, analyzed ${result.analyzed_count}, skipped ${result.skipped_count}, queued ${result.queued_count}`;
+    $("discoveredLinks").innerHTML = result.pages.map(renderSitePage).join("");
+    toast("Site crawl completed");
+    await Promise.all([loadEntities(), loadClaims(), loadTags()]);
+  } catch (error) {
+    $("crawlResult").textContent = `Site crawl failed: ${error.message}`;
+    toast("Site crawl failed");
+  }
+}
+
+function renderSitePage(page) {
+  return `
+    <button class="discovered-link" data-discovered-url="${escapeHtml(page.url)}">
+      <strong>${escapeHtml(page.status)} · ${escapeHtml(page.page_type)} · depth ${page.depth}</strong>
+      <span>${escapeHtml(page.url)}</span>
+      <span>claims ${page.claim_count}, entities ${page.entity_count}, links ${page.discovered_count}${page.error ? `, error: ${escapeHtml(page.error)}` : ""}</span>
+    </button>
+  `;
+}
+
+async function discover() {
+  $("crawlResult").textContent = "Discovering links...";
+  $("discoveredLinks").innerHTML = "";
+  try {
+    const result = await api("/discover", {
+      method: "POST",
+      body: JSON.stringify({
+        config_path: $("configPath").value.trim(),
+        source_name: $("sourceSelect").value,
+        url: $("crawlUrl").value.trim(),
+        limit: 30,
+      }),
+    });
+    if (!result.ok) {
+      $("crawlResult").textContent = result.error ?? "Discovery failed";
+      return;
+    }
+    $("crawlResult").textContent = `Discovered ${result.links.length} links`;
+    $("discoveredLinks").innerHTML = result.links.map(renderDiscoveredLink).join("");
+    document.querySelectorAll("[data-discovered-url]").forEach((button) => {
+      button.addEventListener("click", () => {
+        $("crawlUrl").value = button.dataset.discoveredUrl;
+        toast("URL copied to input");
+      });
+    });
+  } catch (error) {
+    $("crawlResult").textContent = `Discovery failed: ${error.message}`;
+    toast("Discovery failed");
+  }
+}
+
+function renderDiscoveredLink(link) {
+  return `
+    <button class="discovered-link" data-discovered-url="${escapeHtml(link.url)}">
+      <strong>${escapeHtml(link.label)}</strong>
+      <span>${escapeHtml(link.kind)} · ${escapeHtml(link.url)}</span>
+    </button>
+  `;
+}
+
 function updateExtractorOptions() {
  const provider = $("extractorProvider").value;
  $("extractorOptions").classList.toggle("active", provider !== "rule_based");
@@ -164,17 +236,14 @@ function updateExtractorOptions() {
 async function testExtractor() {
  const provider = $("extractorProvider").value;
  const baseUrl = $("extractorBaseUrl").value.trim();
-  $("crawlResult").textContent = "분석기 연결 확인 중...";
+  $("crawlResult").textContent = "Testing analyzer...";
  const result = await api("/extractors/models", {
    method: "POST",
-    body: JSON.stringify({
-      provider,
-      base_url: baseUrl || null,
-    }),
+    body: JSON.stringify({ provider, base_url: baseUrl || null }),
  });
  if (!result.ok) {
-    $("crawlResult").textContent = `분석기 연결 실패: ${result.error}`;
-    toast("분석기 연결 실패");
+    $("crawlResult").textContent = `Analyzer failed: ${result.error}`;
+    toast("Analyzer failed");
    return;
  }
  const models = result.models ?? [];
@@ -182,51 +251,9 @@ async function testExtractor() {
    $("extractorModel").value = models[0].id;
  }
  $("crawlResult").textContent = models.length
-    ? `연결됨. 모델 ${models.length}개: ${models.map((model) => model.id).join(", ")}`
-    : "연결됨. 모델 목록은 비어 있습니다.";
-  toast("분석기 연결 확인 완료");
-}
-
-async function discover() {
-  const sourceName = $("sourceSelect").value;
-  const url = $("crawlUrl").value.trim();
-  $("crawlResult").textContent = "주소 발견 중...";
-  $("discoveredLinks").innerHTML = "";
-  try {
-    const result = await api("/discover", {
-      method: "POST",
-      body: JSON.stringify({
-        config_path: $("configPath").value.trim(),
-        source_name: sourceName,
-        url,
-        limit: 30,
-      }),
-    });
-    if (!result.ok) {
-      $("crawlResult").textContent = result.error ?? "주소 발견 실패";
-      return;
-    }
-    $("crawlResult").textContent = `발견된 주소 ${result.links.length}개`;
-    $("discoveredLinks").innerHTML = result.links.map(renderDiscoveredLink).join("");
-    document.querySelectorAll("[data-discovered-url]").forEach((button) => {
-      button.addEventListener("click", () => {
-        $("crawlUrl").value = button.dataset.discoveredUrl;
-        toast("URL 입력칸에 넣었습니다.");
-      });
-    });
-  } catch (error) {
-    $("crawlResult").textContent = `주소 발견 실패: ${error.message}`;
-    toast("주소 발견 실패");
-  }
-}
-
-function renderDiscoveredLink(link) {
-  return `
-    <button class="discovered-link" data-discovered-url="${escapeHtml(link.url)}">
-      <strong>${escapeHtml(link.label)}</strong>
-      <span>${escapeHtml(link.kind)} · ${escapeHtml(link.url)}</span>
-    </button>
-  `;
+    ? `Analyzer connected. Models: ${models.map((model) => model.id).join(", ")}`
+    : "Analyzer connected. No models returned.";
+  toast("Analyzer connected");
 }

 async function loadEntities() {
@@ -251,7 +278,7 @@ async function mergeEntities() {
  const sourceId = Number($("mergeSourceId").value);
  const targetId = Number($("mergeTargetId").value);
  if (!sourceId || !targetId || sourceId === targetId) {
-    toast("병합할 ID와 남길 ID를 확인하세요.");
+    toast("Check entity IDs");
    return;
  }
  const result = await api("/entities/merge", {
@@ -263,10 +290,10 @@ async function mergeEntities() {
    }),
  });
  if (!result.ok) {
-    toast(result.error ?? "병합 실패");
+    toast(result.error ?? "Merge failed");
    return;
  }
-  toast("Entity 병합 완료");
+  toast("Entity merged");
  $("mergeSourceId").value = "";
  $("mergeTargetId").value = "";
  await Promise.all([loadEntities(), loadClaims(), loadTags()]);
@@ -296,7 +323,7 @@ function renderClaim(claim) {
      <div class="claim-actions">
        <input id="confidence-${claim.id}" type="number" min="0" max="1" step="0.01" value="${claim.confidence}" aria-label="confidence" />
        <input id="reason-${claim.id}" value="${escapeHtml(claim.confidence_reason ?? "")}" aria-label="reason" />
-        <button data-save-claim="${claim.id}">저장</button>
+        <button data-save-claim="${claim.id}">Save</button>
      </div>
    </article>
  `;
@@ -309,7 +336,7 @@ async function updateClaimConfidence(claimId) {
    method: "PATCH",
    body: JSON.stringify({ confidence, reason }),
  });
-  toast("신뢰도 수정 완료");
+  toast("Claim updated");
  await loadClaims();
 }

@@ -357,16 +384,12 @@ function chip(value) {

 function table(headers, rows) {
  if (!rows.length) {
-    return `<table><tbody><tr><td>데이터가 없습니다.</td></tr></tbody></table>`;
+    return `<table><tbody><tr><td>No data.</td></tr></tbody></table>`;
  }
  return `
    <table>
      <thead><tr>${headers.map((header) => `<th>${escapeHtml(header)}</th>`).join("")}</tr></thead>
-      <tbody>
-        ${rows
-          .map((row) => `<tr>${row.map((cell) => `<td>${String(cell)}</td>`).join("")}</tr>`)
-          .join("")}
-      </tbody>
+      <tbody>${rows.map((row) => `<tr>${row.map((cell) => `<td>${String(cell)}</td>`).join("")}</tr>`).join("")}</tbody>
    </table>
  `;
 }
@@ -393,6 +416,7 @@ $("refreshBtn").addEventListener("click", loadProjects);
 $("createProjectBtn").addEventListener("click", createProject);
 $("discoverBtn").addEventListener("click", discover);
 $("crawlBtn").addEventListener("click", crawl);
+$("siteCrawlBtn").addEventListener("click", crawlSite);
 $("extractorProvider").addEventListener("change", updateExtractorOptions);
 $("testExtractorBtn").addEventListener("click", testExtractor);
 $("loadEntitiesBtn").addEventListener("click", loadEntities);
--- a/crawler_platform/app/web/static/index.html
+++ b/crawler_platform/app/web/static/index.html
@@ -10,9 +10,9 @@
    <header class="topbar">
      <div>
        <h1>Ontology Crawler</h1>
-        <p>프로젝트별 수집, Claim 검수, 온톨로지 매핑, 추천 태그 확인</p>
+        <p>Project crawler, ontology mapping, claim review, and recommendation tags</p>
      </div>
-      <button id="refreshBtn" class="icon-button" title="새로고침" aria-label="새로고침">↻</button>
+      <button id="refreshBtn" class="icon-button" title="Refresh" aria-label="Refresh">R</button>
    </header>

    <main class="layout">
@@ -23,7 +23,7 @@
          </div>
          <div class="field-row">
            <input id="configPath" value="configs/perfume_subscription.yaml" aria-label="Config path" />
-            <button id="createProjectBtn" title="프로젝트 생성">＋</button>
+            <button id="createProjectBtn" title="Create project">+</button>
          </div>
          <div id="projectList" class="list"></div>
        </section>
@@ -37,7 +37,7 @@
            <select id="sourceSelect"></select>
          </label>
          <label>
-            URL
+            URL / Seed URL
            <input id="crawlUrl" value="tests/fixtures/sample_perfume.html" />
          </label>
          <label>
@@ -52,18 +52,33 @@
          <div class="extractor-options" id="extractorOptions">
            <label>
              Model
-              <input id="extractorModel" placeholder="예: local model or API model" />
+              <input id="extractorModel" placeholder="local or API model" />
            </label>
            <label>
              Base URL
              <input id="extractorBaseUrl" placeholder="optional provider endpoint" />
            </label>
-            <button id="testExtractorBtn">연결 테스트</button>
+            <button id="testExtractorBtn">Test analyzer</button>
          </div>
          <div class="button-grid">
-            <button id="discoverBtn">주소 발견</button>
-            <button id="crawlBtn" class="primary">수집 실행</button>
+            <button id="discoverBtn">Discover links</button>
+            <button id="crawlBtn" class="primary">Crawl URL</button>
          </div>
+          <div class="site-controls">
+            <label>
+              Max depth
+              <input id="siteMaxDepth" type="number" min="0" max="5" value="2" />
+            </label>
+            <label>
+              Max pages
+              <input id="siteMaxPages" type="number" min="1" max="500" value="25" />
+            </label>
+            <label class="check-row">
+              <input id="sameDomainOnly" type="checkbox" checked />
+              Same domain
+            </label>
+          </div>
+          <button id="siteCrawlBtn" class="primary full">Crawl site from seed</button>
          <div id="crawlResult" class="mini-log"></div>
          <div id="discoveredLinks" class="discovered-links"></div>
        </section>
@@ -120,26 +135,26 @@
        <section id="entities" class="tab-panel">
          <div class="toolbar wrap">
            <select id="entityTypeFilter"></select>
-            <button id="loadEntitiesBtn">조회</button>
+            <button id="loadEntitiesBtn">Load</button>
          </div>
          <div class="merge-bar">
-            <input id="mergeSourceId" placeholder="병합할 Entity ID" aria-label="source entity id" />
-            <input id="mergeTargetId" placeholder="남길 Entity ID" aria-label="target entity id" />
-            <button id="mergeEntitiesBtn">병합</button>
+            <input id="mergeSourceId" placeholder="Entity ID to merge" aria-label="source entity id" />
+            <input id="mergeTargetId" placeholder="Entity ID to keep" aria-label="target entity id" />
+            <button id="mergeEntitiesBtn">Merge</button>
          </div>
          <div id="entityTable" class="table"></div>
        </section>

        <section id="claims" class="tab-panel">
          <div class="toolbar">
-            <button id="loadClaimsBtn">Claim 새로고침</button>
+            <button id="loadClaimsBtn">Refresh claims</button>
          </div>
          <div id="claimTable" class="claim-list"></div>
        </section>

        <section id="tags" class="tab-panel">
          <div class="toolbar">
-            <button id="loadTagsBtn">태그 조회</button>
+            <button id="loadTagsBtn">Load tags</button>
          </div>
          <div id="tagTable" class="table"></div>
        </section>
@@ -167,7 +182,7 @@
              <input id="occasionContext" value="Daily" />
            </label>
          </div>
-          <button id="recommendBtn" class="primary">추천 테스트</button>
+          <button id="recommendBtn" class="primary">Test recommendation</button>
          <div id="recommendTable" class="table"></div>
        </section>
      </section>
--- a/crawler_platform/app/web/static/styles.css
+++ b/crawler_platform/app/web/static/styles.css
@@ -194,6 +194,29 @@ h2 {
  gap: 8px;
 }

+button.full {
+  width: 100%;
+}
+
+.site-controls {
+  display: grid;
+  grid-template-columns: 1fr 1fr;
+  gap: 8px;
+  align-items: end;
+}
+
+.check-row {
+  display: flex;
+  align-items: center;
+  gap: 8px;
+  min-height: 36px;
+}
+
+.check-row input {
+  width: 16px;
+  min-height: 16px;
+}
+
 .extractor-options {
  display: none;
  gap: 8px;
@@ -446,7 +469,8 @@ th {
  .split,
  .recommend-grid,
  .claim-actions,
-  .merge-bar {
+  .merge-bar,
+  .site-controls {
    grid-template-columns: 1fr;
  }
 }
--- a/tests/test_site_crawler.py
+++ b/tests/test_site_crawler.py
@@ -0,0 +1,11 @@
+from crawler_platform.app.core.crawler.site_crawler import classify_page, normalize_url
+
+
+def test_classify_perfume_product_page():
+    text = "Top notes: Bergamot, Neroli\nMiddle notes: Jasmine\nBase notes: Musk\nPrice $89"
+
+    assert classify_page("https://example.com/products/neroli", "Neroli Summer", text) == "product"
+
+
+def test_normalize_url_removes_fragment_and_trailing_slash():
+    assert normalize_url("https://example.com/path/#details") == "https://example.com/path"