diff --git a/.gitignore b/.gitignore index e0fe3e4..cd9d6b2 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,9 @@ __pycache__/ *.py[cod] .pytest_cache/ crawler_platform.db +crawler_platform.db-* *.sqlite *.sqlite3 +*.sqlite-* +*.sqlite3-* uvicorn.*.log diff --git a/README.md b/README.md index 6229077..a492b47 100644 --- a/README.md +++ b/README.md @@ -214,6 +214,7 @@ http://127.0.0.1:8000/ - `POST /projects` - `GET /ontology/{domain}` - `POST /crawl` +- `POST /crawl-site` - `GET /projects/{project_name}/entities` - `GET /projects/{project_name}/claims` - `PATCH /claims/{claim_id}/confidence` @@ -221,6 +222,45 @@ http://127.0.0.1:8000/ - `GET /projects/{project_name}/recommendation-tags` - `POST /recommend` +## 사이트 순회 수집 + +단일 상품 URL뿐 아니라 Seed URL에서 시작해 같은 도메인의 링크를 따라가는 수집도 지원합니다. + +```text +Seed URL +→ robots 확인 +→ 링크 추출 +→ same-domain 필터 +→ URL queue 저장 +→ depth / max pages 제한 +→ 각 페이지 fetch +→ 상품/브랜드/리뷰 페이지 판별 +→ 분석 +→ DB 저장 +→ 다음 링크 반복 +``` + +웹 UI에서는 `Crawl site from seed`를 사용합니다. + +API 예시: + +```bash +curl -X POST http://127.0.0.1:8000/crawl-site \ + -H "Content-Type: application/json" \ + -d '{ + "config_path": "configs/perfume_subscription.yaml", + "source_name": "official_brand_site", + "url": "https://example-brand.com", + "extractor_provider": "rule_based", + "max_depth": 2, + "max_pages": 50, + "same_domain_only": true, + "analyze_page_types": ["product", "brand", "review"] + }' +``` + +주의: 검색 결과 페이지나 robots가 막는 페이지는 수집하지 않습니다. 그런 데이터는 공식 API Provider로 붙이는 방식이 맞습니다. + ## 향수 도메인 MVP 기본 엔티티: diff --git a/crawler_platform.db b/crawler_platform.db index a14e2b2..033f4c2 100644 Binary files a/crawler_platform.db and b/crawler_platform.db differ diff --git a/crawler_platform/app/api/routes.py b/crawler_platform/app/api/routes.py index 8a1dd7c..b67b601 100644 --- a/crawler_platform/app/api/routes.py +++ b/crawler_platform/app/api/routes.py @@ -11,6 +11,7 @@ from crawler_platform.app.config.loader import load_project_config from crawler_platform.app.core.crawler.discovery import discover_links from crawler_platform.app.core.crawler.fetchers import RobotsPolicy, make_fetcher from crawler_platform.app.core.crawler.pipeline import CrawlPipeline +from crawler_platform.app.core.crawler.site_crawler import SiteCrawler from crawler_platform.app.core.database import models from crawler_platform.app.core.database.repository import KnowledgeRepository from crawler_platform.app.core.database.session import session_scope @@ -30,6 +31,13 @@ class CrawlRequest(BaseModel): extractor_base_url: str | None = None +class SiteCrawlRequest(CrawlRequest): + max_depth: int = 2 + max_pages: int = 50 + same_domain_only: bool = True + analyze_page_types: list[str] = Field(default_factory=lambda: ["product", "brand", "review"]) + + class DiscoverRequest(BaseModel): config_path: str source_name: str @@ -158,6 +166,34 @@ def register_routes(app, database_url: str) -> None: raise HTTPException(status_code=400, detail=str(exc)) from exc return {"page_id": result.page_id, "claim_count": result.claim_count, "entity_count": result.entity_count} + @app.post("/crawl-site") + def crawl_site(request: SiteCrawlRequest): + config = load_project_config(request.config_path) + with session_scope(database_url) as session: + repo = KnowledgeRepository(session) + crawler = SiteCrawler( + repo, + extractor_for_domain( + config.domain, + provider=request.extractor_provider, + model=request.extractor_model, + base_url=request.extractor_base_url, + ), + ) + try: + result = crawler.crawl_site( + config, + request.source_name, + request.url, + max_depth=max(request.max_depth, 0), + max_pages=max(min(request.max_pages, 500), 1), + same_domain_only=request.same_domain_only, + analyze_page_types=set(request.analyze_page_types), + ) + except Exception as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + return asdict(result) + @app.post("/discover") def discover(request: DiscoverRequest): config = load_project_config(request.config_path) diff --git a/crawler_platform/app/core/crawler/pipeline.py b/crawler_platform/app/core/crawler/pipeline.py index fc040ea..7650ad4 100644 --- a/crawler_platform/app/core/crawler/pipeline.py +++ b/crawler_platform/app/core/crawler/pipeline.py @@ -30,9 +30,7 @@ class CrawlPipeline: self.robots_policy = robots_policy or RobotsPolicy() def crawl_url(self, project_config: ProjectConfig, source_name: str, url: str) -> CrawlResult: - project = self.repository.upsert_project(project_config) source_config = project_config.source_by_name(source_name) - source = self.repository.get_source(project.id, source_name) if not self.robots_policy.allowed(url, source_config.respect_robots_txt): raise PermissionError(f"robots.txt does not allow crawling: {url}") @@ -40,6 +38,10 @@ class CrawlPipeline: fetch_result = fetcher.fetch(url) parser = self.parser_registry.get(source_config.parser) parsed = parser.parse(fetch_result.html, fetch_result.final_url or url) + bundle = self.extractor.extract(parsed.text, project_config) + + project = self.repository.upsert_project(project_config) + source = self.repository.get_source(project.id, source_name) page = self.repository.upsert_page( project_id=project.id, source_id=source.id, @@ -49,7 +51,5 @@ class CrawlPipeline: cleaned_text=parsed.text, metadata={**parsed.metadata, "final_url": fetch_result.final_url}, ) - bundle = self.extractor.extract(parsed.text, project_config) claims = self.repository.save_extraction_bundle(project.id, source, page, bundle) return CrawlResult(page_id=page.id, claim_count=len(claims), entity_count=len(bundle.entities)) - diff --git a/crawler_platform/app/core/crawler/site_crawler.py b/crawler_platform/app/core/crawler/site_crawler.py new file mode 100644 index 0000000..67d9678 --- /dev/null +++ b/crawler_platform/app/core/crawler/site_crawler.py @@ -0,0 +1,224 @@ +from __future__ import annotations + +from collections import deque +from dataclasses import dataclass, field +from urllib.parse import urldefrag, urlparse + +from crawler_platform.app.config.loader import ProjectConfig +from crawler_platform.app.core.crawler.discovery import discover_links +from crawler_platform.app.core.crawler.fetchers import RobotsPolicy, make_fetcher +from crawler_platform.app.core.crawler.plugins import ParserRegistry, default_parser_registry +from crawler_platform.app.core.database import models +from crawler_platform.app.core.database.repository import KnowledgeRepository +from crawler_platform.app.core.extractor.base import Extractor + + +@dataclass(slots=True) +class SiteCrawlPageResult: + url: str + depth: int + status: str + page_type: str + page_id: int | None = None + claim_count: int = 0 + entity_count: int = 0 + discovered_count: int = 0 + error: str | None = None + + +@dataclass(slots=True) +class SiteCrawlResult: + seed_url: str + visited_count: int = 0 + analyzed_count: int = 0 + queued_count: int = 0 + skipped_count: int = 0 + errors: list[str] = field(default_factory=list) + pages: list[SiteCrawlPageResult] = field(default_factory=list) + + +class SiteCrawler: + def __init__( + self, + repository: KnowledgeRepository, + extractor: Extractor, + parser_registry: ParserRegistry | None = None, + robots_policy: RobotsPolicy | None = None, + ): + self.repository = repository + self.extractor = extractor + self.parser_registry = parser_registry or default_parser_registry() + self.robots_policy = robots_policy or RobotsPolicy() + + def crawl_site( + self, + project_config: ProjectConfig, + source_name: str, + seed_url: str, + max_depth: int = 2, + max_pages: int = 50, + same_domain_only: bool = True, + analyze_page_types: set[str] | None = None, + ) -> SiteCrawlResult: + source_config = project_config.source_by_name(source_name) + fetcher = make_fetcher(source_config.fetcher, source_config.rate_limit_per_minute) + parser = self.parser_registry.get(source_config.parser) + seed_host = normalized_host(seed_url) + analyze_page_types = analyze_page_types or {"product", "brand", "review"} + + result = SiteCrawlResult(seed_url=seed_url) + queue: deque[tuple[str, int]] = deque([(normalize_url(seed_url), 0)]) + queued: set[str] = {normalize_url(seed_url)} + visited: set[str] = set() + + project = self.repository.upsert_project(project_config) + source = self.repository.get_source(project.id, source_name) + + while queue and result.visited_count < max_pages: + url, depth = queue.popleft() + if url in visited: + continue + visited.add(url) + + job = self._create_job(project.id, source.id, url, depth) + if depth > max_depth: + self._finish_job(job, "skipped", "max depth exceeded") + result.skipped_count += 1 + result.pages.append(SiteCrawlPageResult(url=url, depth=depth, status="skipped", page_type="unknown")) + continue + if same_domain_only and normalized_host(url) != seed_host: + self._finish_job(job, "skipped", "outside same-domain filter") + result.skipped_count += 1 + result.pages.append(SiteCrawlPageResult(url=url, depth=depth, status="skipped", page_type="external")) + continue + if not self.robots_policy.allowed(url, source_config.respect_robots_txt): + error = f"robots.txt does not allow crawling: {url}" + self._finish_job(job, "blocked", error) + result.skipped_count += 1 + result.errors.append(error) + result.pages.append(SiteCrawlPageResult(url=url, depth=depth, status="blocked", page_type="unknown", error=error)) + continue + + try: + fetch_result = fetcher.fetch(url) + parsed = parser.parse(fetch_result.html, fetch_result.final_url or url) + page_type = classify_page(fetch_result.final_url or url, parsed.title, parsed.text) + links = discover_links(fetch_result.html, fetch_result.final_url or url, limit=200) + discovered_count = 0 + if depth < max_depth: + for link in links: + next_url = normalize_url(link.url) + if next_url in queued or next_url in visited: + continue + if same_domain_only and normalized_host(next_url) != seed_host: + continue + queue.append((next_url, depth + 1)) + queued.add(next_url) + discovered_count += 1 + + if page_type in analyze_page_types: + bundle = self.extractor.extract(parsed.text, project_config) + page = self.repository.upsert_page( + project_id=project.id, + source_id=source.id, + url=url, + title=parsed.title, + status_code=fetch_result.status_code, + cleaned_text=parsed.text, + metadata={ + **parsed.metadata, + "final_url": fetch_result.final_url, + "page_type": page_type, + "depth": depth, + }, + ) + claims = self.repository.save_extraction_bundle(project.id, source, page, bundle) + self._finish_job(job, "completed") + result.analyzed_count += 1 + result.pages.append( + SiteCrawlPageResult( + url=url, + depth=depth, + status="completed", + page_type=page_type, + page_id=page.id, + claim_count=len(claims), + entity_count=len(bundle.entities), + discovered_count=discovered_count, + ) + ) + else: + self._finish_job(job, "discovered") + result.skipped_count += 1 + result.pages.append( + SiteCrawlPageResult( + url=url, + depth=depth, + status="discovered", + page_type=page_type, + discovered_count=discovered_count, + ) + ) + result.visited_count += 1 + result.queued_count = len(queue) + except Exception as exc: + error = str(exc) + self._finish_job(job, "failed", error) + result.errors.append(error) + result.pages.append(SiteCrawlPageResult(url=url, depth=depth, status="failed", page_type="unknown", error=error)) + + return result + + def _create_job(self, project_id: int, source_id: int, url: str, depth: int) -> models.CrawlJob: + job = models.CrawlJob( + project_id=project_id, + source_id=source_id, + url=url, + status="running", + metadata_json={"depth": depth}, + started_at=models.utcnow(), + ) + self.repository.session.add(job) + self.repository.session.flush() + return job + + def _finish_job(self, job: models.CrawlJob, status: str, error: str | None = None) -> None: + job.status = status + job.error = error + job.finished_at = models.utcnow() + + +def normalize_url(url: str) -> str: + clean, _fragment = urldefrag(url.strip()) + return clean.rstrip("/") + + +def normalized_host(url: str) -> str: + parsed = urlparse(url) + return parsed.netloc.lower() + + +def classify_page(url: str, title: str | None, text: str) -> str: + combined = f"{url}\n{title or ''}\n{text[:3000]}".lower() + product_tokens = [ + "add to cart", + "buy now", + "price", + "top notes", + "middle notes", + "base notes", + "장바구니", + "구매", + "가격", + "탑 노트", + "베이스 노트", + ] + review_tokens = ["review", "reviews", "rating", "stars", "후기", "리뷰", "평점"] + brand_tokens = ["about us", "brand story", "official", "브랜드", "소개"] + if any(token in combined for token in product_tokens): + return "product" + if any(token in combined for token in review_tokens): + return "review" + if any(token in combined for token in brand_tokens): + return "brand" + return "listing" diff --git a/crawler_platform/app/core/database/repository.py b/crawler_platform/app/core/database/repository.py index ceb822a..8e3f601 100644 --- a/crawler_platform/app/core/database/repository.py +++ b/crawler_platform/app/core/database/repository.py @@ -33,9 +33,10 @@ class KnowledgeRepository: self.session.add(project) self.session.flush() else: - project.domain = config.domain - project.config = config_dict - project.updated_at = models.utcnow() + if project.domain != config.domain or project.config != config_dict: + project.domain = config.domain + project.config = config_dict + project.updated_at = models.utcnow() for source_config in config.sources: self.upsert_source(project, source_config) return project @@ -51,12 +52,20 @@ class KnowledgeRepository: source = models.Source(project_id=project.id, name=source_config.name) self.session.add(source) self.session.flush() - source.type = source_config.type - source.base_url = source_config.base_url - source.trust_level = source_config.trust_level - source.respect_robots_txt = source_config.respect_robots_txt - source.rate_limit_per_minute = source_config.rate_limit_per_minute - source.updated_at = models.utcnow() + changed = ( + source.type != source_config.type + or source.base_url != source_config.base_url + or source.trust_level != source_config.trust_level + or source.respect_robots_txt != source_config.respect_robots_txt + or source.rate_limit_per_minute != source_config.rate_limit_per_minute + ) + if changed: + source.type = source_config.type + source.base_url = source_config.base_url + source.trust_level = source_config.trust_level + source.respect_robots_txt = source_config.respect_robots_txt + source.rate_limit_per_minute = source_config.rate_limit_per_minute + source.updated_at = models.utcnow() return source def get_project(self, project_name: str) -> models.Project: diff --git a/crawler_platform/app/core/database/session.py b/crawler_platform/app/core/database/session.py index 40f26cf..141a749 100644 --- a/crawler_platform/app/core/database/session.py +++ b/crawler_platform/app/core/database/session.py @@ -3,15 +3,28 @@ from __future__ import annotations from contextlib import contextmanager from typing import Iterator -from sqlalchemy import create_engine +from sqlalchemy import create_engine, event from sqlalchemy.orm import Session, sessionmaker from crawler_platform.app.core.database.models import Base def make_engine(database_url: str = "sqlite:///crawler_platform.db"): - connect_args = {"check_same_thread": False} if database_url.startswith("sqlite") else {} - return create_engine(database_url, future=True, connect_args=connect_args) + connect_args = {"check_same_thread": False, "timeout": 60} if database_url.startswith("sqlite") else {} + engine = create_engine(database_url, future=True, connect_args=connect_args) + if database_url.startswith("sqlite"): + install_sqlite_pragmas(engine) + return engine + + +def install_sqlite_pragmas(engine) -> None: + @event.listens_for(engine, "connect") + def _set_sqlite_pragmas(dbapi_connection, connection_record): + cursor = dbapi_connection.cursor() + cursor.execute("PRAGMA busy_timeout=60000") + cursor.execute("PRAGMA journal_mode=WAL") + cursor.execute("PRAGMA synchronous=NORMAL") + cursor.close() def init_db(database_url: str = "sqlite:///crawler_platform.db") -> None: @@ -21,7 +34,7 @@ def init_db(database_url: str = "sqlite:///crawler_platform.db") -> None: def make_session_factory(database_url: str = "sqlite:///crawler_platform.db") -> sessionmaker[Session]: engine = make_engine(database_url) - return sessionmaker(bind=engine, expire_on_commit=False, class_=Session, future=True) + return sessionmaker(bind=engine, expire_on_commit=False, autoflush=False, class_=Session, future=True) @contextmanager @@ -36,4 +49,3 @@ def session_scope(database_url: str = "sqlite:///crawler_platform.db") -> Iterat raise finally: session.close() - diff --git a/crawler_platform/app/domains/perfume/extractor.py b/crawler_platform/app/domains/perfume/extractor.py index 931e114..2ddd472 100644 --- a/crawler_platform/app/domains/perfume/extractor.py +++ b/crawler_platform/app/domains/perfume/extractor.py @@ -86,6 +86,30 @@ class PerfumeRuleBasedExtractor(GenericRuleBasedExtractor): name = "perfume_rule_based" def extract_entities(self, page_text: str, project_config: ProjectConfig) -> list[ExtractedEntity]: + product_cards = extract_product_cards(page_text) + if product_cards: + brand = valid_brand(extract_brand(page_text, "")) or infer_site_brand(page_text) + entities: list[ExtractedEntity] = [] + if brand: + entities.append(ExtractedEntity("Brand", brand, confidence=0.62)) + for card in product_cards: + attrs: dict[str, object] = {"name": card["name"]} + if brand: + attrs["brand"] = brand + if card.get("price"): + attrs["price"] = card["price"] + entities.append( + ExtractedEntity( + "Perfume", + str(card["name"]), + attrs, + evidence_text=str(card.get("evidence") or card["name"]), + confidence=0.74, + metadata={"page_pattern": "product_listing"}, + ) + ) + return dedupe_entities(entities) + product_name = extract_product_name(page_text) attrs = {"name": product_name} brand = extract_brand(page_text, product_name) @@ -137,6 +161,38 @@ class PerfumeRuleBasedExtractor(GenericRuleBasedExtractor): page_text: str, project_config: ProjectConfig, ) -> list[ExtractedClaim]: + product_cards = extract_product_cards(page_text) + if product_cards: + claims: list[ExtractedClaim] = [] + brand = next((entity for entity in entities if entity.entity_type == "Brand"), None) + for card in product_cards: + if brand: + claims.append( + ExtractedClaim( + str(card["name"]), + "Perfume", + "hasBrand", + brand.name, + "Brand", + evidence_text=brand.evidence_text or brand.name, + confidence=0.72, + confidence_reason="site brand inferred from listing page", + ) + ) + if card.get("price"): + claims.append( + ExtractedClaim( + str(card["name"]), + "Perfume", + "hasPrice", + object_value=card["price"], + evidence_text=str(card.get("evidence") or card["name"]), + confidence=0.76, + confidence_reason="Korean product listing price pattern matched", + ) + ) + return claims + perfume = next((entity for entity in entities if entity.entity_type == "Perfume"), None) if perfume is None: return [] @@ -195,7 +251,7 @@ class PerfumeRuleBasedExtractor(GenericRuleBasedExtractor): def extract_product_name(page_text: str) -> str: for line in page_text.splitlines()[:8]: clean = line.strip() - if clean and not looks_like_navigation(clean): + if clean and not looks_like_navigation(clean) and not is_template_placeholder(clean): return clean[:240] return first_non_empty_line(page_text) or "Unknown Perfume" @@ -217,6 +273,95 @@ def extract_brand(page_text: str, product_name: str) -> str | None: return None +def extract_product_cards(page_text: str) -> list[dict[str, object]]: + lines = [line.strip() for line in page_text.splitlines() if line.strip()] + cards: list[dict[str, object]] = [] + idx = 0 + while idx < len(lines): + if lines[idx] != "상품명": + idx += 1 + continue + name, name_idx = next_value_after_label(lines, idx) + if not name or is_template_placeholder(name) or name in {":", "상품명"}: + idx += 1 + continue + card: dict[str, object] = {"name": cleanup_value(name), "evidence": f"상품명: {name}"} + scan_end = next_label_index(lines, "상품명", name_idx + 1) or min(len(lines), name_idx + 12) + for price_label in ("할인판매가", "판매가", "price", "Price"): + label_idx = find_label_index(lines, price_label, name_idx + 1, scan_end) + if label_idx is None: + continue + raw_price, _price_idx = next_value_after_label(lines, label_idx) + parsed = parse_price_value(raw_price) + if parsed: + card["price"] = parsed + card["evidence"] = f"{card['evidence']} / {price_label}: {raw_price}" + break + cards.append(card) + idx = scan_end + return dedupe_product_cards(cards) + + +def next_value_after_label(lines: list[str], label_idx: int) -> tuple[str | None, int]: + for idx in range(label_idx + 1, min(len(lines), label_idx + 5)): + value = cleanup_value(lines[idx]) + if not value or value == ":": + continue + return value, idx + return None, label_idx + + +def next_label_index(lines: list[str], label: str, start: int) -> int | None: + for idx in range(start, len(lines)): + if lines[idx] == label: + return idx + return None + + +def find_label_index(lines: list[str], label: str, start: int, end: int) -> int | None: + lower_label = label.lower() + for idx in range(start, min(end, len(lines))): + if lines[idx].lower() == lower_label: + return idx + return None + + +def parse_price_value(raw_price: str | None) -> dict[str, object] | None: + if not raw_price: + return None + match = re.search(r"(?P\d{1,3}(?:,\d{3})*|\d+)\s*(?P원|KRW|₩|USD|\$)?", raw_price) + if not match: + return None + currency = match.group("currency") or "KRW" + if currency in {"원", "₩"}: + currency = "KRW" + return {"amount": float(match.group("amount").replace(",", "")), "currency": currency} + + +def dedupe_product_cards(cards: list[dict[str, object]]) -> list[dict[str, object]]: + seen: set[str] = set() + result: list[dict[str, object]] = [] + for card in cards: + key = str(card["name"]).strip().lower() + if key in seen: + continue + seen.add(key) + result.append(card) + return result + + +def infer_site_brand(page_text: str) -> str | None: + if "912 공식 홈페이지" in page_text or "912" in page_text[:500]: + return "912" + return None + + +def valid_brand(value: str | None) -> str | None: + if not value or is_template_placeholder(value): + return None + return value + + def extract_field_values(field: str, page_text: str) -> list[tuple[str, str]]: if field in NOTE_LABELS: return extract_labeled_values(page_text, NOTE_LABELS[field]) @@ -292,6 +437,15 @@ def looks_like_navigation(value: str) -> bool: return value.lower() in {"home", "shop", "menu", "cart", "login", "검색", "장바구니", "홈"} +def is_template_placeholder(value: str) -> bool: + clean = value.strip() + return clean.startswith("{#") or clean.endswith("}") or clean in { + "CLONE FRAGRANCE", + "NICHE FRAGRANCE", + "HOME FRAGRANCE", + } + + def field_entity_type(field: str) -> str: return { "top_notes": "Note", @@ -347,4 +501,3 @@ def dedupe_entities(entities: list[ExtractedEntity]) -> list[ExtractedEntity]: seen.add(key) result.append(entity) return result - diff --git a/crawler_platform/app/web/static/app.js b/crawler_platform/app/web/static/app.js index 79b9b57..7f1db76 100644 --- a/crawler_platform/app/web/static/app.js +++ b/crawler_platform/app/web/static/app.js @@ -8,10 +8,7 @@ const state = { const $ = (id) => document.getElementById(id); function csv(value) { - return value - .split(",") - .map((item) => item.trim()) - .filter(Boolean); + return value.split(",").map((item) => item.trim()).filter(Boolean); } async function api(path, options = {}) { @@ -25,7 +22,7 @@ async function api(path, options = {}) { const body = await response.json(); detail = body.detail || body.error || detail; } catch { - // Keep the HTTP status text. + // Keep status text. } throw new Error(detail); } @@ -39,16 +36,15 @@ function toast(message) { window.setTimeout(() => node.classList.remove("show"), 2400); } -function renderProjects() { - const list = $("projectList"); - list.innerHTML = ""; - state.projects.forEach((project) => { - const button = document.createElement("button"); - button.className = `project-item ${state.selectedProject === project.name ? "active" : ""}`; - button.innerHTML = `${project.name}${project.domain}`; - button.addEventListener("click", () => selectProject(project.name)); - list.appendChild(button); - }); +function requestBase() { + return { + config_path: $("configPath").value.trim(), + source_name: $("sourceSelect").value, + url: $("crawlUrl").value.trim(), + extractor_provider: $("extractorProvider").value, + extractor_model: $("extractorModel").value.trim() || null, + extractor_base_url: $("extractorBaseUrl").value.trim() || null, + }; } async function loadProjects() { @@ -62,6 +58,18 @@ async function loadProjects() { } } +function renderProjects() { + const list = $("projectList"); + list.innerHTML = ""; + state.projects.forEach((project) => { + const button = document.createElement("button"); + button.className = `project-item ${state.selectedProject === project.name ? "active" : ""}`; + button.innerHTML = `${escapeHtml(project.name)}${escapeHtml(project.domain)}`; + button.addEventListener("click", () => selectProject(project.name)); + list.appendChild(button); + }); +} + async function selectProject(projectName) { state.selectedProject = projectName; state.projectDetail = await api(`/projects/${encodeURIComponent(projectName)}`); @@ -77,6 +85,7 @@ function renderOverview() { $("metricProject").textContent = detail?.name ?? "-"; $("metricDomain").textContent = detail?.domain ?? "-"; $("metricSources").textContent = detail?.sources?.length ?? 0; + const sourceSelect = $("sourceSelect"); sourceSelect.innerHTML = ""; (detail?.sources ?? []).forEach((source) => { @@ -102,8 +111,7 @@ function renderOntology() { const predicates = state.ontology?.predicates ?? []; $("ontologyEntities").innerHTML = entityTypes.map(chip).join(""); $("ontologyPredicates").innerHTML = predicates.map(chip).join(""); - const filter = $("entityTypeFilter"); - filter.innerHTML = `${entityTypes + $("entityTypeFilter").innerHTML = `${entityTypes .map((type) => ``) .join("")}`; } @@ -115,40 +123,104 @@ async function createProject() { method: "POST", body: JSON.stringify({ config_path: configPath }), }); - toast(`프로젝트 생성: ${result.name}`); + toast(`Project created: ${result.name}`); state.selectedProject = result.name; await loadProjects(); } async function crawl() { if (!state.selectedProject) return; - const sourceName = $("sourceSelect").value; - const url = $("crawlUrl").value.trim(); - const provider = $("extractorProvider").value; - const model = $("extractorModel").value.trim(); - const baseUrl = $("extractorBaseUrl").value.trim(); - $("crawlResult").textContent = "수집 중..."; + $("crawlResult").textContent = "Crawling one URL..."; try { const result = await api("/crawl", { method: "POST", - body: JSON.stringify({ - config_path: $("configPath").value.trim(), - source_name: sourceName, - url, - extractor_provider: provider, - extractor_model: model || null, - extractor_base_url: baseUrl || null, - }), + body: JSON.stringify(requestBase()), }); - $("crawlResult").textContent = `analyzer ${provider}, page ${result.page_id}, claims ${result.claim_count}, entities ${result.entity_count}`; - toast("수집 완료"); + $("crawlResult").textContent = `URL done: page ${result.page_id}, claims ${result.claim_count}, entities ${result.entity_count}`; + toast("URL crawl completed"); await Promise.all([loadEntities(), loadClaims(), loadTags()]); } catch (error) { - $("crawlResult").textContent = `수집 실패: ${error.message}`; - toast("수집 실패"); + $("crawlResult").textContent = `Crawl failed: ${error.message}`; + toast("Crawl failed"); } } +async function crawlSite() { + if (!state.selectedProject) return; + $("crawlResult").textContent = "Crawling site from seed..."; + $("discoveredLinks").innerHTML = ""; + try { + const result = await api("/crawl-site", { + method: "POST", + body: JSON.stringify({ + ...requestBase(), + max_depth: Number($("siteMaxDepth").value || 0), + max_pages: Number($("siteMaxPages").value || 1), + same_domain_only: $("sameDomainOnly").checked, + analyze_page_types: ["product", "brand", "review"], + }), + }); + $("crawlResult").textContent = + `Site done: visited ${result.visited_count}, analyzed ${result.analyzed_count}, skipped ${result.skipped_count}, queued ${result.queued_count}`; + $("discoveredLinks").innerHTML = result.pages.map(renderSitePage).join(""); + toast("Site crawl completed"); + await Promise.all([loadEntities(), loadClaims(), loadTags()]); + } catch (error) { + $("crawlResult").textContent = `Site crawl failed: ${error.message}`; + toast("Site crawl failed"); + } +} + +function renderSitePage(page) { + return ` + + `; +} + +async function discover() { + $("crawlResult").textContent = "Discovering links..."; + $("discoveredLinks").innerHTML = ""; + try { + const result = await api("/discover", { + method: "POST", + body: JSON.stringify({ + config_path: $("configPath").value.trim(), + source_name: $("sourceSelect").value, + url: $("crawlUrl").value.trim(), + limit: 30, + }), + }); + if (!result.ok) { + $("crawlResult").textContent = result.error ?? "Discovery failed"; + return; + } + $("crawlResult").textContent = `Discovered ${result.links.length} links`; + $("discoveredLinks").innerHTML = result.links.map(renderDiscoveredLink).join(""); + document.querySelectorAll("[data-discovered-url]").forEach((button) => { + button.addEventListener("click", () => { + $("crawlUrl").value = button.dataset.discoveredUrl; + toast("URL copied to input"); + }); + }); + } catch (error) { + $("crawlResult").textContent = `Discovery failed: ${error.message}`; + toast("Discovery failed"); + } +} + +function renderDiscoveredLink(link) { + return ` + + `; +} + function updateExtractorOptions() { const provider = $("extractorProvider").value; $("extractorOptions").classList.toggle("active", provider !== "rule_based"); @@ -164,17 +236,14 @@ function updateExtractorOptions() { async function testExtractor() { const provider = $("extractorProvider").value; const baseUrl = $("extractorBaseUrl").value.trim(); - $("crawlResult").textContent = "분석기 연결 확인 중..."; + $("crawlResult").textContent = "Testing analyzer..."; const result = await api("/extractors/models", { method: "POST", - body: JSON.stringify({ - provider, - base_url: baseUrl || null, - }), + body: JSON.stringify({ provider, base_url: baseUrl || null }), }); if (!result.ok) { - $("crawlResult").textContent = `분석기 연결 실패: ${result.error}`; - toast("분석기 연결 실패"); + $("crawlResult").textContent = `Analyzer failed: ${result.error}`; + toast("Analyzer failed"); return; } const models = result.models ?? []; @@ -182,51 +251,9 @@ async function testExtractor() { $("extractorModel").value = models[0].id; } $("crawlResult").textContent = models.length - ? `연결됨. 모델 ${models.length}개: ${models.map((model) => model.id).join(", ")}` - : "연결됨. 모델 목록은 비어 있습니다."; - toast("분석기 연결 확인 완료"); -} - -async function discover() { - const sourceName = $("sourceSelect").value; - const url = $("crawlUrl").value.trim(); - $("crawlResult").textContent = "주소 발견 중..."; - $("discoveredLinks").innerHTML = ""; - try { - const result = await api("/discover", { - method: "POST", - body: JSON.stringify({ - config_path: $("configPath").value.trim(), - source_name: sourceName, - url, - limit: 30, - }), - }); - if (!result.ok) { - $("crawlResult").textContent = result.error ?? "주소 발견 실패"; - return; - } - $("crawlResult").textContent = `발견된 주소 ${result.links.length}개`; - $("discoveredLinks").innerHTML = result.links.map(renderDiscoveredLink).join(""); - document.querySelectorAll("[data-discovered-url]").forEach((button) => { - button.addEventListener("click", () => { - $("crawlUrl").value = button.dataset.discoveredUrl; - toast("URL 입력칸에 넣었습니다."); - }); - }); - } catch (error) { - $("crawlResult").textContent = `주소 발견 실패: ${error.message}`; - toast("주소 발견 실패"); - } -} - -function renderDiscoveredLink(link) { - return ` - - `; + ? `Analyzer connected. Models: ${models.map((model) => model.id).join(", ")}` + : "Analyzer connected. No models returned."; + toast("Analyzer connected"); } async function loadEntities() { @@ -251,7 +278,7 @@ async function mergeEntities() { const sourceId = Number($("mergeSourceId").value); const targetId = Number($("mergeTargetId").value); if (!sourceId || !targetId || sourceId === targetId) { - toast("병합할 ID와 남길 ID를 확인하세요."); + toast("Check entity IDs"); return; } const result = await api("/entities/merge", { @@ -263,10 +290,10 @@ async function mergeEntities() { }), }); if (!result.ok) { - toast(result.error ?? "병합 실패"); + toast(result.error ?? "Merge failed"); return; } - toast("Entity 병합 완료"); + toast("Entity merged"); $("mergeSourceId").value = ""; $("mergeTargetId").value = ""; await Promise.all([loadEntities(), loadClaims(), loadTags()]); @@ -296,7 +323,7 @@ function renderClaim(claim) {
- +
`; @@ -309,7 +336,7 @@ async function updateClaimConfidence(claimId) { method: "PATCH", body: JSON.stringify({ confidence, reason }), }); - toast("신뢰도 수정 완료"); + toast("Claim updated"); await loadClaims(); } @@ -357,16 +384,12 @@ function chip(value) { function table(headers, rows) { if (!rows.length) { - return `
데이터가 없습니다.
`; + return `
No data.
`; } return ` ${headers.map((header) => ``).join("")} - - ${rows - .map((row) => `${row.map((cell) => ``).join("")}`) - .join("")} - + ${rows.map((row) => `${row.map((cell) => ``).join("")}`).join("")}
${escapeHtml(header)}
${String(cell)}
${String(cell)}
`; } @@ -393,6 +416,7 @@ $("refreshBtn").addEventListener("click", loadProjects); $("createProjectBtn").addEventListener("click", createProject); $("discoverBtn").addEventListener("click", discover); $("crawlBtn").addEventListener("click", crawl); +$("siteCrawlBtn").addEventListener("click", crawlSite); $("extractorProvider").addEventListener("change", updateExtractorOptions); $("testExtractorBtn").addEventListener("click", testExtractor); $("loadEntitiesBtn").addEventListener("click", loadEntities); diff --git a/crawler_platform/app/web/static/index.html b/crawler_platform/app/web/static/index.html index d444fe7..1ca1e3a 100644 --- a/crawler_platform/app/web/static/index.html +++ b/crawler_platform/app/web/static/index.html @@ -10,9 +10,9 @@

Ontology Crawler

-

프로젝트별 수집, Claim 검수, 온톨로지 매핑, 추천 태그 확인

+

Project crawler, ontology mapping, claim review, and recommendation tags

- +
@@ -23,7 +23,7 @@
- +
@@ -37,7 +37,7 @@ - +
diff --git a/crawler_platform/app/web/static/styles.css b/crawler_platform/app/web/static/styles.css index 79a426d..c19f3ca 100644 --- a/crawler_platform/app/web/static/styles.css +++ b/crawler_platform/app/web/static/styles.css @@ -194,6 +194,29 @@ h2 { gap: 8px; } +button.full { + width: 100%; +} + +.site-controls { + display: grid; + grid-template-columns: 1fr 1fr; + gap: 8px; + align-items: end; +} + +.check-row { + display: flex; + align-items: center; + gap: 8px; + min-height: 36px; +} + +.check-row input { + width: 16px; + min-height: 16px; +} + .extractor-options { display: none; gap: 8px; @@ -446,7 +469,8 @@ th { .split, .recommend-grid, .claim-actions, - .merge-bar { + .merge-bar, + .site-controls { grid-template-columns: 1fr; } } diff --git a/tests/test_site_crawler.py b/tests/test_site_crawler.py new file mode 100644 index 0000000..5e77f3e --- /dev/null +++ b/tests/test_site_crawler.py @@ -0,0 +1,11 @@ +from crawler_platform.app.core.crawler.site_crawler import classify_page, normalize_url + + +def test_classify_perfume_product_page(): + text = "Top notes: Bergamot, Neroli\nMiddle notes: Jasmine\nBase notes: Musk\nPrice $89" + + assert classify_page("https://example.com/products/neroli", "Neroli Summer", text) == "product" + + +def test_normalize_url_removes_fragment_and_trailing_slash(): + assert normalize_url("https://example.com/path/#details") == "https://example.com/path"