Phase 1.3: 시드 크롤 — by-project 엔드포인트 + CrawlPage 폼/폴링/취소

세션 영구화: UI_REBUILD_PLAN.md 신설
- 사용자 비전 5가지 흐름 + Phase 보드 + 작업 재개 가이드
- 세션이 끊겨도 이 파일 + git log로 어디까지 했는지 즉시 복원

백엔드 (crawler_platform/app/api/):
- routes.py: POST /crawl-site/by-project 추가 — config_path 의존 제거
  - SiteCrawlByProjectRequest: project_name 기반, DB project.config 재구성
  - run_site_crawl_job이 __config_dict 메타키 인식하도록 최소 변경
- config/loader.py: project_config_from_dict() 헬퍼 추출 (DB dict ↔ ProjectConfig)

프론트엔드 API (src/lib/api/):
- crawl.ts: crawlApi.startByProject/getJob/cancel + Zod 스키마
  - 종료 상태 판정 헬퍼 isCrawlTerminal()

TanStack Query 훅 (src/hooks/):
- useCrawl.ts: useStartSiteCrawl, useCrawlJob (2초 폴링, 종료 시 자동 중단), useCancelCrawl
- queryKeys에 crawl.job 키

UI 프리미티브 (src/components/ui/):
- select.tsx, progress.tsx, badge.tsx (success/warning/destructive variants 포함)

CrawlPage 완전 교체 (src/pages/):
- 2열 레이아웃: 좌측 크롤 설정 폼 + 우측 진행 상태
- 폼: 소스 선택 (드롭다운, 비어있으면 소스 추가 안내) + 시드 URL + max_depth/max_pages + same_domain_only
  - react-hook-form + zod 검증
- 진행 상태: 상태 배지, 진행률 바, visited/queued/analyzed 카운터, 최근 페이지, 에러 details
- 크롤 종료시 폴링 자동 중단, 완료시 "결과 검토" 버튼
- 취소 버튼 → /crawl-site/jobs/{id}/cancel
- sonner 토스트

i18n: crawl.* 키 (한/영)

다음 단계: Phase 1.4 — 자율 연구 (POST /research/run by-project 마이그레이션 + ResearchPage)

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
lasta
2026-05-14 15:04:41 +09:00
parent 461ebc062b
commit aaaaa054a7
13 changed files with 1009 additions and 28 deletions

View File

@@ -7,7 +7,12 @@ from fastapi import BackgroundTasks, HTTPException
from pydantic import BaseModel, Field
from sqlalchemy import select
from crawler_platform.app.config.loader import ProjectConfig, SourceConfig, load_project_config
from crawler_platform.app.config.loader import (
ProjectConfig,
SourceConfig,
load_project_config,
project_config_from_dict,
)
from crawler_platform.app.core.crawler.discovery import discover_links
from crawler_platform.app.core.crawler.fetchers import RobotsPolicy, make_fetcher
from crawler_platform.app.core.crawler.pipeline import CrawlPipeline
@@ -45,6 +50,42 @@ class SiteCrawlRequest(CrawlRequest):
analyze_page_types: list[str] = Field(default_factory=lambda: ["ProductPage", "BrandStoryPage", "ReviewPage"])
class SiteCrawlByProjectRequest(BaseModel):
"""Site crawl invoked against an existing project (no filesystem config_path)."""
project_name: str
source_name: str
url: str
extractor_provider: str = "lm_studio"
extractor_model: str | None = None
extractor_base_url: str | None = "http://localhost:1234/v1"
check_robots_txt: bool = False
respect_robots_txt: bool | None = None
max_depth: int = 2
max_pages: int = 50
same_domain_only: bool = True
analyze_page_types: list[str] = Field(
default_factory=lambda: ["ProductPage", "BrandStoryPage", "ReviewPage"]
)
def to_site_crawl_request(self, config_path_placeholder: str = "") -> "SiteCrawlRequest":
"""For internal handoff to existing crawl pipeline (config_path is not used)."""
return SiteCrawlRequest(
config_path=config_path_placeholder,
source_name=self.source_name,
url=self.url,
extractor_provider=self.extractor_provider,
extractor_model=self.extractor_model,
extractor_base_url=self.extractor_base_url,
check_robots_txt=self.check_robots_txt,
respect_robots_txt=self.respect_robots_txt,
max_depth=self.max_depth,
max_pages=self.max_pages,
same_domain_only=self.same_domain_only,
analyze_page_types=list(self.analyze_page_types),
)
class DiscoverRequest(BaseModel):
config_path: str
source_name: str
@@ -221,9 +262,13 @@ def is_site_crawl_cancel_requested(session, job_id: int) -> bool:
def run_site_crawl_job(database_url: str, job_id: int, request_data: dict[str, Any]) -> None:
inline_config = request_data.pop("__config_dict", None)
request = SiteCrawlRequest(**request_data)
try:
config = load_project_config(request.config_path)
if inline_config is not None:
config = project_config_from_dict(inline_config)
else:
config = load_project_config(request.config_path)
apply_crawl_request_overrides(config, request)
with session_scope(database_url) as session:
job = session.get(models.CrawlJob, job_id)
@@ -611,6 +656,59 @@ def register_routes(app, database_url: str) -> None:
background_tasks.add_task(run_site_crawl_job, database_url, response["job_id"], request.model_dump())
return response
@app.post("/crawl-site/by-project")
def crawl_site_by_project(
request: SiteCrawlByProjectRequest, background_tasks: BackgroundTasks
):
"""Start a site crawl against an existing DB project (no config_path)."""
with session_scope(database_url) as session:
repo = KnowledgeRepository(session)
project = repo.get_project(request.project_name)
config_dict = project.config or {}
if not config_dict:
raise HTTPException(
status_code=400,
detail=f"Project '{request.project_name}' has no stored config",
)
config = project_config_from_dict(config_dict)
try:
source = repo.get_source(project.id, request.source_name)
except KeyError as exc:
raise HTTPException(status_code=404, detail=str(exc)) from exc
inner_request = request.to_site_crawl_request()
apply_crawl_request_overrides(config, inner_request)
job = models.CrawlJob(
project_id=project.id,
source_id=source.id,
url=request.url,
status="pending",
metadata_json={
"kind": "site_crawl",
"request": inner_request.model_dump(),
"project_name": request.project_name,
"progress": {
"seed_url": request.url,
"visited_count": 0,
"analyzed_count": 0,
"queued_count": 1,
"skipped_count": 0,
"errors": [],
"pages": [],
},
},
)
session.add(job)
session.flush()
response = crawl_job_response(job)
task_payload = {**inner_request.model_dump(), "__config_dict": config_dict}
background_tasks.add_task(
run_site_crawl_job, database_url, response["job_id"], task_payload
)
return response
@app.get("/crawl-site/jobs/{job_id}")
def crawl_site_job(job_id: int):
with session_scope(database_url) as session: