Phase 1.3: 시드 크롤 — by-project 엔드포인트 + CrawlPage 폼/폴링/취소
세션 영구화: UI_REBUILD_PLAN.md 신설
- 사용자 비전 5가지 흐름 + Phase 보드 + 작업 재개 가이드
- 세션이 끊겨도 이 파일 + git log로 어디까지 했는지 즉시 복원
백엔드 (crawler_platform/app/api/):
- routes.py: POST /crawl-site/by-project 추가 — config_path 의존 제거
- SiteCrawlByProjectRequest: project_name 기반, DB project.config 재구성
- run_site_crawl_job이 __config_dict 메타키 인식하도록 최소 변경
- config/loader.py: project_config_from_dict() 헬퍼 추출 (DB dict ↔ ProjectConfig)
프론트엔드 API (src/lib/api/):
- crawl.ts: crawlApi.startByProject/getJob/cancel + Zod 스키마
- 종료 상태 판정 헬퍼 isCrawlTerminal()
TanStack Query 훅 (src/hooks/):
- useCrawl.ts: useStartSiteCrawl, useCrawlJob (2초 폴링, 종료 시 자동 중단), useCancelCrawl
- queryKeys에 crawl.job 키
UI 프리미티브 (src/components/ui/):
- select.tsx, progress.tsx, badge.tsx (success/warning/destructive variants 포함)
CrawlPage 완전 교체 (src/pages/):
- 2열 레이아웃: 좌측 크롤 설정 폼 + 우측 진행 상태
- 폼: 소스 선택 (드롭다운, 비어있으면 소스 추가 안내) + 시드 URL + max_depth/max_pages + same_domain_only
- react-hook-form + zod 검증
- 진행 상태: 상태 배지, 진행률 바, visited/queued/analyzed 카운터, 최근 페이지, 에러 details
- 크롤 종료시 폴링 자동 중단, 완료시 "결과 검토" 버튼
- 취소 버튼 → /crawl-site/jobs/{id}/cancel
- sonner 토스트
i18n: crawl.* 키 (한/영)
다음 단계: Phase 1.4 — 자율 연구 (POST /research/run by-project 마이그레이션 + ResearchPage)
Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
@@ -7,7 +7,12 @@ from fastapi import BackgroundTasks, HTTPException
|
||||
from pydantic import BaseModel, Field
|
||||
from sqlalchemy import select
|
||||
|
||||
from crawler_platform.app.config.loader import ProjectConfig, SourceConfig, load_project_config
|
||||
from crawler_platform.app.config.loader import (
|
||||
ProjectConfig,
|
||||
SourceConfig,
|
||||
load_project_config,
|
||||
project_config_from_dict,
|
||||
)
|
||||
from crawler_platform.app.core.crawler.discovery import discover_links
|
||||
from crawler_platform.app.core.crawler.fetchers import RobotsPolicy, make_fetcher
|
||||
from crawler_platform.app.core.crawler.pipeline import CrawlPipeline
|
||||
@@ -45,6 +50,42 @@ class SiteCrawlRequest(CrawlRequest):
|
||||
analyze_page_types: list[str] = Field(default_factory=lambda: ["ProductPage", "BrandStoryPage", "ReviewPage"])
|
||||
|
||||
|
||||
class SiteCrawlByProjectRequest(BaseModel):
|
||||
"""Site crawl invoked against an existing project (no filesystem config_path)."""
|
||||
|
||||
project_name: str
|
||||
source_name: str
|
||||
url: str
|
||||
extractor_provider: str = "lm_studio"
|
||||
extractor_model: str | None = None
|
||||
extractor_base_url: str | None = "http://localhost:1234/v1"
|
||||
check_robots_txt: bool = False
|
||||
respect_robots_txt: bool | None = None
|
||||
max_depth: int = 2
|
||||
max_pages: int = 50
|
||||
same_domain_only: bool = True
|
||||
analyze_page_types: list[str] = Field(
|
||||
default_factory=lambda: ["ProductPage", "BrandStoryPage", "ReviewPage"]
|
||||
)
|
||||
|
||||
def to_site_crawl_request(self, config_path_placeholder: str = "") -> "SiteCrawlRequest":
|
||||
"""For internal handoff to existing crawl pipeline (config_path is not used)."""
|
||||
return SiteCrawlRequest(
|
||||
config_path=config_path_placeholder,
|
||||
source_name=self.source_name,
|
||||
url=self.url,
|
||||
extractor_provider=self.extractor_provider,
|
||||
extractor_model=self.extractor_model,
|
||||
extractor_base_url=self.extractor_base_url,
|
||||
check_robots_txt=self.check_robots_txt,
|
||||
respect_robots_txt=self.respect_robots_txt,
|
||||
max_depth=self.max_depth,
|
||||
max_pages=self.max_pages,
|
||||
same_domain_only=self.same_domain_only,
|
||||
analyze_page_types=list(self.analyze_page_types),
|
||||
)
|
||||
|
||||
|
||||
class DiscoverRequest(BaseModel):
|
||||
config_path: str
|
||||
source_name: str
|
||||
@@ -221,9 +262,13 @@ def is_site_crawl_cancel_requested(session, job_id: int) -> bool:
|
||||
|
||||
|
||||
def run_site_crawl_job(database_url: str, job_id: int, request_data: dict[str, Any]) -> None:
|
||||
inline_config = request_data.pop("__config_dict", None)
|
||||
request = SiteCrawlRequest(**request_data)
|
||||
try:
|
||||
config = load_project_config(request.config_path)
|
||||
if inline_config is not None:
|
||||
config = project_config_from_dict(inline_config)
|
||||
else:
|
||||
config = load_project_config(request.config_path)
|
||||
apply_crawl_request_overrides(config, request)
|
||||
with session_scope(database_url) as session:
|
||||
job = session.get(models.CrawlJob, job_id)
|
||||
@@ -611,6 +656,59 @@ def register_routes(app, database_url: str) -> None:
|
||||
background_tasks.add_task(run_site_crawl_job, database_url, response["job_id"], request.model_dump())
|
||||
return response
|
||||
|
||||
@app.post("/crawl-site/by-project")
|
||||
def crawl_site_by_project(
|
||||
request: SiteCrawlByProjectRequest, background_tasks: BackgroundTasks
|
||||
):
|
||||
"""Start a site crawl against an existing DB project (no config_path)."""
|
||||
with session_scope(database_url) as session:
|
||||
repo = KnowledgeRepository(session)
|
||||
project = repo.get_project(request.project_name)
|
||||
config_dict = project.config or {}
|
||||
if not config_dict:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=f"Project '{request.project_name}' has no stored config",
|
||||
)
|
||||
config = project_config_from_dict(config_dict)
|
||||
try:
|
||||
source = repo.get_source(project.id, request.source_name)
|
||||
except KeyError as exc:
|
||||
raise HTTPException(status_code=404, detail=str(exc)) from exc
|
||||
|
||||
inner_request = request.to_site_crawl_request()
|
||||
apply_crawl_request_overrides(config, inner_request)
|
||||
|
||||
job = models.CrawlJob(
|
||||
project_id=project.id,
|
||||
source_id=source.id,
|
||||
url=request.url,
|
||||
status="pending",
|
||||
metadata_json={
|
||||
"kind": "site_crawl",
|
||||
"request": inner_request.model_dump(),
|
||||
"project_name": request.project_name,
|
||||
"progress": {
|
||||
"seed_url": request.url,
|
||||
"visited_count": 0,
|
||||
"analyzed_count": 0,
|
||||
"queued_count": 1,
|
||||
"skipped_count": 0,
|
||||
"errors": [],
|
||||
"pages": [],
|
||||
},
|
||||
},
|
||||
)
|
||||
session.add(job)
|
||||
session.flush()
|
||||
response = crawl_job_response(job)
|
||||
|
||||
task_payload = {**inner_request.model_dump(), "__config_dict": config_dict}
|
||||
background_tasks.add_task(
|
||||
run_site_crawl_job, database_url, response["job_id"], task_payload
|
||||
)
|
||||
return response
|
||||
|
||||
@app.get("/crawl-site/jobs/{job_id}")
|
||||
def crawl_site_job(job_id: int):
|
||||
with session_scope(database_url) as session:
|
||||
|
||||
Reference in New Issue
Block a user