Phase 1.3: 시드 크롤 — by-project 엔드포인트 + CrawlPage 폼/폴링/취소
세션 영구화: UI_REBUILD_PLAN.md 신설
- 사용자 비전 5가지 흐름 + Phase 보드 + 작업 재개 가이드
- 세션이 끊겨도 이 파일 + git log로 어디까지 했는지 즉시 복원
백엔드 (crawler_platform/app/api/):
- routes.py: POST /crawl-site/by-project 추가 — config_path 의존 제거
- SiteCrawlByProjectRequest: project_name 기반, DB project.config 재구성
- run_site_crawl_job이 __config_dict 메타키 인식하도록 최소 변경
- config/loader.py: project_config_from_dict() 헬퍼 추출 (DB dict ↔ ProjectConfig)
프론트엔드 API (src/lib/api/):
- crawl.ts: crawlApi.startByProject/getJob/cancel + Zod 스키마
- 종료 상태 판정 헬퍼 isCrawlTerminal()
TanStack Query 훅 (src/hooks/):
- useCrawl.ts: useStartSiteCrawl, useCrawlJob (2초 폴링, 종료 시 자동 중단), useCancelCrawl
- queryKeys에 crawl.job 키
UI 프리미티브 (src/components/ui/):
- select.tsx, progress.tsx, badge.tsx (success/warning/destructive variants 포함)
CrawlPage 완전 교체 (src/pages/):
- 2열 레이아웃: 좌측 크롤 설정 폼 + 우측 진행 상태
- 폼: 소스 선택 (드롭다운, 비어있으면 소스 추가 안내) + 시드 URL + max_depth/max_pages + same_domain_only
- react-hook-form + zod 검증
- 진행 상태: 상태 배지, 진행률 바, visited/queued/analyzed 카운터, 최근 페이지, 에러 details
- 크롤 종료시 폴링 자동 중단, 완료시 "결과 검토" 버튼
- 취소 버튼 → /crawl-site/jobs/{id}/cancel
- sonner 토스트
i18n: crawl.* 키 (한/영)
다음 단계: Phase 1.4 — 자율 연구 (POST /research/run by-project 마이그레이션 + ResearchPage)
Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
lasta
@@ -1,39 +1,474 @@
|
||||
import { useState } from "react";
|
||||
import { useNavigate, useParams } from "react-router-dom";
|
||||
import { useTranslation } from "react-i18next";
|
||||
import { useForm } from "react-hook-form";
|
||||
import { zodResolver } from "@hookform/resolvers/zod";
|
||||
import { z } from "zod";
|
||||
import { toast } from "sonner";
|
||||
import {
|
||||
AlertCircle,
|
||||
ArrowLeft,
|
||||
ArrowRight,
|
||||
Ban,
|
||||
CheckCircle2,
|
||||
Globe,
|
||||
Loader2,
|
||||
PlayCircle,
|
||||
XCircle,
|
||||
} from "lucide-react";
|
||||
import {
|
||||
Card,
|
||||
CardContent,
|
||||
CardDescription,
|
||||
CardHeader,
|
||||
CardTitle,
|
||||
} from "@/components/ui/card";
|
||||
import { Button } from "@/components/ui/button";
|
||||
import { Input } from "@/components/ui/input";
|
||||
import { Label } from "@/components/ui/label";
|
||||
import { Select } from "@/components/ui/select";
|
||||
import { Progress } from "@/components/ui/progress";
|
||||
import { Badge, BadgeProps } from "@/components/ui/badge";
|
||||
import { Skeleton } from "@/components/ui/skeleton";
|
||||
import { useProject } from "@/hooks/useProjects";
|
||||
import {
|
||||
useCancelCrawl,
|
||||
useCrawlJob,
|
||||
useStartSiteCrawl,
|
||||
} from "@/hooks/useCrawl";
|
||||
import { isCrawlTerminal } from "@/lib/api/crawl";
|
||||
|
||||
const startCrawlSchema = z.object({
|
||||
source_name: z.string().min(1, "소스를 선택하세요"),
|
||||
url: z.string().url("올바른 URL 형식이 아닙니다"),
|
||||
max_depth: z.number().int().min(0).max(10),
|
||||
max_pages: z.number().int().min(1).max(500),
|
||||
same_domain_only: z.boolean(),
|
||||
});
|
||||
|
||||
type StartCrawlFormValues = z.infer<typeof startCrawlSchema>;
|
||||
|
||||
function statusBadgeVariant(status: string): BadgeProps["variant"] {
|
||||
switch (status) {
|
||||
case "completed":
|
||||
return "success";
|
||||
case "failed":
|
||||
return "destructive";
|
||||
case "canceled":
|
||||
return "secondary";
|
||||
case "cancel_requested":
|
||||
return "warning";
|
||||
case "running":
|
||||
case "pending":
|
||||
return "default";
|
||||
default:
|
||||
return "outline";
|
||||
}
|
||||
}
|
||||
|
||||
function StatusBadge({ status }: { status: string }) {
|
||||
return <Badge variant={statusBadgeVariant(status)}>{status}</Badge>;
|
||||
}
|
||||
|
||||
export default function CrawlPage() {
|
||||
const navigate = useNavigate();
|
||||
const { projectId } = useParams();
|
||||
const { projectId } = useParams<{ projectId: string }>();
|
||||
const { t } = useTranslation();
|
||||
const projectName = projectId ?? "";
|
||||
|
||||
const { data: project, isLoading, isError, error, refetch } = useProject(projectName);
|
||||
const [activeJobId, setActiveJobId] = useState<string | null>(null);
|
||||
const { data: job } = useCrawlJob(activeJobId);
|
||||
const startCrawl = useStartSiteCrawl();
|
||||
const cancelCrawl = useCancelCrawl();
|
||||
|
||||
const {
|
||||
register,
|
||||
handleSubmit,
|
||||
setValue,
|
||||
watch,
|
||||
formState: { errors, isSubmitting },
|
||||
} = useForm<StartCrawlFormValues>({
|
||||
resolver: zodResolver(startCrawlSchema),
|
||||
defaultValues: {
|
||||
source_name: "",
|
||||
url: "",
|
||||
max_depth: 2,
|
||||
max_pages: 30,
|
||||
same_domain_only: true,
|
||||
},
|
||||
});
|
||||
|
||||
const onStart = async (values: StartCrawlFormValues) => {
|
||||
try {
|
||||
const created = await startCrawl.mutateAsync({
|
||||
project_name: projectName,
|
||||
...values,
|
||||
});
|
||||
setActiveJobId(created.job_id);
|
||||
toast.success(
|
||||
t("crawl.started", "크롤이 시작되었습니다 (job #{{id}})", {
|
||||
id: created.job_id,
|
||||
}),
|
||||
);
|
||||
} catch (e) {
|
||||
toast.error(
|
||||
t("crawl.startFailed", "시작 실패: {{msg}}", {
|
||||
msg: (e as Error).message,
|
||||
}),
|
||||
);
|
||||
}
|
||||
};
|
||||
|
||||
const onCancel = async () => {
|
||||
if (!activeJobId) return;
|
||||
try {
|
||||
await cancelCrawl.mutateAsync(activeJobId);
|
||||
toast.info(t("crawl.cancelRequested", "취소 요청됨"));
|
||||
} catch (e) {
|
||||
toast.error(
|
||||
t("crawl.cancelFailed", "취소 실패: {{msg}}", {
|
||||
msg: (e as Error).message,
|
||||
}),
|
||||
);
|
||||
}
|
||||
};
|
||||
|
||||
const sources = project?.sources ?? [];
|
||||
const sourceName = watch("source_name");
|
||||
const selectedSource = sources.find((s) => s.name === sourceName);
|
||||
const progress = job?.progress;
|
||||
const visited = progress?.visited_count ?? 0;
|
||||
const queued = progress?.queued_count ?? 0;
|
||||
const analyzed = progress?.analyzed_count ?? 0;
|
||||
const errors_ = progress?.errors ?? [];
|
||||
const max_pages = watch("max_pages");
|
||||
const progressPct = job && max_pages
|
||||
? Math.min(100, (visited / max_pages) * 100)
|
||||
: 0;
|
||||
const terminal = job ? isCrawlTerminal(job.status) : false;
|
||||
const running = job && !terminal;
|
||||
|
||||
return (
|
||||
<div className="min-h-screen bg-gray-50 p-4">
|
||||
<div className="max-w-4xl mx-auto">
|
||||
<h1 className="text-3xl font-bold text-gray-900 mb-2">
|
||||
{t("Build Ontology")}
|
||||
</h1>
|
||||
<p className="text-gray-600 mb-6">
|
||||
{t("Step 3 of 4: Auto-crawl and build ontology with Phase 5 + 7")}
|
||||
</p>
|
||||
|
||||
<div className="bg-white rounded-lg shadow p-6">
|
||||
<p className="text-gray-500">{t("Crawl pipeline will appear here")}</p>
|
||||
<div className="mx-auto max-w-6xl px-6 py-10">
|
||||
<div className="mb-6 flex items-center gap-3">
|
||||
<Button
|
||||
variant="ghost"
|
||||
size="icon"
|
||||
onClick={() => navigate(`/sources/${projectName}`)}
|
||||
aria-label={t("common.back", "이전")}
|
||||
>
|
||||
<ArrowLeft className="h-4 w-4" />
|
||||
</Button>
|
||||
<div className="flex-1">
|
||||
<h1 className="text-2xl font-bold tracking-tight">
|
||||
{t("crawl.title", "시드 크롤")}
|
||||
</h1>
|
||||
{project && (
|
||||
<p className="text-sm text-muted-foreground">
|
||||
{project.name}
|
||||
<span className="capitalize text-muted-foreground/70">
|
||||
{" "}
|
||||
· {project.domain}
|
||||
</span>
|
||||
</p>
|
||||
)}
|
||||
</div>
|
||||
{terminal && job?.status === "completed" && (
|
||||
<Button onClick={() => navigate(`/review/${projectName}`)}>
|
||||
{t("crawl.review", "결과 검토")}
|
||||
<ArrowRight className="h-4 w-4" />
|
||||
</Button>
|
||||
)}
|
||||
</div>
|
||||
|
||||
<div className="flex gap-4 mt-6">
|
||||
<button
|
||||
onClick={() => navigate(`/sources/${projectId}`)}
|
||||
className="flex-1 px-4 py-2 border border-gray-300 rounded-lg text-gray-700 hover:bg-gray-50 transition"
|
||||
>
|
||||
{t("Back")}
|
||||
</button>
|
||||
<button
|
||||
onClick={() => navigate(`/review/${projectId}`)}
|
||||
className="flex-1 px-4 py-2 bg-blue-600 text-white rounded-lg hover:bg-blue-700 transition"
|
||||
>
|
||||
{t("Review Results")}
|
||||
</button>
|
||||
</div>
|
||||
{isError && (
|
||||
<Card className="mb-6 border-destructive">
|
||||
<CardContent className="flex items-center justify-between gap-3 py-4">
|
||||
<div className="flex items-center gap-2 text-sm text-destructive">
|
||||
<AlertCircle className="h-4 w-4" />
|
||||
<span>{(error as Error).message}</span>
|
||||
</div>
|
||||
<Button
|
||||
variant="outline"
|
||||
size="sm"
|
||||
type="button"
|
||||
onClick={() => refetch()}
|
||||
>
|
||||
{t("common.retry", "다시 시도")}
|
||||
</Button>
|
||||
</CardContent>
|
||||
</Card>
|
||||
)}
|
||||
|
||||
<div className="grid gap-6 lg:grid-cols-[420px_1fr]">
|
||||
<Card>
|
||||
<CardHeader>
|
||||
<CardTitle>{t("crawl.formTitle", "크롤 설정")}</CardTitle>
|
||||
<CardDescription>
|
||||
{t(
|
||||
"crawl.formDesc",
|
||||
"시드 URL에서 시작해 링크를 따라가며 정보를 추출합니다",
|
||||
)}
|
||||
</CardDescription>
|
||||
</CardHeader>
|
||||
<CardContent>
|
||||
<form
|
||||
onSubmit={handleSubmit(onStart)}
|
||||
className="space-y-4"
|
||||
noValidate
|
||||
>
|
||||
<div className="space-y-1.5">
|
||||
<Label htmlFor="source_name">
|
||||
{t("crawl.source", "참고 소스")}
|
||||
</Label>
|
||||
{isLoading ? (
|
||||
<Skeleton className="h-10" />
|
||||
) : (
|
||||
<Select
|
||||
id="source_name"
|
||||
{...register("source_name")}
|
||||
onChange={(e) =>
|
||||
setValue("source_name", e.target.value, {
|
||||
shouldValidate: true,
|
||||
})
|
||||
}
|
||||
>
|
||||
<option value="">
|
||||
{t("crawl.pickSource", "소스를 선택하세요...")}
|
||||
</option>
|
||||
{sources.map((s) => (
|
||||
<option key={s.id} value={s.name}>
|
||||
{s.name} ({s.type})
|
||||
</option>
|
||||
))}
|
||||
</Select>
|
||||
)}
|
||||
{sources.length === 0 && !isLoading && (
|
||||
<p className="text-xs text-muted-foreground">
|
||||
{t(
|
||||
"crawl.noSources",
|
||||
"등록된 소스가 없습니다. 먼저 참고 소스를 추가하세요.",
|
||||
)}{" "}
|
||||
<button
|
||||
type="button"
|
||||
className="text-primary underline"
|
||||
onClick={() => navigate(`/sources/${projectName}`)}
|
||||
>
|
||||
{t("crawl.addSource", "소스 추가")}
|
||||
</button>
|
||||
</p>
|
||||
)}
|
||||
{errors.source_name && (
|
||||
<p className="text-xs text-destructive">
|
||||
{errors.source_name.message}
|
||||
</p>
|
||||
)}
|
||||
</div>
|
||||
|
||||
<div className="space-y-1.5">
|
||||
<Label htmlFor="url">{t("crawl.seedUrl", "시드 URL")}</Label>
|
||||
<Input
|
||||
id="url"
|
||||
placeholder={
|
||||
selectedSource?.base_url ?? "https://example.com/start"
|
||||
}
|
||||
{...register("url")}
|
||||
/>
|
||||
{errors.url && (
|
||||
<p className="text-xs text-destructive">
|
||||
{errors.url.message}
|
||||
</p>
|
||||
)}
|
||||
</div>
|
||||
|
||||
<div className="grid grid-cols-2 gap-3">
|
||||
<div className="space-y-1.5">
|
||||
<Label htmlFor="max_depth">
|
||||
{t("crawl.maxDepth", "최대 깊이")}
|
||||
</Label>
|
||||
<Input
|
||||
id="max_depth"
|
||||
type="number"
|
||||
min={0}
|
||||
max={10}
|
||||
{...register("max_depth", { valueAsNumber: true })}
|
||||
/>
|
||||
{errors.max_depth && (
|
||||
<p className="text-xs text-destructive">
|
||||
{errors.max_depth.message}
|
||||
</p>
|
||||
)}
|
||||
</div>
|
||||
<div className="space-y-1.5">
|
||||
<Label htmlFor="max_pages">
|
||||
{t("crawl.maxPages", "최대 페이지")}
|
||||
</Label>
|
||||
<Input
|
||||
id="max_pages"
|
||||
type="number"
|
||||
min={1}
|
||||
max={500}
|
||||
{...register("max_pages", { valueAsNumber: true })}
|
||||
/>
|
||||
{errors.max_pages && (
|
||||
<p className="text-xs text-destructive">
|
||||
{errors.max_pages.message}
|
||||
</p>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<label className="flex items-center gap-2 text-sm">
|
||||
<input
|
||||
type="checkbox"
|
||||
className="h-4 w-4 rounded border-input"
|
||||
{...register("same_domain_only")}
|
||||
/>
|
||||
<span>
|
||||
{t("crawl.sameDomainOnly", "동일 도메인만 따라가기")}
|
||||
</span>
|
||||
</label>
|
||||
|
||||
<Button
|
||||
type="submit"
|
||||
className="w-full"
|
||||
disabled={isSubmitting || startCrawl.isPending || Boolean(running)}
|
||||
>
|
||||
{startCrawl.isPending ? (
|
||||
<Loader2 className="h-4 w-4 animate-spin" />
|
||||
) : (
|
||||
<PlayCircle className="h-4 w-4" />
|
||||
)}
|
||||
{t("crawl.start", "크롤 시작")}
|
||||
</Button>
|
||||
</form>
|
||||
</CardContent>
|
||||
</Card>
|
||||
|
||||
<Card>
|
||||
<CardHeader>
|
||||
<div className="flex items-center justify-between">
|
||||
<div>
|
||||
<CardTitle>
|
||||
{t("crawl.progressTitle", "진행 상태")}
|
||||
</CardTitle>
|
||||
{job && (
|
||||
<CardDescription className="flex items-center gap-2">
|
||||
<span>job #{job.job_id}</span>
|
||||
<StatusBadge status={job.status} />
|
||||
</CardDescription>
|
||||
)}
|
||||
</div>
|
||||
{running && (
|
||||
<Button
|
||||
variant="outline"
|
||||
size="sm"
|
||||
onClick={onCancel}
|
||||
disabled={cancelCrawl.isPending}
|
||||
>
|
||||
<Ban className="h-4 w-4" />
|
||||
{t("crawl.cancel", "취소")}
|
||||
</Button>
|
||||
)}
|
||||
</div>
|
||||
</CardHeader>
|
||||
<CardContent>
|
||||
{!job && (
|
||||
<div className="flex flex-col items-center gap-3 py-12 text-center text-muted-foreground">
|
||||
<Globe className="h-12 w-12 opacity-40" />
|
||||
<p>
|
||||
{t(
|
||||
"crawl.idleHint",
|
||||
"왼쪽에서 시드 URL을 입력하고 시작하세요",
|
||||
)}
|
||||
</p>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{job && (
|
||||
<div className="space-y-4">
|
||||
<div>
|
||||
<div className="mb-1 flex items-center justify-between text-sm">
|
||||
<span className="text-muted-foreground">
|
||||
{t("crawl.visited", "방문")} {visited}
|
||||
{" / "}
|
||||
{max_pages}
|
||||
</span>
|
||||
<span className="text-muted-foreground">
|
||||
{t("crawl.queued", "대기")} {queued} ·{" "}
|
||||
{t("crawl.analyzed", "분석")} {analyzed}
|
||||
</span>
|
||||
</div>
|
||||
<Progress value={progressPct} indeterminate={running && visited === 0} />
|
||||
</div>
|
||||
|
||||
{job.url && (
|
||||
<div className="rounded-md bg-secondary/30 px-3 py-2 text-xs">
|
||||
<div className="text-muted-foreground">
|
||||
{t("crawl.seedUrl", "시드 URL")}
|
||||
</div>
|
||||
<a
|
||||
href={job.url}
|
||||
target="_blank"
|
||||
rel="noopener noreferrer"
|
||||
className="break-all text-foreground hover:underline"
|
||||
>
|
||||
{job.url}
|
||||
</a>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{progress?.latest_page && (
|
||||
<div className="rounded-md border bg-background px-3 py-2 text-xs">
|
||||
<div className="mb-1 text-muted-foreground">
|
||||
{t("crawl.latestPage", "최근 페이지")}
|
||||
</div>
|
||||
<div className="truncate font-medium">
|
||||
{progress.latest_page.title || progress.latest_page.url}
|
||||
</div>
|
||||
{progress.latest_page.page_type && (
|
||||
<Badge variant="outline" className="mt-1">
|
||||
{progress.latest_page.page_type}
|
||||
</Badge>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
|
||||
{job.error && (
|
||||
<div className="flex items-start gap-2 rounded-md border border-destructive bg-destructive/5 px-3 py-2 text-sm text-destructive">
|
||||
<XCircle className="mt-0.5 h-4 w-4 flex-shrink-0" />
|
||||
<span>{job.error}</span>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{errors_.length > 0 && (
|
||||
<details className="rounded-md border bg-background">
|
||||
<summary className="cursor-pointer px-3 py-2 text-sm font-medium">
|
||||
{t("crawl.errorsCount", "에러 {{count}}건", {
|
||||
count: errors_.length,
|
||||
})}
|
||||
</summary>
|
||||
<ul className="max-h-40 overflow-y-auto px-4 py-2 text-xs text-muted-foreground">
|
||||
{errors_.map((e, i) => (
|
||||
<li key={i} className="border-b py-1 last:border-0">
|
||||
{e}
|
||||
</li>
|
||||
))}
|
||||
</ul>
|
||||
</details>
|
||||
)}
|
||||
|
||||
{terminal && job.status === "completed" && (
|
||||
<div className="flex items-center gap-2 text-sm text-green-700">
|
||||
<CheckCircle2 className="h-4 w-4" />
|
||||
{t("crawl.doneHint", "크롤 완료. 결과 검토로 이동하세요.")}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
</CardContent>
|
||||
</Card>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
|
||||
Reference in New Issue
Block a user