[crawler_platform 삭제]
This commit is contained in:
LASTA_DEV01\lasta
485
ontology_platform/web/frontend/src/pages/CrawlPage.tsx
Normal file
485
ontology_platform/web/frontend/src/pages/CrawlPage.tsx
Normal file
@@ -0,0 +1,485 @@
|
||||
import { useState } from "react";
|
||||
import { useNavigate, useParams } from "react-router-dom";
|
||||
import { useTranslation } from "react-i18next";
|
||||
import { useForm } from "react-hook-form";
|
||||
import { zodResolver } from "@hookform/resolvers/zod";
|
||||
import { z } from "zod";
|
||||
import { toast } from "sonner";
|
||||
import {
|
||||
AlertCircle,
|
||||
ArrowLeft,
|
||||
ArrowRight,
|
||||
Ban,
|
||||
CheckCircle2,
|
||||
Globe,
|
||||
Loader2,
|
||||
PlayCircle,
|
||||
XCircle,
|
||||
} from "lucide-react";
|
||||
import {
|
||||
Card,
|
||||
CardContent,
|
||||
CardDescription,
|
||||
CardHeader,
|
||||
CardTitle,
|
||||
} from "@/components/ui/card";
|
||||
import { Button } from "@/components/ui/button";
|
||||
import { Input } from "@/components/ui/input";
|
||||
import { Label } from "@/components/ui/label";
|
||||
import { Select } from "@/components/ui/select";
|
||||
import { Progress } from "@/components/ui/progress";
|
||||
import { Badge, BadgeProps } from "@/components/ui/badge";
|
||||
import { Skeleton } from "@/components/ui/skeleton";
|
||||
import { useProject } from "@/hooks/useProjects";
|
||||
import {
|
||||
useCancelCrawl,
|
||||
useCrawlJob,
|
||||
useStartSiteCrawl,
|
||||
} from "@/hooks/useCrawl";
|
||||
import { isCrawlTerminal } from "@/lib/api/crawl";
|
||||
|
||||
const startCrawlSchema = z.object({
|
||||
source_name: z.string().min(1, "소스를 선택하세요"),
|
||||
url: z.string().url("올바른 URL 형식이 아닙니다"),
|
||||
max_depth: z.number().int().min(0).max(10),
|
||||
max_pages: z.number().int().min(1).max(500),
|
||||
same_domain_only: z.boolean(),
|
||||
});
|
||||
|
||||
type StartCrawlFormValues = z.infer<typeof startCrawlSchema>;
|
||||
|
||||
function statusBadgeVariant(status: string): BadgeProps["variant"] {
|
||||
switch (status) {
|
||||
case "completed":
|
||||
return "success";
|
||||
case "failed":
|
||||
return "destructive";
|
||||
case "canceled":
|
||||
return "secondary";
|
||||
case "cancel_requested":
|
||||
return "warning";
|
||||
case "running":
|
||||
case "pending":
|
||||
return "default";
|
||||
default:
|
||||
return "outline";
|
||||
}
|
||||
}
|
||||
|
||||
function StatusBadge({ status }: { status: string }) {
|
||||
return <Badge variant={statusBadgeVariant(status)}>{status}</Badge>;
|
||||
}
|
||||
|
||||
export default function CrawlPage() {
|
||||
const navigate = useNavigate();
|
||||
const { projectId } = useParams<{ projectId: string }>();
|
||||
const { t } = useTranslation();
|
||||
const projectName = projectId ?? "";
|
||||
|
||||
const { data: project, isLoading, isError, error, refetch } = useProject(projectName);
|
||||
const [activeJobId, setActiveJobId] = useState<string | null>(null);
|
||||
const { data: job } = useCrawlJob(activeJobId);
|
||||
const startCrawl = useStartSiteCrawl();
|
||||
const cancelCrawl = useCancelCrawl();
|
||||
|
||||
const {
|
||||
register,
|
||||
handleSubmit,
|
||||
setValue,
|
||||
watch,
|
||||
formState: { errors, isSubmitting },
|
||||
} = useForm<StartCrawlFormValues>({
|
||||
resolver: zodResolver(startCrawlSchema),
|
||||
defaultValues: {
|
||||
source_name: "",
|
||||
url: "",
|
||||
max_depth: 2,
|
||||
max_pages: 30,
|
||||
same_domain_only: true,
|
||||
},
|
||||
});
|
||||
|
||||
const onStart = async (values: StartCrawlFormValues) => {
|
||||
try {
|
||||
const created = await startCrawl.mutateAsync({
|
||||
project_name: projectName,
|
||||
...values,
|
||||
});
|
||||
setActiveJobId(created.job_id);
|
||||
toast.success(
|
||||
t("crawl.started", "크롤이 시작되었습니다 (job #{{id}})", {
|
||||
id: created.job_id,
|
||||
}),
|
||||
);
|
||||
} catch (e) {
|
||||
toast.error(
|
||||
t("crawl.startFailed", "시작 실패: {{msg}}", {
|
||||
msg: (e as Error).message,
|
||||
}),
|
||||
);
|
||||
}
|
||||
};
|
||||
|
||||
const onCancel = async () => {
|
||||
if (!activeJobId) return;
|
||||
try {
|
||||
await cancelCrawl.mutateAsync(activeJobId);
|
||||
toast.info(t("crawl.cancelRequested", "취소 요청됨"));
|
||||
} catch (e) {
|
||||
toast.error(
|
||||
t("crawl.cancelFailed", "취소 실패: {{msg}}", {
|
||||
msg: (e as Error).message,
|
||||
}),
|
||||
);
|
||||
}
|
||||
};
|
||||
|
||||
const sources = project?.sources ?? [];
|
||||
const sourceName = watch("source_name");
|
||||
const selectedSource = sources.find((s) => s.name === sourceName);
|
||||
const progress = job?.progress;
|
||||
const visited = progress?.visited_count ?? 0;
|
||||
const queued = progress?.queued_count ?? 0;
|
||||
const analyzed = progress?.analyzed_count ?? 0;
|
||||
const errors_ = progress?.errors ?? [];
|
||||
const max_pages = watch("max_pages");
|
||||
const progressPct = job && max_pages
|
||||
? Math.min(100, (visited / max_pages) * 100)
|
||||
: 0;
|
||||
const terminal = job ? isCrawlTerminal(job.status) : false;
|
||||
const running = job && !terminal;
|
||||
|
||||
return (
|
||||
<div className="mx-auto max-w-6xl px-6 py-10">
|
||||
<div className="mb-6 flex items-center gap-3">
|
||||
<Button
|
||||
variant="ghost"
|
||||
size="icon"
|
||||
onClick={() => navigate(`/sources/${projectName}`)}
|
||||
aria-label={t("common.back", "이전")}
|
||||
>
|
||||
<ArrowLeft className="h-4 w-4" />
|
||||
</Button>
|
||||
<div className="flex-1">
|
||||
<h1 className="text-2xl font-bold tracking-tight">
|
||||
{t("crawl.title", "시드 크롤")}
|
||||
</h1>
|
||||
{project && (
|
||||
<p className="text-sm text-muted-foreground">
|
||||
{project.name}
|
||||
<span className="capitalize text-muted-foreground/70">
|
||||
{" "}
|
||||
· {project.domain}
|
||||
</span>
|
||||
</p>
|
||||
)}
|
||||
</div>
|
||||
{terminal && job?.status === "completed" && (
|
||||
<Button onClick={() => navigate(`/pipeline/${projectName}`)}>
|
||||
{t("crawl.review", "파이프라인 확인")}
|
||||
<ArrowRight className="h-4 w-4" />
|
||||
</Button>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{isError && (
|
||||
<Card className="mb-6 border-destructive">
|
||||
<CardContent className="flex items-center justify-between gap-3 py-4">
|
||||
<div className="flex items-center gap-2 text-sm text-destructive">
|
||||
<AlertCircle className="h-4 w-4" />
|
||||
<span>{(error as Error).message}</span>
|
||||
</div>
|
||||
<Button
|
||||
variant="outline"
|
||||
size="sm"
|
||||
type="button"
|
||||
onClick={() => refetch()}
|
||||
>
|
||||
{t("common.retry", "다시 시도")}
|
||||
</Button>
|
||||
</CardContent>
|
||||
</Card>
|
||||
)}
|
||||
|
||||
<div className="grid gap-6 lg:grid-cols-[420px_1fr]">
|
||||
<Card>
|
||||
<CardHeader>
|
||||
<CardTitle>{t("crawl.formTitle", "크롤 설정")}</CardTitle>
|
||||
<CardDescription>
|
||||
{t(
|
||||
"crawl.formDesc",
|
||||
"시드 URL에서 시작해 링크를 따라가며 정보를 추출합니다",
|
||||
)}
|
||||
</CardDescription>
|
||||
</CardHeader>
|
||||
<CardContent>
|
||||
<form
|
||||
onSubmit={handleSubmit(onStart)}
|
||||
className="space-y-4"
|
||||
noValidate
|
||||
>
|
||||
<div className="space-y-1.5">
|
||||
<Label htmlFor="source_name">
|
||||
{t("crawl.source", "참고 소스")}
|
||||
</Label>
|
||||
{isLoading ? (
|
||||
<Skeleton className="h-10" />
|
||||
) : (
|
||||
<Select
|
||||
id="source_name"
|
||||
{...register("source_name")}
|
||||
onChange={(e) => {
|
||||
const nextSourceName = e.target.value;
|
||||
const nextSource = sources.find(
|
||||
(source) => source.name === nextSourceName,
|
||||
);
|
||||
|
||||
setValue("source_name", nextSourceName, {
|
||||
shouldValidate: true,
|
||||
shouldDirty: true,
|
||||
});
|
||||
setValue("url", nextSource?.base_url ?? "", {
|
||||
shouldValidate: Boolean(nextSource?.base_url),
|
||||
shouldDirty: true,
|
||||
});
|
||||
}}
|
||||
>
|
||||
<option value="">
|
||||
{t("crawl.pickSource", "소스를 선택하세요...")}
|
||||
</option>
|
||||
{sources.map((s) => (
|
||||
<option key={s.id} value={s.name}>
|
||||
{s.name} ({s.type})
|
||||
</option>
|
||||
))}
|
||||
</Select>
|
||||
)}
|
||||
{sources.length === 0 && !isLoading && (
|
||||
<p className="text-xs text-muted-foreground">
|
||||
{t(
|
||||
"crawl.noSources",
|
||||
"등록된 소스가 없습니다. 먼저 참고 소스를 추가하세요.",
|
||||
)}{" "}
|
||||
<button
|
||||
type="button"
|
||||
className="text-primary underline"
|
||||
onClick={() => navigate(`/sources/${projectName}`)}
|
||||
>
|
||||
{t("crawl.addSource", "소스 추가")}
|
||||
</button>
|
||||
</p>
|
||||
)}
|
||||
{errors.source_name && (
|
||||
<p className="text-xs text-destructive">
|
||||
{errors.source_name.message}
|
||||
</p>
|
||||
)}
|
||||
</div>
|
||||
|
||||
<div className="space-y-1.5">
|
||||
<Label htmlFor="url">{t("crawl.seedUrl", "시드 URL")}</Label>
|
||||
<Input
|
||||
id="url"
|
||||
placeholder={
|
||||
selectedSource?.base_url ?? "https://example.com/start"
|
||||
}
|
||||
{...register("url")}
|
||||
/>
|
||||
{errors.url && (
|
||||
<p className="text-xs text-destructive">
|
||||
{errors.url.message}
|
||||
</p>
|
||||
)}
|
||||
</div>
|
||||
|
||||
<div className="grid grid-cols-2 gap-3">
|
||||
<div className="space-y-1.5">
|
||||
<Label htmlFor="max_depth">
|
||||
{t("crawl.maxDepth", "최대 깊이")}
|
||||
</Label>
|
||||
<Input
|
||||
id="max_depth"
|
||||
type="number"
|
||||
min={0}
|
||||
max={10}
|
||||
{...register("max_depth", { valueAsNumber: true })}
|
||||
/>
|
||||
{errors.max_depth && (
|
||||
<p className="text-xs text-destructive">
|
||||
{errors.max_depth.message}
|
||||
</p>
|
||||
)}
|
||||
</div>
|
||||
<div className="space-y-1.5">
|
||||
<Label htmlFor="max_pages">
|
||||
{t("crawl.maxPages", "최대 페이지")}
|
||||
</Label>
|
||||
<Input
|
||||
id="max_pages"
|
||||
type="number"
|
||||
min={1}
|
||||
max={500}
|
||||
{...register("max_pages", { valueAsNumber: true })}
|
||||
/>
|
||||
{errors.max_pages && (
|
||||
<p className="text-xs text-destructive">
|
||||
{errors.max_pages.message}
|
||||
</p>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<label className="flex items-center gap-2 text-sm">
|
||||
<input
|
||||
type="checkbox"
|
||||
className="h-4 w-4 rounded border-input"
|
||||
{...register("same_domain_only")}
|
||||
/>
|
||||
<span>
|
||||
{t("crawl.sameDomainOnly", "동일 도메인만 따라가기")}
|
||||
</span>
|
||||
</label>
|
||||
|
||||
<Button
|
||||
type="submit"
|
||||
className="w-full"
|
||||
disabled={isSubmitting || startCrawl.isPending || Boolean(running)}
|
||||
>
|
||||
{startCrawl.isPending ? (
|
||||
<Loader2 className="h-4 w-4 animate-spin" />
|
||||
) : (
|
||||
<PlayCircle className="h-4 w-4" />
|
||||
)}
|
||||
{t("crawl.start", "크롤 시작")}
|
||||
</Button>
|
||||
</form>
|
||||
</CardContent>
|
||||
</Card>
|
||||
|
||||
<Card>
|
||||
<CardHeader>
|
||||
<div className="flex items-center justify-between">
|
||||
<div>
|
||||
<CardTitle>
|
||||
{t("crawl.progressTitle", "진행 상태")}
|
||||
</CardTitle>
|
||||
{job && (
|
||||
<CardDescription className="flex items-center gap-2">
|
||||
<span>job #{job.job_id}</span>
|
||||
<StatusBadge status={job.status} />
|
||||
</CardDescription>
|
||||
)}
|
||||
</div>
|
||||
{running && (
|
||||
<Button
|
||||
variant="outline"
|
||||
size="sm"
|
||||
onClick={onCancel}
|
||||
disabled={cancelCrawl.isPending}
|
||||
>
|
||||
<Ban className="h-4 w-4" />
|
||||
{t("crawl.cancel", "취소")}
|
||||
</Button>
|
||||
)}
|
||||
</div>
|
||||
</CardHeader>
|
||||
<CardContent>
|
||||
{!job && (
|
||||
<div className="flex flex-col items-center gap-3 py-12 text-center text-muted-foreground">
|
||||
<Globe className="h-12 w-12 opacity-40" />
|
||||
<p>
|
||||
{t(
|
||||
"crawl.idleHint",
|
||||
"왼쪽에서 시드 URL을 입력하고 시작하세요",
|
||||
)}
|
||||
</p>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{job && (
|
||||
<div className="space-y-4">
|
||||
<div>
|
||||
<div className="mb-1 flex items-center justify-between text-sm">
|
||||
<span className="text-muted-foreground">
|
||||
{t("crawl.visited", "방문")} {visited}
|
||||
{" / "}
|
||||
{max_pages}
|
||||
</span>
|
||||
<span className="text-muted-foreground">
|
||||
{t("crawl.queued", "대기")} {queued} ·{" "}
|
||||
{t("crawl.analyzed", "분석")} {analyzed}
|
||||
</span>
|
||||
</div>
|
||||
<Progress value={progressPct} indeterminate={running && visited === 0} />
|
||||
</div>
|
||||
|
||||
{job.url && (
|
||||
<div className="rounded-md bg-secondary/30 px-3 py-2 text-xs">
|
||||
<div className="text-muted-foreground">
|
||||
{t("crawl.seedUrl", "시드 URL")}
|
||||
</div>
|
||||
<a
|
||||
href={job.url}
|
||||
target="_blank"
|
||||
rel="noopener noreferrer"
|
||||
className="break-all text-foreground hover:underline"
|
||||
>
|
||||
{job.url}
|
||||
</a>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{progress?.latest_page && (
|
||||
<div className="rounded-md border bg-background px-3 py-2 text-xs">
|
||||
<div className="mb-1 text-muted-foreground">
|
||||
{t("crawl.latestPage", "최근 페이지")}
|
||||
</div>
|
||||
<div className="truncate font-medium">
|
||||
{progress.latest_page.title || progress.latest_page.url}
|
||||
</div>
|
||||
{progress.latest_page.page_type && (
|
||||
<Badge variant="outline" className="mt-1">
|
||||
{progress.latest_page.page_type}
|
||||
</Badge>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
|
||||
{job.error && (
|
||||
<div className="flex items-start gap-2 rounded-md border border-destructive bg-destructive/5 px-3 py-2 text-sm text-destructive">
|
||||
<XCircle className="mt-0.5 h-4 w-4 flex-shrink-0" />
|
||||
<span>{job.error}</span>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{errors_.length > 0 && (
|
||||
<details className="rounded-md border bg-background">
|
||||
<summary className="cursor-pointer px-3 py-2 text-sm font-medium">
|
||||
{t("crawl.errorsCount", "에러 {{count}}건", {
|
||||
count: errors_.length,
|
||||
})}
|
||||
</summary>
|
||||
<ul className="max-h-40 overflow-y-auto px-4 py-2 text-xs text-muted-foreground">
|
||||
{errors_.map((e, i) => (
|
||||
<li key={i} className="border-b py-1 last:border-0">
|
||||
{e}
|
||||
</li>
|
||||
))}
|
||||
</ul>
|
||||
</details>
|
||||
)}
|
||||
|
||||
{terminal && job.status === "completed" && (
|
||||
<div className="flex items-center gap-2 text-sm text-green-700">
|
||||
<CheckCircle2 className="h-4 w-4" />
|
||||
{t("crawl.doneHint", "크롤 완료. 구축 파이프라인으로 이동하세요.")}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
</CardContent>
|
||||
</Card>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
Reference in New Issue
Block a user