Phase 1.2: 참고 소스 관리 — 소스 CRUD + UI
백엔드 (crawler_platform/app/api/routes.py):
- POST /projects/{name}/sources: 프로젝트에 소스 추가/업데이트
- InlineSourceConfig 재사용, KnowledgeRepository.upsert_source 호출
- DELETE /projects/{name}/sources/{source_name}: 소스 삭제
- 404 처리 + 외래 키 정리
- GET /projects/{name} 응답에 base_url 필드 추가
프론트엔드 API 레이어 (src/lib/api/):
- sources.ts: sourcesApi.create/delete + Zod 스키마 (Source, deleteSourceResponseSchema)
TanStack Query 훅 (src/hooks/):
- useSources.ts: useCreateSource, useDeleteSource (mutation + 캐시 무효화)
- queryKeys에 sources.byProject 키 팩토리
ConfigureSourcesPage 완전 교체 (src/pages/):
- 2열 레이아웃: 등록된 소스 목록 + 소스 추가 폼
- react-hook-form + zod 검증 (name 정규식, type enum, trust 0~1, rate_limit 1~600)
- 소스 카드 표시: type 배지, 신뢰도, base_url 링크, 삭제 버튼
- 빈 상태/로딩 스켈레톤/에러+재시도 모두 처리
- "크롤 진행" 버튼은 소스 1개 이상일 때만 활성
- sonner 토스트 알림
i18n locale 키 sources.* 추가 (한/영)
다음 단계: Phase 1.3 — URL 시드 크롤 (CrawlPage 실제 구현)
Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
lasta
@@ -35,6 +35,28 @@
|
||||
"created": "Project created: {{name}}",
|
||||
"createFailed": "Create failed: {{msg}}"
|
||||
},
|
||||
"sources": {
|
||||
"title": "Configure Sources",
|
||||
"next": "Proceed to Crawl",
|
||||
"listTitle": "Registered Sources",
|
||||
"listDesc": "Reference sites used for ontology construction",
|
||||
"empty": "No sources yet. Add one using the form on the right.",
|
||||
"addTitle": "Add Source",
|
||||
"addDesc": "Enter information about the reference site",
|
||||
"name": "Name",
|
||||
"type": "Type",
|
||||
"baseUrl": "Base URL",
|
||||
"trust": "Trust",
|
||||
"rateLimit": "rate/min",
|
||||
"respectRobots": "Respect robots.txt",
|
||||
"add": "Add Source",
|
||||
"delete": "Delete",
|
||||
"confirmDelete": "Delete source '{{name}}'?",
|
||||
"added": "Source added: {{name}}",
|
||||
"addFailed": "Add failed: {{msg}}",
|
||||
"deleted": "Source deleted: {{name}}",
|
||||
"deleteFailed": "Delete failed: {{msg}}"
|
||||
},
|
||||
"common": {
|
||||
"retry": "Retry",
|
||||
"cancel": "Cancel",
|
||||
|
||||
@@ -35,6 +35,28 @@
|
||||
"created": "프로젝트가 생성되었습니다: {{name}}",
|
||||
"createFailed": "생성 실패: {{msg}}"
|
||||
},
|
||||
"sources": {
|
||||
"title": "참고 소스 설정",
|
||||
"next": "크롤 진행",
|
||||
"listTitle": "등록된 소스",
|
||||
"listDesc": "프로젝트 온톨로지 구축에 사용할 참고 사이트 목록",
|
||||
"empty": "아직 등록된 소스가 없습니다. 오른쪽 폼에서 추가하세요.",
|
||||
"addTitle": "소스 추가",
|
||||
"addDesc": "참고할 사이트 정보를 입력하세요",
|
||||
"name": "이름",
|
||||
"type": "타입",
|
||||
"baseUrl": "Base URL",
|
||||
"trust": "신뢰도",
|
||||
"rateLimit": "rate/분",
|
||||
"respectRobots": "robots.txt 준수",
|
||||
"add": "소스 추가",
|
||||
"delete": "삭제",
|
||||
"confirmDelete": "정말 '{{name}}' 소스를 삭제하시겠습니까?",
|
||||
"added": "소스가 추가되었습니다: {{name}}",
|
||||
"addFailed": "추가 실패: {{msg}}",
|
||||
"deleted": "소스가 삭제되었습니다: {{name}}",
|
||||
"deleteFailed": "삭제 실패: {{msg}}"
|
||||
},
|
||||
"common": {
|
||||
"retry": "다시 시도",
|
||||
"cancel": "취소",
|
||||
|
||||
@@ -14,4 +14,9 @@ export const queryKeys = {
|
||||
byDomain: (domain: string) =>
|
||||
[...queryKeys.ontology.all, "byDomain", domain] as const,
|
||||
},
|
||||
sources: {
|
||||
all: ["sources"] as const,
|
||||
byProject: (projectName: string) =>
|
||||
[...queryKeys.sources.all, "byProject", projectName] as const,
|
||||
},
|
||||
};
|
||||
|
||||
29
crawler_platform/app/web/frontend/src/hooks/useSources.ts
Normal file
29
crawler_platform/app/web/frontend/src/hooks/useSources.ts
Normal file
@@ -0,0 +1,29 @@
|
||||
import { useMutation, useQueryClient } from "@tanstack/react-query";
|
||||
import { CreateSourceRequest, sourcesApi } from "@/lib/api/sources";
|
||||
import { queryKeys } from "./queryKeys";
|
||||
|
||||
export function useCreateSource(projectName: string) {
|
||||
const queryClient = useQueryClient();
|
||||
return useMutation({
|
||||
mutationFn: (body: CreateSourceRequest) =>
|
||||
sourcesApi.create(projectName, body),
|
||||
onSuccess: () => {
|
||||
queryClient.invalidateQueries({
|
||||
queryKey: queryKeys.projects.detail(projectName),
|
||||
});
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
export function useDeleteSource(projectName: string) {
|
||||
const queryClient = useQueryClient();
|
||||
return useMutation({
|
||||
mutationFn: (sourceName: string) =>
|
||||
sourcesApi.delete(projectName, sourceName),
|
||||
onSuccess: () => {
|
||||
queryClient.invalidateQueries({
|
||||
queryKey: queryKeys.projects.detail(projectName),
|
||||
});
|
||||
},
|
||||
});
|
||||
}
|
||||
@@ -2,3 +2,4 @@ export { apiClient, ApiError } from "./client";
|
||||
export * from "./projects";
|
||||
export * from "./domains";
|
||||
export * from "./ontology";
|
||||
export * from "./sources";
|
||||
|
||||
45
crawler_platform/app/web/frontend/src/lib/api/sources.ts
Normal file
45
crawler_platform/app/web/frontend/src/lib/api/sources.ts
Normal file
@@ -0,0 +1,45 @@
|
||||
import { z } from "zod";
|
||||
import { apiClient } from "./client";
|
||||
|
||||
export const sourceSchema = z.object({
|
||||
id: z.union([z.string(), z.number()]).transform(String),
|
||||
name: z.string(),
|
||||
type: z.string(),
|
||||
base_url: z.string().nullable().optional(),
|
||||
trust_level: z.number().nullable().optional(),
|
||||
respect_robots_txt: z.boolean().nullable().optional(),
|
||||
rate_limit_per_minute: z.number().nullable().optional(),
|
||||
});
|
||||
|
||||
export const deleteSourceResponseSchema = z.object({
|
||||
ok: z.boolean(),
|
||||
deleted: z.string(),
|
||||
});
|
||||
|
||||
export type Source = z.infer<typeof sourceSchema>;
|
||||
|
||||
export interface CreateSourceRequest {
|
||||
name: string;
|
||||
type?: string;
|
||||
trust_level?: number;
|
||||
base_url?: string | null;
|
||||
allowed_paths?: string[];
|
||||
parser?: string;
|
||||
fetcher?: string;
|
||||
rate_limit_per_minute?: number;
|
||||
respect_robots_txt?: boolean;
|
||||
}
|
||||
|
||||
export const sourcesApi = {
|
||||
create: (projectName: string, body: CreateSourceRequest) =>
|
||||
apiClient.post(
|
||||
`/projects/${encodeURIComponent(projectName)}/sources`,
|
||||
sourceSchema,
|
||||
body,
|
||||
),
|
||||
delete: (projectName: string, sourceName: string) =>
|
||||
apiClient.delete(
|
||||
`/projects/${encodeURIComponent(projectName)}/sources/${encodeURIComponent(sourceName)}`,
|
||||
deleteSourceResponseSchema,
|
||||
),
|
||||
};
|
||||
@@ -1,39 +1,410 @@
|
||||
import { useNavigate, useParams } from "react-router-dom";
|
||||
import { useTranslation } from "react-i18next";
|
||||
import { useForm } from "react-hook-form";
|
||||
import { zodResolver } from "@hookform/resolvers/zod";
|
||||
import { z } from "zod";
|
||||
import { toast } from "sonner";
|
||||
import {
|
||||
AlertCircle,
|
||||
ArrowLeft,
|
||||
ArrowRight,
|
||||
ExternalLink,
|
||||
Loader2,
|
||||
Plus,
|
||||
Trash2,
|
||||
} from "lucide-react";
|
||||
import {
|
||||
Card,
|
||||
CardContent,
|
||||
CardDescription,
|
||||
CardHeader,
|
||||
CardTitle,
|
||||
} from "@/components/ui/card";
|
||||
import { Button } from "@/components/ui/button";
|
||||
import { Input } from "@/components/ui/input";
|
||||
import { Label } from "@/components/ui/label";
|
||||
import { Skeleton } from "@/components/ui/skeleton";
|
||||
import { useProject } from "@/hooks/useProjects";
|
||||
import { useCreateSource, useDeleteSource } from "@/hooks/useSources";
|
||||
|
||||
const SOURCE_TYPES = [
|
||||
"official",
|
||||
"review",
|
||||
"blog",
|
||||
"news",
|
||||
"community",
|
||||
"unknown",
|
||||
] as const;
|
||||
|
||||
const sourceSchema = z.object({
|
||||
name: z
|
||||
.string()
|
||||
.min(2, "최소 2자 이상")
|
||||
.max(64, "최대 64자")
|
||||
.regex(/^[a-zA-Z0-9_-]+$/, "영문/숫자/_/-만 허용"),
|
||||
type: z.enum(SOURCE_TYPES),
|
||||
base_url: z
|
||||
.string()
|
||||
.url("올바른 URL 형식이 아닙니다")
|
||||
.or(z.literal(""))
|
||||
.optional(),
|
||||
trust_level: z
|
||||
.number({ invalid_type_error: "0~1 사이의 숫자" })
|
||||
.min(0)
|
||||
.max(1),
|
||||
rate_limit_per_minute: z
|
||||
.number({ invalid_type_error: "양의 정수" })
|
||||
.int()
|
||||
.min(1)
|
||||
.max(600),
|
||||
respect_robots_txt: z.boolean(),
|
||||
});
|
||||
|
||||
type SourceFormValues = z.infer<typeof sourceSchema>;
|
||||
|
||||
export default function ConfigureSourcesPage() {
|
||||
const navigate = useNavigate();
|
||||
const { projectId } = useParams();
|
||||
const { projectId } = useParams<{ projectId: string }>();
|
||||
const { t } = useTranslation();
|
||||
const projectName = projectId ?? "";
|
||||
|
||||
const {
|
||||
data: project,
|
||||
isLoading,
|
||||
isError,
|
||||
error,
|
||||
refetch,
|
||||
} = useProject(projectName);
|
||||
|
||||
const createSource = useCreateSource(projectName);
|
||||
const deleteSource = useDeleteSource(projectName);
|
||||
|
||||
const {
|
||||
register,
|
||||
handleSubmit,
|
||||
reset,
|
||||
formState: { errors, isSubmitting },
|
||||
} = useForm<SourceFormValues>({
|
||||
resolver: zodResolver(sourceSchema),
|
||||
defaultValues: {
|
||||
name: "",
|
||||
type: "official",
|
||||
base_url: "",
|
||||
trust_level: 0.7,
|
||||
rate_limit_per_minute: 30,
|
||||
respect_robots_txt: true,
|
||||
},
|
||||
});
|
||||
|
||||
const onAddSource = async (values: SourceFormValues) => {
|
||||
try {
|
||||
await createSource.mutateAsync({
|
||||
...values,
|
||||
base_url: values.base_url || null,
|
||||
});
|
||||
toast.success(
|
||||
t("sources.added", "소스가 추가되었습니다: {{name}}", {
|
||||
name: values.name,
|
||||
}),
|
||||
);
|
||||
reset();
|
||||
} catch (e) {
|
||||
toast.error(
|
||||
t("sources.addFailed", "추가 실패: {{msg}}", {
|
||||
msg: (e as Error).message,
|
||||
}),
|
||||
);
|
||||
}
|
||||
};
|
||||
|
||||
const onDelete = async (sourceName: string) => {
|
||||
if (
|
||||
!confirm(
|
||||
t("sources.confirmDelete", "정말 '{{name}}' 소스를 삭제하시겠습니까?", {
|
||||
name: sourceName,
|
||||
}),
|
||||
)
|
||||
) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
await deleteSource.mutateAsync(sourceName);
|
||||
toast.success(
|
||||
t("sources.deleted", "소스가 삭제되었습니다: {{name}}", {
|
||||
name: sourceName,
|
||||
}),
|
||||
);
|
||||
} catch (e) {
|
||||
toast.error(
|
||||
t("sources.deleteFailed", "삭제 실패: {{msg}}", {
|
||||
msg: (e as Error).message,
|
||||
}),
|
||||
);
|
||||
}
|
||||
};
|
||||
|
||||
return (
|
||||
<div className="min-h-screen bg-gray-50 p-4">
|
||||
<div className="max-w-4xl mx-auto">
|
||||
<h1 className="text-3xl font-bold text-gray-900 mb-2">
|
||||
{t("Configure Reference Sites")}
|
||||
</h1>
|
||||
<p className="text-gray-600 mb-6">
|
||||
{t("Step 2 of 4: Add 5-20 reference sites for knowledge extraction")}
|
||||
</p>
|
||||
|
||||
<div className="bg-white rounded-lg shadow p-6">
|
||||
<p className="text-gray-500">{t("No sites added yet")}</p>
|
||||
<div className="mx-auto max-w-5xl px-6 py-10">
|
||||
<div className="mb-6 flex items-center gap-3">
|
||||
<Button
|
||||
variant="ghost"
|
||||
size="icon"
|
||||
onClick={() => navigate("/")}
|
||||
aria-label={t("common.back", "이전")}
|
||||
>
|
||||
<ArrowLeft className="h-4 w-4" />
|
||||
</Button>
|
||||
<div className="flex-1">
|
||||
<h1 className="text-2xl font-bold tracking-tight">
|
||||
{t("sources.title", "참고 소스 설정")}
|
||||
</h1>
|
||||
{project && (
|
||||
<p className="text-sm text-muted-foreground">
|
||||
{project.name}{" "}
|
||||
<span className="capitalize text-muted-foreground/70">
|
||||
· {project.domain}
|
||||
</span>
|
||||
</p>
|
||||
)}
|
||||
</div>
|
||||
<Button
|
||||
onClick={() => navigate(`/crawl/${projectName}`)}
|
||||
disabled={!project || (project.sources?.length ?? 0) === 0}
|
||||
>
|
||||
{t("sources.next", "크롤 진행")}
|
||||
<ArrowRight className="h-4 w-4" />
|
||||
</Button>
|
||||
</div>
|
||||
|
||||
<div className="flex gap-4 mt-6">
|
||||
<button
|
||||
onClick={() => navigate("/onboard")}
|
||||
className="flex-1 px-4 py-2 border border-gray-300 rounded-lg text-gray-700 hover:bg-gray-50 transition"
|
||||
>
|
||||
{t("Back")}
|
||||
</button>
|
||||
<button
|
||||
onClick={() => navigate(`/crawl/${projectId}`)}
|
||||
className="flex-1 px-4 py-2 bg-blue-600 text-white rounded-lg hover:bg-blue-700 transition"
|
||||
>
|
||||
{t("Next")}
|
||||
</button>
|
||||
</div>
|
||||
{isError && (
|
||||
<Card className="mb-6 border-destructive">
|
||||
<CardContent className="flex items-center justify-between gap-3 py-4">
|
||||
<div className="flex items-center gap-2 text-sm text-destructive">
|
||||
<AlertCircle className="h-4 w-4" />
|
||||
<span>{(error as Error).message}</span>
|
||||
</div>
|
||||
<Button
|
||||
variant="outline"
|
||||
size="sm"
|
||||
type="button"
|
||||
onClick={() => refetch()}
|
||||
>
|
||||
{t("common.retry", "다시 시도")}
|
||||
</Button>
|
||||
</CardContent>
|
||||
</Card>
|
||||
)}
|
||||
|
||||
<div className="grid gap-6 lg:grid-cols-[1fr_360px]">
|
||||
<section>
|
||||
<Card>
|
||||
<CardHeader>
|
||||
<CardTitle>{t("sources.listTitle", "등록된 소스")}</CardTitle>
|
||||
<CardDescription>
|
||||
{t(
|
||||
"sources.listDesc",
|
||||
"프로젝트 온톨로지 구축에 사용할 참고 사이트 목록",
|
||||
)}
|
||||
</CardDescription>
|
||||
</CardHeader>
|
||||
<CardContent>
|
||||
{isLoading && (
|
||||
<div className="space-y-3">
|
||||
{Array.from({ length: 3 }).map((_, i) => (
|
||||
<Skeleton key={i} className="h-16" />
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
|
||||
{project && project.sources.length === 0 && (
|
||||
<p className="py-8 text-center text-sm text-muted-foreground">
|
||||
{t(
|
||||
"sources.empty",
|
||||
"아직 등록된 소스가 없습니다. 오른쪽 폼에서 추가하세요.",
|
||||
)}
|
||||
</p>
|
||||
)}
|
||||
|
||||
{project && project.sources.length > 0 && (
|
||||
<ul className="divide-y">
|
||||
{project.sources.map((s) => (
|
||||
<li
|
||||
key={s.id}
|
||||
className="flex items-center justify-between gap-4 py-3"
|
||||
>
|
||||
<div className="min-w-0 flex-1">
|
||||
<div className="flex items-center gap-2">
|
||||
<span className="font-medium">{s.name}</span>
|
||||
<span className="rounded bg-secondary px-1.5 py-0.5 text-xs text-secondary-foreground">
|
||||
{s.type}
|
||||
</span>
|
||||
{typeof s.trust_level === "number" && (
|
||||
<span className="text-xs text-muted-foreground">
|
||||
{t("sources.trust", "신뢰도")}{" "}
|
||||
{s.trust_level.toFixed(2)}
|
||||
</span>
|
||||
)}
|
||||
</div>
|
||||
{s.base_url && (
|
||||
<a
|
||||
href={s.base_url}
|
||||
target="_blank"
|
||||
rel="noopener noreferrer"
|
||||
className="mt-1 inline-flex items-center gap-1 text-xs text-muted-foreground hover:text-foreground"
|
||||
>
|
||||
<ExternalLink className="h-3 w-3" />
|
||||
{s.base_url}
|
||||
</a>
|
||||
)}
|
||||
</div>
|
||||
<Button
|
||||
variant="ghost"
|
||||
size="icon"
|
||||
onClick={() => onDelete(s.name)}
|
||||
disabled={deleteSource.isPending}
|
||||
aria-label={t("sources.delete", "삭제")}
|
||||
>
|
||||
<Trash2 className="h-4 w-4 text-destructive" />
|
||||
</Button>
|
||||
</li>
|
||||
))}
|
||||
</ul>
|
||||
)}
|
||||
</CardContent>
|
||||
</Card>
|
||||
</section>
|
||||
|
||||
<aside>
|
||||
<Card>
|
||||
<CardHeader>
|
||||
<CardTitle>{t("sources.addTitle", "소스 추가")}</CardTitle>
|
||||
<CardDescription>
|
||||
{t("sources.addDesc", "참고할 사이트 정보를 입력하세요")}
|
||||
</CardDescription>
|
||||
</CardHeader>
|
||||
<CardContent>
|
||||
<form
|
||||
onSubmit={handleSubmit(onAddSource)}
|
||||
className="space-y-4"
|
||||
noValidate
|
||||
>
|
||||
<div className="space-y-1.5">
|
||||
<Label htmlFor="src_name">
|
||||
{t("sources.name", "이름")}
|
||||
</Label>
|
||||
<Input
|
||||
id="src_name"
|
||||
placeholder="official_brand_site"
|
||||
{...register("name")}
|
||||
/>
|
||||
{errors.name && (
|
||||
<p className="text-xs text-destructive">
|
||||
{errors.name.message}
|
||||
</p>
|
||||
)}
|
||||
</div>
|
||||
|
||||
<div className="space-y-1.5">
|
||||
<Label htmlFor="src_type">
|
||||
{t("sources.type", "타입")}
|
||||
</Label>
|
||||
<select
|
||||
id="src_type"
|
||||
className="flex h-10 w-full rounded-md border border-input bg-background px-3 py-2 text-sm focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring"
|
||||
{...register("type")}
|
||||
>
|
||||
{SOURCE_TYPES.map((t) => (
|
||||
<option key={t} value={t}>
|
||||
{t}
|
||||
</option>
|
||||
))}
|
||||
</select>
|
||||
</div>
|
||||
|
||||
<div className="space-y-1.5">
|
||||
<Label htmlFor="src_url">
|
||||
{t("sources.baseUrl", "Base URL")}
|
||||
</Label>
|
||||
<Input
|
||||
id="src_url"
|
||||
placeholder="https://example.com"
|
||||
type="url"
|
||||
{...register("base_url")}
|
||||
/>
|
||||
{errors.base_url && (
|
||||
<p className="text-xs text-destructive">
|
||||
{errors.base_url.message}
|
||||
</p>
|
||||
)}
|
||||
</div>
|
||||
|
||||
<div className="grid grid-cols-2 gap-3">
|
||||
<div className="space-y-1.5">
|
||||
<Label htmlFor="src_trust">
|
||||
{t("sources.trust", "신뢰도")}
|
||||
</Label>
|
||||
<Input
|
||||
id="src_trust"
|
||||
type="number"
|
||||
step="0.05"
|
||||
min={0}
|
||||
max={1}
|
||||
{...register("trust_level", { valueAsNumber: true })}
|
||||
/>
|
||||
{errors.trust_level && (
|
||||
<p className="text-xs text-destructive">
|
||||
{errors.trust_level.message}
|
||||
</p>
|
||||
)}
|
||||
</div>
|
||||
<div className="space-y-1.5">
|
||||
<Label htmlFor="src_rate">
|
||||
{t("sources.rateLimit", "rate/분")}
|
||||
</Label>
|
||||
<Input
|
||||
id="src_rate"
|
||||
type="number"
|
||||
min={1}
|
||||
max={600}
|
||||
{...register("rate_limit_per_minute", {
|
||||
valueAsNumber: true,
|
||||
})}
|
||||
/>
|
||||
{errors.rate_limit_per_minute && (
|
||||
<p className="text-xs text-destructive">
|
||||
{errors.rate_limit_per_minute.message}
|
||||
</p>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<label className="flex items-center gap-2 text-sm">
|
||||
<input
|
||||
type="checkbox"
|
||||
className="h-4 w-4 rounded border-input"
|
||||
{...register("respect_robots_txt")}
|
||||
/>
|
||||
<span>
|
||||
{t("sources.respectRobots", "robots.txt 준수")}
|
||||
</span>
|
||||
</label>
|
||||
|
||||
<Button
|
||||
type="submit"
|
||||
className="w-full"
|
||||
disabled={isSubmitting || createSource.isPending}
|
||||
>
|
||||
{createSource.isPending ? (
|
||||
<Loader2 className="h-4 w-4 animate-spin" />
|
||||
) : (
|
||||
<Plus className="h-4 w-4" />
|
||||
)}
|
||||
{t("sources.add", "소스 추가")}
|
||||
</Button>
|
||||
</form>
|
||||
</CardContent>
|
||||
</Card>
|
||||
</aside>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
|
||||
Reference in New Issue
Block a user