From 461ebc062b1044421ba8b864da2b92c9a952be84 Mon Sep 17 00:00:00 2001 From: lasta Date: Thu, 14 May 2026 14:53:39 +0900 Subject: [PATCH] =?UTF-8?q?Phase=201.2:=20=EC=B0=B8=EA=B3=A0=20=EC=86=8C?= =?UTF-8?q?=EC=8A=A4=20=EA=B4=80=EB=A6=AC=20=E2=80=94=20=EC=86=8C=EC=8A=A4?= =?UTF-8?q?=20CRUD=20+=20UI?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 백엔드 (crawler_platform/app/api/routes.py): - POST /projects/{name}/sources: 프로젝트에 소스 추가/업데이트 - InlineSourceConfig 재사용, KnowledgeRepository.upsert_source 호출 - DELETE /projects/{name}/sources/{source_name}: 소스 삭제 - 404 처리 + 외래 키 정리 - GET /projects/{name} 응답에 base_url 필드 추가 프론트엔드 API 레이어 (src/lib/api/): - sources.ts: sourcesApi.create/delete + Zod 스키마 (Source, deleteSourceResponseSchema) TanStack Query 훅 (src/hooks/): - useSources.ts: useCreateSource, useDeleteSource (mutation + 캐시 무효화) - queryKeys에 sources.byProject 키 팩토리 ConfigureSourcesPage 완전 교체 (src/pages/): - 2열 레이아웃: 등록된 소스 목록 + 소스 추가 폼 - react-hook-form + zod 검증 (name 정규식, type enum, trust 0~1, rate_limit 1~600) - 소스 카드 표시: type 배지, 신뢰도, base_url 링크, 삭제 버튼 - 빈 상태/로딩 스켈레톤/에러+재시도 모두 처리 - "크롤 진행" 버튼은 소스 1개 이상일 때만 활성 - sonner 토스트 알림 i18n locale 키 sources.* 추가 (한/영) 다음 단계: Phase 1.3 — URL 시드 크롤 (CrawlPage 실제 구현) Co-Authored-By: Claude Opus 4.7 --- crawler_platform/app/api/routes.py | 37 ++ .../frontend/public/locales/en/common.json | 22 + .../frontend/public/locales/ko/common.json | 22 + .../app/web/frontend/src/hooks/queryKeys.ts | 5 + .../app/web/frontend/src/hooks/useSources.ts | 29 ++ .../app/web/frontend/src/lib/api/index.ts | 1 + .../app/web/frontend/src/lib/api/sources.ts | 45 ++ .../src/pages/ConfigureSourcesPage.tsx | 423 ++++++++++++++++-- 8 files changed, 558 insertions(+), 26 deletions(-) create mode 100644 crawler_platform/app/web/frontend/src/hooks/useSources.ts create mode 100644 crawler_platform/app/web/frontend/src/lib/api/sources.ts diff --git a/crawler_platform/app/api/routes.py b/crawler_platform/app/api/routes.py index 5387007..9ed2559 100644 --- a/crawler_platform/app/api/routes.py +++ b/crawler_platform/app/api/routes.py @@ -407,6 +407,7 @@ def register_routes(app, database_url: str) -> None: "id": source.id, "name": source.name, "type": source.type, + "base_url": source.base_url, "trust_level": source.trust_level, "respect_robots_txt": source.respect_robots_txt, "rate_limit_per_minute": source.rate_limit_per_minute, @@ -415,6 +416,42 @@ def register_routes(app, database_url: str) -> None: ], } + @app.post("/projects/{project_name}/sources") + def add_project_source(project_name: str, request: InlineSourceConfig): + """Add or update a source on an existing project.""" + with session_scope(database_url) as session: + repo = KnowledgeRepository(session) + project = repo.get_project(project_name) + source = repo.upsert_source(project, SourceConfig(**request.model_dump())) + return { + "id": source.id, + "name": source.name, + "type": source.type, + "base_url": source.base_url, + "trust_level": source.trust_level, + "respect_robots_txt": source.respect_robots_txt, + "rate_limit_per_minute": source.rate_limit_per_minute, + } + + @app.delete("/projects/{project_name}/sources/{source_name}") + def delete_project_source(project_name: str, source_name: str): + with session_scope(database_url) as session: + repo = KnowledgeRepository(session) + project = repo.get_project(project_name) + source = session.scalar( + select(models.Source).where( + models.Source.project_id == project.id, + models.Source.name == source_name, + ) + ) + if source is None: + raise HTTPException( + status_code=404, + detail=f"Source '{source_name}' not found in project '{project_name}'", + ) + session.delete(source) + return {"ok": True, "deleted": source_name} + @app.get("/ontology/{domain}") def ontology(domain: str): return ontology_to_dict(ontology_for_domain(domain)) diff --git a/crawler_platform/app/web/frontend/public/locales/en/common.json b/crawler_platform/app/web/frontend/public/locales/en/common.json index ee86693..1eaa821 100644 --- a/crawler_platform/app/web/frontend/public/locales/en/common.json +++ b/crawler_platform/app/web/frontend/public/locales/en/common.json @@ -35,6 +35,28 @@ "created": "Project created: {{name}}", "createFailed": "Create failed: {{msg}}" }, + "sources": { + "title": "Configure Sources", + "next": "Proceed to Crawl", + "listTitle": "Registered Sources", + "listDesc": "Reference sites used for ontology construction", + "empty": "No sources yet. Add one using the form on the right.", + "addTitle": "Add Source", + "addDesc": "Enter information about the reference site", + "name": "Name", + "type": "Type", + "baseUrl": "Base URL", + "trust": "Trust", + "rateLimit": "rate/min", + "respectRobots": "Respect robots.txt", + "add": "Add Source", + "delete": "Delete", + "confirmDelete": "Delete source '{{name}}'?", + "added": "Source added: {{name}}", + "addFailed": "Add failed: {{msg}}", + "deleted": "Source deleted: {{name}}", + "deleteFailed": "Delete failed: {{msg}}" + }, "common": { "retry": "Retry", "cancel": "Cancel", diff --git a/crawler_platform/app/web/frontend/public/locales/ko/common.json b/crawler_platform/app/web/frontend/public/locales/ko/common.json index d7009bb..e001837 100644 --- a/crawler_platform/app/web/frontend/public/locales/ko/common.json +++ b/crawler_platform/app/web/frontend/public/locales/ko/common.json @@ -35,6 +35,28 @@ "created": "프로젝트가 생성되었습니다: {{name}}", "createFailed": "생성 실패: {{msg}}" }, + "sources": { + "title": "참고 소스 설정", + "next": "크롤 진행", + "listTitle": "등록된 소스", + "listDesc": "프로젝트 온톨로지 구축에 사용할 참고 사이트 목록", + "empty": "아직 등록된 소스가 없습니다. 오른쪽 폼에서 추가하세요.", + "addTitle": "소스 추가", + "addDesc": "참고할 사이트 정보를 입력하세요", + "name": "이름", + "type": "타입", + "baseUrl": "Base URL", + "trust": "신뢰도", + "rateLimit": "rate/분", + "respectRobots": "robots.txt 준수", + "add": "소스 추가", + "delete": "삭제", + "confirmDelete": "정말 '{{name}}' 소스를 삭제하시겠습니까?", + "added": "소스가 추가되었습니다: {{name}}", + "addFailed": "추가 실패: {{msg}}", + "deleted": "소스가 삭제되었습니다: {{name}}", + "deleteFailed": "삭제 실패: {{msg}}" + }, "common": { "retry": "다시 시도", "cancel": "취소", diff --git a/crawler_platform/app/web/frontend/src/hooks/queryKeys.ts b/crawler_platform/app/web/frontend/src/hooks/queryKeys.ts index f2983a9..5040a9f 100644 --- a/crawler_platform/app/web/frontend/src/hooks/queryKeys.ts +++ b/crawler_platform/app/web/frontend/src/hooks/queryKeys.ts @@ -14,4 +14,9 @@ export const queryKeys = { byDomain: (domain: string) => [...queryKeys.ontology.all, "byDomain", domain] as const, }, + sources: { + all: ["sources"] as const, + byProject: (projectName: string) => + [...queryKeys.sources.all, "byProject", projectName] as const, + }, }; diff --git a/crawler_platform/app/web/frontend/src/hooks/useSources.ts b/crawler_platform/app/web/frontend/src/hooks/useSources.ts new file mode 100644 index 0000000..158f197 --- /dev/null +++ b/crawler_platform/app/web/frontend/src/hooks/useSources.ts @@ -0,0 +1,29 @@ +import { useMutation, useQueryClient } from "@tanstack/react-query"; +import { CreateSourceRequest, sourcesApi } from "@/lib/api/sources"; +import { queryKeys } from "./queryKeys"; + +export function useCreateSource(projectName: string) { + const queryClient = useQueryClient(); + return useMutation({ + mutationFn: (body: CreateSourceRequest) => + sourcesApi.create(projectName, body), + onSuccess: () => { + queryClient.invalidateQueries({ + queryKey: queryKeys.projects.detail(projectName), + }); + }, + }); +} + +export function useDeleteSource(projectName: string) { + const queryClient = useQueryClient(); + return useMutation({ + mutationFn: (sourceName: string) => + sourcesApi.delete(projectName, sourceName), + onSuccess: () => { + queryClient.invalidateQueries({ + queryKey: queryKeys.projects.detail(projectName), + }); + }, + }); +} diff --git a/crawler_platform/app/web/frontend/src/lib/api/index.ts b/crawler_platform/app/web/frontend/src/lib/api/index.ts index 4f123b1..b24ccd4 100644 --- a/crawler_platform/app/web/frontend/src/lib/api/index.ts +++ b/crawler_platform/app/web/frontend/src/lib/api/index.ts @@ -2,3 +2,4 @@ export { apiClient, ApiError } from "./client"; export * from "./projects"; export * from "./domains"; export * from "./ontology"; +export * from "./sources"; diff --git a/crawler_platform/app/web/frontend/src/lib/api/sources.ts b/crawler_platform/app/web/frontend/src/lib/api/sources.ts new file mode 100644 index 0000000..e880259 --- /dev/null +++ b/crawler_platform/app/web/frontend/src/lib/api/sources.ts @@ -0,0 +1,45 @@ +import { z } from "zod"; +import { apiClient } from "./client"; + +export const sourceSchema = z.object({ + id: z.union([z.string(), z.number()]).transform(String), + name: z.string(), + type: z.string(), + base_url: z.string().nullable().optional(), + trust_level: z.number().nullable().optional(), + respect_robots_txt: z.boolean().nullable().optional(), + rate_limit_per_minute: z.number().nullable().optional(), +}); + +export const deleteSourceResponseSchema = z.object({ + ok: z.boolean(), + deleted: z.string(), +}); + +export type Source = z.infer; + +export interface CreateSourceRequest { + name: string; + type?: string; + trust_level?: number; + base_url?: string | null; + allowed_paths?: string[]; + parser?: string; + fetcher?: string; + rate_limit_per_minute?: number; + respect_robots_txt?: boolean; +} + +export const sourcesApi = { + create: (projectName: string, body: CreateSourceRequest) => + apiClient.post( + `/projects/${encodeURIComponent(projectName)}/sources`, + sourceSchema, + body, + ), + delete: (projectName: string, sourceName: string) => + apiClient.delete( + `/projects/${encodeURIComponent(projectName)}/sources/${encodeURIComponent(sourceName)}`, + deleteSourceResponseSchema, + ), +}; diff --git a/crawler_platform/app/web/frontend/src/pages/ConfigureSourcesPage.tsx b/crawler_platform/app/web/frontend/src/pages/ConfigureSourcesPage.tsx index 648e929..cc773b0 100644 --- a/crawler_platform/app/web/frontend/src/pages/ConfigureSourcesPage.tsx +++ b/crawler_platform/app/web/frontend/src/pages/ConfigureSourcesPage.tsx @@ -1,39 +1,410 @@ import { useNavigate, useParams } from "react-router-dom"; import { useTranslation } from "react-i18next"; +import { useForm } from "react-hook-form"; +import { zodResolver } from "@hookform/resolvers/zod"; +import { z } from "zod"; +import { toast } from "sonner"; +import { + AlertCircle, + ArrowLeft, + ArrowRight, + ExternalLink, + Loader2, + Plus, + Trash2, +} from "lucide-react"; +import { + Card, + CardContent, + CardDescription, + CardHeader, + CardTitle, +} from "@/components/ui/card"; +import { Button } from "@/components/ui/button"; +import { Input } from "@/components/ui/input"; +import { Label } from "@/components/ui/label"; +import { Skeleton } from "@/components/ui/skeleton"; +import { useProject } from "@/hooks/useProjects"; +import { useCreateSource, useDeleteSource } from "@/hooks/useSources"; + +const SOURCE_TYPES = [ + "official", + "review", + "blog", + "news", + "community", + "unknown", +] as const; + +const sourceSchema = z.object({ + name: z + .string() + .min(2, "최소 2자 이상") + .max(64, "최대 64자") + .regex(/^[a-zA-Z0-9_-]+$/, "영문/숫자/_/-만 허용"), + type: z.enum(SOURCE_TYPES), + base_url: z + .string() + .url("올바른 URL 형식이 아닙니다") + .or(z.literal("")) + .optional(), + trust_level: z + .number({ invalid_type_error: "0~1 사이의 숫자" }) + .min(0) + .max(1), + rate_limit_per_minute: z + .number({ invalid_type_error: "양의 정수" }) + .int() + .min(1) + .max(600), + respect_robots_txt: z.boolean(), +}); + +type SourceFormValues = z.infer; export default function ConfigureSourcesPage() { const navigate = useNavigate(); - const { projectId } = useParams(); + const { projectId } = useParams<{ projectId: string }>(); const { t } = useTranslation(); + const projectName = projectId ?? ""; + + const { + data: project, + isLoading, + isError, + error, + refetch, + } = useProject(projectName); + + const createSource = useCreateSource(projectName); + const deleteSource = useDeleteSource(projectName); + + const { + register, + handleSubmit, + reset, + formState: { errors, isSubmitting }, + } = useForm({ + resolver: zodResolver(sourceSchema), + defaultValues: { + name: "", + type: "official", + base_url: "", + trust_level: 0.7, + rate_limit_per_minute: 30, + respect_robots_txt: true, + }, + }); + + const onAddSource = async (values: SourceFormValues) => { + try { + await createSource.mutateAsync({ + ...values, + base_url: values.base_url || null, + }); + toast.success( + t("sources.added", "소스가 추가되었습니다: {{name}}", { + name: values.name, + }), + ); + reset(); + } catch (e) { + toast.error( + t("sources.addFailed", "추가 실패: {{msg}}", { + msg: (e as Error).message, + }), + ); + } + }; + + const onDelete = async (sourceName: string) => { + if ( + !confirm( + t("sources.confirmDelete", "정말 '{{name}}' 소스를 삭제하시겠습니까?", { + name: sourceName, + }), + ) + ) { + return; + } + try { + await deleteSource.mutateAsync(sourceName); + toast.success( + t("sources.deleted", "소스가 삭제되었습니다: {{name}}", { + name: sourceName, + }), + ); + } catch (e) { + toast.error( + t("sources.deleteFailed", "삭제 실패: {{msg}}", { + msg: (e as Error).message, + }), + ); + } + }; return ( -
-
-

- {t("Configure Reference Sites")} -

-

- {t("Step 2 of 4: Add 5-20 reference sites for knowledge extraction")} -

- -
-

{t("No sites added yet")}

+
+
+ +
+

+ {t("sources.title", "참고 소스 설정")} +

+ {project && ( +

+ {project.name}{" "} + + · {project.domain} + +

+ )}
+ +
-
- - -
+ {isError && ( + + +
+ + {(error as Error).message} +
+ + + + )} + +
+
+ + + {t("sources.listTitle", "등록된 소스")} + + {t( + "sources.listDesc", + "프로젝트 온톨로지 구축에 사용할 참고 사이트 목록", + )} + + + + {isLoading && ( +
+ {Array.from({ length: 3 }).map((_, i) => ( + + ))} +
+ )} + + {project && project.sources.length === 0 && ( +

+ {t( + "sources.empty", + "아직 등록된 소스가 없습니다. 오른쪽 폼에서 추가하세요.", + )} +

+ )} + + {project && project.sources.length > 0 && ( +
    + {project.sources.map((s) => ( +
  • +
    +
    + {s.name} + + {s.type} + + {typeof s.trust_level === "number" && ( + + {t("sources.trust", "신뢰도")}{" "} + {s.trust_level.toFixed(2)} + + )} +
    + {s.base_url && ( + + + {s.base_url} + + )} +
    + +
    • + ))} +
+ )} + + +
+ +
);