diff --git a/UI_REBUILD_PLAN.md b/UI_REBUILD_PLAN.md index 7fdc785..0cf239e 100644 --- a/UI_REBUILD_PLAN.md +++ b/UI_REBUILD_PLAN.md @@ -21,17 +21,17 @@ | Phase 0.5 | API 클라이언트 + TanStack Query + shadcn UI + AppShell + Dashboard 연결 | `37cad40` | | Phase 1.1 | 프로젝트 생성 (OnboardingPage 폼 + 백엔드 `POST /projects/inline`, `GET /domains`) | `8681ac8` | | Phase 1.2 | 참고 소스 관리 (ConfigureSourcesPage CRUD + 백엔드 `POST/DELETE /projects/{name}/sources`) | `461ebc0` | -| Phase 1.3 | 시드 크롤 (CrawlPage + 폴링 + 취소, 백엔드 `POST /crawl-site/by-project`) | (이번 커밋) | +| Phase 1.3 | 시드 크롤 (CrawlPage + 폴링 + 취소, 백엔드 `POST /crawl-site/by-project`) | `aaaaa05` | +| Phase 1.4 | 자율 연구 (ResearchPage + 세션 이력, 백엔드 `POST /research/run/by-project`) | (이번 커밋) | ### 🚧 진행 중 -(없음 — Phase 1.4 시작 전) +(없음 — Phase 1.5 시작 전) ### ⏳ 대기 | Phase | 내용 | 다음 액션 | |---|---|---| -| **Phase 1.4** | 자율 연구 (research/run 장시간 자율 온톨로지 확장) | `POST /research/run`의 `config_path` 의존을 by-project로 마이그레이션 (Phase 1.3과 동일 패턴) | | Phase 1.5 | 엔티티/클레임 직접 입력 (`POST /projects/{n}/entities`, `/claims` 백엔드 추가 필요) | 독립 가능 | | Phase 2 | 그래프 시각화/편집 (Cytoscape 또는 react-flow 래퍼) | 데이터 일부 있어야 의미있음 | | Phase 3 | JSON Import/Export (대량 데이터 직접 입력) | 독립 가능 | diff --git a/crawler_platform/app/api/routes.py b/crawler_platform/app/api/routes.py index 6079771..86b6b67 100644 --- a/crawler_platform/app/api/routes.py +++ b/crawler_platform/app/api/routes.py @@ -148,6 +148,29 @@ class ResetProjectRequest(BaseModel): project_name: str | None = None +class ResearchRunByProjectRequest(BaseModel): + """Run autonomous research against an existing DB project (no config_path).""" + + project_name: str + source_name: str + url: str | None = None + seed_entity_id: int | None = None + goal: str = "Semantic ontology exploration" + extractor_provider: str = "lm_studio" + extractor_model: str | None = None + extractor_base_url: str | None = "http://localhost:1234/v1" + check_robots_txt: bool = False + respect_robots_txt: bool | None = None + max_depth: int = 2 + max_steps: int = 12 + max_branch: int = 8 + min_relevance: float = 0.35 + same_domain_only: bool = True + analyze_page_types: list[str] = Field( + default_factory=lambda: ["ProductPage", "BrandStoryPage", "ReviewPage"] + ) + + class UpdateClaimConfidenceRequest(BaseModel): confidence: float reason: str | None = None @@ -796,6 +819,56 @@ def register_routes(app, database_url: str) -> None: raise HTTPException(status_code=400, detail=str(exc)) from exc return asdict(result) + @app.post("/research/run/by-project") + def run_research_by_project(request: ResearchRunByProjectRequest): + """Run research against an existing DB project (no config_path).""" + with session_scope(database_url) as session: + repo = KnowledgeRepository(session) + project = repo.get_project(request.project_name) + config_dict = project.config or {} + if not config_dict: + raise HTTPException( + status_code=400, + detail=f"Project '{request.project_name}' has no stored config", + ) + config = project_config_from_dict(config_dict) + try: + source_config = config.source_by_name(request.source_name) + except KeyError as exc: + raise HTTPException(status_code=404, detail=str(exc)) from exc + + check_robots = request.respect_robots_txt + if check_robots is None: + check_robots = request.check_robots_txt + source_config.respect_robots_txt = check_robots + + loop = GraphResearchLoop( + repo, + extractor_for_domain( + config.domain, + provider=request.extractor_provider, + model=request.extractor_model, + base_url=request.extractor_base_url, + ), + ) + try: + result = loop.run( + project_config=config, + source_name=request.source_name, + seed_url=request.url or None, + seed_entity_id=request.seed_entity_id, + goal=request.goal, + max_depth=max(request.max_depth, 0), + max_steps=max(min(request.max_steps, 50), 1), + max_branch=max(min(request.max_branch, 30), 1), + min_relevance=min(max(request.min_relevance, 0.0), 1.0), + same_domain_only=request.same_domain_only, + analyze_page_types=set(request.analyze_page_types), + ) + except Exception as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + return asdict(result) + @app.get("/projects/{project_name}/research/sessions") def research_sessions(project_name: str, limit: int = 25): with session_scope(database_url) as session: diff --git a/crawler_platform/app/web/frontend/public/locales/en/common.json b/crawler_platform/app/web/frontend/public/locales/en/common.json index 858ff6e..dda9a6b 100644 --- a/crawler_platform/app/web/frontend/public/locales/en/common.json +++ b/crawler_platform/app/web/frontend/public/locales/en/common.json @@ -8,6 +8,7 @@ "onboard": "New Project", "sources": "Sources", "crawl": "Crawl", + "research": "Research", "review": "Review", "toggleSidebar": "Toggle sidebar" }, @@ -57,6 +58,40 @@ "deleted": "Source deleted: {{name}}", "deleteFailed": "Delete failed: {{msg}}" }, + "research": { + "title": "Autonomous Research", + "formTitle": "Research Settings", + "formDesc": "AI follows links from a seed to autonomously expand the ontology", + "source": "Source", + "pickSource": "Pick a source...", + "goal": "Goal", + "goalPlaceholder": "e.g. Collect note compositions and seasonal recommendations of popular perfume brands", + "seedUrl": "Seed URL", + "optional": "optional", + "maxSteps": "Max Steps", + "maxBranch": "Branch Width", + "maxDepth": "Max Depth", + "minRelevance": "Min Relevance", + "sameDomainOnly": "Same domain only", + "start": "Start Research", + "runningHint": "This may take a while. Don't close the page until it finishes.", + "runningTitle": "AI is researching...", + "completed": "Research completed", + "failed": "Failed: {{msg}}", + "doneHint": "Done", + "idleHint": "Start research on the left to see results here", + "resultTitle": "Latest Result", + "resultDesc": "Outcome of the research run in this session", + "stepsTaken": "Steps", + "pagesVisited": "Pages", + "entitiesFound": "Entities", + "claimsAdded": "Claims", + "rawResult": "Raw JSON", + "historyTitle": "Session History", + "historyDesc": "Past research sessions for this project", + "historyEmpty": "No sessions yet", + "pages": "pages" + }, "crawl": { "title": "Seed Crawl", "formTitle": "Crawl Settings", diff --git a/crawler_platform/app/web/frontend/public/locales/ko/common.json b/crawler_platform/app/web/frontend/public/locales/ko/common.json index 8f17490..7cbdb19 100644 --- a/crawler_platform/app/web/frontend/public/locales/ko/common.json +++ b/crawler_platform/app/web/frontend/public/locales/ko/common.json @@ -8,6 +8,7 @@ "onboard": "프로젝트 생성", "sources": "참고 소스", "crawl": "크롤 진행", + "research": "자율 연구", "review": "결과 검토", "toggleSidebar": "사이드바 토글" }, @@ -57,6 +58,40 @@ "deleted": "소스가 삭제되었습니다: {{name}}", "deleteFailed": "삭제 실패: {{msg}}" }, + "research": { + "title": "자율 연구", + "formTitle": "자율 연구 설정", + "formDesc": "AI가 시드에서 시작해 스스로 링크를 따라가며 온톨로지를 확장합니다", + "source": "참고 소스", + "pickSource": "소스를 선택하세요...", + "goal": "목표", + "goalPlaceholder": "예: 인기 브랜드 향수의 노트 구성과 시즌 추천 정보 수집", + "seedUrl": "시드 URL", + "optional": "선택", + "maxSteps": "최대 단계", + "maxBranch": "분기 폭", + "maxDepth": "최대 깊이", + "minRelevance": "최소 관련도", + "sameDomainOnly": "동일 도메인만 탐색", + "start": "자율 연구 시작", + "runningHint": "장시간 걸릴 수 있습니다. 완료될 때까지 페이지를 닫지 마세요.", + "runningTitle": "AI가 연구 중입니다...", + "completed": "자율 연구가 완료되었습니다", + "failed": "실패: {{msg}}", + "doneHint": "완료", + "idleHint": "왼쪽에서 자율 연구를 시작하면 결과가 여기에 표시됩니다", + "resultTitle": "최근 결과", + "resultDesc": "이번 세션에서 실행된 연구의 결과", + "stepsTaken": "단계", + "pagesVisited": "페이지", + "entitiesFound": "엔티티", + "claimsAdded": "클레임", + "rawResult": "원시 응답 JSON", + "historyTitle": "세션 이력", + "historyDesc": "이 프로젝트의 자율 연구 세션 기록", + "historyEmpty": "아직 실행된 세션이 없습니다", + "pages": "페이지" + }, "crawl": { "title": "시드 크롤", "formTitle": "크롤 설정", diff --git a/crawler_platform/app/web/frontend/src/App.tsx b/crawler_platform/app/web/frontend/src/App.tsx index c3878ee..14a71c0 100644 --- a/crawler_platform/app/web/frontend/src/App.tsx +++ b/crawler_platform/app/web/frontend/src/App.tsx @@ -3,6 +3,7 @@ import AppShell from "@/components/layout/AppShell"; import OnboardingPage from "@/pages/OnboardingPage"; import ConfigureSourcesPage from "@/pages/ConfigureSourcesPage"; import CrawlPage from "@/pages/CrawlPage"; +import ResearchPage from "@/pages/ResearchPage"; import ReviewPage from "@/pages/ReviewPage"; import DashboardPage from "@/pages/DashboardPage"; @@ -14,6 +15,7 @@ function App() { } /> } /> } /> + } /> } /> diff --git a/crawler_platform/app/web/frontend/src/components/layout/AppShell.tsx b/crawler_platform/app/web/frontend/src/components/layout/AppShell.tsx index c00768b..54dee3e 100644 --- a/crawler_platform/app/web/frontend/src/components/layout/AppShell.tsx +++ b/crawler_platform/app/web/frontend/src/components/layout/AppShell.tsx @@ -1,7 +1,7 @@ import { NavLink, Outlet } from "react-router-dom"; import { useTranslation } from "react-i18next"; import { useSelector, useDispatch } from "react-redux"; -import { LayoutDashboard, UploadCloud, Settings2, Activity, ListChecks, Menu } from "lucide-react"; +import { LayoutDashboard, UploadCloud, Settings2, Activity, Brain, ListChecks, Menu } from "lucide-react"; import { RootState } from "@/stores"; import { toggleSidebar } from "@/stores/slices/uiSlice"; import { cn } from "@/lib/utils"; @@ -18,6 +18,7 @@ const navItems: NavItem[] = [ { to: "/onboard", labelKey: "nav.onboard", icon: UploadCloud }, { to: "/sources/demo-project", labelKey: "nav.sources", icon: Settings2 }, { to: "/crawl/demo-project", labelKey: "nav.crawl", icon: Activity }, + { to: "/research/demo-project", labelKey: "nav.research", icon: Brain }, { to: "/review/demo-project", labelKey: "nav.review", icon: ListChecks }, ]; diff --git a/crawler_platform/app/web/frontend/src/hooks/queryKeys.ts b/crawler_platform/app/web/frontend/src/hooks/queryKeys.ts index ad8e4c6..ec0c863 100644 --- a/crawler_platform/app/web/frontend/src/hooks/queryKeys.ts +++ b/crawler_platform/app/web/frontend/src/hooks/queryKeys.ts @@ -24,4 +24,11 @@ export const queryKeys = { job: (jobId: string) => [...queryKeys.crawl.all, "job", jobId] as const, }, + research: { + all: ["research"] as const, + sessions: (projectName: string) => + [...queryKeys.research.all, "sessions", projectName] as const, + session: (jobId: string) => + [...queryKeys.research.all, "session", jobId] as const, + }, }; diff --git a/crawler_platform/app/web/frontend/src/hooks/useResearch.ts b/crawler_platform/app/web/frontend/src/hooks/useResearch.ts new file mode 100644 index 0000000..c1dc749 --- /dev/null +++ b/crawler_platform/app/web/frontend/src/hooks/useResearch.ts @@ -0,0 +1,37 @@ +import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; +import { + researchApi, + ResearchRunResult, + ResearchSessionDetail, + ResearchSessionItem, + StartResearchRequest, +} from "@/lib/api/research"; +import { queryKeys } from "./queryKeys"; + +export function useStartResearch(projectName: string) { + const queryClient = useQueryClient(); + return useMutation({ + mutationFn: (body) => researchApi.startByProject(body), + onSuccess: () => { + queryClient.invalidateQueries({ + queryKey: queryKeys.research.sessions(projectName), + }); + }, + }); +} + +export function useResearchSessions(projectName: string, limit = 25) { + return useQuery({ + queryKey: queryKeys.research.sessions(projectName), + queryFn: () => researchApi.listSessions(projectName, limit), + enabled: Boolean(projectName), + }); +} + +export function useResearchSession(jobId: string | null | undefined) { + return useQuery({ + queryKey: queryKeys.research.session(jobId ?? ""), + queryFn: () => researchApi.getSession(jobId!), + enabled: Boolean(jobId), + }); +} diff --git a/crawler_platform/app/web/frontend/src/lib/api/index.ts b/crawler_platform/app/web/frontend/src/lib/api/index.ts index 458e141..43219d9 100644 --- a/crawler_platform/app/web/frontend/src/lib/api/index.ts +++ b/crawler_platform/app/web/frontend/src/lib/api/index.ts @@ -4,3 +4,4 @@ export * from "./domains"; export * from "./ontology"; export * from "./sources"; export * from "./crawl"; +export * from "./research"; diff --git a/crawler_platform/app/web/frontend/src/lib/api/research.ts b/crawler_platform/app/web/frontend/src/lib/api/research.ts new file mode 100644 index 0000000..913ef1d --- /dev/null +++ b/crawler_platform/app/web/frontend/src/lib/api/research.ts @@ -0,0 +1,86 @@ +import { z } from "zod"; +import { apiClient } from "./client"; + +const idLike = z.union([z.string(), z.number()]).transform(String); + +/** + * Research result and session payload are loose by design — the backend + * returns `asdict(result)` of a Python dataclass we don't want to mirror + * field-for-field. Keep core fields strict and let extras pass through. + */ +export const researchRunResultSchema = z + .object({ + job_id: idLike.optional(), + project_id: idLike.optional(), + project_name: z.string().optional(), + goal: z.string().optional(), + status: z.string().optional(), + started_at: z.string().nullable().optional(), + finished_at: z.string().nullable().optional(), + steps_taken: z.number().optional(), + pages_visited: z.number().optional(), + entities_found: z.number().optional(), + claims_added: z.number().optional(), + error: z.string().nullable().optional(), + }) + .passthrough(); + +export type ResearchRunResult = z.infer; + +export const researchSessionItemSchema = z + .object({ + job_id: idLike, + status: z.string().optional(), + goal: z.string().nullable().optional(), + seed_url: z.string().nullable().optional(), + started_at: z.string().nullable().optional(), + finished_at: z.string().nullable().optional(), + steps_taken: z.number().nullable().optional(), + pages_visited: z.number().nullable().optional(), + entities_found: z.number().nullable().optional(), + }) + .passthrough(); + +export const researchSessionListSchema = z.array(researchSessionItemSchema); +export type ResearchSessionItem = z.infer; + +export const researchSessionDetailSchema = researchSessionItemSchema.extend({ + trace: z.array(z.record(z.string(), z.unknown())).optional(), +}); + +export type ResearchSessionDetail = z.infer; + +export interface StartResearchRequest { + project_name: string; + source_name: string; + url?: string; + seed_entity_id?: number | null; + goal?: string; + max_depth?: number; + max_steps?: number; + max_branch?: number; + min_relevance?: number; + same_domain_only?: boolean; + analyze_page_types?: string[]; + extractor_provider?: string; + extractor_model?: string | null; + extractor_base_url?: string | null; + check_robots_txt?: boolean; + respect_robots_txt?: boolean | null; +} + +export const researchApi = { + startByProject: (body: StartResearchRequest) => + apiClient.post("/research/run/by-project", researchRunResultSchema, body), + listSessions: (projectName: string, limit = 25) => + apiClient.get( + `/projects/${encodeURIComponent(projectName)}/research/sessions`, + researchSessionListSchema, + { query: { limit } }, + ), + getSession: (jobId: string) => + apiClient.get( + `/research/sessions/${encodeURIComponent(jobId)}`, + researchSessionDetailSchema, + ), +}; diff --git a/crawler_platform/app/web/frontend/src/pages/ResearchPage.tsx b/crawler_platform/app/web/frontend/src/pages/ResearchPage.tsx new file mode 100644 index 0000000..83afc41 --- /dev/null +++ b/crawler_platform/app/web/frontend/src/pages/ResearchPage.tsx @@ -0,0 +1,526 @@ +import { useState } from "react"; +import { useNavigate, useParams } from "react-router-dom"; +import { useTranslation } from "react-i18next"; +import { useForm } from "react-hook-form"; +import { zodResolver } from "@hookform/resolvers/zod"; +import { z } from "zod"; +import { toast } from "sonner"; +import { + AlertCircle, + ArrowLeft, + Brain, + CheckCircle2, + History, + Loader2, + Sparkles, + Target, +} from "lucide-react"; +import { + Card, + CardContent, + CardDescription, + CardHeader, + CardTitle, +} from "@/components/ui/card"; +import { Button } from "@/components/ui/button"; +import { Input } from "@/components/ui/input"; +import { Label } from "@/components/ui/label"; +import { Select } from "@/components/ui/select"; +import { Textarea } from "@/components/ui/textarea"; +import { Badge, BadgeProps } from "@/components/ui/badge"; +import { Skeleton } from "@/components/ui/skeleton"; +import { useProject } from "@/hooks/useProjects"; +import { + useResearchSessions, + useStartResearch, +} from "@/hooks/useResearch"; +import { ResearchRunResult } from "@/lib/api/research"; + +const startResearchSchema = z.object({ + source_name: z.string().min(1, "소스를 선택하세요"), + url: z + .string() + .url("올바른 URL 형식이 아닙니다") + .or(z.literal("")) + .optional(), + goal: z + .string() + .min(3, "최소 3자") + .max(500, "최대 500자"), + max_depth: z.number().int().min(0).max(10), + max_steps: z.number().int().min(1).max(50), + max_branch: z.number().int().min(1).max(30), + min_relevance: z.number().min(0).max(1), + same_domain_only: z.boolean(), +}); + +type StartResearchFormValues = z.infer; + +function sessionStatusVariant(status?: string | null): BadgeProps["variant"] { + switch (status) { + case "completed": + return "success"; + case "failed": + return "destructive"; + case "canceled": + return "secondary"; + case "running": + case "pending": + return "default"; + default: + return "outline"; + } +} + +export default function ResearchPage() { + const navigate = useNavigate(); + const { projectId } = useParams<{ projectId: string }>(); + const { t } = useTranslation(); + const projectName = projectId ?? ""; + + const { data: project, isLoading: projectLoading, isError, error, refetch } = + useProject(projectName); + const sessions = useResearchSessions(projectName); + const startResearch = useStartResearch(projectName); + + const [lastResult, setLastResult] = useState(null); + + const { + register, + handleSubmit, + setValue, + formState: { errors, isSubmitting }, + } = useForm({ + resolver: zodResolver(startResearchSchema), + defaultValues: { + source_name: "", + url: "", + goal: "Semantic ontology exploration", + max_depth: 2, + max_steps: 12, + max_branch: 8, + min_relevance: 0.35, + same_domain_only: true, + }, + }); + + const onSubmit = async (values: StartResearchFormValues) => { + try { + setLastResult(null); + const res = await startResearch.mutateAsync({ + project_name: projectName, + source_name: values.source_name, + url: values.url || undefined, + goal: values.goal, + max_depth: values.max_depth, + max_steps: values.max_steps, + max_branch: values.max_branch, + min_relevance: values.min_relevance, + same_domain_only: values.same_domain_only, + }); + setLastResult(res); + toast.success(t("research.completed", "자율 연구가 완료되었습니다")); + } catch (e) { + toast.error( + t("research.failed", "실패: {{msg}}", { + msg: (e as Error).message, + }), + ); + } + }; + + const sources = project?.sources ?? []; + + return ( +
+
+ +
+

+ + {t("research.title", "자율 연구")} +

+ {project && ( +

+ {project.name} + + {" "} + · {project.domain} + +

+ )} +
+
+ + {isError && ( + + +
+ + {(error as Error).message} +
+ + + + )} + +
+ + + + + {t("research.formTitle", "자율 연구 설정")} + + + {t( + "research.formDesc", + "AI가 시드에서 시작해 스스로 링크를 따라가며 온톨로지를 확장합니다", + )} + + + +
+
+ + {projectLoading ? ( + + ) : ( + + )} + {errors.source_name && ( +

+ {errors.source_name.message} +

+ )} +
+ +
+ +