import { z } from "zod"; import { apiClient } from "./client"; const stringFromAny = z.union([z.string(), z.number()]).transform(String); export const crawlPageItemSchema = z .object({ url: z.string().optional(), status: z.string().optional(), page_type: z.string().optional(), title: z.string().nullable().optional(), extraction_mode: z.string().nullable().optional(), effective_extraction_mode: z.string().nullable().optional(), llm_skipped: z.boolean().optional(), llm_skip_reason: z.string().nullable().optional(), fallback_used: z.boolean().optional(), agreement_claim_count: z.number().optional(), conflict_claim_count: z.number().optional(), error: z.string().nullable().optional(), }) .passthrough(); export const crawlExtractionSummarySchema = z .object({ llm_skipped_count: z.number().default(0), fallback_count: z.number().default(0), conflict_claim_count: z.number().default(0), agreement_claim_count: z.number().default(0), llm_call_count: z.number().default(0), }) .passthrough(); export const crawlProgressSchema = z .object({ seed_url: z.string().optional(), visited_count: z.number().optional(), analyzed_count: z.number().optional(), queued_count: z.number().optional(), skipped_count: z.number().optional(), errors: z.array(z.string()).optional(), pages: z.array(crawlPageItemSchema).optional(), latest_page: crawlPageItemSchema.optional(), extraction_summary: crawlExtractionSummarySchema.optional(), }) .passthrough(); export const crawlJobSchema = z.object({ job_id: stringFromAny, status: z.string(), url: z.string().nullable().optional(), error: z.string().nullable().optional(), scheduled_at: z.string().nullable().optional(), started_at: z.string().nullable().optional(), finished_at: z.string().nullable().optional(), progress: crawlProgressSchema.default({}), request: z.record(z.string(), z.unknown()).default({}), }); export type CrawlJob = z.infer; export type CrawlProgress = z.infer; export interface StartSiteCrawlRequest { project_name: string; source_name: string; url: string; max_depth?: number; max_pages?: number; same_domain_only?: boolean; analyze_page_types?: string[]; extraction_mode?: "rule_only" | "llm_only" | "hybrid" | "compare"; extractor_provider?: string; extractor_model?: string | null; extractor_base_url?: string | null; fallback_to_rules?: boolean; check_robots_txt?: boolean; respect_robots_txt?: boolean | null; } export const TERMINAL_CRAWL_STATUSES = new Set([ "completed", "failed", "canceled", ]); export function isCrawlTerminal(status: string): boolean { return TERMINAL_CRAWL_STATUSES.has(status); } export const crawlApi = { startByProject: (body: StartSiteCrawlRequest) => apiClient.post("/crawl-site/by-project", crawlJobSchema, body), getJob: (jobId: string) => apiClient.get( `/crawl-site/jobs/${encodeURIComponent(jobId)}`, crawlJobSchema, ), cancel: (jobId: string) => apiClient.post( `/crawl-site/jobs/${encodeURIComponent(jobId)}/cancel`, crawlJobSchema, ), };