83 lines
2.3 KiB
TypeScript
83 lines
2.3 KiB
TypeScript
import { z } from "zod";
|
|
import { apiClient } from "./client";
|
|
|
|
const stringFromAny = z.union([z.string(), z.number()]).transform(String);
|
|
|
|
export const crawlPageItemSchema = z
|
|
.object({
|
|
url: z.string().optional(),
|
|
status: z.string().optional(),
|
|
page_type: z.string().optional(),
|
|
title: z.string().nullable().optional(),
|
|
error: z.string().nullable().optional(),
|
|
})
|
|
.passthrough();
|
|
|
|
export const crawlProgressSchema = z
|
|
.object({
|
|
seed_url: z.string().optional(),
|
|
visited_count: z.number().optional(),
|
|
analyzed_count: z.number().optional(),
|
|
queued_count: z.number().optional(),
|
|
skipped_count: z.number().optional(),
|
|
errors: z.array(z.string()).optional(),
|
|
pages: z.array(crawlPageItemSchema).optional(),
|
|
latest_page: crawlPageItemSchema.optional(),
|
|
})
|
|
.passthrough();
|
|
|
|
export const crawlJobSchema = z.object({
|
|
job_id: stringFromAny,
|
|
status: z.string(),
|
|
url: z.string().nullable().optional(),
|
|
error: z.string().nullable().optional(),
|
|
scheduled_at: z.string().nullable().optional(),
|
|
started_at: z.string().nullable().optional(),
|
|
finished_at: z.string().nullable().optional(),
|
|
progress: crawlProgressSchema.default({}),
|
|
request: z.record(z.string(), z.unknown()).default({}),
|
|
});
|
|
|
|
export type CrawlJob = z.infer<typeof crawlJobSchema>;
|
|
export type CrawlProgress = z.infer<typeof crawlProgressSchema>;
|
|
|
|
export interface StartSiteCrawlRequest {
|
|
project_name: string;
|
|
source_name: string;
|
|
url: string;
|
|
max_depth?: number;
|
|
max_pages?: number;
|
|
same_domain_only?: boolean;
|
|
analyze_page_types?: string[];
|
|
extractor_provider?: string;
|
|
extractor_model?: string | null;
|
|
extractor_base_url?: string | null;
|
|
check_robots_txt?: boolean;
|
|
respect_robots_txt?: boolean | null;
|
|
}
|
|
|
|
export const TERMINAL_CRAWL_STATUSES = new Set([
|
|
"completed",
|
|
"failed",
|
|
"canceled",
|
|
]);
|
|
|
|
export function isCrawlTerminal(status: string): boolean {
|
|
return TERMINAL_CRAWL_STATUSES.has(status);
|
|
}
|
|
|
|
export const crawlApi = {
|
|
startByProject: (body: StartSiteCrawlRequest) =>
|
|
apiClient.post("/crawl-site/by-project", crawlJobSchema, body),
|
|
getJob: (jobId: string) =>
|
|
apiClient.get(
|
|
`/crawl-site/jobs/${encodeURIComponent(jobId)}`,
|
|
crawlJobSchema,
|
|
),
|
|
cancel: (jobId: string) =>
|
|
apiClient.post(
|
|
`/crawl-site/jobs/${encodeURIComponent(jobId)}/cancel`,
|
|
crawlJobSchema,
|
|
),
|
|
};
|