128 lines
3.7 KiB
TypeScript
128 lines
3.7 KiB
TypeScript
import { z } from "zod";
|
|
import { apiClient } from "./client";
|
|
|
|
const stringFromAny = z.union([z.string(), z.number()]).transform(String);
|
|
|
|
export const crawlPageItemSchema = z
|
|
.object({
|
|
url: z.string().optional(),
|
|
status: z.string().optional(),
|
|
page_type: z.string().optional(),
|
|
title: z.string().nullable().optional(),
|
|
extraction_mode: z.string().nullable().optional(),
|
|
effective_extraction_mode: z.string().nullable().optional(),
|
|
llm_skipped: z.boolean().optional(),
|
|
llm_skip_reason: z.string().nullable().optional(),
|
|
fallback_used: z.boolean().optional(),
|
|
agreement_claim_count: z.number().optional(),
|
|
conflict_claim_count: z.number().optional(),
|
|
error: z.string().nullable().optional(),
|
|
})
|
|
.passthrough();
|
|
|
|
export const crawlExtractionSummarySchema = z
|
|
.object({
|
|
llm_skipped_count: z.number().default(0),
|
|
fallback_count: z.number().default(0),
|
|
conflict_claim_count: z.number().default(0),
|
|
agreement_claim_count: z.number().default(0),
|
|
llm_call_count: z.number().default(0),
|
|
})
|
|
.passthrough();
|
|
|
|
export const crawlProgressSchema = z
|
|
.object({
|
|
seed_url: z.string().optional(),
|
|
visited_count: z.number().optional(),
|
|
analyzed_count: z.number().optional(),
|
|
queued_count: z.number().optional(),
|
|
skipped_count: z.number().optional(),
|
|
errors: z.array(z.string()).optional(),
|
|
pages: z.array(crawlPageItemSchema).optional(),
|
|
latest_page: crawlPageItemSchema.optional(),
|
|
extraction_summary: crawlExtractionSummarySchema.optional(),
|
|
})
|
|
.passthrough();
|
|
|
|
export const crawlJobSchema = z.object({
|
|
job_id: stringFromAny,
|
|
status: z.string(),
|
|
url: z.string().nullable().optional(),
|
|
error: z.string().nullable().optional(),
|
|
scheduled_at: z.string().nullable().optional(),
|
|
started_at: z.string().nullable().optional(),
|
|
finished_at: z.string().nullable().optional(),
|
|
progress: crawlProgressSchema.default({}),
|
|
request: z.record(z.string(), z.unknown()).default({}),
|
|
});
|
|
|
|
export type CrawlJob = z.infer<typeof crawlJobSchema>;
|
|
export type CrawlProgress = z.infer<typeof crawlProgressSchema>;
|
|
|
|
export interface StartSiteCrawlRequest {
|
|
project_name: string;
|
|
source_name: string;
|
|
url: string;
|
|
max_depth?: number;
|
|
max_pages?: number;
|
|
same_domain_only?: boolean;
|
|
analyze_page_types?: string[];
|
|
extraction_mode?: "rule_only" | "llm_only" | "hybrid" | "compare";
|
|
extractor_provider?: string;
|
|
extractor_model?: string | null;
|
|
extractor_base_url?: string | null;
|
|
fallback_to_rules?: boolean;
|
|
check_robots_txt?: boolean;
|
|
respect_robots_txt?: boolean | null;
|
|
}
|
|
|
|
export const TERMINAL_CRAWL_STATUSES = new Set([
|
|
"completed",
|
|
"failed",
|
|
"canceled",
|
|
]);
|
|
|
|
export function isCrawlTerminal(status: string): boolean {
|
|
return TERMINAL_CRAWL_STATUSES.has(status);
|
|
}
|
|
|
|
export const extractorModelsResponseSchema = z.object({
|
|
ok: z.boolean(),
|
|
error: z.string().optional(),
|
|
models: z
|
|
.array(
|
|
z
|
|
.object({
|
|
id: z.string(),
|
|
owned_by: z.string().nullish(),
|
|
})
|
|
.passthrough(),
|
|
)
|
|
.optional()
|
|
.default([]),
|
|
});
|
|
|
|
export type ExtractorModelsResponse = z.infer<
|
|
typeof extractorModelsResponseSchema
|
|
>;
|
|
|
|
export const crawlApi = {
|
|
startByProject: (body: StartSiteCrawlRequest) =>
|
|
apiClient.post("/crawl-site/by-project", crawlJobSchema, body),
|
|
getJob: (jobId: string) =>
|
|
apiClient.get(
|
|
`/crawl-site/jobs/${encodeURIComponent(jobId)}`,
|
|
crawlJobSchema,
|
|
),
|
|
cancel: (jobId: string) =>
|
|
apiClient.post(
|
|
`/crawl-site/jobs/${encodeURIComponent(jobId)}/cancel`,
|
|
crawlJobSchema,
|
|
),
|
|
listExtractorModels: (provider: string, baseUrl?: string | null) =>
|
|
apiClient.post("/extractors/models", extractorModelsResponseSchema, {
|
|
provider,
|
|
base_url: baseUrl || null,
|
|
}),
|
|
};
|