AI/ontology_platform/web/frontend/src/lib/api/crawl.ts

import { z } from "zod";
import { apiClient } from "./client";

const stringFromAny = z.union([z.string(), z.number()]).transform(String);

export const crawlPageItemSchema = z
  .object({
    url: z.string().optional(),
    status: z.string().optional(),
    page_type: z.string().optional(),
    title: z.string().nullable().optional(),
    extraction_mode: z.string().nullable().optional(),
    effective_extraction_mode: z.string().nullable().optional(),
    llm_skipped: z.boolean().optional(),
    llm_skip_reason: z.string().nullable().optional(),
    fallback_used: z.boolean().optional(),
    agreement_claim_count: z.number().optional(),
    conflict_claim_count: z.number().optional(),
    error: z.string().nullable().optional(),
  })
  .passthrough();

export const crawlExtractionSummarySchema = z
  .object({
    llm_skipped_count: z.number().default(0),
    fallback_count: z.number().default(0),
    conflict_claim_count: z.number().default(0),
    agreement_claim_count: z.number().default(0),
    llm_call_count: z.number().default(0),
  })
  .passthrough();

export const crawlProgressSchema = z
  .object({
    seed_url: z.string().optional(),
    visited_count: z.number().optional(),
    analyzed_count: z.number().optional(),
    queued_count: z.number().optional(),
    skipped_count: z.number().optional(),
    errors: z.array(z.string()).optional(),
    pages: z.array(crawlPageItemSchema).optional(),
    latest_page: crawlPageItemSchema.optional(),
    extraction_summary: crawlExtractionSummarySchema.optional(),
  })
  .passthrough();

export const crawlJobSchema = z.object({
  job_id: stringFromAny,
  status: z.string(),
  url: z.string().nullable().optional(),
  error: z.string().nullable().optional(),
  scheduled_at: z.string().nullable().optional(),
  started_at: z.string().nullable().optional(),
  finished_at: z.string().nullable().optional(),
  progress: crawlProgressSchema.default({}),
  request: z.record(z.string(), z.unknown()).default({}),
});

export type CrawlJob = z.infer<typeof crawlJobSchema>;
export type CrawlProgress = z.infer<typeof crawlProgressSchema>;

export interface StartSiteCrawlRequest {
  project_name: string;
  source_name: string;
  url: string;
  max_depth?: number;
  max_pages?: number;
  same_domain_only?: boolean;
  analyze_page_types?: string[];
  extraction_mode?: "rule_only" | "llm_only" | "hybrid" | "compare";
  extractor_provider?: string;
  extractor_model?: string | null;
  extractor_base_url?: string | null;
  fallback_to_rules?: boolean;
  check_robots_txt?: boolean;
  respect_robots_txt?: boolean | null;
}

export const TERMINAL_CRAWL_STATUSES = new Set([
  "completed",
  "failed",
  "canceled",
]);

export function isCrawlTerminal(status: string): boolean {
  return TERMINAL_CRAWL_STATUSES.has(status);
}

export const extractorModelsResponseSchema = z.object({
  ok: z.boolean(),
  error: z.string().optional(),
  models: z
    .array(
      z
        .object({
          id: z.string(),
          owned_by: z.string().nullish(),
        })
        .passthrough(),
    )
    .optional()
    .default([]),
});

export type ExtractorModelsResponse = z.infer<
  typeof extractorModelsResponseSchema
>;

export const crawlApi = {
  startByProject: (body: StartSiteCrawlRequest) =>
    apiClient.post("/crawl-site/by-project", crawlJobSchema, body),
  getJob: (jobId: string) =>
    apiClient.get(
      `/crawl-site/jobs/${encodeURIComponent(jobId)}`,
      crawlJobSchema,
    ),
  cancel: (jobId: string) =>
    apiClient.post(
      `/crawl-site/jobs/${encodeURIComponent(jobId)}/cancel`,
      crawlJobSchema,
    ),
  listExtractorModels: (provider: string, baseUrl?: string | null) =>
    apiClient.post("/extractors/models", extractorModelsResponseSchema, {
      provider,
      base_url: baseUrl || null,
    }),
};