참고소스 수정본

This commit is contained in:
LASTA_DEV01\lasta
2026-05-12 19:40:31 +09:00
parent 0f34a451fc
commit 2e9204243d
8708 changed files with 3259488 additions and 869 deletions

View File

@@ -0,0 +1,301 @@
import FirecrawlApp, { type CrawlParams, type CrawlResponse, type CrawlStatusResponse, type MapResponse, type ScrapeResponse } from '../../../index';
import { v4 as uuidv4 } from 'uuid';
import dotenv from 'dotenv';
import { describe, test, expect } from '@jest/globals';
dotenv.config();
const TEST_API_KEY = process.env.TEST_API_KEY;
const API_URL = process.env.API_URL ?? "https://api.firecrawl.dev";
describe('FirecrawlApp E2E Tests', () => {
test.concurrent('should throw error for no API key only for cloud service', async () => {
if (API_URL.includes('api.firecrawl.dev')) {
// Should throw for cloud service
expect(() => {
new FirecrawlApp({ apiKey: null, apiUrl: API_URL });
}).toThrow("No API key provided");
} else {
// Should not throw for self-hosted
expect(() => {
new FirecrawlApp({ apiKey: null, apiUrl: API_URL });
}).not.toThrow();
}
});
test.concurrent('should throw error for invalid API key on scrape', async () => {
if (API_URL.includes('api.firecrawl.dev')) {
const invalidApp = new FirecrawlApp({ apiKey: "invalid_api_key", apiUrl: API_URL });
await expect(invalidApp.scrapeUrl('https://roastmywebsite.ai')).rejects.toThrow("Unexpected error occurred while trying to scrape URL. Status code: 401");
} else {
const invalidApp = new FirecrawlApp({ apiKey: "invalid_api_key", apiUrl: API_URL });
await expect(invalidApp.scrapeUrl('https://roastmywebsite.ai')).resolves.not.toThrow();
}
});
test.concurrent('should throw error for unsupported URL on scrape', async () => {
const app = new FirecrawlApp({ apiKey: TEST_API_KEY, apiUrl: API_URL });
const unsupportedUrl = "https://facebook.com/fake-test";
await expect(app.scrapeUrl(unsupportedUrl)).rejects.toThrow("do not support this site");
});
test.concurrent('should return successful response for valid scrape', async () => {
const app = new FirecrawlApp({ apiKey: TEST_API_KEY, apiUrl: API_URL });
const response = await app.scrapeUrl('https://roastmywebsite.ai');
if (!response.success) {
throw new Error(response.error);
}
}, 30000); // 30 seconds timeout
test.concurrent('should return successful response with valid API key and options', async () => {
const app = new FirecrawlApp({ apiKey: TEST_API_KEY, apiUrl: API_URL });
const response = await app.scrapeUrl(
'https://roastmywebsite.ai', {
formats: ['markdown', 'html', 'rawHtml', 'screenshot', 'links'],
headers: { "x-key": "test" },
includeTags: ['h1'],
excludeTags: ['h2'],
onlyMainContent: true,
timeout: 30000,
waitFor: 1000
});
if (!response.success) {
throw new Error(response.error);
}
}, 30000); // 30 seconds timeout
test.concurrent('should return successful response with valid API key and screenshot fullPage', async () => {
const app = new FirecrawlApp({ apiKey: TEST_API_KEY, apiUrl: API_URL });
const response = await app.scrapeUrl(
'https://roastmywebsite.ai', {
formats: ['screenshot@fullPage'],
});
if (!response.success) {
throw new Error(response.error);
}
expect(response.screenshot).not.toBeUndefined();
expect(response.screenshot).not.toBeNull();
expect(response.screenshot).toContain("https://");
}, 30000); // 30 seconds timeout
test.concurrent('should return successful response for valid scrape with PDF file', async () => {
const app = new FirecrawlApp({ apiKey: TEST_API_KEY, apiUrl: API_URL });
const response = await app.scrapeUrl('https://arxiv.org/pdf/astro-ph/9301001.pdf');
if (!response.success) {
throw new Error(response.error);
}
expect(response).not.toBeNull();
expect(response?.markdown).toContain('We present spectrophotometric observations of the Broad Line Radio Galaxy');
}, 30000); // 30 seconds timeout
test.concurrent('should return successful response for valid scrape with PDF file without explicit extension', async () => {
const app = new FirecrawlApp({ apiKey: TEST_API_KEY, apiUrl: API_URL });
const response = await app.scrapeUrl('https://arxiv.org/pdf/astro-ph/9301001');
if (!response.success) {
throw new Error(response.error);
}
expect(response).not.toBeNull();
expect(response?.markdown).toContain('We present spectrophotometric observations of the Broad Line Radio Galaxy');
}, 30000); // 30 seconds timeout
test.concurrent('should return successful response for valid scrape with PDF file and parsePDF true', async () => {
const app = new FirecrawlApp({ apiKey: TEST_API_KEY, apiUrl: API_URL });
const response = await app.scrapeUrl('https://arxiv.org/pdf/astro-ph/9301001.pdf', {
parsePDF: true
});
if (!response.success) {
throw new Error(response.error);
}
expect(response).not.toBeNull();
expect(response?.markdown).toContain('We present spectrophotometric observations of the Broad Line Radio Galaxy');
}, 30000); // 30 seconds timeout
test.concurrent('should return successful response for valid scrape with PDF file and parsePDF false', async () => {
const app = new FirecrawlApp({ apiKey: TEST_API_KEY, apiUrl: API_URL });
const response = await app.scrapeUrl('https://arxiv.org/pdf/astro-ph/9301001.pdf', {
parsePDF: false
});
if (!response.success) {
throw new Error(response.error);
}
expect(response).not.toBeNull();
expect(response?.markdown).toMatch(/^[A-Za-z0-9+/]+=*$/);
}, 30000); // 30 seconds timeout
test.concurrent('should throw error for invalid API key on crawl', async () => {
if (API_URL.includes('api.firecrawl.dev')) {
const invalidApp = new FirecrawlApp({ apiKey: "invalid_api_key", apiUrl: API_URL });
await expect(invalidApp.crawlUrl('https://roastmywebsite.ai')).rejects.toThrow("Request failed with status code 401");
} else {
const invalidApp = new FirecrawlApp({ apiKey: "invalid_api_key", apiUrl: API_URL });
await expect(invalidApp.crawlUrl('https://roastmywebsite.ai')).resolves.not.toThrow();
}
});
test.concurrent('should return successful response for crawl and wait for completion', async () => {
const app = new FirecrawlApp({ apiKey: TEST_API_KEY, apiUrl: API_URL });
const response = await app.crawlUrl('https://roastmywebsite.ai', {}, 30) as CrawlStatusResponse;
expect(response).not.toHaveProperty("next"); // wait until done
expect(response.data.length).toBeGreaterThan(0);
if (response.data[0]) {
expect(response.data[0]).toHaveProperty("markdown");
}
}, 60000); // 60 seconds timeout
test.concurrent('should return successful response for crawl with options and wait for completion', async () => {
const app = new FirecrawlApp({ apiKey: TEST_API_KEY, apiUrl: API_URL });
const response = await app.crawlUrl('https://roastmywebsite.ai', {
excludePaths: ['blog/*'],
includePaths: ['/'],
maxDepth: 2,
ignoreSitemap: true,
limit: 10,
allowBackwardLinks: true,
allowExternalLinks: true,
scrapeOptions: {
formats: ['markdown', 'html', 'rawHtml', 'screenshot', 'links'],
headers: { "x-key": "test" },
includeTags: ['h1'],
excludeTags: ['h2'],
onlyMainContent: true,
waitFor: 1000
}
} as CrawlParams, 30) as CrawlStatusResponse;
expect(response).not.toHaveProperty("next");
expect(response.data.length).toBeGreaterThan(0);
if (response.data[0]) {
expect(response.data[0]).toHaveProperty("markdown");
expect(response.data[0]).not.toHaveProperty('content'); // v0
expect(response.data[0]).toHaveProperty("html");
expect(response.data[0]).toHaveProperty("rawHtml");
expect(response.data[0]).toHaveProperty("screenshot");
expect(response.data[0]).toHaveProperty("links");
}
}, 60000); // 60 seconds timeout
test.concurrent('should handle idempotency key for crawl', async () => {
const app = new FirecrawlApp({ apiKey: TEST_API_KEY, apiUrl: API_URL });
const uniqueIdempotencyKey = uuidv4();
const response = await app.asyncCrawlUrl('https://roastmywebsite.ai', {}, uniqueIdempotencyKey) as CrawlResponse;
expect(response).not.toBeNull();
expect(response.id).toBeDefined();
await expect(app.crawlUrl('https://roastmywebsite.ai', {}, 2, uniqueIdempotencyKey)).rejects.toThrow("Request failed with status code 409");
});
test.concurrent('should check crawl status', async () => {
const app = new FirecrawlApp({ apiKey: TEST_API_KEY, apiUrl: API_URL });
const response = await app.asyncCrawlUrl('https://firecrawl.dev', { limit: 20, scrapeOptions: { formats: ['markdown', 'html', 'rawHtml', 'screenshot', 'links'] } } as CrawlParams) as CrawlResponse;
expect(response).not.toBeNull();
expect(response.id).toBeDefined();
let statusResponse = await app.checkCrawlStatus(response.id);
const maxChecks = 15;
let checks = 0;
expect(statusResponse.success).toBe(true);
while ((statusResponse as any).status === 'scraping' && checks < maxChecks) {
await new Promise(resolve => setTimeout(resolve, 5000));
expect(statusResponse).not.toHaveProperty("partial_data"); // v0
expect(statusResponse).not.toHaveProperty("current"); // v0
expect(statusResponse).toHaveProperty("data");
expect(statusResponse).toHaveProperty("total");
expect(statusResponse).toHaveProperty("creditsUsed");
expect(statusResponse).toHaveProperty("expiresAt");
expect(statusResponse).toHaveProperty("status");
expect(statusResponse).toHaveProperty("next");
expect(statusResponse.success).toBe(true);
if (statusResponse.success === true) {
expect(statusResponse.total).toBeGreaterThan(0);
expect(statusResponse.creditsUsed).toBeGreaterThan(0);
expect(statusResponse.expiresAt.getTime()).toBeGreaterThan(Date.now());
expect(statusResponse.status).toBe("scraping");
expect(statusResponse.next).toContain("/v1/crawl/");
}
statusResponse = await app.checkCrawlStatus(response.id) as CrawlStatusResponse;
expect(statusResponse.success).toBe(true);
checks++;
}
expect(statusResponse).not.toBeNull();
expect(statusResponse).toHaveProperty("total");
expect(statusResponse.success).toBe(true);
if (statusResponse.success === true) {
expect(statusResponse.status).toBe("completed");
expect(statusResponse.data.length).toBeGreaterThan(0);
}
}, 60000); // 60 seconds timeout
test.concurrent('should throw error for invalid API key on map', async () => {
if (API_URL.includes('api.firecrawl.dev')) {
const invalidApp = new FirecrawlApp({ apiKey: "invalid_api_key", apiUrl: API_URL });
await expect(invalidApp.mapUrl('https://roastmywebsite.ai')).rejects.toThrow("Request failed with status code 401");
} else {
const invalidApp = new FirecrawlApp({ apiKey: "invalid_api_key", apiUrl: API_URL });
await expect(invalidApp.mapUrl('https://roastmywebsite.ai')).resolves.not.toThrow();
}
});
test.concurrent('should throw error for unsupported URL on map', async () => {
const app = new FirecrawlApp({ apiKey: TEST_API_KEY, apiUrl: API_URL });
const unsupportedUrl = "https://facebook.com/fake-test";
await expect(app.mapUrl(unsupportedUrl)).rejects.toThrow("403");
});
test.concurrent('should return successful response for valid map', async () => {
const app = new FirecrawlApp({ apiKey: TEST_API_KEY, apiUrl: API_URL }); const response = await app.mapUrl('https://roastmywebsite.ai') as MapResponse;
expect(response).not.toBeNull();
expect(response.links?.length).toBeGreaterThan(0);
expect(response.links?.[0]).toContain("https://");
const filteredLinks = response.links?.filter((link: string) => link.includes("roastmywebsite.ai"));
expect(filteredLinks?.length).toBeGreaterThan(0);
}, 30000); // 30 seconds timeout
test('should search with string query', async () => {
const app = new FirecrawlApp({ apiUrl: API_URL, apiKey: TEST_API_KEY });
const response = await app.search("firecrawl");
expect(response.success).toBe(true);
expect(response.data?.length).toBeGreaterThan(0);
expect(response.data?.[0]?.markdown).not.toBeDefined();
expect(response.data?.[0]?.title).toBeDefined();
expect(response.data?.[0]?.description).toBeDefined();
}, 30000); // 30 seconds timeout
test('should search with params object', async () => {
const app = new FirecrawlApp({ apiUrl: API_URL, apiKey: TEST_API_KEY });
const response = await app.search("firecrawl", {
limit: 3,
lang: 'en',
country: 'us',
scrapeOptions: {
formats: ['markdown', 'html', 'links'],
onlyMainContent: true
}
});
expect(response.success).toBe(true);
expect(response.data.length).toBeLessThanOrEqual(3);
for (const doc of response.data) {
expect(doc.markdown).toBeDefined();
expect(doc.html).toBeDefined();
expect(doc.links).toBeDefined();
expect(doc.title).toBeDefined();
expect(doc.description).toBeDefined();
}
}, 30000); // 30 seconds timeout
test('should handle invalid API key for search', async () => {
const app = new FirecrawlApp({ apiUrl: API_URL, apiKey: "invalid_api_key" });
await expect(app.search("test query")).rejects.toThrow("Request failed with status code 401");
});
});

View File

@@ -0,0 +1,94 @@
/**
* E2E tests for v2 batch scrape (translated from Python tests)
*/
import Firecrawl from "../../../index";
import { config } from "dotenv";
import { getIdentity, getApiUrl } from "./utils/idmux";
import { describe, test, expect, beforeAll } from "@jest/globals";
config();
const API_URL = getApiUrl();
let client: Firecrawl;
beforeAll(async () => {
const { apiKey } = await getIdentity({ name: "js-e2e-batch" });
client = new Firecrawl({ apiKey, apiUrl: API_URL });
});
describe("v2.batch e2e", () => {
test("batch scrape minimal (wait)", async () => {
const urls = [
"https://docs.firecrawl.dev",
"https://firecrawl.dev",
];
const job = await client.batchScrape(urls, { options: { formats: ["markdown"] }, pollInterval: 1, timeout: 180 });
expect(["completed", "failed"]).toContain(job.status);
expect(job.completed).toBeGreaterThanOrEqual(0);
expect(job.total).toBeGreaterThanOrEqual(0);
expect(Array.isArray(job.data)).toBe(true);
}, 240_000);
test("batch scrape with wait returns job id for error retrieval", async () => {
const urls = [
"https://docs.firecrawl.dev",
"https://firecrawl.dev",
];
const job = await client.batchScrape(urls, { options: { formats: ["markdown"] }, pollInterval: 1, timeout: 180 });
// Verify job has id field
expect(job.id).toBeDefined();
expect(typeof job.id).toBe("string");
// Verify we can use the id to retrieve errors
const errors = await client.getBatchScrapeErrors(job.id!);
expect(errors).toHaveProperty("errors");
expect(errors).toHaveProperty("robotsBlocked");
expect(Array.isArray(errors.errors)).toBe(true);
expect(Array.isArray(errors.robotsBlocked)).toBe(true);
}, 240_000);
test("start batch minimal and status", async () => {
const urls = ["https://docs.firecrawl.dev", "https://firecrawl.dev"];
const start = await client.startBatchScrape(urls, { options: { formats: ["markdown"] }, ignoreInvalidURLs: true });
expect(typeof start.id).toBe("string");
expect(typeof start.url).toBe("string");
const status = await client.getBatchScrapeStatus(start.id);
expect(["scraping", "completed", "failed", "cancelled"]).toContain(status.status);
expect(status.total).toBeGreaterThanOrEqual(0);
// Verify status includes id field
expect(status.id).toBeDefined();
expect(status.id).toBe(start.id);
}, 120_000);
test("wait batch with all params", async () => {
const urls = ["https://docs.firecrawl.dev", "https://firecrawl.dev"];
const job = await client.batchScrape(urls, {
options: {
formats: [
"markdown",
{ type: "json", prompt: "Extract page title", schema: { type: "object", properties: { title: { type: "string" } }, required: ["title"] } },
{ type: "changeTracking", prompt: "Track changes", modes: ["json"] },
],
onlyMainContent: true,
mobile: false,
},
ignoreInvalidURLs: true,
maxConcurrency: 2,
zeroDataRetention: false,
pollInterval: 1,
timeout: 180,
});
expect(["completed", "failed", "cancelled"]).toContain(job.status);
expect(job.completed).toBeGreaterThanOrEqual(0);
expect(job.total).toBeGreaterThanOrEqual(0);
expect(Array.isArray(job.data)).toBe(true);
}, 300_000);
test("cancel batch", async () => {
const urls = ["https://docs.firecrawl.dev", "https://firecrawl.dev"];
const start = await client.startBatchScrape(urls, { options: { formats: ["markdown"] }, maxConcurrency: 1 });
expect(typeof start.id).toBe("string");
const cancelled = await client.cancelBatchScrape(start.id);
expect(cancelled).toBe(true);
}, 120_000);
});

View File

@@ -0,0 +1,199 @@
/**
* E2E tests for v2 crawl (translated from Python tests)
*/
import Firecrawl from "../../../index";
import { config } from "dotenv";
import { getIdentity, getApiUrl } from "./utils/idmux";
import { describe, test, expect, beforeAll } from "@jest/globals";
config();
const API_URL = getApiUrl();
let client: Firecrawl;
beforeAll(async () => {
const { apiKey } = await getIdentity({ name: "js-e2e-crawl" });
client = new Firecrawl({ apiKey, apiUrl: API_URL });
});
describe("v2.crawl e2e", () => {
test("start crawl minimal request", async () => {
if (!client) throw new Error();
const job = await client.startCrawl("https://docs.firecrawl.dev", { limit: 3 });
expect(typeof job.id).toBe("string");
expect(typeof job.url).toBe("string");
}, 90_000);
test("start crawl with options", async () => {
if (!client) throw new Error();
const job = await client.startCrawl("https://docs.firecrawl.dev", { limit: 5, maxDiscoveryDepth: 2 });
expect(typeof job.id).toBe("string");
expect(typeof job.url).toBe("string");
}, 90_000);
test("start crawl with prompt", async () => {
if (!client) throw new Error();
const job = await client.startCrawl("https://firecrawl.dev", { prompt: "Extract all blog posts", limit: 3 });
expect(typeof job.id).toBe("string");
expect(typeof job.url).toBe("string");
}, 90_000);
test("get crawl status", async () => {
if (!client) throw new Error();
const start = await client.startCrawl("https://docs.firecrawl.dev", { limit: 3 });
const status = await client.getCrawlStatus(start.id);
expect(["scraping", "completed", "failed", "cancelled"]).toContain(status.status);
expect(status.completed).toBeGreaterThanOrEqual(0);
// Verify status includes id field
expect(status.id).toBeDefined();
expect(status.id).toBe(start.id);
// next/expiresAt may be null/undefined depending on state; check shape
expect(Array.isArray(status.data)).toBe(true);
}, 120_000);
test("cancel crawl", async () => {
if (!client) throw new Error();
const start = await client.startCrawl("https://docs.firecrawl.dev", { limit: 3 });
const ok = await client.cancelCrawl(start.id);
expect(ok).toBe(true);
}, 120_000);
test("get crawl errors", async () => {
if (!client) throw new Error();
const start = await client.startCrawl("https://docs.firecrawl.dev", { limit: 3 });
const resp = await client.getCrawlErrors(start.id);
expect(resp).toHaveProperty("errors");
expect(resp).toHaveProperty("robotsBlocked");
expect(Array.isArray(resp.errors)).toBe(true);
expect(Array.isArray(resp.robotsBlocked)).toBe(true);
for (const e of resp.errors) {
expect(typeof e.id === "string" || e.id == null).toBe(true);
expect(typeof e.timestamp === "string" || e.timestamp == null).toBe(true);
expect(typeof e.url === "string" || e.url == null).toBe(true);
expect(typeof e.error === "string" || e.error == null).toBe(true);
}
}, 120_000);
test("get crawl errors with invalid id should throw", async () => {
if (!client) throw new Error();
await expect(client.getCrawlErrors("invalid-job-id-12345")).rejects.toThrow();
}, 60_000);
test("get active crawls", async () => {
if (!client) throw new Error();
const active = await client.getActiveCrawls();
expect(typeof active.success).toBe("boolean");
expect(Array.isArray(active.crawls)).toBe(true);
for (const c of active.crawls) {
expect(typeof c.id).toBe("string");
expect(typeof c.teamId).toBe("string");
expect(typeof c.url).toBe("string");
if (c.options != null) {
expect(typeof c.options === "object").toBe(true);
}
}
}, 90_000);
test("get active crawls with running crawl", async () => {
if (!client) throw new Error();
const start = await client.startCrawl("https://docs.firecrawl.dev", { limit: 5 });
await new Promise(resolve => setTimeout(resolve, 300));
const active = await client.getActiveCrawls();
expect(Array.isArray(active.crawls)).toBe(true);
const ids = active.crawls.map(c => c.id);
expect(ids.includes(start.id)).toBe(true);
await client.cancelCrawl(start.id);
}, 120_000);
test("crawl with wait", async () => {
if (!client) throw new Error();
const job = await client.crawl("https://docs.firecrawl.dev", { limit: 3, maxDiscoveryDepth: 2, pollInterval: 1, timeout: 120 });
expect(["completed", "failed"]).toContain(job.status);
expect(job.completed).toBeGreaterThanOrEqual(0);
expect(job.total).toBeGreaterThanOrEqual(0);
expect(Array.isArray(job.data)).toBe(true);
}, 180_000);
test("crawl with wait returns job id for error retrieval", async () => {
if (!client) throw new Error();
const job = await client.crawl("https://docs.firecrawl.dev", { limit: 3, maxDiscoveryDepth: 2, pollInterval: 1, timeout: 120 });
// Verify job has id field
expect(job.id).toBeDefined();
expect(typeof job.id).toBe("string");
// Verify we can use the id to retrieve errors
const errors = await client.getCrawlErrors(job.id!);
expect(errors).toHaveProperty("errors");
expect(errors).toHaveProperty("robotsBlocked");
expect(Array.isArray(errors.errors)).toBe(true);
expect(Array.isArray(errors.robotsBlocked)).toBe(true);
}, 180_000);
test("crawl with prompt and wait", async () => {
if (!client) throw new Error();
const job = await client.crawl("https://docs.firecrawl.dev", { prompt: "Extract all blog posts", limit: 3, pollInterval: 1, timeout: 120 });
expect(["completed", "failed"]).toContain(job.status);
expect(job.completed).toBeGreaterThanOrEqual(0);
expect(job.total).toBeGreaterThanOrEqual(0);
expect(Array.isArray(job.data)).toBe(true);
}, 180_000);
test("crawl with scrape options", async () => {
if (!client) throw new Error();
const job = await client.startCrawl("https://docs.firecrawl.dev", {
limit: 2,
scrapeOptions: { formats: ["markdown", "links"], onlyMainContent: false, mobile: true },
});
expect(typeof job.id).toBe("string");
}, 120_000);
test("crawl with json format object", async () => {
if (!client) throw new Error();
const job = await client.startCrawl("https://docs.firecrawl.dev", {
limit: 2,
scrapeOptions: { formats: [{ type: "json", prompt: "Extract page title", schema: { type: "object", properties: { title: { type: "string" } }, required: ["title"] } }] },
});
expect(typeof job.id).toBe("string");
}, 120_000);
test("crawl all parameters", async () => {
if (!client) throw new Error();
const job = await client.startCrawl("https://docs.firecrawl.dev", {
prompt: "Extract all blog posts and documentation",
includePaths: ["/blog/*", "/docs/*"],
excludePaths: ["/admin/*"],
maxDiscoveryDepth: 3,
sitemap: "skip",
ignoreQueryParameters: true,
limit: 5,
crawlEntireDomain: true,
allowExternalLinks: false,
allowSubdomains: true,
delay: 1,
maxConcurrency: 2,
webhook: "https://example.com/hook",
scrapeOptions: {
formats: ["markdown", "html"],
headers: { "User-Agent": "Test Bot" },
includeTags: ["h1", "h2"],
excludeTags: ["nav"],
onlyMainContent: false,
timeout: 15_000,
waitFor: 2000,
mobile: true,
skipTlsVerification: true,
removeBase64Images: false,
},
zeroDataRetention: false,
});
expect(typeof job.id).toBe("string");
}, 180_000);
test("crawl params preview", async () => {
if (!client) throw new Error();
const params = await client.crawlParamsPreview("https://docs.firecrawl.dev", "Extract all blog posts and documentation");
expect(params && typeof params === "object").toBe(true);
// Optional fields may or may not be present; just assert object shape
}, 60_000);
});

View File

@@ -0,0 +1,70 @@
/**
* E2E tests for v2 extract (proxied to v1), translated from Python tests
*/
import Firecrawl from "../../../index";
import { config } from "dotenv";
import { getIdentity, getApiUrl } from "./utils/idmux";
import { describe, test, expect, beforeAll } from "@jest/globals";
import { z } from "zod";
config();
const API_URL = getApiUrl();
let client: Firecrawl;
beforeAll(async () => {
const { apiKey } = await getIdentity({ name: "js-e2e-extract" });
client = new Firecrawl({ apiKey, apiUrl: API_URL });
});
describe("v2.extract e2e", () => {
test("extract minimal with prompt", async () => {
const resp = await client.extract({ urls: ["https://docs.firecrawl.dev"], prompt: "Extract the main page title" });
expect(typeof resp.success === "boolean" || resp.success == null).toBe(true);
}, 120_000);
test("extract with schema", async () => {
const schema = {
type: "object",
properties: { title: { type: "string" } },
required: ["title"],
} as const;
const resp = await client.extract({
urls: ["https://docs.firecrawl.dev"],
schema,
prompt: "Extract the main page title",
showSources: true,
enableWebSearch: false,
});
expect(typeof resp.success === "boolean" || resp.success == null).toBe(true);
if ((resp as any).sources != null) {
expect(typeof (resp as any).sources).toBe("object");
}
if (resp.data != null) {
expect(typeof resp.data).toBe("object");
expect((resp.data as any).title).toBeTruthy();
}
}, 180_000);
test("extract with zod schema", async () => {
const schema = z.object({
title: z.string(),
});
const resp = await client.extract({
urls: ["https://docs.firecrawl.dev"],
schema: schema,
prompt: "Extract the main page title",
showSources: true,
enableWebSearch: false,
});
expect(typeof resp.success === "boolean" || resp.success == null).toBe(true);
if ((resp as any).sources != null) {
expect(typeof (resp as any).sources).toBe("object");
}
if (resp.data != null) {
expect(typeof resp.data).toBe("object");
expect(schema.safeParse(resp.data).success).toBe(true);
}
}, 180_000);
});

View File

@@ -0,0 +1,55 @@
/**
* E2E tests for v2 map (translated from Python tests)
*/
import Firecrawl from "../../../index";
import { config } from "dotenv";
import { getIdentity, getApiUrl } from "./utils/idmux";
import { describe, test, expect, beforeAll } from "@jest/globals";
config();
const API_URL = getApiUrl();
let client: Firecrawl;
beforeAll(async () => {
const { apiKey } = await getIdentity({ name: "js-e2e-map" });
client = new Firecrawl({ apiKey, apiUrl: API_URL });
});
describe("v2.map e2e", () => {
test("minimal request", async () => {
if (!client) throw new Error();
const resp = await client.map("https://docs.firecrawl.dev");
expect(resp).toBeTruthy();
expect(Array.isArray(resp.links)).toBe(true);
if (resp.links.length > 0) {
const first: any = resp.links[0];
expect(typeof first.url).toBe("string");
expect(first.url.startsWith("http")).toBe(true);
}
}, 90_000);
test.each(["only", "skip", "include"]) ("with options sitemap=%s", async (sitemap) => {
if (!client) throw new Error();
const resp = await client.map("https://docs.firecrawl.dev", {
search: "docs",
includeSubdomains: true,
limit: 10,
sitemap: sitemap as any,
timeout: 15_000,
});
expect(resp).toBeTruthy();
expect(Array.isArray(resp.links)).toBe(true);
expect(resp.links.length).toBeLessThanOrEqual(10);
for (const link of resp.links as any[]) {
expect(typeof link.url).toBe("string");
expect(link.url.startsWith("http")).toBe(true);
}
}, 120_000);
});

View File

@@ -0,0 +1,67 @@
import Firecrawl from "../../../index";
import { config } from "dotenv";
import { getIdentity, getApiUrl } from "./utils/idmux";
import { describe, test, expect, beforeAll } from "@jest/globals";
config();
const API_URL = getApiUrl();
let client: Firecrawl;
beforeAll(async () => {
const { apiKey } = await getIdentity({ name: "js-e2e-parse" });
client = new Firecrawl({ apiKey, apiUrl: API_URL });
});
describe("v2.parse e2e", () => {
test(
"parses uploaded HTML files",
async () => {
if (!client) throw new Error();
const doc = await client.parse(
{
data: `
<!DOCTYPE html>
<html>
<body>
<h1>JS SDK Parse E2E</h1>
<p>multipart upload body</p>
</body>
</html>
`,
filename: "parse-e2e.html",
contentType: "text/html",
},
{
formats: ["markdown"],
},
);
expect(doc.markdown).toContain("JS SDK Parse E2E");
expect(doc.metadata?.creditsUsed).toBe(1);
},
60_000,
);
test(
"returns errors for unsupported file types",
async () => {
if (!client) throw new Error();
await expect(
client.parse(
{
data: Buffer.from("image-data"),
filename: "parse-e2e.png",
contentType: "image/png",
},
{
formats: ["markdown"],
},
),
).rejects.toThrow();
},
60_000,
);
});

View File

@@ -0,0 +1,163 @@
/**
* E2E tests for v2 scrape
*/
import Firecrawl from "../../../index";
import { z } from "zod";
import { config } from "dotenv";
import { getIdentity, getApiUrl } from "./utils/idmux";
import { describe, test, expect, beforeAll } from "@jest/globals";
config();
const API_URL = getApiUrl();
let client: Firecrawl;
beforeAll(async () => {
const { apiKey } = await getIdentity({ name: "js-e2e-scrape" });
client = new Firecrawl({ apiKey, apiUrl: API_URL });
});
describe("v2.scrape e2e", () => {
const assertValidDocument = (doc: any) => {
expect(doc).toBeTruthy();
const hasContent = Boolean(doc.markdown?.length) || Boolean(doc.html?.length) || Boolean(doc.rawHtml?.length);
expect(hasContent).toBe(true);
expect(doc.metadata).toBeTruthy();
};
test("minimal: scrape only required params", async () => {
if (!client) throw new Error();
const doc = await client.scrape("https://docs.firecrawl.dev");
assertValidDocument(doc);
}, 60_000);
test("maximal: scrape with all options", async () => {
if (!client) throw new Error();
const doc = await client.scrape("https://docs.firecrawl.dev", {
formats: [
"markdown",
"html",
"rawHtml",
"links",
{ type: "screenshot", fullPage: true, quality: 80, viewport: { width: 1280, height: 800 } },
{
type: "json",
prompt: "Summarize the page and list links",
schema: {
type: "object",
properties: {
summary: { type: "string" },
links: { type: "array", items: { type: "string", format: "uri" } },
},
required: ["summary"],
},
},
],
parsers: ["pdf"],
headers: { "User-Agent": "firecrawl-tests" },
includeTags: ["article"],
excludeTags: ["nav"],
onlyMainContent: true,
waitFor: 1000,
timeout: 30_000,
location: { country: "us", languages: ["en"] },
mobile: false,
skipTlsVerification: false,
removeBase64Images: true,
blockAds: true,
proxy: "auto",
storeInCache: true,
maxAge: 60_000,
});
assertValidDocument(doc);
}, 90_000);
test("json format with zod schema (auto-converted internally)", async () => {
if (!client) throw new Error();
const zodSchema = z.object({
title: z.string().min(1),
items: z.array(z.string().url()).optional(),
});
const doc = await client.scrape("https://docs.firecrawl.dev", {
formats: [
{
type: "json",
prompt: "Extract title and items",
schema: zodSchema,
},
],
});
expect(doc).toBeTruthy();
}, 90_000);
test("summary format returns summary string", async () => {
if (!client) throw new Error();
const doc = await client.scrape("https://firecrawl.dev", { formats: ["summary"] });
expect(typeof doc.summary).toBe("string");
expect((doc.summary || "").length).toBeGreaterThan(10);
}, 90_000);
test.each([
["markdown", "markdown"],
["html", "html"],
["rawHtml", "rawHtml"],
["links", "links"],
["screenshot", "screenshot"],
])("basic format: %s", async (fmt, expectField) => {
if (!client) throw new Error();
const doc = await client.scrape("https://docs.firecrawl.dev", { formats: [fmt as any] });
if (expectField !== "links" && expectField !== "screenshot") {
assertValidDocument(doc);
}
if (expectField === "markdown") expect(doc.markdown).toBeTruthy();
if (expectField === "html") expect(doc.html).toBeTruthy();
if (expectField === "rawHtml") expect(doc.rawHtml).toBeTruthy();
if (expectField === "screenshot") expect(doc.screenshot).toBeTruthy();
if (expectField === "links") {
expect(Array.isArray(doc.links)).toBe(true);
expect((doc.links || []).length).toBeGreaterThan(0);
}
}, 90_000);
test("images format: extract all images from webpage", async () => {
if (!client) throw new Error();
const doc = await client.scrape("https://firecrawl.dev", {
formats: ["images"],
});
expect(doc.images).toBeTruthy();
expect(Array.isArray(doc.images)).toBe(true);
expect(doc.images?.length).toBeGreaterThan(0);
// Should find firecrawl logo/branding images
expect(doc.images?.some(img => img.includes("firecrawl") || img.includes("logo"))).toBe(true);
}, 60_000);
test("images format: works with multiple formats", async () => {
if (!client) throw new Error();
const doc = await client.scrape("https://github.com", {
formats: ["markdown", "links", "images"],
});
expect(doc.markdown).toBeTruthy();
expect(doc.links).toBeTruthy();
expect(doc.images).toBeTruthy();
expect(Array.isArray(doc.images)).toBe(true);
expect(doc.images?.length).toBeGreaterThan(0);
// Images should find things not available in links format
const imageExtensions = ['.jpg', '.jpeg', '.png', '.gif', '.webp', '.svg', '.ico'];
const linkImages = doc.links?.filter(link =>
imageExtensions.some(ext => link.toLowerCase().includes(ext))
) || [];
// Should discover additional images beyond those with obvious extensions
expect(doc.images?.length).toBeGreaterThanOrEqual(linkImages.length);
}, 60_000);
test("invalid url should throw", async () => {
if (!client) throw new Error();
await expect(client.scrape("")).rejects.toThrow("URL cannot be empty");
await expect(client.scrape(" ")).rejects.toThrow("URL cannot be empty");
});
});

View File

@@ -0,0 +1,247 @@
/**
* E2E tests for v2 search (translated from Python tests)
*/
import Firecrawl from "../../../index";
import type { Document, SearchResultWeb, SearchResultNews, SearchResultImages } from "../../../index";
import { config } from "dotenv";
import { getIdentity, getApiUrl } from "./utils/idmux";
import { describe, test, expect, beforeAll } from "@jest/globals";
config();
const API_URL = getApiUrl();
let client: Firecrawl;
beforeAll(async () => {
const { apiKey } = await getIdentity({ name: "js-e2e-search" });
client = new Firecrawl({ apiKey, apiUrl: API_URL });
});
function collectTexts(entries: any[] | undefined): string[] {
const texts: string[] = [];
for (const r of entries || []) {
const title = (r && typeof r === 'object') ? (r.title as unknown as string | undefined) : undefined;
const desc = (r && typeof r === 'object') ? (r.description as unknown as string | undefined) : undefined;
if (title) texts.push(String(title).toLowerCase());
if (desc) texts.push(String(desc).toLowerCase());
}
return texts;
}
function isDocument(entry: Document | SearchResultWeb | SearchResultNews | SearchResultImages | undefined | null): entry is Document {
if (!entry) return false;
const d = entry as Document;
return (
typeof d.markdown === 'string' ||
typeof d.rawHtml === 'string' ||
typeof d.html === 'string' ||
typeof d.links === 'object' ||
typeof d.screenshot === 'string' ||
typeof d.changeTracking === 'object' ||
typeof d.summary === 'string' ||
typeof d.json === 'object'
);
}
describe("v2.search e2e", () => {
test("minimal request", async () => {
if (!client) throw new Error();
const results = await client.search("What is the capital of France?");
expect(results).toBeTruthy();
expect(results).toHaveProperty("web");
expect(results).not.toHaveProperty("news");
expect(results).not.toHaveProperty("images");
expect(results.web).toBeTruthy();
expect((results.web || []).length).toBeGreaterThan(0);
for (const result of results.web || []) {
if (isDocument(result)) {
// documents appear if scraping happens
continue;
}
expect(typeof result.url).toBe("string");
expect(result.url.startsWith("http")).toBe(true);
expect(typeof result.title === "string" || result.title == null).toBe(true);
expect(typeof result.description === "string" || result.description == null).toBe(true);
}
const allText = collectTexts(results.web).join(" ");
expect(allText.includes("paris")).toBe(true);
expect(results.news == null).toBe(true);
expect(results.images == null).toBe(true);
}, 90_000);
test("with sources web+news and limit", async () => {
if (!client) throw new Error();
const results = await client.search("firecrawl", { sources: ["web", "news"], limit: 3 });
expect(results).toBeTruthy();
expect(results.web).toBeTruthy();
expect((results.web || []).length).toBeLessThanOrEqual(3);
if (results.news != null) {
expect((results.news || []).length).toBeLessThanOrEqual(3);
}
expect(results.images == null).toBe(true);
const webTitles = (results.web || [])
.filter((r): r is SearchResultWeb => !isDocument(r))
.map(r => (r.title || "").toString().toLowerCase());
const webDescriptions = (results.web || [])
.filter((r): r is SearchResultWeb => !isDocument(r))
.map(r => (r.description || "").toString().toLowerCase());
const allWebText = (webTitles.concat(webDescriptions)).join(" ");
expect(allWebText.includes("firecrawl")).toBe(true);
}, 90_000);
test("result structure", async () => {
if (!client) throw new Error();
const results = await client.search("test query", { limit: 1 });
if (results.web && results.web.length > 0) {
const result: any = results.web[0];
expect(result).toHaveProperty("url");
expect(result).toHaveProperty("title");
expect(result).toHaveProperty("description");
expect(typeof result.url).toBe("string");
expect(typeof result.title === "string" || result.title == null).toBe(true);
expect(typeof result.description === "string" || result.description == null).toBe(true);
expect(result.url.startsWith("http")).toBe(true);
}
}, 90_000);
test("all parameters (comprehensive)", async () => {
if (!client) throw new Error();
const schema = {
type: "object",
properties: {
title: { type: "string" },
description: { type: "string" },
url: { type: "string" },
},
required: ["title", "description"],
} as const;
const results = await client.search("artificial intelligence", {
sources: [ "web", "news", "images" ],
limit: 3,
tbs: "qdr:m",
location: "US",
ignoreInvalidURLs: true,
timeout: 60_000,
scrapeOptions: {
formats: [
"markdown",
"html",
{ type: "json", prompt: "Extract the title and description from the page", schema },
],
headers: { "User-Agent": "Firecrawl-Test/1.0" },
includeTags: ["h1", "h2", "p"],
excludeTags: ["nav", "footer"],
onlyMainContent: true,
waitFor: 2000,
mobile: false,
skipTlsVerification: false,
removeBase64Images: true,
blockAds: true,
proxy: "basic",
maxAge: 3_600_000,
storeInCache: true,
location: { country: "US", languages: ["en"] },
actions: [{ type: "wait", milliseconds: 1000 }],
},
});
expect(results).toBeTruthy();
expect(results).toHaveProperty("web");
expect(results).toHaveProperty("news");
expect(results).toHaveProperty("images");
expect(results.web).toBeTruthy();
expect((results.web || []).length).toBeLessThanOrEqual(3);
const nonDocEntries = (results.web || []).filter(r => !isDocument(r));
if (nonDocEntries.length > 0) {
const allWebText = collectTexts(nonDocEntries).join(" ");
const aiTerms = ["artificial", "intelligence", "ai", "machine", "learning"];
expect(aiTerms.some(t => allWebText.includes(t))).toBe(true);
}
for (const result of results.web || []) {
if (isDocument(result)) {
expect(Boolean(result.markdown) || Boolean(result.html)).toBe(true);
} else {
expect(typeof result.url).toBe("string");
expect(result.url.startsWith("http")).toBe(true);
}
}
if (results.news != null) {
expect((results.news || []).length).toBeLessThanOrEqual(3);
for (const result of results.news || []) {
if (isDocument(result)) {
expect(Boolean(result.markdown) || Boolean(result.html)).toBe(true);
} else {
expect(typeof result.url).toBe("string");
expect(result.url?.startsWith("http")).toBe(true);
}
}
}
expect(results.images).toBeTruthy();
expect((results.images || []).length).toBeLessThanOrEqual(3);
for (const result of results.images || []) {
if (!isDocument(result)) {
expect(typeof result.url).toBe("string");
expect(result.url?.startsWith("http")).toBe(true);
}
}
}, 120_000);
test("formats flexibility: list vs object", async () => {
if (!client) throw new Error();
const results1 = await client.search("python programming", {
limit: 1,
scrapeOptions: { formats: ["markdown"] },
});
const results2 = await client.search("python programming", {
limit: 1,
scrapeOptions: { formats: ["markdown"] },
});
expect(results1).toBeTruthy();
expect(results2).toBeTruthy();
expect(results1.web).toBeTruthy();
expect(results2.web).toBeTruthy();
}, 90_000);
test("with json format object", async () => {
if (!client) throw new Error();
const jsonSchema = {
type: "object",
properties: { title: { type: "string" } },
required: ["title"],
} as const;
const results = await client.search("site:docs.firecrawl.dev", {
limit: 1,
scrapeOptions: {
formats: [{ type: "json", prompt: "Extract page title", schema: jsonSchema }],
},
});
expect(results).toBeTruthy();
expect(Array.isArray(results.web) || results.web == null).toBe(true);
}, 90_000);
test("with summary format, documents include summary when present", async () => {
if (!client) throw new Error();
const results = await client.search("site:firecrawl.dev", {
limit: 1,
scrapeOptions: { formats: ["summary"] },
});
const docs = (results.web || []).filter(r => isDocument(r)) as Document[];
if (docs.length > 0) {
expect(typeof docs[0].summary).toBe("string");
expect((docs[0].summary || "").length).toBeGreaterThan(5);
}
}, 90_000);
});

View File

@@ -0,0 +1,44 @@
/**
* E2E tests for v2 usage endpoints (translated from Python tests)
*/
import Firecrawl from "../../../index";
import { config } from "dotenv";
import { getIdentity, getApiUrl } from "./utils/idmux";
import { describe, test, expect, beforeAll } from "@jest/globals";
config();
const API_URL = getApiUrl();
let client: Firecrawl;
beforeAll(async () => {
const { apiKey } = await getIdentity({ name: "js-e2e-usage" });
client = new Firecrawl({ apiKey, apiUrl: API_URL });
});
describe("v2.usage e2e", () => {
test("get_concurrency", async () => {
const resp = await client.getConcurrency();
expect(typeof resp.concurrency).toBe("number");
expect(typeof resp.maxConcurrency).toBe("number");
}, 60_000);
test("get_credit_usage", async () => {
const resp = await client.getCreditUsage();
expect(typeof resp.remainingCredits).toBe("number");
}, 60_000);
test("get_token_usage", async () => {
const resp = await client.getTokenUsage();
expect(typeof resp.remainingTokens).toBe("number");
}, 60_000);
test("get_queue_status", async () => {
const resp = await client.getQueueStatus();
expect(typeof resp.jobsInQueue).toBe("number");
expect(typeof resp.activeJobsInQueue).toBe("number");
expect(typeof resp.waitingJobsInQueue).toBe("number");
expect(typeof resp.maxConcurrency).toBe("number");
}, 60_000);
});

View File

@@ -0,0 +1,58 @@
export type IdmuxRequest = {
name?: string;
concurrency?: number;
credits?: number;
tokens?: number;
teamId?: string;
flags?: Record<string, unknown>;
};
export type Identity = {
apiKey: string;
teamId: string;
};
let cachedIdentity: Identity | null = null;
export function getApiUrl(): string {
return process.env.TEST_URL ?? process.env.FIRECRAWL_API_URL ?? "https://api.firecrawl.dev";
}
export async function getIdentity(req: IdmuxRequest = {}): Promise<Identity> {
if (cachedIdentity) return cachedIdentity;
const idmuxUrl = process.env.IDMUX_URL;
if (!idmuxUrl) {
const fallback: Identity = {
apiKey: process.env.TEST_API_KEY ?? process.env.FIRECRAWL_API_KEY ?? "",
teamId: process.env.TEST_TEAM_ID ?? "",
};
cachedIdentity = fallback;
return fallback;
}
const runNumberRaw = process.env.GITHUB_RUN_NUMBER;
const runNumber = runNumberRaw ? Number(runNumberRaw) : 0;
const body = {
refName: process.env.GITHUB_REF_NAME ?? "local",
runNumber: Number.isFinite(runNumber) ? runNumber : 0,
concurrency: req.concurrency ?? 100,
...req,
};
const res = await fetch(`${idmuxUrl}/`, {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify(body),
});
if (!res.ok) {
const text = await res.text();
throw new Error(`idmux request failed: ${res.status} ${text}`);
}
const identity = (await res.json()) as Identity;
cachedIdentity = identity;
return identity;
}

View File

@@ -0,0 +1,96 @@
import Firecrawl from "../../../index";
import { config } from "dotenv";
import { describe, test, expect, beforeAll } from "@jest/globals";
import { getIdentity } from "./utils/idmux";
config();
const API_URL = process.env.FIRECRAWL_API_URL ?? "https://api.firecrawl.dev";
let client: Firecrawl;
beforeAll(async () => {
const { apiKey } = await getIdentity({ name: "js-e2e-watcher" });
client = new Firecrawl({ apiKey, apiUrl: API_URL });
});
describe("v2.watcher e2e", () => {
test("crawl watcher minimal", async () => {
// client is initialized in beforeAll
const start = await client.startCrawl("https://docs.firecrawl.dev", { limit: 3 });
expect(typeof start.id).toBe("string");
const watcher = client.watcher(start.id, { pollInterval: 2 });
let snapshots = 0;
let documents = 0;
watcher.on("snapshot", (snap: any) => {
snapshots += 1;
expect(["scraping", "completed", "failed", "cancelled"]).toContain(snap.status);
expect(typeof snap.completed).toBe("number");
expect(typeof snap.total).toBe("number");
});
watcher.on("document", (_doc: any) => {
documents += 1;
});
const final = await new Promise<any>(async (resolve) => {
watcher.on("done", (payload: any) => {
resolve(payload);
});
watcher.on("error", (err: any) => {
resolve(err);
});
await watcher.start();
});
expect(["completed", "failed", "cancelled"]).toContain(final.status);
expect(Array.isArray(final.data)).toBe(true);
expect(typeof final.id).toBe("string");
expect(snapshots).toBeGreaterThanOrEqual(1);
expect(documents).toBeGreaterThanOrEqual(0);
watcher.close();
}, 240_000);
test("batch watcher with options (kind, pollInterval, timeout)", async () => {
// client is initialized in beforeAll
const urls = [
"https://docs.firecrawl.dev",
"https://firecrawl.dev",
];
const start = await client.startBatchScrape(urls, { options: { formats: ["markdown"] }, ignoreInvalidURLs: true });
expect(typeof start.id).toBe("string");
const watcher = client.watcher(start.id, { kind: "batch", pollInterval: 2, timeout: 180 });
let snapshots = 0;
let gotCompleted = false;
watcher.on("snapshot", (snap: any) => {
snapshots += 1;
if (snap.status === "completed") gotCompleted = true;
expect(["scraping", "completed", "failed", "cancelled"]).toContain(snap.status);
});
const final = await new Promise<any>(async (resolve) => {
watcher.on("done", (payload: any) => {
resolve(payload);
});
watcher.on("error", (err: any) => {
resolve(err);
});
await watcher.start();
});
expect(["completed", "failed", "cancelled"]).toContain(final.status);
expect(Array.isArray(final.data)).toBe(true);
expect(typeof final.id).toBe("string");
expect(snapshots).toBeGreaterThanOrEqual(1);
expect(gotCompleted || final.status !== "completed").toBe(true);
watcher.close();
}, 300_000);
});

View File

@@ -0,0 +1,154 @@
import FirecrawlApp from '../../../index';
import { describe, test, expect, jest, beforeEach, afterEach } from '@jest/globals';
describe('monitorJobStatus retry logic', () => {
let app: FirecrawlApp;
let originalConsoleWarn: typeof console.warn;
beforeEach(() => {
app = new FirecrawlApp({ apiKey: 'test-key', apiUrl: 'https://test.com' });
originalConsoleWarn = console.warn;
console.warn = jest.fn();
});
afterEach(() => {
console.warn = originalConsoleWarn;
jest.clearAllMocks();
});
test('should retry on socket hang up error', async () => {
const socketHangUpError = new Error('socket hang up') as any;
socketHangUpError.code = 'ECONNRESET';
const successResponse = {
status: 200,
data: { status: 'completed', data: [{ url: 'test.com', markdown: 'test' }] }
};
const originalGetRequest = app.getRequest;
let callCount = 0;
app.getRequest = async function(url: string, headers: any) {
callCount++;
if (callCount === 1) {
throw socketHangUpError;
}
return successResponse;
};
const result = await app.monitorJobStatus('test-id', {}, 1);
expect(callCount).toBe(2);
expect(result).toEqual(successResponse.data);
expect(console.warn).toHaveBeenCalledWith(
expect.stringContaining('Network error during job status check (attempt 1/3): socket hang up')
);
});
test('should retry on ETIMEDOUT error', async () => {
const timeoutError = new Error('timeout') as any;
timeoutError.code = 'ETIMEDOUT';
const successResponse = {
status: 200,
data: { status: 'completed', data: [{ url: 'test.com', markdown: 'test' }] }
};
const originalGetRequest = app.getRequest;
let callCount = 0;
app.getRequest = async function(url: string, headers: any) {
callCount++;
if (callCount === 1) {
throw timeoutError;
}
return successResponse;
};
const result = await app.monitorJobStatus('test-id', {}, 1);
expect(callCount).toBe(2);
expect(result).toEqual(successResponse.data);
});
test('should fail after max retries exceeded', async () => {
const socketHangUpError = new Error('socket hang up') as any;
socketHangUpError.code = 'ECONNRESET';
app.getRequest = async function(url: string, headers: any) {
throw socketHangUpError;
};
await expect(app.monitorJobStatus('test-id', {}, 1)).rejects.toThrow('socket hang up');
expect(console.warn).toHaveBeenCalledTimes(3);
}, 15000);
test('should not retry on non-retryable errors', async () => {
const authError = new Error('Unauthorized') as any;
authError.response = { status: 401, data: { error: 'Unauthorized' } };
app.getRequest = async function(url: string, headers: any) {
throw authError;
};
await expect(app.monitorJobStatus('test-id', {}, 1)).rejects.toThrow('Unauthorized');
expect(console.warn).not.toHaveBeenCalled();
});
test('should retry on HTTP timeout status codes', async () => {
const timeoutError = new Error('Request timeout') as any;
timeoutError.response = { status: 408, data: { error: 'Request timeout' } };
const successResponse = {
status: 200,
data: { status: 'completed', data: [{ url: 'test.com', markdown: 'test' }] }
};
const originalGetRequest = app.getRequest;
let callCount = 0;
app.getRequest = async function(url: string, headers: any) {
callCount++;
if (callCount === 1) {
throw timeoutError;
}
return successResponse;
};
const result = await app.monitorJobStatus('test-id', {}, 1);
expect(callCount).toBe(2);
expect(result).toEqual(successResponse.data);
});
test('should use exponential backoff for retries', async () => {
const socketHangUpError = new Error('socket hang up') as any;
socketHangUpError.code = 'ECONNRESET';
const successResponse = {
status: 200,
data: { status: 'completed', data: [{ url: 'test.com', markdown: 'test' }] }
};
const originalGetRequest = app.getRequest;
let callCount = 0;
app.getRequest = async function(url: string, headers: any) {
callCount++;
if (callCount <= 2) {
throw socketHangUpError;
}
return successResponse;
};
const startTime = Date.now();
const result = await app.monitorJobStatus('test-id', {}, 1);
const endTime = Date.now();
expect(callCount).toBe(3);
expect(result).toEqual(successResponse.data);
expect(endTime - startTime).toBeGreaterThan(3000);
});
});

View File

@@ -0,0 +1,80 @@
import { describe, test, expect } from "@jest/globals";
// We need to test the prepareAgentPayload function, but it's not exported.
// Since the function is internal, we'll test the behavior through type checking
// and verify the types are properly exported.
import type { AgentWebhookConfig, AgentWebhookEvent } from "../../../v2/types";
describe("v2 types: Agent webhook types", () => {
test("AgentWebhookConfig accepts string webhook", () => {
// Type check - this should compile without errors
const webhook: string | AgentWebhookConfig = "https://example.com/webhook";
expect(typeof webhook).toBe("string");
});
test("AgentWebhookConfig accepts config object", () => {
const config: AgentWebhookConfig = {
url: "https://example.com/webhook",
headers: { Authorization: "Bearer token" },
events: ["completed", "failed"],
};
expect(config.url).toBe("https://example.com/webhook");
expect(config.headers).toEqual({ Authorization: "Bearer token" });
expect(config.events).toEqual(["completed", "failed"]);
});
test("AgentWebhookConfig accepts minimal config", () => {
const config: AgentWebhookConfig = {
url: "https://example.com/webhook",
};
expect(config.url).toBe("https://example.com/webhook");
expect(config.headers).toBeUndefined();
expect(config.metadata).toBeUndefined();
expect(config.events).toBeUndefined();
});
test("AgentWebhookEvent includes agent-specific events", () => {
const events: AgentWebhookEvent[] = [
"started",
"action",
"completed",
"failed",
"cancelled",
];
expect(events).toContain("action");
expect(events).toContain("cancelled");
expect(events.length).toBe(5);
});
test("AgentWebhookConfig accepts all fields", () => {
const config: AgentWebhookConfig = {
url: "https://example.com/webhook",
headers: {
Authorization: "Bearer token",
"X-Custom-Header": "value",
},
metadata: {
project: "test",
environment: "staging",
},
events: ["started", "action", "completed", "failed", "cancelled"],
};
expect(config.url).toBe("https://example.com/webhook");
expect(Object.keys(config.headers!).length).toBe(2);
expect(config.metadata!.project).toBe("test");
expect(config.events!.length).toBe(5);
});
test("AgentWebhookConfig events are agent-specific (not crawl)", () => {
// Agent has 'action' and 'cancelled', but not 'page'
const config: AgentWebhookConfig = {
url: "https://example.com/webhook",
events: ["action", "cancelled"],
};
expect(config.events).toContain("action");
expect(config.events).toContain("cancelled");
// 'page' is a crawl-specific event, not valid for agent
// This is enforced at the type level
});
});

View File

@@ -0,0 +1,186 @@
import { describe, test, expect, jest } from "@jest/globals";
import { scrape } from "../../../v2/methods/scrape";
describe("JS SDK v2 branding format", () => {
function makeHttp(postImpl: (url: string, data: any) => any) {
return { post: jest.fn(async (u: string, d: any) => postImpl(u, d)) } as any;
}
test("scrape with branding format returns branding data", async () => {
const mockResponse = {
status: 200,
data: {
success: true,
data: {
markdown: "# Example",
branding: {
colorScheme: "light",
colors: {
primary: "#E11D48",
secondary: "#3B82F6",
accent: "#F59E0B"
},
typography: {
fontFamilies: {
primary: "Inter",
heading: "Poppins"
},
fontSizes: {
h1: "2.5rem",
body: "1rem"
}
},
spacing: {
baseUnit: 8
},
components: {
buttonPrimary: {
background: "#E11D48",
textColor: "#FFFFFF",
borderRadius: "0.5rem"
}
}
}
}
}
};
const http = makeHttp(() => mockResponse);
const result = await scrape(http, "https://example.com", { formats: ["branding"] });
expect(result.branding).toBeDefined();
expect(result.branding?.colorScheme).toBe("light");
expect(result.branding?.colors?.primary).toBe("#E11D48");
expect(result.branding?.typography?.fontFamilies?.primary).toBe("Inter");
expect(result.branding?.spacing?.baseUnit).toBe(8);
expect(result.branding?.components?.buttonPrimary?.background).toBe("#E11D48");
});
test("scrape with branding and markdown formats returns both", async () => {
const mockResponse = {
status: 200,
data: {
success: true,
data: {
markdown: "# Example Content",
branding: {
colorScheme: "dark",
colors: {
primary: "#10B981"
},
typography: {
fontFamilies: {
primary: "Roboto"
}
}
}
}
}
};
const http = makeHttp(() => mockResponse);
const result = await scrape(http, "https://example.com", { formats: ["markdown", "branding"] });
expect(result.markdown).toBe("# Example Content");
expect(result.branding).toBeDefined();
expect(result.branding?.colorScheme).toBe("dark");
expect(result.branding?.colors?.primary).toBe("#10B981");
});
test("scrape without branding format does not return branding", async () => {
const mockResponse = {
status: 200,
data: {
success: true,
data: {
markdown: "# Example"
}
}
};
const http = makeHttp(() => mockResponse);
const result = await scrape(http, "https://example.com", { formats: ["markdown"] });
expect(result.markdown).toBe("# Example");
expect(result.branding).toBeUndefined();
});
test("branding format with all nested fields", async () => {
const mockResponse = {
status: 200,
data: {
success: true,
data: {
branding: {
colorScheme: "light",
logo: "https://example.com/logo.png",
fonts: [
{ family: "Inter", weight: 400 },
{ family: "Poppins", weight: 700 }
],
colors: {
primary: "#E11D48",
background: "#FFFFFF"
},
typography: {
fontFamilies: { primary: "Inter" },
fontStacks: { body: ["Inter", "sans-serif"] },
fontSizes: { h1: "2.5rem" },
lineHeights: { body: 1.5 },
fontWeights: { regular: 400 }
},
spacing: {
baseUnit: 8,
padding: { sm: 8, md: 16 }
},
components: {
buttonPrimary: {
background: "#E11D48",
textColor: "#FFFFFF"
}
},
icons: {
style: "outline",
primaryColor: "#E11D48"
},
images: {
logo: "https://example.com/logo.png",
favicon: "https://example.com/favicon.ico"
},
animations: {
transitionDuration: "200ms",
easing: "ease-in-out"
},
layout: {
grid: { columns: 12, maxWidth: "1200px" },
headerHeight: "64px"
},
tone: {
voice: "professional",
emojiUsage: "minimal"
},
personality: {
tone: "professional",
energy: "medium",
targetAudience: "developers"
}
}
}
}
};
const http = makeHttp(() => mockResponse);
const result = await scrape(http, "https://example.com", { formats: ["branding"] });
expect(result.branding).toBeDefined();
expect(result.branding?.logo).toBe("https://example.com/logo.png");
expect(result.branding?.fonts).toHaveLength(2);
expect(result.branding?.typography?.fontStacks?.body).toEqual(["Inter", "sans-serif"]);
expect(result.branding?.spacing?.padding).toEqual({ sm: 8, md: 16 });
expect(result.branding?.icons?.style).toBe("outline");
expect(result.branding?.images?.favicon).toBe("https://example.com/favicon.ico");
expect(result.branding?.animations?.easing).toBe("ease-in-out");
expect(result.branding?.layout?.grid?.columns).toBe(12);
expect(result.branding?.personality?.tone).toBe("professional");
});
});

View File

@@ -0,0 +1,55 @@
import { Firecrawl, type FirecrawlClientOptions } from '../../../index';
describe('Firecrawl v2 Client Options', () => {
it('should accept v2 options including timeoutMs, maxRetries, and backoffFactor', () => {
const options: FirecrawlClientOptions = {
apiKey: 'test-key',
timeoutMs: 300,
maxRetries: 5,
backoffFactor: 0.5,
};
// Should not throw any type errors
const client = new Firecrawl(options);
expect(client).toBeDefined();
expect(client).toBeInstanceOf(Firecrawl);
});
it('should work with minimal options', () => {
const options: FirecrawlClientOptions = {
apiKey: 'test-key',
};
const client = new Firecrawl(options);
expect(client).toBeDefined();
expect(client).toBeInstanceOf(Firecrawl);
});
it('should work with all v2 options', () => {
const options: FirecrawlClientOptions = {
apiKey: 'test-key',
apiUrl: 'https://custom-api.firecrawl.dev',
timeoutMs: 60000,
maxRetries: 3,
backoffFactor: 1.0,
};
const client = new Firecrawl(options);
expect(client).toBeDefined();
expect(client).toBeInstanceOf(Firecrawl);
});
it('should export FirecrawlClientOptions type', () => {
// This test ensures the type is properly exported
const options: FirecrawlClientOptions = {
apiKey: 'test-key',
timeoutMs: 300,
};
expect(options.timeoutMs).toBe(300);
expect(options.apiKey).toBe('test-key');
});
});

View File

@@ -0,0 +1,19 @@
import { describe, test, expect } from "@jest/globals";
import { throwForBadResponse, normalizeAxiosError } from "../../../v2/utils/errorHandler";
describe("v2 utils: errorHandler", () => {
test("throwForBadResponse: throws SdkError with message from body.error", () => {
const resp: any = { status: 400, data: { error: "bad" } };
expect(() => throwForBadResponse(resp, "do thing")).toThrow(/bad/);
});
test("normalizeAxiosError: prefers body.error then err.message", () => {
const err: any = {
isAxiosError: true,
response: { status: 402, data: { error: "payment required" } },
message: "network",
};
expect(() => normalizeAxiosError(err, "action")).toThrow(/payment required/);
});
});

View File

@@ -0,0 +1,133 @@
import { describe, test, expect, jest } from "@jest/globals";
import { getCrawlStatus } from "../../../v2/methods/crawl";
import { getBatchScrapeStatus } from "../../../v2/methods/batch";
import { getMonitorCheck } from "../../../v2/methods/monitor";
describe("JS SDK v2 pagination", () => {
function makeHttp(getImpl: (url: string) => any) {
return { get: jest.fn(async (u: string) => getImpl(u)) } as any;
}
test("crawl: autoPaginate=false returns next", async () => {
const first = { status: 200, data: { success: true, status: "completed", completed: 1, total: 2, next: "https://api/next", data: [{ markdown: "a" }] } };
const http = makeHttp(() => first);
const res = await getCrawlStatus(http, "job1", { autoPaginate: false });
expect(res.data.length).toBe(1);
expect(res.next).toBe("https://api/next");
});
test("crawl: default autoPaginate aggregates and nulls next", async () => {
const first = { status: 200, data: { success: true, status: "completed", completed: 1, total: 3, next: "https://api/n1", data: [{ markdown: "a" }] } };
const second = { status: 200, data: { success: true, next: "https://api/n2", data: [{ markdown: "b" }] } };
const third = { status: 200, data: { success: true, next: null, data: [{ markdown: "c" }] } };
const http = makeHttp((url) => {
if (url.includes("/v2/crawl/")) return first;
if (url.endsWith("n1")) return second;
return third;
});
const res = await getCrawlStatus(http, "job1");
expect(res.data.length).toBe(3);
expect(res.next).toBeNull();
});
test("crawl: respects maxPages and maxResults", async () => {
const first = { status: 200, data: { success: true, status: "completed", completed: 1, total: 10, next: "https://api/n1", data: [{ markdown: "a" }] } };
const page = (n: number) => ({ status: 200, data: { success: true, next: n < 3 ? `https://api/n${n + 1}` : null, data: [{ markdown: `p${n}` }] } });
const http = makeHttp((url) => {
if (url.includes("/v2/crawl/")) return first;
if (url.endsWith("n1")) return page(1);
if (url.endsWith("n2")) return page(2);
return page(3);
});
const res = await getCrawlStatus(http, "job1", { autoPaginate: true, maxPages: 2, maxResults: 2 });
expect(res.data.length).toBe(2);
});
test("batch: default autoPaginate aggregates and nulls next", async () => {
const first = { status: 200, data: { success: true, status: "completed", completed: 1, total: 3, next: "https://api/b1", data: [{ markdown: "a" }] } };
const second = { status: 200, data: { success: true, next: "https://api/b2", data: [{ markdown: "b" }] } };
const third = { status: 200, data: { success: true, next: null, data: [{ markdown: "c" }] } };
const http = makeHttp((url) => {
if (url.includes("/v2/batch/scrape/")) return first;
if (url.endsWith("b1")) return second;
return third;
});
const res = await getBatchScrapeStatus(http, "jobB");
expect(res.data.length).toBe(3);
expect(res.next).toBeNull();
});
test("batch: autoPaginate=false returns next", async () => {
const first = { status: 200, data: { success: true, status: "completed", completed: 1, total: 2, next: "https://api/nextBatch", data: [{ markdown: "a" }] } };
const http = makeHttp(() => first);
const res = await getBatchScrapeStatus(http, "jobB", { autoPaginate: false });
expect(res.data.length).toBe(1);
expect(res.next).toBe("https://api/nextBatch");
});
test("monitor check: default autoPaginate aggregates pages and nulls next", async () => {
const first = { status: 200, data: { success: true, next: "https://api/m1", data: { id: "check1", monitorId: "mon1", status: "completed", trigger: "manual", billingStatus: "confirmed", summary: {}, createdAt: "now", updatedAt: "now", pages: [{ url: "a", status: "changed" }], next: "https://api/m1" } } };
const second = { status: 200, data: { success: true, next: null, data: { pages: [{ url: "b", status: "same" }], next: null } } };
const http = makeHttp((url) => {
if (url.includes("/v2/monitor/")) return first;
return second;
});
const res = await getMonitorCheck(http, "mon1", "check1");
expect(res.pages.length).toBe(2);
expect(res.next).toBeNull();
});
test("monitor check: autoPaginate=false returns next", async () => {
const first = { status: 200, data: { success: true, next: "https://api/m1", data: { id: "check1", monitorId: "mon1", status: "completed", trigger: "manual", billingStatus: "confirmed", summary: {}, createdAt: "now", updatedAt: "now", pages: [{ url: "a", status: "changed" }], next: "https://api/m1" } } };
const http = makeHttp(() => first);
const res = await getMonitorCheck(http, "mon1", "check1", { autoPaginate: false });
expect(res.pages.length).toBe(1);
expect(res.next).toBe("https://api/m1");
});
test("crawl: maxWaitTime stops pagination after first page", async () => {
const first = { status: 200, data: { success: true, status: "completed", completed: 1, total: 5, next: "https://api/n1", data: [{ markdown: "a" }] } };
const p1 = { status: 200, data: { success: true, next: "https://api/n2", data: [{ markdown: "b" }] } };
const http: any = makeHttp((url: string) => {
if (url.includes("/v2/crawl/")) return first;
if (url.endsWith("n1")) return p1;
return { status: 200, data: { success: true, next: null, data: [{ markdown: "c" }] } };
});
const nowSpy = jest.spyOn(Date, "now");
try {
nowSpy
.mockImplementationOnce(() => 0) // started
.mockImplementationOnce(() => 0) // first loop check
.mockImplementationOnce(() => 3000); // second loop check > maxWaitTime
const res = await getCrawlStatus(http, "jobC", { autoPaginate: true, maxWaitTime: 1 });
expect(res.data.length).toBe(2); // initial + first page
expect((http.get as jest.Mock).mock.calls.length).toBe(2); // initial + n1 only
} finally {
nowSpy.mockRestore();
}
});
test("batch: maxWaitTime stops pagination after first page", async () => {
const first = { status: 200, data: { success: true, status: "completed", completed: 1, total: 5, next: "https://api/b1", data: [{ markdown: "a" }] } };
const p1 = { status: 200, data: { success: true, next: "https://api/b2", data: [{ markdown: "b" }] } };
const http: any = makeHttp((url: string) => {
if (url.includes("/v2/batch/scrape/")) return first;
if (url.endsWith("b1")) return p1;
return { status: 200, data: { success: true, next: null, data: [{ markdown: "c" }] } };
});
const nowSpy = jest.spyOn(Date, "now");
try {
nowSpy
.mockImplementationOnce(() => 0) // started
.mockImplementationOnce(() => 0) // first loop check
.mockImplementationOnce(() => 3000); // second loop check > maxWaitTime
const res = await getBatchScrapeStatus(http, "jobB", { autoPaginate: true, maxWaitTime: 1 });
expect(res.data.length).toBe(2);
expect((http.get as jest.Mock).mock.calls.length).toBe(2);
} finally {
nowSpy.mockRestore();
}
});
});

View File

@@ -0,0 +1,58 @@
import { describe, test, expect } from "@jest/globals";
import { FirecrawlClient } from "../../../v2/client";
describe("v2.parse unit", () => {
test("rejects empty filenames before making requests", async () => {
const client = new FirecrawlClient({
apiKey: "test-key",
apiUrl: "https://localhost:3002",
});
await expect(
client.parse(
{
data: "<html><body>test</body></html>",
filename: " ",
contentType: "text/html",
},
{ formats: ["markdown"] },
),
).rejects.toThrow("filename cannot be empty");
});
test("rejects changeTracking format before making requests", async () => {
const client = new FirecrawlClient({
apiKey: "test-key",
apiUrl: "https://localhost:3002",
});
await expect(
client.parse(
{
data: "<html><body>test</body></html>",
filename: "upload.html",
contentType: "text/html",
},
{ formats: ["markdown", { type: "changeTracking" } as any] },
),
).rejects.toThrow("parse does not support changeTracking format");
});
test("rejects lockdown option before making requests", async () => {
const client = new FirecrawlClient({
apiKey: "test-key",
apiUrl: "https://localhost:3002",
});
await expect(
client.parse(
{
data: "<html><body>test</body></html>",
filename: "upload.html",
contentType: "text/html",
},
{ formats: ["markdown"], lockdown: true } as any,
),
).rejects.toThrow("parse does not support cache/index options");
});
});

View File

@@ -0,0 +1,131 @@
import { describe, test, expect, jest } from "@jest/globals";
import { interact, stopInteraction } from "../../../v2/methods/scrape";
import { SdkError } from "../../../v2/types";
describe("JS SDK v2 scrape-browser methods", () => {
test("interact posts to scrape interact endpoint", async () => {
const post = jest.fn(async () => ({
status: 200,
data: {
success: true,
stdout: "ok",
exitCode: 0,
},
}));
const http = { post } as any;
const response = await interact(http, "job-123", {
code: "console.log('ok')",
});
expect(post).toHaveBeenCalledWith(
"/v2/scrape/job-123/interact",
{ code: "console.log('ok')", language: "node" },
{},
);
expect(response.success).toBe(true);
expect(response.exitCode).toBe(0);
});
test("interact with prompt posts prompt to endpoint", async () => {
const post = jest.fn(async () => ({
status: 200,
data: {
success: true,
output: "Clicked the button",
liveViewUrl: "https://live.example.com/view",
interactiveLiveViewUrl: "https://live.example.com/interactive",
stdout: "",
exitCode: 0,
},
}));
const http = { post } as any;
const response = await interact(http, "job-456", {
prompt: "Click the login button",
});
expect(post).toHaveBeenCalledWith(
"/v2/scrape/job-456/interact",
{ prompt: "Click the login button", language: "node" },
{},
);
expect(response.success).toBe(true);
expect(response.output).toBe("Clicked the button");
expect(response.liveViewUrl).toBe("https://live.example.com/view");
expect(response.interactiveLiveViewUrl).toBe(
"https://live.example.com/interactive",
);
});
test("interact throws when neither code nor prompt provided", async () => {
const http = { post: jest.fn() } as any;
await expect(interact(http, "job-123", {})).rejects.toThrow(
"Either 'code' or 'prompt' must be provided",
);
});
test("interact throws on non-200 response", async () => {
const post = jest.fn(async () => ({
status: 400,
data: {
success: false,
error: "Invalid job ID format",
},
}));
const http = { post } as any;
await expect(
interact(http, "bad-id", { code: "console.log('ok')" }),
).rejects.toBeInstanceOf(SdkError);
});
test("stopInteraction calls delete endpoint", async () => {
const del = jest.fn(async () => ({
status: 200,
data: {
success: true,
},
}));
const http = { delete: del } as any;
const response = await stopInteraction(http, "job-123");
expect(del).toHaveBeenCalledWith("/v2/scrape/job-123/interact");
expect(response.success).toBe(true);
});
test("stopInteraction throws on non-200 response", async () => {
const del = jest.fn(async () => ({
status: 404,
data: {
success: false,
error: "Browser session not found.",
},
}));
const http = { delete: del } as any;
await expect(stopInteraction(http, "job-123")).rejects.toBeInstanceOf(
SdkError,
);
});
test("interact converts seconds-based body timeout to ms axios timeout", async () => {
const post = jest.fn(async () => ({
status: 200,
data: { success: true, stdout: "ok", exitCode: 0 },
}));
const http = { post } as any;
await interact(http, "job-123", {
code: "console.log('ok')",
timeout: 150,
});
expect(post).toHaveBeenCalledWith(
"/v2/scrape/job-123/interact",
{ code: "console.log('ok')", language: "node", timeout: 150 },
{ timeoutMs: 155000 },
);
});
});

View File

@@ -0,0 +1,11 @@
/**
* Minimal unit test for v2 scrape (no mocking; sanity check payload path)
*/
import { FirecrawlClient } from "../../../v2/client";
describe("v2.scrape unit", () => {
test("constructor requires apiKey", () => {
expect(() => new FirecrawlClient({ apiKey: "", apiUrl: "https://api.firecrawl.dev" })).toThrow();
});
});

View File

@@ -0,0 +1,105 @@
import { describe, test, expect } from "@jest/globals";
import { ensureValidFormats, ensureValidScrapeOptions } from "../../../v2/utils/validation";
import type { FormatOption } from "../../../v2/types";
import { z } from "zod";
describe("v2 utils: validation", () => {
test("ensureValidFormats: plain 'json' string is invalid", () => {
const formats: FormatOption[] = ["markdown", "json"] as unknown as FormatOption[];
expect(() => ensureValidFormats(formats)).toThrow(/json format must be an object/i);
});
test("ensureValidFormats: json format requires prompt or schema", () => {
// Valid cases - should not throw
const valid1: FormatOption[] = [{ type: "json", prompt: "p" } as any];
const valid2: FormatOption[] = [{ type: "json", schema: {} } as any];
const valid3: FormatOption[] = [{ type: "json", prompt: "p", schema: {} } as any];
expect(() => ensureValidFormats(valid1)).not.toThrow();
expect(() => ensureValidFormats(valid2)).not.toThrow();
expect(() => ensureValidFormats(valid3)).not.toThrow();
// Invalid case - should throw when both are missing
const bad: FormatOption[] = [{ type: "json" } as any];
expect(() => ensureValidFormats(bad)).toThrow(/requires either 'prompt' or 'schema'/i);
});
test("ensureValidFormats: converts zod schema to JSON schema", () => {
const schema = z.object({ title: z.string() });
const formats: FormatOption[] = [
{ type: "json", prompt: "extract", schema } as any,
];
ensureValidFormats(formats);
const jsonFmt = formats[0] as any;
expect(typeof jsonFmt.schema).toBe("object");
expect(jsonFmt.schema?.properties).toBeTruthy();
});
test("ensureValidFormats: screenshot quality must be non-negative number", () => {
const formats: FormatOption[] = [
{ type: "screenshot", quality: -1 } as any,
];
expect(() => ensureValidFormats(formats)).toThrow(/non-negative number/i);
});
test("ensureValidScrapeOptions: validates timeout and waitFor bounds", () => {
expect(() => ensureValidScrapeOptions({ timeout: 0 })).toThrow(/timeout must be positive/i);
expect(() => ensureValidScrapeOptions({ waitFor: -1 })).toThrow(/waitFor must be non-negative/i);
// valid
expect(() => ensureValidScrapeOptions({ timeout: 1000, waitFor: 0 })).not.toThrow();
});
test("ensureValidFormats: accepts screenshot viewport width/height", () => {
const formats: FormatOption[] = [
{ type: "screenshot", viewport: { width: 800, height: 600 } } as any,
];
expect(() => ensureValidFormats(formats)).not.toThrow();
expect((formats[0] as any).viewport).toEqual({ width: 800, height: 600 });
});
test("ensureValidFormats: accepts question, highlights, and deprecated query formats", () => {
const formats: FormatOption[] = [
{ type: "question", question: "What is Firecrawl?" },
{ type: "highlights", query: "What is Firecrawl?" },
{ type: "query", prompt: "What is Firecrawl?", mode: "directQuote" },
];
expect(() => ensureValidFormats(formats)).not.toThrow();
});
test("ensureValidFormats: validates question, highlights, and deprecated query fields", () => {
expect(() =>
ensureValidFormats([{ type: "question", question: "" } as any]),
).toThrow(/question format requires/i);
expect(() =>
ensureValidFormats([{ type: "highlights", query: "" } as any]),
).toThrow(/highlights format requires/i);
expect(() =>
ensureValidFormats([{ type: "query", prompt: "p", mode: "quoted" } as any]),
).toThrow(/query format mode/i);
});
test("ensureValidScrapeOptions: leaves parsers untouched", () => {
const options = { parsers: ["pdf", "images"] as string[] } as any;
const before = [...options.parsers];
expect(() => ensureValidScrapeOptions(options)).not.toThrow();
expect(options.parsers).toEqual(before);
});
test("ensureValidFormats: detects mistaken use of zod schema.shape", () => {
const schema = z.object({ title: z.string(), count: z.number() });
// User mistakenly passes schema.shape instead of schema
const formats: FormatOption[] = [
{ type: "json", prompt: "extract", schema: schema.shape } as any,
];
expect(() => ensureValidFormats(formats)).toThrow(/\.shape property/i);
expect(() => ensureValidFormats(formats)).toThrow(/Pass the Zod schema directly/i);
});
test("ensureValidFormats: detects mistaken use of zod schema.shape in changeTracking", () => {
const schema = z.object({ title: z.string() });
const formats: FormatOption[] = [
{ type: "changeTracking", modes: ["json"], schema: schema.shape } as any,
];
expect(() => ensureValidFormats(formats)).toThrow(/\.shape property/i);
});
});

View File

@@ -0,0 +1,107 @@
import { describe, test, expect } from "@jest/globals";
import { z } from "zod";
import {
isZodSchema,
zodSchemaToJsonSchema,
looksLikeZodShape,
} from "../../../utils/zodSchemaToJson";
describe("zodSchemaToJson utility", () => {
test("isZodSchema detects Zod schemas and rejects non-Zod values", () => {
expect(isZodSchema(z.object({ name: z.string() }))).toBe(true);
expect(isZodSchema(z.string())).toBe(true);
expect(isZodSchema(z.number())).toBe(true);
expect(isZodSchema(z.array(z.string()))).toBe(true);
expect(isZodSchema(z.enum(["A", "B"]))).toBe(true);
expect(isZodSchema(z.union([z.string(), z.number()]))).toBe(true);
expect(isZodSchema(z.string().optional())).toBe(true);
expect(isZodSchema(z.string().nullable())).toBe(true);
expect(isZodSchema(null)).toBe(false);
expect(isZodSchema(undefined)).toBe(false);
expect(isZodSchema({ name: "test" })).toBe(false);
expect(isZodSchema({ type: "object", properties: {} })).toBe(false);
expect(isZodSchema("string")).toBe(false);
expect(isZodSchema(42)).toBe(false);
expect(isZodSchema([1, 2, 3])).toBe(false);
});
test("zodSchemaToJsonSchema converts Zod schemas to JSON Schema", () => {
const simpleSchema = z.object({ name: z.string() });
const simpleResult = zodSchemaToJsonSchema(simpleSchema) as Record<string, unknown>;
expect(simpleResult.type).toBe("object");
expect(simpleResult.properties).toBeDefined();
expect((simpleResult.properties as Record<string, unknown>).name).toBeDefined();
const complexSchema = z.object({
id: z.string().uuid(),
name: z.string().min(1).max(100),
age: z.number().min(0).max(150).optional(),
tags: z.array(z.string()),
status: z.enum(["active", "inactive"]),
metadata: z.object({
createdAt: z.string(),
nested: z.object({ value: z.number() }),
}),
});
const complexResult = zodSchemaToJsonSchema(complexSchema) as Record<string, unknown>;
expect(complexResult.type).toBe("object");
expect(complexResult.properties).toBeDefined();
expect(complexResult.required).toContain("id");
expect(complexResult.required).not.toContain("age");
const enumResult = zodSchemaToJsonSchema(z.enum(["a", "b", "c"])) as Record<string, unknown>;
expect(enumResult.enum).toEqual(["a", "b", "c"]);
const arrayResult = zodSchemaToJsonSchema(z.array(z.number())) as Record<string, unknown>;
expect(arrayResult.type).toBe("array");
expect(arrayResult.items).toBeDefined();
});
test("zodSchemaToJsonSchema passes through non-Zod values unchanged", () => {
const jsonSchema = { type: "object", properties: { name: { type: "string" } } };
expect(zodSchemaToJsonSchema(jsonSchema)).toEqual(jsonSchema);
expect(zodSchemaToJsonSchema(null)).toBe(null);
expect(zodSchemaToJsonSchema(undefined)).toBe(undefined);
expect(zodSchemaToJsonSchema("string")).toBe("string");
expect(zodSchemaToJsonSchema(42)).toBe(42);
expect(zodSchemaToJsonSchema({ foo: "bar" })).toEqual({ foo: "bar" });
});
test("looksLikeZodShape detects .shape property misuse", () => {
const schema = z.object({ title: z.string(), count: z.number() });
expect(looksLikeZodShape(schema.shape)).toBe(true);
expect(looksLikeZodShape(schema)).toBe(false);
expect(looksLikeZodShape(null)).toBe(false);
expect(looksLikeZodShape(undefined)).toBe(false);
expect(looksLikeZodShape({ name: "test" })).toBe(false);
expect(looksLikeZodShape({})).toBe(false);
expect(looksLikeZodShape([1, 2, 3])).toBe(false);
expect(looksLikeZodShape({ type: "object", properties: {} })).toBe(false);
});
test("SDK-like usage: convert Zod schema or pass through JSON schema", () => {
const zodSchema = z.object({
name: z.string(),
email: z.string().email(),
age: z.number().min(0),
});
if (isZodSchema(zodSchema)) {
const result = zodSchemaToJsonSchema(zodSchema) as Record<string, unknown>;
expect(result.type).toBe("object");
expect(result.properties).toBeDefined();
} else {
throw new Error("Should detect Zod schema");
}
const existingJsonSchema = {
type: "object" as const,
properties: { title: { type: "string" as const } },
required: ["title"] as string[],
};
expect(isZodSchema(existingJsonSchema)).toBe(false);
expect(zodSchemaToJsonSchema(existingJsonSchema)).toEqual(existingJsonSchema);
});
});