참고소스 수정본
This commit is contained in:
@@ -0,0 +1,301 @@
|
||||
import FirecrawlApp, { type CrawlParams, type CrawlResponse, type CrawlStatusResponse, type MapResponse, type ScrapeResponse } from '../../../index';
|
||||
import { v4 as uuidv4 } from 'uuid';
|
||||
import dotenv from 'dotenv';
|
||||
import { describe, test, expect } from '@jest/globals';
|
||||
|
||||
dotenv.config();
|
||||
|
||||
const TEST_API_KEY = process.env.TEST_API_KEY;
|
||||
const API_URL = process.env.API_URL ?? "https://api.firecrawl.dev";
|
||||
|
||||
describe('FirecrawlApp E2E Tests', () => {
|
||||
test.concurrent('should throw error for no API key only for cloud service', async () => {
|
||||
if (API_URL.includes('api.firecrawl.dev')) {
|
||||
// Should throw for cloud service
|
||||
expect(() => {
|
||||
new FirecrawlApp({ apiKey: null, apiUrl: API_URL });
|
||||
}).toThrow("No API key provided");
|
||||
} else {
|
||||
// Should not throw for self-hosted
|
||||
expect(() => {
|
||||
new FirecrawlApp({ apiKey: null, apiUrl: API_URL });
|
||||
}).not.toThrow();
|
||||
}
|
||||
});
|
||||
|
||||
test.concurrent('should throw error for invalid API key on scrape', async () => {
|
||||
if (API_URL.includes('api.firecrawl.dev')) {
|
||||
const invalidApp = new FirecrawlApp({ apiKey: "invalid_api_key", apiUrl: API_URL });
|
||||
await expect(invalidApp.scrapeUrl('https://roastmywebsite.ai')).rejects.toThrow("Unexpected error occurred while trying to scrape URL. Status code: 401");
|
||||
} else {
|
||||
const invalidApp = new FirecrawlApp({ apiKey: "invalid_api_key", apiUrl: API_URL });
|
||||
await expect(invalidApp.scrapeUrl('https://roastmywebsite.ai')).resolves.not.toThrow();
|
||||
}
|
||||
});
|
||||
|
||||
test.concurrent('should throw error for unsupported URL on scrape', async () => {
|
||||
const app = new FirecrawlApp({ apiKey: TEST_API_KEY, apiUrl: API_URL });
|
||||
const unsupportedUrl = "https://facebook.com/fake-test";
|
||||
await expect(app.scrapeUrl(unsupportedUrl)).rejects.toThrow("do not support this site");
|
||||
});
|
||||
|
||||
test.concurrent('should return successful response for valid scrape', async () => {
|
||||
const app = new FirecrawlApp({ apiKey: TEST_API_KEY, apiUrl: API_URL });
|
||||
|
||||
const response = await app.scrapeUrl('https://roastmywebsite.ai');
|
||||
if (!response.success) {
|
||||
throw new Error(response.error);
|
||||
}
|
||||
}, 30000); // 30 seconds timeout
|
||||
|
||||
test.concurrent('should return successful response with valid API key and options', async () => {
|
||||
const app = new FirecrawlApp({ apiKey: TEST_API_KEY, apiUrl: API_URL });
|
||||
const response = await app.scrapeUrl(
|
||||
'https://roastmywebsite.ai', {
|
||||
formats: ['markdown', 'html', 'rawHtml', 'screenshot', 'links'],
|
||||
headers: { "x-key": "test" },
|
||||
includeTags: ['h1'],
|
||||
excludeTags: ['h2'],
|
||||
onlyMainContent: true,
|
||||
timeout: 30000,
|
||||
waitFor: 1000
|
||||
});
|
||||
|
||||
if (!response.success) {
|
||||
throw new Error(response.error);
|
||||
}
|
||||
}, 30000); // 30 seconds timeout
|
||||
|
||||
test.concurrent('should return successful response with valid API key and screenshot fullPage', async () => {
|
||||
const app = new FirecrawlApp({ apiKey: TEST_API_KEY, apiUrl: API_URL });
|
||||
const response = await app.scrapeUrl(
|
||||
'https://roastmywebsite.ai', {
|
||||
formats: ['screenshot@fullPage'],
|
||||
});
|
||||
if (!response.success) {
|
||||
throw new Error(response.error);
|
||||
}
|
||||
|
||||
expect(response.screenshot).not.toBeUndefined();
|
||||
expect(response.screenshot).not.toBeNull();
|
||||
expect(response.screenshot).toContain("https://");
|
||||
}, 30000); // 30 seconds timeout
|
||||
|
||||
test.concurrent('should return successful response for valid scrape with PDF file', async () => {
|
||||
const app = new FirecrawlApp({ apiKey: TEST_API_KEY, apiUrl: API_URL });
|
||||
const response = await app.scrapeUrl('https://arxiv.org/pdf/astro-ph/9301001.pdf');
|
||||
if (!response.success) {
|
||||
throw new Error(response.error);
|
||||
}
|
||||
|
||||
expect(response).not.toBeNull();
|
||||
expect(response?.markdown).toContain('We present spectrophotometric observations of the Broad Line Radio Galaxy');
|
||||
}, 30000); // 30 seconds timeout
|
||||
|
||||
test.concurrent('should return successful response for valid scrape with PDF file without explicit extension', async () => {
|
||||
const app = new FirecrawlApp({ apiKey: TEST_API_KEY, apiUrl: API_URL });
|
||||
const response = await app.scrapeUrl('https://arxiv.org/pdf/astro-ph/9301001');
|
||||
if (!response.success) {
|
||||
throw new Error(response.error);
|
||||
}
|
||||
|
||||
expect(response).not.toBeNull();
|
||||
expect(response?.markdown).toContain('We present spectrophotometric observations of the Broad Line Radio Galaxy');
|
||||
}, 30000); // 30 seconds timeout
|
||||
|
||||
test.concurrent('should return successful response for valid scrape with PDF file and parsePDF true', async () => {
|
||||
const app = new FirecrawlApp({ apiKey: TEST_API_KEY, apiUrl: API_URL });
|
||||
const response = await app.scrapeUrl('https://arxiv.org/pdf/astro-ph/9301001.pdf', {
|
||||
parsePDF: true
|
||||
});
|
||||
if (!response.success) {
|
||||
throw new Error(response.error);
|
||||
}
|
||||
|
||||
expect(response).not.toBeNull();
|
||||
expect(response?.markdown).toContain('We present spectrophotometric observations of the Broad Line Radio Galaxy');
|
||||
}, 30000); // 30 seconds timeout
|
||||
|
||||
test.concurrent('should return successful response for valid scrape with PDF file and parsePDF false', async () => {
|
||||
const app = new FirecrawlApp({ apiKey: TEST_API_KEY, apiUrl: API_URL });
|
||||
const response = await app.scrapeUrl('https://arxiv.org/pdf/astro-ph/9301001.pdf', {
|
||||
parsePDF: false
|
||||
});
|
||||
if (!response.success) {
|
||||
throw new Error(response.error);
|
||||
}
|
||||
|
||||
expect(response).not.toBeNull();
|
||||
expect(response?.markdown).toMatch(/^[A-Za-z0-9+/]+=*$/);
|
||||
}, 30000); // 30 seconds timeout
|
||||
|
||||
test.concurrent('should throw error for invalid API key on crawl', async () => {
|
||||
if (API_URL.includes('api.firecrawl.dev')) {
|
||||
const invalidApp = new FirecrawlApp({ apiKey: "invalid_api_key", apiUrl: API_URL });
|
||||
await expect(invalidApp.crawlUrl('https://roastmywebsite.ai')).rejects.toThrow("Request failed with status code 401");
|
||||
} else {
|
||||
const invalidApp = new FirecrawlApp({ apiKey: "invalid_api_key", apiUrl: API_URL });
|
||||
await expect(invalidApp.crawlUrl('https://roastmywebsite.ai')).resolves.not.toThrow();
|
||||
}
|
||||
});
|
||||
|
||||
test.concurrent('should return successful response for crawl and wait for completion', async () => {
|
||||
const app = new FirecrawlApp({ apiKey: TEST_API_KEY, apiUrl: API_URL });
|
||||
const response = await app.crawlUrl('https://roastmywebsite.ai', {}, 30) as CrawlStatusResponse;
|
||||
expect(response).not.toHaveProperty("next"); // wait until done
|
||||
expect(response.data.length).toBeGreaterThan(0);
|
||||
if (response.data[0]) {
|
||||
expect(response.data[0]).toHaveProperty("markdown");
|
||||
}
|
||||
}, 60000); // 60 seconds timeout
|
||||
|
||||
test.concurrent('should return successful response for crawl with options and wait for completion', async () => {
|
||||
const app = new FirecrawlApp({ apiKey: TEST_API_KEY, apiUrl: API_URL });
|
||||
const response = await app.crawlUrl('https://roastmywebsite.ai', {
|
||||
excludePaths: ['blog/*'],
|
||||
includePaths: ['/'],
|
||||
maxDepth: 2,
|
||||
ignoreSitemap: true,
|
||||
limit: 10,
|
||||
allowBackwardLinks: true,
|
||||
allowExternalLinks: true,
|
||||
scrapeOptions: {
|
||||
formats: ['markdown', 'html', 'rawHtml', 'screenshot', 'links'],
|
||||
headers: { "x-key": "test" },
|
||||
includeTags: ['h1'],
|
||||
excludeTags: ['h2'],
|
||||
onlyMainContent: true,
|
||||
waitFor: 1000
|
||||
}
|
||||
} as CrawlParams, 30) as CrawlStatusResponse;
|
||||
expect(response).not.toHaveProperty("next");
|
||||
expect(response.data.length).toBeGreaterThan(0);
|
||||
if (response.data[0]) {
|
||||
expect(response.data[0]).toHaveProperty("markdown");
|
||||
expect(response.data[0]).not.toHaveProperty('content'); // v0
|
||||
expect(response.data[0]).toHaveProperty("html");
|
||||
expect(response.data[0]).toHaveProperty("rawHtml");
|
||||
expect(response.data[0]).toHaveProperty("screenshot");
|
||||
expect(response.data[0]).toHaveProperty("links");
|
||||
}
|
||||
}, 60000); // 60 seconds timeout
|
||||
|
||||
test.concurrent('should handle idempotency key for crawl', async () => {
|
||||
const app = new FirecrawlApp({ apiKey: TEST_API_KEY, apiUrl: API_URL });
|
||||
const uniqueIdempotencyKey = uuidv4();
|
||||
const response = await app.asyncCrawlUrl('https://roastmywebsite.ai', {}, uniqueIdempotencyKey) as CrawlResponse;
|
||||
expect(response).not.toBeNull();
|
||||
expect(response.id).toBeDefined();
|
||||
|
||||
await expect(app.crawlUrl('https://roastmywebsite.ai', {}, 2, uniqueIdempotencyKey)).rejects.toThrow("Request failed with status code 409");
|
||||
});
|
||||
|
||||
test.concurrent('should check crawl status', async () => {
|
||||
const app = new FirecrawlApp({ apiKey: TEST_API_KEY, apiUrl: API_URL });
|
||||
const response = await app.asyncCrawlUrl('https://firecrawl.dev', { limit: 20, scrapeOptions: { formats: ['markdown', 'html', 'rawHtml', 'screenshot', 'links'] } } as CrawlParams) as CrawlResponse;
|
||||
expect(response).not.toBeNull();
|
||||
expect(response.id).toBeDefined();
|
||||
|
||||
let statusResponse = await app.checkCrawlStatus(response.id);
|
||||
const maxChecks = 15;
|
||||
let checks = 0;
|
||||
|
||||
expect(statusResponse.success).toBe(true);
|
||||
while ((statusResponse as any).status === 'scraping' && checks < maxChecks) {
|
||||
await new Promise(resolve => setTimeout(resolve, 5000));
|
||||
expect(statusResponse).not.toHaveProperty("partial_data"); // v0
|
||||
expect(statusResponse).not.toHaveProperty("current"); // v0
|
||||
expect(statusResponse).toHaveProperty("data");
|
||||
expect(statusResponse).toHaveProperty("total");
|
||||
expect(statusResponse).toHaveProperty("creditsUsed");
|
||||
expect(statusResponse).toHaveProperty("expiresAt");
|
||||
expect(statusResponse).toHaveProperty("status");
|
||||
expect(statusResponse).toHaveProperty("next");
|
||||
expect(statusResponse.success).toBe(true);
|
||||
if (statusResponse.success === true) {
|
||||
expect(statusResponse.total).toBeGreaterThan(0);
|
||||
expect(statusResponse.creditsUsed).toBeGreaterThan(0);
|
||||
expect(statusResponse.expiresAt.getTime()).toBeGreaterThan(Date.now());
|
||||
expect(statusResponse.status).toBe("scraping");
|
||||
expect(statusResponse.next).toContain("/v1/crawl/");
|
||||
}
|
||||
statusResponse = await app.checkCrawlStatus(response.id) as CrawlStatusResponse;
|
||||
expect(statusResponse.success).toBe(true);
|
||||
checks++;
|
||||
}
|
||||
|
||||
expect(statusResponse).not.toBeNull();
|
||||
expect(statusResponse).toHaveProperty("total");
|
||||
expect(statusResponse.success).toBe(true);
|
||||
if (statusResponse.success === true) {
|
||||
expect(statusResponse.status).toBe("completed");
|
||||
expect(statusResponse.data.length).toBeGreaterThan(0);
|
||||
}
|
||||
}, 60000); // 60 seconds timeout
|
||||
|
||||
test.concurrent('should throw error for invalid API key on map', async () => {
|
||||
if (API_URL.includes('api.firecrawl.dev')) {
|
||||
const invalidApp = new FirecrawlApp({ apiKey: "invalid_api_key", apiUrl: API_URL });
|
||||
await expect(invalidApp.mapUrl('https://roastmywebsite.ai')).rejects.toThrow("Request failed with status code 401");
|
||||
} else {
|
||||
const invalidApp = new FirecrawlApp({ apiKey: "invalid_api_key", apiUrl: API_URL });
|
||||
await expect(invalidApp.mapUrl('https://roastmywebsite.ai')).resolves.not.toThrow();
|
||||
}
|
||||
});
|
||||
|
||||
test.concurrent('should throw error for unsupported URL on map', async () => {
|
||||
const app = new FirecrawlApp({ apiKey: TEST_API_KEY, apiUrl: API_URL });
|
||||
const unsupportedUrl = "https://facebook.com/fake-test";
|
||||
await expect(app.mapUrl(unsupportedUrl)).rejects.toThrow("403");
|
||||
});
|
||||
|
||||
test.concurrent('should return successful response for valid map', async () => {
|
||||
const app = new FirecrawlApp({ apiKey: TEST_API_KEY, apiUrl: API_URL }); const response = await app.mapUrl('https://roastmywebsite.ai') as MapResponse;
|
||||
expect(response).not.toBeNull();
|
||||
|
||||
expect(response.links?.length).toBeGreaterThan(0);
|
||||
expect(response.links?.[0]).toContain("https://");
|
||||
const filteredLinks = response.links?.filter((link: string) => link.includes("roastmywebsite.ai"));
|
||||
expect(filteredLinks?.length).toBeGreaterThan(0);
|
||||
}, 30000); // 30 seconds timeout
|
||||
|
||||
|
||||
|
||||
test('should search with string query', async () => {
|
||||
const app = new FirecrawlApp({ apiUrl: API_URL, apiKey: TEST_API_KEY });
|
||||
const response = await app.search("firecrawl");
|
||||
expect(response.success).toBe(true);
|
||||
expect(response.data?.length).toBeGreaterThan(0);
|
||||
expect(response.data?.[0]?.markdown).not.toBeDefined();
|
||||
expect(response.data?.[0]?.title).toBeDefined();
|
||||
expect(response.data?.[0]?.description).toBeDefined();
|
||||
}, 30000); // 30 seconds timeout
|
||||
|
||||
test('should search with params object', async () => {
|
||||
const app = new FirecrawlApp({ apiUrl: API_URL, apiKey: TEST_API_KEY });
|
||||
const response = await app.search("firecrawl", {
|
||||
limit: 3,
|
||||
lang: 'en',
|
||||
country: 'us',
|
||||
scrapeOptions: {
|
||||
formats: ['markdown', 'html', 'links'],
|
||||
onlyMainContent: true
|
||||
}
|
||||
});
|
||||
expect(response.success).toBe(true);
|
||||
expect(response.data.length).toBeLessThanOrEqual(3);
|
||||
for (const doc of response.data) {
|
||||
expect(doc.markdown).toBeDefined();
|
||||
expect(doc.html).toBeDefined();
|
||||
expect(doc.links).toBeDefined();
|
||||
expect(doc.title).toBeDefined();
|
||||
expect(doc.description).toBeDefined();
|
||||
}
|
||||
}, 30000); // 30 seconds timeout
|
||||
|
||||
test('should handle invalid API key for search', async () => {
|
||||
const app = new FirecrawlApp({ apiUrl: API_URL, apiKey: "invalid_api_key" });
|
||||
await expect(app.search("test query")).rejects.toThrow("Request failed with status code 401");
|
||||
});
|
||||
|
||||
});
|
||||
@@ -0,0 +1,94 @@
|
||||
/**
|
||||
* E2E tests for v2 batch scrape (translated from Python tests)
|
||||
*/
|
||||
import Firecrawl from "../../../index";
|
||||
import { config } from "dotenv";
|
||||
import { getIdentity, getApiUrl } from "./utils/idmux";
|
||||
import { describe, test, expect, beforeAll } from "@jest/globals";
|
||||
|
||||
config();
|
||||
|
||||
const API_URL = getApiUrl();
|
||||
let client: Firecrawl;
|
||||
|
||||
beforeAll(async () => {
|
||||
const { apiKey } = await getIdentity({ name: "js-e2e-batch" });
|
||||
client = new Firecrawl({ apiKey, apiUrl: API_URL });
|
||||
});
|
||||
|
||||
describe("v2.batch e2e", () => {
|
||||
test("batch scrape minimal (wait)", async () => {
|
||||
const urls = [
|
||||
"https://docs.firecrawl.dev",
|
||||
"https://firecrawl.dev",
|
||||
];
|
||||
const job = await client.batchScrape(urls, { options: { formats: ["markdown"] }, pollInterval: 1, timeout: 180 });
|
||||
expect(["completed", "failed"]).toContain(job.status);
|
||||
expect(job.completed).toBeGreaterThanOrEqual(0);
|
||||
expect(job.total).toBeGreaterThanOrEqual(0);
|
||||
expect(Array.isArray(job.data)).toBe(true);
|
||||
}, 240_000);
|
||||
|
||||
test("batch scrape with wait returns job id for error retrieval", async () => {
|
||||
const urls = [
|
||||
"https://docs.firecrawl.dev",
|
||||
"https://firecrawl.dev",
|
||||
];
|
||||
const job = await client.batchScrape(urls, { options: { formats: ["markdown"] }, pollInterval: 1, timeout: 180 });
|
||||
// Verify job has id field
|
||||
expect(job.id).toBeDefined();
|
||||
expect(typeof job.id).toBe("string");
|
||||
// Verify we can use the id to retrieve errors
|
||||
const errors = await client.getBatchScrapeErrors(job.id!);
|
||||
expect(errors).toHaveProperty("errors");
|
||||
expect(errors).toHaveProperty("robotsBlocked");
|
||||
expect(Array.isArray(errors.errors)).toBe(true);
|
||||
expect(Array.isArray(errors.robotsBlocked)).toBe(true);
|
||||
}, 240_000);
|
||||
|
||||
test("start batch minimal and status", async () => {
|
||||
const urls = ["https://docs.firecrawl.dev", "https://firecrawl.dev"];
|
||||
const start = await client.startBatchScrape(urls, { options: { formats: ["markdown"] }, ignoreInvalidURLs: true });
|
||||
expect(typeof start.id).toBe("string");
|
||||
expect(typeof start.url).toBe("string");
|
||||
const status = await client.getBatchScrapeStatus(start.id);
|
||||
expect(["scraping", "completed", "failed", "cancelled"]).toContain(status.status);
|
||||
expect(status.total).toBeGreaterThanOrEqual(0);
|
||||
// Verify status includes id field
|
||||
expect(status.id).toBeDefined();
|
||||
expect(status.id).toBe(start.id);
|
||||
}, 120_000);
|
||||
|
||||
test("wait batch with all params", async () => {
|
||||
const urls = ["https://docs.firecrawl.dev", "https://firecrawl.dev"];
|
||||
const job = await client.batchScrape(urls, {
|
||||
options: {
|
||||
formats: [
|
||||
"markdown",
|
||||
{ type: "json", prompt: "Extract page title", schema: { type: "object", properties: { title: { type: "string" } }, required: ["title"] } },
|
||||
{ type: "changeTracking", prompt: "Track changes", modes: ["json"] },
|
||||
],
|
||||
onlyMainContent: true,
|
||||
mobile: false,
|
||||
},
|
||||
ignoreInvalidURLs: true,
|
||||
maxConcurrency: 2,
|
||||
zeroDataRetention: false,
|
||||
pollInterval: 1,
|
||||
timeout: 180,
|
||||
});
|
||||
expect(["completed", "failed", "cancelled"]).toContain(job.status);
|
||||
expect(job.completed).toBeGreaterThanOrEqual(0);
|
||||
expect(job.total).toBeGreaterThanOrEqual(0);
|
||||
expect(Array.isArray(job.data)).toBe(true);
|
||||
}, 300_000);
|
||||
|
||||
test("cancel batch", async () => {
|
||||
const urls = ["https://docs.firecrawl.dev", "https://firecrawl.dev"];
|
||||
const start = await client.startBatchScrape(urls, { options: { formats: ["markdown"] }, maxConcurrency: 1 });
|
||||
expect(typeof start.id).toBe("string");
|
||||
const cancelled = await client.cancelBatchScrape(start.id);
|
||||
expect(cancelled).toBe(true);
|
||||
}, 120_000);
|
||||
});
|
||||
|
||||
@@ -0,0 +1,199 @@
|
||||
/**
|
||||
* E2E tests for v2 crawl (translated from Python tests)
|
||||
*/
|
||||
import Firecrawl from "../../../index";
|
||||
import { config } from "dotenv";
|
||||
import { getIdentity, getApiUrl } from "./utils/idmux";
|
||||
import { describe, test, expect, beforeAll } from "@jest/globals";
|
||||
|
||||
config();
|
||||
|
||||
const API_URL = getApiUrl();
|
||||
let client: Firecrawl;
|
||||
|
||||
beforeAll(async () => {
|
||||
const { apiKey } = await getIdentity({ name: "js-e2e-crawl" });
|
||||
client = new Firecrawl({ apiKey, apiUrl: API_URL });
|
||||
});
|
||||
|
||||
describe("v2.crawl e2e", () => {
|
||||
|
||||
test("start crawl minimal request", async () => {
|
||||
if (!client) throw new Error();
|
||||
const job = await client.startCrawl("https://docs.firecrawl.dev", { limit: 3 });
|
||||
expect(typeof job.id).toBe("string");
|
||||
expect(typeof job.url).toBe("string");
|
||||
}, 90_000);
|
||||
|
||||
test("start crawl with options", async () => {
|
||||
if (!client) throw new Error();
|
||||
const job = await client.startCrawl("https://docs.firecrawl.dev", { limit: 5, maxDiscoveryDepth: 2 });
|
||||
expect(typeof job.id).toBe("string");
|
||||
expect(typeof job.url).toBe("string");
|
||||
}, 90_000);
|
||||
|
||||
test("start crawl with prompt", async () => {
|
||||
if (!client) throw new Error();
|
||||
const job = await client.startCrawl("https://firecrawl.dev", { prompt: "Extract all blog posts", limit: 3 });
|
||||
expect(typeof job.id).toBe("string");
|
||||
expect(typeof job.url).toBe("string");
|
||||
}, 90_000);
|
||||
|
||||
test("get crawl status", async () => {
|
||||
if (!client) throw new Error();
|
||||
const start = await client.startCrawl("https://docs.firecrawl.dev", { limit: 3 });
|
||||
const status = await client.getCrawlStatus(start.id);
|
||||
expect(["scraping", "completed", "failed", "cancelled"]).toContain(status.status);
|
||||
expect(status.completed).toBeGreaterThanOrEqual(0);
|
||||
// Verify status includes id field
|
||||
expect(status.id).toBeDefined();
|
||||
expect(status.id).toBe(start.id);
|
||||
// next/expiresAt may be null/undefined depending on state; check shape
|
||||
expect(Array.isArray(status.data)).toBe(true);
|
||||
}, 120_000);
|
||||
|
||||
test("cancel crawl", async () => {
|
||||
if (!client) throw new Error();
|
||||
const start = await client.startCrawl("https://docs.firecrawl.dev", { limit: 3 });
|
||||
const ok = await client.cancelCrawl(start.id);
|
||||
expect(ok).toBe(true);
|
||||
}, 120_000);
|
||||
|
||||
test("get crawl errors", async () => {
|
||||
if (!client) throw new Error();
|
||||
const start = await client.startCrawl("https://docs.firecrawl.dev", { limit: 3 });
|
||||
const resp = await client.getCrawlErrors(start.id);
|
||||
expect(resp).toHaveProperty("errors");
|
||||
expect(resp).toHaveProperty("robotsBlocked");
|
||||
expect(Array.isArray(resp.errors)).toBe(true);
|
||||
expect(Array.isArray(resp.robotsBlocked)).toBe(true);
|
||||
for (const e of resp.errors) {
|
||||
expect(typeof e.id === "string" || e.id == null).toBe(true);
|
||||
expect(typeof e.timestamp === "string" || e.timestamp == null).toBe(true);
|
||||
expect(typeof e.url === "string" || e.url == null).toBe(true);
|
||||
expect(typeof e.error === "string" || e.error == null).toBe(true);
|
||||
}
|
||||
}, 120_000);
|
||||
|
||||
test("get crawl errors with invalid id should throw", async () => {
|
||||
if (!client) throw new Error();
|
||||
await expect(client.getCrawlErrors("invalid-job-id-12345")).rejects.toThrow();
|
||||
}, 60_000);
|
||||
|
||||
test("get active crawls", async () => {
|
||||
if (!client) throw new Error();
|
||||
const active = await client.getActiveCrawls();
|
||||
expect(typeof active.success).toBe("boolean");
|
||||
expect(Array.isArray(active.crawls)).toBe(true);
|
||||
for (const c of active.crawls) {
|
||||
expect(typeof c.id).toBe("string");
|
||||
expect(typeof c.teamId).toBe("string");
|
||||
expect(typeof c.url).toBe("string");
|
||||
if (c.options != null) {
|
||||
expect(typeof c.options === "object").toBe(true);
|
||||
}
|
||||
}
|
||||
}, 90_000);
|
||||
|
||||
test("get active crawls with running crawl", async () => {
|
||||
if (!client) throw new Error();
|
||||
const start = await client.startCrawl("https://docs.firecrawl.dev", { limit: 5 });
|
||||
await new Promise(resolve => setTimeout(resolve, 300));
|
||||
const active = await client.getActiveCrawls();
|
||||
expect(Array.isArray(active.crawls)).toBe(true);
|
||||
const ids = active.crawls.map(c => c.id);
|
||||
expect(ids.includes(start.id)).toBe(true);
|
||||
await client.cancelCrawl(start.id);
|
||||
}, 120_000);
|
||||
|
||||
test("crawl with wait", async () => {
|
||||
if (!client) throw new Error();
|
||||
const job = await client.crawl("https://docs.firecrawl.dev", { limit: 3, maxDiscoveryDepth: 2, pollInterval: 1, timeout: 120 });
|
||||
expect(["completed", "failed"]).toContain(job.status);
|
||||
expect(job.completed).toBeGreaterThanOrEqual(0);
|
||||
expect(job.total).toBeGreaterThanOrEqual(0);
|
||||
expect(Array.isArray(job.data)).toBe(true);
|
||||
}, 180_000);
|
||||
|
||||
test("crawl with wait returns job id for error retrieval", async () => {
|
||||
if (!client) throw new Error();
|
||||
const job = await client.crawl("https://docs.firecrawl.dev", { limit: 3, maxDiscoveryDepth: 2, pollInterval: 1, timeout: 120 });
|
||||
// Verify job has id field
|
||||
expect(job.id).toBeDefined();
|
||||
expect(typeof job.id).toBe("string");
|
||||
// Verify we can use the id to retrieve errors
|
||||
const errors = await client.getCrawlErrors(job.id!);
|
||||
expect(errors).toHaveProperty("errors");
|
||||
expect(errors).toHaveProperty("robotsBlocked");
|
||||
expect(Array.isArray(errors.errors)).toBe(true);
|
||||
expect(Array.isArray(errors.robotsBlocked)).toBe(true);
|
||||
}, 180_000);
|
||||
|
||||
test("crawl with prompt and wait", async () => {
|
||||
if (!client) throw new Error();
|
||||
const job = await client.crawl("https://docs.firecrawl.dev", { prompt: "Extract all blog posts", limit: 3, pollInterval: 1, timeout: 120 });
|
||||
expect(["completed", "failed"]).toContain(job.status);
|
||||
expect(job.completed).toBeGreaterThanOrEqual(0);
|
||||
expect(job.total).toBeGreaterThanOrEqual(0);
|
||||
expect(Array.isArray(job.data)).toBe(true);
|
||||
}, 180_000);
|
||||
|
||||
test("crawl with scrape options", async () => {
|
||||
if (!client) throw new Error();
|
||||
const job = await client.startCrawl("https://docs.firecrawl.dev", {
|
||||
limit: 2,
|
||||
scrapeOptions: { formats: ["markdown", "links"], onlyMainContent: false, mobile: true },
|
||||
});
|
||||
expect(typeof job.id).toBe("string");
|
||||
}, 120_000);
|
||||
|
||||
test("crawl with json format object", async () => {
|
||||
if (!client) throw new Error();
|
||||
const job = await client.startCrawl("https://docs.firecrawl.dev", {
|
||||
limit: 2,
|
||||
scrapeOptions: { formats: [{ type: "json", prompt: "Extract page title", schema: { type: "object", properties: { title: { type: "string" } }, required: ["title"] } }] },
|
||||
});
|
||||
expect(typeof job.id).toBe("string");
|
||||
}, 120_000);
|
||||
|
||||
test("crawl all parameters", async () => {
|
||||
if (!client) throw new Error();
|
||||
const job = await client.startCrawl("https://docs.firecrawl.dev", {
|
||||
prompt: "Extract all blog posts and documentation",
|
||||
includePaths: ["/blog/*", "/docs/*"],
|
||||
excludePaths: ["/admin/*"],
|
||||
maxDiscoveryDepth: 3,
|
||||
sitemap: "skip",
|
||||
ignoreQueryParameters: true,
|
||||
limit: 5,
|
||||
crawlEntireDomain: true,
|
||||
allowExternalLinks: false,
|
||||
allowSubdomains: true,
|
||||
delay: 1,
|
||||
maxConcurrency: 2,
|
||||
webhook: "https://example.com/hook",
|
||||
scrapeOptions: {
|
||||
formats: ["markdown", "html"],
|
||||
headers: { "User-Agent": "Test Bot" },
|
||||
includeTags: ["h1", "h2"],
|
||||
excludeTags: ["nav"],
|
||||
onlyMainContent: false,
|
||||
timeout: 15_000,
|
||||
waitFor: 2000,
|
||||
mobile: true,
|
||||
skipTlsVerification: true,
|
||||
removeBase64Images: false,
|
||||
},
|
||||
zeroDataRetention: false,
|
||||
});
|
||||
expect(typeof job.id).toBe("string");
|
||||
}, 180_000);
|
||||
|
||||
test("crawl params preview", async () => {
|
||||
if (!client) throw new Error();
|
||||
const params = await client.crawlParamsPreview("https://docs.firecrawl.dev", "Extract all blog posts and documentation");
|
||||
expect(params && typeof params === "object").toBe(true);
|
||||
// Optional fields may or may not be present; just assert object shape
|
||||
}, 60_000);
|
||||
});
|
||||
|
||||
@@ -0,0 +1,70 @@
|
||||
/**
|
||||
* E2E tests for v2 extract (proxied to v1), translated from Python tests
|
||||
*/
|
||||
import Firecrawl from "../../../index";
|
||||
import { config } from "dotenv";
|
||||
import { getIdentity, getApiUrl } from "./utils/idmux";
|
||||
import { describe, test, expect, beforeAll } from "@jest/globals";
|
||||
import { z } from "zod";
|
||||
|
||||
config();
|
||||
|
||||
const API_URL = getApiUrl();
|
||||
let client: Firecrawl;
|
||||
|
||||
beforeAll(async () => {
|
||||
const { apiKey } = await getIdentity({ name: "js-e2e-extract" });
|
||||
client = new Firecrawl({ apiKey, apiUrl: API_URL });
|
||||
});
|
||||
|
||||
describe("v2.extract e2e", () => {
|
||||
test("extract minimal with prompt", async () => {
|
||||
const resp = await client.extract({ urls: ["https://docs.firecrawl.dev"], prompt: "Extract the main page title" });
|
||||
expect(typeof resp.success === "boolean" || resp.success == null).toBe(true);
|
||||
}, 120_000);
|
||||
|
||||
test("extract with schema", async () => {
|
||||
const schema = {
|
||||
type: "object",
|
||||
properties: { title: { type: "string" } },
|
||||
required: ["title"],
|
||||
} as const;
|
||||
const resp = await client.extract({
|
||||
urls: ["https://docs.firecrawl.dev"],
|
||||
schema,
|
||||
prompt: "Extract the main page title",
|
||||
showSources: true,
|
||||
enableWebSearch: false,
|
||||
});
|
||||
expect(typeof resp.success === "boolean" || resp.success == null).toBe(true);
|
||||
if ((resp as any).sources != null) {
|
||||
expect(typeof (resp as any).sources).toBe("object");
|
||||
}
|
||||
if (resp.data != null) {
|
||||
expect(typeof resp.data).toBe("object");
|
||||
expect((resp.data as any).title).toBeTruthy();
|
||||
}
|
||||
}, 180_000);
|
||||
|
||||
test("extract with zod schema", async () => {
|
||||
const schema = z.object({
|
||||
title: z.string(),
|
||||
});
|
||||
const resp = await client.extract({
|
||||
urls: ["https://docs.firecrawl.dev"],
|
||||
schema: schema,
|
||||
prompt: "Extract the main page title",
|
||||
showSources: true,
|
||||
enableWebSearch: false,
|
||||
});
|
||||
expect(typeof resp.success === "boolean" || resp.success == null).toBe(true);
|
||||
if ((resp as any).sources != null) {
|
||||
expect(typeof (resp as any).sources).toBe("object");
|
||||
}
|
||||
if (resp.data != null) {
|
||||
expect(typeof resp.data).toBe("object");
|
||||
expect(schema.safeParse(resp.data).success).toBe(true);
|
||||
}
|
||||
}, 180_000);
|
||||
});
|
||||
|
||||
@@ -0,0 +1,55 @@
|
||||
/**
|
||||
* E2E tests for v2 map (translated from Python tests)
|
||||
*/
|
||||
import Firecrawl from "../../../index";
|
||||
import { config } from "dotenv";
|
||||
import { getIdentity, getApiUrl } from "./utils/idmux";
|
||||
import { describe, test, expect, beforeAll } from "@jest/globals";
|
||||
|
||||
config();
|
||||
|
||||
const API_URL = getApiUrl();
|
||||
let client: Firecrawl;
|
||||
|
||||
beforeAll(async () => {
|
||||
const { apiKey } = await getIdentity({ name: "js-e2e-map" });
|
||||
client = new Firecrawl({ apiKey, apiUrl: API_URL });
|
||||
});
|
||||
|
||||
describe("v2.map e2e", () => {
|
||||
|
||||
test("minimal request", async () => {
|
||||
if (!client) throw new Error();
|
||||
const resp = await client.map("https://docs.firecrawl.dev");
|
||||
|
||||
expect(resp).toBeTruthy();
|
||||
expect(Array.isArray(resp.links)).toBe(true);
|
||||
|
||||
if (resp.links.length > 0) {
|
||||
const first: any = resp.links[0];
|
||||
expect(typeof first.url).toBe("string");
|
||||
expect(first.url.startsWith("http")).toBe(true);
|
||||
}
|
||||
}, 90_000);
|
||||
|
||||
test.each(["only", "skip", "include"]) ("with options sitemap=%s", async (sitemap) => {
|
||||
if (!client) throw new Error();
|
||||
const resp = await client.map("https://docs.firecrawl.dev", {
|
||||
search: "docs",
|
||||
includeSubdomains: true,
|
||||
limit: 10,
|
||||
sitemap: sitemap as any,
|
||||
timeout: 15_000,
|
||||
});
|
||||
|
||||
expect(resp).toBeTruthy();
|
||||
expect(Array.isArray(resp.links)).toBe(true);
|
||||
expect(resp.links.length).toBeLessThanOrEqual(10);
|
||||
|
||||
for (const link of resp.links as any[]) {
|
||||
expect(typeof link.url).toBe("string");
|
||||
expect(link.url.startsWith("http")).toBe(true);
|
||||
}
|
||||
}, 120_000);
|
||||
});
|
||||
|
||||
@@ -0,0 +1,67 @@
|
||||
import Firecrawl from "../../../index";
|
||||
import { config } from "dotenv";
|
||||
import { getIdentity, getApiUrl } from "./utils/idmux";
|
||||
import { describe, test, expect, beforeAll } from "@jest/globals";
|
||||
|
||||
config();
|
||||
|
||||
const API_URL = getApiUrl();
|
||||
let client: Firecrawl;
|
||||
|
||||
beforeAll(async () => {
|
||||
const { apiKey } = await getIdentity({ name: "js-e2e-parse" });
|
||||
client = new Firecrawl({ apiKey, apiUrl: API_URL });
|
||||
});
|
||||
|
||||
describe("v2.parse e2e", () => {
|
||||
test(
|
||||
"parses uploaded HTML files",
|
||||
async () => {
|
||||
if (!client) throw new Error();
|
||||
|
||||
const doc = await client.parse(
|
||||
{
|
||||
data: `
|
||||
<!DOCTYPE html>
|
||||
<html>
|
||||
<body>
|
||||
<h1>JS SDK Parse E2E</h1>
|
||||
<p>multipart upload body</p>
|
||||
</body>
|
||||
</html>
|
||||
`,
|
||||
filename: "parse-e2e.html",
|
||||
contentType: "text/html",
|
||||
},
|
||||
{
|
||||
formats: ["markdown"],
|
||||
},
|
||||
);
|
||||
|
||||
expect(doc.markdown).toContain("JS SDK Parse E2E");
|
||||
expect(doc.metadata?.creditsUsed).toBe(1);
|
||||
},
|
||||
60_000,
|
||||
);
|
||||
|
||||
test(
|
||||
"returns errors for unsupported file types",
|
||||
async () => {
|
||||
if (!client) throw new Error();
|
||||
|
||||
await expect(
|
||||
client.parse(
|
||||
{
|
||||
data: Buffer.from("image-data"),
|
||||
filename: "parse-e2e.png",
|
||||
contentType: "image/png",
|
||||
},
|
||||
{
|
||||
formats: ["markdown"],
|
||||
},
|
||||
),
|
||||
).rejects.toThrow();
|
||||
},
|
||||
60_000,
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,163 @@
|
||||
/**
|
||||
* E2E tests for v2 scrape
|
||||
*/
|
||||
import Firecrawl from "../../../index";
|
||||
import { z } from "zod";
|
||||
import { config } from "dotenv";
|
||||
import { getIdentity, getApiUrl } from "./utils/idmux";
|
||||
import { describe, test, expect, beforeAll } from "@jest/globals";
|
||||
|
||||
config();
|
||||
|
||||
const API_URL = getApiUrl();
|
||||
let client: Firecrawl;
|
||||
|
||||
beforeAll(async () => {
|
||||
const { apiKey } = await getIdentity({ name: "js-e2e-scrape" });
|
||||
client = new Firecrawl({ apiKey, apiUrl: API_URL });
|
||||
});
|
||||
|
||||
describe("v2.scrape e2e", () => {
|
||||
|
||||
const assertValidDocument = (doc: any) => {
|
||||
expect(doc).toBeTruthy();
|
||||
const hasContent = Boolean(doc.markdown?.length) || Boolean(doc.html?.length) || Boolean(doc.rawHtml?.length);
|
||||
expect(hasContent).toBe(true);
|
||||
expect(doc.metadata).toBeTruthy();
|
||||
};
|
||||
|
||||
test("minimal: scrape only required params", async () => {
|
||||
if (!client) throw new Error();
|
||||
const doc = await client.scrape("https://docs.firecrawl.dev");
|
||||
assertValidDocument(doc);
|
||||
}, 60_000);
|
||||
|
||||
test("maximal: scrape with all options", async () => {
|
||||
if (!client) throw new Error();
|
||||
const doc = await client.scrape("https://docs.firecrawl.dev", {
|
||||
formats: [
|
||||
"markdown",
|
||||
"html",
|
||||
"rawHtml",
|
||||
"links",
|
||||
{ type: "screenshot", fullPage: true, quality: 80, viewport: { width: 1280, height: 800 } },
|
||||
{
|
||||
type: "json",
|
||||
prompt: "Summarize the page and list links",
|
||||
schema: {
|
||||
type: "object",
|
||||
properties: {
|
||||
summary: { type: "string" },
|
||||
links: { type: "array", items: { type: "string", format: "uri" } },
|
||||
},
|
||||
required: ["summary"],
|
||||
},
|
||||
},
|
||||
],
|
||||
parsers: ["pdf"],
|
||||
headers: { "User-Agent": "firecrawl-tests" },
|
||||
includeTags: ["article"],
|
||||
excludeTags: ["nav"],
|
||||
onlyMainContent: true,
|
||||
waitFor: 1000,
|
||||
timeout: 30_000,
|
||||
location: { country: "us", languages: ["en"] },
|
||||
mobile: false,
|
||||
skipTlsVerification: false,
|
||||
removeBase64Images: true,
|
||||
blockAds: true,
|
||||
proxy: "auto",
|
||||
storeInCache: true,
|
||||
maxAge: 60_000,
|
||||
});
|
||||
assertValidDocument(doc);
|
||||
}, 90_000);
|
||||
|
||||
test("json format with zod schema (auto-converted internally)", async () => {
|
||||
if (!client) throw new Error();
|
||||
const zodSchema = z.object({
|
||||
title: z.string().min(1),
|
||||
items: z.array(z.string().url()).optional(),
|
||||
});
|
||||
|
||||
const doc = await client.scrape("https://docs.firecrawl.dev", {
|
||||
formats: [
|
||||
{
|
||||
type: "json",
|
||||
prompt: "Extract title and items",
|
||||
schema: zodSchema,
|
||||
},
|
||||
],
|
||||
});
|
||||
expect(doc).toBeTruthy();
|
||||
}, 90_000);
|
||||
|
||||
test("summary format returns summary string", async () => {
|
||||
if (!client) throw new Error();
|
||||
const doc = await client.scrape("https://firecrawl.dev", { formats: ["summary"] });
|
||||
expect(typeof doc.summary).toBe("string");
|
||||
expect((doc.summary || "").length).toBeGreaterThan(10);
|
||||
}, 90_000);
|
||||
|
||||
test.each([
|
||||
["markdown", "markdown"],
|
||||
["html", "html"],
|
||||
["rawHtml", "rawHtml"],
|
||||
["links", "links"],
|
||||
["screenshot", "screenshot"],
|
||||
])("basic format: %s", async (fmt, expectField) => {
|
||||
if (!client) throw new Error();
|
||||
const doc = await client.scrape("https://docs.firecrawl.dev", { formats: [fmt as any] });
|
||||
if (expectField !== "links" && expectField !== "screenshot") {
|
||||
assertValidDocument(doc);
|
||||
}
|
||||
if (expectField === "markdown") expect(doc.markdown).toBeTruthy();
|
||||
if (expectField === "html") expect(doc.html).toBeTruthy();
|
||||
if (expectField === "rawHtml") expect(doc.rawHtml).toBeTruthy();
|
||||
if (expectField === "screenshot") expect(doc.screenshot).toBeTruthy();
|
||||
if (expectField === "links") {
|
||||
expect(Array.isArray(doc.links)).toBe(true);
|
||||
expect((doc.links || []).length).toBeGreaterThan(0);
|
||||
}
|
||||
}, 90_000);
|
||||
|
||||
test("images format: extract all images from webpage", async () => {
|
||||
if (!client) throw new Error();
|
||||
const doc = await client.scrape("https://firecrawl.dev", {
|
||||
formats: ["images"],
|
||||
});
|
||||
expect(doc.images).toBeTruthy();
|
||||
expect(Array.isArray(doc.images)).toBe(true);
|
||||
expect(doc.images?.length).toBeGreaterThan(0);
|
||||
// Should find firecrawl logo/branding images
|
||||
expect(doc.images?.some(img => img.includes("firecrawl") || img.includes("logo"))).toBe(true);
|
||||
}, 60_000);
|
||||
|
||||
test("images format: works with multiple formats", async () => {
|
||||
if (!client) throw new Error();
|
||||
const doc = await client.scrape("https://github.com", {
|
||||
formats: ["markdown", "links", "images"],
|
||||
});
|
||||
expect(doc.markdown).toBeTruthy();
|
||||
expect(doc.links).toBeTruthy();
|
||||
expect(doc.images).toBeTruthy();
|
||||
expect(Array.isArray(doc.images)).toBe(true);
|
||||
expect(doc.images?.length).toBeGreaterThan(0);
|
||||
|
||||
// Images should find things not available in links format
|
||||
const imageExtensions = ['.jpg', '.jpeg', '.png', '.gif', '.webp', '.svg', '.ico'];
|
||||
const linkImages = doc.links?.filter(link =>
|
||||
imageExtensions.some(ext => link.toLowerCase().includes(ext))
|
||||
) || [];
|
||||
|
||||
// Should discover additional images beyond those with obvious extensions
|
||||
expect(doc.images?.length).toBeGreaterThanOrEqual(linkImages.length);
|
||||
}, 60_000);
|
||||
|
||||
test("invalid url should throw", async () => {
|
||||
if (!client) throw new Error();
|
||||
await expect(client.scrape("")).rejects.toThrow("URL cannot be empty");
|
||||
await expect(client.scrape(" ")).rejects.toThrow("URL cannot be empty");
|
||||
});
|
||||
});
|
||||
|
||||
@@ -0,0 +1,247 @@
|
||||
/**
|
||||
* E2E tests for v2 search (translated from Python tests)
|
||||
*/
|
||||
import Firecrawl from "../../../index";
|
||||
import type { Document, SearchResultWeb, SearchResultNews, SearchResultImages } from "../../../index";
|
||||
import { config } from "dotenv";
|
||||
import { getIdentity, getApiUrl } from "./utils/idmux";
|
||||
import { describe, test, expect, beforeAll } from "@jest/globals";
|
||||
|
||||
config();
|
||||
|
||||
const API_URL = getApiUrl();
|
||||
let client: Firecrawl;
|
||||
|
||||
beforeAll(async () => {
|
||||
const { apiKey } = await getIdentity({ name: "js-e2e-search" });
|
||||
client = new Firecrawl({ apiKey, apiUrl: API_URL });
|
||||
});
|
||||
|
||||
function collectTexts(entries: any[] | undefined): string[] {
|
||||
const texts: string[] = [];
|
||||
for (const r of entries || []) {
|
||||
const title = (r && typeof r === 'object') ? (r.title as unknown as string | undefined) : undefined;
|
||||
const desc = (r && typeof r === 'object') ? (r.description as unknown as string | undefined) : undefined;
|
||||
if (title) texts.push(String(title).toLowerCase());
|
||||
if (desc) texts.push(String(desc).toLowerCase());
|
||||
}
|
||||
return texts;
|
||||
}
|
||||
|
||||
function isDocument(entry: Document | SearchResultWeb | SearchResultNews | SearchResultImages | undefined | null): entry is Document {
|
||||
if (!entry) return false;
|
||||
const d = entry as Document;
|
||||
return (
|
||||
typeof d.markdown === 'string' ||
|
||||
typeof d.rawHtml === 'string' ||
|
||||
typeof d.html === 'string' ||
|
||||
typeof d.links === 'object' ||
|
||||
typeof d.screenshot === 'string' ||
|
||||
typeof d.changeTracking === 'object' ||
|
||||
typeof d.summary === 'string' ||
|
||||
typeof d.json === 'object'
|
||||
);
|
||||
}
|
||||
|
||||
describe("v2.search e2e", () => {
|
||||
|
||||
test("minimal request", async () => {
|
||||
if (!client) throw new Error();
|
||||
const results = await client.search("What is the capital of France?");
|
||||
expect(results).toBeTruthy();
|
||||
expect(results).toHaveProperty("web");
|
||||
expect(results).not.toHaveProperty("news");
|
||||
expect(results).not.toHaveProperty("images");
|
||||
|
||||
expect(results.web).toBeTruthy();
|
||||
expect((results.web || []).length).toBeGreaterThan(0);
|
||||
|
||||
for (const result of results.web || []) {
|
||||
if (isDocument(result)) {
|
||||
// documents appear if scraping happens
|
||||
continue;
|
||||
}
|
||||
expect(typeof result.url).toBe("string");
|
||||
expect(result.url.startsWith("http")).toBe(true);
|
||||
expect(typeof result.title === "string" || result.title == null).toBe(true);
|
||||
expect(typeof result.description === "string" || result.description == null).toBe(true);
|
||||
}
|
||||
|
||||
const allText = collectTexts(results.web).join(" ");
|
||||
expect(allText.includes("paris")).toBe(true);
|
||||
|
||||
expect(results.news == null).toBe(true);
|
||||
expect(results.images == null).toBe(true);
|
||||
}, 90_000);
|
||||
|
||||
test("with sources web+news and limit", async () => {
|
||||
if (!client) throw new Error();
|
||||
const results = await client.search("firecrawl", { sources: ["web", "news"], limit: 3 });
|
||||
expect(results).toBeTruthy();
|
||||
expect(results.web).toBeTruthy();
|
||||
expect((results.web || []).length).toBeLessThanOrEqual(3);
|
||||
if (results.news != null) {
|
||||
expect((results.news || []).length).toBeLessThanOrEqual(3);
|
||||
}
|
||||
expect(results.images == null).toBe(true);
|
||||
|
||||
const webTitles = (results.web || [])
|
||||
.filter((r): r is SearchResultWeb => !isDocument(r))
|
||||
.map(r => (r.title || "").toString().toLowerCase());
|
||||
const webDescriptions = (results.web || [])
|
||||
.filter((r): r is SearchResultWeb => !isDocument(r))
|
||||
.map(r => (r.description || "").toString().toLowerCase());
|
||||
const allWebText = (webTitles.concat(webDescriptions)).join(" ");
|
||||
expect(allWebText.includes("firecrawl")).toBe(true);
|
||||
}, 90_000);
|
||||
|
||||
test("result structure", async () => {
|
||||
if (!client) throw new Error();
|
||||
const results = await client.search("test query", { limit: 1 });
|
||||
if (results.web && results.web.length > 0) {
|
||||
const result: any = results.web[0];
|
||||
expect(result).toHaveProperty("url");
|
||||
expect(result).toHaveProperty("title");
|
||||
expect(result).toHaveProperty("description");
|
||||
expect(typeof result.url).toBe("string");
|
||||
expect(typeof result.title === "string" || result.title == null).toBe(true);
|
||||
expect(typeof result.description === "string" || result.description == null).toBe(true);
|
||||
expect(result.url.startsWith("http")).toBe(true);
|
||||
}
|
||||
}, 90_000);
|
||||
|
||||
test("all parameters (comprehensive)", async () => {
|
||||
if (!client) throw new Error();
|
||||
const schema = {
|
||||
type: "object",
|
||||
properties: {
|
||||
title: { type: "string" },
|
||||
description: { type: "string" },
|
||||
url: { type: "string" },
|
||||
},
|
||||
required: ["title", "description"],
|
||||
} as const;
|
||||
|
||||
const results = await client.search("artificial intelligence", {
|
||||
sources: [ "web", "news", "images" ],
|
||||
limit: 3,
|
||||
tbs: "qdr:m",
|
||||
location: "US",
|
||||
ignoreInvalidURLs: true,
|
||||
timeout: 60_000,
|
||||
scrapeOptions: {
|
||||
formats: [
|
||||
"markdown",
|
||||
"html",
|
||||
{ type: "json", prompt: "Extract the title and description from the page", schema },
|
||||
],
|
||||
headers: { "User-Agent": "Firecrawl-Test/1.0" },
|
||||
includeTags: ["h1", "h2", "p"],
|
||||
excludeTags: ["nav", "footer"],
|
||||
onlyMainContent: true,
|
||||
waitFor: 2000,
|
||||
mobile: false,
|
||||
skipTlsVerification: false,
|
||||
removeBase64Images: true,
|
||||
blockAds: true,
|
||||
proxy: "basic",
|
||||
maxAge: 3_600_000,
|
||||
storeInCache: true,
|
||||
location: { country: "US", languages: ["en"] },
|
||||
actions: [{ type: "wait", milliseconds: 1000 }],
|
||||
},
|
||||
});
|
||||
|
||||
expect(results).toBeTruthy();
|
||||
expect(results).toHaveProperty("web");
|
||||
expect(results).toHaveProperty("news");
|
||||
expect(results).toHaveProperty("images");
|
||||
|
||||
expect(results.web).toBeTruthy();
|
||||
expect((results.web || []).length).toBeLessThanOrEqual(3);
|
||||
|
||||
const nonDocEntries = (results.web || []).filter(r => !isDocument(r));
|
||||
if (nonDocEntries.length > 0) {
|
||||
const allWebText = collectTexts(nonDocEntries).join(" ");
|
||||
const aiTerms = ["artificial", "intelligence", "ai", "machine", "learning"];
|
||||
expect(aiTerms.some(t => allWebText.includes(t))).toBe(true);
|
||||
}
|
||||
|
||||
for (const result of results.web || []) {
|
||||
if (isDocument(result)) {
|
||||
expect(Boolean(result.markdown) || Boolean(result.html)).toBe(true);
|
||||
} else {
|
||||
expect(typeof result.url).toBe("string");
|
||||
expect(result.url.startsWith("http")).toBe(true);
|
||||
}
|
||||
}
|
||||
|
||||
if (results.news != null) {
|
||||
expect((results.news || []).length).toBeLessThanOrEqual(3);
|
||||
for (const result of results.news || []) {
|
||||
if (isDocument(result)) {
|
||||
expect(Boolean(result.markdown) || Boolean(result.html)).toBe(true);
|
||||
} else {
|
||||
expect(typeof result.url).toBe("string");
|
||||
expect(result.url?.startsWith("http")).toBe(true);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
expect(results.images).toBeTruthy();
|
||||
expect((results.images || []).length).toBeLessThanOrEqual(3);
|
||||
for (const result of results.images || []) {
|
||||
if (!isDocument(result)) {
|
||||
expect(typeof result.url).toBe("string");
|
||||
expect(result.url?.startsWith("http")).toBe(true);
|
||||
}
|
||||
}
|
||||
}, 120_000);
|
||||
|
||||
test("formats flexibility: list vs object", async () => {
|
||||
if (!client) throw new Error();
|
||||
const results1 = await client.search("python programming", {
|
||||
limit: 1,
|
||||
scrapeOptions: { formats: ["markdown"] },
|
||||
});
|
||||
const results2 = await client.search("python programming", {
|
||||
limit: 1,
|
||||
scrapeOptions: { formats: ["markdown"] },
|
||||
});
|
||||
expect(results1).toBeTruthy();
|
||||
expect(results2).toBeTruthy();
|
||||
expect(results1.web).toBeTruthy();
|
||||
expect(results2.web).toBeTruthy();
|
||||
}, 90_000);
|
||||
|
||||
test("with json format object", async () => {
|
||||
if (!client) throw new Error();
|
||||
const jsonSchema = {
|
||||
type: "object",
|
||||
properties: { title: { type: "string" } },
|
||||
required: ["title"],
|
||||
} as const;
|
||||
const results = await client.search("site:docs.firecrawl.dev", {
|
||||
limit: 1,
|
||||
scrapeOptions: {
|
||||
formats: [{ type: "json", prompt: "Extract page title", schema: jsonSchema }],
|
||||
},
|
||||
});
|
||||
expect(results).toBeTruthy();
|
||||
expect(Array.isArray(results.web) || results.web == null).toBe(true);
|
||||
}, 90_000);
|
||||
|
||||
test("with summary format, documents include summary when present", async () => {
|
||||
if (!client) throw new Error();
|
||||
const results = await client.search("site:firecrawl.dev", {
|
||||
limit: 1,
|
||||
scrapeOptions: { formats: ["summary"] },
|
||||
});
|
||||
const docs = (results.web || []).filter(r => isDocument(r)) as Document[];
|
||||
if (docs.length > 0) {
|
||||
expect(typeof docs[0].summary).toBe("string");
|
||||
expect((docs[0].summary || "").length).toBeGreaterThan(5);
|
||||
}
|
||||
}, 90_000);
|
||||
});
|
||||
|
||||
@@ -0,0 +1,44 @@
|
||||
/**
|
||||
* E2E tests for v2 usage endpoints (translated from Python tests)
|
||||
*/
|
||||
import Firecrawl from "../../../index";
|
||||
import { config } from "dotenv";
|
||||
import { getIdentity, getApiUrl } from "./utils/idmux";
|
||||
import { describe, test, expect, beforeAll } from "@jest/globals";
|
||||
|
||||
config();
|
||||
|
||||
const API_URL = getApiUrl();
|
||||
let client: Firecrawl;
|
||||
|
||||
beforeAll(async () => {
|
||||
const { apiKey } = await getIdentity({ name: "js-e2e-usage" });
|
||||
client = new Firecrawl({ apiKey, apiUrl: API_URL });
|
||||
});
|
||||
|
||||
describe("v2.usage e2e", () => {
|
||||
test("get_concurrency", async () => {
|
||||
const resp = await client.getConcurrency();
|
||||
expect(typeof resp.concurrency).toBe("number");
|
||||
expect(typeof resp.maxConcurrency).toBe("number");
|
||||
}, 60_000);
|
||||
|
||||
test("get_credit_usage", async () => {
|
||||
const resp = await client.getCreditUsage();
|
||||
expect(typeof resp.remainingCredits).toBe("number");
|
||||
}, 60_000);
|
||||
|
||||
test("get_token_usage", async () => {
|
||||
const resp = await client.getTokenUsage();
|
||||
expect(typeof resp.remainingTokens).toBe("number");
|
||||
}, 60_000);
|
||||
|
||||
test("get_queue_status", async () => {
|
||||
const resp = await client.getQueueStatus();
|
||||
expect(typeof resp.jobsInQueue).toBe("number");
|
||||
expect(typeof resp.activeJobsInQueue).toBe("number");
|
||||
expect(typeof resp.waitingJobsInQueue).toBe("number");
|
||||
expect(typeof resp.maxConcurrency).toBe("number");
|
||||
}, 60_000);
|
||||
});
|
||||
|
||||
@@ -0,0 +1,58 @@
|
||||
export type IdmuxRequest = {
|
||||
name?: string;
|
||||
concurrency?: number;
|
||||
credits?: number;
|
||||
tokens?: number;
|
||||
teamId?: string;
|
||||
flags?: Record<string, unknown>;
|
||||
};
|
||||
|
||||
export type Identity = {
|
||||
apiKey: string;
|
||||
teamId: string;
|
||||
};
|
||||
|
||||
let cachedIdentity: Identity | null = null;
|
||||
|
||||
export function getApiUrl(): string {
|
||||
return process.env.TEST_URL ?? process.env.FIRECRAWL_API_URL ?? "https://api.firecrawl.dev";
|
||||
}
|
||||
|
||||
export async function getIdentity(req: IdmuxRequest = {}): Promise<Identity> {
|
||||
if (cachedIdentity) return cachedIdentity;
|
||||
|
||||
const idmuxUrl = process.env.IDMUX_URL;
|
||||
if (!idmuxUrl) {
|
||||
const fallback: Identity = {
|
||||
apiKey: process.env.TEST_API_KEY ?? process.env.FIRECRAWL_API_KEY ?? "",
|
||||
teamId: process.env.TEST_TEAM_ID ?? "",
|
||||
};
|
||||
cachedIdentity = fallback;
|
||||
return fallback;
|
||||
}
|
||||
|
||||
const runNumberRaw = process.env.GITHUB_RUN_NUMBER;
|
||||
const runNumber = runNumberRaw ? Number(runNumberRaw) : 0;
|
||||
const body = {
|
||||
refName: process.env.GITHUB_REF_NAME ?? "local",
|
||||
runNumber: Number.isFinite(runNumber) ? runNumber : 0,
|
||||
concurrency: req.concurrency ?? 100,
|
||||
...req,
|
||||
};
|
||||
|
||||
const res = await fetch(`${idmuxUrl}/`, {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify(body),
|
||||
});
|
||||
|
||||
if (!res.ok) {
|
||||
const text = await res.text();
|
||||
throw new Error(`idmux request failed: ${res.status} ${text}`);
|
||||
}
|
||||
|
||||
const identity = (await res.json()) as Identity;
|
||||
cachedIdentity = identity;
|
||||
return identity;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,96 @@
|
||||
import Firecrawl from "../../../index";
|
||||
import { config } from "dotenv";
|
||||
import { describe, test, expect, beforeAll } from "@jest/globals";
|
||||
import { getIdentity } from "./utils/idmux";
|
||||
|
||||
config();
|
||||
|
||||
const API_URL = process.env.FIRECRAWL_API_URL ?? "https://api.firecrawl.dev";
|
||||
let client: Firecrawl;
|
||||
|
||||
beforeAll(async () => {
|
||||
const { apiKey } = await getIdentity({ name: "js-e2e-watcher" });
|
||||
client = new Firecrawl({ apiKey, apiUrl: API_URL });
|
||||
});
|
||||
|
||||
describe("v2.watcher e2e", () => {
|
||||
test("crawl watcher minimal", async () => {
|
||||
// client is initialized in beforeAll
|
||||
const start = await client.startCrawl("https://docs.firecrawl.dev", { limit: 3 });
|
||||
|
||||
expect(typeof start.id).toBe("string");
|
||||
|
||||
const watcher = client.watcher(start.id, { pollInterval: 2 });
|
||||
|
||||
let snapshots = 0;
|
||||
let documents = 0;
|
||||
|
||||
watcher.on("snapshot", (snap: any) => {
|
||||
snapshots += 1;
|
||||
expect(["scraping", "completed", "failed", "cancelled"]).toContain(snap.status);
|
||||
expect(typeof snap.completed).toBe("number");
|
||||
expect(typeof snap.total).toBe("number");
|
||||
});
|
||||
|
||||
watcher.on("document", (_doc: any) => {
|
||||
documents += 1;
|
||||
});
|
||||
|
||||
const final = await new Promise<any>(async (resolve) => {
|
||||
watcher.on("done", (payload: any) => {
|
||||
resolve(payload);
|
||||
});
|
||||
watcher.on("error", (err: any) => {
|
||||
resolve(err);
|
||||
});
|
||||
await watcher.start();
|
||||
});
|
||||
|
||||
expect(["completed", "failed", "cancelled"]).toContain(final.status);
|
||||
expect(Array.isArray(final.data)).toBe(true);
|
||||
expect(typeof final.id).toBe("string");
|
||||
expect(snapshots).toBeGreaterThanOrEqual(1);
|
||||
expect(documents).toBeGreaterThanOrEqual(0);
|
||||
watcher.close();
|
||||
}, 240_000);
|
||||
|
||||
test("batch watcher with options (kind, pollInterval, timeout)", async () => {
|
||||
// client is initialized in beforeAll
|
||||
const urls = [
|
||||
"https://docs.firecrawl.dev",
|
||||
"https://firecrawl.dev",
|
||||
];
|
||||
|
||||
const start = await client.startBatchScrape(urls, { options: { formats: ["markdown"] }, ignoreInvalidURLs: true });
|
||||
expect(typeof start.id).toBe("string");
|
||||
|
||||
const watcher = client.watcher(start.id, { kind: "batch", pollInterval: 2, timeout: 180 });
|
||||
|
||||
let snapshots = 0;
|
||||
let gotCompleted = false;
|
||||
|
||||
watcher.on("snapshot", (snap: any) => {
|
||||
snapshots += 1;
|
||||
if (snap.status === "completed") gotCompleted = true;
|
||||
expect(["scraping", "completed", "failed", "cancelled"]).toContain(snap.status);
|
||||
});
|
||||
|
||||
const final = await new Promise<any>(async (resolve) => {
|
||||
watcher.on("done", (payload: any) => {
|
||||
resolve(payload);
|
||||
});
|
||||
watcher.on("error", (err: any) => {
|
||||
resolve(err);
|
||||
});
|
||||
await watcher.start();
|
||||
});
|
||||
|
||||
expect(["completed", "failed", "cancelled"]).toContain(final.status);
|
||||
expect(Array.isArray(final.data)).toBe(true);
|
||||
expect(typeof final.id).toBe("string");
|
||||
expect(snapshots).toBeGreaterThanOrEqual(1);
|
||||
expect(gotCompleted || final.status !== "completed").toBe(true);
|
||||
watcher.close();
|
||||
}, 300_000);
|
||||
});
|
||||
|
||||
@@ -0,0 +1,154 @@
|
||||
import FirecrawlApp from '../../../index';
|
||||
import { describe, test, expect, jest, beforeEach, afterEach } from '@jest/globals';
|
||||
|
||||
describe('monitorJobStatus retry logic', () => {
|
||||
let app: FirecrawlApp;
|
||||
let originalConsoleWarn: typeof console.warn;
|
||||
|
||||
beforeEach(() => {
|
||||
app = new FirecrawlApp({ apiKey: 'test-key', apiUrl: 'https://test.com' });
|
||||
originalConsoleWarn = console.warn;
|
||||
console.warn = jest.fn();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
console.warn = originalConsoleWarn;
|
||||
jest.clearAllMocks();
|
||||
});
|
||||
|
||||
test('should retry on socket hang up error', async () => {
|
||||
const socketHangUpError = new Error('socket hang up') as any;
|
||||
socketHangUpError.code = 'ECONNRESET';
|
||||
|
||||
const successResponse = {
|
||||
status: 200,
|
||||
data: { status: 'completed', data: [{ url: 'test.com', markdown: 'test' }] }
|
||||
};
|
||||
|
||||
const originalGetRequest = app.getRequest;
|
||||
let callCount = 0;
|
||||
|
||||
app.getRequest = async function(url: string, headers: any) {
|
||||
callCount++;
|
||||
if (callCount === 1) {
|
||||
throw socketHangUpError;
|
||||
}
|
||||
return successResponse;
|
||||
};
|
||||
|
||||
const result = await app.monitorJobStatus('test-id', {}, 1);
|
||||
|
||||
expect(callCount).toBe(2);
|
||||
expect(result).toEqual(successResponse.data);
|
||||
expect(console.warn).toHaveBeenCalledWith(
|
||||
expect.stringContaining('Network error during job status check (attempt 1/3): socket hang up')
|
||||
);
|
||||
});
|
||||
|
||||
test('should retry on ETIMEDOUT error', async () => {
|
||||
const timeoutError = new Error('timeout') as any;
|
||||
timeoutError.code = 'ETIMEDOUT';
|
||||
|
||||
const successResponse = {
|
||||
status: 200,
|
||||
data: { status: 'completed', data: [{ url: 'test.com', markdown: 'test' }] }
|
||||
};
|
||||
|
||||
const originalGetRequest = app.getRequest;
|
||||
let callCount = 0;
|
||||
|
||||
app.getRequest = async function(url: string, headers: any) {
|
||||
callCount++;
|
||||
if (callCount === 1) {
|
||||
throw timeoutError;
|
||||
}
|
||||
return successResponse;
|
||||
};
|
||||
|
||||
const result = await app.monitorJobStatus('test-id', {}, 1);
|
||||
|
||||
expect(callCount).toBe(2);
|
||||
expect(result).toEqual(successResponse.data);
|
||||
});
|
||||
|
||||
test('should fail after max retries exceeded', async () => {
|
||||
const socketHangUpError = new Error('socket hang up') as any;
|
||||
socketHangUpError.code = 'ECONNRESET';
|
||||
|
||||
app.getRequest = async function(url: string, headers: any) {
|
||||
throw socketHangUpError;
|
||||
};
|
||||
|
||||
await expect(app.monitorJobStatus('test-id', {}, 1)).rejects.toThrow('socket hang up');
|
||||
|
||||
expect(console.warn).toHaveBeenCalledTimes(3);
|
||||
}, 15000);
|
||||
|
||||
test('should not retry on non-retryable errors', async () => {
|
||||
const authError = new Error('Unauthorized') as any;
|
||||
authError.response = { status: 401, data: { error: 'Unauthorized' } };
|
||||
|
||||
app.getRequest = async function(url: string, headers: any) {
|
||||
throw authError;
|
||||
};
|
||||
|
||||
await expect(app.monitorJobStatus('test-id', {}, 1)).rejects.toThrow('Unauthorized');
|
||||
|
||||
expect(console.warn).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
test('should retry on HTTP timeout status codes', async () => {
|
||||
const timeoutError = new Error('Request timeout') as any;
|
||||
timeoutError.response = { status: 408, data: { error: 'Request timeout' } };
|
||||
|
||||
const successResponse = {
|
||||
status: 200,
|
||||
data: { status: 'completed', data: [{ url: 'test.com', markdown: 'test' }] }
|
||||
};
|
||||
|
||||
const originalGetRequest = app.getRequest;
|
||||
let callCount = 0;
|
||||
|
||||
app.getRequest = async function(url: string, headers: any) {
|
||||
callCount++;
|
||||
if (callCount === 1) {
|
||||
throw timeoutError;
|
||||
}
|
||||
return successResponse;
|
||||
};
|
||||
|
||||
const result = await app.monitorJobStatus('test-id', {}, 1);
|
||||
|
||||
expect(callCount).toBe(2);
|
||||
expect(result).toEqual(successResponse.data);
|
||||
});
|
||||
|
||||
test('should use exponential backoff for retries', async () => {
|
||||
const socketHangUpError = new Error('socket hang up') as any;
|
||||
socketHangUpError.code = 'ECONNRESET';
|
||||
|
||||
const successResponse = {
|
||||
status: 200,
|
||||
data: { status: 'completed', data: [{ url: 'test.com', markdown: 'test' }] }
|
||||
};
|
||||
|
||||
const originalGetRequest = app.getRequest;
|
||||
let callCount = 0;
|
||||
|
||||
app.getRequest = async function(url: string, headers: any) {
|
||||
callCount++;
|
||||
if (callCount <= 2) {
|
||||
throw socketHangUpError;
|
||||
}
|
||||
return successResponse;
|
||||
};
|
||||
|
||||
const startTime = Date.now();
|
||||
const result = await app.monitorJobStatus('test-id', {}, 1);
|
||||
const endTime = Date.now();
|
||||
|
||||
expect(callCount).toBe(3);
|
||||
expect(result).toEqual(successResponse.data);
|
||||
expect(endTime - startTime).toBeGreaterThan(3000);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,80 @@
|
||||
import { describe, test, expect } from "@jest/globals";
|
||||
|
||||
// We need to test the prepareAgentPayload function, but it's not exported.
|
||||
// Since the function is internal, we'll test the behavior through type checking
|
||||
// and verify the types are properly exported.
|
||||
|
||||
import type { AgentWebhookConfig, AgentWebhookEvent } from "../../../v2/types";
|
||||
|
||||
describe("v2 types: Agent webhook types", () => {
|
||||
test("AgentWebhookConfig accepts string webhook", () => {
|
||||
// Type check - this should compile without errors
|
||||
const webhook: string | AgentWebhookConfig = "https://example.com/webhook";
|
||||
expect(typeof webhook).toBe("string");
|
||||
});
|
||||
|
||||
test("AgentWebhookConfig accepts config object", () => {
|
||||
const config: AgentWebhookConfig = {
|
||||
url: "https://example.com/webhook",
|
||||
headers: { Authorization: "Bearer token" },
|
||||
events: ["completed", "failed"],
|
||||
};
|
||||
expect(config.url).toBe("https://example.com/webhook");
|
||||
expect(config.headers).toEqual({ Authorization: "Bearer token" });
|
||||
expect(config.events).toEqual(["completed", "failed"]);
|
||||
});
|
||||
|
||||
test("AgentWebhookConfig accepts minimal config", () => {
|
||||
const config: AgentWebhookConfig = {
|
||||
url: "https://example.com/webhook",
|
||||
};
|
||||
expect(config.url).toBe("https://example.com/webhook");
|
||||
expect(config.headers).toBeUndefined();
|
||||
expect(config.metadata).toBeUndefined();
|
||||
expect(config.events).toBeUndefined();
|
||||
});
|
||||
|
||||
test("AgentWebhookEvent includes agent-specific events", () => {
|
||||
const events: AgentWebhookEvent[] = [
|
||||
"started",
|
||||
"action",
|
||||
"completed",
|
||||
"failed",
|
||||
"cancelled",
|
||||
];
|
||||
expect(events).toContain("action");
|
||||
expect(events).toContain("cancelled");
|
||||
expect(events.length).toBe(5);
|
||||
});
|
||||
|
||||
test("AgentWebhookConfig accepts all fields", () => {
|
||||
const config: AgentWebhookConfig = {
|
||||
url: "https://example.com/webhook",
|
||||
headers: {
|
||||
Authorization: "Bearer token",
|
||||
"X-Custom-Header": "value",
|
||||
},
|
||||
metadata: {
|
||||
project: "test",
|
||||
environment: "staging",
|
||||
},
|
||||
events: ["started", "action", "completed", "failed", "cancelled"],
|
||||
};
|
||||
expect(config.url).toBe("https://example.com/webhook");
|
||||
expect(Object.keys(config.headers!).length).toBe(2);
|
||||
expect(config.metadata!.project).toBe("test");
|
||||
expect(config.events!.length).toBe(5);
|
||||
});
|
||||
|
||||
test("AgentWebhookConfig events are agent-specific (not crawl)", () => {
|
||||
// Agent has 'action' and 'cancelled', but not 'page'
|
||||
const config: AgentWebhookConfig = {
|
||||
url: "https://example.com/webhook",
|
||||
events: ["action", "cancelled"],
|
||||
};
|
||||
expect(config.events).toContain("action");
|
||||
expect(config.events).toContain("cancelled");
|
||||
// 'page' is a crawl-specific event, not valid for agent
|
||||
// This is enforced at the type level
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,186 @@
|
||||
import { describe, test, expect, jest } from "@jest/globals";
|
||||
import { scrape } from "../../../v2/methods/scrape";
|
||||
|
||||
describe("JS SDK v2 branding format", () => {
|
||||
function makeHttp(postImpl: (url: string, data: any) => any) {
|
||||
return { post: jest.fn(async (u: string, d: any) => postImpl(u, d)) } as any;
|
||||
}
|
||||
|
||||
test("scrape with branding format returns branding data", async () => {
|
||||
const mockResponse = {
|
||||
status: 200,
|
||||
data: {
|
||||
success: true,
|
||||
data: {
|
||||
markdown: "# Example",
|
||||
branding: {
|
||||
colorScheme: "light",
|
||||
colors: {
|
||||
primary: "#E11D48",
|
||||
secondary: "#3B82F6",
|
||||
accent: "#F59E0B"
|
||||
},
|
||||
typography: {
|
||||
fontFamilies: {
|
||||
primary: "Inter",
|
||||
heading: "Poppins"
|
||||
},
|
||||
fontSizes: {
|
||||
h1: "2.5rem",
|
||||
body: "1rem"
|
||||
}
|
||||
},
|
||||
spacing: {
|
||||
baseUnit: 8
|
||||
},
|
||||
components: {
|
||||
buttonPrimary: {
|
||||
background: "#E11D48",
|
||||
textColor: "#FFFFFF",
|
||||
borderRadius: "0.5rem"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
const http = makeHttp(() => mockResponse);
|
||||
const result = await scrape(http, "https://example.com", { formats: ["branding"] });
|
||||
|
||||
expect(result.branding).toBeDefined();
|
||||
expect(result.branding?.colorScheme).toBe("light");
|
||||
expect(result.branding?.colors?.primary).toBe("#E11D48");
|
||||
expect(result.branding?.typography?.fontFamilies?.primary).toBe("Inter");
|
||||
expect(result.branding?.spacing?.baseUnit).toBe(8);
|
||||
expect(result.branding?.components?.buttonPrimary?.background).toBe("#E11D48");
|
||||
});
|
||||
|
||||
test("scrape with branding and markdown formats returns both", async () => {
|
||||
const mockResponse = {
|
||||
status: 200,
|
||||
data: {
|
||||
success: true,
|
||||
data: {
|
||||
markdown: "# Example Content",
|
||||
branding: {
|
||||
colorScheme: "dark",
|
||||
colors: {
|
||||
primary: "#10B981"
|
||||
},
|
||||
typography: {
|
||||
fontFamilies: {
|
||||
primary: "Roboto"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
const http = makeHttp(() => mockResponse);
|
||||
const result = await scrape(http, "https://example.com", { formats: ["markdown", "branding"] });
|
||||
|
||||
expect(result.markdown).toBe("# Example Content");
|
||||
expect(result.branding).toBeDefined();
|
||||
expect(result.branding?.colorScheme).toBe("dark");
|
||||
expect(result.branding?.colors?.primary).toBe("#10B981");
|
||||
});
|
||||
|
||||
test("scrape without branding format does not return branding", async () => {
|
||||
const mockResponse = {
|
||||
status: 200,
|
||||
data: {
|
||||
success: true,
|
||||
data: {
|
||||
markdown: "# Example"
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
const http = makeHttp(() => mockResponse);
|
||||
const result = await scrape(http, "https://example.com", { formats: ["markdown"] });
|
||||
|
||||
expect(result.markdown).toBe("# Example");
|
||||
expect(result.branding).toBeUndefined();
|
||||
});
|
||||
|
||||
test("branding format with all nested fields", async () => {
|
||||
const mockResponse = {
|
||||
status: 200,
|
||||
data: {
|
||||
success: true,
|
||||
data: {
|
||||
branding: {
|
||||
colorScheme: "light",
|
||||
logo: "https://example.com/logo.png",
|
||||
fonts: [
|
||||
{ family: "Inter", weight: 400 },
|
||||
{ family: "Poppins", weight: 700 }
|
||||
],
|
||||
colors: {
|
||||
primary: "#E11D48",
|
||||
background: "#FFFFFF"
|
||||
},
|
||||
typography: {
|
||||
fontFamilies: { primary: "Inter" },
|
||||
fontStacks: { body: ["Inter", "sans-serif"] },
|
||||
fontSizes: { h1: "2.5rem" },
|
||||
lineHeights: { body: 1.5 },
|
||||
fontWeights: { regular: 400 }
|
||||
},
|
||||
spacing: {
|
||||
baseUnit: 8,
|
||||
padding: { sm: 8, md: 16 }
|
||||
},
|
||||
components: {
|
||||
buttonPrimary: {
|
||||
background: "#E11D48",
|
||||
textColor: "#FFFFFF"
|
||||
}
|
||||
},
|
||||
icons: {
|
||||
style: "outline",
|
||||
primaryColor: "#E11D48"
|
||||
},
|
||||
images: {
|
||||
logo: "https://example.com/logo.png",
|
||||
favicon: "https://example.com/favicon.ico"
|
||||
},
|
||||
animations: {
|
||||
transitionDuration: "200ms",
|
||||
easing: "ease-in-out"
|
||||
},
|
||||
layout: {
|
||||
grid: { columns: 12, maxWidth: "1200px" },
|
||||
headerHeight: "64px"
|
||||
},
|
||||
tone: {
|
||||
voice: "professional",
|
||||
emojiUsage: "minimal"
|
||||
},
|
||||
personality: {
|
||||
tone: "professional",
|
||||
energy: "medium",
|
||||
targetAudience: "developers"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
const http = makeHttp(() => mockResponse);
|
||||
const result = await scrape(http, "https://example.com", { formats: ["branding"] });
|
||||
|
||||
expect(result.branding).toBeDefined();
|
||||
expect(result.branding?.logo).toBe("https://example.com/logo.png");
|
||||
expect(result.branding?.fonts).toHaveLength(2);
|
||||
expect(result.branding?.typography?.fontStacks?.body).toEqual(["Inter", "sans-serif"]);
|
||||
expect(result.branding?.spacing?.padding).toEqual({ sm: 8, md: 16 });
|
||||
expect(result.branding?.icons?.style).toBe("outline");
|
||||
expect(result.branding?.images?.favicon).toBe("https://example.com/favicon.ico");
|
||||
expect(result.branding?.animations?.easing).toBe("ease-in-out");
|
||||
expect(result.branding?.layout?.grid?.columns).toBe(12);
|
||||
expect(result.branding?.personality?.tone).toBe("professional");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,55 @@
|
||||
import { Firecrawl, type FirecrawlClientOptions } from '../../../index';
|
||||
|
||||
describe('Firecrawl v2 Client Options', () => {
|
||||
it('should accept v2 options including timeoutMs, maxRetries, and backoffFactor', () => {
|
||||
const options: FirecrawlClientOptions = {
|
||||
apiKey: 'test-key',
|
||||
timeoutMs: 300,
|
||||
maxRetries: 5,
|
||||
backoffFactor: 0.5,
|
||||
};
|
||||
|
||||
// Should not throw any type errors
|
||||
const client = new Firecrawl(options);
|
||||
|
||||
expect(client).toBeDefined();
|
||||
expect(client).toBeInstanceOf(Firecrawl);
|
||||
});
|
||||
|
||||
it('should work with minimal options', () => {
|
||||
const options: FirecrawlClientOptions = {
|
||||
apiKey: 'test-key',
|
||||
};
|
||||
|
||||
const client = new Firecrawl(options);
|
||||
|
||||
expect(client).toBeDefined();
|
||||
expect(client).toBeInstanceOf(Firecrawl);
|
||||
});
|
||||
|
||||
it('should work with all v2 options', () => {
|
||||
const options: FirecrawlClientOptions = {
|
||||
apiKey: 'test-key',
|
||||
apiUrl: 'https://custom-api.firecrawl.dev',
|
||||
timeoutMs: 60000,
|
||||
maxRetries: 3,
|
||||
backoffFactor: 1.0,
|
||||
};
|
||||
|
||||
const client = new Firecrawl(options);
|
||||
|
||||
expect(client).toBeDefined();
|
||||
expect(client).toBeInstanceOf(Firecrawl);
|
||||
});
|
||||
|
||||
it('should export FirecrawlClientOptions type', () => {
|
||||
// This test ensures the type is properly exported
|
||||
const options: FirecrawlClientOptions = {
|
||||
apiKey: 'test-key',
|
||||
timeoutMs: 300,
|
||||
};
|
||||
|
||||
expect(options.timeoutMs).toBe(300);
|
||||
expect(options.apiKey).toBe('test-key');
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,19 @@
|
||||
import { describe, test, expect } from "@jest/globals";
|
||||
import { throwForBadResponse, normalizeAxiosError } from "../../../v2/utils/errorHandler";
|
||||
|
||||
describe("v2 utils: errorHandler", () => {
|
||||
test("throwForBadResponse: throws SdkError with message from body.error", () => {
|
||||
const resp: any = { status: 400, data: { error: "bad" } };
|
||||
expect(() => throwForBadResponse(resp, "do thing")).toThrow(/bad/);
|
||||
});
|
||||
|
||||
test("normalizeAxiosError: prefers body.error then err.message", () => {
|
||||
const err: any = {
|
||||
isAxiosError: true,
|
||||
response: { status: 402, data: { error: "payment required" } },
|
||||
message: "network",
|
||||
};
|
||||
expect(() => normalizeAxiosError(err, "action")).toThrow(/payment required/);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -0,0 +1,133 @@
|
||||
import { describe, test, expect, jest } from "@jest/globals";
|
||||
import { getCrawlStatus } from "../../../v2/methods/crawl";
|
||||
import { getBatchScrapeStatus } from "../../../v2/methods/batch";
|
||||
import { getMonitorCheck } from "../../../v2/methods/monitor";
|
||||
|
||||
describe("JS SDK v2 pagination", () => {
|
||||
function makeHttp(getImpl: (url: string) => any) {
|
||||
return { get: jest.fn(async (u: string) => getImpl(u)) } as any;
|
||||
}
|
||||
|
||||
test("crawl: autoPaginate=false returns next", async () => {
|
||||
const first = { status: 200, data: { success: true, status: "completed", completed: 1, total: 2, next: "https://api/next", data: [{ markdown: "a" }] } };
|
||||
const http = makeHttp(() => first);
|
||||
const res = await getCrawlStatus(http, "job1", { autoPaginate: false });
|
||||
expect(res.data.length).toBe(1);
|
||||
expect(res.next).toBe("https://api/next");
|
||||
});
|
||||
|
||||
test("crawl: default autoPaginate aggregates and nulls next", async () => {
|
||||
const first = { status: 200, data: { success: true, status: "completed", completed: 1, total: 3, next: "https://api/n1", data: [{ markdown: "a" }] } };
|
||||
const second = { status: 200, data: { success: true, next: "https://api/n2", data: [{ markdown: "b" }] } };
|
||||
const third = { status: 200, data: { success: true, next: null, data: [{ markdown: "c" }] } };
|
||||
const http = makeHttp((url) => {
|
||||
if (url.includes("/v2/crawl/")) return first;
|
||||
if (url.endsWith("n1")) return second;
|
||||
return third;
|
||||
});
|
||||
const res = await getCrawlStatus(http, "job1");
|
||||
expect(res.data.length).toBe(3);
|
||||
expect(res.next).toBeNull();
|
||||
});
|
||||
|
||||
test("crawl: respects maxPages and maxResults", async () => {
|
||||
const first = { status: 200, data: { success: true, status: "completed", completed: 1, total: 10, next: "https://api/n1", data: [{ markdown: "a" }] } };
|
||||
const page = (n: number) => ({ status: 200, data: { success: true, next: n < 3 ? `https://api/n${n + 1}` : null, data: [{ markdown: `p${n}` }] } });
|
||||
const http = makeHttp((url) => {
|
||||
if (url.includes("/v2/crawl/")) return first;
|
||||
if (url.endsWith("n1")) return page(1);
|
||||
if (url.endsWith("n2")) return page(2);
|
||||
return page(3);
|
||||
});
|
||||
const res = await getCrawlStatus(http, "job1", { autoPaginate: true, maxPages: 2, maxResults: 2 });
|
||||
expect(res.data.length).toBe(2);
|
||||
});
|
||||
|
||||
test("batch: default autoPaginate aggregates and nulls next", async () => {
|
||||
const first = { status: 200, data: { success: true, status: "completed", completed: 1, total: 3, next: "https://api/b1", data: [{ markdown: "a" }] } };
|
||||
const second = { status: 200, data: { success: true, next: "https://api/b2", data: [{ markdown: "b" }] } };
|
||||
const third = { status: 200, data: { success: true, next: null, data: [{ markdown: "c" }] } };
|
||||
const http = makeHttp((url) => {
|
||||
if (url.includes("/v2/batch/scrape/")) return first;
|
||||
if (url.endsWith("b1")) return second;
|
||||
return third;
|
||||
});
|
||||
const res = await getBatchScrapeStatus(http, "jobB");
|
||||
expect(res.data.length).toBe(3);
|
||||
expect(res.next).toBeNull();
|
||||
});
|
||||
|
||||
test("batch: autoPaginate=false returns next", async () => {
|
||||
const first = { status: 200, data: { success: true, status: "completed", completed: 1, total: 2, next: "https://api/nextBatch", data: [{ markdown: "a" }] } };
|
||||
const http = makeHttp(() => first);
|
||||
const res = await getBatchScrapeStatus(http, "jobB", { autoPaginate: false });
|
||||
expect(res.data.length).toBe(1);
|
||||
expect(res.next).toBe("https://api/nextBatch");
|
||||
});
|
||||
|
||||
test("monitor check: default autoPaginate aggregates pages and nulls next", async () => {
|
||||
const first = { status: 200, data: { success: true, next: "https://api/m1", data: { id: "check1", monitorId: "mon1", status: "completed", trigger: "manual", billingStatus: "confirmed", summary: {}, createdAt: "now", updatedAt: "now", pages: [{ url: "a", status: "changed" }], next: "https://api/m1" } } };
|
||||
const second = { status: 200, data: { success: true, next: null, data: { pages: [{ url: "b", status: "same" }], next: null } } };
|
||||
const http = makeHttp((url) => {
|
||||
if (url.includes("/v2/monitor/")) return first;
|
||||
return second;
|
||||
});
|
||||
const res = await getMonitorCheck(http, "mon1", "check1");
|
||||
expect(res.pages.length).toBe(2);
|
||||
expect(res.next).toBeNull();
|
||||
});
|
||||
|
||||
test("monitor check: autoPaginate=false returns next", async () => {
|
||||
const first = { status: 200, data: { success: true, next: "https://api/m1", data: { id: "check1", monitorId: "mon1", status: "completed", trigger: "manual", billingStatus: "confirmed", summary: {}, createdAt: "now", updatedAt: "now", pages: [{ url: "a", status: "changed" }], next: "https://api/m1" } } };
|
||||
const http = makeHttp(() => first);
|
||||
const res = await getMonitorCheck(http, "mon1", "check1", { autoPaginate: false });
|
||||
expect(res.pages.length).toBe(1);
|
||||
expect(res.next).toBe("https://api/m1");
|
||||
});
|
||||
|
||||
test("crawl: maxWaitTime stops pagination after first page", async () => {
|
||||
const first = { status: 200, data: { success: true, status: "completed", completed: 1, total: 5, next: "https://api/n1", data: [{ markdown: "a" }] } };
|
||||
const p1 = { status: 200, data: { success: true, next: "https://api/n2", data: [{ markdown: "b" }] } };
|
||||
const http: any = makeHttp((url: string) => {
|
||||
if (url.includes("/v2/crawl/")) return first;
|
||||
if (url.endsWith("n1")) return p1;
|
||||
return { status: 200, data: { success: true, next: null, data: [{ markdown: "c" }] } };
|
||||
});
|
||||
const nowSpy = jest.spyOn(Date, "now");
|
||||
try {
|
||||
nowSpy
|
||||
.mockImplementationOnce(() => 0) // started
|
||||
.mockImplementationOnce(() => 0) // first loop check
|
||||
.mockImplementationOnce(() => 3000); // second loop check > maxWaitTime
|
||||
const res = await getCrawlStatus(http, "jobC", { autoPaginate: true, maxWaitTime: 1 });
|
||||
expect(res.data.length).toBe(2); // initial + first page
|
||||
expect((http.get as jest.Mock).mock.calls.length).toBe(2); // initial + n1 only
|
||||
} finally {
|
||||
nowSpy.mockRestore();
|
||||
}
|
||||
});
|
||||
|
||||
test("batch: maxWaitTime stops pagination after first page", async () => {
|
||||
const first = { status: 200, data: { success: true, status: "completed", completed: 1, total: 5, next: "https://api/b1", data: [{ markdown: "a" }] } };
|
||||
const p1 = { status: 200, data: { success: true, next: "https://api/b2", data: [{ markdown: "b" }] } };
|
||||
const http: any = makeHttp((url: string) => {
|
||||
if (url.includes("/v2/batch/scrape/")) return first;
|
||||
if (url.endsWith("b1")) return p1;
|
||||
return { status: 200, data: { success: true, next: null, data: [{ markdown: "c" }] } };
|
||||
});
|
||||
const nowSpy = jest.spyOn(Date, "now");
|
||||
try {
|
||||
nowSpy
|
||||
.mockImplementationOnce(() => 0) // started
|
||||
.mockImplementationOnce(() => 0) // first loop check
|
||||
.mockImplementationOnce(() => 3000); // second loop check > maxWaitTime
|
||||
const res = await getBatchScrapeStatus(http, "jobB", { autoPaginate: true, maxWaitTime: 1 });
|
||||
expect(res.data.length).toBe(2);
|
||||
expect((http.get as jest.Mock).mock.calls.length).toBe(2);
|
||||
} finally {
|
||||
nowSpy.mockRestore();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
@@ -0,0 +1,58 @@
|
||||
import { describe, test, expect } from "@jest/globals";
|
||||
import { FirecrawlClient } from "../../../v2/client";
|
||||
|
||||
describe("v2.parse unit", () => {
|
||||
test("rejects empty filenames before making requests", async () => {
|
||||
const client = new FirecrawlClient({
|
||||
apiKey: "test-key",
|
||||
apiUrl: "https://localhost:3002",
|
||||
});
|
||||
|
||||
await expect(
|
||||
client.parse(
|
||||
{
|
||||
data: "<html><body>test</body></html>",
|
||||
filename: " ",
|
||||
contentType: "text/html",
|
||||
},
|
||||
{ formats: ["markdown"] },
|
||||
),
|
||||
).rejects.toThrow("filename cannot be empty");
|
||||
});
|
||||
|
||||
test("rejects changeTracking format before making requests", async () => {
|
||||
const client = new FirecrawlClient({
|
||||
apiKey: "test-key",
|
||||
apiUrl: "https://localhost:3002",
|
||||
});
|
||||
|
||||
await expect(
|
||||
client.parse(
|
||||
{
|
||||
data: "<html><body>test</body></html>",
|
||||
filename: "upload.html",
|
||||
contentType: "text/html",
|
||||
},
|
||||
{ formats: ["markdown", { type: "changeTracking" } as any] },
|
||||
),
|
||||
).rejects.toThrow("parse does not support changeTracking format");
|
||||
});
|
||||
|
||||
test("rejects lockdown option before making requests", async () => {
|
||||
const client = new FirecrawlClient({
|
||||
apiKey: "test-key",
|
||||
apiUrl: "https://localhost:3002",
|
||||
});
|
||||
|
||||
await expect(
|
||||
client.parse(
|
||||
{
|
||||
data: "<html><body>test</body></html>",
|
||||
filename: "upload.html",
|
||||
contentType: "text/html",
|
||||
},
|
||||
{ formats: ["markdown"], lockdown: true } as any,
|
||||
),
|
||||
).rejects.toThrow("parse does not support cache/index options");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,131 @@
|
||||
import { describe, test, expect, jest } from "@jest/globals";
|
||||
import { interact, stopInteraction } from "../../../v2/methods/scrape";
|
||||
import { SdkError } from "../../../v2/types";
|
||||
|
||||
describe("JS SDK v2 scrape-browser methods", () => {
|
||||
test("interact posts to scrape interact endpoint", async () => {
|
||||
const post = jest.fn(async () => ({
|
||||
status: 200,
|
||||
data: {
|
||||
success: true,
|
||||
stdout: "ok",
|
||||
exitCode: 0,
|
||||
},
|
||||
}));
|
||||
|
||||
const http = { post } as any;
|
||||
const response = await interact(http, "job-123", {
|
||||
code: "console.log('ok')",
|
||||
});
|
||||
|
||||
expect(post).toHaveBeenCalledWith(
|
||||
"/v2/scrape/job-123/interact",
|
||||
{ code: "console.log('ok')", language: "node" },
|
||||
{},
|
||||
);
|
||||
expect(response.success).toBe(true);
|
||||
expect(response.exitCode).toBe(0);
|
||||
});
|
||||
|
||||
test("interact with prompt posts prompt to endpoint", async () => {
|
||||
const post = jest.fn(async () => ({
|
||||
status: 200,
|
||||
data: {
|
||||
success: true,
|
||||
output: "Clicked the button",
|
||||
liveViewUrl: "https://live.example.com/view",
|
||||
interactiveLiveViewUrl: "https://live.example.com/interactive",
|
||||
stdout: "",
|
||||
exitCode: 0,
|
||||
},
|
||||
}));
|
||||
|
||||
const http = { post } as any;
|
||||
const response = await interact(http, "job-456", {
|
||||
prompt: "Click the login button",
|
||||
});
|
||||
|
||||
expect(post).toHaveBeenCalledWith(
|
||||
"/v2/scrape/job-456/interact",
|
||||
{ prompt: "Click the login button", language: "node" },
|
||||
{},
|
||||
);
|
||||
expect(response.success).toBe(true);
|
||||
expect(response.output).toBe("Clicked the button");
|
||||
expect(response.liveViewUrl).toBe("https://live.example.com/view");
|
||||
expect(response.interactiveLiveViewUrl).toBe(
|
||||
"https://live.example.com/interactive",
|
||||
);
|
||||
});
|
||||
|
||||
test("interact throws when neither code nor prompt provided", async () => {
|
||||
const http = { post: jest.fn() } as any;
|
||||
await expect(interact(http, "job-123", {})).rejects.toThrow(
|
||||
"Either 'code' or 'prompt' must be provided",
|
||||
);
|
||||
});
|
||||
|
||||
test("interact throws on non-200 response", async () => {
|
||||
const post = jest.fn(async () => ({
|
||||
status: 400,
|
||||
data: {
|
||||
success: false,
|
||||
error: "Invalid job ID format",
|
||||
},
|
||||
}));
|
||||
|
||||
const http = { post } as any;
|
||||
await expect(
|
||||
interact(http, "bad-id", { code: "console.log('ok')" }),
|
||||
).rejects.toBeInstanceOf(SdkError);
|
||||
});
|
||||
|
||||
test("stopInteraction calls delete endpoint", async () => {
|
||||
const del = jest.fn(async () => ({
|
||||
status: 200,
|
||||
data: {
|
||||
success: true,
|
||||
},
|
||||
}));
|
||||
|
||||
const http = { delete: del } as any;
|
||||
const response = await stopInteraction(http, "job-123");
|
||||
|
||||
expect(del).toHaveBeenCalledWith("/v2/scrape/job-123/interact");
|
||||
expect(response.success).toBe(true);
|
||||
});
|
||||
|
||||
test("stopInteraction throws on non-200 response", async () => {
|
||||
const del = jest.fn(async () => ({
|
||||
status: 404,
|
||||
data: {
|
||||
success: false,
|
||||
error: "Browser session not found.",
|
||||
},
|
||||
}));
|
||||
|
||||
const http = { delete: del } as any;
|
||||
await expect(stopInteraction(http, "job-123")).rejects.toBeInstanceOf(
|
||||
SdkError,
|
||||
);
|
||||
});
|
||||
|
||||
test("interact converts seconds-based body timeout to ms axios timeout", async () => {
|
||||
const post = jest.fn(async () => ({
|
||||
status: 200,
|
||||
data: { success: true, stdout: "ok", exitCode: 0 },
|
||||
}));
|
||||
|
||||
const http = { post } as any;
|
||||
await interact(http, "job-123", {
|
||||
code: "console.log('ok')",
|
||||
timeout: 150,
|
||||
});
|
||||
|
||||
expect(post).toHaveBeenCalledWith(
|
||||
"/v2/scrape/job-123/interact",
|
||||
{ code: "console.log('ok')", language: "node", timeout: 150 },
|
||||
{ timeoutMs: 155000 },
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,11 @@
|
||||
/**
|
||||
* Minimal unit test for v2 scrape (no mocking; sanity check payload path)
|
||||
*/
|
||||
import { FirecrawlClient } from "../../../v2/client";
|
||||
|
||||
describe("v2.scrape unit", () => {
|
||||
test("constructor requires apiKey", () => {
|
||||
expect(() => new FirecrawlClient({ apiKey: "", apiUrl: "https://api.firecrawl.dev" })).toThrow();
|
||||
});
|
||||
});
|
||||
|
||||
@@ -0,0 +1,105 @@
|
||||
import { describe, test, expect } from "@jest/globals";
|
||||
import { ensureValidFormats, ensureValidScrapeOptions } from "../../../v2/utils/validation";
|
||||
import type { FormatOption } from "../../../v2/types";
|
||||
import { z } from "zod";
|
||||
|
||||
describe("v2 utils: validation", () => {
|
||||
test("ensureValidFormats: plain 'json' string is invalid", () => {
|
||||
const formats: FormatOption[] = ["markdown", "json"] as unknown as FormatOption[];
|
||||
expect(() => ensureValidFormats(formats)).toThrow(/json format must be an object/i);
|
||||
});
|
||||
|
||||
test("ensureValidFormats: json format requires prompt or schema", () => {
|
||||
// Valid cases - should not throw
|
||||
const valid1: FormatOption[] = [{ type: "json", prompt: "p" } as any];
|
||||
const valid2: FormatOption[] = [{ type: "json", schema: {} } as any];
|
||||
const valid3: FormatOption[] = [{ type: "json", prompt: "p", schema: {} } as any];
|
||||
expect(() => ensureValidFormats(valid1)).not.toThrow();
|
||||
expect(() => ensureValidFormats(valid2)).not.toThrow();
|
||||
expect(() => ensureValidFormats(valid3)).not.toThrow();
|
||||
|
||||
// Invalid case - should throw when both are missing
|
||||
const bad: FormatOption[] = [{ type: "json" } as any];
|
||||
expect(() => ensureValidFormats(bad)).toThrow(/requires either 'prompt' or 'schema'/i);
|
||||
});
|
||||
|
||||
test("ensureValidFormats: converts zod schema to JSON schema", () => {
|
||||
const schema = z.object({ title: z.string() });
|
||||
const formats: FormatOption[] = [
|
||||
{ type: "json", prompt: "extract", schema } as any,
|
||||
];
|
||||
ensureValidFormats(formats);
|
||||
const jsonFmt = formats[0] as any;
|
||||
expect(typeof jsonFmt.schema).toBe("object");
|
||||
expect(jsonFmt.schema?.properties).toBeTruthy();
|
||||
});
|
||||
|
||||
test("ensureValidFormats: screenshot quality must be non-negative number", () => {
|
||||
const formats: FormatOption[] = [
|
||||
{ type: "screenshot", quality: -1 } as any,
|
||||
];
|
||||
expect(() => ensureValidFormats(formats)).toThrow(/non-negative number/i);
|
||||
});
|
||||
|
||||
test("ensureValidScrapeOptions: validates timeout and waitFor bounds", () => {
|
||||
expect(() => ensureValidScrapeOptions({ timeout: 0 })).toThrow(/timeout must be positive/i);
|
||||
expect(() => ensureValidScrapeOptions({ waitFor: -1 })).toThrow(/waitFor must be non-negative/i);
|
||||
// valid
|
||||
expect(() => ensureValidScrapeOptions({ timeout: 1000, waitFor: 0 })).not.toThrow();
|
||||
});
|
||||
|
||||
test("ensureValidFormats: accepts screenshot viewport width/height", () => {
|
||||
const formats: FormatOption[] = [
|
||||
{ type: "screenshot", viewport: { width: 800, height: 600 } } as any,
|
||||
];
|
||||
expect(() => ensureValidFormats(formats)).not.toThrow();
|
||||
expect((formats[0] as any).viewport).toEqual({ width: 800, height: 600 });
|
||||
});
|
||||
|
||||
test("ensureValidFormats: accepts question, highlights, and deprecated query formats", () => {
|
||||
const formats: FormatOption[] = [
|
||||
{ type: "question", question: "What is Firecrawl?" },
|
||||
{ type: "highlights", query: "What is Firecrawl?" },
|
||||
{ type: "query", prompt: "What is Firecrawl?", mode: "directQuote" },
|
||||
];
|
||||
expect(() => ensureValidFormats(formats)).not.toThrow();
|
||||
});
|
||||
|
||||
test("ensureValidFormats: validates question, highlights, and deprecated query fields", () => {
|
||||
expect(() =>
|
||||
ensureValidFormats([{ type: "question", question: "" } as any]),
|
||||
).toThrow(/question format requires/i);
|
||||
expect(() =>
|
||||
ensureValidFormats([{ type: "highlights", query: "" } as any]),
|
||||
).toThrow(/highlights format requires/i);
|
||||
expect(() =>
|
||||
ensureValidFormats([{ type: "query", prompt: "p", mode: "quoted" } as any]),
|
||||
).toThrow(/query format mode/i);
|
||||
});
|
||||
|
||||
test("ensureValidScrapeOptions: leaves parsers untouched", () => {
|
||||
const options = { parsers: ["pdf", "images"] as string[] } as any;
|
||||
const before = [...options.parsers];
|
||||
expect(() => ensureValidScrapeOptions(options)).not.toThrow();
|
||||
expect(options.parsers).toEqual(before);
|
||||
});
|
||||
|
||||
test("ensureValidFormats: detects mistaken use of zod schema.shape", () => {
|
||||
const schema = z.object({ title: z.string(), count: z.number() });
|
||||
// User mistakenly passes schema.shape instead of schema
|
||||
const formats: FormatOption[] = [
|
||||
{ type: "json", prompt: "extract", schema: schema.shape } as any,
|
||||
];
|
||||
expect(() => ensureValidFormats(formats)).toThrow(/\.shape property/i);
|
||||
expect(() => ensureValidFormats(formats)).toThrow(/Pass the Zod schema directly/i);
|
||||
});
|
||||
|
||||
test("ensureValidFormats: detects mistaken use of zod schema.shape in changeTracking", () => {
|
||||
const schema = z.object({ title: z.string() });
|
||||
const formats: FormatOption[] = [
|
||||
{ type: "changeTracking", modes: ["json"], schema: schema.shape } as any,
|
||||
];
|
||||
expect(() => ensureValidFormats(formats)).toThrow(/\.shape property/i);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -0,0 +1,107 @@
|
||||
import { describe, test, expect } from "@jest/globals";
|
||||
import { z } from "zod";
|
||||
import {
|
||||
isZodSchema,
|
||||
zodSchemaToJsonSchema,
|
||||
looksLikeZodShape,
|
||||
} from "../../../utils/zodSchemaToJson";
|
||||
|
||||
describe("zodSchemaToJson utility", () => {
|
||||
test("isZodSchema detects Zod schemas and rejects non-Zod values", () => {
|
||||
expect(isZodSchema(z.object({ name: z.string() }))).toBe(true);
|
||||
expect(isZodSchema(z.string())).toBe(true);
|
||||
expect(isZodSchema(z.number())).toBe(true);
|
||||
expect(isZodSchema(z.array(z.string()))).toBe(true);
|
||||
expect(isZodSchema(z.enum(["A", "B"]))).toBe(true);
|
||||
expect(isZodSchema(z.union([z.string(), z.number()]))).toBe(true);
|
||||
expect(isZodSchema(z.string().optional())).toBe(true);
|
||||
expect(isZodSchema(z.string().nullable())).toBe(true);
|
||||
|
||||
expect(isZodSchema(null)).toBe(false);
|
||||
expect(isZodSchema(undefined)).toBe(false);
|
||||
expect(isZodSchema({ name: "test" })).toBe(false);
|
||||
expect(isZodSchema({ type: "object", properties: {} })).toBe(false);
|
||||
expect(isZodSchema("string")).toBe(false);
|
||||
expect(isZodSchema(42)).toBe(false);
|
||||
expect(isZodSchema([1, 2, 3])).toBe(false);
|
||||
});
|
||||
|
||||
test("zodSchemaToJsonSchema converts Zod schemas to JSON Schema", () => {
|
||||
const simpleSchema = z.object({ name: z.string() });
|
||||
const simpleResult = zodSchemaToJsonSchema(simpleSchema) as Record<string, unknown>;
|
||||
expect(simpleResult.type).toBe("object");
|
||||
expect(simpleResult.properties).toBeDefined();
|
||||
expect((simpleResult.properties as Record<string, unknown>).name).toBeDefined();
|
||||
|
||||
const complexSchema = z.object({
|
||||
id: z.string().uuid(),
|
||||
name: z.string().min(1).max(100),
|
||||
age: z.number().min(0).max(150).optional(),
|
||||
tags: z.array(z.string()),
|
||||
status: z.enum(["active", "inactive"]),
|
||||
metadata: z.object({
|
||||
createdAt: z.string(),
|
||||
nested: z.object({ value: z.number() }),
|
||||
}),
|
||||
});
|
||||
const complexResult = zodSchemaToJsonSchema(complexSchema) as Record<string, unknown>;
|
||||
expect(complexResult.type).toBe("object");
|
||||
expect(complexResult.properties).toBeDefined();
|
||||
expect(complexResult.required).toContain("id");
|
||||
expect(complexResult.required).not.toContain("age");
|
||||
|
||||
const enumResult = zodSchemaToJsonSchema(z.enum(["a", "b", "c"])) as Record<string, unknown>;
|
||||
expect(enumResult.enum).toEqual(["a", "b", "c"]);
|
||||
|
||||
const arrayResult = zodSchemaToJsonSchema(z.array(z.number())) as Record<string, unknown>;
|
||||
expect(arrayResult.type).toBe("array");
|
||||
expect(arrayResult.items).toBeDefined();
|
||||
});
|
||||
|
||||
test("zodSchemaToJsonSchema passes through non-Zod values unchanged", () => {
|
||||
const jsonSchema = { type: "object", properties: { name: { type: "string" } } };
|
||||
expect(zodSchemaToJsonSchema(jsonSchema)).toEqual(jsonSchema);
|
||||
expect(zodSchemaToJsonSchema(null)).toBe(null);
|
||||
expect(zodSchemaToJsonSchema(undefined)).toBe(undefined);
|
||||
expect(zodSchemaToJsonSchema("string")).toBe("string");
|
||||
expect(zodSchemaToJsonSchema(42)).toBe(42);
|
||||
expect(zodSchemaToJsonSchema({ foo: "bar" })).toEqual({ foo: "bar" });
|
||||
});
|
||||
|
||||
test("looksLikeZodShape detects .shape property misuse", () => {
|
||||
const schema = z.object({ title: z.string(), count: z.number() });
|
||||
expect(looksLikeZodShape(schema.shape)).toBe(true);
|
||||
expect(looksLikeZodShape(schema)).toBe(false);
|
||||
expect(looksLikeZodShape(null)).toBe(false);
|
||||
expect(looksLikeZodShape(undefined)).toBe(false);
|
||||
expect(looksLikeZodShape({ name: "test" })).toBe(false);
|
||||
expect(looksLikeZodShape({})).toBe(false);
|
||||
expect(looksLikeZodShape([1, 2, 3])).toBe(false);
|
||||
expect(looksLikeZodShape({ type: "object", properties: {} })).toBe(false);
|
||||
});
|
||||
|
||||
test("SDK-like usage: convert Zod schema or pass through JSON schema", () => {
|
||||
const zodSchema = z.object({
|
||||
name: z.string(),
|
||||
email: z.string().email(),
|
||||
age: z.number().min(0),
|
||||
});
|
||||
|
||||
if (isZodSchema(zodSchema)) {
|
||||
const result = zodSchemaToJsonSchema(zodSchema) as Record<string, unknown>;
|
||||
expect(result.type).toBe("object");
|
||||
expect(result.properties).toBeDefined();
|
||||
} else {
|
||||
throw new Error("Should detect Zod schema");
|
||||
}
|
||||
|
||||
const existingJsonSchema = {
|
||||
type: "object" as const,
|
||||
properties: { title: { type: "string" as const } },
|
||||
required: ["title"] as string[],
|
||||
};
|
||||
|
||||
expect(isZodSchema(existingJsonSchema)).toBe(false);
|
||||
expect(zodSchemaToJsonSchema(existingJsonSchema)).toEqual(existingJsonSchema);
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user