참고소스 수정본
This commit is contained in:
5
참고/firecrawl-main/apps/js-sdk/firecrawl/.env.example
Normal file
5
참고/firecrawl-main/apps/js-sdk/firecrawl/.env.example
Normal file
@@ -0,0 +1,5 @@
|
||||
FIRECRAWL_API_KEY=<your api key>
|
||||
FIRECRAWL_API_URL=https://api.firecrawl.dev
|
||||
|
||||
# TESTING PURPOSE:
|
||||
IDMUX_URL=
|
||||
132
참고/firecrawl-main/apps/js-sdk/firecrawl/.gitignore
vendored
Normal file
132
참고/firecrawl-main/apps/js-sdk/firecrawl/.gitignore
vendored
Normal file
@@ -0,0 +1,132 @@
|
||||
# Logs
|
||||
logs
|
||||
*.log
|
||||
npm-debug.log*
|
||||
yarn-debug.log*
|
||||
yarn-error.log*
|
||||
lerna-debug.log*
|
||||
.pnpm-debug.log*
|
||||
|
||||
# Diagnostic reports (https://nodejs.org/api/report.html)
|
||||
report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json
|
||||
|
||||
# Runtime data
|
||||
pids
|
||||
*.pid
|
||||
*.seed
|
||||
*.pid.lock
|
||||
|
||||
# Directory for instrumented libs generated by jscoverage/JSCover
|
||||
lib-cov
|
||||
|
||||
# Coverage directory used by tools like istanbul
|
||||
coverage
|
||||
*.lcov
|
||||
|
||||
# nyc test coverage
|
||||
.nyc_output
|
||||
|
||||
# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files)
|
||||
.grunt
|
||||
|
||||
# Bower dependency directory (https://bower.io/)
|
||||
bower_components
|
||||
|
||||
# node-waf configuration
|
||||
.lock-wscript
|
||||
|
||||
# Compiled binary addons (https://nodejs.org/api/addons.html)
|
||||
build/Release
|
||||
|
||||
# Dependency directories
|
||||
node_modules/
|
||||
jspm_packages/
|
||||
|
||||
# Snowpack dependency directory (https://snowpack.dev/)
|
||||
web_modules/
|
||||
|
||||
# TypeScript cache
|
||||
*.tsbuildinfo
|
||||
|
||||
# Optional npm cache directory
|
||||
.npm
|
||||
|
||||
# Optional eslint cache
|
||||
.eslintcache
|
||||
|
||||
# Optional stylelint cache
|
||||
.stylelintcache
|
||||
|
||||
# Microbundle cache
|
||||
.rpt2_cache/
|
||||
.rts2_cache_cjs/
|
||||
.rts2_cache_es/
|
||||
.rts2_cache_umd/
|
||||
|
||||
# Optional REPL history
|
||||
.node_repl_history
|
||||
|
||||
# Output of 'npm pack'
|
||||
*.tgz
|
||||
|
||||
# Yarn Integrity file
|
||||
.yarn-integrity
|
||||
|
||||
# dotenv environment variable files
|
||||
.env
|
||||
.env.development.local
|
||||
.env.test.local
|
||||
.env.production.local
|
||||
.env.local
|
||||
|
||||
# parcel-bundler cache (https://parceljs.org/)
|
||||
.cache
|
||||
.parcel-cache
|
||||
|
||||
# Next.js build output
|
||||
.next
|
||||
out
|
||||
|
||||
# Nuxt.js build / generate output
|
||||
.nuxt
|
||||
dist
|
||||
|
||||
# Gatsby files
|
||||
.cache/
|
||||
# Comment in the public line in if your project uses Gatsby and not Next.js
|
||||
# https://nextjs.org/blog/next-9-1#public-directory-support
|
||||
# public
|
||||
|
||||
# vuepress build output
|
||||
.vuepress/dist
|
||||
|
||||
# vuepress v2.x temp and cache directory
|
||||
.temp
|
||||
.cache
|
||||
|
||||
# Docusaurus cache and generated files
|
||||
.docusaurus
|
||||
|
||||
# Serverless directories
|
||||
.serverless/
|
||||
|
||||
# FuseBox cache
|
||||
.fusebox/
|
||||
|
||||
# DynamoDB Local files
|
||||
.dynamodb/
|
||||
|
||||
# TernJS port file
|
||||
.tern-port
|
||||
|
||||
# Stores VSCode versions used for testing VSCode extensions
|
||||
.vscode-test
|
||||
|
||||
# yarn v2
|
||||
.yarn/cache
|
||||
.yarn/unplugged
|
||||
.yarn/build-state.yml
|
||||
.yarn/install-state.gz
|
||||
.pnp.*
|
||||
|
||||
build
|
||||
21
참고/firecrawl-main/apps/js-sdk/firecrawl/LICENSE
Normal file
21
참고/firecrawl-main/apps/js-sdk/firecrawl/LICENSE
Normal file
@@ -0,0 +1,21 @@
|
||||
MIT License
|
||||
|
||||
Copyright (c) 2024 Sideguide Technologies Inc.
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
249
참고/firecrawl-main/apps/js-sdk/firecrawl/README.md
Normal file
249
참고/firecrawl-main/apps/js-sdk/firecrawl/README.md
Normal file
@@ -0,0 +1,249 @@
|
||||
# Firecrawl Node SDK
|
||||
|
||||
The Firecrawl Node SDK is a library that allows you to easily search, scrape, and interact with the web, and output the data in a format ready for use with language models (LLMs). It provides a simple and intuitive interface for the Firecrawl API.
|
||||
|
||||
## Installation
|
||||
|
||||
To install the Firecrawl Node SDK, you can use npm:
|
||||
|
||||
```bash
|
||||
npm install @mendable/firecrawl-js
|
||||
```
|
||||
|
||||
## Usage
|
||||
|
||||
1. Get an API key from [firecrawl.dev](https://firecrawl.dev)
|
||||
2. Set the API key as an environment variable named `FIRECRAWL_API_KEY` or pass it as a parameter to the `FirecrawlApp` class.
|
||||
|
||||
Here's an example of how to use the SDK with error handling:
|
||||
|
||||
```js
|
||||
import Firecrawl from '@mendable/firecrawl-js';
|
||||
|
||||
const app = new Firecrawl({ apiKey: 'fc-YOUR_API_KEY' });
|
||||
|
||||
// Scrape a website
|
||||
const scrapeResponse = await app.scrape('https://firecrawl.dev', {
|
||||
formats: ['markdown', 'html'],
|
||||
});
|
||||
console.log(scrapeResponse);
|
||||
|
||||
// Crawl a website (waiter)
|
||||
const crawlResponse = await app.crawl('https://firecrawl.dev', {
|
||||
limit: 100,
|
||||
scrapeOptions: { formats: ['markdown', 'html'] },
|
||||
pollInterval: 2,
|
||||
});
|
||||
console.log(crawlResponse);
|
||||
```
|
||||
|
||||
### Scraping a URL
|
||||
|
||||
To scrape a single URL with error handling, use the `scrape` method. It takes the URL as a parameter and returns the scraped data.
|
||||
|
||||
```js
|
||||
const url = 'https://example.com';
|
||||
const scrapedData = await app.scrape(url);
|
||||
```
|
||||
|
||||
### Parsing uploaded files
|
||||
|
||||
Use `parse` to upload a file (`html`, `pdf`, `docx`, etc.) as multipart form data and process it through the same parsing pipeline.
|
||||
Parse does not support browser-only formats/options like `changeTracking`, `screenshot`, `branding`, `actions`, `waitFor`, `location`, or `mobile`.
|
||||
|
||||
```js
|
||||
const parsed = await app.parse(
|
||||
{
|
||||
data: '<html><body><h1>Hello parse</h1></body></html>',
|
||||
filename: 'upload.html',
|
||||
contentType: 'text/html',
|
||||
},
|
||||
{
|
||||
formats: ['markdown'],
|
||||
}
|
||||
);
|
||||
|
||||
console.log(parsed.markdown);
|
||||
```
|
||||
|
||||
### Crawling a Website
|
||||
|
||||
To crawl a website with error handling, use the `crawl` method. It takes the starting URL and optional parameters, including limits and per‑page `scrapeOptions`.
|
||||
|
||||
```js
|
||||
const crawlResponse = await app.crawl('https://firecrawl.dev', {
|
||||
limit: 100,
|
||||
scrapeOptions: { formats: ['markdown', 'html'] },
|
||||
});
|
||||
```
|
||||
|
||||
|
||||
### Asynchronous Crawl
|
||||
|
||||
To start an asynchronous crawl, use `startCrawl`. It returns a job ID you can poll with `getCrawlStatus`.
|
||||
|
||||
```js
|
||||
const start = await app.startCrawl('https://mendable.ai', {
|
||||
excludePaths: ['blog/*'],
|
||||
limit: 5,
|
||||
});
|
||||
```
|
||||
|
||||
### Checking Crawl Status
|
||||
|
||||
To check the status of a crawl job with error handling, use the `getCrawlStatus` method. It takes the job ID as a parameter and returns the current status.
|
||||
|
||||
```js
|
||||
const status = await app.getCrawlStatus(id);
|
||||
```
|
||||
|
||||
### Extracting structured data from URLs
|
||||
|
||||
Use `extract` with a prompt and schema. Zod schemas are supported directly.
|
||||
|
||||
```js
|
||||
import Firecrawl from '@mendable/firecrawl-js';
|
||||
import { z } from 'zod';
|
||||
|
||||
const app = new Firecrawl({ apiKey: 'fc-YOUR_API_KEY' });
|
||||
|
||||
const schema = z.object({
|
||||
title: z.string(),
|
||||
});
|
||||
|
||||
const result = await app.extract({
|
||||
urls: ['https://firecrawl.dev'],
|
||||
prompt: 'Extract the page title',
|
||||
schema,
|
||||
showSources: true,
|
||||
});
|
||||
|
||||
console.log(result.data);
|
||||
```
|
||||
|
||||
### Map a Website
|
||||
|
||||
Use `map` to generate a list of URLs from a website. Options let you customize the mapping process, including whether to utilize the sitemap or include subdomains.
|
||||
|
||||
```js
|
||||
const mapResult = await app.map('https://example.com');
|
||||
console.log(mapResult);
|
||||
```
|
||||
|
||||
### Scrape-bound interactive browsing (v2)
|
||||
|
||||
Use a scrape job ID to keep interacting with the replayed browser context:
|
||||
|
||||
```js
|
||||
const doc = await app.scrape('https://example.com', {
|
||||
actions: [{ type: 'click', selector: 'a[href="/pricing"]' }],
|
||||
});
|
||||
|
||||
const scrapeJobId = doc?.metadata?.scrapeId;
|
||||
if (!scrapeJobId) throw new Error('Missing scrapeId');
|
||||
|
||||
const run = await app.interact(scrapeJobId, {
|
||||
code: 'console.log(await page.url())',
|
||||
language: 'node',
|
||||
timeout: 60,
|
||||
});
|
||||
console.log(run.stdout);
|
||||
|
||||
await app.stopInteraction(scrapeJobId);
|
||||
```
|
||||
|
||||
### Crawl a website with real‑time updates
|
||||
|
||||
To receive real‑time updates, start a crawl and attach a watcher.
|
||||
|
||||
```js
|
||||
const start = await app.startCrawl('https://mendable.ai', { excludePaths: ['blog/*'], limit: 5 });
|
||||
const watch = app.watcher(start.id, { kind: 'crawl', pollInterval: 2 });
|
||||
|
||||
watch.on('document', (doc) => {
|
||||
console.log('DOC', doc);
|
||||
});
|
||||
|
||||
watch.on('error', (err) => {
|
||||
console.error('ERR', err);
|
||||
});
|
||||
|
||||
watch.on('done', (state) => {
|
||||
console.log('DONE', state.status);
|
||||
});
|
||||
|
||||
await watch.start();
|
||||
```
|
||||
|
||||
### Batch scraping multiple URLs
|
||||
|
||||
To batch scrape multiple URLs with error handling, use the `batchScrape` method.
|
||||
|
||||
```js
|
||||
const batchScrapeResponse = await app.batchScrape(['https://firecrawl.dev', 'https://mendable.ai'], {
|
||||
formats: ['markdown', 'html'],
|
||||
});
|
||||
```
|
||||
|
||||
|
||||
#### Asynchronous batch scrape
|
||||
|
||||
To start an asynchronous batch scrape, use `startBatchScrape` and poll with `getBatchScrapeStatus`.
|
||||
|
||||
```js
|
||||
const asyncBatchScrapeResult = await app.startBatchScrape(['https://firecrawl.dev', 'https://mendable.ai'], {
|
||||
formats: ['markdown', 'html'],
|
||||
});
|
||||
```
|
||||
|
||||
#### Batch scrape with real‑time updates
|
||||
|
||||
To use batch scrape with real‑time updates, start the job and watch it using the watcher.
|
||||
|
||||
```js
|
||||
const start = await app.startBatchScrape(['https://firecrawl.dev', 'https://mendable.ai'], { formats: ['markdown', 'html'] });
|
||||
const watch = app.watcher(start.id, { kind: 'batch', pollInterval: 2 });
|
||||
|
||||
watch.on('document', (doc) => {
|
||||
console.log('DOC', doc);
|
||||
});
|
||||
|
||||
watch.on('error', (err) => {
|
||||
console.error('ERR', err);
|
||||
});
|
||||
|
||||
watch.on('done', (state) => {
|
||||
console.log('DONE', state.status);
|
||||
});
|
||||
|
||||
await watch.start();
|
||||
```
|
||||
|
||||
## v1 compatibility
|
||||
|
||||
The feature‑frozen v1 is still available under `app.v1` with the original method names.
|
||||
|
||||
```js
|
||||
import Firecrawl from '@mendable/firecrawl-js';
|
||||
|
||||
const app = new Firecrawl({ apiKey: 'fc-YOUR_API_KEY' });
|
||||
|
||||
// v1 methods (feature‑frozen)
|
||||
const scrapeV1 = await app.v1.scrapeUrl('https://firecrawl.dev', { formats: ['markdown', 'html'] });
|
||||
const crawlV1 = await app.v1.crawlUrl('https://firecrawl.dev', { limit: 100 });
|
||||
const mapV1 = await app.v1.mapUrl('https://firecrawl.dev');
|
||||
```
|
||||
|
||||
## Error Handling
|
||||
|
||||
The SDK handles errors returned by the Firecrawl API and raises appropriate exceptions. If an error occurs during a request, an exception will be raised with a descriptive error message. The examples above demonstrate how to handle these errors using `try/catch` blocks.
|
||||
|
||||
## License
|
||||
|
||||
The Firecrawl Node SDK is licensed under the MIT License. This means you are free to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the SDK, subject to the following conditions:
|
||||
|
||||
- The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
|
||||
Please note that while this SDK is MIT licensed, it is part of a larger project which may be under different licensing terms. Always refer to the license information in the root directory of the main project for overall licensing details.
|
||||
7
참고/firecrawl-main/apps/js-sdk/firecrawl/audit-ci.jsonc
Normal file
7
참고/firecrawl-main/apps/js-sdk/firecrawl/audit-ci.jsonc
Normal file
@@ -0,0 +1,7 @@
|
||||
{
|
||||
"$schema": "https://github.com/IBM/audit-ci/raw/main/docs/schema.json",
|
||||
"low": true,
|
||||
"allowlist": [
|
||||
"GHSA-w5hq-g745-h8pq"
|
||||
]
|
||||
}
|
||||
16
참고/firecrawl-main/apps/js-sdk/firecrawl/jest.config.js
Normal file
16
참고/firecrawl-main/apps/js-sdk/firecrawl/jest.config.js
Normal file
@@ -0,0 +1,16 @@
|
||||
/** @type {import('ts-jest').JestConfigWithTsJest} **/
|
||||
export default {
|
||||
testEnvironment: "node",
|
||||
"moduleNameMapper": {
|
||||
"^(\\.{1,2}/.*)\\.js$": "$1",
|
||||
},
|
||||
"extensionsToTreatAsEsm": [".ts"],
|
||||
"transform": {
|
||||
"^.+\\.(mt|t|cj|j)s$": [
|
||||
"ts-jest",
|
||||
{
|
||||
"useESM": true
|
||||
}
|
||||
]
|
||||
},
|
||||
};
|
||||
77
참고/firecrawl-main/apps/js-sdk/firecrawl/package.json
Normal file
77
참고/firecrawl-main/apps/js-sdk/firecrawl/package.json
Normal file
@@ -0,0 +1,77 @@
|
||||
{
|
||||
"name": "@mendable/firecrawl-js",
|
||||
"version": "4.22.2",
|
||||
"description": "JavaScript SDK for Firecrawl API",
|
||||
"main": "dist/index.js",
|
||||
"types": "dist/index.d.ts",
|
||||
"exports": {
|
||||
"./package.json": "./package.json",
|
||||
".": {
|
||||
"import": "./dist/index.js",
|
||||
"default": "./dist/index.cjs"
|
||||
}
|
||||
},
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
"build": "tsup",
|
||||
"build-and-publish": "pnpm run build && pnpm publish --access public",
|
||||
"publish-beta": "pnpm run build && pnpm publish --access public --tag beta",
|
||||
"test": "NODE_OPTIONS=--experimental-vm-modules jest --verbose src/__tests__/e2e/v2/*.test.ts --detectOpenHandles",
|
||||
"test:unit": "NODE_OPTIONS=--experimental-vm-modules jest --verbose src/__tests__/unit/v2/*.test.ts"
|
||||
},
|
||||
"repository": {
|
||||
"type": "git",
|
||||
"url": "git+https://github.com/firecrawl/firecrawl.git"
|
||||
},
|
||||
"author": "Mendable.ai",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"axios": "1.15.2",
|
||||
"firecrawl": "4.16.0",
|
||||
"typescript-event-target": "^1.1.1",
|
||||
"zod": "^3.23.8",
|
||||
"zod-to-json-schema": "^3.23.0"
|
||||
},
|
||||
"bugs": {
|
||||
"url": "https://github.com/firecrawl/firecrawl/issues"
|
||||
},
|
||||
"homepage": "https://github.com/firecrawl/firecrawl#readme",
|
||||
"devDependencies": {
|
||||
"@jest/globals": "^30.2.0",
|
||||
"@types/dotenv": "^8.2.0",
|
||||
"@types/jest": "^30.0.0",
|
||||
"@types/mocha": "^10.0.6",
|
||||
"@types/node": "^20.12.12",
|
||||
"@types/uuid": "^9.0.8",
|
||||
"dotenv": "^16.4.5",
|
||||
"jest": "^30.2.0",
|
||||
"ts-jest": "^29.4.5",
|
||||
"tsup": "^8.5.0",
|
||||
"typescript": "^5.4.5",
|
||||
"uuid": "^9.0.1"
|
||||
},
|
||||
"keywords": [
|
||||
"firecrawl",
|
||||
"mendable",
|
||||
"crawler",
|
||||
"web",
|
||||
"scraper",
|
||||
"api",
|
||||
"sdk"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">=22.0.0"
|
||||
},
|
||||
"pnpm": {
|
||||
"overrides": {
|
||||
"@isaacs/brace-expansion@<=5.0.0": ">=5.0.1",
|
||||
"minimatch@<10.2.3": ">=10.2.3",
|
||||
"rollup@<4.59.0": ">=4.59.0",
|
||||
"picomatch@<4.0.4": ">=4.0.4",
|
||||
"handlebars": ">=4.7.9",
|
||||
"brace-expansion": ">=5.0.5",
|
||||
"axios@<1.15.2": "1.15.2",
|
||||
"follow-redirects@<1.16.0": ">=1.16.0 <2.0.0"
|
||||
}
|
||||
}
|
||||
}
|
||||
4138
참고/firecrawl-main/apps/js-sdk/firecrawl/pnpm-lock.yaml
generated
Normal file
4138
참고/firecrawl-main/apps/js-sdk/firecrawl/pnpm-lock.yaml
generated
Normal file
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,301 @@
|
||||
import FirecrawlApp, { type CrawlParams, type CrawlResponse, type CrawlStatusResponse, type MapResponse, type ScrapeResponse } from '../../../index';
|
||||
import { v4 as uuidv4 } from 'uuid';
|
||||
import dotenv from 'dotenv';
|
||||
import { describe, test, expect } from '@jest/globals';
|
||||
|
||||
dotenv.config();
|
||||
|
||||
const TEST_API_KEY = process.env.TEST_API_KEY;
|
||||
const API_URL = process.env.API_URL ?? "https://api.firecrawl.dev";
|
||||
|
||||
describe('FirecrawlApp E2E Tests', () => {
|
||||
test.concurrent('should throw error for no API key only for cloud service', async () => {
|
||||
if (API_URL.includes('api.firecrawl.dev')) {
|
||||
// Should throw for cloud service
|
||||
expect(() => {
|
||||
new FirecrawlApp({ apiKey: null, apiUrl: API_URL });
|
||||
}).toThrow("No API key provided");
|
||||
} else {
|
||||
// Should not throw for self-hosted
|
||||
expect(() => {
|
||||
new FirecrawlApp({ apiKey: null, apiUrl: API_URL });
|
||||
}).not.toThrow();
|
||||
}
|
||||
});
|
||||
|
||||
test.concurrent('should throw error for invalid API key on scrape', async () => {
|
||||
if (API_URL.includes('api.firecrawl.dev')) {
|
||||
const invalidApp = new FirecrawlApp({ apiKey: "invalid_api_key", apiUrl: API_URL });
|
||||
await expect(invalidApp.scrapeUrl('https://roastmywebsite.ai')).rejects.toThrow("Unexpected error occurred while trying to scrape URL. Status code: 401");
|
||||
} else {
|
||||
const invalidApp = new FirecrawlApp({ apiKey: "invalid_api_key", apiUrl: API_URL });
|
||||
await expect(invalidApp.scrapeUrl('https://roastmywebsite.ai')).resolves.not.toThrow();
|
||||
}
|
||||
});
|
||||
|
||||
test.concurrent('should throw error for unsupported URL on scrape', async () => {
|
||||
const app = new FirecrawlApp({ apiKey: TEST_API_KEY, apiUrl: API_URL });
|
||||
const unsupportedUrl = "https://facebook.com/fake-test";
|
||||
await expect(app.scrapeUrl(unsupportedUrl)).rejects.toThrow("do not support this site");
|
||||
});
|
||||
|
||||
test.concurrent('should return successful response for valid scrape', async () => {
|
||||
const app = new FirecrawlApp({ apiKey: TEST_API_KEY, apiUrl: API_URL });
|
||||
|
||||
const response = await app.scrapeUrl('https://roastmywebsite.ai');
|
||||
if (!response.success) {
|
||||
throw new Error(response.error);
|
||||
}
|
||||
}, 30000); // 30 seconds timeout
|
||||
|
||||
test.concurrent('should return successful response with valid API key and options', async () => {
|
||||
const app = new FirecrawlApp({ apiKey: TEST_API_KEY, apiUrl: API_URL });
|
||||
const response = await app.scrapeUrl(
|
||||
'https://roastmywebsite.ai', {
|
||||
formats: ['markdown', 'html', 'rawHtml', 'screenshot', 'links'],
|
||||
headers: { "x-key": "test" },
|
||||
includeTags: ['h1'],
|
||||
excludeTags: ['h2'],
|
||||
onlyMainContent: true,
|
||||
timeout: 30000,
|
||||
waitFor: 1000
|
||||
});
|
||||
|
||||
if (!response.success) {
|
||||
throw new Error(response.error);
|
||||
}
|
||||
}, 30000); // 30 seconds timeout
|
||||
|
||||
test.concurrent('should return successful response with valid API key and screenshot fullPage', async () => {
|
||||
const app = new FirecrawlApp({ apiKey: TEST_API_KEY, apiUrl: API_URL });
|
||||
const response = await app.scrapeUrl(
|
||||
'https://roastmywebsite.ai', {
|
||||
formats: ['screenshot@fullPage'],
|
||||
});
|
||||
if (!response.success) {
|
||||
throw new Error(response.error);
|
||||
}
|
||||
|
||||
expect(response.screenshot).not.toBeUndefined();
|
||||
expect(response.screenshot).not.toBeNull();
|
||||
expect(response.screenshot).toContain("https://");
|
||||
}, 30000); // 30 seconds timeout
|
||||
|
||||
test.concurrent('should return successful response for valid scrape with PDF file', async () => {
|
||||
const app = new FirecrawlApp({ apiKey: TEST_API_KEY, apiUrl: API_URL });
|
||||
const response = await app.scrapeUrl('https://arxiv.org/pdf/astro-ph/9301001.pdf');
|
||||
if (!response.success) {
|
||||
throw new Error(response.error);
|
||||
}
|
||||
|
||||
expect(response).not.toBeNull();
|
||||
expect(response?.markdown).toContain('We present spectrophotometric observations of the Broad Line Radio Galaxy');
|
||||
}, 30000); // 30 seconds timeout
|
||||
|
||||
test.concurrent('should return successful response for valid scrape with PDF file without explicit extension', async () => {
|
||||
const app = new FirecrawlApp({ apiKey: TEST_API_KEY, apiUrl: API_URL });
|
||||
const response = await app.scrapeUrl('https://arxiv.org/pdf/astro-ph/9301001');
|
||||
if (!response.success) {
|
||||
throw new Error(response.error);
|
||||
}
|
||||
|
||||
expect(response).not.toBeNull();
|
||||
expect(response?.markdown).toContain('We present spectrophotometric observations of the Broad Line Radio Galaxy');
|
||||
}, 30000); // 30 seconds timeout
|
||||
|
||||
test.concurrent('should return successful response for valid scrape with PDF file and parsePDF true', async () => {
|
||||
const app = new FirecrawlApp({ apiKey: TEST_API_KEY, apiUrl: API_URL });
|
||||
const response = await app.scrapeUrl('https://arxiv.org/pdf/astro-ph/9301001.pdf', {
|
||||
parsePDF: true
|
||||
});
|
||||
if (!response.success) {
|
||||
throw new Error(response.error);
|
||||
}
|
||||
|
||||
expect(response).not.toBeNull();
|
||||
expect(response?.markdown).toContain('We present spectrophotometric observations of the Broad Line Radio Galaxy');
|
||||
}, 30000); // 30 seconds timeout
|
||||
|
||||
test.concurrent('should return successful response for valid scrape with PDF file and parsePDF false', async () => {
|
||||
const app = new FirecrawlApp({ apiKey: TEST_API_KEY, apiUrl: API_URL });
|
||||
const response = await app.scrapeUrl('https://arxiv.org/pdf/astro-ph/9301001.pdf', {
|
||||
parsePDF: false
|
||||
});
|
||||
if (!response.success) {
|
||||
throw new Error(response.error);
|
||||
}
|
||||
|
||||
expect(response).not.toBeNull();
|
||||
expect(response?.markdown).toMatch(/^[A-Za-z0-9+/]+=*$/);
|
||||
}, 30000); // 30 seconds timeout
|
||||
|
||||
test.concurrent('should throw error for invalid API key on crawl', async () => {
|
||||
if (API_URL.includes('api.firecrawl.dev')) {
|
||||
const invalidApp = new FirecrawlApp({ apiKey: "invalid_api_key", apiUrl: API_URL });
|
||||
await expect(invalidApp.crawlUrl('https://roastmywebsite.ai')).rejects.toThrow("Request failed with status code 401");
|
||||
} else {
|
||||
const invalidApp = new FirecrawlApp({ apiKey: "invalid_api_key", apiUrl: API_URL });
|
||||
await expect(invalidApp.crawlUrl('https://roastmywebsite.ai')).resolves.not.toThrow();
|
||||
}
|
||||
});
|
||||
|
||||
test.concurrent('should return successful response for crawl and wait for completion', async () => {
|
||||
const app = new FirecrawlApp({ apiKey: TEST_API_KEY, apiUrl: API_URL });
|
||||
const response = await app.crawlUrl('https://roastmywebsite.ai', {}, 30) as CrawlStatusResponse;
|
||||
expect(response).not.toHaveProperty("next"); // wait until done
|
||||
expect(response.data.length).toBeGreaterThan(0);
|
||||
if (response.data[0]) {
|
||||
expect(response.data[0]).toHaveProperty("markdown");
|
||||
}
|
||||
}, 60000); // 60 seconds timeout
|
||||
|
||||
test.concurrent('should return successful response for crawl with options and wait for completion', async () => {
|
||||
const app = new FirecrawlApp({ apiKey: TEST_API_KEY, apiUrl: API_URL });
|
||||
const response = await app.crawlUrl('https://roastmywebsite.ai', {
|
||||
excludePaths: ['blog/*'],
|
||||
includePaths: ['/'],
|
||||
maxDepth: 2,
|
||||
ignoreSitemap: true,
|
||||
limit: 10,
|
||||
allowBackwardLinks: true,
|
||||
allowExternalLinks: true,
|
||||
scrapeOptions: {
|
||||
formats: ['markdown', 'html', 'rawHtml', 'screenshot', 'links'],
|
||||
headers: { "x-key": "test" },
|
||||
includeTags: ['h1'],
|
||||
excludeTags: ['h2'],
|
||||
onlyMainContent: true,
|
||||
waitFor: 1000
|
||||
}
|
||||
} as CrawlParams, 30) as CrawlStatusResponse;
|
||||
expect(response).not.toHaveProperty("next");
|
||||
expect(response.data.length).toBeGreaterThan(0);
|
||||
if (response.data[0]) {
|
||||
expect(response.data[0]).toHaveProperty("markdown");
|
||||
expect(response.data[0]).not.toHaveProperty('content'); // v0
|
||||
expect(response.data[0]).toHaveProperty("html");
|
||||
expect(response.data[0]).toHaveProperty("rawHtml");
|
||||
expect(response.data[0]).toHaveProperty("screenshot");
|
||||
expect(response.data[0]).toHaveProperty("links");
|
||||
}
|
||||
}, 60000); // 60 seconds timeout
|
||||
|
||||
test.concurrent('should handle idempotency key for crawl', async () => {
|
||||
const app = new FirecrawlApp({ apiKey: TEST_API_KEY, apiUrl: API_URL });
|
||||
const uniqueIdempotencyKey = uuidv4();
|
||||
const response = await app.asyncCrawlUrl('https://roastmywebsite.ai', {}, uniqueIdempotencyKey) as CrawlResponse;
|
||||
expect(response).not.toBeNull();
|
||||
expect(response.id).toBeDefined();
|
||||
|
||||
await expect(app.crawlUrl('https://roastmywebsite.ai', {}, 2, uniqueIdempotencyKey)).rejects.toThrow("Request failed with status code 409");
|
||||
});
|
||||
|
||||
test.concurrent('should check crawl status', async () => {
|
||||
const app = new FirecrawlApp({ apiKey: TEST_API_KEY, apiUrl: API_URL });
|
||||
const response = await app.asyncCrawlUrl('https://firecrawl.dev', { limit: 20, scrapeOptions: { formats: ['markdown', 'html', 'rawHtml', 'screenshot', 'links'] } } as CrawlParams) as CrawlResponse;
|
||||
expect(response).not.toBeNull();
|
||||
expect(response.id).toBeDefined();
|
||||
|
||||
let statusResponse = await app.checkCrawlStatus(response.id);
|
||||
const maxChecks = 15;
|
||||
let checks = 0;
|
||||
|
||||
expect(statusResponse.success).toBe(true);
|
||||
while ((statusResponse as any).status === 'scraping' && checks < maxChecks) {
|
||||
await new Promise(resolve => setTimeout(resolve, 5000));
|
||||
expect(statusResponse).not.toHaveProperty("partial_data"); // v0
|
||||
expect(statusResponse).not.toHaveProperty("current"); // v0
|
||||
expect(statusResponse).toHaveProperty("data");
|
||||
expect(statusResponse).toHaveProperty("total");
|
||||
expect(statusResponse).toHaveProperty("creditsUsed");
|
||||
expect(statusResponse).toHaveProperty("expiresAt");
|
||||
expect(statusResponse).toHaveProperty("status");
|
||||
expect(statusResponse).toHaveProperty("next");
|
||||
expect(statusResponse.success).toBe(true);
|
||||
if (statusResponse.success === true) {
|
||||
expect(statusResponse.total).toBeGreaterThan(0);
|
||||
expect(statusResponse.creditsUsed).toBeGreaterThan(0);
|
||||
expect(statusResponse.expiresAt.getTime()).toBeGreaterThan(Date.now());
|
||||
expect(statusResponse.status).toBe("scraping");
|
||||
expect(statusResponse.next).toContain("/v1/crawl/");
|
||||
}
|
||||
statusResponse = await app.checkCrawlStatus(response.id) as CrawlStatusResponse;
|
||||
expect(statusResponse.success).toBe(true);
|
||||
checks++;
|
||||
}
|
||||
|
||||
expect(statusResponse).not.toBeNull();
|
||||
expect(statusResponse).toHaveProperty("total");
|
||||
expect(statusResponse.success).toBe(true);
|
||||
if (statusResponse.success === true) {
|
||||
expect(statusResponse.status).toBe("completed");
|
||||
expect(statusResponse.data.length).toBeGreaterThan(0);
|
||||
}
|
||||
}, 60000); // 60 seconds timeout
|
||||
|
||||
test.concurrent('should throw error for invalid API key on map', async () => {
|
||||
if (API_URL.includes('api.firecrawl.dev')) {
|
||||
const invalidApp = new FirecrawlApp({ apiKey: "invalid_api_key", apiUrl: API_URL });
|
||||
await expect(invalidApp.mapUrl('https://roastmywebsite.ai')).rejects.toThrow("Request failed with status code 401");
|
||||
} else {
|
||||
const invalidApp = new FirecrawlApp({ apiKey: "invalid_api_key", apiUrl: API_URL });
|
||||
await expect(invalidApp.mapUrl('https://roastmywebsite.ai')).resolves.not.toThrow();
|
||||
}
|
||||
});
|
||||
|
||||
test.concurrent('should throw error for unsupported URL on map', async () => {
|
||||
const app = new FirecrawlApp({ apiKey: TEST_API_KEY, apiUrl: API_URL });
|
||||
const unsupportedUrl = "https://facebook.com/fake-test";
|
||||
await expect(app.mapUrl(unsupportedUrl)).rejects.toThrow("403");
|
||||
});
|
||||
|
||||
test.concurrent('should return successful response for valid map', async () => {
|
||||
const app = new FirecrawlApp({ apiKey: TEST_API_KEY, apiUrl: API_URL }); const response = await app.mapUrl('https://roastmywebsite.ai') as MapResponse;
|
||||
expect(response).not.toBeNull();
|
||||
|
||||
expect(response.links?.length).toBeGreaterThan(0);
|
||||
expect(response.links?.[0]).toContain("https://");
|
||||
const filteredLinks = response.links?.filter((link: string) => link.includes("roastmywebsite.ai"));
|
||||
expect(filteredLinks?.length).toBeGreaterThan(0);
|
||||
}, 30000); // 30 seconds timeout
|
||||
|
||||
|
||||
|
||||
test('should search with string query', async () => {
|
||||
const app = new FirecrawlApp({ apiUrl: API_URL, apiKey: TEST_API_KEY });
|
||||
const response = await app.search("firecrawl");
|
||||
expect(response.success).toBe(true);
|
||||
expect(response.data?.length).toBeGreaterThan(0);
|
||||
expect(response.data?.[0]?.markdown).not.toBeDefined();
|
||||
expect(response.data?.[0]?.title).toBeDefined();
|
||||
expect(response.data?.[0]?.description).toBeDefined();
|
||||
}, 30000); // 30 seconds timeout
|
||||
|
||||
test('should search with params object', async () => {
|
||||
const app = new FirecrawlApp({ apiUrl: API_URL, apiKey: TEST_API_KEY });
|
||||
const response = await app.search("firecrawl", {
|
||||
limit: 3,
|
||||
lang: 'en',
|
||||
country: 'us',
|
||||
scrapeOptions: {
|
||||
formats: ['markdown', 'html', 'links'],
|
||||
onlyMainContent: true
|
||||
}
|
||||
});
|
||||
expect(response.success).toBe(true);
|
||||
expect(response.data.length).toBeLessThanOrEqual(3);
|
||||
for (const doc of response.data) {
|
||||
expect(doc.markdown).toBeDefined();
|
||||
expect(doc.html).toBeDefined();
|
||||
expect(doc.links).toBeDefined();
|
||||
expect(doc.title).toBeDefined();
|
||||
expect(doc.description).toBeDefined();
|
||||
}
|
||||
}, 30000); // 30 seconds timeout
|
||||
|
||||
test('should handle invalid API key for search', async () => {
|
||||
const app = new FirecrawlApp({ apiUrl: API_URL, apiKey: "invalid_api_key" });
|
||||
await expect(app.search("test query")).rejects.toThrow("Request failed with status code 401");
|
||||
});
|
||||
|
||||
});
|
||||
@@ -0,0 +1,94 @@
|
||||
/**
|
||||
* E2E tests for v2 batch scrape (translated from Python tests)
|
||||
*/
|
||||
import Firecrawl from "../../../index";
|
||||
import { config } from "dotenv";
|
||||
import { getIdentity, getApiUrl } from "./utils/idmux";
|
||||
import { describe, test, expect, beforeAll } from "@jest/globals";
|
||||
|
||||
config();
|
||||
|
||||
const API_URL = getApiUrl();
|
||||
let client: Firecrawl;
|
||||
|
||||
beforeAll(async () => {
|
||||
const { apiKey } = await getIdentity({ name: "js-e2e-batch" });
|
||||
client = new Firecrawl({ apiKey, apiUrl: API_URL });
|
||||
});
|
||||
|
||||
describe("v2.batch e2e", () => {
|
||||
test("batch scrape minimal (wait)", async () => {
|
||||
const urls = [
|
||||
"https://docs.firecrawl.dev",
|
||||
"https://firecrawl.dev",
|
||||
];
|
||||
const job = await client.batchScrape(urls, { options: { formats: ["markdown"] }, pollInterval: 1, timeout: 180 });
|
||||
expect(["completed", "failed"]).toContain(job.status);
|
||||
expect(job.completed).toBeGreaterThanOrEqual(0);
|
||||
expect(job.total).toBeGreaterThanOrEqual(0);
|
||||
expect(Array.isArray(job.data)).toBe(true);
|
||||
}, 240_000);
|
||||
|
||||
test("batch scrape with wait returns job id for error retrieval", async () => {
|
||||
const urls = [
|
||||
"https://docs.firecrawl.dev",
|
||||
"https://firecrawl.dev",
|
||||
];
|
||||
const job = await client.batchScrape(urls, { options: { formats: ["markdown"] }, pollInterval: 1, timeout: 180 });
|
||||
// Verify job has id field
|
||||
expect(job.id).toBeDefined();
|
||||
expect(typeof job.id).toBe("string");
|
||||
// Verify we can use the id to retrieve errors
|
||||
const errors = await client.getBatchScrapeErrors(job.id!);
|
||||
expect(errors).toHaveProperty("errors");
|
||||
expect(errors).toHaveProperty("robotsBlocked");
|
||||
expect(Array.isArray(errors.errors)).toBe(true);
|
||||
expect(Array.isArray(errors.robotsBlocked)).toBe(true);
|
||||
}, 240_000);
|
||||
|
||||
test("start batch minimal and status", async () => {
|
||||
const urls = ["https://docs.firecrawl.dev", "https://firecrawl.dev"];
|
||||
const start = await client.startBatchScrape(urls, { options: { formats: ["markdown"] }, ignoreInvalidURLs: true });
|
||||
expect(typeof start.id).toBe("string");
|
||||
expect(typeof start.url).toBe("string");
|
||||
const status = await client.getBatchScrapeStatus(start.id);
|
||||
expect(["scraping", "completed", "failed", "cancelled"]).toContain(status.status);
|
||||
expect(status.total).toBeGreaterThanOrEqual(0);
|
||||
// Verify status includes id field
|
||||
expect(status.id).toBeDefined();
|
||||
expect(status.id).toBe(start.id);
|
||||
}, 120_000);
|
||||
|
||||
test("wait batch with all params", async () => {
|
||||
const urls = ["https://docs.firecrawl.dev", "https://firecrawl.dev"];
|
||||
const job = await client.batchScrape(urls, {
|
||||
options: {
|
||||
formats: [
|
||||
"markdown",
|
||||
{ type: "json", prompt: "Extract page title", schema: { type: "object", properties: { title: { type: "string" } }, required: ["title"] } },
|
||||
{ type: "changeTracking", prompt: "Track changes", modes: ["json"] },
|
||||
],
|
||||
onlyMainContent: true,
|
||||
mobile: false,
|
||||
},
|
||||
ignoreInvalidURLs: true,
|
||||
maxConcurrency: 2,
|
||||
zeroDataRetention: false,
|
||||
pollInterval: 1,
|
||||
timeout: 180,
|
||||
});
|
||||
expect(["completed", "failed", "cancelled"]).toContain(job.status);
|
||||
expect(job.completed).toBeGreaterThanOrEqual(0);
|
||||
expect(job.total).toBeGreaterThanOrEqual(0);
|
||||
expect(Array.isArray(job.data)).toBe(true);
|
||||
}, 300_000);
|
||||
|
||||
test("cancel batch", async () => {
|
||||
const urls = ["https://docs.firecrawl.dev", "https://firecrawl.dev"];
|
||||
const start = await client.startBatchScrape(urls, { options: { formats: ["markdown"] }, maxConcurrency: 1 });
|
||||
expect(typeof start.id).toBe("string");
|
||||
const cancelled = await client.cancelBatchScrape(start.id);
|
||||
expect(cancelled).toBe(true);
|
||||
}, 120_000);
|
||||
});
|
||||
|
||||
@@ -0,0 +1,199 @@
|
||||
/**
|
||||
* E2E tests for v2 crawl (translated from Python tests)
|
||||
*/
|
||||
import Firecrawl from "../../../index";
|
||||
import { config } from "dotenv";
|
||||
import { getIdentity, getApiUrl } from "./utils/idmux";
|
||||
import { describe, test, expect, beforeAll } from "@jest/globals";
|
||||
|
||||
config();
|
||||
|
||||
const API_URL = getApiUrl();
|
||||
let client: Firecrawl;
|
||||
|
||||
beforeAll(async () => {
|
||||
const { apiKey } = await getIdentity({ name: "js-e2e-crawl" });
|
||||
client = new Firecrawl({ apiKey, apiUrl: API_URL });
|
||||
});
|
||||
|
||||
describe("v2.crawl e2e", () => {
|
||||
|
||||
test("start crawl minimal request", async () => {
|
||||
if (!client) throw new Error();
|
||||
const job = await client.startCrawl("https://docs.firecrawl.dev", { limit: 3 });
|
||||
expect(typeof job.id).toBe("string");
|
||||
expect(typeof job.url).toBe("string");
|
||||
}, 90_000);
|
||||
|
||||
test("start crawl with options", async () => {
|
||||
if (!client) throw new Error();
|
||||
const job = await client.startCrawl("https://docs.firecrawl.dev", { limit: 5, maxDiscoveryDepth: 2 });
|
||||
expect(typeof job.id).toBe("string");
|
||||
expect(typeof job.url).toBe("string");
|
||||
}, 90_000);
|
||||
|
||||
test("start crawl with prompt", async () => {
|
||||
if (!client) throw new Error();
|
||||
const job = await client.startCrawl("https://firecrawl.dev", { prompt: "Extract all blog posts", limit: 3 });
|
||||
expect(typeof job.id).toBe("string");
|
||||
expect(typeof job.url).toBe("string");
|
||||
}, 90_000);
|
||||
|
||||
test("get crawl status", async () => {
|
||||
if (!client) throw new Error();
|
||||
const start = await client.startCrawl("https://docs.firecrawl.dev", { limit: 3 });
|
||||
const status = await client.getCrawlStatus(start.id);
|
||||
expect(["scraping", "completed", "failed", "cancelled"]).toContain(status.status);
|
||||
expect(status.completed).toBeGreaterThanOrEqual(0);
|
||||
// Verify status includes id field
|
||||
expect(status.id).toBeDefined();
|
||||
expect(status.id).toBe(start.id);
|
||||
// next/expiresAt may be null/undefined depending on state; check shape
|
||||
expect(Array.isArray(status.data)).toBe(true);
|
||||
}, 120_000);
|
||||
|
||||
test("cancel crawl", async () => {
|
||||
if (!client) throw new Error();
|
||||
const start = await client.startCrawl("https://docs.firecrawl.dev", { limit: 3 });
|
||||
const ok = await client.cancelCrawl(start.id);
|
||||
expect(ok).toBe(true);
|
||||
}, 120_000);
|
||||
|
||||
test("get crawl errors", async () => {
|
||||
if (!client) throw new Error();
|
||||
const start = await client.startCrawl("https://docs.firecrawl.dev", { limit: 3 });
|
||||
const resp = await client.getCrawlErrors(start.id);
|
||||
expect(resp).toHaveProperty("errors");
|
||||
expect(resp).toHaveProperty("robotsBlocked");
|
||||
expect(Array.isArray(resp.errors)).toBe(true);
|
||||
expect(Array.isArray(resp.robotsBlocked)).toBe(true);
|
||||
for (const e of resp.errors) {
|
||||
expect(typeof e.id === "string" || e.id == null).toBe(true);
|
||||
expect(typeof e.timestamp === "string" || e.timestamp == null).toBe(true);
|
||||
expect(typeof e.url === "string" || e.url == null).toBe(true);
|
||||
expect(typeof e.error === "string" || e.error == null).toBe(true);
|
||||
}
|
||||
}, 120_000);
|
||||
|
||||
test("get crawl errors with invalid id should throw", async () => {
|
||||
if (!client) throw new Error();
|
||||
await expect(client.getCrawlErrors("invalid-job-id-12345")).rejects.toThrow();
|
||||
}, 60_000);
|
||||
|
||||
test("get active crawls", async () => {
|
||||
if (!client) throw new Error();
|
||||
const active = await client.getActiveCrawls();
|
||||
expect(typeof active.success).toBe("boolean");
|
||||
expect(Array.isArray(active.crawls)).toBe(true);
|
||||
for (const c of active.crawls) {
|
||||
expect(typeof c.id).toBe("string");
|
||||
expect(typeof c.teamId).toBe("string");
|
||||
expect(typeof c.url).toBe("string");
|
||||
if (c.options != null) {
|
||||
expect(typeof c.options === "object").toBe(true);
|
||||
}
|
||||
}
|
||||
}, 90_000);
|
||||
|
||||
test("get active crawls with running crawl", async () => {
|
||||
if (!client) throw new Error();
|
||||
const start = await client.startCrawl("https://docs.firecrawl.dev", { limit: 5 });
|
||||
await new Promise(resolve => setTimeout(resolve, 300));
|
||||
const active = await client.getActiveCrawls();
|
||||
expect(Array.isArray(active.crawls)).toBe(true);
|
||||
const ids = active.crawls.map(c => c.id);
|
||||
expect(ids.includes(start.id)).toBe(true);
|
||||
await client.cancelCrawl(start.id);
|
||||
}, 120_000);
|
||||
|
||||
test("crawl with wait", async () => {
|
||||
if (!client) throw new Error();
|
||||
const job = await client.crawl("https://docs.firecrawl.dev", { limit: 3, maxDiscoveryDepth: 2, pollInterval: 1, timeout: 120 });
|
||||
expect(["completed", "failed"]).toContain(job.status);
|
||||
expect(job.completed).toBeGreaterThanOrEqual(0);
|
||||
expect(job.total).toBeGreaterThanOrEqual(0);
|
||||
expect(Array.isArray(job.data)).toBe(true);
|
||||
}, 180_000);
|
||||
|
||||
test("crawl with wait returns job id for error retrieval", async () => {
|
||||
if (!client) throw new Error();
|
||||
const job = await client.crawl("https://docs.firecrawl.dev", { limit: 3, maxDiscoveryDepth: 2, pollInterval: 1, timeout: 120 });
|
||||
// Verify job has id field
|
||||
expect(job.id).toBeDefined();
|
||||
expect(typeof job.id).toBe("string");
|
||||
// Verify we can use the id to retrieve errors
|
||||
const errors = await client.getCrawlErrors(job.id!);
|
||||
expect(errors).toHaveProperty("errors");
|
||||
expect(errors).toHaveProperty("robotsBlocked");
|
||||
expect(Array.isArray(errors.errors)).toBe(true);
|
||||
expect(Array.isArray(errors.robotsBlocked)).toBe(true);
|
||||
}, 180_000);
|
||||
|
||||
test("crawl with prompt and wait", async () => {
|
||||
if (!client) throw new Error();
|
||||
const job = await client.crawl("https://docs.firecrawl.dev", { prompt: "Extract all blog posts", limit: 3, pollInterval: 1, timeout: 120 });
|
||||
expect(["completed", "failed"]).toContain(job.status);
|
||||
expect(job.completed).toBeGreaterThanOrEqual(0);
|
||||
expect(job.total).toBeGreaterThanOrEqual(0);
|
||||
expect(Array.isArray(job.data)).toBe(true);
|
||||
}, 180_000);
|
||||
|
||||
test("crawl with scrape options", async () => {
|
||||
if (!client) throw new Error();
|
||||
const job = await client.startCrawl("https://docs.firecrawl.dev", {
|
||||
limit: 2,
|
||||
scrapeOptions: { formats: ["markdown", "links"], onlyMainContent: false, mobile: true },
|
||||
});
|
||||
expect(typeof job.id).toBe("string");
|
||||
}, 120_000);
|
||||
|
||||
test("crawl with json format object", async () => {
|
||||
if (!client) throw new Error();
|
||||
const job = await client.startCrawl("https://docs.firecrawl.dev", {
|
||||
limit: 2,
|
||||
scrapeOptions: { formats: [{ type: "json", prompt: "Extract page title", schema: { type: "object", properties: { title: { type: "string" } }, required: ["title"] } }] },
|
||||
});
|
||||
expect(typeof job.id).toBe("string");
|
||||
}, 120_000);
|
||||
|
||||
test("crawl all parameters", async () => {
|
||||
if (!client) throw new Error();
|
||||
const job = await client.startCrawl("https://docs.firecrawl.dev", {
|
||||
prompt: "Extract all blog posts and documentation",
|
||||
includePaths: ["/blog/*", "/docs/*"],
|
||||
excludePaths: ["/admin/*"],
|
||||
maxDiscoveryDepth: 3,
|
||||
sitemap: "skip",
|
||||
ignoreQueryParameters: true,
|
||||
limit: 5,
|
||||
crawlEntireDomain: true,
|
||||
allowExternalLinks: false,
|
||||
allowSubdomains: true,
|
||||
delay: 1,
|
||||
maxConcurrency: 2,
|
||||
webhook: "https://example.com/hook",
|
||||
scrapeOptions: {
|
||||
formats: ["markdown", "html"],
|
||||
headers: { "User-Agent": "Test Bot" },
|
||||
includeTags: ["h1", "h2"],
|
||||
excludeTags: ["nav"],
|
||||
onlyMainContent: false,
|
||||
timeout: 15_000,
|
||||
waitFor: 2000,
|
||||
mobile: true,
|
||||
skipTlsVerification: true,
|
||||
removeBase64Images: false,
|
||||
},
|
||||
zeroDataRetention: false,
|
||||
});
|
||||
expect(typeof job.id).toBe("string");
|
||||
}, 180_000);
|
||||
|
||||
test("crawl params preview", async () => {
|
||||
if (!client) throw new Error();
|
||||
const params = await client.crawlParamsPreview("https://docs.firecrawl.dev", "Extract all blog posts and documentation");
|
||||
expect(params && typeof params === "object").toBe(true);
|
||||
// Optional fields may or may not be present; just assert object shape
|
||||
}, 60_000);
|
||||
});
|
||||
|
||||
@@ -0,0 +1,70 @@
|
||||
/**
|
||||
* E2E tests for v2 extract (proxied to v1), translated from Python tests
|
||||
*/
|
||||
import Firecrawl from "../../../index";
|
||||
import { config } from "dotenv";
|
||||
import { getIdentity, getApiUrl } from "./utils/idmux";
|
||||
import { describe, test, expect, beforeAll } from "@jest/globals";
|
||||
import { z } from "zod";
|
||||
|
||||
config();
|
||||
|
||||
const API_URL = getApiUrl();
|
||||
let client: Firecrawl;
|
||||
|
||||
beforeAll(async () => {
|
||||
const { apiKey } = await getIdentity({ name: "js-e2e-extract" });
|
||||
client = new Firecrawl({ apiKey, apiUrl: API_URL });
|
||||
});
|
||||
|
||||
describe("v2.extract e2e", () => {
|
||||
test("extract minimal with prompt", async () => {
|
||||
const resp = await client.extract({ urls: ["https://docs.firecrawl.dev"], prompt: "Extract the main page title" });
|
||||
expect(typeof resp.success === "boolean" || resp.success == null).toBe(true);
|
||||
}, 120_000);
|
||||
|
||||
test("extract with schema", async () => {
|
||||
const schema = {
|
||||
type: "object",
|
||||
properties: { title: { type: "string" } },
|
||||
required: ["title"],
|
||||
} as const;
|
||||
const resp = await client.extract({
|
||||
urls: ["https://docs.firecrawl.dev"],
|
||||
schema,
|
||||
prompt: "Extract the main page title",
|
||||
showSources: true,
|
||||
enableWebSearch: false,
|
||||
});
|
||||
expect(typeof resp.success === "boolean" || resp.success == null).toBe(true);
|
||||
if ((resp as any).sources != null) {
|
||||
expect(typeof (resp as any).sources).toBe("object");
|
||||
}
|
||||
if (resp.data != null) {
|
||||
expect(typeof resp.data).toBe("object");
|
||||
expect((resp.data as any).title).toBeTruthy();
|
||||
}
|
||||
}, 180_000);
|
||||
|
||||
test("extract with zod schema", async () => {
|
||||
const schema = z.object({
|
||||
title: z.string(),
|
||||
});
|
||||
const resp = await client.extract({
|
||||
urls: ["https://docs.firecrawl.dev"],
|
||||
schema: schema,
|
||||
prompt: "Extract the main page title",
|
||||
showSources: true,
|
||||
enableWebSearch: false,
|
||||
});
|
||||
expect(typeof resp.success === "boolean" || resp.success == null).toBe(true);
|
||||
if ((resp as any).sources != null) {
|
||||
expect(typeof (resp as any).sources).toBe("object");
|
||||
}
|
||||
if (resp.data != null) {
|
||||
expect(typeof resp.data).toBe("object");
|
||||
expect(schema.safeParse(resp.data).success).toBe(true);
|
||||
}
|
||||
}, 180_000);
|
||||
});
|
||||
|
||||
@@ -0,0 +1,55 @@
|
||||
/**
|
||||
* E2E tests for v2 map (translated from Python tests)
|
||||
*/
|
||||
import Firecrawl from "../../../index";
|
||||
import { config } from "dotenv";
|
||||
import { getIdentity, getApiUrl } from "./utils/idmux";
|
||||
import { describe, test, expect, beforeAll } from "@jest/globals";
|
||||
|
||||
config();
|
||||
|
||||
const API_URL = getApiUrl();
|
||||
let client: Firecrawl;
|
||||
|
||||
beforeAll(async () => {
|
||||
const { apiKey } = await getIdentity({ name: "js-e2e-map" });
|
||||
client = new Firecrawl({ apiKey, apiUrl: API_URL });
|
||||
});
|
||||
|
||||
describe("v2.map e2e", () => {
|
||||
|
||||
test("minimal request", async () => {
|
||||
if (!client) throw new Error();
|
||||
const resp = await client.map("https://docs.firecrawl.dev");
|
||||
|
||||
expect(resp).toBeTruthy();
|
||||
expect(Array.isArray(resp.links)).toBe(true);
|
||||
|
||||
if (resp.links.length > 0) {
|
||||
const first: any = resp.links[0];
|
||||
expect(typeof first.url).toBe("string");
|
||||
expect(first.url.startsWith("http")).toBe(true);
|
||||
}
|
||||
}, 90_000);
|
||||
|
||||
test.each(["only", "skip", "include"]) ("with options sitemap=%s", async (sitemap) => {
|
||||
if (!client) throw new Error();
|
||||
const resp = await client.map("https://docs.firecrawl.dev", {
|
||||
search: "docs",
|
||||
includeSubdomains: true,
|
||||
limit: 10,
|
||||
sitemap: sitemap as any,
|
||||
timeout: 15_000,
|
||||
});
|
||||
|
||||
expect(resp).toBeTruthy();
|
||||
expect(Array.isArray(resp.links)).toBe(true);
|
||||
expect(resp.links.length).toBeLessThanOrEqual(10);
|
||||
|
||||
for (const link of resp.links as any[]) {
|
||||
expect(typeof link.url).toBe("string");
|
||||
expect(link.url.startsWith("http")).toBe(true);
|
||||
}
|
||||
}, 120_000);
|
||||
});
|
||||
|
||||
@@ -0,0 +1,67 @@
|
||||
import Firecrawl from "../../../index";
|
||||
import { config } from "dotenv";
|
||||
import { getIdentity, getApiUrl } from "./utils/idmux";
|
||||
import { describe, test, expect, beforeAll } from "@jest/globals";
|
||||
|
||||
config();
|
||||
|
||||
const API_URL = getApiUrl();
|
||||
let client: Firecrawl;
|
||||
|
||||
beforeAll(async () => {
|
||||
const { apiKey } = await getIdentity({ name: "js-e2e-parse" });
|
||||
client = new Firecrawl({ apiKey, apiUrl: API_URL });
|
||||
});
|
||||
|
||||
describe("v2.parse e2e", () => {
|
||||
test(
|
||||
"parses uploaded HTML files",
|
||||
async () => {
|
||||
if (!client) throw new Error();
|
||||
|
||||
const doc = await client.parse(
|
||||
{
|
||||
data: `
|
||||
<!DOCTYPE html>
|
||||
<html>
|
||||
<body>
|
||||
<h1>JS SDK Parse E2E</h1>
|
||||
<p>multipart upload body</p>
|
||||
</body>
|
||||
</html>
|
||||
`,
|
||||
filename: "parse-e2e.html",
|
||||
contentType: "text/html",
|
||||
},
|
||||
{
|
||||
formats: ["markdown"],
|
||||
},
|
||||
);
|
||||
|
||||
expect(doc.markdown).toContain("JS SDK Parse E2E");
|
||||
expect(doc.metadata?.creditsUsed).toBe(1);
|
||||
},
|
||||
60_000,
|
||||
);
|
||||
|
||||
test(
|
||||
"returns errors for unsupported file types",
|
||||
async () => {
|
||||
if (!client) throw new Error();
|
||||
|
||||
await expect(
|
||||
client.parse(
|
||||
{
|
||||
data: Buffer.from("image-data"),
|
||||
filename: "parse-e2e.png",
|
||||
contentType: "image/png",
|
||||
},
|
||||
{
|
||||
formats: ["markdown"],
|
||||
},
|
||||
),
|
||||
).rejects.toThrow();
|
||||
},
|
||||
60_000,
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,163 @@
|
||||
/**
|
||||
* E2E tests for v2 scrape
|
||||
*/
|
||||
import Firecrawl from "../../../index";
|
||||
import { z } from "zod";
|
||||
import { config } from "dotenv";
|
||||
import { getIdentity, getApiUrl } from "./utils/idmux";
|
||||
import { describe, test, expect, beforeAll } from "@jest/globals";
|
||||
|
||||
config();
|
||||
|
||||
const API_URL = getApiUrl();
|
||||
let client: Firecrawl;
|
||||
|
||||
beforeAll(async () => {
|
||||
const { apiKey } = await getIdentity({ name: "js-e2e-scrape" });
|
||||
client = new Firecrawl({ apiKey, apiUrl: API_URL });
|
||||
});
|
||||
|
||||
describe("v2.scrape e2e", () => {
|
||||
|
||||
const assertValidDocument = (doc: any) => {
|
||||
expect(doc).toBeTruthy();
|
||||
const hasContent = Boolean(doc.markdown?.length) || Boolean(doc.html?.length) || Boolean(doc.rawHtml?.length);
|
||||
expect(hasContent).toBe(true);
|
||||
expect(doc.metadata).toBeTruthy();
|
||||
};
|
||||
|
||||
test("minimal: scrape only required params", async () => {
|
||||
if (!client) throw new Error();
|
||||
const doc = await client.scrape("https://docs.firecrawl.dev");
|
||||
assertValidDocument(doc);
|
||||
}, 60_000);
|
||||
|
||||
test("maximal: scrape with all options", async () => {
|
||||
if (!client) throw new Error();
|
||||
const doc = await client.scrape("https://docs.firecrawl.dev", {
|
||||
formats: [
|
||||
"markdown",
|
||||
"html",
|
||||
"rawHtml",
|
||||
"links",
|
||||
{ type: "screenshot", fullPage: true, quality: 80, viewport: { width: 1280, height: 800 } },
|
||||
{
|
||||
type: "json",
|
||||
prompt: "Summarize the page and list links",
|
||||
schema: {
|
||||
type: "object",
|
||||
properties: {
|
||||
summary: { type: "string" },
|
||||
links: { type: "array", items: { type: "string", format: "uri" } },
|
||||
},
|
||||
required: ["summary"],
|
||||
},
|
||||
},
|
||||
],
|
||||
parsers: ["pdf"],
|
||||
headers: { "User-Agent": "firecrawl-tests" },
|
||||
includeTags: ["article"],
|
||||
excludeTags: ["nav"],
|
||||
onlyMainContent: true,
|
||||
waitFor: 1000,
|
||||
timeout: 30_000,
|
||||
location: { country: "us", languages: ["en"] },
|
||||
mobile: false,
|
||||
skipTlsVerification: false,
|
||||
removeBase64Images: true,
|
||||
blockAds: true,
|
||||
proxy: "auto",
|
||||
storeInCache: true,
|
||||
maxAge: 60_000,
|
||||
});
|
||||
assertValidDocument(doc);
|
||||
}, 90_000);
|
||||
|
||||
test("json format with zod schema (auto-converted internally)", async () => {
|
||||
if (!client) throw new Error();
|
||||
const zodSchema = z.object({
|
||||
title: z.string().min(1),
|
||||
items: z.array(z.string().url()).optional(),
|
||||
});
|
||||
|
||||
const doc = await client.scrape("https://docs.firecrawl.dev", {
|
||||
formats: [
|
||||
{
|
||||
type: "json",
|
||||
prompt: "Extract title and items",
|
||||
schema: zodSchema,
|
||||
},
|
||||
],
|
||||
});
|
||||
expect(doc).toBeTruthy();
|
||||
}, 90_000);
|
||||
|
||||
test("summary format returns summary string", async () => {
|
||||
if (!client) throw new Error();
|
||||
const doc = await client.scrape("https://firecrawl.dev", { formats: ["summary"] });
|
||||
expect(typeof doc.summary).toBe("string");
|
||||
expect((doc.summary || "").length).toBeGreaterThan(10);
|
||||
}, 90_000);
|
||||
|
||||
test.each([
|
||||
["markdown", "markdown"],
|
||||
["html", "html"],
|
||||
["rawHtml", "rawHtml"],
|
||||
["links", "links"],
|
||||
["screenshot", "screenshot"],
|
||||
])("basic format: %s", async (fmt, expectField) => {
|
||||
if (!client) throw new Error();
|
||||
const doc = await client.scrape("https://docs.firecrawl.dev", { formats: [fmt as any] });
|
||||
if (expectField !== "links" && expectField !== "screenshot") {
|
||||
assertValidDocument(doc);
|
||||
}
|
||||
if (expectField === "markdown") expect(doc.markdown).toBeTruthy();
|
||||
if (expectField === "html") expect(doc.html).toBeTruthy();
|
||||
if (expectField === "rawHtml") expect(doc.rawHtml).toBeTruthy();
|
||||
if (expectField === "screenshot") expect(doc.screenshot).toBeTruthy();
|
||||
if (expectField === "links") {
|
||||
expect(Array.isArray(doc.links)).toBe(true);
|
||||
expect((doc.links || []).length).toBeGreaterThan(0);
|
||||
}
|
||||
}, 90_000);
|
||||
|
||||
test("images format: extract all images from webpage", async () => {
|
||||
if (!client) throw new Error();
|
||||
const doc = await client.scrape("https://firecrawl.dev", {
|
||||
formats: ["images"],
|
||||
});
|
||||
expect(doc.images).toBeTruthy();
|
||||
expect(Array.isArray(doc.images)).toBe(true);
|
||||
expect(doc.images?.length).toBeGreaterThan(0);
|
||||
// Should find firecrawl logo/branding images
|
||||
expect(doc.images?.some(img => img.includes("firecrawl") || img.includes("logo"))).toBe(true);
|
||||
}, 60_000);
|
||||
|
||||
test("images format: works with multiple formats", async () => {
|
||||
if (!client) throw new Error();
|
||||
const doc = await client.scrape("https://github.com", {
|
||||
formats: ["markdown", "links", "images"],
|
||||
});
|
||||
expect(doc.markdown).toBeTruthy();
|
||||
expect(doc.links).toBeTruthy();
|
||||
expect(doc.images).toBeTruthy();
|
||||
expect(Array.isArray(doc.images)).toBe(true);
|
||||
expect(doc.images?.length).toBeGreaterThan(0);
|
||||
|
||||
// Images should find things not available in links format
|
||||
const imageExtensions = ['.jpg', '.jpeg', '.png', '.gif', '.webp', '.svg', '.ico'];
|
||||
const linkImages = doc.links?.filter(link =>
|
||||
imageExtensions.some(ext => link.toLowerCase().includes(ext))
|
||||
) || [];
|
||||
|
||||
// Should discover additional images beyond those with obvious extensions
|
||||
expect(doc.images?.length).toBeGreaterThanOrEqual(linkImages.length);
|
||||
}, 60_000);
|
||||
|
||||
test("invalid url should throw", async () => {
|
||||
if (!client) throw new Error();
|
||||
await expect(client.scrape("")).rejects.toThrow("URL cannot be empty");
|
||||
await expect(client.scrape(" ")).rejects.toThrow("URL cannot be empty");
|
||||
});
|
||||
});
|
||||
|
||||
@@ -0,0 +1,247 @@
|
||||
/**
|
||||
* E2E tests for v2 search (translated from Python tests)
|
||||
*/
|
||||
import Firecrawl from "../../../index";
|
||||
import type { Document, SearchResultWeb, SearchResultNews, SearchResultImages } from "../../../index";
|
||||
import { config } from "dotenv";
|
||||
import { getIdentity, getApiUrl } from "./utils/idmux";
|
||||
import { describe, test, expect, beforeAll } from "@jest/globals";
|
||||
|
||||
config();
|
||||
|
||||
const API_URL = getApiUrl();
|
||||
let client: Firecrawl;
|
||||
|
||||
beforeAll(async () => {
|
||||
const { apiKey } = await getIdentity({ name: "js-e2e-search" });
|
||||
client = new Firecrawl({ apiKey, apiUrl: API_URL });
|
||||
});
|
||||
|
||||
function collectTexts(entries: any[] | undefined): string[] {
|
||||
const texts: string[] = [];
|
||||
for (const r of entries || []) {
|
||||
const title = (r && typeof r === 'object') ? (r.title as unknown as string | undefined) : undefined;
|
||||
const desc = (r && typeof r === 'object') ? (r.description as unknown as string | undefined) : undefined;
|
||||
if (title) texts.push(String(title).toLowerCase());
|
||||
if (desc) texts.push(String(desc).toLowerCase());
|
||||
}
|
||||
return texts;
|
||||
}
|
||||
|
||||
function isDocument(entry: Document | SearchResultWeb | SearchResultNews | SearchResultImages | undefined | null): entry is Document {
|
||||
if (!entry) return false;
|
||||
const d = entry as Document;
|
||||
return (
|
||||
typeof d.markdown === 'string' ||
|
||||
typeof d.rawHtml === 'string' ||
|
||||
typeof d.html === 'string' ||
|
||||
typeof d.links === 'object' ||
|
||||
typeof d.screenshot === 'string' ||
|
||||
typeof d.changeTracking === 'object' ||
|
||||
typeof d.summary === 'string' ||
|
||||
typeof d.json === 'object'
|
||||
);
|
||||
}
|
||||
|
||||
describe("v2.search e2e", () => {
|
||||
|
||||
test("minimal request", async () => {
|
||||
if (!client) throw new Error();
|
||||
const results = await client.search("What is the capital of France?");
|
||||
expect(results).toBeTruthy();
|
||||
expect(results).toHaveProperty("web");
|
||||
expect(results).not.toHaveProperty("news");
|
||||
expect(results).not.toHaveProperty("images");
|
||||
|
||||
expect(results.web).toBeTruthy();
|
||||
expect((results.web || []).length).toBeGreaterThan(0);
|
||||
|
||||
for (const result of results.web || []) {
|
||||
if (isDocument(result)) {
|
||||
// documents appear if scraping happens
|
||||
continue;
|
||||
}
|
||||
expect(typeof result.url).toBe("string");
|
||||
expect(result.url.startsWith("http")).toBe(true);
|
||||
expect(typeof result.title === "string" || result.title == null).toBe(true);
|
||||
expect(typeof result.description === "string" || result.description == null).toBe(true);
|
||||
}
|
||||
|
||||
const allText = collectTexts(results.web).join(" ");
|
||||
expect(allText.includes("paris")).toBe(true);
|
||||
|
||||
expect(results.news == null).toBe(true);
|
||||
expect(results.images == null).toBe(true);
|
||||
}, 90_000);
|
||||
|
||||
test("with sources web+news and limit", async () => {
|
||||
if (!client) throw new Error();
|
||||
const results = await client.search("firecrawl", { sources: ["web", "news"], limit: 3 });
|
||||
expect(results).toBeTruthy();
|
||||
expect(results.web).toBeTruthy();
|
||||
expect((results.web || []).length).toBeLessThanOrEqual(3);
|
||||
if (results.news != null) {
|
||||
expect((results.news || []).length).toBeLessThanOrEqual(3);
|
||||
}
|
||||
expect(results.images == null).toBe(true);
|
||||
|
||||
const webTitles = (results.web || [])
|
||||
.filter((r): r is SearchResultWeb => !isDocument(r))
|
||||
.map(r => (r.title || "").toString().toLowerCase());
|
||||
const webDescriptions = (results.web || [])
|
||||
.filter((r): r is SearchResultWeb => !isDocument(r))
|
||||
.map(r => (r.description || "").toString().toLowerCase());
|
||||
const allWebText = (webTitles.concat(webDescriptions)).join(" ");
|
||||
expect(allWebText.includes("firecrawl")).toBe(true);
|
||||
}, 90_000);
|
||||
|
||||
test("result structure", async () => {
|
||||
if (!client) throw new Error();
|
||||
const results = await client.search("test query", { limit: 1 });
|
||||
if (results.web && results.web.length > 0) {
|
||||
const result: any = results.web[0];
|
||||
expect(result).toHaveProperty("url");
|
||||
expect(result).toHaveProperty("title");
|
||||
expect(result).toHaveProperty("description");
|
||||
expect(typeof result.url).toBe("string");
|
||||
expect(typeof result.title === "string" || result.title == null).toBe(true);
|
||||
expect(typeof result.description === "string" || result.description == null).toBe(true);
|
||||
expect(result.url.startsWith("http")).toBe(true);
|
||||
}
|
||||
}, 90_000);
|
||||
|
||||
test("all parameters (comprehensive)", async () => {
|
||||
if (!client) throw new Error();
|
||||
const schema = {
|
||||
type: "object",
|
||||
properties: {
|
||||
title: { type: "string" },
|
||||
description: { type: "string" },
|
||||
url: { type: "string" },
|
||||
},
|
||||
required: ["title", "description"],
|
||||
} as const;
|
||||
|
||||
const results = await client.search("artificial intelligence", {
|
||||
sources: [ "web", "news", "images" ],
|
||||
limit: 3,
|
||||
tbs: "qdr:m",
|
||||
location: "US",
|
||||
ignoreInvalidURLs: true,
|
||||
timeout: 60_000,
|
||||
scrapeOptions: {
|
||||
formats: [
|
||||
"markdown",
|
||||
"html",
|
||||
{ type: "json", prompt: "Extract the title and description from the page", schema },
|
||||
],
|
||||
headers: { "User-Agent": "Firecrawl-Test/1.0" },
|
||||
includeTags: ["h1", "h2", "p"],
|
||||
excludeTags: ["nav", "footer"],
|
||||
onlyMainContent: true,
|
||||
waitFor: 2000,
|
||||
mobile: false,
|
||||
skipTlsVerification: false,
|
||||
removeBase64Images: true,
|
||||
blockAds: true,
|
||||
proxy: "basic",
|
||||
maxAge: 3_600_000,
|
||||
storeInCache: true,
|
||||
location: { country: "US", languages: ["en"] },
|
||||
actions: [{ type: "wait", milliseconds: 1000 }],
|
||||
},
|
||||
});
|
||||
|
||||
expect(results).toBeTruthy();
|
||||
expect(results).toHaveProperty("web");
|
||||
expect(results).toHaveProperty("news");
|
||||
expect(results).toHaveProperty("images");
|
||||
|
||||
expect(results.web).toBeTruthy();
|
||||
expect((results.web || []).length).toBeLessThanOrEqual(3);
|
||||
|
||||
const nonDocEntries = (results.web || []).filter(r => !isDocument(r));
|
||||
if (nonDocEntries.length > 0) {
|
||||
const allWebText = collectTexts(nonDocEntries).join(" ");
|
||||
const aiTerms = ["artificial", "intelligence", "ai", "machine", "learning"];
|
||||
expect(aiTerms.some(t => allWebText.includes(t))).toBe(true);
|
||||
}
|
||||
|
||||
for (const result of results.web || []) {
|
||||
if (isDocument(result)) {
|
||||
expect(Boolean(result.markdown) || Boolean(result.html)).toBe(true);
|
||||
} else {
|
||||
expect(typeof result.url).toBe("string");
|
||||
expect(result.url.startsWith("http")).toBe(true);
|
||||
}
|
||||
}
|
||||
|
||||
if (results.news != null) {
|
||||
expect((results.news || []).length).toBeLessThanOrEqual(3);
|
||||
for (const result of results.news || []) {
|
||||
if (isDocument(result)) {
|
||||
expect(Boolean(result.markdown) || Boolean(result.html)).toBe(true);
|
||||
} else {
|
||||
expect(typeof result.url).toBe("string");
|
||||
expect(result.url?.startsWith("http")).toBe(true);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
expect(results.images).toBeTruthy();
|
||||
expect((results.images || []).length).toBeLessThanOrEqual(3);
|
||||
for (const result of results.images || []) {
|
||||
if (!isDocument(result)) {
|
||||
expect(typeof result.url).toBe("string");
|
||||
expect(result.url?.startsWith("http")).toBe(true);
|
||||
}
|
||||
}
|
||||
}, 120_000);
|
||||
|
||||
test("formats flexibility: list vs object", async () => {
|
||||
if (!client) throw new Error();
|
||||
const results1 = await client.search("python programming", {
|
||||
limit: 1,
|
||||
scrapeOptions: { formats: ["markdown"] },
|
||||
});
|
||||
const results2 = await client.search("python programming", {
|
||||
limit: 1,
|
||||
scrapeOptions: { formats: ["markdown"] },
|
||||
});
|
||||
expect(results1).toBeTruthy();
|
||||
expect(results2).toBeTruthy();
|
||||
expect(results1.web).toBeTruthy();
|
||||
expect(results2.web).toBeTruthy();
|
||||
}, 90_000);
|
||||
|
||||
test("with json format object", async () => {
|
||||
if (!client) throw new Error();
|
||||
const jsonSchema = {
|
||||
type: "object",
|
||||
properties: { title: { type: "string" } },
|
||||
required: ["title"],
|
||||
} as const;
|
||||
const results = await client.search("site:docs.firecrawl.dev", {
|
||||
limit: 1,
|
||||
scrapeOptions: {
|
||||
formats: [{ type: "json", prompt: "Extract page title", schema: jsonSchema }],
|
||||
},
|
||||
});
|
||||
expect(results).toBeTruthy();
|
||||
expect(Array.isArray(results.web) || results.web == null).toBe(true);
|
||||
}, 90_000);
|
||||
|
||||
test("with summary format, documents include summary when present", async () => {
|
||||
if (!client) throw new Error();
|
||||
const results = await client.search("site:firecrawl.dev", {
|
||||
limit: 1,
|
||||
scrapeOptions: { formats: ["summary"] },
|
||||
});
|
||||
const docs = (results.web || []).filter(r => isDocument(r)) as Document[];
|
||||
if (docs.length > 0) {
|
||||
expect(typeof docs[0].summary).toBe("string");
|
||||
expect((docs[0].summary || "").length).toBeGreaterThan(5);
|
||||
}
|
||||
}, 90_000);
|
||||
});
|
||||
|
||||
@@ -0,0 +1,44 @@
|
||||
/**
|
||||
* E2E tests for v2 usage endpoints (translated from Python tests)
|
||||
*/
|
||||
import Firecrawl from "../../../index";
|
||||
import { config } from "dotenv";
|
||||
import { getIdentity, getApiUrl } from "./utils/idmux";
|
||||
import { describe, test, expect, beforeAll } from "@jest/globals";
|
||||
|
||||
config();
|
||||
|
||||
const API_URL = getApiUrl();
|
||||
let client: Firecrawl;
|
||||
|
||||
beforeAll(async () => {
|
||||
const { apiKey } = await getIdentity({ name: "js-e2e-usage" });
|
||||
client = new Firecrawl({ apiKey, apiUrl: API_URL });
|
||||
});
|
||||
|
||||
describe("v2.usage e2e", () => {
|
||||
test("get_concurrency", async () => {
|
||||
const resp = await client.getConcurrency();
|
||||
expect(typeof resp.concurrency).toBe("number");
|
||||
expect(typeof resp.maxConcurrency).toBe("number");
|
||||
}, 60_000);
|
||||
|
||||
test("get_credit_usage", async () => {
|
||||
const resp = await client.getCreditUsage();
|
||||
expect(typeof resp.remainingCredits).toBe("number");
|
||||
}, 60_000);
|
||||
|
||||
test("get_token_usage", async () => {
|
||||
const resp = await client.getTokenUsage();
|
||||
expect(typeof resp.remainingTokens).toBe("number");
|
||||
}, 60_000);
|
||||
|
||||
test("get_queue_status", async () => {
|
||||
const resp = await client.getQueueStatus();
|
||||
expect(typeof resp.jobsInQueue).toBe("number");
|
||||
expect(typeof resp.activeJobsInQueue).toBe("number");
|
||||
expect(typeof resp.waitingJobsInQueue).toBe("number");
|
||||
expect(typeof resp.maxConcurrency).toBe("number");
|
||||
}, 60_000);
|
||||
});
|
||||
|
||||
@@ -0,0 +1,58 @@
|
||||
export type IdmuxRequest = {
|
||||
name?: string;
|
||||
concurrency?: number;
|
||||
credits?: number;
|
||||
tokens?: number;
|
||||
teamId?: string;
|
||||
flags?: Record<string, unknown>;
|
||||
};
|
||||
|
||||
export type Identity = {
|
||||
apiKey: string;
|
||||
teamId: string;
|
||||
};
|
||||
|
||||
let cachedIdentity: Identity | null = null;
|
||||
|
||||
export function getApiUrl(): string {
|
||||
return process.env.TEST_URL ?? process.env.FIRECRAWL_API_URL ?? "https://api.firecrawl.dev";
|
||||
}
|
||||
|
||||
export async function getIdentity(req: IdmuxRequest = {}): Promise<Identity> {
|
||||
if (cachedIdentity) return cachedIdentity;
|
||||
|
||||
const idmuxUrl = process.env.IDMUX_URL;
|
||||
if (!idmuxUrl) {
|
||||
const fallback: Identity = {
|
||||
apiKey: process.env.TEST_API_KEY ?? process.env.FIRECRAWL_API_KEY ?? "",
|
||||
teamId: process.env.TEST_TEAM_ID ?? "",
|
||||
};
|
||||
cachedIdentity = fallback;
|
||||
return fallback;
|
||||
}
|
||||
|
||||
const runNumberRaw = process.env.GITHUB_RUN_NUMBER;
|
||||
const runNumber = runNumberRaw ? Number(runNumberRaw) : 0;
|
||||
const body = {
|
||||
refName: process.env.GITHUB_REF_NAME ?? "local",
|
||||
runNumber: Number.isFinite(runNumber) ? runNumber : 0,
|
||||
concurrency: req.concurrency ?? 100,
|
||||
...req,
|
||||
};
|
||||
|
||||
const res = await fetch(`${idmuxUrl}/`, {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify(body),
|
||||
});
|
||||
|
||||
if (!res.ok) {
|
||||
const text = await res.text();
|
||||
throw new Error(`idmux request failed: ${res.status} ${text}`);
|
||||
}
|
||||
|
||||
const identity = (await res.json()) as Identity;
|
||||
cachedIdentity = identity;
|
||||
return identity;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,96 @@
|
||||
import Firecrawl from "../../../index";
|
||||
import { config } from "dotenv";
|
||||
import { describe, test, expect, beforeAll } from "@jest/globals";
|
||||
import { getIdentity } from "./utils/idmux";
|
||||
|
||||
config();
|
||||
|
||||
const API_URL = process.env.FIRECRAWL_API_URL ?? "https://api.firecrawl.dev";
|
||||
let client: Firecrawl;
|
||||
|
||||
beforeAll(async () => {
|
||||
const { apiKey } = await getIdentity({ name: "js-e2e-watcher" });
|
||||
client = new Firecrawl({ apiKey, apiUrl: API_URL });
|
||||
});
|
||||
|
||||
describe("v2.watcher e2e", () => {
|
||||
test("crawl watcher minimal", async () => {
|
||||
// client is initialized in beforeAll
|
||||
const start = await client.startCrawl("https://docs.firecrawl.dev", { limit: 3 });
|
||||
|
||||
expect(typeof start.id).toBe("string");
|
||||
|
||||
const watcher = client.watcher(start.id, { pollInterval: 2 });
|
||||
|
||||
let snapshots = 0;
|
||||
let documents = 0;
|
||||
|
||||
watcher.on("snapshot", (snap: any) => {
|
||||
snapshots += 1;
|
||||
expect(["scraping", "completed", "failed", "cancelled"]).toContain(snap.status);
|
||||
expect(typeof snap.completed).toBe("number");
|
||||
expect(typeof snap.total).toBe("number");
|
||||
});
|
||||
|
||||
watcher.on("document", (_doc: any) => {
|
||||
documents += 1;
|
||||
});
|
||||
|
||||
const final = await new Promise<any>(async (resolve) => {
|
||||
watcher.on("done", (payload: any) => {
|
||||
resolve(payload);
|
||||
});
|
||||
watcher.on("error", (err: any) => {
|
||||
resolve(err);
|
||||
});
|
||||
await watcher.start();
|
||||
});
|
||||
|
||||
expect(["completed", "failed", "cancelled"]).toContain(final.status);
|
||||
expect(Array.isArray(final.data)).toBe(true);
|
||||
expect(typeof final.id).toBe("string");
|
||||
expect(snapshots).toBeGreaterThanOrEqual(1);
|
||||
expect(documents).toBeGreaterThanOrEqual(0);
|
||||
watcher.close();
|
||||
}, 240_000);
|
||||
|
||||
test("batch watcher with options (kind, pollInterval, timeout)", async () => {
|
||||
// client is initialized in beforeAll
|
||||
const urls = [
|
||||
"https://docs.firecrawl.dev",
|
||||
"https://firecrawl.dev",
|
||||
];
|
||||
|
||||
const start = await client.startBatchScrape(urls, { options: { formats: ["markdown"] }, ignoreInvalidURLs: true });
|
||||
expect(typeof start.id).toBe("string");
|
||||
|
||||
const watcher = client.watcher(start.id, { kind: "batch", pollInterval: 2, timeout: 180 });
|
||||
|
||||
let snapshots = 0;
|
||||
let gotCompleted = false;
|
||||
|
||||
watcher.on("snapshot", (snap: any) => {
|
||||
snapshots += 1;
|
||||
if (snap.status === "completed") gotCompleted = true;
|
||||
expect(["scraping", "completed", "failed", "cancelled"]).toContain(snap.status);
|
||||
});
|
||||
|
||||
const final = await new Promise<any>(async (resolve) => {
|
||||
watcher.on("done", (payload: any) => {
|
||||
resolve(payload);
|
||||
});
|
||||
watcher.on("error", (err: any) => {
|
||||
resolve(err);
|
||||
});
|
||||
await watcher.start();
|
||||
});
|
||||
|
||||
expect(["completed", "failed", "cancelled"]).toContain(final.status);
|
||||
expect(Array.isArray(final.data)).toBe(true);
|
||||
expect(typeof final.id).toBe("string");
|
||||
expect(snapshots).toBeGreaterThanOrEqual(1);
|
||||
expect(gotCompleted || final.status !== "completed").toBe(true);
|
||||
watcher.close();
|
||||
}, 300_000);
|
||||
});
|
||||
|
||||
@@ -0,0 +1,154 @@
|
||||
import FirecrawlApp from '../../../index';
|
||||
import { describe, test, expect, jest, beforeEach, afterEach } from '@jest/globals';
|
||||
|
||||
describe('monitorJobStatus retry logic', () => {
|
||||
let app: FirecrawlApp;
|
||||
let originalConsoleWarn: typeof console.warn;
|
||||
|
||||
beforeEach(() => {
|
||||
app = new FirecrawlApp({ apiKey: 'test-key', apiUrl: 'https://test.com' });
|
||||
originalConsoleWarn = console.warn;
|
||||
console.warn = jest.fn();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
console.warn = originalConsoleWarn;
|
||||
jest.clearAllMocks();
|
||||
});
|
||||
|
||||
test('should retry on socket hang up error', async () => {
|
||||
const socketHangUpError = new Error('socket hang up') as any;
|
||||
socketHangUpError.code = 'ECONNRESET';
|
||||
|
||||
const successResponse = {
|
||||
status: 200,
|
||||
data: { status: 'completed', data: [{ url: 'test.com', markdown: 'test' }] }
|
||||
};
|
||||
|
||||
const originalGetRequest = app.getRequest;
|
||||
let callCount = 0;
|
||||
|
||||
app.getRequest = async function(url: string, headers: any) {
|
||||
callCount++;
|
||||
if (callCount === 1) {
|
||||
throw socketHangUpError;
|
||||
}
|
||||
return successResponse;
|
||||
};
|
||||
|
||||
const result = await app.monitorJobStatus('test-id', {}, 1);
|
||||
|
||||
expect(callCount).toBe(2);
|
||||
expect(result).toEqual(successResponse.data);
|
||||
expect(console.warn).toHaveBeenCalledWith(
|
||||
expect.stringContaining('Network error during job status check (attempt 1/3): socket hang up')
|
||||
);
|
||||
});
|
||||
|
||||
test('should retry on ETIMEDOUT error', async () => {
|
||||
const timeoutError = new Error('timeout') as any;
|
||||
timeoutError.code = 'ETIMEDOUT';
|
||||
|
||||
const successResponse = {
|
||||
status: 200,
|
||||
data: { status: 'completed', data: [{ url: 'test.com', markdown: 'test' }] }
|
||||
};
|
||||
|
||||
const originalGetRequest = app.getRequest;
|
||||
let callCount = 0;
|
||||
|
||||
app.getRequest = async function(url: string, headers: any) {
|
||||
callCount++;
|
||||
if (callCount === 1) {
|
||||
throw timeoutError;
|
||||
}
|
||||
return successResponse;
|
||||
};
|
||||
|
||||
const result = await app.monitorJobStatus('test-id', {}, 1);
|
||||
|
||||
expect(callCount).toBe(2);
|
||||
expect(result).toEqual(successResponse.data);
|
||||
});
|
||||
|
||||
test('should fail after max retries exceeded', async () => {
|
||||
const socketHangUpError = new Error('socket hang up') as any;
|
||||
socketHangUpError.code = 'ECONNRESET';
|
||||
|
||||
app.getRequest = async function(url: string, headers: any) {
|
||||
throw socketHangUpError;
|
||||
};
|
||||
|
||||
await expect(app.monitorJobStatus('test-id', {}, 1)).rejects.toThrow('socket hang up');
|
||||
|
||||
expect(console.warn).toHaveBeenCalledTimes(3);
|
||||
}, 15000);
|
||||
|
||||
test('should not retry on non-retryable errors', async () => {
|
||||
const authError = new Error('Unauthorized') as any;
|
||||
authError.response = { status: 401, data: { error: 'Unauthorized' } };
|
||||
|
||||
app.getRequest = async function(url: string, headers: any) {
|
||||
throw authError;
|
||||
};
|
||||
|
||||
await expect(app.monitorJobStatus('test-id', {}, 1)).rejects.toThrow('Unauthorized');
|
||||
|
||||
expect(console.warn).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
test('should retry on HTTP timeout status codes', async () => {
|
||||
const timeoutError = new Error('Request timeout') as any;
|
||||
timeoutError.response = { status: 408, data: { error: 'Request timeout' } };
|
||||
|
||||
const successResponse = {
|
||||
status: 200,
|
||||
data: { status: 'completed', data: [{ url: 'test.com', markdown: 'test' }] }
|
||||
};
|
||||
|
||||
const originalGetRequest = app.getRequest;
|
||||
let callCount = 0;
|
||||
|
||||
app.getRequest = async function(url: string, headers: any) {
|
||||
callCount++;
|
||||
if (callCount === 1) {
|
||||
throw timeoutError;
|
||||
}
|
||||
return successResponse;
|
||||
};
|
||||
|
||||
const result = await app.monitorJobStatus('test-id', {}, 1);
|
||||
|
||||
expect(callCount).toBe(2);
|
||||
expect(result).toEqual(successResponse.data);
|
||||
});
|
||||
|
||||
test('should use exponential backoff for retries', async () => {
|
||||
const socketHangUpError = new Error('socket hang up') as any;
|
||||
socketHangUpError.code = 'ECONNRESET';
|
||||
|
||||
const successResponse = {
|
||||
status: 200,
|
||||
data: { status: 'completed', data: [{ url: 'test.com', markdown: 'test' }] }
|
||||
};
|
||||
|
||||
const originalGetRequest = app.getRequest;
|
||||
let callCount = 0;
|
||||
|
||||
app.getRequest = async function(url: string, headers: any) {
|
||||
callCount++;
|
||||
if (callCount <= 2) {
|
||||
throw socketHangUpError;
|
||||
}
|
||||
return successResponse;
|
||||
};
|
||||
|
||||
const startTime = Date.now();
|
||||
const result = await app.monitorJobStatus('test-id', {}, 1);
|
||||
const endTime = Date.now();
|
||||
|
||||
expect(callCount).toBe(3);
|
||||
expect(result).toEqual(successResponse.data);
|
||||
expect(endTime - startTime).toBeGreaterThan(3000);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,80 @@
|
||||
import { describe, test, expect } from "@jest/globals";
|
||||
|
||||
// We need to test the prepareAgentPayload function, but it's not exported.
|
||||
// Since the function is internal, we'll test the behavior through type checking
|
||||
// and verify the types are properly exported.
|
||||
|
||||
import type { AgentWebhookConfig, AgentWebhookEvent } from "../../../v2/types";
|
||||
|
||||
describe("v2 types: Agent webhook types", () => {
|
||||
test("AgentWebhookConfig accepts string webhook", () => {
|
||||
// Type check - this should compile without errors
|
||||
const webhook: string | AgentWebhookConfig = "https://example.com/webhook";
|
||||
expect(typeof webhook).toBe("string");
|
||||
});
|
||||
|
||||
test("AgentWebhookConfig accepts config object", () => {
|
||||
const config: AgentWebhookConfig = {
|
||||
url: "https://example.com/webhook",
|
||||
headers: { Authorization: "Bearer token" },
|
||||
events: ["completed", "failed"],
|
||||
};
|
||||
expect(config.url).toBe("https://example.com/webhook");
|
||||
expect(config.headers).toEqual({ Authorization: "Bearer token" });
|
||||
expect(config.events).toEqual(["completed", "failed"]);
|
||||
});
|
||||
|
||||
test("AgentWebhookConfig accepts minimal config", () => {
|
||||
const config: AgentWebhookConfig = {
|
||||
url: "https://example.com/webhook",
|
||||
};
|
||||
expect(config.url).toBe("https://example.com/webhook");
|
||||
expect(config.headers).toBeUndefined();
|
||||
expect(config.metadata).toBeUndefined();
|
||||
expect(config.events).toBeUndefined();
|
||||
});
|
||||
|
||||
test("AgentWebhookEvent includes agent-specific events", () => {
|
||||
const events: AgentWebhookEvent[] = [
|
||||
"started",
|
||||
"action",
|
||||
"completed",
|
||||
"failed",
|
||||
"cancelled",
|
||||
];
|
||||
expect(events).toContain("action");
|
||||
expect(events).toContain("cancelled");
|
||||
expect(events.length).toBe(5);
|
||||
});
|
||||
|
||||
test("AgentWebhookConfig accepts all fields", () => {
|
||||
const config: AgentWebhookConfig = {
|
||||
url: "https://example.com/webhook",
|
||||
headers: {
|
||||
Authorization: "Bearer token",
|
||||
"X-Custom-Header": "value",
|
||||
},
|
||||
metadata: {
|
||||
project: "test",
|
||||
environment: "staging",
|
||||
},
|
||||
events: ["started", "action", "completed", "failed", "cancelled"],
|
||||
};
|
||||
expect(config.url).toBe("https://example.com/webhook");
|
||||
expect(Object.keys(config.headers!).length).toBe(2);
|
||||
expect(config.metadata!.project).toBe("test");
|
||||
expect(config.events!.length).toBe(5);
|
||||
});
|
||||
|
||||
test("AgentWebhookConfig events are agent-specific (not crawl)", () => {
|
||||
// Agent has 'action' and 'cancelled', but not 'page'
|
||||
const config: AgentWebhookConfig = {
|
||||
url: "https://example.com/webhook",
|
||||
events: ["action", "cancelled"],
|
||||
};
|
||||
expect(config.events).toContain("action");
|
||||
expect(config.events).toContain("cancelled");
|
||||
// 'page' is a crawl-specific event, not valid for agent
|
||||
// This is enforced at the type level
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,186 @@
|
||||
import { describe, test, expect, jest } from "@jest/globals";
|
||||
import { scrape } from "../../../v2/methods/scrape";
|
||||
|
||||
describe("JS SDK v2 branding format", () => {
|
||||
function makeHttp(postImpl: (url: string, data: any) => any) {
|
||||
return { post: jest.fn(async (u: string, d: any) => postImpl(u, d)) } as any;
|
||||
}
|
||||
|
||||
test("scrape with branding format returns branding data", async () => {
|
||||
const mockResponse = {
|
||||
status: 200,
|
||||
data: {
|
||||
success: true,
|
||||
data: {
|
||||
markdown: "# Example",
|
||||
branding: {
|
||||
colorScheme: "light",
|
||||
colors: {
|
||||
primary: "#E11D48",
|
||||
secondary: "#3B82F6",
|
||||
accent: "#F59E0B"
|
||||
},
|
||||
typography: {
|
||||
fontFamilies: {
|
||||
primary: "Inter",
|
||||
heading: "Poppins"
|
||||
},
|
||||
fontSizes: {
|
||||
h1: "2.5rem",
|
||||
body: "1rem"
|
||||
}
|
||||
},
|
||||
spacing: {
|
||||
baseUnit: 8
|
||||
},
|
||||
components: {
|
||||
buttonPrimary: {
|
||||
background: "#E11D48",
|
||||
textColor: "#FFFFFF",
|
||||
borderRadius: "0.5rem"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
const http = makeHttp(() => mockResponse);
|
||||
const result = await scrape(http, "https://example.com", { formats: ["branding"] });
|
||||
|
||||
expect(result.branding).toBeDefined();
|
||||
expect(result.branding?.colorScheme).toBe("light");
|
||||
expect(result.branding?.colors?.primary).toBe("#E11D48");
|
||||
expect(result.branding?.typography?.fontFamilies?.primary).toBe("Inter");
|
||||
expect(result.branding?.spacing?.baseUnit).toBe(8);
|
||||
expect(result.branding?.components?.buttonPrimary?.background).toBe("#E11D48");
|
||||
});
|
||||
|
||||
test("scrape with branding and markdown formats returns both", async () => {
|
||||
const mockResponse = {
|
||||
status: 200,
|
||||
data: {
|
||||
success: true,
|
||||
data: {
|
||||
markdown: "# Example Content",
|
||||
branding: {
|
||||
colorScheme: "dark",
|
||||
colors: {
|
||||
primary: "#10B981"
|
||||
},
|
||||
typography: {
|
||||
fontFamilies: {
|
||||
primary: "Roboto"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
const http = makeHttp(() => mockResponse);
|
||||
const result = await scrape(http, "https://example.com", { formats: ["markdown", "branding"] });
|
||||
|
||||
expect(result.markdown).toBe("# Example Content");
|
||||
expect(result.branding).toBeDefined();
|
||||
expect(result.branding?.colorScheme).toBe("dark");
|
||||
expect(result.branding?.colors?.primary).toBe("#10B981");
|
||||
});
|
||||
|
||||
test("scrape without branding format does not return branding", async () => {
|
||||
const mockResponse = {
|
||||
status: 200,
|
||||
data: {
|
||||
success: true,
|
||||
data: {
|
||||
markdown: "# Example"
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
const http = makeHttp(() => mockResponse);
|
||||
const result = await scrape(http, "https://example.com", { formats: ["markdown"] });
|
||||
|
||||
expect(result.markdown).toBe("# Example");
|
||||
expect(result.branding).toBeUndefined();
|
||||
});
|
||||
|
||||
test("branding format with all nested fields", async () => {
|
||||
const mockResponse = {
|
||||
status: 200,
|
||||
data: {
|
||||
success: true,
|
||||
data: {
|
||||
branding: {
|
||||
colorScheme: "light",
|
||||
logo: "https://example.com/logo.png",
|
||||
fonts: [
|
||||
{ family: "Inter", weight: 400 },
|
||||
{ family: "Poppins", weight: 700 }
|
||||
],
|
||||
colors: {
|
||||
primary: "#E11D48",
|
||||
background: "#FFFFFF"
|
||||
},
|
||||
typography: {
|
||||
fontFamilies: { primary: "Inter" },
|
||||
fontStacks: { body: ["Inter", "sans-serif"] },
|
||||
fontSizes: { h1: "2.5rem" },
|
||||
lineHeights: { body: 1.5 },
|
||||
fontWeights: { regular: 400 }
|
||||
},
|
||||
spacing: {
|
||||
baseUnit: 8,
|
||||
padding: { sm: 8, md: 16 }
|
||||
},
|
||||
components: {
|
||||
buttonPrimary: {
|
||||
background: "#E11D48",
|
||||
textColor: "#FFFFFF"
|
||||
}
|
||||
},
|
||||
icons: {
|
||||
style: "outline",
|
||||
primaryColor: "#E11D48"
|
||||
},
|
||||
images: {
|
||||
logo: "https://example.com/logo.png",
|
||||
favicon: "https://example.com/favicon.ico"
|
||||
},
|
||||
animations: {
|
||||
transitionDuration: "200ms",
|
||||
easing: "ease-in-out"
|
||||
},
|
||||
layout: {
|
||||
grid: { columns: 12, maxWidth: "1200px" },
|
||||
headerHeight: "64px"
|
||||
},
|
||||
tone: {
|
||||
voice: "professional",
|
||||
emojiUsage: "minimal"
|
||||
},
|
||||
personality: {
|
||||
tone: "professional",
|
||||
energy: "medium",
|
||||
targetAudience: "developers"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
const http = makeHttp(() => mockResponse);
|
||||
const result = await scrape(http, "https://example.com", { formats: ["branding"] });
|
||||
|
||||
expect(result.branding).toBeDefined();
|
||||
expect(result.branding?.logo).toBe("https://example.com/logo.png");
|
||||
expect(result.branding?.fonts).toHaveLength(2);
|
||||
expect(result.branding?.typography?.fontStacks?.body).toEqual(["Inter", "sans-serif"]);
|
||||
expect(result.branding?.spacing?.padding).toEqual({ sm: 8, md: 16 });
|
||||
expect(result.branding?.icons?.style).toBe("outline");
|
||||
expect(result.branding?.images?.favicon).toBe("https://example.com/favicon.ico");
|
||||
expect(result.branding?.animations?.easing).toBe("ease-in-out");
|
||||
expect(result.branding?.layout?.grid?.columns).toBe(12);
|
||||
expect(result.branding?.personality?.tone).toBe("professional");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,55 @@
|
||||
import { Firecrawl, type FirecrawlClientOptions } from '../../../index';
|
||||
|
||||
describe('Firecrawl v2 Client Options', () => {
|
||||
it('should accept v2 options including timeoutMs, maxRetries, and backoffFactor', () => {
|
||||
const options: FirecrawlClientOptions = {
|
||||
apiKey: 'test-key',
|
||||
timeoutMs: 300,
|
||||
maxRetries: 5,
|
||||
backoffFactor: 0.5,
|
||||
};
|
||||
|
||||
// Should not throw any type errors
|
||||
const client = new Firecrawl(options);
|
||||
|
||||
expect(client).toBeDefined();
|
||||
expect(client).toBeInstanceOf(Firecrawl);
|
||||
});
|
||||
|
||||
it('should work with minimal options', () => {
|
||||
const options: FirecrawlClientOptions = {
|
||||
apiKey: 'test-key',
|
||||
};
|
||||
|
||||
const client = new Firecrawl(options);
|
||||
|
||||
expect(client).toBeDefined();
|
||||
expect(client).toBeInstanceOf(Firecrawl);
|
||||
});
|
||||
|
||||
it('should work with all v2 options', () => {
|
||||
const options: FirecrawlClientOptions = {
|
||||
apiKey: 'test-key',
|
||||
apiUrl: 'https://custom-api.firecrawl.dev',
|
||||
timeoutMs: 60000,
|
||||
maxRetries: 3,
|
||||
backoffFactor: 1.0,
|
||||
};
|
||||
|
||||
const client = new Firecrawl(options);
|
||||
|
||||
expect(client).toBeDefined();
|
||||
expect(client).toBeInstanceOf(Firecrawl);
|
||||
});
|
||||
|
||||
it('should export FirecrawlClientOptions type', () => {
|
||||
// This test ensures the type is properly exported
|
||||
const options: FirecrawlClientOptions = {
|
||||
apiKey: 'test-key',
|
||||
timeoutMs: 300,
|
||||
};
|
||||
|
||||
expect(options.timeoutMs).toBe(300);
|
||||
expect(options.apiKey).toBe('test-key');
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,19 @@
|
||||
import { describe, test, expect } from "@jest/globals";
|
||||
import { throwForBadResponse, normalizeAxiosError } from "../../../v2/utils/errorHandler";
|
||||
|
||||
describe("v2 utils: errorHandler", () => {
|
||||
test("throwForBadResponse: throws SdkError with message from body.error", () => {
|
||||
const resp: any = { status: 400, data: { error: "bad" } };
|
||||
expect(() => throwForBadResponse(resp, "do thing")).toThrow(/bad/);
|
||||
});
|
||||
|
||||
test("normalizeAxiosError: prefers body.error then err.message", () => {
|
||||
const err: any = {
|
||||
isAxiosError: true,
|
||||
response: { status: 402, data: { error: "payment required" } },
|
||||
message: "network",
|
||||
};
|
||||
expect(() => normalizeAxiosError(err, "action")).toThrow(/payment required/);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -0,0 +1,133 @@
|
||||
import { describe, test, expect, jest } from "@jest/globals";
|
||||
import { getCrawlStatus } from "../../../v2/methods/crawl";
|
||||
import { getBatchScrapeStatus } from "../../../v2/methods/batch";
|
||||
import { getMonitorCheck } from "../../../v2/methods/monitor";
|
||||
|
||||
describe("JS SDK v2 pagination", () => {
|
||||
function makeHttp(getImpl: (url: string) => any) {
|
||||
return { get: jest.fn(async (u: string) => getImpl(u)) } as any;
|
||||
}
|
||||
|
||||
test("crawl: autoPaginate=false returns next", async () => {
|
||||
const first = { status: 200, data: { success: true, status: "completed", completed: 1, total: 2, next: "https://api/next", data: [{ markdown: "a" }] } };
|
||||
const http = makeHttp(() => first);
|
||||
const res = await getCrawlStatus(http, "job1", { autoPaginate: false });
|
||||
expect(res.data.length).toBe(1);
|
||||
expect(res.next).toBe("https://api/next");
|
||||
});
|
||||
|
||||
test("crawl: default autoPaginate aggregates and nulls next", async () => {
|
||||
const first = { status: 200, data: { success: true, status: "completed", completed: 1, total: 3, next: "https://api/n1", data: [{ markdown: "a" }] } };
|
||||
const second = { status: 200, data: { success: true, next: "https://api/n2", data: [{ markdown: "b" }] } };
|
||||
const third = { status: 200, data: { success: true, next: null, data: [{ markdown: "c" }] } };
|
||||
const http = makeHttp((url) => {
|
||||
if (url.includes("/v2/crawl/")) return first;
|
||||
if (url.endsWith("n1")) return second;
|
||||
return third;
|
||||
});
|
||||
const res = await getCrawlStatus(http, "job1");
|
||||
expect(res.data.length).toBe(3);
|
||||
expect(res.next).toBeNull();
|
||||
});
|
||||
|
||||
test("crawl: respects maxPages and maxResults", async () => {
|
||||
const first = { status: 200, data: { success: true, status: "completed", completed: 1, total: 10, next: "https://api/n1", data: [{ markdown: "a" }] } };
|
||||
const page = (n: number) => ({ status: 200, data: { success: true, next: n < 3 ? `https://api/n${n + 1}` : null, data: [{ markdown: `p${n}` }] } });
|
||||
const http = makeHttp((url) => {
|
||||
if (url.includes("/v2/crawl/")) return first;
|
||||
if (url.endsWith("n1")) return page(1);
|
||||
if (url.endsWith("n2")) return page(2);
|
||||
return page(3);
|
||||
});
|
||||
const res = await getCrawlStatus(http, "job1", { autoPaginate: true, maxPages: 2, maxResults: 2 });
|
||||
expect(res.data.length).toBe(2);
|
||||
});
|
||||
|
||||
test("batch: default autoPaginate aggregates and nulls next", async () => {
|
||||
const first = { status: 200, data: { success: true, status: "completed", completed: 1, total: 3, next: "https://api/b1", data: [{ markdown: "a" }] } };
|
||||
const second = { status: 200, data: { success: true, next: "https://api/b2", data: [{ markdown: "b" }] } };
|
||||
const third = { status: 200, data: { success: true, next: null, data: [{ markdown: "c" }] } };
|
||||
const http = makeHttp((url) => {
|
||||
if (url.includes("/v2/batch/scrape/")) return first;
|
||||
if (url.endsWith("b1")) return second;
|
||||
return third;
|
||||
});
|
||||
const res = await getBatchScrapeStatus(http, "jobB");
|
||||
expect(res.data.length).toBe(3);
|
||||
expect(res.next).toBeNull();
|
||||
});
|
||||
|
||||
test("batch: autoPaginate=false returns next", async () => {
|
||||
const first = { status: 200, data: { success: true, status: "completed", completed: 1, total: 2, next: "https://api/nextBatch", data: [{ markdown: "a" }] } };
|
||||
const http = makeHttp(() => first);
|
||||
const res = await getBatchScrapeStatus(http, "jobB", { autoPaginate: false });
|
||||
expect(res.data.length).toBe(1);
|
||||
expect(res.next).toBe("https://api/nextBatch");
|
||||
});
|
||||
|
||||
test("monitor check: default autoPaginate aggregates pages and nulls next", async () => {
|
||||
const first = { status: 200, data: { success: true, next: "https://api/m1", data: { id: "check1", monitorId: "mon1", status: "completed", trigger: "manual", billingStatus: "confirmed", summary: {}, createdAt: "now", updatedAt: "now", pages: [{ url: "a", status: "changed" }], next: "https://api/m1" } } };
|
||||
const second = { status: 200, data: { success: true, next: null, data: { pages: [{ url: "b", status: "same" }], next: null } } };
|
||||
const http = makeHttp((url) => {
|
||||
if (url.includes("/v2/monitor/")) return first;
|
||||
return second;
|
||||
});
|
||||
const res = await getMonitorCheck(http, "mon1", "check1");
|
||||
expect(res.pages.length).toBe(2);
|
||||
expect(res.next).toBeNull();
|
||||
});
|
||||
|
||||
test("monitor check: autoPaginate=false returns next", async () => {
|
||||
const first = { status: 200, data: { success: true, next: "https://api/m1", data: { id: "check1", monitorId: "mon1", status: "completed", trigger: "manual", billingStatus: "confirmed", summary: {}, createdAt: "now", updatedAt: "now", pages: [{ url: "a", status: "changed" }], next: "https://api/m1" } } };
|
||||
const http = makeHttp(() => first);
|
||||
const res = await getMonitorCheck(http, "mon1", "check1", { autoPaginate: false });
|
||||
expect(res.pages.length).toBe(1);
|
||||
expect(res.next).toBe("https://api/m1");
|
||||
});
|
||||
|
||||
test("crawl: maxWaitTime stops pagination after first page", async () => {
|
||||
const first = { status: 200, data: { success: true, status: "completed", completed: 1, total: 5, next: "https://api/n1", data: [{ markdown: "a" }] } };
|
||||
const p1 = { status: 200, data: { success: true, next: "https://api/n2", data: [{ markdown: "b" }] } };
|
||||
const http: any = makeHttp((url: string) => {
|
||||
if (url.includes("/v2/crawl/")) return first;
|
||||
if (url.endsWith("n1")) return p1;
|
||||
return { status: 200, data: { success: true, next: null, data: [{ markdown: "c" }] } };
|
||||
});
|
||||
const nowSpy = jest.spyOn(Date, "now");
|
||||
try {
|
||||
nowSpy
|
||||
.mockImplementationOnce(() => 0) // started
|
||||
.mockImplementationOnce(() => 0) // first loop check
|
||||
.mockImplementationOnce(() => 3000); // second loop check > maxWaitTime
|
||||
const res = await getCrawlStatus(http, "jobC", { autoPaginate: true, maxWaitTime: 1 });
|
||||
expect(res.data.length).toBe(2); // initial + first page
|
||||
expect((http.get as jest.Mock).mock.calls.length).toBe(2); // initial + n1 only
|
||||
} finally {
|
||||
nowSpy.mockRestore();
|
||||
}
|
||||
});
|
||||
|
||||
test("batch: maxWaitTime stops pagination after first page", async () => {
|
||||
const first = { status: 200, data: { success: true, status: "completed", completed: 1, total: 5, next: "https://api/b1", data: [{ markdown: "a" }] } };
|
||||
const p1 = { status: 200, data: { success: true, next: "https://api/b2", data: [{ markdown: "b" }] } };
|
||||
const http: any = makeHttp((url: string) => {
|
||||
if (url.includes("/v2/batch/scrape/")) return first;
|
||||
if (url.endsWith("b1")) return p1;
|
||||
return { status: 200, data: { success: true, next: null, data: [{ markdown: "c" }] } };
|
||||
});
|
||||
const nowSpy = jest.spyOn(Date, "now");
|
||||
try {
|
||||
nowSpy
|
||||
.mockImplementationOnce(() => 0) // started
|
||||
.mockImplementationOnce(() => 0) // first loop check
|
||||
.mockImplementationOnce(() => 3000); // second loop check > maxWaitTime
|
||||
const res = await getBatchScrapeStatus(http, "jobB", { autoPaginate: true, maxWaitTime: 1 });
|
||||
expect(res.data.length).toBe(2);
|
||||
expect((http.get as jest.Mock).mock.calls.length).toBe(2);
|
||||
} finally {
|
||||
nowSpy.mockRestore();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
@@ -0,0 +1,58 @@
|
||||
import { describe, test, expect } from "@jest/globals";
|
||||
import { FirecrawlClient } from "../../../v2/client";
|
||||
|
||||
describe("v2.parse unit", () => {
|
||||
test("rejects empty filenames before making requests", async () => {
|
||||
const client = new FirecrawlClient({
|
||||
apiKey: "test-key",
|
||||
apiUrl: "https://localhost:3002",
|
||||
});
|
||||
|
||||
await expect(
|
||||
client.parse(
|
||||
{
|
||||
data: "<html><body>test</body></html>",
|
||||
filename: " ",
|
||||
contentType: "text/html",
|
||||
},
|
||||
{ formats: ["markdown"] },
|
||||
),
|
||||
).rejects.toThrow("filename cannot be empty");
|
||||
});
|
||||
|
||||
test("rejects changeTracking format before making requests", async () => {
|
||||
const client = new FirecrawlClient({
|
||||
apiKey: "test-key",
|
||||
apiUrl: "https://localhost:3002",
|
||||
});
|
||||
|
||||
await expect(
|
||||
client.parse(
|
||||
{
|
||||
data: "<html><body>test</body></html>",
|
||||
filename: "upload.html",
|
||||
contentType: "text/html",
|
||||
},
|
||||
{ formats: ["markdown", { type: "changeTracking" } as any] },
|
||||
),
|
||||
).rejects.toThrow("parse does not support changeTracking format");
|
||||
});
|
||||
|
||||
test("rejects lockdown option before making requests", async () => {
|
||||
const client = new FirecrawlClient({
|
||||
apiKey: "test-key",
|
||||
apiUrl: "https://localhost:3002",
|
||||
});
|
||||
|
||||
await expect(
|
||||
client.parse(
|
||||
{
|
||||
data: "<html><body>test</body></html>",
|
||||
filename: "upload.html",
|
||||
contentType: "text/html",
|
||||
},
|
||||
{ formats: ["markdown"], lockdown: true } as any,
|
||||
),
|
||||
).rejects.toThrow("parse does not support cache/index options");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,131 @@
|
||||
import { describe, test, expect, jest } from "@jest/globals";
|
||||
import { interact, stopInteraction } from "../../../v2/methods/scrape";
|
||||
import { SdkError } from "../../../v2/types";
|
||||
|
||||
describe("JS SDK v2 scrape-browser methods", () => {
|
||||
test("interact posts to scrape interact endpoint", async () => {
|
||||
const post = jest.fn(async () => ({
|
||||
status: 200,
|
||||
data: {
|
||||
success: true,
|
||||
stdout: "ok",
|
||||
exitCode: 0,
|
||||
},
|
||||
}));
|
||||
|
||||
const http = { post } as any;
|
||||
const response = await interact(http, "job-123", {
|
||||
code: "console.log('ok')",
|
||||
});
|
||||
|
||||
expect(post).toHaveBeenCalledWith(
|
||||
"/v2/scrape/job-123/interact",
|
||||
{ code: "console.log('ok')", language: "node" },
|
||||
{},
|
||||
);
|
||||
expect(response.success).toBe(true);
|
||||
expect(response.exitCode).toBe(0);
|
||||
});
|
||||
|
||||
test("interact with prompt posts prompt to endpoint", async () => {
|
||||
const post = jest.fn(async () => ({
|
||||
status: 200,
|
||||
data: {
|
||||
success: true,
|
||||
output: "Clicked the button",
|
||||
liveViewUrl: "https://live.example.com/view",
|
||||
interactiveLiveViewUrl: "https://live.example.com/interactive",
|
||||
stdout: "",
|
||||
exitCode: 0,
|
||||
},
|
||||
}));
|
||||
|
||||
const http = { post } as any;
|
||||
const response = await interact(http, "job-456", {
|
||||
prompt: "Click the login button",
|
||||
});
|
||||
|
||||
expect(post).toHaveBeenCalledWith(
|
||||
"/v2/scrape/job-456/interact",
|
||||
{ prompt: "Click the login button", language: "node" },
|
||||
{},
|
||||
);
|
||||
expect(response.success).toBe(true);
|
||||
expect(response.output).toBe("Clicked the button");
|
||||
expect(response.liveViewUrl).toBe("https://live.example.com/view");
|
||||
expect(response.interactiveLiveViewUrl).toBe(
|
||||
"https://live.example.com/interactive",
|
||||
);
|
||||
});
|
||||
|
||||
test("interact throws when neither code nor prompt provided", async () => {
|
||||
const http = { post: jest.fn() } as any;
|
||||
await expect(interact(http, "job-123", {})).rejects.toThrow(
|
||||
"Either 'code' or 'prompt' must be provided",
|
||||
);
|
||||
});
|
||||
|
||||
test("interact throws on non-200 response", async () => {
|
||||
const post = jest.fn(async () => ({
|
||||
status: 400,
|
||||
data: {
|
||||
success: false,
|
||||
error: "Invalid job ID format",
|
||||
},
|
||||
}));
|
||||
|
||||
const http = { post } as any;
|
||||
await expect(
|
||||
interact(http, "bad-id", { code: "console.log('ok')" }),
|
||||
).rejects.toBeInstanceOf(SdkError);
|
||||
});
|
||||
|
||||
test("stopInteraction calls delete endpoint", async () => {
|
||||
const del = jest.fn(async () => ({
|
||||
status: 200,
|
||||
data: {
|
||||
success: true,
|
||||
},
|
||||
}));
|
||||
|
||||
const http = { delete: del } as any;
|
||||
const response = await stopInteraction(http, "job-123");
|
||||
|
||||
expect(del).toHaveBeenCalledWith("/v2/scrape/job-123/interact");
|
||||
expect(response.success).toBe(true);
|
||||
});
|
||||
|
||||
test("stopInteraction throws on non-200 response", async () => {
|
||||
const del = jest.fn(async () => ({
|
||||
status: 404,
|
||||
data: {
|
||||
success: false,
|
||||
error: "Browser session not found.",
|
||||
},
|
||||
}));
|
||||
|
||||
const http = { delete: del } as any;
|
||||
await expect(stopInteraction(http, "job-123")).rejects.toBeInstanceOf(
|
||||
SdkError,
|
||||
);
|
||||
});
|
||||
|
||||
test("interact converts seconds-based body timeout to ms axios timeout", async () => {
|
||||
const post = jest.fn(async () => ({
|
||||
status: 200,
|
||||
data: { success: true, stdout: "ok", exitCode: 0 },
|
||||
}));
|
||||
|
||||
const http = { post } as any;
|
||||
await interact(http, "job-123", {
|
||||
code: "console.log('ok')",
|
||||
timeout: 150,
|
||||
});
|
||||
|
||||
expect(post).toHaveBeenCalledWith(
|
||||
"/v2/scrape/job-123/interact",
|
||||
{ code: "console.log('ok')", language: "node", timeout: 150 },
|
||||
{ timeoutMs: 155000 },
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,11 @@
|
||||
/**
|
||||
* Minimal unit test for v2 scrape (no mocking; sanity check payload path)
|
||||
*/
|
||||
import { FirecrawlClient } from "../../../v2/client";
|
||||
|
||||
describe("v2.scrape unit", () => {
|
||||
test("constructor requires apiKey", () => {
|
||||
expect(() => new FirecrawlClient({ apiKey: "", apiUrl: "https://api.firecrawl.dev" })).toThrow();
|
||||
});
|
||||
});
|
||||
|
||||
@@ -0,0 +1,105 @@
|
||||
import { describe, test, expect } from "@jest/globals";
|
||||
import { ensureValidFormats, ensureValidScrapeOptions } from "../../../v2/utils/validation";
|
||||
import type { FormatOption } from "../../../v2/types";
|
||||
import { z } from "zod";
|
||||
|
||||
describe("v2 utils: validation", () => {
|
||||
test("ensureValidFormats: plain 'json' string is invalid", () => {
|
||||
const formats: FormatOption[] = ["markdown", "json"] as unknown as FormatOption[];
|
||||
expect(() => ensureValidFormats(formats)).toThrow(/json format must be an object/i);
|
||||
});
|
||||
|
||||
test("ensureValidFormats: json format requires prompt or schema", () => {
|
||||
// Valid cases - should not throw
|
||||
const valid1: FormatOption[] = [{ type: "json", prompt: "p" } as any];
|
||||
const valid2: FormatOption[] = [{ type: "json", schema: {} } as any];
|
||||
const valid3: FormatOption[] = [{ type: "json", prompt: "p", schema: {} } as any];
|
||||
expect(() => ensureValidFormats(valid1)).not.toThrow();
|
||||
expect(() => ensureValidFormats(valid2)).not.toThrow();
|
||||
expect(() => ensureValidFormats(valid3)).not.toThrow();
|
||||
|
||||
// Invalid case - should throw when both are missing
|
||||
const bad: FormatOption[] = [{ type: "json" } as any];
|
||||
expect(() => ensureValidFormats(bad)).toThrow(/requires either 'prompt' or 'schema'/i);
|
||||
});
|
||||
|
||||
test("ensureValidFormats: converts zod schema to JSON schema", () => {
|
||||
const schema = z.object({ title: z.string() });
|
||||
const formats: FormatOption[] = [
|
||||
{ type: "json", prompt: "extract", schema } as any,
|
||||
];
|
||||
ensureValidFormats(formats);
|
||||
const jsonFmt = formats[0] as any;
|
||||
expect(typeof jsonFmt.schema).toBe("object");
|
||||
expect(jsonFmt.schema?.properties).toBeTruthy();
|
||||
});
|
||||
|
||||
test("ensureValidFormats: screenshot quality must be non-negative number", () => {
|
||||
const formats: FormatOption[] = [
|
||||
{ type: "screenshot", quality: -1 } as any,
|
||||
];
|
||||
expect(() => ensureValidFormats(formats)).toThrow(/non-negative number/i);
|
||||
});
|
||||
|
||||
test("ensureValidScrapeOptions: validates timeout and waitFor bounds", () => {
|
||||
expect(() => ensureValidScrapeOptions({ timeout: 0 })).toThrow(/timeout must be positive/i);
|
||||
expect(() => ensureValidScrapeOptions({ waitFor: -1 })).toThrow(/waitFor must be non-negative/i);
|
||||
// valid
|
||||
expect(() => ensureValidScrapeOptions({ timeout: 1000, waitFor: 0 })).not.toThrow();
|
||||
});
|
||||
|
||||
test("ensureValidFormats: accepts screenshot viewport width/height", () => {
|
||||
const formats: FormatOption[] = [
|
||||
{ type: "screenshot", viewport: { width: 800, height: 600 } } as any,
|
||||
];
|
||||
expect(() => ensureValidFormats(formats)).not.toThrow();
|
||||
expect((formats[0] as any).viewport).toEqual({ width: 800, height: 600 });
|
||||
});
|
||||
|
||||
test("ensureValidFormats: accepts question, highlights, and deprecated query formats", () => {
|
||||
const formats: FormatOption[] = [
|
||||
{ type: "question", question: "What is Firecrawl?" },
|
||||
{ type: "highlights", query: "What is Firecrawl?" },
|
||||
{ type: "query", prompt: "What is Firecrawl?", mode: "directQuote" },
|
||||
];
|
||||
expect(() => ensureValidFormats(formats)).not.toThrow();
|
||||
});
|
||||
|
||||
test("ensureValidFormats: validates question, highlights, and deprecated query fields", () => {
|
||||
expect(() =>
|
||||
ensureValidFormats([{ type: "question", question: "" } as any]),
|
||||
).toThrow(/question format requires/i);
|
||||
expect(() =>
|
||||
ensureValidFormats([{ type: "highlights", query: "" } as any]),
|
||||
).toThrow(/highlights format requires/i);
|
||||
expect(() =>
|
||||
ensureValidFormats([{ type: "query", prompt: "p", mode: "quoted" } as any]),
|
||||
).toThrow(/query format mode/i);
|
||||
});
|
||||
|
||||
test("ensureValidScrapeOptions: leaves parsers untouched", () => {
|
||||
const options = { parsers: ["pdf", "images"] as string[] } as any;
|
||||
const before = [...options.parsers];
|
||||
expect(() => ensureValidScrapeOptions(options)).not.toThrow();
|
||||
expect(options.parsers).toEqual(before);
|
||||
});
|
||||
|
||||
test("ensureValidFormats: detects mistaken use of zod schema.shape", () => {
|
||||
const schema = z.object({ title: z.string(), count: z.number() });
|
||||
// User mistakenly passes schema.shape instead of schema
|
||||
const formats: FormatOption[] = [
|
||||
{ type: "json", prompt: "extract", schema: schema.shape } as any,
|
||||
];
|
||||
expect(() => ensureValidFormats(formats)).toThrow(/\.shape property/i);
|
||||
expect(() => ensureValidFormats(formats)).toThrow(/Pass the Zod schema directly/i);
|
||||
});
|
||||
|
||||
test("ensureValidFormats: detects mistaken use of zod schema.shape in changeTracking", () => {
|
||||
const schema = z.object({ title: z.string() });
|
||||
const formats: FormatOption[] = [
|
||||
{ type: "changeTracking", modes: ["json"], schema: schema.shape } as any,
|
||||
];
|
||||
expect(() => ensureValidFormats(formats)).toThrow(/\.shape property/i);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -0,0 +1,107 @@
|
||||
import { describe, test, expect } from "@jest/globals";
|
||||
import { z } from "zod";
|
||||
import {
|
||||
isZodSchema,
|
||||
zodSchemaToJsonSchema,
|
||||
looksLikeZodShape,
|
||||
} from "../../../utils/zodSchemaToJson";
|
||||
|
||||
describe("zodSchemaToJson utility", () => {
|
||||
test("isZodSchema detects Zod schemas and rejects non-Zod values", () => {
|
||||
expect(isZodSchema(z.object({ name: z.string() }))).toBe(true);
|
||||
expect(isZodSchema(z.string())).toBe(true);
|
||||
expect(isZodSchema(z.number())).toBe(true);
|
||||
expect(isZodSchema(z.array(z.string()))).toBe(true);
|
||||
expect(isZodSchema(z.enum(["A", "B"]))).toBe(true);
|
||||
expect(isZodSchema(z.union([z.string(), z.number()]))).toBe(true);
|
||||
expect(isZodSchema(z.string().optional())).toBe(true);
|
||||
expect(isZodSchema(z.string().nullable())).toBe(true);
|
||||
|
||||
expect(isZodSchema(null)).toBe(false);
|
||||
expect(isZodSchema(undefined)).toBe(false);
|
||||
expect(isZodSchema({ name: "test" })).toBe(false);
|
||||
expect(isZodSchema({ type: "object", properties: {} })).toBe(false);
|
||||
expect(isZodSchema("string")).toBe(false);
|
||||
expect(isZodSchema(42)).toBe(false);
|
||||
expect(isZodSchema([1, 2, 3])).toBe(false);
|
||||
});
|
||||
|
||||
test("zodSchemaToJsonSchema converts Zod schemas to JSON Schema", () => {
|
||||
const simpleSchema = z.object({ name: z.string() });
|
||||
const simpleResult = zodSchemaToJsonSchema(simpleSchema) as Record<string, unknown>;
|
||||
expect(simpleResult.type).toBe("object");
|
||||
expect(simpleResult.properties).toBeDefined();
|
||||
expect((simpleResult.properties as Record<string, unknown>).name).toBeDefined();
|
||||
|
||||
const complexSchema = z.object({
|
||||
id: z.string().uuid(),
|
||||
name: z.string().min(1).max(100),
|
||||
age: z.number().min(0).max(150).optional(),
|
||||
tags: z.array(z.string()),
|
||||
status: z.enum(["active", "inactive"]),
|
||||
metadata: z.object({
|
||||
createdAt: z.string(),
|
||||
nested: z.object({ value: z.number() }),
|
||||
}),
|
||||
});
|
||||
const complexResult = zodSchemaToJsonSchema(complexSchema) as Record<string, unknown>;
|
||||
expect(complexResult.type).toBe("object");
|
||||
expect(complexResult.properties).toBeDefined();
|
||||
expect(complexResult.required).toContain("id");
|
||||
expect(complexResult.required).not.toContain("age");
|
||||
|
||||
const enumResult = zodSchemaToJsonSchema(z.enum(["a", "b", "c"])) as Record<string, unknown>;
|
||||
expect(enumResult.enum).toEqual(["a", "b", "c"]);
|
||||
|
||||
const arrayResult = zodSchemaToJsonSchema(z.array(z.number())) as Record<string, unknown>;
|
||||
expect(arrayResult.type).toBe("array");
|
||||
expect(arrayResult.items).toBeDefined();
|
||||
});
|
||||
|
||||
test("zodSchemaToJsonSchema passes through non-Zod values unchanged", () => {
|
||||
const jsonSchema = { type: "object", properties: { name: { type: "string" } } };
|
||||
expect(zodSchemaToJsonSchema(jsonSchema)).toEqual(jsonSchema);
|
||||
expect(zodSchemaToJsonSchema(null)).toBe(null);
|
||||
expect(zodSchemaToJsonSchema(undefined)).toBe(undefined);
|
||||
expect(zodSchemaToJsonSchema("string")).toBe("string");
|
||||
expect(zodSchemaToJsonSchema(42)).toBe(42);
|
||||
expect(zodSchemaToJsonSchema({ foo: "bar" })).toEqual({ foo: "bar" });
|
||||
});
|
||||
|
||||
test("looksLikeZodShape detects .shape property misuse", () => {
|
||||
const schema = z.object({ title: z.string(), count: z.number() });
|
||||
expect(looksLikeZodShape(schema.shape)).toBe(true);
|
||||
expect(looksLikeZodShape(schema)).toBe(false);
|
||||
expect(looksLikeZodShape(null)).toBe(false);
|
||||
expect(looksLikeZodShape(undefined)).toBe(false);
|
||||
expect(looksLikeZodShape({ name: "test" })).toBe(false);
|
||||
expect(looksLikeZodShape({})).toBe(false);
|
||||
expect(looksLikeZodShape([1, 2, 3])).toBe(false);
|
||||
expect(looksLikeZodShape({ type: "object", properties: {} })).toBe(false);
|
||||
});
|
||||
|
||||
test("SDK-like usage: convert Zod schema or pass through JSON schema", () => {
|
||||
const zodSchema = z.object({
|
||||
name: z.string(),
|
||||
email: z.string().email(),
|
||||
age: z.number().min(0),
|
||||
});
|
||||
|
||||
if (isZodSchema(zodSchema)) {
|
||||
const result = zodSchemaToJsonSchema(zodSchema) as Record<string, unknown>;
|
||||
expect(result.type).toBe("object");
|
||||
expect(result.properties).toBeDefined();
|
||||
} else {
|
||||
throw new Error("Should detect Zod schema");
|
||||
}
|
||||
|
||||
const existingJsonSchema = {
|
||||
type: "object" as const,
|
||||
properties: { title: { type: "string" as const } },
|
||||
required: ["title"] as string[],
|
||||
};
|
||||
|
||||
expect(isZodSchema(existingJsonSchema)).toBe(false);
|
||||
expect(zodSchemaToJsonSchema(existingJsonSchema)).toEqual(existingJsonSchema);
|
||||
});
|
||||
});
|
||||
2146
참고/firecrawl-main/apps/js-sdk/firecrawl/src/index.backup.ts
Normal file
2146
참고/firecrawl-main/apps/js-sdk/firecrawl/src/index.backup.ts
Normal file
File diff suppressed because it is too large
Load Diff
47
참고/firecrawl-main/apps/js-sdk/firecrawl/src/index.ts
Normal file
47
참고/firecrawl-main/apps/js-sdk/firecrawl/src/index.ts
Normal file
@@ -0,0 +1,47 @@
|
||||
/**
|
||||
* Firecrawl JS/TS SDK — unified entrypoint.
|
||||
* - v2 by default on the top‑level client
|
||||
* - v1 available under `.v1` (feature‑frozen)
|
||||
* - Exports: `Firecrawl` (default), `FirecrawlClient` (v2), `FirecrawlAppV1` (v1), and v2 types
|
||||
*/
|
||||
|
||||
/** Direct v2 client. */
|
||||
export { FirecrawlClient } from "./v2/client";
|
||||
/** Public v2 request/response types. */
|
||||
export * from "./v2/types";
|
||||
/** Watcher class and options for crawl/batch job monitoring. */
|
||||
export { Watcher, type WatcherOptions } from "./v2/watcher";
|
||||
/** Legacy v1 client (feature‑frozen). */
|
||||
export { default as FirecrawlAppV1 } from "./v1";
|
||||
|
||||
import V1 from "./v1";
|
||||
import { FirecrawlClient as V2, type FirecrawlClientOptions } from "./v2/client";
|
||||
import type { FirecrawlAppConfig } from "./v1";
|
||||
|
||||
// Re-export v2 client options for convenience
|
||||
export type { FirecrawlClientOptions } from "./v2/client";
|
||||
|
||||
/** Unified client: extends v2 and adds `.v1` for backward compatibility. */
|
||||
export class Firecrawl extends V2 {
|
||||
/** Feature‑frozen v1 client (lazy). */
|
||||
private _v1?: V1;
|
||||
private _v1Opts: FirecrawlAppConfig;
|
||||
|
||||
/** @param opts API credentials and base URL. */
|
||||
constructor(opts: FirecrawlClientOptions = {}) {
|
||||
super(opts);
|
||||
this._v1Opts = {
|
||||
apiKey: opts.apiKey,
|
||||
apiUrl: opts.apiUrl,
|
||||
};
|
||||
}
|
||||
|
||||
/** Access the legacy v1 client (instantiated on first access). */
|
||||
get v1(): V1 {
|
||||
if (!this._v1) this._v1 = new V1(this._v1Opts);
|
||||
return this._v1;
|
||||
}
|
||||
}
|
||||
|
||||
export default Firecrawl;
|
||||
|
||||
10
참고/firecrawl-main/apps/js-sdk/firecrawl/src/types/node-undici.d.ts
vendored
Normal file
10
참고/firecrawl-main/apps/js-sdk/firecrawl/src/types/node-undici.d.ts
vendored
Normal file
@@ -0,0 +1,10 @@
|
||||
type WebSocketCtor = typeof globalThis.WebSocket;
|
||||
|
||||
declare module "node:undici" {
|
||||
export const WebSocket: WebSocketCtor;
|
||||
const _default: {
|
||||
WebSocket: WebSocketCtor;
|
||||
};
|
||||
export default _default;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,67 @@
|
||||
import { zodToJsonSchema as zodToJsonSchemaLib } from "zod-to-json-schema";
|
||||
|
||||
type SchemaConverter = (schema: unknown) => unknown;
|
||||
|
||||
export function isZodSchema(value: unknown): boolean {
|
||||
if (!value || typeof value !== "object") return false;
|
||||
const schema = value as Record<string, unknown>;
|
||||
|
||||
const hasV3Markers =
|
||||
"_def" in schema &&
|
||||
(typeof schema.safeParse === "function" ||
|
||||
typeof schema.parse === "function");
|
||||
|
||||
const hasV4Markers = "_zod" in schema && typeof schema._zod === "object";
|
||||
|
||||
return hasV3Markers || hasV4Markers;
|
||||
}
|
||||
|
||||
function isZodV4Schema(schema: unknown): boolean {
|
||||
if (!schema || typeof schema !== "object") return false;
|
||||
return "_zod" in schema && typeof (schema as Record<string, unknown>)._zod === "object";
|
||||
}
|
||||
|
||||
function tryZodV4Conversion(schema: unknown): Record<string, unknown> | null {
|
||||
if (!isZodV4Schema(schema)) return null;
|
||||
|
||||
try {
|
||||
const zodModule = (schema as Record<string, unknown>).constructor?.prototype?.constructor;
|
||||
if (zodModule && typeof (zodModule as Record<string, unknown>).toJSONSchema === "function") {
|
||||
return (zodModule as { toJSONSchema: SchemaConverter }).toJSONSchema(schema) as Record<string, unknown>;
|
||||
}
|
||||
} catch {
|
||||
// V4 conversion not available
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
export function zodSchemaToJsonSchema(schema: unknown): Record<string, unknown> | unknown {
|
||||
if (!isZodSchema(schema)) {
|
||||
return schema;
|
||||
}
|
||||
|
||||
const v4Result = tryZodV4Conversion(schema);
|
||||
if (v4Result) {
|
||||
return v4Result;
|
||||
}
|
||||
|
||||
try {
|
||||
return zodToJsonSchemaLib(schema as Parameters<typeof zodToJsonSchemaLib>[0]) as Record<string, unknown>;
|
||||
} catch {
|
||||
return schema;
|
||||
}
|
||||
}
|
||||
|
||||
export function looksLikeZodShape(obj: unknown): boolean {
|
||||
if (!obj || typeof obj !== "object" || Array.isArray(obj)) return false;
|
||||
const values = Object.values(obj);
|
||||
if (values.length === 0) return false;
|
||||
return values.some(
|
||||
(v) =>
|
||||
v &&
|
||||
typeof v === "object" &&
|
||||
(v as Record<string, unknown>)._def &&
|
||||
typeof (v as Record<string, unknown>).safeParse === "function"
|
||||
);
|
||||
}
|
||||
2298
참고/firecrawl-main/apps/js-sdk/firecrawl/src/v1/index.ts
Normal file
2298
참고/firecrawl-main/apps/js-sdk/firecrawl/src/v1/index.ts
Normal file
File diff suppressed because it is too large
Load Diff
552
참고/firecrawl-main/apps/js-sdk/firecrawl/src/v2/client.ts
Normal file
552
참고/firecrawl-main/apps/js-sdk/firecrawl/src/v2/client.ts
Normal file
@@ -0,0 +1,552 @@
|
||||
import { HttpClient } from "./utils/httpClient";
|
||||
import {
|
||||
scrape,
|
||||
interact as interactMethod,
|
||||
stopInteraction as stopInteractionMethod,
|
||||
} from "./methods/scrape";
|
||||
import { parse as parseMethod } from "./methods/parse";
|
||||
import { search } from "./methods/search";
|
||||
import { map as mapMethod } from "./methods/map";
|
||||
import {
|
||||
startCrawl,
|
||||
getCrawlStatus,
|
||||
cancelCrawl,
|
||||
crawl as crawlWaiter,
|
||||
getCrawlErrors,
|
||||
getActiveCrawls,
|
||||
crawlParamsPreview,
|
||||
} from "./methods/crawl";
|
||||
import {
|
||||
startBatchScrape,
|
||||
getBatchScrapeStatus,
|
||||
getBatchScrapeErrors,
|
||||
cancelBatchScrape,
|
||||
batchScrape as batchWaiter,
|
||||
} from "./methods/batch";
|
||||
import { startExtract, getExtractStatus, extract as extractWaiter } from "./methods/extract";
|
||||
import { startAgent, getAgentStatus, cancelAgent, agent as agentWaiter } from "./methods/agent";
|
||||
import {
|
||||
browser as browserMethod,
|
||||
browserExecute,
|
||||
deleteBrowser,
|
||||
listBrowsers,
|
||||
} from "./methods/browser";
|
||||
import { getConcurrency, getCreditUsage, getQueueStatus, getTokenUsage, getCreditUsageHistorical, getTokenUsageHistorical } from "./methods/usage";
|
||||
import {
|
||||
createMonitor as createMonitorMethod,
|
||||
deleteMonitor as deleteMonitorMethod,
|
||||
getMonitor as getMonitorMethod,
|
||||
getMonitorCheck as getMonitorCheckMethod,
|
||||
listMonitorChecks as listMonitorChecksMethod,
|
||||
listMonitors as listMonitorsMethod,
|
||||
runMonitor as runMonitorMethod,
|
||||
updateMonitor as updateMonitorMethod,
|
||||
} from "./methods/monitor";
|
||||
import type {
|
||||
Document,
|
||||
ParseFile,
|
||||
ParseOptions,
|
||||
ScrapeOptions,
|
||||
SearchData,
|
||||
SearchRequest,
|
||||
MapData,
|
||||
MapOptions,
|
||||
CrawlResponse,
|
||||
CrawlJob,
|
||||
CrawlErrorsResponse,
|
||||
ActiveCrawlsResponse,
|
||||
BatchScrapeResponse,
|
||||
BatchScrapeJob,
|
||||
ExtractResponse,
|
||||
AgentResponse,
|
||||
AgentStatusResponse,
|
||||
CrawlOptions,
|
||||
BatchScrapeOptions,
|
||||
PaginationConfig,
|
||||
BrowserCreateResponse,
|
||||
BrowserExecuteResponse,
|
||||
BrowserDeleteResponse,
|
||||
BrowserListResponse,
|
||||
ScrapeExecuteRequest,
|
||||
ScrapeExecuteResponse,
|
||||
ScrapeBrowserDeleteResponse,
|
||||
CreateMonitorRequest,
|
||||
ListMonitorChecksOptions,
|
||||
ListMonitorsOptions,
|
||||
Monitor,
|
||||
MonitorCheck,
|
||||
MonitorCheckDetail,
|
||||
GetMonitorCheckOptions,
|
||||
UpdateMonitorRequest,
|
||||
} from "./types";
|
||||
import { Watcher } from "./watcher";
|
||||
import type { WatcherOptions } from "./watcher";
|
||||
import * as zt from "zod";
|
||||
|
||||
// Helper types to infer the `json` field from a Zod schema included in `formats`
|
||||
type ExtractJsonSchemaFromFormats<Formats> = Formats extends readonly any[]
|
||||
? Extract<Formats[number], { type: "json"; schema?: unknown }>["schema"]
|
||||
: never;
|
||||
|
||||
type InferredJsonFromOptions<Opts> = Opts extends { formats?: infer Fmts }
|
||||
? ExtractJsonSchemaFromFormats<Fmts> extends zt.ZodTypeAny
|
||||
? zt.infer<ExtractJsonSchemaFromFormats<Fmts>>
|
||||
: unknown
|
||||
: unknown;
|
||||
|
||||
/**
|
||||
* Configuration for the v2 client transport.
|
||||
*/
|
||||
export interface FirecrawlClientOptions {
|
||||
/** API key (falls back to FIRECRAWL_API_KEY). */
|
||||
apiKey?: string | null;
|
||||
/** API base URL (falls back to FIRECRAWL_API_URL or https://api.firecrawl.dev). */
|
||||
apiUrl?: string | null;
|
||||
/** Per-request timeout in milliseconds (optional). */
|
||||
timeoutMs?: number;
|
||||
/** Max automatic retries for transient failures (optional). */
|
||||
maxRetries?: number;
|
||||
/** Exponential backoff factor for retries (optional). */
|
||||
backoffFactor?: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Firecrawl v2 client. Provides typed access to all v2 endpoints and utilities.
|
||||
*/
|
||||
|
||||
export class FirecrawlClient {
|
||||
private readonly http: HttpClient;
|
||||
|
||||
private isCloudService(url: string): boolean {
|
||||
return url.includes('api.firecrawl.dev');
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a v2 client.
|
||||
* @param options Transport configuration (API key, base URL, timeouts, retries).
|
||||
*/
|
||||
constructor(options: FirecrawlClientOptions = {}) {
|
||||
const apiKey = options.apiKey ?? process.env.FIRECRAWL_API_KEY ?? "";
|
||||
const apiUrl = (options.apiUrl ?? process.env.FIRECRAWL_API_URL ?? "https://api.firecrawl.dev").replace(/\/$/, "");
|
||||
|
||||
if (this.isCloudService(apiUrl) && !apiKey) {
|
||||
throw new Error("API key is required for the cloud API. Set FIRECRAWL_API_KEY env or pass apiKey.");
|
||||
}
|
||||
|
||||
this.http = new HttpClient({
|
||||
apiKey,
|
||||
apiUrl,
|
||||
timeoutMs: options.timeoutMs,
|
||||
maxRetries: options.maxRetries,
|
||||
backoffFactor: options.backoffFactor,
|
||||
});
|
||||
}
|
||||
|
||||
// Scrape
|
||||
/**
|
||||
* Scrape a single URL.
|
||||
* @param url Target URL.
|
||||
* @param options Optional scrape options (formats, headers, etc.).
|
||||
* @returns Resolved document with requested formats.
|
||||
*/
|
||||
async scrape<Opts extends ScrapeOptions>(
|
||||
url: string,
|
||||
options: Opts
|
||||
): Promise<Omit<Document, "json"> & { json?: InferredJsonFromOptions<Opts> }>;
|
||||
async scrape(url: string, options?: ScrapeOptions): Promise<Document>;
|
||||
async scrape(url: string, options?: ScrapeOptions): Promise<Document> {
|
||||
return scrape(this.http, url, options);
|
||||
}
|
||||
/**
|
||||
* Interact with the browser session associated with a scrape job.
|
||||
* @param jobId Scrape job id.
|
||||
* @param args Code or prompt to execute, with language/timeout options.
|
||||
* @returns Execution result including output, stdout, stderr, exitCode, and killed status.
|
||||
*/
|
||||
async interact(
|
||||
jobId: string,
|
||||
args: ScrapeExecuteRequest
|
||||
): Promise<ScrapeExecuteResponse> {
|
||||
return interactMethod(this.http, jobId, args);
|
||||
}
|
||||
/**
|
||||
* Stop the interaction session associated with a scrape job.
|
||||
* @param jobId Scrape job id.
|
||||
*/
|
||||
async stopInteraction(jobId: string): Promise<ScrapeBrowserDeleteResponse> {
|
||||
return stopInteractionMethod(this.http, jobId);
|
||||
}
|
||||
/**
|
||||
* @deprecated Use interact().
|
||||
*/
|
||||
async scrapeExecute(
|
||||
jobId: string,
|
||||
args: ScrapeExecuteRequest
|
||||
): Promise<ScrapeExecuteResponse> {
|
||||
return this.interact(jobId, args);
|
||||
}
|
||||
/**
|
||||
* @deprecated Use stopInteraction().
|
||||
*/
|
||||
async stopInteractiveBrowser(jobId: string): Promise<ScrapeBrowserDeleteResponse> {
|
||||
return this.stopInteraction(jobId);
|
||||
}
|
||||
/**
|
||||
* @deprecated Use stopInteraction().
|
||||
*/
|
||||
async deleteScrapeBrowser(jobId: string): Promise<ScrapeBrowserDeleteResponse> {
|
||||
return this.stopInteraction(jobId);
|
||||
}
|
||||
|
||||
// Parse
|
||||
/**
|
||||
* Parse an uploaded file via the v2 parse endpoint.
|
||||
* @param file File payload (data, filename, optional contentType).
|
||||
* @param options Optional parse options (formats, parsers, etc.).
|
||||
* Note: parse does not support changeTracking, screenshot, branding,
|
||||
* actions, waitFor, location, or mobile options.
|
||||
* @returns Parsed document with requested formats.
|
||||
*/
|
||||
async parse<Opts extends ParseOptions>(
|
||||
file: ParseFile,
|
||||
options: Opts
|
||||
): Promise<Omit<Document, "json"> & { json?: InferredJsonFromOptions<Opts> }>;
|
||||
async parse(file: ParseFile, options?: ParseOptions): Promise<Document>;
|
||||
async parse(file: ParseFile, options?: ParseOptions): Promise<Document> {
|
||||
return parseMethod(this.http, file, options);
|
||||
}
|
||||
|
||||
// Search
|
||||
/**
|
||||
* Search the web and optionally scrape each result.
|
||||
* @param query Search query string.
|
||||
* @param req Additional search options (sources, limit, scrapeOptions, etc.).
|
||||
* @returns Structured search results.
|
||||
*/
|
||||
async search(query: string, req: Omit<SearchRequest, "query"> = {}): Promise<SearchData> {
|
||||
return search(this.http, { query, ...req });
|
||||
}
|
||||
|
||||
// Map
|
||||
/**
|
||||
* Map a site to discover URLs (sitemap-aware).
|
||||
* @param url Root URL to map.
|
||||
* @param options Mapping options (sitemap mode, includeSubdomains, limit, timeout).
|
||||
* @returns Discovered links.
|
||||
*/
|
||||
async map(url: string, options?: MapOptions): Promise<MapData> {
|
||||
return mapMethod(this.http, url, options);
|
||||
}
|
||||
|
||||
// Crawl
|
||||
/**
|
||||
* Start a crawl job (async).
|
||||
* @param url Root URL to crawl.
|
||||
* @param req Crawl configuration (paths, limits, scrapeOptions, webhook, etc.).
|
||||
* @returns Job id and url.
|
||||
*/
|
||||
async startCrawl(url: string, req: CrawlOptions = {}): Promise<CrawlResponse> {
|
||||
return startCrawl(this.http, { url, ...req });
|
||||
}
|
||||
/**
|
||||
* Get the status and partial data of a crawl job.
|
||||
* @param jobId Crawl job id.
|
||||
*/
|
||||
async getCrawlStatus(jobId: string, pagination?: PaginationConfig): Promise<CrawlJob> {
|
||||
return getCrawlStatus(this.http, jobId, pagination);
|
||||
}
|
||||
/**
|
||||
* Cancel a crawl job.
|
||||
* @param jobId Crawl job id.
|
||||
* @returns True if cancelled.
|
||||
*/
|
||||
async cancelCrawl(jobId: string): Promise<boolean> {
|
||||
return cancelCrawl(this.http, jobId);
|
||||
}
|
||||
/**
|
||||
* Convenience waiter: start a crawl and poll until it finishes.
|
||||
* @param url Root URL to crawl.
|
||||
* @param req Crawl configuration plus waiter controls (pollInterval, timeout seconds).
|
||||
* @returns Final job snapshot.
|
||||
*/
|
||||
async crawl(url: string, req: CrawlOptions & { pollInterval?: number; timeout?: number } = {}): Promise<CrawlJob> {
|
||||
return crawlWaiter(this.http, { url, ...req }, req.pollInterval, req.timeout);
|
||||
}
|
||||
/**
|
||||
* Retrieve crawl errors and robots.txt blocks.
|
||||
* @param crawlId Crawl job id.
|
||||
*/
|
||||
async getCrawlErrors(crawlId: string): Promise<CrawlErrorsResponse> {
|
||||
return getCrawlErrors(this.http, crawlId);
|
||||
}
|
||||
/**
|
||||
* List active crawls for the authenticated team.
|
||||
*/
|
||||
async getActiveCrawls(): Promise<ActiveCrawlsResponse> {
|
||||
return getActiveCrawls(this.http);
|
||||
}
|
||||
/**
|
||||
* Preview normalized crawl parameters produced by a natural-language prompt.
|
||||
* @param url Root URL.
|
||||
* @param prompt Natural-language instruction.
|
||||
*/
|
||||
async crawlParamsPreview(url: string, prompt: string): Promise<Record<string, unknown>> {
|
||||
return crawlParamsPreview(this.http, url, prompt);
|
||||
}
|
||||
|
||||
// Monitor
|
||||
/**
|
||||
* Create a scheduled monitor.
|
||||
*/
|
||||
async createMonitor(request: CreateMonitorRequest): Promise<Monitor> {
|
||||
return createMonitorMethod(this.http, request);
|
||||
}
|
||||
|
||||
/**
|
||||
* List monitors for the authenticated team.
|
||||
*/
|
||||
async listMonitors(options?: ListMonitorsOptions): Promise<Monitor[]> {
|
||||
return listMonitorsMethod(this.http, options);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get a monitor by id.
|
||||
*/
|
||||
async getMonitor(monitorId: string): Promise<Monitor> {
|
||||
return getMonitorMethod(this.http, monitorId);
|
||||
}
|
||||
|
||||
/**
|
||||
* Update a monitor.
|
||||
*/
|
||||
async updateMonitor(
|
||||
monitorId: string,
|
||||
request: UpdateMonitorRequest,
|
||||
): Promise<Monitor> {
|
||||
return updateMonitorMethod(this.http, monitorId, request);
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete a monitor.
|
||||
*/
|
||||
async deleteMonitor(monitorId: string): Promise<boolean> {
|
||||
return deleteMonitorMethod(this.http, monitorId);
|
||||
}
|
||||
|
||||
/**
|
||||
* Trigger a manual monitor check.
|
||||
*/
|
||||
async runMonitor(monitorId: string): Promise<MonitorCheck> {
|
||||
return runMonitorMethod(this.http, monitorId);
|
||||
}
|
||||
|
||||
/**
|
||||
* List checks for a monitor.
|
||||
*/
|
||||
async listMonitorChecks(
|
||||
monitorId: string,
|
||||
options?: ListMonitorChecksOptions,
|
||||
): Promise<MonitorCheck[]> {
|
||||
return listMonitorChecksMethod(this.http, monitorId, options);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get a monitor check with paginated page results and inline diffs.
|
||||
*/
|
||||
async getMonitorCheck(
|
||||
monitorId: string,
|
||||
checkId: string,
|
||||
options?: GetMonitorCheckOptions,
|
||||
): Promise<MonitorCheckDetail> {
|
||||
return getMonitorCheckMethod(this.http, monitorId, checkId, options);
|
||||
}
|
||||
|
||||
// Batch
|
||||
/**
|
||||
* Start a batch scrape job for multiple URLs (async).
|
||||
* @param urls URLs to scrape.
|
||||
* @param opts Batch options (scrape options, webhook, concurrency, idempotency key, etc.).
|
||||
* @returns Job id and url.
|
||||
*/
|
||||
async startBatchScrape(urls: string[], opts?: BatchScrapeOptions): Promise<BatchScrapeResponse> {
|
||||
return startBatchScrape(this.http, urls, opts);
|
||||
}
|
||||
/**
|
||||
* Get the status and partial data of a batch scrape job.
|
||||
* @param jobId Batch job id.
|
||||
*/
|
||||
async getBatchScrapeStatus(jobId: string, pagination?: PaginationConfig): Promise<BatchScrapeJob> {
|
||||
return getBatchScrapeStatus(this.http, jobId, pagination);
|
||||
}
|
||||
/**
|
||||
* Retrieve batch scrape errors and robots.txt blocks.
|
||||
* @param jobId Batch job id.
|
||||
*/
|
||||
async getBatchScrapeErrors(jobId: string): Promise<CrawlErrorsResponse> {
|
||||
return getBatchScrapeErrors(this.http, jobId);
|
||||
}
|
||||
/**
|
||||
* Cancel a batch scrape job.
|
||||
* @param jobId Batch job id.
|
||||
* @returns True if cancelled.
|
||||
*/
|
||||
async cancelBatchScrape(jobId: string): Promise<boolean> {
|
||||
return cancelBatchScrape(this.http, jobId);
|
||||
}
|
||||
/**
|
||||
* Convenience waiter: start a batch scrape and poll until it finishes.
|
||||
* @param urls URLs to scrape.
|
||||
* @param opts Batch options plus waiter controls (pollInterval, timeout seconds).
|
||||
* @returns Final job snapshot.
|
||||
*/
|
||||
async batchScrape(urls: string[], opts?: BatchScrapeOptions & { pollInterval?: number; timeout?: number }): Promise<BatchScrapeJob> {
|
||||
return batchWaiter(this.http, urls, opts);
|
||||
}
|
||||
|
||||
// Extract
|
||||
/**
|
||||
* Start an extract job (async).
|
||||
* @param args Extraction request (urls, schema or prompt, flags).
|
||||
* @returns Job id or processing state.
|
||||
* @deprecated The extract endpoint is in maintenance mode and its use is discouraged.
|
||||
* Review https://docs.firecrawl.dev/developer-guides/usage-guides/choosing-the-data-extractor to find a replacement.
|
||||
*/
|
||||
async startExtract(args: Parameters<typeof startExtract>[1]): Promise<ExtractResponse> {
|
||||
return startExtract(this.http, args);
|
||||
}
|
||||
/**
|
||||
* Get extract job status/data.
|
||||
* @param jobId Extract job id.
|
||||
* @deprecated The extract endpoint is in maintenance mode and its use is discouraged.
|
||||
* Review https://docs.firecrawl.dev/developer-guides/usage-guides/choosing-the-data-extractor to find a replacement.
|
||||
*/
|
||||
async getExtractStatus(jobId: string): Promise<ExtractResponse> {
|
||||
return getExtractStatus(this.http, jobId);
|
||||
}
|
||||
/**
|
||||
* Convenience waiter: start an extract and poll until it finishes.
|
||||
* @param args Extraction request plus waiter controls (pollInterval, timeout seconds).
|
||||
* @returns Final extract response.
|
||||
* @deprecated The extract endpoint is in maintenance mode and its use is discouraged.
|
||||
* Review https://docs.firecrawl.dev/developer-guides/usage-guides/choosing-the-data-extractor to find a replacement.
|
||||
*/
|
||||
async extract(args: Parameters<typeof startExtract>[1] & { pollInterval?: number; timeout?: number }): Promise<ExtractResponse> {
|
||||
return extractWaiter(this.http, args);
|
||||
}
|
||||
|
||||
// Agent
|
||||
/**
|
||||
* Start an agent job (async).
|
||||
* @param args Agent request (urls, prompt, schema).
|
||||
* @returns Job id or processing state.
|
||||
*/
|
||||
async startAgent(args: Parameters<typeof startAgent>[1]): Promise<AgentResponse> {
|
||||
return startAgent(this.http, args);
|
||||
}
|
||||
/**
|
||||
* Get agent job status/data.
|
||||
* @param jobId Agent job id.
|
||||
*/
|
||||
async getAgentStatus(jobId: string): Promise<AgentStatusResponse> {
|
||||
return getAgentStatus(this.http, jobId);
|
||||
}
|
||||
/**
|
||||
* Convenience waiter: start an agent and poll until it finishes.
|
||||
* @param args Agent request plus waiter controls (pollInterval, timeout seconds).
|
||||
* @returns Final agent response.
|
||||
*/
|
||||
async agent(args: Parameters<typeof startAgent>[1] & { pollInterval?: number; timeout?: number }): Promise<AgentStatusResponse> {
|
||||
return agentWaiter(this.http, args);
|
||||
}
|
||||
/**
|
||||
* Cancel an agent job.
|
||||
* @param jobId Agent job id.
|
||||
* @returns True if cancelled.
|
||||
*/
|
||||
async cancelAgent(jobId: string): Promise<boolean> {
|
||||
return cancelAgent(this.http, jobId);
|
||||
}
|
||||
|
||||
// Browser
|
||||
/**
|
||||
* Create a new browser session.
|
||||
* @param args Session options (ttl, activityTtl, streamWebView, profile).
|
||||
* @returns Session id, CDP URL, live view URL, and expiration time.
|
||||
*/
|
||||
async browser(
|
||||
args: Parameters<typeof browserMethod>[1] = {}
|
||||
): Promise<BrowserCreateResponse> {
|
||||
return browserMethod(this.http, args);
|
||||
}
|
||||
/**
|
||||
* Execute code in a browser session.
|
||||
* @param sessionId Browser session id.
|
||||
* @param args Code, language ("python" | "node" | "bash"), and optional timeout.
|
||||
* @returns Execution result including stdout, stderr, exitCode, and killed status.
|
||||
*/
|
||||
async browserExecute(
|
||||
sessionId: string,
|
||||
args: Parameters<typeof browserExecute>[2]
|
||||
): Promise<BrowserExecuteResponse> {
|
||||
return browserExecute(this.http, sessionId, args);
|
||||
}
|
||||
/**
|
||||
* Delete a browser session.
|
||||
* @param sessionId Browser session id.
|
||||
*/
|
||||
async deleteBrowser(sessionId: string): Promise<BrowserDeleteResponse> {
|
||||
return deleteBrowser(this.http, sessionId);
|
||||
}
|
||||
/**
|
||||
* List browser sessions.
|
||||
* @param args Optional filter (status: "active" | "destroyed").
|
||||
* @returns List of browser sessions.
|
||||
*/
|
||||
async listBrowsers(
|
||||
args: Parameters<typeof listBrowsers>[1] = {}
|
||||
): Promise<BrowserListResponse> {
|
||||
return listBrowsers(this.http, args);
|
||||
}
|
||||
|
||||
// Usage
|
||||
/** Current concurrency usage. */
|
||||
async getConcurrency() {
|
||||
return getConcurrency(this.http);
|
||||
}
|
||||
/** Current credit usage. */
|
||||
async getCreditUsage() {
|
||||
return getCreditUsage(this.http);
|
||||
}
|
||||
/** Recent token usage. */
|
||||
async getTokenUsage() {
|
||||
return getTokenUsage(this.http);
|
||||
}
|
||||
|
||||
/** Historical credit usage by month; set byApiKey to true to break down by API key. */
|
||||
async getCreditUsageHistorical(byApiKey?: boolean) {
|
||||
return getCreditUsageHistorical(this.http, byApiKey);
|
||||
}
|
||||
|
||||
/** Historical token usage by month; set byApiKey to true to break down by API key. */
|
||||
async getTokenUsageHistorical(byApiKey?: boolean) {
|
||||
return getTokenUsageHistorical(this.http, byApiKey);
|
||||
}
|
||||
|
||||
/** Metrics about the team's scrape queue. */
|
||||
async getQueueStatus() {
|
||||
return getQueueStatus(this.http);
|
||||
}
|
||||
|
||||
// Watcher
|
||||
/**
|
||||
* Create a watcher for a crawl or batch job. Emits: `document`, `snapshot`, `done`, `error`.
|
||||
* @param jobId Job id.
|
||||
* @param opts Watcher options (kind, pollInterval, timeout seconds).
|
||||
*/
|
||||
watcher(jobId: string, opts: WatcherOptions = {}): Watcher {
|
||||
return new Watcher(this.http, jobId, opts);
|
||||
}
|
||||
}
|
||||
|
||||
export default FirecrawlClient;
|
||||
|
||||
@@ -0,0 +1,90 @@
|
||||
import { type AgentResponse, type AgentStatusResponse, type AgentWebhookConfig } from "../types";
|
||||
import { HttpClient } from "../utils/httpClient";
|
||||
import { normalizeAxiosError, throwForBadResponse } from "../utils/errorHandler";
|
||||
import { isZodSchema, zodSchemaToJsonSchema } from "../../utils/zodSchemaToJson";
|
||||
import type { ZodTypeAny } from "zod";
|
||||
|
||||
function prepareAgentPayload(args: {
|
||||
urls?: string[];
|
||||
prompt: string;
|
||||
schema?: Record<string, unknown> | ZodTypeAny;
|
||||
integration?: string;
|
||||
origin?: string;
|
||||
maxCredits?: number;
|
||||
strictConstrainToURLs?: boolean;
|
||||
model?: "spark-1-pro" | "spark-1-mini";
|
||||
webhook?: string | AgentWebhookConfig;
|
||||
}): Record<string, unknown> {
|
||||
const body: Record<string, unknown> = {};
|
||||
if (args.urls) body.urls = args.urls;
|
||||
body.prompt = args.prompt;
|
||||
if (args.schema != null) {
|
||||
body.schema = isZodSchema(args.schema) ? zodSchemaToJsonSchema(args.schema) : args.schema;
|
||||
}
|
||||
if (args.integration && args.integration.trim()) body.integration = args.integration.trim();
|
||||
if (args.origin) body.origin = args.origin;
|
||||
if (args.maxCredits !== null && args.maxCredits !== undefined) body.maxCredits = args.maxCredits;
|
||||
if (args.strictConstrainToURLs !== null && args.strictConstrainToURLs !== undefined) body.strictConstrainToURLs = args.strictConstrainToURLs;
|
||||
if (args.model !== null && args.model !== undefined) body.model = args.model;
|
||||
if (args.webhook != null) body.webhook = args.webhook;
|
||||
return body;
|
||||
}
|
||||
|
||||
export async function startAgent(http: HttpClient, args: Parameters<typeof prepareAgentPayload>[0]): Promise<AgentResponse> {
|
||||
const payload = prepareAgentPayload(args);
|
||||
try {
|
||||
const res = await http.post<AgentResponse>("/v2/agent", payload);
|
||||
if (res.status !== 200) throwForBadResponse(res, "agent");
|
||||
return res.data;
|
||||
} catch (err: any) {
|
||||
if (err?.isAxiosError) return normalizeAxiosError(err, "agent");
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
export async function getAgentStatus(http: HttpClient, jobId: string): Promise<AgentStatusResponse> {
|
||||
try {
|
||||
const res = await http.get<AgentStatusResponse>(`/v2/agent/${jobId}`);
|
||||
if (res.status !== 200) throwForBadResponse(res, "agent status");
|
||||
return res.data;
|
||||
} catch (err: any) {
|
||||
if (err?.isAxiosError) return normalizeAxiosError(err, "agent status");
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
export async function waitAgent(
|
||||
http: HttpClient,
|
||||
jobId: string,
|
||||
pollInterval = 2,
|
||||
timeout?: number
|
||||
): Promise<AgentStatusResponse> {
|
||||
const start = Date.now();
|
||||
while (true) {
|
||||
const status = await getAgentStatus(http, jobId);
|
||||
if (["completed", "failed", "cancelled"].includes(status.status || "")) return status;
|
||||
if (timeout != null && Date.now() - start > timeout * 1000) return status;
|
||||
await new Promise((r) => setTimeout(r, Math.max(1000, pollInterval * 1000)));
|
||||
}
|
||||
}
|
||||
|
||||
export async function agent(
|
||||
http: HttpClient,
|
||||
args: Parameters<typeof prepareAgentPayload>[0] & { pollInterval?: number; timeout?: number }
|
||||
): Promise<AgentStatusResponse> {
|
||||
const started = await startAgent(http, args);
|
||||
const jobId = started.id;
|
||||
if (!jobId) return started as unknown as AgentStatusResponse;
|
||||
return waitAgent(http, jobId, args.pollInterval ?? 2, args.timeout);
|
||||
}
|
||||
|
||||
export async function cancelAgent(http: HttpClient, jobId: string): Promise<boolean> {
|
||||
try {
|
||||
const res = await http.delete<{ success: boolean }>(`/v2/agent/${jobId}`);
|
||||
if (res.status !== 200) throwForBadResponse(res, "cancel agent");
|
||||
return res.data?.success === true;
|
||||
} catch (err: any) {
|
||||
if (err?.isAxiosError) return normalizeAxiosError(err, "cancel agent");
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
231
참고/firecrawl-main/apps/js-sdk/firecrawl/src/v2/methods/batch.ts
Normal file
231
참고/firecrawl-main/apps/js-sdk/firecrawl/src/v2/methods/batch.ts
Normal file
@@ -0,0 +1,231 @@
|
||||
import {
|
||||
type BatchScrapeJob,
|
||||
type BatchScrapeResponse,
|
||||
type CrawlErrorsResponse,
|
||||
type Document,
|
||||
type BatchScrapeOptions,
|
||||
type PaginationConfig,
|
||||
JobTimeoutError,
|
||||
SdkError,
|
||||
} from "../types";
|
||||
import { HttpClient } from "../utils/httpClient";
|
||||
import { ensureValidScrapeOptions } from "../utils/validation";
|
||||
import { fetchAllPages } from "../utils/pagination";
|
||||
import {
|
||||
normalizeAxiosError,
|
||||
throwForBadResponse,
|
||||
isRetryableError,
|
||||
} from "../utils/errorHandler";
|
||||
|
||||
export async function startBatchScrape(
|
||||
http: HttpClient,
|
||||
urls: string[],
|
||||
{
|
||||
options,
|
||||
webhook,
|
||||
appendToId,
|
||||
ignoreInvalidURLs,
|
||||
maxConcurrency,
|
||||
zeroDataRetention,
|
||||
idempotencyKey,
|
||||
integration,
|
||||
origin,
|
||||
}: BatchScrapeOptions = {},
|
||||
): Promise<BatchScrapeResponse> {
|
||||
if (!Array.isArray(urls) || urls.length === 0)
|
||||
throw new Error("URLs list cannot be empty");
|
||||
const payload: Record<string, unknown> = { urls };
|
||||
if (options) {
|
||||
ensureValidScrapeOptions(options);
|
||||
Object.assign(payload, options);
|
||||
}
|
||||
if (webhook != null) payload.webhook = webhook;
|
||||
if (appendToId != null) payload.appendToId = appendToId;
|
||||
if (ignoreInvalidURLs != null) payload.ignoreInvalidURLs = ignoreInvalidURLs;
|
||||
if (maxConcurrency != null) payload.maxConcurrency = maxConcurrency;
|
||||
if (zeroDataRetention != null) payload.zeroDataRetention = zeroDataRetention;
|
||||
if (integration != null && integration.trim())
|
||||
payload.integration = integration.trim();
|
||||
if (origin) payload.origin = origin;
|
||||
|
||||
try {
|
||||
const headers = http.prepareHeaders(idempotencyKey);
|
||||
const res = await http.post<{
|
||||
success: boolean;
|
||||
id: string;
|
||||
url: string;
|
||||
invalidURLs?: string[];
|
||||
error?: string;
|
||||
}>("/v2/batch/scrape", payload, { headers });
|
||||
if (res.status !== 200 || !res.data?.success)
|
||||
throwForBadResponse(res, "start batch scrape");
|
||||
return {
|
||||
id: res.data.id,
|
||||
url: res.data.url,
|
||||
invalidURLs: res.data.invalidURLs || undefined,
|
||||
};
|
||||
} catch (err: any) {
|
||||
if (err?.isAxiosError)
|
||||
return normalizeAxiosError(err, "start batch scrape");
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
export async function getBatchScrapeStatus(
|
||||
http: HttpClient,
|
||||
jobId: string,
|
||||
pagination?: PaginationConfig,
|
||||
): Promise<BatchScrapeJob> {
|
||||
try {
|
||||
const res = await http.get<{
|
||||
success: boolean;
|
||||
status: BatchScrapeJob["status"];
|
||||
completed?: number;
|
||||
total?: number;
|
||||
creditsUsed?: number;
|
||||
expiresAt?: string;
|
||||
next?: string | null;
|
||||
data?: Document[];
|
||||
}>(`/v2/batch/scrape/${jobId}`);
|
||||
if (res.status !== 200 || !res.data?.success)
|
||||
throwForBadResponse(res, "get batch scrape status");
|
||||
const body = res.data;
|
||||
const initialDocs = (body.data || []) as Document[];
|
||||
const auto = pagination?.autoPaginate ?? true;
|
||||
if (!auto || !body.next) {
|
||||
return {
|
||||
id: jobId,
|
||||
status: body.status,
|
||||
completed: body.completed ?? 0,
|
||||
total: body.total ?? 0,
|
||||
creditsUsed: body.creditsUsed,
|
||||
expiresAt: body.expiresAt,
|
||||
next: body.next ?? null,
|
||||
data: initialDocs,
|
||||
};
|
||||
}
|
||||
|
||||
const aggregated = await fetchAllPages(
|
||||
http,
|
||||
body.next,
|
||||
initialDocs,
|
||||
pagination,
|
||||
);
|
||||
return {
|
||||
id: jobId,
|
||||
status: body.status,
|
||||
completed: body.completed ?? 0,
|
||||
total: body.total ?? 0,
|
||||
creditsUsed: body.creditsUsed,
|
||||
expiresAt: body.expiresAt,
|
||||
next: null,
|
||||
data: aggregated,
|
||||
};
|
||||
} catch (err: any) {
|
||||
if (err?.isAxiosError)
|
||||
return normalizeAxiosError(err, "get batch scrape status");
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
export async function cancelBatchScrape(
|
||||
http: HttpClient,
|
||||
jobId: string,
|
||||
): Promise<boolean> {
|
||||
try {
|
||||
const res = await http.delete<{ status: string }>(
|
||||
`/v2/batch/scrape/${jobId}`,
|
||||
);
|
||||
if (res.status !== 200) throwForBadResponse(res, "cancel batch scrape");
|
||||
return res.data?.status === "cancelled";
|
||||
} catch (err: any) {
|
||||
if (err?.isAxiosError)
|
||||
return normalizeAxiosError(err, "cancel batch scrape");
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
export async function getBatchScrapeErrors(
|
||||
http: HttpClient,
|
||||
jobId: string,
|
||||
): Promise<CrawlErrorsResponse> {
|
||||
try {
|
||||
const res = await http.get<{
|
||||
success?: boolean;
|
||||
data?: { errors: Array<Record<string, string>>; robotsBlocked: string[] };
|
||||
}>(`/v2/batch/scrape/${jobId}/errors`);
|
||||
if (res.status !== 200) throwForBadResponse(res, "get batch scrape errors");
|
||||
const payload = res.data?.data ?? (res.data as any);
|
||||
return {
|
||||
errors: payload.errors || [],
|
||||
robotsBlocked: payload.robotsBlocked || [],
|
||||
};
|
||||
} catch (err: any) {
|
||||
if (err?.isAxiosError)
|
||||
return normalizeAxiosError(err, "get batch scrape errors");
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
export async function waitForBatchCompletion(
|
||||
http: HttpClient,
|
||||
jobId: string,
|
||||
pollInterval = 2,
|
||||
timeout?: number,
|
||||
): Promise<BatchScrapeJob> {
|
||||
const start = Date.now();
|
||||
|
||||
while (true) {
|
||||
try {
|
||||
const status = await getBatchScrapeStatus(http, jobId);
|
||||
|
||||
if (["completed", "failed", "cancelled"].includes(status.status)) {
|
||||
return status;
|
||||
}
|
||||
} catch (err: any) {
|
||||
// Don't retry on permanent errors (4xx) - re-throw immediately with jobId context
|
||||
if (!isRetryableError(err)) {
|
||||
// Create new error with jobId for better debugging (non-retryable errors like 404)
|
||||
if (err instanceof SdkError) {
|
||||
const errorWithJobId = new SdkError(
|
||||
err.message,
|
||||
err.status,
|
||||
err.code,
|
||||
err.details,
|
||||
jobId,
|
||||
);
|
||||
throw errorWithJobId;
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
// Otherwise, retry after delay - error might be transient (network issue, timeout, 5xx, etc.)
|
||||
}
|
||||
|
||||
if (timeout != null && Date.now() - start > timeout * 1000) {
|
||||
throw new JobTimeoutError(jobId, timeout, "batch");
|
||||
}
|
||||
|
||||
await new Promise(r => setTimeout(r, Math.max(1000, pollInterval * 1000)));
|
||||
}
|
||||
}
|
||||
|
||||
export async function batchScrape(
|
||||
http: HttpClient,
|
||||
urls: string[],
|
||||
opts: BatchScrapeOptions & { pollInterval?: number; timeout?: number } = {},
|
||||
): Promise<BatchScrapeJob> {
|
||||
const start = await startBatchScrape(http, urls, opts);
|
||||
return waitForBatchCompletion(
|
||||
http,
|
||||
start.id,
|
||||
opts.pollInterval ?? 2,
|
||||
opts.timeout,
|
||||
);
|
||||
}
|
||||
|
||||
export function chunkUrls(urls: string[], chunkSize = 100): string[][] {
|
||||
const chunks: string[][] = [];
|
||||
for (let i = 0; i < urls.length; i += chunkSize)
|
||||
chunks.push(urls.slice(i, i + chunkSize));
|
||||
return chunks;
|
||||
}
|
||||
@@ -0,0 +1,111 @@
|
||||
import type {
|
||||
BrowserCreateResponse,
|
||||
BrowserExecuteResponse,
|
||||
BrowserDeleteResponse,
|
||||
BrowserListResponse,
|
||||
} from "../types";
|
||||
import { HttpClient } from "../utils/httpClient";
|
||||
import {
|
||||
normalizeAxiosError,
|
||||
throwForBadResponse,
|
||||
} from "../utils/errorHandler";
|
||||
|
||||
export async function browser(
|
||||
http: HttpClient,
|
||||
args: {
|
||||
ttl?: number;
|
||||
activityTtl?: number;
|
||||
streamWebView?: boolean;
|
||||
profile?: {
|
||||
name: string;
|
||||
saveChanges?: boolean;
|
||||
};
|
||||
integration?: string;
|
||||
origin?: string;
|
||||
} = {},
|
||||
): Promise<BrowserCreateResponse> {
|
||||
const body: Record<string, unknown> = {};
|
||||
if (args.ttl != null) body.ttl = args.ttl;
|
||||
if (args.activityTtl != null) body.activityTtl = args.activityTtl;
|
||||
if (args.streamWebView != null) body.streamWebView = args.streamWebView;
|
||||
if (args.profile != null) body.profile = args.profile;
|
||||
if (args.integration != null) body.integration = args.integration;
|
||||
if (args.origin) body.origin = args.origin;
|
||||
|
||||
try {
|
||||
const res = await http.post<BrowserCreateResponse>("/v2/browser", body);
|
||||
if (res.status !== 200) throwForBadResponse(res, "create browser session");
|
||||
return res.data;
|
||||
} catch (err: any) {
|
||||
if (err?.isAxiosError)
|
||||
return normalizeAxiosError(err, "create browser session");
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
export async function browserExecute(
|
||||
http: HttpClient,
|
||||
sessionId: string,
|
||||
args: {
|
||||
code: string;
|
||||
language?: "python" | "node" | "bash";
|
||||
timeout?: number;
|
||||
},
|
||||
): Promise<BrowserExecuteResponse> {
|
||||
const body: Record<string, unknown> = {
|
||||
code: args.code,
|
||||
language: args.language ?? "bash",
|
||||
};
|
||||
if (args.timeout != null) body.timeout = args.timeout;
|
||||
|
||||
try {
|
||||
const res = await http.post<BrowserExecuteResponse>(
|
||||
`/v2/browser/${sessionId}/execute`,
|
||||
body,
|
||||
args.timeout != null ? { timeoutMs: args.timeout * 1000 + 5000 } : {},
|
||||
);
|
||||
if (res.status !== 200) throwForBadResponse(res, "execute browser code");
|
||||
return res.data;
|
||||
} catch (err: any) {
|
||||
if (err?.isAxiosError)
|
||||
return normalizeAxiosError(err, "execute browser code");
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
export async function deleteBrowser(
|
||||
http: HttpClient,
|
||||
sessionId: string,
|
||||
): Promise<BrowserDeleteResponse> {
|
||||
try {
|
||||
const res = await http.delete<BrowserDeleteResponse>(
|
||||
`/v2/browser/${sessionId}`,
|
||||
);
|
||||
if (res.status !== 200) throwForBadResponse(res, "delete browser session");
|
||||
return res.data;
|
||||
} catch (err: any) {
|
||||
if (err?.isAxiosError)
|
||||
return normalizeAxiosError(err, "delete browser session");
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
export async function listBrowsers(
|
||||
http: HttpClient,
|
||||
args: {
|
||||
status?: "active" | "destroyed";
|
||||
} = {},
|
||||
): Promise<BrowserListResponse> {
|
||||
let endpoint = "/v2/browser";
|
||||
if (args.status) endpoint += `?status=${args.status}`;
|
||||
|
||||
try {
|
||||
const res = await http.get<BrowserListResponse>(endpoint);
|
||||
if (res.status !== 200) throwForBadResponse(res, "list browser sessions");
|
||||
return res.data;
|
||||
} catch (err: any) {
|
||||
if (err?.isAxiosError)
|
||||
return normalizeAxiosError(err, "list browser sessions");
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
201
참고/firecrawl-main/apps/js-sdk/firecrawl/src/v2/methods/crawl.ts
Normal file
201
참고/firecrawl-main/apps/js-sdk/firecrawl/src/v2/methods/crawl.ts
Normal file
@@ -0,0 +1,201 @@
|
||||
import {
|
||||
type ActiveCrawlsResponse,
|
||||
type CrawlErrorsResponse,
|
||||
type CrawlJob,
|
||||
type CrawlResponse,
|
||||
type Document,
|
||||
type CrawlOptions,
|
||||
type PaginationConfig,
|
||||
JobTimeoutError,
|
||||
SdkError,
|
||||
} from "../types";
|
||||
import { HttpClient } from "../utils/httpClient";
|
||||
import { ensureValidScrapeOptions } from "../utils/validation";
|
||||
import { normalizeAxiosError, throwForBadResponse, isRetryableError } from "../utils/errorHandler";
|
||||
import type { HttpClient as _Http } from "../utils/httpClient";
|
||||
import { fetchAllPages } from "../utils/pagination";
|
||||
|
||||
export type CrawlRequest = CrawlOptions & {
|
||||
url: string;
|
||||
};
|
||||
|
||||
function prepareCrawlPayload(request: CrawlRequest): Record<string, unknown> {
|
||||
if (!request.url || !request.url.trim()) throw new Error("URL cannot be empty");
|
||||
const data: Record<string, unknown> = { url: request.url.trim() };
|
||||
if (request.prompt) data.prompt = request.prompt;
|
||||
if (request.excludePaths) data.excludePaths = request.excludePaths;
|
||||
if (request.includePaths) data.includePaths = request.includePaths;
|
||||
if (request.maxDiscoveryDepth != null) data.maxDiscoveryDepth = request.maxDiscoveryDepth;
|
||||
if (request.sitemap != null) data.sitemap = request.sitemap;
|
||||
if (request.robotsUserAgent != null) data.robotsUserAgent = request.robotsUserAgent;
|
||||
if (request.ignoreQueryParameters != null) data.ignoreQueryParameters = request.ignoreQueryParameters;
|
||||
if (request.deduplicateSimilarURLs != null) data.deduplicateSimilarURLs = request.deduplicateSimilarURLs;
|
||||
if (request.limit != null) data.limit = request.limit;
|
||||
if (request.crawlEntireDomain != null) data.crawlEntireDomain = request.crawlEntireDomain;
|
||||
if (request.allowExternalLinks != null) data.allowExternalLinks = request.allowExternalLinks;
|
||||
if (request.allowSubdomains != null) data.allowSubdomains = request.allowSubdomains;
|
||||
if (request.delay != null) data.delay = request.delay;
|
||||
if (request.maxConcurrency != null) data.maxConcurrency = request.maxConcurrency;
|
||||
if (request.regexOnFullURL != null) data.regexOnFullURL = request.regexOnFullURL;
|
||||
if (request.webhook != null) data.webhook = request.webhook;
|
||||
if (request.integration != null && request.integration.trim()) data.integration = request.integration.trim();
|
||||
if (request.origin) data.origin = request.origin;
|
||||
if (request.scrapeOptions) {
|
||||
ensureValidScrapeOptions(request.scrapeOptions);
|
||||
data.scrapeOptions = request.scrapeOptions;
|
||||
}
|
||||
if (request.zeroDataRetention != null) data.zeroDataRetention = request.zeroDataRetention;
|
||||
return data;
|
||||
}
|
||||
|
||||
export async function startCrawl(http: HttpClient, request: CrawlRequest): Promise<CrawlResponse> {
|
||||
const payload = prepareCrawlPayload(request);
|
||||
try {
|
||||
const res = await http.post<{ success: boolean; id: string; url: string; error?: string }>("/v2/crawl", payload);
|
||||
if (res.status !== 200 || !res.data?.success) {
|
||||
throwForBadResponse(res, "start crawl");
|
||||
}
|
||||
return { id: res.data.id, url: res.data.url };
|
||||
} catch (err: any) {
|
||||
if (err?.isAxiosError) return normalizeAxiosError(err, "start crawl");
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
export async function getCrawlStatus(
|
||||
http: HttpClient,
|
||||
jobId: string,
|
||||
pagination?: PaginationConfig
|
||||
): Promise<CrawlJob> {
|
||||
try {
|
||||
const res = await http.get<{ success: boolean; status: CrawlJob["status"]; completed?: number; total?: number; creditsUsed?: number; expiresAt?: string; next?: string | null; data?: Document[] }>(`/v2/crawl/${jobId}`);
|
||||
if (res.status !== 200 || !res.data?.success) {
|
||||
throwForBadResponse(res, "get crawl status");
|
||||
}
|
||||
const body = res.data;
|
||||
const initialDocs = (body.data || []) as Document[];
|
||||
|
||||
const auto = pagination?.autoPaginate ?? true;
|
||||
if (!auto || !body.next) {
|
||||
return {
|
||||
id: jobId,
|
||||
status: body.status,
|
||||
completed: body.completed ?? 0,
|
||||
total: body.total ?? 0,
|
||||
creditsUsed: body.creditsUsed,
|
||||
expiresAt: body.expiresAt,
|
||||
next: body.next ?? null,
|
||||
data: initialDocs,
|
||||
};
|
||||
}
|
||||
|
||||
const aggregated = await fetchAllPages(http, body.next, initialDocs, pagination);
|
||||
|
||||
return {
|
||||
id: jobId,
|
||||
status: body.status,
|
||||
completed: body.completed ?? 0,
|
||||
total: body.total ?? 0,
|
||||
creditsUsed: body.creditsUsed,
|
||||
expiresAt: body.expiresAt,
|
||||
next: null,
|
||||
data: aggregated,
|
||||
};
|
||||
} catch (err: any) {
|
||||
if (err?.isAxiosError) return normalizeAxiosError(err, "get crawl status");
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
export async function cancelCrawl(http: HttpClient, jobId: string): Promise<boolean> {
|
||||
try {
|
||||
const res = await http.delete<{ status: string }>(`/v2/crawl/${jobId}`);
|
||||
if (res.status !== 200) throwForBadResponse(res, "cancel crawl");
|
||||
return res.data?.status === "cancelled";
|
||||
} catch (err: any) {
|
||||
if (err?.isAxiosError) return normalizeAxiosError(err, "cancel crawl");
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
export async function waitForCrawlCompletion(http: HttpClient, jobId: string, pollInterval = 2, timeout?: number): Promise<CrawlJob> {
|
||||
const start = Date.now();
|
||||
|
||||
while (true) {
|
||||
try {
|
||||
const status = await getCrawlStatus(http, jobId);
|
||||
|
||||
if (["completed", "failed", "cancelled"].includes(status.status)) {
|
||||
return status;
|
||||
}
|
||||
} catch (err: any) {
|
||||
// Don't retry on permanent errors (4xx) - re-throw immediately with jobId context
|
||||
if (!isRetryableError(err)) {
|
||||
// Create new error with jobId for better debugging (non-retryable errors like 404)
|
||||
if (err instanceof SdkError) {
|
||||
const errorWithJobId = new SdkError(
|
||||
err.message,
|
||||
err.status,
|
||||
err.code,
|
||||
err.details,
|
||||
jobId
|
||||
);
|
||||
throw errorWithJobId;
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
// Otherwise, retry after delay - error might be transient (network issue, timeout, 5xx, etc.)
|
||||
}
|
||||
|
||||
if (timeout != null && Date.now() - start > timeout * 1000) {
|
||||
throw new JobTimeoutError(jobId, timeout, 'crawl');
|
||||
}
|
||||
|
||||
await new Promise((r) => setTimeout(r, Math.max(1000, pollInterval * 1000)));
|
||||
}
|
||||
}
|
||||
|
||||
export async function crawl(http: HttpClient, request: CrawlRequest, pollInterval = 2, timeout?: number): Promise<CrawlJob> {
|
||||
const started = await startCrawl(http, request);
|
||||
return waitForCrawlCompletion(http, started.id, pollInterval, timeout);
|
||||
}
|
||||
|
||||
export async function getCrawlErrors(http: HttpClient, crawlId: string): Promise<CrawlErrorsResponse> {
|
||||
try {
|
||||
const res = await http.get<{ success?: boolean; data?: { errors: Array<Record<string, string>>; robotsBlocked: string[] } }>(`/v2/crawl/${crawlId}/errors`);
|
||||
if (res.status !== 200) throwForBadResponse(res, "get crawl errors");
|
||||
const payload = res.data?.data ?? (res.data as any);
|
||||
return { errors: payload.errors || [], robotsBlocked: payload.robotsBlocked || [] };
|
||||
} catch (err: any) {
|
||||
if (err?.isAxiosError) return normalizeAxiosError(err, "get crawl errors");
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
export async function getActiveCrawls(http: HttpClient): Promise<ActiveCrawlsResponse> {
|
||||
try {
|
||||
const res = await http.get<{ success: boolean; crawls: Array<{ id: string; teamId?: string; team_id?: string; url: string; options?: any }> }>(`/v2/crawl/active`);
|
||||
if (res.status !== 200 || !res.data?.success) throwForBadResponse(res, "get active crawls");
|
||||
const crawlsIn = res.data?.crawls || [];
|
||||
const crawls = crawlsIn.map((c) => ({ id: c.id, teamId: (c as any).teamId ?? (c as any).team_id, url: c.url, options: c.options ?? null }));
|
||||
return { success: true, crawls };
|
||||
} catch (err: any) {
|
||||
if (err?.isAxiosError) return normalizeAxiosError(err, "get active crawls");
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
export async function crawlParamsPreview(http: HttpClient, url: string, prompt: string): Promise<Record<string, unknown>> {
|
||||
if (!url || !url.trim()) throw new Error("URL cannot be empty");
|
||||
if (!prompt || !prompt.trim()) throw new Error("Prompt cannot be empty");
|
||||
try {
|
||||
const res = await http.post<{ success: boolean; data?: Record<string, unknown>; warning?: string }>("/v2/crawl/params-preview", { url: url.trim(), prompt });
|
||||
if (res.status !== 200 || !res.data?.success) throwForBadResponse(res, "crawl params preview");
|
||||
const data = res.data.data || {};
|
||||
if (res.data.warning) (data as any).warning = res.data.warning;
|
||||
return data;
|
||||
} catch (err: any) {
|
||||
if (err?.isAxiosError) return normalizeAxiosError(err, "crawl params preview");
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,106 @@
|
||||
import { type ExtractResponse, type ScrapeOptions, type AgentOptions } from "../types";
|
||||
import { HttpClient } from "../utils/httpClient";
|
||||
import { ensureValidScrapeOptions } from "../utils/validation";
|
||||
import { normalizeAxiosError, throwForBadResponse } from "../utils/errorHandler";
|
||||
import { isZodSchema, zodSchemaToJsonSchema } from "../../utils/zodSchemaToJson";
|
||||
import type { ZodTypeAny } from "zod";
|
||||
|
||||
function prepareExtractPayload(args: {
|
||||
urls?: string[];
|
||||
prompt?: string;
|
||||
schema?: Record<string, unknown> | ZodTypeAny;
|
||||
systemPrompt?: string;
|
||||
allowExternalLinks?: boolean;
|
||||
enableWebSearch?: boolean;
|
||||
showSources?: boolean;
|
||||
scrapeOptions?: ScrapeOptions;
|
||||
ignoreInvalidURLs?: boolean;
|
||||
integration?: string;
|
||||
origin?: string;
|
||||
agent?: AgentOptions;
|
||||
}): Record<string, unknown> {
|
||||
const body: Record<string, unknown> = {};
|
||||
if (args.urls) body.urls = args.urls;
|
||||
if (args.prompt != null) body.prompt = args.prompt;
|
||||
if (args.schema != null) {
|
||||
body.schema = isZodSchema(args.schema) ? zodSchemaToJsonSchema(args.schema) : args.schema;
|
||||
}
|
||||
if (args.systemPrompt != null) body.systemPrompt = args.systemPrompt;
|
||||
if (args.allowExternalLinks != null) body.allowExternalLinks = args.allowExternalLinks;
|
||||
if (args.enableWebSearch != null) body.enableWebSearch = args.enableWebSearch;
|
||||
if (args.showSources != null) body.showSources = args.showSources;
|
||||
if (args.ignoreInvalidURLs != null) body.ignoreInvalidURLs = args.ignoreInvalidURLs;
|
||||
if (args.integration && args.integration.trim()) body.integration = args.integration.trim();
|
||||
if (args.origin) body.origin = args.origin;
|
||||
if (args.agent) body.agent = args.agent;
|
||||
if (args.scrapeOptions) {
|
||||
ensureValidScrapeOptions(args.scrapeOptions);
|
||||
body.scrapeOptions = args.scrapeOptions;
|
||||
}
|
||||
return body;
|
||||
}
|
||||
|
||||
/**
|
||||
* @deprecated The extract endpoint is in maintenance mode and its use is discouraged.
|
||||
* Review https://docs.firecrawl.dev/developer-guides/usage-guides/choosing-the-data-extractor to find a replacement.
|
||||
*/
|
||||
export async function startExtract(http: HttpClient, args: Parameters<typeof prepareExtractPayload>[0]): Promise<ExtractResponse> {
|
||||
const payload = prepareExtractPayload(args);
|
||||
try {
|
||||
const res = await http.post<ExtractResponse>("/v2/extract", payload);
|
||||
if (res.status !== 200) throwForBadResponse(res, "extract");
|
||||
return res.data;
|
||||
} catch (err: any) {
|
||||
if (err?.isAxiosError) return normalizeAxiosError(err, "extract");
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @deprecated The extract endpoint is in maintenance mode and its use is discouraged.
|
||||
* Review https://docs.firecrawl.dev/developer-guides/usage-guides/choosing-the-data-extractor to find a replacement.
|
||||
*/
|
||||
export async function getExtractStatus(http: HttpClient, jobId: string): Promise<ExtractResponse> {
|
||||
try {
|
||||
const res = await http.get<ExtractResponse>(`/v2/extract/${jobId}`);
|
||||
if (res.status !== 200) throwForBadResponse(res, "extract status");
|
||||
return res.data;
|
||||
} catch (err: any) {
|
||||
if (err?.isAxiosError) return normalizeAxiosError(err, "extract status");
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @deprecated The extract endpoint is in maintenance mode and its use is discouraged.
|
||||
* Review https://docs.firecrawl.dev/developer-guides/usage-guides/choosing-the-data-extractor to find a replacement.
|
||||
*/
|
||||
export async function waitExtract(
|
||||
http: HttpClient,
|
||||
jobId: string,
|
||||
pollInterval = 2,
|
||||
timeout?: number
|
||||
): Promise<ExtractResponse> {
|
||||
const start = Date.now();
|
||||
while (true) {
|
||||
const status = await getExtractStatus(http, jobId);
|
||||
if (["completed", "failed", "cancelled"].includes(status.status || "")) return status;
|
||||
if (timeout != null && Date.now() - start > timeout * 1000) return status;
|
||||
await new Promise((r) => setTimeout(r, Math.max(1000, pollInterval * 1000)));
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @deprecated The extract endpoint is in maintenance mode and its use is discouraged.
|
||||
* Review https://docs.firecrawl.dev/developer-guides/usage-guides/choosing-the-data-extractor to find a replacement.
|
||||
*/
|
||||
export async function extract(
|
||||
http: HttpClient,
|
||||
args: Parameters<typeof prepareExtractPayload>[0] & { pollInterval?: number; timeout?: number }
|
||||
): Promise<ExtractResponse> {
|
||||
const started = await startExtract(http, args);
|
||||
const jobId = started.id;
|
||||
if (!jobId) return started;
|
||||
return waitExtract(http, jobId, args.pollInterval ?? 2, args.timeout);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,68 @@
|
||||
import { type MapData, type MapOptions, type SearchResultWeb } from "../types";
|
||||
import { HttpClient } from "../utils/httpClient";
|
||||
import {
|
||||
throwForBadResponse,
|
||||
normalizeAxiosError,
|
||||
} from "../utils/errorHandler";
|
||||
|
||||
function prepareMapPayload(
|
||||
url: string,
|
||||
options?: MapOptions,
|
||||
): Record<string, unknown> {
|
||||
if (!url || !url.trim()) throw new Error("URL cannot be empty");
|
||||
const payload: Record<string, unknown> = { url: url.trim() };
|
||||
if (options) {
|
||||
if (options.sitemap != null) payload.sitemap = options.sitemap;
|
||||
if (options.search != null) payload.search = options.search;
|
||||
if (options.includeSubdomains != null)
|
||||
payload.includeSubdomains = options.includeSubdomains;
|
||||
if (options.ignoreQueryParameters != null)
|
||||
payload.ignoreQueryParameters = options.ignoreQueryParameters;
|
||||
if (options.limit != null) payload.limit = options.limit;
|
||||
if (options.timeout != null) payload.timeout = options.timeout;
|
||||
if (options.integration != null && options.integration.trim())
|
||||
payload.integration = options.integration.trim();
|
||||
if (options.origin) payload.origin = options.origin;
|
||||
if (options.location != null) payload.location = options.location;
|
||||
}
|
||||
return payload;
|
||||
}
|
||||
|
||||
export async function map(
|
||||
http: HttpClient,
|
||||
url: string,
|
||||
options?: MapOptions,
|
||||
): Promise<MapData> {
|
||||
const payload = prepareMapPayload(url, options);
|
||||
try {
|
||||
const res = await http.post<{
|
||||
success: boolean;
|
||||
error?: string;
|
||||
links?: Array<string | SearchResultWeb>;
|
||||
}>(
|
||||
"/v2/map",
|
||||
payload,
|
||||
typeof options?.timeout === "number"
|
||||
? { timeoutMs: options.timeout + 5000 }
|
||||
: {},
|
||||
);
|
||||
if (res.status !== 200 || !res.data?.success) {
|
||||
throwForBadResponse(res, "map");
|
||||
}
|
||||
const linksIn = res.data.links || [];
|
||||
const links: SearchResultWeb[] = [];
|
||||
for (const item of linksIn) {
|
||||
if (typeof item === "string") links.push({ url: item });
|
||||
else if (item && typeof item === "object")
|
||||
links.push({
|
||||
url: item.url,
|
||||
title: (item as any).title,
|
||||
description: (item as any).description,
|
||||
});
|
||||
}
|
||||
return { links };
|
||||
} catch (err: any) {
|
||||
if (err?.isAxiosError) return normalizeAxiosError(err, "map");
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,181 @@
|
||||
import {
|
||||
type CreateMonitorRequest,
|
||||
type ListMonitorsOptions,
|
||||
type ListMonitorChecksOptions,
|
||||
type Monitor,
|
||||
type MonitorCheck,
|
||||
type MonitorCheckDetail,
|
||||
type MonitorCheckPage,
|
||||
type GetMonitorCheckOptions,
|
||||
type UpdateMonitorRequest,
|
||||
} from "../types";
|
||||
import { HttpClient } from "../utils/httpClient";
|
||||
import {
|
||||
throwForBadResponse,
|
||||
normalizeAxiosError,
|
||||
} from "../utils/errorHandler";
|
||||
import { fetchAllPages } from "../utils/pagination";
|
||||
|
||||
type ApiResponse<T> = {
|
||||
success: boolean;
|
||||
data?: T;
|
||||
id?: string;
|
||||
error?: string;
|
||||
};
|
||||
|
||||
function queryString(params?: Record<string, unknown>): string {
|
||||
if (!params) return "";
|
||||
const query = new URLSearchParams();
|
||||
for (const [key, value] of Object.entries(params)) {
|
||||
if (value !== undefined && value !== null) query.set(key, String(value));
|
||||
}
|
||||
const str = query.toString();
|
||||
return str ? `?${str}` : "";
|
||||
}
|
||||
|
||||
function dataOrThrow<T>(res: { status: number; data?: ApiResponse<T> }, action: string): T {
|
||||
if (res.status !== 200 || !res.data?.success || res.data.data == null) {
|
||||
throwForBadResponse(res as any, action);
|
||||
}
|
||||
return res.data.data;
|
||||
}
|
||||
|
||||
export async function createMonitor(
|
||||
http: HttpClient,
|
||||
request: CreateMonitorRequest,
|
||||
): Promise<Monitor> {
|
||||
try {
|
||||
const res = await http.post<ApiResponse<Monitor>>("/v2/monitor", request as any);
|
||||
return dataOrThrow(res, "create monitor");
|
||||
} catch (err: any) {
|
||||
if (err?.isAxiosError) return normalizeAxiosError(err, "create monitor");
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
export async function listMonitors(
|
||||
http: HttpClient,
|
||||
options?: ListMonitorsOptions,
|
||||
): Promise<Monitor[]> {
|
||||
try {
|
||||
const res = await http.get<ApiResponse<Monitor[]>>(
|
||||
`/v2/monitor${queryString(options as Record<string, unknown>)}`,
|
||||
);
|
||||
return dataOrThrow(res, "list monitors");
|
||||
} catch (err: any) {
|
||||
if (err?.isAxiosError) return normalizeAxiosError(err, "list monitors");
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
export async function getMonitor(
|
||||
http: HttpClient,
|
||||
monitorId: string,
|
||||
): Promise<Monitor> {
|
||||
try {
|
||||
const res = await http.get<ApiResponse<Monitor>>(`/v2/monitor/${monitorId}`);
|
||||
return dataOrThrow(res, "get monitor");
|
||||
} catch (err: any) {
|
||||
if (err?.isAxiosError) return normalizeAxiosError(err, "get monitor");
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
export async function updateMonitor(
|
||||
http: HttpClient,
|
||||
monitorId: string,
|
||||
request: UpdateMonitorRequest,
|
||||
): Promise<Monitor> {
|
||||
try {
|
||||
const res = await http.patch<ApiResponse<Monitor>>(
|
||||
`/v2/monitor/${monitorId}`,
|
||||
request as any,
|
||||
);
|
||||
return dataOrThrow(res, "update monitor");
|
||||
} catch (err: any) {
|
||||
if (err?.isAxiosError) return normalizeAxiosError(err, "update monitor");
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
export async function deleteMonitor(
|
||||
http: HttpClient,
|
||||
monitorId: string,
|
||||
): Promise<boolean> {
|
||||
try {
|
||||
const res = await http.delete<ApiResponse<unknown>>(`/v2/monitor/${monitorId}`);
|
||||
if (res.status !== 200 || !res.data?.success) {
|
||||
throwForBadResponse(res, "delete monitor");
|
||||
}
|
||||
return true;
|
||||
} catch (err: any) {
|
||||
if (err?.isAxiosError) return normalizeAxiosError(err, "delete monitor");
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
export async function runMonitor(
|
||||
http: HttpClient,
|
||||
monitorId: string,
|
||||
): Promise<MonitorCheck> {
|
||||
try {
|
||||
const res = await http.post<ApiResponse<MonitorCheck>>(
|
||||
`/v2/monitor/${monitorId}/run`,
|
||||
{},
|
||||
);
|
||||
return dataOrThrow(res, "run monitor");
|
||||
} catch (err: any) {
|
||||
if (err?.isAxiosError) return normalizeAxiosError(err, "run monitor");
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
export async function listMonitorChecks(
|
||||
http: HttpClient,
|
||||
monitorId: string,
|
||||
options?: ListMonitorChecksOptions,
|
||||
): Promise<MonitorCheck[]> {
|
||||
try {
|
||||
const res = await http.get<ApiResponse<MonitorCheck[]>>(
|
||||
`/v2/monitor/${monitorId}/checks${queryString(options as Record<string, unknown>)}`,
|
||||
);
|
||||
return dataOrThrow(res, "list monitor checks");
|
||||
} catch (err: any) {
|
||||
if (err?.isAxiosError) return normalizeAxiosError(err, "list monitor checks");
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
export async function getMonitorCheck(
|
||||
http: HttpClient,
|
||||
monitorId: string,
|
||||
checkId: string,
|
||||
options?: GetMonitorCheckOptions,
|
||||
): Promise<MonitorCheckDetail> {
|
||||
try {
|
||||
const { autoPaginate: _autoPaginate, maxPages: _maxPages, maxResults: _maxResults, maxWaitTime: _maxWaitTime, ...query } = options ?? {};
|
||||
const res = await http.get<ApiResponse<MonitorCheckDetail>>(
|
||||
`/v2/monitor/${monitorId}/checks/${checkId}${queryString(query as Record<string, unknown>)}`,
|
||||
);
|
||||
const detail = dataOrThrow(res, "get monitor check");
|
||||
const next = res.data?.next ?? detail.next ?? null;
|
||||
const auto = options?.autoPaginate ?? true;
|
||||
if (!auto || !next) {
|
||||
return { ...detail, next };
|
||||
}
|
||||
|
||||
return {
|
||||
...detail,
|
||||
pages: await fetchAllPages<MonitorCheckPage>(
|
||||
http,
|
||||
next,
|
||||
detail.pages || [],
|
||||
options,
|
||||
),
|
||||
next: null,
|
||||
};
|
||||
} catch (err: any) {
|
||||
if (err?.isAxiosError) return normalizeAxiosError(err, "get monitor check");
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,96 @@
|
||||
import { type Document, type ParseFile, type ParseOptions } from "../types";
|
||||
import { HttpClient } from "../utils/httpClient";
|
||||
import { ensureValidParseOptions } from "../utils/validation";
|
||||
import {
|
||||
throwForBadResponse,
|
||||
normalizeAxiosError,
|
||||
} from "../utils/errorHandler";
|
||||
import { getVersion } from "../utils/getVersion";
|
||||
|
||||
function toUploadBlob(input: ParseFile["data"], contentType?: string): Blob {
|
||||
if (typeof Blob !== "undefined" && input instanceof Blob) {
|
||||
if (contentType && input.type !== contentType) {
|
||||
return new Blob([input], { type: contentType });
|
||||
}
|
||||
return input;
|
||||
}
|
||||
|
||||
if (typeof Buffer !== "undefined" && Buffer.isBuffer(input)) {
|
||||
return new Blob([input], { type: contentType });
|
||||
}
|
||||
|
||||
if (input instanceof ArrayBuffer) {
|
||||
return new Blob([input], { type: contentType });
|
||||
}
|
||||
|
||||
if (ArrayBuffer.isView(input)) {
|
||||
return new Blob([input], { type: contentType });
|
||||
}
|
||||
|
||||
if (typeof input === "string") {
|
||||
return new Blob([input], {
|
||||
type: contentType ?? "text/plain; charset=utf-8",
|
||||
});
|
||||
}
|
||||
|
||||
throw new Error("Unsupported parse file data type");
|
||||
}
|
||||
|
||||
export async function parse(
|
||||
http: HttpClient,
|
||||
file: ParseFile,
|
||||
options?: ParseOptions,
|
||||
): Promise<Document> {
|
||||
if (!file || !file.filename || !file.filename.trim()) {
|
||||
throw new Error("filename cannot be empty");
|
||||
}
|
||||
|
||||
if (file.data == null) {
|
||||
throw new Error("file data cannot be empty");
|
||||
}
|
||||
|
||||
const blob = toUploadBlob(file.data, file.contentType);
|
||||
if (blob.size === 0) {
|
||||
throw new Error("file data cannot be empty");
|
||||
}
|
||||
|
||||
if (options) ensureValidParseOptions(options);
|
||||
|
||||
const version = getVersion();
|
||||
const normalizedOptions: ParseOptions = {
|
||||
...(options ?? {}),
|
||||
origin:
|
||||
typeof options?.origin === "string" && options.origin.includes("mcp")
|
||||
? options.origin
|
||||
: (options?.origin ?? `js-sdk@${version}`),
|
||||
};
|
||||
|
||||
const formData = new FormData();
|
||||
formData.append("options", JSON.stringify(normalizedOptions));
|
||||
formData.append(
|
||||
"file",
|
||||
toUploadBlob(file.data, file.contentType),
|
||||
file.filename.trim(),
|
||||
);
|
||||
|
||||
try {
|
||||
const res = await http.postMultipart<{
|
||||
success: boolean;
|
||||
data?: Document;
|
||||
error?: string;
|
||||
}>(
|
||||
"/v2/parse",
|
||||
formData,
|
||||
typeof normalizedOptions.timeout === "number"
|
||||
? { timeoutMs: normalizedOptions.timeout + 5000 }
|
||||
: {},
|
||||
);
|
||||
if (res.status !== 200 || !res.data?.success) {
|
||||
throwForBadResponse(res, "parse");
|
||||
}
|
||||
return (res.data.data || {}) as Document;
|
||||
} catch (err: any) {
|
||||
if (err?.isAxiosError) return normalizeAxiosError(err, "parse");
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
130
참고/firecrawl-main/apps/js-sdk/firecrawl/src/v2/methods/scrape.ts
Normal file
130
참고/firecrawl-main/apps/js-sdk/firecrawl/src/v2/methods/scrape.ts
Normal file
@@ -0,0 +1,130 @@
|
||||
import {
|
||||
type Document,
|
||||
type ScrapeBrowserDeleteResponse,
|
||||
type ScrapeExecuteRequest,
|
||||
type ScrapeExecuteResponse,
|
||||
type ScrapeOptions,
|
||||
} from "../types";
|
||||
import { HttpClient } from "../utils/httpClient";
|
||||
import { ensureValidScrapeOptions } from "../utils/validation";
|
||||
import {
|
||||
throwForBadResponse,
|
||||
normalizeAxiosError,
|
||||
} from "../utils/errorHandler";
|
||||
|
||||
export async function scrape(
|
||||
http: HttpClient,
|
||||
url: string,
|
||||
options?: ScrapeOptions,
|
||||
): Promise<Document> {
|
||||
if (!url || !url.trim()) {
|
||||
throw new Error("URL cannot be empty");
|
||||
}
|
||||
if (options) ensureValidScrapeOptions(options);
|
||||
|
||||
const payload: Record<string, unknown> = { url: url.trim() };
|
||||
if (options) Object.assign(payload, options);
|
||||
|
||||
try {
|
||||
const res = await http.post<{
|
||||
success: boolean;
|
||||
data?: Document;
|
||||
error?: string;
|
||||
}>(
|
||||
"/v2/scrape",
|
||||
payload,
|
||||
typeof options?.timeout === "number"
|
||||
? { timeoutMs: options.timeout + 5000 }
|
||||
: {},
|
||||
);
|
||||
if (res.status !== 200 || !res.data?.success) {
|
||||
throwForBadResponse(res, "scrape");
|
||||
}
|
||||
return (res.data.data || {}) as Document;
|
||||
} catch (err: any) {
|
||||
if (err?.isAxiosError) return normalizeAxiosError(err, "scrape");
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
export async function interact(
|
||||
http: HttpClient,
|
||||
jobId: string,
|
||||
args: ScrapeExecuteRequest,
|
||||
): Promise<ScrapeExecuteResponse> {
|
||||
if (!jobId || !jobId.trim()) {
|
||||
throw new Error("Job ID cannot be empty");
|
||||
}
|
||||
const hasCode = args?.code && args.code.trim();
|
||||
const hasPrompt = args?.prompt && args.prompt.trim();
|
||||
if (!hasCode && !hasPrompt) {
|
||||
throw new Error("Either 'code' or 'prompt' must be provided");
|
||||
}
|
||||
|
||||
const body: Record<string, unknown> = {};
|
||||
if (hasCode) body.code = args.code;
|
||||
if (hasPrompt) body.prompt = args.prompt;
|
||||
body.language = args.language ?? "node";
|
||||
if (args.timeout != null) body.timeout = args.timeout;
|
||||
if (args.origin) body.origin = args.origin;
|
||||
|
||||
try {
|
||||
const res = await http.post<ScrapeExecuteResponse>(
|
||||
`/v2/scrape/${jobId}/interact`,
|
||||
body,
|
||||
args.timeout != null ? { timeoutMs: args.timeout * 1000 + 5000 } : {},
|
||||
);
|
||||
if (res.status !== 200)
|
||||
throwForBadResponse(res, "interact with scrape browser");
|
||||
return res.data;
|
||||
} catch (err: any) {
|
||||
if (err?.isAxiosError)
|
||||
return normalizeAxiosError(err, "interact with scrape browser");
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
export async function stopInteraction(
|
||||
http: HttpClient,
|
||||
jobId: string,
|
||||
): Promise<ScrapeBrowserDeleteResponse> {
|
||||
if (!jobId || !jobId.trim()) {
|
||||
throw new Error("Job ID cannot be empty");
|
||||
}
|
||||
|
||||
try {
|
||||
const res = await http.delete<ScrapeBrowserDeleteResponse>(
|
||||
`/v2/scrape/${jobId}/interact`,
|
||||
);
|
||||
if (res.status !== 200) throwForBadResponse(res, "stop interaction");
|
||||
return res.data;
|
||||
} catch (err: any) {
|
||||
if (err?.isAxiosError) return normalizeAxiosError(err, "stop interaction");
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
/** @deprecated Use interact(). */
|
||||
export async function scrapeExecute(
|
||||
http: HttpClient,
|
||||
jobId: string,
|
||||
args: ScrapeExecuteRequest,
|
||||
): Promise<ScrapeExecuteResponse> {
|
||||
return interact(http, jobId, args);
|
||||
}
|
||||
|
||||
/** @deprecated Use stopInteraction(). */
|
||||
export async function stopInteractiveBrowser(
|
||||
http: HttpClient,
|
||||
jobId: string,
|
||||
): Promise<ScrapeBrowserDeleteResponse> {
|
||||
return stopInteraction(http, jobId);
|
||||
}
|
||||
|
||||
/** @deprecated Use stopInteraction(). */
|
||||
export async function deleteScrapeBrowser(
|
||||
http: HttpClient,
|
||||
jobId: string,
|
||||
): Promise<ScrapeBrowserDeleteResponse> {
|
||||
return stopInteraction(http, jobId);
|
||||
}
|
||||
106
참고/firecrawl-main/apps/js-sdk/firecrawl/src/v2/methods/search.ts
Normal file
106
참고/firecrawl-main/apps/js-sdk/firecrawl/src/v2/methods/search.ts
Normal file
@@ -0,0 +1,106 @@
|
||||
import {
|
||||
type Document,
|
||||
type SearchData,
|
||||
type SearchRequest,
|
||||
type SearchResultWeb,
|
||||
type ScrapeOptions,
|
||||
type SearchResultNews,
|
||||
type SearchResultImages,
|
||||
} from "../types";
|
||||
import { HttpClient } from "../utils/httpClient";
|
||||
import { ensureValidScrapeOptions } from "../utils/validation";
|
||||
import {
|
||||
throwForBadResponse,
|
||||
normalizeAxiosError,
|
||||
} from "../utils/errorHandler";
|
||||
|
||||
function prepareSearchPayload(req: SearchRequest): Record<string, unknown> {
|
||||
if (!req.query || !req.query.trim()) throw new Error("Query cannot be empty");
|
||||
if (req.limit != null && req.limit <= 0)
|
||||
throw new Error("limit must be positive");
|
||||
if (req.timeout != null && req.timeout <= 0)
|
||||
throw new Error("timeout must be positive");
|
||||
if (req.includeDomains?.length && req.excludeDomains?.length)
|
||||
throw new Error(
|
||||
"includeDomains and excludeDomains cannot both be specified",
|
||||
);
|
||||
const payload: Record<string, unknown> = {
|
||||
query: req.query,
|
||||
};
|
||||
if (req.sources) payload.sources = req.sources;
|
||||
if (req.categories) payload.categories = req.categories;
|
||||
if (req.includeDomains) payload.includeDomains = req.includeDomains;
|
||||
if (req.excludeDomains) payload.excludeDomains = req.excludeDomains;
|
||||
if (req.limit != null) payload.limit = req.limit;
|
||||
if (req.tbs != null) payload.tbs = req.tbs;
|
||||
if (req.location != null) payload.location = req.location;
|
||||
if (req.ignoreInvalidURLs != null)
|
||||
payload.ignoreInvalidURLs = req.ignoreInvalidURLs;
|
||||
if (req.timeout != null) payload.timeout = req.timeout;
|
||||
if (req.integration && req.integration.trim())
|
||||
payload.integration = req.integration.trim();
|
||||
if (req.origin) payload.origin = req.origin;
|
||||
if (req.scrapeOptions) {
|
||||
ensureValidScrapeOptions(req.scrapeOptions as ScrapeOptions);
|
||||
payload.scrapeOptions = req.scrapeOptions;
|
||||
}
|
||||
return payload;
|
||||
}
|
||||
|
||||
function transformArray<ResultType>(arr: any[]): Array<ResultType | Document> {
|
||||
const results: Array<ResultType | Document> = [] as any;
|
||||
for (const item of arr) {
|
||||
if (item && typeof item === "object") {
|
||||
if (
|
||||
"markdown" in item ||
|
||||
"html" in item ||
|
||||
"rawHtml" in item ||
|
||||
"links" in item ||
|
||||
"screenshot" in item ||
|
||||
"changeTracking" in item ||
|
||||
"summary" in item ||
|
||||
"json" in item
|
||||
) {
|
||||
results.push(item as Document);
|
||||
} else {
|
||||
results.push(item as ResultType);
|
||||
}
|
||||
} else {
|
||||
results.push({ url: item } as ResultType);
|
||||
}
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
||||
export async function search(
|
||||
http: HttpClient,
|
||||
request: SearchRequest,
|
||||
): Promise<SearchData> {
|
||||
const payload = prepareSearchPayload(request);
|
||||
try {
|
||||
const res = await http.post<{
|
||||
success: boolean;
|
||||
data?: Record<string, unknown>;
|
||||
error?: string;
|
||||
}>(
|
||||
"/v2/search",
|
||||
payload,
|
||||
typeof request.timeout === "number"
|
||||
? { timeoutMs: request.timeout + 5000 }
|
||||
: {},
|
||||
);
|
||||
if (res.status !== 200 || !res.data?.success) {
|
||||
throwForBadResponse(res, "search");
|
||||
}
|
||||
const data = (res.data.data || {}) as Record<string, any>;
|
||||
const out: SearchData = {};
|
||||
if (data.web) out.web = transformArray<SearchResultWeb>(data.web);
|
||||
if (data.news) out.news = transformArray<SearchResultNews>(data.news);
|
||||
if (data.images)
|
||||
out.images = transformArray<SearchResultImages>(data.images);
|
||||
return out;
|
||||
} catch (err: any) {
|
||||
if (err?.isAxiosError) return normalizeAxiosError(err, "search");
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,84 @@
|
||||
import type { ConcurrencyCheck, CreditUsage, QueueStatusResponse, TokenUsage, CreditUsageHistoricalResponse, TokenUsageHistoricalResponse } from "../types";
|
||||
import { HttpClient } from "../utils/httpClient";
|
||||
import { normalizeAxiosError, throwForBadResponse } from "../utils/errorHandler";
|
||||
|
||||
export async function getConcurrency(http: HttpClient): Promise<ConcurrencyCheck> {
|
||||
try {
|
||||
const res = await http.get<{ success: boolean; data?: { concurrency: number; maxConcurrency: number } }>("/v2/concurrency-check");
|
||||
if (res.status !== 200 || !res.data?.success) throwForBadResponse(res, "get concurrency");
|
||||
const d = res.data.data || (res.data as any);
|
||||
return { concurrency: d.concurrency, maxConcurrency: d.maxConcurrency ?? d.max_concurrency };
|
||||
} catch (err: any) {
|
||||
if (err?.isAxiosError) return normalizeAxiosError(err, "get concurrency");
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
export async function getCreditUsage(http: HttpClient): Promise<CreditUsage> {
|
||||
try {
|
||||
const res = await http.get<{ success: boolean; data?: { remainingCredits?: number; remaining_credits?: number; planCredits?: number; plan_credits?: number; billingPeriodStart?: string | null; billing_period_start?: string | null; billingPeriodEnd?: string | null; billing_period_end?: string | null } }>("/v2/team/credit-usage");
|
||||
if (res.status !== 200 || !res.data?.success) throwForBadResponse(res, "get credit usage");
|
||||
const d = res.data.data || (res.data as any);
|
||||
return {
|
||||
remainingCredits: d.remainingCredits ?? d.remaining_credits ?? 0,
|
||||
planCredits: d.planCredits ?? d.plan_credits,
|
||||
billingPeriodStart: d.billingPeriodStart ?? d.billing_period_start ?? null,
|
||||
billingPeriodEnd: d.billingPeriodEnd ?? d.billing_period_end ?? null,
|
||||
};
|
||||
} catch (err: any) {
|
||||
if (err?.isAxiosError) return normalizeAxiosError(err, "get credit usage");
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
export async function getTokenUsage(http: HttpClient): Promise<TokenUsage> {
|
||||
try {
|
||||
const res = await http.get<{ success: boolean; data?: { remainingTokens?: number; planTokens?: number; billingPeriodStart?: string | null; billingPeriodEnd?: string | null; remaining_tokens?: number; plan_tokens?: number; billing_period_start?: string | null; billing_period_end?: string | null } }>("/v2/team/token-usage");
|
||||
if (res.status !== 200 || !res.data?.success) throwForBadResponse(res, "get token usage");
|
||||
const d = res.data.data || (res.data as any);
|
||||
return {
|
||||
remainingTokens: d.remainingTokens ?? d.remaining_tokens ?? 0,
|
||||
planTokens: d.planTokens ?? d.plan_tokens,
|
||||
billingPeriodStart: d.billingPeriodStart ?? d.billing_period_start ?? null,
|
||||
billingPeriodEnd: d.billingPeriodEnd ?? d.billing_period_end ?? null,
|
||||
};
|
||||
} catch (err: any) {
|
||||
if (err?.isAxiosError) return normalizeAxiosError(err, "get token usage");
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
export async function getQueueStatus(http: HttpClient): Promise<QueueStatusResponse> {
|
||||
try {
|
||||
const res = await http.get<QueueStatusResponse>("/v2/team/queue-status");
|
||||
if (res.status !== 200 || !res.data?.success) throwForBadResponse(res, "get queue status");
|
||||
return res.data;
|
||||
} catch (err: any) {
|
||||
if (err?.isAxiosError) return normalizeAxiosError(err, "get queue status");
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
export async function getCreditUsageHistorical(http: HttpClient, byApiKey?: boolean): Promise<CreditUsageHistoricalResponse> {
|
||||
try {
|
||||
const query = byApiKey ? "?byApiKey=true" : "";
|
||||
const res = await http.get<CreditUsageHistoricalResponse>(`/v2/team/credit-usage/historical${query}`);
|
||||
if (res.status !== 200 || !res.data?.success) throwForBadResponse(res, "get credit usage historical");
|
||||
return res.data;
|
||||
} catch (err: any) {
|
||||
if (err?.isAxiosError) return normalizeAxiosError(err, "get credit usage historical");
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
export async function getTokenUsageHistorical(http: HttpClient, byApiKey?: boolean): Promise<TokenUsageHistoricalResponse> {
|
||||
try {
|
||||
const query = byApiKey ? "?byApiKey=true" : "";
|
||||
const res = await http.get<TokenUsageHistoricalResponse>(`/v2/team/token-usage/historical${query}`);
|
||||
if (res.status !== 200 || !res.data?.success) throwForBadResponse(res, "get token usage historical");
|
||||
return res.data;
|
||||
} catch (err: any) {
|
||||
if (err?.isAxiosError) return normalizeAxiosError(err, "get token usage historical");
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
983
참고/firecrawl-main/apps/js-sdk/firecrawl/src/v2/types.ts
Normal file
983
참고/firecrawl-main/apps/js-sdk/firecrawl/src/v2/types.ts
Normal file
@@ -0,0 +1,983 @@
|
||||
import type { ZodTypeAny } from 'zod';
|
||||
// Public types for Firecrawl JS/TS SDK v2 (camelCase only)
|
||||
|
||||
export type FormatString =
|
||||
| 'markdown'
|
||||
| 'html'
|
||||
| 'rawHtml'
|
||||
| 'links'
|
||||
| 'images'
|
||||
| 'screenshot'
|
||||
| 'summary'
|
||||
| 'changeTracking'
|
||||
| 'json'
|
||||
| 'attributes'
|
||||
| 'branding'
|
||||
| 'audio';
|
||||
|
||||
export interface Viewport {
|
||||
width: number;
|
||||
height: number;
|
||||
}
|
||||
|
||||
export interface Format {
|
||||
type: FormatString;
|
||||
}
|
||||
|
||||
export interface JsonFormat extends Format {
|
||||
type: 'json';
|
||||
prompt?: string;
|
||||
schema?: Record<string, unknown> | ZodTypeAny;
|
||||
}
|
||||
|
||||
export interface ScreenshotFormat {
|
||||
type: 'screenshot';
|
||||
fullPage?: boolean;
|
||||
quality?: number;
|
||||
viewport?: Viewport | { width: number; height: number };
|
||||
}
|
||||
|
||||
export interface ChangeTrackingFormat extends Format {
|
||||
type: 'changeTracking';
|
||||
modes: ('git-diff' | 'json')[];
|
||||
schema?: Record<string, unknown>;
|
||||
prompt?: string;
|
||||
tag?: string;
|
||||
}
|
||||
export interface AttributesFormat extends Format {
|
||||
type: 'attributes';
|
||||
selectors: Array<{
|
||||
selector: string;
|
||||
attribute: string;
|
||||
}>;
|
||||
}
|
||||
|
||||
export interface QuestionFormat {
|
||||
type: 'question';
|
||||
question: string;
|
||||
}
|
||||
|
||||
export interface HighlightsFormat {
|
||||
type: 'highlights';
|
||||
query: string;
|
||||
}
|
||||
|
||||
/** @deprecated Use QuestionFormat or HighlightsFormat instead. */
|
||||
export interface QueryFormat {
|
||||
type: 'query';
|
||||
prompt: string;
|
||||
mode?: 'freeform' | 'directQuote';
|
||||
}
|
||||
|
||||
export type FormatOption =
|
||||
| FormatString
|
||||
| Format
|
||||
| JsonFormat
|
||||
| ChangeTrackingFormat
|
||||
| ScreenshotFormat
|
||||
| AttributesFormat
|
||||
| QuestionFormat
|
||||
| HighlightsFormat
|
||||
| QueryFormat;
|
||||
|
||||
export type ParseFormatString = Exclude<
|
||||
FormatString,
|
||||
'screenshot' | 'changeTracking' | 'branding'
|
||||
>;
|
||||
|
||||
export interface ParseFormat {
|
||||
type: ParseFormatString;
|
||||
}
|
||||
|
||||
export type ParseFormatOption =
|
||||
| ParseFormatString
|
||||
| ParseFormat
|
||||
| JsonFormat
|
||||
| AttributesFormat
|
||||
| QuestionFormat
|
||||
| HighlightsFormat
|
||||
| QueryFormat;
|
||||
|
||||
export interface LocationConfig {
|
||||
country?: string;
|
||||
languages?: string[];
|
||||
}
|
||||
|
||||
export interface WaitAction {
|
||||
type: 'wait';
|
||||
milliseconds?: number;
|
||||
selector?: string;
|
||||
}
|
||||
|
||||
export interface ScreenshotAction {
|
||||
type: 'screenshot';
|
||||
fullPage?: boolean;
|
||||
quality?: number;
|
||||
viewport?: Viewport | { width: number; height: number };
|
||||
}
|
||||
|
||||
export interface ClickAction {
|
||||
type: 'click';
|
||||
selector: string;
|
||||
}
|
||||
|
||||
export interface WriteAction {
|
||||
type: 'write';
|
||||
text: string;
|
||||
}
|
||||
|
||||
export interface PressAction {
|
||||
type: 'press';
|
||||
key: string;
|
||||
}
|
||||
|
||||
export interface ScrollAction {
|
||||
type: 'scroll';
|
||||
direction: 'up' | 'down';
|
||||
selector?: string;
|
||||
}
|
||||
|
||||
export interface ScrapeAction {
|
||||
type: 'scrape';
|
||||
}
|
||||
|
||||
export interface ExecuteJavascriptAction {
|
||||
type: 'executeJavascript';
|
||||
script: string;
|
||||
}
|
||||
|
||||
export interface PDFAction {
|
||||
type: 'pdf';
|
||||
format?:
|
||||
| 'A0'
|
||||
| 'A1'
|
||||
| 'A2'
|
||||
| 'A3'
|
||||
| 'A4'
|
||||
| 'A5'
|
||||
| 'A6'
|
||||
| 'Letter'
|
||||
| 'Legal'
|
||||
| 'Tabloid'
|
||||
| 'Ledger';
|
||||
landscape?: boolean;
|
||||
scale?: number;
|
||||
}
|
||||
|
||||
export type ActionOption =
|
||||
| WaitAction
|
||||
| ScreenshotAction
|
||||
| ClickAction
|
||||
| WriteAction
|
||||
| PressAction
|
||||
| ScrollAction
|
||||
| ScrapeAction
|
||||
| ExecuteJavascriptAction
|
||||
| PDFAction;
|
||||
|
||||
export interface ScrapeOptions {
|
||||
formats?: FormatOption[];
|
||||
headers?: Record<string, string>;
|
||||
includeTags?: string[];
|
||||
excludeTags?: string[];
|
||||
onlyMainContent?: boolean;
|
||||
timeout?: number;
|
||||
waitFor?: number;
|
||||
mobile?: boolean;
|
||||
parsers?: Array<string | { type: 'pdf'; mode?: 'fast' | 'auto' | 'ocr'; maxPages?: number }>;
|
||||
actions?: ActionOption[];
|
||||
location?: LocationConfig;
|
||||
skipTlsVerification?: boolean;
|
||||
removeBase64Images?: boolean;
|
||||
fastMode?: boolean;
|
||||
useMock?: string;
|
||||
blockAds?: boolean;
|
||||
proxy?: 'basic' | 'stealth' | 'enhanced' | 'auto' | string;
|
||||
maxAge?: number;
|
||||
minAge?: number;
|
||||
storeInCache?: boolean;
|
||||
lockdown?: boolean;
|
||||
profile?: {
|
||||
name: string;
|
||||
saveChanges?: boolean;
|
||||
};
|
||||
integration?: string;
|
||||
origin?: string;
|
||||
}
|
||||
|
||||
export type ParseFileData =
|
||||
| Blob
|
||||
| File
|
||||
| Buffer
|
||||
| Uint8Array
|
||||
| ArrayBuffer
|
||||
| string;
|
||||
|
||||
export interface ParseFile {
|
||||
data: ParseFileData;
|
||||
filename: string;
|
||||
contentType?: string;
|
||||
}
|
||||
|
||||
export type ParseOptions = Omit<
|
||||
ScrapeOptions,
|
||||
| 'formats'
|
||||
| 'waitFor'
|
||||
| 'mobile'
|
||||
| 'actions'
|
||||
| 'location'
|
||||
| 'maxAge'
|
||||
| 'minAge'
|
||||
| 'storeInCache'
|
||||
| 'lockdown'
|
||||
| 'proxy'
|
||||
> & {
|
||||
formats?: ParseFormatOption[];
|
||||
proxy?: 'basic' | 'auto';
|
||||
};
|
||||
|
||||
export interface WebhookConfig {
|
||||
url: string;
|
||||
headers?: Record<string, string>;
|
||||
metadata?: Record<string, string>;
|
||||
events?: Array<'completed' | 'failed' | 'page' | 'started'>;
|
||||
}
|
||||
|
||||
// Agent webhook events differ from crawl: has 'action' and 'cancelled', no 'page'
|
||||
export type AgentWebhookEvent = 'started' | 'action' | 'completed' | 'failed' | 'cancelled';
|
||||
|
||||
export interface AgentWebhookConfig {
|
||||
url: string;
|
||||
headers?: Record<string, string>;
|
||||
metadata?: Record<string, string>;
|
||||
events?: AgentWebhookEvent[];
|
||||
}
|
||||
|
||||
export interface BrandingProfile {
|
||||
colorScheme?: 'light' | 'dark';
|
||||
logo?: string | null;
|
||||
fonts?: Array<{
|
||||
family: string;
|
||||
[key: string]: unknown;
|
||||
}>;
|
||||
colors?: {
|
||||
primary?: string;
|
||||
secondary?: string;
|
||||
accent?: string;
|
||||
background?: string;
|
||||
textPrimary?: string;
|
||||
textSecondary?: string;
|
||||
link?: string;
|
||||
success?: string;
|
||||
warning?: string;
|
||||
error?: string;
|
||||
[key: string]: string | undefined;
|
||||
};
|
||||
typography?: {
|
||||
fontFamilies?: {
|
||||
primary?: string;
|
||||
heading?: string;
|
||||
code?: string;
|
||||
[key: string]: string | undefined;
|
||||
};
|
||||
fontStacks?: {
|
||||
primary?: string[];
|
||||
heading?: string[];
|
||||
body?: string[];
|
||||
paragraph?: string[];
|
||||
[key: string]: string[] | undefined;
|
||||
};
|
||||
fontSizes?: {
|
||||
h1?: string;
|
||||
h2?: string;
|
||||
h3?: string;
|
||||
body?: string;
|
||||
small?: string;
|
||||
[key: string]: string | undefined;
|
||||
};
|
||||
lineHeights?: {
|
||||
heading?: number;
|
||||
body?: number;
|
||||
[key: string]: number | undefined;
|
||||
};
|
||||
fontWeights?: {
|
||||
light?: number;
|
||||
regular?: number;
|
||||
medium?: number;
|
||||
bold?: number;
|
||||
[key: string]: number | undefined;
|
||||
};
|
||||
};
|
||||
spacing?: {
|
||||
baseUnit?: number;
|
||||
padding?: Record<string, number>;
|
||||
margins?: Record<string, number>;
|
||||
gridGutter?: number;
|
||||
borderRadius?: string;
|
||||
[key: string]: number | string | Record<string, number> | undefined;
|
||||
};
|
||||
components?: {
|
||||
buttonPrimary?: {
|
||||
background?: string;
|
||||
textColor?: string;
|
||||
borderColor?: string;
|
||||
borderRadius?: string;
|
||||
[key: string]: string | undefined;
|
||||
};
|
||||
buttonSecondary?: {
|
||||
background?: string;
|
||||
textColor?: string;
|
||||
borderColor?: string;
|
||||
borderRadius?: string;
|
||||
[key: string]: string | undefined;
|
||||
};
|
||||
input?: {
|
||||
borderColor?: string;
|
||||
focusBorderColor?: string;
|
||||
borderRadius?: string;
|
||||
[key: string]: string | undefined;
|
||||
};
|
||||
[key: string]: unknown;
|
||||
};
|
||||
icons?: {
|
||||
style?: string;
|
||||
primaryColor?: string;
|
||||
[key: string]: string | undefined;
|
||||
};
|
||||
images?: {
|
||||
logo?: string | null;
|
||||
favicon?: string | null;
|
||||
ogImage?: string | null;
|
||||
[key: string]: string | null | undefined;
|
||||
};
|
||||
animations?: {
|
||||
transitionDuration?: string;
|
||||
easing?: string;
|
||||
[key: string]: string | undefined;
|
||||
};
|
||||
layout?: {
|
||||
grid?: {
|
||||
columns?: number;
|
||||
maxWidth?: string;
|
||||
[key: string]: number | string | undefined;
|
||||
};
|
||||
headerHeight?: string;
|
||||
footerHeight?: string;
|
||||
[key: string]:
|
||||
| number
|
||||
| string
|
||||
| Record<string, number | string | undefined>
|
||||
| undefined;
|
||||
};
|
||||
tone?: {
|
||||
voice?: string;
|
||||
emojiUsage?: string;
|
||||
[key: string]: string | undefined;
|
||||
};
|
||||
personality?: {
|
||||
tone:
|
||||
| 'professional'
|
||||
| 'playful'
|
||||
| 'modern'
|
||||
| 'traditional'
|
||||
| 'minimalist'
|
||||
| 'bold';
|
||||
energy: 'low' | 'medium' | 'high';
|
||||
targetAudience: string;
|
||||
};
|
||||
[key: string]: unknown;
|
||||
}
|
||||
|
||||
export interface DocumentMetadata {
|
||||
// Common metadata fields
|
||||
title?: string;
|
||||
description?: string;
|
||||
url?: string;
|
||||
language?: string;
|
||||
keywords?: string | string[];
|
||||
robots?: string;
|
||||
|
||||
// OpenGraph and social metadata
|
||||
ogTitle?: string;
|
||||
ogDescription?: string;
|
||||
ogUrl?: string;
|
||||
ogImage?: string;
|
||||
ogAudio?: string;
|
||||
ogDeterminer?: string;
|
||||
ogLocale?: string;
|
||||
ogLocaleAlternate?: string[];
|
||||
ogSiteName?: string;
|
||||
ogVideo?: string;
|
||||
|
||||
// Dublin Core and other site metadata
|
||||
favicon?: string;
|
||||
dcTermsCreated?: string;
|
||||
dcDateCreated?: string;
|
||||
dcDate?: string;
|
||||
dcTermsType?: string;
|
||||
dcType?: string;
|
||||
dcTermsAudience?: string;
|
||||
dcTermsSubject?: string;
|
||||
dcSubject?: string;
|
||||
dcDescription?: string;
|
||||
dcTermsKeywords?: string;
|
||||
|
||||
modifiedTime?: string;
|
||||
publishedTime?: string;
|
||||
articleTag?: string;
|
||||
articleSection?: string;
|
||||
|
||||
// Response-level metadata
|
||||
sourceURL?: string;
|
||||
statusCode?: number;
|
||||
scrapeId?: string;
|
||||
numPages?: number;
|
||||
contentType?: string;
|
||||
timezone?: string;
|
||||
proxyUsed?: 'basic' | 'stealth';
|
||||
cacheState?: 'hit' | 'miss';
|
||||
cachedAt?: string;
|
||||
creditsUsed?: number;
|
||||
concurrencyLimited?: boolean;
|
||||
concurrencyQueueDurationMs?: number;
|
||||
|
||||
// Error information
|
||||
error?: string;
|
||||
|
||||
[key: string]: unknown;
|
||||
}
|
||||
|
||||
export interface Document {
|
||||
markdown?: string;
|
||||
html?: string;
|
||||
rawHtml?: string;
|
||||
json?: unknown;
|
||||
summary?: string;
|
||||
metadata?: DocumentMetadata;
|
||||
links?: string[];
|
||||
images?: string[];
|
||||
screenshot?: string;
|
||||
audio?: string;
|
||||
attributes?: Array<{
|
||||
selector: string;
|
||||
attribute: string;
|
||||
values: string[];
|
||||
}>;
|
||||
actions?: Record<string, unknown>;
|
||||
answer?: string;
|
||||
highlights?: string;
|
||||
warning?: string;
|
||||
changeTracking?: Record<string, unknown>;
|
||||
branding?: BrandingProfile;
|
||||
}
|
||||
|
||||
// Pagination configuration for auto-fetching pages from v2 endpoints that return a `next` URL
|
||||
export interface PaginationConfig {
|
||||
/** When true (default), automatically follow `next` links and aggregate all documents. */
|
||||
autoPaginate?: boolean;
|
||||
/** Maximum number of additional pages to fetch after the first response. */
|
||||
maxPages?: number;
|
||||
/** Maximum total number of documents to return across all pages. */
|
||||
maxResults?: number;
|
||||
/** Maximum time to spend fetching additional pages (in seconds). */
|
||||
maxWaitTime?: number;
|
||||
}
|
||||
|
||||
export interface SearchResultWeb {
|
||||
url: string;
|
||||
title?: string;
|
||||
description?: string;
|
||||
category?: string;
|
||||
}
|
||||
|
||||
export interface SearchResultNews {
|
||||
title?: string;
|
||||
url?: string;
|
||||
snippet?: string;
|
||||
date?: string;
|
||||
imageUrl?: string;
|
||||
position?: number;
|
||||
category?: string;
|
||||
}
|
||||
|
||||
export interface SearchResultImages {
|
||||
title?: string;
|
||||
imageUrl?: string;
|
||||
imageWidth?: number;
|
||||
imageHeight?: number;
|
||||
url?: string;
|
||||
position?: number;
|
||||
}
|
||||
|
||||
export interface SearchData {
|
||||
web?: Array<SearchResultWeb | Document>;
|
||||
news?: Array<SearchResultNews | Document>;
|
||||
images?: Array<SearchResultImages | Document>;
|
||||
}
|
||||
|
||||
export interface CategoryOption {
|
||||
type: 'github' | 'research' | 'pdf';
|
||||
}
|
||||
|
||||
export interface SearchRequest {
|
||||
query: string;
|
||||
sources?: Array<
|
||||
'web' | 'news' | 'images' | { type: 'web' | 'news' | 'images' }
|
||||
>;
|
||||
categories?: Array<'github' | 'research' | 'pdf' | CategoryOption>;
|
||||
includeDomains?: string[];
|
||||
excludeDomains?: string[];
|
||||
limit?: number;
|
||||
tbs?: string;
|
||||
location?: string;
|
||||
ignoreInvalidURLs?: boolean;
|
||||
timeout?: number; // ms
|
||||
scrapeOptions?: ScrapeOptions;
|
||||
integration?: string;
|
||||
origin?: string;
|
||||
}
|
||||
|
||||
export interface CrawlOptions {
|
||||
prompt?: string | null;
|
||||
excludePaths?: string[] | null;
|
||||
includePaths?: string[] | null;
|
||||
maxDiscoveryDepth?: number | null;
|
||||
sitemap?: 'skip' | 'include' | 'only';
|
||||
ignoreQueryParameters?: boolean;
|
||||
deduplicateSimilarURLs?: boolean;
|
||||
limit?: number | null;
|
||||
crawlEntireDomain?: boolean;
|
||||
allowExternalLinks?: boolean;
|
||||
allowSubdomains?: boolean;
|
||||
ignoreRobotsTxt?: boolean;
|
||||
robotsUserAgent?: string | null;
|
||||
delay?: number | null;
|
||||
maxConcurrency?: number | null;
|
||||
webhook?: string | WebhookConfig | null;
|
||||
scrapeOptions?: ScrapeOptions | null;
|
||||
regexOnFullURL?: boolean;
|
||||
zeroDataRetention?: boolean;
|
||||
integration?: string;
|
||||
origin?: string;
|
||||
}
|
||||
|
||||
export interface CrawlResponse {
|
||||
id: string;
|
||||
url: string;
|
||||
}
|
||||
|
||||
export interface CrawlJob {
|
||||
id: string;
|
||||
status: 'scraping' | 'completed' | 'failed' | 'cancelled';
|
||||
total: number;
|
||||
completed: number;
|
||||
creditsUsed?: number;
|
||||
expiresAt?: string;
|
||||
next?: string | null;
|
||||
data: Document[];
|
||||
}
|
||||
|
||||
export interface BatchScrapeOptions {
|
||||
options?: ScrapeOptions;
|
||||
webhook?: string | WebhookConfig;
|
||||
appendToId?: string;
|
||||
ignoreInvalidURLs?: boolean;
|
||||
maxConcurrency?: number;
|
||||
zeroDataRetention?: boolean;
|
||||
idempotencyKey?: string;
|
||||
integration?: string;
|
||||
origin?: string;
|
||||
}
|
||||
|
||||
export interface BatchScrapeResponse {
|
||||
id: string;
|
||||
url: string;
|
||||
invalidURLs?: string[];
|
||||
}
|
||||
|
||||
export interface BatchScrapeJob {
|
||||
id: string;
|
||||
status: 'scraping' | 'completed' | 'failed' | 'cancelled';
|
||||
completed: number;
|
||||
total: number;
|
||||
creditsUsed?: number;
|
||||
expiresAt?: string;
|
||||
next?: string | null;
|
||||
data: Document[];
|
||||
}
|
||||
|
||||
export interface MapData {
|
||||
links: SearchResultWeb[];
|
||||
}
|
||||
|
||||
export interface MapOptions {
|
||||
search?: string;
|
||||
sitemap?: 'only' | 'include' | 'skip';
|
||||
includeSubdomains?: boolean;
|
||||
ignoreQueryParameters?: boolean;
|
||||
limit?: number;
|
||||
timeout?: number;
|
||||
integration?: string;
|
||||
origin?: string;
|
||||
location?: LocationConfig;
|
||||
}
|
||||
|
||||
export interface MonitorSchedule {
|
||||
cron: string;
|
||||
timezone?: string;
|
||||
}
|
||||
|
||||
export interface MonitorEmailNotification {
|
||||
enabled?: boolean;
|
||||
recipients?: string[];
|
||||
includeDiffs?: boolean;
|
||||
}
|
||||
|
||||
export interface MonitorNotification {
|
||||
email?: MonitorEmailNotification;
|
||||
}
|
||||
|
||||
export interface MonitorWebhookConfig {
|
||||
url: string;
|
||||
headers?: Record<string, string>;
|
||||
metadata?: Record<string, string>;
|
||||
events?: string[];
|
||||
}
|
||||
|
||||
export interface MonitorScrapeTarget {
|
||||
id?: string;
|
||||
type: 'scrape';
|
||||
urls: string[];
|
||||
scrapeOptions?: ScrapeOptions;
|
||||
}
|
||||
|
||||
export interface MonitorCrawlTarget {
|
||||
id?: string;
|
||||
type: 'crawl';
|
||||
url: string;
|
||||
crawlOptions?: CrawlOptions;
|
||||
scrapeOptions?: ScrapeOptions;
|
||||
}
|
||||
|
||||
export type MonitorTarget = MonitorScrapeTarget | MonitorCrawlTarget;
|
||||
|
||||
export interface CreateMonitorRequest {
|
||||
name: string;
|
||||
schedule: MonitorSchedule;
|
||||
webhook?: MonitorWebhookConfig;
|
||||
notification?: MonitorNotification;
|
||||
targets: MonitorTarget[];
|
||||
retentionDays?: number;
|
||||
}
|
||||
|
||||
export interface UpdateMonitorRequest {
|
||||
name?: string;
|
||||
status?: 'active' | 'paused';
|
||||
schedule?: MonitorSchedule;
|
||||
webhook?: MonitorWebhookConfig | null;
|
||||
notification?: MonitorNotification | null;
|
||||
targets?: MonitorTarget[];
|
||||
retentionDays?: number;
|
||||
}
|
||||
|
||||
export interface MonitorSummary {
|
||||
totalPages: number;
|
||||
same: number;
|
||||
changed: number;
|
||||
new: number;
|
||||
removed: number;
|
||||
error: number;
|
||||
}
|
||||
|
||||
export interface Monitor {
|
||||
id: string;
|
||||
name: string;
|
||||
status: 'active' | 'paused' | 'deleted';
|
||||
schedule: MonitorSchedule;
|
||||
nextRunAt?: string | null;
|
||||
lastRunAt?: string | null;
|
||||
currentCheckId?: string | null;
|
||||
targets: MonitorTarget[];
|
||||
webhook?: MonitorWebhookConfig | null;
|
||||
notification?: MonitorNotification | null;
|
||||
retentionDays: number;
|
||||
estimatedCreditsPerMonth?: number | null;
|
||||
lastCheckSummary?: MonitorSummary | null;
|
||||
createdAt: string;
|
||||
updatedAt: string;
|
||||
}
|
||||
|
||||
export interface MonitorCheck {
|
||||
id: string;
|
||||
monitorId: string;
|
||||
status:
|
||||
| 'queued'
|
||||
| 'running'
|
||||
| 'completed'
|
||||
| 'failed'
|
||||
| 'partial'
|
||||
| 'skipped_overlap';
|
||||
trigger: 'scheduled' | 'manual';
|
||||
scheduledFor?: string | null;
|
||||
startedAt?: string | null;
|
||||
finishedAt?: string | null;
|
||||
estimatedCredits?: number | null;
|
||||
reservedCredits?: number | null;
|
||||
actualCredits?: number | null;
|
||||
billingStatus:
|
||||
| 'not_applicable'
|
||||
| 'reserved'
|
||||
| 'confirmed'
|
||||
| 'released'
|
||||
| 'failed';
|
||||
summary: MonitorSummary;
|
||||
targetResults?: unknown;
|
||||
notificationStatus?: unknown;
|
||||
error?: string | null;
|
||||
createdAt: string;
|
||||
updatedAt: string;
|
||||
}
|
||||
|
||||
export interface MonitorCheckPage {
|
||||
id: string;
|
||||
targetId: string;
|
||||
url: string;
|
||||
status: 'same' | 'new' | 'changed' | 'removed' | 'error';
|
||||
previousScrapeId?: string | null;
|
||||
currentScrapeId?: string | null;
|
||||
statusCode?: number | null;
|
||||
error?: string | null;
|
||||
metadata?: unknown;
|
||||
diff?: unknown;
|
||||
createdAt: string;
|
||||
}
|
||||
|
||||
export interface MonitorCheckDetail extends MonitorCheck {
|
||||
pages: MonitorCheckPage[];
|
||||
next?: string | null;
|
||||
}
|
||||
|
||||
export interface ListMonitorsOptions {
|
||||
limit?: number;
|
||||
offset?: number;
|
||||
}
|
||||
|
||||
export type ListMonitorChecksOptions = ListMonitorsOptions;
|
||||
|
||||
export type GetMonitorCheckOptions = PaginationConfig & {
|
||||
limit?: number;
|
||||
skip?: number;
|
||||
status?: MonitorCheckPage["status"];
|
||||
};
|
||||
|
||||
export interface ExtractResponse {
|
||||
success?: boolean;
|
||||
id?: string;
|
||||
status?: 'processing' | 'completed' | 'failed' | 'cancelled';
|
||||
data?: unknown;
|
||||
error?: string;
|
||||
warning?: string;
|
||||
warnings?: string[];
|
||||
replacement?: string;
|
||||
sources?: Record<string, unknown>;
|
||||
expiresAt?: string;
|
||||
creditsUsed?: number;
|
||||
}
|
||||
|
||||
export interface AgentResponse {
|
||||
success: boolean;
|
||||
id: string;
|
||||
error?: string;
|
||||
}
|
||||
|
||||
export interface AgentStatusResponse {
|
||||
success: boolean;
|
||||
status: 'processing' | 'completed' | 'failed';
|
||||
error?: string;
|
||||
data?: unknown;
|
||||
model?: 'spark-1-pro' | 'spark-1-mini';
|
||||
expiresAt: string;
|
||||
creditsUsed?: number;
|
||||
}
|
||||
|
||||
export interface AgentOptions {
|
||||
model: 'FIRE-1' | 'v3-beta';
|
||||
}
|
||||
|
||||
export interface ConcurrencyCheck {
|
||||
concurrency: number;
|
||||
maxConcurrency: number;
|
||||
}
|
||||
|
||||
export interface CreditUsage {
|
||||
remainingCredits: number;
|
||||
planCredits?: number;
|
||||
billingPeriodStart?: string | null;
|
||||
billingPeriodEnd?: string | null;
|
||||
}
|
||||
|
||||
export interface TokenUsage {
|
||||
remainingTokens: number;
|
||||
planTokens?: number;
|
||||
billingPeriodStart?: string | null;
|
||||
billingPeriodEnd?: string | null;
|
||||
}
|
||||
|
||||
export interface CreditUsageHistoricalPeriod {
|
||||
startDate: string | null;
|
||||
endDate: string | null;
|
||||
apiKey?: string;
|
||||
creditsUsed: number;
|
||||
}
|
||||
|
||||
export interface CreditUsageHistoricalResponse {
|
||||
success: boolean;
|
||||
periods: CreditUsageHistoricalPeriod[];
|
||||
}
|
||||
|
||||
export interface TokenUsageHistoricalPeriod {
|
||||
startDate: string | null;
|
||||
endDate: string | null;
|
||||
apiKey?: string;
|
||||
tokensUsed: number;
|
||||
}
|
||||
|
||||
export interface TokenUsageHistoricalResponse {
|
||||
success: boolean;
|
||||
periods: TokenUsageHistoricalPeriod[];
|
||||
}
|
||||
|
||||
export interface CrawlErrorsResponse {
|
||||
errors: {
|
||||
id: string;
|
||||
timestamp?: string;
|
||||
url: string;
|
||||
code?: string;
|
||||
error: string;
|
||||
}[];
|
||||
robotsBlocked: string[];
|
||||
}
|
||||
|
||||
export interface ActiveCrawl {
|
||||
id: string;
|
||||
teamId: string;
|
||||
url: string;
|
||||
options?: Record<string, unknown> | null;
|
||||
}
|
||||
|
||||
export interface ActiveCrawlsResponse {
|
||||
success: boolean;
|
||||
crawls: ActiveCrawl[];
|
||||
}
|
||||
|
||||
export interface ErrorDetails {
|
||||
code?: string;
|
||||
message: string;
|
||||
details?: Record<string, unknown>;
|
||||
status?: number;
|
||||
}
|
||||
|
||||
export class SdkError extends Error {
|
||||
status?: number;
|
||||
code?: string;
|
||||
details?: unknown;
|
||||
jobId?: string;
|
||||
constructor(
|
||||
message: string,
|
||||
status?: number,
|
||||
code?: string,
|
||||
details?: unknown,
|
||||
jobId?: string
|
||||
) {
|
||||
super(message);
|
||||
this.name = 'FirecrawlSdkError';
|
||||
this.status = status;
|
||||
this.code = code;
|
||||
this.details = details;
|
||||
this.jobId = jobId;
|
||||
}
|
||||
}
|
||||
|
||||
export class JobTimeoutError extends SdkError {
|
||||
timeoutSeconds: number;
|
||||
constructor(jobId: string, timeoutSeconds: number, jobType: 'batch' | 'crawl' = 'batch') {
|
||||
const jobTypeLabel = jobType === 'batch' ? 'batch scrape' : 'crawl';
|
||||
super(
|
||||
`${jobTypeLabel.charAt(0).toUpperCase() + jobTypeLabel.slice(1)} job ${jobId} did not complete within ${timeoutSeconds} seconds`,
|
||||
undefined,
|
||||
'JOB_TIMEOUT',
|
||||
undefined,
|
||||
jobId
|
||||
);
|
||||
this.name = 'JobTimeoutError';
|
||||
this.timeoutSeconds = timeoutSeconds;
|
||||
}
|
||||
}
|
||||
|
||||
export interface QueueStatusResponse {
|
||||
success: boolean;
|
||||
jobsInQueue: number;
|
||||
activeJobsInQueue: number;
|
||||
waitingJobsInQueue: number;
|
||||
maxConcurrency: number;
|
||||
mostRecentSuccess: string | null;
|
||||
}
|
||||
|
||||
// Browser types
|
||||
export interface BrowserCreateResponse {
|
||||
success: boolean;
|
||||
id?: string;
|
||||
cdpUrl?: string;
|
||||
liveViewUrl?: string;
|
||||
interactiveLiveViewUrl?: string;
|
||||
expiresAt?: string;
|
||||
error?: string;
|
||||
}
|
||||
|
||||
export interface BrowserExecuteResponse {
|
||||
success: boolean;
|
||||
liveViewUrl?: string;
|
||||
interactiveLiveViewUrl?: string;
|
||||
output?: string;
|
||||
stdout?: string;
|
||||
result?: string;
|
||||
stderr?: string;
|
||||
exitCode?: number;
|
||||
killed?: boolean;
|
||||
error?: string;
|
||||
}
|
||||
|
||||
export interface BrowserDeleteResponse {
|
||||
success: boolean;
|
||||
sessionDurationMs?: number;
|
||||
creditsBilled?: number;
|
||||
error?: string;
|
||||
}
|
||||
|
||||
export interface ScrapeExecuteRequest {
|
||||
code?: string;
|
||||
prompt?: string;
|
||||
language?: "python" | "node" | "bash";
|
||||
timeout?: number;
|
||||
origin?: string;
|
||||
}
|
||||
|
||||
export type ScrapeExecuteResponse = BrowserExecuteResponse;
|
||||
export type ScrapeBrowserDeleteResponse = BrowserDeleteResponse;
|
||||
|
||||
export interface BrowserSession {
|
||||
id: string;
|
||||
status: string;
|
||||
cdpUrl: string;
|
||||
liveViewUrl: string;
|
||||
interactiveLiveViewUrl?: string;
|
||||
streamWebView: boolean;
|
||||
createdAt: string;
|
||||
lastActivity: string;
|
||||
}
|
||||
|
||||
export interface BrowserListResponse {
|
||||
success: boolean;
|
||||
sessions?: BrowserSession[];
|
||||
error?: string;
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
import { type AxiosError, type AxiosResponse } from "axios";
|
||||
import { SdkError, JobTimeoutError } from "../types";
|
||||
|
||||
export function throwForBadResponse(resp: AxiosResponse, action: string): never {
|
||||
const status = resp.status;
|
||||
const body = resp.data || {};
|
||||
const msg = body?.error || body?.message || `Request failed (${status}) while trying to ${action}`;
|
||||
throw new SdkError(msg, status, undefined, body?.details);
|
||||
}
|
||||
|
||||
export function normalizeAxiosError(err: AxiosError, action: string): never {
|
||||
const status = err.response?.status;
|
||||
const body: any = err.response?.data;
|
||||
const message = body?.error || err.message || `Request failed${status ? ` (${status})` : ""} while trying to ${action}`;
|
||||
const code = (body?.code as string) || err.code;
|
||||
throw new SdkError(message, status, code, body?.details ?? body);
|
||||
}
|
||||
|
||||
export function isRetryableError(err: any): boolean {
|
||||
// JobTimeoutError should never be retried - it's the overall timeout
|
||||
if (err instanceof JobTimeoutError) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// If it's an SdkError with a status code, check if it's retryable
|
||||
if (err instanceof SdkError || (err && typeof err === 'object' && 'status' in err)) {
|
||||
const status = err.status;
|
||||
// 4xx errors are client errors and shouldn't be retried
|
||||
if (status && status >= 400 && status < 500) {
|
||||
return false; // Don't retry client errors (401, 404, etc.)
|
||||
}
|
||||
// 5xx errors are server errors and can be retried
|
||||
if (status && status >= 500) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
// Network errors (no response) are retryable
|
||||
if (err?.isAxiosError && !err.response) {
|
||||
return true;
|
||||
}
|
||||
|
||||
// HTTP timeout errors are retryable (different from JobTimeoutError)
|
||||
if (err?.code === 'ECONNABORTED' || err?.message?.includes('timeout')) {
|
||||
return true;
|
||||
}
|
||||
|
||||
// Default: retry on unknown errors (safer than not retrying)
|
||||
return true;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,14 @@
|
||||
export function getVersion(): string {
|
||||
try {
|
||||
if (typeof process !== "undefined" && process.env && process.env.npm_package_version) {
|
||||
return process.env.npm_package_version as string;
|
||||
}
|
||||
|
||||
// eslint-disable-next-line @typescript-eslint/no-var-requires
|
||||
const pkg = require("../../../package.json");
|
||||
return (pkg?.version as string) || "3.x.x";
|
||||
} catch {
|
||||
return "3.x.x";
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,171 @@
|
||||
import axios, {
|
||||
type AxiosInstance,
|
||||
type AxiosRequestConfig,
|
||||
type AxiosResponse,
|
||||
} from "axios";
|
||||
import { getVersion } from "./getVersion";
|
||||
|
||||
export interface HttpClientOptions {
|
||||
apiKey: string;
|
||||
apiUrl: string;
|
||||
timeoutMs?: number;
|
||||
maxRetries?: number;
|
||||
backoffFactor?: number; // seconds factor for 0.5, 1, 2...
|
||||
}
|
||||
|
||||
export interface RequestOptions {
|
||||
headers?: Record<string, string>;
|
||||
timeoutMs?: number;
|
||||
}
|
||||
|
||||
export class HttpClient {
|
||||
private instance: AxiosInstance;
|
||||
private readonly apiKey: string;
|
||||
private readonly apiUrl: string;
|
||||
private readonly maxRetries: number;
|
||||
private readonly backoffFactor: number;
|
||||
|
||||
constructor(options: HttpClientOptions) {
|
||||
this.apiKey = options.apiKey;
|
||||
this.apiUrl = options.apiUrl.replace(/\/$/, "");
|
||||
this.maxRetries = options.maxRetries ?? 3;
|
||||
this.backoffFactor = options.backoffFactor ?? 0.5;
|
||||
this.instance = axios.create({
|
||||
baseURL: this.apiUrl,
|
||||
timeout: options.timeoutMs ?? 300000,
|
||||
headers: {
|
||||
Authorization: `Bearer ${this.apiKey}`,
|
||||
},
|
||||
transitional: { clarifyTimeoutError: true },
|
||||
});
|
||||
}
|
||||
|
||||
getApiUrl(): string {
|
||||
return this.apiUrl;
|
||||
}
|
||||
|
||||
getApiKey(): string {
|
||||
return this.apiKey;
|
||||
}
|
||||
|
||||
private async request<T = any>(
|
||||
config: AxiosRequestConfig,
|
||||
): Promise<AxiosResponse<T>> {
|
||||
const version = getVersion();
|
||||
config.headers = {
|
||||
...(config.headers || {}),
|
||||
};
|
||||
|
||||
let lastError: any;
|
||||
for (let attempt = 0; attempt < this.maxRetries; attempt++) {
|
||||
try {
|
||||
const cfg: AxiosRequestConfig = { ...config };
|
||||
const isFormDataBody =
|
||||
typeof FormData !== "undefined" && cfg.data instanceof FormData;
|
||||
const isPlainObjectBody =
|
||||
!isFormDataBody &&
|
||||
cfg.data != null &&
|
||||
typeof cfg.data === "object" &&
|
||||
!Array.isArray(cfg.data);
|
||||
|
||||
// For JSON POST/PUT/PATCH, ensure origin is present in body
|
||||
if (
|
||||
isPlainObjectBody &&
|
||||
cfg.method &&
|
||||
["post", "put", "patch"].includes(cfg.method.toLowerCase())
|
||||
) {
|
||||
const data = (cfg.data ?? {}) as Record<string, unknown>;
|
||||
cfg.data = {
|
||||
...data,
|
||||
origin:
|
||||
typeof data.origin === "string" && data.origin.includes("mcp")
|
||||
? data.origin
|
||||
: `js-sdk@${version}`,
|
||||
};
|
||||
}
|
||||
|
||||
if (isFormDataBody) {
|
||||
cfg.headers = { ...(cfg.headers || {}) };
|
||||
delete (cfg.headers as Record<string, unknown>)["Content-Type"];
|
||||
delete (cfg.headers as Record<string, unknown>)["content-type"];
|
||||
}
|
||||
|
||||
const res = await this.instance.request<T>(cfg);
|
||||
if (res.status === 502 && attempt < this.maxRetries - 1) {
|
||||
await this.sleep(this.backoffFactor * Math.pow(2, attempt));
|
||||
continue;
|
||||
}
|
||||
return res;
|
||||
} catch (err: any) {
|
||||
lastError = err;
|
||||
const status = err?.response?.status;
|
||||
if (status === 502 && attempt < this.maxRetries - 1) {
|
||||
await this.sleep(this.backoffFactor * Math.pow(2, attempt));
|
||||
continue;
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
throw lastError ?? new Error("Unexpected HTTP client error");
|
||||
}
|
||||
|
||||
private sleep(seconds: number): Promise<void> {
|
||||
return new Promise(r => setTimeout(r, seconds * 1000));
|
||||
}
|
||||
|
||||
post<T = any>(
|
||||
endpoint: string,
|
||||
body: Record<string, unknown>,
|
||||
options?: RequestOptions,
|
||||
) {
|
||||
return this.request<T>({
|
||||
method: "post",
|
||||
url: endpoint,
|
||||
data: body,
|
||||
headers: options?.headers,
|
||||
timeout: options?.timeoutMs,
|
||||
});
|
||||
}
|
||||
|
||||
postMultipart<T = any>(
|
||||
endpoint: string,
|
||||
formData: FormData,
|
||||
options?: RequestOptions,
|
||||
) {
|
||||
return this.request<T>({
|
||||
method: "post",
|
||||
url: endpoint,
|
||||
data: formData,
|
||||
headers: options?.headers,
|
||||
timeout: options?.timeoutMs,
|
||||
});
|
||||
}
|
||||
|
||||
get<T = any>(endpoint: string, headers?: Record<string, string>) {
|
||||
return this.request<T>({ method: "get", url: endpoint, headers });
|
||||
}
|
||||
|
||||
delete<T = any>(endpoint: string, headers?: Record<string, string>) {
|
||||
return this.request<T>({ method: "delete", url: endpoint, headers });
|
||||
}
|
||||
|
||||
patch<T = any>(
|
||||
endpoint: string,
|
||||
body: Record<string, unknown>,
|
||||
options?: RequestOptions,
|
||||
) {
|
||||
return this.request<T>({
|
||||
method: "patch",
|
||||
url: endpoint,
|
||||
data: body,
|
||||
headers: options?.headers,
|
||||
timeout: options?.timeoutMs,
|
||||
});
|
||||
}
|
||||
|
||||
prepareHeaders(idempotencyKey?: string): Record<string, string> {
|
||||
const headers: Record<string, string> = {};
|
||||
if (idempotencyKey) headers["x-idempotency-key"] = idempotencyKey;
|
||||
return headers;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,48 @@
|
||||
import type { HttpClient } from "../utils/httpClient";
|
||||
import type { Document, PaginationConfig } from "../types";
|
||||
|
||||
/**
|
||||
* Shared helper to follow `next` URLs and aggregate paginated result arrays.
|
||||
*/
|
||||
export async function fetchAllPages<T = Document>(
|
||||
http: HttpClient,
|
||||
nextUrl: string,
|
||||
initial: T[],
|
||||
pagination?: PaginationConfig
|
||||
): Promise<T[]> {
|
||||
const docs = initial.slice();
|
||||
let current: string | null = nextUrl;
|
||||
let pageCount = 0;
|
||||
const maxPages = pagination?.maxPages ?? undefined;
|
||||
const maxResults = pagination?.maxResults ?? undefined;
|
||||
const maxWaitTime = pagination?.maxWaitTime ?? undefined;
|
||||
const started = Date.now();
|
||||
|
||||
while (current) {
|
||||
if (maxPages != null && pageCount >= maxPages) break;
|
||||
if (maxWaitTime != null && (Date.now() - started) / 1000 > maxWaitTime) break;
|
||||
|
||||
let payload: { success: boolean; next?: string | null; data?: T[] | { pages?: T[]; next?: string | null } } | null = null;
|
||||
try {
|
||||
const res = await http.get<{ success: boolean; next?: string | null; data?: T[] | { pages?: T[]; next?: string | null } }>(current);
|
||||
payload = res.data;
|
||||
} catch {
|
||||
break; // axios rejects on non-2xx; stop pagination gracefully
|
||||
}
|
||||
if (!payload?.success) break;
|
||||
|
||||
const pageData = Array.isArray(payload.data)
|
||||
? payload.data
|
||||
: payload.data?.pages || [];
|
||||
for (const d of pageData) {
|
||||
if (maxResults != null && docs.length >= maxResults) break;
|
||||
docs.push(d as T);
|
||||
}
|
||||
if (maxResults != null && docs.length >= maxResults) break;
|
||||
current = (payload.next ?? (Array.isArray(payload.data) ? null : payload.data?.next) ?? null) as string | null;
|
||||
pageCount += 1;
|
||||
}
|
||||
return docs;
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,208 @@
|
||||
import {
|
||||
type ChangeTrackingFormat,
|
||||
type FormatOption,
|
||||
type JsonFormat,
|
||||
type ParseFormatOption,
|
||||
type ParseOptions,
|
||||
type QuestionFormat,
|
||||
type HighlightsFormat,
|
||||
type QueryFormat,
|
||||
type ScrapeOptions,
|
||||
type ScreenshotFormat,
|
||||
} from "../types";
|
||||
import { isZodSchema, zodSchemaToJsonSchema, looksLikeZodShape } from "../../utils/zodSchemaToJson";
|
||||
|
||||
export function ensureValidFormats(formats?: FormatOption[]): void {
|
||||
if (!formats) return;
|
||||
for (const fmt of formats) {
|
||||
if (typeof fmt === "string") {
|
||||
if (fmt === "json") {
|
||||
throw new Error("json format must be an object with { type: 'json', prompt, schema }");
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if ((fmt as JsonFormat).type === "json") {
|
||||
const j = fmt as JsonFormat;
|
||||
if (!j.prompt && !j.schema) {
|
||||
throw new Error("json format requires either 'prompt' or 'schema' (or both)");
|
||||
}
|
||||
const maybeSchema = j.schema;
|
||||
if (isZodSchema(maybeSchema)) {
|
||||
(j as any).schema = zodSchemaToJsonSchema(maybeSchema);
|
||||
} else if (looksLikeZodShape(maybeSchema)) {
|
||||
throw new Error(
|
||||
"json format schema appears to be a Zod schema's .shape property. " +
|
||||
"Pass the Zod schema directly (e.g., `schema: MySchema`) instead of `schema: MySchema.shape`. " +
|
||||
"The SDK will automatically convert Zod schemas to JSON Schema format."
|
||||
);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if ((fmt as ChangeTrackingFormat).type === "changeTracking") {
|
||||
const ct = fmt as ChangeTrackingFormat;
|
||||
const maybeSchema = ct.schema;
|
||||
if (isZodSchema(maybeSchema)) {
|
||||
(ct as any).schema = zodSchemaToJsonSchema(maybeSchema);
|
||||
} else if (looksLikeZodShape(maybeSchema)) {
|
||||
throw new Error(
|
||||
"changeTracking format schema appears to be a Zod schema's .shape property. " +
|
||||
"Pass the Zod schema directly (e.g., `schema: MySchema`) instead of `schema: MySchema.shape`. " +
|
||||
"The SDK will automatically convert Zod schemas to JSON Schema format."
|
||||
);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if ((fmt as QuestionFormat).type === "question") {
|
||||
const q = fmt as QuestionFormat;
|
||||
if (typeof q.question !== "string" || q.question.trim().length === 0) {
|
||||
throw new Error("question format requires a non-empty 'question' string");
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if ((fmt as HighlightsFormat).type === "highlights") {
|
||||
const h = fmt as HighlightsFormat;
|
||||
if (typeof h.query !== "string" || h.query.trim().length === 0) {
|
||||
throw new Error("highlights format requires a non-empty 'query' string");
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if ((fmt as QueryFormat).type === "query") {
|
||||
const q = fmt as QueryFormat;
|
||||
if (typeof q.prompt !== "string" || q.prompt.trim().length === 0) {
|
||||
throw new Error("query format requires a non-empty 'prompt' string");
|
||||
}
|
||||
if (q.mode != null && q.mode !== "freeform" && q.mode !== "directQuote") {
|
||||
throw new Error("query format mode must be 'freeform' or 'directQuote'");
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if ((fmt as ScreenshotFormat).type === "screenshot") {
|
||||
// no-op; already camelCase; validate numeric fields if present
|
||||
const s = fmt as ScreenshotFormat;
|
||||
if (s.quality != null && (typeof s.quality !== "number" || s.quality < 0)) {
|
||||
throw new Error("screenshot.quality must be a non-negative number");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export function ensureValidScrapeOptions(options?: ScrapeOptions): void {
|
||||
if (!options) return;
|
||||
if (options.timeout != null && options.timeout <= 0) {
|
||||
throw new Error("timeout must be positive");
|
||||
}
|
||||
if (options.waitFor != null && options.waitFor < 0) {
|
||||
throw new Error("waitFor must be non-negative");
|
||||
}
|
||||
ensureValidFormats(options.formats);
|
||||
}
|
||||
|
||||
export function ensureValidParseFormats(formats?: ParseFormatOption[]): void {
|
||||
if (!formats) return;
|
||||
|
||||
for (const fmt of formats) {
|
||||
if (typeof fmt === "string") {
|
||||
if (fmt === "json") {
|
||||
throw new Error("json format must be an object with { type: 'json', prompt, schema }");
|
||||
}
|
||||
if (fmt === "screenshot") {
|
||||
throw new Error("parse does not support screenshot format");
|
||||
}
|
||||
if (fmt === "changeTracking") {
|
||||
throw new Error("parse does not support changeTracking format");
|
||||
}
|
||||
if (fmt === "branding") {
|
||||
throw new Error("parse does not support branding format");
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
const type = (fmt as any).type;
|
||||
if (type === "changeTracking") {
|
||||
throw new Error("parse does not support changeTracking format");
|
||||
}
|
||||
if (type === "screenshot") {
|
||||
throw new Error("parse does not support screenshot format");
|
||||
}
|
||||
if (type === "branding") {
|
||||
throw new Error("parse does not support branding format");
|
||||
}
|
||||
|
||||
if ((fmt as JsonFormat).type === "json") {
|
||||
const j = fmt as JsonFormat;
|
||||
if (!j.prompt && !j.schema) {
|
||||
throw new Error("json format requires either 'prompt' or 'schema' (or both)");
|
||||
}
|
||||
const maybeSchema = j.schema;
|
||||
if (isZodSchema(maybeSchema)) {
|
||||
(j as any).schema = zodSchemaToJsonSchema(maybeSchema);
|
||||
} else if (looksLikeZodShape(maybeSchema)) {
|
||||
throw new Error(
|
||||
"json format schema appears to be a Zod schema's .shape property. " +
|
||||
"Pass the Zod schema directly (e.g., `schema: MySchema`) instead of `schema: MySchema.shape`. " +
|
||||
"The SDK will automatically convert Zod schemas to JSON Schema format."
|
||||
);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
if ((fmt as QuestionFormat).type === "question") {
|
||||
const q = fmt as QuestionFormat;
|
||||
if (typeof q.question !== "string" || q.question.trim().length === 0) {
|
||||
throw new Error("question format requires a non-empty 'question' string");
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if ((fmt as HighlightsFormat).type === "highlights") {
|
||||
const h = fmt as HighlightsFormat;
|
||||
if (typeof h.query !== "string" || h.query.trim().length === 0) {
|
||||
throw new Error("highlights format requires a non-empty 'query' string");
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if ((fmt as QueryFormat).type === "query") {
|
||||
const q = fmt as QueryFormat;
|
||||
if (typeof q.prompt !== "string" || q.prompt.trim().length === 0) {
|
||||
throw new Error("query format requires a non-empty 'prompt' string");
|
||||
}
|
||||
if (q.mode != null && q.mode !== "freeform" && q.mode !== "directQuote") {
|
||||
throw new Error("query format mode must be 'freeform' or 'directQuote'");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export function ensureValidParseOptions(options?: ParseOptions): void {
|
||||
if (!options) return;
|
||||
if (options.timeout != null && options.timeout <= 0) {
|
||||
throw new Error("timeout must be positive");
|
||||
}
|
||||
|
||||
const raw = options as Record<string, unknown>;
|
||||
if (raw.waitFor !== undefined) {
|
||||
throw new Error("parse does not support waitFor");
|
||||
}
|
||||
if (raw.actions !== undefined) {
|
||||
throw new Error("parse does not support actions");
|
||||
}
|
||||
if (raw.location !== undefined) {
|
||||
throw new Error("parse does not support location overrides");
|
||||
}
|
||||
if (raw.mobile !== undefined) {
|
||||
throw new Error("parse does not support mobile rendering");
|
||||
}
|
||||
if (
|
||||
raw.maxAge !== undefined ||
|
||||
raw.minAge !== undefined ||
|
||||
raw.storeInCache !== undefined ||
|
||||
raw.lockdown !== undefined
|
||||
) {
|
||||
throw new Error("parse does not support cache/index options");
|
||||
}
|
||||
if (raw.proxy !== undefined && raw.proxy !== "basic" && raw.proxy !== "auto") {
|
||||
throw new Error("parse only supports proxy values of 'basic' or 'auto'");
|
||||
}
|
||||
|
||||
ensureValidParseFormats(options.formats);
|
||||
}
|
||||
|
||||
283
참고/firecrawl-main/apps/js-sdk/firecrawl/src/v2/watcher.ts
Normal file
283
참고/firecrawl-main/apps/js-sdk/firecrawl/src/v2/watcher.ts
Normal file
@@ -0,0 +1,283 @@
|
||||
import { EventEmitter } from "events";
|
||||
import type { BatchScrapeJob, CrawlJob, Document } from "./types";
|
||||
import type { HttpClient } from "./utils/httpClient";
|
||||
import { getBatchScrapeStatus } from "./methods/batch";
|
||||
import { getCrawlStatus } from "./methods/crawl";
|
||||
// Note: browsers/Deno expose globalThis.WebSocket, but many Node runtimes (<22.4 or without
|
||||
// experimental flags) do not. We lazily fall back to node:undici.
|
||||
|
||||
type WebSocketConstructor = new (url: string, protocols?: string | string[]) => WebSocket;
|
||||
|
||||
const hasGlobalWebSocket = (): WebSocketConstructor | undefined => {
|
||||
if (typeof globalThis === "undefined") return undefined;
|
||||
const candidate = (globalThis as any).WebSocket;
|
||||
return typeof candidate === "function" ? (candidate as WebSocketConstructor) : undefined;
|
||||
};
|
||||
|
||||
const isNodeRuntime = () => typeof process !== "undefined" && !!process.versions?.node;
|
||||
|
||||
let cachedWebSocket: WebSocketConstructor | undefined;
|
||||
let loadPromise: Promise<WebSocketConstructor | undefined> | undefined;
|
||||
|
||||
const loadNodeWebSocket = async (): Promise<WebSocketConstructor | undefined> => {
|
||||
if (!isNodeRuntime()) return undefined;
|
||||
try {
|
||||
const undici = await import("node:undici");
|
||||
const ctor = (undici as any).WebSocket ?? (undici as any).default?.WebSocket;
|
||||
return typeof ctor === "function" ? (ctor as WebSocketConstructor) : undefined;
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
};
|
||||
|
||||
const getWebSocketCtor = async (): Promise<WebSocketConstructor | undefined> => {
|
||||
if (cachedWebSocket) return cachedWebSocket;
|
||||
const globalWs = hasGlobalWebSocket();
|
||||
if (globalWs) {
|
||||
cachedWebSocket = globalWs;
|
||||
return cachedWebSocket;
|
||||
}
|
||||
if (!loadPromise) {
|
||||
loadPromise = loadNodeWebSocket();
|
||||
}
|
||||
cachedWebSocket = await loadPromise;
|
||||
return cachedWebSocket;
|
||||
};
|
||||
|
||||
const decoder = typeof TextDecoder !== "undefined" ? new TextDecoder() : undefined;
|
||||
|
||||
const ensureUtf8String = (data: unknown): string | undefined => {
|
||||
if (typeof data === "string") return data;
|
||||
|
||||
if (typeof Buffer !== "undefined" && Buffer.isBuffer(data)) {
|
||||
return data.toString("utf8");
|
||||
}
|
||||
|
||||
const convertView = (view: ArrayBufferView): string | undefined => {
|
||||
if (typeof Buffer !== "undefined") {
|
||||
return Buffer.from(view.buffer, view.byteOffset, view.byteLength).toString("utf8");
|
||||
}
|
||||
return decoder?.decode(view);
|
||||
};
|
||||
|
||||
if (ArrayBuffer.isView(data)) {
|
||||
return convertView(data);
|
||||
}
|
||||
|
||||
if (data instanceof ArrayBuffer) {
|
||||
return convertView(new Uint8Array(data));
|
||||
}
|
||||
|
||||
return undefined;
|
||||
};
|
||||
|
||||
type JobKind = "crawl" | "batch";
|
||||
|
||||
export interface WatcherOptions {
|
||||
kind?: JobKind;
|
||||
pollInterval?: number; // seconds
|
||||
timeout?: number; // seconds
|
||||
}
|
||||
|
||||
type Snapshot = CrawlJob | BatchScrapeJob;
|
||||
|
||||
export class Watcher extends EventEmitter {
|
||||
private readonly http: HttpClient;
|
||||
private readonly jobId: string;
|
||||
private readonly kind: JobKind;
|
||||
private readonly pollInterval: number;
|
||||
private readonly timeout?: number;
|
||||
private ws?: WebSocket;
|
||||
private closed = false;
|
||||
private readonly emittedDocumentKeys = new Set<string>();
|
||||
|
||||
constructor(http: HttpClient, jobId: string, opts: WatcherOptions = {}) {
|
||||
super();
|
||||
this.http = http;
|
||||
this.jobId = jobId;
|
||||
this.kind = opts.kind ?? "crawl";
|
||||
this.pollInterval = opts.pollInterval ?? 2;
|
||||
this.timeout = opts.timeout;
|
||||
}
|
||||
|
||||
private buildWsUrl(): string {
|
||||
// replace http/https with ws/wss
|
||||
const apiUrl = this.http.getApiUrl();
|
||||
const wsBase = apiUrl.replace(/^http/, "ws");
|
||||
const path = this.kind === "crawl" ? `/v2/crawl/${this.jobId}` : `/v2/batch/scrape/${this.jobId}`;
|
||||
return `${wsBase}${path}`;
|
||||
}
|
||||
|
||||
async start(): Promise<void> {
|
||||
return new Promise<void>((resolve, reject) => {
|
||||
const onDone = () => { cleanup(); resolve(); };
|
||||
const onError = (err: any) => { cleanup(); resolve(); };
|
||||
const cleanup = () => {
|
||||
this.removeListener("done", onDone);
|
||||
this.removeListener("error", onError);
|
||||
};
|
||||
this.on("done", onDone);
|
||||
this.on("error", onError);
|
||||
|
||||
(async () => {
|
||||
try {
|
||||
const url = this.buildWsUrl();
|
||||
const wsCtor = await getWebSocketCtor();
|
||||
if (!wsCtor) {
|
||||
this.pollLoop();
|
||||
return;
|
||||
}
|
||||
this.ws = new wsCtor(url, this.http.getApiKey()) as any;
|
||||
if (this.ws && "binaryType" in this.ws) {
|
||||
(this.ws as any).binaryType = "arraybuffer";
|
||||
}
|
||||
|
||||
if (this.ws) {
|
||||
this.attachWsHandlers(this.ws);
|
||||
}
|
||||
} catch (err) {
|
||||
this.pollLoop();
|
||||
}
|
||||
})();
|
||||
});
|
||||
}
|
||||
|
||||
private attachWsHandlers(ws: WebSocket) {
|
||||
let startTs = Date.now();
|
||||
const timeoutMs = this.timeout ? this.timeout * 1000 : undefined;
|
||||
ws.onmessage = (ev: MessageEvent) => {
|
||||
try {
|
||||
const raw = ensureUtf8String(ev.data);
|
||||
if (!raw) return;
|
||||
const body = JSON.parse(raw);
|
||||
const type = body.type as string | undefined;
|
||||
if (type === "error") {
|
||||
this.emit("error", { status: "failed", data: [], error: body.error, id: this.jobId });
|
||||
return;
|
||||
}
|
||||
if (type === "catchup") {
|
||||
const payload = body.data || {};
|
||||
this.emitDocuments(payload.data || []);
|
||||
this.emitSnapshot(payload);
|
||||
return;
|
||||
}
|
||||
if (type === "document") {
|
||||
const doc = body.data;
|
||||
if (doc) this.emitDocuments([doc]);
|
||||
return;
|
||||
}
|
||||
if (type === "done") {
|
||||
const payload = body.data || body;
|
||||
const data = (payload.data || []) as Document[];
|
||||
if (data.length) this.emitDocuments(data);
|
||||
this.emit("done", { status: "completed", data, id: this.jobId, total: payload.total, completed: payload.completed, creditsUsed: payload.creditsUsed });
|
||||
this.close();
|
||||
return;
|
||||
}
|
||||
const payload = body.data || body;
|
||||
if (payload && payload.status) this.emitSnapshot(payload);
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
if (timeoutMs && Date.now() - startTs > timeoutMs) {
|
||||
this.emit("error", { status: "failed", data: [], error: "Watcher timeout", id: this.jobId });
|
||||
this.close();
|
||||
}
|
||||
};
|
||||
ws.onerror = () => {
|
||||
this.emit("error", { status: "failed", data: [], error: "WebSocket error", id: this.jobId });
|
||||
this.close();
|
||||
};
|
||||
ws.onclose = () => {
|
||||
if (!this.closed) this.pollLoop();
|
||||
};
|
||||
}
|
||||
|
||||
private documentKey(doc: Document): string {
|
||||
if (doc && typeof doc === "object") {
|
||||
const explicitId = (doc as any).id ?? (doc as any).docId ?? (doc as any).url;
|
||||
if (typeof explicitId === "string" && explicitId.length) {
|
||||
return explicitId;
|
||||
}
|
||||
}
|
||||
try {
|
||||
return JSON.stringify(doc);
|
||||
} catch {
|
||||
return `${Date.now()}-${Math.random()}`;
|
||||
}
|
||||
}
|
||||
|
||||
private emitDocuments(docs: Document[]) {
|
||||
for (const doc of docs) {
|
||||
if (!doc) continue;
|
||||
const key = this.documentKey(doc);
|
||||
if (this.emittedDocumentKeys.has(key)) continue;
|
||||
this.emittedDocumentKeys.add(key);
|
||||
this.emit("document", { ...(doc as any), id: this.jobId });
|
||||
}
|
||||
}
|
||||
|
||||
private emitSnapshot(payload: any) {
|
||||
const status = payload.status as Snapshot["status"];
|
||||
const data = (payload.data || []) as Document[];
|
||||
const snap: Snapshot = this.kind === "crawl"
|
||||
? {
|
||||
id: this.jobId,
|
||||
status,
|
||||
completed: payload.completed ?? 0,
|
||||
total: payload.total ?? 0,
|
||||
creditsUsed: payload.creditsUsed,
|
||||
expiresAt: payload.expiresAt,
|
||||
next: payload.next ?? null,
|
||||
data,
|
||||
}
|
||||
: {
|
||||
id: this.jobId,
|
||||
status,
|
||||
completed: payload.completed ?? 0,
|
||||
total: payload.total ?? 0,
|
||||
creditsUsed: payload.creditsUsed,
|
||||
expiresAt: payload.expiresAt,
|
||||
next: payload.next ?? null,
|
||||
data,
|
||||
};
|
||||
this.emit("snapshot", snap);
|
||||
if (["completed", "failed", "cancelled"].includes(status)) {
|
||||
this.emit("done", { status, data, id: this.jobId, total: payload.total ?? 0, completed: payload.completed ?? 0, creditsUsed: payload.creditsUsed });
|
||||
this.close();
|
||||
}
|
||||
}
|
||||
|
||||
private async pollLoop() {
|
||||
const startTs = Date.now();
|
||||
const timeoutMs = this.timeout ? this.timeout * 1000 : undefined;
|
||||
while (!this.closed) {
|
||||
try {
|
||||
const snap = this.kind === "crawl"
|
||||
? await getCrawlStatus(this.http as any, this.jobId)
|
||||
: await getBatchScrapeStatus(this.http as any, this.jobId);
|
||||
this.emitDocuments((snap.data || []) as Document[]);
|
||||
this.emit("snapshot", snap);
|
||||
if (["completed", "failed", "cancelled"].includes(snap.status)) {
|
||||
this.emit("done", { status: snap.status, data: snap.data, id: this.jobId, total: (snap as any).total ?? 0, completed: (snap as any).completed ?? 0, creditsUsed: (snap as any).creditsUsed });
|
||||
this.close();
|
||||
break;
|
||||
}
|
||||
} catch {
|
||||
// ignore polling errors
|
||||
}
|
||||
if (timeoutMs && Date.now() - startTs > timeoutMs) {
|
||||
this.emit("error", { status: "failed", data: [], error: "Watcher timeout", id: this.jobId });
|
||||
this.close();
|
||||
break;
|
||||
}
|
||||
await new Promise((r) => setTimeout(r, Math.max(1000, this.pollInterval * 1000)));
|
||||
}
|
||||
}
|
||||
|
||||
close() {
|
||||
this.closed = true;
|
||||
if (this.ws && (this.ws as any).close) (this.ws as any).close();
|
||||
}
|
||||
}
|
||||
|
||||
26
참고/firecrawl-main/apps/js-sdk/firecrawl/tsconfig.json
Normal file
26
참고/firecrawl-main/apps/js-sdk/firecrawl/tsconfig.json
Normal file
@@ -0,0 +1,26 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
// See https://www.totaltypescript.com/tsconfig-cheat-sheet
|
||||
/* Base Options: */
|
||||
"esModuleInterop": true,
|
||||
"skipLibCheck": true,
|
||||
"target": "es2022",
|
||||
"allowJs": true,
|
||||
"resolveJsonModule": true,
|
||||
"moduleDetection": "force",
|
||||
"isolatedModules": true,
|
||||
"verbatimModuleSyntax": true,
|
||||
|
||||
/* Strictness */
|
||||
"strict": true,
|
||||
"noUncheckedIndexedAccess": true,
|
||||
"noImplicitOverride": true,
|
||||
|
||||
/* If NOT transpiling with TypeScript: */
|
||||
"module": "ESNext",
|
||||
"moduleResolution": "Bundler",
|
||||
"noEmit": true,
|
||||
},
|
||||
"include": ["src/**/*"],
|
||||
"exclude": ["node_modules", "dist", "**/__tests__/*"]
|
||||
}
|
||||
18
참고/firecrawl-main/apps/js-sdk/firecrawl/tsup.config.ts
Normal file
18
참고/firecrawl-main/apps/js-sdk/firecrawl/tsup.config.ts
Normal file
@@ -0,0 +1,18 @@
|
||||
import { defineConfig } from "tsup";
|
||||
|
||||
export default defineConfig({
|
||||
entryPoints: ["src/index.ts"],
|
||||
format: ["cjs", "esm"],
|
||||
dts: true,
|
||||
outDir: "dist",
|
||||
clean: true,
|
||||
platform: "node",
|
||||
target: "node22",
|
||||
noExternal: ["typescript-event-target"],
|
||||
esbuildOptions(options) {
|
||||
options.define = {
|
||||
...options.define,
|
||||
"process.env.NODE_ENV": JSON.stringify(process.env.NODE_ENV || "production"),
|
||||
};
|
||||
},
|
||||
});
|
||||
Reference in New Issue
Block a user