참고소스 수정본

This commit is contained in:
LASTA_DEV01\lasta
2026-05-12 19:40:31 +09:00
parent 0f34a451fc
commit 2e9204243d
8708 changed files with 3259488 additions and 869 deletions

View File

@@ -0,0 +1,287 @@
package com.firecrawl.client;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.DeserializationFeature;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.datatype.jdk8.Jdk8Module;
import com.firecrawl.errors.AuthenticationException;
import com.firecrawl.errors.FirecrawlException;
import com.firecrawl.errors.RateLimitException;
import okhttp3.*;
import java.io.IOException;
import java.util.Collections;
import java.util.Map;
import java.util.concurrent.TimeUnit;
/**
* Internal HTTP client for making authenticated requests to the Firecrawl API.
* Handles retry logic with exponential backoff.
*/
class FirecrawlHttpClient {
private static final MediaType JSON = MediaType.get("application/json; charset=utf-8");
private final OkHttpClient httpClient;
private final String apiKey;
private final String baseUrl;
private final int maxRetries;
private final double backoffFactor;
final ObjectMapper objectMapper;
FirecrawlHttpClient(String apiKey, String baseUrl, long timeoutMs, int maxRetries, double backoffFactor) {
this(apiKey, baseUrl, timeoutMs, maxRetries, backoffFactor, null);
}
FirecrawlHttpClient(String apiKey, String baseUrl, long timeoutMs, int maxRetries, double backoffFactor,
OkHttpClient httpClient) {
this.apiKey = apiKey;
this.baseUrl = baseUrl.endsWith("/") ? baseUrl.substring(0, baseUrl.length() - 1) : baseUrl;
this.maxRetries = maxRetries;
this.backoffFactor = backoffFactor;
if (httpClient != null) {
this.httpClient = httpClient;
} else {
this.httpClient = new OkHttpClient.Builder()
.connectTimeout(timeoutMs, TimeUnit.MILLISECONDS)
.readTimeout(timeoutMs, TimeUnit.MILLISECONDS)
.writeTimeout(timeoutMs, TimeUnit.MILLISECONDS)
.build();
}
this.objectMapper = new ObjectMapper()
.registerModule(new Jdk8Module())
.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
}
/**
* Sends a POST request with JSON body.
*/
<T> T post(String path, Object body, Class<T> responseType) {
return post(path, body, responseType, Collections.emptyMap());
}
/**
* Sends a POST request with JSON body and extra headers.
*/
<T> T post(String path, Object body, Class<T> responseType, Map<String, String> extraHeaders) {
String url = baseUrl + path;
String json;
try {
json = objectMapper.writeValueAsString(body);
} catch (JsonProcessingException e) {
throw new FirecrawlException("Failed to serialize request body", e);
}
RequestBody requestBody = RequestBody.create(json, JSON);
Request.Builder builder = new Request.Builder()
.url(url)
.header("Authorization", "Bearer " + apiKey)
.header("Content-Type", "application/json")
.post(requestBody);
for (Map.Entry<String, String> entry : extraHeaders.entrySet()) {
builder.header(entry.getKey(), entry.getValue());
}
Request request = builder.build();
return executeWithRetry(request, responseType);
}
/**
* Sends a PATCH request with JSON body.
*/
<T> T patch(String path, Object body, Class<T> responseType) {
String url = baseUrl + path;
String json;
try {
json = objectMapper.writeValueAsString(body);
} catch (JsonProcessingException e) {
throw new FirecrawlException("Failed to serialize request body", e);
}
RequestBody requestBody = RequestBody.create(json, JSON);
Request request = new Request.Builder()
.url(url)
.header("Authorization", "Bearer " + apiKey)
.header("Content-Type", "application/json")
.patch(requestBody)
.build();
return executeWithRetry(request, responseType);
}
/**
* Sends a POST multipart/form-data request.
*/
<T> T postMultipart(
String path,
Map<String, String> fields,
String fileFieldName,
byte[] fileContent,
String filename,
String contentType,
Class<T> responseType
) {
String url = baseUrl + path;
MultipartBody.Builder multipart = new MultipartBody.Builder()
.setType(MultipartBody.FORM);
for (Map.Entry<String, String> entry : fields.entrySet()) {
multipart.addFormDataPart(entry.getKey(), entry.getValue());
}
MediaType mediaType;
if (contentType != null && !contentType.isBlank()) {
try {
mediaType = MediaType.get(contentType);
} catch (IllegalArgumentException ignored) {
mediaType = MediaType.get("application/octet-stream");
}
} else {
mediaType = MediaType.get("application/octet-stream");
}
RequestBody fileBody = RequestBody.create(fileContent, mediaType);
multipart.addFormDataPart(fileFieldName, filename, fileBody);
Request request = new Request.Builder()
.url(url)
.header("Authorization", "Bearer " + apiKey)
.post(multipart.build())
.build();
return executeWithRetry(request, responseType);
}
/**
* Sends a GET request.
*/
<T> T get(String path, Class<T> responseType) {
String url = baseUrl + path;
Request request = new Request.Builder()
.url(url)
.header("Authorization", "Bearer " + apiKey)
.get()
.build();
return executeWithRetry(request, responseType);
}
/**
* Sends a GET request with full URL (for following next-page cursors).
*/
<T> T getAbsolute(String absoluteUrl, Class<T> responseType) {
Request request = new Request.Builder()
.url(absoluteUrl)
.header("Authorization", "Bearer " + apiKey)
.get()
.build();
return executeWithRetry(request, responseType);
}
/**
* Sends a DELETE request.
*/
<T> T delete(String path, Class<T> responseType) {
String url = baseUrl + path;
Request request = new Request.Builder()
.url(url)
.header("Authorization", "Bearer " + apiKey)
.delete()
.build();
return executeWithRetry(request, responseType);
}
/**
* Sends a raw GET request and returns the response body as a parsed Map.
*/
@SuppressWarnings("unchecked")
Map<String, Object> getRaw(String path) {
return get(path, Map.class);
}
private <T> T executeWithRetry(Request request, Class<T> responseType) {
int attempt = 0;
while (true) {
try {
try (Response response = httpClient.newCall(request).execute()) {
ResponseBody responseBody = response.body();
String bodyStr = responseBody != null ? responseBody.string() : "";
if (response.isSuccessful()) {
if (responseType == Void.class || responseType == void.class) {
return null;
}
return objectMapper.readValue(bodyStr, responseType);
}
int code = response.code();
// Parse error details from response
String errorMessage = extractErrorMessage(bodyStr, code);
String errorCode = extractErrorCode(bodyStr);
// Non-retryable client errors
if (code == 401) {
throw new AuthenticationException(errorMessage, errorCode, null);
}
if (code == 429) {
throw new RateLimitException(errorMessage, errorCode, null);
}
if (code >= 400 && code < 500 && code != 408 && code != 409) {
throw new FirecrawlException(errorMessage, code, errorCode, null);
}
// Retryable errors: 408, 409, 502, 5xx
if (attempt < maxRetries) {
attempt++;
sleepWithBackoff(attempt);
continue;
}
throw new FirecrawlException(errorMessage, code, errorCode, null);
}
} catch (FirecrawlException e) {
throw e;
} catch (IOException e) {
if (attempt < maxRetries) {
attempt++;
sleepWithBackoff(attempt);
continue;
}
throw new FirecrawlException("Request failed: " + e.getMessage(), e);
}
}
}
@SuppressWarnings("unchecked")
private String extractErrorMessage(String body, int statusCode) {
try {
Map<String, Object> parsed = objectMapper.readValue(body, Map.class);
if (parsed.containsKey("error")) {
return String.valueOf(parsed.get("error"));
}
if (parsed.containsKey("message")) {
return String.valueOf(parsed.get("message"));
}
} catch (Exception ignored) {
}
return "HTTP " + statusCode + " error";
}
@SuppressWarnings("unchecked")
private String extractErrorCode(String body) {
try {
Map<String, Object> parsed = objectMapper.readValue(body, Map.class);
Object code = parsed.get("code");
return code != null ? String.valueOf(code) : null;
} catch (Exception ignored) {
}
return null;
}
private void sleepWithBackoff(int attempt) {
long delayMs = (long) (backoffFactor * 1000 * Math.pow(2, attempt - 1));
try {
Thread.sleep(delayMs);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
throw new FirecrawlException("Request interrupted during retry backoff", e);
}
}
}

View File

@@ -0,0 +1,15 @@
package com.firecrawl.errors;
/**
* Thrown when the API returns a 401 Unauthorized response.
*/
public class AuthenticationException extends FirecrawlException {
public AuthenticationException(String message) {
super(message, 401);
}
public AuthenticationException(String message, String errorCode, Object details) {
super(message, 401, errorCode, details);
}
}

View File

@@ -0,0 +1,42 @@
package com.firecrawl.errors;
/**
* Base exception for all Firecrawl SDK errors.
*/
public class FirecrawlException extends RuntimeException {
private final int statusCode;
private final String errorCode;
private final Object details;
public FirecrawlException(String message) {
this(message, 0, null, null);
}
public FirecrawlException(String message, int statusCode) {
this(message, statusCode, null, null);
}
public FirecrawlException(String message, int statusCode, String errorCode, Object details) {
super(message);
this.statusCode = statusCode;
this.errorCode = errorCode;
this.details = details;
}
public FirecrawlException(String message, Throwable cause) {
super(message, cause);
this.statusCode = 0;
this.errorCode = null;
this.details = null;
}
/** HTTP status code (0 if not an HTTP error). */
public int getStatusCode() { return statusCode; }
/** Error code from the API response, if any. */
public String getErrorCode() { return errorCode; }
/** Additional error details from the API response, if any. */
public Object getDetails() { return details; }
}

View File

@@ -0,0 +1,22 @@
package com.firecrawl.errors;
/**
* Thrown when an async job (crawl, batch, agent) does not complete within the specified timeout.
*/
public class JobTimeoutException extends FirecrawlException {
private final String jobId;
private final int timeoutSeconds;
public JobTimeoutException(String jobId, int timeoutSeconds, String jobType) {
super(jobType + " job " + jobId + " did not complete within " + timeoutSeconds + " seconds");
this.jobId = jobId;
this.timeoutSeconds = timeoutSeconds;
}
/** The ID of the timed-out job. */
public String getJobId() { return jobId; }
/** The timeout in seconds that was exceeded. */
public int getTimeoutSeconds() { return timeoutSeconds; }
}

View File

@@ -0,0 +1,15 @@
package com.firecrawl.errors;
/**
* Thrown when the API returns a 429 Too Many Requests response.
*/
public class RateLimitException extends FirecrawlException {
public RateLimitException(String message) {
super(message, 429);
}
public RateLimitException(String message, String errorCode, Object details) {
super(message, 429, errorCode, details);
}
}

View File

@@ -0,0 +1,80 @@
package com.firecrawl.models;
import com.fasterxml.jackson.annotation.JsonInclude;
import java.util.List;
import java.util.Map;
/**
* Options for starting an agent task.
*/
@JsonInclude(JsonInclude.Include.NON_NULL)
public class AgentOptions {
private List<String> urls;
private String prompt;
private Map<String, Object> schema;
private String integration;
private Integer maxCredits;
private Boolean strictConstrainToURLs;
private String model;
private WebhookConfig webhook;
private AgentOptions() {}
public List<String> getUrls() { return urls; }
public String getPrompt() { return prompt; }
public Map<String, Object> getSchema() { return schema; }
public String getIntegration() { return integration; }
public Integer getMaxCredits() { return maxCredits; }
public Boolean getStrictConstrainToURLs() { return strictConstrainToURLs; }
public String getModel() { return model; }
public WebhookConfig getWebhook() { return webhook; }
public static Builder builder() { return new Builder(); }
public static final class Builder {
private List<String> urls;
private String prompt;
private Map<String, Object> schema;
private String integration;
private Integer maxCredits;
private Boolean strictConstrainToURLs;
private String model;
private WebhookConfig webhook;
private Builder() {}
/** Optional URLs to constrain the agent to. */
public Builder urls(List<String> urls) { this.urls = urls; return this; }
/** Natural language prompt describing what data to find. */
public Builder prompt(String prompt) { this.prompt = prompt; return this; }
/** JSON Schema for structured output. */
public Builder schema(Map<String, Object> schema) { this.schema = schema; return this; }
/** Integration identifier. */
public Builder integration(String integration) { this.integration = integration; return this; }
/** Maximum credits to spend. */
public Builder maxCredits(Integer maxCredits) { this.maxCredits = maxCredits; return this; }
/** Don't navigate outside provided URLs. */
public Builder strictConstrainToURLs(Boolean strictConstrainToURLs) { this.strictConstrainToURLs = strictConstrainToURLs; return this; }
/** Agent model: "spark-1-pro" or "spark-1-mini". */
public Builder model(String model) { this.model = model; return this; }
/** Webhook configuration. */
public Builder webhook(WebhookConfig webhook) { this.webhook = webhook; return this; }
public AgentOptions build() {
if (prompt == null || prompt.isEmpty()) {
throw new IllegalArgumentException("Agent prompt is required");
}
AgentOptions o = new AgentOptions();
o.urls = this.urls;
o.prompt = this.prompt;
o.schema = this.schema;
o.integration = this.integration;
o.maxCredits = this.maxCredits;
o.strictConstrainToURLs = this.strictConstrainToURLs;
o.model = this.model;
o.webhook = this.webhook;
return o;
}
}
}

View File

@@ -0,0 +1,23 @@
package com.firecrawl.models;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
/**
* Response from starting an agent task.
*/
@JsonIgnoreProperties(ignoreUnknown = true)
public class AgentResponse {
private boolean success;
private String id;
private String error;
public boolean isSuccess() { return success; }
public String getId() { return id; }
public String getError() { return error; }
@Override
public String toString() {
return "AgentResponse{success=" + success + ", id=" + id + "}";
}
}

View File

@@ -0,0 +1,35 @@
package com.firecrawl.models;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
/**
* Status response for an agent task.
*/
@JsonIgnoreProperties(ignoreUnknown = true)
public class AgentStatusResponse {
private boolean success;
private String status;
private String error;
private Object data;
private String model;
private String expiresAt;
private Integer creditsUsed;
public boolean isSuccess() { return success; }
public String getStatus() { return status; }
public String getError() { return error; }
public Object getData() { return data; }
public String getModel() { return model; }
public String getExpiresAt() { return expiresAt; }
public Integer getCreditsUsed() { return creditsUsed; }
public boolean isDone() {
return "completed".equals(status) || "failed".equals(status) || "cancelled".equals(status);
}
@Override
public String toString() {
return "AgentStatusResponse{status=" + status + ", model=" + model + "}";
}
}

View File

@@ -0,0 +1,39 @@
package com.firecrawl.models;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import java.util.List;
/**
* Status and results of a batch scrape job.
*/
@JsonIgnoreProperties(ignoreUnknown = true)
public class BatchScrapeJob {
private String id;
private String status;
private int completed;
private int total;
private Integer creditsUsed;
private String expiresAt;
private String next;
private List<Document> data;
public String getId() { return id; }
public String getStatus() { return status; }
public int getCompleted() { return completed; }
public int getTotal() { return total; }
public Integer getCreditsUsed() { return creditsUsed; }
public String getExpiresAt() { return expiresAt; }
public String getNext() { return next; }
public List<Document> getData() { return data; }
public void setData(List<Document> data) { this.data = data; }
public boolean isDone() {
return "completed".equals(status) || "failed".equals(status) || "cancelled".equals(status);
}
@Override
public String toString() {
return "BatchScrapeJob{id=" + id + ", status=" + status + ", completed=" + completed + "/" + total + "}";
}
}

View File

@@ -0,0 +1,78 @@
package com.firecrawl.models;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonInclude;
/**
* Options for a batch scrape job.
*/
@JsonInclude(JsonInclude.Include.NON_NULL)
public class BatchScrapeOptions {
private ScrapeOptions options;
private Object webhook;
private String appendToId;
private Boolean ignoreInvalidURLs;
private Integer maxConcurrency;
private Boolean zeroDataRetention;
@JsonIgnore
private String idempotencyKey;
private String integration;
private BatchScrapeOptions() {}
public ScrapeOptions getOptions() { return options; }
public Object getWebhook() { return webhook; }
public String getAppendToId() { return appendToId; }
public Boolean getIgnoreInvalidURLs() { return ignoreInvalidURLs; }
public Integer getMaxConcurrency() { return maxConcurrency; }
public Boolean getZeroDataRetention() { return zeroDataRetention; }
@JsonIgnore
public String getIdempotencyKey() { return idempotencyKey; }
public String getIntegration() { return integration; }
public static Builder builder() { return new Builder(); }
public static final class Builder {
private ScrapeOptions options;
private Object webhook;
private String appendToId;
private Boolean ignoreInvalidURLs;
private Integer maxConcurrency;
private Boolean zeroDataRetention;
private String idempotencyKey;
private String integration;
private Builder() {}
/** Scrape options applied to each URL. */
public Builder options(ScrapeOptions options) { this.options = options; return this; }
/** Webhook URL string or {@link WebhookConfig} object. */
public Builder webhook(Object webhook) { this.webhook = webhook; return this; }
/** Append URLs to an existing batch job. */
public Builder appendToId(String appendToId) { this.appendToId = appendToId; return this; }
/** Ignore invalid URLs instead of failing. */
public Builder ignoreInvalidURLs(Boolean ignoreInvalidURLs) { this.ignoreInvalidURLs = ignoreInvalidURLs; return this; }
/** Max concurrent scrapes. */
public Builder maxConcurrency(Integer maxConcurrency) { this.maxConcurrency = maxConcurrency; return this; }
/** Do not store any data on Firecrawl servers. */
public Builder zeroDataRetention(Boolean zeroDataRetention) { this.zeroDataRetention = zeroDataRetention; return this; }
/** Idempotency key to prevent duplicate batch jobs. */
public Builder idempotencyKey(String idempotencyKey) { this.idempotencyKey = idempotencyKey; return this; }
/** Integration identifier. */
public Builder integration(String integration) { this.integration = integration; return this; }
public BatchScrapeOptions build() {
BatchScrapeOptions o = new BatchScrapeOptions();
o.options = this.options;
o.webhook = this.webhook;
o.appendToId = this.appendToId;
o.ignoreInvalidURLs = this.ignoreInvalidURLs;
o.maxConcurrency = this.maxConcurrency;
o.zeroDataRetention = this.zeroDataRetention;
o.idempotencyKey = this.idempotencyKey;
o.integration = this.integration;
return o;
}
}
}

View File

@@ -0,0 +1,24 @@
package com.firecrawl.models;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import java.util.List;
/**
* Response from starting an async batch scrape job.
*/
@JsonIgnoreProperties(ignoreUnknown = true)
public class BatchScrapeResponse {
private String id;
private String url;
private List<String> invalidURLs;
public String getId() { return id; }
public String getUrl() { return url; }
public List<String> getInvalidURLs() { return invalidURLs; }
@Override
public String toString() {
return "BatchScrapeResponse{id=" + id + ", url=" + url + "}";
}
}

View File

@@ -0,0 +1,29 @@
package com.firecrawl.models;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
/**
* Response from creating a new browser session.
*/
@JsonIgnoreProperties(ignoreUnknown = true)
public class BrowserCreateResponse {
private boolean success;
private String id;
private String cdpUrl;
private String liveViewUrl;
private String expiresAt;
private String error;
public boolean isSuccess() { return success; }
public String getId() { return id; }
public String getCdpUrl() { return cdpUrl; }
public String getLiveViewUrl() { return liveViewUrl; }
public String getExpiresAt() { return expiresAt; }
public String getError() { return error; }
@Override
public String toString() {
return "BrowserCreateResponse{id=" + id + ", success=" + success + "}";
}
}

View File

@@ -0,0 +1,25 @@
package com.firecrawl.models;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
/**
* Response from deleting a browser session.
*/
@JsonIgnoreProperties(ignoreUnknown = true)
public class BrowserDeleteResponse {
private boolean success;
private Long sessionDurationMs;
private Integer creditsBilled;
private String error;
public boolean isSuccess() { return success; }
public Long getSessionDurationMs() { return sessionDurationMs; }
public Integer getCreditsBilled() { return creditsBilled; }
public String getError() { return error; }
@Override
public String toString() {
return "BrowserDeleteResponse{success=" + success + ", creditsBilled=" + creditsBilled + "}";
}
}

View File

@@ -0,0 +1,31 @@
package com.firecrawl.models;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
/**
* Response from executing code in a browser session.
*/
@JsonIgnoreProperties(ignoreUnknown = true)
public class BrowserExecuteResponse {
private boolean success;
private String stdout;
private String result;
private String stderr;
private Integer exitCode;
private Boolean killed;
private String error;
public boolean isSuccess() { return success; }
public String getStdout() { return stdout; }
public String getResult() { return result; }
public String getStderr() { return stderr; }
public Integer getExitCode() { return exitCode; }
public Boolean getKilled() { return killed; }
public String getError() { return error; }
@Override
public String toString() {
return "BrowserExecuteResponse{success=" + success + ", exitCode=" + exitCode + "}";
}
}

View File

@@ -0,0 +1,25 @@
package com.firecrawl.models;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import java.util.List;
/**
* Response from listing browser sessions.
*/
@JsonIgnoreProperties(ignoreUnknown = true)
public class BrowserListResponse {
private boolean success;
private List<BrowserSession> sessions;
private String error;
public boolean isSuccess() { return success; }
public List<BrowserSession> getSessions() { return sessions; }
public String getError() { return error; }
@Override
public String toString() {
int count = sessions != null ? sessions.size() : 0;
return "BrowserListResponse{success=" + success + ", sessions=" + count + "}";
}
}

View File

@@ -0,0 +1,31 @@
package com.firecrawl.models;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
/**
* Represents a browser session's metadata.
*/
@JsonIgnoreProperties(ignoreUnknown = true)
public class BrowserSession {
private String id;
private String status;
private String cdpUrl;
private String liveViewUrl;
private boolean streamWebView;
private String createdAt;
private String lastActivity;
public String getId() { return id; }
public String getStatus() { return status; }
public String getCdpUrl() { return cdpUrl; }
public String getLiveViewUrl() { return liveViewUrl; }
public boolean isStreamWebView() { return streamWebView; }
public String getCreatedAt() { return createdAt; }
public String getLastActivity() { return lastActivity; }
@Override
public String toString() {
return "BrowserSession{id=" + id + ", status=" + status + "}";
}
}

View File

@@ -0,0 +1,21 @@
package com.firecrawl.models;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
/**
* Current concurrency usage.
*/
@JsonIgnoreProperties(ignoreUnknown = true)
public class ConcurrencyCheck {
private int concurrency;
private int maxConcurrency;
public int getConcurrency() { return concurrency; }
public int getMaxConcurrency() { return maxConcurrency; }
@Override
public String toString() {
return "ConcurrencyCheck{concurrency=" + concurrency + "/" + maxConcurrency + "}";
}
}

View File

@@ -0,0 +1,40 @@
package com.firecrawl.models;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import java.util.List;
/**
* Status and results of a crawl job.
*/
@JsonIgnoreProperties(ignoreUnknown = true)
public class CrawlJob {
private String id;
private String status;
private int total;
private int completed;
private Integer creditsUsed;
private String expiresAt;
private String next;
private List<Document> data;
public String getId() { return id; }
public String getStatus() { return status; }
public int getTotal() { return total; }
public int getCompleted() { return completed; }
public Integer getCreditsUsed() { return creditsUsed; }
public String getExpiresAt() { return expiresAt; }
public String getNext() { return next; }
public List<Document> getData() { return data; }
public void setData(List<Document> data) { this.data = data; }
/** Returns true if the job has finished (completed, failed, or cancelled). */
public boolean isDone() {
return "completed".equals(status) || "failed".equals(status) || "cancelled".equals(status);
}
@Override
public String toString() {
return "CrawlJob{id=" + id + ", status=" + status + ", completed=" + completed + "/" + total + "}";
}
}

View File

@@ -0,0 +1,168 @@
package com.firecrawl.models;
import com.fasterxml.jackson.annotation.JsonInclude;
import java.util.List;
import java.util.Map;
/**
* Options for crawling a website.
*/
@JsonInclude(JsonInclude.Include.NON_NULL)
public class CrawlOptions {
private String prompt;
private List<String> excludePaths;
private List<String> includePaths;
private Integer maxDiscoveryDepth;
private String sitemap;
private Boolean ignoreQueryParameters;
private Boolean deduplicateSimilarURLs;
private Integer limit;
private Boolean crawlEntireDomain;
private Boolean allowExternalLinks;
private Boolean allowSubdomains;
private Boolean ignoreRobotsTxt;
private String robotsUserAgent;
private Integer delay;
private Integer maxConcurrency;
private Object webhook;
private ScrapeOptions scrapeOptions;
private Boolean regexOnFullURL;
private Boolean zeroDataRetention;
private String integration;
private CrawlOptions() {}
public String getPrompt() { return prompt; }
public List<String> getExcludePaths() { return excludePaths; }
public List<String> getIncludePaths() { return includePaths; }
public Integer getMaxDiscoveryDepth() { return maxDiscoveryDepth; }
public String getSitemap() { return sitemap; }
public Boolean getIgnoreQueryParameters() { return ignoreQueryParameters; }
public Boolean getDeduplicateSimilarURLs() { return deduplicateSimilarURLs; }
public Integer getLimit() { return limit; }
public Boolean getCrawlEntireDomain() { return crawlEntireDomain; }
public Boolean getAllowExternalLinks() { return allowExternalLinks; }
public Boolean getAllowSubdomains() { return allowSubdomains; }
public Boolean getIgnoreRobotsTxt() { return ignoreRobotsTxt; }
public String getRobotsUserAgent() { return robotsUserAgent; }
public Integer getDelay() { return delay; }
public Integer getMaxConcurrency() { return maxConcurrency; }
public Object getWebhook() { return webhook; }
public ScrapeOptions getScrapeOptions() { return scrapeOptions; }
public Boolean getRegexOnFullURL() { return regexOnFullURL; }
public Boolean getZeroDataRetention() { return zeroDataRetention; }
public String getIntegration() { return integration; }
public static Builder builder() { return new Builder(); }
public static final class Builder {
private String prompt;
private List<String> excludePaths;
private List<String> includePaths;
private Integer maxDiscoveryDepth;
private String sitemap;
private Boolean ignoreQueryParameters;
private Boolean deduplicateSimilarURLs;
private Integer limit;
private Boolean crawlEntireDomain;
private Boolean allowExternalLinks;
private Boolean allowSubdomains;
private Boolean ignoreRobotsTxt;
private String robotsUserAgent;
private Integer delay;
private Integer maxConcurrency;
private Object webhook;
private ScrapeOptions scrapeOptions;
private Boolean regexOnFullURL;
private Boolean zeroDataRetention;
private String integration;
private Builder() {}
/** Natural language prompt to guide crawling. */
public Builder prompt(String prompt) { this.prompt = prompt; return this; }
/** URL path patterns to exclude from crawling. */
public Builder excludePaths(List<String> excludePaths) { this.excludePaths = excludePaths; return this; }
/** URL path patterns to include in crawling. */
public Builder includePaths(List<String> includePaths) { this.includePaths = includePaths; return this; }
/** Maximum depth to discover links. */
public Builder maxDiscoveryDepth(Integer maxDiscoveryDepth) { this.maxDiscoveryDepth = maxDiscoveryDepth; return this; }
/** Sitemap handling: "skip", "include", or "only". */
public Builder sitemap(String sitemap) { this.sitemap = sitemap; return this; }
/** Ignore query parameters when deduplicating URLs. */
public Builder ignoreQueryParameters(Boolean ignoreQueryParameters) { this.ignoreQueryParameters = ignoreQueryParameters; return this; }
/** Deduplicate URLs that are similar. */
public Builder deduplicateSimilarURLs(Boolean deduplicateSimilarURLs) { this.deduplicateSimilarURLs = deduplicateSimilarURLs; return this; }
/** Maximum number of pages to crawl. */
public Builder limit(Integer limit) { this.limit = limit; return this; }
/** Whether to crawl the entire domain. */
public Builder crawlEntireDomain(Boolean crawlEntireDomain) { this.crawlEntireDomain = crawlEntireDomain; return this; }
/** Follow external links. */
public Builder allowExternalLinks(Boolean allowExternalLinks) { this.allowExternalLinks = allowExternalLinks; return this; }
/** Follow subdomains. */
public Builder allowSubdomains(Boolean allowSubdomains) { this.allowSubdomains = allowSubdomains; return this; }
/** Ignore the website's robots.txt rules. Enterprise only. */
public Builder ignoreRobotsTxt(Boolean ignoreRobotsTxt) { this.ignoreRobotsTxt = ignoreRobotsTxt; return this; }
/** Custom User-Agent string for robots.txt evaluation. Enterprise only. */
public Builder robotsUserAgent(String robotsUserAgent) { this.robotsUserAgent = robotsUserAgent; return this; }
/** Delay in milliseconds between requests. */
public Builder delay(Integer delay) { this.delay = delay; return this; }
/** Maximum concurrent requests. */
public Builder maxConcurrency(Integer maxConcurrency) { this.maxConcurrency = maxConcurrency; return this; }
/** Webhook URL string or {@link WebhookConfig} object. */
public Builder webhook(Object webhook) { this.webhook = webhook; return this; }
/** Scrape options applied to each crawled page. */
public Builder scrapeOptions(ScrapeOptions scrapeOptions) { this.scrapeOptions = scrapeOptions; return this; }
/** Apply regex patterns to the full URL, not just the path. */
public Builder regexOnFullURL(Boolean regexOnFullURL) { this.regexOnFullURL = regexOnFullURL; return this; }
/** Do not store any scraped data on Firecrawl servers. */
public Builder zeroDataRetention(Boolean zeroDataRetention) { this.zeroDataRetention = zeroDataRetention; return this; }
/** Integration identifier. */
public Builder integration(String integration) { this.integration = integration; return this; }
public CrawlOptions build() {
CrawlOptions o = new CrawlOptions();
o.prompt = this.prompt;
o.excludePaths = this.excludePaths;
o.includePaths = this.includePaths;
o.maxDiscoveryDepth = this.maxDiscoveryDepth;
o.sitemap = this.sitemap;
o.ignoreQueryParameters = this.ignoreQueryParameters;
o.deduplicateSimilarURLs = this.deduplicateSimilarURLs;
o.limit = this.limit;
o.crawlEntireDomain = this.crawlEntireDomain;
o.allowExternalLinks = this.allowExternalLinks;
o.allowSubdomains = this.allowSubdomains;
o.ignoreRobotsTxt = this.ignoreRobotsTxt;
o.robotsUserAgent = this.robotsUserAgent;
o.delay = this.delay;
o.maxConcurrency = this.maxConcurrency;
o.webhook = this.webhook;
o.scrapeOptions = this.scrapeOptions;
o.regexOnFullURL = this.regexOnFullURL;
o.zeroDataRetention = this.zeroDataRetention;
o.integration = this.integration;
return o;
}
}
}

View File

@@ -0,0 +1,21 @@
package com.firecrawl.models;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
/**
* Response from starting an async crawl job.
*/
@JsonIgnoreProperties(ignoreUnknown = true)
public class CrawlResponse {
private String id;
private String url;
public String getId() { return id; }
public String getUrl() { return url; }
@Override
public String toString() {
return "CrawlResponse{id=" + id + ", url=" + url + "}";
}
}

View File

@@ -0,0 +1,25 @@
package com.firecrawl.models;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
/**
* Current credit usage information.
*/
@JsonIgnoreProperties(ignoreUnknown = true)
public class CreditUsage {
private int remainingCredits;
private Integer planCredits;
private String billingPeriodStart;
private String billingPeriodEnd;
public int getRemainingCredits() { return remainingCredits; }
public Integer getPlanCredits() { return planCredits; }
public String getBillingPeriodStart() { return billingPeriodStart; }
public String getBillingPeriodEnd() { return billingPeriodEnd; }
@Override
public String toString() {
return "CreditUsage{remaining=" + remainingCredits + ", plan=" + planCredits + "}";
}
}

View File

@@ -0,0 +1,55 @@
package com.firecrawl.models;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import java.util.List;
import java.util.Map;
/**
* A scraped document returned by scrape, crawl, and batch endpoints.
*/
@JsonIgnoreProperties(ignoreUnknown = true)
public class Document {
private String markdown;
private String html;
private String rawHtml;
private Object json;
private String summary;
private Map<String, Object> metadata;
private List<String> links;
private List<String> images;
private String screenshot;
private String audio;
private List<Map<String, Object>> attributes;
private Map<String, Object> actions;
private String answer;
private String highlights;
private String warning;
private Map<String, Object> changeTracking;
private Map<String, Object> branding;
public String getMarkdown() { return markdown; }
public String getHtml() { return html; }
public String getRawHtml() { return rawHtml; }
public Object getJson() { return json; }
public String getSummary() { return summary; }
public Map<String, Object> getMetadata() { return metadata; }
public List<String> getLinks() { return links; }
public List<String> getImages() { return images; }
public String getScreenshot() { return screenshot; }
public String getAudio() { return audio; }
public List<Map<String, Object>> getAttributes() { return attributes; }
public Map<String, Object> getActions() { return actions; }
public String getAnswer() { return answer; }
public String getHighlights() { return highlights; }
public String getWarning() { return warning; }
public Map<String, Object> getChangeTracking() { return changeTracking; }
public Map<String, Object> getBranding() { return branding; }
@Override
public String toString() {
String title = metadata != null ? String.valueOf(metadata.get("title")) : "untitled";
String url = metadata != null ? String.valueOf(metadata.get("sourceURL")) : "unknown";
return "Document{title=" + title + ", url=" + url + "}";
}
}

View File

@@ -0,0 +1,35 @@
package com.firecrawl.models;
import com.fasterxml.jackson.annotation.JsonInclude;
/**
* Highlights format for extracting direct highlights from page content.
*/
@JsonInclude(JsonInclude.Include.NON_NULL)
public class HighlightsFormat {
private final String type = "highlights";
private String query;
private HighlightsFormat() {}
public String getType() { return type; }
public String getQuery() { return query; }
public static Builder builder() { return new Builder(); }
public static final class Builder {
private String query;
private Builder() {}
/** Query used to select highlights from the page content. */
public Builder query(String query) { this.query = query; return this; }
public HighlightsFormat build() {
HighlightsFormat f = new HighlightsFormat();
f.query = this.query;
return f;
}
}
}

View File

@@ -0,0 +1,57 @@
package com.firecrawl.models;
import com.fasterxml.jackson.annotation.JsonInclude;
import java.util.Map;
/**
* JSON extraction format with optional schema and prompt.
*
* <p>Usage:
* <pre>{@code
* JsonFormat jsonFmt = JsonFormat.builder()
* .prompt("Extract the product name and price")
* .schema(Map.of(
* "type", "object",
* "properties", Map.of(
* "name", Map.of("type", "string"),
* "price", Map.of("type", "number")
* )
* ))
* .build();
* }</pre>
*/
@JsonInclude(JsonInclude.Include.NON_NULL)
public class JsonFormat {
private final String type = "json";
private String prompt;
private Map<String, Object> schema;
private JsonFormat() {}
public String getType() { return type; }
public String getPrompt() { return prompt; }
public Map<String, Object> getSchema() { return schema; }
public static Builder builder() { return new Builder(); }
public static final class Builder {
private String prompt;
private Map<String, Object> schema;
private Builder() {}
/** LLM prompt for extraction. */
public Builder prompt(String prompt) { this.prompt = prompt; return this; }
/** JSON Schema for structured extraction. */
public Builder schema(Map<String, Object> schema) { this.schema = schema; return this; }
public JsonFormat build() {
JsonFormat f = new JsonFormat();
f.prompt = this.prompt;
f.schema = this.schema;
return f;
}
}
}

View File

@@ -0,0 +1,38 @@
package com.firecrawl.models;
import com.fasterxml.jackson.annotation.JsonInclude;
import java.util.List;
/**
* Geolocation configuration for requests.
*/
@JsonInclude(JsonInclude.Include.NON_NULL)
public class LocationConfig {
private String country;
private List<String> languages;
private LocationConfig() {}
public String getCountry() { return country; }
public List<String> getLanguages() { return languages; }
public static Builder builder() { return new Builder(); }
public static final class Builder {
private String country;
private List<String> languages;
private Builder() {}
public Builder country(String country) { this.country = country; return this; }
public Builder languages(List<String> languages) { this.languages = languages; return this; }
public LocationConfig build() {
LocationConfig c = new LocationConfig();
c.country = this.country;
c.languages = this.languages;
return c;
}
}
}

View File

@@ -0,0 +1,53 @@
package com.firecrawl.models;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import java.util.ArrayList;
import java.util.Collections;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
/**
* Result of a map operation containing discovered URLs.
*
* <p>The v2 API may return {@code links} as either plain URL strings or
* objects with {@code url}, {@code title}, and {@code description} fields.
* This class normalises both representations into a uniform
* {@code List<Map<String, Object>>} where each entry always contains at
* least a {@code "url"} key.
*/
@JsonIgnoreProperties(ignoreUnknown = true)
public class MapData {
private List<Object> links;
/**
* Returns the discovered links, normalised so that every entry is a
* {@code Map<String, Object>} containing at least a {@code "url"} key.
* Plain-string entries returned by the API are wrapped as
* {@code {"url": "<value>"}}.
*/
@SuppressWarnings("unchecked")
public List<Map<String, Object>> getLinks() {
if (links == null) {
return null;
}
List<Map<String, Object>> result = new ArrayList<>(links.size());
for (Object item : links) {
if (item instanceof Map) {
result.add((Map<String, Object>) item);
} else if (item instanceof String) {
Map<String, Object> wrapped = new LinkedHashMap<>();
wrapped.put("url", item);
result.add(wrapped);
}
}
return Collections.unmodifiableList(result);
}
@Override
public String toString() {
int count = links != null ? links.size() : 0;
return "MapData{links=" + count + "}";
}
}

View File

@@ -0,0 +1,76 @@
package com.firecrawl.models;
import com.fasterxml.jackson.annotation.JsonInclude;
/**
* Options for mapping (discovering URLs on) a website.
*/
@JsonInclude(JsonInclude.Include.NON_NULL)
public class MapOptions {
private String search;
private String sitemap;
private Boolean includeSubdomains;
private Boolean ignoreQueryParameters;
private Integer limit;
private Integer timeout;
private String integration;
private LocationConfig location;
private MapOptions() {}
public String getSearch() { return search; }
/** Sitemap mode: "only", "include", or "skip". */
public String getSitemap() { return sitemap; }
public Boolean getIncludeSubdomains() { return includeSubdomains; }
public Boolean getIgnoreQueryParameters() { return ignoreQueryParameters; }
public Integer getLimit() { return limit; }
public Integer getTimeout() { return timeout; }
public String getIntegration() { return integration; }
public LocationConfig getLocation() { return location; }
public static Builder builder() { return new Builder(); }
public static final class Builder {
private String search;
private String sitemap;
private Boolean includeSubdomains;
private Boolean ignoreQueryParameters;
private Integer limit;
private Integer timeout;
private String integration;
private LocationConfig location;
private Builder() {}
/** Filter discovered URLs by keyword. */
public Builder search(String search) { this.search = search; return this; }
/** Sitemap mode: "only", "include", or "skip". */
public Builder sitemap(String sitemap) { this.sitemap = sitemap; return this; }
/** Include subdomains. */
public Builder includeSubdomains(Boolean includeSubdomains) { this.includeSubdomains = includeSubdomains; return this; }
/** Ignore query parameters when deduplicating URLs. */
public Builder ignoreQueryParameters(Boolean ignoreQueryParameters) { this.ignoreQueryParameters = ignoreQueryParameters; return this; }
/** Maximum number of URLs to return. */
public Builder limit(Integer limit) { this.limit = limit; return this; }
/** Timeout in milliseconds. */
public Builder timeout(Integer timeout) { this.timeout = timeout; return this; }
/** Integration identifier. */
public Builder integration(String integration) { this.integration = integration; return this; }
/** Geolocation configuration. */
public Builder location(LocationConfig location) { this.location = location; return this; }
public MapOptions build() {
MapOptions o = new MapOptions();
o.search = this.search;
o.sitemap = this.sitemap;
o.includeSubdomains = this.includeSubdomains;
o.ignoreQueryParameters = this.ignoreQueryParameters;
o.limit = this.limit;
o.timeout = this.timeout;
o.integration = this.integration;
o.location = this.location;
return o;
}
}
}

View File

@@ -0,0 +1,40 @@
package com.firecrawl.models;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import java.util.List;
import java.util.Map;
@JsonIgnoreProperties(ignoreUnknown = true)
public class Monitor {
private String id;
private String name;
private String status;
private MonitorSchedule schedule;
private String nextRunAt;
private String lastRunAt;
private String currentCheckId;
private List<Map<String, Object>> targets;
private Map<String, Object> webhook;
private Map<String, Object> notification;
private int retentionDays;
private Integer estimatedCreditsPerMonth;
private MonitorSummary lastCheckSummary;
private String createdAt;
private String updatedAt;
public String getId() { return id; }
public String getName() { return name; }
public String getStatus() { return status; }
public MonitorSchedule getSchedule() { return schedule; }
public String getNextRunAt() { return nextRunAt; }
public String getLastRunAt() { return lastRunAt; }
public String getCurrentCheckId() { return currentCheckId; }
public List<Map<String, Object>> getTargets() { return targets; }
public Map<String, Object> getWebhook() { return webhook; }
public Map<String, Object> getNotification() { return notification; }
public int getRetentionDays() { return retentionDays; }
public Integer getEstimatedCreditsPerMonth() { return estimatedCreditsPerMonth; }
public MonitorSummary getLastCheckSummary() { return lastCheckSummary; }
public String getCreatedAt() { return createdAt; }
public String getUpdatedAt() { return updatedAt; }
}

View File

@@ -0,0 +1,42 @@
package com.firecrawl.models;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
@JsonIgnoreProperties(ignoreUnknown = true)
public class MonitorCheck {
private String id;
private String monitorId;
private String status;
private String trigger;
private String scheduledFor;
private String startedAt;
private String finishedAt;
private Integer estimatedCredits;
private Integer reservedCredits;
private Integer actualCredits;
private String billingStatus;
private MonitorSummary summary;
private Object targetResults;
private Object notificationStatus;
private String error;
private String createdAt;
private String updatedAt;
public String getId() { return id; }
public String getMonitorId() { return monitorId; }
public String getStatus() { return status; }
public String getTrigger() { return trigger; }
public String getScheduledFor() { return scheduledFor; }
public String getStartedAt() { return startedAt; }
public String getFinishedAt() { return finishedAt; }
public Integer getEstimatedCredits() { return estimatedCredits; }
public Integer getReservedCredits() { return reservedCredits; }
public Integer getActualCredits() { return actualCredits; }
public String getBillingStatus() { return billingStatus; }
public MonitorSummary getSummary() { return summary; }
public Object getTargetResults() { return targetResults; }
public Object getNotificationStatus() { return notificationStatus; }
public String getError() { return error; }
public String getCreatedAt() { return createdAt; }
public String getUpdatedAt() { return updatedAt; }
}

View File

@@ -0,0 +1,15 @@
package com.firecrawl.models;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import java.util.List;
@JsonIgnoreProperties(ignoreUnknown = true)
public class MonitorCheckDetail extends MonitorCheck {
private List<MonitorCheckPage> pages;
private String next;
public List<MonitorCheckPage> getPages() { return pages; }
public void setPages(List<MonitorCheckPage> pages) { this.pages = pages; }
public String getNext() { return next; }
public void setNext(String next) { this.next = next; }
}

View File

@@ -0,0 +1,30 @@
package com.firecrawl.models;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
@JsonIgnoreProperties(ignoreUnknown = true)
public class MonitorCheckPage {
private String id;
private String targetId;
private String url;
private String status;
private String previousScrapeId;
private String currentScrapeId;
private Integer statusCode;
private String error;
private Object metadata;
private Object diff;
private String createdAt;
public String getId() { return id; }
public String getTargetId() { return targetId; }
public String getUrl() { return url; }
public String getStatus() { return status; }
public String getPreviousScrapeId() { return previousScrapeId; }
public String getCurrentScrapeId() { return currentScrapeId; }
public Integer getStatusCode() { return statusCode; }
public String getError() { return error; }
public Object getMetadata() { return metadata; }
public Object getDiff() { return diff; }
public String getCreatedAt() { return createdAt; }
}

View File

@@ -0,0 +1,12 @@
package com.firecrawl.models;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
@JsonIgnoreProperties(ignoreUnknown = true)
public class MonitorSchedule {
private String cron;
private String timezone;
public String getCron() { return cron; }
public String getTimezone() { return timezone; }
}

View File

@@ -0,0 +1,25 @@
package com.firecrawl.models;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
@JsonIgnoreProperties(ignoreUnknown = true)
public class MonitorSummary {
private int totalPages;
private int same;
private int changed;
private int newCount;
private int removed;
private int error;
public int getTotalPages() { return totalPages; }
public int getSame() { return same; }
public int getChanged() { return changed; }
public int getNew() { return newCount; }
public int getRemoved() { return removed; }
public int getError() { return error; }
@com.fasterxml.jackson.annotation.JsonProperty("new")
private void setNewCount(int value) {
this.newCount = value;
}
}

View File

@@ -0,0 +1,77 @@
package com.firecrawl.models;
import java.util.Arrays;
import java.util.Objects;
/**
* Binary upload payload for the v2 parse endpoint.
*/
public class ParseFile {
private final byte[] content;
private final String filename;
private final String contentType;
private ParseFile(byte[] content, String filename, String contentType) {
this.content = content;
this.filename = filename;
this.contentType = contentType;
}
public byte[] getContent() {
return Arrays.copyOf(content, content.length);
}
public String getFilename() {
return filename;
}
public String getContentType() {
return contentType;
}
public static Builder builder() {
return new Builder();
}
public static final class Builder {
private byte[] content;
private String filename;
private String contentType;
private Builder() {}
/** Raw file content bytes. */
public Builder content(byte[] content) {
this.content = content != null ? Arrays.copyOf(content, content.length) : null;
return this;
}
/** Uploaded filename (e.g., "document.pdf"). */
public Builder filename(String filename) {
this.filename = filename;
return this;
}
/** Optional MIME type hint (e.g., "application/pdf"). */
public Builder contentType(String contentType) {
this.contentType = contentType;
return this;
}
public ParseFile build() {
Objects.requireNonNull(content, "File content is required");
if (content.length == 0) {
throw new IllegalArgumentException("File content cannot be empty");
}
Objects.requireNonNull(filename, "Filename is required");
if (filename.isBlank()) {
throw new IllegalArgumentException("Filename cannot be blank");
}
return new ParseFile(
Arrays.copyOf(content, content.length),
filename.trim(),
contentType
);
}
}
}

View File

@@ -0,0 +1,155 @@
package com.firecrawl.models;
import com.fasterxml.jackson.annotation.JsonInclude;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
/**
* Options for parsing uploaded files via /v2/parse.
*
* <p>Parse does not support browser-rendering formats/options such as
* change tracking, screenshot, branding, actions, waitFor, location, or mobile.
*/
@JsonInclude(JsonInclude.Include.NON_NULL)
public class ParseOptions {
private List<Object> formats;
private Map<String, String> headers;
private List<String> includeTags;
private List<String> excludeTags;
private Boolean onlyMainContent;
private Integer timeout;
private List<Object> parsers;
private Boolean skipTlsVerification;
private Boolean removeBase64Images;
private Boolean blockAds;
private String proxy;
private String integration;
private ParseOptions() {}
public List<Object> getFormats() { return formats; }
public Map<String, String> getHeaders() { return headers; }
public List<String> getIncludeTags() { return includeTags; }
public List<String> getExcludeTags() { return excludeTags; }
public Boolean getOnlyMainContent() { return onlyMainContent; }
public Integer getTimeout() { return timeout; }
public List<Object> getParsers() { return parsers; }
public Boolean getSkipTlsVerification() { return skipTlsVerification; }
public Boolean getRemoveBase64Images() { return removeBase64Images; }
public Boolean getBlockAds() { return blockAds; }
public String getProxy() { return proxy; }
public String getIntegration() { return integration; }
public static Builder builder() { return new Builder(); }
public Builder toBuilder() {
Builder b = new Builder();
b.formats = this.formats != null ? new ArrayList<>(this.formats) : null;
b.headers = this.headers != null ? new HashMap<>(this.headers) : null;
b.includeTags = this.includeTags != null ? new ArrayList<>(this.includeTags) : null;
b.excludeTags = this.excludeTags != null ? new ArrayList<>(this.excludeTags) : null;
b.onlyMainContent = this.onlyMainContent;
b.timeout = this.timeout;
b.parsers = this.parsers != null ? new ArrayList<>(this.parsers) : null;
b.skipTlsVerification = this.skipTlsVerification;
b.removeBase64Images = this.removeBase64Images;
b.blockAds = this.blockAds;
b.proxy = this.proxy;
b.integration = this.integration;
return b;
}
private static String extractFormatType(Object fmt) {
if (fmt instanceof String) return (String) fmt;
if (fmt instanceof Map<?, ?>) {
Map<?, ?> mapObj = (Map<?, ?>) fmt;
Object type = mapObj.get("type");
if (type instanceof String) return (String) type;
}
try {
Object type = fmt.getClass().getMethod("getType").invoke(fmt);
if (type instanceof String) return (String) type;
} catch (ReflectiveOperationException ignored) {
// Ignore: format object doesn't expose a getType() method.
}
return null;
}
private static boolean isUnsupportedParseFormat(String formatType) {
if (formatType == null) return false;
String normalized = formatType.trim();
return normalized.equals("changeTracking")
|| normalized.equals("change_tracking")
|| normalized.equals("screenshot")
|| normalized.equals("screenshot@fullPage")
|| normalized.equals("branding");
}
public static final class Builder {
private List<Object> formats;
private Map<String, String> headers;
private List<String> includeTags;
private List<String> excludeTags;
private Boolean onlyMainContent;
private Integer timeout;
private List<Object> parsers;
private Boolean skipTlsVerification;
private Boolean removeBase64Images;
private Boolean blockAds;
private String proxy;
private String integration;
private Builder() {}
public Builder formats(List<Object> formats) { this.formats = formats; return this; }
public Builder headers(Map<String, String> headers) { this.headers = headers; return this; }
public Builder includeTags(List<String> includeTags) { this.includeTags = includeTags; return this; }
public Builder excludeTags(List<String> excludeTags) { this.excludeTags = excludeTags; return this; }
public Builder onlyMainContent(Boolean onlyMainContent) { this.onlyMainContent = onlyMainContent; return this; }
public Builder timeout(Integer timeout) { this.timeout = timeout; return this; }
public Builder parsers(List<Object> parsers) { this.parsers = parsers; return this; }
public Builder skipTlsVerification(Boolean skipTlsVerification) { this.skipTlsVerification = skipTlsVerification; return this; }
public Builder removeBase64Images(Boolean removeBase64Images) { this.removeBase64Images = removeBase64Images; return this; }
public Builder blockAds(Boolean blockAds) { this.blockAds = blockAds; return this; }
public Builder proxy(String proxy) { this.proxy = proxy; return this; }
public Builder integration(String integration) { this.integration = integration; return this; }
public ParseOptions build() {
if (timeout != null && timeout <= 0) {
throw new IllegalArgumentException("timeout must be positive");
}
if (proxy != null && !proxy.isBlank()) {
if (!proxy.equals("basic") && !proxy.equals("auto")) {
throw new IllegalArgumentException("parse only supports proxy values 'basic' or 'auto'");
}
}
if (formats != null) {
for (Object fmt : formats) {
String formatType = extractFormatType(fmt);
if (isUnsupportedParseFormat(formatType)) {
throw new IllegalArgumentException("parse does not support format: " + formatType);
}
}
}
ParseOptions o = new ParseOptions();
o.formats = this.formats != null ? Collections.unmodifiableList(new ArrayList<>(this.formats)) : null;
o.headers = this.headers != null ? Collections.unmodifiableMap(new HashMap<>(this.headers)) : null;
o.includeTags = this.includeTags != null ? Collections.unmodifiableList(new ArrayList<>(this.includeTags)) : null;
o.excludeTags = this.excludeTags != null ? Collections.unmodifiableList(new ArrayList<>(this.excludeTags)) : null;
o.onlyMainContent = this.onlyMainContent;
o.timeout = this.timeout;
o.parsers = this.parsers != null ? Collections.unmodifiableList(new ArrayList<>(this.parsers)) : null;
o.skipTlsVerification = this.skipTlsVerification;
o.removeBase64Images = this.removeBase64Images;
o.blockAds = this.blockAds;
o.proxy = this.proxy;
o.integration = this.integration;
return o;
}
}
}

View File

@@ -0,0 +1,62 @@
package com.firecrawl.models;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonValue;
/**
* Deprecated query format for asking a question about page content.
*
* @deprecated Use {@link QuestionFormat} or {@link HighlightsFormat} instead.
*/
@Deprecated
@JsonInclude(JsonInclude.Include.NON_NULL)
public class QueryFormat {
public enum Mode {
FREEFORM("freeform"),
DIRECT_QUOTE("directQuote");
private final String value;
Mode(String value) {
this.value = value;
}
@JsonValue
public String getValue() {
return value;
}
}
private final String type = "query";
private String prompt;
private Mode mode;
private QueryFormat() {}
public String getType() { return type; }
public String getPrompt() { return prompt; }
public Mode getMode() { return mode; }
public static Builder builder() { return new Builder(); }
public static final class Builder {
private String prompt;
private Mode mode;
private Builder() {}
/** Question to answer from the page content. */
public Builder prompt(String prompt) { this.prompt = prompt; return this; }
/** Query answer mode: freeform or direct quote. */
public Builder mode(Mode mode) { this.mode = mode; return this; }
public QueryFormat build() {
QueryFormat f = new QueryFormat();
f.prompt = this.prompt;
f.mode = this.mode;
return f;
}
}
}

View File

@@ -0,0 +1,35 @@
package com.firecrawl.models;
import com.fasterxml.jackson.annotation.JsonInclude;
/**
* Question format for asking a question about page content.
*/
@JsonInclude(JsonInclude.Include.NON_NULL)
public class QuestionFormat {
private final String type = "question";
private String question;
private QuestionFormat() {}
public String getType() { return type; }
public String getQuestion() { return question; }
public static Builder builder() { return new Builder(); }
public static final class Builder {
private String question;
private Builder() {}
/** Question to answer from the page content. */
public Builder question(String question) { this.question = question; return this; }
public QuestionFormat build() {
QuestionFormat f = new QuestionFormat();
f.question = this.question;
return f;
}
}
}

View File

@@ -0,0 +1,194 @@
package com.firecrawl.models;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
/**
* Options for scraping a single URL.
*/
@JsonInclude(JsonInclude.Include.NON_NULL)
public class ScrapeOptions {
private List<Object> formats;
private Map<String, String> headers;
private List<String> includeTags;
private List<String> excludeTags;
private Boolean onlyMainContent;
private Integer timeout;
private Integer waitFor;
private Boolean mobile;
private List<Object> parsers;
private List<Map<String, Object>> actions;
private LocationConfig location;
private Boolean skipTlsVerification;
private Boolean removeBase64Images;
private Boolean blockAds;
private String proxy;
@JsonProperty("maxAge")
private Long maxAge;
private Boolean storeInCache;
private Boolean lockdown;
private String integration;
private ScrapeOptions() {}
public List<Object> getFormats() { return formats; }
public Map<String, String> getHeaders() { return headers; }
public List<String> getIncludeTags() { return includeTags; }
public List<String> getExcludeTags() { return excludeTags; }
public Boolean getOnlyMainContent() { return onlyMainContent; }
public Integer getTimeout() { return timeout; }
public Integer getWaitFor() { return waitFor; }
public Boolean getMobile() { return mobile; }
public List<Object> getParsers() { return parsers; }
public List<Map<String, Object>> getActions() { return actions; }
public LocationConfig getLocation() { return location; }
public Boolean getSkipTlsVerification() { return skipTlsVerification; }
public Boolean getRemoveBase64Images() { return removeBase64Images; }
public Boolean getBlockAds() { return blockAds; }
public String getProxy() { return proxy; }
public Long getMaxAge() { return maxAge; }
public Boolean getStoreInCache() { return storeInCache; }
public Boolean getLockdown() { return lockdown; }
public String getIntegration() { return integration; }
public static Builder builder() { return new Builder(); }
public Builder toBuilder() {
Builder b = new Builder();
b.formats = this.formats != null ? new ArrayList<>(this.formats) : null;
b.headers = this.headers != null ? new HashMap<>(this.headers) : null;
b.includeTags = this.includeTags != null ? new ArrayList<>(this.includeTags) : null;
b.excludeTags = this.excludeTags != null ? new ArrayList<>(this.excludeTags) : null;
b.onlyMainContent = this.onlyMainContent;
b.timeout = this.timeout;
b.waitFor = this.waitFor;
b.mobile = this.mobile;
b.parsers = this.parsers != null ? new ArrayList<>(this.parsers) : null;
b.actions = this.actions != null ? new ArrayList<>(this.actions) : null;
b.location = this.location;
b.skipTlsVerification = this.skipTlsVerification;
b.removeBase64Images = this.removeBase64Images;
b.blockAds = this.blockAds;
b.proxy = this.proxy;
b.maxAge = this.maxAge;
b.storeInCache = this.storeInCache;
b.lockdown = this.lockdown;
b.integration = this.integration;
return b;
}
public static final class Builder {
private List<Object> formats;
private Map<String, String> headers;
private List<String> includeTags;
private List<String> excludeTags;
private Boolean onlyMainContent;
private Integer timeout;
private Integer waitFor;
private Boolean mobile;
private List<Object> parsers;
private List<Map<String, Object>> actions;
private LocationConfig location;
private Boolean skipTlsVerification;
private Boolean removeBase64Images;
private Boolean blockAds;
private String proxy;
private Long maxAge;
private Boolean storeInCache;
private Boolean lockdown;
private String integration;
private Builder() {}
/**
* Output formats to request. Accepts strings like "markdown", "html", "rawHtml",
* "links", "screenshot", "json", "audio", etc., or format configuration maps/objects for
* advanced formats (e.g., JsonFormat, QuestionFormat, HighlightsFormat).
*/
public Builder formats(List<Object> formats) { this.formats = formats; return this; }
/** Custom HTTP headers to send with the request. */
public Builder headers(Map<String, String> headers) { this.headers = headers; return this; }
/** Only include content from these HTML tags. */
public Builder includeTags(List<String> includeTags) { this.includeTags = includeTags; return this; }
/** Exclude content from these HTML tags. */
public Builder excludeTags(List<String> excludeTags) { this.excludeTags = excludeTags; return this; }
/** Only return the main content of the page, excluding navbars/footers. */
public Builder onlyMainContent(Boolean onlyMainContent) { this.onlyMainContent = onlyMainContent; return this; }
/** Timeout in milliseconds for the scrape request. */
public Builder timeout(Integer timeout) { this.timeout = timeout; return this; }
/** Wait time in milliseconds before scraping (for JS rendering). */
public Builder waitFor(Integer waitFor) { this.waitFor = waitFor; return this; }
/** Scrape as a mobile device. */
public Builder mobile(Boolean mobile) { this.mobile = mobile; return this; }
/** Parsers to use (e.g., "pdf" or {"type": "pdf", "maxPages": 10}). */
public Builder parsers(List<Object> parsers) { this.parsers = parsers; return this; }
/** Actions to execute before/during scraping. */
public Builder actions(List<Map<String, Object>> actions) { this.actions = actions; return this; }
/** Geolocation configuration. */
public Builder location(LocationConfig location) { this.location = location; return this; }
/** Skip TLS certificate verification. */
public Builder skipTlsVerification(Boolean skipTlsVerification) { this.skipTlsVerification = skipTlsVerification; return this; }
/** Remove base64-encoded images from the response. */
public Builder removeBase64Images(Boolean removeBase64Images) { this.removeBase64Images = removeBase64Images; return this; }
/** Block advertisements during scraping. */
public Builder blockAds(Boolean blockAds) { this.blockAds = blockAds; return this; }
/** Proxy mode: "basic", "stealth", "enhanced", "auto", or a custom proxy URL. */
public Builder proxy(String proxy) { this.proxy = proxy; return this; }
/** Use cached result if younger than this many milliseconds. */
public Builder maxAge(Long maxAge) { this.maxAge = maxAge; return this; }
/** Whether to cache the result. */
public Builder storeInCache(Boolean storeInCache) { this.storeInCache = storeInCache; return this; }
/** Lockdown mode: serve only previously cached results, never make outbound requests. */
public Builder lockdown(Boolean lockdown) { this.lockdown = lockdown; return this; }
/** Integration identifier. */
public Builder integration(String integration) { this.integration = integration; return this; }
public ScrapeOptions build() {
ScrapeOptions o = new ScrapeOptions();
o.formats = this.formats != null ? Collections.unmodifiableList(new ArrayList<>(this.formats)) : null;
o.headers = this.headers != null ? Collections.unmodifiableMap(new HashMap<>(this.headers)) : null;
o.includeTags = this.includeTags != null ? Collections.unmodifiableList(new ArrayList<>(this.includeTags)) : null;
o.excludeTags = this.excludeTags != null ? Collections.unmodifiableList(new ArrayList<>(this.excludeTags)) : null;
o.onlyMainContent = this.onlyMainContent;
o.timeout = this.timeout;
o.waitFor = this.waitFor;
o.mobile = this.mobile;
o.parsers = this.parsers != null ? Collections.unmodifiableList(new ArrayList<>(this.parsers)) : null;
o.actions = this.actions != null ? Collections.unmodifiableList(new ArrayList<>(this.actions)) : null;
o.location = this.location;
o.skipTlsVerification = this.skipTlsVerification;
o.removeBase64Images = this.removeBase64Images;
o.blockAds = this.blockAds;
o.proxy = this.proxy;
o.maxAge = this.maxAge;
o.storeInCache = this.storeInCache;
o.lockdown = this.lockdown;
o.integration = this.integration;
return o;
}
}
}

View File

@@ -0,0 +1,37 @@
package com.firecrawl.models;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import java.util.List;
import java.util.Map;
/**
* Search results from the v2 search API.
* The API returns an object with web, news, and images arrays.
*/
@JsonIgnoreProperties(ignoreUnknown = true)
public class SearchData {
private List<Map<String, Object>> web;
private List<Map<String, Object>> news;
private List<Map<String, Object>> images;
/** Web search results. */
public List<Map<String, Object>> getWeb() { return web; }
public void setWeb(List<Map<String, Object>> web) { this.web = web; }
/** News search results. */
public List<Map<String, Object>> getNews() { return news; }
public void setNews(List<Map<String, Object>> news) { this.news = news; }
/** Image search results. */
public List<Map<String, Object>> getImages() { return images; }
public void setImages(List<Map<String, Object>> images) { this.images = images; }
@Override
public String toString() {
int webCount = web != null ? web.size() : 0;
int newsCount = news != null ? news.size() : 0;
int imageCount = images != null ? images.size() : 0;
return "SearchData{web=" + webCount + ", news=" + newsCount + ", images=" + imageCount + "}";
}
}

View File

@@ -0,0 +1,94 @@
package com.firecrawl.models;
import com.fasterxml.jackson.annotation.JsonInclude;
import java.util.List;
/**
* Options for a web search request.
*/
@JsonInclude(JsonInclude.Include.NON_NULL)
public class SearchOptions {
private List<Object> sources;
private List<Object> categories;
private List<String> includeDomains;
private List<String> excludeDomains;
private Integer limit;
private String tbs;
private String location;
private Boolean ignoreInvalidURLs;
private Integer timeout;
private ScrapeOptions scrapeOptions;
private String integration;
private SearchOptions() {}
public List<Object> getSources() { return sources; }
public List<Object> getCategories() { return categories; }
public List<String> getIncludeDomains() { return includeDomains; }
public List<String> getExcludeDomains() { return excludeDomains; }
public Integer getLimit() { return limit; }
public String getTbs() { return tbs; }
public String getLocation() { return location; }
public Boolean getIgnoreInvalidURLs() { return ignoreInvalidURLs; }
public Integer getTimeout() { return timeout; }
public ScrapeOptions getScrapeOptions() { return scrapeOptions; }
public String getIntegration() { return integration; }
public static Builder builder() { return new Builder(); }
public static final class Builder {
private List<Object> sources;
private List<Object> categories;
private List<String> includeDomains;
private List<String> excludeDomains;
private Integer limit;
private String tbs;
private String location;
private Boolean ignoreInvalidURLs;
private Integer timeout;
private ScrapeOptions scrapeOptions;
private String integration;
private Builder() {}
/** Source types: "web", "news", "images" as strings or {type: "web"} maps. */
public Builder sources(List<Object> sources) { this.sources = sources; return this; }
/** Categories: "github", "research", "pdf". */
public Builder categories(List<Object> categories) { this.categories = categories; return this; }
/** Domains to include in search results. */
public Builder includeDomains(List<String> includeDomains) { this.includeDomains = includeDomains; return this; }
/** Domains to exclude from search results. */
public Builder excludeDomains(List<String> excludeDomains) { this.excludeDomains = excludeDomains; return this; }
/** Maximum number of results. */
public Builder limit(Integer limit) { this.limit = limit; return this; }
/** Time-based search filter (e.g., "qdr:d" for past day, "qdr:w" for past week). */
public Builder tbs(String tbs) { this.tbs = tbs; return this; }
/** Location for search results (e.g., "US"). */
public Builder location(String location) { this.location = location; return this; }
/** Ignore invalid URLs in results. */
public Builder ignoreInvalidURLs(Boolean ignoreInvalidURLs) { this.ignoreInvalidURLs = ignoreInvalidURLs; return this; }
/** Timeout in milliseconds. */
public Builder timeout(Integer timeout) { this.timeout = timeout; return this; }
/** Scrape options applied to search result pages. */
public Builder scrapeOptions(ScrapeOptions scrapeOptions) { this.scrapeOptions = scrapeOptions; return this; }
/** Integration identifier. */
public Builder integration(String integration) { this.integration = integration; return this; }
public SearchOptions build() {
SearchOptions o = new SearchOptions();
o.sources = this.sources;
o.categories = this.categories;
o.includeDomains = this.includeDomains;
o.excludeDomains = this.excludeDomains;
o.limit = this.limit;
o.tbs = this.tbs;
o.location = this.location;
o.ignoreInvalidURLs = this.ignoreInvalidURLs;
o.timeout = this.timeout;
o.scrapeOptions = this.scrapeOptions;
o.integration = this.integration;
return o;
}
}
}

View File

@@ -0,0 +1,57 @@
package com.firecrawl.models;
import com.fasterxml.jackson.annotation.JsonInclude;
import java.util.List;
import java.util.Map;
/**
* Webhook configuration for async jobs.
*/
@JsonInclude(JsonInclude.Include.NON_NULL)
public class WebhookConfig {
private String url;
private Map<String, String> headers;
private Map<String, String> metadata;
private List<String> events;
private WebhookConfig() {}
public String getUrl() { return url; }
public Map<String, String> getHeaders() { return headers; }
public Map<String, String> getMetadata() { return metadata; }
public List<String> getEvents() { return events; }
public static Builder builder() { return new Builder(); }
public static final class Builder {
private String url;
private Map<String, String> headers;
private Map<String, String> metadata;
private List<String> events;
private Builder() {}
public Builder url(String url) { this.url = url; return this; }
public Builder headers(Map<String, String> headers) { this.headers = headers; return this; }
public Builder metadata(Map<String, String> metadata) { this.metadata = metadata; return this; }
/**
* Events to subscribe to. Crawl/batch events: "completed", "failed", "page", "started".
* Agent events: "started", "action", "completed", "failed", "cancelled".
*/
public Builder events(List<String> events) { this.events = events; return this; }
public WebhookConfig build() {
if (url == null || url.isEmpty()) {
throw new IllegalArgumentException("Webhook URL is required");
}
WebhookConfig c = new WebhookConfig();
c.url = this.url;
c.headers = this.headers;
c.metadata = this.metadata;
c.events = this.events;
return c;
}
}
}

View File

@@ -0,0 +1,23 @@
/**
* Firecrawl Java SDK — a type-safe client for the Firecrawl v2 web scraping API.
*
* <p>Quick start:
* <pre>{@code
* import com.firecrawl.client.FirecrawlClient;
* import com.firecrawl.models.*;
*
* FirecrawlClient client = FirecrawlClient.builder()
* .apiKey("fc-your-api-key")
* .build();
*
* Document doc = client.scrape("https://example.com",
* ScrapeOptions.builder()
* .formats(List.of("markdown"))
* .build());
*
* System.out.println(doc.getMarkdown());
* }</pre>
*
* @see com.firecrawl.client.FirecrawlClient
*/
package com.firecrawl;