lasta/AI
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

참고소스 수정본

Browse Source
This commit is contained in:
LASTA_DEV01\lasta
2026-05-12 19:40:31 +09:00
parent 0f34a451fc
commit 2e9204243d
8708 changed files with 3259488 additions and 869 deletions
Download Patch File Download Diff File Expand all files Collapse all files

674
참고/firecrawl-main/apps/rust-sdk/src/agent.rs Normal file
View File

@@ -0,0 +1,674 @@
//! Agent endpoint for Firecrawl API v2.
//!
//! The Agent endpoint provides autonomous web browsing capabilities using AI
//! to accomplish complex tasks that may require multiple page interactions.
use serde::{Deserialize, Serialize};
use serde_json::Value;
use crate::client::Client;
use crate::types::{AgentModel, AgentWebhookConfig};
use crate::FirecrawlError;
/// Options for running an agent task.
#[serde_with::skip_serializing_none]
#[derive(Deserialize, Serialize, Debug, Default, Clone)]
#[serde(rename_all = "camelCase")]
pub struct AgentOptions {
/// Starting URLs for the agent to explore.
pub urls: Option<Vec<String>>,
/// The prompt describing what the agent should accomplish.
pub prompt: String,
/// JSON schema for the expected output structure.
pub schema: Option<Value>,
/// Integration identifier for tracking.
pub integration: Option<String>,
/// Maximum credits the agent can use.
pub max_credits: Option<u32>,
/// Strictly constrain the agent to the provided URLs.
pub strict_constrain_to_urls: Option<bool>,
/// Agent model to use.
pub model: Option<AgentModel>,
/// Webhook configuration for agent notifications.
pub webhook: Option<AgentWebhookConfig>,
/// Poll interval for synchronous agent execution (milliseconds).
#[serde(skip)]
pub poll_interval: Option<u64>,
/// Timeout for synchronous agent execution (seconds).
#[serde(skip)]
pub timeout: Option<u64>,
}
/// Response from starting an agent task.
#[derive(Deserialize, Serialize, Debug, Clone)]
#[serde(rename_all = "camelCase")]
pub struct AgentResponse {
/// Whether the request was successful.
pub success: bool,
/// The agent task ID.
pub id: String,
/// Error message if the request failed.
pub error: Option<String>,
}
/// Agent task status.
#[derive(Deserialize, Serialize, Clone, Copy, Debug, PartialEq, Eq)]
#[serde(rename_all = "camelCase")]
pub enum AgentStatus {
/// The agent is still processing.
Processing,
/// The agent has completed its task.
Completed,
/// The agent task failed.
Failed,
/// The agent task was cancelled.
Cancelled,
}
/// Status response from an agent task.
#[serde_with::skip_serializing_none]
#[derive(Deserialize, Serialize, Debug, Clone)]
#[serde(rename_all = "camelCase")]
pub struct AgentStatusResponse {
/// Whether the status check was successful.
pub success: bool,
/// Current status of the agent task.
pub status: AgentStatus,
/// Error message if the task failed.
pub error: Option<String>,
/// Extracted data (if schema was provided) or task results.
pub data: Option<Value>,
/// Model used for the agent task.
pub model: Option<AgentModel>,
/// Expiry time of the task data.
pub expires_at: Option<String>,
/// Credits used by the agent task.
pub credits_used: Option<u32>,
}
impl Client {
/// Starts an agent task asynchronously.
///
/// Returns immediately with a task ID that can be used to check status.
///
/// # Arguments
///
/// * `options` - Agent task configuration including the prompt.
///
/// # Returns
///
/// An `AgentResponse` containing the task ID.
///
/// # Example
///
/// ```no_run
/// use firecrawl::{Client, AgentOptions};
///
/// #[tokio::main]
/// async fn main() -> Result<(), Box<dyn std::error::Error>> {
/// let client = Client::new("your-api-key")?;
///
/// let options = AgentOptions {
/// urls: Some(vec!["https://example.com".to_string()]),
/// prompt: "Find the pricing information on this website".to_string(),
/// ..Default::default()
/// };
///
/// let response = client.start_agent(options).await?;
/// println!("Agent task started: {}", response.id);
///
/// Ok(())
/// }
/// ```
pub async fn start_agent(
&self,
options: AgentOptions,
) -> Result<AgentResponse, FirecrawlError> {
let headers = self.prepare_headers(None);
let response = self
.client
.post(self.url("/agent"))
.headers(headers)
.json(&options)
.send()
.await
.map_err(|e| FirecrawlError::HttpError("Starting agent task".to_string(), e))?;
self.handle_response(response, "start agent").await
}
/// Gets the status of an agent task.
///
/// # Arguments
///
/// * `id` - The agent task ID.
///
/// # Returns
///
/// An `AgentStatusResponse` containing the current status and any results.
///
/// # Example
///
/// ```no_run
/// use firecrawl::Client;
///
/// #[tokio::main]
/// async fn main() -> Result<(), Box<dyn std::error::Error>> {
/// let client = Client::new("your-api-key")?;
///
/// let status = client.get_agent_status("task-id").await?;
/// println!("Status: {:?}", status.status);
///
/// if let Some(data) = status.data {
/// println!("Result: {}", data);
/// }
///
/// Ok(())
/// }
/// ```
pub async fn get_agent_status(
&self,
id: impl AsRef<str>,
) -> Result<AgentStatusResponse, FirecrawlError> {
let response = self
.client
.get(self.url(&format!("/agent/{}", id.as_ref())))
.headers(self.prepare_headers(None))
.send()
.await
.map_err(|e| {
FirecrawlError::HttpError(format!("Getting agent status {}", id.as_ref()), e)
})?;
self.handle_response(response, format!("agent status {}", id.as_ref()))
.await
}
/// Runs an agent task and waits for completion.
///
/// This method starts an agent task and polls until it completes, fails, or times out.
///
/// # Arguments
///
/// * `options` - Agent task configuration including the prompt.
///
/// # Returns
///
/// An `AgentStatusResponse` containing the final status and results.
///
/// # Example
///
/// ```no_run
/// use firecrawl::{Client, AgentOptions, AgentModel};
/// use serde_json::json;
///
/// #[tokio::main]
/// async fn main() -> Result<(), Box<dyn std::error::Error>> {
/// let client = Client::new("your-api-key")?;
///
/// let options = AgentOptions {
/// urls: Some(vec!["https://example.com/pricing".to_string()]),
/// prompt: "Extract the pricing tiers and their features".to_string(),
/// schema: Some(json!({
/// "type": "object",
/// "properties": {
/// "tiers": {
/// "type": "array",
/// "items": {
/// "type": "object",
/// "properties": {
/// "name": { "type": "string" },
/// "price": { "type": "number" },
/// "features": { "type": "array", "items": { "type": "string" } }
/// }
/// }
/// }
/// }
/// })),
/// model: Some(AgentModel::Spark1Pro),
/// poll_interval: Some(3000),
/// timeout: Some(300),
/// ..Default::default()
/// };
///
/// let result = client.agent(options).await?;
///
/// if let Some(data) = result.data {
/// println!("Extracted pricing: {}", serde_json::to_string_pretty(&data)?);
/// }
///
/// Ok(())
/// }
/// ```
pub async fn agent(
&self,
options: AgentOptions,
) -> Result<AgentStatusResponse, FirecrawlError> {
let poll_interval = options.poll_interval.unwrap_or(2000);
let timeout = options.timeout;
let response = self.start_agent(options).await?;
self.wait_for_agent(&response.id, poll_interval, timeout)
.await
}
/// Waits for an agent task to complete.
async fn wait_for_agent(
&self,
id: &str,
poll_interval: u64,
timeout: Option<u64>,
) -> Result<AgentStatusResponse, FirecrawlError> {
let start = std::time::Instant::now();
loop {
let status = self.get_agent_status(id).await?;
match status.status {
AgentStatus::Completed | AgentStatus::Failed | AgentStatus::Cancelled => {
return Ok(status);
}
AgentStatus::Processing => {
// Check timeout
if let Some(timeout_secs) = timeout {
if start.elapsed().as_secs() > timeout_secs {
return Ok(status);
}
}
tokio::time::sleep(tokio::time::Duration::from_millis(poll_interval)).await;
}
}
}
}
/// Cancels a running agent task.
///
/// # Arguments
///
/// * `id` - The agent task ID to cancel.
///
/// # Returns
///
/// `true` if the cancellation was successful.
///
/// # Example
///
/// ```no_run
/// use firecrawl::Client;
///
/// #[tokio::main]
/// async fn main() -> Result<(), Box<dyn std::error::Error>> {
/// let client = Client::new("your-api-key")?;
///
/// let cancelled = client.cancel_agent("task-id").await?;
/// println!("Cancelled: {}", cancelled);
///
/// Ok(())
/// }
/// ```
pub async fn cancel_agent(&self, id: impl AsRef<str>) -> Result<bool, FirecrawlError> {
let response = self
.client
.delete(self.url(&format!("/agent/{}", id.as_ref())))
.headers(self.prepare_headers(None))
.send()
.await
.map_err(|e| {
FirecrawlError::HttpError(format!("Cancelling agent {}", id.as_ref()), e)
})?;
#[derive(Deserialize)]
struct CancelResponse {
success: bool,
}
let result: CancelResponse = self
.handle_response(response, format!("cancel agent {}", id.as_ref()))
.await?;
Ok(result.success)
}
/// Runs an agent with a typed schema for structured output.
///
/// This is a convenience method that automatically converts the result
/// to the specified type.
///
/// # Arguments
///
/// * `urls` - Starting URLs for the agent.
/// * `prompt` - The task description.
/// * `schema` - JSON schema for the expected output.
///
/// # Returns
///
/// The extracted data as the specified type, or `None` if extraction failed.
///
/// # Example
///
/// ```no_run
/// use firecrawl::Client;
/// use serde::Deserialize;
/// use serde_json::json;
///
/// #[derive(Debug, Deserialize)]
/// struct ProductInfo {
/// name: String,
/// price: f64,
/// description: Option<String>,
/// }
///
/// #[tokio::main]
/// async fn main() -> Result<(), Box<dyn std::error::Error>> {
/// let client = Client::new("your-api-key")?;
///
/// let schema = json!({
/// "type": "object",
/// "properties": {
/// "name": { "type": "string" },
/// "price": { "type": "number" },
/// "description": { "type": "string" }
/// },
/// "required": ["name", "price"]
/// });
///
/// let result: Option<ProductInfo> = client.agent_with_schema(
/// vec!["https://example.com/product".to_string()],
/// "Extract the product information",
/// schema,
/// ).await?;
///
/// if let Some(product) = result {
/// println!("Product: {} - ${}", product.name, product.price);
/// }
///
/// Ok(())
/// }
/// ```
pub async fn agent_with_schema<T: serde::de::DeserializeOwned>(
&self,
urls: Vec<String>,
prompt: impl AsRef<str>,
schema: Value,
) -> Result<Option<T>, FirecrawlError> {
let options = AgentOptions {
urls: Some(urls),
prompt: prompt.as_ref().to_string(),
schema: Some(schema),
..Default::default()
};
let result = self.agent(options).await?;
if result.status != AgentStatus::Completed {
return Ok(None);
}
match result.data {
Some(data) => {
let typed: T =
serde_json::from_value(data).map_err(FirecrawlError::ResponseParseError)?;
Ok(Some(typed))
}
None => Ok(None),
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use serde_json::json;
#[tokio::test]
async fn test_start_agent_with_mock() {
let mut server = mockito::Server::new_async().await;
let mock = server
.mock("POST", "/v2/agent")
.with_status(200)
.with_header("content-type", "application/json")
.with_body(
json!({
"success": true,
"id": "agent-123"
})
.to_string(),
)
.create();
let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();
let options = AgentOptions {
urls: Some(vec!["https://example.com".to_string()]),
prompt: "Find the contact information".to_string(),
..Default::default()
};
let response = client.start_agent(options).await.unwrap();
assert!(response.success);
assert_eq!(response.id, "agent-123");
mock.assert();
}
#[tokio::test]
async fn test_get_agent_status_with_mock() {
let mut server = mockito::Server::new_async().await;
let mock = server
.mock("GET", "/v2/agent/agent-123")
.with_status(200)
.with_header("content-type", "application/json")
.with_body(
json!({
"success": true,
"status": "completed",
"data": {
"email": "contact@example.com",
"phone": "555-1234"
},
"creditsUsed": 5,
"expiresAt": "2024-12-31T23:59:59Z"
})
.to_string(),
)
.create();
let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();
let status = client.get_agent_status("agent-123").await.unwrap();
assert!(status.success);
assert_eq!(status.status, AgentStatus::Completed);
assert!(status.data.is_some());
assert_eq!(status.credits_used, Some(5));
mock.assert();
}
#[tokio::test]
async fn test_agent_sync_with_mock() {
let mut server = mockito::Server::new_async().await;
// Mock the start endpoint
let start_mock = server
.mock("POST", "/v2/agent")
.with_status(200)
.with_header("content-type", "application/json")
.with_body(
json!({
"success": true,
"id": "agent-456"
})
.to_string(),
)
.create();
// Mock the status endpoint (completed immediately)
let status_mock = server
.mock("GET", "/v2/agent/agent-456")
.with_status(200)
.with_header("content-type", "application/json")
.with_body(
json!({
"success": true,
"status": "completed",
"data": {
"result": "Task completed successfully"
}
})
.to_string(),
)
.create();
let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();
let options = AgentOptions {
urls: Some(vec!["https://example.com".to_string()]),
prompt: "Test task".to_string(),
..Default::default()
};
let result = client.agent(options).await.unwrap();
assert_eq!(result.status, AgentStatus::Completed);
assert!(result.data.is_some());
start_mock.assert();
status_mock.assert();
}
#[tokio::test]
async fn test_cancel_agent_with_mock() {
let mut server = mockito::Server::new_async().await;
let mock = server
.mock("DELETE", "/v2/agent/agent-789")
.with_status(200)
.with_header("content-type", "application/json")
.with_body(
json!({
"success": true
})
.to_string(),
)
.create();
let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();
let cancelled = client.cancel_agent("agent-789").await.unwrap();
assert!(cancelled);
mock.assert();
}
#[tokio::test]
async fn test_agent_with_schema() {
let mut server = mockito::Server::new_async().await;
// Mock the start endpoint
let start_mock = server
.mock("POST", "/v2/agent")
.with_status(200)
.with_header("content-type", "application/json")
.with_body(
json!({
"success": true,
"id": "agent-schema"
})
.to_string(),
)
.create();
// Mock the status endpoint
let status_mock = server
.mock("GET", "/v2/agent/agent-schema")
.with_status(200)
.with_header("content-type", "application/json")
.with_body(
json!({
"success": true,
"status": "completed",
"data": {
"name": "Test Product",
"price": 29.99
}
})
.to_string(),
)
.create();
#[derive(Debug, serde::Deserialize, PartialEq)]
struct Product {
name: String,
price: f64,
}
let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();
let schema = json!({
"type": "object",
"properties": {
"name": { "type": "string" },
"price": { "type": "number" }
}
});
let result: Option<Product> = client
.agent_with_schema(
vec!["https://example.com".to_string()],
"Extract product info",
schema,
)
.await
.unwrap();
assert_eq!(
result,
Some(Product {
name: "Test Product".to_string(),
price: 29.99
})
);
start_mock.assert();
status_mock.assert();
}
#[tokio::test]
async fn test_agent_with_model_option() {
let mut server = mockito::Server::new_async().await;
let mock = server
.mock("POST", "/v2/agent")
.with_status(200)
.with_header("content-type", "application/json")
.with_body(
json!({
"success": true,
"id": "agent-model"
})
.to_string(),
)
.create();
let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();
let options = AgentOptions {
urls: Some(vec!["https://example.com".to_string()]),
prompt: "Task with specific model".to_string(),
model: Some(AgentModel::Spark1Pro),
max_credits: Some(100),
..Default::default()
};
let response = client.start_agent(options).await.unwrap();
assert!(response.success);
mock.assert();
}
}

562
참고/firecrawl-main/apps/rust-sdk/src/batch_scrape.rs Normal file
View File

@@ -0,0 +1,562 @@
//! Batch scrape endpoint for Firecrawl API v2.
use serde::{Deserialize, Serialize};
use crate::client::Client;
use crate::scrape::ScrapeOptions;
use crate::types::{CrawlErrorsResponse, Document, JobStatus, WebhookConfig};
use crate::FirecrawlError;
/// Options for batch scraping.
#[serde_with::skip_serializing_none]
#[derive(Deserialize, Serialize, Debug, Default, Clone)]
#[serde(rename_all = "camelCase")]
pub struct BatchScrapeOptions {
/// Scrape options to apply to all URLs.
#[serde(flatten)]
pub options: Option<ScrapeOptions>,
/// Webhook configuration for job notifications.
pub webhook: Option<WebhookConfig>,
/// ID of an existing batch job to append URLs to.
pub append_to_id: Option<String>,
/// Whether to ignore invalid URLs instead of failing.
#[serde(rename = "ignoreInvalidURLs")]
pub ignore_invalid_urls: Option<bool>,
/// Maximum concurrent requests.
pub max_concurrency: Option<u32>,
/// Enable zero data retention mode.
pub zero_data_retention: Option<bool>,
/// Idempotency key for the request.
#[serde(skip)]
pub idempotency_key: Option<String>,
/// Integration identifier for tracking.
pub integration: Option<String>,
/// Poll interval for synchronous batch scrape (milliseconds).
#[serde(skip)]
pub poll_interval: Option<u64>,
}
/// Request body for batch scrape endpoint.
#[derive(Deserialize, Serialize, Debug, Default)]
#[serde(rename_all = "camelCase")]
struct BatchScrapeRequest {
urls: Vec<String>,
#[serde(flatten)]
options: BatchScrapeOptions,
}
/// Response from starting a batch scrape job.
#[derive(Deserialize, Serialize, Debug, Clone)]
#[serde(rename_all = "camelCase")]
pub struct BatchScrapeResponse {
/// Whether the request was successful.
pub success: bool,
/// The batch scrape job ID.
pub id: String,
/// URL to check the batch scrape status.
pub url: String,
/// URLs that were invalid and ignored.
#[serde(rename = "invalidURLs")]
pub invalid_urls: Option<Vec<String>>,
}
/// Status of a batch scrape job.
#[serde_with::skip_serializing_none]
#[derive(Deserialize, Serialize, Debug, Clone)]
#[serde(rename_all = "camelCase")]
pub struct BatchScrapeJob {
/// Current status of the batch scrape job.
pub status: JobStatus,
/// Number of URLs completed.
pub completed: u32,
/// Total number of URLs to scrape.
pub total: u32,
/// Credits used by the batch scrape.
pub credits_used: Option<u32>,
/// Expiry time of the batch data.
pub expires_at: Option<String>,
/// URL for the next page of results.
pub next: Option<String>,
/// Scraped documents.
pub data: Vec<Document>,
}
impl Client {
/// Starts a batch scrape job asynchronously.
///
/// Returns immediately with a job ID that can be used to check status.
///
/// # Arguments
///
/// * `urls` - List of URLs to scrape.
/// * `options` - Optional batch scrape configuration.
///
/// # Returns
///
/// A `BatchScrapeResponse` containing the job ID.
///
/// # Example
///
/// ```no_run
/// use firecrawl::{Client, BatchScrapeOptions};
///
/// #[tokio::main]
/// async fn main() -> Result<(), Box<dyn std::error::Error>> {
/// let client = Client::new("your-api-key")?;
///
/// let urls = vec![
/// "https://example.com".to_string(),
/// "https://example.org".to_string(),
/// ];
///
/// let response = client.start_batch_scrape(urls, None).await?;
/// println!("Batch job started: {}", response.id);
///
/// // Check status later
/// let status = client.get_batch_scrape_status(&response.id).await?;
/// println!("Completed: {}/{}", status.completed, status.total);
///
/// Ok(())
/// }
/// ```
pub async fn start_batch_scrape(
&self,
urls: Vec<String>,
options: impl Into<Option<BatchScrapeOptions>>,
) -> Result<BatchScrapeResponse, FirecrawlError> {
let options = options.into().unwrap_or_default();
let body = BatchScrapeRequest {
urls,
options: options.clone(),
};
let headers = self.prepare_headers(options.idempotency_key.as_ref());
let response = self
.client
.post(self.url("/batch/scrape"))
.headers(headers)
.json(&body)
.send()
.await
.map_err(|e| FirecrawlError::HttpError("Starting batch scrape".to_string(), e))?;
self.handle_response(response, "start batch scrape").await
}
/// Gets the status of a batch scrape job.
///
/// If the job is completed, this will automatically fetch all pages of results.
///
/// # Arguments
///
/// * `id` - The batch scrape job ID.
///
/// # Returns
///
/// A `BatchScrapeJob` containing the current status and any available documents.
///
/// # Example
///
/// ```no_run
/// use firecrawl::Client;
///
/// #[tokio::main]
/// async fn main() -> Result<(), Box<dyn std::error::Error>> {
/// let client = Client::new("your-api-key")?;
///
/// let status = client.get_batch_scrape_status("job-id").await?;
/// println!("Status: {:?}", status.status);
/// println!("Completed: {}/{}", status.completed, status.total);
/// println!("Documents: {}", status.data.len());
///
/// Ok(())
/// }
/// ```
pub async fn get_batch_scrape_status(
&self,
id: impl AsRef<str>,
) -> Result<BatchScrapeJob, FirecrawlError> {
let response = self
.client
.get(self.url(&format!("/batch/scrape/{}", id.as_ref())))
.headers(self.prepare_headers(None))
.send()
.await
.map_err(|e| {
FirecrawlError::HttpError(
format!("Checking batch scrape status {}", id.as_ref()),
e,
)
})?;
let mut status: BatchScrapeJob = self
.handle_response(response, format!("batch scrape status {}", id.as_ref()))
.await?;
// Auto-paginate if completed
if status.status == JobStatus::Completed {
while let Some(next) = status.next.take() {
let next_status = self.get_batch_scrape_status_next(&next).await?;
status.data.extend(next_status.data);
status.next = next_status.next;
}
}
Ok(status)
}
/// Fetches the next page of batch scrape results.
async fn get_batch_scrape_status_next(
&self,
next: &str,
) -> Result<BatchScrapeJob, FirecrawlError> {
let response = self
.client
.get(next)
.headers(self.prepare_headers(None))
.send()
.await
.map_err(|e| {
FirecrawlError::HttpError(format!("Paginating batch scrape at {}", next), e)
})?;
self.handle_response(response, "batch scrape pagination")
.await
}
/// Scrapes multiple URLs and waits for completion.
///
/// This method starts a batch scrape and polls until it completes or fails.
///
/// # Arguments
///
/// * `urls` - List of URLs to scrape.
/// * `options` - Optional batch scrape configuration.
///
/// # Returns
///
/// A `BatchScrapeJob` containing all scraped documents.
///
/// # Example
///
/// ```no_run
/// use firecrawl::{Client, BatchScrapeOptions, ScrapeOptions, Format};
///
/// #[tokio::main]
/// async fn main() -> Result<(), Box<dyn std::error::Error>> {
/// let client = Client::new("your-api-key")?;
///
/// let urls = vec![
/// "https://example.com/page1".to_string(),
/// "https://example.com/page2".to_string(),
/// "https://example.com/page3".to_string(),
/// ];
///
/// let options = BatchScrapeOptions {
/// options: Some(ScrapeOptions {
/// formats: Some(vec![Format::Markdown, Format::Links]),
/// ..Default::default()
/// }),
/// ignore_invalid_urls: Some(true),
/// poll_interval: Some(3000),
/// ..Default::default()
/// };
///
/// let result = client.batch_scrape(urls, options).await?;
/// println!("Scraped {} pages", result.data.len());
///
/// for doc in result.data {
/// println!("URL: {:?}", doc.metadata.and_then(|m| m.source_url));
/// }
///
/// Ok(())
/// }
/// ```
pub async fn batch_scrape(
&self,
urls: Vec<String>,
options: impl Into<Option<BatchScrapeOptions>>,
) -> Result<BatchScrapeJob, FirecrawlError> {
let options = options.into().unwrap_or_default();
let poll_interval = options.poll_interval.unwrap_or(2000);
let response = self.start_batch_scrape(urls, options).await?;
self.wait_for_batch_scrape(&response.id, poll_interval)
.await
}
/// Waits for a batch scrape job to complete.
async fn wait_for_batch_scrape(
&self,
id: &str,
poll_interval: u64,
) -> Result<BatchScrapeJob, FirecrawlError> {
loop {
let status = self.get_batch_scrape_status(id).await?;
match status.status {
JobStatus::Completed => return Ok(status),
JobStatus::Scraping => {
tokio::time::sleep(tokio::time::Duration::from_millis(poll_interval)).await;
}
JobStatus::Failed => {
return Err(FirecrawlError::JobFailed(
"Batch scrape job failed".to_string(),
JobStatus::Failed,
));
}
JobStatus::Cancelled => {
return Err(FirecrawlError::JobFailed(
"Batch scrape job was cancelled".to_string(),
JobStatus::Cancelled,
));
}
}
}
}
/// Gets errors from a batch scrape job.
///
/// # Arguments
///
/// * `id` - The batch scrape job ID.
///
/// # Returns
///
/// A `CrawlErrorsResponse` containing error details.
///
/// # Example
///
/// ```no_run
/// use firecrawl::Client;
///
/// #[tokio::main]
/// async fn main() -> Result<(), Box<dyn std::error::Error>> {
/// let client = Client::new("your-api-key")?;
///
/// let errors = client.get_batch_scrape_errors("job-id").await?;
/// for error in errors.errors {
/// println!("Error on {}: {}", error.url, error.error);
/// }
///
/// Ok(())
/// }
/// ```
pub async fn get_batch_scrape_errors(
&self,
id: impl AsRef<str>,
) -> Result<CrawlErrorsResponse, FirecrawlError> {
let response = self
.client
.get(self.url(&format!("/batch/scrape/{}/errors", id.as_ref())))
.headers(self.prepare_headers(None))
.send()
.await
.map_err(|e| {
FirecrawlError::HttpError(format!("Getting batch scrape errors {}", id.as_ref()), e)
})?;
self.handle_response(response, "batch scrape errors").await
}
}
#[cfg(test)]
mod tests {
use super::*;
use serde_json::json;
#[tokio::test]
async fn test_start_batch_scrape_with_mock() {
let mut server = mockito::Server::new_async().await;
let mock = server
.mock("POST", "/v2/batch/scrape")
.with_status(200)
.with_header("content-type", "application/json")
.with_body(
json!({
"success": true,
"id": "batch-123",
"url": "https://api.firecrawl.dev/v2/batch/scrape/batch-123"
})
.to_string(),
)
.create();
let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();
let urls = vec![
"https://example.com".to_string(),
"https://example.org".to_string(),
];
let response = client.start_batch_scrape(urls, None).await.unwrap();
assert!(response.success);
assert_eq!(response.id, "batch-123");
mock.assert();
}
#[tokio::test]
async fn test_get_batch_scrape_status_with_mock() {
let mut server = mockito::Server::new_async().await;
let mock = server
.mock("GET", "/v2/batch/scrape/batch-123")
.with_status(200)
.with_header("content-type", "application/json")
.with_body(
json!({
"status": "completed",
"total": 3,
"completed": 3,
"creditsUsed": 3,
"data": [
{
"markdown": "# Page 1",
"metadata": { "sourceURL": "https://example.com/1", "statusCode": 200 }
},
{
"markdown": "# Page 2",
"metadata": { "sourceURL": "https://example.com/2", "statusCode": 200 }
}
]
})
.to_string(),
)
.create();
let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();
let status = client.get_batch_scrape_status("batch-123").await.unwrap();
assert_eq!(status.status, JobStatus::Completed);
assert_eq!(status.total, 3);
assert_eq!(status.completed, 3);
assert_eq!(status.data.len(), 2);
mock.assert();
}
#[tokio::test]
async fn test_batch_scrape_with_invalid_urls() {
let mut server = mockito::Server::new_async().await;
let mock = server
.mock("POST", "/v2/batch/scrape")
.with_status(200)
.with_header("content-type", "application/json")
.with_body(
json!({
"success": true,
"id": "batch-456",
"url": "https://api.firecrawl.dev/v2/batch/scrape/batch-456",
"invalidURLs": ["not-a-url"]
})
.to_string(),
)
.create();
let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();
let urls = vec!["https://example.com".to_string(), "not-a-url".to_string()];
let options = BatchScrapeOptions {
ignore_invalid_urls: Some(true),
..Default::default()
};
let response = client.start_batch_scrape(urls, options).await.unwrap();
assert!(response.success);
assert_eq!(response.invalid_urls, Some(vec!["not-a-url".to_string()]));
mock.assert();
}
#[tokio::test]
async fn test_batch_scrape_sync() {
let mut server = mockito::Server::new_async().await;
// Mock the start endpoint
let start_mock = server
.mock("POST", "/v2/batch/scrape")
.with_status(200)
.with_header("content-type", "application/json")
.with_body(
json!({
"success": true,
"id": "batch-789",
"url": "https://api.firecrawl.dev/v2/batch/scrape/batch-789"
})
.to_string(),
)
.create();
// Mock the status endpoint (completed immediately)
let status_mock = server
.mock("GET", "/v2/batch/scrape/batch-789")
.with_status(200)
.with_header("content-type", "application/json")
.with_body(
json!({
"status": "completed",
"total": 2,
"completed": 2,
"data": [
{
"markdown": "# Content",
"metadata": { "sourceURL": "https://example.com", "statusCode": 200 }
}
]
})
.to_string(),
)
.create();
let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();
let urls = vec!["https://example.com".to_string()];
let result = client.batch_scrape(urls, None).await.unwrap();
assert_eq!(result.status, JobStatus::Completed);
assert_eq!(result.data.len(), 1);
start_mock.assert();
status_mock.assert();
}
#[tokio::test]
async fn test_get_batch_scrape_errors() {
let mut server = mockito::Server::new_async().await;
let mock = server
.mock("GET", "/v2/batch/scrape/batch-123/errors")
.with_status(200)
.with_header("content-type", "application/json")
.with_body(
json!({
"errors": [
{
"id": "err-1",
"url": "https://example.com/broken",
"error": "Connection timeout"
}
],
"robotsBlocked": []
})
.to_string(),
)
.create();
let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();
let errors = client.get_batch_scrape_errors("batch-123").await.unwrap();
assert_eq!(errors.errors.len(), 1);
assert_eq!(errors.errors[0].error, "Connection timeout");
mock.assert();
}
}

280
참고/firecrawl-main/apps/rust-sdk/src/client.rs Normal file
View File

@@ -0,0 +1,280 @@
//! Firecrawl API v2 client.
use reqwest::Response;
use serde::de::DeserializeOwned;
use serde_json::Value;
use crate::error::{FirecrawlAPIError, FirecrawlError};
pub(crate) const API_VERSION: &str = "/v2";
const CLOUD_API_URL: &str = "https://api.firecrawl.dev";
/// Firecrawl API v2 client.
///
/// This client provides access to all v2 API endpoints including scrape, crawl,
/// search, map, batch scrape, and agent operations.
///
/// # Example
///
/// ```no_run
/// use firecrawl::Client;
///
/// #[tokio::main]
/// async fn main() -> Result<(), Box<dyn std::error::Error>> {
/// // Create a client for the Firecrawl cloud service
/// let client = Client::new("your-api-key")?;
///
/// // Or create a client for a self-hosted instance
/// let client = Client::new_selfhosted("http://localhost:3000", Some("api-key"))?;
///
/// Ok(())
/// }
/// ```
#[derive(Clone, Debug)]
pub struct Client {
pub(crate) api_key: Option<String>,
pub(crate) api_url: String,
pub(crate) client: reqwest::Client,
}
impl Client {
/// Creates a new client for the Firecrawl cloud service.
///
/// # Arguments
///
/// * `api_key` - Your Firecrawl API key.
///
/// # Errors
///
/// Returns an error if the API key is empty.
///
/// # Example
///
/// ```no_run
/// use firecrawl::Client;
///
/// let client = Client::new("your-api-key").unwrap();
/// ```
pub fn new(api_key: impl AsRef<str>) -> Result<Self, FirecrawlError> {
Client::new_selfhosted(CLOUD_API_URL, Some(api_key))
}
/// Creates a new client for a self-hosted Firecrawl instance.
///
/// # Arguments
///
/// * `api_url` - The base URL of your Firecrawl instance.
/// * `api_key` - Optional API key (required for cloud, optional for self-hosted).
///
/// # Errors
///
/// Returns an error if using the cloud service without an API key.
///
/// # Example
///
/// ```no_run
/// use firecrawl::Client;
///
/// // Self-hosted without authentication
/// let client = Client::new_selfhosted("http://localhost:3000", None::<&str>).unwrap();
///
/// // Self-hosted with authentication
/// let client = Client::new_selfhosted("http://localhost:3000", Some("api-key")).unwrap();
/// ```
pub fn new_selfhosted(
api_url: impl AsRef<str>,
api_key: Option<impl AsRef<str>>,
) -> Result<Self, FirecrawlError> {
// Normalize URL by trimming trailing slashes for consistent comparison
let url = api_url.as_ref().trim_end_matches('/').to_string();
let api_key = api_key.map(|k| k.as_ref().to_string());
// Reject empty or missing API key for cloud service
if url == CLOUD_API_URL {
match &api_key {
None => {
return Err(FirecrawlError::APIError(
"Configuration".to_string(),
FirecrawlAPIError {
success: false,
error: "API key is required for cloud service".to_string(),
details: None,
},
));
}
Some(key) if key.trim().is_empty() => {
return Err(FirecrawlError::APIError(
"Configuration".to_string(),
FirecrawlAPIError {
success: false,
error: "API key cannot be empty for cloud service".to_string(),
details: None,
},
));
}
_ => {}
}
}
Ok(Client {
api_key,
api_url: url,
client: reqwest::Client::new(),
})
}
/// Prepares headers for API requests.
pub(crate) fn prepare_headers(
&self,
idempotency_key: Option<&String>,
) -> reqwest::header::HeaderMap {
use reqwest::header::HeaderValue;
let mut headers = reqwest::header::HeaderMap::new();
// Static string is always valid ASCII
headers.insert("Content-Type", HeaderValue::from_static("application/json"));
if let Some(api_key) = self.api_key.as_ref() {
// API key is validated at client creation, so this should always succeed.
// Use if-let to gracefully handle edge cases without panicking.
if let Ok(value) = format!("Bearer {}", api_key).parse() {
headers.insert("Authorization", value);
}
}
if let Some(key) = idempotency_key {
// Gracefully skip invalid idempotency keys instead of panicking
if let Ok(value) = key.parse() {
headers.insert("x-idempotency-key", value);
}
}
headers
}
/// Prepares headers for multipart requests (without a fixed content type).
pub(crate) fn prepare_multipart_headers(
&self,
idempotency_key: Option<&String>,
) -> reqwest::header::HeaderMap {
let mut headers = self.prepare_headers(idempotency_key);
headers.remove(reqwest::header::CONTENT_TYPE);
headers
}
/// Handles API responses, parsing JSON and handling errors.
pub(crate) async fn handle_response<T: DeserializeOwned>(
&self,
response: Response,
action: impl AsRef<str>,
) -> Result<T, FirecrawlError> {
let (is_success, status) = (response.status().is_success(), response.status());
let response = response
.text()
.await
.map_err(FirecrawlError::ResponseParseErrorText)
.and_then(|response_json| {
serde_json::from_str::<Value>(&response_json)
.map_err(FirecrawlError::ResponseParseError)
})
.and_then(|response_value| {
// Check for success field, or allow responses without it for status checks
if action.as_ref().contains("status")
|| action.as_ref().contains("cancel")
|| response_value["success"].as_bool().unwrap_or(false)
|| response_value.get("success").is_none()
{
serde_json::from_value::<T>(response_value)
.map_err(FirecrawlError::ResponseParseError)
} else {
Err(FirecrawlError::APIError(
action.as_ref().to_string(),
serde_json::from_value(response_value)
.map_err(FirecrawlError::ResponseParseError)?,
))
}
});
match &response {
Ok(_) => response,
Err(FirecrawlError::ResponseParseError(_))
| Err(FirecrawlError::ResponseParseErrorText(_)) => {
if is_success {
response
} else {
Err(FirecrawlError::HttpRequestFailed(
action.as_ref().to_string(),
status.as_u16(),
status.as_str().to_string(),
))
}
}
Err(_) => response,
}
}
/// Builds the full URL for an API endpoint.
pub(crate) fn url(&self, path: &str) -> String {
format!("{}{}{}", self.api_url, API_VERSION, path)
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_new_client() {
let client = Client::new("test-api-key").unwrap();
assert_eq!(client.api_key, Some("test-api-key".to_string()));
assert_eq!(client.api_url, CLOUD_API_URL);
}
#[test]
fn test_new_client_requires_api_key_for_cloud() {
let result = Client::new_selfhosted(CLOUD_API_URL, None::<&str>);
assert!(result.is_err());
}
#[test]
fn test_new_client_rejects_empty_api_key_for_cloud() {
let result = Client::new_selfhosted(CLOUD_API_URL, Some(""));
assert!(result.is_err());
let result = Client::new_selfhosted(CLOUD_API_URL, Some(" "));
assert!(result.is_err());
}
#[test]
fn test_new_selfhosted_client() {
let client = Client::new_selfhosted("http://localhost:3000", Some("api-key")).unwrap();
assert_eq!(client.api_key, Some("api-key".to_string()));
assert_eq!(client.api_url, "http://localhost:3000");
}
#[test]
fn test_selfhosted_without_api_key() {
let client = Client::new_selfhosted("http://localhost:3000", None::<&str>).unwrap();
assert_eq!(client.api_key, None);
assert_eq!(client.api_url, "http://localhost:3000");
}
#[test]
fn test_url_builder() {
let client = Client::new("test-key").unwrap();
assert_eq!(client.url("/scrape"), "https://api.firecrawl.dev/v2/scrape");
}
#[test]
fn test_url_normalization_trailing_slash() {
// Cloud URL with trailing slash should still require API key
let result = Client::new_selfhosted("https://api.firecrawl.dev/", None::<&str>);
assert!(result.is_err());
// Should work with API key
let client = Client::new_selfhosted("https://api.firecrawl.dev/", Some("key")).unwrap();
assert_eq!(client.api_url, "https://api.firecrawl.dev");
// Self-hosted URL normalization
let client = Client::new_selfhosted("http://localhost:3000/", None::<&str>).unwrap();
assert_eq!(client.api_url, "http://localhost:3000");
}
}

611
참고/firecrawl-main/apps/rust-sdk/src/crawl.rs Normal file
View File

@@ -0,0 +1,611 @@
//! Crawl endpoint for Firecrawl API v2.
use serde::{Deserialize, Serialize};
use crate::client::Client;
use crate::scrape::ScrapeOptions;
use crate::types::{CrawlErrorsResponse, Document, JobStatus, SitemapMode, WebhookConfig};
use crate::FirecrawlError;
/// Options for crawling a website.
#[serde_with::skip_serializing_none]
#[derive(Deserialize, Serialize, Debug, Default, Clone)]
#[serde(rename_all = "camelCase")]
pub struct CrawlOptions {
/// Natural language prompt to guide crawl behavior.
pub prompt: Option<String>,
/// URL path patterns to exclude from crawling.
pub exclude_paths: Option<Vec<String>>,
/// URL path patterns to include in crawling.
pub include_paths: Option<Vec<String>>,
/// Maximum depth of links to follow from the initial URL.
pub max_discovery_depth: Option<u32>,
/// How to handle the sitemap.
pub sitemap: Option<SitemapMode>,
/// Ignore query parameters when deduplicating URLs.
pub ignore_query_parameters: Option<bool>,
/// Maximum number of pages to crawl.
pub limit: Option<u32>,
/// Crawl the entire domain regardless of path structure.
pub crawl_entire_domain: Option<bool>,
/// Allow following links to external domains.
pub allow_external_links: Option<bool>,
/// Allow following links to subdomains.
pub allow_subdomains: Option<bool>,
/// Delay between requests in seconds.
pub delay: Option<u32>,
/// Maximum concurrent requests.
pub max_concurrency: Option<u32>,
/// Webhook configuration for job notifications.
pub webhook: Option<WebhookConfig>,
/// Scrape options to apply to each page.
pub scrape_options: Option<ScrapeOptions>,
/// Enable zero data retention mode.
pub zero_data_retention: Option<bool>,
/// Integration identifier for tracking.
pub integration: Option<String>,
/// Idempotency key for the request.
#[serde(skip)]
pub idempotency_key: Option<String>,
/// Poll interval for synchronous crawl (milliseconds).
#[serde(skip)]
pub poll_interval: Option<u64>,
}
/// Request body for crawl endpoint.
#[derive(Deserialize, Serialize, Debug, Default)]
#[serde(rename_all = "camelCase")]
struct CrawlRequest {
url: String,
#[serde(flatten)]
options: CrawlOptions,
}
/// Response from starting a crawl job.
#[derive(Deserialize, Serialize, Debug, Clone)]
#[serde(rename_all = "camelCase")]
pub struct CrawlResponse {
/// Whether the request was successful.
pub success: bool,
/// The crawl job ID.
pub id: String,
/// URL to check the crawl status.
pub url: String,
}
/// Status of a crawl job.
#[serde_with::skip_serializing_none]
#[derive(Deserialize, Serialize, Debug, Clone)]
#[serde(rename_all = "camelCase")]
pub struct CrawlJob {
/// Current status of the crawl job.
pub status: JobStatus,
/// Total number of pages to crawl.
pub total: u32,
/// Number of pages completed.
pub completed: u32,
/// Credits used by the crawl.
pub credits_used: Option<u32>,
/// Expiry time of the crawl data.
pub expires_at: Option<String>,
/// URL for the next page of results.
pub next: Option<String>,
/// Crawled documents.
pub data: Vec<Document>,
}
/// Response from canceling a crawl.
#[derive(Deserialize, Serialize, Debug, Clone)]
#[serde(rename_all = "camelCase")]
pub struct CancelCrawlResponse {
/// Status of the cancellation.
pub status: String,
}
impl Client {
/// Starts a crawl job asynchronously.
///
/// Returns immediately with a job ID that can be used to check status.
///
/// # Arguments
///
/// * `url` - The URL to start crawling from.
/// * `options` - Optional crawl configuration.
///
/// # Returns
///
/// A `CrawlResponse` containing the job ID.
///
/// # Example
///
/// ```no_run
/// use firecrawl::{Client, CrawlOptions};
///
/// #[tokio::main]
/// async fn main() -> Result<(), Box<dyn std::error::Error>> {
/// let client = Client::new("your-api-key")?;
///
/// let response = client.start_crawl("https://example.com", None).await?;
/// println!("Crawl job started: {}", response.id);
///
/// // Check status later
/// let status = client.get_crawl_status(&response.id).await?;
/// println!("Status: {:?}, Completed: {}/{}", status.status, status.completed, status.total);
///
/// Ok(())
/// }
/// ```
pub async fn start_crawl(
&self,
url: impl AsRef<str>,
options: impl Into<Option<CrawlOptions>>,
) -> Result<CrawlResponse, FirecrawlError> {
let options = options.into().unwrap_or_default();
let body = CrawlRequest {
url: url.as_ref().to_string(),
options: options.clone(),
};
let headers = self.prepare_headers(options.idempotency_key.as_ref());
let response = self
.client
.post(self.url("/crawl"))
.headers(headers)
.json(&body)
.send()
.await
.map_err(|e| {
FirecrawlError::HttpError(format!("Starting crawl of {:?}", url.as_ref()), e)
})?;
self.handle_response(response, "start crawl").await
}
/// Gets the status of a crawl job.
///
/// If the job is completed, this will automatically fetch all pages of results.
///
/// # Arguments
///
/// * `id` - The crawl job ID.
///
/// # Returns
///
/// A `CrawlJob` containing the current status and any available documents.
///
/// # Example
///
/// ```no_run
/// use firecrawl::Client;
///
/// #[tokio::main]
/// async fn main() -> Result<(), Box<dyn std::error::Error>> {
/// let client = Client::new("your-api-key")?;
///
/// let status = client.get_crawl_status("job-id").await?;
/// println!("Status: {:?}", status.status);
/// println!("Completed: {}/{}", status.completed, status.total);
/// println!("Documents: {}", status.data.len());
///
/// Ok(())
/// }
/// ```
pub async fn get_crawl_status(&self, id: impl AsRef<str>) -> Result<CrawlJob, FirecrawlError> {
let response = self
.client
.get(self.url(&format!("/crawl/{}", id.as_ref())))
.headers(self.prepare_headers(None))
.send()
.await
.map_err(|e| {
FirecrawlError::HttpError(format!("Checking crawl status {}", id.as_ref()), e)
})?;
let mut status: CrawlJob = self
.handle_response(response, format!("crawl status {}", id.as_ref()))
.await?;
// Auto-paginate if completed
if status.status == JobStatus::Completed {
while let Some(next) = status.next.take() {
let next_status = self.get_crawl_status_next(&next).await?;
status.data.extend(next_status.data);
status.next = next_status.next;
}
}
Ok(status)
}
/// Fetches the next page of crawl results.
async fn get_crawl_status_next(&self, next: &str) -> Result<CrawlJob, FirecrawlError> {
let response = self
.client
.get(next)
.headers(self.prepare_headers(None))
.send()
.await
.map_err(|e| FirecrawlError::HttpError(format!("Paginating crawl at {}", next), e))?;
self.handle_response(response, "crawl pagination").await
}
/// Crawls a website and waits for completion.
///
/// This method starts a crawl and polls until it completes or fails.
///
/// # Arguments
///
/// * `url` - The URL to start crawling from.
/// * `options` - Optional crawl configuration.
///
/// # Returns
///
/// A `CrawlJob` containing all crawled documents.
///
/// # Example
///
/// ```no_run
/// use firecrawl::{Client, CrawlOptions, SitemapMode};
///
/// #[tokio::main]
/// async fn main() -> Result<(), Box<dyn std::error::Error>> {
/// let client = Client::new("your-api-key")?;
///
/// let options = CrawlOptions {
/// sitemap: Some(SitemapMode::Include),
/// limit: Some(100),
/// poll_interval: Some(5000), // Check every 5 seconds
/// ..Default::default()
/// };
///
/// let result = client.crawl("https://example.com", options).await?;
/// println!("Crawled {} pages", result.data.len());
///
/// for doc in result.data {
/// println!("URL: {:?}", doc.metadata.and_then(|m| m.source_url));
/// }
///
/// Ok(())
/// }
/// ```
pub async fn crawl(
&self,
url: impl AsRef<str>,
options: impl Into<Option<CrawlOptions>>,
) -> Result<CrawlJob, FirecrawlError> {
let options = options.into().unwrap_or_default();
let poll_interval = options.poll_interval.unwrap_or(2000);
let response = self.start_crawl(url, options).await?;
self.wait_for_crawl(&response.id, poll_interval).await
}
/// Waits for a crawl job to complete.
async fn wait_for_crawl(
&self,
id: &str,
poll_interval: u64,
) -> Result<CrawlJob, FirecrawlError> {
loop {
let status = self.get_crawl_status(id).await?;
match status.status {
JobStatus::Completed => return Ok(status),
JobStatus::Scraping => {
tokio::time::sleep(tokio::time::Duration::from_millis(poll_interval)).await;
}
JobStatus::Failed => {
return Err(FirecrawlError::JobFailed(
"Crawl job failed".to_string(),
JobStatus::Failed,
));
}
JobStatus::Cancelled => {
return Err(FirecrawlError::JobFailed(
"Crawl job was cancelled".to_string(),
JobStatus::Cancelled,
));
}
}
}
}
/// Cancels a running crawl job.
///
/// # Arguments
///
/// * `id` - The crawl job ID to cancel.
///
/// # Returns
///
/// A `CancelCrawlResponse` indicating the cancellation status.
///
/// # Example
///
/// ```no_run
/// use firecrawl::Client;
///
/// #[tokio::main]
/// async fn main() -> Result<(), Box<dyn std::error::Error>> {
/// let client = Client::new("your-api-key")?;
///
/// let response = client.cancel_crawl("job-id").await?;
/// println!("Cancellation status: {}", response.status);
///
/// Ok(())
/// }
/// ```
pub async fn cancel_crawl(
&self,
id: impl AsRef<str>,
) -> Result<CancelCrawlResponse, FirecrawlError> {
let response = self
.client
.delete(self.url(&format!("/crawl/{}", id.as_ref())))
.headers(self.prepare_headers(None))
.send()
.await
.map_err(|e| {
FirecrawlError::HttpError(format!("Cancelling crawl {}", id.as_ref()), e)
})?;
self.handle_response(response, "cancel crawl").await
}
/// Gets errors from a crawl job.
///
/// # Arguments
///
/// * `id` - The crawl job ID.
///
/// # Returns
///
/// A `CrawlErrorsResponse` containing error details.
///
/// # Example
///
/// ```no_run
/// use firecrawl::Client;
///
/// #[tokio::main]
/// async fn main() -> Result<(), Box<dyn std::error::Error>> {
/// let client = Client::new("your-api-key")?;
///
/// let errors = client.get_crawl_errors("job-id").await?;
/// for error in errors.errors {
/// println!("Error on {}: {}", error.url, error.error);
/// }
///
/// Ok(())
/// }
/// ```
pub async fn get_crawl_errors(
&self,
id: impl AsRef<str>,
) -> Result<CrawlErrorsResponse, FirecrawlError> {
let response = self
.client
.get(self.url(&format!("/crawl/{}/errors", id.as_ref())))
.headers(self.prepare_headers(None))
.send()
.await
.map_err(|e| {
FirecrawlError::HttpError(format!("Getting crawl errors {}", id.as_ref()), e)
})?;
self.handle_response(response, "crawl errors").await
}
}
#[cfg(test)]
mod tests {
use super::*;
use serde_json::json;
#[tokio::test]
async fn test_start_crawl_with_mock() {
let mut server = mockito::Server::new_async().await;
let mock = server
.mock("POST", "/v2/crawl")
.with_status(200)
.with_header("content-type", "application/json")
.with_body(
json!({
"success": true,
"id": "crawl-123",
"url": "https://api.firecrawl.dev/v2/crawl/crawl-123"
})
.to_string(),
)
.create();
let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();
let response = client
.start_crawl("https://example.com", None)
.await
.unwrap();
assert!(response.success);
assert_eq!(response.id, "crawl-123");
mock.assert();
}
#[tokio::test]
async fn test_get_crawl_status_with_mock() {
let mut server = mockito::Server::new_async().await;
let mock = server
.mock("GET", "/v2/crawl/crawl-123")
.with_status(200)
.with_header("content-type", "application/json")
.with_body(
json!({
"status": "completed",
"total": 5,
"completed": 5,
"creditsUsed": 5,
"expiresAt": "2024-12-31T23:59:59Z",
"data": [
{
"markdown": "# Page 1",
"metadata": {
"sourceURL": "https://example.com/page1",
"statusCode": 200
}
},
{
"markdown": "# Page 2",
"metadata": {
"sourceURL": "https://example.com/page2",
"statusCode": 200
}
}
]
})
.to_string(),
)
.create();
let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();
let status = client.get_crawl_status("crawl-123").await.unwrap();
assert_eq!(status.status, JobStatus::Completed);
assert_eq!(status.total, 5);
assert_eq!(status.completed, 5);
assert_eq!(status.data.len(), 2);
mock.assert();
}
#[tokio::test]
async fn test_cancel_crawl_with_mock() {
let mut server = mockito::Server::new_async().await;
let mock = server
.mock("DELETE", "/v2/crawl/crawl-123")
.with_status(200)
.with_header("content-type", "application/json")
.with_body(
json!({
"status": "cancelled"
})
.to_string(),
)
.create();
let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();
let response = client.cancel_crawl("crawl-123").await.unwrap();
assert_eq!(response.status, "cancelled");
mock.assert();
}
#[tokio::test]
async fn test_get_crawl_errors_with_mock() {
let mut server = mockito::Server::new_async().await;
let mock = server
.mock("GET", "/v2/crawl/crawl-123/errors")
.with_status(200)
.with_header("content-type", "application/json")
.with_body(
json!({
"errors": [
{
"id": "error-1",
"timestamp": "2024-01-01T00:00:00Z",
"url": "https://example.com/broken",
"error": "404 Not Found"
}
],
"robotsBlocked": [
"https://example.com/admin"
]
})
.to_string(),
)
.create();
let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();
let errors = client.get_crawl_errors("crawl-123").await.unwrap();
assert_eq!(errors.errors.len(), 1);
assert_eq!(errors.errors[0].url, "https://example.com/broken");
assert_eq!(errors.robots_blocked.len(), 1);
mock.assert();
}
#[tokio::test]
async fn test_crawl_with_options() {
let mut server = mockito::Server::new_async().await;
// Mock the start endpoint
let start_mock = server
.mock("POST", "/v2/crawl")
.with_status(200)
.with_header("content-type", "application/json")
.with_body(
json!({
"success": true,
"id": "crawl-456",
"url": "https://api.firecrawl.dev/v2/crawl/crawl-456"
})
.to_string(),
)
.create();
// Mock the status endpoint (completed immediately)
let status_mock = server
.mock("GET", "/v2/crawl/crawl-456")
.with_status(200)
.with_header("content-type", "application/json")
.with_body(
json!({
"status": "completed",
"total": 2,
"completed": 2,
"data": [
{
"markdown": "# Page 1",
"metadata": { "sourceURL": "https://example.com/1", "statusCode": 200 }
}
]
})
.to_string(),
)
.create();
let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();
let options = CrawlOptions {
limit: Some(10),
sitemap: Some(SitemapMode::Include),
..Default::default()
};
let result = client.crawl("https://example.com", options).await.unwrap();
assert_eq!(result.status, JobStatus::Completed);
assert_eq!(result.data.len(), 1);
start_mock.assert();
status_mock.assert();
}
}

45
참고/firecrawl-main/apps/rust-sdk/src/error.rs Normal file
View File

@@ -0,0 +1,45 @@
use std::fmt::Display;
use serde::{Deserialize, Serialize};
use serde_json::Value;
use thiserror::Error;
#[derive(Debug, Deserialize, Serialize, Clone)]
pub struct FirecrawlAPIError {
/// Always false.
pub success: bool,
/// Error message
pub error: String,
/// Additional details of this error. Schema depends on the error itself.
pub details: Option<Value>,
}
impl Display for FirecrawlAPIError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
if let Some(details) = self.details.as_ref() {
write!(f, "{} ({})", self.error, details)
} else {
write!(f, "{}", self.error)
}
}
}
#[derive(Error, Debug)]
pub enum FirecrawlError {
#[error("{0} failed: HTTP error {1}: {2}")]
HttpRequestFailed(String, u16, String),
#[error("{0} failed: HTTP error: {1}")]
HttpError(String, reqwest::Error),
#[error("Failed to parse response as text: {0}")]
ResponseParseErrorText(reqwest::Error),
#[error("Failed to parse response: {0}")]
ResponseParseError(serde_json::Error),
#[error("{0} failed: {1}")]
APIError(String, FirecrawlAPIError),
#[error("Job failed: {0} (status: {1:?})")]
JobFailed(String, crate::types::JobStatus),
#[error("Misuse: {0}")]
Misuse(String),
}

44
참고/firecrawl-main/apps/rust-sdk/src/lib.rs Normal file
View File

@@ -0,0 +1,44 @@
//! Firecrawl Rust SDK
//!
//! This SDK provides access to the Firecrawl v2 API for web scraping, crawling,
//! searching, mapping, batch scraping, and agent operations.
//!
//! # Quick Start
//!
//! ```no_run
//! use firecrawl::Client;
//!
//! #[tokio::main]
//! async fn main() -> Result<(), Box<dyn std::error::Error>> {
//! let client = Client::new("your-api-key")?;
//! let document = client.scrape("https://example.com", None).await?;
//! println!("{:?}", document.markdown);
//! Ok(())
//! }
//! ```
pub mod error;
pub(crate) mod serde_helpers;
mod agent;
mod batch_scrape;
mod client;
mod crawl;
mod map;
mod monitor;
mod parse;
mod scrape;
mod search;
mod types;
pub use agent::*;
pub use batch_scrape::*;
pub use client::Client;
pub use crawl::*;
pub use error::FirecrawlError;
pub use map::*;
pub use monitor::*;
pub use parse::*;
pub use scrape::*;
pub use search::*;
pub use types::*;

333
참고/firecrawl-main/apps/rust-sdk/src/map.rs Normal file
View File

@@ -0,0 +1,333 @@
//! Map endpoint for Firecrawl API v2.
use serde::{Deserialize, Serialize};
use crate::client::Client;
use crate::types::{LocationConfig, SearchResultWeb, SitemapMode};
use crate::FirecrawlError;
/// Options for mapping a URL.
#[serde_with::skip_serializing_none]
#[derive(Deserialize, Serialize, Debug, Default, Clone)]
#[serde(rename_all = "camelCase")]
pub struct MapOptions {
/// Search query to filter discovered links.
pub search: Option<String>,
/// How to handle the sitemap.
pub sitemap: Option<SitemapMode>,
/// Include subdomains in the mapping.
pub include_subdomains: Option<bool>,
/// Ignore query parameters when deduplicating URLs.
pub ignore_query_parameters: Option<bool>,
/// Maximum number of links to return.
pub limit: Option<u32>,
/// Timeout in milliseconds.
pub timeout: Option<u32>,
/// Integration identifier for tracking.
pub integration: Option<String>,
/// Location configuration for proxy routing.
pub location: Option<LocationConfig>,
}
/// Request body for map endpoint.
#[derive(Deserialize, Serialize, Debug, Default)]
#[serde(rename_all = "camelCase")]
struct MapRequest {
url: String,
#[serde(flatten)]
options: MapOptions,
}
/// Response from map endpoint.
#[derive(Deserialize, Serialize, Debug)]
#[serde(rename_all = "camelCase")]
pub struct MapResponse {
/// Whether the request was successful.
pub success: bool,
/// Discovered links with metadata.
pub links: Vec<SearchResultWeb>,
/// Warning message if any.
pub warning: Option<String>,
}
impl Client {
/// Maps a URL to discover all associated links.
///
/// This endpoint discovers links from a website's sitemap, page content,
/// and other sources without fully scraping each page.
///
/// # Arguments
///
/// * `url` - The URL to map.
/// * `options` - Optional mapping configuration.
///
/// # Returns
///
/// A `MapResponse` containing the discovered links.
///
/// # Example
///
/// ```no_run
/// use firecrawl::{Client, MapOptions, SitemapMode};
///
/// #[tokio::main]
/// async fn main() -> Result<(), Box<dyn std::error::Error>> {
/// let client = Client::new("your-api-key")?;
///
/// // Simple map
/// let response = client.map("https://example.com", None).await?;
/// println!("Found {} links", response.links.len());
///
/// // Map with options
/// let options = MapOptions {
/// sitemap: Some(SitemapMode::Include),
/// include_subdomains: Some(true),
/// limit: Some(1000),
/// ..Default::default()
/// };
/// let response = client.map("https://example.com", options).await?;
///
/// for link in response.links {
/// println!("URL: {}, Title: {:?}", link.url, link.title);
/// }
///
/// Ok(())
/// }
/// ```
pub async fn map(
&self,
url: impl AsRef<str>,
options: impl Into<Option<MapOptions>>,
) -> Result<MapResponse, FirecrawlError> {
let body = MapRequest {
url: url.as_ref().to_string(),
options: options.into().unwrap_or_default(),
};
let headers = self.prepare_headers(None);
let response = self
.client
.post(self.url("/map"))
.headers(headers)
.json(&body)
.send()
.await
.map_err(|e| FirecrawlError::HttpError(format!("Mapping {:?}", url.as_ref()), e))?;
self.handle_response(response, "map").await
}
/// Maps a URL and returns just the list of URLs.
///
/// This is a convenience method that returns only the URL strings.
///
/// # Arguments
///
/// * `url` - The URL to map.
/// * `options` - Optional mapping configuration.
///
/// # Returns
///
/// A vector of discovered URL strings.
///
/// # Example
///
/// ```no_run
/// use firecrawl::Client;
///
/// #[tokio::main]
/// async fn main() -> Result<(), Box<dyn std::error::Error>> {
/// let client = Client::new("your-api-key")?;
///
/// let urls = client.map_urls("https://example.com", None).await?;
/// for url in urls {
/// println!("{}", url);
/// }
///
/// Ok(())
/// }
/// ```
pub async fn map_urls(
&self,
url: impl AsRef<str>,
options: impl Into<Option<MapOptions>>,
) -> Result<Vec<String>, FirecrawlError> {
let response = self.map(url, options).await?;
Ok(response.links.into_iter().map(|link| link.url).collect())
}
}
#[cfg(test)]
mod tests {
use super::*;
use serde_json::json;
#[tokio::test]
async fn test_map_with_mock() {
let mut server = mockito::Server::new_async().await;
let mock = server
.mock("POST", "/v2/map")
.with_status(200)
.with_header("content-type", "application/json")
.with_body(
json!({
"success": true,
"links": [
{
"url": "https://example.com/",
"title": "Example Domain",
"description": "Home page"
},
{
"url": "https://example.com/about",
"title": "About Us"
},
{
"url": "https://example.com/contact"
}
]
})
.to_string(),
)
.create();
let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();
let response = client.map("https://example.com", None).await.unwrap();
assert!(response.success);
assert_eq!(response.links.len(), 3);
assert_eq!(response.links[0].url, "https://example.com/");
assert_eq!(response.links[0].title, Some("Example Domain".to_string()));
mock.assert();
}
#[tokio::test]
async fn test_map_with_options() {
let mut server = mockito::Server::new_async().await;
let mock = server
.mock("POST", "/v2/map")
.with_status(200)
.with_header("content-type", "application/json")
.with_body(
json!({
"success": true,
"links": [
{ "url": "https://example.com/page1" },
{ "url": "https://example.com/page2" }
]
})
.to_string(),
)
.create();
let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();
let options = MapOptions {
sitemap: Some(SitemapMode::Include),
include_subdomains: Some(true),
limit: Some(100),
..Default::default()
};
let response = client.map("https://example.com", options).await.unwrap();
assert!(response.success);
assert_eq!(response.links.len(), 2);
mock.assert();
}
#[tokio::test]
async fn test_map_urls() {
let mut server = mockito::Server::new_async().await;
let mock = server
.mock("POST", "/v2/map")
.with_status(200)
.with_header("content-type", "application/json")
.with_body(
json!({
"success": true,
"links": [
{ "url": "https://example.com/page1" },
{ "url": "https://example.com/page2" },
{ "url": "https://example.com/page3" }
]
})
.to_string(),
)
.create();
let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();
let urls = client.map_urls("https://example.com", None).await.unwrap();
assert_eq!(urls.len(), 3);
assert_eq!(urls[0], "https://example.com/page1");
mock.assert();
}
#[tokio::test]
async fn test_map_with_search_filter() {
let mut server = mockito::Server::new_async().await;
let mock = server
.mock("POST", "/v2/map")
.with_status(200)
.with_header("content-type", "application/json")
.with_body(
json!({
"success": true,
"links": [
{ "url": "https://example.com/blog/post1" },
{ "url": "https://example.com/blog/post2" }
]
})
.to_string(),
)
.create();
let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();
let options = MapOptions {
search: Some("blog".to_string()),
..Default::default()
};
let response = client.map("https://example.com", options).await.unwrap();
assert!(response.success);
assert_eq!(response.links.len(), 2);
mock.assert();
}
#[tokio::test]
async fn test_map_error_response() {
let mut server = mockito::Server::new_async().await;
let mock = server
.mock("POST", "/v2/map")
.with_status(400)
.with_header("content-type", "application/json")
.with_body(
json!({
"success": false,
"error": "Invalid URL"
})
.to_string(),
)
.create();
let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();
let result = client.map("invalid-url", None).await;
assert!(result.is_err());
mock.assert();
}
}

370
참고/firecrawl-main/apps/rust-sdk/src/monitor.rs Normal file
View File

@@ -0,0 +1,370 @@
//! Monitor endpoint for Firecrawl API v2.
use serde::{Deserialize, Serialize};
use serde_json::Value;
use crate::client::Client;
use crate::FirecrawlError;
#[derive(Deserialize, Serialize, Debug, Clone)]
#[serde(rename_all = "camelCase")]
pub struct MonitorSchedule {
pub cron: String,
pub timezone: Option<String>,
}
#[serde_with::skip_serializing_none]
#[derive(Deserialize, Serialize, Debug, Clone)]
#[serde(rename_all = "camelCase")]
pub struct CreateMonitorRequest {
pub name: String,
pub schedule: MonitorSchedule,
pub targets: Vec<Value>,
pub webhook: Option<Value>,
pub notification: Option<Value>,
pub retention_days: Option<u32>,
}
#[serde_with::skip_serializing_none]
#[derive(Deserialize, Serialize, Debug, Default, Clone)]
#[serde(rename_all = "camelCase")]
pub struct UpdateMonitorRequest {
pub name: Option<String>,
pub status: Option<String>,
pub schedule: Option<MonitorSchedule>,
pub targets: Option<Vec<Value>>,
pub webhook: Option<Value>,
pub notification: Option<Value>,
pub retention_days: Option<u32>,
}
#[derive(Deserialize, Serialize, Debug, Default, Clone)]
#[serde(rename_all = "camelCase")]
pub struct MonitorSummary {
pub total_pages: u32,
pub same: u32,
pub changed: u32,
pub new: u32,
pub removed: u32,
pub error: u32,
}
#[derive(Deserialize, Serialize, Debug, Clone)]
#[serde(rename_all = "camelCase")]
pub struct Monitor {
pub id: String,
pub name: String,
pub status: String,
pub schedule: MonitorSchedule,
pub next_run_at: Option<String>,
pub last_run_at: Option<String>,
pub current_check_id: Option<String>,
pub targets: Vec<Value>,
pub webhook: Option<Value>,
pub notification: Option<Value>,
pub retention_days: u32,
pub estimated_credits_per_month: Option<u32>,
pub last_check_summary: Option<MonitorSummary>,
pub created_at: String,
pub updated_at: String,
}
#[derive(Deserialize, Serialize, Debug, Clone)]
#[serde(rename_all = "camelCase")]
pub struct MonitorCheck {
pub id: String,
pub monitor_id: String,
pub status: String,
pub trigger: String,
pub scheduled_for: Option<String>,
pub started_at: Option<String>,
pub finished_at: Option<String>,
pub estimated_credits: Option<u32>,
pub reserved_credits: Option<u32>,
pub actual_credits: Option<u32>,
pub billing_status: String,
pub summary: MonitorSummary,
pub target_results: Option<Value>,
pub notification_status: Option<Value>,
pub error: Option<String>,
pub created_at: String,
pub updated_at: String,
}
#[derive(Deserialize, Serialize, Debug, Clone)]
#[serde(rename_all = "camelCase")]
pub struct MonitorCheckPage {
pub id: String,
pub target_id: String,
pub url: String,
pub status: String,
pub previous_scrape_id: Option<String>,
pub current_scrape_id: Option<String>,
pub status_code: Option<u16>,
pub error: Option<String>,
pub metadata: Option<Value>,
pub diff: Option<Value>,
pub created_at: String,
}
#[derive(Deserialize, Serialize, Debug, Clone)]
#[serde(rename_all = "camelCase")]
pub struct MonitorCheckDetail {
#[serde(flatten)]
pub check: MonitorCheck,
pub pages: Vec<MonitorCheckPage>,
pub next: Option<String>,
}
#[derive(Deserialize, Debug)]
#[serde(rename_all = "camelCase")]
struct DataResponse<T> {
data: T,
}
#[derive(Deserialize, Debug)]
#[serde(rename_all = "camelCase")]
struct SuccessResponse {
success: bool,
}
fn query(limit: Option<u32>, offset: Option<u32>, status: Option<&str>) -> String {
let mut params = Vec::new();
if let Some(limit) = limit {
params.push(format!("limit={}", limit));
}
if let Some(offset) = offset {
params.push(format!("offset={}", offset));
}
if let Some(status) = status {
params.push(format!("status={}", status));
}
if params.is_empty() {
String::new()
} else {
format!("?{}", params.join("&"))
}
}
fn check_page_query(limit: Option<u32>, skip: Option<u32>, status: Option<&str>) -> String {
let mut params = Vec::new();
if let Some(limit) = limit {
params.push(format!("limit={}", limit));
}
if let Some(skip) = skip {
params.push(format!("skip={}", skip));
}
if let Some(status) = status {
params.push(format!("status={}", status));
}
if params.is_empty() {
String::new()
} else {
format!("?{}", params.join("&"))
}
}
impl Client {
pub async fn create_monitor(
&self,
request: CreateMonitorRequest,
) -> Result<Monitor, FirecrawlError> {
let response = self
.client
.post(self.url("/monitor"))
.headers(self.prepare_headers(None))
.json(&request)
.send()
.await
.map_err(|e| FirecrawlError::HttpError("Creating monitor".to_string(), e))?;
let response: DataResponse<Monitor> =
self.handle_response(response, "create monitor").await?;
Ok(response.data)
}
pub async fn list_monitors(
&self,
limit: Option<u32>,
offset: Option<u32>,
) -> Result<Vec<Monitor>, FirecrawlError> {
let response = self
.client
.get(self.url(&format!("/monitor{}", query(limit, offset, None))))
.headers(self.prepare_headers(None))
.send()
.await
.map_err(|e| FirecrawlError::HttpError("Listing monitors".to_string(), e))?;
let response: DataResponse<Vec<Monitor>> =
self.handle_response(response, "list monitors").await?;
Ok(response.data)
}
pub async fn get_monitor(
&self,
monitor_id: impl AsRef<str>,
) -> Result<Monitor, FirecrawlError> {
let response = self
.client
.get(self.url(&format!("/monitor/{}", monitor_id.as_ref())))
.headers(self.prepare_headers(None))
.send()
.await
.map_err(|e| FirecrawlError::HttpError("Getting monitor".to_string(), e))?;
let response: DataResponse<Monitor> = self.handle_response(response, "get monitor").await?;
Ok(response.data)
}
pub async fn update_monitor(
&self,
monitor_id: impl AsRef<str>,
request: UpdateMonitorRequest,
) -> Result<Monitor, FirecrawlError> {
let response = self
.client
.patch(self.url(&format!("/monitor/{}", monitor_id.as_ref())))
.headers(self.prepare_headers(None))
.json(&request)
.send()
.await
.map_err(|e| FirecrawlError::HttpError("Updating monitor".to_string(), e))?;
let response: DataResponse<Monitor> =
self.handle_response(response, "update monitor").await?;
Ok(response.data)
}
pub async fn delete_monitor(
&self,
monitor_id: impl AsRef<str>,
) -> Result<bool, FirecrawlError> {
let response = self
.client
.delete(self.url(&format!("/monitor/{}", monitor_id.as_ref())))
.headers(self.prepare_headers(None))
.send()
.await
.map_err(|e| FirecrawlError::HttpError("Deleting monitor".to_string(), e))?;
let response: SuccessResponse = self.handle_response(response, "delete monitor").await?;
Ok(response.success)
}
pub async fn run_monitor(
&self,
monitor_id: impl AsRef<str>,
) -> Result<MonitorCheck, FirecrawlError> {
let response = self
.client
.post(self.url(&format!("/monitor/{}/run", monitor_id.as_ref())))
.headers(self.prepare_headers(None))
.json(&serde_json::json!({}))
.send()
.await
.map_err(|e| FirecrawlError::HttpError("Running monitor".to_string(), e))?;
let response: DataResponse<MonitorCheck> =
self.handle_response(response, "run monitor").await?;
Ok(response.data)
}
pub async fn list_monitor_checks(
&self,
monitor_id: impl AsRef<str>,
limit: Option<u32>,
offset: Option<u32>,
) -> Result<Vec<MonitorCheck>, FirecrawlError> {
let path = format!(
"/monitor/{}/checks{}",
monitor_id.as_ref(),
query(limit, offset, None)
);
let response = self
.client
.get(self.url(&path))
.headers(self.prepare_headers(None))
.send()
.await
.map_err(|e| FirecrawlError::HttpError("Listing monitor checks".to_string(), e))?;
let response: DataResponse<Vec<MonitorCheck>> = self
.handle_response(response, "list monitor checks")
.await?;
Ok(response.data)
}
pub async fn get_monitor_check(
&self,
monitor_id: impl AsRef<str>,
check_id: impl AsRef<str>,
limit: Option<u32>,
skip: Option<u32>,
status: Option<&str>,
) -> Result<MonitorCheckDetail, FirecrawlError> {
let path = format!(
"/monitor/{}/checks/{}{}",
monitor_id.as_ref(),
check_id.as_ref(),
check_page_query(limit, skip, status)
);
let response = self
.client
.get(self.url(&path))
.headers(self.prepare_headers(None))
.send()
.await
.map_err(|e| FirecrawlError::HttpError("Getting monitor check".to_string(), e))?;
let response: DataResponse<MonitorCheckDetail> =
self.handle_response(response, "get monitor check").await?;
let mut check = response.data;
while let Some(next) = check.next.clone() {
let response = self
.client
.get(next)
.headers(self.prepare_headers(None))
.send()
.await
.map_err(|e| {
FirecrawlError::HttpError("Getting monitor check page".to_string(), e)
})?;
let response: DataResponse<MonitorCheckDetail> = self
.handle_response(response, "get monitor check page")
.await?;
check.pages.extend(response.data.pages);
check.next = response.data.next;
}
Ok(check)
}
pub async fn get_monitor_check_page(
&self,
monitor_id: impl AsRef<str>,
check_id: impl AsRef<str>,
limit: Option<u32>,
skip: Option<u32>,
status: Option<&str>,
) -> Result<MonitorCheckDetail, FirecrawlError> {
let path = format!(
"/monitor/{}/checks/{}{}",
monitor_id.as_ref(),
check_id.as_ref(),
check_page_query(limit, skip, status)
);
let response = self
.client
.get(self.url(&path))
.headers(self.prepare_headers(None))
.send()
.await
.map_err(|e| FirecrawlError::HttpError("Getting monitor check".to_string(), e))?;
let response: DataResponse<MonitorCheckDetail> =
self.handle_response(response, "get monitor check").await?;
Ok(response.data)
}
}

284
참고/firecrawl-main/apps/rust-sdk/src/parse.rs Normal file
View File

@@ -0,0 +1,284 @@
//! Parse endpoint for Firecrawl API v2.
use std::path::Path;
use reqwest::multipart::{Form, Part};
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
use super::client::Client;
use super::scrape::ParserConfig;
use super::types::{AttributeSelector, Document, JsonOptions};
use crate::FirecrawlError;
/// Uploaded file payload for the `/v2/parse` endpoint.
#[derive(Debug, Clone)]
pub struct ParseFile {
pub filename: String,
pub bytes: Vec<u8>,
pub content_type: Option<String>,
}
impl ParseFile {
/// Build a parse file from in-memory bytes.
pub fn from_bytes(filename: impl Into<String>, bytes: Vec<u8>) -> Self {
Self {
filename: filename.into(),
bytes,
content_type: None,
}
}
/// Build a parse file by reading bytes from disk.
pub fn from_path(path: impl AsRef<Path>) -> Result<Self, FirecrawlError> {
let path_ref = path.as_ref();
let bytes = std::fs::read(path_ref).map_err(|e| {
FirecrawlError::Misuse(format!(
"Failed to read parse file {}: {}",
path_ref.display(),
e
))
})?;
let filename = path_ref
.file_name()
.and_then(|name| name.to_str())
.ok_or_else(|| {
FirecrawlError::Misuse("Could not derive a valid filename from path".to_string())
})?
.to_string();
Ok(Self {
filename,
bytes,
content_type: None,
})
}
/// Attach a content type hint (e.g. `text/html`, `application/pdf`).
pub fn with_content_type(mut self, content_type: impl Into<String>) -> Self {
self.content_type = Some(content_type.into());
self
}
}
/// Response from parse endpoint.
#[derive(Deserialize, Serialize, Debug)]
#[serde(rename_all = "camelCase")]
struct ParseResponse {
success: bool,
data: Document,
#[serde(skip_serializing_if = "Option::is_none")]
warning: Option<String>,
}
/// Proxy settings accepted by `/v2/parse`.
#[derive(Deserialize, Serialize, Clone, Copy, Debug, PartialEq, Eq)]
#[serde(rename_all = "lowercase")]
pub enum ParseProxyType {
Basic,
Auto,
}
/// Output formats accepted by `/v2/parse`.
#[derive(Deserialize, Serialize, Clone, Copy, Debug, PartialEq, Eq)]
#[serde(rename_all = "camelCase")]
pub enum ParseFormat {
Markdown,
Html,
RawHtml,
Links,
Images,
Summary,
Json,
Attributes,
}
/// Options accepted by the `/v2/parse` endpoint.
///
/// This intentionally omits scrape-only fields that `/v2/parse` rejects
/// (e.g. actions, waitFor, location, and screenshot/branding/changeTracking options).
#[serde_with::skip_serializing_none]
#[derive(Deserialize, Serialize, Debug, Default, Clone)]
#[serde(rename_all = "camelCase")]
pub struct ParseOptions {
/// Output formats to include in the response.
pub formats: Option<Vec<ParseFormat>>,
/// Additional HTTP headers.
pub headers: Option<HashMap<String, String>>,
/// HTML tags to include.
pub include_tags: Option<Vec<String>>,
/// HTML tags to exclude.
pub exclude_tags: Option<Vec<String>>,
/// Extract only the main content.
pub only_main_content: Option<bool>,
/// Timeout in milliseconds.
pub timeout: Option<u32>,
/// Parser configurations (e.g. PDF parser).
pub parsers: Option<Vec<ParserConfig>>,
/// Skip TLS verification.
pub skip_tls_verification: Option<bool>,
/// Remove base64 images.
pub remove_base64_images: Option<bool>,
/// Fast mode.
pub fast_mode: Option<bool>,
/// Mock fixture id to use.
pub use_mock: Option<String>,
/// Block ads.
pub block_ads: Option<bool>,
/// Proxy type.
pub proxy: Option<ParseProxyType>,
/// Integration identifier.
pub integration: Option<String>,
/// Request origin identifier.
pub origin: Option<String>,
/// Zero data retention mode.
pub zero_data_retention: Option<bool>,
/// JSON extraction options.
pub json_options: Option<JsonOptions>,
/// Attribute selectors for extraction.
pub attribute_selectors: Option<Vec<AttributeSelector>>,
}
impl Client {
/// Parse an uploaded file and return the extracted document.
pub async fn parse(
&self,
file: ParseFile,
options: impl Into<Option<ParseOptions>>,
) -> Result<Document, FirecrawlError> {
let resolved_filename = file.filename.trim().to_string();
if resolved_filename.is_empty() {
return Err(FirecrawlError::Misuse(
"filename cannot be empty".to_string(),
));
}
if file.bytes.is_empty() {
return Err(FirecrawlError::Misuse(
"file content cannot be empty".to_string(),
));
}
let options = options.into().unwrap_or_default();
let options_json =
serde_json::to_string(&options).map_err(FirecrawlError::ResponseParseError)?;
let mut part = Part::bytes(file.bytes).file_name(resolved_filename);
if let Some(content_type) = file.content_type {
part = part.mime_str(&content_type).map_err(|e| {
FirecrawlError::Misuse(format!("Invalid content type for parse file: {}", e))
})?;
}
let form = Form::new().text("options", options_json).part("file", part);
let headers = self.prepare_multipart_headers(None);
let response = self
.client
.post(self.url("/parse"))
.headers(headers)
.multipart(form)
.send()
.await
.map_err(|e| FirecrawlError::HttpError("Parsing uploaded file".to_string(), e))?;
let response: ParseResponse = self.handle_response(response, "parse").await?;
Ok(response.data)
}
}
#[cfg(test)]
mod tests {
use super::*;
use mockito::Matcher;
use serde_json::json;
#[tokio::test]
async fn test_parse_with_mock() {
let mut server = mockito::Server::new_async().await;
let mock = server
.mock("POST", "/v2/parse")
.match_header(
"content-type",
Matcher::Regex("multipart/form-data".to_string()),
)
.with_status(200)
.with_header("content-type", "application/json")
.with_body(
json!({
"success": true,
"data": {
"markdown": "# Parsed File",
"metadata": {
"sourceURL": "https://parse.firecrawl.dev/uploads/upload.html",
"statusCode": 200
}
}
})
.to_string(),
)
.create();
let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();
let file = ParseFile::from_bytes("upload.html", b"<html><body>ok</body></html>".to_vec())
.with_content_type("text/html");
let doc = client.parse(file, None).await.unwrap();
assert!(doc.markdown.is_some());
assert!(doc.markdown.unwrap().contains("Parsed File"));
mock.assert();
}
#[test]
fn test_parse_file_from_missing_path() {
let result = ParseFile::from_path("/tmp/this-file-should-not-exist-for-parse-sdk-test");
assert!(result.is_err());
}
#[tokio::test]
async fn test_parse_error_response() {
let mut server = mockito::Server::new_async().await;
let mock = server
.mock("POST", "/v2/parse")
.match_header(
"content-type",
Matcher::Regex("multipart/form-data".to_string()),
)
.with_status(400)
.with_header("content-type", "application/json")
.with_body(
json!({
"success": false,
"error": "Unsupported upload type."
})
.to_string(),
)
.create();
let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();
let file = ParseFile::from_bytes("upload.xyz", b"not a real file".to_vec());
let result = client.parse(file, None).await;
assert!(result.is_err());
mock.assert();
}
#[test]
fn test_parse_rejects_empty_bytes() {
let file = ParseFile::from_bytes("empty.html", vec![]);
let rt = tokio::runtime::Runtime::new().unwrap();
let result = rt.block_on(async {
let client = Client::new_selfhosted("http://localhost:9999", Some("k")).unwrap();
client.parse(file, None).await
});
assert!(result.is_err());
let err_msg = format!("{}", result.unwrap_err());
assert!(
err_msg.contains("empty"),
"Expected empty file error, got: {}",
err_msg
);
}
}

789
참고/firecrawl-main/apps/rust-sdk/src/scrape.rs Normal file
View File

@@ -0,0 +1,789 @@
//! Scrape endpoint for Firecrawl API v2.
use serde::{Deserialize, Serialize};
use serde_json::Value;
use std::collections::HashMap;
use crate::client::Client;
use crate::types::{
Action, AttributeSelector, ChangeTrackingOptions, Document, Format, JsonOptions,
LocationConfig, ProfileConfig, ProxyType, ScreenshotOptions,
};
use crate::FirecrawlError;
/// Options for scraping a URL.
#[serde_with::skip_serializing_none]
#[derive(Deserialize, Serialize, Debug, Default, Clone)]
#[serde(rename_all = "camelCase")]
pub struct ScrapeOptions {
/// Output formats to include in the response.
pub formats: Option<Vec<Format>>,
/// Additional HTTP headers to send with the request.
pub headers: Option<HashMap<String, String>>,
/// HTML tags to exclusively include in the output.
pub include_tags: Option<Vec<String>>,
/// HTML tags to exclude from the output.
pub exclude_tags: Option<Vec<String>>,
/// Only extract the main content of the page.
pub only_main_content: Option<bool>,
/// Timeout in milliseconds before returning an error.
pub timeout: Option<u32>,
/// Time to wait after page load before scraping (milliseconds).
pub wait_for: Option<u32>,
/// Emulate a mobile device.
pub mobile: Option<bool>,
/// Parser configurations (e.g., for PDFs).
pub parsers: Option<Vec<ParserConfig>>,
/// Browser automation actions to perform before scraping.
pub actions: Option<Vec<Action>>,
/// Location configuration for proxy routing.
pub location: Option<LocationConfig>,
/// Skip TLS certificate verification.
pub skip_tls_verification: Option<bool>,
/// Remove base64-encoded images from the output.
pub remove_base64_images: Option<bool>,
/// Enable fast mode for quicker scrapes with reduced accuracy.
pub fast_mode: Option<bool>,
/// Block advertisements on the page.
pub block_ads: Option<bool>,
/// Proxy type to use.
pub proxy: Option<ProxyType>,
/// Maximum age of cached content to accept (seconds).
pub max_age: Option<u32>,
/// Minimum age of cached content to accept (seconds).
pub min_age: Option<u32>,
/// Store the result in cache for future requests.
pub store_in_cache: Option<bool>,
/// Lockdown mode: serve only previously cached results, never make outbound requests.
pub lockdown: Option<bool>,
/// Persistent browser profile for maintaining state across scrapes.
pub profile: Option<ProfileConfig>,
/// Integration identifier for tracking.
pub integration: Option<String>,
/// JSON extraction options.
pub json_options: Option<JsonOptions>,
/// Screenshot options.
pub screenshot_options: Option<ScreenshotOptions>,
/// Change tracking options.
pub change_tracking_options: Option<ChangeTrackingOptions>,
/// Attribute selectors for extraction.
pub attribute_selectors: Option<Vec<AttributeSelector>>,
}
/// Parser configuration for document parsing.
#[derive(Deserialize, Serialize, Debug, Clone)]
#[serde(untagged)]
pub enum ParserConfig {
/// Simple parser type string.
Simple(String),
/// PDF parser with options.
Pdf {
#[serde(rename = "type")]
parser_type: String,
#[serde(skip_serializing_if = "Option::is_none")]
mode: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
max_pages: Option<u32>,
},
}
/// Request body for scrape endpoint.
#[derive(Deserialize, Serialize, Debug, Default)]
#[serde(rename_all = "camelCase")]
struct ScrapeRequest {
url: String,
#[serde(flatten)]
options: ScrapeOptions,
}
/// Response from scrape endpoint.
#[derive(Deserialize, Serialize, Debug)]
#[serde(rename_all = "camelCase")]
struct ScrapeResponse {
success: bool,
data: Document,
#[serde(skip_serializing_if = "Option::is_none")]
warning: Option<String>,
}
/// Supported languages for scrape-bound browser execution.
#[derive(Deserialize, Serialize, Clone, Copy, Debug, PartialEq, Eq)]
#[serde(rename_all = "lowercase")]
pub enum ScrapeExecuteLanguage {
Python,
Node,
Bash,
}
/// Options for executing code or a prompt in a scrape-bound browser session.
///
/// At least one of `code` or `prompt` must be provided.
#[serde_with::skip_serializing_none]
#[derive(Deserialize, Serialize, Debug, Default, Clone)]
#[serde(rename_all = "camelCase")]
pub struct ScrapeExecuteOptions {
/// Code to execute (optional if `prompt` is provided).
pub code: Option<String>,
/// Natural-language instruction for the browser agent (optional if `code` is provided).
pub prompt: Option<String>,
/// Runtime language for the code.
pub language: Option<ScrapeExecuteLanguage>,
/// Execution timeout in seconds.
pub timeout: Option<u32>,
/// Optional origin tag for request attribution.
pub origin: Option<String>,
}
/// Response from scrape-bound browser execution.
#[serde_with::skip_serializing_none]
#[derive(Deserialize, Serialize, Debug, Clone)]
#[serde(rename_all = "camelCase")]
pub struct ScrapeExecuteResponse {
/// Whether the request succeeded.
pub success: bool,
/// Live-view URL for the browser session.
pub live_view_url: Option<String>,
/// Interactive live-view URL for the browser session.
pub interactive_live_view_url: Option<String>,
/// Agent output when a prompt was used.
pub output: Option<String>,
/// Captured stdout from execution.
pub stdout: Option<String>,
/// Optional execution result payload.
pub result: Option<String>,
/// Captured stderr from execution.
pub stderr: Option<String>,
/// Process exit code.
pub exit_code: Option<i32>,
/// Whether execution was killed by timeout or system.
pub killed: Option<bool>,
/// Error message when execution fails.
pub error: Option<String>,
}
/// Response from deleting a scrape-bound browser session.
#[serde_with::skip_serializing_none]
#[derive(Deserialize, Serialize, Debug, Clone)]
#[serde(rename_all = "camelCase")]
pub struct ScrapeBrowserDeleteResponse {
/// Whether the delete request succeeded.
pub success: bool,
/// Session duration in milliseconds when available.
pub session_duration_ms: Option<u64>,
/// Credits billed when available.
pub credits_billed: Option<u32>,
/// Error message when deletion fails.
pub error: Option<String>,
}
impl Client {
/// Scrapes a URL and returns the content in the requested formats.
///
/// # Arguments
///
/// * `url` - The URL to scrape.
/// * `options` - Optional scrape configuration.
///
/// # Returns
///
/// A `Document` containing the scraped content.
///
/// # Example
///
/// ```no_run
/// use firecrawl::{Client, ScrapeOptions, Format};
///
/// #[tokio::main]
/// async fn main() -> Result<(), Box<dyn std::error::Error>> {
/// let client = Client::new("your-api-key")?;
///
/// // Simple scrape
/// let document = client.scrape("https://example.com", None).await?;
/// println!("Markdown: {:?}", document.markdown);
///
/// // Scrape with options
/// let options = ScrapeOptions {
/// formats: Some(vec![Format::Markdown, Format::Html, Format::Links]),
/// only_main_content: Some(true),
/// ..Default::default()
/// };
/// let document = client.scrape("https://example.com", options).await?;
/// println!("Links: {:?}", document.links);
///
/// Ok(())
/// }
/// ```
pub async fn scrape(
&self,
url: impl AsRef<str>,
options: impl Into<Option<ScrapeOptions>>,
) -> Result<Document, FirecrawlError> {
let body = ScrapeRequest {
url: url.as_ref().to_string(),
options: options.into().unwrap_or_default(),
};
let headers = self.prepare_headers(None);
let response = self
.client
.post(self.url("/scrape"))
.headers(headers)
.json(&body)
.send()
.await
.map_err(|e| FirecrawlError::HttpError(format!("Scraping {:?}", url.as_ref()), e))?;
let response: ScrapeResponse = self.handle_response(response, "scrape").await?;
Ok(response.data)
}
/// Scrapes a URL with a JSON schema for structured extraction.
///
/// This is a convenience method that combines scraping with JSON extraction.
///
/// # Arguments
///
/// * `url` - The URL to scrape.
/// * `schema` - JSON schema for the extraction.
/// * `prompt` - Optional extraction prompt.
///
/// # Returns
///
/// The extracted JSON value matching the schema.
///
/// # Example
///
/// ```no_run
/// use firecrawl::Client;
/// use serde_json::json;
///
/// #[tokio::main]
/// async fn main() -> Result<(), Box<dyn std::error::Error>> {
/// let client = Client::new("your-api-key")?;
///
/// let schema = json!({
/// "type": "object",
/// "properties": {
/// "title": { "type": "string" },
/// "price": { "type": "number" }
/// }
/// });
///
/// let data = client.scrape_with_schema(
/// "https://example.com/product",
/// schema,
/// Some("Extract the product title and price")
/// ).await?;
///
/// println!("Extracted: {}", data);
///
/// Ok(())
/// }
/// ```
pub async fn scrape_with_schema(
&self,
url: impl AsRef<str>,
schema: Value,
prompt: Option<impl AsRef<str>>,
) -> Result<Value, FirecrawlError> {
let options = ScrapeOptions {
formats: Some(vec![Format::Json]),
json_options: Some(JsonOptions {
schema: Some(schema),
prompt: prompt.map(|p| p.as_ref().to_string()),
..Default::default()
}),
..Default::default()
};
let document = self.scrape(url, options).await?;
Ok(document.json.unwrap_or(Value::Null))
}
/// Interacts with the browser session associated with a scrape job.
///
/// # Arguments
///
/// * `job_id` - The scrape job ID.
/// * `options` - Execution options including code and runtime config.
///
/// # Returns
///
/// A `ScrapeExecuteResponse` containing execution output.
pub async fn interact(
&self,
job_id: impl AsRef<str>,
options: ScrapeExecuteOptions,
) -> Result<ScrapeExecuteResponse, FirecrawlError> {
let has_code = options.code.as_ref().is_some_and(|c| !c.trim().is_empty());
let has_prompt = options
.prompt
.as_ref()
.is_some_and(|p| !p.trim().is_empty());
if !has_code && !has_prompt {
return Err(FirecrawlError::Misuse(
"Either 'code' or 'prompt' must be provided".into(),
));
}
let mut body = options;
if body.language.is_none() {
body.language = Some(ScrapeExecuteLanguage::Node);
}
let response = self
.client
.post(self.url(&format!("/scrape/{}/interact", job_id.as_ref())))
.headers(self.prepare_headers(None))
.json(&body)
.send()
.await
.map_err(|e| {
FirecrawlError::HttpError(
format!("Interacting with scrape browser for {}", job_id.as_ref()),
e,
)
})?;
self.handle_response(response, "scrape interact").await
}
/// Stops the interaction session associated with a scrape job.
///
/// # Arguments
///
/// * `job_id` - The scrape job ID.
///
/// # Returns
///
/// A `ScrapeBrowserDeleteResponse` indicating stop status.
pub async fn stop_interaction(
&self,
job_id: impl AsRef<str>,
) -> Result<ScrapeBrowserDeleteResponse, FirecrawlError> {
let response = self
.client
.delete(self.url(&format!("/scrape/{}/interact", job_id.as_ref())))
.headers(self.prepare_headers(None))
.send()
.await
.map_err(|e| {
FirecrawlError::HttpError(
format!("Stopping interaction for {}", job_id.as_ref()),
e,
)
})?;
self.handle_response(response, "stop interaction").await
}
/// Deprecated alias for [`Client::interact`].
#[deprecated(note = "Use interact() instead")]
pub async fn scrape_execute(
&self,
job_id: impl AsRef<str>,
options: ScrapeExecuteOptions,
) -> Result<ScrapeExecuteResponse, FirecrawlError> {
self.interact(job_id, options).await
}
/// Deprecated alias for [`Client::stop_interaction`].
#[deprecated(note = "Use stop_interaction() instead")]
pub async fn stop_interactive_browser(
&self,
job_id: impl AsRef<str>,
) -> Result<ScrapeBrowserDeleteResponse, FirecrawlError> {
self.stop_interaction(job_id).await
}
/// Deprecated alias for [`Client::stop_interaction`].
#[deprecated(note = "Use stop_interaction() instead")]
pub async fn delete_scrape_browser(
&self,
job_id: impl AsRef<str>,
) -> Result<ScrapeBrowserDeleteResponse, FirecrawlError> {
self.stop_interaction(job_id).await
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::{HighlightsFormat, QueryFormat, QueryFormatMode, QuestionFormat};
use serde_json::json;
#[test]
fn test_query_format_serializes_mode() {
let options = ScrapeOptions {
formats: Some(vec![Format::Query(QueryFormat {
prompt: "What is Firecrawl?".to_string(),
mode: Some(QueryFormatMode::DirectQuote),
})]),
..Default::default()
};
let payload = serde_json::to_value(options).unwrap();
assert_eq!(
payload["formats"][0],
json!({
"type": "query",
"prompt": "What is Firecrawl?",
"mode": "directQuote"
})
);
}
#[test]
fn test_question_and_highlights_formats_serialize() {
let options = ScrapeOptions {
formats: Some(vec![
Format::Question(QuestionFormat {
question: "What is Firecrawl?".to_string(),
}),
Format::Highlights(HighlightsFormat {
query: "What is Firecrawl?".to_string(),
}),
]),
..Default::default()
};
let payload = serde_json::to_value(options).unwrap();
assert_eq!(
payload["formats"][0],
json!({
"type": "question",
"question": "What is Firecrawl?"
})
);
assert_eq!(
payload["formats"][1],
json!({
"type": "highlights",
"query": "What is Firecrawl?"
})
);
}
#[tokio::test]
async fn test_scrape_with_mock() {
let mut server = mockito::Server::new_async().await;
let mock = server
.mock("POST", "/v2/scrape")
.with_status(200)
.with_header("content-type", "application/json")
.with_body(
json!({
"success": true,
"data": {
"markdown": "# Example Domain\n\nThis is an example.",
"metadata": {
"sourceURL": "https://example.com",
"statusCode": 200,
"title": "Example Domain"
}
}
})
.to_string(),
)
.create();
let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();
let document = client.scrape("https://example.com", None).await.unwrap();
assert!(document.markdown.is_some());
assert!(document.markdown.unwrap().contains("Example Domain"));
mock.assert();
}
#[tokio::test]
async fn test_scrape_with_options() {
let mut server = mockito::Server::new_async().await;
let mock = server
.mock("POST", "/v2/scrape")
.with_status(200)
.with_header("content-type", "application/json")
.with_body(
json!({
"success": true,
"data": {
"markdown": "# Test",
"html": "<h1>Test</h1>",
"links": ["https://example.com/page1", "https://example.com/page2"],
"metadata": {
"sourceURL": "https://example.com",
"statusCode": 200
}
}
})
.to_string(),
)
.create();
let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();
let options = ScrapeOptions {
formats: Some(vec![Format::Markdown, Format::Html, Format::Links]),
only_main_content: Some(true),
..Default::default()
};
let document = client.scrape("https://example.com", options).await.unwrap();
assert!(document.markdown.is_some());
assert!(document.html.is_some());
assert!(document.links.is_some());
assert_eq!(document.links.unwrap().len(), 2);
mock.assert();
}
#[tokio::test]
async fn test_scrape_with_schema() {
let mut server = mockito::Server::new_async().await;
let mock = server
.mock("POST", "/v2/scrape")
.with_status(200)
.with_header("content-type", "application/json")
.with_body(
json!({
"success": true,
"data": {
"json": {
"title": "Product Name",
"price": 99.99
},
"metadata": {
"sourceURL": "https://example.com/product",
"statusCode": 200
}
}
})
.to_string(),
)
.create();
let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();
let schema = json!({
"type": "object",
"properties": {
"title": { "type": "string" },
"price": { "type": "number" }
}
});
let data = client
.scrape_with_schema(
"https://example.com/product",
schema,
Some("Extract product info"),
)
.await
.unwrap();
assert_eq!(data["title"], "Product Name");
assert_eq!(data["price"], 99.99);
mock.assert();
}
#[tokio::test]
async fn test_scrape_error_response() {
let mut server = mockito::Server::new_async().await;
let mock = server
.mock("POST", "/v2/scrape")
.with_status(400)
.with_header("content-type", "application/json")
.with_body(
json!({
"success": false,
"error": "Invalid URL"
})
.to_string(),
)
.create();
let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();
let result = client.scrape("invalid-url", None).await;
assert!(result.is_err());
mock.assert();
}
#[tokio::test]
async fn test_interact_with_mock() {
let mut server = mockito::Server::new_async().await;
let mock = server
.mock("POST", "/v2/scrape/job-123/interact")
.with_status(200)
.with_header("content-type", "application/json")
.with_body(
json!({
"success": true,
"stdout": "ok",
"result": "done",
"stderr": "",
"exitCode": 0,
"killed": false
})
.to_string(),
)
.create();
let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();
let response = client
.interact(
"job-123",
ScrapeExecuteOptions {
code: Some("console.log('ok')".to_string()),
timeout: Some(30),
..Default::default()
},
)
.await
.unwrap();
assert!(response.success);
assert_eq!(response.exit_code, Some(0));
assert_eq!(response.result, Some("done".to_string()));
mock.assert();
}
#[tokio::test]
async fn test_interact_with_prompt() {
let mut server = mockito::Server::new_async().await;
let mock = server
.mock("POST", "/v2/scrape/job-789/interact")
.with_status(200)
.with_header("content-type", "application/json")
.with_body(
json!({
"success": true,
"output": "Clicked the login button",
"liveViewUrl": "https://live.example.com/view",
"interactiveLiveViewUrl": "https://live.example.com/interactive",
"stdout": "",
"exitCode": 0,
"killed": false
})
.to_string(),
)
.create();
let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();
let response = client
.interact(
"job-789",
ScrapeExecuteOptions {
prompt: Some("Click the login button".to_string()),
..Default::default()
},
)
.await
.unwrap();
assert!(response.success);
assert_eq!(
response.output,
Some("Clicked the login button".to_string())
);
assert_eq!(
response.live_view_url,
Some("https://live.example.com/view".to_string())
);
assert_eq!(
response.interactive_live_view_url,
Some("https://live.example.com/interactive".to_string())
);
mock.assert();
}
#[tokio::test]
async fn test_stop_interaction_with_mock() {
let mut server = mockito::Server::new_async().await;
let mock = server
.mock("DELETE", "/v2/scrape/job-123/interact")
.with_status(200)
.with_header("content-type", "application/json")
.with_body(
json!({
"success": true,
"sessionDurationMs": 1200,
"creditsBilled": 3
})
.to_string(),
)
.create();
let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();
let response = client.stop_interaction("job-123").await.unwrap();
assert!(response.success);
assert_eq!(response.session_duration_ms, Some(1200));
assert_eq!(response.credits_billed, Some(3));
mock.assert();
}
#[tokio::test]
async fn test_interact_error_response() {
let mut server = mockito::Server::new_async().await;
let mock = server
.mock("POST", "/v2/scrape/job-404/interact")
.with_status(404)
.with_header("content-type", "application/json")
.with_body(
json!({
"success": false,
"error": "Job not found."
})
.to_string(),
)
.create();
let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();
let result = client
.interact(
"job-404",
ScrapeExecuteOptions {
code: Some("console.log('ok')".to_string()),
..Default::default()
},
)
.await;
assert!(result.is_err());
mock.assert();
}
}

472
참고/firecrawl-main/apps/rust-sdk/src/search.rs Normal file
View File

@@ -0,0 +1,472 @@
//! Search endpoint for Firecrawl API v2.
use serde::{Deserialize, Serialize};
use crate::client::Client;
use crate::scrape::ScrapeOptions;
use crate::types::{
Document, SearchCategory, SearchResultImage, SearchResultNews, SearchResultWeb, SearchSource,
};
use crate::FirecrawlError;
/// Options for search requests.
#[serde_with::skip_serializing_none]
#[derive(Deserialize, Serialize, Debug, Default, Clone)]
#[serde(rename_all = "camelCase")]
pub struct SearchOptions {
/// Maximum number of results to return. Default: 5, Max: 20.
pub limit: Option<u32>,
/// Search sources to query (web, news, images).
pub sources: Option<Vec<SearchSource>>,
/// Categories to filter results (github, research, pdf).
pub categories: Option<Vec<SearchCategory>>,
/// Domains to include in search results.
pub include_domains: Option<Vec<String>>,
/// Domains to exclude from search results.
pub exclude_domains: Option<Vec<String>>,
/// Time-based search filter (e.g., "qdr:d" for past day).
pub tbs: Option<String>,
/// Geographic location string for local search results.
pub location: Option<String>,
/// Whether to ignore invalid URLs in results.
pub ignore_invalid_urls: Option<bool>,
/// Timeout in milliseconds.
pub timeout: Option<u32>,
/// Scrape options to apply to each search result.
pub scrape_options: Option<ScrapeOptions>,
/// Integration identifier for tracking.
pub integration: Option<String>,
}
/// Request body for search endpoint.
#[derive(Deserialize, Serialize, Debug, Default)]
#[serde(rename_all = "camelCase")]
struct SearchRequest {
query: String,
#[serde(flatten)]
options: SearchOptions,
}
/// Search results data structure.
#[serde_with::skip_serializing_none]
#[derive(Deserialize, Serialize, Debug, Default, Clone)]
#[serde(rename_all = "camelCase")]
pub struct SearchData {
/// Web search results (may include scraped documents).
pub web: Option<Vec<SearchResultOrDocument>>,
/// News search results.
pub news: Option<Vec<SearchResultNews>>,
/// Image search results.
pub images: Option<Vec<SearchResultImage>>,
}
/// A search result that may be a simple result or a full document.
///
/// Uses custom deserialization to properly distinguish between web results
/// and scraped documents by checking for document-specific fields.
#[derive(Serialize, Debug, Clone)]
#[serde(untagged)]
pub enum SearchResultOrDocument {
/// Simple web search result.
WebResult(SearchResultWeb),
/// Full scraped document.
Document(Document),
}
impl<'de> serde::Deserialize<'de> for SearchResultOrDocument {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: serde::Deserializer<'de>,
{
use serde_json::Value;
let value = Value::deserialize(deserializer)?;
// Check for document-specific fields that indicate scraped content
// If any of these exist, it's a Document, not a simple WebResult
let is_document = value.get("markdown").is_some()
|| value.get("html").is_some()
|| value.get("rawHtml").is_some()
|| value.get("metadata").is_some();
if is_document {
Document::deserialize(value)
.map(SearchResultOrDocument::Document)
.map_err(serde::de::Error::custom)
} else {
SearchResultWeb::deserialize(value)
.map(SearchResultOrDocument::WebResult)
.map_err(serde::de::Error::custom)
}
}
}
/// Response from search endpoint.
#[derive(Deserialize, Serialize, Debug)]
#[serde(rename_all = "camelCase")]
pub struct SearchResponse {
/// Whether the request was successful.
pub success: bool,
/// Search results data.
pub data: SearchData,
/// Warning message if any.
pub warning: Option<String>,
}
impl Client {
/// Searches the web and optionally scrapes the results.
///
/// # Arguments
///
/// * `query` - The search query string.
/// * `options` - Optional search configuration.
///
/// # Returns
///
/// A `SearchResponse` containing the search results.
///
/// # Example
///
/// ```no_run
/// use firecrawl::{Client, SearchOptions, SearchSource};
///
/// #[tokio::main]
/// async fn main() -> Result<(), Box<dyn std::error::Error>> {
/// let client = Client::new("your-api-key")?;
///
/// // Simple search
/// let results = client.search("rust programming", None).await?;
/// for result in results.data.web.unwrap_or_default() {
/// match result {
/// firecrawl::SearchResultOrDocument::WebResult(r) => {
/// println!("URL: {}", r.url);
/// }
/// firecrawl::SearchResultOrDocument::Document(d) => {
/// println!("Content: {:?}", d.markdown);
/// }
/// }
/// }
///
/// // Search with options
/// let options = SearchOptions {
/// limit: Some(10),
/// sources: Some(vec![SearchSource::Web, SearchSource::News]),
/// ..Default::default()
/// };
/// let results = client.search("rust programming", options).await?;
///
/// Ok(())
/// }
/// ```
pub async fn search(
&self,
query: impl AsRef<str>,
options: impl Into<Option<SearchOptions>>,
) -> Result<SearchResponse, FirecrawlError> {
let body = SearchRequest {
query: query.as_ref().to_string(),
options: options.into().unwrap_or_default(),
};
let headers = self.prepare_headers(None);
let response = self
.client
.post(self.url("/search"))
.headers(headers)
.json(&body)
.send()
.await
.map_err(|e| {
FirecrawlError::HttpError(format!("Searching for {:?}", query.as_ref()), e)
})?;
self.handle_response(response, "search").await
}
/// Searches the web and scrapes the results.
///
/// This is a convenience method that enables scraping for all results.
///
/// # Arguments
///
/// * `query` - The search query string.
/// * `limit` - Maximum number of results to return.
///
/// # Returns
///
/// A vector of scraped documents.
///
/// # Example
///
/// ```no_run
/// use firecrawl::Client;
///
/// #[tokio::main]
/// async fn main() -> Result<(), Box<dyn std::error::Error>> {
/// let client = Client::new("your-api-key")?;
///
/// let documents = client.search_and_scrape("rust programming", 5).await?;
/// for doc in documents {
/// println!("Title: {:?}", doc.metadata.and_then(|m| m.title));
/// println!("Content: {:?}", doc.markdown);
/// }
///
/// Ok(())
/// }
/// ```
pub async fn search_and_scrape(
&self,
query: impl AsRef<str>,
limit: u32,
) -> Result<Vec<Document>, FirecrawlError> {
let options = SearchOptions {
limit: Some(limit),
scrape_options: Some(ScrapeOptions::default()),
..Default::default()
};
let response = self.search(query, options).await?;
let documents: Vec<Document> = response
.data
.web
.unwrap_or_default()
.into_iter()
.filter_map(|result| match result {
SearchResultOrDocument::Document(doc) => Some(doc),
SearchResultOrDocument::WebResult(_) => None,
})
.collect();
Ok(documents)
}
}
#[cfg(test)]
mod tests {
use super::*;
use serde_json::json;
#[tokio::test]
async fn test_search_with_mock() {
let mut server = mockito::Server::new_async().await;
let mock = server
.mock("POST", "/v2/search")
.with_status(200)
.with_header("content-type", "application/json")
.with_body(
json!({
"success": true,
"data": {
"web": [
{
"url": "https://example.com",
"title": "Example Domain",
"description": "This domain is for examples"
},
{
"url": "https://example.org",
"title": "Another Example",
"description": "More examples"
}
]
}
})
.to_string(),
)
.create();
let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();
let response = client.search("test query", None).await.unwrap();
assert!(response.success);
let web_results = response.data.web.unwrap();
assert_eq!(web_results.len(), 2);
mock.assert();
}
#[tokio::test]
async fn test_search_with_options() {
let mut server = mockito::Server::new_async().await;
let mock = server
.mock("POST", "/v2/search")
.with_status(200)
.with_header("content-type", "application/json")
.with_body(
json!({
"success": true,
"data": {
"web": [
{
"url": "https://example.com",
"title": "Test Result",
"description": "A test result"
}
],
"news": [
{
"title": "Breaking News",
"url": "https://news.example.com",
"snippet": "Something happened",
"date": "2024-01-01"
}
]
}
})
.to_string(),
)
.create();
let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();
let options = SearchOptions {
limit: Some(10),
sources: Some(vec![SearchSource::Web, SearchSource::News]),
..Default::default()
};
let response = client.search("test", options).await.unwrap();
assert!(response.success);
assert!(response.data.web.is_some());
assert!(response.data.news.is_some());
mock.assert();
}
#[tokio::test]
async fn test_search_and_scrape() {
let mut server = mockito::Server::new_async().await;
let mock = server
.mock("POST", "/v2/search")
.with_status(200)
.with_header("content-type", "application/json")
.with_body(
json!({
"success": true,
"data": {
"web": [
{
"markdown": "# Example\n\nThis is the scraped content.",
"metadata": {
"sourceURL": "https://example.com",
"statusCode": 200,
"title": "Example"
}
}
]
}
})
.to_string(),
)
.create();
let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();
let documents = client.search_and_scrape("test", 5).await.unwrap();
assert_eq!(documents.len(), 1);
assert!(documents[0].markdown.is_some());
mock.assert();
}
#[tokio::test]
async fn test_search_error_response() {
let mut server = mockito::Server::new_async().await;
let mock = server
.mock("POST", "/v2/search")
.with_status(400)
.with_header("content-type", "application/json")
.with_body(
json!({
"success": false,
"error": "Invalid query"
})
.to_string(),
)
.create();
let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();
let result = client.search("", None).await;
assert!(result.is_err());
mock.assert();
}
#[tokio::test]
async fn test_search_mixed_results_deserialization() {
// Test that results with markdown/metadata are correctly identified as Documents
// and simple results are identified as WebResults
let mut server = mockito::Server::new_async().await;
let mock = server
.mock("POST", "/v2/search")
.with_status(200)
.with_header("content-type", "application/json")
.with_body(
json!({
"success": true,
"data": {
"web": [
// Simple web result (no markdown/metadata)
{
"url": "https://example.com/simple",
"title": "Simple Result",
"description": "Just a search result"
},
// Scraped document with url AND markdown
{
"url": "https://example.com/scraped",
"markdown": "# Scraped Content\n\nThis has content.",
"metadata": {
"sourceURL": "https://example.com/scraped",
"statusCode": 200,
"title": "Scraped Page"
}
}
]
}
})
.to_string(),
)
.create();
let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();
let response = client.search("test", None).await.unwrap();
let web_results = response.data.web.unwrap();
assert_eq!(web_results.len(), 2);
// First should be WebResult
match &web_results[0] {
SearchResultOrDocument::WebResult(r) => {
assert_eq!(r.url, "https://example.com/simple");
assert_eq!(r.title, Some("Simple Result".to_string()));
}
SearchResultOrDocument::Document(_) => panic!("Expected WebResult, got Document"),
}
// Second should be Document (has markdown)
match &web_results[1] {
SearchResultOrDocument::Document(d) => {
assert!(d.markdown.is_some());
assert!(d.markdown.as_ref().unwrap().contains("Scraped Content"));
}
SearchResultOrDocument::WebResult(_) => panic!("Expected Document, got WebResult"),
}
mock.assert();
}
}

85
참고/firecrawl-main/apps/rust-sdk/src/serde_helpers.rs Normal file
View File

@@ -0,0 +1,85 @@
use serde::Deserialize;
use serde_json::Value;
/// Handles metadata fields that the API may return as either a string or an array of strings.
/// Arrays are joined with ", ".
pub(crate) fn deserialize_string_or_array<'de, D>(
deserializer: D,
) -> Result<Option<String>, D::Error>
where
D: serde::Deserializer<'de>,
{
match Option::<Value>::deserialize(deserializer)? {
None | Some(Value::Null) => Ok(None),
Some(Value::String(s)) => Ok(Some(s)),
Some(Value::Array(arr)) => {
let strings: Vec<String> = arr
.into_iter()
.map(|v| match v {
Value::String(s) => s,
other => other.to_string(),
})
.collect();
if strings.is_empty() {
Ok(None)
} else {
Ok(Some(strings.join(", ")))
}
}
Some(other) => Ok(Some(other.to_string())),
}
}
#[cfg(test)]
mod tests {
use serde::Deserialize;
use serde_json::json;
#[derive(Deserialize)]
struct Helper {
#[serde(default, deserialize_with = "super::deserialize_string_or_array")]
field: Option<String>,
}
#[test]
fn test_string_value() {
let h: Helper = serde_json::from_value(json!({"field": "hello"})).unwrap();
assert_eq!(h.field, Some("hello".to_string()));
}
#[test]
fn test_array_value() {
let h: Helper = serde_json::from_value(json!({"field": ["index", "follow"]})).unwrap();
assert_eq!(h.field, Some("index, follow".to_string()));
}
#[test]
fn test_null_value() {
let h: Helper = serde_json::from_value(json!({"field": null})).unwrap();
assert_eq!(h.field, None);
}
#[test]
fn test_missing_field() {
let h: Helper = serde_json::from_value(json!({})).unwrap();
assert_eq!(h.field, None);
}
#[test]
fn test_empty_array() {
let h: Helper = serde_json::from_value(json!({"field": []})).unwrap();
assert_eq!(h.field, None);
}
#[test]
fn test_single_element_array() {
let h: Helper = serde_json::from_value(json!({"field": ["noindex"]})).unwrap();
assert_eq!(h.field, Some("noindex".to_string()));
}
#[test]
fn test_serialization_roundtrip() {
let h: Helper = serde_json::from_value(json!({"field": "index, follow"})).unwrap();
assert_eq!(h.field, Some("index, follow".to_string()));
}
}

820
참고/firecrawl-main/apps/rust-sdk/src/types.rs Normal file
View File

@@ -0,0 +1,820 @@
//! Type definitions for Firecrawl API v2.
use serde::{de, Deserialize, Deserializer, Serialize, Serializer};
use serde_json::Value;
use std::collections::HashMap;
use crate::serde_helpers::deserialize_string_or_array;
/// Available output formats for scraping operations.
#[derive(Clone, Debug, PartialEq, Eq)]
pub enum Format {
/// Markdown content of the page.
Markdown,
/// Filtered, content-only HTML.
Html,
/// Original, untouched HTML.
RawHtml,
/// List of URLs found on the page.
Links,
/// List of image URLs found on the page.
Images,
/// Screenshot of the visible viewport.
Screenshot,
/// AI-generated summary of the page content.
Summary,
/// Change tracking information.
ChangeTracking,
/// Structured JSON extraction via LLM.
Json,
/// Custom attribute extraction.
Attributes,
/// Brand analysis of the page.
Branding,
/// Audio extraction (MP3) from YouTube videos.
Audio,
/// Question answer generated from the page content.
Question(QuestionFormat),
/// Direct highlights selected from the page content.
Highlights(HighlightsFormat),
/// Deprecated query answer generated from the page content.
Query(QueryFormat),
}
impl Serialize for Format {
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where
S: Serializer,
{
match self {
Format::Markdown => serializer.serialize_str("markdown"),
Format::Html => serializer.serialize_str("html"),
Format::RawHtml => serializer.serialize_str("rawHtml"),
Format::Links => serializer.serialize_str("links"),
Format::Images => serializer.serialize_str("images"),
Format::Screenshot => serializer.serialize_str("screenshot"),
Format::Summary => serializer.serialize_str("summary"),
Format::ChangeTracking => serializer.serialize_str("changeTracking"),
Format::Json => serializer.serialize_str("json"),
Format::Attributes => serializer.serialize_str("attributes"),
Format::Branding => serializer.serialize_str("branding"),
Format::Audio => serializer.serialize_str("audio"),
Format::Question(question) => question.serialize(serializer),
Format::Highlights(highlights) => highlights.serialize(serializer),
Format::Query(query) => query.serialize(serializer),
}
}
}
impl<'de> Deserialize<'de> for Format {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: Deserializer<'de>,
{
let value = Value::deserialize(deserializer)?;
match value {
Value::String(format) => match format.as_str() {
"markdown" => Ok(Format::Markdown),
"html" => Ok(Format::Html),
"rawHtml" => Ok(Format::RawHtml),
"links" => Ok(Format::Links),
"images" => Ok(Format::Images),
"screenshot" => Ok(Format::Screenshot),
"summary" => Ok(Format::Summary),
"changeTracking" => Ok(Format::ChangeTracking),
"json" => Ok(Format::Json),
"attributes" => Ok(Format::Attributes),
"branding" => Ok(Format::Branding),
"audio" => Ok(Format::Audio),
_ => Err(de::Error::custom(format!("unknown format: {}", format))),
},
Value::Object(_) => match value.get("type").and_then(Value::as_str) {
Some("question") => QuestionFormat::deserialize(value)
.map(Format::Question)
.map_err(de::Error::custom),
Some("highlights") => HighlightsFormat::deserialize(value)
.map(Format::Highlights)
.map_err(de::Error::custom),
Some("query") => QueryFormat::deserialize(value)
.map(Format::Query)
.map_err(de::Error::custom),
Some(format_type) => Err(de::Error::custom(format!(
"unknown object format: {}",
format_type
))),
None => Err(de::Error::custom("object format must have a type")),
},
_ => Err(de::Error::custom("format must be a string or object")),
}
}
}
/// Question format for asking a question about page content.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct QuestionFormat {
pub question: String,
}
#[derive(Deserialize, Serialize)]
#[serde(rename_all = "camelCase")]
struct QuestionFormatWire {
#[serde(rename = "type")]
format_type: String,
question: String,
}
impl Serialize for QuestionFormat {
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where
S: Serializer,
{
QuestionFormatWire {
format_type: "question".to_string(),
question: self.question.clone(),
}
.serialize(serializer)
}
}
impl<'de> Deserialize<'de> for QuestionFormat {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: Deserializer<'de>,
{
let wire = QuestionFormatWire::deserialize(deserializer)?;
if wire.format_type != "question" {
return Err(de::Error::custom(
"question format object must have type question",
));
}
Ok(Self {
question: wire.question,
})
}
}
/// Highlights format for selecting direct highlights from page content.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct HighlightsFormat {
pub query: String,
}
#[derive(Deserialize, Serialize)]
#[serde(rename_all = "camelCase")]
struct HighlightsFormatWire {
#[serde(rename = "type")]
format_type: String,
query: String,
}
impl Serialize for HighlightsFormat {
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where
S: Serializer,
{
HighlightsFormatWire {
format_type: "highlights".to_string(),
query: self.query.clone(),
}
.serialize(serializer)
}
}
impl<'de> Deserialize<'de> for HighlightsFormat {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: Deserializer<'de>,
{
let wire = HighlightsFormatWire::deserialize(deserializer)?;
if wire.format_type != "highlights" {
return Err(de::Error::custom(
"highlights format object must have type highlights",
));
}
Ok(Self { query: wire.query })
}
}
/// Deprecated query format for asking a question about page content.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct QueryFormat {
pub prompt: String,
pub mode: Option<QueryFormatMode>,
}
#[derive(Deserialize, Serialize)]
#[serde(rename_all = "camelCase")]
struct QueryFormatWire {
#[serde(rename = "type")]
format_type: String,
prompt: String,
#[serde(skip_serializing_if = "Option::is_none")]
mode: Option<QueryFormatMode>,
}
impl Serialize for QueryFormat {
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where
S: Serializer,
{
QueryFormatWire {
format_type: "query".to_string(),
prompt: self.prompt.clone(),
mode: self.mode,
}
.serialize(serializer)
}
}
impl<'de> Deserialize<'de> for QueryFormat {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: Deserializer<'de>,
{
let wire = QueryFormatWire::deserialize(deserializer)?;
if wire.format_type != "query" {
return Err(de::Error::custom(
"query format object must have type query",
));
}
Ok(Self {
prompt: wire.prompt,
mode: wire.mode,
})
}
}
/// Query answer mode.
#[derive(Deserialize, Serialize, Clone, Copy, Debug, PartialEq, Eq)]
pub enum QueryFormatMode {
#[serde(rename = "freeform")]
Freeform,
#[serde(rename = "directQuote")]
DirectQuote,
}
/// Viewport dimensions for screenshots.
#[serde_with::skip_serializing_none]
#[derive(Deserialize, Serialize, Debug, Default, Clone)]
pub struct Viewport {
pub width: u32,
pub height: u32,
}
/// Screenshot format options.
#[serde_with::skip_serializing_none]
#[derive(Deserialize, Serialize, Debug, Default, Clone)]
#[serde(rename_all = "camelCase")]
pub struct ScreenshotOptions {
/// Take a full-page screenshot instead of just the visible viewport.
pub full_page: Option<bool>,
/// Quality of the screenshot (1-100).
pub quality: Option<u8>,
/// Custom viewport dimensions.
pub viewport: Option<Viewport>,
}
/// Change tracking format options.
#[serde_with::skip_serializing_none]
#[derive(Deserialize, Serialize, Debug, Default, Clone)]
#[serde(rename_all = "camelCase")]
pub struct ChangeTrackingOptions {
/// Modes for change tracking output.
pub modes: Option<Vec<ChangeTrackingMode>>,
/// JSON schema for structured change output.
pub schema: Option<Value>,
/// Prompt for LLM-based change analysis.
pub prompt: Option<String>,
/// Tag to identify this tracking session.
pub tag: Option<String>,
}
/// Available change tracking modes.
#[derive(Deserialize, Serialize, Clone, Copy, Debug, PartialEq, Eq)]
#[serde(rename_all = "kebab-case")]
pub enum ChangeTrackingMode {
GitDiff,
Json,
}
/// Attribute extraction selector.
#[serde_with::skip_serializing_none]
#[derive(Deserialize, Serialize, Debug, Default, Clone)]
pub struct AttributeSelector {
/// CSS selector for the element.
pub selector: String,
/// Attribute name to extract.
pub attribute: String,
}
/// JSON extraction options.
#[serde_with::skip_serializing_none]
#[derive(Deserialize, Serialize, Debug, Default, Clone)]
#[serde(rename_all = "camelCase")]
pub struct JsonOptions {
/// JSON schema the output should adhere to.
pub schema: Option<Value>,
/// System prompt for the LLM agent.
pub system_prompt: Option<String>,
/// Extraction prompt for the LLM agent.
pub prompt: Option<String>,
}
/// Location configuration for proxy routing.
#[serde_with::skip_serializing_none]
#[derive(Deserialize, Serialize, Debug, Default, Clone)]
#[serde(rename_all = "camelCase")]
pub struct LocationConfig {
/// Country code (ISO 3166-1 alpha-2).
pub country: Option<String>,
/// List of preferred language codes.
pub languages: Option<Vec<String>>,
}
/// Persistent browser profile for maintaining state across scrapes.
#[serde_with::skip_serializing_none]
#[derive(Deserialize, Serialize, Debug, Default, Clone)]
#[serde(rename_all = "camelCase")]
pub struct ProfileConfig {
/// Profile name (1–128 characters).
pub name: String,
/// Whether to persist changes made during the session (defaults to true).
pub save_changes: Option<bool>,
}
/// Proxy type for scraping.
#[derive(Deserialize, Serialize, Clone, Copy, Debug, PartialEq, Eq)]
#[serde(rename_all = "lowercase")]
pub enum ProxyType {
Basic,
Stealth,
Enhanced,
Auto,
}
/// Browser action types for automation.
#[derive(Deserialize, Serialize, Debug, Clone)]
#[serde(tag = "type", rename_all = "camelCase")]
pub enum Action {
/// Wait for a specified time or element.
Wait {
/// Milliseconds to wait.
#[serde(skip_serializing_if = "Option::is_none")]
milliseconds: Option<u32>,
/// CSS selector to wait for.
#[serde(skip_serializing_if = "Option::is_none")]
selector: Option<String>,
},
/// Take a screenshot.
Screenshot {
#[serde(skip_serializing_if = "Option::is_none")]
full_page: Option<bool>,
#[serde(skip_serializing_if = "Option::is_none")]
quality: Option<u8>,
#[serde(skip_serializing_if = "Option::is_none")]
viewport: Option<Viewport>,
},
/// Click an element.
Click {
/// CSS selector of the element to click.
selector: String,
},
/// Write text to the focused input.
Write {
/// Text to write.
text: String,
},
/// Press a keyboard key.
Press {
/// Key name to press.
key: String,
},
/// Scroll the page.
Scroll {
/// Direction to scroll.
direction: ScrollDirection,
/// Optional selector to scroll within.
#[serde(skip_serializing_if = "Option::is_none")]
selector: Option<String>,
},
/// Trigger a scrape action.
Scrape,
/// Execute custom JavaScript.
#[serde(rename = "executeJavascript")]
ExecuteJavascript {
/// JavaScript code to execute.
script: String,
},
/// Generate a PDF.
Pdf {
#[serde(skip_serializing_if = "Option::is_none")]
format: Option<PdfFormat>,
#[serde(skip_serializing_if = "Option::is_none")]
landscape: Option<bool>,
#[serde(skip_serializing_if = "Option::is_none")]
scale: Option<f32>,
},
}
/// Scroll direction for scroll actions.
#[derive(Deserialize, Serialize, Clone, Copy, Debug, PartialEq, Eq)]
#[serde(rename_all = "lowercase")]
pub enum ScrollDirection {
Up,
Down,
}
/// PDF format options.
#[derive(Deserialize, Serialize, Clone, Copy, Debug, PartialEq, Eq)]
pub enum PdfFormat {
A0,
A1,
A2,
A3,
A4,
A5,
A6,
Letter,
Legal,
Tabloid,
Ledger,
}
/// Webhook configuration for async operations.
#[serde_with::skip_serializing_none]
#[derive(Deserialize, Serialize, Debug, Default, Clone)]
#[serde(rename_all = "camelCase")]
pub struct WebhookConfig {
/// URL to send webhook notifications to.
pub url: String,
/// Custom headers to include in webhook requests.
pub headers: Option<HashMap<String, String>>,
/// Custom metadata to include in webhook payloads.
pub metadata: Option<HashMap<String, String>>,
/// Event types to receive notifications for.
pub events: Option<Vec<WebhookEvent>>,
}
impl From<String> for WebhookConfig {
fn from(url: String) -> Self {
Self {
url,
..Default::default()
}
}
}
impl From<&str> for WebhookConfig {
fn from(url: &str) -> Self {
Self {
url: url.to_string(),
..Default::default()
}
}
}
/// Webhook event types for crawl/batch operations.
#[derive(Deserialize, Serialize, Clone, Copy, Debug, PartialEq, Eq)]
#[serde(rename_all = "camelCase")]
pub enum WebhookEvent {
Completed,
Failed,
Page,
Started,
}
/// Agent-specific webhook event types.
#[derive(Deserialize, Serialize, Clone, Copy, Debug, PartialEq, Eq)]
#[serde(rename_all = "camelCase")]
pub enum AgentWebhookEvent {
Started,
Action,
Completed,
Failed,
Cancelled,
}
/// Agent webhook configuration.
#[serde_with::skip_serializing_none]
#[derive(Deserialize, Serialize, Debug, Default, Clone)]
#[serde(rename_all = "camelCase")]
pub struct AgentWebhookConfig {
/// URL to send webhook notifications to.
pub url: String,
/// Custom headers to include in webhook requests.
pub headers: Option<HashMap<String, String>>,
/// Custom metadata to include in webhook payloads.
pub metadata: Option<HashMap<String, String>>,
/// Event types to receive notifications for.
pub events: Option<Vec<AgentWebhookEvent>>,
}
impl From<String> for AgentWebhookConfig {
fn from(url: String) -> Self {
Self {
url,
..Default::default()
}
}
}
impl From<&str> for AgentWebhookConfig {
fn from(url: &str) -> Self {
Self {
url: url.to_string(),
..Default::default()
}
}
}
/// Document metadata returned from scrape operations.
#[serde_with::skip_serializing_none]
#[derive(Deserialize, Serialize, Debug, Default, Clone)]
#[serde(rename_all = "camelCase")]
pub struct DocumentMetadata {
// Firecrawl specific
#[serde(rename = "sourceURL")]
pub source_url: Option<String>,
pub status_code: Option<u16>,
pub error: Option<String>,
// Basic meta tags
#[serde(default, deserialize_with = "deserialize_string_or_array")]
pub title: Option<String>,
#[serde(default, deserialize_with = "deserialize_string_or_array")]
pub description: Option<String>,
#[serde(default, deserialize_with = "deserialize_string_or_array")]
pub language: Option<String>,
#[serde(default, deserialize_with = "deserialize_string_or_array")]
pub keywords: Option<String>,
#[serde(default, deserialize_with = "deserialize_string_or_array")]
pub robots: Option<String>,
// OpenGraph namespace
#[serde(default, deserialize_with = "deserialize_string_or_array")]
pub og_title: Option<String>,
#[serde(default, deserialize_with = "deserialize_string_or_array")]
pub og_description: Option<String>,
#[serde(default, deserialize_with = "deserialize_string_or_array")]
pub og_url: Option<String>,
#[serde(default, deserialize_with = "deserialize_string_or_array")]
pub og_image: Option<String>,
#[serde(default, deserialize_with = "deserialize_string_or_array")]
pub og_audio: Option<String>,
#[serde(default, deserialize_with = "deserialize_string_or_array")]
pub og_determiner: Option<String>,
#[serde(default, deserialize_with = "deserialize_string_or_array")]
pub og_locale: Option<String>,
pub og_locale_alternate: Option<Vec<String>>,
#[serde(default, deserialize_with = "deserialize_string_or_array")]
pub og_site_name: Option<String>,
#[serde(default, deserialize_with = "deserialize_string_or_array")]
pub og_video: Option<String>,
// Article namespace
#[serde(default, deserialize_with = "deserialize_string_or_array")]
pub article_section: Option<String>,
#[serde(default, deserialize_with = "deserialize_string_or_array")]
pub article_tag: Option<String>,
#[serde(default, deserialize_with = "deserialize_string_or_array")]
pub published_time: Option<String>,
#[serde(default, deserialize_with = "deserialize_string_or_array")]
pub modified_time: Option<String>,
// Dublin Core namespace
#[serde(default, deserialize_with = "deserialize_string_or_array")]
pub dcterms_keywords: Option<String>,
#[serde(default, deserialize_with = "deserialize_string_or_array")]
pub dc_description: Option<String>,
#[serde(default, deserialize_with = "deserialize_string_or_array")]
pub dc_subject: Option<String>,
#[serde(default, deserialize_with = "deserialize_string_or_array")]
pub dcterms_subject: Option<String>,
#[serde(default, deserialize_with = "deserialize_string_or_array")]
pub dcterms_audience: Option<String>,
#[serde(default, deserialize_with = "deserialize_string_or_array")]
pub dc_type: Option<String>,
#[serde(default, deserialize_with = "deserialize_string_or_array")]
pub dcterms_type: Option<String>,
#[serde(default, deserialize_with = "deserialize_string_or_array")]
pub dc_date: Option<String>,
#[serde(default, deserialize_with = "deserialize_string_or_array")]
pub dc_date_created: Option<String>,
#[serde(default, deserialize_with = "deserialize_string_or_array")]
pub dcterms_created: Option<String>,
// Response metadata
#[serde(default, deserialize_with = "deserialize_string_or_array")]
pub scrape_id: Option<String>,
pub num_pages: Option<u32>,
#[serde(default, deserialize_with = "deserialize_string_or_array")]
pub content_type: Option<String>,
#[serde(default, deserialize_with = "deserialize_string_or_array")]
pub timezone: Option<String>,
#[serde(default, deserialize_with = "deserialize_string_or_array")]
pub proxy_used: Option<String>,
#[serde(default, deserialize_with = "deserialize_string_or_array")]
pub cache_state: Option<String>,
#[serde(default, deserialize_with = "deserialize_string_or_array")]
pub cached_at: Option<String>,
pub credits_used: Option<u32>,
pub concurrency_limited: Option<bool>,
}
/// Extracted attribute result.
#[serde_with::skip_serializing_none]
#[derive(Deserialize, Serialize, Debug, Default, Clone)]
pub struct AttributeResult {
pub selector: String,
pub attribute: String,
pub values: Vec<String>,
}
/// Document returned from scrape operations.
#[serde_with::skip_serializing_none]
#[derive(Deserialize, Serialize, Debug, Default, Clone)]
#[serde(rename_all = "camelCase")]
pub struct Document {
/// Markdown content of the page.
pub markdown: Option<String>,
/// Filtered HTML content.
pub html: Option<String>,
/// Raw HTML content.
pub raw_html: Option<String>,
/// Structured JSON extraction result.
pub json: Option<Value>,
/// AI-generated summary.
pub summary: Option<String>,
/// Document metadata.
pub metadata: Option<DocumentMetadata>,
/// Links found on the page.
pub links: Option<Vec<String>>,
/// Images found on the page.
pub images: Option<Vec<String>>,
/// Screenshot URL or base64 data.
pub screenshot: Option<String>,
/// Audio download URL (signed GCS link for MP3).
pub audio: Option<String>,
/// Extracted attributes.
pub attributes: Option<Vec<AttributeResult>>,
/// Action results.
pub actions: Option<HashMap<String, Value>>,
/// Answer generated by the question or deprecated query format.
pub answer: Option<String>,
/// Highlights generated by the highlights format.
pub highlights: Option<String>,
/// Warning message.
pub warning: Option<String>,
/// Change tracking data.
pub change_tracking: Option<Value>,
/// Branding analysis.
pub branding: Option<Value>,
}
/// Job status types for crawl and batch operations.
#[derive(Deserialize, Serialize, Clone, Copy, Debug, PartialEq, Eq)]
#[serde(rename_all = "camelCase")]
pub enum JobStatus {
Scraping,
Completed,
Failed,
Cancelled,
}
/// Sitemap handling mode.
#[derive(Deserialize, Serialize, Clone, Copy, Debug, PartialEq, Eq)]
#[serde(rename_all = "lowercase")]
pub enum SitemapMode {
/// Skip sitemap entirely.
Skip,
/// Include sitemap links alongside discovered links.
Include,
/// Only use links from the sitemap.
Only,
}
/// Agent model types.
#[derive(Deserialize, Serialize, Clone, Copy, Debug, PartialEq, Eq)]
#[serde(rename_all = "kebab-case")]
pub enum AgentModel {
#[serde(rename = "spark-1-pro")]
Spark1Pro,
#[serde(rename = "spark-1-mini")]
Spark1Mini,
}
/// Search source types.
#[derive(Deserialize, Serialize, Clone, Copy, Debug, PartialEq, Eq)]
#[serde(rename_all = "lowercase")]
pub enum SearchSource {
Web,
News,
Images,
}
/// Search category types.
#[derive(Deserialize, Serialize, Clone, Copy, Debug, PartialEq, Eq)]
#[serde(rename_all = "lowercase")]
pub enum SearchCategory {
Github,
Research,
Pdf,
}
/// Web search result.
#[serde_with::skip_serializing_none]
#[derive(Deserialize, Serialize, Debug, Default, Clone)]
#[serde(rename_all = "camelCase")]
pub struct SearchResultWeb {
pub url: String,
pub title: Option<String>,
pub description: Option<String>,
pub category: Option<String>,
}
/// News search result.
#[serde_with::skip_serializing_none]
#[derive(Deserialize, Serialize, Debug, Default, Clone)]
#[serde(rename_all = "camelCase")]
pub struct SearchResultNews {
pub title: Option<String>,
pub url: Option<String>,
pub snippet: Option<String>,
pub date: Option<String>,
pub image_url: Option<String>,
pub position: Option<u32>,
pub category: Option<String>,
}
/// Image search result.
#[serde_with::skip_serializing_none]
#[derive(Deserialize, Serialize, Debug, Default, Clone)]
#[serde(rename_all = "camelCase")]
pub struct SearchResultImage {
pub title: Option<String>,
pub image_url: Option<String>,
pub image_width: Option<u32>,
pub image_height: Option<u32>,
pub url: Option<String>,
pub position: Option<u32>,
}
/// Crawl error information.
#[serde_with::skip_serializing_none]
#[derive(Deserialize, Serialize, Debug, Clone)]
#[serde(rename_all = "camelCase")]
pub struct CrawlError {
pub id: String,
pub timestamp: Option<String>,
pub url: String,
pub code: Option<String>,
pub error: String,
}
/// Crawl errors response.
#[serde_with::skip_serializing_none]
#[derive(Deserialize, Serialize, Debug, Clone)]
#[serde(rename_all = "camelCase")]
pub struct CrawlErrorsResponse {
pub errors: Vec<CrawlError>,
#[serde(rename = "robotsBlocked")]
pub robots_blocked: Vec<String>,
}
#[cfg(test)]
mod tests {
use super::*;
use serde_json::json;
#[test]
fn test_full_document_with_array_metadata() {
let json = json!({
"markdown": "# Hello",
"metadata": {
"sourceURL": "https://example.com",
"statusCode": 200,
"title": "Example Page",
"description": ["A great page", "with multiple descriptions"],
"robots": ["index", "follow"],
"ogImage": ["https://img.jpg"],
"language": "en",
"keywords": ["rust", "sdk", "firecrawl"]
}
});
let doc: Document = serde_json::from_value(json).unwrap();
assert_eq!(doc.markdown, Some("# Hello".to_string()));
let meta = doc.metadata.unwrap();
assert_eq!(meta.title, Some("Example Page".to_string()));
assert_eq!(
meta.description,
Some("A great page, with multiple descriptions".to_string())
);
assert_eq!(meta.robots, Some("index, follow".to_string()));
assert_eq!(meta.og_image, Some("https://img.jpg".to_string()));
assert_eq!(meta.language, Some("en".to_string()));
assert_eq!(meta.keywords, Some("rust, sdk, firecrawl".to_string()));
}
}