참고소스 수정본
This commit is contained in:
LASTA_DEV01\lasta
161
참고/firecrawl-main/apps/python-sdk/firecrawl/v2/methods/agent.py
Normal file
161
참고/firecrawl-main/apps/python-sdk/firecrawl/v2/methods/agent.py
Normal file
@@ -0,0 +1,161 @@
|
||||
from typing import Any, Dict, List, Literal, Optional, Union
|
||||
import time
|
||||
|
||||
from ..types import AgentResponse, AgentWebhookConfig
|
||||
from ..utils.http_client import HttpClient
|
||||
from ..utils.error_handler import handle_response_error
|
||||
from ..utils.validation import _normalize_schema
|
||||
|
||||
|
||||
def _prepare_agent_request(
|
||||
urls: Optional[List[str]],
|
||||
*,
|
||||
prompt: str,
|
||||
schema: Optional[Any] = None,
|
||||
integration: Optional[str] = None,
|
||||
max_credits: Optional[int] = None,
|
||||
strict_constrain_to_urls: Optional[bool] = None,
|
||||
model: Optional[Literal["spark-1-pro", "spark-1-mini"]] = None,
|
||||
webhook: Optional[Union[str, AgentWebhookConfig]] = None,
|
||||
) -> Dict[str, Any]:
|
||||
body: Dict[str, Any] = {}
|
||||
if urls is not None:
|
||||
body["urls"] = urls
|
||||
body["prompt"] = prompt
|
||||
if schema is not None:
|
||||
normalized_schema = _normalize_schema(schema)
|
||||
if normalized_schema is not None:
|
||||
body["schema"] = normalized_schema
|
||||
else:
|
||||
raise ValueError(
|
||||
f"Invalid schema type: {type(schema).__name__}. "
|
||||
"Schema must be a dict, Pydantic BaseModel class, or Pydantic model instance."
|
||||
)
|
||||
if integration is not None and str(integration).strip():
|
||||
body["integration"] = str(integration).strip()
|
||||
if max_credits is not None and max_credits > 0:
|
||||
body["maxCredits"] = max_credits
|
||||
if strict_constrain_to_urls is not None and strict_constrain_to_urls:
|
||||
body["strictConstrainToURLs"] = strict_constrain_to_urls
|
||||
if model is not None:
|
||||
body["model"] = model
|
||||
if webhook is not None:
|
||||
if isinstance(webhook, str):
|
||||
body["webhook"] = webhook
|
||||
else:
|
||||
body["webhook"] = webhook.model_dump(exclude_none=True)
|
||||
return body
|
||||
|
||||
|
||||
def _normalize_agent_response_payload(payload: Dict[str, Any]) -> Dict[str, Any]:
|
||||
out = dict(payload)
|
||||
if "expiresAt" in out and "expires_at" not in out:
|
||||
out["expires_at"] = out["expiresAt"]
|
||||
if "creditsUsed" in out and "credits_used" not in out:
|
||||
out["credits_used"] = out["creditsUsed"]
|
||||
return out
|
||||
|
||||
|
||||
def start_agent(
|
||||
client: HttpClient,
|
||||
urls: Optional[List[str]],
|
||||
*,
|
||||
prompt: str,
|
||||
schema: Optional[Any] = None,
|
||||
integration: Optional[str] = None,
|
||||
max_credits: Optional[int] = None,
|
||||
strict_constrain_to_urls: Optional[bool] = None,
|
||||
model: Optional[Literal["spark-1-pro", "spark-1-mini"]] = None,
|
||||
webhook: Optional[Union[str, AgentWebhookConfig]] = None,
|
||||
) -> AgentResponse:
|
||||
body = _prepare_agent_request(
|
||||
urls,
|
||||
prompt=prompt,
|
||||
schema=schema,
|
||||
integration=integration,
|
||||
max_credits=max_credits,
|
||||
strict_constrain_to_urls=strict_constrain_to_urls,
|
||||
model=model,
|
||||
webhook=webhook,
|
||||
)
|
||||
resp = client.post("/v2/agent", body)
|
||||
if not resp.ok:
|
||||
handle_response_error(resp, "agent")
|
||||
payload = _normalize_agent_response_payload(resp.json())
|
||||
return AgentResponse(**payload)
|
||||
|
||||
|
||||
def get_agent_status(client: HttpClient, job_id: str) -> AgentResponse:
|
||||
resp = client.get(f"/v2/agent/{job_id}")
|
||||
if not resp.ok:
|
||||
handle_response_error(resp, "agent-status")
|
||||
payload = _normalize_agent_response_payload(resp.json())
|
||||
return AgentResponse(**payload)
|
||||
|
||||
|
||||
def wait_agent(
|
||||
client: HttpClient,
|
||||
job_id: str,
|
||||
*,
|
||||
poll_interval: int = 2,
|
||||
timeout: Optional[int] = None,
|
||||
) -> AgentResponse:
|
||||
start_ts = time.time()
|
||||
while True:
|
||||
status = get_agent_status(client, job_id)
|
||||
if status.status in ("completed", "failed", "cancelled"):
|
||||
return status
|
||||
if timeout is not None and (time.time() - start_ts) > timeout:
|
||||
return status
|
||||
time.sleep(max(1, poll_interval))
|
||||
|
||||
|
||||
def agent(
|
||||
client: HttpClient,
|
||||
urls: Optional[List[str]],
|
||||
*,
|
||||
prompt: str,
|
||||
schema: Optional[Any] = None,
|
||||
integration: Optional[str] = None,
|
||||
poll_interval: int = 2,
|
||||
timeout: Optional[int] = None,
|
||||
max_credits: Optional[int] = None,
|
||||
strict_constrain_to_urls: Optional[bool] = None,
|
||||
model: Optional[Literal["spark-1-pro", "spark-1-mini"]] = None,
|
||||
webhook: Optional[Union[str, AgentWebhookConfig]] = None,
|
||||
) -> AgentResponse:
|
||||
started = start_agent(
|
||||
client,
|
||||
urls,
|
||||
prompt=prompt,
|
||||
schema=schema,
|
||||
integration=integration,
|
||||
max_credits=max_credits,
|
||||
strict_constrain_to_urls=strict_constrain_to_urls,
|
||||
model=model,
|
||||
webhook=webhook,
|
||||
)
|
||||
job_id = getattr(started, "id", None)
|
||||
if not job_id:
|
||||
return started
|
||||
return wait_agent(client, job_id, poll_interval=poll_interval, timeout=timeout)
|
||||
|
||||
|
||||
def cancel_agent(client: HttpClient, job_id: str) -> bool:
|
||||
"""
|
||||
Cancel a running agent job.
|
||||
|
||||
Args:
|
||||
client: HTTP client instance
|
||||
job_id: ID of the agent job to cancel
|
||||
|
||||
Returns:
|
||||
bool: True if the agent was cancelled, False otherwise
|
||||
|
||||
Raises:
|
||||
Exception: If the cancellation fails
|
||||
"""
|
||||
resp = client.delete(f"/v2/agent/{job_id}")
|
||||
if not resp.ok:
|
||||
handle_response_error(resp, "cancel agent")
|
||||
return resp.json().get("success", False)
|
||||
@@ -0,0 +1 @@
|
||||
# Async (aio) method modules for v2
|
||||
@@ -0,0 +1,154 @@
|
||||
from typing import Any, Dict, List, Literal, Optional, Union
|
||||
import asyncio
|
||||
|
||||
from ...types import AgentResponse, AgentWebhookConfig
|
||||
from ...utils.http_client_async import AsyncHttpClient
|
||||
from ...utils.validation import _normalize_schema
|
||||
|
||||
|
||||
def _prepare_agent_request(
|
||||
urls: Optional[List[str]],
|
||||
*,
|
||||
prompt: str,
|
||||
schema: Optional[Any] = None,
|
||||
integration: Optional[str] = None,
|
||||
max_credits: Optional[int] = None,
|
||||
strict_constrain_to_urls: Optional[bool] = None,
|
||||
model: Optional[Literal["spark-1-pro", "spark-1-mini"]] = None,
|
||||
webhook: Optional[Union[str, AgentWebhookConfig]] = None,
|
||||
) -> Dict[str, Any]:
|
||||
body: Dict[str, Any] = {}
|
||||
if urls is not None:
|
||||
body["urls"] = urls
|
||||
body["prompt"] = prompt
|
||||
if schema is not None:
|
||||
normalized_schema = _normalize_schema(schema)
|
||||
if normalized_schema is not None:
|
||||
body["schema"] = normalized_schema
|
||||
else:
|
||||
raise ValueError(
|
||||
f"Invalid schema type: {type(schema).__name__}. "
|
||||
"Schema must be a dict, Pydantic BaseModel class, or Pydantic model instance."
|
||||
)
|
||||
if integration is not None and str(integration).strip():
|
||||
body["integration"] = str(integration).strip()
|
||||
if max_credits is not None and max_credits > 0:
|
||||
body["maxCredits"] = max_credits
|
||||
if strict_constrain_to_urls is not None and strict_constrain_to_urls:
|
||||
body["strictConstrainToURLs"] = strict_constrain_to_urls
|
||||
if model is not None:
|
||||
body["model"] = model
|
||||
if webhook is not None:
|
||||
if isinstance(webhook, str):
|
||||
body["webhook"] = webhook
|
||||
else:
|
||||
body["webhook"] = webhook.model_dump(exclude_none=True)
|
||||
return body
|
||||
|
||||
|
||||
def _normalize_agent_response_payload(payload: Dict[str, Any]) -> Dict[str, Any]:
|
||||
out = dict(payload)
|
||||
if "expiresAt" in out and "expires_at" not in out:
|
||||
out["expires_at"] = out["expiresAt"]
|
||||
if "creditsUsed" in out and "credits_used" not in out:
|
||||
out["credits_used"] = out["creditsUsed"]
|
||||
return out
|
||||
|
||||
|
||||
async def start_agent(
|
||||
client: AsyncHttpClient,
|
||||
urls: Optional[List[str]],
|
||||
*,
|
||||
prompt: str,
|
||||
schema: Optional[Any] = None,
|
||||
integration: Optional[str] = None,
|
||||
max_credits: Optional[int] = None,
|
||||
strict_constrain_to_urls: Optional[bool] = None,
|
||||
model: Optional[Literal["spark-1-pro", "spark-1-mini"]] = None,
|
||||
webhook: Optional[Union[str, AgentWebhookConfig]] = None,
|
||||
) -> AgentResponse:
|
||||
body = _prepare_agent_request(
|
||||
urls,
|
||||
prompt=prompt,
|
||||
schema=schema,
|
||||
integration=integration,
|
||||
max_credits=max_credits,
|
||||
strict_constrain_to_urls=strict_constrain_to_urls,
|
||||
model=model,
|
||||
webhook=webhook,
|
||||
)
|
||||
resp = await client.post("/v2/agent", body)
|
||||
payload = _normalize_agent_response_payload(resp.json())
|
||||
return AgentResponse(**payload)
|
||||
|
||||
|
||||
async def get_agent_status(client: AsyncHttpClient, job_id: str) -> AgentResponse:
|
||||
resp = await client.get(f"/v2/agent/{job_id}")
|
||||
payload = _normalize_agent_response_payload(resp.json())
|
||||
return AgentResponse(**payload)
|
||||
|
||||
|
||||
async def wait_agent(
|
||||
client: AsyncHttpClient,
|
||||
job_id: str,
|
||||
*,
|
||||
poll_interval: int = 2,
|
||||
timeout: Optional[int] = None,
|
||||
) -> AgentResponse:
|
||||
start_ts = asyncio.get_event_loop().time()
|
||||
while True:
|
||||
status = await get_agent_status(client, job_id)
|
||||
if status.status in ("completed", "failed", "cancelled"):
|
||||
return status
|
||||
if timeout is not None and (asyncio.get_event_loop().time() - start_ts) > timeout:
|
||||
return status
|
||||
await asyncio.sleep(max(1, poll_interval))
|
||||
|
||||
|
||||
async def agent(
|
||||
client: AsyncHttpClient,
|
||||
urls: Optional[List[str]],
|
||||
*,
|
||||
prompt: str,
|
||||
schema: Optional[Any] = None,
|
||||
integration: Optional[str] = None,
|
||||
poll_interval: int = 2,
|
||||
timeout: Optional[int] = None,
|
||||
max_credits: Optional[int] = None,
|
||||
strict_constrain_to_urls: Optional[bool] = None,
|
||||
model: Optional[Literal["spark-1-pro", "spark-1-mini"]] = None,
|
||||
webhook: Optional[Union[str, AgentWebhookConfig]] = None,
|
||||
) -> AgentResponse:
|
||||
started = await start_agent(
|
||||
client,
|
||||
urls,
|
||||
prompt=prompt,
|
||||
schema=schema,
|
||||
integration=integration,
|
||||
max_credits=max_credits,
|
||||
strict_constrain_to_urls=strict_constrain_to_urls,
|
||||
model=model,
|
||||
webhook=webhook,
|
||||
)
|
||||
job_id = getattr(started, "id", None)
|
||||
if not job_id:
|
||||
return started
|
||||
return await wait_agent(client, job_id, poll_interval=poll_interval, timeout=timeout)
|
||||
|
||||
|
||||
async def cancel_agent(client: AsyncHttpClient, job_id: str) -> bool:
|
||||
"""
|
||||
Cancel a running agent job.
|
||||
|
||||
Args:
|
||||
client: Async HTTP client instance
|
||||
job_id: ID of the agent job to cancel
|
||||
|
||||
Returns:
|
||||
bool: True if the agent was cancelled, False otherwise
|
||||
|
||||
Raises:
|
||||
Exception: If the cancellation fails
|
||||
"""
|
||||
resp = await client.delete(f"/v2/agent/{job_id}")
|
||||
return resp.json().get("success", False)
|
||||
@@ -0,0 +1,240 @@
|
||||
from typing import Optional, List, Dict, Any
|
||||
from ...types import ScrapeOptions, WebhookConfig, Document, BatchScrapeResponse, BatchScrapeJob, PaginationConfig
|
||||
from ...utils.http_client_async import AsyncHttpClient
|
||||
from ...utils.validation import prepare_scrape_options
|
||||
from ...utils.error_handler import handle_response_error
|
||||
from ...utils.normalize import normalize_document_input
|
||||
from ...methods.batch import validate_batch_urls
|
||||
import time
|
||||
|
||||
def _parse_batch_scrape_documents(data_list: Optional[List[Any]]) -> List[Document]:
|
||||
documents: List[Document] = []
|
||||
for doc in data_list or []:
|
||||
if isinstance(doc, dict):
|
||||
normalized = normalize_document_input(doc)
|
||||
documents.append(Document(**normalized))
|
||||
return documents
|
||||
|
||||
|
||||
def _parse_batch_scrape_status_response(body: Dict[str, Any]) -> Dict[str, Any]:
|
||||
if not body.get("success"):
|
||||
raise Exception(body.get("error", "Unknown error occurred"))
|
||||
|
||||
return {
|
||||
"status": body.get("status"),
|
||||
"completed": body.get("completed", 0),
|
||||
"total": body.get("total", 0),
|
||||
"credits_used": body.get("creditsUsed"),
|
||||
"expires_at": body.get("expiresAt"),
|
||||
"next": body.get("next"),
|
||||
"data": _parse_batch_scrape_documents(body.get("data", []) or []),
|
||||
}
|
||||
|
||||
def _prepare(urls: List[str], *, options: Optional[ScrapeOptions] = None, **kwargs) -> Dict[str, Any]:
|
||||
if not urls:
|
||||
raise ValueError("URLs list cannot be empty")
|
||||
|
||||
validated_urls = validate_batch_urls([u.strip() if isinstance(u, str) else u for u in urls])
|
||||
payload: Dict[str, Any] = {"urls": validated_urls}
|
||||
if options:
|
||||
opts = prepare_scrape_options(options)
|
||||
if opts:
|
||||
payload.update(opts)
|
||||
if (w := kwargs.get("webhook")) is not None:
|
||||
payload["webhook"] = w if isinstance(w, str) else w.model_dump(exclude_none=True)
|
||||
if (v := kwargs.get("append_to_id")) is not None:
|
||||
payload["appendToId"] = v
|
||||
if (v := kwargs.get("ignore_invalid_urls")) is not None:
|
||||
payload["ignoreInvalidURLs"] = v
|
||||
if (v := kwargs.get("max_concurrency")) is not None:
|
||||
payload["maxConcurrency"] = v
|
||||
if (v := kwargs.get("zero_data_retention")) is not None:
|
||||
payload["zeroDataRetention"] = v
|
||||
if (v := kwargs.get("integration")) is not None:
|
||||
trimmed_integration = str(v).strip()
|
||||
if trimmed_integration:
|
||||
payload["integration"] = trimmed_integration
|
||||
return payload
|
||||
|
||||
|
||||
async def start_batch_scrape(client: AsyncHttpClient, urls: List[str], **kwargs) -> BatchScrapeResponse:
|
||||
payload = _prepare(urls, **kwargs)
|
||||
response = await client.post("/v2/batch/scrape", payload)
|
||||
if response.status_code >= 400:
|
||||
handle_response_error(response, "start batch scrape")
|
||||
body = response.json()
|
||||
if not body.get("success"):
|
||||
raise Exception(body.get("error", "Unknown error occurred"))
|
||||
return BatchScrapeResponse(id=body.get("id"), url=body.get("url"), invalid_urls=body.get("invalidURLs"))
|
||||
|
||||
|
||||
async def get_batch_scrape_status(
|
||||
client: AsyncHttpClient,
|
||||
job_id: str,
|
||||
pagination_config: Optional[PaginationConfig] = None
|
||||
) -> BatchScrapeJob:
|
||||
"""
|
||||
Get the status of a batch scrape job.
|
||||
|
||||
Args:
|
||||
client: Async HTTP client instance
|
||||
job_id: ID of the batch scrape job
|
||||
pagination_config: Optional configuration for pagination behavior
|
||||
|
||||
Returns:
|
||||
BatchScrapeJob containing job status and data
|
||||
|
||||
Raises:
|
||||
Exception: If the status check fails
|
||||
"""
|
||||
response = await client.get(f"/v2/batch/scrape/{job_id}")
|
||||
if response.status_code >= 400:
|
||||
handle_response_error(response, "get batch scrape status")
|
||||
body = response.json()
|
||||
payload = _parse_batch_scrape_status_response(body)
|
||||
docs = payload["data"]
|
||||
|
||||
# Handle pagination if requested
|
||||
auto_paginate = pagination_config.auto_paginate if pagination_config else True
|
||||
if auto_paginate and payload["next"]:
|
||||
docs = await _fetch_all_batch_pages_async(
|
||||
client,
|
||||
payload["next"],
|
||||
docs,
|
||||
pagination_config
|
||||
)
|
||||
|
||||
return BatchScrapeJob(
|
||||
status=payload["status"],
|
||||
completed=payload["completed"],
|
||||
total=payload["total"],
|
||||
credits_used=payload["credits_used"],
|
||||
expires_at=payload["expires_at"],
|
||||
next=payload["next"] if not auto_paginate else None,
|
||||
data=docs,
|
||||
)
|
||||
|
||||
|
||||
async def get_batch_scrape_status_page(
|
||||
client: AsyncHttpClient,
|
||||
next_url: str,
|
||||
*,
|
||||
request_timeout: Optional[float] = None,
|
||||
) -> BatchScrapeJob:
|
||||
"""
|
||||
Fetch a single page of batch scrape results using the provided next URL.
|
||||
|
||||
Args:
|
||||
client: Async HTTP client instance
|
||||
next_url: Opaque next URL from a prior batch scrape status response
|
||||
request_timeout: Timeout (in seconds) for the HTTP request
|
||||
|
||||
Returns:
|
||||
BatchScrapeJob with the page data and next URL (if any)
|
||||
|
||||
Raises:
|
||||
Exception: If the request fails or returns an error response
|
||||
"""
|
||||
response = await client.get(next_url, timeout=request_timeout)
|
||||
if response.status_code >= 400:
|
||||
handle_response_error(response, "get batch scrape status page")
|
||||
body = response.json()
|
||||
payload = _parse_batch_scrape_status_response(body)
|
||||
return BatchScrapeJob(
|
||||
status=payload["status"],
|
||||
completed=payload["completed"],
|
||||
total=payload["total"],
|
||||
credits_used=payload["credits_used"],
|
||||
expires_at=payload["expires_at"],
|
||||
next=payload["next"],
|
||||
data=payload["data"],
|
||||
)
|
||||
|
||||
|
||||
async def _fetch_all_batch_pages_async(
|
||||
client: AsyncHttpClient,
|
||||
next_url: str,
|
||||
initial_documents: List[Document],
|
||||
pagination_config: Optional[PaginationConfig] = None
|
||||
) -> List[Document]:
|
||||
"""
|
||||
Fetch all pages of batch scrape results asynchronously.
|
||||
|
||||
Args:
|
||||
client: Async HTTP client instance
|
||||
next_url: URL for the next page
|
||||
initial_documents: Documents from the first page
|
||||
pagination_config: Optional configuration for pagination limits
|
||||
|
||||
Returns:
|
||||
List of all documents from all pages
|
||||
"""
|
||||
documents = initial_documents.copy()
|
||||
current_url = next_url
|
||||
page_count = 0
|
||||
|
||||
# Apply pagination limits
|
||||
max_pages = pagination_config.max_pages if pagination_config else None
|
||||
max_results = pagination_config.max_results if pagination_config else None
|
||||
max_wait_time = pagination_config.max_wait_time if pagination_config else None
|
||||
|
||||
start_time = time.monotonic()
|
||||
|
||||
while current_url:
|
||||
# Check pagination limits
|
||||
if (max_pages is not None) and (page_count >= max_pages):
|
||||
break
|
||||
|
||||
if (max_wait_time is not None) and (time.monotonic() - start_time) > max_wait_time:
|
||||
break
|
||||
|
||||
# Fetch next page
|
||||
response = await client.get(current_url)
|
||||
|
||||
if response.status_code >= 400:
|
||||
# Log error but continue with what we have
|
||||
import logging
|
||||
logger = logging.getLogger("firecrawl")
|
||||
logger.warning(f"Failed to fetch next page: {response.status_code}")
|
||||
break
|
||||
|
||||
page_data = response.json()
|
||||
try:
|
||||
page_payload = _parse_batch_scrape_status_response(page_data)
|
||||
except Exception:
|
||||
break
|
||||
|
||||
# Add documents from this page
|
||||
for document in page_payload["data"]:
|
||||
# Check max_results limit
|
||||
if (max_results is not None) and (len(documents) >= max_results):
|
||||
break
|
||||
documents.append(document)
|
||||
|
||||
# Check if we hit max_results limit
|
||||
if (max_results is not None) and (len(documents) >= max_results):
|
||||
break
|
||||
|
||||
# Get next URL
|
||||
current_url = page_payload["next"]
|
||||
page_count += 1
|
||||
|
||||
return documents
|
||||
|
||||
|
||||
async def cancel_batch_scrape(client: AsyncHttpClient, job_id: str) -> bool:
|
||||
response = await client.delete(f"/v2/batch/scrape/{job_id}")
|
||||
if response.status_code >= 400:
|
||||
handle_response_error(response, "cancel batch scrape")
|
||||
body = response.json()
|
||||
return body.get("status") == "cancelled"
|
||||
|
||||
|
||||
async def get_batch_scrape_errors(client: AsyncHttpClient, job_id: str) -> Dict[str, Any]:
|
||||
response = await client.get(f"/v2/batch/scrape/{job_id}/errors")
|
||||
if response.status_code >= 400:
|
||||
handle_response_error(response, "get batch scrape errors")
|
||||
body = response.json()
|
||||
if not body.get("success"):
|
||||
raise Exception(body.get("error", "Unknown error occurred"))
|
||||
return body
|
||||
@@ -0,0 +1,179 @@
|
||||
"""
|
||||
Async browser session methods for Firecrawl v2 API.
|
||||
|
||||
Provides async create, execute, delete, and list operations for browser sessions.
|
||||
"""
|
||||
|
||||
from typing import Any, Dict, Literal, Optional
|
||||
|
||||
from ...types import (
|
||||
BrowserCreateResponse,
|
||||
BrowserExecuteResponse,
|
||||
BrowserDeleteResponse,
|
||||
BrowserListResponse,
|
||||
)
|
||||
from ...utils.http_client_async import AsyncHttpClient
|
||||
|
||||
|
||||
def _normalize_browser_create_response(payload: Dict[str, Any]) -> Dict[str, Any]:
|
||||
out = dict(payload)
|
||||
if "cdpUrl" in out and "cdp_url" not in out:
|
||||
out["cdp_url"] = out["cdpUrl"]
|
||||
if "liveViewUrl" in out and "live_view_url" not in out:
|
||||
out["live_view_url"] = out["liveViewUrl"]
|
||||
if "interactiveLiveViewUrl" in out and "interactive_live_view_url" not in out:
|
||||
out["interactive_live_view_url"] = out["interactiveLiveViewUrl"]
|
||||
if "expiresAt" in out and "expires_at" not in out:
|
||||
out["expires_at"] = out["expiresAt"]
|
||||
return out
|
||||
|
||||
|
||||
def _normalize_browser_list_response(payload: Dict[str, Any]) -> Dict[str, Any]:
|
||||
out = dict(payload)
|
||||
if "sessions" in out and isinstance(out["sessions"], list):
|
||||
normalized_sessions = []
|
||||
for s in out["sessions"]:
|
||||
ns = dict(s)
|
||||
if "cdpUrl" in ns and "cdp_url" not in ns:
|
||||
ns["cdp_url"] = ns["cdpUrl"]
|
||||
if "liveViewUrl" in ns and "live_view_url" not in ns:
|
||||
ns["live_view_url"] = ns["liveViewUrl"]
|
||||
if "interactiveLiveViewUrl" in ns and "interactive_live_view_url" not in ns:
|
||||
ns["interactive_live_view_url"] = ns["interactiveLiveViewUrl"]
|
||||
if "streamWebView" in ns and "stream_web_view" not in ns:
|
||||
ns["stream_web_view"] = ns["streamWebView"]
|
||||
if "createdAt" in ns and "created_at" not in ns:
|
||||
ns["created_at"] = ns["createdAt"]
|
||||
if "lastActivity" in ns and "last_activity" not in ns:
|
||||
ns["last_activity"] = ns["lastActivity"]
|
||||
normalized_sessions.append(ns)
|
||||
out["sessions"] = normalized_sessions
|
||||
return out
|
||||
|
||||
|
||||
async def browser(
|
||||
client: AsyncHttpClient,
|
||||
*,
|
||||
ttl: Optional[int] = None,
|
||||
activity_ttl: Optional[int] = None,
|
||||
stream_web_view: Optional[bool] = None,
|
||||
profile: Optional[Dict[str, Any]] = None,
|
||||
) -> BrowserCreateResponse:
|
||||
"""Create a new browser session.
|
||||
|
||||
Args:
|
||||
client: Async HTTP client instance
|
||||
ttl: Total time-to-live in seconds (30-3600, default 300)
|
||||
activity_ttl: Inactivity TTL in seconds (10-3600)
|
||||
stream_web_view: Whether to enable webview streaming
|
||||
profile: Profile config with ``name`` (str) and
|
||||
optional ``save_changes`` (bool, default ``True``)
|
||||
|
||||
Returns:
|
||||
BrowserCreateResponse with session id and CDP URL
|
||||
"""
|
||||
body: Dict[str, Any] = {}
|
||||
if ttl is not None:
|
||||
body["ttl"] = ttl
|
||||
if activity_ttl is not None:
|
||||
body["activityTtl"] = activity_ttl
|
||||
if stream_web_view is not None:
|
||||
body["streamWebView"] = stream_web_view
|
||||
if profile is not None:
|
||||
body["profile"] = {
|
||||
"name": profile["name"],
|
||||
"saveChanges": profile.get("save_changes", True),
|
||||
}
|
||||
|
||||
resp = await client.post("/v2/browser", body)
|
||||
payload = _normalize_browser_create_response(resp.json())
|
||||
return BrowserCreateResponse(**payload)
|
||||
|
||||
|
||||
def _normalize_browser_execute_response(payload: Dict[str, Any]) -> Dict[str, Any]:
|
||||
out = dict(payload)
|
||||
if "exitCode" in out and "exit_code" not in out:
|
||||
out["exit_code"] = out["exitCode"]
|
||||
return out
|
||||
|
||||
|
||||
def _normalize_browser_delete_response(payload: Dict[str, Any]) -> Dict[str, Any]:
|
||||
out = dict(payload)
|
||||
if "sessionDurationMs" in out and "session_duration_ms" not in out:
|
||||
out["session_duration_ms"] = out["sessionDurationMs"]
|
||||
if "creditsBilled" in out and "credits_billed" not in out:
|
||||
out["credits_billed"] = out["creditsBilled"]
|
||||
return out
|
||||
|
||||
|
||||
async def browser_execute(
|
||||
client: AsyncHttpClient,
|
||||
session_id: str,
|
||||
code: str,
|
||||
*,
|
||||
language: Literal["python", "node", "bash"] = "bash",
|
||||
timeout: Optional[int] = None,
|
||||
) -> BrowserExecuteResponse:
|
||||
"""Execute code in a browser session.
|
||||
|
||||
Args:
|
||||
client: Async HTTP client instance
|
||||
session_id: Browser session ID
|
||||
code: Code to execute
|
||||
language: Programming language ("python", "node", or "bash")
|
||||
timeout: Execution timeout in seconds (1-300, default 30)
|
||||
|
||||
Returns:
|
||||
BrowserExecuteResponse with execution result
|
||||
"""
|
||||
body: Dict[str, Any] = {
|
||||
"code": code,
|
||||
"language": language,
|
||||
}
|
||||
if timeout is not None:
|
||||
body["timeout"] = timeout
|
||||
|
||||
resp = await client.post(f"/v2/browser/{session_id}/execute", body)
|
||||
payload = _normalize_browser_execute_response(resp.json())
|
||||
return BrowserExecuteResponse(**payload)
|
||||
|
||||
|
||||
async def delete_browser(
|
||||
client: AsyncHttpClient,
|
||||
session_id: str,
|
||||
) -> BrowserDeleteResponse:
|
||||
"""Delete a browser session.
|
||||
|
||||
Args:
|
||||
client: Async HTTP client instance
|
||||
session_id: Browser session ID
|
||||
|
||||
Returns:
|
||||
BrowserDeleteResponse
|
||||
"""
|
||||
resp = await client.delete(f"/v2/browser/{session_id}")
|
||||
payload = _normalize_browser_delete_response(resp.json())
|
||||
return BrowserDeleteResponse(**payload)
|
||||
|
||||
|
||||
async def list_browsers(
|
||||
client: AsyncHttpClient,
|
||||
*,
|
||||
status: Optional[Literal["active", "destroyed"]] = None,
|
||||
) -> BrowserListResponse:
|
||||
"""List browser sessions.
|
||||
|
||||
Args:
|
||||
client: Async HTTP client instance
|
||||
status: Filter by session status ("active" or "destroyed")
|
||||
|
||||
Returns:
|
||||
BrowserListResponse with list of sessions
|
||||
"""
|
||||
endpoint = "/v2/browser"
|
||||
if status is not None:
|
||||
endpoint = f"{endpoint}?status={status}"
|
||||
|
||||
resp = await client.get(endpoint)
|
||||
payload = _normalize_browser_list_response(resp.json())
|
||||
return BrowserListResponse(**payload)
|
||||
@@ -0,0 +1,414 @@
|
||||
from typing import Optional, Dict, Any, List
|
||||
from ...types import (
|
||||
CrawlRequest,
|
||||
CrawlJob,
|
||||
CrawlResponse,
|
||||
Document,
|
||||
CrawlParamsRequest,
|
||||
CrawlParamsData,
|
||||
WebhookConfig,
|
||||
CrawlErrorsResponse,
|
||||
ActiveCrawlsResponse,
|
||||
ActiveCrawl,
|
||||
PaginationConfig,
|
||||
)
|
||||
from ...utils.error_handler import handle_response_error
|
||||
from ...utils.validation import prepare_scrape_options
|
||||
from ...utils.http_client_async import AsyncHttpClient
|
||||
from ...utils.normalize import normalize_document_input
|
||||
import time
|
||||
|
||||
|
||||
def _prepare_crawl_request(request: CrawlRequest) -> dict:
|
||||
if not request.url or not request.url.strip():
|
||||
raise ValueError("URL cannot be empty")
|
||||
if request.limit is not None and request.limit <= 0:
|
||||
raise ValueError("Limit must be positive")
|
||||
data = {"url": request.url}
|
||||
if request.prompt:
|
||||
data["prompt"] = request.prompt
|
||||
if request.scrape_options is not None:
|
||||
opts = prepare_scrape_options(request.scrape_options)
|
||||
if opts:
|
||||
data["scrapeOptions"] = opts
|
||||
# Webhook conversion
|
||||
if request.webhook is not None:
|
||||
if isinstance(request.webhook, str):
|
||||
data["webhook"] = request.webhook
|
||||
else:
|
||||
data["webhook"] = request.webhook.model_dump(exclude_none=True)
|
||||
request_data = request.model_dump(exclude_none=True, exclude_unset=True)
|
||||
request_data.pop("url", None)
|
||||
request_data.pop("prompt", None)
|
||||
request_data.pop("scrape_options", None)
|
||||
field_mappings = {
|
||||
"include_paths": "includePaths",
|
||||
"exclude_paths": "excludePaths",
|
||||
"max_discovery_depth": "maxDiscoveryDepth",
|
||||
"sitemap": "sitemap",
|
||||
"ignore_query_parameters": "ignoreQueryParameters",
|
||||
"deduplicate_similar_urls": "deduplicateSimilarURLs",
|
||||
"crawl_entire_domain": "crawlEntireDomain",
|
||||
"allow_external_links": "allowExternalLinks",
|
||||
"allow_subdomains": "allowSubdomains",
|
||||
"ignore_robots_txt": "ignoreRobotsTxt",
|
||||
"robots_user_agent": "robotsUserAgent",
|
||||
"delay": "delay",
|
||||
"max_concurrency": "maxConcurrency",
|
||||
"regex_on_full_url": "regexOnFullURL",
|
||||
"zero_data_retention": "zeroDataRetention",
|
||||
}
|
||||
for snake, camel in field_mappings.items():
|
||||
if snake in request_data:
|
||||
data[camel] = request_data.pop(snake)
|
||||
data.update(request_data)
|
||||
if getattr(request, "integration", None) is not None:
|
||||
data["integration"] = str(getattr(request, "integration")).strip()
|
||||
return data
|
||||
|
||||
|
||||
def _parse_crawl_documents(data_list: Optional[List[Any]]) -> List[Document]:
|
||||
documents: List[Document] = []
|
||||
for doc_data in data_list or []:
|
||||
if isinstance(doc_data, dict):
|
||||
normalized = normalize_document_input(doc_data)
|
||||
documents.append(Document(**normalized))
|
||||
return documents
|
||||
|
||||
|
||||
def _parse_crawl_status_response(body: Dict[str, Any]) -> Dict[str, Any]:
|
||||
if not body.get("success"):
|
||||
raise Exception(body.get("error", "Unknown error occurred"))
|
||||
|
||||
return {
|
||||
"status": body.get("status"),
|
||||
"completed": body.get("completed", 0),
|
||||
"total": body.get("total", 0),
|
||||
"credits_used": body.get("creditsUsed", 0),
|
||||
"expires_at": body.get("expiresAt"),
|
||||
"next": body.get("next"),
|
||||
"data": _parse_crawl_documents(body.get("data", [])),
|
||||
}
|
||||
|
||||
|
||||
async def start_crawl(client: AsyncHttpClient, request: CrawlRequest) -> CrawlResponse:
|
||||
"""
|
||||
Start a crawl job for a website.
|
||||
|
||||
Args:
|
||||
client: Async HTTP client instance
|
||||
request: CrawlRequest containing URL and options
|
||||
|
||||
Returns:
|
||||
CrawlResponse with job information
|
||||
|
||||
Raises:
|
||||
ValueError: If request is invalid
|
||||
Exception: If the crawl operation fails to start
|
||||
"""
|
||||
payload = _prepare_crawl_request(request)
|
||||
response = await client.post("/v2/crawl", payload)
|
||||
if response.status_code >= 400:
|
||||
handle_response_error(response, "start crawl")
|
||||
body = response.json()
|
||||
if body.get("success"):
|
||||
return CrawlResponse(id=body.get("id"), url=body.get("url"))
|
||||
raise Exception(body.get("error", "Unknown error occurred"))
|
||||
|
||||
|
||||
async def get_crawl_status(
|
||||
client: AsyncHttpClient,
|
||||
job_id: str,
|
||||
pagination_config: Optional[PaginationConfig] = None,
|
||||
*,
|
||||
request_timeout: Optional[float] = None,
|
||||
) -> CrawlJob:
|
||||
"""
|
||||
Get the status of a crawl job.
|
||||
|
||||
Args:
|
||||
client: Async HTTP client instance
|
||||
job_id: ID of the crawl job
|
||||
pagination_config: Optional configuration for pagination limits
|
||||
request_timeout: Timeout (in seconds) for each individual HTTP request. When auto-pagination
|
||||
is enabled (default) and there are multiple pages of results, this timeout applies to
|
||||
each page request separately, not to the entire operation
|
||||
|
||||
Returns:
|
||||
CrawlJob with job information
|
||||
|
||||
Raises:
|
||||
Exception: If the status check fails
|
||||
"""
|
||||
response = await client.get(f"/v2/crawl/{job_id}", timeout=request_timeout)
|
||||
if response.status_code >= 400:
|
||||
handle_response_error(response, "get crawl status")
|
||||
body = response.json()
|
||||
payload = _parse_crawl_status_response(body)
|
||||
|
||||
documents = payload["data"]
|
||||
|
||||
# Handle pagination if requested
|
||||
auto_paginate = pagination_config.auto_paginate if pagination_config else True
|
||||
if auto_paginate and payload["next"]:
|
||||
documents = await _fetch_all_pages_async(
|
||||
client,
|
||||
payload["next"],
|
||||
documents,
|
||||
pagination_config,
|
||||
request_timeout=request_timeout,
|
||||
)
|
||||
|
||||
return CrawlJob(
|
||||
status=payload["status"],
|
||||
completed=payload["completed"],
|
||||
total=payload["total"],
|
||||
credits_used=payload["credits_used"],
|
||||
expires_at=payload["expires_at"],
|
||||
next=payload["next"] if not auto_paginate else None,
|
||||
data=documents,
|
||||
)
|
||||
|
||||
|
||||
async def get_crawl_status_page(
|
||||
client: AsyncHttpClient,
|
||||
next_url: str,
|
||||
*,
|
||||
request_timeout: Optional[float] = None,
|
||||
) -> CrawlJob:
|
||||
"""
|
||||
Fetch a single page of crawl results using the provided next URL.
|
||||
|
||||
Args:
|
||||
client: Async HTTP client instance
|
||||
next_url: Opaque next URL from a prior crawl status response
|
||||
request_timeout: Timeout (in seconds) for the HTTP request
|
||||
|
||||
Returns:
|
||||
CrawlJob with the page data and next URL (if any)
|
||||
|
||||
Raises:
|
||||
Exception: If the request fails or returns an error response
|
||||
"""
|
||||
response = await client.get(next_url, timeout=request_timeout)
|
||||
if response.status_code >= 400:
|
||||
handle_response_error(response, "get crawl status page")
|
||||
body = response.json()
|
||||
payload = _parse_crawl_status_response(body)
|
||||
return CrawlJob(
|
||||
status=payload["status"],
|
||||
completed=payload["completed"],
|
||||
total=payload["total"],
|
||||
credits_used=payload["credits_used"],
|
||||
expires_at=payload["expires_at"],
|
||||
next=payload["next"],
|
||||
data=payload["data"],
|
||||
)
|
||||
|
||||
|
||||
async def _fetch_all_pages_async(
|
||||
client: AsyncHttpClient,
|
||||
next_url: str,
|
||||
initial_documents: List[Document],
|
||||
pagination_config: Optional[PaginationConfig] = None,
|
||||
*,
|
||||
request_timeout: Optional[float] = None,
|
||||
) -> List[Document]:
|
||||
"""
|
||||
Fetch all pages of crawl results asynchronously.
|
||||
|
||||
Args:
|
||||
client: Async HTTP client instance
|
||||
next_url: URL for the next page
|
||||
initial_documents: Documents from the first page
|
||||
pagination_config: Optional configuration for pagination limits
|
||||
request_timeout: Optional timeout (in seconds) for the underlying HTTP request
|
||||
|
||||
Returns:
|
||||
List of all documents from all pages
|
||||
"""
|
||||
documents = initial_documents.copy()
|
||||
current_url = next_url
|
||||
page_count = 0
|
||||
|
||||
# Apply pagination limits
|
||||
max_pages = pagination_config.max_pages if pagination_config else None
|
||||
max_results = pagination_config.max_results if pagination_config else None
|
||||
max_wait_time = pagination_config.max_wait_time if pagination_config else None
|
||||
|
||||
start_time = time.monotonic()
|
||||
|
||||
while current_url:
|
||||
# Check pagination limits (treat 0 as a valid limit)
|
||||
if (max_pages is not None) and page_count >= max_pages:
|
||||
break
|
||||
|
||||
if (max_wait_time is not None) and (time.monotonic() - start_time) > max_wait_time:
|
||||
break
|
||||
|
||||
# Fetch next page
|
||||
response = await client.get(current_url, timeout=request_timeout)
|
||||
|
||||
if response.status_code >= 400:
|
||||
# Log error but continue with what we have
|
||||
import logging
|
||||
logger = logging.getLogger("firecrawl")
|
||||
logger.warning("Failed to fetch next page", extra={"status_code": response.status_code})
|
||||
break
|
||||
|
||||
page_data = response.json()
|
||||
try:
|
||||
page_payload = _parse_crawl_status_response(page_data)
|
||||
except Exception:
|
||||
break
|
||||
|
||||
# Add documents from this page
|
||||
for document in page_payload["data"]:
|
||||
# Check max_results limit
|
||||
if (max_results is not None) and (len(documents) >= max_results):
|
||||
break
|
||||
documents.append(document)
|
||||
|
||||
# Check if we hit max_results limit
|
||||
if (max_results is not None) and (len(documents) >= max_results):
|
||||
break
|
||||
|
||||
# Get next URL
|
||||
current_url = page_payload["next"]
|
||||
page_count += 1
|
||||
|
||||
return documents
|
||||
|
||||
|
||||
async def cancel_crawl(client: AsyncHttpClient, job_id: str) -> bool:
|
||||
"""
|
||||
Cancel a crawl job.
|
||||
|
||||
Args:
|
||||
client: Async HTTP client instance
|
||||
job_id: ID of the crawl job
|
||||
|
||||
Returns:
|
||||
True if cancellation was successful
|
||||
|
||||
Raises:
|
||||
Exception: If the cancellation operation fails
|
||||
"""
|
||||
response = await client.delete(f"/v2/crawl/{job_id}")
|
||||
if response.status_code >= 400:
|
||||
handle_response_error(response, "cancel crawl")
|
||||
body = response.json()
|
||||
return body.get("status") == "cancelled"
|
||||
|
||||
|
||||
async def crawl_params_preview(client: AsyncHttpClient, request: CrawlParamsRequest) -> CrawlParamsData:
|
||||
"""
|
||||
Preview crawl parameters before starting a crawl job.
|
||||
|
||||
Args:
|
||||
client: Async HTTP client instance
|
||||
request: CrawlParamsRequest containing URL and prompt
|
||||
|
||||
Returns:
|
||||
CrawlParamsData containing crawl configuration
|
||||
|
||||
Raises:
|
||||
ValueError: If request is invalid
|
||||
Exception: If the parameter preview fails
|
||||
"""
|
||||
if not request.url or not request.url.strip():
|
||||
raise ValueError("URL cannot be empty")
|
||||
if not request.prompt or not request.prompt.strip():
|
||||
raise ValueError("Prompt cannot be empty")
|
||||
payload = {"url": request.url, "prompt": request.prompt}
|
||||
response = await client.post("/v2/crawl/params-preview", payload)
|
||||
if response.status_code >= 400:
|
||||
handle_response_error(response, "crawl params preview")
|
||||
body = response.json()
|
||||
if not body.get("success"):
|
||||
raise Exception(body.get("error", "Unknown error occurred"))
|
||||
params_data = body.get("data", {})
|
||||
converted: Dict[str, Any] = {}
|
||||
mapping = {
|
||||
"includePaths": "include_paths",
|
||||
"excludePaths": "exclude_paths",
|
||||
"maxDiscoveryDepth": "max_discovery_depth",
|
||||
"sitemap": "sitemap",
|
||||
"ignoreQueryParameters": "ignore_query_parameters",
|
||||
"deduplicateSimilarURLs": "deduplicate_similar_urls",
|
||||
"crawlEntireDomain": "crawl_entire_domain",
|
||||
"allowExternalLinks": "allow_external_links",
|
||||
"allowSubdomains": "allow_subdomains",
|
||||
"ignoreRobotsTxt": "ignore_robots_txt",
|
||||
"robotsUserAgent": "robots_user_agent",
|
||||
"maxConcurrency": "max_concurrency",
|
||||
"scrapeOptions": "scrape_options",
|
||||
"zeroDataRetention": "zero_data_retention",
|
||||
}
|
||||
for camel, snake in mapping.items():
|
||||
if camel in params_data:
|
||||
converted[snake] = params_data[camel]
|
||||
if "webhook" in params_data:
|
||||
wk = params_data["webhook"]
|
||||
converted["webhook"] = wk
|
||||
if "warning" in body:
|
||||
converted["warning"] = body["warning"]
|
||||
return CrawlParamsData(**converted)
|
||||
|
||||
|
||||
async def get_crawl_errors(client: AsyncHttpClient, crawl_id: str) -> CrawlErrorsResponse:
|
||||
"""
|
||||
Get errors from a crawl job.
|
||||
|
||||
Args:
|
||||
client: Async HTTP client instance
|
||||
crawl_id: ID of the crawl job
|
||||
|
||||
Returns:
|
||||
CrawlErrorsResponse with errors and robots blocked
|
||||
|
||||
Raises:
|
||||
Exception: If the error check operation fails
|
||||
"""
|
||||
response = await client.get(f"/v2/crawl/{crawl_id}/errors")
|
||||
if response.status_code >= 400:
|
||||
handle_response_error(response, "check crawl errors")
|
||||
body = response.json()
|
||||
payload = body.get("data", body)
|
||||
normalized = {
|
||||
"errors": payload.get("errors", []),
|
||||
"robots_blocked": payload.get("robotsBlocked", payload.get("robots_blocked", [])),
|
||||
}
|
||||
return CrawlErrorsResponse(**normalized)
|
||||
|
||||
|
||||
async def get_active_crawls(client: AsyncHttpClient) -> ActiveCrawlsResponse:
|
||||
"""
|
||||
Get active crawl jobs.
|
||||
|
||||
Args:
|
||||
client: Async HTTP client instance
|
||||
|
||||
Returns:
|
||||
ActiveCrawlsResponse with active crawl jobs
|
||||
|
||||
Raises:
|
||||
Exception: If the active crawl jobs operation fails
|
||||
"""
|
||||
response = await client.get("/v2/crawl/active")
|
||||
if response.status_code >= 400:
|
||||
handle_response_error(response, "get active crawls")
|
||||
body = response.json()
|
||||
if not body.get("success"):
|
||||
raise Exception(body.get("error", "Unknown error occurred"))
|
||||
crawls_in = body.get("crawls", [])
|
||||
normalized = []
|
||||
for c in crawls_in:
|
||||
if isinstance(c, dict):
|
||||
normalized.append({
|
||||
"id": c.get("id"),
|
||||
"team_id": c.get("teamId", c.get("team_id")),
|
||||
"url": c.get("url"),
|
||||
"options": c.get("options"),
|
||||
})
|
||||
return ActiveCrawlsResponse(success=True, crawls=[ActiveCrawl(**nc) for nc in normalized])
|
||||
@@ -0,0 +1,164 @@
|
||||
from typing import Any, Dict, List, Optional
|
||||
import asyncio
|
||||
import warnings
|
||||
|
||||
from ...types import ExtractResponse, ScrapeOptions
|
||||
from ...utils.http_client_async import AsyncHttpClient
|
||||
from ...utils.validation import prepare_scrape_options
|
||||
|
||||
_EXTRACT_DEPRECATION_MSG = (
|
||||
"The extract endpoint is in maintenance mode and its use is discouraged. "
|
||||
"Review https://docs.firecrawl.dev/developer-guides/usage-guides/choosing-the-data-extractor "
|
||||
"to find a replacement."
|
||||
)
|
||||
|
||||
|
||||
def _prepare_extract_request(
|
||||
urls: Optional[List[str]],
|
||||
*,
|
||||
prompt: Optional[str] = None,
|
||||
schema: Optional[Dict[str, Any]] = None,
|
||||
system_prompt: Optional[str] = None,
|
||||
allow_external_links: Optional[bool] = None,
|
||||
enable_web_search: Optional[bool] = None,
|
||||
show_sources: Optional[bool] = None,
|
||||
scrape_options: Optional[ScrapeOptions] = None,
|
||||
ignore_invalid_urls: Optional[bool] = None,
|
||||
integration: Optional[str] = None,
|
||||
) -> Dict[str, Any]:
|
||||
body: Dict[str, Any] = {}
|
||||
if urls is not None:
|
||||
body["urls"] = urls
|
||||
if prompt is not None:
|
||||
body["prompt"] = prompt
|
||||
if schema is not None:
|
||||
body["schema"] = schema
|
||||
if system_prompt is not None:
|
||||
body["systemPrompt"] = system_prompt
|
||||
if allow_external_links is not None:
|
||||
body["allowExternalLinks"] = allow_external_links
|
||||
if enable_web_search is not None:
|
||||
body["enableWebSearch"] = enable_web_search
|
||||
if show_sources is not None:
|
||||
body["showSources"] = show_sources
|
||||
if ignore_invalid_urls is not None:
|
||||
body["ignoreInvalidURLs"] = ignore_invalid_urls
|
||||
if scrape_options is not None:
|
||||
prepared = prepare_scrape_options(scrape_options)
|
||||
if prepared:
|
||||
body["scrapeOptions"] = prepared
|
||||
if integration is not None and str(integration).strip():
|
||||
body["integration"] = str(integration).strip()
|
||||
return body
|
||||
|
||||
|
||||
async def start_extract(
|
||||
client: AsyncHttpClient,
|
||||
urls: Optional[List[str]],
|
||||
*,
|
||||
prompt: Optional[str] = None,
|
||||
schema: Optional[Dict[str, Any]] = None,
|
||||
system_prompt: Optional[str] = None,
|
||||
allow_external_links: Optional[bool] = None,
|
||||
enable_web_search: Optional[bool] = None,
|
||||
show_sources: Optional[bool] = None,
|
||||
scrape_options: Optional[ScrapeOptions] = None,
|
||||
ignore_invalid_urls: Optional[bool] = None,
|
||||
integration: Optional[str] = None,
|
||||
) -> ExtractResponse:
|
||||
"""Start an extract job (non-blocking, async).
|
||||
|
||||
.. deprecated::
|
||||
The extract endpoint is in maintenance mode and its use is discouraged.
|
||||
Review https://docs.firecrawl.dev/developer-guides/usage-guides/choosing-the-data-extractor
|
||||
to find a replacement.
|
||||
"""
|
||||
warnings.warn(_EXTRACT_DEPRECATION_MSG, DeprecationWarning, stacklevel=2)
|
||||
body = _prepare_extract_request(
|
||||
urls,
|
||||
prompt=prompt,
|
||||
schema=schema,
|
||||
system_prompt=system_prompt,
|
||||
allow_external_links=allow_external_links,
|
||||
enable_web_search=enable_web_search,
|
||||
show_sources=show_sources,
|
||||
scrape_options=scrape_options,
|
||||
ignore_invalid_urls=ignore_invalid_urls,
|
||||
integration=integration,
|
||||
)
|
||||
resp = await client.post("/v2/extract", body)
|
||||
return ExtractResponse(**resp.json())
|
||||
|
||||
|
||||
async def get_extract_status(client: AsyncHttpClient, job_id: str) -> ExtractResponse:
|
||||
"""Get the current status of an extract job (async).
|
||||
|
||||
.. deprecated::
|
||||
The extract endpoint is in maintenance mode and its use is discouraged.
|
||||
Review https://docs.firecrawl.dev/developer-guides/usage-guides/choosing-the-data-extractor
|
||||
to find a replacement.
|
||||
"""
|
||||
warnings.warn(_EXTRACT_DEPRECATION_MSG, DeprecationWarning, stacklevel=2)
|
||||
resp = await client.get(f"/v2/extract/{job_id}")
|
||||
return ExtractResponse(**resp.json())
|
||||
|
||||
|
||||
async def wait_extract(
|
||||
client: AsyncHttpClient,
|
||||
job_id: str,
|
||||
*,
|
||||
poll_interval: int = 2,
|
||||
timeout: Optional[int] = None,
|
||||
) -> ExtractResponse:
|
||||
start_ts = asyncio.get_event_loop().time()
|
||||
while True:
|
||||
status = await get_extract_status(client, job_id)
|
||||
if status.status in ("completed", "failed", "cancelled"):
|
||||
return status
|
||||
if timeout is not None and (asyncio.get_event_loop().time() - start_ts) > timeout:
|
||||
return status
|
||||
await asyncio.sleep(max(1, poll_interval))
|
||||
|
||||
|
||||
async def extract(
|
||||
client: AsyncHttpClient,
|
||||
urls: Optional[List[str]],
|
||||
*,
|
||||
prompt: Optional[str] = None,
|
||||
schema: Optional[Dict[str, Any]] = None,
|
||||
system_prompt: Optional[str] = None,
|
||||
allow_external_links: Optional[bool] = None,
|
||||
enable_web_search: Optional[bool] = None,
|
||||
show_sources: Optional[bool] = None,
|
||||
scrape_options: Optional[ScrapeOptions] = None,
|
||||
ignore_invalid_urls: Optional[bool] = None,
|
||||
poll_interval: int = 2,
|
||||
timeout: Optional[int] = None,
|
||||
integration: Optional[str] = None,
|
||||
) -> ExtractResponse:
|
||||
"""Extract structured data and wait until completion (async).
|
||||
|
||||
.. deprecated::
|
||||
The extract endpoint is in maintenance mode and its use is discouraged.
|
||||
Review https://docs.firecrawl.dev/developer-guides/usage-guides/choosing-the-data-extractor
|
||||
to find a replacement.
|
||||
"""
|
||||
warnings.warn(_EXTRACT_DEPRECATION_MSG, DeprecationWarning, stacklevel=2)
|
||||
started = await start_extract(
|
||||
client,
|
||||
urls,
|
||||
prompt=prompt,
|
||||
schema=schema,
|
||||
system_prompt=system_prompt,
|
||||
allow_external_links=allow_external_links,
|
||||
enable_web_search=enable_web_search,
|
||||
show_sources=show_sources,
|
||||
scrape_options=scrape_options,
|
||||
ignore_invalid_urls=ignore_invalid_urls,
|
||||
integration=integration,
|
||||
)
|
||||
job_id = getattr(started, "id", None)
|
||||
if not job_id:
|
||||
return started
|
||||
return await wait_extract(client, job_id, poll_interval=poll_interval, timeout=timeout)
|
||||
|
||||
@@ -0,0 +1,67 @@
|
||||
from typing import Optional, Dict, Any
|
||||
from ...types import MapOptions, MapData, LinkResult
|
||||
from ...utils.http_client_async import AsyncHttpClient
|
||||
from ...utils.error_handler import handle_response_error
|
||||
|
||||
|
||||
def _prepare_map_request(url: str, options: Optional[MapOptions] = None) -> Dict[str, Any]:
|
||||
if not url or not url.strip():
|
||||
raise ValueError("URL cannot be empty")
|
||||
payload: Dict[str, Any] = {"url": url.strip()}
|
||||
if options is not None:
|
||||
data: Dict[str, Any] = {}
|
||||
if getattr(options, "sitemap", None) is not None:
|
||||
data["sitemap"] = options.sitemap
|
||||
if options.search is not None:
|
||||
data["search"] = options.search
|
||||
if options.include_subdomains is not None:
|
||||
data["includeSubdomains"] = options.include_subdomains
|
||||
if options.ignore_query_parameters is not None:
|
||||
data["ignoreQueryParameters"] = options.ignore_query_parameters
|
||||
if options.limit is not None:
|
||||
if options.limit <= 0:
|
||||
raise ValueError("Limit must be positive")
|
||||
data["limit"] = options.limit
|
||||
if options.timeout is not None:
|
||||
data["timeout"] = options.timeout
|
||||
if options.integration is not None:
|
||||
data["integration"] = options.integration.strip()
|
||||
if options.location is not None:
|
||||
data["location"] = options.location.model_dump(exclude_none=True)
|
||||
payload.update(data)
|
||||
return payload
|
||||
|
||||
|
||||
async def map(client: AsyncHttpClient, url: str, options: Optional[MapOptions] = None) -> MapData:
|
||||
request_data = _prepare_map_request(url, options)
|
||||
response = await client.post("/v2/map", request_data)
|
||||
if response.status_code >= 400:
|
||||
handle_response_error(response, "map")
|
||||
body = response.json()
|
||||
if not body.get("success"):
|
||||
raise Exception(body.get("error", "Unknown error occurred"))
|
||||
|
||||
|
||||
# data = body.get("data", {})
|
||||
# result_links: list[LinkResult] = []
|
||||
# for item in data.get("links", []):
|
||||
# if isinstance(item, dict):
|
||||
# result_links.append(
|
||||
# LinkResult(
|
||||
# url=item.get("url", ""),
|
||||
# title=item.get("title"),
|
||||
# description=item.get("description"),
|
||||
# )
|
||||
# )
|
||||
# elif isinstance(item, str):
|
||||
# result_links.append(LinkResult(url=item))
|
||||
|
||||
result_links: list[LinkResult] = []
|
||||
for item in body.get("links", []):
|
||||
if isinstance(item, dict):
|
||||
result_links.append(LinkResult(url=item.get("url", ""), title=item.get("title"), description=item.get("description")))
|
||||
elif isinstance(item, str):
|
||||
result_links.append(LinkResult(url=item))
|
||||
|
||||
return MapData(links=result_links)
|
||||
|
||||
@@ -0,0 +1,215 @@
|
||||
import time
|
||||
from typing import Any, Dict, List, Optional
|
||||
from pydantic import BaseModel
|
||||
|
||||
from ...types import (
|
||||
Monitor,
|
||||
MonitorCheck,
|
||||
MonitorCheckDetail,
|
||||
MonitorCheckPage,
|
||||
MonitorCreateRequest,
|
||||
PaginationConfig,
|
||||
MonitorTarget,
|
||||
MonitorUpdateRequest,
|
||||
ScrapeOptions,
|
||||
)
|
||||
from ...utils.error_handler import handle_response_error
|
||||
from ...utils.http_client_async import AsyncHttpClient
|
||||
from ...utils.validation import prepare_scrape_options
|
||||
|
||||
|
||||
def _dump(value: Any) -> Any:
|
||||
if isinstance(value, ScrapeOptions):
|
||||
return prepare_scrape_options(value)
|
||||
if isinstance(value, MonitorTarget):
|
||||
data = value.model_dump(exclude_none=True, by_alias=True)
|
||||
if isinstance(value.scrape_options, ScrapeOptions):
|
||||
data["scrapeOptions"] = prepare_scrape_options(value.scrape_options)
|
||||
return _prepare_target(data)
|
||||
if isinstance(value, BaseModel):
|
||||
return value.model_dump(exclude_none=True, by_alias=True)
|
||||
if isinstance(value, list):
|
||||
return [_dump(item) for item in value]
|
||||
if isinstance(value, dict):
|
||||
return {key: _dump(item) for key, item in value.items() if item is not None}
|
||||
return value
|
||||
|
||||
|
||||
def _prepare_target(target: Dict[str, Any]) -> Dict[str, Any]:
|
||||
prepared = dict(target)
|
||||
if "scrapeOptions" in prepared and isinstance(prepared["scrapeOptions"], ScrapeOptions):
|
||||
prepared["scrapeOptions"] = prepare_scrape_options(prepared["scrapeOptions"])
|
||||
if "crawlOptions" in prepared:
|
||||
prepared["crawlOptions"] = _dump(prepared["crawlOptions"])
|
||||
return prepared
|
||||
|
||||
|
||||
def _prepare_payload(request: Any) -> Dict[str, Any]:
|
||||
payload = _dump(request)
|
||||
if not isinstance(payload, dict):
|
||||
raise ValueError("Monitor request must be an object")
|
||||
if "targets" in payload:
|
||||
payload["targets"] = [
|
||||
_prepare_target(_dump(target))
|
||||
for target in payload.get("targets", [])
|
||||
]
|
||||
return payload
|
||||
|
||||
|
||||
async def _data_or_error(response, action: str) -> Any:
|
||||
if response.status_code >= 400:
|
||||
handle_response_error(response, action)
|
||||
body = response.json()
|
||||
if not body.get("success"):
|
||||
raise Exception(body.get("error", "Unknown error occurred"))
|
||||
return body.get("data")
|
||||
|
||||
|
||||
async def _monitor_check_data_or_error(response, action: str) -> Dict[str, Any]:
|
||||
if response.status_code >= 400:
|
||||
handle_response_error(response, action)
|
||||
body = response.json()
|
||||
if not body.get("success"):
|
||||
raise Exception(body.get("error", "Unknown error occurred"))
|
||||
data = body.get("data") or {}
|
||||
if body.get("next") is not None:
|
||||
data["next"] = body.get("next")
|
||||
return data
|
||||
|
||||
|
||||
async def _fetch_all_monitor_check_pages(
|
||||
client: AsyncHttpClient,
|
||||
next_url: str,
|
||||
initial_pages: List[MonitorCheckPage],
|
||||
pagination_config: Optional[PaginationConfig] = None,
|
||||
) -> List[MonitorCheckPage]:
|
||||
pages = initial_pages.copy()
|
||||
current_url = next_url
|
||||
page_count = 0
|
||||
max_pages = pagination_config.max_pages if pagination_config else None
|
||||
max_results = pagination_config.max_results if pagination_config else None
|
||||
max_wait_time = pagination_config.max_wait_time if pagination_config else None
|
||||
start_time = time.monotonic()
|
||||
|
||||
while current_url:
|
||||
if max_pages is not None and page_count >= max_pages:
|
||||
break
|
||||
if max_wait_time is not None and (time.monotonic() - start_time) > max_wait_time:
|
||||
break
|
||||
|
||||
response = await client.get(current_url)
|
||||
if response.status_code >= 400:
|
||||
break
|
||||
try:
|
||||
data = await _monitor_check_data_or_error(response, "get monitor check page")
|
||||
except Exception:
|
||||
break
|
||||
|
||||
for page in data.get("pages") or []:
|
||||
if max_results is not None and len(pages) >= max_results:
|
||||
break
|
||||
pages.append(MonitorCheckPage(**page))
|
||||
|
||||
if max_results is not None and len(pages) >= max_results:
|
||||
break
|
||||
|
||||
current_url = data.get("next")
|
||||
page_count += 1
|
||||
|
||||
return pages
|
||||
|
||||
|
||||
async def create_monitor(client: AsyncHttpClient, request: MonitorCreateRequest) -> Monitor:
|
||||
data = await _data_or_error(await client.post("/v2/monitor", _prepare_payload(request)), "create monitor")
|
||||
return Monitor(**data)
|
||||
|
||||
|
||||
async def list_monitors(client: AsyncHttpClient, *, limit: Optional[int] = None, offset: Optional[int] = None) -> List[Monitor]:
|
||||
params = []
|
||||
if limit is not None:
|
||||
params.append(f"limit={limit}")
|
||||
if offset is not None:
|
||||
params.append(f"offset={offset}")
|
||||
suffix = f"?{'&'.join(params)}" if params else ""
|
||||
data = await _data_or_error(await client.get(f"/v2/monitor{suffix}"), "list monitors")
|
||||
return [Monitor(**item) for item in data or []]
|
||||
|
||||
|
||||
async def get_monitor(client: AsyncHttpClient, monitor_id: str) -> Monitor:
|
||||
data = await _data_or_error(await client.get(f"/v2/monitor/{monitor_id}"), "get monitor")
|
||||
return Monitor(**data)
|
||||
|
||||
|
||||
async def update_monitor(client: AsyncHttpClient, monitor_id: str, request: MonitorUpdateRequest) -> Monitor:
|
||||
data = await _data_or_error(await client.patch(f"/v2/monitor/{monitor_id}", _prepare_payload(request)), "update monitor")
|
||||
return Monitor(**data)
|
||||
|
||||
|
||||
async def delete_monitor(client: AsyncHttpClient, monitor_id: str) -> bool:
|
||||
response = await client.delete(f"/v2/monitor/{monitor_id}")
|
||||
if response.status_code >= 400:
|
||||
handle_response_error(response, "delete monitor")
|
||||
body = response.json()
|
||||
if not body.get("success"):
|
||||
raise Exception(body.get("error", "Unknown error occurred"))
|
||||
return True
|
||||
|
||||
|
||||
async def run_monitor(client: AsyncHttpClient, monitor_id: str) -> MonitorCheck:
|
||||
data = await _data_or_error(await client.post(f"/v2/monitor/{monitor_id}/run", {}), "run monitor")
|
||||
return MonitorCheck(**data)
|
||||
|
||||
|
||||
async def list_monitor_checks(
|
||||
client: AsyncHttpClient,
|
||||
monitor_id: str,
|
||||
*,
|
||||
limit: Optional[int] = None,
|
||||
offset: Optional[int] = None,
|
||||
) -> List[MonitorCheck]:
|
||||
params = []
|
||||
if limit is not None:
|
||||
params.append(f"limit={limit}")
|
||||
if offset is not None:
|
||||
params.append(f"offset={offset}")
|
||||
suffix = f"?{'&'.join(params)}" if params else ""
|
||||
data = await _data_or_error(await client.get(f"/v2/monitor/{monitor_id}/checks{suffix}"), "list monitor checks")
|
||||
return [MonitorCheck(**item) for item in data or []]
|
||||
|
||||
|
||||
async def get_monitor_check(
|
||||
client: AsyncHttpClient,
|
||||
monitor_id: str,
|
||||
check_id: str,
|
||||
*,
|
||||
limit: Optional[int] = None,
|
||||
skip: Optional[int] = None,
|
||||
status: Optional[str] = None,
|
||||
pagination_config: Optional[PaginationConfig] = None,
|
||||
) -> MonitorCheckDetail:
|
||||
params = []
|
||||
if limit is not None:
|
||||
params.append(f"limit={limit}")
|
||||
if skip is not None:
|
||||
params.append(f"skip={skip}")
|
||||
if status is not None:
|
||||
params.append(f"status={status}")
|
||||
suffix = f"?{'&'.join(params)}" if params else ""
|
||||
data = await _monitor_check_data_or_error(await client.get(f"/v2/monitor/{monitor_id}/checks/{check_id}{suffix}"), "get monitor check")
|
||||
detail = MonitorCheckDetail(**data)
|
||||
|
||||
auto_paginate = pagination_config.auto_paginate if pagination_config else True
|
||||
if auto_paginate and detail.next and not (
|
||||
pagination_config
|
||||
and pagination_config.max_results is not None
|
||||
and len(detail.pages) >= pagination_config.max_results
|
||||
):
|
||||
detail.pages = await _fetch_all_monitor_check_pages(
|
||||
client,
|
||||
detail.next,
|
||||
detail.pages,
|
||||
pagination_config,
|
||||
)
|
||||
detail.next = None
|
||||
|
||||
return detail
|
||||
@@ -0,0 +1,63 @@
|
||||
import asyncio
|
||||
from functools import partial
|
||||
import json
|
||||
from typing import Optional, Dict, Any, Tuple
|
||||
|
||||
from ...types import Document, ParseOptions
|
||||
from ...utils.normalize import normalize_document_input
|
||||
from ...utils.error_handler import handle_response_error
|
||||
from ...utils.http_client_async import AsyncHttpClient
|
||||
from ..parse import (
|
||||
ParseFileInput,
|
||||
_prepare_file_payload,
|
||||
_prepare_parse_options_payload,
|
||||
)
|
||||
|
||||
async def _prepare_parse_request(
|
||||
file: ParseFileInput,
|
||||
options: Optional[ParseOptions] = None,
|
||||
*,
|
||||
filename: Optional[str] = None,
|
||||
content_type: Optional[str] = None,
|
||||
) -> Tuple[Dict[str, Any], Dict[str, Tuple[str, bytes, str]]]:
|
||||
request_data = _prepare_parse_options_payload(options)
|
||||
multipart_fields = {"options": json.dumps(request_data)}
|
||||
loop = asyncio.get_running_loop()
|
||||
multipart_files = await loop.run_in_executor(
|
||||
None,
|
||||
partial(
|
||||
_prepare_file_payload,
|
||||
file,
|
||||
filename=filename,
|
||||
content_type=content_type,
|
||||
),
|
||||
)
|
||||
return multipart_fields, multipart_files
|
||||
|
||||
|
||||
async def parse(
|
||||
client: AsyncHttpClient,
|
||||
file: ParseFileInput,
|
||||
options: Optional[ParseOptions] = None,
|
||||
*,
|
||||
filename: Optional[str] = None,
|
||||
content_type: Optional[str] = None,
|
||||
) -> Document:
|
||||
fields, files = await _prepare_parse_request(
|
||||
file,
|
||||
options,
|
||||
filename=filename,
|
||||
content_type=content_type,
|
||||
)
|
||||
|
||||
response = await client.post_multipart("/v2/parse", data=fields, files=files)
|
||||
if response.status_code >= 400:
|
||||
handle_response_error(response, "parse")
|
||||
|
||||
body = response.json()
|
||||
if not body.get("success"):
|
||||
raise Exception(body.get("error", "Unknown error occurred"))
|
||||
|
||||
document_data = body.get("data", {})
|
||||
normalized = normalize_document_input(document_data)
|
||||
return Document(**normalized)
|
||||
@@ -0,0 +1,144 @@
|
||||
from typing import Optional, Dict, Any, Literal
|
||||
from ...types import (
|
||||
ScrapeOptions,
|
||||
Document,
|
||||
BrowserExecuteResponse,
|
||||
BrowserDeleteResponse,
|
||||
)
|
||||
from ...utils.normalize import normalize_document_input
|
||||
from ...utils.error_handler import handle_response_error
|
||||
from ...utils.validation import prepare_scrape_options, validate_scrape_options
|
||||
from ...utils.http_client_async import AsyncHttpClient
|
||||
|
||||
|
||||
async def _prepare_scrape_request(url: str, options: Optional[ScrapeOptions] = None) -> Dict[str, Any]:
|
||||
if not url or not url.strip():
|
||||
raise ValueError("URL cannot be empty")
|
||||
payload: Dict[str, Any] = {"url": url.strip()}
|
||||
if options is not None:
|
||||
validated = validate_scrape_options(options)
|
||||
if validated is not None:
|
||||
opts = prepare_scrape_options(validated)
|
||||
if opts:
|
||||
payload.update(opts)
|
||||
return payload
|
||||
|
||||
|
||||
async def scrape(client: AsyncHttpClient, url: str, options: Optional[ScrapeOptions] = None) -> Document:
|
||||
payload = await _prepare_scrape_request(url, options)
|
||||
response = await client.post("/v2/scrape", payload)
|
||||
if response.status_code >= 400:
|
||||
handle_response_error(response, "scrape")
|
||||
body = response.json()
|
||||
if not body.get("success"):
|
||||
raise Exception(body.get("error", "Unknown error occurred"))
|
||||
document_data = body.get("data", {})
|
||||
normalized = normalize_document_input(document_data)
|
||||
return Document(**normalized)
|
||||
|
||||
|
||||
async def interact(
|
||||
client: AsyncHttpClient,
|
||||
job_id: str,
|
||||
code: Optional[str] = None,
|
||||
*,
|
||||
prompt: Optional[str] = None,
|
||||
language: Literal["python", "node", "bash"] = "node",
|
||||
timeout: Optional[int] = None,
|
||||
origin: Optional[str] = None,
|
||||
) -> BrowserExecuteResponse:
|
||||
if not job_id or not job_id.strip():
|
||||
raise ValueError("Job ID cannot be empty")
|
||||
has_code = code and code.strip()
|
||||
has_prompt = prompt and prompt.strip()
|
||||
if not has_code and not has_prompt:
|
||||
raise ValueError("Either 'code' or 'prompt' must be provided")
|
||||
|
||||
payload: Dict[str, Any] = {
|
||||
"language": language,
|
||||
}
|
||||
if has_code:
|
||||
payload["code"] = code
|
||||
if has_prompt:
|
||||
payload["prompt"] = prompt
|
||||
if timeout is not None:
|
||||
payload["timeout"] = timeout
|
||||
if origin is not None:
|
||||
payload["origin"] = origin
|
||||
|
||||
response = await client.post(f"/v2/scrape/{job_id}/interact", payload)
|
||||
if response.status_code >= 400:
|
||||
handle_response_error(response, "interact with scrape browser")
|
||||
|
||||
body = response.json()
|
||||
if not body.get("success"):
|
||||
raise Exception(body.get("error", "Unknown error occurred"))
|
||||
|
||||
normalized = dict(body)
|
||||
if "exitCode" in normalized and "exit_code" not in normalized:
|
||||
normalized["exit_code"] = normalized["exitCode"]
|
||||
if "liveViewUrl" in normalized and "live_view_url" not in normalized:
|
||||
normalized["live_view_url"] = normalized["liveViewUrl"]
|
||||
if "interactiveLiveViewUrl" in normalized and "interactive_live_view_url" not in normalized:
|
||||
normalized["interactive_live_view_url"] = normalized["interactiveLiveViewUrl"]
|
||||
return BrowserExecuteResponse(**normalized)
|
||||
|
||||
|
||||
async def stop_interaction(
|
||||
client: AsyncHttpClient,
|
||||
job_id: str,
|
||||
) -> BrowserDeleteResponse:
|
||||
if not job_id or not job_id.strip():
|
||||
raise ValueError("Job ID cannot be empty")
|
||||
|
||||
response = await client.delete(f"/v2/scrape/{job_id}/interact")
|
||||
if response.status_code >= 400:
|
||||
handle_response_error(response, "stop interaction")
|
||||
|
||||
body = response.json()
|
||||
normalized = dict(body)
|
||||
if "sessionDurationMs" in normalized and "session_duration_ms" not in normalized:
|
||||
normalized["session_duration_ms"] = normalized["sessionDurationMs"]
|
||||
if "creditsBilled" in normalized and "credits_billed" not in normalized:
|
||||
normalized["credits_billed"] = normalized["creditsBilled"]
|
||||
|
||||
return BrowserDeleteResponse(**normalized)
|
||||
|
||||
|
||||
async def stop_interactive_browser(
|
||||
client: AsyncHttpClient,
|
||||
job_id: str,
|
||||
) -> BrowserDeleteResponse:
|
||||
"""Deprecated alias for stop_interaction()."""
|
||||
return await stop_interaction(client, job_id)
|
||||
|
||||
|
||||
async def scrape_execute(
|
||||
client: AsyncHttpClient,
|
||||
job_id: str,
|
||||
code: Optional[str] = None,
|
||||
*,
|
||||
prompt: Optional[str] = None,
|
||||
language: Literal["python", "node", "bash"] = "node",
|
||||
timeout: Optional[int] = None,
|
||||
origin: Optional[str] = None,
|
||||
) -> BrowserExecuteResponse:
|
||||
"""Deprecated alias for interact()."""
|
||||
return await interact(
|
||||
client,
|
||||
job_id,
|
||||
code,
|
||||
prompt=prompt,
|
||||
language=language,
|
||||
timeout=timeout,
|
||||
origin=origin,
|
||||
)
|
||||
|
||||
|
||||
async def delete_scrape_browser(
|
||||
client: AsyncHttpClient,
|
||||
job_id: str,
|
||||
) -> BrowserDeleteResponse:
|
||||
"""Deprecated alias for stop_interaction()."""
|
||||
return await stop_interaction(client, job_id)
|
||||
|
||||
@@ -0,0 +1,178 @@
|
||||
from typing import Dict, Any, Union, List, TypeVar, Type
|
||||
from ...types import (
|
||||
SearchRequest,
|
||||
SearchData,
|
||||
Document,
|
||||
SearchResultWeb,
|
||||
SearchResultNews,
|
||||
SearchResultImages,
|
||||
)
|
||||
from ...utils.http_client_async import AsyncHttpClient
|
||||
from ...utils.error_handler import handle_response_error
|
||||
from ...utils.normalize import normalize_document_input
|
||||
from ...utils.validation import validate_scrape_options, prepare_scrape_options
|
||||
|
||||
T = TypeVar("T")
|
||||
|
||||
async def search(
|
||||
client: AsyncHttpClient,
|
||||
request: SearchRequest
|
||||
) -> SearchData:
|
||||
"""
|
||||
Async search for documents.
|
||||
|
||||
Args:
|
||||
client: Async HTTP client instance
|
||||
request: Search request
|
||||
|
||||
Returns:
|
||||
SearchData with search results grouped by source type
|
||||
|
||||
Raises:
|
||||
FirecrawlError: If the search operation fails
|
||||
"""
|
||||
request_data = _prepare_search_request(request)
|
||||
try:
|
||||
response = await client.post("/v2/search", request_data)
|
||||
if response.status_code != 200:
|
||||
handle_response_error(response, "search")
|
||||
response_data = response.json()
|
||||
if not response_data.get("success"):
|
||||
handle_response_error(response, "search")
|
||||
data = response_data.get("data", {}) or {}
|
||||
out = SearchData()
|
||||
if "web" in data:
|
||||
out.web = _transform_array(data["web"], SearchResultWeb)
|
||||
if "news" in data:
|
||||
out.news = _transform_array(data["news"], SearchResultNews)
|
||||
if "images" in data:
|
||||
out.images = _transform_array(data["images"], SearchResultImages)
|
||||
return out
|
||||
except Exception as err:
|
||||
if hasattr(err, "response"):
|
||||
handle_response_error(getattr(err, "response"), "search")
|
||||
raise err
|
||||
|
||||
def _transform_array(arr: List[Any], result_type: Type[T]) -> List[Union[T, Document]]:
|
||||
"""
|
||||
Transforms an array of items into a list of result_type or Document.
|
||||
If the item dict contains any of the special keys, it is treated as a Document.
|
||||
Otherwise, it is treated as result_type.
|
||||
If the item is not a dict, it is wrapped as result_type with url=item.
|
||||
"""
|
||||
results: List[Union[T, Document]] = []
|
||||
for item in arr:
|
||||
if item and isinstance(item, dict):
|
||||
if (
|
||||
"markdown" in item or
|
||||
"html" in item or
|
||||
"rawHtml" in item or
|
||||
"links" in item or
|
||||
"screenshot" in item or
|
||||
"changeTracking" in item or
|
||||
"summary" in item or
|
||||
"json" in item
|
||||
):
|
||||
results.append(Document(**normalize_document_input(item)))
|
||||
else:
|
||||
results.append(result_type(**item))
|
||||
else:
|
||||
results.append(result_type(url=item))
|
||||
return results
|
||||
|
||||
def _validate_search_request(request: SearchRequest) -> SearchRequest:
|
||||
"""
|
||||
Validate and normalize search request.
|
||||
|
||||
Args:
|
||||
request: Search request to validate
|
||||
|
||||
Returns:
|
||||
Validated request
|
||||
|
||||
Raises:
|
||||
ValueError: If request is invalid
|
||||
"""
|
||||
if not request.query or not request.query.strip():
|
||||
raise ValueError("Query cannot be empty")
|
||||
|
||||
if request.limit is not None:
|
||||
if request.limit <= 0:
|
||||
raise ValueError("Limit must be positive")
|
||||
if request.limit > 100:
|
||||
raise ValueError("Limit cannot exceed 100")
|
||||
|
||||
if request.timeout is not None:
|
||||
if request.timeout <= 0:
|
||||
raise ValueError("Timeout must be positive")
|
||||
if request.timeout > 300000:
|
||||
raise ValueError("Timeout cannot exceed 300000ms (5 minutes)")
|
||||
|
||||
if request.sources is not None:
|
||||
valid_sources = {"web", "news", "images"}
|
||||
for source in request.sources:
|
||||
if isinstance(source, str):
|
||||
if source not in valid_sources:
|
||||
raise ValueError(f"Invalid source type: {source}. Valid types: {valid_sources}")
|
||||
elif hasattr(source, 'type'):
|
||||
if source.type not in valid_sources:
|
||||
raise ValueError(f"Invalid source type: {source.type}. Valid types: {valid_sources}")
|
||||
|
||||
if request.include_domains and request.exclude_domains:
|
||||
raise ValueError(
|
||||
"include_domains and exclude_domains cannot both be specified"
|
||||
)
|
||||
|
||||
if request.location is not None:
|
||||
if not isinstance(request.location, str) or len(request.location.strip()) == 0:
|
||||
raise ValueError("Location must be a non-empty string")
|
||||
|
||||
if request.tbs is not None:
|
||||
if not isinstance(request.tbs, str) or len(request.tbs.strip()) == 0:
|
||||
raise ValueError("tbs must be a non-empty string")
|
||||
|
||||
if request.scrape_options is not None:
|
||||
validate_scrape_options(request.scrape_options)
|
||||
|
||||
return request
|
||||
|
||||
def _prepare_search_request(request: SearchRequest) -> Dict[str, Any]:
|
||||
"""
|
||||
Prepare a search request payload.
|
||||
|
||||
Args:
|
||||
request: Search request
|
||||
|
||||
Returns:
|
||||
Request payload dictionary
|
||||
"""
|
||||
validated_request = _validate_search_request(request)
|
||||
data = validated_request.model_dump(exclude_none=True, by_alias=True)
|
||||
|
||||
if "limit" not in data and validated_request.limit is not None:
|
||||
data["limit"] = validated_request.limit
|
||||
if "timeout" not in data and validated_request.timeout is not None:
|
||||
data["timeout"] = validated_request.timeout
|
||||
|
||||
if validated_request.ignore_invalid_urls is not None:
|
||||
data["ignoreInvalidURLs"] = validated_request.ignore_invalid_urls
|
||||
data.pop("ignore_invalid_urls", None)
|
||||
|
||||
if validated_request.include_domains is not None:
|
||||
data["includeDomains"] = validated_request.include_domains
|
||||
data.pop("include_domains", None)
|
||||
|
||||
if validated_request.exclude_domains is not None:
|
||||
data["excludeDomains"] = validated_request.exclude_domains
|
||||
data.pop("exclude_domains", None)
|
||||
|
||||
if validated_request.scrape_options is not None:
|
||||
scrape_data = prepare_scrape_options(validated_request.scrape_options)
|
||||
if scrape_data:
|
||||
data["scrapeOptions"] = scrape_data
|
||||
data.pop("scrape_options", None)
|
||||
|
||||
if (v := getattr(validated_request, "integration", None)) is not None and str(v).strip():
|
||||
data["integration"] = str(validated_request.integration).strip()
|
||||
|
||||
return data
|
||||
@@ -0,0 +1,89 @@
|
||||
from ...utils.http_client_async import AsyncHttpClient
|
||||
from ...utils.error_handler import handle_response_error
|
||||
from ...types import ConcurrencyCheck, CreditUsage, TokenUsage, CreditUsageHistoricalResponse, TokenUsageHistoricalResponse, QueueStatusResponse
|
||||
|
||||
|
||||
async def get_concurrency(client: AsyncHttpClient) -> ConcurrencyCheck:
|
||||
resp = await client.get("/v2/concurrency-check")
|
||||
if resp.status_code >= 400:
|
||||
handle_response_error(resp, "get concurrency")
|
||||
body = resp.json()
|
||||
if not body.get("success"):
|
||||
raise Exception(body.get("error", "Unknown error"))
|
||||
data = body.get("data", body)
|
||||
return ConcurrencyCheck(
|
||||
concurrency=data.get("concurrency"),
|
||||
max_concurrency=data.get("maxConcurrency", data.get("max_concurrency")),
|
||||
)
|
||||
|
||||
|
||||
async def get_credit_usage(client: AsyncHttpClient) -> CreditUsage:
|
||||
resp = await client.get("/v2/team/credit-usage")
|
||||
if resp.status_code >= 400:
|
||||
handle_response_error(resp, "get credit usage")
|
||||
body = resp.json()
|
||||
if not body.get("success"):
|
||||
raise Exception(body.get("error", "Unknown error"))
|
||||
data = body.get("data", body)
|
||||
return CreditUsage(
|
||||
remaining_credits=data.get("remainingCredits", data.get("remaining_credits", 0)),
|
||||
plan_credits=data.get("planCredits", data.get("plan_credits")),
|
||||
billing_period_start=data.get("billingPeriodStart", data.get("billing_period_start")),
|
||||
billing_period_end=data.get("billingPeriodEnd", data.get("billing_period_end")),
|
||||
)
|
||||
|
||||
|
||||
async def get_token_usage(client: AsyncHttpClient) -> TokenUsage:
|
||||
resp = await client.get("/v2/team/token-usage")
|
||||
if resp.status_code >= 400:
|
||||
handle_response_error(resp, "get token usage")
|
||||
body = resp.json()
|
||||
if not body.get("success"):
|
||||
raise Exception(body.get("error", "Unknown error"))
|
||||
data = body.get("data", body)
|
||||
return TokenUsage(
|
||||
remaining_tokens=data.get("remainingTokens", data.get("remaining_tokens", 0)),
|
||||
plan_tokens=data.get("planTokens", data.get("plan_tokens")),
|
||||
billing_period_start=data.get("billingPeriodStart", data.get("billing_period_start")),
|
||||
billing_period_end=data.get("billingPeriodEnd", data.get("billing_period_end")),
|
||||
)
|
||||
|
||||
|
||||
async def get_queue_status(client: AsyncHttpClient) -> QueueStatusResponse:
|
||||
resp = await client.get("/v2/team/queue-status")
|
||||
if resp.status_code >= 400:
|
||||
handle_response_error(resp, "get queue status")
|
||||
body = resp.json()
|
||||
if not body.get("success"):
|
||||
raise Exception(body.get("error", "Unknown error"))
|
||||
data = body.get("data", body)
|
||||
return QueueStatusResponse(
|
||||
jobs_in_queue=data.get("jobsInQueue", 0),
|
||||
active_jobs_in_queue=data.get("activeJobsInQueue", 0),
|
||||
waiting_jobs_in_queue=data.get("waitingJobsInQueue", 0),
|
||||
max_concurrency=data.get("maxConcurrency", 0),
|
||||
most_recent_success=data.get("mostRecentSuccess", None),
|
||||
)
|
||||
|
||||
|
||||
async def get_credit_usage_historical(client: AsyncHttpClient, by_api_key: bool = False) -> CreditUsageHistoricalResponse:
|
||||
query = "?byApiKey=true" if by_api_key else ""
|
||||
resp = await client.get(f"/v2/team/credit-usage/historical{query}")
|
||||
if resp.status_code >= 400:
|
||||
handle_response_error(resp, "get credit usage historical")
|
||||
body = resp.json()
|
||||
if not body.get("success"):
|
||||
raise Exception(body.get("error", "Unknown error"))
|
||||
return CreditUsageHistoricalResponse(**body)
|
||||
|
||||
|
||||
async def get_token_usage_historical(client: AsyncHttpClient, by_api_key: bool = False) -> TokenUsageHistoricalResponse:
|
||||
query = "?byApiKey=true" if by_api_key else ""
|
||||
resp = await client.get(f"/v2/team/token-usage/historical{query}")
|
||||
if resp.status_code >= 400:
|
||||
handle_response_error(resp, "get token usage historical")
|
||||
body = resp.json()
|
||||
if not body.get("success"):
|
||||
raise Exception(body.get("error", "Unknown error"))
|
||||
return TokenUsageHistoricalResponse(**body)
|
||||
|
||||
554
참고/firecrawl-main/apps/python-sdk/firecrawl/v2/methods/batch.py
Normal file
554
참고/firecrawl-main/apps/python-sdk/firecrawl/v2/methods/batch.py
Normal file
@@ -0,0 +1,554 @@
|
||||
"""
|
||||
Batch scraping functionality for Firecrawl v2 API.
|
||||
"""
|
||||
|
||||
import time
|
||||
from typing import Optional, List, Callable, Dict, Any, Union
|
||||
from ..types import (
|
||||
BatchScrapeRequest,
|
||||
BatchScrapeResponse,
|
||||
BatchScrapeJob,
|
||||
ScrapeOptions,
|
||||
Document,
|
||||
WebhookConfig,
|
||||
PaginationConfig,
|
||||
)
|
||||
from ..utils import HttpClient, handle_response_error, validate_scrape_options, prepare_scrape_options
|
||||
from ..utils.normalize import normalize_document_input
|
||||
from ..types import CrawlErrorsResponse
|
||||
|
||||
|
||||
def _parse_batch_scrape_documents(data_list: Optional[List[Any]]) -> List[Document]:
|
||||
documents: List[Document] = []
|
||||
for doc in data_list or []:
|
||||
if isinstance(doc, dict):
|
||||
normalized = normalize_document_input(doc)
|
||||
documents.append(Document(**normalized))
|
||||
return documents
|
||||
|
||||
|
||||
def _parse_batch_scrape_status_response(body: Dict[str, Any]) -> Dict[str, Any]:
|
||||
if not body.get("success"):
|
||||
raise Exception(body.get("error", "Unknown error occurred"))
|
||||
|
||||
return {
|
||||
"status": body.get("status"),
|
||||
"completed": body.get("completed", 0),
|
||||
"total": body.get("total", 0),
|
||||
"credits_used": body.get("creditsUsed"),
|
||||
"expires_at": body.get("expiresAt"),
|
||||
"next": body.get("next"),
|
||||
"data": _parse_batch_scrape_documents(body.get("data", []) or []),
|
||||
}
|
||||
|
||||
|
||||
def start_batch_scrape(
|
||||
client: HttpClient,
|
||||
urls: List[str],
|
||||
*,
|
||||
options: Optional[ScrapeOptions] = None,
|
||||
webhook: Optional[Union[str, WebhookConfig]] = None,
|
||||
append_to_id: Optional[str] = None,
|
||||
ignore_invalid_urls: Optional[bool] = None,
|
||||
max_concurrency: Optional[int] = None,
|
||||
zero_data_retention: Optional[bool] = None,
|
||||
integration: Optional[str] = None,
|
||||
idempotency_key: Optional[str] = None,
|
||||
) -> BatchScrapeResponse:
|
||||
"""
|
||||
Start a batch scrape job for multiple URLs.
|
||||
|
||||
Args:
|
||||
client: HTTP client instance
|
||||
urls: List of URLs to scrape
|
||||
options: Scraping options
|
||||
|
||||
Returns:
|
||||
BatchScrapeResponse containing job information
|
||||
|
||||
Raises:
|
||||
FirecrawlError: If the batch scrape operation fails to start
|
||||
"""
|
||||
# Prepare request data
|
||||
request_data = prepare_batch_scrape_request(
|
||||
urls,
|
||||
options=options,
|
||||
webhook=webhook,
|
||||
append_to_id=append_to_id,
|
||||
ignore_invalid_urls=ignore_invalid_urls,
|
||||
max_concurrency=max_concurrency,
|
||||
zero_data_retention=zero_data_retention,
|
||||
integration=integration,
|
||||
)
|
||||
|
||||
# Make the API request
|
||||
headers = client._prepare_headers(idempotency_key) # type: ignore[attr-defined]
|
||||
response = client.post("/v2/batch/scrape", request_data, headers=headers)
|
||||
|
||||
# Handle errors
|
||||
if not response.ok:
|
||||
handle_response_error(response, "start batch scrape")
|
||||
|
||||
# Parse response
|
||||
body = response.json()
|
||||
if not body.get("success"):
|
||||
raise Exception(body.get("error", "Unknown error occurred"))
|
||||
return BatchScrapeResponse(
|
||||
id=body.get("id"),
|
||||
url=body.get("url"),
|
||||
invalid_urls=body.get("invalidURLs") or None,
|
||||
)
|
||||
|
||||
|
||||
def get_batch_scrape_status(
|
||||
client: HttpClient,
|
||||
job_id: str,
|
||||
pagination_config: Optional[PaginationConfig] = None
|
||||
) -> BatchScrapeJob:
|
||||
"""
|
||||
Get the status of a batch scrape job.
|
||||
|
||||
Args:
|
||||
client: HTTP client instance
|
||||
job_id: ID of the batch scrape job
|
||||
pagination_config: Optional configuration for pagination behavior
|
||||
|
||||
Returns:
|
||||
BatchScrapeJob containing job status and data
|
||||
|
||||
Raises:
|
||||
FirecrawlError: If the status check fails
|
||||
"""
|
||||
# Make the API request
|
||||
response = client.get(f"/v2/batch/scrape/{job_id}")
|
||||
|
||||
# Handle errors
|
||||
if not response.ok:
|
||||
handle_response_error(response, "get batch scrape status")
|
||||
|
||||
# Parse response
|
||||
body = response.json()
|
||||
payload = _parse_batch_scrape_status_response(body)
|
||||
documents = payload["data"]
|
||||
|
||||
# Handle pagination if requested
|
||||
auto_paginate = pagination_config.auto_paginate if pagination_config else True
|
||||
if auto_paginate and payload["next"]:
|
||||
documents = _fetch_all_batch_pages(
|
||||
client,
|
||||
payload["next"],
|
||||
documents,
|
||||
pagination_config
|
||||
)
|
||||
|
||||
return BatchScrapeJob(
|
||||
status=payload["status"],
|
||||
completed=payload["completed"],
|
||||
total=payload["total"],
|
||||
credits_used=payload["credits_used"],
|
||||
expires_at=payload["expires_at"],
|
||||
next=payload["next"] if not auto_paginate else None,
|
||||
data=documents,
|
||||
)
|
||||
|
||||
|
||||
def get_batch_scrape_status_page(
|
||||
client: HttpClient,
|
||||
next_url: str,
|
||||
*,
|
||||
request_timeout: Optional[float] = None,
|
||||
) -> BatchScrapeJob:
|
||||
"""
|
||||
Fetch a single page of batch scrape results using the provided next URL.
|
||||
|
||||
Args:
|
||||
client: HTTP client instance
|
||||
next_url: Opaque next URL from a prior batch scrape status response
|
||||
request_timeout: Timeout (in seconds) for the HTTP request
|
||||
|
||||
Returns:
|
||||
BatchScrapeJob with the page data and next URL (if any)
|
||||
|
||||
Raises:
|
||||
Exception: If the request fails or returns an error response
|
||||
"""
|
||||
response = client.get(next_url, timeout=request_timeout)
|
||||
|
||||
if not response.ok:
|
||||
handle_response_error(response, "get batch scrape status page")
|
||||
|
||||
body = response.json()
|
||||
payload = _parse_batch_scrape_status_response(body)
|
||||
|
||||
return BatchScrapeJob(
|
||||
status=payload["status"],
|
||||
completed=payload["completed"],
|
||||
total=payload["total"],
|
||||
credits_used=payload["credits_used"],
|
||||
expires_at=payload["expires_at"],
|
||||
next=payload["next"],
|
||||
data=payload["data"],
|
||||
)
|
||||
|
||||
|
||||
def _fetch_all_batch_pages(
|
||||
client: HttpClient,
|
||||
next_url: str,
|
||||
initial_documents: List[Document],
|
||||
pagination_config: Optional[PaginationConfig] = None
|
||||
) -> List[Document]:
|
||||
"""
|
||||
Fetch all pages of batch scrape results.
|
||||
|
||||
Args:
|
||||
client: HTTP client instance
|
||||
next_url: URL for the next page
|
||||
initial_documents: Documents from the first page
|
||||
pagination_config: Optional configuration for pagination limits
|
||||
|
||||
Returns:
|
||||
List of all documents from all pages
|
||||
"""
|
||||
documents = initial_documents.copy()
|
||||
current_url = next_url
|
||||
page_count = 0
|
||||
|
||||
# Apply pagination limits
|
||||
max_pages = pagination_config.max_pages if pagination_config else None
|
||||
max_results = pagination_config.max_results if pagination_config else None
|
||||
max_wait_time = pagination_config.max_wait_time if pagination_config else None
|
||||
|
||||
start_time = time.monotonic()
|
||||
|
||||
while current_url:
|
||||
# Check pagination limits (treat 0 as a valid limit)
|
||||
if (max_pages is not None) and page_count >= max_pages:
|
||||
break
|
||||
|
||||
if (max_wait_time is not None) and (time.monotonic() - start_time) > max_wait_time:
|
||||
break
|
||||
|
||||
# Fetch next page
|
||||
response = client.get(current_url)
|
||||
|
||||
if not response.ok:
|
||||
# Log error but continue with what we have
|
||||
import logging
|
||||
logger = logging.getLogger("firecrawl")
|
||||
logger.warning("Failed to fetch next page", extra={"status_code": response.status_code})
|
||||
break
|
||||
|
||||
page_data = response.json()
|
||||
try:
|
||||
page_payload = _parse_batch_scrape_status_response(page_data)
|
||||
except Exception:
|
||||
break
|
||||
|
||||
# Add documents from this page
|
||||
for document in page_payload["data"]:
|
||||
# Check max_results limit
|
||||
if max_results is not None and len(documents) >= max_results:
|
||||
break
|
||||
documents.append(document)
|
||||
|
||||
# Check if we hit max_results limit after adding all docs from this page
|
||||
if max_results is not None and len(documents) >= max_results:
|
||||
break
|
||||
|
||||
# Get next URL
|
||||
current_url = page_payload["next"]
|
||||
page_count += 1
|
||||
|
||||
return documents
|
||||
|
||||
|
||||
def cancel_batch_scrape(
|
||||
client: HttpClient,
|
||||
job_id: str
|
||||
) -> bool:
|
||||
"""
|
||||
Cancel a running batch scrape job.
|
||||
|
||||
Args:
|
||||
client: HTTP client instance
|
||||
job_id: ID of the batch scrape job to cancel
|
||||
|
||||
Returns:
|
||||
BatchScrapeStatusResponse with updated status
|
||||
|
||||
Raises:
|
||||
FirecrawlError: If the cancellation fails
|
||||
"""
|
||||
# Make the API request
|
||||
response = client.delete(f"/v2/batch/scrape/{job_id}")
|
||||
|
||||
# Handle errors
|
||||
if not response.ok:
|
||||
handle_response_error(response, "cancel batch scrape")
|
||||
|
||||
# Parse response
|
||||
body = response.json()
|
||||
return body.get("status") == "cancelled"
|
||||
|
||||
|
||||
def wait_for_batch_completion(
|
||||
client: HttpClient,
|
||||
job_id: str,
|
||||
poll_interval: int = 2,
|
||||
timeout: Optional[int] = None
|
||||
) -> BatchScrapeJob:
|
||||
"""
|
||||
Wait for a batch scrape job to complete, polling for status updates.
|
||||
|
||||
Args:
|
||||
client: HTTP client instance
|
||||
job_id: ID of the batch scrape job
|
||||
poll_interval: Seconds between status checks
|
||||
timeout: Maximum seconds to wait (None for no timeout)
|
||||
|
||||
Returns:
|
||||
BatchScrapeStatusResponse when job completes
|
||||
|
||||
Raises:
|
||||
FirecrawlError: If the job fails or timeout is reached
|
||||
TimeoutError: If timeout is reached
|
||||
"""
|
||||
start_time = time.monotonic()
|
||||
|
||||
while True:
|
||||
status_job = get_batch_scrape_status(client, job_id)
|
||||
|
||||
# Check if job is complete
|
||||
if status_job.status in ["completed", "failed", "cancelled"]:
|
||||
return status_job
|
||||
|
||||
# Check timeout
|
||||
if timeout and (time.monotonic() - start_time) > timeout:
|
||||
raise TimeoutError(f"Batch scrape job {job_id} did not complete within {timeout} seconds")
|
||||
|
||||
# Wait before next poll
|
||||
time.sleep(poll_interval)
|
||||
|
||||
|
||||
def batch_scrape(
|
||||
client: HttpClient,
|
||||
urls: List[str],
|
||||
*,
|
||||
options: Optional[ScrapeOptions] = None,
|
||||
webhook: Optional[Union[str, WebhookConfig]] = None,
|
||||
append_to_id: Optional[str] = None,
|
||||
ignore_invalid_urls: Optional[bool] = None,
|
||||
max_concurrency: Optional[int] = None,
|
||||
zero_data_retention: Optional[bool] = None,
|
||||
integration: Optional[str] = None,
|
||||
idempotency_key: Optional[str] = None,
|
||||
poll_interval: int = 2,
|
||||
timeout: Optional[int] = None
|
||||
) -> BatchScrapeJob:
|
||||
"""
|
||||
Start a batch scrape job and wait for it to complete.
|
||||
|
||||
Args:
|
||||
client: HTTP client instance
|
||||
urls: List of URLs to scrape
|
||||
options: Scraping options
|
||||
poll_interval: Seconds between status checks
|
||||
timeout: Maximum seconds to wait (None for no timeout)
|
||||
|
||||
Returns:
|
||||
BatchScrapeStatusResponse when job completes
|
||||
|
||||
Raises:
|
||||
FirecrawlError: If the batch scrape fails to start or complete
|
||||
TimeoutError: If timeout is reached
|
||||
"""
|
||||
# Start the batch scrape
|
||||
start = start_batch_scrape(
|
||||
client,
|
||||
urls,
|
||||
options=options,
|
||||
webhook=webhook,
|
||||
append_to_id=append_to_id,
|
||||
ignore_invalid_urls=ignore_invalid_urls,
|
||||
max_concurrency=max_concurrency,
|
||||
zero_data_retention=zero_data_retention,
|
||||
integration=integration,
|
||||
idempotency_key=idempotency_key,
|
||||
)
|
||||
|
||||
job_id = start.id
|
||||
|
||||
# Wait for completion
|
||||
return wait_for_batch_completion(
|
||||
client, job_id, poll_interval, timeout
|
||||
)
|
||||
|
||||
|
||||
def validate_batch_urls(urls: List[str]) -> List[str]:
|
||||
"""
|
||||
Validate and normalize a list of URLs for batch scraping.
|
||||
|
||||
Args:
|
||||
urls: List of URLs to validate
|
||||
|
||||
Returns:
|
||||
Validated list of URLs
|
||||
|
||||
Raises:
|
||||
ValueError: If URLs are invalid
|
||||
"""
|
||||
if not urls:
|
||||
raise ValueError("URLs list cannot be empty")
|
||||
|
||||
validated_urls = []
|
||||
for url in urls:
|
||||
if not url or not isinstance(url, str):
|
||||
raise ValueError(f"Invalid URL: {url}")
|
||||
|
||||
# Basic URL validation
|
||||
if not (url.startswith("http://") or url.startswith("https://")):
|
||||
raise ValueError(f"URL must start with http:// or https://: {url}")
|
||||
|
||||
validated_urls.append(url.strip())
|
||||
|
||||
return validated_urls
|
||||
|
||||
|
||||
def prepare_batch_scrape_request(
|
||||
urls: List[str],
|
||||
*,
|
||||
options: Optional[ScrapeOptions] = None,
|
||||
webhook: Optional[Union[str, WebhookConfig]] = None,
|
||||
append_to_id: Optional[str] = None,
|
||||
ignore_invalid_urls: Optional[bool] = None,
|
||||
max_concurrency: Optional[int] = None,
|
||||
zero_data_retention: Optional[bool] = None,
|
||||
integration: Optional[str] = None,
|
||||
) -> dict:
|
||||
"""
|
||||
Prepare a batch scrape request payload.
|
||||
|
||||
Args:
|
||||
urls: List of URLs to scrape
|
||||
options: Scraping options
|
||||
|
||||
Returns:
|
||||
Request payload dictionary
|
||||
"""
|
||||
validated_urls = validate_batch_urls(urls)
|
||||
request_data: Dict[str, Any] = {"urls": validated_urls}
|
||||
|
||||
# Flatten scrape options at the top level (v2 behavior)
|
||||
if options:
|
||||
scrape_data = prepare_scrape_options(options)
|
||||
if scrape_data:
|
||||
request_data.update(scrape_data)
|
||||
|
||||
# Batch-specific fields
|
||||
if webhook is not None:
|
||||
if isinstance(webhook, str):
|
||||
request_data["webhook"] = webhook
|
||||
else:
|
||||
request_data["webhook"] = webhook.model_dump(exclude_none=True)
|
||||
if append_to_id is not None:
|
||||
request_data["appendToId"] = append_to_id
|
||||
if ignore_invalid_urls is not None:
|
||||
request_data["ignoreInvalidURLs"] = ignore_invalid_urls
|
||||
if max_concurrency is not None:
|
||||
request_data["maxConcurrency"] = max_concurrency
|
||||
if zero_data_retention is not None:
|
||||
request_data["zeroDataRetention"] = zero_data_retention
|
||||
if integration is not None:
|
||||
request_data["integration"] = str(integration).strip()
|
||||
|
||||
return request_data
|
||||
|
||||
|
||||
def chunk_urls(urls: List[str], chunk_size: int = 100) -> List[List[str]]:
|
||||
"""
|
||||
Split a large list of URLs into smaller chunks for batch processing.
|
||||
|
||||
Args:
|
||||
urls: List of URLs to chunk
|
||||
chunk_size: Maximum size of each chunk
|
||||
|
||||
Returns:
|
||||
List of URL chunks
|
||||
"""
|
||||
chunks = []
|
||||
for i in range(0, len(urls), chunk_size):
|
||||
chunks.append(urls[i:i + chunk_size])
|
||||
return chunks
|
||||
|
||||
|
||||
def process_large_batch(
|
||||
client: HttpClient,
|
||||
urls: List[str],
|
||||
options: Optional[ScrapeOptions] = None,
|
||||
chunk_size: int = 100,
|
||||
poll_interval: int = 2,
|
||||
timeout: Optional[int] = None
|
||||
) -> List[Document]:
|
||||
"""
|
||||
Process a large batch of URLs by splitting into smaller chunks.
|
||||
|
||||
Args:
|
||||
client: HTTP client instance
|
||||
urls: List of URLs to scrape
|
||||
options: Scraping options
|
||||
chunk_size: Size of each batch chunk
|
||||
poll_interval: Seconds between status checks
|
||||
timeout: Maximum seconds to wait per chunk
|
||||
|
||||
Returns:
|
||||
List of all scraped documents
|
||||
|
||||
Raises:
|
||||
FirecrawlError: If any chunk fails
|
||||
"""
|
||||
url_chunks = chunk_urls(urls, chunk_size)
|
||||
all_documents = []
|
||||
completed_chunks = 0
|
||||
|
||||
for chunk in url_chunks:
|
||||
# Process this chunk
|
||||
result = batch_scrape(
|
||||
client,
|
||||
chunk,
|
||||
options=options,
|
||||
poll_interval=poll_interval,
|
||||
timeout=timeout,
|
||||
)
|
||||
|
||||
# Add documents from this chunk
|
||||
if result.data:
|
||||
all_documents.extend(result.data)
|
||||
|
||||
completed_chunks += 1
|
||||
|
||||
return all_documents
|
||||
|
||||
|
||||
def get_batch_scrape_errors(client: HttpClient, job_id: str) -> CrawlErrorsResponse:
|
||||
"""
|
||||
Get errors for a batch scrape job.
|
||||
|
||||
Args:
|
||||
client: HTTP client instance
|
||||
job_id: ID of the batch scrape job
|
||||
|
||||
Returns:
|
||||
CrawlErrorsResponse with errors and robots-blocked URLs
|
||||
"""
|
||||
response = client.get(f"/v2/batch/scrape/{job_id}/errors")
|
||||
|
||||
if not response.ok:
|
||||
handle_response_error(response, "get batch scrape errors")
|
||||
|
||||
body = response.json()
|
||||
payload = body.get("data", body)
|
||||
normalized = {
|
||||
"errors": payload.get("errors", []),
|
||||
"robots_blocked": payload.get("robotsBlocked", payload.get("robots_blocked", [])),
|
||||
}
|
||||
return CrawlErrorsResponse(**normalized)
|
||||
@@ -0,0 +1,188 @@
|
||||
"""
|
||||
Browser session methods for Firecrawl v2 API.
|
||||
|
||||
Provides create, execute, delete, and list operations for browser sessions.
|
||||
"""
|
||||
|
||||
from typing import Any, Dict, List, Literal, Optional
|
||||
|
||||
from ..types import (
|
||||
BrowserCreateResponse,
|
||||
BrowserExecuteResponse,
|
||||
BrowserDeleteResponse,
|
||||
BrowserListResponse,
|
||||
)
|
||||
from ..utils.http_client import HttpClient
|
||||
from ..utils.error_handler import handle_response_error
|
||||
|
||||
|
||||
def _normalize_browser_create_response(payload: Dict[str, Any]) -> Dict[str, Any]:
|
||||
out = dict(payload)
|
||||
if "cdpUrl" in out and "cdp_url" not in out:
|
||||
out["cdp_url"] = out["cdpUrl"]
|
||||
if "liveViewUrl" in out and "live_view_url" not in out:
|
||||
out["live_view_url"] = out["liveViewUrl"]
|
||||
if "interactiveLiveViewUrl" in out and "interactive_live_view_url" not in out:
|
||||
out["interactive_live_view_url"] = out["interactiveLiveViewUrl"]
|
||||
if "expiresAt" in out and "expires_at" not in out:
|
||||
out["expires_at"] = out["expiresAt"]
|
||||
return out
|
||||
|
||||
|
||||
def _normalize_browser_list_response(payload: Dict[str, Any]) -> Dict[str, Any]:
|
||||
out = dict(payload)
|
||||
if "sessions" in out and isinstance(out["sessions"], list):
|
||||
normalized_sessions = []
|
||||
for s in out["sessions"]:
|
||||
ns = dict(s)
|
||||
if "cdpUrl" in ns and "cdp_url" not in ns:
|
||||
ns["cdp_url"] = ns["cdpUrl"]
|
||||
if "liveViewUrl" in ns and "live_view_url" not in ns:
|
||||
ns["live_view_url"] = ns["liveViewUrl"]
|
||||
if "interactiveLiveViewUrl" in ns and "interactive_live_view_url" not in ns:
|
||||
ns["interactive_live_view_url"] = ns["interactiveLiveViewUrl"]
|
||||
if "streamWebView" in ns and "stream_web_view" not in ns:
|
||||
ns["stream_web_view"] = ns["streamWebView"]
|
||||
if "createdAt" in ns and "created_at" not in ns:
|
||||
ns["created_at"] = ns["createdAt"]
|
||||
if "lastActivity" in ns and "last_activity" not in ns:
|
||||
ns["last_activity"] = ns["lastActivity"]
|
||||
normalized_sessions.append(ns)
|
||||
out["sessions"] = normalized_sessions
|
||||
return out
|
||||
|
||||
|
||||
def browser(
|
||||
client: HttpClient,
|
||||
*,
|
||||
ttl: Optional[int] = None,
|
||||
activity_ttl: Optional[int] = None,
|
||||
stream_web_view: Optional[bool] = None,
|
||||
profile: Optional[Dict[str, Any]] = None,
|
||||
) -> BrowserCreateResponse:
|
||||
"""Create a new browser session.
|
||||
|
||||
Args:
|
||||
client: HTTP client instance
|
||||
ttl: Total time-to-live in seconds (30-3600, default 300)
|
||||
activity_ttl: Inactivity TTL in seconds (10-3600)
|
||||
stream_web_view: Whether to enable webview streaming
|
||||
profile: Profile config with ``name`` (str) and
|
||||
optional ``save_changes`` (bool, default ``True``)
|
||||
|
||||
Returns:
|
||||
BrowserCreateResponse with session id and CDP URL
|
||||
"""
|
||||
body: Dict[str, Any] = {}
|
||||
if ttl is not None:
|
||||
body["ttl"] = ttl
|
||||
if activity_ttl is not None:
|
||||
body["activityTtl"] = activity_ttl
|
||||
if stream_web_view is not None:
|
||||
body["streamWebView"] = stream_web_view
|
||||
if profile is not None:
|
||||
body["profile"] = {
|
||||
"name": profile["name"],
|
||||
"saveChanges": profile.get("save_changes", True),
|
||||
}
|
||||
|
||||
resp = client.post("/v2/browser", body)
|
||||
if not resp.ok:
|
||||
handle_response_error(resp, "create browser session")
|
||||
payload = _normalize_browser_create_response(resp.json())
|
||||
return BrowserCreateResponse(**payload)
|
||||
|
||||
|
||||
def _normalize_browser_execute_response(payload: Dict[str, Any]) -> Dict[str, Any]:
|
||||
out = dict(payload)
|
||||
if "exitCode" in out and "exit_code" not in out:
|
||||
out["exit_code"] = out["exitCode"]
|
||||
return out
|
||||
|
||||
|
||||
def browser_execute(
|
||||
client: HttpClient,
|
||||
session_id: str,
|
||||
code: str,
|
||||
*,
|
||||
language: Literal["python", "node", "bash"] = "bash",
|
||||
timeout: Optional[int] = None,
|
||||
) -> BrowserExecuteResponse:
|
||||
"""Execute code in a browser session.
|
||||
|
||||
Args:
|
||||
client: HTTP client instance
|
||||
session_id: Browser session ID
|
||||
code: Code to execute
|
||||
language: Programming language ("python", "node", or "bash")
|
||||
timeout: Execution timeout in seconds (1-300, default 30)
|
||||
|
||||
Returns:
|
||||
BrowserExecuteResponse with execution result
|
||||
"""
|
||||
body: Dict[str, Any] = {
|
||||
"code": code,
|
||||
"language": language,
|
||||
}
|
||||
if timeout is not None:
|
||||
body["timeout"] = timeout
|
||||
|
||||
resp = client.post(f"/v2/browser/{session_id}/execute", body)
|
||||
if not resp.ok:
|
||||
handle_response_error(resp, "execute browser code")
|
||||
payload = _normalize_browser_execute_response(resp.json())
|
||||
return BrowserExecuteResponse(**payload)
|
||||
|
||||
|
||||
def _normalize_browser_delete_response(payload: Dict[str, Any]) -> Dict[str, Any]:
|
||||
out = dict(payload)
|
||||
if "sessionDurationMs" in out and "session_duration_ms" not in out:
|
||||
out["session_duration_ms"] = out["sessionDurationMs"]
|
||||
if "creditsBilled" in out and "credits_billed" not in out:
|
||||
out["credits_billed"] = out["creditsBilled"]
|
||||
return out
|
||||
|
||||
|
||||
def delete_browser(
|
||||
client: HttpClient,
|
||||
session_id: str,
|
||||
) -> BrowserDeleteResponse:
|
||||
"""Delete a browser session.
|
||||
|
||||
Args:
|
||||
client: HTTP client instance
|
||||
session_id: Browser session ID
|
||||
|
||||
Returns:
|
||||
BrowserDeleteResponse
|
||||
"""
|
||||
resp = client.delete(f"/v2/browser/{session_id}")
|
||||
if not resp.ok:
|
||||
handle_response_error(resp, "delete browser session")
|
||||
payload = _normalize_browser_delete_response(resp.json())
|
||||
return BrowserDeleteResponse(**payload)
|
||||
|
||||
|
||||
def list_browsers(
|
||||
client: HttpClient,
|
||||
*,
|
||||
status: Optional[Literal["active", "destroyed"]] = None,
|
||||
) -> BrowserListResponse:
|
||||
"""List browser sessions.
|
||||
|
||||
Args:
|
||||
client: HTTP client instance
|
||||
status: Filter by session status ("active" or "destroyed")
|
||||
|
||||
Returns:
|
||||
BrowserListResponse with list of sessions
|
||||
"""
|
||||
endpoint = "/v2/browser"
|
||||
if status is not None:
|
||||
endpoint = f"{endpoint}?status={status}"
|
||||
|
||||
resp = client.get(endpoint)
|
||||
if not resp.ok:
|
||||
handle_response_error(resp, "list browser sessions")
|
||||
payload = _normalize_browser_list_response(resp.json())
|
||||
return BrowserListResponse(**payload)
|
||||
647
참고/firecrawl-main/apps/python-sdk/firecrawl/v2/methods/crawl.py
Normal file
647
참고/firecrawl-main/apps/python-sdk/firecrawl/v2/methods/crawl.py
Normal file
@@ -0,0 +1,647 @@
|
||||
"""
|
||||
Crawling functionality for Firecrawl v2 API.
|
||||
"""
|
||||
|
||||
import time
|
||||
from typing import Optional, Dict, Any, List
|
||||
from ..types import (
|
||||
CrawlRequest,
|
||||
CrawlJob,
|
||||
CrawlResponse, Document, CrawlParamsRequest, CrawlParamsResponse, CrawlParamsData,
|
||||
WebhookConfig, CrawlErrorsResponse, ActiveCrawlsResponse, ActiveCrawl, PaginationConfig
|
||||
)
|
||||
from ..utils import HttpClient, handle_response_error, validate_scrape_options, prepare_scrape_options
|
||||
from ..utils.normalize import normalize_document_input
|
||||
|
||||
|
||||
def _validate_crawl_request(request: CrawlRequest) -> None:
|
||||
"""
|
||||
Validate crawl request parameters.
|
||||
|
||||
Args:
|
||||
request: CrawlRequest to validate
|
||||
|
||||
Raises:
|
||||
ValueError: If request is invalid
|
||||
"""
|
||||
if not request.url or not request.url.strip():
|
||||
raise ValueError("URL cannot be empty")
|
||||
|
||||
if request.limit is not None and request.limit <= 0:
|
||||
raise ValueError("Limit must be positive")
|
||||
|
||||
# Validate scrape_options (if provided)
|
||||
if request.scrape_options is not None:
|
||||
validate_scrape_options(request.scrape_options)
|
||||
|
||||
|
||||
def _prepare_crawl_request(request: CrawlRequest) -> dict:
|
||||
"""
|
||||
Prepare crawl request for API submission.
|
||||
|
||||
Args:
|
||||
request: CrawlRequest to prepare
|
||||
|
||||
Returns:
|
||||
Dictionary ready for API submission
|
||||
"""
|
||||
# Validate request
|
||||
_validate_crawl_request(request)
|
||||
|
||||
# Start with basic data
|
||||
data = {"url": request.url}
|
||||
|
||||
# Add prompt if present
|
||||
if request.prompt:
|
||||
data["prompt"] = request.prompt
|
||||
|
||||
# Handle scrape_options conversion first (before model_dump)
|
||||
if request.scrape_options is not None:
|
||||
scrape_data = prepare_scrape_options(request.scrape_options)
|
||||
if scrape_data:
|
||||
data["scrapeOptions"] = scrape_data
|
||||
|
||||
# Convert request to dict
|
||||
request_data = request.model_dump(exclude_none=True, exclude_unset=True)
|
||||
|
||||
# Remove url, prompt, and scrape_options (already handled)
|
||||
request_data.pop("url", None)
|
||||
request_data.pop("prompt", None)
|
||||
request_data.pop("scrape_options", None)
|
||||
|
||||
# Handle webhook conversion first (before model_dump)
|
||||
if request.webhook is not None:
|
||||
if isinstance(request.webhook, str):
|
||||
data["webhook"] = request.webhook
|
||||
else:
|
||||
# Convert WebhookConfig to dict
|
||||
data["webhook"] = request.webhook.model_dump(exclude_none=True)
|
||||
|
||||
# Convert other snake_case fields to camelCase
|
||||
field_mappings = {
|
||||
"include_paths": "includePaths",
|
||||
"exclude_paths": "excludePaths",
|
||||
"max_discovery_depth": "maxDiscoveryDepth",
|
||||
"sitemap": "sitemap",
|
||||
"ignore_query_parameters": "ignoreQueryParameters",
|
||||
"deduplicate_similar_urls": "deduplicateSimilarURLs",
|
||||
"crawl_entire_domain": "crawlEntireDomain",
|
||||
"allow_external_links": "allowExternalLinks",
|
||||
"allow_subdomains": "allowSubdomains",
|
||||
"ignore_robots_txt": "ignoreRobotsTxt",
|
||||
"robots_user_agent": "robotsUserAgent",
|
||||
"delay": "delay",
|
||||
"max_concurrency": "maxConcurrency",
|
||||
"regex_on_full_url": "regexOnFullURL",
|
||||
"zero_data_retention": "zeroDataRetention"
|
||||
}
|
||||
|
||||
# Apply field mappings
|
||||
for snake_case, camel_case in field_mappings.items():
|
||||
if snake_case in request_data:
|
||||
data[camel_case] = request_data.pop(snake_case)
|
||||
|
||||
# Add any remaining fields that don't need conversion (like limit)
|
||||
data.update(request_data)
|
||||
# Trim integration if present
|
||||
if "integration" in data and isinstance(data["integration"], str):
|
||||
data["integration"] = data["integration"].strip()
|
||||
|
||||
return data
|
||||
|
||||
|
||||
def _parse_crawl_documents(data_list: Optional[List[Any]]) -> List[Document]:
|
||||
documents: List[Document] = []
|
||||
for doc_data in data_list or []:
|
||||
if isinstance(doc_data, dict):
|
||||
documents.append(Document(**normalize_document_input(doc_data)))
|
||||
return documents
|
||||
|
||||
|
||||
def _parse_crawl_status_response(response_data: Dict[str, Any]) -> Dict[str, Any]:
|
||||
if not response_data.get("success"):
|
||||
raise Exception(response_data.get("error", "Unknown error occurred"))
|
||||
|
||||
return {
|
||||
"status": response_data.get("status"),
|
||||
"completed": response_data.get("completed", 0),
|
||||
"total": response_data.get("total", 0),
|
||||
"credits_used": response_data.get("creditsUsed", 0),
|
||||
"expires_at": response_data.get("expiresAt"),
|
||||
"next": response_data.get("next"),
|
||||
"data": _parse_crawl_documents(response_data.get("data", [])),
|
||||
}
|
||||
|
||||
|
||||
def start_crawl(client: HttpClient, request: CrawlRequest) -> CrawlResponse:
|
||||
"""
|
||||
Start a crawl job for a website.
|
||||
|
||||
Args:
|
||||
client: HTTP client instance
|
||||
request: CrawlRequest containing URL and options
|
||||
|
||||
Returns:
|
||||
CrawlResponse with job information
|
||||
|
||||
Raises:
|
||||
ValueError: If request is invalid
|
||||
Exception: If the crawl operation fails to start
|
||||
"""
|
||||
request_data = _prepare_crawl_request(request)
|
||||
|
||||
response = client.post("/v2/crawl", request_data)
|
||||
|
||||
if not response.ok:
|
||||
handle_response_error(response, "start crawl")
|
||||
|
||||
response_data = response.json()
|
||||
|
||||
if response_data.get("success"):
|
||||
job_data = {
|
||||
"id": response_data.get("id"),
|
||||
"url": response_data.get("url")
|
||||
}
|
||||
|
||||
return CrawlResponse(**job_data)
|
||||
else:
|
||||
raise Exception(response_data.get("error", "Unknown error occurred"))
|
||||
|
||||
|
||||
def get_crawl_status(
|
||||
client: HttpClient,
|
||||
job_id: str,
|
||||
pagination_config: Optional[PaginationConfig] = None,
|
||||
*,
|
||||
request_timeout: Optional[float] = None,
|
||||
) -> CrawlJob:
|
||||
"""
|
||||
Get the status of a crawl job.
|
||||
|
||||
Args:
|
||||
client: HTTP client instance
|
||||
job_id: ID of the crawl job
|
||||
pagination_config: Optional configuration for pagination behavior
|
||||
request_timeout: Timeout (in seconds) for each individual HTTP request. When auto-pagination
|
||||
is enabled (default) and there are multiple pages of results, this timeout applies to
|
||||
each page request separately, not to the entire operation
|
||||
|
||||
Returns:
|
||||
CrawlJob with current status and data
|
||||
|
||||
Raises:
|
||||
Exception: If the status check fails
|
||||
"""
|
||||
# Make the API request
|
||||
response = client.get(f"/v2/crawl/{job_id}", timeout=request_timeout)
|
||||
|
||||
# Handle errors
|
||||
if not response.ok:
|
||||
handle_response_error(response, "get crawl status")
|
||||
|
||||
# Parse response
|
||||
response_data = response.json()
|
||||
|
||||
payload = _parse_crawl_status_response(response_data)
|
||||
|
||||
documents = payload["data"]
|
||||
|
||||
# Handle pagination if requested
|
||||
auto_paginate = pagination_config.auto_paginate if pagination_config else True
|
||||
if auto_paginate and payload["next"] and not (
|
||||
pagination_config
|
||||
and pagination_config.max_results is not None
|
||||
and len(documents) >= pagination_config.max_results
|
||||
):
|
||||
documents = _fetch_all_pages(
|
||||
client,
|
||||
payload["next"],
|
||||
documents,
|
||||
pagination_config,
|
||||
request_timeout=request_timeout,
|
||||
)
|
||||
|
||||
# Create CrawlJob with current status and data
|
||||
return CrawlJob(
|
||||
status=payload["status"],
|
||||
completed=payload["completed"],
|
||||
total=payload["total"],
|
||||
credits_used=payload["credits_used"],
|
||||
expires_at=payload["expires_at"],
|
||||
next=payload["next"] if not auto_paginate else None,
|
||||
data=documents,
|
||||
)
|
||||
|
||||
|
||||
def get_crawl_status_page(
|
||||
client: HttpClient,
|
||||
next_url: str,
|
||||
*,
|
||||
request_timeout: Optional[float] = None,
|
||||
) -> CrawlJob:
|
||||
"""
|
||||
Fetch a single page of crawl results using the provided next URL.
|
||||
|
||||
Args:
|
||||
client: HTTP client instance
|
||||
next_url: Opaque next URL from a prior crawl status response
|
||||
request_timeout: Timeout (in seconds) for the HTTP request
|
||||
|
||||
Returns:
|
||||
CrawlJob with the page data and next URL (if any)
|
||||
|
||||
Raises:
|
||||
Exception: If the request fails or returns an error response
|
||||
"""
|
||||
response = client.get(next_url, timeout=request_timeout)
|
||||
|
||||
if not response.ok:
|
||||
handle_response_error(response, "get crawl status page")
|
||||
|
||||
response_data = response.json()
|
||||
payload = _parse_crawl_status_response(response_data)
|
||||
|
||||
return CrawlJob(
|
||||
status=payload["status"],
|
||||
completed=payload["completed"],
|
||||
total=payload["total"],
|
||||
credits_used=payload["credits_used"],
|
||||
expires_at=payload["expires_at"],
|
||||
next=payload["next"],
|
||||
data=payload["data"],
|
||||
)
|
||||
|
||||
|
||||
def _fetch_all_pages(
|
||||
client: HttpClient,
|
||||
next_url: str,
|
||||
initial_documents: List[Document],
|
||||
pagination_config: Optional[PaginationConfig] = None,
|
||||
*,
|
||||
request_timeout: Optional[float] = None,
|
||||
) -> List[Document]:
|
||||
"""
|
||||
Fetch all pages of crawl results.
|
||||
|
||||
Args:
|
||||
client: HTTP client instance
|
||||
next_url: URL for the next page
|
||||
initial_documents: Documents from the first page
|
||||
pagination_config: Optional configuration for pagination limits
|
||||
request_timeout: Optional timeout (in seconds) for the underlying HTTP request
|
||||
|
||||
Returns:
|
||||
List of all documents from all pages
|
||||
"""
|
||||
documents = initial_documents.copy()
|
||||
current_url = next_url
|
||||
page_count = 0
|
||||
|
||||
# Apply pagination limits
|
||||
max_pages = pagination_config.max_pages if pagination_config else None
|
||||
max_results = pagination_config.max_results if pagination_config else None
|
||||
max_wait_time = pagination_config.max_wait_time if pagination_config else None
|
||||
|
||||
start_time = time.monotonic()
|
||||
|
||||
while current_url:
|
||||
# Check pagination limits (treat 0 as a valid limit)
|
||||
if (max_pages is not None) and page_count >= max_pages:
|
||||
break
|
||||
|
||||
if (max_wait_time is not None) and (time.monotonic() - start_time) > max_wait_time:
|
||||
break
|
||||
|
||||
# Fetch next page
|
||||
response = client.get(current_url, timeout=request_timeout)
|
||||
|
||||
if not response.ok:
|
||||
# Log error but continue with what we have
|
||||
import logging
|
||||
logger = logging.getLogger("firecrawl")
|
||||
logger.warning("Failed to fetch next page", extra={"status_code": response.status_code})
|
||||
break
|
||||
|
||||
page_data = response.json()
|
||||
|
||||
try:
|
||||
page_payload = _parse_crawl_status_response(page_data)
|
||||
except Exception:
|
||||
break
|
||||
|
||||
# Add documents from this page
|
||||
for document in page_payload["data"]:
|
||||
# Check max_results limit BEFORE adding each document
|
||||
if max_results is not None and len(documents) >= max_results:
|
||||
break
|
||||
documents.append(document)
|
||||
|
||||
# Check if we hit max_results limit
|
||||
if max_results is not None and len(documents) >= max_results:
|
||||
break
|
||||
|
||||
# Get next URL
|
||||
current_url = page_payload["next"]
|
||||
page_count += 1
|
||||
|
||||
return documents
|
||||
|
||||
|
||||
def cancel_crawl(client: HttpClient, job_id: str) -> bool:
|
||||
"""
|
||||
Cancel a running crawl job.
|
||||
|
||||
Args:
|
||||
client: HTTP client instance
|
||||
job_id: ID of the crawl job to cancel
|
||||
|
||||
Returns:
|
||||
bool: True if the crawl was cancelled, False otherwise
|
||||
|
||||
Raises:
|
||||
Exception: If the cancellation fails
|
||||
"""
|
||||
response = client.delete(f"/v2/crawl/{job_id}")
|
||||
|
||||
if not response.ok:
|
||||
handle_response_error(response, "cancel crawl")
|
||||
|
||||
response_data = response.json()
|
||||
|
||||
return response_data.get("status") == "cancelled"
|
||||
|
||||
def wait_for_crawl_completion(
|
||||
client: HttpClient,
|
||||
job_id: str,
|
||||
poll_interval: int = 2,
|
||||
timeout: Optional[int] = None,
|
||||
*,
|
||||
request_timeout: Optional[float] = None,
|
||||
) -> CrawlJob:
|
||||
"""
|
||||
Wait for a crawl job to complete, polling for status updates.
|
||||
|
||||
Args:
|
||||
client: HTTP client instance
|
||||
job_id: ID of the crawl job
|
||||
poll_interval: Seconds between status checks
|
||||
timeout: Maximum seconds to wait (None for no timeout)
|
||||
request_timeout: Optional timeout (in seconds) for each status request
|
||||
|
||||
Returns:
|
||||
CrawlJob when job completes
|
||||
|
||||
Raises:
|
||||
Exception: If the job fails
|
||||
TimeoutError: If timeout is reached
|
||||
"""
|
||||
start_time = time.monotonic()
|
||||
|
||||
while True:
|
||||
crawl_job = get_crawl_status(
|
||||
client,
|
||||
job_id,
|
||||
request_timeout=request_timeout,
|
||||
)
|
||||
|
||||
# Check if job is complete
|
||||
if crawl_job.status in ["completed", "failed", "cancelled"]:
|
||||
return crawl_job
|
||||
|
||||
# Check timeout
|
||||
if timeout is not None and (time.monotonic() - start_time) > timeout:
|
||||
raise TimeoutError(f"Crawl job {job_id} did not complete within {timeout} seconds")
|
||||
|
||||
# Wait before next poll
|
||||
time.sleep(poll_interval)
|
||||
|
||||
|
||||
def crawl(
|
||||
client: HttpClient,
|
||||
request: CrawlRequest,
|
||||
poll_interval: int = 2,
|
||||
timeout: Optional[int] = None,
|
||||
*,
|
||||
request_timeout: Optional[float] = None,
|
||||
) -> CrawlJob:
|
||||
"""
|
||||
Start a crawl job and wait for it to complete.
|
||||
|
||||
Args:
|
||||
client: HTTP client instance
|
||||
request: CrawlRequest containing URL and options
|
||||
poll_interval: Seconds between status checks
|
||||
timeout: Maximum seconds to wait for the entire crawl job to complete (None for no timeout)
|
||||
request_timeout: Timeout (in seconds) for each individual HTTP request, including pagination
|
||||
requests when fetching results. If there are multiple pages, each page request gets this timeout
|
||||
|
||||
Returns:
|
||||
CrawlJob when job completes
|
||||
|
||||
Raises:
|
||||
ValueError: If request is invalid
|
||||
Exception: If the crawl fails to start or complete
|
||||
TimeoutError: If timeout is reached
|
||||
"""
|
||||
# Start the crawl
|
||||
crawl_job = start_crawl(client, request)
|
||||
job_id = crawl_job.id
|
||||
|
||||
# Determine the per-request timeout. If not provided, reuse the overall timeout value.
|
||||
effective_request_timeout = request_timeout if request_timeout is not None else timeout
|
||||
|
||||
# Wait for completion
|
||||
return wait_for_crawl_completion(
|
||||
client,
|
||||
job_id,
|
||||
poll_interval,
|
||||
timeout,
|
||||
request_timeout=effective_request_timeout,
|
||||
)
|
||||
|
||||
|
||||
def crawl_params_preview(client: HttpClient, request: CrawlParamsRequest) -> CrawlParamsData:
|
||||
"""
|
||||
Get crawl parameters from LLM based on URL and prompt.
|
||||
|
||||
Args:
|
||||
client: HTTP client instance
|
||||
request: CrawlParamsRequest containing URL and prompt
|
||||
|
||||
Returns:
|
||||
CrawlParamsData containing suggested crawl options
|
||||
|
||||
Raises:
|
||||
ValueError: If request is invalid
|
||||
Exception: If the operation fails
|
||||
"""
|
||||
# Validate request
|
||||
if not request.url or not request.url.strip():
|
||||
raise ValueError("URL cannot be empty")
|
||||
|
||||
if not request.prompt or not request.prompt.strip():
|
||||
raise ValueError("Prompt cannot be empty")
|
||||
|
||||
# Prepare request data
|
||||
request_data = {
|
||||
"url": request.url,
|
||||
"prompt": request.prompt
|
||||
}
|
||||
|
||||
# Make the API request
|
||||
response = client.post("/v2/crawl/params-preview", request_data)
|
||||
|
||||
# Handle errors
|
||||
if not response.ok:
|
||||
handle_response_error(response, "crawl params preview")
|
||||
|
||||
# Parse response
|
||||
response_data = response.json()
|
||||
|
||||
if response_data.get("success"):
|
||||
params_data = response_data.get("data", {})
|
||||
|
||||
# Convert camelCase to snake_case for CrawlParamsData
|
||||
converted_params = {}
|
||||
field_mappings = {
|
||||
"includePaths": "include_paths",
|
||||
"excludePaths": "exclude_paths",
|
||||
"maxDiscoveryDepth": "max_discovery_depth",
|
||||
"sitemap": "sitemap",
|
||||
"ignoreQueryParameters": "ignore_query_parameters",
|
||||
"deduplicateSimilarURLs": "deduplicate_similar_urls",
|
||||
"crawlEntireDomain": "crawl_entire_domain",
|
||||
"allowExternalLinks": "allow_external_links",
|
||||
"allowSubdomains": "allow_subdomains",
|
||||
"ignoreRobotsTxt": "ignore_robots_txt",
|
||||
"robotsUserAgent": "robots_user_agent",
|
||||
"maxConcurrency": "max_concurrency",
|
||||
"scrapeOptions": "scrape_options",
|
||||
"zeroDataRetention": "zero_data_retention"
|
||||
}
|
||||
|
||||
# Handle webhook conversion
|
||||
if "webhook" in params_data:
|
||||
webhook_data = params_data["webhook"]
|
||||
if isinstance(webhook_data, dict):
|
||||
converted_params["webhook"] = WebhookConfig(**webhook_data)
|
||||
else:
|
||||
converted_params["webhook"] = webhook_data
|
||||
|
||||
for camel_case, snake_case in field_mappings.items():
|
||||
if camel_case in params_data:
|
||||
if camel_case == "scrapeOptions" and params_data[camel_case] is not None:
|
||||
# Handle nested scrapeOptions conversion
|
||||
scrape_opts_data = params_data[camel_case]
|
||||
converted_scrape_opts = {}
|
||||
scrape_field_mappings = {
|
||||
"includeTags": "include_tags",
|
||||
"excludeTags": "exclude_tags",
|
||||
"onlyMainContent": "only_main_content",
|
||||
"waitFor": "wait_for",
|
||||
"skipTlsVerification": "skip_tls_verification",
|
||||
"removeBase64Images": "remove_base64_images"
|
||||
}
|
||||
|
||||
for scrape_camel, scrape_snake in scrape_field_mappings.items():
|
||||
if scrape_camel in scrape_opts_data:
|
||||
converted_scrape_opts[scrape_snake] = scrape_opts_data[scrape_camel]
|
||||
|
||||
# Handle formats field - if it's a list, convert to ScrapeFormats
|
||||
if "formats" in scrape_opts_data:
|
||||
formats_data = scrape_opts_data["formats"]
|
||||
if isinstance(formats_data, list):
|
||||
# Convert list to ScrapeFormats object
|
||||
from ..types import ScrapeFormats
|
||||
converted_scrape_opts["formats"] = ScrapeFormats(formats=formats_data)
|
||||
else:
|
||||
converted_scrape_opts["formats"] = formats_data
|
||||
|
||||
# Add fields that don't need conversion
|
||||
for key, value in scrape_opts_data.items():
|
||||
if key not in scrape_field_mappings and key != "formats":
|
||||
converted_scrape_opts[key] = value
|
||||
|
||||
converted_params[snake_case] = converted_scrape_opts
|
||||
else:
|
||||
converted_params[snake_case] = params_data[camel_case]
|
||||
|
||||
# Add fields that don't need conversion
|
||||
for key, value in params_data.items():
|
||||
if key not in field_mappings:
|
||||
converted_params[key] = value
|
||||
|
||||
# Add warning if present
|
||||
if "warning" in response_data:
|
||||
converted_params["warning"] = response_data["warning"]
|
||||
|
||||
return CrawlParamsData(**converted_params)
|
||||
else:
|
||||
raise Exception(response_data.get("error", "Unknown error occurred"))
|
||||
|
||||
|
||||
def get_crawl_errors(http_client: HttpClient, crawl_id: str) -> CrawlErrorsResponse:
|
||||
"""
|
||||
Get errors from a crawl job.
|
||||
|
||||
Args:
|
||||
http_client: HTTP client for making requests
|
||||
crawl_id: The ID of the crawl job
|
||||
|
||||
Returns:
|
||||
CrawlErrorsResponse containing errors and robots blocked URLs
|
||||
|
||||
Raises:
|
||||
Exception: If the request fails
|
||||
"""
|
||||
response = http_client.get(f"/v2/crawl/{crawl_id}/errors")
|
||||
|
||||
if not response.ok:
|
||||
handle_response_error(response, "check crawl errors")
|
||||
|
||||
try:
|
||||
body = response.json()
|
||||
payload = body.get("data", body)
|
||||
# Manual key normalization since we avoid Pydantic aliases
|
||||
normalized = {
|
||||
"errors": payload.get("errors", []),
|
||||
"robots_blocked": payload.get("robotsBlocked", payload.get("robots_blocked", [])),
|
||||
}
|
||||
return CrawlErrorsResponse(**normalized)
|
||||
except Exception as e:
|
||||
raise Exception(f"Failed to parse crawl errors response: {e}")
|
||||
|
||||
|
||||
def get_active_crawls(client: HttpClient) -> ActiveCrawlsResponse:
|
||||
"""
|
||||
Get a list of currently active crawl jobs.
|
||||
|
||||
Args:
|
||||
client: HTTP client instance
|
||||
|
||||
Returns:
|
||||
ActiveCrawlsResponse containing a list of active crawl jobs
|
||||
|
||||
Raises:
|
||||
Exception: If the request fails
|
||||
"""
|
||||
response = client.get("/v2/crawl/active")
|
||||
|
||||
if not response.ok:
|
||||
handle_response_error(response, "get active crawls")
|
||||
|
||||
body = response.json()
|
||||
if not body.get("success"):
|
||||
raise Exception(body.get("error", "Unknown error occurred"))
|
||||
|
||||
crawls_in = body.get("crawls", [])
|
||||
normalized_crawls = []
|
||||
for c in crawls_in:
|
||||
if isinstance(c, dict):
|
||||
normalized_crawls.append({
|
||||
"id": c.get("id"),
|
||||
"team_id": c.get("teamId", c.get("team_id")),
|
||||
"url": c.get("url"),
|
||||
"options": c.get("options"),
|
||||
})
|
||||
return ActiveCrawlsResponse(success=True, crawls=[ActiveCrawl(**nc) for nc in normalized_crawls])
|
||||
@@ -0,0 +1,192 @@
|
||||
from typing import Any, Dict, List, Optional
|
||||
import time
|
||||
import warnings
|
||||
|
||||
from ..types import ExtractResponse, ScrapeOptions
|
||||
from ..types import AgentOptions
|
||||
from ..utils.http_client import HttpClient
|
||||
from ..utils.validation import prepare_scrape_options
|
||||
from ..utils.error_handler import handle_response_error
|
||||
|
||||
_EXTRACT_DEPRECATION_MSG = (
|
||||
"The extract endpoint is in maintenance mode and its use is discouraged. "
|
||||
"Review https://docs.firecrawl.dev/developer-guides/usage-guides/choosing-the-data-extractor "
|
||||
"to find a replacement."
|
||||
)
|
||||
|
||||
|
||||
def _prepare_extract_request(
|
||||
urls: Optional[List[str]],
|
||||
*,
|
||||
prompt: Optional[str] = None,
|
||||
schema: Optional[Dict[str, Any]] = None,
|
||||
system_prompt: Optional[str] = None,
|
||||
allow_external_links: Optional[bool] = None,
|
||||
enable_web_search: Optional[bool] = None,
|
||||
show_sources: Optional[bool] = None,
|
||||
scrape_options: Optional[ScrapeOptions] = None,
|
||||
ignore_invalid_urls: Optional[bool] = None,
|
||||
integration: Optional[str] = None,
|
||||
agent: Optional[AgentOptions] = None,
|
||||
) -> Dict[str, Any]:
|
||||
body: Dict[str, Any] = {}
|
||||
if urls is not None:
|
||||
body["urls"] = urls
|
||||
if prompt is not None:
|
||||
body["prompt"] = prompt
|
||||
if schema is not None:
|
||||
body["schema"] = schema
|
||||
if system_prompt is not None:
|
||||
body["systemPrompt"] = system_prompt
|
||||
if allow_external_links is not None:
|
||||
body["allowExternalLinks"] = allow_external_links
|
||||
if enable_web_search is not None:
|
||||
body["enableWebSearch"] = enable_web_search
|
||||
if show_sources is not None:
|
||||
body["showSources"] = show_sources
|
||||
if ignore_invalid_urls is not None:
|
||||
body["ignoreInvalidURLs"] = ignore_invalid_urls
|
||||
if scrape_options is not None:
|
||||
prepared = prepare_scrape_options(scrape_options)
|
||||
if prepared:
|
||||
body["scrapeOptions"] = prepared
|
||||
if integration is not None and str(integration).strip():
|
||||
body["integration"] = str(integration).strip()
|
||||
if agent is not None:
|
||||
try:
|
||||
body["agent"] = agent.model_dump(exclude_none=True) # type: ignore[attr-defined]
|
||||
except AttributeError:
|
||||
body["agent"] = agent # fallback
|
||||
return body
|
||||
|
||||
|
||||
def _normalize_extract_response_payload(payload: Dict[str, Any]) -> Dict[str, Any]:
|
||||
out = dict(payload)
|
||||
if "expiresAt" in out and "expires_at" not in out:
|
||||
out["expires_at"] = out["expiresAt"]
|
||||
if "creditsUsed" in out and "credits_used" not in out:
|
||||
out["credits_used"] = out["creditsUsed"]
|
||||
if "tokensUsed" in out and "tokens_used" not in out:
|
||||
out["tokens_used"] = out["tokensUsed"]
|
||||
return out
|
||||
|
||||
|
||||
def start_extract(
|
||||
client: HttpClient,
|
||||
urls: Optional[List[str]],
|
||||
*,
|
||||
prompt: Optional[str] = None,
|
||||
schema: Optional[Dict[str, Any]] = None,
|
||||
system_prompt: Optional[str] = None,
|
||||
allow_external_links: Optional[bool] = None,
|
||||
enable_web_search: Optional[bool] = None,
|
||||
show_sources: Optional[bool] = None,
|
||||
scrape_options: Optional[ScrapeOptions] = None,
|
||||
ignore_invalid_urls: Optional[bool] = None,
|
||||
integration: Optional[str] = None,
|
||||
agent: Optional[AgentOptions] = None,
|
||||
) -> ExtractResponse:
|
||||
"""Start an extract job (non-blocking).
|
||||
|
||||
.. deprecated::
|
||||
The extract endpoint is in maintenance mode and its use is discouraged.
|
||||
Review https://docs.firecrawl.dev/developer-guides/usage-guides/choosing-the-data-extractor
|
||||
to find a replacement.
|
||||
"""
|
||||
warnings.warn(_EXTRACT_DEPRECATION_MSG, DeprecationWarning, stacklevel=2)
|
||||
body = _prepare_extract_request(
|
||||
urls,
|
||||
prompt=prompt,
|
||||
schema=schema,
|
||||
system_prompt=system_prompt,
|
||||
allow_external_links=allow_external_links,
|
||||
enable_web_search=enable_web_search,
|
||||
show_sources=show_sources,
|
||||
scrape_options=scrape_options,
|
||||
ignore_invalid_urls=ignore_invalid_urls,
|
||||
integration=integration,
|
||||
agent=agent,
|
||||
)
|
||||
resp = client.post("/v2/extract", body)
|
||||
if not resp.ok:
|
||||
handle_response_error(resp, "extract")
|
||||
payload = _normalize_extract_response_payload(resp.json())
|
||||
return ExtractResponse(**payload)
|
||||
|
||||
|
||||
def get_extract_status(client: HttpClient, job_id: str) -> ExtractResponse:
|
||||
"""Get the current status of an extract job.
|
||||
|
||||
.. deprecated::
|
||||
The extract endpoint is in maintenance mode and its use is discouraged.
|
||||
Review https://docs.firecrawl.dev/developer-guides/usage-guides/choosing-the-data-extractor
|
||||
to find a replacement.
|
||||
"""
|
||||
warnings.warn(_EXTRACT_DEPRECATION_MSG, DeprecationWarning, stacklevel=2)
|
||||
resp = client.get(f"/v2/extract/{job_id}")
|
||||
if not resp.ok:
|
||||
handle_response_error(resp, "extract-status")
|
||||
payload = _normalize_extract_response_payload(resp.json())
|
||||
return ExtractResponse(**payload)
|
||||
|
||||
|
||||
def wait_extract(
|
||||
client: HttpClient,
|
||||
job_id: str,
|
||||
*,
|
||||
poll_interval: int = 2,
|
||||
timeout: Optional[int] = None,
|
||||
) -> ExtractResponse:
|
||||
start_ts = time.time()
|
||||
while True:
|
||||
status = get_extract_status(client, job_id)
|
||||
if status.status in ("completed", "failed", "cancelled"):
|
||||
return status
|
||||
if timeout is not None and (time.time() - start_ts) > timeout:
|
||||
return status
|
||||
time.sleep(max(1, poll_interval))
|
||||
|
||||
|
||||
def extract(
|
||||
client: HttpClient,
|
||||
urls: Optional[List[str]],
|
||||
*,
|
||||
prompt: Optional[str] = None,
|
||||
schema: Optional[Dict[str, Any]] = None,
|
||||
system_prompt: Optional[str] = None,
|
||||
allow_external_links: Optional[bool] = None,
|
||||
enable_web_search: Optional[bool] = None,
|
||||
show_sources: Optional[bool] = None,
|
||||
scrape_options: Optional[ScrapeOptions] = None,
|
||||
ignore_invalid_urls: Optional[bool] = None,
|
||||
poll_interval: int = 2,
|
||||
timeout: Optional[int] = None,
|
||||
integration: Optional[str] = None,
|
||||
agent: Optional[AgentOptions] = None,
|
||||
) -> ExtractResponse:
|
||||
"""Extract structured data and wait until completion.
|
||||
|
||||
.. deprecated::
|
||||
The extract endpoint is in maintenance mode and its use is discouraged.
|
||||
Review https://docs.firecrawl.dev/developer-guides/usage-guides/choosing-the-data-extractor
|
||||
to find a replacement.
|
||||
"""
|
||||
warnings.warn(_EXTRACT_DEPRECATION_MSG, DeprecationWarning, stacklevel=2)
|
||||
started = start_extract(
|
||||
client,
|
||||
urls,
|
||||
prompt=prompt,
|
||||
schema=schema,
|
||||
system_prompt=system_prompt,
|
||||
allow_external_links=allow_external_links,
|
||||
enable_web_search=enable_web_search,
|
||||
show_sources=show_sources,
|
||||
scrape_options=scrape_options,
|
||||
ignore_invalid_urls=ignore_invalid_urls,
|
||||
integration=integration,
|
||||
agent=agent,
|
||||
)
|
||||
job_id = getattr(started, "id", None)
|
||||
if not job_id:
|
||||
return started
|
||||
return wait_extract(client, job_id, poll_interval=poll_interval, timeout=timeout)
|
||||
@@ -0,0 +1,85 @@
|
||||
"""
|
||||
Mapping functionality for Firecrawl v2 API.
|
||||
"""
|
||||
|
||||
from typing import Optional, Dict, Any
|
||||
from ..types import MapOptions, MapData, LinkResult
|
||||
from ..utils import HttpClient, handle_response_error
|
||||
|
||||
|
||||
def _prepare_map_request(url: str, options: Optional[MapOptions] = None) -> Dict[str, Any]:
|
||||
if not url or not url.strip():
|
||||
raise ValueError("URL cannot be empty")
|
||||
|
||||
payload: Dict[str, Any] = {"url": url.strip()}
|
||||
|
||||
if options is not None:
|
||||
# Unified sitemap parameter already provided in options
|
||||
data: Dict[str, Any] = {}
|
||||
if getattr(options, "sitemap", None) is not None:
|
||||
data["sitemap"] = options.sitemap
|
||||
|
||||
if options.search is not None:
|
||||
data["search"] = options.search
|
||||
if options.include_subdomains is not None:
|
||||
data["includeSubdomains"] = options.include_subdomains
|
||||
if options.ignore_query_parameters is not None:
|
||||
data["ignoreQueryParameters"] = options.ignore_query_parameters
|
||||
if options.limit is not None:
|
||||
if options.limit <= 0:
|
||||
raise ValueError("Limit must be positive")
|
||||
data["limit"] = options.limit
|
||||
if options.timeout is not None:
|
||||
data["timeout"] = options.timeout
|
||||
if options.integration is not None and options.integration.strip():
|
||||
data["integration"] = options.integration.strip()
|
||||
if options.location is not None:
|
||||
data["location"] = options.location.model_dump(exclude_none=True)
|
||||
payload.update(data)
|
||||
|
||||
return payload
|
||||
|
||||
|
||||
def map(client: HttpClient, url: str, options: Optional[MapOptions] = None) -> MapData:
|
||||
"""
|
||||
Map a URL and return MapData (links list with optional titles/descriptions).
|
||||
"""
|
||||
request_data = _prepare_map_request(url, options)
|
||||
response = client.post("/v2/map", request_data)
|
||||
if not response.ok:
|
||||
handle_response_error(response, "map")
|
||||
|
||||
body = response.json()
|
||||
if not body.get("success"):
|
||||
raise Exception(body.get("error", "Unknown error occurred"))
|
||||
|
||||
# shouldnt return inside data?
|
||||
# data = body.get("data", {})
|
||||
# result_links: list[LinkResult] = []
|
||||
# for item in data.get("links", []):
|
||||
# if isinstance(item, dict):
|
||||
# result_links.append(
|
||||
# LinkResult(
|
||||
# url=item.get("url", ""),
|
||||
# title=item.get("title"),
|
||||
# description=item.get("description"),
|
||||
# )
|
||||
# )
|
||||
# elif isinstance(item, str):
|
||||
# result_links.append(LinkResult(url=item))
|
||||
|
||||
result_links: list[LinkResult] = []
|
||||
for item in body.get("links", []):
|
||||
if isinstance(item, dict):
|
||||
result_links.append(
|
||||
LinkResult(
|
||||
url=item.get("url", ""),
|
||||
title=item.get("title"),
|
||||
description=item.get("description"),
|
||||
)
|
||||
)
|
||||
elif isinstance(item, str):
|
||||
result_links.append(LinkResult(url=item))
|
||||
|
||||
return MapData(links=result_links)
|
||||
|
||||
@@ -0,0 +1,214 @@
|
||||
import time
|
||||
from typing import Any, Dict, List, Optional
|
||||
from pydantic import BaseModel
|
||||
|
||||
from ..types import (
|
||||
Monitor,
|
||||
MonitorCheck,
|
||||
MonitorCheckDetail,
|
||||
MonitorCheckPage,
|
||||
MonitorCreateRequest,
|
||||
PaginationConfig,
|
||||
MonitorTarget,
|
||||
MonitorUpdateRequest,
|
||||
ScrapeOptions,
|
||||
)
|
||||
from ..utils import HttpClient, handle_response_error
|
||||
from ..utils.validation import prepare_scrape_options
|
||||
|
||||
|
||||
def _dump(value: Any) -> Any:
|
||||
if isinstance(value, ScrapeOptions):
|
||||
return prepare_scrape_options(value)
|
||||
if isinstance(value, MonitorTarget):
|
||||
data = value.model_dump(exclude_none=True, by_alias=True)
|
||||
if isinstance(value.scrape_options, ScrapeOptions):
|
||||
data["scrapeOptions"] = prepare_scrape_options(value.scrape_options)
|
||||
return _prepare_target(data)
|
||||
if isinstance(value, BaseModel):
|
||||
return value.model_dump(exclude_none=True, by_alias=True)
|
||||
if isinstance(value, list):
|
||||
return [_dump(item) for item in value]
|
||||
if isinstance(value, dict):
|
||||
return {key: _dump(item) for key, item in value.items() if item is not None}
|
||||
return value
|
||||
|
||||
|
||||
def _prepare_target(target: Dict[str, Any]) -> Dict[str, Any]:
|
||||
prepared = dict(target)
|
||||
if "scrapeOptions" in prepared and isinstance(prepared["scrapeOptions"], ScrapeOptions):
|
||||
prepared["scrapeOptions"] = prepare_scrape_options(prepared["scrapeOptions"])
|
||||
if "crawlOptions" in prepared:
|
||||
prepared["crawlOptions"] = _dump(prepared["crawlOptions"])
|
||||
return prepared
|
||||
|
||||
|
||||
def _prepare_payload(request: Any) -> Dict[str, Any]:
|
||||
payload = _dump(request)
|
||||
if not isinstance(payload, dict):
|
||||
raise ValueError("Monitor request must be an object")
|
||||
if "targets" in payload:
|
||||
payload["targets"] = [
|
||||
_prepare_target(_dump(target))
|
||||
for target in payload.get("targets", [])
|
||||
]
|
||||
return payload
|
||||
|
||||
|
||||
def _data_or_error(response, action: str) -> Any:
|
||||
if not response.ok:
|
||||
handle_response_error(response, action)
|
||||
body = response.json()
|
||||
if not body.get("success"):
|
||||
raise Exception(body.get("error", "Unknown error occurred"))
|
||||
return body.get("data")
|
||||
|
||||
|
||||
def _monitor_check_data_or_error(response, action: str) -> Dict[str, Any]:
|
||||
if not response.ok:
|
||||
handle_response_error(response, action)
|
||||
body = response.json()
|
||||
if not body.get("success"):
|
||||
raise Exception(body.get("error", "Unknown error occurred"))
|
||||
data = body.get("data") or {}
|
||||
if body.get("next") is not None:
|
||||
data["next"] = body.get("next")
|
||||
return data
|
||||
|
||||
|
||||
def _fetch_all_monitor_check_pages(
|
||||
client: HttpClient,
|
||||
next_url: str,
|
||||
initial_pages: List[MonitorCheckPage],
|
||||
pagination_config: Optional[PaginationConfig] = None,
|
||||
) -> List[MonitorCheckPage]:
|
||||
pages = initial_pages.copy()
|
||||
current_url = next_url
|
||||
page_count = 0
|
||||
max_pages = pagination_config.max_pages if pagination_config else None
|
||||
max_results = pagination_config.max_results if pagination_config else None
|
||||
max_wait_time = pagination_config.max_wait_time if pagination_config else None
|
||||
start_time = time.monotonic()
|
||||
|
||||
while current_url:
|
||||
if max_pages is not None and page_count >= max_pages:
|
||||
break
|
||||
if max_wait_time is not None and (time.monotonic() - start_time) > max_wait_time:
|
||||
break
|
||||
|
||||
response = client.get(current_url)
|
||||
if not response.ok:
|
||||
break
|
||||
try:
|
||||
data = _monitor_check_data_or_error(response, "get monitor check page")
|
||||
except Exception:
|
||||
break
|
||||
|
||||
for page in data.get("pages") or []:
|
||||
if max_results is not None and len(pages) >= max_results:
|
||||
break
|
||||
pages.append(MonitorCheckPage(**page))
|
||||
|
||||
if max_results is not None and len(pages) >= max_results:
|
||||
break
|
||||
|
||||
current_url = data.get("next")
|
||||
page_count += 1
|
||||
|
||||
return pages
|
||||
|
||||
|
||||
def create_monitor(client: HttpClient, request: MonitorCreateRequest) -> Monitor:
|
||||
data = _data_or_error(client.post("/v2/monitor", _prepare_payload(request)), "create monitor")
|
||||
return Monitor(**data)
|
||||
|
||||
|
||||
def list_monitors(client: HttpClient, *, limit: Optional[int] = None, offset: Optional[int] = None) -> List[Monitor]:
|
||||
params = []
|
||||
if limit is not None:
|
||||
params.append(f"limit={limit}")
|
||||
if offset is not None:
|
||||
params.append(f"offset={offset}")
|
||||
suffix = f"?{'&'.join(params)}" if params else ""
|
||||
data = _data_or_error(client.get(f"/v2/monitor{suffix}"), "list monitors")
|
||||
return [Monitor(**item) for item in data or []]
|
||||
|
||||
|
||||
def get_monitor(client: HttpClient, monitor_id: str) -> Monitor:
|
||||
data = _data_or_error(client.get(f"/v2/monitor/{monitor_id}"), "get monitor")
|
||||
return Monitor(**data)
|
||||
|
||||
|
||||
def update_monitor(client: HttpClient, monitor_id: str, request: MonitorUpdateRequest) -> Monitor:
|
||||
data = _data_or_error(client.patch(f"/v2/monitor/{monitor_id}", _prepare_payload(request)), "update monitor")
|
||||
return Monitor(**data)
|
||||
|
||||
|
||||
def delete_monitor(client: HttpClient, monitor_id: str) -> bool:
|
||||
response = client.delete(f"/v2/monitor/{monitor_id}")
|
||||
if not response.ok:
|
||||
handle_response_error(response, "delete monitor")
|
||||
body = response.json()
|
||||
if not body.get("success"):
|
||||
raise Exception(body.get("error", "Unknown error occurred"))
|
||||
return True
|
||||
|
||||
|
||||
def run_monitor(client: HttpClient, monitor_id: str) -> MonitorCheck:
|
||||
data = _data_or_error(client.post(f"/v2/monitor/{monitor_id}/run", {}), "run monitor")
|
||||
return MonitorCheck(**data)
|
||||
|
||||
|
||||
def list_monitor_checks(
|
||||
client: HttpClient,
|
||||
monitor_id: str,
|
||||
*,
|
||||
limit: Optional[int] = None,
|
||||
offset: Optional[int] = None,
|
||||
) -> List[MonitorCheck]:
|
||||
params = []
|
||||
if limit is not None:
|
||||
params.append(f"limit={limit}")
|
||||
if offset is not None:
|
||||
params.append(f"offset={offset}")
|
||||
suffix = f"?{'&'.join(params)}" if params else ""
|
||||
data = _data_or_error(client.get(f"/v2/monitor/{monitor_id}/checks{suffix}"), "list monitor checks")
|
||||
return [MonitorCheck(**item) for item in data or []]
|
||||
|
||||
|
||||
def get_monitor_check(
|
||||
client: HttpClient,
|
||||
monitor_id: str,
|
||||
check_id: str,
|
||||
*,
|
||||
limit: Optional[int] = None,
|
||||
skip: Optional[int] = None,
|
||||
status: Optional[str] = None,
|
||||
pagination_config: Optional[PaginationConfig] = None,
|
||||
) -> MonitorCheckDetail:
|
||||
params = []
|
||||
if limit is not None:
|
||||
params.append(f"limit={limit}")
|
||||
if skip is not None:
|
||||
params.append(f"skip={skip}")
|
||||
if status is not None:
|
||||
params.append(f"status={status}")
|
||||
suffix = f"?{'&'.join(params)}" if params else ""
|
||||
data = _monitor_check_data_or_error(client.get(f"/v2/monitor/{monitor_id}/checks/{check_id}{suffix}"), "get monitor check")
|
||||
detail = MonitorCheckDetail(**data)
|
||||
|
||||
auto_paginate = pagination_config.auto_paginate if pagination_config else True
|
||||
if auto_paginate and detail.next and not (
|
||||
pagination_config
|
||||
and pagination_config.max_results is not None
|
||||
and len(detail.pages) >= pagination_config.max_results
|
||||
):
|
||||
detail.pages = _fetch_all_monitor_check_pages(
|
||||
client,
|
||||
detail.next,
|
||||
detail.pages,
|
||||
pagination_config,
|
||||
)
|
||||
detail.next = None
|
||||
|
||||
return detail
|
||||
162
참고/firecrawl-main/apps/python-sdk/firecrawl/v2/methods/parse.py
Normal file
162
참고/firecrawl-main/apps/python-sdk/firecrawl/v2/methods/parse.py
Normal file
@@ -0,0 +1,162 @@
|
||||
"""
|
||||
Parse (multipart upload) functionality for Firecrawl v2 API.
|
||||
"""
|
||||
|
||||
import json
|
||||
import mimetypes
|
||||
from pathlib import Path
|
||||
from typing import Optional, Dict, Any, BinaryIO, Union, Tuple
|
||||
|
||||
from ..types import Document, ParseOptions
|
||||
from ..utils.normalize import normalize_document_input
|
||||
from ..utils import HttpClient, handle_response_error, prepare_scrape_options, validate_scrape_options
|
||||
from ..utils.get_version import get_version
|
||||
|
||||
version = get_version()
|
||||
|
||||
ParseFileInput = Union[str, bytes, bytearray, Path, BinaryIO]
|
||||
UNSUPPORTED_PARSE_FORMATS = {"changeTracking", "screenshot", "branding"}
|
||||
|
||||
|
||||
def _extract_format_type(format_item: Any) -> Optional[str]:
|
||||
if isinstance(format_item, str):
|
||||
return format_item
|
||||
if isinstance(format_item, dict):
|
||||
fmt_type = format_item.get("type")
|
||||
return fmt_type if isinstance(fmt_type, str) else None
|
||||
return None
|
||||
|
||||
|
||||
def _validate_parse_options_payload(options_payload: Dict[str, Any]) -> None:
|
||||
actions = options_payload.get("actions")
|
||||
if isinstance(actions, list) and len(actions) > 0:
|
||||
raise ValueError("Parse uploads do not support actions.")
|
||||
|
||||
wait_for = options_payload.get("waitFor")
|
||||
if isinstance(wait_for, (int, float)) and wait_for > 0:
|
||||
raise ValueError("Parse uploads do not support waitFor.")
|
||||
|
||||
if options_payload.get("location") is not None:
|
||||
raise ValueError("Parse uploads do not support location overrides.")
|
||||
|
||||
if options_payload.get("mobile"):
|
||||
raise ValueError("Parse uploads do not support mobile rendering.")
|
||||
|
||||
proxy = options_payload.get("proxy")
|
||||
if proxy not in (None, "auto", "basic"):
|
||||
raise ValueError("Parse uploads only support proxy values of auto or basic.")
|
||||
|
||||
for fmt in options_payload.get("formats") or []:
|
||||
fmt_type = _extract_format_type(fmt)
|
||||
if fmt_type in UNSUPPORTED_PARSE_FORMATS:
|
||||
if fmt_type == "changeTracking":
|
||||
raise ValueError("Parse uploads do not support change tracking.")
|
||||
if fmt_type == "screenshot":
|
||||
raise ValueError("Parse uploads do not support screenshot output.")
|
||||
raise ValueError("Parse uploads do not support branding output.")
|
||||
|
||||
|
||||
def _prepare_parse_options_payload(
|
||||
options: Optional[ParseOptions],
|
||||
) -> Dict[str, Any]:
|
||||
request_data: Dict[str, Any] = {}
|
||||
|
||||
if options is not None:
|
||||
validated = validate_scrape_options(options)
|
||||
if validated is not None:
|
||||
opts = prepare_scrape_options(validated) or {}
|
||||
_validate_parse_options_payload(opts)
|
||||
# Parse is always uncached server-side; avoid sending cache/index hints.
|
||||
opts.pop("maxAge", None)
|
||||
opts.pop("minAge", None)
|
||||
opts.pop("storeInCache", None)
|
||||
opts.pop("lockdown", None)
|
||||
request_data.update(opts)
|
||||
|
||||
request_data["origin"] = request_data.get("origin") or f"python-sdk@{version}"
|
||||
return request_data
|
||||
|
||||
|
||||
def _prepare_file_payload(
|
||||
file: ParseFileInput,
|
||||
filename: Optional[str] = None,
|
||||
content_type: Optional[str] = None,
|
||||
) -> Dict[str, Tuple[str, bytes, str]]:
|
||||
if isinstance(file, (str, Path)):
|
||||
file_path = Path(file)
|
||||
if not file_path.exists() or not file_path.is_file():
|
||||
raise ValueError(f"File path does not exist: {file_path}")
|
||||
file_bytes = file_path.read_bytes()
|
||||
resolved_filename = filename or file_path.name
|
||||
elif isinstance(file, (bytes, bytearray)):
|
||||
file_bytes = bytes(file)
|
||||
resolved_filename = filename or "upload"
|
||||
elif hasattr(file, "read"):
|
||||
raw_bytes = file.read()
|
||||
if isinstance(raw_bytes, str):
|
||||
file_bytes = raw_bytes.encode("utf-8")
|
||||
else:
|
||||
file_bytes = bytes(raw_bytes)
|
||||
guessed_name = getattr(file, "name", None)
|
||||
resolved_filename = filename or (Path(guessed_name).name if guessed_name else "upload")
|
||||
else:
|
||||
raise ValueError("Unsupported file input type. Use a file path, bytes, bytearray, or binary file object.")
|
||||
|
||||
if not resolved_filename or not resolved_filename.strip():
|
||||
raise ValueError("filename cannot be empty")
|
||||
|
||||
resolved_filename = resolved_filename.strip()
|
||||
resolved_content_type = (
|
||||
content_type
|
||||
or mimetypes.guess_type(resolved_filename)[0]
|
||||
or "application/octet-stream"
|
||||
)
|
||||
|
||||
return {
|
||||
"file": (resolved_filename, file_bytes, resolved_content_type),
|
||||
}
|
||||
|
||||
|
||||
def _prepare_parse_request(
|
||||
file: ParseFileInput,
|
||||
options: Optional[ParseOptions] = None,
|
||||
*,
|
||||
filename: Optional[str] = None,
|
||||
content_type: Optional[str] = None,
|
||||
) -> Tuple[Dict[str, Any], Dict[str, Tuple[str, bytes, str]]]:
|
||||
request_data = _prepare_parse_options_payload(options)
|
||||
multipart_fields = {"options": json.dumps(request_data)}
|
||||
multipart_files = _prepare_file_payload(
|
||||
file,
|
||||
filename=filename,
|
||||
content_type=content_type,
|
||||
)
|
||||
return multipart_fields, multipart_files
|
||||
|
||||
|
||||
def parse(
|
||||
client: HttpClient,
|
||||
file: ParseFileInput,
|
||||
options: Optional[ParseOptions] = None,
|
||||
*,
|
||||
filename: Optional[str] = None,
|
||||
content_type: Optional[str] = None,
|
||||
) -> Document:
|
||||
fields, files = _prepare_parse_request(
|
||||
file,
|
||||
options,
|
||||
filename=filename,
|
||||
content_type=content_type,
|
||||
)
|
||||
|
||||
response = client.post_multipart("/v2/parse", data=fields, files=files)
|
||||
if not response.ok:
|
||||
handle_response_error(response, "parse")
|
||||
|
||||
body = response.json()
|
||||
if not body.get("success"):
|
||||
raise Exception(body.get("error", "Unknown error occurred"))
|
||||
|
||||
document_data = body.get("data", {})
|
||||
normalized = normalize_document_input(document_data)
|
||||
return Document(**normalized)
|
||||
204
참고/firecrawl-main/apps/python-sdk/firecrawl/v2/methods/scrape.py
Normal file
204
참고/firecrawl-main/apps/python-sdk/firecrawl/v2/methods/scrape.py
Normal file
@@ -0,0 +1,204 @@
|
||||
"""
|
||||
Scraping functionality for Firecrawl v2 API.
|
||||
"""
|
||||
|
||||
from typing import Optional, Dict, Any, Literal
|
||||
from ..types import (
|
||||
ScrapeOptions,
|
||||
Document,
|
||||
BrowserExecuteResponse,
|
||||
BrowserDeleteResponse,
|
||||
)
|
||||
from ..utils.normalize import normalize_document_input
|
||||
from ..utils import HttpClient, handle_response_error, prepare_scrape_options, validate_scrape_options
|
||||
|
||||
|
||||
def _prepare_scrape_request(url: str, options: Optional[ScrapeOptions] = None) -> Dict[str, Any]:
|
||||
"""
|
||||
Prepare a scrape request payload for v2 API.
|
||||
|
||||
Args:
|
||||
url: URL to scrape
|
||||
options: ScrapeOptions (snake_case) to convert and include
|
||||
|
||||
Returns:
|
||||
Request payload dictionary with camelCase fields
|
||||
"""
|
||||
if not url or not url.strip():
|
||||
raise ValueError("URL cannot be empty")
|
||||
|
||||
request_data: Dict[str, Any] = {"url": url.strip()}
|
||||
|
||||
if options is not None:
|
||||
validated = validate_scrape_options(options)
|
||||
if validated is not None:
|
||||
opts = prepare_scrape_options(validated)
|
||||
if opts:
|
||||
request_data.update(opts)
|
||||
|
||||
return request_data
|
||||
|
||||
def scrape(client: HttpClient, url: str, options: Optional[ScrapeOptions] = None) -> Document:
|
||||
"""
|
||||
Scrape a single URL and return the document.
|
||||
|
||||
The v2 API returns: { success: boolean, data: Document }
|
||||
We surface just the Document to callers.
|
||||
|
||||
Args:
|
||||
client: HTTP client instance
|
||||
url: URL to scrape
|
||||
options: Scraping options (snake_case)
|
||||
|
||||
Returns:
|
||||
Document
|
||||
"""
|
||||
payload = _prepare_scrape_request(url, options)
|
||||
|
||||
response = client.post("/v2/scrape", payload)
|
||||
|
||||
if not response.ok:
|
||||
handle_response_error(response, "scrape")
|
||||
|
||||
body = response.json()
|
||||
if not body.get("success"):
|
||||
raise Exception(body.get("error", "Unknown error occurred"))
|
||||
|
||||
document_data = body.get("data", {})
|
||||
normalized = normalize_document_input(document_data)
|
||||
return Document(**normalized)
|
||||
|
||||
|
||||
def interact(
|
||||
client: HttpClient,
|
||||
job_id: str,
|
||||
code: Optional[str] = None,
|
||||
*,
|
||||
prompt: Optional[str] = None,
|
||||
language: Literal["python", "node", "bash"] = "node",
|
||||
timeout: Optional[int] = None,
|
||||
origin: Optional[str] = None,
|
||||
) -> BrowserExecuteResponse:
|
||||
"""
|
||||
Interact with the scrape-bound browser session for a scrape job.
|
||||
|
||||
Either ``code`` or ``prompt`` must be provided. When ``prompt`` is given
|
||||
the server runs an AI agent that translates the natural-language instruction
|
||||
into browser actions.
|
||||
|
||||
Args:
|
||||
client: HTTP client instance
|
||||
job_id: Scrape job ID
|
||||
code: Code to execute (optional if prompt is provided)
|
||||
prompt: Natural-language instruction for the browser agent (optional if code is provided)
|
||||
language: Programming language ("python", "node", or "bash")
|
||||
timeout: Execution timeout in seconds (1-300)
|
||||
origin: Optional request origin tag
|
||||
|
||||
Returns:
|
||||
BrowserExecuteResponse with execution output
|
||||
"""
|
||||
if not job_id or not job_id.strip():
|
||||
raise ValueError("Job ID cannot be empty")
|
||||
has_code = code and code.strip()
|
||||
has_prompt = prompt and prompt.strip()
|
||||
if not has_code and not has_prompt:
|
||||
raise ValueError("Either 'code' or 'prompt' must be provided")
|
||||
|
||||
body: Dict[str, Any] = {
|
||||
"language": language,
|
||||
}
|
||||
if has_code:
|
||||
body["code"] = code
|
||||
if has_prompt:
|
||||
body["prompt"] = prompt
|
||||
if timeout is not None:
|
||||
body["timeout"] = timeout
|
||||
if origin is not None:
|
||||
body["origin"] = origin
|
||||
|
||||
response = client.post(f"/v2/scrape/{job_id}/interact", body)
|
||||
if not response.ok:
|
||||
handle_response_error(response, "interact with scrape browser")
|
||||
|
||||
payload = response.json()
|
||||
if not payload.get("success"):
|
||||
raise Exception(payload.get("error", "Unknown error occurred"))
|
||||
|
||||
normalized = dict(payload)
|
||||
if "exitCode" in normalized and "exit_code" not in normalized:
|
||||
normalized["exit_code"] = normalized["exitCode"]
|
||||
if "liveViewUrl" in normalized and "live_view_url" not in normalized:
|
||||
normalized["live_view_url"] = normalized["liveViewUrl"]
|
||||
if "interactiveLiveViewUrl" in normalized and "interactive_live_view_url" not in normalized:
|
||||
normalized["interactive_live_view_url"] = normalized["interactiveLiveViewUrl"]
|
||||
return BrowserExecuteResponse(**normalized)
|
||||
|
||||
|
||||
def stop_interaction(
|
||||
client: HttpClient,
|
||||
job_id: str,
|
||||
) -> BrowserDeleteResponse:
|
||||
"""
|
||||
Stop the interaction session for a scrape job.
|
||||
|
||||
Args:
|
||||
client: HTTP client instance
|
||||
job_id: Scrape job ID
|
||||
|
||||
Returns:
|
||||
BrowserDeleteResponse
|
||||
"""
|
||||
if not job_id or not job_id.strip():
|
||||
raise ValueError("Job ID cannot be empty")
|
||||
|
||||
response = client.delete(f"/v2/scrape/{job_id}/interact")
|
||||
if not response.ok:
|
||||
handle_response_error(response, "stop interaction")
|
||||
|
||||
payload = response.json()
|
||||
normalized = dict(payload)
|
||||
if "sessionDurationMs" in normalized and "session_duration_ms" not in normalized:
|
||||
normalized["session_duration_ms"] = normalized["sessionDurationMs"]
|
||||
if "creditsBilled" in normalized and "credits_billed" not in normalized:
|
||||
normalized["credits_billed"] = normalized["creditsBilled"]
|
||||
|
||||
return BrowserDeleteResponse(**normalized)
|
||||
|
||||
|
||||
def stop_interactive_browser(
|
||||
client: HttpClient,
|
||||
job_id: str,
|
||||
) -> BrowserDeleteResponse:
|
||||
"""Deprecated alias for stop_interaction()."""
|
||||
return stop_interaction(client, job_id)
|
||||
|
||||
|
||||
def scrape_execute(
|
||||
client: HttpClient,
|
||||
job_id: str,
|
||||
code: Optional[str] = None,
|
||||
*,
|
||||
prompt: Optional[str] = None,
|
||||
language: Literal["python", "node", "bash"] = "node",
|
||||
timeout: Optional[int] = None,
|
||||
origin: Optional[str] = None,
|
||||
) -> BrowserExecuteResponse:
|
||||
"""Deprecated alias for interact()."""
|
||||
return interact(
|
||||
client,
|
||||
job_id,
|
||||
code,
|
||||
prompt=prompt,
|
||||
language=language,
|
||||
timeout=timeout,
|
||||
origin=origin,
|
||||
)
|
||||
|
||||
|
||||
def delete_scrape_browser(
|
||||
client: HttpClient,
|
||||
job_id: str,
|
||||
) -> BrowserDeleteResponse:
|
||||
"""Deprecated alias for stop_interaction()."""
|
||||
return stop_interaction(client, job_id)
|
||||
218
참고/firecrawl-main/apps/python-sdk/firecrawl/v2/methods/search.py
Normal file
218
참고/firecrawl-main/apps/python-sdk/firecrawl/v2/methods/search.py
Normal file
@@ -0,0 +1,218 @@
|
||||
"""
|
||||
Search functionality for Firecrawl v2 API.
|
||||
"""
|
||||
|
||||
from typing import Dict, Any, Union, List, TypeVar, Type
|
||||
from ..types import SearchRequest, SearchData, Document, SearchResultWeb, SearchResultNews, SearchResultImages
|
||||
from ..utils.normalize import normalize_document_input, _map_search_result_keys
|
||||
from ..utils import HttpClient, handle_response_error, validate_scrape_options, prepare_scrape_options
|
||||
|
||||
T = TypeVar("T")
|
||||
|
||||
def search(
|
||||
client: HttpClient,
|
||||
request: SearchRequest
|
||||
) -> SearchData:
|
||||
"""
|
||||
Search for documents.
|
||||
|
||||
Args:
|
||||
client: HTTP client instance
|
||||
request: Search request
|
||||
|
||||
Returns:
|
||||
SearchData with search results grouped by source type
|
||||
|
||||
Raises:
|
||||
FirecrawlError: If the search operation fails
|
||||
"""
|
||||
request_data = _prepare_search_request(request)
|
||||
try:
|
||||
response = client.post("/v2/search", request_data)
|
||||
if response.status_code != 200:
|
||||
handle_response_error(response, "search")
|
||||
response_data = response.json()
|
||||
if not response_data.get("success"):
|
||||
handle_response_error(response, "search")
|
||||
data = response_data.get("data", {}) or {}
|
||||
out = SearchData()
|
||||
if "web" in data:
|
||||
out.web = _transform_array(data["web"], SearchResultWeb)
|
||||
if "news" in data:
|
||||
out.news = _transform_array(data["news"], SearchResultNews)
|
||||
if "images" in data:
|
||||
out.images = _transform_array(data["images"], SearchResultImages)
|
||||
return out
|
||||
except Exception as err:
|
||||
# If the error is an HTTP error from requests, handle it
|
||||
# (simulate isAxiosError by checking for requests' HTTPError or Response)
|
||||
if hasattr(err, "response"):
|
||||
handle_response_error(getattr(err, "response"), "search")
|
||||
raise err
|
||||
|
||||
def _transform_array(arr: List[Any], result_type: Type[T]) -> List[Union[T, 'Document']]:
|
||||
"""
|
||||
Transforms an array of items into a list of result_type or Document.
|
||||
If the item dict contains any of the special keys, it is treated as a Document.
|
||||
Otherwise, it is treated as result_type.
|
||||
If the item is not a dict, it is wrapped as result_type with url=item.
|
||||
"""
|
||||
results: List[Union[T, 'Document']] = []
|
||||
for item in arr:
|
||||
if item and isinstance(item, dict):
|
||||
if (
|
||||
"markdown" in item or
|
||||
"html" in item or
|
||||
"rawHtml" in item or
|
||||
"links" in item or
|
||||
"screenshot" in item or
|
||||
"changeTracking" in item or
|
||||
"summary" in item or
|
||||
"json" in item
|
||||
):
|
||||
results.append(Document(**normalize_document_input(item)))
|
||||
else:
|
||||
result_type_name = None
|
||||
if result_type == SearchResultImages:
|
||||
result_type_name = "images"
|
||||
elif result_type == SearchResultNews:
|
||||
result_type_name = "news"
|
||||
elif result_type == SearchResultWeb:
|
||||
result_type_name = "web"
|
||||
|
||||
if result_type_name:
|
||||
normalized_item = _map_search_result_keys(item, result_type_name)
|
||||
results.append(result_type(**normalized_item))
|
||||
else:
|
||||
results.append(result_type(**item))
|
||||
else:
|
||||
results.append(result_type(url=item))
|
||||
return results
|
||||
|
||||
def _validate_search_request(request: SearchRequest) -> SearchRequest:
|
||||
"""
|
||||
Validate and normalize search request.
|
||||
|
||||
Args:
|
||||
request: Search request to validate
|
||||
|
||||
Returns:
|
||||
Validated request
|
||||
|
||||
Raises:
|
||||
ValueError: If request is invalid
|
||||
"""
|
||||
# Validate query
|
||||
if not request.query or not request.query.strip():
|
||||
raise ValueError("Query cannot be empty")
|
||||
|
||||
# Validate limit
|
||||
if request.limit is not None:
|
||||
if request.limit <= 0:
|
||||
raise ValueError("Limit must be positive")
|
||||
if request.limit > 100:
|
||||
raise ValueError("Limit cannot exceed 100")
|
||||
|
||||
# Validate timeout
|
||||
if request.timeout is not None:
|
||||
if request.timeout <= 0:
|
||||
raise ValueError("Timeout must be positive")
|
||||
if request.timeout > 300000: # 5 minutes max
|
||||
raise ValueError("Timeout cannot exceed 300000ms (5 minutes)")
|
||||
|
||||
# Validate sources (if provided)
|
||||
if request.sources is not None:
|
||||
valid_sources = {"web", "news", "images"}
|
||||
for source in request.sources:
|
||||
if isinstance(source, str):
|
||||
if source not in valid_sources:
|
||||
raise ValueError(f"Invalid source type: {source}. Valid types: {valid_sources}")
|
||||
elif hasattr(source, 'type'):
|
||||
if source.type not in valid_sources:
|
||||
raise ValueError(f"Invalid source type: {source.type}. Valid types: {valid_sources}")
|
||||
|
||||
# Validate categories (if provided)
|
||||
if request.categories is not None:
|
||||
valid_categories = {"github", "research", "pdf"}
|
||||
for category in request.categories:
|
||||
if isinstance(category, str):
|
||||
if category not in valid_categories:
|
||||
raise ValueError(f"Invalid category type: {category}. Valid types: {valid_categories}")
|
||||
elif hasattr(category, 'type'):
|
||||
if category.type not in valid_categories:
|
||||
raise ValueError(f"Invalid category type: {category.type}. Valid types: {valid_categories}")
|
||||
|
||||
if request.include_domains and request.exclude_domains:
|
||||
raise ValueError(
|
||||
"include_domains and exclude_domains cannot both be specified"
|
||||
)
|
||||
|
||||
# Validate location (if provided)
|
||||
if request.location is not None:
|
||||
if not isinstance(request.location, str) or len(request.location.strip()) == 0:
|
||||
raise ValueError("Location must be a non-empty string")
|
||||
|
||||
# Validate tbs (time-based search, if provided)
|
||||
if request.tbs is not None:
|
||||
if not isinstance(request.tbs, str) or len(request.tbs.strip()) == 0:
|
||||
raise ValueError("tbs must be a non-empty string")
|
||||
|
||||
# Validate scrape_options (if provided)
|
||||
if request.scrape_options is not None:
|
||||
validate_scrape_options(request.scrape_options)
|
||||
|
||||
return request
|
||||
|
||||
|
||||
def _prepare_search_request(request: SearchRequest) -> Dict[str, Any]:
|
||||
"""
|
||||
Prepare a search request payload.
|
||||
|
||||
Args:
|
||||
request: Search request
|
||||
|
||||
Returns:
|
||||
Request payload dictionary
|
||||
"""
|
||||
validated_request = _validate_search_request(request)
|
||||
data = validated_request.model_dump(exclude_none=True, by_alias=True)
|
||||
|
||||
# Ensure default values are included only if not explicitly set to None
|
||||
if "limit" not in data and validated_request.limit is not None:
|
||||
data["limit"] = validated_request.limit
|
||||
if "timeout" not in data and validated_request.timeout is not None:
|
||||
data["timeout"] = validated_request.timeout
|
||||
|
||||
# Handle snake_case to camelCase conversions manually
|
||||
# (Pydantic Field() aliases interfere with value assignment)
|
||||
|
||||
# ignore_invalid_urls → ignoreInvalidURLs
|
||||
if validated_request.ignore_invalid_urls is not None:
|
||||
data["ignoreInvalidURLs"] = validated_request.ignore_invalid_urls
|
||||
data.pop("ignore_invalid_urls", None)
|
||||
|
||||
# include_domains → includeDomains
|
||||
if validated_request.include_domains is not None:
|
||||
data["includeDomains"] = validated_request.include_domains
|
||||
data.pop("include_domains", None)
|
||||
|
||||
# exclude_domains → excludeDomains
|
||||
if validated_request.exclude_domains is not None:
|
||||
data["excludeDomains"] = validated_request.exclude_domains
|
||||
data.pop("exclude_domains", None)
|
||||
|
||||
# scrape_options → scrapeOptions
|
||||
if validated_request.scrape_options is not None:
|
||||
scrape_data = prepare_scrape_options(validated_request.scrape_options)
|
||||
if scrape_data:
|
||||
data["scrapeOptions"] = scrape_data
|
||||
data.pop("scrape_options", None)
|
||||
|
||||
# Only include integration if it was explicitly provided and non-empty
|
||||
integration_value = getattr(validated_request, "integration", None)
|
||||
if integration_value is not None:
|
||||
integration_str = str(integration_value).strip()
|
||||
if integration_str:
|
||||
data["integration"] = integration_str
|
||||
|
||||
return data
|
||||
@@ -0,0 +1,84 @@
|
||||
from ..utils import HttpClient, handle_response_error
|
||||
from ..types import ConcurrencyCheck, CreditUsage, QueueStatusResponse, TokenUsage, CreditUsageHistoricalResponse, TokenUsageHistoricalResponse
|
||||
|
||||
|
||||
def get_concurrency(client: HttpClient) -> ConcurrencyCheck:
|
||||
resp = client.get("/v2/concurrency-check")
|
||||
if not resp.ok:
|
||||
handle_response_error(resp, "get concurrency")
|
||||
body = resp.json()
|
||||
if not body.get("success"):
|
||||
raise Exception(body.get("error", "Unknown error"))
|
||||
data = body.get("data", body)
|
||||
return ConcurrencyCheck(
|
||||
concurrency=data.get("concurrency"),
|
||||
max_concurrency=data.get("maxConcurrency", data.get("max_concurrency")),
|
||||
)
|
||||
|
||||
|
||||
def get_credit_usage(client: HttpClient) -> CreditUsage:
|
||||
resp = client.get("/v2/team/credit-usage")
|
||||
if not resp.ok:
|
||||
handle_response_error(resp, "get credit usage")
|
||||
body = resp.json()
|
||||
if not body.get("success"):
|
||||
raise Exception(body.get("error", "Unknown error"))
|
||||
data = body.get("data", body)
|
||||
return CreditUsage(
|
||||
remaining_credits=data.get("remainingCredits", data.get("remaining_credits", 0)),
|
||||
plan_credits=data.get("planCredits", data.get("plan_credits")),
|
||||
billing_period_start=data.get("billingPeriodStart", data.get("billing_period_start")),
|
||||
billing_period_end=data.get("billingPeriodEnd", data.get("billing_period_end")),
|
||||
)
|
||||
|
||||
|
||||
def get_token_usage(client: HttpClient) -> TokenUsage:
|
||||
resp = client.get("/v2/team/token-usage")
|
||||
if not resp.ok:
|
||||
handle_response_error(resp, "get token usage")
|
||||
body = resp.json()
|
||||
if not body.get("success"):
|
||||
raise Exception(body.get("error", "Unknown error"))
|
||||
data = body.get("data", body)
|
||||
return TokenUsage(
|
||||
remaining_tokens=data.get("remainingTokens", data.get("remaining_tokens", 0)),
|
||||
plan_tokens=data.get("planTokens", data.get("plan_tokens")),
|
||||
billing_period_start=data.get("billingPeriodStart", data.get("billing_period_start")),
|
||||
billing_period_end=data.get("billingPeriodEnd", data.get("billing_period_end")),
|
||||
)
|
||||
|
||||
def get_queue_status(client: HttpClient) -> QueueStatusResponse:
|
||||
resp = client.get("/v2/team/queue-status")
|
||||
if not resp.ok:
|
||||
handle_response_error(resp, "get queue status")
|
||||
body = resp.json()
|
||||
if not body.get("success"):
|
||||
raise Exception(body.get("error", "Unknown error"))
|
||||
data = body.get("data", body)
|
||||
return QueueStatusResponse(
|
||||
jobs_in_queue=data.get("jobsInQueue", 0),
|
||||
active_jobs_in_queue=data.get("activeJobsInQueue", 0),
|
||||
waiting_jobs_in_queue=data.get("waitingJobsInQueue", 0),
|
||||
max_concurrency=data.get("maxConcurrency", 0),
|
||||
most_recent_success=data.get("mostRecentSuccess", None),
|
||||
)
|
||||
|
||||
|
||||
def get_credit_usage_historical(client: HttpClient, by_api_key: bool = False) -> CreditUsageHistoricalResponse:
|
||||
resp = client.get(f"/v2/team/credit-usage/historical{'?byApiKey=true' if by_api_key else ''}")
|
||||
if not resp.ok:
|
||||
handle_response_error(resp, "get credit usage historical")
|
||||
body = resp.json()
|
||||
if not body.get("success"):
|
||||
raise Exception(body.get("error", "Unknown error"))
|
||||
return CreditUsageHistoricalResponse(**body)
|
||||
|
||||
|
||||
def get_token_usage_historical(client: HttpClient, by_api_key: bool = False) -> TokenUsageHistoricalResponse:
|
||||
resp = client.get(f"/v2/team/token-usage/historical{'?byApiKey=true' if by_api_key else ''}")
|
||||
if not resp.ok:
|
||||
handle_response_error(resp, "get token usage historical")
|
||||
body = resp.json()
|
||||
if not body.get("success"):
|
||||
raise Exception(body.get("error", "Unknown error"))
|
||||
return TokenUsageHistoricalResponse(**body)
|
||||
Reference in New Issue
Block a user