참고소스 수정본
This commit is contained in:
LASTA_DEV01\lasta
@@ -0,0 +1 @@
|
||||
# Async (aio) method modules for v2
|
||||
@@ -0,0 +1,154 @@
|
||||
from typing import Any, Dict, List, Literal, Optional, Union
|
||||
import asyncio
|
||||
|
||||
from ...types import AgentResponse, AgentWebhookConfig
|
||||
from ...utils.http_client_async import AsyncHttpClient
|
||||
from ...utils.validation import _normalize_schema
|
||||
|
||||
|
||||
def _prepare_agent_request(
|
||||
urls: Optional[List[str]],
|
||||
*,
|
||||
prompt: str,
|
||||
schema: Optional[Any] = None,
|
||||
integration: Optional[str] = None,
|
||||
max_credits: Optional[int] = None,
|
||||
strict_constrain_to_urls: Optional[bool] = None,
|
||||
model: Optional[Literal["spark-1-pro", "spark-1-mini"]] = None,
|
||||
webhook: Optional[Union[str, AgentWebhookConfig]] = None,
|
||||
) -> Dict[str, Any]:
|
||||
body: Dict[str, Any] = {}
|
||||
if urls is not None:
|
||||
body["urls"] = urls
|
||||
body["prompt"] = prompt
|
||||
if schema is not None:
|
||||
normalized_schema = _normalize_schema(schema)
|
||||
if normalized_schema is not None:
|
||||
body["schema"] = normalized_schema
|
||||
else:
|
||||
raise ValueError(
|
||||
f"Invalid schema type: {type(schema).__name__}. "
|
||||
"Schema must be a dict, Pydantic BaseModel class, or Pydantic model instance."
|
||||
)
|
||||
if integration is not None and str(integration).strip():
|
||||
body["integration"] = str(integration).strip()
|
||||
if max_credits is not None and max_credits > 0:
|
||||
body["maxCredits"] = max_credits
|
||||
if strict_constrain_to_urls is not None and strict_constrain_to_urls:
|
||||
body["strictConstrainToURLs"] = strict_constrain_to_urls
|
||||
if model is not None:
|
||||
body["model"] = model
|
||||
if webhook is not None:
|
||||
if isinstance(webhook, str):
|
||||
body["webhook"] = webhook
|
||||
else:
|
||||
body["webhook"] = webhook.model_dump(exclude_none=True)
|
||||
return body
|
||||
|
||||
|
||||
def _normalize_agent_response_payload(payload: Dict[str, Any]) -> Dict[str, Any]:
|
||||
out = dict(payload)
|
||||
if "expiresAt" in out and "expires_at" not in out:
|
||||
out["expires_at"] = out["expiresAt"]
|
||||
if "creditsUsed" in out and "credits_used" not in out:
|
||||
out["credits_used"] = out["creditsUsed"]
|
||||
return out
|
||||
|
||||
|
||||
async def start_agent(
|
||||
client: AsyncHttpClient,
|
||||
urls: Optional[List[str]],
|
||||
*,
|
||||
prompt: str,
|
||||
schema: Optional[Any] = None,
|
||||
integration: Optional[str] = None,
|
||||
max_credits: Optional[int] = None,
|
||||
strict_constrain_to_urls: Optional[bool] = None,
|
||||
model: Optional[Literal["spark-1-pro", "spark-1-mini"]] = None,
|
||||
webhook: Optional[Union[str, AgentWebhookConfig]] = None,
|
||||
) -> AgentResponse:
|
||||
body = _prepare_agent_request(
|
||||
urls,
|
||||
prompt=prompt,
|
||||
schema=schema,
|
||||
integration=integration,
|
||||
max_credits=max_credits,
|
||||
strict_constrain_to_urls=strict_constrain_to_urls,
|
||||
model=model,
|
||||
webhook=webhook,
|
||||
)
|
||||
resp = await client.post("/v2/agent", body)
|
||||
payload = _normalize_agent_response_payload(resp.json())
|
||||
return AgentResponse(**payload)
|
||||
|
||||
|
||||
async def get_agent_status(client: AsyncHttpClient, job_id: str) -> AgentResponse:
|
||||
resp = await client.get(f"/v2/agent/{job_id}")
|
||||
payload = _normalize_agent_response_payload(resp.json())
|
||||
return AgentResponse(**payload)
|
||||
|
||||
|
||||
async def wait_agent(
|
||||
client: AsyncHttpClient,
|
||||
job_id: str,
|
||||
*,
|
||||
poll_interval: int = 2,
|
||||
timeout: Optional[int] = None,
|
||||
) -> AgentResponse:
|
||||
start_ts = asyncio.get_event_loop().time()
|
||||
while True:
|
||||
status = await get_agent_status(client, job_id)
|
||||
if status.status in ("completed", "failed", "cancelled"):
|
||||
return status
|
||||
if timeout is not None and (asyncio.get_event_loop().time() - start_ts) > timeout:
|
||||
return status
|
||||
await asyncio.sleep(max(1, poll_interval))
|
||||
|
||||
|
||||
async def agent(
|
||||
client: AsyncHttpClient,
|
||||
urls: Optional[List[str]],
|
||||
*,
|
||||
prompt: str,
|
||||
schema: Optional[Any] = None,
|
||||
integration: Optional[str] = None,
|
||||
poll_interval: int = 2,
|
||||
timeout: Optional[int] = None,
|
||||
max_credits: Optional[int] = None,
|
||||
strict_constrain_to_urls: Optional[bool] = None,
|
||||
model: Optional[Literal["spark-1-pro", "spark-1-mini"]] = None,
|
||||
webhook: Optional[Union[str, AgentWebhookConfig]] = None,
|
||||
) -> AgentResponse:
|
||||
started = await start_agent(
|
||||
client,
|
||||
urls,
|
||||
prompt=prompt,
|
||||
schema=schema,
|
||||
integration=integration,
|
||||
max_credits=max_credits,
|
||||
strict_constrain_to_urls=strict_constrain_to_urls,
|
||||
model=model,
|
||||
webhook=webhook,
|
||||
)
|
||||
job_id = getattr(started, "id", None)
|
||||
if not job_id:
|
||||
return started
|
||||
return await wait_agent(client, job_id, poll_interval=poll_interval, timeout=timeout)
|
||||
|
||||
|
||||
async def cancel_agent(client: AsyncHttpClient, job_id: str) -> bool:
|
||||
"""
|
||||
Cancel a running agent job.
|
||||
|
||||
Args:
|
||||
client: Async HTTP client instance
|
||||
job_id: ID of the agent job to cancel
|
||||
|
||||
Returns:
|
||||
bool: True if the agent was cancelled, False otherwise
|
||||
|
||||
Raises:
|
||||
Exception: If the cancellation fails
|
||||
"""
|
||||
resp = await client.delete(f"/v2/agent/{job_id}")
|
||||
return resp.json().get("success", False)
|
||||
@@ -0,0 +1,240 @@
|
||||
from typing import Optional, List, Dict, Any
|
||||
from ...types import ScrapeOptions, WebhookConfig, Document, BatchScrapeResponse, BatchScrapeJob, PaginationConfig
|
||||
from ...utils.http_client_async import AsyncHttpClient
|
||||
from ...utils.validation import prepare_scrape_options
|
||||
from ...utils.error_handler import handle_response_error
|
||||
from ...utils.normalize import normalize_document_input
|
||||
from ...methods.batch import validate_batch_urls
|
||||
import time
|
||||
|
||||
def _parse_batch_scrape_documents(data_list: Optional[List[Any]]) -> List[Document]:
|
||||
documents: List[Document] = []
|
||||
for doc in data_list or []:
|
||||
if isinstance(doc, dict):
|
||||
normalized = normalize_document_input(doc)
|
||||
documents.append(Document(**normalized))
|
||||
return documents
|
||||
|
||||
|
||||
def _parse_batch_scrape_status_response(body: Dict[str, Any]) -> Dict[str, Any]:
|
||||
if not body.get("success"):
|
||||
raise Exception(body.get("error", "Unknown error occurred"))
|
||||
|
||||
return {
|
||||
"status": body.get("status"),
|
||||
"completed": body.get("completed", 0),
|
||||
"total": body.get("total", 0),
|
||||
"credits_used": body.get("creditsUsed"),
|
||||
"expires_at": body.get("expiresAt"),
|
||||
"next": body.get("next"),
|
||||
"data": _parse_batch_scrape_documents(body.get("data", []) or []),
|
||||
}
|
||||
|
||||
def _prepare(urls: List[str], *, options: Optional[ScrapeOptions] = None, **kwargs) -> Dict[str, Any]:
|
||||
if not urls:
|
||||
raise ValueError("URLs list cannot be empty")
|
||||
|
||||
validated_urls = validate_batch_urls([u.strip() if isinstance(u, str) else u for u in urls])
|
||||
payload: Dict[str, Any] = {"urls": validated_urls}
|
||||
if options:
|
||||
opts = prepare_scrape_options(options)
|
||||
if opts:
|
||||
payload.update(opts)
|
||||
if (w := kwargs.get("webhook")) is not None:
|
||||
payload["webhook"] = w if isinstance(w, str) else w.model_dump(exclude_none=True)
|
||||
if (v := kwargs.get("append_to_id")) is not None:
|
||||
payload["appendToId"] = v
|
||||
if (v := kwargs.get("ignore_invalid_urls")) is not None:
|
||||
payload["ignoreInvalidURLs"] = v
|
||||
if (v := kwargs.get("max_concurrency")) is not None:
|
||||
payload["maxConcurrency"] = v
|
||||
if (v := kwargs.get("zero_data_retention")) is not None:
|
||||
payload["zeroDataRetention"] = v
|
||||
if (v := kwargs.get("integration")) is not None:
|
||||
trimmed_integration = str(v).strip()
|
||||
if trimmed_integration:
|
||||
payload["integration"] = trimmed_integration
|
||||
return payload
|
||||
|
||||
|
||||
async def start_batch_scrape(client: AsyncHttpClient, urls: List[str], **kwargs) -> BatchScrapeResponse:
|
||||
payload = _prepare(urls, **kwargs)
|
||||
response = await client.post("/v2/batch/scrape", payload)
|
||||
if response.status_code >= 400:
|
||||
handle_response_error(response, "start batch scrape")
|
||||
body = response.json()
|
||||
if not body.get("success"):
|
||||
raise Exception(body.get("error", "Unknown error occurred"))
|
||||
return BatchScrapeResponse(id=body.get("id"), url=body.get("url"), invalid_urls=body.get("invalidURLs"))
|
||||
|
||||
|
||||
async def get_batch_scrape_status(
|
||||
client: AsyncHttpClient,
|
||||
job_id: str,
|
||||
pagination_config: Optional[PaginationConfig] = None
|
||||
) -> BatchScrapeJob:
|
||||
"""
|
||||
Get the status of a batch scrape job.
|
||||
|
||||
Args:
|
||||
client: Async HTTP client instance
|
||||
job_id: ID of the batch scrape job
|
||||
pagination_config: Optional configuration for pagination behavior
|
||||
|
||||
Returns:
|
||||
BatchScrapeJob containing job status and data
|
||||
|
||||
Raises:
|
||||
Exception: If the status check fails
|
||||
"""
|
||||
response = await client.get(f"/v2/batch/scrape/{job_id}")
|
||||
if response.status_code >= 400:
|
||||
handle_response_error(response, "get batch scrape status")
|
||||
body = response.json()
|
||||
payload = _parse_batch_scrape_status_response(body)
|
||||
docs = payload["data"]
|
||||
|
||||
# Handle pagination if requested
|
||||
auto_paginate = pagination_config.auto_paginate if pagination_config else True
|
||||
if auto_paginate and payload["next"]:
|
||||
docs = await _fetch_all_batch_pages_async(
|
||||
client,
|
||||
payload["next"],
|
||||
docs,
|
||||
pagination_config
|
||||
)
|
||||
|
||||
return BatchScrapeJob(
|
||||
status=payload["status"],
|
||||
completed=payload["completed"],
|
||||
total=payload["total"],
|
||||
credits_used=payload["credits_used"],
|
||||
expires_at=payload["expires_at"],
|
||||
next=payload["next"] if not auto_paginate else None,
|
||||
data=docs,
|
||||
)
|
||||
|
||||
|
||||
async def get_batch_scrape_status_page(
|
||||
client: AsyncHttpClient,
|
||||
next_url: str,
|
||||
*,
|
||||
request_timeout: Optional[float] = None,
|
||||
) -> BatchScrapeJob:
|
||||
"""
|
||||
Fetch a single page of batch scrape results using the provided next URL.
|
||||
|
||||
Args:
|
||||
client: Async HTTP client instance
|
||||
next_url: Opaque next URL from a prior batch scrape status response
|
||||
request_timeout: Timeout (in seconds) for the HTTP request
|
||||
|
||||
Returns:
|
||||
BatchScrapeJob with the page data and next URL (if any)
|
||||
|
||||
Raises:
|
||||
Exception: If the request fails or returns an error response
|
||||
"""
|
||||
response = await client.get(next_url, timeout=request_timeout)
|
||||
if response.status_code >= 400:
|
||||
handle_response_error(response, "get batch scrape status page")
|
||||
body = response.json()
|
||||
payload = _parse_batch_scrape_status_response(body)
|
||||
return BatchScrapeJob(
|
||||
status=payload["status"],
|
||||
completed=payload["completed"],
|
||||
total=payload["total"],
|
||||
credits_used=payload["credits_used"],
|
||||
expires_at=payload["expires_at"],
|
||||
next=payload["next"],
|
||||
data=payload["data"],
|
||||
)
|
||||
|
||||
|
||||
async def _fetch_all_batch_pages_async(
|
||||
client: AsyncHttpClient,
|
||||
next_url: str,
|
||||
initial_documents: List[Document],
|
||||
pagination_config: Optional[PaginationConfig] = None
|
||||
) -> List[Document]:
|
||||
"""
|
||||
Fetch all pages of batch scrape results asynchronously.
|
||||
|
||||
Args:
|
||||
client: Async HTTP client instance
|
||||
next_url: URL for the next page
|
||||
initial_documents: Documents from the first page
|
||||
pagination_config: Optional configuration for pagination limits
|
||||
|
||||
Returns:
|
||||
List of all documents from all pages
|
||||
"""
|
||||
documents = initial_documents.copy()
|
||||
current_url = next_url
|
||||
page_count = 0
|
||||
|
||||
# Apply pagination limits
|
||||
max_pages = pagination_config.max_pages if pagination_config else None
|
||||
max_results = pagination_config.max_results if pagination_config else None
|
||||
max_wait_time = pagination_config.max_wait_time if pagination_config else None
|
||||
|
||||
start_time = time.monotonic()
|
||||
|
||||
while current_url:
|
||||
# Check pagination limits
|
||||
if (max_pages is not None) and (page_count >= max_pages):
|
||||
break
|
||||
|
||||
if (max_wait_time is not None) and (time.monotonic() - start_time) > max_wait_time:
|
||||
break
|
||||
|
||||
# Fetch next page
|
||||
response = await client.get(current_url)
|
||||
|
||||
if response.status_code >= 400:
|
||||
# Log error but continue with what we have
|
||||
import logging
|
||||
logger = logging.getLogger("firecrawl")
|
||||
logger.warning(f"Failed to fetch next page: {response.status_code}")
|
||||
break
|
||||
|
||||
page_data = response.json()
|
||||
try:
|
||||
page_payload = _parse_batch_scrape_status_response(page_data)
|
||||
except Exception:
|
||||
break
|
||||
|
||||
# Add documents from this page
|
||||
for document in page_payload["data"]:
|
||||
# Check max_results limit
|
||||
if (max_results is not None) and (len(documents) >= max_results):
|
||||
break
|
||||
documents.append(document)
|
||||
|
||||
# Check if we hit max_results limit
|
||||
if (max_results is not None) and (len(documents) >= max_results):
|
||||
break
|
||||
|
||||
# Get next URL
|
||||
current_url = page_payload["next"]
|
||||
page_count += 1
|
||||
|
||||
return documents
|
||||
|
||||
|
||||
async def cancel_batch_scrape(client: AsyncHttpClient, job_id: str) -> bool:
|
||||
response = await client.delete(f"/v2/batch/scrape/{job_id}")
|
||||
if response.status_code >= 400:
|
||||
handle_response_error(response, "cancel batch scrape")
|
||||
body = response.json()
|
||||
return body.get("status") == "cancelled"
|
||||
|
||||
|
||||
async def get_batch_scrape_errors(client: AsyncHttpClient, job_id: str) -> Dict[str, Any]:
|
||||
response = await client.get(f"/v2/batch/scrape/{job_id}/errors")
|
||||
if response.status_code >= 400:
|
||||
handle_response_error(response, "get batch scrape errors")
|
||||
body = response.json()
|
||||
if not body.get("success"):
|
||||
raise Exception(body.get("error", "Unknown error occurred"))
|
||||
return body
|
||||
@@ -0,0 +1,179 @@
|
||||
"""
|
||||
Async browser session methods for Firecrawl v2 API.
|
||||
|
||||
Provides async create, execute, delete, and list operations for browser sessions.
|
||||
"""
|
||||
|
||||
from typing import Any, Dict, Literal, Optional
|
||||
|
||||
from ...types import (
|
||||
BrowserCreateResponse,
|
||||
BrowserExecuteResponse,
|
||||
BrowserDeleteResponse,
|
||||
BrowserListResponse,
|
||||
)
|
||||
from ...utils.http_client_async import AsyncHttpClient
|
||||
|
||||
|
||||
def _normalize_browser_create_response(payload: Dict[str, Any]) -> Dict[str, Any]:
|
||||
out = dict(payload)
|
||||
if "cdpUrl" in out and "cdp_url" not in out:
|
||||
out["cdp_url"] = out["cdpUrl"]
|
||||
if "liveViewUrl" in out and "live_view_url" not in out:
|
||||
out["live_view_url"] = out["liveViewUrl"]
|
||||
if "interactiveLiveViewUrl" in out and "interactive_live_view_url" not in out:
|
||||
out["interactive_live_view_url"] = out["interactiveLiveViewUrl"]
|
||||
if "expiresAt" in out and "expires_at" not in out:
|
||||
out["expires_at"] = out["expiresAt"]
|
||||
return out
|
||||
|
||||
|
||||
def _normalize_browser_list_response(payload: Dict[str, Any]) -> Dict[str, Any]:
|
||||
out = dict(payload)
|
||||
if "sessions" in out and isinstance(out["sessions"], list):
|
||||
normalized_sessions = []
|
||||
for s in out["sessions"]:
|
||||
ns = dict(s)
|
||||
if "cdpUrl" in ns and "cdp_url" not in ns:
|
||||
ns["cdp_url"] = ns["cdpUrl"]
|
||||
if "liveViewUrl" in ns and "live_view_url" not in ns:
|
||||
ns["live_view_url"] = ns["liveViewUrl"]
|
||||
if "interactiveLiveViewUrl" in ns and "interactive_live_view_url" not in ns:
|
||||
ns["interactive_live_view_url"] = ns["interactiveLiveViewUrl"]
|
||||
if "streamWebView" in ns and "stream_web_view" not in ns:
|
||||
ns["stream_web_view"] = ns["streamWebView"]
|
||||
if "createdAt" in ns and "created_at" not in ns:
|
||||
ns["created_at"] = ns["createdAt"]
|
||||
if "lastActivity" in ns and "last_activity" not in ns:
|
||||
ns["last_activity"] = ns["lastActivity"]
|
||||
normalized_sessions.append(ns)
|
||||
out["sessions"] = normalized_sessions
|
||||
return out
|
||||
|
||||
|
||||
async def browser(
|
||||
client: AsyncHttpClient,
|
||||
*,
|
||||
ttl: Optional[int] = None,
|
||||
activity_ttl: Optional[int] = None,
|
||||
stream_web_view: Optional[bool] = None,
|
||||
profile: Optional[Dict[str, Any]] = None,
|
||||
) -> BrowserCreateResponse:
|
||||
"""Create a new browser session.
|
||||
|
||||
Args:
|
||||
client: Async HTTP client instance
|
||||
ttl: Total time-to-live in seconds (30-3600, default 300)
|
||||
activity_ttl: Inactivity TTL in seconds (10-3600)
|
||||
stream_web_view: Whether to enable webview streaming
|
||||
profile: Profile config with ``name`` (str) and
|
||||
optional ``save_changes`` (bool, default ``True``)
|
||||
|
||||
Returns:
|
||||
BrowserCreateResponse with session id and CDP URL
|
||||
"""
|
||||
body: Dict[str, Any] = {}
|
||||
if ttl is not None:
|
||||
body["ttl"] = ttl
|
||||
if activity_ttl is not None:
|
||||
body["activityTtl"] = activity_ttl
|
||||
if stream_web_view is not None:
|
||||
body["streamWebView"] = stream_web_view
|
||||
if profile is not None:
|
||||
body["profile"] = {
|
||||
"name": profile["name"],
|
||||
"saveChanges": profile.get("save_changes", True),
|
||||
}
|
||||
|
||||
resp = await client.post("/v2/browser", body)
|
||||
payload = _normalize_browser_create_response(resp.json())
|
||||
return BrowserCreateResponse(**payload)
|
||||
|
||||
|
||||
def _normalize_browser_execute_response(payload: Dict[str, Any]) -> Dict[str, Any]:
|
||||
out = dict(payload)
|
||||
if "exitCode" in out and "exit_code" not in out:
|
||||
out["exit_code"] = out["exitCode"]
|
||||
return out
|
||||
|
||||
|
||||
def _normalize_browser_delete_response(payload: Dict[str, Any]) -> Dict[str, Any]:
|
||||
out = dict(payload)
|
||||
if "sessionDurationMs" in out and "session_duration_ms" not in out:
|
||||
out["session_duration_ms"] = out["sessionDurationMs"]
|
||||
if "creditsBilled" in out and "credits_billed" not in out:
|
||||
out["credits_billed"] = out["creditsBilled"]
|
||||
return out
|
||||
|
||||
|
||||
async def browser_execute(
|
||||
client: AsyncHttpClient,
|
||||
session_id: str,
|
||||
code: str,
|
||||
*,
|
||||
language: Literal["python", "node", "bash"] = "bash",
|
||||
timeout: Optional[int] = None,
|
||||
) -> BrowserExecuteResponse:
|
||||
"""Execute code in a browser session.
|
||||
|
||||
Args:
|
||||
client: Async HTTP client instance
|
||||
session_id: Browser session ID
|
||||
code: Code to execute
|
||||
language: Programming language ("python", "node", or "bash")
|
||||
timeout: Execution timeout in seconds (1-300, default 30)
|
||||
|
||||
Returns:
|
||||
BrowserExecuteResponse with execution result
|
||||
"""
|
||||
body: Dict[str, Any] = {
|
||||
"code": code,
|
||||
"language": language,
|
||||
}
|
||||
if timeout is not None:
|
||||
body["timeout"] = timeout
|
||||
|
||||
resp = await client.post(f"/v2/browser/{session_id}/execute", body)
|
||||
payload = _normalize_browser_execute_response(resp.json())
|
||||
return BrowserExecuteResponse(**payload)
|
||||
|
||||
|
||||
async def delete_browser(
|
||||
client: AsyncHttpClient,
|
||||
session_id: str,
|
||||
) -> BrowserDeleteResponse:
|
||||
"""Delete a browser session.
|
||||
|
||||
Args:
|
||||
client: Async HTTP client instance
|
||||
session_id: Browser session ID
|
||||
|
||||
Returns:
|
||||
BrowserDeleteResponse
|
||||
"""
|
||||
resp = await client.delete(f"/v2/browser/{session_id}")
|
||||
payload = _normalize_browser_delete_response(resp.json())
|
||||
return BrowserDeleteResponse(**payload)
|
||||
|
||||
|
||||
async def list_browsers(
|
||||
client: AsyncHttpClient,
|
||||
*,
|
||||
status: Optional[Literal["active", "destroyed"]] = None,
|
||||
) -> BrowserListResponse:
|
||||
"""List browser sessions.
|
||||
|
||||
Args:
|
||||
client: Async HTTP client instance
|
||||
status: Filter by session status ("active" or "destroyed")
|
||||
|
||||
Returns:
|
||||
BrowserListResponse with list of sessions
|
||||
"""
|
||||
endpoint = "/v2/browser"
|
||||
if status is not None:
|
||||
endpoint = f"{endpoint}?status={status}"
|
||||
|
||||
resp = await client.get(endpoint)
|
||||
payload = _normalize_browser_list_response(resp.json())
|
||||
return BrowserListResponse(**payload)
|
||||
@@ -0,0 +1,414 @@
|
||||
from typing import Optional, Dict, Any, List
|
||||
from ...types import (
|
||||
CrawlRequest,
|
||||
CrawlJob,
|
||||
CrawlResponse,
|
||||
Document,
|
||||
CrawlParamsRequest,
|
||||
CrawlParamsData,
|
||||
WebhookConfig,
|
||||
CrawlErrorsResponse,
|
||||
ActiveCrawlsResponse,
|
||||
ActiveCrawl,
|
||||
PaginationConfig,
|
||||
)
|
||||
from ...utils.error_handler import handle_response_error
|
||||
from ...utils.validation import prepare_scrape_options
|
||||
from ...utils.http_client_async import AsyncHttpClient
|
||||
from ...utils.normalize import normalize_document_input
|
||||
import time
|
||||
|
||||
|
||||
def _prepare_crawl_request(request: CrawlRequest) -> dict:
|
||||
if not request.url or not request.url.strip():
|
||||
raise ValueError("URL cannot be empty")
|
||||
if request.limit is not None and request.limit <= 0:
|
||||
raise ValueError("Limit must be positive")
|
||||
data = {"url": request.url}
|
||||
if request.prompt:
|
||||
data["prompt"] = request.prompt
|
||||
if request.scrape_options is not None:
|
||||
opts = prepare_scrape_options(request.scrape_options)
|
||||
if opts:
|
||||
data["scrapeOptions"] = opts
|
||||
# Webhook conversion
|
||||
if request.webhook is not None:
|
||||
if isinstance(request.webhook, str):
|
||||
data["webhook"] = request.webhook
|
||||
else:
|
||||
data["webhook"] = request.webhook.model_dump(exclude_none=True)
|
||||
request_data = request.model_dump(exclude_none=True, exclude_unset=True)
|
||||
request_data.pop("url", None)
|
||||
request_data.pop("prompt", None)
|
||||
request_data.pop("scrape_options", None)
|
||||
field_mappings = {
|
||||
"include_paths": "includePaths",
|
||||
"exclude_paths": "excludePaths",
|
||||
"max_discovery_depth": "maxDiscoveryDepth",
|
||||
"sitemap": "sitemap",
|
||||
"ignore_query_parameters": "ignoreQueryParameters",
|
||||
"deduplicate_similar_urls": "deduplicateSimilarURLs",
|
||||
"crawl_entire_domain": "crawlEntireDomain",
|
||||
"allow_external_links": "allowExternalLinks",
|
||||
"allow_subdomains": "allowSubdomains",
|
||||
"ignore_robots_txt": "ignoreRobotsTxt",
|
||||
"robots_user_agent": "robotsUserAgent",
|
||||
"delay": "delay",
|
||||
"max_concurrency": "maxConcurrency",
|
||||
"regex_on_full_url": "regexOnFullURL",
|
||||
"zero_data_retention": "zeroDataRetention",
|
||||
}
|
||||
for snake, camel in field_mappings.items():
|
||||
if snake in request_data:
|
||||
data[camel] = request_data.pop(snake)
|
||||
data.update(request_data)
|
||||
if getattr(request, "integration", None) is not None:
|
||||
data["integration"] = str(getattr(request, "integration")).strip()
|
||||
return data
|
||||
|
||||
|
||||
def _parse_crawl_documents(data_list: Optional[List[Any]]) -> List[Document]:
|
||||
documents: List[Document] = []
|
||||
for doc_data in data_list or []:
|
||||
if isinstance(doc_data, dict):
|
||||
normalized = normalize_document_input(doc_data)
|
||||
documents.append(Document(**normalized))
|
||||
return documents
|
||||
|
||||
|
||||
def _parse_crawl_status_response(body: Dict[str, Any]) -> Dict[str, Any]:
|
||||
if not body.get("success"):
|
||||
raise Exception(body.get("error", "Unknown error occurred"))
|
||||
|
||||
return {
|
||||
"status": body.get("status"),
|
||||
"completed": body.get("completed", 0),
|
||||
"total": body.get("total", 0),
|
||||
"credits_used": body.get("creditsUsed", 0),
|
||||
"expires_at": body.get("expiresAt"),
|
||||
"next": body.get("next"),
|
||||
"data": _parse_crawl_documents(body.get("data", [])),
|
||||
}
|
||||
|
||||
|
||||
async def start_crawl(client: AsyncHttpClient, request: CrawlRequest) -> CrawlResponse:
|
||||
"""
|
||||
Start a crawl job for a website.
|
||||
|
||||
Args:
|
||||
client: Async HTTP client instance
|
||||
request: CrawlRequest containing URL and options
|
||||
|
||||
Returns:
|
||||
CrawlResponse with job information
|
||||
|
||||
Raises:
|
||||
ValueError: If request is invalid
|
||||
Exception: If the crawl operation fails to start
|
||||
"""
|
||||
payload = _prepare_crawl_request(request)
|
||||
response = await client.post("/v2/crawl", payload)
|
||||
if response.status_code >= 400:
|
||||
handle_response_error(response, "start crawl")
|
||||
body = response.json()
|
||||
if body.get("success"):
|
||||
return CrawlResponse(id=body.get("id"), url=body.get("url"))
|
||||
raise Exception(body.get("error", "Unknown error occurred"))
|
||||
|
||||
|
||||
async def get_crawl_status(
|
||||
client: AsyncHttpClient,
|
||||
job_id: str,
|
||||
pagination_config: Optional[PaginationConfig] = None,
|
||||
*,
|
||||
request_timeout: Optional[float] = None,
|
||||
) -> CrawlJob:
|
||||
"""
|
||||
Get the status of a crawl job.
|
||||
|
||||
Args:
|
||||
client: Async HTTP client instance
|
||||
job_id: ID of the crawl job
|
||||
pagination_config: Optional configuration for pagination limits
|
||||
request_timeout: Timeout (in seconds) for each individual HTTP request. When auto-pagination
|
||||
is enabled (default) and there are multiple pages of results, this timeout applies to
|
||||
each page request separately, not to the entire operation
|
||||
|
||||
Returns:
|
||||
CrawlJob with job information
|
||||
|
||||
Raises:
|
||||
Exception: If the status check fails
|
||||
"""
|
||||
response = await client.get(f"/v2/crawl/{job_id}", timeout=request_timeout)
|
||||
if response.status_code >= 400:
|
||||
handle_response_error(response, "get crawl status")
|
||||
body = response.json()
|
||||
payload = _parse_crawl_status_response(body)
|
||||
|
||||
documents = payload["data"]
|
||||
|
||||
# Handle pagination if requested
|
||||
auto_paginate = pagination_config.auto_paginate if pagination_config else True
|
||||
if auto_paginate and payload["next"]:
|
||||
documents = await _fetch_all_pages_async(
|
||||
client,
|
||||
payload["next"],
|
||||
documents,
|
||||
pagination_config,
|
||||
request_timeout=request_timeout,
|
||||
)
|
||||
|
||||
return CrawlJob(
|
||||
status=payload["status"],
|
||||
completed=payload["completed"],
|
||||
total=payload["total"],
|
||||
credits_used=payload["credits_used"],
|
||||
expires_at=payload["expires_at"],
|
||||
next=payload["next"] if not auto_paginate else None,
|
||||
data=documents,
|
||||
)
|
||||
|
||||
|
||||
async def get_crawl_status_page(
|
||||
client: AsyncHttpClient,
|
||||
next_url: str,
|
||||
*,
|
||||
request_timeout: Optional[float] = None,
|
||||
) -> CrawlJob:
|
||||
"""
|
||||
Fetch a single page of crawl results using the provided next URL.
|
||||
|
||||
Args:
|
||||
client: Async HTTP client instance
|
||||
next_url: Opaque next URL from a prior crawl status response
|
||||
request_timeout: Timeout (in seconds) for the HTTP request
|
||||
|
||||
Returns:
|
||||
CrawlJob with the page data and next URL (if any)
|
||||
|
||||
Raises:
|
||||
Exception: If the request fails or returns an error response
|
||||
"""
|
||||
response = await client.get(next_url, timeout=request_timeout)
|
||||
if response.status_code >= 400:
|
||||
handle_response_error(response, "get crawl status page")
|
||||
body = response.json()
|
||||
payload = _parse_crawl_status_response(body)
|
||||
return CrawlJob(
|
||||
status=payload["status"],
|
||||
completed=payload["completed"],
|
||||
total=payload["total"],
|
||||
credits_used=payload["credits_used"],
|
||||
expires_at=payload["expires_at"],
|
||||
next=payload["next"],
|
||||
data=payload["data"],
|
||||
)
|
||||
|
||||
|
||||
async def _fetch_all_pages_async(
|
||||
client: AsyncHttpClient,
|
||||
next_url: str,
|
||||
initial_documents: List[Document],
|
||||
pagination_config: Optional[PaginationConfig] = None,
|
||||
*,
|
||||
request_timeout: Optional[float] = None,
|
||||
) -> List[Document]:
|
||||
"""
|
||||
Fetch all pages of crawl results asynchronously.
|
||||
|
||||
Args:
|
||||
client: Async HTTP client instance
|
||||
next_url: URL for the next page
|
||||
initial_documents: Documents from the first page
|
||||
pagination_config: Optional configuration for pagination limits
|
||||
request_timeout: Optional timeout (in seconds) for the underlying HTTP request
|
||||
|
||||
Returns:
|
||||
List of all documents from all pages
|
||||
"""
|
||||
documents = initial_documents.copy()
|
||||
current_url = next_url
|
||||
page_count = 0
|
||||
|
||||
# Apply pagination limits
|
||||
max_pages = pagination_config.max_pages if pagination_config else None
|
||||
max_results = pagination_config.max_results if pagination_config else None
|
||||
max_wait_time = pagination_config.max_wait_time if pagination_config else None
|
||||
|
||||
start_time = time.monotonic()
|
||||
|
||||
while current_url:
|
||||
# Check pagination limits (treat 0 as a valid limit)
|
||||
if (max_pages is not None) and page_count >= max_pages:
|
||||
break
|
||||
|
||||
if (max_wait_time is not None) and (time.monotonic() - start_time) > max_wait_time:
|
||||
break
|
||||
|
||||
# Fetch next page
|
||||
response = await client.get(current_url, timeout=request_timeout)
|
||||
|
||||
if response.status_code >= 400:
|
||||
# Log error but continue with what we have
|
||||
import logging
|
||||
logger = logging.getLogger("firecrawl")
|
||||
logger.warning("Failed to fetch next page", extra={"status_code": response.status_code})
|
||||
break
|
||||
|
||||
page_data = response.json()
|
||||
try:
|
||||
page_payload = _parse_crawl_status_response(page_data)
|
||||
except Exception:
|
||||
break
|
||||
|
||||
# Add documents from this page
|
||||
for document in page_payload["data"]:
|
||||
# Check max_results limit
|
||||
if (max_results is not None) and (len(documents) >= max_results):
|
||||
break
|
||||
documents.append(document)
|
||||
|
||||
# Check if we hit max_results limit
|
||||
if (max_results is not None) and (len(documents) >= max_results):
|
||||
break
|
||||
|
||||
# Get next URL
|
||||
current_url = page_payload["next"]
|
||||
page_count += 1
|
||||
|
||||
return documents
|
||||
|
||||
|
||||
async def cancel_crawl(client: AsyncHttpClient, job_id: str) -> bool:
|
||||
"""
|
||||
Cancel a crawl job.
|
||||
|
||||
Args:
|
||||
client: Async HTTP client instance
|
||||
job_id: ID of the crawl job
|
||||
|
||||
Returns:
|
||||
True if cancellation was successful
|
||||
|
||||
Raises:
|
||||
Exception: If the cancellation operation fails
|
||||
"""
|
||||
response = await client.delete(f"/v2/crawl/{job_id}")
|
||||
if response.status_code >= 400:
|
||||
handle_response_error(response, "cancel crawl")
|
||||
body = response.json()
|
||||
return body.get("status") == "cancelled"
|
||||
|
||||
|
||||
async def crawl_params_preview(client: AsyncHttpClient, request: CrawlParamsRequest) -> CrawlParamsData:
|
||||
"""
|
||||
Preview crawl parameters before starting a crawl job.
|
||||
|
||||
Args:
|
||||
client: Async HTTP client instance
|
||||
request: CrawlParamsRequest containing URL and prompt
|
||||
|
||||
Returns:
|
||||
CrawlParamsData containing crawl configuration
|
||||
|
||||
Raises:
|
||||
ValueError: If request is invalid
|
||||
Exception: If the parameter preview fails
|
||||
"""
|
||||
if not request.url or not request.url.strip():
|
||||
raise ValueError("URL cannot be empty")
|
||||
if not request.prompt or not request.prompt.strip():
|
||||
raise ValueError("Prompt cannot be empty")
|
||||
payload = {"url": request.url, "prompt": request.prompt}
|
||||
response = await client.post("/v2/crawl/params-preview", payload)
|
||||
if response.status_code >= 400:
|
||||
handle_response_error(response, "crawl params preview")
|
||||
body = response.json()
|
||||
if not body.get("success"):
|
||||
raise Exception(body.get("error", "Unknown error occurred"))
|
||||
params_data = body.get("data", {})
|
||||
converted: Dict[str, Any] = {}
|
||||
mapping = {
|
||||
"includePaths": "include_paths",
|
||||
"excludePaths": "exclude_paths",
|
||||
"maxDiscoveryDepth": "max_discovery_depth",
|
||||
"sitemap": "sitemap",
|
||||
"ignoreQueryParameters": "ignore_query_parameters",
|
||||
"deduplicateSimilarURLs": "deduplicate_similar_urls",
|
||||
"crawlEntireDomain": "crawl_entire_domain",
|
||||
"allowExternalLinks": "allow_external_links",
|
||||
"allowSubdomains": "allow_subdomains",
|
||||
"ignoreRobotsTxt": "ignore_robots_txt",
|
||||
"robotsUserAgent": "robots_user_agent",
|
||||
"maxConcurrency": "max_concurrency",
|
||||
"scrapeOptions": "scrape_options",
|
||||
"zeroDataRetention": "zero_data_retention",
|
||||
}
|
||||
for camel, snake in mapping.items():
|
||||
if camel in params_data:
|
||||
converted[snake] = params_data[camel]
|
||||
if "webhook" in params_data:
|
||||
wk = params_data["webhook"]
|
||||
converted["webhook"] = wk
|
||||
if "warning" in body:
|
||||
converted["warning"] = body["warning"]
|
||||
return CrawlParamsData(**converted)
|
||||
|
||||
|
||||
async def get_crawl_errors(client: AsyncHttpClient, crawl_id: str) -> CrawlErrorsResponse:
|
||||
"""
|
||||
Get errors from a crawl job.
|
||||
|
||||
Args:
|
||||
client: Async HTTP client instance
|
||||
crawl_id: ID of the crawl job
|
||||
|
||||
Returns:
|
||||
CrawlErrorsResponse with errors and robots blocked
|
||||
|
||||
Raises:
|
||||
Exception: If the error check operation fails
|
||||
"""
|
||||
response = await client.get(f"/v2/crawl/{crawl_id}/errors")
|
||||
if response.status_code >= 400:
|
||||
handle_response_error(response, "check crawl errors")
|
||||
body = response.json()
|
||||
payload = body.get("data", body)
|
||||
normalized = {
|
||||
"errors": payload.get("errors", []),
|
||||
"robots_blocked": payload.get("robotsBlocked", payload.get("robots_blocked", [])),
|
||||
}
|
||||
return CrawlErrorsResponse(**normalized)
|
||||
|
||||
|
||||
async def get_active_crawls(client: AsyncHttpClient) -> ActiveCrawlsResponse:
|
||||
"""
|
||||
Get active crawl jobs.
|
||||
|
||||
Args:
|
||||
client: Async HTTP client instance
|
||||
|
||||
Returns:
|
||||
ActiveCrawlsResponse with active crawl jobs
|
||||
|
||||
Raises:
|
||||
Exception: If the active crawl jobs operation fails
|
||||
"""
|
||||
response = await client.get("/v2/crawl/active")
|
||||
if response.status_code >= 400:
|
||||
handle_response_error(response, "get active crawls")
|
||||
body = response.json()
|
||||
if not body.get("success"):
|
||||
raise Exception(body.get("error", "Unknown error occurred"))
|
||||
crawls_in = body.get("crawls", [])
|
||||
normalized = []
|
||||
for c in crawls_in:
|
||||
if isinstance(c, dict):
|
||||
normalized.append({
|
||||
"id": c.get("id"),
|
||||
"team_id": c.get("teamId", c.get("team_id")),
|
||||
"url": c.get("url"),
|
||||
"options": c.get("options"),
|
||||
})
|
||||
return ActiveCrawlsResponse(success=True, crawls=[ActiveCrawl(**nc) for nc in normalized])
|
||||
@@ -0,0 +1,164 @@
|
||||
from typing import Any, Dict, List, Optional
|
||||
import asyncio
|
||||
import warnings
|
||||
|
||||
from ...types import ExtractResponse, ScrapeOptions
|
||||
from ...utils.http_client_async import AsyncHttpClient
|
||||
from ...utils.validation import prepare_scrape_options
|
||||
|
||||
_EXTRACT_DEPRECATION_MSG = (
|
||||
"The extract endpoint is in maintenance mode and its use is discouraged. "
|
||||
"Review https://docs.firecrawl.dev/developer-guides/usage-guides/choosing-the-data-extractor "
|
||||
"to find a replacement."
|
||||
)
|
||||
|
||||
|
||||
def _prepare_extract_request(
|
||||
urls: Optional[List[str]],
|
||||
*,
|
||||
prompt: Optional[str] = None,
|
||||
schema: Optional[Dict[str, Any]] = None,
|
||||
system_prompt: Optional[str] = None,
|
||||
allow_external_links: Optional[bool] = None,
|
||||
enable_web_search: Optional[bool] = None,
|
||||
show_sources: Optional[bool] = None,
|
||||
scrape_options: Optional[ScrapeOptions] = None,
|
||||
ignore_invalid_urls: Optional[bool] = None,
|
||||
integration: Optional[str] = None,
|
||||
) -> Dict[str, Any]:
|
||||
body: Dict[str, Any] = {}
|
||||
if urls is not None:
|
||||
body["urls"] = urls
|
||||
if prompt is not None:
|
||||
body["prompt"] = prompt
|
||||
if schema is not None:
|
||||
body["schema"] = schema
|
||||
if system_prompt is not None:
|
||||
body["systemPrompt"] = system_prompt
|
||||
if allow_external_links is not None:
|
||||
body["allowExternalLinks"] = allow_external_links
|
||||
if enable_web_search is not None:
|
||||
body["enableWebSearch"] = enable_web_search
|
||||
if show_sources is not None:
|
||||
body["showSources"] = show_sources
|
||||
if ignore_invalid_urls is not None:
|
||||
body["ignoreInvalidURLs"] = ignore_invalid_urls
|
||||
if scrape_options is not None:
|
||||
prepared = prepare_scrape_options(scrape_options)
|
||||
if prepared:
|
||||
body["scrapeOptions"] = prepared
|
||||
if integration is not None and str(integration).strip():
|
||||
body["integration"] = str(integration).strip()
|
||||
return body
|
||||
|
||||
|
||||
async def start_extract(
|
||||
client: AsyncHttpClient,
|
||||
urls: Optional[List[str]],
|
||||
*,
|
||||
prompt: Optional[str] = None,
|
||||
schema: Optional[Dict[str, Any]] = None,
|
||||
system_prompt: Optional[str] = None,
|
||||
allow_external_links: Optional[bool] = None,
|
||||
enable_web_search: Optional[bool] = None,
|
||||
show_sources: Optional[bool] = None,
|
||||
scrape_options: Optional[ScrapeOptions] = None,
|
||||
ignore_invalid_urls: Optional[bool] = None,
|
||||
integration: Optional[str] = None,
|
||||
) -> ExtractResponse:
|
||||
"""Start an extract job (non-blocking, async).
|
||||
|
||||
.. deprecated::
|
||||
The extract endpoint is in maintenance mode and its use is discouraged.
|
||||
Review https://docs.firecrawl.dev/developer-guides/usage-guides/choosing-the-data-extractor
|
||||
to find a replacement.
|
||||
"""
|
||||
warnings.warn(_EXTRACT_DEPRECATION_MSG, DeprecationWarning, stacklevel=2)
|
||||
body = _prepare_extract_request(
|
||||
urls,
|
||||
prompt=prompt,
|
||||
schema=schema,
|
||||
system_prompt=system_prompt,
|
||||
allow_external_links=allow_external_links,
|
||||
enable_web_search=enable_web_search,
|
||||
show_sources=show_sources,
|
||||
scrape_options=scrape_options,
|
||||
ignore_invalid_urls=ignore_invalid_urls,
|
||||
integration=integration,
|
||||
)
|
||||
resp = await client.post("/v2/extract", body)
|
||||
return ExtractResponse(**resp.json())
|
||||
|
||||
|
||||
async def get_extract_status(client: AsyncHttpClient, job_id: str) -> ExtractResponse:
|
||||
"""Get the current status of an extract job (async).
|
||||
|
||||
.. deprecated::
|
||||
The extract endpoint is in maintenance mode and its use is discouraged.
|
||||
Review https://docs.firecrawl.dev/developer-guides/usage-guides/choosing-the-data-extractor
|
||||
to find a replacement.
|
||||
"""
|
||||
warnings.warn(_EXTRACT_DEPRECATION_MSG, DeprecationWarning, stacklevel=2)
|
||||
resp = await client.get(f"/v2/extract/{job_id}")
|
||||
return ExtractResponse(**resp.json())
|
||||
|
||||
|
||||
async def wait_extract(
|
||||
client: AsyncHttpClient,
|
||||
job_id: str,
|
||||
*,
|
||||
poll_interval: int = 2,
|
||||
timeout: Optional[int] = None,
|
||||
) -> ExtractResponse:
|
||||
start_ts = asyncio.get_event_loop().time()
|
||||
while True:
|
||||
status = await get_extract_status(client, job_id)
|
||||
if status.status in ("completed", "failed", "cancelled"):
|
||||
return status
|
||||
if timeout is not None and (asyncio.get_event_loop().time() - start_ts) > timeout:
|
||||
return status
|
||||
await asyncio.sleep(max(1, poll_interval))
|
||||
|
||||
|
||||
async def extract(
|
||||
client: AsyncHttpClient,
|
||||
urls: Optional[List[str]],
|
||||
*,
|
||||
prompt: Optional[str] = None,
|
||||
schema: Optional[Dict[str, Any]] = None,
|
||||
system_prompt: Optional[str] = None,
|
||||
allow_external_links: Optional[bool] = None,
|
||||
enable_web_search: Optional[bool] = None,
|
||||
show_sources: Optional[bool] = None,
|
||||
scrape_options: Optional[ScrapeOptions] = None,
|
||||
ignore_invalid_urls: Optional[bool] = None,
|
||||
poll_interval: int = 2,
|
||||
timeout: Optional[int] = None,
|
||||
integration: Optional[str] = None,
|
||||
) -> ExtractResponse:
|
||||
"""Extract structured data and wait until completion (async).
|
||||
|
||||
.. deprecated::
|
||||
The extract endpoint is in maintenance mode and its use is discouraged.
|
||||
Review https://docs.firecrawl.dev/developer-guides/usage-guides/choosing-the-data-extractor
|
||||
to find a replacement.
|
||||
"""
|
||||
warnings.warn(_EXTRACT_DEPRECATION_MSG, DeprecationWarning, stacklevel=2)
|
||||
started = await start_extract(
|
||||
client,
|
||||
urls,
|
||||
prompt=prompt,
|
||||
schema=schema,
|
||||
system_prompt=system_prompt,
|
||||
allow_external_links=allow_external_links,
|
||||
enable_web_search=enable_web_search,
|
||||
show_sources=show_sources,
|
||||
scrape_options=scrape_options,
|
||||
ignore_invalid_urls=ignore_invalid_urls,
|
||||
integration=integration,
|
||||
)
|
||||
job_id = getattr(started, "id", None)
|
||||
if not job_id:
|
||||
return started
|
||||
return await wait_extract(client, job_id, poll_interval=poll_interval, timeout=timeout)
|
||||
|
||||
@@ -0,0 +1,67 @@
|
||||
from typing import Optional, Dict, Any
|
||||
from ...types import MapOptions, MapData, LinkResult
|
||||
from ...utils.http_client_async import AsyncHttpClient
|
||||
from ...utils.error_handler import handle_response_error
|
||||
|
||||
|
||||
def _prepare_map_request(url: str, options: Optional[MapOptions] = None) -> Dict[str, Any]:
|
||||
if not url or not url.strip():
|
||||
raise ValueError("URL cannot be empty")
|
||||
payload: Dict[str, Any] = {"url": url.strip()}
|
||||
if options is not None:
|
||||
data: Dict[str, Any] = {}
|
||||
if getattr(options, "sitemap", None) is not None:
|
||||
data["sitemap"] = options.sitemap
|
||||
if options.search is not None:
|
||||
data["search"] = options.search
|
||||
if options.include_subdomains is not None:
|
||||
data["includeSubdomains"] = options.include_subdomains
|
||||
if options.ignore_query_parameters is not None:
|
||||
data["ignoreQueryParameters"] = options.ignore_query_parameters
|
||||
if options.limit is not None:
|
||||
if options.limit <= 0:
|
||||
raise ValueError("Limit must be positive")
|
||||
data["limit"] = options.limit
|
||||
if options.timeout is not None:
|
||||
data["timeout"] = options.timeout
|
||||
if options.integration is not None:
|
||||
data["integration"] = options.integration.strip()
|
||||
if options.location is not None:
|
||||
data["location"] = options.location.model_dump(exclude_none=True)
|
||||
payload.update(data)
|
||||
return payload
|
||||
|
||||
|
||||
async def map(client: AsyncHttpClient, url: str, options: Optional[MapOptions] = None) -> MapData:
|
||||
request_data = _prepare_map_request(url, options)
|
||||
response = await client.post("/v2/map", request_data)
|
||||
if response.status_code >= 400:
|
||||
handle_response_error(response, "map")
|
||||
body = response.json()
|
||||
if not body.get("success"):
|
||||
raise Exception(body.get("error", "Unknown error occurred"))
|
||||
|
||||
|
||||
# data = body.get("data", {})
|
||||
# result_links: list[LinkResult] = []
|
||||
# for item in data.get("links", []):
|
||||
# if isinstance(item, dict):
|
||||
# result_links.append(
|
||||
# LinkResult(
|
||||
# url=item.get("url", ""),
|
||||
# title=item.get("title"),
|
||||
# description=item.get("description"),
|
||||
# )
|
||||
# )
|
||||
# elif isinstance(item, str):
|
||||
# result_links.append(LinkResult(url=item))
|
||||
|
||||
result_links: list[LinkResult] = []
|
||||
for item in body.get("links", []):
|
||||
if isinstance(item, dict):
|
||||
result_links.append(LinkResult(url=item.get("url", ""), title=item.get("title"), description=item.get("description")))
|
||||
elif isinstance(item, str):
|
||||
result_links.append(LinkResult(url=item))
|
||||
|
||||
return MapData(links=result_links)
|
||||
|
||||
@@ -0,0 +1,215 @@
|
||||
import time
|
||||
from typing import Any, Dict, List, Optional
|
||||
from pydantic import BaseModel
|
||||
|
||||
from ...types import (
|
||||
Monitor,
|
||||
MonitorCheck,
|
||||
MonitorCheckDetail,
|
||||
MonitorCheckPage,
|
||||
MonitorCreateRequest,
|
||||
PaginationConfig,
|
||||
MonitorTarget,
|
||||
MonitorUpdateRequest,
|
||||
ScrapeOptions,
|
||||
)
|
||||
from ...utils.error_handler import handle_response_error
|
||||
from ...utils.http_client_async import AsyncHttpClient
|
||||
from ...utils.validation import prepare_scrape_options
|
||||
|
||||
|
||||
def _dump(value: Any) -> Any:
|
||||
if isinstance(value, ScrapeOptions):
|
||||
return prepare_scrape_options(value)
|
||||
if isinstance(value, MonitorTarget):
|
||||
data = value.model_dump(exclude_none=True, by_alias=True)
|
||||
if isinstance(value.scrape_options, ScrapeOptions):
|
||||
data["scrapeOptions"] = prepare_scrape_options(value.scrape_options)
|
||||
return _prepare_target(data)
|
||||
if isinstance(value, BaseModel):
|
||||
return value.model_dump(exclude_none=True, by_alias=True)
|
||||
if isinstance(value, list):
|
||||
return [_dump(item) for item in value]
|
||||
if isinstance(value, dict):
|
||||
return {key: _dump(item) for key, item in value.items() if item is not None}
|
||||
return value
|
||||
|
||||
|
||||
def _prepare_target(target: Dict[str, Any]) -> Dict[str, Any]:
|
||||
prepared = dict(target)
|
||||
if "scrapeOptions" in prepared and isinstance(prepared["scrapeOptions"], ScrapeOptions):
|
||||
prepared["scrapeOptions"] = prepare_scrape_options(prepared["scrapeOptions"])
|
||||
if "crawlOptions" in prepared:
|
||||
prepared["crawlOptions"] = _dump(prepared["crawlOptions"])
|
||||
return prepared
|
||||
|
||||
|
||||
def _prepare_payload(request: Any) -> Dict[str, Any]:
|
||||
payload = _dump(request)
|
||||
if not isinstance(payload, dict):
|
||||
raise ValueError("Monitor request must be an object")
|
||||
if "targets" in payload:
|
||||
payload["targets"] = [
|
||||
_prepare_target(_dump(target))
|
||||
for target in payload.get("targets", [])
|
||||
]
|
||||
return payload
|
||||
|
||||
|
||||
async def _data_or_error(response, action: str) -> Any:
|
||||
if response.status_code >= 400:
|
||||
handle_response_error(response, action)
|
||||
body = response.json()
|
||||
if not body.get("success"):
|
||||
raise Exception(body.get("error", "Unknown error occurred"))
|
||||
return body.get("data")
|
||||
|
||||
|
||||
async def _monitor_check_data_or_error(response, action: str) -> Dict[str, Any]:
|
||||
if response.status_code >= 400:
|
||||
handle_response_error(response, action)
|
||||
body = response.json()
|
||||
if not body.get("success"):
|
||||
raise Exception(body.get("error", "Unknown error occurred"))
|
||||
data = body.get("data") or {}
|
||||
if body.get("next") is not None:
|
||||
data["next"] = body.get("next")
|
||||
return data
|
||||
|
||||
|
||||
async def _fetch_all_monitor_check_pages(
|
||||
client: AsyncHttpClient,
|
||||
next_url: str,
|
||||
initial_pages: List[MonitorCheckPage],
|
||||
pagination_config: Optional[PaginationConfig] = None,
|
||||
) -> List[MonitorCheckPage]:
|
||||
pages = initial_pages.copy()
|
||||
current_url = next_url
|
||||
page_count = 0
|
||||
max_pages = pagination_config.max_pages if pagination_config else None
|
||||
max_results = pagination_config.max_results if pagination_config else None
|
||||
max_wait_time = pagination_config.max_wait_time if pagination_config else None
|
||||
start_time = time.monotonic()
|
||||
|
||||
while current_url:
|
||||
if max_pages is not None and page_count >= max_pages:
|
||||
break
|
||||
if max_wait_time is not None and (time.monotonic() - start_time) > max_wait_time:
|
||||
break
|
||||
|
||||
response = await client.get(current_url)
|
||||
if response.status_code >= 400:
|
||||
break
|
||||
try:
|
||||
data = await _monitor_check_data_or_error(response, "get monitor check page")
|
||||
except Exception:
|
||||
break
|
||||
|
||||
for page in data.get("pages") or []:
|
||||
if max_results is not None and len(pages) >= max_results:
|
||||
break
|
||||
pages.append(MonitorCheckPage(**page))
|
||||
|
||||
if max_results is not None and len(pages) >= max_results:
|
||||
break
|
||||
|
||||
current_url = data.get("next")
|
||||
page_count += 1
|
||||
|
||||
return pages
|
||||
|
||||
|
||||
async def create_monitor(client: AsyncHttpClient, request: MonitorCreateRequest) -> Monitor:
|
||||
data = await _data_or_error(await client.post("/v2/monitor", _prepare_payload(request)), "create monitor")
|
||||
return Monitor(**data)
|
||||
|
||||
|
||||
async def list_monitors(client: AsyncHttpClient, *, limit: Optional[int] = None, offset: Optional[int] = None) -> List[Monitor]:
|
||||
params = []
|
||||
if limit is not None:
|
||||
params.append(f"limit={limit}")
|
||||
if offset is not None:
|
||||
params.append(f"offset={offset}")
|
||||
suffix = f"?{'&'.join(params)}" if params else ""
|
||||
data = await _data_or_error(await client.get(f"/v2/monitor{suffix}"), "list monitors")
|
||||
return [Monitor(**item) for item in data or []]
|
||||
|
||||
|
||||
async def get_monitor(client: AsyncHttpClient, monitor_id: str) -> Monitor:
|
||||
data = await _data_or_error(await client.get(f"/v2/monitor/{monitor_id}"), "get monitor")
|
||||
return Monitor(**data)
|
||||
|
||||
|
||||
async def update_monitor(client: AsyncHttpClient, monitor_id: str, request: MonitorUpdateRequest) -> Monitor:
|
||||
data = await _data_or_error(await client.patch(f"/v2/monitor/{monitor_id}", _prepare_payload(request)), "update monitor")
|
||||
return Monitor(**data)
|
||||
|
||||
|
||||
async def delete_monitor(client: AsyncHttpClient, monitor_id: str) -> bool:
|
||||
response = await client.delete(f"/v2/monitor/{monitor_id}")
|
||||
if response.status_code >= 400:
|
||||
handle_response_error(response, "delete monitor")
|
||||
body = response.json()
|
||||
if not body.get("success"):
|
||||
raise Exception(body.get("error", "Unknown error occurred"))
|
||||
return True
|
||||
|
||||
|
||||
async def run_monitor(client: AsyncHttpClient, monitor_id: str) -> MonitorCheck:
|
||||
data = await _data_or_error(await client.post(f"/v2/monitor/{monitor_id}/run", {}), "run monitor")
|
||||
return MonitorCheck(**data)
|
||||
|
||||
|
||||
async def list_monitor_checks(
|
||||
client: AsyncHttpClient,
|
||||
monitor_id: str,
|
||||
*,
|
||||
limit: Optional[int] = None,
|
||||
offset: Optional[int] = None,
|
||||
) -> List[MonitorCheck]:
|
||||
params = []
|
||||
if limit is not None:
|
||||
params.append(f"limit={limit}")
|
||||
if offset is not None:
|
||||
params.append(f"offset={offset}")
|
||||
suffix = f"?{'&'.join(params)}" if params else ""
|
||||
data = await _data_or_error(await client.get(f"/v2/monitor/{monitor_id}/checks{suffix}"), "list monitor checks")
|
||||
return [MonitorCheck(**item) for item in data or []]
|
||||
|
||||
|
||||
async def get_monitor_check(
|
||||
client: AsyncHttpClient,
|
||||
monitor_id: str,
|
||||
check_id: str,
|
||||
*,
|
||||
limit: Optional[int] = None,
|
||||
skip: Optional[int] = None,
|
||||
status: Optional[str] = None,
|
||||
pagination_config: Optional[PaginationConfig] = None,
|
||||
) -> MonitorCheckDetail:
|
||||
params = []
|
||||
if limit is not None:
|
||||
params.append(f"limit={limit}")
|
||||
if skip is not None:
|
||||
params.append(f"skip={skip}")
|
||||
if status is not None:
|
||||
params.append(f"status={status}")
|
||||
suffix = f"?{'&'.join(params)}" if params else ""
|
||||
data = await _monitor_check_data_or_error(await client.get(f"/v2/monitor/{monitor_id}/checks/{check_id}{suffix}"), "get monitor check")
|
||||
detail = MonitorCheckDetail(**data)
|
||||
|
||||
auto_paginate = pagination_config.auto_paginate if pagination_config else True
|
||||
if auto_paginate and detail.next and not (
|
||||
pagination_config
|
||||
and pagination_config.max_results is not None
|
||||
and len(detail.pages) >= pagination_config.max_results
|
||||
):
|
||||
detail.pages = await _fetch_all_monitor_check_pages(
|
||||
client,
|
||||
detail.next,
|
||||
detail.pages,
|
||||
pagination_config,
|
||||
)
|
||||
detail.next = None
|
||||
|
||||
return detail
|
||||
@@ -0,0 +1,63 @@
|
||||
import asyncio
|
||||
from functools import partial
|
||||
import json
|
||||
from typing import Optional, Dict, Any, Tuple
|
||||
|
||||
from ...types import Document, ParseOptions
|
||||
from ...utils.normalize import normalize_document_input
|
||||
from ...utils.error_handler import handle_response_error
|
||||
from ...utils.http_client_async import AsyncHttpClient
|
||||
from ..parse import (
|
||||
ParseFileInput,
|
||||
_prepare_file_payload,
|
||||
_prepare_parse_options_payload,
|
||||
)
|
||||
|
||||
async def _prepare_parse_request(
|
||||
file: ParseFileInput,
|
||||
options: Optional[ParseOptions] = None,
|
||||
*,
|
||||
filename: Optional[str] = None,
|
||||
content_type: Optional[str] = None,
|
||||
) -> Tuple[Dict[str, Any], Dict[str, Tuple[str, bytes, str]]]:
|
||||
request_data = _prepare_parse_options_payload(options)
|
||||
multipart_fields = {"options": json.dumps(request_data)}
|
||||
loop = asyncio.get_running_loop()
|
||||
multipart_files = await loop.run_in_executor(
|
||||
None,
|
||||
partial(
|
||||
_prepare_file_payload,
|
||||
file,
|
||||
filename=filename,
|
||||
content_type=content_type,
|
||||
),
|
||||
)
|
||||
return multipart_fields, multipart_files
|
||||
|
||||
|
||||
async def parse(
|
||||
client: AsyncHttpClient,
|
||||
file: ParseFileInput,
|
||||
options: Optional[ParseOptions] = None,
|
||||
*,
|
||||
filename: Optional[str] = None,
|
||||
content_type: Optional[str] = None,
|
||||
) -> Document:
|
||||
fields, files = await _prepare_parse_request(
|
||||
file,
|
||||
options,
|
||||
filename=filename,
|
||||
content_type=content_type,
|
||||
)
|
||||
|
||||
response = await client.post_multipart("/v2/parse", data=fields, files=files)
|
||||
if response.status_code >= 400:
|
||||
handle_response_error(response, "parse")
|
||||
|
||||
body = response.json()
|
||||
if not body.get("success"):
|
||||
raise Exception(body.get("error", "Unknown error occurred"))
|
||||
|
||||
document_data = body.get("data", {})
|
||||
normalized = normalize_document_input(document_data)
|
||||
return Document(**normalized)
|
||||
@@ -0,0 +1,144 @@
|
||||
from typing import Optional, Dict, Any, Literal
|
||||
from ...types import (
|
||||
ScrapeOptions,
|
||||
Document,
|
||||
BrowserExecuteResponse,
|
||||
BrowserDeleteResponse,
|
||||
)
|
||||
from ...utils.normalize import normalize_document_input
|
||||
from ...utils.error_handler import handle_response_error
|
||||
from ...utils.validation import prepare_scrape_options, validate_scrape_options
|
||||
from ...utils.http_client_async import AsyncHttpClient
|
||||
|
||||
|
||||
async def _prepare_scrape_request(url: str, options: Optional[ScrapeOptions] = None) -> Dict[str, Any]:
|
||||
if not url or not url.strip():
|
||||
raise ValueError("URL cannot be empty")
|
||||
payload: Dict[str, Any] = {"url": url.strip()}
|
||||
if options is not None:
|
||||
validated = validate_scrape_options(options)
|
||||
if validated is not None:
|
||||
opts = prepare_scrape_options(validated)
|
||||
if opts:
|
||||
payload.update(opts)
|
||||
return payload
|
||||
|
||||
|
||||
async def scrape(client: AsyncHttpClient, url: str, options: Optional[ScrapeOptions] = None) -> Document:
|
||||
payload = await _prepare_scrape_request(url, options)
|
||||
response = await client.post("/v2/scrape", payload)
|
||||
if response.status_code >= 400:
|
||||
handle_response_error(response, "scrape")
|
||||
body = response.json()
|
||||
if not body.get("success"):
|
||||
raise Exception(body.get("error", "Unknown error occurred"))
|
||||
document_data = body.get("data", {})
|
||||
normalized = normalize_document_input(document_data)
|
||||
return Document(**normalized)
|
||||
|
||||
|
||||
async def interact(
|
||||
client: AsyncHttpClient,
|
||||
job_id: str,
|
||||
code: Optional[str] = None,
|
||||
*,
|
||||
prompt: Optional[str] = None,
|
||||
language: Literal["python", "node", "bash"] = "node",
|
||||
timeout: Optional[int] = None,
|
||||
origin: Optional[str] = None,
|
||||
) -> BrowserExecuteResponse:
|
||||
if not job_id or not job_id.strip():
|
||||
raise ValueError("Job ID cannot be empty")
|
||||
has_code = code and code.strip()
|
||||
has_prompt = prompt and prompt.strip()
|
||||
if not has_code and not has_prompt:
|
||||
raise ValueError("Either 'code' or 'prompt' must be provided")
|
||||
|
||||
payload: Dict[str, Any] = {
|
||||
"language": language,
|
||||
}
|
||||
if has_code:
|
||||
payload["code"] = code
|
||||
if has_prompt:
|
||||
payload["prompt"] = prompt
|
||||
if timeout is not None:
|
||||
payload["timeout"] = timeout
|
||||
if origin is not None:
|
||||
payload["origin"] = origin
|
||||
|
||||
response = await client.post(f"/v2/scrape/{job_id}/interact", payload)
|
||||
if response.status_code >= 400:
|
||||
handle_response_error(response, "interact with scrape browser")
|
||||
|
||||
body = response.json()
|
||||
if not body.get("success"):
|
||||
raise Exception(body.get("error", "Unknown error occurred"))
|
||||
|
||||
normalized = dict(body)
|
||||
if "exitCode" in normalized and "exit_code" not in normalized:
|
||||
normalized["exit_code"] = normalized["exitCode"]
|
||||
if "liveViewUrl" in normalized and "live_view_url" not in normalized:
|
||||
normalized["live_view_url"] = normalized["liveViewUrl"]
|
||||
if "interactiveLiveViewUrl" in normalized and "interactive_live_view_url" not in normalized:
|
||||
normalized["interactive_live_view_url"] = normalized["interactiveLiveViewUrl"]
|
||||
return BrowserExecuteResponse(**normalized)
|
||||
|
||||
|
||||
async def stop_interaction(
|
||||
client: AsyncHttpClient,
|
||||
job_id: str,
|
||||
) -> BrowserDeleteResponse:
|
||||
if not job_id or not job_id.strip():
|
||||
raise ValueError("Job ID cannot be empty")
|
||||
|
||||
response = await client.delete(f"/v2/scrape/{job_id}/interact")
|
||||
if response.status_code >= 400:
|
||||
handle_response_error(response, "stop interaction")
|
||||
|
||||
body = response.json()
|
||||
normalized = dict(body)
|
||||
if "sessionDurationMs" in normalized and "session_duration_ms" not in normalized:
|
||||
normalized["session_duration_ms"] = normalized["sessionDurationMs"]
|
||||
if "creditsBilled" in normalized and "credits_billed" not in normalized:
|
||||
normalized["credits_billed"] = normalized["creditsBilled"]
|
||||
|
||||
return BrowserDeleteResponse(**normalized)
|
||||
|
||||
|
||||
async def stop_interactive_browser(
|
||||
client: AsyncHttpClient,
|
||||
job_id: str,
|
||||
) -> BrowserDeleteResponse:
|
||||
"""Deprecated alias for stop_interaction()."""
|
||||
return await stop_interaction(client, job_id)
|
||||
|
||||
|
||||
async def scrape_execute(
|
||||
client: AsyncHttpClient,
|
||||
job_id: str,
|
||||
code: Optional[str] = None,
|
||||
*,
|
||||
prompt: Optional[str] = None,
|
||||
language: Literal["python", "node", "bash"] = "node",
|
||||
timeout: Optional[int] = None,
|
||||
origin: Optional[str] = None,
|
||||
) -> BrowserExecuteResponse:
|
||||
"""Deprecated alias for interact()."""
|
||||
return await interact(
|
||||
client,
|
||||
job_id,
|
||||
code,
|
||||
prompt=prompt,
|
||||
language=language,
|
||||
timeout=timeout,
|
||||
origin=origin,
|
||||
)
|
||||
|
||||
|
||||
async def delete_scrape_browser(
|
||||
client: AsyncHttpClient,
|
||||
job_id: str,
|
||||
) -> BrowserDeleteResponse:
|
||||
"""Deprecated alias for stop_interaction()."""
|
||||
return await stop_interaction(client, job_id)
|
||||
|
||||
@@ -0,0 +1,178 @@
|
||||
from typing import Dict, Any, Union, List, TypeVar, Type
|
||||
from ...types import (
|
||||
SearchRequest,
|
||||
SearchData,
|
||||
Document,
|
||||
SearchResultWeb,
|
||||
SearchResultNews,
|
||||
SearchResultImages,
|
||||
)
|
||||
from ...utils.http_client_async import AsyncHttpClient
|
||||
from ...utils.error_handler import handle_response_error
|
||||
from ...utils.normalize import normalize_document_input
|
||||
from ...utils.validation import validate_scrape_options, prepare_scrape_options
|
||||
|
||||
T = TypeVar("T")
|
||||
|
||||
async def search(
|
||||
client: AsyncHttpClient,
|
||||
request: SearchRequest
|
||||
) -> SearchData:
|
||||
"""
|
||||
Async search for documents.
|
||||
|
||||
Args:
|
||||
client: Async HTTP client instance
|
||||
request: Search request
|
||||
|
||||
Returns:
|
||||
SearchData with search results grouped by source type
|
||||
|
||||
Raises:
|
||||
FirecrawlError: If the search operation fails
|
||||
"""
|
||||
request_data = _prepare_search_request(request)
|
||||
try:
|
||||
response = await client.post("/v2/search", request_data)
|
||||
if response.status_code != 200:
|
||||
handle_response_error(response, "search")
|
||||
response_data = response.json()
|
||||
if not response_data.get("success"):
|
||||
handle_response_error(response, "search")
|
||||
data = response_data.get("data", {}) or {}
|
||||
out = SearchData()
|
||||
if "web" in data:
|
||||
out.web = _transform_array(data["web"], SearchResultWeb)
|
||||
if "news" in data:
|
||||
out.news = _transform_array(data["news"], SearchResultNews)
|
||||
if "images" in data:
|
||||
out.images = _transform_array(data["images"], SearchResultImages)
|
||||
return out
|
||||
except Exception as err:
|
||||
if hasattr(err, "response"):
|
||||
handle_response_error(getattr(err, "response"), "search")
|
||||
raise err
|
||||
|
||||
def _transform_array(arr: List[Any], result_type: Type[T]) -> List[Union[T, Document]]:
|
||||
"""
|
||||
Transforms an array of items into a list of result_type or Document.
|
||||
If the item dict contains any of the special keys, it is treated as a Document.
|
||||
Otherwise, it is treated as result_type.
|
||||
If the item is not a dict, it is wrapped as result_type with url=item.
|
||||
"""
|
||||
results: List[Union[T, Document]] = []
|
||||
for item in arr:
|
||||
if item and isinstance(item, dict):
|
||||
if (
|
||||
"markdown" in item or
|
||||
"html" in item or
|
||||
"rawHtml" in item or
|
||||
"links" in item or
|
||||
"screenshot" in item or
|
||||
"changeTracking" in item or
|
||||
"summary" in item or
|
||||
"json" in item
|
||||
):
|
||||
results.append(Document(**normalize_document_input(item)))
|
||||
else:
|
||||
results.append(result_type(**item))
|
||||
else:
|
||||
results.append(result_type(url=item))
|
||||
return results
|
||||
|
||||
def _validate_search_request(request: SearchRequest) -> SearchRequest:
|
||||
"""
|
||||
Validate and normalize search request.
|
||||
|
||||
Args:
|
||||
request: Search request to validate
|
||||
|
||||
Returns:
|
||||
Validated request
|
||||
|
||||
Raises:
|
||||
ValueError: If request is invalid
|
||||
"""
|
||||
if not request.query or not request.query.strip():
|
||||
raise ValueError("Query cannot be empty")
|
||||
|
||||
if request.limit is not None:
|
||||
if request.limit <= 0:
|
||||
raise ValueError("Limit must be positive")
|
||||
if request.limit > 100:
|
||||
raise ValueError("Limit cannot exceed 100")
|
||||
|
||||
if request.timeout is not None:
|
||||
if request.timeout <= 0:
|
||||
raise ValueError("Timeout must be positive")
|
||||
if request.timeout > 300000:
|
||||
raise ValueError("Timeout cannot exceed 300000ms (5 minutes)")
|
||||
|
||||
if request.sources is not None:
|
||||
valid_sources = {"web", "news", "images"}
|
||||
for source in request.sources:
|
||||
if isinstance(source, str):
|
||||
if source not in valid_sources:
|
||||
raise ValueError(f"Invalid source type: {source}. Valid types: {valid_sources}")
|
||||
elif hasattr(source, 'type'):
|
||||
if source.type not in valid_sources:
|
||||
raise ValueError(f"Invalid source type: {source.type}. Valid types: {valid_sources}")
|
||||
|
||||
if request.include_domains and request.exclude_domains:
|
||||
raise ValueError(
|
||||
"include_domains and exclude_domains cannot both be specified"
|
||||
)
|
||||
|
||||
if request.location is not None:
|
||||
if not isinstance(request.location, str) or len(request.location.strip()) == 0:
|
||||
raise ValueError("Location must be a non-empty string")
|
||||
|
||||
if request.tbs is not None:
|
||||
if not isinstance(request.tbs, str) or len(request.tbs.strip()) == 0:
|
||||
raise ValueError("tbs must be a non-empty string")
|
||||
|
||||
if request.scrape_options is not None:
|
||||
validate_scrape_options(request.scrape_options)
|
||||
|
||||
return request
|
||||
|
||||
def _prepare_search_request(request: SearchRequest) -> Dict[str, Any]:
|
||||
"""
|
||||
Prepare a search request payload.
|
||||
|
||||
Args:
|
||||
request: Search request
|
||||
|
||||
Returns:
|
||||
Request payload dictionary
|
||||
"""
|
||||
validated_request = _validate_search_request(request)
|
||||
data = validated_request.model_dump(exclude_none=True, by_alias=True)
|
||||
|
||||
if "limit" not in data and validated_request.limit is not None:
|
||||
data["limit"] = validated_request.limit
|
||||
if "timeout" not in data and validated_request.timeout is not None:
|
||||
data["timeout"] = validated_request.timeout
|
||||
|
||||
if validated_request.ignore_invalid_urls is not None:
|
||||
data["ignoreInvalidURLs"] = validated_request.ignore_invalid_urls
|
||||
data.pop("ignore_invalid_urls", None)
|
||||
|
||||
if validated_request.include_domains is not None:
|
||||
data["includeDomains"] = validated_request.include_domains
|
||||
data.pop("include_domains", None)
|
||||
|
||||
if validated_request.exclude_domains is not None:
|
||||
data["excludeDomains"] = validated_request.exclude_domains
|
||||
data.pop("exclude_domains", None)
|
||||
|
||||
if validated_request.scrape_options is not None:
|
||||
scrape_data = prepare_scrape_options(validated_request.scrape_options)
|
||||
if scrape_data:
|
||||
data["scrapeOptions"] = scrape_data
|
||||
data.pop("scrape_options", None)
|
||||
|
||||
if (v := getattr(validated_request, "integration", None)) is not None and str(v).strip():
|
||||
data["integration"] = str(validated_request.integration).strip()
|
||||
|
||||
return data
|
||||
@@ -0,0 +1,89 @@
|
||||
from ...utils.http_client_async import AsyncHttpClient
|
||||
from ...utils.error_handler import handle_response_error
|
||||
from ...types import ConcurrencyCheck, CreditUsage, TokenUsage, CreditUsageHistoricalResponse, TokenUsageHistoricalResponse, QueueStatusResponse
|
||||
|
||||
|
||||
async def get_concurrency(client: AsyncHttpClient) -> ConcurrencyCheck:
|
||||
resp = await client.get("/v2/concurrency-check")
|
||||
if resp.status_code >= 400:
|
||||
handle_response_error(resp, "get concurrency")
|
||||
body = resp.json()
|
||||
if not body.get("success"):
|
||||
raise Exception(body.get("error", "Unknown error"))
|
||||
data = body.get("data", body)
|
||||
return ConcurrencyCheck(
|
||||
concurrency=data.get("concurrency"),
|
||||
max_concurrency=data.get("maxConcurrency", data.get("max_concurrency")),
|
||||
)
|
||||
|
||||
|
||||
async def get_credit_usage(client: AsyncHttpClient) -> CreditUsage:
|
||||
resp = await client.get("/v2/team/credit-usage")
|
||||
if resp.status_code >= 400:
|
||||
handle_response_error(resp, "get credit usage")
|
||||
body = resp.json()
|
||||
if not body.get("success"):
|
||||
raise Exception(body.get("error", "Unknown error"))
|
||||
data = body.get("data", body)
|
||||
return CreditUsage(
|
||||
remaining_credits=data.get("remainingCredits", data.get("remaining_credits", 0)),
|
||||
plan_credits=data.get("planCredits", data.get("plan_credits")),
|
||||
billing_period_start=data.get("billingPeriodStart", data.get("billing_period_start")),
|
||||
billing_period_end=data.get("billingPeriodEnd", data.get("billing_period_end")),
|
||||
)
|
||||
|
||||
|
||||
async def get_token_usage(client: AsyncHttpClient) -> TokenUsage:
|
||||
resp = await client.get("/v2/team/token-usage")
|
||||
if resp.status_code >= 400:
|
||||
handle_response_error(resp, "get token usage")
|
||||
body = resp.json()
|
||||
if not body.get("success"):
|
||||
raise Exception(body.get("error", "Unknown error"))
|
||||
data = body.get("data", body)
|
||||
return TokenUsage(
|
||||
remaining_tokens=data.get("remainingTokens", data.get("remaining_tokens", 0)),
|
||||
plan_tokens=data.get("planTokens", data.get("plan_tokens")),
|
||||
billing_period_start=data.get("billingPeriodStart", data.get("billing_period_start")),
|
||||
billing_period_end=data.get("billingPeriodEnd", data.get("billing_period_end")),
|
||||
)
|
||||
|
||||
|
||||
async def get_queue_status(client: AsyncHttpClient) -> QueueStatusResponse:
|
||||
resp = await client.get("/v2/team/queue-status")
|
||||
if resp.status_code >= 400:
|
||||
handle_response_error(resp, "get queue status")
|
||||
body = resp.json()
|
||||
if not body.get("success"):
|
||||
raise Exception(body.get("error", "Unknown error"))
|
||||
data = body.get("data", body)
|
||||
return QueueStatusResponse(
|
||||
jobs_in_queue=data.get("jobsInQueue", 0),
|
||||
active_jobs_in_queue=data.get("activeJobsInQueue", 0),
|
||||
waiting_jobs_in_queue=data.get("waitingJobsInQueue", 0),
|
||||
max_concurrency=data.get("maxConcurrency", 0),
|
||||
most_recent_success=data.get("mostRecentSuccess", None),
|
||||
)
|
||||
|
||||
|
||||
async def get_credit_usage_historical(client: AsyncHttpClient, by_api_key: bool = False) -> CreditUsageHistoricalResponse:
|
||||
query = "?byApiKey=true" if by_api_key else ""
|
||||
resp = await client.get(f"/v2/team/credit-usage/historical{query}")
|
||||
if resp.status_code >= 400:
|
||||
handle_response_error(resp, "get credit usage historical")
|
||||
body = resp.json()
|
||||
if not body.get("success"):
|
||||
raise Exception(body.get("error", "Unknown error"))
|
||||
return CreditUsageHistoricalResponse(**body)
|
||||
|
||||
|
||||
async def get_token_usage_historical(client: AsyncHttpClient, by_api_key: bool = False) -> TokenUsageHistoricalResponse:
|
||||
query = "?byApiKey=true" if by_api_key else ""
|
||||
resp = await client.get(f"/v2/team/token-usage/historical{query}")
|
||||
if resp.status_code >= 400:
|
||||
handle_response_error(resp, "get token usage historical")
|
||||
body = resp.json()
|
||||
if not body.get("success"):
|
||||
raise Exception(body.get("error", "Unknown error"))
|
||||
return TokenUsageHistoricalResponse(**body)
|
||||
|
||||
Reference in New Issue
Block a user