참고소스 수정본
This commit is contained in:
LASTA_DEV01\lasta
@@ -0,0 +1,9 @@
|
||||
"""
|
||||
Utility modules for v2 API client.
|
||||
"""
|
||||
|
||||
from .http_client import HttpClient
|
||||
from .error_handler import FirecrawlError, handle_response_error
|
||||
from .validation import validate_scrape_options, prepare_scrape_options
|
||||
|
||||
__all__ = ['HttpClient', 'FirecrawlError', 'handle_response_error', 'validate_scrape_options', 'prepare_scrape_options']
|
||||
@@ -0,0 +1,107 @@
|
||||
"""
|
||||
Error handling utilities for v2 API.
|
||||
"""
|
||||
|
||||
import requests
|
||||
from typing import Dict, Any, Optional
|
||||
|
||||
|
||||
class FirecrawlError(Exception):
|
||||
"""Base exception for Firecrawl API errors."""
|
||||
|
||||
def __init__(self, message: str, status_code: Optional[int] = None, response: Optional[requests.Response] = None):
|
||||
super().__init__(message)
|
||||
self.status_code = status_code
|
||||
self.response = response
|
||||
|
||||
|
||||
class BadRequestError(FirecrawlError):
|
||||
"""Raised when the request is invalid (400)."""
|
||||
pass
|
||||
|
||||
|
||||
|
||||
class UnauthorizedError(FirecrawlError):
|
||||
"""Raised when the request is unauthorized (401)."""
|
||||
pass
|
||||
|
||||
|
||||
class PaymentRequiredError(FirecrawlError):
|
||||
"""Raised when payment is required (402)."""
|
||||
pass
|
||||
|
||||
|
||||
class WebsiteNotSupportedError(FirecrawlError):
|
||||
"""Raised when website is not supported (403)."""
|
||||
pass
|
||||
|
||||
|
||||
class RequestTimeoutError(FirecrawlError):
|
||||
"""Raised when request times out (408)."""
|
||||
pass
|
||||
|
||||
|
||||
class RateLimitError(FirecrawlError):
|
||||
"""Raised when the rate limit is exceeded (429)."""
|
||||
pass
|
||||
|
||||
|
||||
class InternalServerError(FirecrawlError):
|
||||
"""Raised when there's an internal server error (500)."""
|
||||
pass
|
||||
|
||||
|
||||
def handle_response_error(response: requests.Response, action: str) -> None:
|
||||
"""
|
||||
Handle API response errors and raise appropriate exceptions.
|
||||
|
||||
Args:
|
||||
response: The HTTP response object
|
||||
action: Description of the action being performed
|
||||
|
||||
Raises:
|
||||
FirecrawlError: Appropriate error based on status code
|
||||
"""
|
||||
try:
|
||||
response_json = response.json()
|
||||
error_message = response_json.get('error', 'No error message provided.')
|
||||
error_details = response_json.get('details', 'No additional error details provided.')
|
||||
except:
|
||||
# If we can't parse JSON, provide a helpful error message
|
||||
try:
|
||||
response_text = response.text[:500] # Limit to first 500 chars
|
||||
if response_text.strip():
|
||||
error_message = f"Server returned non-JSON response: {response_text}"
|
||||
error_details = f"Full response status: {response.status_code}"
|
||||
else:
|
||||
error_message = f"Server returned empty response with status {response.status_code}"
|
||||
error_details = "No additional details available"
|
||||
except:
|
||||
error_message = f"Server returned unreadable response with status {response.status_code}"
|
||||
error_details = "No additional details available"
|
||||
|
||||
# Create appropriate error message
|
||||
if response.status_code == 400:
|
||||
message = f"Bad Request: Failed to {action}. {error_message} - {error_details}"
|
||||
raise BadRequestError(message, response.status_code, response)
|
||||
elif response.status_code == 401:
|
||||
message = f"Unauthorized: Failed to {action}. {error_message} - {error_details}"
|
||||
raise UnauthorizedError(message, response.status_code, response)
|
||||
elif response.status_code == 402:
|
||||
message = f"Payment Required: Failed to {action}. {error_message} - {error_details}"
|
||||
raise PaymentRequiredError(message, response.status_code, response)
|
||||
elif response.status_code == 403:
|
||||
message = f"Website Not Supported: Failed to {action}. {error_message} - {error_details}"
|
||||
raise WebsiteNotSupportedError(message, response.status_code, response)
|
||||
elif response.status_code == 408:
|
||||
message = f"Request Timeout: Failed to {action} as the request timed out. {error_message} - {error_details}"
|
||||
raise RequestTimeoutError(message, response.status_code, response)
|
||||
elif response.status_code == 429:
|
||||
message = f"Rate Limit Exceeded: Failed to {action}. {error_message} - {error_details}"
|
||||
raise RateLimitError(message, response.status_code, response)
|
||||
elif response.status_code == 500:
|
||||
message = f"Internal Server Error: Failed to {action}. {error_message} - {error_details}"
|
||||
raise InternalServerError(message, response.status_code, response)
|
||||
else:
|
||||
message = f"Unexpected error during {action}: Status code {response.status_code}. {error_message} - {error_details}"
|
||||
raise FirecrawlError(message, response.status_code, response)
|
||||
@@ -0,0 +1,15 @@
|
||||
import os
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
def get_version():
|
||||
try:
|
||||
package_path = Path(__file__).parents[2]
|
||||
version_file = (package_path / "__init__.py").read_text()
|
||||
version_match = re.search(r"^__version__ = ['\"]([^'\"]*)['\"]", version_file, re.M)
|
||||
if version_match:
|
||||
return version_match.group(1).strip()
|
||||
return "3.x.x"
|
||||
except Exception as e:
|
||||
print(f"Failed to get version from __init__.py: {e}")
|
||||
return "3.x.x"
|
||||
@@ -0,0 +1,312 @@
|
||||
"""
|
||||
HTTP client utilities for v2 API.
|
||||
"""
|
||||
|
||||
import time
|
||||
from typing import Dict, Any, Optional
|
||||
from urllib.parse import urlparse, urlunparse, urljoin
|
||||
import requests
|
||||
from .get_version import get_version
|
||||
|
||||
version = get_version()
|
||||
|
||||
class HttpClient:
|
||||
"""HTTP client with retry logic and error handling."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
api_key: Optional[str],
|
||||
api_url: str,
|
||||
timeout: Optional[float] = None,
|
||||
max_retries: int = 3,
|
||||
backoff_factor: float = 0.5,
|
||||
):
|
||||
self.api_key = api_key
|
||||
self.api_url = api_url
|
||||
self.timeout = timeout
|
||||
self.max_retries = max_retries
|
||||
self.backoff_factor = backoff_factor
|
||||
|
||||
def _build_url(self, endpoint: str) -> str:
|
||||
base = urlparse(self.api_url)
|
||||
ep = urlparse(endpoint)
|
||||
|
||||
# Absolute or protocol-relative (has netloc)
|
||||
if ep.netloc:
|
||||
# Different host: keep path/query but force base host/scheme (no token leakage)
|
||||
path = ep.path or "/"
|
||||
if (ep.hostname or "") != (base.hostname or ""):
|
||||
return urlunparse((base.scheme or "https", base.netloc, path, "", ep.query, ""))
|
||||
# Same host: normalize scheme to base
|
||||
return urlunparse((base.scheme or "https", base.netloc, path, "", ep.query, ""))
|
||||
|
||||
# Relative (including leading slash or not)
|
||||
base_str = self.api_url if self.api_url.endswith("/") else f"{self.api_url}/"
|
||||
# Guard protocol-relative like //host/path slipping through as “relative”
|
||||
if endpoint.startswith("//"):
|
||||
ep2 = urlparse(f"https:{endpoint}")
|
||||
path = ep2.path or "/"
|
||||
return urlunparse((base.scheme or "https", base.netloc, path, "", ep2.query, ""))
|
||||
return urljoin(base_str, endpoint)
|
||||
|
||||
def _prepare_headers(
|
||||
self,
|
||||
idempotency_key: Optional[str] = None,
|
||||
include_json_content_type: bool = True,
|
||||
) -> Dict[str, str]:
|
||||
"""Prepare headers for API requests."""
|
||||
headers: Dict[str, str] = {}
|
||||
|
||||
if include_json_content_type:
|
||||
headers['Content-Type'] = 'application/json'
|
||||
|
||||
if self.api_key:
|
||||
headers['Authorization'] = f'Bearer {self.api_key}'
|
||||
|
||||
if idempotency_key:
|
||||
headers['x-idempotency-key'] = idempotency_key
|
||||
|
||||
return headers
|
||||
|
||||
def post(
|
||||
self,
|
||||
endpoint: str,
|
||||
data: Dict[str, Any],
|
||||
headers: Optional[Dict[str, str]] = None,
|
||||
timeout: Optional[float] = None,
|
||||
retries: Optional[int] = None,
|
||||
backoff_factor: Optional[float] = None,
|
||||
) -> requests.Response:
|
||||
"""Make a POST request with retry logic."""
|
||||
if headers is None:
|
||||
headers = self._prepare_headers()
|
||||
if timeout is None:
|
||||
timeout = self.timeout
|
||||
if retries is None:
|
||||
retries = self.max_retries
|
||||
if backoff_factor is None:
|
||||
backoff_factor = self.backoff_factor
|
||||
|
||||
payload = dict(data)
|
||||
payload['origin'] = f'python-sdk@{version}'
|
||||
|
||||
url = self._build_url(endpoint)
|
||||
|
||||
last_exception = None
|
||||
num_attempts = max(1, retries)
|
||||
|
||||
for attempt in range(num_attempts):
|
||||
try:
|
||||
response = requests.post(
|
||||
url,
|
||||
headers=headers,
|
||||
json=payload,
|
||||
timeout=timeout
|
||||
)
|
||||
|
||||
if response.status_code == 502:
|
||||
if attempt < num_attempts - 1:
|
||||
time.sleep(backoff_factor * (2 ** attempt))
|
||||
continue
|
||||
|
||||
return response
|
||||
|
||||
except requests.RequestException as e:
|
||||
last_exception = e
|
||||
if attempt == num_attempts - 1:
|
||||
raise e
|
||||
time.sleep(backoff_factor * (2 ** attempt))
|
||||
|
||||
# This should never be reached due to the exception handling above
|
||||
raise last_exception or Exception("Unexpected error in POST request")
|
||||
|
||||
def post_multipart(
|
||||
self,
|
||||
endpoint: str,
|
||||
data: Dict[str, Any],
|
||||
files: Dict[str, Any],
|
||||
headers: Optional[Dict[str, str]] = None,
|
||||
timeout: Optional[float] = None,
|
||||
retries: Optional[int] = None,
|
||||
backoff_factor: Optional[float] = None,
|
||||
) -> requests.Response:
|
||||
"""Make a multipart/form-data POST request with retry logic."""
|
||||
multipart_headers = self._prepare_headers(include_json_content_type=False)
|
||||
if headers:
|
||||
multipart_headers.update(headers)
|
||||
multipart_headers.pop("Content-Type", None)
|
||||
multipart_headers.pop("content-type", None)
|
||||
if timeout is None:
|
||||
timeout = self.timeout
|
||||
if retries is None:
|
||||
retries = self.max_retries
|
||||
if backoff_factor is None:
|
||||
backoff_factor = self.backoff_factor
|
||||
|
||||
url = self._build_url(endpoint)
|
||||
last_exception = None
|
||||
num_attempts = max(1, retries)
|
||||
|
||||
for attempt in range(num_attempts):
|
||||
try:
|
||||
response = requests.post(
|
||||
url,
|
||||
headers=multipart_headers,
|
||||
data=data,
|
||||
files=files,
|
||||
timeout=timeout,
|
||||
)
|
||||
|
||||
if response.status_code == 502:
|
||||
if attempt < num_attempts - 1:
|
||||
time.sleep(backoff_factor * (2 ** attempt))
|
||||
continue
|
||||
|
||||
return response
|
||||
except requests.RequestException as e:
|
||||
last_exception = e
|
||||
if attempt == num_attempts - 1:
|
||||
raise e
|
||||
time.sleep(backoff_factor * (2 ** attempt))
|
||||
|
||||
raise last_exception or Exception("Unexpected error in multipart POST request")
|
||||
|
||||
def get(
|
||||
self,
|
||||
endpoint: str,
|
||||
headers: Optional[Dict[str, str]] = None,
|
||||
timeout: Optional[float] = None,
|
||||
retries: Optional[int] = None,
|
||||
backoff_factor: Optional[float] = None,
|
||||
) -> requests.Response:
|
||||
"""Make a GET request with retry logic."""
|
||||
if headers is None:
|
||||
headers = self._prepare_headers()
|
||||
if timeout is None:
|
||||
timeout = self.timeout
|
||||
if retries is None:
|
||||
retries = self.max_retries
|
||||
if backoff_factor is None:
|
||||
backoff_factor = self.backoff_factor
|
||||
|
||||
url = self._build_url(endpoint)
|
||||
|
||||
last_exception = None
|
||||
num_attempts = max(1, retries)
|
||||
|
||||
for attempt in range(num_attempts):
|
||||
try:
|
||||
response = requests.get(
|
||||
url,
|
||||
headers=headers,
|
||||
timeout=timeout
|
||||
)
|
||||
|
||||
if response.status_code == 502:
|
||||
if attempt < num_attempts - 1:
|
||||
time.sleep(backoff_factor * (2 ** attempt))
|
||||
continue
|
||||
|
||||
return response
|
||||
|
||||
except requests.RequestException as e:
|
||||
last_exception = e
|
||||
if attempt == num_attempts - 1:
|
||||
raise e
|
||||
time.sleep(backoff_factor * (2 ** attempt))
|
||||
|
||||
# This should never be reached due to the exception handling above
|
||||
raise last_exception or Exception("Unexpected error in GET request")
|
||||
|
||||
def delete(
|
||||
self,
|
||||
endpoint: str,
|
||||
headers: Optional[Dict[str, str]] = None,
|
||||
timeout: Optional[float] = None,
|
||||
retries: Optional[int] = None,
|
||||
backoff_factor: Optional[float] = None,
|
||||
) -> requests.Response:
|
||||
"""Make a DELETE request with retry logic."""
|
||||
if headers is None:
|
||||
headers = self._prepare_headers()
|
||||
if timeout is None:
|
||||
timeout = self.timeout
|
||||
if retries is None:
|
||||
retries = self.max_retries
|
||||
if backoff_factor is None:
|
||||
backoff_factor = self.backoff_factor
|
||||
|
||||
url = self._build_url(endpoint)
|
||||
|
||||
last_exception = None
|
||||
num_attempts = max(1, retries)
|
||||
|
||||
for attempt in range(num_attempts):
|
||||
try:
|
||||
response = requests.delete(
|
||||
url,
|
||||
headers=headers,
|
||||
timeout=timeout
|
||||
)
|
||||
|
||||
if response.status_code == 502:
|
||||
if attempt < num_attempts - 1:
|
||||
time.sleep(backoff_factor * (2 ** attempt))
|
||||
continue
|
||||
|
||||
return response
|
||||
|
||||
except requests.RequestException as e:
|
||||
last_exception = e
|
||||
if attempt == num_attempts - 1:
|
||||
raise e
|
||||
time.sleep(backoff_factor * (2 ** attempt))
|
||||
|
||||
# This should never be reached due to the exception handling above
|
||||
raise last_exception or Exception("Unexpected error in DELETE request")
|
||||
|
||||
def patch(
|
||||
self,
|
||||
endpoint: str,
|
||||
data: Dict[str, Any],
|
||||
headers: Optional[Dict[str, str]] = None,
|
||||
timeout: Optional[float] = None,
|
||||
retries: Optional[int] = None,
|
||||
backoff_factor: Optional[float] = None,
|
||||
) -> requests.Response:
|
||||
"""Make a PATCH request with retry logic."""
|
||||
if headers is None:
|
||||
headers = self._prepare_headers()
|
||||
if timeout is None:
|
||||
timeout = self.timeout
|
||||
if retries is None:
|
||||
retries = self.max_retries
|
||||
if backoff_factor is None:
|
||||
backoff_factor = self.backoff_factor
|
||||
|
||||
payload = dict(data)
|
||||
payload['origin'] = f'python-sdk@{version}'
|
||||
url = self._build_url(endpoint)
|
||||
|
||||
last_exception = None
|
||||
num_attempts = max(1, retries)
|
||||
|
||||
for attempt in range(num_attempts):
|
||||
try:
|
||||
response = requests.patch(
|
||||
url,
|
||||
json=payload,
|
||||
headers=headers,
|
||||
timeout=timeout
|
||||
)
|
||||
if response.status_code == 502 and attempt < num_attempts - 1:
|
||||
time.sleep(backoff_factor * (2 ** attempt))
|
||||
continue
|
||||
return response
|
||||
except requests.RequestException as e:
|
||||
last_exception = e
|
||||
if attempt == num_attempts - 1:
|
||||
raise e
|
||||
time.sleep(backoff_factor * (2 ** attempt))
|
||||
|
||||
raise last_exception or Exception("Unexpected error in PATCH request")
|
||||
@@ -0,0 +1,245 @@
|
||||
import asyncio
|
||||
import httpx
|
||||
from typing import Optional, Dict, Any
|
||||
from .get_version import get_version
|
||||
|
||||
version = get_version()
|
||||
|
||||
|
||||
class AsyncHttpClient:
|
||||
def __init__(
|
||||
self,
|
||||
api_key: Optional[str],
|
||||
api_url: str,
|
||||
timeout: Optional[float] = None,
|
||||
max_retries: int = 3,
|
||||
backoff_factor: float = 0.5,
|
||||
):
|
||||
self.api_key = api_key
|
||||
self.api_url = api_url
|
||||
self.timeout = timeout
|
||||
self.max_retries = max_retries
|
||||
self.backoff_factor = backoff_factor
|
||||
|
||||
headers = {}
|
||||
|
||||
if api_key:
|
||||
headers["Authorization"] = f"Bearer {api_key}"
|
||||
|
||||
self._client = httpx.AsyncClient(
|
||||
base_url=api_url,
|
||||
headers=headers,
|
||||
limits=httpx.Limits(max_keepalive_connections=0),
|
||||
)
|
||||
|
||||
async def close(self) -> None:
|
||||
await self._client.aclose()
|
||||
|
||||
def _headers(self, idempotency_key: Optional[str] = None) -> Dict[str, str]:
|
||||
headers: Dict[str, str] = {}
|
||||
if idempotency_key:
|
||||
headers["x-idempotency-key"] = idempotency_key
|
||||
return headers
|
||||
|
||||
async def post(
|
||||
self,
|
||||
endpoint: str,
|
||||
data: Dict[str, Any],
|
||||
headers: Optional[Dict[str, str]] = None,
|
||||
timeout: Optional[float] = None,
|
||||
retries: Optional[int] = None,
|
||||
backoff_factor: Optional[float] = None,
|
||||
) -> httpx.Response:
|
||||
if timeout is None:
|
||||
timeout = self.timeout
|
||||
if retries is None:
|
||||
retries = self.max_retries
|
||||
if backoff_factor is None:
|
||||
backoff_factor = self.backoff_factor
|
||||
|
||||
payload = dict(data)
|
||||
payload["origin"] = f"python-sdk@{version}"
|
||||
|
||||
last_exception = None
|
||||
num_attempts = max(1, retries)
|
||||
|
||||
for attempt in range(num_attempts):
|
||||
try:
|
||||
response = await self._client.post(
|
||||
endpoint,
|
||||
json=payload,
|
||||
headers={**self._headers(), **(headers or {})},
|
||||
timeout=timeout,
|
||||
)
|
||||
if response.status_code == 502:
|
||||
if attempt < num_attempts - 1:
|
||||
await asyncio.sleep(backoff_factor * (2 ** attempt))
|
||||
continue
|
||||
return response
|
||||
except httpx.HTTPError as e:
|
||||
last_exception = e
|
||||
if attempt == num_attempts - 1:
|
||||
raise e
|
||||
await asyncio.sleep(backoff_factor * (2 ** attempt))
|
||||
|
||||
raise last_exception or Exception("Unexpected error in POST request")
|
||||
|
||||
async def post_multipart(
|
||||
self,
|
||||
endpoint: str,
|
||||
data: Dict[str, Any],
|
||||
files: Dict[str, Any],
|
||||
headers: Optional[Dict[str, str]] = None,
|
||||
timeout: Optional[float] = None,
|
||||
retries: Optional[int] = None,
|
||||
backoff_factor: Optional[float] = None,
|
||||
) -> httpx.Response:
|
||||
if timeout is None:
|
||||
timeout = self.timeout
|
||||
if retries is None:
|
||||
retries = self.max_retries
|
||||
if backoff_factor is None:
|
||||
backoff_factor = self.backoff_factor
|
||||
|
||||
last_exception = None
|
||||
num_attempts = max(1, retries)
|
||||
|
||||
for attempt in range(num_attempts):
|
||||
try:
|
||||
response = await self._client.post(
|
||||
endpoint,
|
||||
data=data,
|
||||
files=files,
|
||||
headers={**self._headers(), **(headers or {})},
|
||||
timeout=timeout,
|
||||
)
|
||||
if response.status_code == 502:
|
||||
if attempt < num_attempts - 1:
|
||||
await asyncio.sleep(backoff_factor * (2 ** attempt))
|
||||
continue
|
||||
return response
|
||||
except httpx.HTTPError as e:
|
||||
last_exception = e
|
||||
if attempt == num_attempts - 1:
|
||||
raise e
|
||||
await asyncio.sleep(backoff_factor * (2 ** attempt))
|
||||
|
||||
raise last_exception or Exception("Unexpected error in multipart POST request")
|
||||
|
||||
async def get(
|
||||
self,
|
||||
endpoint: str,
|
||||
headers: Optional[Dict[str, str]] = None,
|
||||
timeout: Optional[float] = None,
|
||||
retries: Optional[int] = None,
|
||||
backoff_factor: Optional[float] = None,
|
||||
) -> httpx.Response:
|
||||
if timeout is None:
|
||||
timeout = self.timeout
|
||||
if retries is None:
|
||||
retries = self.max_retries
|
||||
if backoff_factor is None:
|
||||
backoff_factor = self.backoff_factor
|
||||
|
||||
last_exception = None
|
||||
num_attempts = max(1, retries)
|
||||
|
||||
for attempt in range(num_attempts):
|
||||
try:
|
||||
response = await self._client.get(
|
||||
endpoint,
|
||||
headers={**self._headers(), **(headers or {})},
|
||||
timeout=timeout,
|
||||
)
|
||||
if response.status_code == 502:
|
||||
if attempt < num_attempts - 1:
|
||||
await asyncio.sleep(backoff_factor * (2 ** attempt))
|
||||
continue
|
||||
return response
|
||||
except httpx.HTTPError as e:
|
||||
last_exception = e
|
||||
if attempt == num_attempts - 1:
|
||||
raise e
|
||||
await asyncio.sleep(backoff_factor * (2 ** attempt))
|
||||
|
||||
raise last_exception or Exception("Unexpected error in GET request")
|
||||
|
||||
async def delete(
|
||||
self,
|
||||
endpoint: str,
|
||||
headers: Optional[Dict[str, str]] = None,
|
||||
timeout: Optional[float] = None,
|
||||
retries: Optional[int] = None,
|
||||
backoff_factor: Optional[float] = None,
|
||||
) -> httpx.Response:
|
||||
if timeout is None:
|
||||
timeout = self.timeout
|
||||
if retries is None:
|
||||
retries = self.max_retries
|
||||
if backoff_factor is None:
|
||||
backoff_factor = self.backoff_factor
|
||||
|
||||
last_exception = None
|
||||
num_attempts = max(1, retries)
|
||||
|
||||
for attempt in range(num_attempts):
|
||||
try:
|
||||
response = await self._client.delete(
|
||||
endpoint,
|
||||
headers={**self._headers(), **(headers or {})},
|
||||
timeout=timeout,
|
||||
)
|
||||
if response.status_code == 502:
|
||||
if attempt < num_attempts - 1:
|
||||
await asyncio.sleep(backoff_factor * (2 ** attempt))
|
||||
continue
|
||||
return response
|
||||
except httpx.HTTPError as e:
|
||||
last_exception = e
|
||||
if attempt == num_attempts - 1:
|
||||
raise e
|
||||
await asyncio.sleep(backoff_factor * (2 ** attempt))
|
||||
|
||||
raise last_exception or Exception("Unexpected error in DELETE request")
|
||||
|
||||
async def patch(
|
||||
self,
|
||||
endpoint: str,
|
||||
data: Dict[str, Any],
|
||||
headers: Optional[Dict[str, str]] = None,
|
||||
timeout: Optional[float] = None,
|
||||
retries: Optional[int] = None,
|
||||
backoff_factor: Optional[float] = None,
|
||||
) -> httpx.Response:
|
||||
if timeout is None:
|
||||
timeout = self.timeout
|
||||
if retries is None:
|
||||
retries = self.max_retries
|
||||
if backoff_factor is None:
|
||||
backoff_factor = self.backoff_factor
|
||||
|
||||
payload = dict(data)
|
||||
payload["origin"] = f"python-sdk@{version}"
|
||||
|
||||
last_exception = None
|
||||
num_attempts = max(1, retries)
|
||||
|
||||
for attempt in range(num_attempts):
|
||||
try:
|
||||
response = await self._client.patch(
|
||||
endpoint,
|
||||
json=payload,
|
||||
headers={**self._headers(), **(headers or {})},
|
||||
timeout=timeout,
|
||||
)
|
||||
if response.status_code == 502 and attempt < num_attempts - 1:
|
||||
await asyncio.sleep(backoff_factor * (2 ** attempt))
|
||||
continue
|
||||
return response
|
||||
except httpx.HTTPError as e:
|
||||
last_exception = e
|
||||
if attempt == num_attempts - 1:
|
||||
raise e
|
||||
await asyncio.sleep(backoff_factor * (2 ** attempt))
|
||||
|
||||
raise last_exception or Exception("Unexpected error in PATCH request")
|
||||
@@ -0,0 +1,125 @@
|
||||
"""
|
||||
Normalization helpers for v2 API payloads to avoid relying on Pydantic aliases.
|
||||
"""
|
||||
|
||||
from typing import Any, Dict, List
|
||||
from ..types import DocumentMetadata
|
||||
|
||||
|
||||
def _map_metadata_keys(md: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""
|
||||
Convert API v2 camelCase metadata keys to snake_case expected by DocumentMetadata.
|
||||
Leaves unknown keys as-is.
|
||||
"""
|
||||
mapping = {
|
||||
# OpenGraph
|
||||
"ogTitle": "og_title",
|
||||
"ogDescription": "og_description",
|
||||
"ogUrl": "og_url",
|
||||
"ogImage": "og_image",
|
||||
"ogAudio": "og_audio",
|
||||
"ogDeterminer": "og_determiner",
|
||||
"ogLocale": "og_locale",
|
||||
"ogLocaleAlternate": "og_locale_alternate",
|
||||
"ogSiteName": "og_site_name",
|
||||
"ogVideo": "og_video",
|
||||
# Dublin Core and misc
|
||||
"dcTermsCreated": "dc_terms_created",
|
||||
"dcDateCreated": "dc_date_created",
|
||||
"dcDate": "dc_date",
|
||||
"dcTermsType": "dc_terms_type",
|
||||
"dcType": "dc_type",
|
||||
"dcTermsAudience": "dc_terms_audience",
|
||||
"dcTermsSubject": "dc_terms_subject",
|
||||
"dcSubject": "dc_subject",
|
||||
"dcDescription": "dc_description",
|
||||
"dcTermsKeywords": "dc_terms_keywords",
|
||||
"modifiedTime": "modified_time",
|
||||
"publishedTime": "published_time",
|
||||
"articleTag": "article_tag",
|
||||
"articleSection": "article_section",
|
||||
# Response-level
|
||||
"sourceURL": "source_url",
|
||||
"statusCode": "status_code",
|
||||
"scrapeId": "scrape_id",
|
||||
"numPages": "num_pages",
|
||||
"contentType": "content_type",
|
||||
"proxyUsed": "proxy_used",
|
||||
"cacheState": "cache_state",
|
||||
"cachedAt": "cached_at",
|
||||
"creditsUsed": "credits_used",
|
||||
"concurrencyLimited": "concurrency_limited",
|
||||
"concurrencyQueueDurationMs": "concurrency_queue_duration_ms",
|
||||
}
|
||||
|
||||
out: Dict[str, Any] = {}
|
||||
for k, v in md.items():
|
||||
snake = mapping.get(k, k)
|
||||
out[snake] = v
|
||||
|
||||
# Light coercions where server may send strings/lists
|
||||
if isinstance(out.get("status_code"), str):
|
||||
try:
|
||||
out["status_code"] = int(out["status_code"]) # type: ignore
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
# Preserve list values for unknown keys; only lightweight coercions above
|
||||
return out
|
||||
|
||||
|
||||
def normalize_document_input(doc: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""
|
||||
Normalize a raw Document dict from the API into the Python SDK's expected shape:
|
||||
- Convert top-level keys rawHtml->raw_html, changeTracking->change_tracking
|
||||
- Convert metadata keys from camelCase to snake_case
|
||||
- Convert branding.colorScheme to branding.color_scheme
|
||||
"""
|
||||
normalized = dict(doc)
|
||||
|
||||
if "rawHtml" in normalized and "raw_html" not in normalized:
|
||||
normalized["raw_html"] = normalized.pop("rawHtml")
|
||||
|
||||
if "changeTracking" in normalized and "change_tracking" not in normalized:
|
||||
normalized["change_tracking"] = normalized.pop("changeTracking")
|
||||
|
||||
md = normalized.get("metadata")
|
||||
if isinstance(md, dict):
|
||||
mapped = _map_metadata_keys(md)
|
||||
# Construct a typed DocumentMetadata; extras allowed/preserved
|
||||
try:
|
||||
normalized["metadata"] = DocumentMetadata.model_validate(mapped)
|
||||
except Exception:
|
||||
normalized["metadata"] = mapped
|
||||
|
||||
# Normalize branding top-level camelCase keys
|
||||
branding = normalized.get("branding")
|
||||
if isinstance(branding, dict):
|
||||
if "colorScheme" in branding and "color_scheme" not in branding:
|
||||
branding["color_scheme"] = branding.pop("colorScheme")
|
||||
|
||||
return normalized
|
||||
|
||||
|
||||
def _map_search_result_keys(result: Dict[str, Any], result_type: str) -> Dict[str, Any]:
|
||||
if result_type == "images":
|
||||
mapping = {
|
||||
"imageUrl": "image_url",
|
||||
"imageWidth": "image_width",
|
||||
"imageHeight": "image_height",
|
||||
}
|
||||
elif result_type == "news":
|
||||
mapping = {
|
||||
"imageUrl": "image_url",
|
||||
}
|
||||
elif result_type == "web":
|
||||
mapping = {}
|
||||
else:
|
||||
mapping = {}
|
||||
|
||||
out: Dict[str, Any] = {}
|
||||
for k, v in result.items():
|
||||
snake = mapping.get(k, k)
|
||||
out[snake] = v
|
||||
|
||||
return out
|
||||
@@ -0,0 +1,790 @@
|
||||
"""
|
||||
Shared validation functions for Firecrawl v2 API.
|
||||
"""
|
||||
|
||||
from typing import Optional, Dict, Any, List
|
||||
from ..types import ScrapeOptions, ScrapeFormats
|
||||
|
||||
|
||||
def _convert_format_string(format_str: str) -> str:
|
||||
"""
|
||||
Convert format string from snake_case to camelCase.
|
||||
|
||||
Args:
|
||||
format_str: Format string in snake_case
|
||||
|
||||
Returns:
|
||||
Format string in camelCase
|
||||
"""
|
||||
format_mapping = {
|
||||
"raw_html": "rawHtml",
|
||||
"change_tracking": "changeTracking",
|
||||
"screenshot_full_page": "screenshot@fullPage"
|
||||
}
|
||||
return format_mapping.get(format_str, format_str)
|
||||
|
||||
|
||||
def normalize_schema_for_openai(schema: Any) -> Any:
|
||||
"""
|
||||
Normalize a schema for OpenAI compatibility by handling recursive references.
|
||||
|
||||
Args:
|
||||
schema: Schema to normalize
|
||||
|
||||
Returns:
|
||||
Normalized schema
|
||||
"""
|
||||
if not schema or not isinstance(schema, dict):
|
||||
return schema
|
||||
|
||||
visited = set()
|
||||
|
||||
def normalize_object(obj: Any) -> Any:
|
||||
if not isinstance(obj, dict):
|
||||
if isinstance(obj, list):
|
||||
return [normalize_object(item) for item in obj]
|
||||
return obj
|
||||
|
||||
obj_id = id(obj)
|
||||
if obj_id in visited:
|
||||
return obj
|
||||
visited.add(obj_id)
|
||||
|
||||
normalized = dict(obj)
|
||||
|
||||
# Handle $ref recursion
|
||||
if "$ref" in normalized:
|
||||
visited.discard(obj_id)
|
||||
return normalized
|
||||
|
||||
if "$defs" in normalized:
|
||||
defs = normalized.pop("$defs")
|
||||
processed_rest = {}
|
||||
|
||||
for key, value in normalized.items():
|
||||
if isinstance(value, dict) and "$ref" not in value:
|
||||
processed_rest[key] = normalize_object(value)
|
||||
else:
|
||||
processed_rest[key] = value
|
||||
|
||||
normalized_defs = {}
|
||||
for key, value in defs.items():
|
||||
normalized_defs[key] = normalize_object(value)
|
||||
|
||||
result = {**processed_rest, "$defs": normalized_defs}
|
||||
visited.discard(obj_id)
|
||||
return result
|
||||
|
||||
if (normalized.get("type") == "object" and
|
||||
"properties" in normalized and
|
||||
normalized.get("additionalProperties") is True):
|
||||
del normalized["additionalProperties"]
|
||||
|
||||
if (normalized.get("type") == "object" and
|
||||
"required" in normalized and
|
||||
"properties" in normalized):
|
||||
if (isinstance(normalized["required"], list) and
|
||||
isinstance(normalized["properties"], dict)):
|
||||
valid_required = [field for field in normalized["required"]
|
||||
if field in normalized["properties"]]
|
||||
if valid_required:
|
||||
normalized["required"] = valid_required
|
||||
else:
|
||||
del normalized["required"]
|
||||
else:
|
||||
del normalized["required"]
|
||||
|
||||
for key, value in list(normalized.items()):
|
||||
if isinstance(value, dict) and "$ref" not in value:
|
||||
normalized[key] = normalize_object(value)
|
||||
elif isinstance(value, list):
|
||||
normalized[key] = [normalize_object(item) if isinstance(item, dict) else item for item in value]
|
||||
|
||||
visited.discard(obj_id)
|
||||
return normalized
|
||||
|
||||
return normalize_object(schema)
|
||||
|
||||
|
||||
def validate_schema_for_openai(schema: Any) -> bool:
|
||||
"""
|
||||
Validate schema for OpenAI compatibility.
|
||||
|
||||
Args:
|
||||
schema: Schema to validate
|
||||
|
||||
Returns:
|
||||
True if schema is valid, False otherwise
|
||||
"""
|
||||
if not schema or not isinstance(schema, dict):
|
||||
return True
|
||||
|
||||
visited = set()
|
||||
|
||||
def has_invalid_structure(obj: Any) -> bool:
|
||||
if not isinstance(obj, dict):
|
||||
return False
|
||||
|
||||
obj_id = id(obj)
|
||||
if obj_id in visited:
|
||||
return False
|
||||
visited.add(obj_id)
|
||||
|
||||
if "$ref" in obj:
|
||||
visited.discard(obj_id)
|
||||
return False
|
||||
|
||||
if (obj.get("type") == "object" and
|
||||
"properties" not in obj and
|
||||
"patternProperties" not in obj and
|
||||
obj.get("additionalProperties") is True):
|
||||
visited.discard(obj_id)
|
||||
return True
|
||||
|
||||
for value in obj.values():
|
||||
if isinstance(value, dict) and "$ref" not in value:
|
||||
if has_invalid_structure(value):
|
||||
visited.discard(obj_id)
|
||||
return True
|
||||
elif isinstance(value, list):
|
||||
for item in value:
|
||||
if isinstance(item, dict) and "$ref" not in item:
|
||||
if has_invalid_structure(item):
|
||||
visited.discard(obj_id)
|
||||
return True
|
||||
|
||||
visited.discard(obj_id)
|
||||
return False
|
||||
|
||||
return not has_invalid_structure(schema)
|
||||
|
||||
|
||||
OPENAI_SCHEMA_ERROR_MESSAGE = (
|
||||
"Schema contains invalid structure for OpenAI: object type with no 'properties' defined "
|
||||
"but 'additionalProperties: true' (schema-less dictionary not supported by OpenAI). "
|
||||
"Please define specific properties for your object. Note: Recursive schemas using '$ref' are supported."
|
||||
)
|
||||
|
||||
|
||||
def _contains_recursive_ref(obj: Any, target_def_name: str, defs: Dict[str, Any], visited: Optional[set] = None) -> bool:
|
||||
"""
|
||||
Check if an object contains a recursive reference to a specific definition.
|
||||
|
||||
Args:
|
||||
obj: Object to check
|
||||
target_def_name: Name of the definition to check for recursion
|
||||
defs: Dictionary of definitions
|
||||
visited: Set of visited object keys to detect cycles
|
||||
|
||||
Returns:
|
||||
True if recursive reference is found, False otherwise
|
||||
"""
|
||||
if not obj or not isinstance(obj, (dict, list)):
|
||||
return False
|
||||
|
||||
if visited is None:
|
||||
visited = set()
|
||||
|
||||
import json
|
||||
obj_key = json.dumps(obj, sort_keys=True, default=str)
|
||||
if obj_key in visited:
|
||||
return False
|
||||
visited.add(obj_key)
|
||||
|
||||
try:
|
||||
if isinstance(obj, dict):
|
||||
if "$ref" in obj and isinstance(obj["$ref"], str):
|
||||
ref_path = obj["$ref"].split("/")
|
||||
if len(ref_path) >= 3 and ref_path[0] == "#" and ref_path[1] == "$defs":
|
||||
def_name = ref_path[-1]
|
||||
if def_name == target_def_name:
|
||||
return True
|
||||
if def_name in defs:
|
||||
return _contains_recursive_ref(defs[def_name], target_def_name, defs, visited)
|
||||
|
||||
for value in obj.values():
|
||||
if _contains_recursive_ref(value, target_def_name, defs, visited):
|
||||
return True
|
||||
|
||||
elif isinstance(obj, list):
|
||||
for item in obj:
|
||||
if _contains_recursive_ref(item, target_def_name, defs, visited):
|
||||
return True
|
||||
|
||||
finally:
|
||||
visited.discard(obj_key)
|
||||
|
||||
return False
|
||||
|
||||
|
||||
def _check_for_circular_defs(defs: Dict[str, Any]) -> bool:
|
||||
"""
|
||||
Check if $defs contain circular references.
|
||||
|
||||
Args:
|
||||
defs: Dictionary of definitions to check
|
||||
|
||||
Returns:
|
||||
True if circular references are found, False otherwise
|
||||
"""
|
||||
if not defs:
|
||||
return False
|
||||
|
||||
for def_name, def_value in defs.items():
|
||||
if _contains_recursive_ref(def_value, def_name, defs):
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
|
||||
def resolve_refs(obj: Any, defs: Dict[str, Any], visited: Optional[set] = None, depth: int = 0) -> Any:
|
||||
"""
|
||||
Resolve $ref references in a JSON schema object.
|
||||
|
||||
Args:
|
||||
obj: Object to resolve references in
|
||||
defs: Dictionary of definitions
|
||||
visited: Set to track visited objects and prevent infinite recursion
|
||||
depth: Current recursion depth
|
||||
|
||||
Returns:
|
||||
Object with resolved references
|
||||
"""
|
||||
if not obj or not isinstance(obj, (dict, list)) or depth > 10:
|
||||
return obj
|
||||
|
||||
if visited is None:
|
||||
visited = set()
|
||||
|
||||
obj_id = id(obj)
|
||||
if obj_id in visited:
|
||||
return obj
|
||||
|
||||
visited.add(obj_id)
|
||||
|
||||
try:
|
||||
if isinstance(obj, dict):
|
||||
if "$ref" in obj and isinstance(obj["$ref"], str):
|
||||
ref_path = obj["$ref"].split("/")
|
||||
if len(ref_path) >= 3 and ref_path[0] == "#" and ref_path[1] == "$defs":
|
||||
def_name = ref_path[-1]
|
||||
if def_name in defs:
|
||||
return resolve_refs(dict(defs[def_name]), defs, visited, depth + 1)
|
||||
return obj
|
||||
|
||||
resolved = {}
|
||||
for key, value in obj.items():
|
||||
if key == "$defs":
|
||||
continue
|
||||
resolved[key] = resolve_refs(value, defs, visited, depth + 1)
|
||||
return resolved
|
||||
|
||||
elif isinstance(obj, list):
|
||||
return [resolve_refs(item, defs, visited, depth + 1) for item in obj]
|
||||
|
||||
finally:
|
||||
visited.discard(obj_id)
|
||||
|
||||
return obj
|
||||
|
||||
|
||||
def detect_recursive_schema(schema: Any) -> bool:
|
||||
"""
|
||||
Detect if a schema contains recursive references.
|
||||
|
||||
Args:
|
||||
schema: Schema to analyze
|
||||
|
||||
Returns:
|
||||
True if schema has recursive patterns, False otherwise
|
||||
"""
|
||||
if not schema or not isinstance(schema, dict):
|
||||
return False
|
||||
|
||||
import json
|
||||
schema_string = json.dumps(schema)
|
||||
has_refs = (
|
||||
'"$ref"' in schema_string or
|
||||
"#/$defs/" in schema_string or
|
||||
"#/definitions/" in schema_string
|
||||
)
|
||||
has_defs = bool(schema.get("$defs") or schema.get("definitions"))
|
||||
|
||||
return has_refs or has_defs
|
||||
|
||||
|
||||
def select_model_for_schema(schema: Any = None) -> Dict[str, str]:
|
||||
"""
|
||||
Select appropriate model based on schema complexity.
|
||||
|
||||
Args:
|
||||
schema: Schema to analyze
|
||||
|
||||
Returns:
|
||||
Dict with modelName and reason
|
||||
"""
|
||||
if not schema:
|
||||
return {"modelName": "gpt-4o-mini", "reason": "no_schema"}
|
||||
|
||||
if detect_recursive_schema(schema):
|
||||
return {"modelName": "gpt-4o", "reason": "recursive_schema_detected"}
|
||||
|
||||
return {"modelName": "gpt-4o-mini", "reason": "simple_schema"}
|
||||
|
||||
|
||||
def _normalize_schema(schema: Any) -> Optional[Dict[str, Any]]:
|
||||
"""
|
||||
Normalize a schema object which may be a dict, Pydantic BaseModel subclass,
|
||||
or a Pydantic model instance into a plain dict.
|
||||
"""
|
||||
try:
|
||||
# Pydantic v2 BaseModel subclass: has "model_json_schema"
|
||||
if hasattr(schema, "model_json_schema") and callable(schema.model_json_schema):
|
||||
return schema.model_json_schema()
|
||||
# Pydantic v2 BaseModel instance: has "model_dump" or "model_json_schema"
|
||||
if hasattr(schema, "model_dump") and callable(schema.model_dump):
|
||||
# Try to get JSON schema if available on the class
|
||||
mjs = getattr(schema.__class__, "model_json_schema", None)
|
||||
if callable(mjs):
|
||||
return schema.__class__.model_json_schema()
|
||||
# Fallback to data shape (not ideal, but better than dropping)
|
||||
return schema.model_dump()
|
||||
# Pydantic v1 BaseModel subclass: has "schema"
|
||||
if hasattr(schema, "schema") and callable(schema.schema):
|
||||
return schema.schema()
|
||||
# Pydantic v1 BaseModel instance
|
||||
if hasattr(schema, "dict") and callable(schema.dict):
|
||||
# Prefer class-level schema if present
|
||||
sch = getattr(schema.__class__, "schema", None)
|
||||
if callable(sch):
|
||||
return schema.__class__.schema()
|
||||
return schema.dict()
|
||||
except Exception:
|
||||
pass
|
||||
# Already a dict or unsupported type
|
||||
return schema if isinstance(schema, dict) else None
|
||||
|
||||
|
||||
def _validate_json_format(format_obj: Any) -> Dict[str, Any]:
|
||||
"""
|
||||
Validate and prepare json format object.
|
||||
|
||||
Args:
|
||||
format_obj: Format object that should be json type
|
||||
|
||||
Returns:
|
||||
Validated json format dict
|
||||
|
||||
Raises:
|
||||
ValueError: If json format is missing required fields
|
||||
"""
|
||||
if not isinstance(format_obj, dict):
|
||||
raise ValueError("json format must be an object with 'type', 'prompt', and 'schema' fields")
|
||||
|
||||
if format_obj.get('type') != 'json':
|
||||
raise ValueError("json format must have type='json'")
|
||||
|
||||
# prompt is optional in v2; only normalize when present
|
||||
# schema is recommended; if provided, normalize Pydantic forms
|
||||
schema = format_obj.get('schema')
|
||||
normalized = dict(format_obj)
|
||||
if schema is not None:
|
||||
normalized_schema = _normalize_schema(schema)
|
||||
if normalized_schema is not None:
|
||||
# Handle schema reference resolution similar to TypeScript implementation
|
||||
if isinstance(normalized_schema, dict):
|
||||
defs = normalized_schema.get("$defs", {})
|
||||
import json
|
||||
schema_string = json.dumps(normalized_schema)
|
||||
has_any_refs = (
|
||||
normalized_schema.get("$defs") or
|
||||
'"$ref"' in schema_string or
|
||||
"#/$defs/" in schema_string
|
||||
)
|
||||
|
||||
if has_any_refs:
|
||||
try:
|
||||
resolved_schema = resolve_refs(normalized_schema, defs)
|
||||
resolved_string = json.dumps(resolved_schema)
|
||||
has_remaining_refs = '"$ref"' in resolved_string or "#/$defs/" in resolved_string
|
||||
|
||||
if not has_remaining_refs:
|
||||
normalized_schema = resolved_schema
|
||||
# Remove $defs after successful resolution
|
||||
if isinstance(normalized_schema, dict) and "$defs" in normalized_schema:
|
||||
del normalized_schema["$defs"]
|
||||
# If refs remain, preserve original schema
|
||||
except Exception:
|
||||
# Failed to resolve refs, preserve original schema
|
||||
pass
|
||||
else:
|
||||
# No recursive references detected, resolve refs anyway
|
||||
try:
|
||||
normalized_schema = resolve_refs(normalized_schema, defs)
|
||||
if isinstance(normalized_schema, dict) and "$defs" in normalized_schema:
|
||||
del normalized_schema["$defs"]
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Apply OpenAI normalization and validation
|
||||
openai_normalized_schema = normalize_schema_for_openai(normalized_schema)
|
||||
if not validate_schema_for_openai(openai_normalized_schema):
|
||||
raise ValueError(OPENAI_SCHEMA_ERROR_MESSAGE)
|
||||
|
||||
normalized['schema'] = openai_normalized_schema
|
||||
return normalized
|
||||
|
||||
|
||||
def _validate_query_format(format_obj: Any) -> Dict[str, Any]:
|
||||
"""
|
||||
Validate and prepare query format object.
|
||||
|
||||
Args:
|
||||
format_obj: Format object that should be query type
|
||||
|
||||
Returns:
|
||||
Validated query format dict
|
||||
|
||||
Raises:
|
||||
ValueError: If query format is missing required 'prompt' field
|
||||
"""
|
||||
if not isinstance(format_obj, dict):
|
||||
raise ValueError("query format must be an object with 'type' and 'prompt' fields")
|
||||
|
||||
if not isinstance(format_obj.get('prompt'), str) or not format_obj['prompt'].strip():
|
||||
raise ValueError("query format requires a non-empty 'prompt' string")
|
||||
|
||||
if "directQuote" in format_obj:
|
||||
raise ValueError("query format uses 'mode' instead of 'directQuote'")
|
||||
|
||||
mode = format_obj.get("mode")
|
||||
if mode is not None and mode not in ("freeform", "directQuote"):
|
||||
raise ValueError("query format mode must be 'freeform' or 'directQuote'")
|
||||
|
||||
return format_obj
|
||||
|
||||
|
||||
def _validate_question_format(format_obj: Any) -> Dict[str, Any]:
|
||||
"""Validate and prepare question format object."""
|
||||
if not isinstance(format_obj, dict):
|
||||
raise ValueError("question format must be an object with 'type' and 'question' fields")
|
||||
|
||||
if not isinstance(format_obj.get('question'), str) or not format_obj['question'].strip():
|
||||
raise ValueError("question format requires a non-empty 'question' string")
|
||||
|
||||
return format_obj
|
||||
|
||||
|
||||
def _validate_highlights_format(format_obj: Any) -> Dict[str, Any]:
|
||||
"""Validate and prepare highlights format object."""
|
||||
if not isinstance(format_obj, dict):
|
||||
raise ValueError("highlights format must be an object with 'type' and 'query' fields")
|
||||
|
||||
if not isinstance(format_obj.get('query'), str) or not format_obj['query'].strip():
|
||||
raise ValueError("highlights format requires a non-empty 'query' string")
|
||||
|
||||
return format_obj
|
||||
|
||||
|
||||
def validate_scrape_options(options: Optional[ScrapeOptions]) -> Optional[ScrapeOptions]:
|
||||
"""
|
||||
Validate and normalize scrape options.
|
||||
|
||||
Args:
|
||||
options: Scraping options to validate
|
||||
|
||||
Returns:
|
||||
Validated options or None
|
||||
|
||||
Raises:
|
||||
ValueError: If options are invalid
|
||||
"""
|
||||
if options is None:
|
||||
return None
|
||||
|
||||
# Validate timeout
|
||||
if options.timeout is not None and options.timeout <= 0:
|
||||
raise ValueError("Timeout must be positive")
|
||||
|
||||
# Validate wait_for
|
||||
if options.wait_for is not None and options.wait_for < 0:
|
||||
raise ValueError("wait_for must be non-negative")
|
||||
|
||||
return options
|
||||
|
||||
|
||||
def prepare_scrape_options(options: Optional[ScrapeOptions]) -> Optional[Dict[str, Any]]:
|
||||
"""
|
||||
Prepare ScrapeOptions for API submission with manual snake_case to camelCase conversion.
|
||||
|
||||
Args:
|
||||
options: ScrapeOptions to prepare
|
||||
|
||||
Returns:
|
||||
Dictionary ready for API submission or None if options is None
|
||||
"""
|
||||
if options is None:
|
||||
return None
|
||||
|
||||
# Validate options first
|
||||
validated_options = validate_scrape_options(options)
|
||||
if validated_options is None:
|
||||
return None
|
||||
|
||||
# Apply default values for None fields
|
||||
default_values = {
|
||||
"only_main_content": True,
|
||||
"mobile": False,
|
||||
"skip_tls_verification": True,
|
||||
"remove_base64_images": True,
|
||||
"fast_mode": False,
|
||||
"block_ads": True,
|
||||
"max_age": 14400000,
|
||||
"store_in_cache": True
|
||||
}
|
||||
|
||||
# Convert to dict and handle manual snake_case to camelCase conversion
|
||||
options_data = validated_options.model_dump(exclude_none=True)
|
||||
|
||||
# Apply defaults for None fields
|
||||
for field, default_value in default_values.items():
|
||||
if field not in options_data:
|
||||
options_data[field] = default_value
|
||||
|
||||
scrape_data = {}
|
||||
|
||||
# Manual field mapping for snake_case to camelCase conversion
|
||||
field_mappings = {
|
||||
"include_tags": "includeTags",
|
||||
"exclude_tags": "excludeTags",
|
||||
"only_main_content": "onlyMainContent",
|
||||
"wait_for": "waitFor",
|
||||
"skip_tls_verification": "skipTlsVerification",
|
||||
"remove_base64_images": "removeBase64Images",
|
||||
"fast_mode": "fastMode",
|
||||
"use_mock": "useMock",
|
||||
"block_ads": "blockAds",
|
||||
"store_in_cache": "storeInCache",
|
||||
"max_age": "maxAge"
|
||||
}
|
||||
|
||||
# Apply field mappings
|
||||
for snake_case, camel_case in field_mappings.items():
|
||||
if snake_case in options_data:
|
||||
scrape_data[camel_case] = options_data.pop(snake_case)
|
||||
|
||||
# Handle special cases
|
||||
for key, value in options_data.items():
|
||||
if value is not None:
|
||||
if key == "integration":
|
||||
scrape_data["integration"] = (str(value).strip() or None)
|
||||
continue
|
||||
if key == "formats":
|
||||
# Handle formats conversion
|
||||
converted_formats: List[Any] = []
|
||||
|
||||
# Prefer using original object to detect ScrapeFormats vs list
|
||||
original_formats = getattr(options, 'formats', None)
|
||||
|
||||
if isinstance(original_formats, ScrapeFormats):
|
||||
# Include explicit list first
|
||||
if original_formats.formats:
|
||||
for fmt in original_formats.formats:
|
||||
if isinstance(fmt, str):
|
||||
if fmt == "json":
|
||||
raise ValueError("json format must be an object with 'type', 'prompt', and 'schema' fields")
|
||||
if fmt == "query":
|
||||
raise ValueError("query format must be an object with 'type' and 'prompt' fields")
|
||||
if fmt == "question":
|
||||
raise ValueError("question format must be an object with 'type' and 'question' fields")
|
||||
if fmt == "highlights":
|
||||
raise ValueError("highlights format must be an object with 'type' and 'query' fields")
|
||||
converted_formats.append(_convert_format_string(fmt))
|
||||
elif isinstance(fmt, dict):
|
||||
fmt_type = _convert_format_string(fmt.get('type')) if fmt.get('type') else None
|
||||
if fmt_type == 'json':
|
||||
validated_json = _validate_json_format({**fmt, 'type': 'json'})
|
||||
converted_formats.append(validated_json)
|
||||
elif fmt_type == 'question':
|
||||
converted_formats.append(_validate_question_format(fmt))
|
||||
elif fmt_type == 'highlights':
|
||||
converted_formats.append(_validate_highlights_format(fmt))
|
||||
elif fmt_type == 'query':
|
||||
converted_formats.append(_validate_query_format(fmt))
|
||||
elif fmt_type == 'screenshot':
|
||||
# Normalize screenshot options
|
||||
normalized = {**fmt, 'type': 'screenshot'}
|
||||
if 'full_page' in normalized:
|
||||
normalized['fullPage'] = normalized.pop('full_page')
|
||||
# Normalize viewport if it's a model instance
|
||||
vp = normalized.get('viewport')
|
||||
if hasattr(vp, 'model_dump'):
|
||||
normalized['viewport'] = vp.model_dump(exclude_none=True)
|
||||
converted_formats.append(normalized)
|
||||
else:
|
||||
if 'type' in fmt:
|
||||
fmt['type'] = fmt_type or fmt['type']
|
||||
converted_formats.append(fmt)
|
||||
elif hasattr(fmt, 'type'):
|
||||
if fmt.type == 'json':
|
||||
converted_formats.append(_validate_json_format(fmt.model_dump()))
|
||||
elif fmt.type == 'question':
|
||||
converted_formats.append(_validate_question_format(fmt.model_dump(exclude_none=True)))
|
||||
elif fmt.type == 'highlights':
|
||||
converted_formats.append(_validate_highlights_format(fmt.model_dump(exclude_none=True)))
|
||||
elif fmt.type == 'query':
|
||||
converted_formats.append(_validate_query_format(fmt.model_dump(exclude_none=True)))
|
||||
else:
|
||||
converted_formats.append(_convert_format_string(fmt.type))
|
||||
else:
|
||||
converted_formats.append(fmt)
|
||||
|
||||
# Add booleans from ScrapeFormats
|
||||
if original_formats.markdown:
|
||||
converted_formats.append("markdown")
|
||||
if original_formats.html:
|
||||
converted_formats.append("html")
|
||||
if original_formats.raw_html:
|
||||
converted_formats.append("rawHtml")
|
||||
if original_formats.summary:
|
||||
converted_formats.append("summary")
|
||||
if original_formats.links:
|
||||
converted_formats.append("links")
|
||||
if original_formats.screenshot:
|
||||
converted_formats.append("screenshot")
|
||||
if original_formats.change_tracking:
|
||||
converted_formats.append("changeTracking")
|
||||
# Note: We intentionally do not auto-include 'json' when boolean is set,
|
||||
# because JSON requires an object with schema/prompt. The caller must
|
||||
# supply the full json format object explicitly.
|
||||
elif isinstance(original_formats, list):
|
||||
for fmt in original_formats:
|
||||
if isinstance(fmt, str):
|
||||
if fmt == "json":
|
||||
raise ValueError("json format must be an object with 'type', 'prompt', and 'schema' fields")
|
||||
if fmt == "query":
|
||||
raise ValueError("query format must be an object with 'type' and 'prompt' fields")
|
||||
if fmt == "question":
|
||||
raise ValueError("question format must be an object with 'type' and 'question' fields")
|
||||
if fmt == "highlights":
|
||||
raise ValueError("highlights format must be an object with 'type' and 'query' fields")
|
||||
converted_formats.append(_convert_format_string(fmt))
|
||||
elif isinstance(fmt, dict):
|
||||
fmt_type = _convert_format_string(fmt.get('type')) if fmt.get('type') else None
|
||||
if fmt_type == 'json':
|
||||
validated_json = _validate_json_format({**fmt, 'type': 'json'})
|
||||
converted_formats.append(validated_json)
|
||||
elif fmt_type == 'question':
|
||||
converted_formats.append(_validate_question_format(fmt))
|
||||
elif fmt_type == 'highlights':
|
||||
converted_formats.append(_validate_highlights_format(fmt))
|
||||
elif fmt_type == 'query':
|
||||
converted_formats.append(_validate_query_format(fmt))
|
||||
elif fmt_type == 'screenshot':
|
||||
normalized = {**fmt, 'type': 'screenshot'}
|
||||
if 'full_page' in normalized:
|
||||
normalized['fullPage'] = normalized.pop('full_page')
|
||||
vp = normalized.get('viewport')
|
||||
if hasattr(vp, 'model_dump'):
|
||||
normalized['viewport'] = vp.model_dump(exclude_none=True)
|
||||
converted_formats.append(normalized)
|
||||
else:
|
||||
if 'type' in fmt:
|
||||
fmt['type'] = fmt_type or fmt['type']
|
||||
converted_formats.append(fmt)
|
||||
elif hasattr(fmt, 'type'):
|
||||
if fmt.type == 'json':
|
||||
converted_formats.append(_validate_json_format(fmt.model_dump()))
|
||||
elif fmt.type == 'question':
|
||||
converted_formats.append(_validate_question_format(fmt.model_dump(exclude_none=True)))
|
||||
elif fmt.type == 'highlights':
|
||||
converted_formats.append(_validate_highlights_format(fmt.model_dump(exclude_none=True)))
|
||||
elif fmt.type == 'screenshot':
|
||||
normalized = {'type': 'screenshot'}
|
||||
if getattr(fmt, 'full_page', None) is not None:
|
||||
normalized['fullPage'] = fmt.full_page
|
||||
if getattr(fmt, 'quality', None) is not None:
|
||||
normalized['quality'] = fmt.quality
|
||||
vp = getattr(fmt, 'viewport', None)
|
||||
if vp is not None:
|
||||
normalized['viewport'] = vp.model_dump(exclude_none=True) if hasattr(vp, 'model_dump') else vp
|
||||
converted_formats.append(normalized)
|
||||
elif fmt.type == 'query':
|
||||
converted_formats.append(_validate_query_format(fmt.model_dump(exclude_none=True)))
|
||||
else:
|
||||
converted_formats.append(_convert_format_string(fmt.type))
|
||||
else:
|
||||
converted_formats.append(fmt)
|
||||
else:
|
||||
# Fallback: try to iterate over value if it's a list-like
|
||||
try:
|
||||
for fmt in value:
|
||||
converted_formats.append(fmt)
|
||||
except TypeError:
|
||||
pass
|
||||
|
||||
if converted_formats:
|
||||
scrape_data["formats"] = converted_formats
|
||||
elif key == "actions":
|
||||
# Handle actions conversion
|
||||
converted_actions = []
|
||||
for action in value:
|
||||
if isinstance(action, dict):
|
||||
# Convert action dict
|
||||
converted_action = {}
|
||||
for action_key, action_value in action.items():
|
||||
if action_key == "full_page":
|
||||
converted_action["fullPage"] = action_value
|
||||
else:
|
||||
converted_action[action_key] = action_value
|
||||
converted_actions.append(converted_action)
|
||||
else:
|
||||
# Handle action objects
|
||||
action_data = action.model_dump(exclude_none=True)
|
||||
converted_action = {}
|
||||
for action_key, action_value in action_data.items():
|
||||
if action_key == "full_page":
|
||||
converted_action["fullPage"] = action_value
|
||||
else:
|
||||
converted_action[action_key] = action_value
|
||||
converted_actions.append(converted_action)
|
||||
scrape_data["actions"] = converted_actions
|
||||
elif key == "parsers":
|
||||
converted_parsers = []
|
||||
for parser in value:
|
||||
if isinstance(parser, str):
|
||||
converted_parsers.append(parser)
|
||||
elif isinstance(parser, dict):
|
||||
parser_data = dict(parser)
|
||||
if "max_pages" in parser_data:
|
||||
parser_data["maxPages"] = parser_data.pop("max_pages")
|
||||
converted_parsers.append(parser_data)
|
||||
else:
|
||||
parser_data = parser.model_dump(exclude_none=True)
|
||||
# Convert snake_case to camelCase for API
|
||||
if "max_pages" in parser_data:
|
||||
parser_data["maxPages"] = parser_data.pop("max_pages")
|
||||
converted_parsers.append(parser_data)
|
||||
scrape_data["parsers"] = converted_parsers
|
||||
elif key == "location":
|
||||
if isinstance(value, dict):
|
||||
scrape_data["location"] = value
|
||||
else:
|
||||
scrape_data["location"] = value.model_dump(exclude_none=True)
|
||||
elif key == "profile":
|
||||
if isinstance(value, dict):
|
||||
profile_data = {
|
||||
"name": value["name"],
|
||||
"saveChanges": value.get("save_changes", value.get("saveChanges", True)),
|
||||
}
|
||||
else:
|
||||
profile_data = {
|
||||
"name": value.name,
|
||||
"saveChanges": getattr(value, "save_changes", getattr(value, "saveChanges", True)),
|
||||
}
|
||||
scrape_data["profile"] = profile_data
|
||||
else:
|
||||
# For fields that don't need conversion, use as-is
|
||||
scrape_data[key] = value
|
||||
|
||||
return scrape_data
|
||||
Reference in New Issue
Block a user