참고소스 수정본

This commit is contained in:
LASTA_DEV01\lasta
2026-05-12 19:40:31 +09:00
parent 0f34a451fc
commit 2e9204243d
8708 changed files with 3259488 additions and 869 deletions

View File

@@ -0,0 +1,12 @@
namespace Firecrawl.Exceptions;
/// <summary>
/// Thrown when the API returns a 401 Unauthorized response.
/// </summary>
public class AuthenticationException : FirecrawlException
{
public AuthenticationException(string message, string? errorCode = null, object? details = null)
: base(message, 401, errorCode, details)
{
}
}

View File

@@ -0,0 +1,46 @@
namespace Firecrawl.Exceptions;
/// <summary>
/// Base exception for all Firecrawl SDK errors.
/// </summary>
public class FirecrawlException : Exception
{
/// <summary>
/// HTTP status code (0 if not an HTTP error).
/// </summary>
public int StatusCode { get; }
/// <summary>
/// Error code returned by the API, if any.
/// </summary>
public string? ErrorCode { get; }
/// <summary>
/// Additional error details from the API response.
/// </summary>
public object? Details { get; }
public FirecrawlException(string message)
: base(message)
{
}
public FirecrawlException(string message, int statusCode)
: base(message)
{
StatusCode = statusCode;
}
public FirecrawlException(string message, int statusCode, string? errorCode, object? details)
: base(message)
{
StatusCode = statusCode;
ErrorCode = errorCode;
Details = details;
}
public FirecrawlException(string message, Exception innerException)
: base(message, innerException)
{
}
}

View File

@@ -0,0 +1,24 @@
namespace Firecrawl.Exceptions;
/// <summary>
/// Thrown when an async job (crawl, batch scrape, agent) does not complete within the specified timeout.
/// </summary>
public class JobTimeoutException : FirecrawlException
{
/// <summary>
/// The ID of the job that timed out.
/// </summary>
public string JobId { get; }
/// <summary>
/// The timeout in seconds that was exceeded.
/// </summary>
public int TimeoutSeconds { get; }
public JobTimeoutException(string jobId, int timeoutSeconds, string jobType)
: base($"{jobType} job {jobId} did not complete within {timeoutSeconds} seconds")
{
JobId = jobId;
TimeoutSeconds = timeoutSeconds;
}
}

View File

@@ -0,0 +1,12 @@
namespace Firecrawl.Exceptions;
/// <summary>
/// Thrown when the API returns a 429 Too Many Requests response.
/// </summary>
public class RateLimitException : FirecrawlException
{
public RateLimitException(string message, string? errorCode = null, object? details = null)
: base(message, 429, errorCode, details)
{
}
}

View File

@@ -0,0 +1,35 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net8.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
<RootNamespace>Firecrawl</RootNamespace>
<!-- NuGet package metadata -->
<PackageId>firecrawl-sdk</PackageId>
<Version>1.3.1</Version>
<Authors>Firecrawl</Authors>
<Company>Firecrawl</Company>
<Description>.NET SDK for the Firecrawl API - web scraping, crawling, and data extraction</Description>
<PackageLicenseExpression>MIT</PackageLicenseExpression>
<PackageProjectUrl>https://github.com/firecrawl/firecrawl</PackageProjectUrl>
<RepositoryUrl>https://github.com/firecrawl/firecrawl</RepositoryUrl>
<RepositoryType>git</RepositoryType>
<PackageTags>firecrawl;web-scraping;crawling;api;sdk</PackageTags>
<PackageReadmeFile>README.md</PackageReadmeFile>
</PropertyGroup>
<ItemGroup>
<InternalsVisibleTo Include="Firecrawl.Tests" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="System.Text.Json" Version="9.0.3" />
</ItemGroup>
<ItemGroup>
<None Include="../README.md" Pack="true" PackagePath="/" />
</ItemGroup>
</Project>

View File

@@ -0,0 +1,670 @@
using System.Text.Json;
using Firecrawl.Exceptions;
using Firecrawl.Models;
using MonitorModel = Firecrawl.Models.Monitor;
namespace Firecrawl;
/// <summary>
/// Client for the Firecrawl v2 API.
///
/// <example>
/// <code>
/// var client = new FirecrawlClient("fc-your-api-key");
///
/// // Scrape a single page
/// var doc = await client.ScrapeAsync("https://example.com",
/// new ScrapeOptions { Formats = new List&lt;object&gt; { "markdown" } });
///
/// // Crawl a website
/// var job = await client.CrawlAsync("https://example.com",
/// new CrawlOptions { Limit = 50 });
/// </code>
/// </example>
/// </summary>
public class FirecrawlClient
{
private const string DefaultApiUrl = "https://api.firecrawl.dev";
private static readonly TimeSpan DefaultTimeout = TimeSpan.FromMinutes(5);
private const int DefaultMaxRetries = 3;
private const double DefaultBackoffFactor = 0.5;
private const int DefaultPollIntervalSec = 2;
private const int DefaultJobTimeoutSec = 300;
private readonly FirecrawlHttpClient _http;
/// <summary>
/// Creates a new FirecrawlClient with the specified API key.
/// </summary>
/// <param name="apiKey">The Firecrawl API key.</param>
/// <param name="apiUrl">Optional API base URL (defaults to https://api.firecrawl.dev).</param>
/// <param name="timeout">Optional HTTP request timeout.</param>
/// <param name="maxRetries">Optional maximum number of retries for transient failures.</param>
/// <param name="backoffFactor">Optional exponential backoff factor in seconds.</param>
/// <param name="httpClient">Optional pre-configured HttpClient instance.</param>
public FirecrawlClient(
string? apiKey = null,
string? apiUrl = null,
TimeSpan? timeout = null,
int maxRetries = DefaultMaxRetries,
double backoffFactor = DefaultBackoffFactor,
HttpClient? httpClient = null)
{
var resolvedKey = ResolveApiKey(apiKey);
var resolvedUrl = ResolveApiUrl(apiUrl);
_http = new FirecrawlHttpClient(
resolvedKey,
resolvedUrl,
timeout ?? DefaultTimeout,
maxRetries,
backoffFactor,
httpClient);
}
// ================================================================
// SCRAPE
// ================================================================
/// <summary>
/// Scrapes a single URL and returns the document.
/// </summary>
public async Task<Document> ScrapeAsync(
string url,
ScrapeOptions? options = null,
CancellationToken cancellationToken = default)
{
ArgumentNullException.ThrowIfNull(url);
var body = BuildBody(options);
body["url"] = url;
var response = await _http.PostAsync<ApiResponse<Document>>(
"/v2/scrape", body, cancellationToken: cancellationToken);
return response.Data ?? throw new FirecrawlException("Scrape response contained no data");
}
// ================================================================
// CRAWL
// ================================================================
/// <summary>
/// Starts an async crawl job and returns immediately.
/// </summary>
public async Task<CrawlResponse> StartCrawlAsync(
string url,
CrawlOptions? options = null,
CancellationToken cancellationToken = default)
{
ArgumentNullException.ThrowIfNull(url);
var body = BuildBody(options);
body["url"] = url;
return await _http.PostAsync<CrawlResponse>(
"/v2/crawl", body, cancellationToken: cancellationToken);
}
/// <summary>
/// Gets the status and results of a crawl job.
/// </summary>
public async Task<CrawlJob> GetCrawlStatusAsync(
string jobId,
CancellationToken cancellationToken = default)
{
ArgumentNullException.ThrowIfNull(jobId);
return await _http.GetAsync<CrawlJob>(
$"/v2/crawl/{jobId}", cancellationToken);
}
/// <summary>
/// Crawls a website and waits for completion (auto-polling).
/// </summary>
public async Task<CrawlJob> CrawlAsync(
string url,
CrawlOptions? options = null,
int pollIntervalSec = DefaultPollIntervalSec,
int timeoutSec = DefaultJobTimeoutSec,
CancellationToken cancellationToken = default)
{
var start = await StartCrawlAsync(url, options, cancellationToken);
return await PollCrawlAsync(
start.Id ?? throw new FirecrawlException("Crawl start did not return a job ID"),
pollIntervalSec, timeoutSec, cancellationToken);
}
/// <summary>
/// Cancels a running crawl job.
/// </summary>
public async Task<Dictionary<string, object>> CancelCrawlAsync(
string jobId,
CancellationToken cancellationToken = default)
{
ArgumentNullException.ThrowIfNull(jobId);
return await _http.DeleteAsync<Dictionary<string, object>>(
$"/v2/crawl/{jobId}", cancellationToken);
}
/// <summary>
/// Gets errors from a crawl job.
/// </summary>
public async Task<Dictionary<string, object>> GetCrawlErrorsAsync(
string jobId,
CancellationToken cancellationToken = default)
{
ArgumentNullException.ThrowIfNull(jobId);
return await _http.GetAsync<Dictionary<string, object>>(
$"/v2/crawl/{jobId}/errors", cancellationToken);
}
// ================================================================
// BATCH SCRAPE
// ================================================================
/// <summary>
/// Starts an async batch scrape job.
/// </summary>
public async Task<BatchScrapeResponse> StartBatchScrapeAsync(
List<string> urls,
BatchScrapeOptions? options = null,
CancellationToken cancellationToken = default)
{
ArgumentNullException.ThrowIfNull(urls);
var body = BuildBody(options);
body["urls"] = urls;
// The API expects scrape options flattened at the top level
if (body.TryGetValue("options", out var nested) && nested is JsonElement nestedElement)
{
body.Remove("options");
var nestedDict = JsonSerializer.Deserialize<Dictionary<string, object>>(
nestedElement.GetRawText(), FirecrawlHttpClient.JsonOptions);
if (nestedDict != null)
{
var batchFields = new Dictionary<string, object>(body);
foreach (var kv in nestedDict)
body.TryAdd(kv.Key, kv.Value);
foreach (var kv in batchFields)
body[kv.Key] = kv.Value;
}
}
Dictionary<string, string>? extraHeaders = null;
if (options?.IdempotencyKey is { Length: > 0 } idempotencyKey)
{
extraHeaders = new Dictionary<string, string>
{
["x-idempotency-key"] = idempotencyKey
};
}
return await _http.PostAsync<BatchScrapeResponse>(
"/v2/batch/scrape", body, extraHeaders, cancellationToken);
}
/// <summary>
/// Gets the status and results of a batch scrape job.
/// </summary>
public async Task<BatchScrapeJob> GetBatchScrapeStatusAsync(
string jobId,
CancellationToken cancellationToken = default)
{
ArgumentNullException.ThrowIfNull(jobId);
return await _http.GetAsync<BatchScrapeJob>(
$"/v2/batch/scrape/{jobId}", cancellationToken);
}
/// <summary>
/// Batch-scrapes URLs and waits for completion (auto-polling).
/// </summary>
public async Task<BatchScrapeJob> BatchScrapeAsync(
List<string> urls,
BatchScrapeOptions? options = null,
int pollIntervalSec = DefaultPollIntervalSec,
int timeoutSec = DefaultJobTimeoutSec,
CancellationToken cancellationToken = default)
{
var start = await StartBatchScrapeAsync(urls, options, cancellationToken);
return await PollBatchScrapeAsync(
start.Id ?? throw new FirecrawlException("Batch scrape start did not return a job ID"),
pollIntervalSec, timeoutSec, cancellationToken);
}
/// <summary>
/// Cancels a running batch scrape job.
/// </summary>
public async Task<Dictionary<string, object>> CancelBatchScrapeAsync(
string jobId,
CancellationToken cancellationToken = default)
{
ArgumentNullException.ThrowIfNull(jobId);
return await _http.DeleteAsync<Dictionary<string, object>>(
$"/v2/batch/scrape/{jobId}", cancellationToken);
}
// ================================================================
// PARSE
// ================================================================
/// <summary>
/// Parses an uploaded file (HTML, PDF, DOCX, etc.) via <c>/v2/parse</c>
/// and returns the extracted document.
/// </summary>
/// <param name="file">The file to upload.</param>
/// <param name="options">Optional parse options. Browser-only formats
/// (changeTracking, screenshot, branding), actions, waitFor, location,
/// and mobile are rejected.</param>
/// <param name="cancellationToken">Cancellation token.</param>
public async Task<Document> ParseAsync(
ParseFile file,
ParseOptions? options = null,
CancellationToken cancellationToken = default)
{
ArgumentNullException.ThrowIfNull(file);
var filename = file.Filename?.Trim();
if (string.IsNullOrEmpty(filename))
throw new ArgumentException("filename cannot be empty", nameof(file));
if (file.Content is null || file.Content.Length == 0)
throw new ArgumentException("file content cannot be empty", nameof(file));
options?.Validate();
var optionsJson = JsonSerializer.Serialize(
options ?? new ParseOptions(),
FirecrawlHttpClient.JsonOptions);
var fields = new Dictionary<string, string>
{
["options"] = optionsJson,
};
var response = await _http.PostMultipartAsync<ApiResponse<Document>>(
"/v2/parse",
fields,
fileField: "file",
fileName: filename,
fileContentType: file.ResolveContentType(),
fileContent: file.Content,
cancellationToken: cancellationToken);
return response.Data ?? throw new FirecrawlException("Parse response contained no data");
}
// ================================================================
// MAP
// ================================================================
/// <summary>
/// Discovers URLs on a website.
/// </summary>
public async Task<MapData> MapAsync(
string url,
MapOptions? options = null,
CancellationToken cancellationToken = default)
{
ArgumentNullException.ThrowIfNull(url);
var body = BuildBody(options);
body["url"] = url;
var response = await _http.PostAsync<ApiResponse<MapData>>(
"/v2/map", body, cancellationToken: cancellationToken);
return response.Data ?? throw new FirecrawlException("Map response contained no data");
}
// ================================================================
// MONITOR
// ================================================================
public async Task<MonitorModel> CreateMonitorAsync(
CreateMonitorRequest request,
CancellationToken cancellationToken = default)
{
ArgumentNullException.ThrowIfNull(request);
var response = await _http.PostAsync<ApiResponse<MonitorModel>>(
"/v2/monitor", request, cancellationToken: cancellationToken);
return response.Data ?? throw new FirecrawlException("Create monitor response contained no data");
}
public async Task<List<MonitorModel>> ListMonitorsAsync(
int? limit = null,
int? offset = null,
CancellationToken cancellationToken = default)
{
var response = await _http.GetAsync<ApiResponse<List<MonitorModel>>>(
$"/v2/monitor{BuildQuery(limit, offset)}", cancellationToken);
return response.Data ?? new List<MonitorModel>();
}
public async Task<MonitorModel> GetMonitorAsync(
string monitorId,
CancellationToken cancellationToken = default)
{
ArgumentNullException.ThrowIfNull(monitorId);
var response = await _http.GetAsync<ApiResponse<MonitorModel>>(
$"/v2/monitor/{monitorId}", cancellationToken);
return response.Data ?? throw new FirecrawlException("Get monitor response contained no data");
}
public async Task<MonitorModel> UpdateMonitorAsync(
string monitorId,
UpdateMonitorRequest request,
CancellationToken cancellationToken = default)
{
ArgumentNullException.ThrowIfNull(monitorId);
ArgumentNullException.ThrowIfNull(request);
var response = await _http.PatchAsync<ApiResponse<MonitorModel>>(
$"/v2/monitor/{monitorId}", request, cancellationToken);
return response.Data ?? throw new FirecrawlException("Update monitor response contained no data");
}
public async Task<bool> DeleteMonitorAsync(
string monitorId,
CancellationToken cancellationToken = default)
{
ArgumentNullException.ThrowIfNull(monitorId);
var response = await _http.DeleteAsync<Dictionary<string, object>>(
$"/v2/monitor/{monitorId}", cancellationToken);
return response.TryGetValue("success", out var success) && success switch
{
bool value => value,
JsonElement element when element.ValueKind == JsonValueKind.True => true,
_ => false
};
}
public async Task<MonitorCheck> RunMonitorAsync(
string monitorId,
CancellationToken cancellationToken = default)
{
ArgumentNullException.ThrowIfNull(monitorId);
var response = await _http.PostAsync<ApiResponse<MonitorCheck>>(
$"/v2/monitor/{monitorId}/run", new Dictionary<string, object>(), cancellationToken: cancellationToken);
return response.Data ?? throw new FirecrawlException("Run monitor response contained no data");
}
public async Task<List<MonitorCheck>> ListMonitorChecksAsync(
string monitorId,
int? limit = null,
int? offset = null,
CancellationToken cancellationToken = default)
{
ArgumentNullException.ThrowIfNull(monitorId);
var response = await _http.GetAsync<ApiResponse<List<MonitorCheck>>>(
$"/v2/monitor/{monitorId}/checks{BuildQuery(limit, offset)}", cancellationToken);
return response.Data ?? new List<MonitorCheck>();
}
public async Task<MonitorCheckDetail> GetMonitorCheckAsync(
string monitorId,
string checkId,
int? limit = null,
int? skip = null,
string? status = null,
bool autoPaginate = true,
CancellationToken cancellationToken = default)
{
ArgumentNullException.ThrowIfNull(monitorId);
ArgumentNullException.ThrowIfNull(checkId);
var response = await _http.GetAsync<ApiResponse<MonitorCheckDetail>>(
$"/v2/monitor/{monitorId}/checks/{checkId}{BuildMonitorCheckQuery(limit, skip, status)}",
cancellationToken);
var check = response.Data ?? throw new FirecrawlException("Get monitor check response contained no data");
return autoPaginate ? await PaginateMonitorCheckAsync(check, cancellationToken) : check;
}
// ================================================================
// SEARCH
// ================================================================
/// <summary>
/// Performs a web search.
/// </summary>
public async Task<SearchData> SearchAsync(
string query,
SearchOptions? options = null,
CancellationToken cancellationToken = default)
{
ArgumentNullException.ThrowIfNull(query);
var body = BuildBody(options);
body["query"] = query;
var response = await _http.PostAsync<ApiResponse<SearchData>>(
"/v2/search", body, cancellationToken: cancellationToken);
return response.Data ?? throw new FirecrawlException("Search response contained no data");
}
// ================================================================
// USAGE & METRICS
// ================================================================
/// <summary>
/// Gets current concurrency usage.
/// </summary>
public async Task<ConcurrencyCheck> GetConcurrencyAsync(
CancellationToken cancellationToken = default)
{
return await _http.GetAsync<ConcurrencyCheck>(
"/v2/concurrency-check", cancellationToken);
}
/// <summary>
/// Gets current credit usage.
/// </summary>
public async Task<CreditUsage> GetCreditUsageAsync(
CancellationToken cancellationToken = default)
{
return await _http.GetAsync<CreditUsage>(
"/v2/team/credit-usage", cancellationToken);
}
// ================================================================
// INTERNAL POLLING HELPERS
// ================================================================
private async Task<CrawlJob> PollCrawlAsync(
string jobId,
int pollIntervalSec,
int timeoutSec,
CancellationToken cancellationToken)
{
var deadline = DateTime.UtcNow.AddSeconds(timeoutSec);
while (DateTime.UtcNow < deadline)
{
cancellationToken.ThrowIfCancellationRequested();
var job = await GetCrawlStatusAsync(jobId, cancellationToken);
if (job.IsDone)
return await PaginateCrawlAsync(job, cancellationToken);
await Task.Delay(TimeSpan.FromSeconds(pollIntervalSec), cancellationToken);
}
throw new JobTimeoutException(jobId, timeoutSec, "Crawl");
}
private async Task<BatchScrapeJob> PollBatchScrapeAsync(
string jobId,
int pollIntervalSec,
int timeoutSec,
CancellationToken cancellationToken)
{
var deadline = DateTime.UtcNow.AddSeconds(timeoutSec);
while (DateTime.UtcNow < deadline)
{
cancellationToken.ThrowIfCancellationRequested();
var job = await GetBatchScrapeStatusAsync(jobId, cancellationToken);
if (job.IsDone)
return await PaginateBatchScrapeAsync(job, cancellationToken);
await Task.Delay(TimeSpan.FromSeconds(pollIntervalSec), cancellationToken);
}
throw new JobTimeoutException(jobId, timeoutSec, "Batch scrape");
}
private async Task<CrawlJob> PaginateCrawlAsync(
CrawlJob job,
CancellationToken cancellationToken)
{
job.Data ??= new List<Document>();
var current = job;
while (!string.IsNullOrEmpty(current.Next))
{
var nextPage = await _http.GetAbsoluteAsync<CrawlJob>(
current.Next, cancellationToken);
if (nextPage.Data is { Count: > 0 })
job.Data.AddRange(nextPage.Data);
current = nextPage;
}
job.Next = null;
return job;
}
private async Task<BatchScrapeJob> PaginateBatchScrapeAsync(
BatchScrapeJob job,
CancellationToken cancellationToken)
{
job.Data ??= new List<Document>();
var current = job;
while (!string.IsNullOrEmpty(current.Next))
{
var nextPage = await _http.GetAbsoluteAsync<BatchScrapeJob>(
current.Next, cancellationToken);
if (nextPage.Data is { Count: > 0 })
job.Data.AddRange(nextPage.Data);
current = nextPage;
}
job.Next = null;
return job;
}
private async Task<MonitorCheckDetail> PaginateMonitorCheckAsync(
MonitorCheckDetail check,
CancellationToken cancellationToken)
{
check.Pages ??= new List<MonitorCheckPage>();
var current = check;
while (!string.IsNullOrEmpty(current.Next))
{
var response = await _http.GetAbsoluteAsync<ApiResponse<MonitorCheckDetail>>(
current.Next, cancellationToken);
if (response.Data == null)
break;
var nextPage = response.Data;
if (nextPage.Pages is { Count: > 0 })
check.Pages.AddRange(nextPage.Pages);
current = nextPage;
}
check.Next = null;
return check;
}
// ================================================================
// INTERNAL UTILITIES
// ================================================================
private static Dictionary<string, object> BuildBody(object? options)
{
if (options == null)
return new Dictionary<string, object>();
var json = JsonSerializer.Serialize(options, FirecrawlHttpClient.JsonOptions);
return JsonSerializer.Deserialize<Dictionary<string, object>>(json, FirecrawlHttpClient.JsonOptions)
?? new Dictionary<string, object>();
}
private static string BuildQuery(int? limit = null, int? offset = null, string? status = null)
{
var query = new List<string>();
if (limit.HasValue)
query.Add($"limit={Uri.EscapeDataString(limit.Value.ToString())}");
if (offset.HasValue)
query.Add($"offset={Uri.EscapeDataString(offset.Value.ToString())}");
if (!string.IsNullOrWhiteSpace(status))
query.Add($"status={Uri.EscapeDataString(status)}");
return query.Count == 0 ? string.Empty : "?" + string.Join("&", query);
}
private static string BuildMonitorCheckQuery(int? limit = null, int? skip = null, string? status = null)
{
var query = new List<string>();
if (limit.HasValue)
query.Add($"limit={Uri.EscapeDataString(limit.Value.ToString())}");
if (skip.HasValue)
query.Add($"skip={Uri.EscapeDataString(skip.Value.ToString())}");
if (!string.IsNullOrWhiteSpace(status))
query.Add($"status={Uri.EscapeDataString(status)}");
return query.Count == 0 ? string.Empty : "?" + string.Join("&", query);
}
private static string ResolveApiKey(string? apiKey)
{
if (!string.IsNullOrWhiteSpace(apiKey))
return apiKey;
var envKey = Environment.GetEnvironmentVariable("FIRECRAWL_API_KEY");
if (!string.IsNullOrWhiteSpace(envKey))
return envKey;
throw new FirecrawlException(
"API key is required. Pass it to the constructor or set the FIRECRAWL_API_KEY environment variable.");
}
private static string ResolveApiUrl(string? apiUrl)
{
if (!string.IsNullOrWhiteSpace(apiUrl))
return apiUrl;
var envUrl = Environment.GetEnvironmentVariable("FIRECRAWL_API_URL");
if (!string.IsNullOrWhiteSpace(envUrl))
return envUrl;
return DefaultApiUrl;
}
}

View File

@@ -0,0 +1,318 @@
using System.Net;
using System.Net.Http.Headers;
using System.Text;
using System.Text.Json;
using System.Text.Json.Serialization;
using Firecrawl.Exceptions;
namespace Firecrawl;
/// <summary>
/// Internal HTTP client for making authenticated requests to the Firecrawl API.
/// Handles retry logic with exponential backoff.
/// </summary>
internal class FirecrawlHttpClient
{
private readonly HttpClient _httpClient;
private readonly string _apiKey;
private readonly string _baseUrl;
private readonly int _maxRetries;
private readonly double _backoffFactor;
internal static readonly JsonSerializerOptions JsonOptions = new()
{
PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
PropertyNameCaseInsensitive = true
};
internal FirecrawlHttpClient(
string apiKey,
string baseUrl,
TimeSpan timeout,
int maxRetries,
double backoffFactor,
HttpClient? httpClient = null)
{
_apiKey = apiKey;
_baseUrl = baseUrl.TrimEnd('/');
_maxRetries = maxRetries;
_backoffFactor = backoffFactor;
if (httpClient != null)
{
_httpClient = httpClient;
}
else
{
_httpClient = new HttpClient { Timeout = timeout };
}
}
internal async Task<T> PostAsync<T>(
string path,
object body,
Dictionary<string, string>? extraHeaders = null,
CancellationToken cancellationToken = default)
{
var url = _baseUrl + path;
var json = JsonSerializer.Serialize(body, JsonOptions);
HttpRequestMessage BuildRequest()
{
var content = new StringContent(json, Encoding.UTF8, "application/json");
var request = new HttpRequestMessage(HttpMethod.Post, url) { Content = content };
ApplyStandardHeaders(request);
if (extraHeaders != null)
{
foreach (var header in extraHeaders)
{
request.Headers.TryAddWithoutValidation(header.Key, header.Value);
}
}
return request;
}
return await ExecuteWithRetryAsync<T>(BuildRequest, cancellationToken);
}
internal async Task<T> PostMultipartAsync<T>(
string path,
Dictionary<string, string> fields,
string fileField,
string fileName,
string fileContentType,
byte[] fileContent,
Dictionary<string, string>? extraHeaders = null,
CancellationToken cancellationToken = default)
{
var url = _baseUrl + path;
HttpRequestMessage BuildRequest()
{
var content = new MultipartFormDataContent();
foreach (var kv in fields)
{
var fieldContent = new StringContent(kv.Value, Encoding.UTF8);
fieldContent.Headers.ContentType = null;
content.Add(fieldContent, kv.Key);
}
var fileBytes = new ByteArrayContent(fileContent);
fileBytes.Headers.ContentType =
MediaTypeHeaderValue.Parse(string.IsNullOrWhiteSpace(fileContentType)
? "application/octet-stream"
: fileContentType);
content.Add(fileBytes, fileField, fileName);
var request = new HttpRequestMessage(HttpMethod.Post, url) { Content = content };
ApplyStandardHeaders(request);
if (extraHeaders != null)
{
foreach (var header in extraHeaders)
{
request.Headers.TryAddWithoutValidation(header.Key, header.Value);
}
}
return request;
}
return await ExecuteWithRetryAsync<T>(BuildRequest, cancellationToken);
}
internal async Task<T> GetAsync<T>(string path, CancellationToken cancellationToken = default)
{
var url = _baseUrl + path;
HttpRequestMessage BuildRequest()
{
var request = new HttpRequestMessage(HttpMethod.Get, url);
ApplyStandardHeaders(request);
return request;
}
return await ExecuteWithRetryAsync<T>(BuildRequest, cancellationToken);
}
internal async Task<T> PatchAsync<T>(
string path,
object body,
CancellationToken cancellationToken = default)
{
var url = _baseUrl + path;
var json = JsonSerializer.Serialize(body, JsonOptions);
HttpRequestMessage BuildRequest()
{
var content = new StringContent(json, Encoding.UTF8, "application/json");
var request = new HttpRequestMessage(HttpMethod.Patch, url) { Content = content };
ApplyStandardHeaders(request);
return request;
}
return await ExecuteWithRetryAsync<T>(BuildRequest, cancellationToken);
}
internal async Task<T> GetAbsoluteAsync<T>(string absoluteUrl, CancellationToken cancellationToken = default)
{
// Validate that the pagination URL belongs to the same host to prevent API key exfiltration
var targetUri = new Uri(absoluteUrl);
var baseUri = new Uri(_baseUrl);
if (!string.Equals(targetUri.Scheme, baseUri.Scheme, StringComparison.OrdinalIgnoreCase) ||
!string.Equals(targetUri.Host, baseUri.Host, StringComparison.OrdinalIgnoreCase) ||
targetUri.Port != baseUri.Port)
{
throw new FirecrawlException(
$"Pagination URL origin '{targetUri.Scheme}://{targetUri.Host}:{targetUri.Port}' does not match API base URL origin '{baseUri.Scheme}://{baseUri.Host}:{baseUri.Port}'. " +
"Refusing to send credentials to a different origin.");
}
HttpRequestMessage BuildRequest()
{
var request = new HttpRequestMessage(HttpMethod.Get, absoluteUrl);
ApplyStandardHeaders(request);
return request;
}
return await ExecuteWithRetryAsync<T>(BuildRequest, cancellationToken);
}
internal async Task<T> DeleteAsync<T>(string path, CancellationToken cancellationToken = default)
{
var url = _baseUrl + path;
HttpRequestMessage BuildRequest()
{
var request = new HttpRequestMessage(HttpMethod.Delete, url);
ApplyStandardHeaders(request);
return request;
}
return await ExecuteWithRetryAsync<T>(BuildRequest, cancellationToken);
}
private void ApplyStandardHeaders(HttpRequestMessage request)
{
request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", _apiKey);
request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json"));
}
private async Task<T> ExecuteWithRetryAsync<T>(
Func<HttpRequestMessage> requestBuilder,
CancellationToken cancellationToken)
{
var attempt = 0;
while (true)
{
// Build a fresh request for each attempt (HttpRequestMessage can only be sent once,
// and multipart content is not cheaply cloneable).
using var request = requestBuilder();
HttpResponseMessage? response = null;
try
{
response = await _httpClient.SendAsync(request, cancellationToken);
var bodyStr = await response.Content.ReadAsStringAsync(cancellationToken);
if (response.IsSuccessStatusCode)
{
return JsonSerializer.Deserialize<T>(bodyStr, JsonOptions)
?? throw new FirecrawlException("Failed to deserialize response");
}
var code = (int)response.StatusCode;
var errorMessage = ExtractErrorMessage(bodyStr, code);
var errorCode = ExtractErrorCode(bodyStr);
// Non-retryable client errors
if (code == 401)
throw new AuthenticationException(errorMessage, errorCode);
if (code == 429)
throw new RateLimitException(errorMessage, errorCode);
if (code >= 400 && code < 500 && code != 408 && code != 409)
throw new FirecrawlException(errorMessage, code, errorCode, null);
// Retryable errors: 408, 409, 502, 5xx
if (attempt < _maxRetries)
{
attempt++;
await SleepWithBackoffAsync(attempt, cancellationToken);
continue;
}
throw new FirecrawlException(errorMessage, code, errorCode, null);
}
catch (FirecrawlException)
{
throw;
}
catch (OperationCanceledException)
{
throw;
}
catch (HttpRequestException ex)
{
if (attempt < _maxRetries)
{
attempt++;
await SleepWithBackoffAsync(attempt, cancellationToken);
continue;
}
throw new FirecrawlException($"Request failed: {ex.Message}", ex);
}
finally
{
response?.Dispose();
}
}
}
private static string ExtractErrorMessage(string body, int statusCode)
{
try
{
using var doc = JsonDocument.Parse(body);
var root = doc.RootElement;
if (root.TryGetProperty("error", out var errorProp))
return errorProp.GetString() ?? $"HTTP {statusCode} error";
if (root.TryGetProperty("message", out var messageProp))
return messageProp.GetString() ?? $"HTTP {statusCode} error";
}
catch
{
// ignored
}
return $"HTTP {statusCode} error";
}
private static string? ExtractErrorCode(string body)
{
try
{
using var doc = JsonDocument.Parse(body);
if (doc.RootElement.TryGetProperty("code", out var codeProp))
return codeProp.GetString();
}
catch
{
// ignored
}
return null;
}
private async Task SleepWithBackoffAsync(int attempt, CancellationToken cancellationToken)
{
var delayMs = (int)(_backoffFactor * 1000 * Math.Pow(2, attempt - 1));
await Task.Delay(delayMs, cancellationToken);
}
}

View File

@@ -0,0 +1,15 @@
using System.Text.Json.Serialization;
namespace Firecrawl.Models;
/// <summary>
/// Internal wrapper for API responses that contain a "data" field.
/// </summary>
internal class ApiResponse<T>
{
[JsonPropertyName("success")]
public bool Success { get; set; }
[JsonPropertyName("data")]
public T? Data { get; set; }
}

View File

@@ -0,0 +1,39 @@
using System.Text.Json.Serialization;
namespace Firecrawl.Models;
/// <summary>
/// Status and results of a batch scrape job.
/// </summary>
public class BatchScrapeJob
{
[JsonPropertyName("id")]
public string? Id { get; set; }
[JsonPropertyName("status")]
public string? Status { get; set; }
[JsonPropertyName("completed")]
public int? Completed { get; set; }
[JsonPropertyName("total")]
public int? Total { get; set; }
[JsonPropertyName("creditsUsed")]
public int? CreditsUsed { get; set; }
[JsonPropertyName("expiresAt")]
public string? ExpiresAt { get; set; }
[JsonPropertyName("next")]
public string? Next { get; set; }
[JsonPropertyName("data")]
public List<Document>? Data { get; set; }
/// <summary>
/// Returns true if the batch scrape job has reached a terminal state.
/// </summary>
public bool IsDone =>
Status == "completed" || Status == "cancelled" || Status == "failed";
}

View File

@@ -0,0 +1,43 @@
using System.Text.Json.Serialization;
namespace Firecrawl.Models;
/// <summary>
/// Configuration options for batch scraping multiple URLs.
/// </summary>
public class BatchScrapeOptions
{
[JsonPropertyName("options")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public ScrapeOptions? Options { get; set; }
[JsonPropertyName("webhook")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public object? Webhook { get; set; }
[JsonPropertyName("appendToId")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? AppendToId { get; set; }
[JsonPropertyName("ignoreInvalidURLs")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public bool? IgnoreInvalidURLs { get; set; }
[JsonPropertyName("maxConcurrency")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public int? MaxConcurrency { get; set; }
[JsonPropertyName("zeroDataRetention")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public bool? ZeroDataRetention { get; set; }
[JsonPropertyName("integration")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? Integration { get; set; }
/// <summary>
/// Idempotency key sent as the x-idempotency-key HTTP header (not in the JSON body).
/// </summary>
[JsonIgnore]
public string? IdempotencyKey { get; set; }
}

View File

@@ -0,0 +1,18 @@
using System.Text.Json.Serialization;
namespace Firecrawl.Models;
/// <summary>
/// Response from starting an async batch scrape job.
/// </summary>
public class BatchScrapeResponse
{
[JsonPropertyName("success")]
public bool Success { get; set; }
[JsonPropertyName("id")]
public string? Id { get; set; }
[JsonPropertyName("invalidURLs")]
public List<string>? InvalidURLs { get; set; }
}

View File

@@ -0,0 +1,15 @@
using System.Text.Json.Serialization;
namespace Firecrawl.Models;
/// <summary>
/// Current concurrency usage information.
/// </summary>
public class ConcurrencyCheck
{
[JsonPropertyName("current")]
public int Current { get; set; }
[JsonPropertyName("max")]
public int MaxConcurrency { get; set; }
}

View File

@@ -0,0 +1,39 @@
using System.Text.Json.Serialization;
namespace Firecrawl.Models;
/// <summary>
/// Status and results of a crawl job.
/// </summary>
public class CrawlJob
{
[JsonPropertyName("id")]
public string? Id { get; set; }
[JsonPropertyName("status")]
public string? Status { get; set; }
[JsonPropertyName("completed")]
public int? Completed { get; set; }
[JsonPropertyName("total")]
public int? Total { get; set; }
[JsonPropertyName("creditsUsed")]
public int? CreditsUsed { get; set; }
[JsonPropertyName("expiresAt")]
public string? ExpiresAt { get; set; }
[JsonPropertyName("next")]
public string? Next { get; set; }
[JsonPropertyName("data")]
public List<Document>? Data { get; set; }
/// <summary>
/// Returns true if the crawl job has reached a terminal state.
/// </summary>
public bool IsDone =>
Status == "completed" || Status == "cancelled" || Status == "failed";
}

View File

@@ -0,0 +1,81 @@
using System.Text.Json.Serialization;
namespace Firecrawl.Models;
/// <summary>
/// Configuration options for crawling a website.
/// </summary>
public class CrawlOptions
{
[JsonPropertyName("prompt")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? Prompt { get; set; }
[JsonPropertyName("excludePaths")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public List<string>? ExcludePaths { get; set; }
[JsonPropertyName("includePaths")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public List<string>? IncludePaths { get; set; }
[JsonPropertyName("maxDiscoveryDepth")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public int? MaxDiscoveryDepth { get; set; }
[JsonPropertyName("sitemap")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? Sitemap { get; set; }
[JsonPropertyName("ignoreQueryParameters")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public bool? IgnoreQueryParameters { get; set; }
[JsonPropertyName("deduplicateSimilarURLs")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public bool? DeduplicateSimilarURLs { get; set; }
[JsonPropertyName("limit")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public int? Limit { get; set; }
[JsonPropertyName("crawlEntireDomain")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public bool? CrawlEntireDomain { get; set; }
[JsonPropertyName("allowExternalLinks")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public bool? AllowExternalLinks { get; set; }
[JsonPropertyName("allowSubdomains")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public bool? AllowSubdomains { get; set; }
[JsonPropertyName("delay")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public int? Delay { get; set; }
[JsonPropertyName("maxConcurrency")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public int? MaxConcurrency { get; set; }
[JsonPropertyName("webhook")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public object? Webhook { get; set; }
[JsonPropertyName("scrapeOptions")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public ScrapeOptions? ScrapeOptions { get; set; }
[JsonPropertyName("regexOnFullURL")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public bool? RegexOnFullURL { get; set; }
[JsonPropertyName("zeroDataRetention")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public bool? ZeroDataRetention { get; set; }
[JsonPropertyName("integration")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? Integration { get; set; }
}

View File

@@ -0,0 +1,18 @@
using System.Text.Json.Serialization;
namespace Firecrawl.Models;
/// <summary>
/// Response from starting an async crawl job.
/// </summary>
public class CrawlResponse
{
[JsonPropertyName("success")]
public bool Success { get; set; }
[JsonPropertyName("id")]
public string? Id { get; set; }
[JsonPropertyName("url")]
public string? Url { get; set; }
}

View File

@@ -0,0 +1,21 @@
using System.Text.Json.Serialization;
namespace Firecrawl.Models;
/// <summary>
/// Account credit usage information.
/// </summary>
public class CreditUsage
{
[JsonPropertyName("remaining_credits")]
public int? RemainingCredits { get; set; }
[JsonPropertyName("total_credits_used")]
public int? TotalCreditsUsed { get; set; }
[JsonPropertyName("billing_period_start")]
public string? BillingPeriodStart { get; set; }
[JsonPropertyName("billing_period_end")]
public string? BillingPeriodEnd { get; set; }
}

View File

@@ -0,0 +1,57 @@
using System.Text.Json.Serialization;
namespace Firecrawl.Models;
/// <summary>
/// Represents a scraped document returned by the Firecrawl API.
/// </summary>
public class Document
{
[JsonPropertyName("markdown")]
public string? Markdown { get; set; }
[JsonPropertyName("html")]
public string? Html { get; set; }
[JsonPropertyName("rawHtml")]
public string? RawHtml { get; set; }
[JsonPropertyName("json")]
public object? Json { get; set; }
[JsonPropertyName("summary")]
public string? Summary { get; set; }
[JsonPropertyName("metadata")]
public Dictionary<string, object>? Metadata { get; set; }
[JsonPropertyName("links")]
public List<string>? Links { get; set; }
[JsonPropertyName("images")]
public List<string>? Images { get; set; }
[JsonPropertyName("screenshot")]
public string? Screenshot { get; set; }
[JsonPropertyName("audio")]
public object? Audio { get; set; }
[JsonPropertyName("actions")]
public object? Actions { get; set; }
[JsonPropertyName("answer")]
public string? Answer { get; set; }
[JsonPropertyName("highlights")]
public string? Highlights { get; set; }
[JsonPropertyName("warning")]
public string? Warning { get; set; }
[JsonPropertyName("changeTracking")]
public object? ChangeTracking { get; set; }
[JsonPropertyName("branding")]
public object? Branding { get; set; }
}

View File

@@ -0,0 +1,15 @@
using System.Text.Json.Serialization;
namespace Firecrawl.Models;
/// <summary>
/// Highlights format specification for use in ScrapeOptions.Formats.
/// </summary>
public class HighlightsFormat
{
[JsonPropertyName("type")]
public string Type { get; } = "highlights";
[JsonPropertyName("query")]
public required string Query { get; set; }
}

View File

@@ -0,0 +1,20 @@
using System.Text.Json.Serialization;
namespace Firecrawl.Models;
/// <summary>
/// JSON extraction format specification for use in ScrapeOptions.Formats.
/// </summary>
public class JsonFormat
{
[JsonPropertyName("type")]
public string Type { get; } = "json";
[JsonPropertyName("prompt")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? Prompt { get; set; }
[JsonPropertyName("schema")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public Dictionary<string, object>? Schema { get; set; }
}

View File

@@ -0,0 +1,17 @@
using System.Text.Json.Serialization;
namespace Firecrawl.Models;
/// <summary>
/// Geolocation configuration for requests.
/// </summary>
public class LocationConfig
{
[JsonPropertyName("country")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? Country { get; set; }
[JsonPropertyName("languages")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public List<string>? Languages { get; set; }
}

View File

@@ -0,0 +1,12 @@
using System.Text.Json.Serialization;
namespace Firecrawl.Models;
/// <summary>
/// URL discovery (map) results.
/// </summary>
public class MapData
{
[JsonPropertyName("links")]
public List<string>? Links { get; set; }
}

View File

@@ -0,0 +1,45 @@
using System.Text.Json.Serialization;
namespace Firecrawl.Models;
/// <summary>
/// Configuration options for URL discovery (map).
/// </summary>
public class MapOptions
{
[JsonPropertyName("search")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? Search { get; set; }
[JsonPropertyName("sitemap")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? Sitemap { get; set; }
[JsonPropertyName("includeSubdomains")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public bool? IncludeSubdomains { get; set; }
[JsonPropertyName("ignoreQueryParameters")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public bool? IgnoreQueryParameters { get; set; }
[JsonPropertyName("limit")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public int? Limit { get; set; }
[JsonPropertyName("timeout")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public int? Timeout { get; set; }
[JsonPropertyName("integration")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? Integration { get; set; }
[JsonPropertyName("location")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public LocationConfig? Location { get; set; }
[JsonPropertyName("ignoreCache")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public bool? IgnoreCache { get; set; }
}

View File

@@ -0,0 +1,225 @@
using System.Text.Json.Serialization;
namespace Firecrawl.Models;
public class MonitorSchedule
{
[JsonPropertyName("cron")]
public string? Cron { get; set; }
[JsonPropertyName("timezone")]
public string? Timezone { get; set; }
}
public class CreateMonitorRequest
{
[JsonPropertyName("name")]
public string? Name { get; set; }
[JsonPropertyName("schedule")]
public MonitorSchedule? Schedule { get; set; }
[JsonPropertyName("targets")]
public List<Dictionary<string, object>>? Targets { get; set; }
[JsonPropertyName("webhook")]
public Dictionary<string, object>? Webhook { get; set; }
[JsonPropertyName("notification")]
public Dictionary<string, object>? Notification { get; set; }
[JsonPropertyName("retentionDays")]
public int? RetentionDays { get; set; }
}
public class UpdateMonitorRequest
{
[JsonPropertyName("name")]
public string? Name { get; set; }
[JsonPropertyName("status")]
public string? Status { get; set; }
[JsonPropertyName("schedule")]
public MonitorSchedule? Schedule { get; set; }
[JsonPropertyName("targets")]
public List<Dictionary<string, object>>? Targets { get; set; }
[JsonPropertyName("webhook")]
public Dictionary<string, object>? Webhook { get; set; }
[JsonPropertyName("notification")]
public Dictionary<string, object>? Notification { get; set; }
[JsonPropertyName("retentionDays")]
public int? RetentionDays { get; set; }
}
public class MonitorSummary
{
[JsonPropertyName("totalPages")]
public int TotalPages { get; set; }
[JsonPropertyName("same")]
public int Same { get; set; }
[JsonPropertyName("changed")]
public int Changed { get; set; }
[JsonPropertyName("new")]
public int New { get; set; }
[JsonPropertyName("removed")]
public int Removed { get; set; }
[JsonPropertyName("error")]
public int Error { get; set; }
}
public class Monitor
{
[JsonPropertyName("id")]
public string? Id { get; set; }
[JsonPropertyName("name")]
public string? Name { get; set; }
[JsonPropertyName("status")]
public string? Status { get; set; }
[JsonPropertyName("schedule")]
public MonitorSchedule? Schedule { get; set; }
[JsonPropertyName("nextRunAt")]
public string? NextRunAt { get; set; }
[JsonPropertyName("lastRunAt")]
public string? LastRunAt { get; set; }
[JsonPropertyName("currentCheckId")]
public string? CurrentCheckId { get; set; }
[JsonPropertyName("targets")]
public List<Dictionary<string, object>>? Targets { get; set; }
[JsonPropertyName("webhook")]
public Dictionary<string, object>? Webhook { get; set; }
[JsonPropertyName("notification")]
public Dictionary<string, object>? Notification { get; set; }
[JsonPropertyName("retentionDays")]
public int RetentionDays { get; set; }
[JsonPropertyName("estimatedCreditsPerMonth")]
public int? EstimatedCreditsPerMonth { get; set; }
[JsonPropertyName("lastCheckSummary")]
public MonitorSummary? LastCheckSummary { get; set; }
[JsonPropertyName("createdAt")]
public string? CreatedAt { get; set; }
[JsonPropertyName("updatedAt")]
public string? UpdatedAt { get; set; }
}
public class MonitorCheck
{
[JsonPropertyName("id")]
public string? Id { get; set; }
[JsonPropertyName("monitorId")]
public string? MonitorId { get; set; }
[JsonPropertyName("status")]
public string? Status { get; set; }
[JsonPropertyName("trigger")]
public string? Trigger { get; set; }
[JsonPropertyName("scheduledFor")]
public string? ScheduledFor { get; set; }
[JsonPropertyName("startedAt")]
public string? StartedAt { get; set; }
[JsonPropertyName("finishedAt")]
public string? FinishedAt { get; set; }
[JsonPropertyName("estimatedCredits")]
public int? EstimatedCredits { get; set; }
[JsonPropertyName("reservedCredits")]
public int? ReservedCredits { get; set; }
[JsonPropertyName("actualCredits")]
public int? ActualCredits { get; set; }
[JsonPropertyName("billingStatus")]
public string? BillingStatus { get; set; }
[JsonPropertyName("summary")]
public MonitorSummary? Summary { get; set; }
[JsonPropertyName("targetResults")]
public object? TargetResults { get; set; }
[JsonPropertyName("notificationStatus")]
public object? NotificationStatus { get; set; }
[JsonPropertyName("error")]
public string? Error { get; set; }
[JsonPropertyName("createdAt")]
public string? CreatedAt { get; set; }
[JsonPropertyName("updatedAt")]
public string? UpdatedAt { get; set; }
}
public class MonitorCheckPage
{
[JsonPropertyName("id")]
public string? Id { get; set; }
[JsonPropertyName("targetId")]
public string? TargetId { get; set; }
[JsonPropertyName("url")]
public string? Url { get; set; }
[JsonPropertyName("status")]
public string? Status { get; set; }
[JsonPropertyName("previousScrapeId")]
public string? PreviousScrapeId { get; set; }
[JsonPropertyName("currentScrapeId")]
public string? CurrentScrapeId { get; set; }
[JsonPropertyName("statusCode")]
public int? StatusCode { get; set; }
[JsonPropertyName("error")]
public string? Error { get; set; }
[JsonPropertyName("metadata")]
public object? Metadata { get; set; }
[JsonPropertyName("diff")]
public object? Diff { get; set; }
[JsonPropertyName("createdAt")]
public string? CreatedAt { get; set; }
}
public class MonitorCheckDetail : MonitorCheck
{
[JsonPropertyName("pages")]
public List<MonitorCheckPage>? Pages { get; set; }
[JsonPropertyName("next")]
public string? Next { get; set; }
}

View File

@@ -0,0 +1,80 @@
namespace Firecrawl.Models;
/// <summary>
/// Uploaded file payload for the <c>/v2/parse</c> endpoint.
/// </summary>
public class ParseFile
{
/// <summary>
/// Filename used in the multipart upload (e.g. <c>upload.pdf</c>).
/// </summary>
public string Filename { get; set; }
/// <summary>
/// Raw bytes of the file to be parsed.
/// </summary>
public byte[] Content { get; set; }
/// <summary>
/// Optional MIME type hint (e.g. <c>application/pdf</c>).
/// When null, the value is guessed from the filename extension,
/// falling back to <c>application/octet-stream</c>.
/// </summary>
public string? ContentType { get; set; }
public ParseFile(string filename, byte[] content, string? contentType = null)
{
Filename = filename;
Content = content;
ContentType = contentType;
}
/// <summary>
/// Build a <see cref="ParseFile"/> from raw bytes.
/// </summary>
public static ParseFile FromBytes(string filename, byte[] content, string? contentType = null)
=> new(filename, content, contentType);
/// <summary>
/// Build a <see cref="ParseFile"/> by reading a file from disk.
/// The filename is derived from the path unless overridden.
/// </summary>
public static ParseFile FromPath(string path, string? filename = null, string? contentType = null)
{
if (string.IsNullOrWhiteSpace(path))
throw new ArgumentException("Path cannot be empty.", nameof(path));
if (!File.Exists(path))
throw new FileNotFoundException($"Parse file not found: {path}", path);
var bytes = File.ReadAllBytes(path);
var resolvedName = filename ?? Path.GetFileName(path);
return new ParseFile(resolvedName, bytes, contentType);
}
internal string ResolveContentType()
{
if (!string.IsNullOrWhiteSpace(ContentType))
return ContentType;
var extension = Path.GetExtension(Filename).ToLowerInvariant();
return extension switch
{
".html" or ".htm" => "text/html",
".pdf" => "application/pdf",
".docx" => "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
".doc" => "application/msword",
".xlsx" => "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
".xls" => "application/vnd.ms-excel",
".pptx" => "application/vnd.openxmlformats-officedocument.presentationml.presentation",
".ppt" => "application/vnd.ms-powerpoint",
".txt" => "text/plain",
".md" => "text/markdown",
".csv" => "text/csv",
".json" => "application/json",
".xml" => "application/xml",
".rtf" => "application/rtf",
_ => "application/octet-stream",
};
}
}

View File

@@ -0,0 +1,134 @@
using System.Text.Json;
using System.Text.Json.Serialization;
namespace Firecrawl.Models;
/// <summary>
/// Options for the <c>/v2/parse</c> endpoint.
///
/// <para>
/// Parse does not support browser-rendering formats/options such as change
/// tracking, screenshot, branding, actions, waitFor, location, or mobile.
/// These are rejected client-side in <see cref="Validate"/>.
/// </para>
/// </summary>
public class ParseOptions
{
internal static readonly HashSet<string> UnsupportedFormats = new(StringComparer.OrdinalIgnoreCase)
{
"changeTracking",
"change_tracking",
"screenshot",
"screenshot@fullPage",
"branding",
};
internal static readonly HashSet<string> SupportedProxies = new(StringComparer.OrdinalIgnoreCase)
{
"auto",
"basic",
};
[JsonPropertyName("formats")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public List<object>? Formats { get; set; }
[JsonPropertyName("headers")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public Dictionary<string, string>? Headers { get; set; }
[JsonPropertyName("includeTags")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public List<string>? IncludeTags { get; set; }
[JsonPropertyName("excludeTags")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public List<string>? ExcludeTags { get; set; }
[JsonPropertyName("onlyMainContent")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public bool? OnlyMainContent { get; set; }
[JsonPropertyName("timeout")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public int? Timeout { get; set; }
[JsonPropertyName("parsers")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public List<object>? Parsers { get; set; }
[JsonPropertyName("skipTlsVerification")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public bool? SkipTlsVerification { get; set; }
[JsonPropertyName("removeBase64Images")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public bool? RemoveBase64Images { get; set; }
[JsonPropertyName("blockAds")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public bool? BlockAds { get; set; }
[JsonPropertyName("proxy")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? Proxy { get; set; }
[JsonPropertyName("integration")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? Integration { get; set; }
/// <summary>
/// Validate the options against /v2/parse's supported surface.
/// </summary>
/// <exception cref="ArgumentException">Thrown when an unsupported format,
/// proxy, or timeout value is set.</exception>
public void Validate()
{
if (Timeout is not null and <= 0)
throw new ArgumentException("timeout must be positive", nameof(Timeout));
if (!string.IsNullOrWhiteSpace(Proxy) && !SupportedProxies.Contains(Proxy))
throw new ArgumentException(
"parse only supports proxy values 'basic' or 'auto'", nameof(Proxy));
if (Formats is { Count: > 0 })
{
foreach (var fmt in Formats)
{
var type = ExtractFormatType(fmt);
if (type is not null && UnsupportedFormats.Contains(type))
throw new ArgumentException($"parse does not support format: {type}", nameof(Formats));
}
}
}
private static string? ExtractFormatType(object? format)
{
if (format is null)
return null;
if (format is string s)
return s;
if (format is IDictionary<string, object?> dict &&
dict.TryGetValue("type", out var typeObj) &&
typeObj is string ts)
{
return ts;
}
if (format is JsonElement element)
{
if (element.ValueKind == JsonValueKind.String)
return element.GetString();
if (element.ValueKind == JsonValueKind.Object &&
element.TryGetProperty("type", out var typeProp) &&
typeProp.ValueKind == JsonValueKind.String)
{
return typeProp.GetString();
}
}
return null;
}
}

View File

@@ -0,0 +1,23 @@
using System.Text.Json.Serialization;
namespace Firecrawl.Models;
/// <summary>
/// Deprecated query format specification for use in ScrapeOptions.Formats.
/// </summary>
[Obsolete("Use QuestionFormat or HighlightsFormat instead.")]
public class QueryFormat
{
public const string FreeformMode = "freeform";
public const string DirectQuoteMode = "directQuote";
[JsonPropertyName("type")]
public string Type { get; } = "query";
[JsonPropertyName("prompt")]
public required string Prompt { get; set; }
[JsonPropertyName("mode")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? Mode { get; set; }
}

View File

@@ -0,0 +1,15 @@
using System.Text.Json.Serialization;
namespace Firecrawl.Models;
/// <summary>
/// Question format specification for use in ScrapeOptions.Formats.
/// </summary>
public class QuestionFormat
{
[JsonPropertyName("type")]
public string Type { get; } = "question";
[JsonPropertyName("question")]
public required string Question { get; set; }
}

View File

@@ -0,0 +1,81 @@
using System.Text.Json.Serialization;
namespace Firecrawl.Models;
/// <summary>
/// Configuration options for scraping a single URL.
/// </summary>
public class ScrapeOptions
{
[JsonPropertyName("formats")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public List<object>? Formats { get; set; }
[JsonPropertyName("headers")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public Dictionary<string, string>? Headers { get; set; }
[JsonPropertyName("includeTags")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public List<string>? IncludeTags { get; set; }
[JsonPropertyName("excludeTags")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public List<string>? ExcludeTags { get; set; }
[JsonPropertyName("onlyMainContent")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public bool? OnlyMainContent { get; set; }
[JsonPropertyName("timeout")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public int? Timeout { get; set; }
[JsonPropertyName("waitFor")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public int? WaitFor { get; set; }
[JsonPropertyName("mobile")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public bool? Mobile { get; set; }
[JsonPropertyName("parsers")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public List<object>? Parsers { get; set; }
[JsonPropertyName("actions")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public List<Dictionary<string, object>>? Actions { get; set; }
[JsonPropertyName("location")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public LocationConfig? Location { get; set; }
[JsonPropertyName("skipTlsVerification")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public bool? SkipTlsVerification { get; set; }
[JsonPropertyName("removeBase64Images")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public bool? RemoveBase64Images { get; set; }
[JsonPropertyName("blockAds")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public bool? BlockAds { get; set; }
[JsonPropertyName("proxy")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? Proxy { get; set; }
[JsonPropertyName("maxAge")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public long? MaxAge { get; set; }
[JsonPropertyName("storeInCache")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public bool? StoreInCache { get; set; }
[JsonPropertyName("integration")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? Integration { get; set; }
}

View File

@@ -0,0 +1,144 @@
using System.Text.Json.Serialization;
namespace Firecrawl.Models;
/// <summary>
/// Represents a single web search hit.
/// </summary>
public class WebSearchHit
{
[JsonPropertyName("url")]
public string? Url { get; set; }
[JsonPropertyName("title")]
public string? Title { get; set; }
[JsonPropertyName("description")]
public string? Description { get; set; }
[JsonPropertyName("position")]
public int? Position { get; set; }
[JsonPropertyName("category")]
public string? Category { get; set; }
[JsonPropertyName("markdown")]
public string? Markdown { get; set; }
[JsonPropertyName("html")]
public string? Html { get; set; }
[JsonPropertyName("rawHtml")]
public string? RawHtml { get; set; }
[JsonPropertyName("links")]
public List<string>? Links { get; set; }
[JsonPropertyName("screenshot")]
public string? Screenshot { get; set; }
[JsonPropertyName("metadata")]
public Dictionary<string, object>? Metadata { get; set; }
[JsonPropertyName("answer")]
public string? Answer { get; set; }
[JsonPropertyName("highlights")]
public string? Highlights { get; set; }
}
/// <summary>
/// Represents a news search result with news-specific fields.
/// </summary>
public class NewsSearchHit
{
[JsonPropertyName("url")]
public string? Url { get; set; }
[JsonPropertyName("title")]
public string? Title { get; set; }
[JsonPropertyName("snippet")]
public string? Snippet { get; set; }
[JsonPropertyName("date")]
public string? Date { get; set; }
[JsonPropertyName("imageUrl")]
public string? ImageUrl { get; set; }
[JsonPropertyName("position")]
public int? Position { get; set; }
[JsonPropertyName("category")]
public string? Category { get; set; }
[JsonPropertyName("markdown")]
public string? Markdown { get; set; }
[JsonPropertyName("html")]
public string? Html { get; set; }
[JsonPropertyName("rawHtml")]
public string? RawHtml { get; set; }
[JsonPropertyName("links")]
public List<string>? Links { get; set; }
[JsonPropertyName("screenshot")]
public string? Screenshot { get; set; }
[JsonPropertyName("metadata")]
public Dictionary<string, object>? Metadata { get; set; }
[JsonPropertyName("answer")]
public string? Answer { get; set; }
[JsonPropertyName("highlights")]
public string? Highlights { get; set; }
}
/// <summary>
/// Represents an image search result.
/// </summary>
public class ImageSearchHit
{
[JsonPropertyName("title")]
public string? Title { get; set; }
[JsonPropertyName("imageUrl")]
public string? ImageUrl { get; set; }
[JsonPropertyName("imageWidth")]
public int? ImageWidth { get; set; }
[JsonPropertyName("imageHeight")]
public int? ImageHeight { get; set; }
[JsonPropertyName("url")]
public string? Url { get; set; }
[JsonPropertyName("position")]
public int? Position { get; set; }
[JsonPropertyName("answer")]
public string? Answer { get; set; }
[JsonPropertyName("highlights")]
public string? Highlights { get; set; }
}
/// <summary>
/// Web search results.
/// </summary>
public class SearchData
{
[JsonPropertyName("web")]
public List<WebSearchHit>? Web { get; set; }
[JsonPropertyName("news")]
public List<NewsSearchHit>? News { get; set; }
[JsonPropertyName("images")]
public List<ImageSearchHit>? Images { get; set; }
}

View File

@@ -0,0 +1,61 @@
using System.Text.Json.Serialization;
namespace Firecrawl.Models;
/// <summary>
/// Configuration options for web search.
/// </summary>
public class SearchOptions
{
[JsonPropertyName("sources")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public List<object>? Sources { get; set; }
[JsonPropertyName("categories")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public List<object>? Categories { get; set; }
[JsonPropertyName("includeDomains")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public List<string>? IncludeDomains { get; set; }
[JsonPropertyName("excludeDomains")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public List<string>? ExcludeDomains { get; set; }
[JsonPropertyName("limit")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public int? Limit { get; set; }
[JsonPropertyName("tbs")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? Tbs { get; set; }
[JsonPropertyName("location")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? Location { get; set; }
[JsonPropertyName("ignoreInvalidURLs")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public bool? IgnoreInvalidURLs { get; set; }
[JsonPropertyName("timeout")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public int? Timeout { get; set; }
[JsonPropertyName("scrapeOptions")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public ScrapeOptions? ScrapeOptions { get; set; }
[JsonPropertyName("integration")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? Integration { get; set; }
[JsonPropertyName("country")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? Country { get; set; }
[JsonPropertyName("enterprise")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public bool? Enterprise { get; set; }
}

View File

@@ -0,0 +1,24 @@
using System.Text.Json.Serialization;
namespace Firecrawl.Models;
/// <summary>
/// Webhook configuration for async jobs.
/// </summary>
public class WebhookConfig
{
[JsonPropertyName("url")]
public required string Url { get; set; }
[JsonPropertyName("headers")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public Dictionary<string, string>? Headers { get; set; }
[JsonPropertyName("metadata")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public Dictionary<string, string>? Metadata { get; set; }
[JsonPropertyName("events")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public List<string>? Events { get; set; }
}