참고소스 수정본
This commit is contained in:
@@ -0,0 +1,12 @@
|
||||
namespace Firecrawl.Exceptions;
|
||||
|
||||
/// <summary>
|
||||
/// Thrown when the API returns a 401 Unauthorized response.
|
||||
/// </summary>
|
||||
public class AuthenticationException : FirecrawlException
|
||||
{
|
||||
public AuthenticationException(string message, string? errorCode = null, object? details = null)
|
||||
: base(message, 401, errorCode, details)
|
||||
{
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
namespace Firecrawl.Exceptions;
|
||||
|
||||
/// <summary>
|
||||
/// Base exception for all Firecrawl SDK errors.
|
||||
/// </summary>
|
||||
public class FirecrawlException : Exception
|
||||
{
|
||||
/// <summary>
|
||||
/// HTTP status code (0 if not an HTTP error).
|
||||
/// </summary>
|
||||
public int StatusCode { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Error code returned by the API, if any.
|
||||
/// </summary>
|
||||
public string? ErrorCode { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Additional error details from the API response.
|
||||
/// </summary>
|
||||
public object? Details { get; }
|
||||
|
||||
public FirecrawlException(string message)
|
||||
: base(message)
|
||||
{
|
||||
}
|
||||
|
||||
public FirecrawlException(string message, int statusCode)
|
||||
: base(message)
|
||||
{
|
||||
StatusCode = statusCode;
|
||||
}
|
||||
|
||||
public FirecrawlException(string message, int statusCode, string? errorCode, object? details)
|
||||
: base(message)
|
||||
{
|
||||
StatusCode = statusCode;
|
||||
ErrorCode = errorCode;
|
||||
Details = details;
|
||||
}
|
||||
|
||||
public FirecrawlException(string message, Exception innerException)
|
||||
: base(message, innerException)
|
||||
{
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
namespace Firecrawl.Exceptions;
|
||||
|
||||
/// <summary>
|
||||
/// Thrown when an async job (crawl, batch scrape, agent) does not complete within the specified timeout.
|
||||
/// </summary>
|
||||
public class JobTimeoutException : FirecrawlException
|
||||
{
|
||||
/// <summary>
|
||||
/// The ID of the job that timed out.
|
||||
/// </summary>
|
||||
public string JobId { get; }
|
||||
|
||||
/// <summary>
|
||||
/// The timeout in seconds that was exceeded.
|
||||
/// </summary>
|
||||
public int TimeoutSeconds { get; }
|
||||
|
||||
public JobTimeoutException(string jobId, int timeoutSeconds, string jobType)
|
||||
: base($"{jobType} job {jobId} did not complete within {timeoutSeconds} seconds")
|
||||
{
|
||||
JobId = jobId;
|
||||
TimeoutSeconds = timeoutSeconds;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
namespace Firecrawl.Exceptions;
|
||||
|
||||
/// <summary>
|
||||
/// Thrown when the API returns a 429 Too Many Requests response.
|
||||
/// </summary>
|
||||
public class RateLimitException : FirecrawlException
|
||||
{
|
||||
public RateLimitException(string message, string? errorCode = null, object? details = null)
|
||||
: base(message, 429, errorCode, details)
|
||||
{
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<PropertyGroup>
|
||||
<TargetFramework>net8.0</TargetFramework>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<Nullable>enable</Nullable>
|
||||
<RootNamespace>Firecrawl</RootNamespace>
|
||||
|
||||
<!-- NuGet package metadata -->
|
||||
<PackageId>firecrawl-sdk</PackageId>
|
||||
<Version>1.3.1</Version>
|
||||
<Authors>Firecrawl</Authors>
|
||||
<Company>Firecrawl</Company>
|
||||
<Description>.NET SDK for the Firecrawl API - web scraping, crawling, and data extraction</Description>
|
||||
<PackageLicenseExpression>MIT</PackageLicenseExpression>
|
||||
<PackageProjectUrl>https://github.com/firecrawl/firecrawl</PackageProjectUrl>
|
||||
<RepositoryUrl>https://github.com/firecrawl/firecrawl</RepositoryUrl>
|
||||
<RepositoryType>git</RepositoryType>
|
||||
<PackageTags>firecrawl;web-scraping;crawling;api;sdk</PackageTags>
|
||||
<PackageReadmeFile>README.md</PackageReadmeFile>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<InternalsVisibleTo Include="Firecrawl.Tests" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="System.Text.Json" Version="9.0.3" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<None Include="../README.md" Pack="true" PackagePath="/" />
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
||||
670
참고/firecrawl-main/apps/dot-net-sdk/Firecrawl/FirecrawlClient.cs
Normal file
670
참고/firecrawl-main/apps/dot-net-sdk/Firecrawl/FirecrawlClient.cs
Normal file
@@ -0,0 +1,670 @@
|
||||
using System.Text.Json;
|
||||
using Firecrawl.Exceptions;
|
||||
using Firecrawl.Models;
|
||||
using MonitorModel = Firecrawl.Models.Monitor;
|
||||
|
||||
namespace Firecrawl;
|
||||
|
||||
/// <summary>
|
||||
/// Client for the Firecrawl v2 API.
|
||||
///
|
||||
/// <example>
|
||||
/// <code>
|
||||
/// var client = new FirecrawlClient("fc-your-api-key");
|
||||
///
|
||||
/// // Scrape a single page
|
||||
/// var doc = await client.ScrapeAsync("https://example.com",
|
||||
/// new ScrapeOptions { Formats = new List<object> { "markdown" } });
|
||||
///
|
||||
/// // Crawl a website
|
||||
/// var job = await client.CrawlAsync("https://example.com",
|
||||
/// new CrawlOptions { Limit = 50 });
|
||||
/// </code>
|
||||
/// </example>
|
||||
/// </summary>
|
||||
public class FirecrawlClient
|
||||
{
|
||||
private const string DefaultApiUrl = "https://api.firecrawl.dev";
|
||||
private static readonly TimeSpan DefaultTimeout = TimeSpan.FromMinutes(5);
|
||||
private const int DefaultMaxRetries = 3;
|
||||
private const double DefaultBackoffFactor = 0.5;
|
||||
private const int DefaultPollIntervalSec = 2;
|
||||
private const int DefaultJobTimeoutSec = 300;
|
||||
|
||||
private readonly FirecrawlHttpClient _http;
|
||||
|
||||
/// <summary>
|
||||
/// Creates a new FirecrawlClient with the specified API key.
|
||||
/// </summary>
|
||||
/// <param name="apiKey">The Firecrawl API key.</param>
|
||||
/// <param name="apiUrl">Optional API base URL (defaults to https://api.firecrawl.dev).</param>
|
||||
/// <param name="timeout">Optional HTTP request timeout.</param>
|
||||
/// <param name="maxRetries">Optional maximum number of retries for transient failures.</param>
|
||||
/// <param name="backoffFactor">Optional exponential backoff factor in seconds.</param>
|
||||
/// <param name="httpClient">Optional pre-configured HttpClient instance.</param>
|
||||
public FirecrawlClient(
|
||||
string? apiKey = null,
|
||||
string? apiUrl = null,
|
||||
TimeSpan? timeout = null,
|
||||
int maxRetries = DefaultMaxRetries,
|
||||
double backoffFactor = DefaultBackoffFactor,
|
||||
HttpClient? httpClient = null)
|
||||
{
|
||||
var resolvedKey = ResolveApiKey(apiKey);
|
||||
var resolvedUrl = ResolveApiUrl(apiUrl);
|
||||
|
||||
_http = new FirecrawlHttpClient(
|
||||
resolvedKey,
|
||||
resolvedUrl,
|
||||
timeout ?? DefaultTimeout,
|
||||
maxRetries,
|
||||
backoffFactor,
|
||||
httpClient);
|
||||
}
|
||||
|
||||
// ================================================================
|
||||
// SCRAPE
|
||||
// ================================================================
|
||||
|
||||
/// <summary>
|
||||
/// Scrapes a single URL and returns the document.
|
||||
/// </summary>
|
||||
public async Task<Document> ScrapeAsync(
|
||||
string url,
|
||||
ScrapeOptions? options = null,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(url);
|
||||
|
||||
var body = BuildBody(options);
|
||||
body["url"] = url;
|
||||
|
||||
var response = await _http.PostAsync<ApiResponse<Document>>(
|
||||
"/v2/scrape", body, cancellationToken: cancellationToken);
|
||||
|
||||
return response.Data ?? throw new FirecrawlException("Scrape response contained no data");
|
||||
}
|
||||
|
||||
// ================================================================
|
||||
// CRAWL
|
||||
// ================================================================
|
||||
|
||||
/// <summary>
|
||||
/// Starts an async crawl job and returns immediately.
|
||||
/// </summary>
|
||||
public async Task<CrawlResponse> StartCrawlAsync(
|
||||
string url,
|
||||
CrawlOptions? options = null,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(url);
|
||||
|
||||
var body = BuildBody(options);
|
||||
body["url"] = url;
|
||||
|
||||
return await _http.PostAsync<CrawlResponse>(
|
||||
"/v2/crawl", body, cancellationToken: cancellationToken);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets the status and results of a crawl job.
|
||||
/// </summary>
|
||||
public async Task<CrawlJob> GetCrawlStatusAsync(
|
||||
string jobId,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(jobId);
|
||||
|
||||
return await _http.GetAsync<CrawlJob>(
|
||||
$"/v2/crawl/{jobId}", cancellationToken);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Crawls a website and waits for completion (auto-polling).
|
||||
/// </summary>
|
||||
public async Task<CrawlJob> CrawlAsync(
|
||||
string url,
|
||||
CrawlOptions? options = null,
|
||||
int pollIntervalSec = DefaultPollIntervalSec,
|
||||
int timeoutSec = DefaultJobTimeoutSec,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var start = await StartCrawlAsync(url, options, cancellationToken);
|
||||
return await PollCrawlAsync(
|
||||
start.Id ?? throw new FirecrawlException("Crawl start did not return a job ID"),
|
||||
pollIntervalSec, timeoutSec, cancellationToken);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Cancels a running crawl job.
|
||||
/// </summary>
|
||||
public async Task<Dictionary<string, object>> CancelCrawlAsync(
|
||||
string jobId,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(jobId);
|
||||
|
||||
return await _http.DeleteAsync<Dictionary<string, object>>(
|
||||
$"/v2/crawl/{jobId}", cancellationToken);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets errors from a crawl job.
|
||||
/// </summary>
|
||||
public async Task<Dictionary<string, object>> GetCrawlErrorsAsync(
|
||||
string jobId,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(jobId);
|
||||
|
||||
return await _http.GetAsync<Dictionary<string, object>>(
|
||||
$"/v2/crawl/{jobId}/errors", cancellationToken);
|
||||
}
|
||||
|
||||
// ================================================================
|
||||
// BATCH SCRAPE
|
||||
// ================================================================
|
||||
|
||||
/// <summary>
|
||||
/// Starts an async batch scrape job.
|
||||
/// </summary>
|
||||
public async Task<BatchScrapeResponse> StartBatchScrapeAsync(
|
||||
List<string> urls,
|
||||
BatchScrapeOptions? options = null,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(urls);
|
||||
|
||||
var body = BuildBody(options);
|
||||
body["urls"] = urls;
|
||||
|
||||
// The API expects scrape options flattened at the top level
|
||||
if (body.TryGetValue("options", out var nested) && nested is JsonElement nestedElement)
|
||||
{
|
||||
body.Remove("options");
|
||||
var nestedDict = JsonSerializer.Deserialize<Dictionary<string, object>>(
|
||||
nestedElement.GetRawText(), FirecrawlHttpClient.JsonOptions);
|
||||
if (nestedDict != null)
|
||||
{
|
||||
var batchFields = new Dictionary<string, object>(body);
|
||||
foreach (var kv in nestedDict)
|
||||
body.TryAdd(kv.Key, kv.Value);
|
||||
foreach (var kv in batchFields)
|
||||
body[kv.Key] = kv.Value;
|
||||
}
|
||||
}
|
||||
|
||||
Dictionary<string, string>? extraHeaders = null;
|
||||
if (options?.IdempotencyKey is { Length: > 0 } idempotencyKey)
|
||||
{
|
||||
extraHeaders = new Dictionary<string, string>
|
||||
{
|
||||
["x-idempotency-key"] = idempotencyKey
|
||||
};
|
||||
}
|
||||
|
||||
return await _http.PostAsync<BatchScrapeResponse>(
|
||||
"/v2/batch/scrape", body, extraHeaders, cancellationToken);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets the status and results of a batch scrape job.
|
||||
/// </summary>
|
||||
public async Task<BatchScrapeJob> GetBatchScrapeStatusAsync(
|
||||
string jobId,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(jobId);
|
||||
|
||||
return await _http.GetAsync<BatchScrapeJob>(
|
||||
$"/v2/batch/scrape/{jobId}", cancellationToken);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Batch-scrapes URLs and waits for completion (auto-polling).
|
||||
/// </summary>
|
||||
public async Task<BatchScrapeJob> BatchScrapeAsync(
|
||||
List<string> urls,
|
||||
BatchScrapeOptions? options = null,
|
||||
int pollIntervalSec = DefaultPollIntervalSec,
|
||||
int timeoutSec = DefaultJobTimeoutSec,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var start = await StartBatchScrapeAsync(urls, options, cancellationToken);
|
||||
return await PollBatchScrapeAsync(
|
||||
start.Id ?? throw new FirecrawlException("Batch scrape start did not return a job ID"),
|
||||
pollIntervalSec, timeoutSec, cancellationToken);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Cancels a running batch scrape job.
|
||||
/// </summary>
|
||||
public async Task<Dictionary<string, object>> CancelBatchScrapeAsync(
|
||||
string jobId,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(jobId);
|
||||
|
||||
return await _http.DeleteAsync<Dictionary<string, object>>(
|
||||
$"/v2/batch/scrape/{jobId}", cancellationToken);
|
||||
}
|
||||
|
||||
// ================================================================
|
||||
// PARSE
|
||||
// ================================================================
|
||||
|
||||
/// <summary>
|
||||
/// Parses an uploaded file (HTML, PDF, DOCX, etc.) via <c>/v2/parse</c>
|
||||
/// and returns the extracted document.
|
||||
/// </summary>
|
||||
/// <param name="file">The file to upload.</param>
|
||||
/// <param name="options">Optional parse options. Browser-only formats
|
||||
/// (changeTracking, screenshot, branding), actions, waitFor, location,
|
||||
/// and mobile are rejected.</param>
|
||||
/// <param name="cancellationToken">Cancellation token.</param>
|
||||
public async Task<Document> ParseAsync(
|
||||
ParseFile file,
|
||||
ParseOptions? options = null,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(file);
|
||||
|
||||
var filename = file.Filename?.Trim();
|
||||
if (string.IsNullOrEmpty(filename))
|
||||
throw new ArgumentException("filename cannot be empty", nameof(file));
|
||||
|
||||
if (file.Content is null || file.Content.Length == 0)
|
||||
throw new ArgumentException("file content cannot be empty", nameof(file));
|
||||
|
||||
options?.Validate();
|
||||
|
||||
var optionsJson = JsonSerializer.Serialize(
|
||||
options ?? new ParseOptions(),
|
||||
FirecrawlHttpClient.JsonOptions);
|
||||
|
||||
var fields = new Dictionary<string, string>
|
||||
{
|
||||
["options"] = optionsJson,
|
||||
};
|
||||
|
||||
var response = await _http.PostMultipartAsync<ApiResponse<Document>>(
|
||||
"/v2/parse",
|
||||
fields,
|
||||
fileField: "file",
|
||||
fileName: filename,
|
||||
fileContentType: file.ResolveContentType(),
|
||||
fileContent: file.Content,
|
||||
cancellationToken: cancellationToken);
|
||||
|
||||
return response.Data ?? throw new FirecrawlException("Parse response contained no data");
|
||||
}
|
||||
|
||||
// ================================================================
|
||||
// MAP
|
||||
// ================================================================
|
||||
|
||||
/// <summary>
|
||||
/// Discovers URLs on a website.
|
||||
/// </summary>
|
||||
public async Task<MapData> MapAsync(
|
||||
string url,
|
||||
MapOptions? options = null,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(url);
|
||||
|
||||
var body = BuildBody(options);
|
||||
body["url"] = url;
|
||||
|
||||
var response = await _http.PostAsync<ApiResponse<MapData>>(
|
||||
"/v2/map", body, cancellationToken: cancellationToken);
|
||||
|
||||
return response.Data ?? throw new FirecrawlException("Map response contained no data");
|
||||
}
|
||||
|
||||
// ================================================================
|
||||
// MONITOR
|
||||
// ================================================================
|
||||
|
||||
public async Task<MonitorModel> CreateMonitorAsync(
|
||||
CreateMonitorRequest request,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(request);
|
||||
|
||||
var response = await _http.PostAsync<ApiResponse<MonitorModel>>(
|
||||
"/v2/monitor", request, cancellationToken: cancellationToken);
|
||||
|
||||
return response.Data ?? throw new FirecrawlException("Create monitor response contained no data");
|
||||
}
|
||||
|
||||
public async Task<List<MonitorModel>> ListMonitorsAsync(
|
||||
int? limit = null,
|
||||
int? offset = null,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var response = await _http.GetAsync<ApiResponse<List<MonitorModel>>>(
|
||||
$"/v2/monitor{BuildQuery(limit, offset)}", cancellationToken);
|
||||
|
||||
return response.Data ?? new List<MonitorModel>();
|
||||
}
|
||||
|
||||
public async Task<MonitorModel> GetMonitorAsync(
|
||||
string monitorId,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(monitorId);
|
||||
|
||||
var response = await _http.GetAsync<ApiResponse<MonitorModel>>(
|
||||
$"/v2/monitor/{monitorId}", cancellationToken);
|
||||
|
||||
return response.Data ?? throw new FirecrawlException("Get monitor response contained no data");
|
||||
}
|
||||
|
||||
public async Task<MonitorModel> UpdateMonitorAsync(
|
||||
string monitorId,
|
||||
UpdateMonitorRequest request,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(monitorId);
|
||||
ArgumentNullException.ThrowIfNull(request);
|
||||
|
||||
var response = await _http.PatchAsync<ApiResponse<MonitorModel>>(
|
||||
$"/v2/monitor/{monitorId}", request, cancellationToken);
|
||||
|
||||
return response.Data ?? throw new FirecrawlException("Update monitor response contained no data");
|
||||
}
|
||||
|
||||
public async Task<bool> DeleteMonitorAsync(
|
||||
string monitorId,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(monitorId);
|
||||
|
||||
var response = await _http.DeleteAsync<Dictionary<string, object>>(
|
||||
$"/v2/monitor/{monitorId}", cancellationToken);
|
||||
|
||||
return response.TryGetValue("success", out var success) && success switch
|
||||
{
|
||||
bool value => value,
|
||||
JsonElement element when element.ValueKind == JsonValueKind.True => true,
|
||||
_ => false
|
||||
};
|
||||
}
|
||||
|
||||
public async Task<MonitorCheck> RunMonitorAsync(
|
||||
string monitorId,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(monitorId);
|
||||
|
||||
var response = await _http.PostAsync<ApiResponse<MonitorCheck>>(
|
||||
$"/v2/monitor/{monitorId}/run", new Dictionary<string, object>(), cancellationToken: cancellationToken);
|
||||
|
||||
return response.Data ?? throw new FirecrawlException("Run monitor response contained no data");
|
||||
}
|
||||
|
||||
public async Task<List<MonitorCheck>> ListMonitorChecksAsync(
|
||||
string monitorId,
|
||||
int? limit = null,
|
||||
int? offset = null,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(monitorId);
|
||||
|
||||
var response = await _http.GetAsync<ApiResponse<List<MonitorCheck>>>(
|
||||
$"/v2/monitor/{monitorId}/checks{BuildQuery(limit, offset)}", cancellationToken);
|
||||
|
||||
return response.Data ?? new List<MonitorCheck>();
|
||||
}
|
||||
|
||||
public async Task<MonitorCheckDetail> GetMonitorCheckAsync(
|
||||
string monitorId,
|
||||
string checkId,
|
||||
int? limit = null,
|
||||
int? skip = null,
|
||||
string? status = null,
|
||||
bool autoPaginate = true,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(monitorId);
|
||||
ArgumentNullException.ThrowIfNull(checkId);
|
||||
|
||||
var response = await _http.GetAsync<ApiResponse<MonitorCheckDetail>>(
|
||||
$"/v2/monitor/{monitorId}/checks/{checkId}{BuildMonitorCheckQuery(limit, skip, status)}",
|
||||
cancellationToken);
|
||||
|
||||
var check = response.Data ?? throw new FirecrawlException("Get monitor check response contained no data");
|
||||
return autoPaginate ? await PaginateMonitorCheckAsync(check, cancellationToken) : check;
|
||||
}
|
||||
|
||||
// ================================================================
|
||||
// SEARCH
|
||||
// ================================================================
|
||||
|
||||
/// <summary>
|
||||
/// Performs a web search.
|
||||
/// </summary>
|
||||
public async Task<SearchData> SearchAsync(
|
||||
string query,
|
||||
SearchOptions? options = null,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(query);
|
||||
|
||||
var body = BuildBody(options);
|
||||
body["query"] = query;
|
||||
|
||||
var response = await _http.PostAsync<ApiResponse<SearchData>>(
|
||||
"/v2/search", body, cancellationToken: cancellationToken);
|
||||
|
||||
return response.Data ?? throw new FirecrawlException("Search response contained no data");
|
||||
}
|
||||
|
||||
// ================================================================
|
||||
// USAGE & METRICS
|
||||
// ================================================================
|
||||
|
||||
/// <summary>
|
||||
/// Gets current concurrency usage.
|
||||
/// </summary>
|
||||
public async Task<ConcurrencyCheck> GetConcurrencyAsync(
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
return await _http.GetAsync<ConcurrencyCheck>(
|
||||
"/v2/concurrency-check", cancellationToken);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets current credit usage.
|
||||
/// </summary>
|
||||
public async Task<CreditUsage> GetCreditUsageAsync(
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
return await _http.GetAsync<CreditUsage>(
|
||||
"/v2/team/credit-usage", cancellationToken);
|
||||
}
|
||||
|
||||
// ================================================================
|
||||
// INTERNAL POLLING HELPERS
|
||||
// ================================================================
|
||||
|
||||
private async Task<CrawlJob> PollCrawlAsync(
|
||||
string jobId,
|
||||
int pollIntervalSec,
|
||||
int timeoutSec,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var deadline = DateTime.UtcNow.AddSeconds(timeoutSec);
|
||||
|
||||
while (DateTime.UtcNow < deadline)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
var job = await GetCrawlStatusAsync(jobId, cancellationToken);
|
||||
if (job.IsDone)
|
||||
return await PaginateCrawlAsync(job, cancellationToken);
|
||||
|
||||
await Task.Delay(TimeSpan.FromSeconds(pollIntervalSec), cancellationToken);
|
||||
}
|
||||
|
||||
throw new JobTimeoutException(jobId, timeoutSec, "Crawl");
|
||||
}
|
||||
|
||||
private async Task<BatchScrapeJob> PollBatchScrapeAsync(
|
||||
string jobId,
|
||||
int pollIntervalSec,
|
||||
int timeoutSec,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var deadline = DateTime.UtcNow.AddSeconds(timeoutSec);
|
||||
|
||||
while (DateTime.UtcNow < deadline)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
var job = await GetBatchScrapeStatusAsync(jobId, cancellationToken);
|
||||
if (job.IsDone)
|
||||
return await PaginateBatchScrapeAsync(job, cancellationToken);
|
||||
|
||||
await Task.Delay(TimeSpan.FromSeconds(pollIntervalSec), cancellationToken);
|
||||
}
|
||||
|
||||
throw new JobTimeoutException(jobId, timeoutSec, "Batch scrape");
|
||||
}
|
||||
|
||||
private async Task<CrawlJob> PaginateCrawlAsync(
|
||||
CrawlJob job,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
job.Data ??= new List<Document>();
|
||||
var current = job;
|
||||
|
||||
while (!string.IsNullOrEmpty(current.Next))
|
||||
{
|
||||
var nextPage = await _http.GetAbsoluteAsync<CrawlJob>(
|
||||
current.Next, cancellationToken);
|
||||
|
||||
if (nextPage.Data is { Count: > 0 })
|
||||
job.Data.AddRange(nextPage.Data);
|
||||
|
||||
current = nextPage;
|
||||
}
|
||||
|
||||
job.Next = null;
|
||||
return job;
|
||||
}
|
||||
|
||||
private async Task<BatchScrapeJob> PaginateBatchScrapeAsync(
|
||||
BatchScrapeJob job,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
job.Data ??= new List<Document>();
|
||||
var current = job;
|
||||
|
||||
while (!string.IsNullOrEmpty(current.Next))
|
||||
{
|
||||
var nextPage = await _http.GetAbsoluteAsync<BatchScrapeJob>(
|
||||
current.Next, cancellationToken);
|
||||
|
||||
if (nextPage.Data is { Count: > 0 })
|
||||
job.Data.AddRange(nextPage.Data);
|
||||
|
||||
current = nextPage;
|
||||
}
|
||||
|
||||
job.Next = null;
|
||||
return job;
|
||||
}
|
||||
|
||||
private async Task<MonitorCheckDetail> PaginateMonitorCheckAsync(
|
||||
MonitorCheckDetail check,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
check.Pages ??= new List<MonitorCheckPage>();
|
||||
var current = check;
|
||||
|
||||
while (!string.IsNullOrEmpty(current.Next))
|
||||
{
|
||||
var response = await _http.GetAbsoluteAsync<ApiResponse<MonitorCheckDetail>>(
|
||||
current.Next, cancellationToken);
|
||||
if (response.Data == null)
|
||||
break;
|
||||
|
||||
var nextPage = response.Data;
|
||||
|
||||
if (nextPage.Pages is { Count: > 0 })
|
||||
check.Pages.AddRange(nextPage.Pages);
|
||||
|
||||
current = nextPage;
|
||||
}
|
||||
|
||||
check.Next = null;
|
||||
return check;
|
||||
}
|
||||
|
||||
// ================================================================
|
||||
// INTERNAL UTILITIES
|
||||
// ================================================================
|
||||
|
||||
private static Dictionary<string, object> BuildBody(object? options)
|
||||
{
|
||||
if (options == null)
|
||||
return new Dictionary<string, object>();
|
||||
|
||||
var json = JsonSerializer.Serialize(options, FirecrawlHttpClient.JsonOptions);
|
||||
return JsonSerializer.Deserialize<Dictionary<string, object>>(json, FirecrawlHttpClient.JsonOptions)
|
||||
?? new Dictionary<string, object>();
|
||||
}
|
||||
|
||||
private static string BuildQuery(int? limit = null, int? offset = null, string? status = null)
|
||||
{
|
||||
var query = new List<string>();
|
||||
if (limit.HasValue)
|
||||
query.Add($"limit={Uri.EscapeDataString(limit.Value.ToString())}");
|
||||
if (offset.HasValue)
|
||||
query.Add($"offset={Uri.EscapeDataString(offset.Value.ToString())}");
|
||||
if (!string.IsNullOrWhiteSpace(status))
|
||||
query.Add($"status={Uri.EscapeDataString(status)}");
|
||||
|
||||
return query.Count == 0 ? string.Empty : "?" + string.Join("&", query);
|
||||
}
|
||||
|
||||
private static string BuildMonitorCheckQuery(int? limit = null, int? skip = null, string? status = null)
|
||||
{
|
||||
var query = new List<string>();
|
||||
if (limit.HasValue)
|
||||
query.Add($"limit={Uri.EscapeDataString(limit.Value.ToString())}");
|
||||
if (skip.HasValue)
|
||||
query.Add($"skip={Uri.EscapeDataString(skip.Value.ToString())}");
|
||||
if (!string.IsNullOrWhiteSpace(status))
|
||||
query.Add($"status={Uri.EscapeDataString(status)}");
|
||||
|
||||
return query.Count == 0 ? string.Empty : "?" + string.Join("&", query);
|
||||
}
|
||||
|
||||
private static string ResolveApiKey(string? apiKey)
|
||||
{
|
||||
if (!string.IsNullOrWhiteSpace(apiKey))
|
||||
return apiKey;
|
||||
|
||||
var envKey = Environment.GetEnvironmentVariable("FIRECRAWL_API_KEY");
|
||||
if (!string.IsNullOrWhiteSpace(envKey))
|
||||
return envKey;
|
||||
|
||||
throw new FirecrawlException(
|
||||
"API key is required. Pass it to the constructor or set the FIRECRAWL_API_KEY environment variable.");
|
||||
}
|
||||
|
||||
private static string ResolveApiUrl(string? apiUrl)
|
||||
{
|
||||
if (!string.IsNullOrWhiteSpace(apiUrl))
|
||||
return apiUrl;
|
||||
|
||||
var envUrl = Environment.GetEnvironmentVariable("FIRECRAWL_API_URL");
|
||||
if (!string.IsNullOrWhiteSpace(envUrl))
|
||||
return envUrl;
|
||||
|
||||
return DefaultApiUrl;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,318 @@
|
||||
using System.Net;
|
||||
using System.Net.Http.Headers;
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Serialization;
|
||||
using Firecrawl.Exceptions;
|
||||
|
||||
namespace Firecrawl;
|
||||
|
||||
/// <summary>
|
||||
/// Internal HTTP client for making authenticated requests to the Firecrawl API.
|
||||
/// Handles retry logic with exponential backoff.
|
||||
/// </summary>
|
||||
internal class FirecrawlHttpClient
|
||||
{
|
||||
private readonly HttpClient _httpClient;
|
||||
private readonly string _apiKey;
|
||||
private readonly string _baseUrl;
|
||||
private readonly int _maxRetries;
|
||||
private readonly double _backoffFactor;
|
||||
|
||||
internal static readonly JsonSerializerOptions JsonOptions = new()
|
||||
{
|
||||
PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
|
||||
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
|
||||
PropertyNameCaseInsensitive = true
|
||||
};
|
||||
|
||||
internal FirecrawlHttpClient(
|
||||
string apiKey,
|
||||
string baseUrl,
|
||||
TimeSpan timeout,
|
||||
int maxRetries,
|
||||
double backoffFactor,
|
||||
HttpClient? httpClient = null)
|
||||
{
|
||||
_apiKey = apiKey;
|
||||
_baseUrl = baseUrl.TrimEnd('/');
|
||||
_maxRetries = maxRetries;
|
||||
_backoffFactor = backoffFactor;
|
||||
|
||||
if (httpClient != null)
|
||||
{
|
||||
_httpClient = httpClient;
|
||||
}
|
||||
else
|
||||
{
|
||||
_httpClient = new HttpClient { Timeout = timeout };
|
||||
}
|
||||
}
|
||||
|
||||
internal async Task<T> PostAsync<T>(
|
||||
string path,
|
||||
object body,
|
||||
Dictionary<string, string>? extraHeaders = null,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var url = _baseUrl + path;
|
||||
var json = JsonSerializer.Serialize(body, JsonOptions);
|
||||
|
||||
HttpRequestMessage BuildRequest()
|
||||
{
|
||||
var content = new StringContent(json, Encoding.UTF8, "application/json");
|
||||
var request = new HttpRequestMessage(HttpMethod.Post, url) { Content = content };
|
||||
ApplyStandardHeaders(request);
|
||||
|
||||
if (extraHeaders != null)
|
||||
{
|
||||
foreach (var header in extraHeaders)
|
||||
{
|
||||
request.Headers.TryAddWithoutValidation(header.Key, header.Value);
|
||||
}
|
||||
}
|
||||
|
||||
return request;
|
||||
}
|
||||
|
||||
return await ExecuteWithRetryAsync<T>(BuildRequest, cancellationToken);
|
||||
}
|
||||
|
||||
internal async Task<T> PostMultipartAsync<T>(
|
||||
string path,
|
||||
Dictionary<string, string> fields,
|
||||
string fileField,
|
||||
string fileName,
|
||||
string fileContentType,
|
||||
byte[] fileContent,
|
||||
Dictionary<string, string>? extraHeaders = null,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var url = _baseUrl + path;
|
||||
|
||||
HttpRequestMessage BuildRequest()
|
||||
{
|
||||
var content = new MultipartFormDataContent();
|
||||
|
||||
foreach (var kv in fields)
|
||||
{
|
||||
var fieldContent = new StringContent(kv.Value, Encoding.UTF8);
|
||||
fieldContent.Headers.ContentType = null;
|
||||
content.Add(fieldContent, kv.Key);
|
||||
}
|
||||
|
||||
var fileBytes = new ByteArrayContent(fileContent);
|
||||
fileBytes.Headers.ContentType =
|
||||
MediaTypeHeaderValue.Parse(string.IsNullOrWhiteSpace(fileContentType)
|
||||
? "application/octet-stream"
|
||||
: fileContentType);
|
||||
content.Add(fileBytes, fileField, fileName);
|
||||
|
||||
var request = new HttpRequestMessage(HttpMethod.Post, url) { Content = content };
|
||||
ApplyStandardHeaders(request);
|
||||
|
||||
if (extraHeaders != null)
|
||||
{
|
||||
foreach (var header in extraHeaders)
|
||||
{
|
||||
request.Headers.TryAddWithoutValidation(header.Key, header.Value);
|
||||
}
|
||||
}
|
||||
|
||||
return request;
|
||||
}
|
||||
|
||||
return await ExecuteWithRetryAsync<T>(BuildRequest, cancellationToken);
|
||||
}
|
||||
|
||||
internal async Task<T> GetAsync<T>(string path, CancellationToken cancellationToken = default)
|
||||
{
|
||||
var url = _baseUrl + path;
|
||||
|
||||
HttpRequestMessage BuildRequest()
|
||||
{
|
||||
var request = new HttpRequestMessage(HttpMethod.Get, url);
|
||||
ApplyStandardHeaders(request);
|
||||
return request;
|
||||
}
|
||||
|
||||
return await ExecuteWithRetryAsync<T>(BuildRequest, cancellationToken);
|
||||
}
|
||||
|
||||
internal async Task<T> PatchAsync<T>(
|
||||
string path,
|
||||
object body,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var url = _baseUrl + path;
|
||||
var json = JsonSerializer.Serialize(body, JsonOptions);
|
||||
|
||||
HttpRequestMessage BuildRequest()
|
||||
{
|
||||
var content = new StringContent(json, Encoding.UTF8, "application/json");
|
||||
var request = new HttpRequestMessage(HttpMethod.Patch, url) { Content = content };
|
||||
ApplyStandardHeaders(request);
|
||||
return request;
|
||||
}
|
||||
|
||||
return await ExecuteWithRetryAsync<T>(BuildRequest, cancellationToken);
|
||||
}
|
||||
|
||||
internal async Task<T> GetAbsoluteAsync<T>(string absoluteUrl, CancellationToken cancellationToken = default)
|
||||
{
|
||||
// Validate that the pagination URL belongs to the same host to prevent API key exfiltration
|
||||
var targetUri = new Uri(absoluteUrl);
|
||||
var baseUri = new Uri(_baseUrl);
|
||||
if (!string.Equals(targetUri.Scheme, baseUri.Scheme, StringComparison.OrdinalIgnoreCase) ||
|
||||
!string.Equals(targetUri.Host, baseUri.Host, StringComparison.OrdinalIgnoreCase) ||
|
||||
targetUri.Port != baseUri.Port)
|
||||
{
|
||||
throw new FirecrawlException(
|
||||
$"Pagination URL origin '{targetUri.Scheme}://{targetUri.Host}:{targetUri.Port}' does not match API base URL origin '{baseUri.Scheme}://{baseUri.Host}:{baseUri.Port}'. " +
|
||||
"Refusing to send credentials to a different origin.");
|
||||
}
|
||||
|
||||
HttpRequestMessage BuildRequest()
|
||||
{
|
||||
var request = new HttpRequestMessage(HttpMethod.Get, absoluteUrl);
|
||||
ApplyStandardHeaders(request);
|
||||
return request;
|
||||
}
|
||||
|
||||
return await ExecuteWithRetryAsync<T>(BuildRequest, cancellationToken);
|
||||
}
|
||||
|
||||
internal async Task<T> DeleteAsync<T>(string path, CancellationToken cancellationToken = default)
|
||||
{
|
||||
var url = _baseUrl + path;
|
||||
|
||||
HttpRequestMessage BuildRequest()
|
||||
{
|
||||
var request = new HttpRequestMessage(HttpMethod.Delete, url);
|
||||
ApplyStandardHeaders(request);
|
||||
return request;
|
||||
}
|
||||
|
||||
return await ExecuteWithRetryAsync<T>(BuildRequest, cancellationToken);
|
||||
}
|
||||
|
||||
private void ApplyStandardHeaders(HttpRequestMessage request)
|
||||
{
|
||||
request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", _apiKey);
|
||||
request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json"));
|
||||
}
|
||||
|
||||
private async Task<T> ExecuteWithRetryAsync<T>(
|
||||
Func<HttpRequestMessage> requestBuilder,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var attempt = 0;
|
||||
|
||||
while (true)
|
||||
{
|
||||
// Build a fresh request for each attempt (HttpRequestMessage can only be sent once,
|
||||
// and multipart content is not cheaply cloneable).
|
||||
using var request = requestBuilder();
|
||||
|
||||
HttpResponseMessage? response = null;
|
||||
try
|
||||
{
|
||||
response = await _httpClient.SendAsync(request, cancellationToken);
|
||||
var bodyStr = await response.Content.ReadAsStringAsync(cancellationToken);
|
||||
|
||||
if (response.IsSuccessStatusCode)
|
||||
{
|
||||
return JsonSerializer.Deserialize<T>(bodyStr, JsonOptions)
|
||||
?? throw new FirecrawlException("Failed to deserialize response");
|
||||
}
|
||||
|
||||
var code = (int)response.StatusCode;
|
||||
var errorMessage = ExtractErrorMessage(bodyStr, code);
|
||||
var errorCode = ExtractErrorCode(bodyStr);
|
||||
|
||||
// Non-retryable client errors
|
||||
if (code == 401)
|
||||
throw new AuthenticationException(errorMessage, errorCode);
|
||||
if (code == 429)
|
||||
throw new RateLimitException(errorMessage, errorCode);
|
||||
if (code >= 400 && code < 500 && code != 408 && code != 409)
|
||||
throw new FirecrawlException(errorMessage, code, errorCode, null);
|
||||
|
||||
// Retryable errors: 408, 409, 502, 5xx
|
||||
if (attempt < _maxRetries)
|
||||
{
|
||||
attempt++;
|
||||
await SleepWithBackoffAsync(attempt, cancellationToken);
|
||||
continue;
|
||||
}
|
||||
|
||||
throw new FirecrawlException(errorMessage, code, errorCode, null);
|
||||
}
|
||||
catch (FirecrawlException)
|
||||
{
|
||||
throw;
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
throw;
|
||||
}
|
||||
catch (HttpRequestException ex)
|
||||
{
|
||||
if (attempt < _maxRetries)
|
||||
{
|
||||
attempt++;
|
||||
await SleepWithBackoffAsync(attempt, cancellationToken);
|
||||
continue;
|
||||
}
|
||||
|
||||
throw new FirecrawlException($"Request failed: {ex.Message}", ex);
|
||||
}
|
||||
finally
|
||||
{
|
||||
response?.Dispose();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static string ExtractErrorMessage(string body, int statusCode)
|
||||
{
|
||||
try
|
||||
{
|
||||
using var doc = JsonDocument.Parse(body);
|
||||
var root = doc.RootElement;
|
||||
|
||||
if (root.TryGetProperty("error", out var errorProp))
|
||||
return errorProp.GetString() ?? $"HTTP {statusCode} error";
|
||||
if (root.TryGetProperty("message", out var messageProp))
|
||||
return messageProp.GetString() ?? $"HTTP {statusCode} error";
|
||||
}
|
||||
catch
|
||||
{
|
||||
// ignored
|
||||
}
|
||||
|
||||
return $"HTTP {statusCode} error";
|
||||
}
|
||||
|
||||
private static string? ExtractErrorCode(string body)
|
||||
{
|
||||
try
|
||||
{
|
||||
using var doc = JsonDocument.Parse(body);
|
||||
if (doc.RootElement.TryGetProperty("code", out var codeProp))
|
||||
return codeProp.GetString();
|
||||
}
|
||||
catch
|
||||
{
|
||||
// ignored
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
private async Task SleepWithBackoffAsync(int attempt, CancellationToken cancellationToken)
|
||||
{
|
||||
var delayMs = (int)(_backoffFactor * 1000 * Math.Pow(2, attempt - 1));
|
||||
await Task.Delay(delayMs, cancellationToken);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace Firecrawl.Models;
|
||||
|
||||
/// <summary>
|
||||
/// Internal wrapper for API responses that contain a "data" field.
|
||||
/// </summary>
|
||||
internal class ApiResponse<T>
|
||||
{
|
||||
[JsonPropertyName("success")]
|
||||
public bool Success { get; set; }
|
||||
|
||||
[JsonPropertyName("data")]
|
||||
public T? Data { get; set; }
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace Firecrawl.Models;
|
||||
|
||||
/// <summary>
|
||||
/// Status and results of a batch scrape job.
|
||||
/// </summary>
|
||||
public class BatchScrapeJob
|
||||
{
|
||||
[JsonPropertyName("id")]
|
||||
public string? Id { get; set; }
|
||||
|
||||
[JsonPropertyName("status")]
|
||||
public string? Status { get; set; }
|
||||
|
||||
[JsonPropertyName("completed")]
|
||||
public int? Completed { get; set; }
|
||||
|
||||
[JsonPropertyName("total")]
|
||||
public int? Total { get; set; }
|
||||
|
||||
[JsonPropertyName("creditsUsed")]
|
||||
public int? CreditsUsed { get; set; }
|
||||
|
||||
[JsonPropertyName("expiresAt")]
|
||||
public string? ExpiresAt { get; set; }
|
||||
|
||||
[JsonPropertyName("next")]
|
||||
public string? Next { get; set; }
|
||||
|
||||
[JsonPropertyName("data")]
|
||||
public List<Document>? Data { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Returns true if the batch scrape job has reached a terminal state.
|
||||
/// </summary>
|
||||
public bool IsDone =>
|
||||
Status == "completed" || Status == "cancelled" || Status == "failed";
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace Firecrawl.Models;
|
||||
|
||||
/// <summary>
|
||||
/// Configuration options for batch scraping multiple URLs.
|
||||
/// </summary>
|
||||
public class BatchScrapeOptions
|
||||
{
|
||||
[JsonPropertyName("options")]
|
||||
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
|
||||
public ScrapeOptions? Options { get; set; }
|
||||
|
||||
[JsonPropertyName("webhook")]
|
||||
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
|
||||
public object? Webhook { get; set; }
|
||||
|
||||
[JsonPropertyName("appendToId")]
|
||||
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
|
||||
public string? AppendToId { get; set; }
|
||||
|
||||
[JsonPropertyName("ignoreInvalidURLs")]
|
||||
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
|
||||
public bool? IgnoreInvalidURLs { get; set; }
|
||||
|
||||
[JsonPropertyName("maxConcurrency")]
|
||||
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
|
||||
public int? MaxConcurrency { get; set; }
|
||||
|
||||
[JsonPropertyName("zeroDataRetention")]
|
||||
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
|
||||
public bool? ZeroDataRetention { get; set; }
|
||||
|
||||
[JsonPropertyName("integration")]
|
||||
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
|
||||
public string? Integration { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Idempotency key sent as the x-idempotency-key HTTP header (not in the JSON body).
|
||||
/// </summary>
|
||||
[JsonIgnore]
|
||||
public string? IdempotencyKey { get; set; }
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace Firecrawl.Models;
|
||||
|
||||
/// <summary>
|
||||
/// Response from starting an async batch scrape job.
|
||||
/// </summary>
|
||||
public class BatchScrapeResponse
|
||||
{
|
||||
[JsonPropertyName("success")]
|
||||
public bool Success { get; set; }
|
||||
|
||||
[JsonPropertyName("id")]
|
||||
public string? Id { get; set; }
|
||||
|
||||
[JsonPropertyName("invalidURLs")]
|
||||
public List<string>? InvalidURLs { get; set; }
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace Firecrawl.Models;
|
||||
|
||||
/// <summary>
|
||||
/// Current concurrency usage information.
|
||||
/// </summary>
|
||||
public class ConcurrencyCheck
|
||||
{
|
||||
[JsonPropertyName("current")]
|
||||
public int Current { get; set; }
|
||||
|
||||
[JsonPropertyName("max")]
|
||||
public int MaxConcurrency { get; set; }
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace Firecrawl.Models;
|
||||
|
||||
/// <summary>
|
||||
/// Status and results of a crawl job.
|
||||
/// </summary>
|
||||
public class CrawlJob
|
||||
{
|
||||
[JsonPropertyName("id")]
|
||||
public string? Id { get; set; }
|
||||
|
||||
[JsonPropertyName("status")]
|
||||
public string? Status { get; set; }
|
||||
|
||||
[JsonPropertyName("completed")]
|
||||
public int? Completed { get; set; }
|
||||
|
||||
[JsonPropertyName("total")]
|
||||
public int? Total { get; set; }
|
||||
|
||||
[JsonPropertyName("creditsUsed")]
|
||||
public int? CreditsUsed { get; set; }
|
||||
|
||||
[JsonPropertyName("expiresAt")]
|
||||
public string? ExpiresAt { get; set; }
|
||||
|
||||
[JsonPropertyName("next")]
|
||||
public string? Next { get; set; }
|
||||
|
||||
[JsonPropertyName("data")]
|
||||
public List<Document>? Data { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Returns true if the crawl job has reached a terminal state.
|
||||
/// </summary>
|
||||
public bool IsDone =>
|
||||
Status == "completed" || Status == "cancelled" || Status == "failed";
|
||||
}
|
||||
@@ -0,0 +1,81 @@
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace Firecrawl.Models;
|
||||
|
||||
/// <summary>
|
||||
/// Configuration options for crawling a website.
|
||||
/// </summary>
|
||||
public class CrawlOptions
|
||||
{
|
||||
[JsonPropertyName("prompt")]
|
||||
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
|
||||
public string? Prompt { get; set; }
|
||||
|
||||
[JsonPropertyName("excludePaths")]
|
||||
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
|
||||
public List<string>? ExcludePaths { get; set; }
|
||||
|
||||
[JsonPropertyName("includePaths")]
|
||||
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
|
||||
public List<string>? IncludePaths { get; set; }
|
||||
|
||||
[JsonPropertyName("maxDiscoveryDepth")]
|
||||
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
|
||||
public int? MaxDiscoveryDepth { get; set; }
|
||||
|
||||
[JsonPropertyName("sitemap")]
|
||||
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
|
||||
public string? Sitemap { get; set; }
|
||||
|
||||
[JsonPropertyName("ignoreQueryParameters")]
|
||||
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
|
||||
public bool? IgnoreQueryParameters { get; set; }
|
||||
|
||||
[JsonPropertyName("deduplicateSimilarURLs")]
|
||||
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
|
||||
public bool? DeduplicateSimilarURLs { get; set; }
|
||||
|
||||
[JsonPropertyName("limit")]
|
||||
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
|
||||
public int? Limit { get; set; }
|
||||
|
||||
[JsonPropertyName("crawlEntireDomain")]
|
||||
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
|
||||
public bool? CrawlEntireDomain { get; set; }
|
||||
|
||||
[JsonPropertyName("allowExternalLinks")]
|
||||
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
|
||||
public bool? AllowExternalLinks { get; set; }
|
||||
|
||||
[JsonPropertyName("allowSubdomains")]
|
||||
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
|
||||
public bool? AllowSubdomains { get; set; }
|
||||
|
||||
[JsonPropertyName("delay")]
|
||||
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
|
||||
public int? Delay { get; set; }
|
||||
|
||||
[JsonPropertyName("maxConcurrency")]
|
||||
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
|
||||
public int? MaxConcurrency { get; set; }
|
||||
|
||||
[JsonPropertyName("webhook")]
|
||||
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
|
||||
public object? Webhook { get; set; }
|
||||
|
||||
[JsonPropertyName("scrapeOptions")]
|
||||
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
|
||||
public ScrapeOptions? ScrapeOptions { get; set; }
|
||||
|
||||
[JsonPropertyName("regexOnFullURL")]
|
||||
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
|
||||
public bool? RegexOnFullURL { get; set; }
|
||||
|
||||
[JsonPropertyName("zeroDataRetention")]
|
||||
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
|
||||
public bool? ZeroDataRetention { get; set; }
|
||||
|
||||
[JsonPropertyName("integration")]
|
||||
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
|
||||
public string? Integration { get; set; }
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace Firecrawl.Models;
|
||||
|
||||
/// <summary>
|
||||
/// Response from starting an async crawl job.
|
||||
/// </summary>
|
||||
public class CrawlResponse
|
||||
{
|
||||
[JsonPropertyName("success")]
|
||||
public bool Success { get; set; }
|
||||
|
||||
[JsonPropertyName("id")]
|
||||
public string? Id { get; set; }
|
||||
|
||||
[JsonPropertyName("url")]
|
||||
public string? Url { get; set; }
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace Firecrawl.Models;
|
||||
|
||||
/// <summary>
|
||||
/// Account credit usage information.
|
||||
/// </summary>
|
||||
public class CreditUsage
|
||||
{
|
||||
[JsonPropertyName("remaining_credits")]
|
||||
public int? RemainingCredits { get; set; }
|
||||
|
||||
[JsonPropertyName("total_credits_used")]
|
||||
public int? TotalCreditsUsed { get; set; }
|
||||
|
||||
[JsonPropertyName("billing_period_start")]
|
||||
public string? BillingPeriodStart { get; set; }
|
||||
|
||||
[JsonPropertyName("billing_period_end")]
|
||||
public string? BillingPeriodEnd { get; set; }
|
||||
}
|
||||
@@ -0,0 +1,57 @@
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace Firecrawl.Models;
|
||||
|
||||
/// <summary>
|
||||
/// Represents a scraped document returned by the Firecrawl API.
|
||||
/// </summary>
|
||||
public class Document
|
||||
{
|
||||
[JsonPropertyName("markdown")]
|
||||
public string? Markdown { get; set; }
|
||||
|
||||
[JsonPropertyName("html")]
|
||||
public string? Html { get; set; }
|
||||
|
||||
[JsonPropertyName("rawHtml")]
|
||||
public string? RawHtml { get; set; }
|
||||
|
||||
[JsonPropertyName("json")]
|
||||
public object? Json { get; set; }
|
||||
|
||||
[JsonPropertyName("summary")]
|
||||
public string? Summary { get; set; }
|
||||
|
||||
[JsonPropertyName("metadata")]
|
||||
public Dictionary<string, object>? Metadata { get; set; }
|
||||
|
||||
[JsonPropertyName("links")]
|
||||
public List<string>? Links { get; set; }
|
||||
|
||||
[JsonPropertyName("images")]
|
||||
public List<string>? Images { get; set; }
|
||||
|
||||
[JsonPropertyName("screenshot")]
|
||||
public string? Screenshot { get; set; }
|
||||
|
||||
[JsonPropertyName("audio")]
|
||||
public object? Audio { get; set; }
|
||||
|
||||
[JsonPropertyName("actions")]
|
||||
public object? Actions { get; set; }
|
||||
|
||||
[JsonPropertyName("answer")]
|
||||
public string? Answer { get; set; }
|
||||
|
||||
[JsonPropertyName("highlights")]
|
||||
public string? Highlights { get; set; }
|
||||
|
||||
[JsonPropertyName("warning")]
|
||||
public string? Warning { get; set; }
|
||||
|
||||
[JsonPropertyName("changeTracking")]
|
||||
public object? ChangeTracking { get; set; }
|
||||
|
||||
[JsonPropertyName("branding")]
|
||||
public object? Branding { get; set; }
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace Firecrawl.Models;
|
||||
|
||||
/// <summary>
|
||||
/// Highlights format specification for use in ScrapeOptions.Formats.
|
||||
/// </summary>
|
||||
public class HighlightsFormat
|
||||
{
|
||||
[JsonPropertyName("type")]
|
||||
public string Type { get; } = "highlights";
|
||||
|
||||
[JsonPropertyName("query")]
|
||||
public required string Query { get; set; }
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace Firecrawl.Models;
|
||||
|
||||
/// <summary>
|
||||
/// JSON extraction format specification for use in ScrapeOptions.Formats.
|
||||
/// </summary>
|
||||
public class JsonFormat
|
||||
{
|
||||
[JsonPropertyName("type")]
|
||||
public string Type { get; } = "json";
|
||||
|
||||
[JsonPropertyName("prompt")]
|
||||
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
|
||||
public string? Prompt { get; set; }
|
||||
|
||||
[JsonPropertyName("schema")]
|
||||
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
|
||||
public Dictionary<string, object>? Schema { get; set; }
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace Firecrawl.Models;
|
||||
|
||||
/// <summary>
|
||||
/// Geolocation configuration for requests.
|
||||
/// </summary>
|
||||
public class LocationConfig
|
||||
{
|
||||
[JsonPropertyName("country")]
|
||||
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
|
||||
public string? Country { get; set; }
|
||||
|
||||
[JsonPropertyName("languages")]
|
||||
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
|
||||
public List<string>? Languages { get; set; }
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace Firecrawl.Models;
|
||||
|
||||
/// <summary>
|
||||
/// URL discovery (map) results.
|
||||
/// </summary>
|
||||
public class MapData
|
||||
{
|
||||
[JsonPropertyName("links")]
|
||||
public List<string>? Links { get; set; }
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace Firecrawl.Models;
|
||||
|
||||
/// <summary>
|
||||
/// Configuration options for URL discovery (map).
|
||||
/// </summary>
|
||||
public class MapOptions
|
||||
{
|
||||
[JsonPropertyName("search")]
|
||||
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
|
||||
public string? Search { get; set; }
|
||||
|
||||
[JsonPropertyName("sitemap")]
|
||||
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
|
||||
public string? Sitemap { get; set; }
|
||||
|
||||
[JsonPropertyName("includeSubdomains")]
|
||||
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
|
||||
public bool? IncludeSubdomains { get; set; }
|
||||
|
||||
[JsonPropertyName("ignoreQueryParameters")]
|
||||
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
|
||||
public bool? IgnoreQueryParameters { get; set; }
|
||||
|
||||
[JsonPropertyName("limit")]
|
||||
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
|
||||
public int? Limit { get; set; }
|
||||
|
||||
[JsonPropertyName("timeout")]
|
||||
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
|
||||
public int? Timeout { get; set; }
|
||||
|
||||
[JsonPropertyName("integration")]
|
||||
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
|
||||
public string? Integration { get; set; }
|
||||
|
||||
[JsonPropertyName("location")]
|
||||
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
|
||||
public LocationConfig? Location { get; set; }
|
||||
|
||||
[JsonPropertyName("ignoreCache")]
|
||||
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
|
||||
public bool? IgnoreCache { get; set; }
|
||||
}
|
||||
@@ -0,0 +1,225 @@
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace Firecrawl.Models;
|
||||
|
||||
public class MonitorSchedule
|
||||
{
|
||||
[JsonPropertyName("cron")]
|
||||
public string? Cron { get; set; }
|
||||
|
||||
[JsonPropertyName("timezone")]
|
||||
public string? Timezone { get; set; }
|
||||
}
|
||||
|
||||
public class CreateMonitorRequest
|
||||
{
|
||||
[JsonPropertyName("name")]
|
||||
public string? Name { get; set; }
|
||||
|
||||
[JsonPropertyName("schedule")]
|
||||
public MonitorSchedule? Schedule { get; set; }
|
||||
|
||||
[JsonPropertyName("targets")]
|
||||
public List<Dictionary<string, object>>? Targets { get; set; }
|
||||
|
||||
[JsonPropertyName("webhook")]
|
||||
public Dictionary<string, object>? Webhook { get; set; }
|
||||
|
||||
[JsonPropertyName("notification")]
|
||||
public Dictionary<string, object>? Notification { get; set; }
|
||||
|
||||
[JsonPropertyName("retentionDays")]
|
||||
public int? RetentionDays { get; set; }
|
||||
}
|
||||
|
||||
public class UpdateMonitorRequest
|
||||
{
|
||||
[JsonPropertyName("name")]
|
||||
public string? Name { get; set; }
|
||||
|
||||
[JsonPropertyName("status")]
|
||||
public string? Status { get; set; }
|
||||
|
||||
[JsonPropertyName("schedule")]
|
||||
public MonitorSchedule? Schedule { get; set; }
|
||||
|
||||
[JsonPropertyName("targets")]
|
||||
public List<Dictionary<string, object>>? Targets { get; set; }
|
||||
|
||||
[JsonPropertyName("webhook")]
|
||||
public Dictionary<string, object>? Webhook { get; set; }
|
||||
|
||||
[JsonPropertyName("notification")]
|
||||
public Dictionary<string, object>? Notification { get; set; }
|
||||
|
||||
[JsonPropertyName("retentionDays")]
|
||||
public int? RetentionDays { get; set; }
|
||||
}
|
||||
|
||||
public class MonitorSummary
|
||||
{
|
||||
[JsonPropertyName("totalPages")]
|
||||
public int TotalPages { get; set; }
|
||||
|
||||
[JsonPropertyName("same")]
|
||||
public int Same { get; set; }
|
||||
|
||||
[JsonPropertyName("changed")]
|
||||
public int Changed { get; set; }
|
||||
|
||||
[JsonPropertyName("new")]
|
||||
public int New { get; set; }
|
||||
|
||||
[JsonPropertyName("removed")]
|
||||
public int Removed { get; set; }
|
||||
|
||||
[JsonPropertyName("error")]
|
||||
public int Error { get; set; }
|
||||
}
|
||||
|
||||
public class Monitor
|
||||
{
|
||||
[JsonPropertyName("id")]
|
||||
public string? Id { get; set; }
|
||||
|
||||
[JsonPropertyName("name")]
|
||||
public string? Name { get; set; }
|
||||
|
||||
[JsonPropertyName("status")]
|
||||
public string? Status { get; set; }
|
||||
|
||||
[JsonPropertyName("schedule")]
|
||||
public MonitorSchedule? Schedule { get; set; }
|
||||
|
||||
[JsonPropertyName("nextRunAt")]
|
||||
public string? NextRunAt { get; set; }
|
||||
|
||||
[JsonPropertyName("lastRunAt")]
|
||||
public string? LastRunAt { get; set; }
|
||||
|
||||
[JsonPropertyName("currentCheckId")]
|
||||
public string? CurrentCheckId { get; set; }
|
||||
|
||||
[JsonPropertyName("targets")]
|
||||
public List<Dictionary<string, object>>? Targets { get; set; }
|
||||
|
||||
[JsonPropertyName("webhook")]
|
||||
public Dictionary<string, object>? Webhook { get; set; }
|
||||
|
||||
[JsonPropertyName("notification")]
|
||||
public Dictionary<string, object>? Notification { get; set; }
|
||||
|
||||
[JsonPropertyName("retentionDays")]
|
||||
public int RetentionDays { get; set; }
|
||||
|
||||
[JsonPropertyName("estimatedCreditsPerMonth")]
|
||||
public int? EstimatedCreditsPerMonth { get; set; }
|
||||
|
||||
[JsonPropertyName("lastCheckSummary")]
|
||||
public MonitorSummary? LastCheckSummary { get; set; }
|
||||
|
||||
[JsonPropertyName("createdAt")]
|
||||
public string? CreatedAt { get; set; }
|
||||
|
||||
[JsonPropertyName("updatedAt")]
|
||||
public string? UpdatedAt { get; set; }
|
||||
}
|
||||
|
||||
public class MonitorCheck
|
||||
{
|
||||
[JsonPropertyName("id")]
|
||||
public string? Id { get; set; }
|
||||
|
||||
[JsonPropertyName("monitorId")]
|
||||
public string? MonitorId { get; set; }
|
||||
|
||||
[JsonPropertyName("status")]
|
||||
public string? Status { get; set; }
|
||||
|
||||
[JsonPropertyName("trigger")]
|
||||
public string? Trigger { get; set; }
|
||||
|
||||
[JsonPropertyName("scheduledFor")]
|
||||
public string? ScheduledFor { get; set; }
|
||||
|
||||
[JsonPropertyName("startedAt")]
|
||||
public string? StartedAt { get; set; }
|
||||
|
||||
[JsonPropertyName("finishedAt")]
|
||||
public string? FinishedAt { get; set; }
|
||||
|
||||
[JsonPropertyName("estimatedCredits")]
|
||||
public int? EstimatedCredits { get; set; }
|
||||
|
||||
[JsonPropertyName("reservedCredits")]
|
||||
public int? ReservedCredits { get; set; }
|
||||
|
||||
[JsonPropertyName("actualCredits")]
|
||||
public int? ActualCredits { get; set; }
|
||||
|
||||
[JsonPropertyName("billingStatus")]
|
||||
public string? BillingStatus { get; set; }
|
||||
|
||||
[JsonPropertyName("summary")]
|
||||
public MonitorSummary? Summary { get; set; }
|
||||
|
||||
[JsonPropertyName("targetResults")]
|
||||
public object? TargetResults { get; set; }
|
||||
|
||||
[JsonPropertyName("notificationStatus")]
|
||||
public object? NotificationStatus { get; set; }
|
||||
|
||||
[JsonPropertyName("error")]
|
||||
public string? Error { get; set; }
|
||||
|
||||
[JsonPropertyName("createdAt")]
|
||||
public string? CreatedAt { get; set; }
|
||||
|
||||
[JsonPropertyName("updatedAt")]
|
||||
public string? UpdatedAt { get; set; }
|
||||
}
|
||||
|
||||
public class MonitorCheckPage
|
||||
{
|
||||
[JsonPropertyName("id")]
|
||||
public string? Id { get; set; }
|
||||
|
||||
[JsonPropertyName("targetId")]
|
||||
public string? TargetId { get; set; }
|
||||
|
||||
[JsonPropertyName("url")]
|
||||
public string? Url { get; set; }
|
||||
|
||||
[JsonPropertyName("status")]
|
||||
public string? Status { get; set; }
|
||||
|
||||
[JsonPropertyName("previousScrapeId")]
|
||||
public string? PreviousScrapeId { get; set; }
|
||||
|
||||
[JsonPropertyName("currentScrapeId")]
|
||||
public string? CurrentScrapeId { get; set; }
|
||||
|
||||
[JsonPropertyName("statusCode")]
|
||||
public int? StatusCode { get; set; }
|
||||
|
||||
[JsonPropertyName("error")]
|
||||
public string? Error { get; set; }
|
||||
|
||||
[JsonPropertyName("metadata")]
|
||||
public object? Metadata { get; set; }
|
||||
|
||||
[JsonPropertyName("diff")]
|
||||
public object? Diff { get; set; }
|
||||
|
||||
[JsonPropertyName("createdAt")]
|
||||
public string? CreatedAt { get; set; }
|
||||
}
|
||||
|
||||
public class MonitorCheckDetail : MonitorCheck
|
||||
{
|
||||
[JsonPropertyName("pages")]
|
||||
public List<MonitorCheckPage>? Pages { get; set; }
|
||||
|
||||
[JsonPropertyName("next")]
|
||||
public string? Next { get; set; }
|
||||
}
|
||||
@@ -0,0 +1,80 @@
|
||||
namespace Firecrawl.Models;
|
||||
|
||||
/// <summary>
|
||||
/// Uploaded file payload for the <c>/v2/parse</c> endpoint.
|
||||
/// </summary>
|
||||
public class ParseFile
|
||||
{
|
||||
/// <summary>
|
||||
/// Filename used in the multipart upload (e.g. <c>upload.pdf</c>).
|
||||
/// </summary>
|
||||
public string Filename { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Raw bytes of the file to be parsed.
|
||||
/// </summary>
|
||||
public byte[] Content { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Optional MIME type hint (e.g. <c>application/pdf</c>).
|
||||
/// When null, the value is guessed from the filename extension,
|
||||
/// falling back to <c>application/octet-stream</c>.
|
||||
/// </summary>
|
||||
public string? ContentType { get; set; }
|
||||
|
||||
public ParseFile(string filename, byte[] content, string? contentType = null)
|
||||
{
|
||||
Filename = filename;
|
||||
Content = content;
|
||||
ContentType = contentType;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Build a <see cref="ParseFile"/> from raw bytes.
|
||||
/// </summary>
|
||||
public static ParseFile FromBytes(string filename, byte[] content, string? contentType = null)
|
||||
=> new(filename, content, contentType);
|
||||
|
||||
/// <summary>
|
||||
/// Build a <see cref="ParseFile"/> by reading a file from disk.
|
||||
/// The filename is derived from the path unless overridden.
|
||||
/// </summary>
|
||||
public static ParseFile FromPath(string path, string? filename = null, string? contentType = null)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(path))
|
||||
throw new ArgumentException("Path cannot be empty.", nameof(path));
|
||||
|
||||
if (!File.Exists(path))
|
||||
throw new FileNotFoundException($"Parse file not found: {path}", path);
|
||||
|
||||
var bytes = File.ReadAllBytes(path);
|
||||
var resolvedName = filename ?? Path.GetFileName(path);
|
||||
return new ParseFile(resolvedName, bytes, contentType);
|
||||
}
|
||||
|
||||
internal string ResolveContentType()
|
||||
{
|
||||
if (!string.IsNullOrWhiteSpace(ContentType))
|
||||
return ContentType;
|
||||
|
||||
var extension = Path.GetExtension(Filename).ToLowerInvariant();
|
||||
return extension switch
|
||||
{
|
||||
".html" or ".htm" => "text/html",
|
||||
".pdf" => "application/pdf",
|
||||
".docx" => "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
|
||||
".doc" => "application/msword",
|
||||
".xlsx" => "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
|
||||
".xls" => "application/vnd.ms-excel",
|
||||
".pptx" => "application/vnd.openxmlformats-officedocument.presentationml.presentation",
|
||||
".ppt" => "application/vnd.ms-powerpoint",
|
||||
".txt" => "text/plain",
|
||||
".md" => "text/markdown",
|
||||
".csv" => "text/csv",
|
||||
".json" => "application/json",
|
||||
".xml" => "application/xml",
|
||||
".rtf" => "application/rtf",
|
||||
_ => "application/octet-stream",
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,134 @@
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace Firecrawl.Models;
|
||||
|
||||
/// <summary>
|
||||
/// Options for the <c>/v2/parse</c> endpoint.
|
||||
///
|
||||
/// <para>
|
||||
/// Parse does not support browser-rendering formats/options such as change
|
||||
/// tracking, screenshot, branding, actions, waitFor, location, or mobile.
|
||||
/// These are rejected client-side in <see cref="Validate"/>.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
public class ParseOptions
|
||||
{
|
||||
internal static readonly HashSet<string> UnsupportedFormats = new(StringComparer.OrdinalIgnoreCase)
|
||||
{
|
||||
"changeTracking",
|
||||
"change_tracking",
|
||||
"screenshot",
|
||||
"screenshot@fullPage",
|
||||
"branding",
|
||||
};
|
||||
|
||||
internal static readonly HashSet<string> SupportedProxies = new(StringComparer.OrdinalIgnoreCase)
|
||||
{
|
||||
"auto",
|
||||
"basic",
|
||||
};
|
||||
|
||||
[JsonPropertyName("formats")]
|
||||
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
|
||||
public List<object>? Formats { get; set; }
|
||||
|
||||
[JsonPropertyName("headers")]
|
||||
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
|
||||
public Dictionary<string, string>? Headers { get; set; }
|
||||
|
||||
[JsonPropertyName("includeTags")]
|
||||
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
|
||||
public List<string>? IncludeTags { get; set; }
|
||||
|
||||
[JsonPropertyName("excludeTags")]
|
||||
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
|
||||
public List<string>? ExcludeTags { get; set; }
|
||||
|
||||
[JsonPropertyName("onlyMainContent")]
|
||||
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
|
||||
public bool? OnlyMainContent { get; set; }
|
||||
|
||||
[JsonPropertyName("timeout")]
|
||||
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
|
||||
public int? Timeout { get; set; }
|
||||
|
||||
[JsonPropertyName("parsers")]
|
||||
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
|
||||
public List<object>? Parsers { get; set; }
|
||||
|
||||
[JsonPropertyName("skipTlsVerification")]
|
||||
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
|
||||
public bool? SkipTlsVerification { get; set; }
|
||||
|
||||
[JsonPropertyName("removeBase64Images")]
|
||||
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
|
||||
public bool? RemoveBase64Images { get; set; }
|
||||
|
||||
[JsonPropertyName("blockAds")]
|
||||
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
|
||||
public bool? BlockAds { get; set; }
|
||||
|
||||
[JsonPropertyName("proxy")]
|
||||
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
|
||||
public string? Proxy { get; set; }
|
||||
|
||||
[JsonPropertyName("integration")]
|
||||
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
|
||||
public string? Integration { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Validate the options against /v2/parse's supported surface.
|
||||
/// </summary>
|
||||
/// <exception cref="ArgumentException">Thrown when an unsupported format,
|
||||
/// proxy, or timeout value is set.</exception>
|
||||
public void Validate()
|
||||
{
|
||||
if (Timeout is not null and <= 0)
|
||||
throw new ArgumentException("timeout must be positive", nameof(Timeout));
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(Proxy) && !SupportedProxies.Contains(Proxy))
|
||||
throw new ArgumentException(
|
||||
"parse only supports proxy values 'basic' or 'auto'", nameof(Proxy));
|
||||
|
||||
if (Formats is { Count: > 0 })
|
||||
{
|
||||
foreach (var fmt in Formats)
|
||||
{
|
||||
var type = ExtractFormatType(fmt);
|
||||
if (type is not null && UnsupportedFormats.Contains(type))
|
||||
throw new ArgumentException($"parse does not support format: {type}", nameof(Formats));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static string? ExtractFormatType(object? format)
|
||||
{
|
||||
if (format is null)
|
||||
return null;
|
||||
|
||||
if (format is string s)
|
||||
return s;
|
||||
|
||||
if (format is IDictionary<string, object?> dict &&
|
||||
dict.TryGetValue("type", out var typeObj) &&
|
||||
typeObj is string ts)
|
||||
{
|
||||
return ts;
|
||||
}
|
||||
|
||||
if (format is JsonElement element)
|
||||
{
|
||||
if (element.ValueKind == JsonValueKind.String)
|
||||
return element.GetString();
|
||||
if (element.ValueKind == JsonValueKind.Object &&
|
||||
element.TryGetProperty("type", out var typeProp) &&
|
||||
typeProp.ValueKind == JsonValueKind.String)
|
||||
{
|
||||
return typeProp.GetString();
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace Firecrawl.Models;
|
||||
|
||||
/// <summary>
|
||||
/// Deprecated query format specification for use in ScrapeOptions.Formats.
|
||||
/// </summary>
|
||||
[Obsolete("Use QuestionFormat or HighlightsFormat instead.")]
|
||||
public class QueryFormat
|
||||
{
|
||||
public const string FreeformMode = "freeform";
|
||||
public const string DirectQuoteMode = "directQuote";
|
||||
|
||||
[JsonPropertyName("type")]
|
||||
public string Type { get; } = "query";
|
||||
|
||||
[JsonPropertyName("prompt")]
|
||||
public required string Prompt { get; set; }
|
||||
|
||||
[JsonPropertyName("mode")]
|
||||
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
|
||||
public string? Mode { get; set; }
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace Firecrawl.Models;
|
||||
|
||||
/// <summary>
|
||||
/// Question format specification for use in ScrapeOptions.Formats.
|
||||
/// </summary>
|
||||
public class QuestionFormat
|
||||
{
|
||||
[JsonPropertyName("type")]
|
||||
public string Type { get; } = "question";
|
||||
|
||||
[JsonPropertyName("question")]
|
||||
public required string Question { get; set; }
|
||||
}
|
||||
@@ -0,0 +1,81 @@
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace Firecrawl.Models;
|
||||
|
||||
/// <summary>
|
||||
/// Configuration options for scraping a single URL.
|
||||
/// </summary>
|
||||
public class ScrapeOptions
|
||||
{
|
||||
[JsonPropertyName("formats")]
|
||||
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
|
||||
public List<object>? Formats { get; set; }
|
||||
|
||||
[JsonPropertyName("headers")]
|
||||
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
|
||||
public Dictionary<string, string>? Headers { get; set; }
|
||||
|
||||
[JsonPropertyName("includeTags")]
|
||||
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
|
||||
public List<string>? IncludeTags { get; set; }
|
||||
|
||||
[JsonPropertyName("excludeTags")]
|
||||
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
|
||||
public List<string>? ExcludeTags { get; set; }
|
||||
|
||||
[JsonPropertyName("onlyMainContent")]
|
||||
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
|
||||
public bool? OnlyMainContent { get; set; }
|
||||
|
||||
[JsonPropertyName("timeout")]
|
||||
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
|
||||
public int? Timeout { get; set; }
|
||||
|
||||
[JsonPropertyName("waitFor")]
|
||||
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
|
||||
public int? WaitFor { get; set; }
|
||||
|
||||
[JsonPropertyName("mobile")]
|
||||
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
|
||||
public bool? Mobile { get; set; }
|
||||
|
||||
[JsonPropertyName("parsers")]
|
||||
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
|
||||
public List<object>? Parsers { get; set; }
|
||||
|
||||
[JsonPropertyName("actions")]
|
||||
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
|
||||
public List<Dictionary<string, object>>? Actions { get; set; }
|
||||
|
||||
[JsonPropertyName("location")]
|
||||
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
|
||||
public LocationConfig? Location { get; set; }
|
||||
|
||||
[JsonPropertyName("skipTlsVerification")]
|
||||
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
|
||||
public bool? SkipTlsVerification { get; set; }
|
||||
|
||||
[JsonPropertyName("removeBase64Images")]
|
||||
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
|
||||
public bool? RemoveBase64Images { get; set; }
|
||||
|
||||
[JsonPropertyName("blockAds")]
|
||||
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
|
||||
public bool? BlockAds { get; set; }
|
||||
|
||||
[JsonPropertyName("proxy")]
|
||||
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
|
||||
public string? Proxy { get; set; }
|
||||
|
||||
[JsonPropertyName("maxAge")]
|
||||
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
|
||||
public long? MaxAge { get; set; }
|
||||
|
||||
[JsonPropertyName("storeInCache")]
|
||||
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
|
||||
public bool? StoreInCache { get; set; }
|
||||
|
||||
[JsonPropertyName("integration")]
|
||||
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
|
||||
public string? Integration { get; set; }
|
||||
}
|
||||
@@ -0,0 +1,144 @@
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace Firecrawl.Models;
|
||||
|
||||
/// <summary>
|
||||
/// Represents a single web search hit.
|
||||
/// </summary>
|
||||
public class WebSearchHit
|
||||
{
|
||||
[JsonPropertyName("url")]
|
||||
public string? Url { get; set; }
|
||||
|
||||
[JsonPropertyName("title")]
|
||||
public string? Title { get; set; }
|
||||
|
||||
[JsonPropertyName("description")]
|
||||
public string? Description { get; set; }
|
||||
|
||||
[JsonPropertyName("position")]
|
||||
public int? Position { get; set; }
|
||||
|
||||
[JsonPropertyName("category")]
|
||||
public string? Category { get; set; }
|
||||
|
||||
[JsonPropertyName("markdown")]
|
||||
public string? Markdown { get; set; }
|
||||
|
||||
[JsonPropertyName("html")]
|
||||
public string? Html { get; set; }
|
||||
|
||||
[JsonPropertyName("rawHtml")]
|
||||
public string? RawHtml { get; set; }
|
||||
|
||||
[JsonPropertyName("links")]
|
||||
public List<string>? Links { get; set; }
|
||||
|
||||
[JsonPropertyName("screenshot")]
|
||||
public string? Screenshot { get; set; }
|
||||
|
||||
[JsonPropertyName("metadata")]
|
||||
public Dictionary<string, object>? Metadata { get; set; }
|
||||
|
||||
[JsonPropertyName("answer")]
|
||||
public string? Answer { get; set; }
|
||||
|
||||
[JsonPropertyName("highlights")]
|
||||
public string? Highlights { get; set; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Represents a news search result with news-specific fields.
|
||||
/// </summary>
|
||||
public class NewsSearchHit
|
||||
{
|
||||
[JsonPropertyName("url")]
|
||||
public string? Url { get; set; }
|
||||
|
||||
[JsonPropertyName("title")]
|
||||
public string? Title { get; set; }
|
||||
|
||||
[JsonPropertyName("snippet")]
|
||||
public string? Snippet { get; set; }
|
||||
|
||||
[JsonPropertyName("date")]
|
||||
public string? Date { get; set; }
|
||||
|
||||
[JsonPropertyName("imageUrl")]
|
||||
public string? ImageUrl { get; set; }
|
||||
|
||||
[JsonPropertyName("position")]
|
||||
public int? Position { get; set; }
|
||||
|
||||
[JsonPropertyName("category")]
|
||||
public string? Category { get; set; }
|
||||
|
||||
[JsonPropertyName("markdown")]
|
||||
public string? Markdown { get; set; }
|
||||
|
||||
[JsonPropertyName("html")]
|
||||
public string? Html { get; set; }
|
||||
|
||||
[JsonPropertyName("rawHtml")]
|
||||
public string? RawHtml { get; set; }
|
||||
|
||||
[JsonPropertyName("links")]
|
||||
public List<string>? Links { get; set; }
|
||||
|
||||
[JsonPropertyName("screenshot")]
|
||||
public string? Screenshot { get; set; }
|
||||
|
||||
[JsonPropertyName("metadata")]
|
||||
public Dictionary<string, object>? Metadata { get; set; }
|
||||
|
||||
[JsonPropertyName("answer")]
|
||||
public string? Answer { get; set; }
|
||||
|
||||
[JsonPropertyName("highlights")]
|
||||
public string? Highlights { get; set; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Represents an image search result.
|
||||
/// </summary>
|
||||
public class ImageSearchHit
|
||||
{
|
||||
[JsonPropertyName("title")]
|
||||
public string? Title { get; set; }
|
||||
|
||||
[JsonPropertyName("imageUrl")]
|
||||
public string? ImageUrl { get; set; }
|
||||
|
||||
[JsonPropertyName("imageWidth")]
|
||||
public int? ImageWidth { get; set; }
|
||||
|
||||
[JsonPropertyName("imageHeight")]
|
||||
public int? ImageHeight { get; set; }
|
||||
|
||||
[JsonPropertyName("url")]
|
||||
public string? Url { get; set; }
|
||||
|
||||
[JsonPropertyName("position")]
|
||||
public int? Position { get; set; }
|
||||
|
||||
[JsonPropertyName("answer")]
|
||||
public string? Answer { get; set; }
|
||||
|
||||
[JsonPropertyName("highlights")]
|
||||
public string? Highlights { get; set; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Web search results.
|
||||
/// </summary>
|
||||
public class SearchData
|
||||
{
|
||||
[JsonPropertyName("web")]
|
||||
public List<WebSearchHit>? Web { get; set; }
|
||||
|
||||
[JsonPropertyName("news")]
|
||||
public List<NewsSearchHit>? News { get; set; }
|
||||
|
||||
[JsonPropertyName("images")]
|
||||
public List<ImageSearchHit>? Images { get; set; }
|
||||
}
|
||||
@@ -0,0 +1,61 @@
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace Firecrawl.Models;
|
||||
|
||||
/// <summary>
|
||||
/// Configuration options for web search.
|
||||
/// </summary>
|
||||
public class SearchOptions
|
||||
{
|
||||
[JsonPropertyName("sources")]
|
||||
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
|
||||
public List<object>? Sources { get; set; }
|
||||
|
||||
[JsonPropertyName("categories")]
|
||||
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
|
||||
public List<object>? Categories { get; set; }
|
||||
|
||||
[JsonPropertyName("includeDomains")]
|
||||
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
|
||||
public List<string>? IncludeDomains { get; set; }
|
||||
|
||||
[JsonPropertyName("excludeDomains")]
|
||||
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
|
||||
public List<string>? ExcludeDomains { get; set; }
|
||||
|
||||
[JsonPropertyName("limit")]
|
||||
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
|
||||
public int? Limit { get; set; }
|
||||
|
||||
[JsonPropertyName("tbs")]
|
||||
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
|
||||
public string? Tbs { get; set; }
|
||||
|
||||
[JsonPropertyName("location")]
|
||||
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
|
||||
public string? Location { get; set; }
|
||||
|
||||
[JsonPropertyName("ignoreInvalidURLs")]
|
||||
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
|
||||
public bool? IgnoreInvalidURLs { get; set; }
|
||||
|
||||
[JsonPropertyName("timeout")]
|
||||
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
|
||||
public int? Timeout { get; set; }
|
||||
|
||||
[JsonPropertyName("scrapeOptions")]
|
||||
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
|
||||
public ScrapeOptions? ScrapeOptions { get; set; }
|
||||
|
||||
[JsonPropertyName("integration")]
|
||||
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
|
||||
public string? Integration { get; set; }
|
||||
|
||||
[JsonPropertyName("country")]
|
||||
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
|
||||
public string? Country { get; set; }
|
||||
|
||||
[JsonPropertyName("enterprise")]
|
||||
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
|
||||
public bool? Enterprise { get; set; }
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace Firecrawl.Models;
|
||||
|
||||
/// <summary>
|
||||
/// Webhook configuration for async jobs.
|
||||
/// </summary>
|
||||
public class WebhookConfig
|
||||
{
|
||||
[JsonPropertyName("url")]
|
||||
public required string Url { get; set; }
|
||||
|
||||
[JsonPropertyName("headers")]
|
||||
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
|
||||
public Dictionary<string, string>? Headers { get; set; }
|
||||
|
||||
[JsonPropertyName("metadata")]
|
||||
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
|
||||
public Dictionary<string, string>? Metadata { get; set; }
|
||||
|
||||
[JsonPropertyName("events")]
|
||||
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
|
||||
public List<string>? Events { get; set; }
|
||||
}
|
||||
Reference in New Issue
Block a user