참고소스 수정본

This commit is contained in:
LASTA_DEV01\lasta
2026-05-12 19:40:31 +09:00
parent 0f34a451fc
commit 2e9204243d
8708 changed files with 3259488 additions and 869 deletions

View File

@@ -0,0 +1,70 @@
using Firecrawl.Exceptions;
using Xunit;
namespace Firecrawl.Tests;
public class ExceptionsTests
{
[Fact]
public void FirecrawlException_HasMessage()
{
var ex = new FirecrawlException("test error");
Assert.Equal("test error", ex.Message);
}
[Fact]
public void FirecrawlException_HasStatusCode()
{
var ex = new FirecrawlException("test error", 500);
Assert.Equal(500, ex.StatusCode);
}
[Fact]
public void FirecrawlException_HasErrorCode()
{
var ex = new FirecrawlException("test error", 400, "INVALID_REQUEST", null);
Assert.Equal(400, ex.StatusCode);
Assert.Equal("INVALID_REQUEST", ex.ErrorCode);
}
[Fact]
public void FirecrawlException_HasInnerException()
{
var inner = new InvalidOperationException("inner");
var ex = new FirecrawlException("wrapper", inner);
Assert.Equal("wrapper", ex.Message);
Assert.Same(inner, ex.InnerException);
}
[Fact]
public void AuthenticationException_Has401StatusCode()
{
var ex = new AuthenticationException("Unauthorized");
Assert.Equal(401, ex.StatusCode);
}
[Fact]
public void AuthenticationException_HasErrorCode()
{
var ex = new AuthenticationException("Unauthorized", "AUTH_FAILED");
Assert.Equal("AUTH_FAILED", ex.ErrorCode);
}
[Fact]
public void RateLimitException_Has429StatusCode()
{
var ex = new RateLimitException("Too many requests");
Assert.Equal(429, ex.StatusCode);
}
[Fact]
public void JobTimeoutException_HasJobIdAndTimeout()
{
var ex = new JobTimeoutException("job-123", 300, "Crawl");
Assert.Equal("job-123", ex.JobId);
Assert.Equal(300, ex.TimeoutSeconds);
Assert.Contains("job-123", ex.Message);
Assert.Contains("300", ex.Message);
Assert.Contains("Crawl", ex.Message);
}
}

View File

@@ -0,0 +1,23 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net8.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
<IsPackable>false</IsPackable>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.12.0" />
<PackageReference Include="xunit" Version="2.9.3" />
<PackageReference Include="xunit.runner.visualstudio" Version="2.8.2">
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
<PrivateAssets>all</PrivateAssets>
</PackageReference>
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\Firecrawl\Firecrawl.csproj" />
</ItemGroup>
</Project>

View File

@@ -0,0 +1,132 @@
using Firecrawl.Exceptions;
using Firecrawl.Models;
using Xunit;
namespace Firecrawl.Tests;
public class FirecrawlClientTests
{
[Fact]
public void Constructor_RequiresApiKey()
{
// Clear env variable in case it's set
Environment.SetEnvironmentVariable("FIRECRAWL_API_KEY", null);
var ex = Assert.Throws<FirecrawlException>(() => new FirecrawlClient(apiKey: ""));
Assert.Contains("API key is required", ex.Message);
}
[Fact]
public void Constructor_RequiresApiKey_WhenNull()
{
Environment.SetEnvironmentVariable("FIRECRAWL_API_KEY", null);
var ex = Assert.Throws<FirecrawlException>(() => new FirecrawlClient(apiKey: null));
Assert.Contains("API key is required", ex.Message);
}
[Fact]
public void Constructor_AcceptsApiKey()
{
var client = new FirecrawlClient(apiKey: "fc-test-key");
Assert.NotNull(client);
}
[Fact]
public void Constructor_AcceptsCustomHttpClient()
{
var httpClient = new HttpClient { Timeout = TimeSpan.FromSeconds(10) };
var client = new FirecrawlClient(
apiKey: "fc-test-key",
httpClient: httpClient);
Assert.NotNull(client);
}
[Fact]
public void Constructor_AcceptsCustomApiUrl()
{
var client = new FirecrawlClient(
apiKey: "fc-test-key",
apiUrl: "https://custom-api.firecrawl.dev");
Assert.NotNull(client);
}
[Fact]
public void Constructor_ReadsFromEnvironmentVariable()
{
Environment.SetEnvironmentVariable("FIRECRAWL_API_KEY", "fc-env-key");
try
{
var client = new FirecrawlClient();
Assert.NotNull(client);
}
finally
{
Environment.SetEnvironmentVariable("FIRECRAWL_API_KEY", null);
}
}
[Fact]
public async Task ScrapeAsync_RequiresUrl()
{
var client = new FirecrawlClient(apiKey: "fc-test-key");
await Assert.ThrowsAsync<ArgumentNullException>(
() => client.ScrapeAsync(null!));
}
[Fact]
public async Task StartCrawlAsync_RequiresUrl()
{
var client = new FirecrawlClient(apiKey: "fc-test-key");
await Assert.ThrowsAsync<ArgumentNullException>(
() => client.StartCrawlAsync(null!));
}
[Fact]
public async Task MapAsync_RequiresUrl()
{
var client = new FirecrawlClient(apiKey: "fc-test-key");
await Assert.ThrowsAsync<ArgumentNullException>(
() => client.MapAsync(null!));
}
[Fact]
public async Task SearchAsync_RequiresQuery()
{
var client = new FirecrawlClient(apiKey: "fc-test-key");
await Assert.ThrowsAsync<ArgumentNullException>(
() => client.SearchAsync(null!));
}
[Fact]
public async Task StartBatchScrapeAsync_RequiresUrls()
{
var client = new FirecrawlClient(apiKey: "fc-test-key");
await Assert.ThrowsAsync<ArgumentNullException>(
() => client.StartBatchScrapeAsync(null!));
}
[Fact]
public async Task CancelCrawlAsync_RequiresJobId()
{
var client = new FirecrawlClient(apiKey: "fc-test-key");
await Assert.ThrowsAsync<ArgumentNullException>(
() => client.CancelCrawlAsync(null!));
}
[Fact]
public async Task GetCrawlStatusAsync_RequiresJobId()
{
var client = new FirecrawlClient(apiKey: "fc-test-key");
await Assert.ThrowsAsync<ArgumentNullException>(
() => client.GetCrawlStatusAsync(null!));
}
}

View File

@@ -0,0 +1,347 @@
using System.Text.Json;
using Firecrawl.Models;
using Xunit;
namespace Firecrawl.Tests;
public class ModelsTests
{
private static readonly JsonSerializerOptions JsonOptions = FirecrawlHttpClient.JsonOptions;
[Fact]
public void ScrapeOptions_SerializesCorrectly()
{
var options = new ScrapeOptions
{
Formats = new List<object> { "markdown", "html" },
OnlyMainContent = true,
Timeout = 30000,
Mobile = false
};
var json = JsonSerializer.Serialize(options, JsonOptions);
Assert.Contains("\"formats\"", json);
Assert.Contains("\"markdown\"", json);
Assert.Contains("\"html\"", json);
Assert.Contains("\"onlyMainContent\":true", json);
Assert.Contains("\"timeout\":30000", json);
Assert.Contains("\"mobile\":false", json);
}
[Fact]
public void ScrapeOptions_OmitsNullProperties()
{
var options = new ScrapeOptions
{
Formats = new List<object> { "markdown" }
};
var json = JsonSerializer.Serialize(options, JsonOptions);
Assert.Contains("\"formats\"", json);
Assert.DoesNotContain("\"timeout\"", json);
Assert.DoesNotContain("\"mobile\"", json);
Assert.DoesNotContain("\"headers\"", json);
}
[Fact]
public void CrawlOptions_SerializesCorrectly()
{
var options = new CrawlOptions
{
Limit = 100,
MaxDiscoveryDepth = 3,
Sitemap = "include",
ExcludePaths = new List<string> { "/admin/*" }
};
var json = JsonSerializer.Serialize(options, JsonOptions);
Assert.Contains("\"limit\":100", json);
Assert.Contains("\"maxDiscoveryDepth\":3", json);
Assert.Contains("\"sitemap\":\"include\"", json);
Assert.Contains("\"/admin/*\"", json);
}
[Fact]
public void MapOptions_SerializesCorrectly()
{
var options = new MapOptions
{
Search = "pricing",
Limit = 10,
IncludeSubdomains = true
};
var json = JsonSerializer.Serialize(options, JsonOptions);
Assert.Contains("\"search\":\"pricing\"", json);
Assert.Contains("\"limit\":10", json);
Assert.Contains("\"includeSubdomains\":true", json);
}
[Fact]
public void SearchOptions_SerializesCorrectly()
{
var options = new SearchOptions
{
Limit = 5,
Location = "US",
Tbs = "qdr:w",
IncludeDomains = new() { "firecrawl.dev" },
ExcludeDomains = new() { "example.com" }
};
var json = JsonSerializer.Serialize(options, JsonOptions);
Assert.Contains("\"limit\":5", json);
Assert.Contains("\"location\":\"US\"", json);
Assert.Contains("\"tbs\":\"qdr:w\"", json);
Assert.Contains("\"includeDomains\":[\"firecrawl.dev\"]", json);
Assert.Contains("\"excludeDomains\":[\"example.com\"]", json);
}
[Fact]
public void BatchScrapeOptions_IdempotencyKey_NotSerialized()
{
var options = new BatchScrapeOptions
{
IdempotencyKey = "my-key-123",
IgnoreInvalidURLs = true
};
var json = JsonSerializer.Serialize(options, JsonOptions);
Assert.DoesNotContain("idempotencyKey", json);
Assert.DoesNotContain("my-key-123", json);
Assert.Contains("\"ignoreInvalidURLs\":true", json);
}
[Fact]
public void Document_DeserializesCorrectly()
{
var json = """
{
"markdown": "# Hello World",
"html": "<h1>Hello World</h1>",
"metadata": {
"title": "Test",
"sourceURL": "https://example.com"
},
"warning": null
}
""";
var doc = JsonSerializer.Deserialize<Document>(json, JsonOptions);
Assert.NotNull(doc);
Assert.Equal("# Hello World", doc.Markdown);
Assert.Equal("<h1>Hello World</h1>", doc.Html);
Assert.NotNull(doc.Metadata);
Assert.Null(doc.Warning);
}
[Fact]
public void Document_IgnoresUnknownProperties()
{
var json = """
{
"markdown": "# Test",
"futureField": "should be ignored",
"anotherNewField": 42
}
""";
var doc = JsonSerializer.Deserialize<Document>(json, JsonOptions);
Assert.NotNull(doc);
Assert.Equal("# Test", doc.Markdown);
}
[Fact]
public void CrawlJob_IsDone_Completed()
{
var job = new CrawlJob { Status = "completed" };
Assert.True(job.IsDone);
}
[Fact]
public void CrawlJob_IsDone_Failed()
{
var job = new CrawlJob { Status = "failed" };
Assert.True(job.IsDone);
}
[Fact]
public void CrawlJob_IsDone_Cancelled()
{
var job = new CrawlJob { Status = "cancelled" };
Assert.True(job.IsDone);
}
[Fact]
public void CrawlJob_NotDone_Scraping()
{
var job = new CrawlJob { Status = "scraping" };
Assert.False(job.IsDone);
}
[Fact]
public void BatchScrapeJob_IsDone_Completed()
{
var job = new BatchScrapeJob { Status = "completed" };
Assert.True(job.IsDone);
}
[Fact]
public void BatchScrapeJob_NotDone_Scraping()
{
var job = new BatchScrapeJob { Status = "scraping" };
Assert.False(job.IsDone);
}
[Fact]
public void CrawlJob_DeserializesCorrectly()
{
var json = """
{
"id": "crawl-123",
"status": "completed",
"completed": 5,
"total": 5,
"creditsUsed": 5,
"data": [
{ "markdown": "# Page 1" },
{ "markdown": "# Page 2" }
],
"next": null
}
""";
var job = JsonSerializer.Deserialize<CrawlJob>(json, JsonOptions);
Assert.NotNull(job);
Assert.Equal("crawl-123", job.Id);
Assert.Equal("completed", job.Status);
Assert.Equal(5, job.Completed);
Assert.Equal(5, job.Total);
Assert.True(job.IsDone);
Assert.NotNull(job.Data);
Assert.Equal(2, job.Data.Count);
}
[Fact]
public void JsonFormat_HasCorrectType()
{
var format = new JsonFormat
{
Prompt = "Extract the main content",
Schema = new Dictionary<string, object>
{
["type"] = "object",
["properties"] = new Dictionary<string, object>
{
["title"] = new Dictionary<string, object> { ["type"] = "string" }
}
}
};
var json = JsonSerializer.Serialize(format, JsonOptions);
Assert.Contains("\"type\":\"json\"", json);
Assert.Contains("\"prompt\"", json);
Assert.Contains("\"schema\"", json);
}
[Fact]
public void QueryFormat_HasCorrectMode()
{
var format = new QueryFormat
{
Prompt = "What is Firecrawl?",
Mode = QueryFormat.DirectQuoteMode
};
var json = JsonSerializer.Serialize(format, JsonOptions);
Assert.Contains("\"type\":\"query\"", json);
Assert.Contains("\"prompt\":\"What is Firecrawl?\"", json);
Assert.Contains("\"mode\":\"directQuote\"", json);
}
[Fact]
public void QuestionAndHighlightsFormats_SerializeCorrectly()
{
var question = new QuestionFormat
{
Question = "What is Firecrawl?"
};
var highlights = new HighlightsFormat
{
Query = "What is Firecrawl?"
};
var questionJson = JsonSerializer.Serialize(question, JsonOptions);
Assert.Contains("\"type\":\"question\"", questionJson);
Assert.Contains("\"question\":\"What is Firecrawl?\"", questionJson);
var highlightsJson = JsonSerializer.Serialize(highlights, JsonOptions);
Assert.Contains("\"type\":\"highlights\"", highlightsJson);
Assert.Contains("\"query\":\"What is Firecrawl?\"", highlightsJson);
}
[Fact]
public void WebhookConfig_SerializesCorrectly()
{
var config = new WebhookConfig
{
Url = "https://example.com/webhook",
Events = new List<string> { "completed", "failed" }
};
var json = JsonSerializer.Serialize(config, JsonOptions);
Assert.Contains("\"url\":\"https://example.com/webhook\"", json);
Assert.Contains("\"completed\"", json);
Assert.Contains("\"failed\"", json);
}
[Fact]
public void LocationConfig_SerializesCorrectly()
{
var config = new LocationConfig
{
Country = "US",
Languages = new List<string> { "en" }
};
var json = JsonSerializer.Serialize(config, JsonOptions);
Assert.Contains("\"country\":\"US\"", json);
Assert.Contains("\"en\"", json);
}
[Fact]
public void CrawlResponse_DeserializesCorrectly()
{
var json = """
{
"success": true,
"id": "crawl-abc",
"url": "https://api.firecrawl.dev/v2/crawl/crawl-abc"
}
""";
var response = JsonSerializer.Deserialize<CrawlResponse>(json, JsonOptions);
Assert.NotNull(response);
Assert.True(response.Success);
Assert.Equal("crawl-abc", response.Id);
}
[Fact]
public void BatchScrapeResponse_DeserializesCorrectly()
{
var json = """
{
"success": true,
"id": "batch-abc",
"invalidURLs": ["not-a-url"]
}
""";
var response = JsonSerializer.Deserialize<BatchScrapeResponse>(json, JsonOptions);
Assert.NotNull(response);
Assert.True(response.Success);
Assert.Equal("batch-abc", response.Id);
Assert.NotNull(response.InvalidURLs);
Assert.Single(response.InvalidURLs);
}
}

View File

@@ -0,0 +1,231 @@
using System.Net;
using System.Net.Http.Headers;
using System.Text;
using System.Text.Json;
using Firecrawl.Exceptions;
using Firecrawl.Models;
using Xunit;
namespace Firecrawl.Tests;
public class ParseTests
{
[Fact]
public void ParseFile_FromBytes_SetsProperties()
{
var bytes = Encoding.UTF8.GetBytes("<html><body>ok</body></html>");
var file = ParseFile.FromBytes("upload.html", bytes, "text/html");
Assert.Equal("upload.html", file.Filename);
Assert.Equal(bytes, file.Content);
Assert.Equal("text/html", file.ContentType);
}
[Fact]
public void ParseFile_FromPath_LoadsFile()
{
var tempPath = Path.Combine(Path.GetTempPath(), $"parse-test-{Guid.NewGuid():N}.html");
try
{
File.WriteAllText(tempPath, "<html>hi</html>", Encoding.UTF8);
var file = ParseFile.FromPath(tempPath);
Assert.Equal(Path.GetFileName(tempPath), file.Filename);
Assert.NotEmpty(file.Content);
}
finally
{
if (File.Exists(tempPath))
File.Delete(tempPath);
}
}
[Fact]
public void ParseFile_FromPath_ThrowsWhenMissing()
{
var bogusPath = Path.Combine(Path.GetTempPath(), $"missing-parse-{Guid.NewGuid():N}.html");
Assert.Throws<FileNotFoundException>(() => ParseFile.FromPath(bogusPath));
}
[Fact]
public void ParseOptions_Serializes_SupportedFields()
{
var options = new ParseOptions
{
Formats = new List<object> { "markdown" },
OnlyMainContent = true,
Timeout = 30000,
Proxy = "auto",
};
var json = JsonSerializer.Serialize(options, FirecrawlHttpClient.JsonOptions);
Assert.Contains("\"formats\"", json);
Assert.Contains("\"onlyMainContent\":true", json);
Assert.Contains("\"timeout\":30000", json);
Assert.Contains("\"proxy\":\"auto\"", json);
}
[Fact]
public void ParseOptions_Validate_RejectsUnsupportedFormats()
{
var options = new ParseOptions
{
Formats = new List<object> { "markdown", "screenshot" }
};
var ex = Assert.Throws<ArgumentException>(() => options.Validate());
Assert.Contains("screenshot", ex.Message);
}
[Fact]
public void ParseOptions_Validate_RejectsUnsupportedProxy()
{
var options = new ParseOptions { Proxy = "stealth" };
var ex = Assert.Throws<ArgumentException>(() => options.Validate());
Assert.Contains("proxy", ex.Message);
}
[Fact]
public void ParseOptions_Validate_RejectsNonPositiveTimeout()
{
var options = new ParseOptions { Timeout = 0 };
Assert.Throws<ArgumentException>(() => options.Validate());
}
[Fact]
public void ParseOptions_Validate_AllowsSupportedProxy()
{
var options = new ParseOptions { Proxy = "basic" };
options.Validate();
}
[Fact]
public async Task ParseAsync_RequiresFile()
{
var client = new FirecrawlClient(apiKey: "fc-test-key");
await Assert.ThrowsAsync<ArgumentNullException>(() => client.ParseAsync(null!));
}
[Fact]
public async Task ParseAsync_RejectsEmptyFilename()
{
var client = new FirecrawlClient(apiKey: "fc-test-key");
var file = new ParseFile("", new byte[] { 1, 2, 3 });
await Assert.ThrowsAsync<ArgumentException>(() => client.ParseAsync(file));
}
[Fact]
public async Task ParseAsync_RejectsEmptyContent()
{
var client = new FirecrawlClient(apiKey: "fc-test-key");
var file = new ParseFile("upload.html", Array.Empty<byte>());
await Assert.ThrowsAsync<ArgumentException>(() => client.ParseAsync(file));
}
[Fact]
public async Task ParseAsync_SendsMultipartRequest()
{
var handler = new CapturingHandler((req, ct) =>
{
var response = new HttpResponseMessage(HttpStatusCode.OK)
{
Content = new StringContent(
"{\"success\":true,\"data\":{\"markdown\":\"# Parsed\"}}",
Encoding.UTF8,
"application/json"),
};
return Task.FromResult(response);
});
var httpClient = new HttpClient(handler);
var client = new FirecrawlClient(
apiKey: "fc-test-key",
apiUrl: "https://api.firecrawl.test",
httpClient: httpClient);
var file = ParseFile.FromBytes(
"upload.html",
Encoding.UTF8.GetBytes("<html>hi</html>"),
"text/html");
var doc = await client.ParseAsync(file,
new ParseOptions { Formats = new List<object> { "markdown" } });
Assert.NotNull(handler.LastRequest);
Assert.Equal(HttpMethod.Post, handler.LastRequest!.Method);
Assert.Equal("/v2/parse", handler.LastRequest.RequestUri!.AbsolutePath);
var contentType = handler.LastRequest.Content!.Headers.ContentType!;
Assert.Equal("multipart/form-data", contentType.MediaType);
var rawBody = handler.LastRequestBody!;
Assert.Matches("name=\"?options\"?", rawBody);
Assert.Contains("\"markdown\"", rawBody);
Assert.Matches("name=\"?file\"?", rawBody);
Assert.Matches("filename=\"?upload\\.html\"?", rawBody);
Assert.Contains("<html>hi</html>", rawBody);
Assert.Equal("# Parsed", doc.Markdown);
}
[Fact]
public async Task ParseAsync_PropagatesApiError()
{
var handler = new CapturingHandler((req, ct) =>
{
var response = new HttpResponseMessage(HttpStatusCode.BadRequest)
{
Content = new StringContent(
"{\"success\":false,\"error\":\"Unsupported upload type.\"}",
Encoding.UTF8,
"application/json"),
};
return Task.FromResult(response);
});
var httpClient = new HttpClient(handler);
var client = new FirecrawlClient(
apiKey: "fc-test-key",
apiUrl: "https://api.firecrawl.test",
httpClient: httpClient);
var file = ParseFile.FromBytes("upload.xyz", new byte[] { 1, 2, 3 });
var ex = await Assert.ThrowsAsync<FirecrawlException>(
() => client.ParseAsync(file));
Assert.Contains("Unsupported upload type", ex.Message);
}
private sealed class CapturingHandler : HttpMessageHandler
{
private readonly Func<HttpRequestMessage, CancellationToken, Task<HttpResponseMessage>> _responder;
public HttpRequestMessage? LastRequest { get; private set; }
public string? LastRequestBody { get; private set; }
public CapturingHandler(Func<HttpRequestMessage, CancellationToken, Task<HttpResponseMessage>> responder)
{
_responder = responder;
}
protected override async Task<HttpResponseMessage> SendAsync(
HttpRequestMessage request,
CancellationToken cancellationToken)
{
// Read the request body eagerly because HttpClient disposes
// request.Content after SendAsync returns.
if (request.Content != null)
{
LastRequestBody = await request.Content.ReadAsStringAsync(cancellationToken);
}
LastRequest = request;
return await _responder(request, cancellationToken);
}
}
}