참고소스 수정본
This commit is contained in:
LASTA_DEV01\lasta
382
참고/firecrawl-main/apps/go-sdk/README.md
Normal file
382
참고/firecrawl-main/apps/go-sdk/README.md
Normal file
@@ -0,0 +1,382 @@
|
||||
# Firecrawl Go SDK
|
||||
|
||||
Go SDK for the [Firecrawl](https://firecrawl.dev) v2 web scraping API.
|
||||
|
||||
## Requirements
|
||||
|
||||
- **Go:** 1.23 or later
|
||||
|
||||
## Installation
|
||||
|
||||
```bash
|
||||
go get github.com/firecrawl/firecrawl/apps/go-sdk
|
||||
```
|
||||
|
||||
## API Key Setup
|
||||
|
||||
Get your API key from the [Firecrawl Dashboard](https://firecrawl.dev) and set it as an environment variable:
|
||||
|
||||
```bash
|
||||
export FIRECRAWL_API_KEY="fc-your-api-key-here"
|
||||
```
|
||||
|
||||
## Quick Start
|
||||
|
||||
```go
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log"
|
||||
|
||||
firecrawl "github.com/firecrawl/firecrawl/apps/go-sdk"
|
||||
"github.com/firecrawl/firecrawl/apps/go-sdk/option"
|
||||
)
|
||||
|
||||
func main() {
|
||||
// Create a client (reads FIRECRAWL_API_KEY from environment)
|
||||
client, err := firecrawl.NewClient()
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
// Or provide the API key directly
|
||||
client, err = firecrawl.NewClient(
|
||||
option.WithAPIKey("fc-your-api-key"),
|
||||
)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
// Scrape a single page
|
||||
doc, err := client.Scrape(ctx, "https://example.com", &firecrawl.ScrapeOptions{
|
||||
Formats: []string{"markdown"},
|
||||
})
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
fmt.Println(doc.Markdown)
|
||||
}
|
||||
```
|
||||
|
||||
## Configuration
|
||||
|
||||
```go
|
||||
client, err := firecrawl.NewClient(
|
||||
option.WithAPIKey("fc-your-api-key"), // API key (or set FIRECRAWL_API_KEY env var)
|
||||
option.WithAPIURL("https://api.firecrawl.dev"), // Custom API URL
|
||||
option.WithMaxRetries(3), // Max retry attempts (default: 3)
|
||||
option.WithBackoffFactor(0.5), // Backoff factor in seconds (default: 0.5)
|
||||
option.WithTimeout(5 * time.Minute), // HTTP timeout (default: 5 minutes)
|
||||
option.WithHTTPClient(customHTTPClient), // Custom *http.Client
|
||||
)
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
### Scrape
|
||||
|
||||
Scrape a single URL and get its content.
|
||||
|
||||
```go
|
||||
// Basic scrape
|
||||
doc, err := client.Scrape(ctx, "https://example.com", nil)
|
||||
|
||||
// With options
|
||||
doc, err := client.Scrape(ctx, "https://example.com", &firecrawl.ScrapeOptions{
|
||||
Formats: []string{"markdown", "html"},
|
||||
OnlyMainContent: firecrawl.Bool(true),
|
||||
WaitFor: firecrawl.Int(5000),
|
||||
Location: &firecrawl.LocationConfig{Country: "US"},
|
||||
})
|
||||
```
|
||||
|
||||
#### Interactive Browser
|
||||
|
||||
Execute code in a scrape-bound browser session:
|
||||
|
||||
```go
|
||||
resp, err := client.Interact(ctx, scrapeJobID, "document.title", &firecrawl.InteractParams{
|
||||
Language: "node",
|
||||
Timeout: firecrawl.Int(30),
|
||||
})
|
||||
|
||||
// Stop the browser session
|
||||
deleteResp, err := client.StopInteractiveBrowser(ctx, scrapeJobID)
|
||||
```
|
||||
|
||||
### Parse
|
||||
|
||||
Upload a local file (`html`, `pdf`, `docx`, etc.) via multipart form data and
|
||||
parse it synchronously. Parse options intentionally exclude browser-only
|
||||
features such as change tracking, screenshot, branding, actions, waitFor,
|
||||
location, and mobile. The `Proxy` option only accepts `"auto"` or `"basic"`.
|
||||
|
||||
```go
|
||||
// From disk
|
||||
file, err := firecrawl.NewParseFileFromPath("./document.pdf")
|
||||
|
||||
// Or from memory
|
||||
file := firecrawl.NewParseFileFromBytes("upload.html", []byte("<html>hi</html>"))
|
||||
file.ContentType = "text/html"
|
||||
|
||||
doc, err := client.Parse(ctx, file, &firecrawl.ParseOptions{
|
||||
Formats: []string{"markdown"},
|
||||
})
|
||||
fmt.Println(doc.Markdown)
|
||||
```
|
||||
|
||||
### Crawl
|
||||
|
||||
Crawl a website and get content from multiple pages.
|
||||
|
||||
```go
|
||||
// Auto-polling: starts the crawl and waits for completion
|
||||
job, err := client.Crawl(ctx, "https://example.com", &firecrawl.CrawlOptions{
|
||||
Limit: firecrawl.Int(50),
|
||||
MaxDiscoveryDepth: firecrawl.Int(3),
|
||||
ScrapeOptions: &firecrawl.ScrapeOptions{
|
||||
Formats: []string{"markdown"},
|
||||
},
|
||||
})
|
||||
|
||||
// Or manage polling manually
|
||||
resp, err := client.StartCrawl(ctx, "https://example.com", &firecrawl.CrawlOptions{
|
||||
Limit: firecrawl.Int(50),
|
||||
})
|
||||
|
||||
// Check status
|
||||
status, err := client.GetCrawlStatus(ctx, resp.ID)
|
||||
|
||||
// Cancel
|
||||
_, err = client.CancelCrawl(ctx, resp.ID)
|
||||
|
||||
// Get errors
|
||||
errors, err := client.GetCrawlErrors(ctx, resp.ID)
|
||||
```
|
||||
|
||||
### Batch Scrape
|
||||
|
||||
Scrape multiple URLs in a single batch job.
|
||||
|
||||
```go
|
||||
urls := []string{
|
||||
"https://example.com/page1",
|
||||
"https://example.com/page2",
|
||||
"https://example.com/page3",
|
||||
}
|
||||
|
||||
// Auto-polling
|
||||
job, err := client.BatchScrape(ctx, urls, &firecrawl.BatchScrapeOptions{
|
||||
ScrapeOptions: &firecrawl.ScrapeOptions{
|
||||
Formats: []string{"markdown"},
|
||||
},
|
||||
})
|
||||
|
||||
// Or manage manually
|
||||
resp, err := client.StartBatchScrape(ctx, urls, nil)
|
||||
status, err := client.GetBatchScrapeStatus(ctx, resp.ID)
|
||||
_, err = client.CancelBatchScrape(ctx, resp.ID)
|
||||
```
|
||||
|
||||
### Map
|
||||
|
||||
Discover URLs on a website.
|
||||
|
||||
```go
|
||||
mapData, err := client.Map(ctx, "https://example.com", &firecrawl.MapOptions{
|
||||
Search: firecrawl.String("pricing"),
|
||||
IncludeSubdomains: firecrawl.Bool(true),
|
||||
Limit: firecrawl.Int(100),
|
||||
})
|
||||
```
|
||||
|
||||
### Search
|
||||
|
||||
Search the web and get scraped results.
|
||||
|
||||
```go
|
||||
results, err := client.Search(ctx, "firecrawl web scraping", &firecrawl.SearchOptions{
|
||||
Limit: firecrawl.Int(5),
|
||||
ScrapeOptions: &firecrawl.ScrapeOptions{
|
||||
Formats: []string{"markdown"},
|
||||
},
|
||||
})
|
||||
```
|
||||
|
||||
### Agent
|
||||
|
||||
Run an AI-powered agent to extract structured data.
|
||||
|
||||
```go
|
||||
// Auto-polling
|
||||
status, err := client.Agent(ctx, &firecrawl.AgentOptions{
|
||||
Prompt: "Find all pricing plans and their features",
|
||||
URLs: []string{"https://example.com/pricing"},
|
||||
Schema: map[string]interface{}{
|
||||
"type": "object",
|
||||
"properties": map[string]interface{}{
|
||||
"plans": map[string]interface{}{
|
||||
"type": "array",
|
||||
"items": map[string]interface{}{
|
||||
"type": "object",
|
||||
"properties": map[string]interface{}{
|
||||
"name": map[string]interface{}{"type": "string"},
|
||||
"price": map[string]interface{}{"type": "string"},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
// Or manage manually
|
||||
resp, err := client.StartAgent(ctx, &firecrawl.AgentOptions{
|
||||
Prompt: "Extract product information",
|
||||
})
|
||||
status, err := client.GetAgentStatus(ctx, resp.ID)
|
||||
_, err = client.CancelAgent(ctx, resp.ID)
|
||||
```
|
||||
|
||||
### Browser
|
||||
|
||||
Create and manage standalone browser sessions.
|
||||
|
||||
```go
|
||||
// Create a browser session
|
||||
session, err := client.Browser(ctx, &firecrawl.BrowserOptions{
|
||||
TTL: firecrawl.Int(300),
|
||||
StreamWebView: firecrawl.Bool(true),
|
||||
})
|
||||
|
||||
// Execute code
|
||||
result, err := client.BrowserExecute(ctx, session.ID, "echo 'hello'", &firecrawl.BrowserExecuteParams{
|
||||
Language: "bash",
|
||||
Timeout: firecrawl.Int(30),
|
||||
})
|
||||
|
||||
// List sessions
|
||||
list, err := client.ListBrowsers(ctx, "active")
|
||||
|
||||
// Delete session
|
||||
_, err = client.DeleteBrowser(ctx, session.ID)
|
||||
```
|
||||
|
||||
### Usage & Metrics
|
||||
|
||||
```go
|
||||
// Check concurrency
|
||||
concurrency, err := client.GetConcurrency(ctx)
|
||||
fmt.Printf("Using %d of %d\n", concurrency.Concurrency, concurrency.MaxConcurrency)
|
||||
|
||||
// Check credit usage
|
||||
credits, err := client.GetCreditUsage(ctx)
|
||||
fmt.Printf("Remaining: %d of %d\n", credits.RemainingCredits, credits.PlanCredits)
|
||||
```
|
||||
|
||||
## Error Handling
|
||||
|
||||
The SDK uses typed errors for different failure scenarios:
|
||||
|
||||
```go
|
||||
doc, err := client.Scrape(ctx, "https://example.com", nil)
|
||||
if err != nil {
|
||||
var authErr *firecrawl.AuthenticationError
|
||||
var rateErr *firecrawl.RateLimitError
|
||||
var timeoutErr *firecrawl.JobTimeoutError
|
||||
var fcErr *firecrawl.FirecrawlError
|
||||
|
||||
switch {
|
||||
case errors.As(err, &authErr):
|
||||
fmt.Println("Invalid API key:", authErr.Message)
|
||||
case errors.As(err, &rateErr):
|
||||
fmt.Println("Rate limited:", rateErr.Message)
|
||||
case errors.As(err, &timeoutErr):
|
||||
fmt.Printf("Job %s timed out after %ds\n", timeoutErr.JobID, timeoutErr.TimeoutSeconds)
|
||||
case errors.As(err, &fcErr):
|
||||
fmt.Printf("API error (HTTP %d): %s\n", fcErr.StatusCode, fcErr.Message)
|
||||
default:
|
||||
fmt.Println("Unexpected error:", err)
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### Retry Logic
|
||||
|
||||
The SDK automatically retries transient failures:
|
||||
- **Retried:** 408, 409, 5xx errors, and connection failures
|
||||
- **Not retried:** 401, 429, and other 4xx errors
|
||||
- **Backoff:** Exponential backoff with configurable factor
|
||||
|
||||
## Context Support
|
||||
|
||||
All methods accept a `context.Context` for cancellation and deadline control:
|
||||
|
||||
```go
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
doc, err := client.Scrape(ctx, "https://example.com", nil)
|
||||
```
|
||||
|
||||
## Pointer Helpers
|
||||
|
||||
The SDK provides convenience functions for optional fields:
|
||||
|
||||
```go
|
||||
firecrawl.Bool(true) // *bool
|
||||
firecrawl.Int(50) // *int
|
||||
firecrawl.Int64(1000) // *int64
|
||||
firecrawl.String("test") // *string
|
||||
firecrawl.Float64(0.5) // *float64
|
||||
```
|
||||
|
||||
## Releases
|
||||
|
||||
The Go SDK lives in a monorepo subdirectory, so releases follow Go's
|
||||
[nested module tagging](https://go.dev/ref/mod#vcs-version) convention. Tags
|
||||
**must** be prefixed with the module subdirectory path:
|
||||
|
||||
```
|
||||
apps/go-sdk/v1.0.0
|
||||
```
|
||||
|
||||
A bare `v1.0.0` tag will not be resolvable by the Go module proxy.
|
||||
|
||||
### Release workflow
|
||||
|
||||
The SDK version is the single source of truth in
|
||||
[`version.go`](./version.go):
|
||||
|
||||
```go
|
||||
const Version = "1.0.0"
|
||||
```
|
||||
|
||||
To cut a release:
|
||||
|
||||
1. Bump the `Version` constant in `apps/go-sdk/version.go`
|
||||
2. Merge to `main`
|
||||
3. The [`publish-go-sdk`](../../.github/workflows/publish-go-sdk.yml) workflow
|
||||
will automatically:
|
||||
- create the `apps/go-sdk/v{Version}` tag on the merge commit,
|
||||
- push it to the repository,
|
||||
- warm `proxy.golang.org` to trigger indexing on
|
||||
[pkg.go.dev](https://pkg.go.dev/github.com/firecrawl/firecrawl/apps/go-sdk).
|
||||
|
||||
The workflow is idempotent: if the tag already exists, it is a no-op.
|
||||
|
||||
### Consuming a specific version
|
||||
|
||||
```bash
|
||||
go get github.com/firecrawl/firecrawl/apps/go-sdk@v1.1.0
|
||||
```
|
||||
|
||||
Users pin via the semantic version suffix; they never reference the
|
||||
`apps/go-sdk/` tag prefix directly — Go's toolchain handles the translation.
|
||||
|
||||
## License
|
||||
|
||||
MIT
|
||||
1029
참고/firecrawl-main/apps/go-sdk/firecrawl.go
Normal file
1029
참고/firecrawl-main/apps/go-sdk/firecrawl.go
Normal file
File diff suppressed because it is too large
Load Diff
46
참고/firecrawl-main/apps/go-sdk/firecrawl_error.go
Normal file
46
참고/firecrawl-main/apps/go-sdk/firecrawl_error.go
Normal file
@@ -0,0 +1,46 @@
|
||||
package firecrawl
|
||||
|
||||
import "fmt"
|
||||
|
||||
// FirecrawlError represents an error returned by the Firecrawl API.
|
||||
type FirecrawlError struct {
|
||||
// StatusCode is the HTTP status code (0 if not an HTTP error).
|
||||
StatusCode int
|
||||
// ErrorCode is the API error code, if any.
|
||||
ErrorCode string
|
||||
// Message is the human-readable error message.
|
||||
Message string
|
||||
}
|
||||
|
||||
func (e *FirecrawlError) Error() string {
|
||||
if e.ErrorCode != "" {
|
||||
return fmt.Sprintf("firecrawl: HTTP %d [%s]: %s", e.StatusCode, e.ErrorCode, e.Message)
|
||||
}
|
||||
if e.StatusCode != 0 {
|
||||
return fmt.Sprintf("firecrawl: HTTP %d: %s", e.StatusCode, e.Message)
|
||||
}
|
||||
return fmt.Sprintf("firecrawl: %s", e.Message)
|
||||
}
|
||||
|
||||
// AuthenticationError is returned when the API key is invalid (HTTP 401).
|
||||
type AuthenticationError struct {
|
||||
FirecrawlError
|
||||
}
|
||||
|
||||
// RateLimitError is returned when the rate limit is exceeded (HTTP 429).
|
||||
type RateLimitError struct {
|
||||
FirecrawlError
|
||||
}
|
||||
|
||||
// JobTimeoutError is returned when an async job exceeds its timeout.
|
||||
type JobTimeoutError struct {
|
||||
FirecrawlError
|
||||
// JobID is the ID of the timed-out job.
|
||||
JobID string
|
||||
// TimeoutSeconds is the timeout that was exceeded.
|
||||
TimeoutSeconds int
|
||||
}
|
||||
|
||||
func (e *JobTimeoutError) Error() string {
|
||||
return fmt.Sprintf("firecrawl: job %s timed out after %d seconds", e.JobID, e.TimeoutSeconds)
|
||||
}
|
||||
3
참고/firecrawl-main/apps/go-sdk/go.mod
Normal file
3
참고/firecrawl-main/apps/go-sdk/go.mod
Normal file
@@ -0,0 +1,3 @@
|
||||
module github.com/firecrawl/firecrawl/apps/go-sdk
|
||||
|
||||
go 1.23
|
||||
0
참고/firecrawl-main/apps/go-sdk/go.sum
Normal file
0
참고/firecrawl-main/apps/go-sdk/go.sum
Normal file
319
참고/firecrawl-main/apps/go-sdk/http_client.go
Normal file
319
참고/firecrawl-main/apps/go-sdk/http_client.go
Normal file
@@ -0,0 +1,319 @@
|
||||
package firecrawl
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"math"
|
||||
"mime/multipart"
|
||||
"net/http"
|
||||
"net/textproto"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
defaultAPIURL = "https://api.firecrawl.dev"
|
||||
defaultTimeout = 5 * time.Minute
|
||||
defaultMaxRetries = 3
|
||||
defaultBackoffFactor = 0.5
|
||||
)
|
||||
|
||||
// httpClient is the internal HTTP client for the Firecrawl API.
|
||||
type httpClient struct {
|
||||
client *http.Client
|
||||
apiKey string
|
||||
baseURL string
|
||||
maxRetries int
|
||||
backoffFactor float64
|
||||
extraHeaders map[string]string
|
||||
}
|
||||
|
||||
func newHTTPClient(apiKey, baseURL string, client *http.Client, maxRetries int, backoffFactor float64, extraHeaders map[string]string) *httpClient {
|
||||
baseURL = strings.TrimRight(baseURL, "/")
|
||||
return &httpClient{
|
||||
client: client,
|
||||
apiKey: apiKey,
|
||||
baseURL: baseURL,
|
||||
maxRetries: maxRetries,
|
||||
backoffFactor: backoffFactor,
|
||||
extraHeaders: extraHeaders,
|
||||
}
|
||||
}
|
||||
|
||||
// post sends a POST request with a JSON body.
|
||||
func (h *httpClient) post(ctx context.Context, path string, body interface{}, extraHeaders map[string]string) (json.RawMessage, error) {
|
||||
url := h.baseURL + path
|
||||
return h.doJSON(ctx, "POST", url, body, extraHeaders)
|
||||
}
|
||||
|
||||
// patch sends a PATCH request.
|
||||
func (h *httpClient) patch(ctx context.Context, path string, body interface{}) (json.RawMessage, error) {
|
||||
url := h.baseURL + path
|
||||
return h.doJSON(ctx, "PATCH", url, body, nil)
|
||||
}
|
||||
|
||||
// get sends a GET request.
|
||||
func (h *httpClient) get(ctx context.Context, path string) (json.RawMessage, error) {
|
||||
url := h.baseURL + path
|
||||
return h.doJSON(ctx, "GET", url, nil, nil)
|
||||
}
|
||||
|
||||
// getAbsolute sends a GET request to an absolute URL (for pagination cursors).
|
||||
func (h *httpClient) getAbsolute(ctx context.Context, absoluteURL string) (json.RawMessage, error) {
|
||||
return h.doJSON(ctx, "GET", absoluteURL, nil, nil)
|
||||
}
|
||||
|
||||
// delete sends a DELETE request.
|
||||
func (h *httpClient) delete(ctx context.Context, path string) (json.RawMessage, error) {
|
||||
url := h.baseURL + path
|
||||
return h.doJSON(ctx, "DELETE", url, nil, nil)
|
||||
}
|
||||
|
||||
// postMultipart sends a POST request with a multipart/form-data body. The extra
|
||||
// text `fields` are written first, followed by a single file part.
|
||||
func (h *httpClient) postMultipart(
|
||||
ctx context.Context,
|
||||
path string,
|
||||
fields map[string]string,
|
||||
fileField, fileName, fileContentType string,
|
||||
fileContent []byte,
|
||||
) (json.RawMessage, error) {
|
||||
url := h.baseURL + path
|
||||
|
||||
buildBody := func() (io.Reader, string, error) {
|
||||
buf := &bytes.Buffer{}
|
||||
writer := multipart.NewWriter(buf)
|
||||
|
||||
for k, v := range fields {
|
||||
if err := writer.WriteField(k, v); err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
}
|
||||
|
||||
partHeader := make(textproto.MIMEHeader)
|
||||
partHeader.Set(
|
||||
"Content-Disposition",
|
||||
fmt.Sprintf(`form-data; name=%q; filename=%q`, fileField, fileName),
|
||||
)
|
||||
if fileContentType != "" {
|
||||
partHeader.Set("Content-Type", fileContentType)
|
||||
}
|
||||
part, err := writer.CreatePart(partHeader)
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
if _, err := part.Write(fileContent); err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
if err := writer.Close(); err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
return buf, writer.FormDataContentType(), nil
|
||||
}
|
||||
|
||||
body, contentType, err := buildBody()
|
||||
if err != nil {
|
||||
return nil, &FirecrawlError{Message: fmt.Sprintf("failed to build multipart body: %v", err)}
|
||||
}
|
||||
|
||||
var lastErr error
|
||||
for attempt := 0; attempt <= h.maxRetries; attempt++ {
|
||||
if attempt > 0 {
|
||||
if err := h.sleepBackoff(ctx, attempt); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
body, contentType, err = buildBody()
|
||||
if err != nil {
|
||||
return nil, &FirecrawlError{Message: fmt.Sprintf("failed to rebuild multipart body: %v", err)}
|
||||
}
|
||||
}
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, "POST", url, body)
|
||||
if err != nil {
|
||||
return nil, &FirecrawlError{Message: fmt.Sprintf("failed to create request: %v", err)}
|
||||
}
|
||||
|
||||
req.Header.Set("Authorization", "Bearer "+h.apiKey)
|
||||
req.Header.Set("Content-Type", contentType)
|
||||
req.Header.Set("User-Agent", "firecrawl-go/"+Version)
|
||||
for k, v := range h.extraHeaders {
|
||||
req.Header.Set(k, v)
|
||||
}
|
||||
|
||||
resp, err := h.client.Do(req)
|
||||
if err != nil {
|
||||
if ctx.Err() != nil {
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
lastErr = err
|
||||
continue
|
||||
}
|
||||
|
||||
respBody, err := io.ReadAll(resp.Body)
|
||||
resp.Body.Close()
|
||||
if err != nil {
|
||||
lastErr = err
|
||||
continue
|
||||
}
|
||||
|
||||
if resp.StatusCode >= 200 && resp.StatusCode < 300 {
|
||||
return json.RawMessage(respBody), nil
|
||||
}
|
||||
|
||||
errMsg, errCode := extractError(respBody, resp.StatusCode)
|
||||
|
||||
switch resp.StatusCode {
|
||||
case 401:
|
||||
return nil, &AuthenticationError{
|
||||
FirecrawlError: FirecrawlError{StatusCode: 401, ErrorCode: errCode, Message: errMsg},
|
||||
}
|
||||
case 429:
|
||||
return nil, &RateLimitError{
|
||||
FirecrawlError: FirecrawlError{StatusCode: 429, ErrorCode: errCode, Message: errMsg},
|
||||
}
|
||||
}
|
||||
|
||||
if resp.StatusCode >= 400 && resp.StatusCode < 500 && resp.StatusCode != 408 && resp.StatusCode != 409 {
|
||||
return nil, &FirecrawlError{StatusCode: resp.StatusCode, ErrorCode: errCode, Message: errMsg}
|
||||
}
|
||||
|
||||
lastErr = &FirecrawlError{StatusCode: resp.StatusCode, ErrorCode: errCode, Message: errMsg}
|
||||
}
|
||||
|
||||
if lastErr != nil {
|
||||
if fe, ok := lastErr.(*FirecrawlError); ok {
|
||||
return nil, fe
|
||||
}
|
||||
return nil, &FirecrawlError{Message: fmt.Sprintf("request failed after %d retries: %v", h.maxRetries, lastErr)}
|
||||
}
|
||||
return nil, &FirecrawlError{Message: "request failed"}
|
||||
}
|
||||
|
||||
func (h *httpClient) doJSON(ctx context.Context, method, url string, body interface{}, extraHeaders map[string]string) (json.RawMessage, error) {
|
||||
var bodyReader io.Reader
|
||||
if body != nil {
|
||||
data, err := json.Marshal(body)
|
||||
if err != nil {
|
||||
return nil, &FirecrawlError{Message: fmt.Sprintf("failed to serialize request body: %v", err)}
|
||||
}
|
||||
bodyReader = bytes.NewReader(data)
|
||||
}
|
||||
|
||||
var lastErr error
|
||||
for attempt := 0; attempt <= h.maxRetries; attempt++ {
|
||||
if attempt > 0 {
|
||||
if err := h.sleepBackoff(ctx, attempt); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Reset the body reader for retries.
|
||||
if body != nil {
|
||||
data, _ := json.Marshal(body)
|
||||
bodyReader = bytes.NewReader(data)
|
||||
}
|
||||
}
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, method, url, bodyReader)
|
||||
if err != nil {
|
||||
return nil, &FirecrawlError{Message: fmt.Sprintf("failed to create request: %v", err)}
|
||||
}
|
||||
|
||||
req.Header.Set("Authorization", "Bearer "+h.apiKey)
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req.Header.Set("User-Agent", "firecrawl-go/"+Version)
|
||||
// Apply client-level headers.
|
||||
for k, v := range h.extraHeaders {
|
||||
req.Header.Set(k, v)
|
||||
}
|
||||
// Apply per-request headers (override client-level).
|
||||
for k, v := range extraHeaders {
|
||||
req.Header.Set(k, v)
|
||||
}
|
||||
|
||||
resp, err := h.client.Do(req)
|
||||
if err != nil {
|
||||
// If context is cancelled, return immediately instead of retrying.
|
||||
if ctx.Err() != nil {
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
lastErr = err
|
||||
continue // Retry on transport errors.
|
||||
}
|
||||
|
||||
respBody, err := io.ReadAll(resp.Body)
|
||||
resp.Body.Close()
|
||||
if err != nil {
|
||||
lastErr = err
|
||||
continue
|
||||
}
|
||||
|
||||
if resp.StatusCode >= 200 && resp.StatusCode < 300 {
|
||||
return json.RawMessage(respBody), nil
|
||||
}
|
||||
|
||||
// Parse error details from the response.
|
||||
errMsg, errCode := extractError(respBody, resp.StatusCode)
|
||||
|
||||
// Non-retryable client errors.
|
||||
switch resp.StatusCode {
|
||||
case 401:
|
||||
return nil, &AuthenticationError{
|
||||
FirecrawlError: FirecrawlError{StatusCode: 401, ErrorCode: errCode, Message: errMsg},
|
||||
}
|
||||
case 429:
|
||||
return nil, &RateLimitError{
|
||||
FirecrawlError: FirecrawlError{StatusCode: 429, ErrorCode: errCode, Message: errMsg},
|
||||
}
|
||||
}
|
||||
|
||||
if resp.StatusCode >= 400 && resp.StatusCode < 500 && resp.StatusCode != 408 && resp.StatusCode != 409 {
|
||||
return nil, &FirecrawlError{StatusCode: resp.StatusCode, ErrorCode: errCode, Message: errMsg}
|
||||
}
|
||||
|
||||
// Retryable: 408, 409, 5xx
|
||||
lastErr = &FirecrawlError{StatusCode: resp.StatusCode, ErrorCode: errCode, Message: errMsg}
|
||||
}
|
||||
|
||||
if lastErr != nil {
|
||||
if fe, ok := lastErr.(*FirecrawlError); ok {
|
||||
return nil, fe
|
||||
}
|
||||
return nil, &FirecrawlError{Message: fmt.Sprintf("request failed after %d retries: %v", h.maxRetries, lastErr)}
|
||||
}
|
||||
return nil, &FirecrawlError{Message: "request failed"}
|
||||
}
|
||||
|
||||
func (h *httpClient) sleepBackoff(ctx context.Context, attempt int) error {
|
||||
delay := time.Duration(h.backoffFactor*1000*math.Pow(2, float64(attempt-1))) * time.Millisecond
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-time.After(delay):
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// extractError parses an API error response to get the message and error code.
|
||||
func extractError(body []byte, statusCode int) (string, string) {
|
||||
var parsed map[string]interface{}
|
||||
if err := json.Unmarshal(body, &parsed); err != nil {
|
||||
return fmt.Sprintf("HTTP %d error", statusCode), ""
|
||||
}
|
||||
|
||||
msg := fmt.Sprintf("HTTP %d error", statusCode)
|
||||
if v, ok := parsed["error"]; ok {
|
||||
msg = fmt.Sprintf("%v", v)
|
||||
} else if v, ok := parsed["message"]; ok {
|
||||
msg = fmt.Sprintf("%v", v)
|
||||
}
|
||||
|
||||
var errCode string
|
||||
if v, ok := parsed["code"]; ok && v != nil {
|
||||
errCode = fmt.Sprintf("%v", v)
|
||||
}
|
||||
|
||||
return msg, errCode
|
||||
}
|
||||
314
참고/firecrawl-main/apps/go-sdk/models.go
Normal file
314
참고/firecrawl-main/apps/go-sdk/models.go
Normal file
@@ -0,0 +1,314 @@
|
||||
package firecrawl
|
||||
|
||||
import "encoding/json"
|
||||
|
||||
// Document represents a scraped web page.
|
||||
type Document struct {
|
||||
Markdown string `json:"markdown,omitempty"`
|
||||
HTML string `json:"html,omitempty"`
|
||||
RawHTML string `json:"rawHtml,omitempty"`
|
||||
JSON interface{} `json:"json,omitempty"`
|
||||
Summary string `json:"summary,omitempty"`
|
||||
Metadata map[string]interface{} `json:"metadata,omitempty"`
|
||||
Links []string `json:"links,omitempty"`
|
||||
Images []string `json:"images,omitempty"`
|
||||
Screenshot string `json:"screenshot,omitempty"`
|
||||
Audio string `json:"audio,omitempty"`
|
||||
Attributes []map[string]interface{} `json:"attributes,omitempty"`
|
||||
Actions map[string]interface{} `json:"actions,omitempty"`
|
||||
Answer string `json:"answer,omitempty"`
|
||||
Highlights string `json:"highlights,omitempty"`
|
||||
Warning string `json:"warning,omitempty"`
|
||||
ChangeTracking map[string]interface{} `json:"changeTracking,omitempty"`
|
||||
Branding map[string]interface{} `json:"branding,omitempty"`
|
||||
}
|
||||
|
||||
// CrawlResponse is returned when starting an async crawl.
|
||||
type CrawlResponse struct {
|
||||
ID string `json:"id"`
|
||||
URL string `json:"url,omitempty"`
|
||||
}
|
||||
|
||||
// CrawlJob represents the status and results of a crawl job.
|
||||
type CrawlJob struct {
|
||||
ID string `json:"id,omitempty"`
|
||||
Status string `json:"status"`
|
||||
Total int `json:"total"`
|
||||
Completed int `json:"completed"`
|
||||
CreditsUsed *int `json:"creditsUsed,omitempty"`
|
||||
ExpiresAt string `json:"expiresAt,omitempty"`
|
||||
Next string `json:"next,omitempty"`
|
||||
Data []Document `json:"data,omitempty"`
|
||||
}
|
||||
|
||||
// IsDone returns true if the crawl job has finished (completed, failed, or cancelled).
|
||||
func (c *CrawlJob) IsDone() bool {
|
||||
return c.Status == "completed" || c.Status == "failed" || c.Status == "cancelled"
|
||||
}
|
||||
|
||||
// BatchScrapeResponse is returned when starting an async batch scrape.
|
||||
type BatchScrapeResponse struct {
|
||||
ID string `json:"id"`
|
||||
URL string `json:"url,omitempty"`
|
||||
InvalidURLs []string `json:"invalidURLs,omitempty"`
|
||||
}
|
||||
|
||||
// BatchScrapeJob represents the status and results of a batch scrape job.
|
||||
type BatchScrapeJob struct {
|
||||
ID string `json:"id,omitempty"`
|
||||
Status string `json:"status"`
|
||||
Total int `json:"total"`
|
||||
Completed int `json:"completed"`
|
||||
CreditsUsed *int `json:"creditsUsed,omitempty"`
|
||||
ExpiresAt string `json:"expiresAt,omitempty"`
|
||||
Next string `json:"next,omitempty"`
|
||||
Data []Document `json:"data,omitempty"`
|
||||
}
|
||||
|
||||
// IsDone returns true if the batch scrape job has finished.
|
||||
func (b *BatchScrapeJob) IsDone() bool {
|
||||
return b.Status == "completed" || b.Status == "failed" || b.Status == "cancelled"
|
||||
}
|
||||
|
||||
// LinkResult represents a discovered URL from a map request.
|
||||
// The API may return links as plain strings or as objects with url/title/description.
|
||||
type LinkResult struct {
|
||||
URL string `json:"url,omitempty"`
|
||||
Title string `json:"title,omitempty"`
|
||||
Description string `json:"description,omitempty"`
|
||||
}
|
||||
|
||||
// UnmarshalJSON handles both string and object link elements from the API.
|
||||
func (l *LinkResult) UnmarshalJSON(data []byte) error {
|
||||
// Try as a plain string first.
|
||||
var s string
|
||||
if err := json.Unmarshal(data, &s); err == nil {
|
||||
l.URL = s
|
||||
return nil
|
||||
}
|
||||
|
||||
// Otherwise unmarshal as an object.
|
||||
type linkAlias LinkResult
|
||||
var alias linkAlias
|
||||
if err := json.Unmarshal(data, &alias); err != nil {
|
||||
return err
|
||||
}
|
||||
*l = LinkResult(alias)
|
||||
return nil
|
||||
}
|
||||
|
||||
// MapData represents the result of a map (URL discovery) request.
|
||||
type MapData struct {
|
||||
Links []LinkResult `json:"links,omitempty"`
|
||||
}
|
||||
|
||||
// MonitorSchedule configures when a monitor runs.
|
||||
type MonitorSchedule struct {
|
||||
Cron string `json:"cron"`
|
||||
Timezone string `json:"timezone,omitempty"`
|
||||
}
|
||||
|
||||
// MonitorCreateRequest creates a scheduled monitor.
|
||||
type MonitorCreateRequest struct {
|
||||
Name string `json:"name"`
|
||||
Schedule MonitorSchedule `json:"schedule"`
|
||||
Targets []map[string]interface{} `json:"targets"`
|
||||
Webhook map[string]interface{} `json:"webhook,omitempty"`
|
||||
Notification map[string]interface{} `json:"notification,omitempty"`
|
||||
RetentionDays *int `json:"retentionDays,omitempty"`
|
||||
}
|
||||
|
||||
// MonitorUpdateRequest updates a scheduled monitor.
|
||||
type MonitorUpdateRequest struct {
|
||||
Name string `json:"name,omitempty"`
|
||||
Status string `json:"status,omitempty"`
|
||||
Schedule *MonitorSchedule `json:"schedule,omitempty"`
|
||||
Targets []map[string]interface{} `json:"targets,omitempty"`
|
||||
Webhook map[string]interface{} `json:"webhook,omitempty"`
|
||||
Notification map[string]interface{} `json:"notification,omitempty"`
|
||||
RetentionDays *int `json:"retentionDays,omitempty"`
|
||||
}
|
||||
|
||||
// Monitor represents a scheduled monitor.
|
||||
type Monitor struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
Status string `json:"status"`
|
||||
Schedule MonitorSchedule `json:"schedule"`
|
||||
NextRunAt string `json:"nextRunAt,omitempty"`
|
||||
LastRunAt string `json:"lastRunAt,omitempty"`
|
||||
CurrentCheckID string `json:"currentCheckId,omitempty"`
|
||||
Targets []map[string]interface{} `json:"targets,omitempty"`
|
||||
Webhook map[string]interface{} `json:"webhook,omitempty"`
|
||||
Notification map[string]interface{} `json:"notification,omitempty"`
|
||||
RetentionDays int `json:"retentionDays"`
|
||||
EstimatedCreditsPerMonth *int `json:"estimatedCreditsPerMonth,omitempty"`
|
||||
LastCheckSummary *MonitorSummary `json:"lastCheckSummary,omitempty"`
|
||||
CreatedAt string `json:"createdAt,omitempty"`
|
||||
UpdatedAt string `json:"updatedAt,omitempty"`
|
||||
}
|
||||
|
||||
// MonitorSummary summarizes page statuses in a check.
|
||||
type MonitorSummary struct {
|
||||
TotalPages int `json:"totalPages"`
|
||||
Same int `json:"same"`
|
||||
Changed int `json:"changed"`
|
||||
New int `json:"new"`
|
||||
Removed int `json:"removed"`
|
||||
Error int `json:"error"`
|
||||
}
|
||||
|
||||
// MonitorCheck represents a single monitor run.
|
||||
type MonitorCheck struct {
|
||||
ID string `json:"id"`
|
||||
MonitorID string `json:"monitorId"`
|
||||
Status string `json:"status"`
|
||||
Trigger string `json:"trigger"`
|
||||
ScheduledFor string `json:"scheduledFor,omitempty"`
|
||||
StartedAt string `json:"startedAt,omitempty"`
|
||||
FinishedAt string `json:"finishedAt,omitempty"`
|
||||
EstimatedCredits *int `json:"estimatedCredits,omitempty"`
|
||||
ReservedCredits *int `json:"reservedCredits,omitempty"`
|
||||
ActualCredits *int `json:"actualCredits,omitempty"`
|
||||
BillingStatus string `json:"billingStatus,omitempty"`
|
||||
Summary MonitorSummary `json:"summary"`
|
||||
TargetResults interface{} `json:"targetResults,omitempty"`
|
||||
NotificationStatus interface{} `json:"notificationStatus,omitempty"`
|
||||
Error string `json:"error,omitempty"`
|
||||
CreatedAt string `json:"createdAt,omitempty"`
|
||||
UpdatedAt string `json:"updatedAt,omitempty"`
|
||||
}
|
||||
|
||||
// MonitorCheckPage is a single page result in a monitor check.
|
||||
type MonitorCheckPage struct {
|
||||
ID string `json:"id"`
|
||||
TargetID string `json:"targetId"`
|
||||
URL string `json:"url"`
|
||||
Status string `json:"status"`
|
||||
PreviousScrapeID string `json:"previousScrapeId,omitempty"`
|
||||
CurrentScrapeID string `json:"currentScrapeId,omitempty"`
|
||||
StatusCode *int `json:"statusCode,omitempty"`
|
||||
Error string `json:"error,omitempty"`
|
||||
Metadata interface{} `json:"metadata,omitempty"`
|
||||
Diff interface{} `json:"diff,omitempty"`
|
||||
CreatedAt string `json:"createdAt,omitempty"`
|
||||
}
|
||||
|
||||
// MonitorCheckDetail includes paginated page results and inline diffs.
|
||||
type MonitorCheckDetail struct {
|
||||
MonitorCheck
|
||||
Pages []MonitorCheckPage `json:"pages,omitempty"`
|
||||
Next string `json:"next,omitempty"`
|
||||
}
|
||||
|
||||
// ListMonitorsOptions controls monitor list pagination.
|
||||
type ListMonitorsOptions struct {
|
||||
Limit *int
|
||||
Offset *int
|
||||
}
|
||||
|
||||
// ListMonitorChecksOptions controls monitor check pagination/filtering.
|
||||
type ListMonitorChecksOptions struct {
|
||||
Limit *int
|
||||
Offset *int
|
||||
Status string
|
||||
}
|
||||
|
||||
// GetMonitorCheckOptions controls monitor check page pagination/filtering.
|
||||
type GetMonitorCheckOptions struct {
|
||||
Limit *int
|
||||
Skip *int
|
||||
Status string
|
||||
AutoPaginate *bool
|
||||
}
|
||||
|
||||
// SearchData represents the result of a search request.
|
||||
type SearchData struct {
|
||||
Web []map[string]interface{} `json:"web,omitempty"`
|
||||
News []map[string]interface{} `json:"news,omitempty"`
|
||||
Images []map[string]interface{} `json:"images,omitempty"`
|
||||
}
|
||||
|
||||
// AgentResponse is returned when starting an async agent task.
|
||||
type AgentResponse struct {
|
||||
Success bool `json:"success"`
|
||||
ID string `json:"id,omitempty"`
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
// AgentStatusResponse represents the status and results of an agent task.
|
||||
type AgentStatusResponse struct {
|
||||
Success bool `json:"success"`
|
||||
Status string `json:"status"`
|
||||
Error string `json:"error,omitempty"`
|
||||
Data interface{} `json:"data,omitempty"`
|
||||
Model string `json:"model,omitempty"`
|
||||
ExpiresAt string `json:"expiresAt,omitempty"`
|
||||
CreditsUsed *int `json:"creditsUsed,omitempty"`
|
||||
}
|
||||
|
||||
// IsDone returns true if the agent task has finished.
|
||||
func (a *AgentStatusResponse) IsDone() bool {
|
||||
return a.Status == "completed" || a.Status == "failed" || a.Status == "cancelled"
|
||||
}
|
||||
|
||||
// BrowserCreateResponse is returned when creating a browser session.
|
||||
type BrowserCreateResponse struct {
|
||||
Success bool `json:"success"`
|
||||
ID string `json:"id,omitempty"`
|
||||
CDPUrl string `json:"cdpUrl,omitempty"`
|
||||
LiveViewURL string `json:"liveViewUrl,omitempty"`
|
||||
ExpiresAt string `json:"expiresAt,omitempty"`
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
// BrowserExecuteResponse is returned when executing code in a browser session.
|
||||
type BrowserExecuteResponse struct {
|
||||
Success bool `json:"success"`
|
||||
Stdout string `json:"stdout,omitempty"`
|
||||
Result string `json:"result,omitempty"`
|
||||
Stderr string `json:"stderr,omitempty"`
|
||||
ExitCode *int `json:"exitCode,omitempty"`
|
||||
Killed *bool `json:"killed,omitempty"`
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
// BrowserDeleteResponse is returned when deleting a browser session.
|
||||
type BrowserDeleteResponse struct {
|
||||
Success bool `json:"success"`
|
||||
SessionDurationMs *int64 `json:"sessionDurationMs,omitempty"`
|
||||
CreditsBilled *int `json:"creditsBilled,omitempty"`
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
// BrowserListResponse is returned when listing browser sessions.
|
||||
type BrowserListResponse struct {
|
||||
Success bool `json:"success"`
|
||||
Sessions []BrowserSession `json:"sessions,omitempty"`
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
// BrowserSession represents a browser session.
|
||||
type BrowserSession struct {
|
||||
ID string `json:"id"`
|
||||
Status string `json:"status"`
|
||||
CDPUrl string `json:"cdpUrl,omitempty"`
|
||||
LiveViewURL string `json:"liveViewUrl,omitempty"`
|
||||
StreamWebView bool `json:"streamWebView,omitempty"`
|
||||
CreatedAt string `json:"createdAt,omitempty"`
|
||||
LastActivity string `json:"lastActivity,omitempty"`
|
||||
}
|
||||
|
||||
// ConcurrencyCheck represents concurrency usage information.
|
||||
type ConcurrencyCheck struct {
|
||||
Concurrency int `json:"concurrency"`
|
||||
MaxConcurrency int `json:"maxConcurrency"`
|
||||
}
|
||||
|
||||
// CreditUsage represents credit usage information.
|
||||
type CreditUsage struct {
|
||||
RemainingCredits int `json:"remainingCredits"`
|
||||
PlanCredits int `json:"planCredits"`
|
||||
BillingPeriodStart string `json:"billingPeriodStart,omitempty"`
|
||||
BillingPeriodEnd string `json:"billingPeriodEnd,omitempty"`
|
||||
}
|
||||
76
참고/firecrawl-main/apps/go-sdk/option/option.go
Normal file
76
참고/firecrawl-main/apps/go-sdk/option/option.go
Normal file
@@ -0,0 +1,76 @@
|
||||
// Package option provides functional options for configuring the Firecrawl client.
|
||||
package option
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"time"
|
||||
)
|
||||
|
||||
// RequestOption configures an individual request or the client.
|
||||
type RequestOption func(*RequestConfig)
|
||||
|
||||
// RequestConfig holds the configuration for an HTTP request.
|
||||
type RequestConfig struct {
|
||||
APIKey string
|
||||
APIURL string
|
||||
HTTPClient *http.Client
|
||||
MaxRetries int
|
||||
BackoffFactor float64
|
||||
ExtraHeaders map[string]string
|
||||
}
|
||||
|
||||
// WithAPIKey sets the API key. Defaults to the FIRECRAWL_API_KEY environment variable.
|
||||
func WithAPIKey(key string) RequestOption {
|
||||
return func(c *RequestConfig) {
|
||||
c.APIKey = key
|
||||
}
|
||||
}
|
||||
|
||||
// WithAPIURL sets the API base URL. Defaults to https://api.firecrawl.dev.
|
||||
func WithAPIURL(url string) RequestOption {
|
||||
return func(c *RequestConfig) {
|
||||
c.APIURL = url
|
||||
}
|
||||
}
|
||||
|
||||
// WithHTTPClient sets a custom *http.Client for all requests.
|
||||
func WithHTTPClient(client *http.Client) RequestOption {
|
||||
return func(c *RequestConfig) {
|
||||
c.HTTPClient = client
|
||||
}
|
||||
}
|
||||
|
||||
// WithMaxRetries sets the maximum number of automatic retries for transient failures.
|
||||
// Default: 3.
|
||||
func WithMaxRetries(n int) RequestOption {
|
||||
return func(c *RequestConfig) {
|
||||
c.MaxRetries = n
|
||||
}
|
||||
}
|
||||
|
||||
// WithBackoffFactor sets the exponential backoff factor in seconds. Default: 0.5.
|
||||
func WithBackoffFactor(f float64) RequestOption {
|
||||
return func(c *RequestConfig) {
|
||||
c.BackoffFactor = f
|
||||
}
|
||||
}
|
||||
|
||||
// WithTimeout sets the HTTP client timeout. Default: 5 minutes.
|
||||
func WithTimeout(d time.Duration) RequestOption {
|
||||
return func(c *RequestConfig) {
|
||||
if c.HTTPClient == nil {
|
||||
c.HTTPClient = &http.Client{}
|
||||
}
|
||||
c.HTTPClient.Timeout = d
|
||||
}
|
||||
}
|
||||
|
||||
// WithHeader adds an extra header to all requests.
|
||||
func WithHeader(key, value string) RequestOption {
|
||||
return func(c *RequestConfig) {
|
||||
if c.ExtraHeaders == nil {
|
||||
c.ExtraHeaders = make(map[string]string)
|
||||
}
|
||||
c.ExtraHeaders[key] = value
|
||||
}
|
||||
}
|
||||
224
참고/firecrawl-main/apps/go-sdk/options.go
Normal file
224
참고/firecrawl-main/apps/go-sdk/options.go
Normal file
@@ -0,0 +1,224 @@
|
||||
package firecrawl
|
||||
|
||||
import "encoding/json"
|
||||
|
||||
// QueryFormatMode selects how deprecated query answers are generated.
|
||||
type QueryFormatMode string
|
||||
|
||||
const (
|
||||
QueryModeFreeform QueryFormatMode = "freeform"
|
||||
QueryModeDirectQuote QueryFormatMode = "directQuote"
|
||||
)
|
||||
|
||||
// QuestionFormat asks a question about page content.
|
||||
type QuestionFormat struct {
|
||||
Question string `json:"question"`
|
||||
}
|
||||
|
||||
// MarshalJSON always emits the API-required question format type.
|
||||
func (q QuestionFormat) MarshalJSON() ([]byte, error) {
|
||||
type questionFormat struct {
|
||||
Type string `json:"type"`
|
||||
Question string `json:"question"`
|
||||
}
|
||||
|
||||
return json.Marshal(questionFormat{
|
||||
Type: "question",
|
||||
Question: q.Question,
|
||||
})
|
||||
}
|
||||
|
||||
// HighlightsFormat extracts direct highlights from page content.
|
||||
type HighlightsFormat struct {
|
||||
Query string `json:"query"`
|
||||
}
|
||||
|
||||
// MarshalJSON always emits the API-required highlights format type.
|
||||
func (h HighlightsFormat) MarshalJSON() ([]byte, error) {
|
||||
type highlightsFormat struct {
|
||||
Type string `json:"type"`
|
||||
Query string `json:"query"`
|
||||
}
|
||||
|
||||
return json.Marshal(highlightsFormat{
|
||||
Type: "highlights",
|
||||
Query: h.Query,
|
||||
})
|
||||
}
|
||||
|
||||
// QueryFormat asks a question about page content.
|
||||
//
|
||||
// Deprecated: use QuestionFormat or HighlightsFormat instead.
|
||||
type QueryFormat struct {
|
||||
Prompt string `json:"prompt"`
|
||||
Mode QueryFormatMode `json:"mode,omitempty"`
|
||||
}
|
||||
|
||||
// MarshalJSON always emits the API-required query format type.
|
||||
func (q QueryFormat) MarshalJSON() ([]byte, error) {
|
||||
type queryFormat struct {
|
||||
Type string `json:"type"`
|
||||
Prompt string `json:"prompt"`
|
||||
Mode QueryFormatMode `json:"mode,omitempty"`
|
||||
}
|
||||
|
||||
return json.Marshal(queryFormat{
|
||||
Type: "query",
|
||||
Prompt: q.Prompt,
|
||||
Mode: q.Mode,
|
||||
})
|
||||
}
|
||||
|
||||
// ScrapeOptions configures a single-page scrape request.
|
||||
type ScrapeOptions struct {
|
||||
Formats []string `json:"-"`
|
||||
FormatOptions []interface{} `json:"-"`
|
||||
Headers map[string]string `json:"headers,omitempty"`
|
||||
IncludeTags []string `json:"includeTags,omitempty"`
|
||||
ExcludeTags []string `json:"excludeTags,omitempty"`
|
||||
OnlyMainContent *bool `json:"onlyMainContent,omitempty"`
|
||||
Timeout *int `json:"timeout,omitempty"`
|
||||
WaitFor *int `json:"waitFor,omitempty"`
|
||||
Mobile *bool `json:"mobile,omitempty"`
|
||||
Parsers []interface{} `json:"parsers,omitempty"`
|
||||
Actions []map[string]interface{} `json:"actions,omitempty"`
|
||||
Location *LocationConfig `json:"location,omitempty"`
|
||||
SkipTLSVerification *bool `json:"skipTlsVerification,omitempty"`
|
||||
RemoveBase64Images *bool `json:"removeBase64Images,omitempty"`
|
||||
BlockAds *bool `json:"blockAds,omitempty"`
|
||||
Proxy *string `json:"proxy,omitempty"`
|
||||
MaxAge *int64 `json:"maxAge,omitempty"`
|
||||
StoreInCache *bool `json:"storeInCache,omitempty"`
|
||||
Lockdown *bool `json:"lockdown,omitempty"`
|
||||
Integration *string `json:"integration,omitempty"`
|
||||
JsonOptions *JsonOptions `json:"jsonOptions,omitempty"`
|
||||
}
|
||||
|
||||
// MarshalJSON preserves string formats while allowing object formats such as QuestionFormat.
|
||||
func (o ScrapeOptions) MarshalJSON() ([]byte, error) {
|
||||
type scrapeOptions ScrapeOptions
|
||||
payload := struct {
|
||||
scrapeOptions
|
||||
Formats interface{} `json:"formats,omitempty"`
|
||||
}{
|
||||
scrapeOptions: scrapeOptions(o),
|
||||
}
|
||||
|
||||
if len(o.FormatOptions) > 0 {
|
||||
payload.Formats = o.FormatOptions
|
||||
} else if len(o.Formats) > 0 {
|
||||
payload.Formats = o.Formats
|
||||
}
|
||||
|
||||
return json.Marshal(payload)
|
||||
}
|
||||
|
||||
// CrawlOptions configures a crawl request.
|
||||
type CrawlOptions struct {
|
||||
Prompt *string `json:"prompt,omitempty"`
|
||||
ExcludePaths []string `json:"excludePaths,omitempty"`
|
||||
IncludePaths []string `json:"includePaths,omitempty"`
|
||||
MaxDiscoveryDepth *int `json:"maxDiscoveryDepth,omitempty"`
|
||||
Sitemap *string `json:"sitemap,omitempty"`
|
||||
IgnoreQueryParameters *bool `json:"ignoreQueryParameters,omitempty"`
|
||||
DeduplicateSimilarURLs *bool `json:"deduplicateSimilarURLs,omitempty"`
|
||||
Limit *int `json:"limit,omitempty"`
|
||||
CrawlEntireDomain *bool `json:"crawlEntireDomain,omitempty"`
|
||||
AllowExternalLinks *bool `json:"allowExternalLinks,omitempty"`
|
||||
AllowSubdomains *bool `json:"allowSubdomains,omitempty"`
|
||||
Delay *int `json:"delay,omitempty"`
|
||||
MaxConcurrency *int `json:"maxConcurrency,omitempty"`
|
||||
Webhook interface{} `json:"webhook,omitempty"`
|
||||
ScrapeOptions *ScrapeOptions `json:"scrapeOptions,omitempty"`
|
||||
RegexOnFullURL *bool `json:"regexOnFullURL,omitempty"`
|
||||
ZeroDataRetention *bool `json:"zeroDataRetention,omitempty"`
|
||||
Integration *string `json:"integration,omitempty"`
|
||||
}
|
||||
|
||||
// BatchScrapeOptions configures a batch scrape request.
|
||||
type BatchScrapeOptions struct {
|
||||
ScrapeOptions *ScrapeOptions `json:"options,omitempty"`
|
||||
Webhook interface{} `json:"webhook,omitempty"`
|
||||
AppendToID *string `json:"appendToId,omitempty"`
|
||||
IgnoreInvalidURLs *bool `json:"ignoreInvalidURLs,omitempty"`
|
||||
MaxConcurrency *int `json:"maxConcurrency,omitempty"`
|
||||
ZeroDataRetention *bool `json:"zeroDataRetention,omitempty"`
|
||||
IdempotencyKey *string `json:"-"` // Sent as HTTP header, not in body
|
||||
Integration *string `json:"integration,omitempty"`
|
||||
}
|
||||
|
||||
// MapOptions configures a map (URL discovery) request.
|
||||
type MapOptions struct {
|
||||
Search *string `json:"search,omitempty"`
|
||||
Sitemap *string `json:"sitemap,omitempty"`
|
||||
IncludeSubdomains *bool `json:"includeSubdomains,omitempty"`
|
||||
IgnoreQueryParameters *bool `json:"ignoreQueryParameters,omitempty"`
|
||||
Limit *int `json:"limit,omitempty"`
|
||||
Timeout *int `json:"timeout,omitempty"`
|
||||
Integration *string `json:"integration,omitempty"`
|
||||
Location *LocationConfig `json:"location,omitempty"`
|
||||
}
|
||||
|
||||
// SearchOptions configures a search request.
|
||||
type SearchOptions struct {
|
||||
Sources []interface{} `json:"sources,omitempty"`
|
||||
Categories []interface{} `json:"categories,omitempty"`
|
||||
IncludeDomains []string `json:"includeDomains,omitempty"`
|
||||
ExcludeDomains []string `json:"excludeDomains,omitempty"`
|
||||
Limit *int `json:"limit,omitempty"`
|
||||
TBS *string `json:"tbs,omitempty"`
|
||||
Location *string `json:"location,omitempty"`
|
||||
IgnoreInvalidURLs *bool `json:"ignoreInvalidURLs,omitempty"`
|
||||
Timeout *int `json:"timeout,omitempty"`
|
||||
ScrapeOptions *ScrapeOptions `json:"scrapeOptions,omitempty"`
|
||||
Integration *string `json:"integration,omitempty"`
|
||||
}
|
||||
|
||||
// AgentOptions configures an agent request.
|
||||
type AgentOptions struct {
|
||||
URLs []string `json:"urls,omitempty"`
|
||||
Prompt string `json:"prompt"`
|
||||
Schema map[string]interface{} `json:"schema,omitempty"`
|
||||
Integration *string `json:"integration,omitempty"`
|
||||
MaxCredits *int `json:"maxCredits,omitempty"`
|
||||
StrictConstrainToURLs *bool `json:"strictConstrainToURLs,omitempty"`
|
||||
Model *string `json:"model,omitempty"`
|
||||
Webhook *WebhookConfig `json:"webhook,omitempty"`
|
||||
}
|
||||
|
||||
// LocationConfig specifies geolocation for requests.
|
||||
type LocationConfig struct {
|
||||
Country string `json:"country,omitempty"`
|
||||
Languages []string `json:"languages,omitempty"`
|
||||
}
|
||||
|
||||
// WebhookConfig configures webhook notifications.
|
||||
type WebhookConfig struct {
|
||||
URL string `json:"url"`
|
||||
Headers map[string]string `json:"headers,omitempty"`
|
||||
Metadata map[string]string `json:"metadata,omitempty"`
|
||||
Events []string `json:"events,omitempty"`
|
||||
}
|
||||
|
||||
// JsonOptions configures JSON extraction within formats.
|
||||
type JsonOptions struct {
|
||||
Prompt string `json:"prompt,omitempty"`
|
||||
Schema map[string]interface{} `json:"schema,omitempty"`
|
||||
}
|
||||
|
||||
// Pointer helpers for optional fields.
|
||||
|
||||
// Bool returns a pointer to the given bool value.
|
||||
func Bool(v bool) *bool { return &v }
|
||||
|
||||
// Int returns a pointer to the given int value.
|
||||
func Int(v int) *int { return &v }
|
||||
|
||||
// Int64 returns a pointer to the given int64 value.
|
||||
func Int64(v int64) *int64 { return &v }
|
||||
|
||||
// String returns a pointer to the given string value.
|
||||
func String(v string) *string { return &v }
|
||||
|
||||
// Float64 returns a pointer to the given float64 value.
|
||||
func Float64(v float64) *float64 { return &v }
|
||||
65
참고/firecrawl-main/apps/go-sdk/options_test.go
Normal file
65
참고/firecrawl-main/apps/go-sdk/options_test.go
Normal file
@@ -0,0 +1,65 @@
|
||||
package firecrawl
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestScrapeOptionsSerializesQueryFormatMode(t *testing.T) {
|
||||
payload, err := json.Marshal(ScrapeOptions{
|
||||
FormatOptions: []interface{}{
|
||||
QueryFormat{
|
||||
Prompt: "What is Firecrawl?",
|
||||
Mode: QueryModeDirectQuote,
|
||||
},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Marshal ScrapeOptions: %v", err)
|
||||
}
|
||||
|
||||
jsonBody := string(payload)
|
||||
for _, want := range []string{
|
||||
`"formats":[{"type":"query","prompt":"What is Firecrawl?","mode":"directQuote"}]`,
|
||||
} {
|
||||
if !strings.Contains(jsonBody, want) {
|
||||
t.Fatalf("serialized query format = %s, want to contain %s", jsonBody, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestScrapeOptionsSerializesQuestionAndHighlightsFormats(t *testing.T) {
|
||||
payload, err := json.Marshal(ScrapeOptions{
|
||||
FormatOptions: []interface{}{
|
||||
QuestionFormat{Question: "What is Firecrawl?"},
|
||||
HighlightsFormat{Query: "What is Firecrawl?"},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Marshal ScrapeOptions: %v", err)
|
||||
}
|
||||
|
||||
jsonBody := string(payload)
|
||||
for _, want := range []string{
|
||||
`{"type":"question","question":"What is Firecrawl?"}`,
|
||||
`{"type":"highlights","query":"What is Firecrawl?"}`,
|
||||
} {
|
||||
if !strings.Contains(jsonBody, want) {
|
||||
t.Fatalf("serialized formats = %s, want to contain %s", jsonBody, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestScrapeOptionsPreservesStringFormats(t *testing.T) {
|
||||
payload, err := json.Marshal(ScrapeOptions{
|
||||
Formats: []string{"markdown"},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Marshal ScrapeOptions: %v", err)
|
||||
}
|
||||
|
||||
if !strings.Contains(string(payload), `"formats":["markdown"]`) {
|
||||
t.Fatalf("serialized string formats = %s", payload)
|
||||
}
|
||||
}
|
||||
127
참고/firecrawl-main/apps/go-sdk/parse.go
Normal file
127
참고/firecrawl-main/apps/go-sdk/parse.go
Normal file
@@ -0,0 +1,127 @@
|
||||
package firecrawl
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"mime"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// ParseFile is a binary upload payload for the `/v2/parse` endpoint.
|
||||
//
|
||||
// Supported file extensions: .html, .htm, .pdf, .docx, .doc, .odt, .rtf, .xlsx, .xls
|
||||
type ParseFile struct {
|
||||
// Filename for the upload (e.g., "document.pdf"). Required.
|
||||
Filename string
|
||||
// Raw file bytes. Required (non-empty).
|
||||
Content []byte
|
||||
// Optional MIME type hint (e.g., "application/pdf").
|
||||
ContentType string
|
||||
}
|
||||
|
||||
// NewParseFileFromPath reads a file from disk and returns a ParseFile ready for upload.
|
||||
func NewParseFileFromPath(path string) (*ParseFile, error) {
|
||||
if path == "" {
|
||||
return nil, &FirecrawlError{Message: "file path is required"}
|
||||
}
|
||||
|
||||
content, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return nil, &FirecrawlError{Message: fmt.Sprintf("failed to read parse file %q: %v", path, err)}
|
||||
}
|
||||
|
||||
filename := filepath.Base(path)
|
||||
contentType := mime.TypeByExtension(filepath.Ext(path))
|
||||
|
||||
return &ParseFile{
|
||||
Filename: filename,
|
||||
Content: content,
|
||||
ContentType: contentType,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// NewParseFileFromBytes builds a ParseFile from in-memory bytes.
|
||||
func NewParseFileFromBytes(filename string, content []byte) *ParseFile {
|
||||
return &ParseFile{
|
||||
Filename: filename,
|
||||
Content: content,
|
||||
}
|
||||
}
|
||||
|
||||
// ParseOptions configures a parse request.
|
||||
//
|
||||
// Parse does not support browser-rendering features (actions, waitFor, location,
|
||||
// mobile) nor the screenshot, branding, or changeTracking formats. The proxy
|
||||
// field only accepts "auto" or "basic".
|
||||
type ParseOptions struct {
|
||||
Formats []string `json:"-"`
|
||||
FormatOptions []interface{} `json:"-"`
|
||||
Headers map[string]string `json:"headers,omitempty"`
|
||||
IncludeTags []string `json:"includeTags,omitempty"`
|
||||
ExcludeTags []string `json:"excludeTags,omitempty"`
|
||||
OnlyMainContent *bool `json:"onlyMainContent,omitempty"`
|
||||
Timeout *int `json:"timeout,omitempty"`
|
||||
Parsers []interface{} `json:"parsers,omitempty"`
|
||||
SkipTLSVerification *bool `json:"skipTlsVerification,omitempty"`
|
||||
RemoveBase64Images *bool `json:"removeBase64Images,omitempty"`
|
||||
BlockAds *bool `json:"blockAds,omitempty"`
|
||||
Proxy *string `json:"proxy,omitempty"`
|
||||
Integration *string `json:"integration,omitempty"`
|
||||
JsonOptions *JsonOptions `json:"jsonOptions,omitempty"`
|
||||
}
|
||||
|
||||
// MarshalJSON preserves string formats while allowing object formats such as QuestionFormat.
|
||||
func (o ParseOptions) MarshalJSON() ([]byte, error) {
|
||||
type parseOptions ParseOptions
|
||||
payload := struct {
|
||||
parseOptions
|
||||
Formats interface{} `json:"formats,omitempty"`
|
||||
}{
|
||||
parseOptions: parseOptions(o),
|
||||
}
|
||||
|
||||
if len(o.FormatOptions) > 0 {
|
||||
payload.Formats = o.FormatOptions
|
||||
} else if len(o.Formats) > 0 {
|
||||
payload.Formats = o.Formats
|
||||
}
|
||||
|
||||
return json.Marshal(payload)
|
||||
}
|
||||
|
||||
// Parse uploads a file to the `/v2/parse` endpoint and returns the extracted document.
|
||||
func (c *Client) Parse(ctx context.Context, file *ParseFile, opts *ParseOptions) (*Document, error) {
|
||||
if file == nil {
|
||||
return nil, &FirecrawlError{Message: "parse file is required"}
|
||||
}
|
||||
filename := strings.TrimSpace(file.Filename)
|
||||
if filename == "" {
|
||||
return nil, &FirecrawlError{Message: "filename cannot be empty"}
|
||||
}
|
||||
if len(file.Content) == 0 {
|
||||
return nil, &FirecrawlError{Message: "file content cannot be empty"}
|
||||
}
|
||||
|
||||
optionsMap := map[string]interface{}{}
|
||||
mergeOptions(optionsMap, opts)
|
||||
|
||||
optionsJSON, err := json.Marshal(optionsMap)
|
||||
if err != nil {
|
||||
return nil, &FirecrawlError{Message: fmt.Sprintf("failed to serialize parse options: %v", err)}
|
||||
}
|
||||
|
||||
fields := map[string]string{"options": string(optionsJSON)}
|
||||
raw, err := c.http.postMultipart(ctx, "/v2/parse", fields, "file", filename, file.ContentType, file.Content)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
doc, err := extractDataAs[Document](raw)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return doc, nil
|
||||
}
|
||||
130
참고/firecrawl-main/apps/go-sdk/parse_test.go
Normal file
130
참고/firecrawl-main/apps/go-sdk/parse_test.go
Normal file
@@ -0,0 +1,130 @@
|
||||
package firecrawl
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"mime"
|
||||
"mime/multipart"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/firecrawl/firecrawl/apps/go-sdk/option"
|
||||
)
|
||||
|
||||
func TestParseSendsMultipartRequest(t *testing.T) {
|
||||
var (
|
||||
gotOptions string
|
||||
gotFilename string
|
||||
gotFileBody string
|
||||
gotFileType string
|
||||
)
|
||||
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != http.MethodPost || r.URL.Path != "/v2/parse" {
|
||||
t.Fatalf("unexpected request: %s %s", r.Method, r.URL.Path)
|
||||
}
|
||||
|
||||
mediaType, params, err := mime.ParseMediaType(r.Header.Get("Content-Type"))
|
||||
if err != nil {
|
||||
t.Fatalf("parse content-type: %v", err)
|
||||
}
|
||||
if mediaType != "multipart/form-data" {
|
||||
t.Fatalf("expected multipart/form-data, got %q", mediaType)
|
||||
}
|
||||
|
||||
mr := multipart.NewReader(r.Body, params["boundary"])
|
||||
for {
|
||||
part, err := mr.NextPart()
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatalf("read part: %v", err)
|
||||
}
|
||||
data, _ := io.ReadAll(part)
|
||||
switch part.FormName() {
|
||||
case "options":
|
||||
gotOptions = string(data)
|
||||
case "file":
|
||||
gotFilename = part.FileName()
|
||||
gotFileBody = string(data)
|
||||
gotFileType = part.Header.Get("Content-Type")
|
||||
}
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
_, _ = w.Write([]byte(`{"success":true,"data":{"markdown":"# Hello"}}`))
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
client, err := NewClient(
|
||||
option.WithAPIKey("fc-test"),
|
||||
option.WithAPIURL(server.URL),
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("NewClient: %v", err)
|
||||
}
|
||||
|
||||
file := NewParseFileFromBytes("upload.html", []byte("<html>hi</html>"))
|
||||
file.ContentType = "text/html"
|
||||
|
||||
doc, err := client.Parse(context.Background(), file, &ParseOptions{
|
||||
Formats: []string{"markdown"},
|
||||
OnlyMainContent: Bool(true),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Parse: %v", err)
|
||||
}
|
||||
|
||||
if doc.Markdown != "# Hello" {
|
||||
t.Errorf("markdown = %q, want %q", doc.Markdown, "# Hello")
|
||||
}
|
||||
if !strings.Contains(gotOptions, `"formats":["markdown"]`) {
|
||||
t.Errorf("options missing formats: %q", gotOptions)
|
||||
}
|
||||
if !strings.Contains(gotOptions, `"onlyMainContent":true`) {
|
||||
t.Errorf("options missing onlyMainContent: %q", gotOptions)
|
||||
}
|
||||
if gotFilename != "upload.html" {
|
||||
t.Errorf("filename = %q, want upload.html", gotFilename)
|
||||
}
|
||||
if gotFileBody != "<html>hi</html>" {
|
||||
t.Errorf("file body = %q", gotFileBody)
|
||||
}
|
||||
if gotFileType != "text/html" {
|
||||
t.Errorf("file content-type = %q, want text/html", gotFileType)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseRejectsEmptyFilename(t *testing.T) {
|
||||
client, err := NewClient(
|
||||
option.WithAPIKey("fc-test"),
|
||||
option.WithAPIURL("http://localhost:0"),
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("NewClient: %v", err)
|
||||
}
|
||||
|
||||
_, err = client.Parse(context.Background(), &ParseFile{Filename: " ", Content: []byte("x")}, nil)
|
||||
if err == nil {
|
||||
t.Fatalf("expected error for empty filename")
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseRejectsEmptyContent(t *testing.T) {
|
||||
client, err := NewClient(
|
||||
option.WithAPIKey("fc-test"),
|
||||
option.WithAPIURL("http://localhost:0"),
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("NewClient: %v", err)
|
||||
}
|
||||
|
||||
_, err = client.Parse(context.Background(), &ParseFile{Filename: "doc.pdf"}, nil)
|
||||
if err == nil {
|
||||
t.Fatalf("expected error for empty content")
|
||||
}
|
||||
}
|
||||
12
참고/firecrawl-main/apps/go-sdk/version.go
Normal file
12
참고/firecrawl-main/apps/go-sdk/version.go
Normal file
@@ -0,0 +1,12 @@
|
||||
package firecrawl
|
||||
|
||||
// Version is the SDK version. It is used as the source of truth for release
|
||||
// tags (apps/go-sdk/v{Version}) and as the User-Agent suffix on API requests.
|
||||
//
|
||||
// Note: this version tracks the SDK release cycle, not the Firecrawl API
|
||||
// version. The SDK targets the Firecrawl v2 API.
|
||||
//
|
||||
// Bump this when preparing a new release. The publish-go-sdk GitHub workflow
|
||||
// reads this value and creates the corresponding monorepo-prefixed tag on
|
||||
// merge to main.
|
||||
const Version = "1.2.2"
|
||||
Reference in New Issue
Block a user