참고소스 수정본
This commit is contained in:
LASTA_DEV01\lasta
17
참고/firecrawl-main/apps/java-sdk/.gitignore
vendored
Normal file
17
참고/firecrawl-main/apps/java-sdk/.gitignore
vendored
Normal file
@@ -0,0 +1,17 @@
|
||||
.gradle/
|
||||
build/
|
||||
*.class
|
||||
*.jar
|
||||
!gradle/wrapper/gradle-wrapper.jar
|
||||
*.war
|
||||
*.ear
|
||||
*.iml
|
||||
.idea/
|
||||
*.ipr
|
||||
*.iws
|
||||
out/
|
||||
.settings/
|
||||
.classpath
|
||||
.project
|
||||
bin/
|
||||
local.properties
|
||||
475
참고/firecrawl-main/apps/java-sdk/README.md
Normal file
475
참고/firecrawl-main/apps/java-sdk/README.md
Normal file
@@ -0,0 +1,475 @@
|
||||
# Firecrawl Java SDK
|
||||
|
||||
Java SDK for [Firecrawl](https://firecrawl.dev) — search, scrape, and interact with the web.
|
||||
|
||||
## Prerequisites
|
||||
|
||||
Before using the Java SDK, ensure you have the following installed:
|
||||
|
||||
### Java Development Kit (JDK)
|
||||
|
||||
- **Required:** Java 11 or later
|
||||
- **Installation (macOS):**
|
||||
```bash
|
||||
brew install openjdk
|
||||
```
|
||||
|
||||
Then add Java to your PATH:
|
||||
```bash
|
||||
echo 'export PATH="/opt/homebrew/opt/openjdk/bin:$PATH"' >> ~/.zshrc
|
||||
source ~/.zshrc
|
||||
```
|
||||
|
||||
- **Installation (Linux):**
|
||||
```bash
|
||||
# Ubuntu/Debian
|
||||
sudo apt-get update
|
||||
sudo apt-get install openjdk-11-jdk
|
||||
|
||||
# Fedora/RHEL
|
||||
sudo dnf install java-11-openjdk-devel
|
||||
```
|
||||
|
||||
- **Verify Installation:**
|
||||
```bash
|
||||
java --version
|
||||
```
|
||||
|
||||
### Gradle (for building from source)
|
||||
|
||||
- **Required:** Gradle 8+
|
||||
- **Installation (macOS):**
|
||||
```bash
|
||||
brew install gradle
|
||||
```
|
||||
|
||||
- **Installation (Linux):**
|
||||
```bash
|
||||
# Ubuntu/Debian
|
||||
sudo apt-get install gradle
|
||||
|
||||
# Or use SDKMAN
|
||||
curl -s "https://get.sdkman.io" | bash
|
||||
sdk install gradle
|
||||
```
|
||||
|
||||
- **Verify Installation:**
|
||||
```bash
|
||||
gradle --version
|
||||
```
|
||||
|
||||
### API Key Setup
|
||||
|
||||
1. Get your API key from [Firecrawl Dashboard](https://firecrawl.dev)
|
||||
2. Set it as an environment variable:
|
||||
```bash
|
||||
export FIRECRAWL_API_KEY="fc-your-api-key-here"
|
||||
```
|
||||
|
||||
3. **Or** add it to your shell profile for persistence:
|
||||
```bash
|
||||
# For Zsh (macOS/Linux)
|
||||
echo 'export FIRECRAWL_API_KEY="fc-your-api-key-here"' >> ~/.zshrc
|
||||
source ~/.zshrc
|
||||
|
||||
# For Bash
|
||||
echo 'export FIRECRAWL_API_KEY="fc-your-api-key-here"' >> ~/.bashrc
|
||||
source ~/.bashrc
|
||||
```
|
||||
|
||||
## Installation
|
||||
|
||||
### Gradle (Kotlin DSL)
|
||||
|
||||
```kotlin
|
||||
implementation("com.firecrawl:firecrawl-java:1.1.1")
|
||||
```
|
||||
|
||||
### Gradle (Groovy)
|
||||
|
||||
```groovy
|
||||
implementation 'com.firecrawl:firecrawl-java:1.1.1'
|
||||
```
|
||||
|
||||
### Maven
|
||||
|
||||
```xml
|
||||
<dependency>
|
||||
<groupId>com.firecrawl</groupId>
|
||||
<artifactId>firecrawl-java</artifactId>
|
||||
<version>1.1.1</version>
|
||||
</dependency>
|
||||
```
|
||||
|
||||
## Quick Start
|
||||
|
||||
```java
|
||||
import com.firecrawl.client.FirecrawlClient;
|
||||
import com.firecrawl.models.*;
|
||||
import java.util.List;
|
||||
|
||||
// Create client with explicit API key
|
||||
FirecrawlClient client = FirecrawlClient.builder()
|
||||
.apiKey("fc-your-api-key")
|
||||
.build();
|
||||
|
||||
// Scrape a page
|
||||
Document doc = client.scrape("https://example.com",
|
||||
ScrapeOptions.builder()
|
||||
.formats(List.of("markdown"))
|
||||
.build());
|
||||
|
||||
System.out.println(doc.getMarkdown());
|
||||
```
|
||||
|
||||
Or create a client from the environment variable:
|
||||
|
||||
```java
|
||||
// export FIRECRAWL_API_KEY=fc-your-api-key
|
||||
FirecrawlClient client = FirecrawlClient.fromEnv();
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
### Scrape
|
||||
|
||||
Scrape a single URL and get the content in various formats.
|
||||
|
||||
```java
|
||||
Document doc = client.scrape("https://example.com",
|
||||
ScrapeOptions.builder()
|
||||
.formats(List.of("markdown", "html"))
|
||||
.onlyMainContent(true)
|
||||
.waitFor(5000)
|
||||
.build());
|
||||
|
||||
System.out.println(doc.getMarkdown());
|
||||
System.out.println(doc.getMetadata().get("title"));
|
||||
```
|
||||
|
||||
### Parse Uploaded Files
|
||||
|
||||
Upload local files (`html`, `pdf`, `docx`, etc.) via multipart form data and parse them synchronously.
|
||||
Parse options intentionally exclude browser-only features like change tracking, screenshot, branding, actions, waitFor, location, and mobile.
|
||||
|
||||
```java
|
||||
ParseFile file = ParseFile.builder()
|
||||
.filename("upload.html")
|
||||
.content("<!DOCTYPE html><html><body><h1>Java Parse</h1></body></html>".getBytes())
|
||||
.contentType("text/html")
|
||||
.build();
|
||||
|
||||
Document parsed = client.parse(file,
|
||||
ParseOptions.builder()
|
||||
.formats(List.of("markdown"))
|
||||
.build());
|
||||
|
||||
System.out.println(parsed.getMarkdown());
|
||||
```
|
||||
|
||||
#### JSON Extraction
|
||||
|
||||
```java
|
||||
import com.firecrawl.models.JsonFormat;
|
||||
|
||||
JsonFormat jsonFmt = JsonFormat.builder()
|
||||
.prompt("Extract the product name and price")
|
||||
.schema(Map.of(
|
||||
"type", "object",
|
||||
"properties", Map.of(
|
||||
"name", Map.of("type", "string"),
|
||||
"price", Map.of("type", "number")
|
||||
)
|
||||
))
|
||||
.build();
|
||||
|
||||
Document doc = client.scrape("https://example.com/product",
|
||||
ScrapeOptions.builder()
|
||||
.formats(List.of(jsonFmt))
|
||||
.build());
|
||||
|
||||
System.out.println(doc.getJson());
|
||||
```
|
||||
|
||||
#### Scrape-Bound Interactive Session
|
||||
|
||||
Run browser automation against the page context captured by a scrape job:
|
||||
|
||||
```java
|
||||
Document doc = client.scrape("https://example.com");
|
||||
String scrapeId = String.valueOf(doc.getMetadata().get("scrapeId"));
|
||||
|
||||
BrowserExecuteResponse exec = client.interact(
|
||||
scrapeId,
|
||||
"console.log(await page.title());",
|
||||
"node",
|
||||
30
|
||||
);
|
||||
|
||||
System.out.println(exec.getStdout());
|
||||
|
||||
BrowserDeleteResponse deleted = client.stopInteractiveBrowser(scrapeId);
|
||||
System.out.println("Deleted: " + deleted.isSuccess());
|
||||
```
|
||||
|
||||
### Crawl
|
||||
|
||||
Crawl an entire website. The `crawl()` method polls until completion.
|
||||
|
||||
```java
|
||||
// Convenience method — polls until done
|
||||
CrawlJob job = client.crawl("https://example.com",
|
||||
CrawlOptions.builder()
|
||||
.limit(50)
|
||||
.maxDiscoveryDepth(3)
|
||||
.scrapeOptions(ScrapeOptions.builder()
|
||||
.formats(List.of("markdown"))
|
||||
.build())
|
||||
.build());
|
||||
|
||||
for (Document doc : job.getData()) {
|
||||
System.out.println(doc.getMetadata().get("sourceURL"));
|
||||
}
|
||||
```
|
||||
|
||||
#### Async Crawl (manual polling)
|
||||
|
||||
```java
|
||||
CrawlResponse start = client.startCrawl("https://example.com",
|
||||
CrawlOptions.builder().limit(100).build());
|
||||
|
||||
System.out.println("Job started: " + start.getId());
|
||||
|
||||
// Poll manually
|
||||
CrawlJob status;
|
||||
do {
|
||||
try { Thread.sleep(2000); } catch (InterruptedException e) { Thread.currentThread().interrupt(); break; }
|
||||
status = client.getCrawlStatus(start.getId());
|
||||
System.out.println(status.getCompleted() + "/" + status.getTotal());
|
||||
} while (!status.isDone());
|
||||
```
|
||||
|
||||
### Batch Scrape
|
||||
|
||||
Scrape multiple URLs in parallel.
|
||||
|
||||
```java
|
||||
BatchScrapeJob job = client.batchScrape(
|
||||
List.of("https://example.com", "https://example.org"),
|
||||
BatchScrapeOptions.builder()
|
||||
.options(ScrapeOptions.builder()
|
||||
.formats(List.of("markdown"))
|
||||
.build())
|
||||
.build());
|
||||
|
||||
for (Document doc : job.getData()) {
|
||||
System.out.println(doc.getMarkdown());
|
||||
}
|
||||
```
|
||||
|
||||
### Map
|
||||
|
||||
Discover all URLs on a website.
|
||||
|
||||
```java
|
||||
MapData data = client.map("https://example.com",
|
||||
MapOptions.builder()
|
||||
.limit(100)
|
||||
.search("blog")
|
||||
.build());
|
||||
|
||||
for (Map<String, Object> link : data.getLinks()) {
|
||||
System.out.println(link.get("url") + " - " + link.get("title"));
|
||||
}
|
||||
```
|
||||
|
||||
### Search
|
||||
|
||||
Search the web and optionally scrape results.
|
||||
|
||||
```java
|
||||
SearchData results = client.search("firecrawl",
|
||||
SearchOptions.builder()
|
||||
.limit(10)
|
||||
.build());
|
||||
|
||||
if (results.getWeb() != null) {
|
||||
for (Map<String, Object> result : results.getWeb()) {
|
||||
System.out.println(result.get("title") + " — " + result.get("url"));
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### Agent
|
||||
|
||||
Run an AI-powered agent to research and extract data from the web.
|
||||
|
||||
```java
|
||||
AgentStatusResponse result = client.agent(
|
||||
AgentOptions.builder()
|
||||
.prompt("Find the pricing plans for Firecrawl and compare them")
|
||||
.build());
|
||||
|
||||
System.out.println(result.getData());
|
||||
```
|
||||
|
||||
### Usage & Metrics
|
||||
|
||||
```java
|
||||
ConcurrencyCheck conc = client.getConcurrency();
|
||||
System.out.println("Concurrency: " + conc.getConcurrency() + "/" + conc.getMaxConcurrency());
|
||||
|
||||
CreditUsage credits = client.getCreditUsage();
|
||||
System.out.println("Remaining credits: " + credits.getRemainingCredits());
|
||||
```
|
||||
|
||||
## Async Support
|
||||
|
||||
All methods have async variants that return `CompletableFuture`:
|
||||
|
||||
```java
|
||||
import java.util.concurrent.CompletableFuture;
|
||||
|
||||
CompletableFuture<Document> future = client.scrapeAsync(
|
||||
"https://example.com",
|
||||
ScrapeOptions.builder().formats(List.of("markdown")).build());
|
||||
|
||||
future.thenAccept(doc -> System.out.println(doc.getMarkdown()));
|
||||
```
|
||||
|
||||
## Error Handling
|
||||
|
||||
The SDK throws unchecked exceptions:
|
||||
|
||||
```java
|
||||
import com.firecrawl.errors.*;
|
||||
|
||||
try {
|
||||
Document doc = client.scrape("https://example.com");
|
||||
} catch (AuthenticationException e) {
|
||||
// 401 — invalid API key
|
||||
System.err.println("Auth failed: " + e.getMessage());
|
||||
} catch (RateLimitException e) {
|
||||
// 429 — too many requests
|
||||
System.err.println("Rate limited: " + e.getMessage());
|
||||
} catch (JobTimeoutException e) {
|
||||
// Async job timed out
|
||||
System.err.println("Job " + e.getJobId() + " timed out after " + e.getTimeoutSeconds() + "s");
|
||||
} catch (FirecrawlException e) {
|
||||
// All other API errors
|
||||
System.err.println("Error " + e.getStatusCode() + ": " + e.getMessage());
|
||||
}
|
||||
```
|
||||
|
||||
## Configuration
|
||||
|
||||
```java
|
||||
FirecrawlClient client = FirecrawlClient.builder()
|
||||
.apiKey("fc-your-api-key") // Required (or set FIRECRAWL_API_KEY env var)
|
||||
.apiUrl("https://api.firecrawl.dev") // Optional (or set FIRECRAWL_API_URL env var)
|
||||
.timeoutMs(300_000) // HTTP timeout: 5 min default
|
||||
.maxRetries(3) // Auto-retries for transient failures
|
||||
.backoffFactor(0.5) // Exponential backoff factor (seconds)
|
||||
.asyncExecutor(myExecutor) // Custom executor for async methods
|
||||
.build();
|
||||
```
|
||||
|
||||
## Building from Source
|
||||
|
||||
### Clone and Build
|
||||
|
||||
```bash
|
||||
# Clone the repository (if you haven't already)
|
||||
git clone https://github.com/firecrawl/firecrawl.git
|
||||
cd firecrawl/apps/java-sdk
|
||||
|
||||
# Build the project
|
||||
gradle build
|
||||
```
|
||||
|
||||
### Generate JAR
|
||||
|
||||
```bash
|
||||
gradle jar
|
||||
# Output: build/libs/firecrawl-java-1.1.1.jar
|
||||
```
|
||||
|
||||
### Install Locally
|
||||
|
||||
```bash
|
||||
gradle publishToMavenLocal
|
||||
# Now available as: com.firecrawl:firecrawl-java:1.1.1 in local Maven repository
|
||||
```
|
||||
|
||||
## Running Tests
|
||||
|
||||
The SDK includes both unit tests and E2E integration tests.
|
||||
|
||||
### Unit Tests (No API Key Required)
|
||||
|
||||
Unit tests verify SDK functionality without making actual API calls:
|
||||
|
||||
```bash
|
||||
gradle test
|
||||
```
|
||||
|
||||
### E2E Integration Tests (API Key Required)
|
||||
|
||||
E2E tests make real API calls and require a valid API key. These tests will be **skipped** if `FIRECRAWL_API_KEY` is not set:
|
||||
|
||||
```bash
|
||||
# Set your API key
|
||||
export FIRECRAWL_API_KEY="fc-your-api-key-here"
|
||||
|
||||
# Run all tests including E2E
|
||||
gradle test
|
||||
```
|
||||
|
||||
### Run Specific Tests
|
||||
|
||||
```bash
|
||||
# Run only scrape tests
|
||||
gradle test --tests "*testScrape*"
|
||||
|
||||
# Run only E2E tests
|
||||
gradle test --tests "*E2E"
|
||||
|
||||
# Run specific test class
|
||||
gradle test --tests "com.firecrawl.FirecrawlClientTest"
|
||||
```
|
||||
|
||||
### View Test Results
|
||||
|
||||
After running tests, view the detailed report:
|
||||
|
||||
```bash
|
||||
open build/reports/tests/test/index.html # macOS
|
||||
xdg-open build/reports/tests/test/index.html # Linux
|
||||
```
|
||||
|
||||
## Development Setup
|
||||
|
||||
If you're contributing to the SDK or testing local changes:
|
||||
|
||||
1. **Install Prerequisites** (see Prerequisites section above)
|
||||
|
||||
2. **Set Environment Variables:**
|
||||
```bash
|
||||
export FIRECRAWL_API_KEY="fc-your-api-key"
|
||||
# Optional: use local API server
|
||||
export FIRECRAWL_API_URL="http://localhost:3002"
|
||||
```
|
||||
|
||||
3. **Build and Test:**
|
||||
```bash
|
||||
gradle clean build test
|
||||
```
|
||||
|
||||
4. **Make Changes and Retest:**
|
||||
```bash
|
||||
# Quick compilation check
|
||||
gradle compileJava
|
||||
|
||||
# Run tests
|
||||
gradle test --tests "*testYourFeature*"
|
||||
```
|
||||
71
참고/firecrawl-main/apps/java-sdk/build.gradle.kts
Normal file
71
참고/firecrawl-main/apps/java-sdk/build.gradle.kts
Normal file
@@ -0,0 +1,71 @@
|
||||
plugins {
|
||||
`java-library`
|
||||
id("com.vanniktech.maven.publish") version "0.30.0"
|
||||
}
|
||||
|
||||
group = "com.firecrawl"
|
||||
version = "1.5.1"
|
||||
|
||||
java {
|
||||
sourceCompatibility = JavaVersion.VERSION_11
|
||||
targetCompatibility = JavaVersion.VERSION_11
|
||||
}
|
||||
|
||||
repositories {
|
||||
mavenCentral()
|
||||
}
|
||||
|
||||
dependencies {
|
||||
api("com.squareup.okhttp3:okhttp:4.12.0")
|
||||
api("com.fasterxml.jackson.core:jackson-databind:2.17.2")
|
||||
api("com.fasterxml.jackson.core:jackson-annotations:2.17.2")
|
||||
api("com.fasterxml.jackson.datatype:jackson-datatype-jdk8:2.17.2")
|
||||
|
||||
testImplementation("org.junit.jupiter:junit-jupiter:5.10.3")
|
||||
testRuntimeOnly("org.junit.platform:junit-platform-launcher:1.10.3")
|
||||
}
|
||||
|
||||
tasks.test {
|
||||
useJUnitPlatform()
|
||||
}
|
||||
|
||||
tasks.withType<Javadoc> {
|
||||
options {
|
||||
(this as StandardJavadocDocletOptions).apply {
|
||||
addStringOption("Xdoclint:none", "-quiet")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
mavenPublishing {
|
||||
publishToMavenCentral(com.vanniktech.maven.publish.SonatypeHost.CENTRAL_PORTAL)
|
||||
signAllPublications()
|
||||
|
||||
coordinates("com.firecrawl", "firecrawl-java", version.toString())
|
||||
|
||||
pom {
|
||||
name.set("Firecrawl Java SDK")
|
||||
description.set("Java SDK for the Firecrawl API")
|
||||
url.set("https://github.com/firecrawl/firecrawl")
|
||||
|
||||
licenses {
|
||||
license {
|
||||
name.set("MIT License")
|
||||
url.set("https://opensource.org/licenses/MIT")
|
||||
}
|
||||
}
|
||||
|
||||
developers {
|
||||
developer {
|
||||
name.set("Firecrawl")
|
||||
url.set("https://firecrawl.dev")
|
||||
}
|
||||
}
|
||||
|
||||
scm {
|
||||
url.set("https://github.com/firecrawl/firecrawl")
|
||||
connection.set("scm:git:git://github.com/firecrawl/firecrawl.git")
|
||||
developerConnection.set("scm:git:ssh://github.com/firecrawl/firecrawl.git")
|
||||
}
|
||||
}
|
||||
}
|
||||
BIN
참고/firecrawl-main/apps/java-sdk/gradle/wrapper/gradle-wrapper.jar
vendored
Normal file
BIN
참고/firecrawl-main/apps/java-sdk/gradle/wrapper/gradle-wrapper.jar
vendored
Normal file
Binary file not shown.
7
참고/firecrawl-main/apps/java-sdk/gradle/wrapper/gradle-wrapper.properties
vendored
Normal file
7
참고/firecrawl-main/apps/java-sdk/gradle/wrapper/gradle-wrapper.properties
vendored
Normal file
@@ -0,0 +1,7 @@
|
||||
distributionBase=GRADLE_USER_HOME
|
||||
distributionPath=wrapper/dists
|
||||
distributionUrl=https\://services.gradle.org/distributions/gradle-8.10-bin.zip
|
||||
networkTimeout=10000
|
||||
validateDistributionUrl=true
|
||||
zipStoreBase=GRADLE_USER_HOME
|
||||
zipStorePath=wrapper/dists
|
||||
120
참고/firecrawl-main/apps/java-sdk/gradlew
vendored
Normal file
120
참고/firecrawl-main/apps/java-sdk/gradlew
vendored
Normal file
@@ -0,0 +1,120 @@
|
||||
#!/bin/sh
|
||||
|
||||
#
|
||||
# Copyright © 2015-2021 the original authors.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# https://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
##############################################################################
|
||||
#
|
||||
# Gradle start up script for POSIX generated by Gradle.
|
||||
#
|
||||
##############################################################################
|
||||
|
||||
# Attempt to set APP_HOME
|
||||
# Resolve links: $0 may be a link
|
||||
app_path=$0
|
||||
while
|
||||
APP_HOME=${app_path%"${app_path##*/}"} # leaves a trailing /; empty if no leading path
|
||||
[ -h "$app_path" ]
|
||||
do
|
||||
ls=$( ls -ld -- "$app_path" )
|
||||
link=${ls#*' -> '}
|
||||
case $link in #(
|
||||
/*) app_path=$link ;; #(
|
||||
*) app_path=$APP_HOME$link ;;
|
||||
esac
|
||||
done
|
||||
|
||||
# This is normally unused
|
||||
# shellcheck disable=SC2034
|
||||
APP_BASE_NAME=${0##*/}
|
||||
# Discard cd standard output in case $CDPATH is set (https://github.com/gradle/gradle/issues/25036)
|
||||
APP_HOME=$( cd "${APP_HOME:-./}" > /dev/null && pwd -P ) || exit
|
||||
|
||||
# Use the maximum available, or set MAX_FD != -1 to use that value.
|
||||
MAX_FD=maximum
|
||||
|
||||
warn () {
|
||||
echo "$*"
|
||||
} >&2
|
||||
|
||||
die () {
|
||||
echo
|
||||
echo "$*"
|
||||
echo
|
||||
exit 1
|
||||
} >&2
|
||||
|
||||
# OS specific support (must be 'true' or 'false').
|
||||
cygwin=false
|
||||
msys=false
|
||||
darwin=false
|
||||
nonstop=false
|
||||
case "$( uname )" in #(
|
||||
CYGWIN* ) cygwin=true ;; #(
|
||||
Darwin* ) darwin=true ;; #(
|
||||
MSYS* | MINGW* ) msys=true ;; #(
|
||||
NonStop* ) nonstop=true ;;
|
||||
esac
|
||||
|
||||
CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar
|
||||
|
||||
# Determine the Java command to use to start the JVM.
|
||||
if [ -n "$JAVA_HOME" ] ; then
|
||||
if [ -x "$JAVA_HOME/jre/sh/java" ] ; then
|
||||
# IBM's JDK on AIX uses strange locations for the executables
|
||||
JAVACMD=$JAVA_HOME/jre/sh/java
|
||||
else
|
||||
JAVACMD=$JAVA_HOME/bin/java
|
||||
fi
|
||||
if [ ! -x "$JAVACMD" ] ; then
|
||||
die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME
|
||||
|
||||
Please set the JAVA_HOME variable in your environment to match the
|
||||
location of your Java installation."
|
||||
fi
|
||||
else
|
||||
JAVACMD=java
|
||||
if ! command -v java >/dev/null 2>&1 ; then
|
||||
die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
|
||||
|
||||
Please set the JAVA_HOME variable in your environment to match the
|
||||
location of your Java installation."
|
||||
fi
|
||||
fi
|
||||
|
||||
# Increase the maximum file descriptors if we can.
|
||||
if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then
|
||||
case $MAX_FD in #(
|
||||
max*)
|
||||
# In POSIX sh, ulimit -H is://undefined. That's://why the redirect is://done to /dev/null 2>&1
|
||||
MAX_FD=$( ulimit -H -n ) ||
|
||||
warn "Could not query maximum file descriptor limit"
|
||||
;;
|
||||
esac
|
||||
case $MAX_FD in #(
|
||||
'' | soft) :;; #(
|
||||
*)
|
||||
# In POSIX sh, ulimit -n is://undefined. That's://why the redirect is://done to /dev/null 2>&1
|
||||
ulimit -n "$MAX_FD" ||
|
||||
warn "Could not set maximum file descriptor limit to $MAX_FD"
|
||||
;;
|
||||
esac
|
||||
fi
|
||||
|
||||
# Collect all arguments for the java command, stracks://the style://of://arguments://after://the://class name
|
||||
eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$@"
|
||||
|
||||
exec "$JAVACMD" "$@"
|
||||
1
참고/firecrawl-main/apps/java-sdk/settings.gradle.kts
Normal file
1
참고/firecrawl-main/apps/java-sdk/settings.gradle.kts
Normal file
@@ -0,0 +1 @@
|
||||
rootProject.name = "firecrawl-java"
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,287 @@
|
||||
package com.firecrawl.client;
|
||||
|
||||
import com.fasterxml.jackson.core.JsonProcessingException;
|
||||
import com.fasterxml.jackson.databind.DeserializationFeature;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.fasterxml.jackson.datatype.jdk8.Jdk8Module;
|
||||
import com.firecrawl.errors.AuthenticationException;
|
||||
import com.firecrawl.errors.FirecrawlException;
|
||||
import com.firecrawl.errors.RateLimitException;
|
||||
import okhttp3.*;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.Collections;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
/**
|
||||
* Internal HTTP client for making authenticated requests to the Firecrawl API.
|
||||
* Handles retry logic with exponential backoff.
|
||||
*/
|
||||
class FirecrawlHttpClient {
|
||||
|
||||
private static final MediaType JSON = MediaType.get("application/json; charset=utf-8");
|
||||
|
||||
private final OkHttpClient httpClient;
|
||||
private final String apiKey;
|
||||
private final String baseUrl;
|
||||
private final int maxRetries;
|
||||
private final double backoffFactor;
|
||||
final ObjectMapper objectMapper;
|
||||
|
||||
FirecrawlHttpClient(String apiKey, String baseUrl, long timeoutMs, int maxRetries, double backoffFactor) {
|
||||
this(apiKey, baseUrl, timeoutMs, maxRetries, backoffFactor, null);
|
||||
}
|
||||
|
||||
FirecrawlHttpClient(String apiKey, String baseUrl, long timeoutMs, int maxRetries, double backoffFactor,
|
||||
OkHttpClient httpClient) {
|
||||
this.apiKey = apiKey;
|
||||
this.baseUrl = baseUrl.endsWith("/") ? baseUrl.substring(0, baseUrl.length() - 1) : baseUrl;
|
||||
this.maxRetries = maxRetries;
|
||||
this.backoffFactor = backoffFactor;
|
||||
|
||||
if (httpClient != null) {
|
||||
this.httpClient = httpClient;
|
||||
} else {
|
||||
this.httpClient = new OkHttpClient.Builder()
|
||||
.connectTimeout(timeoutMs, TimeUnit.MILLISECONDS)
|
||||
.readTimeout(timeoutMs, TimeUnit.MILLISECONDS)
|
||||
.writeTimeout(timeoutMs, TimeUnit.MILLISECONDS)
|
||||
.build();
|
||||
}
|
||||
|
||||
this.objectMapper = new ObjectMapper()
|
||||
.registerModule(new Jdk8Module())
|
||||
.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
|
||||
}
|
||||
|
||||
/**
|
||||
* Sends a POST request with JSON body.
|
||||
*/
|
||||
<T> T post(String path, Object body, Class<T> responseType) {
|
||||
return post(path, body, responseType, Collections.emptyMap());
|
||||
}
|
||||
|
||||
/**
|
||||
* Sends a POST request with JSON body and extra headers.
|
||||
*/
|
||||
<T> T post(String path, Object body, Class<T> responseType, Map<String, String> extraHeaders) {
|
||||
String url = baseUrl + path;
|
||||
String json;
|
||||
try {
|
||||
json = objectMapper.writeValueAsString(body);
|
||||
} catch (JsonProcessingException e) {
|
||||
throw new FirecrawlException("Failed to serialize request body", e);
|
||||
}
|
||||
RequestBody requestBody = RequestBody.create(json, JSON);
|
||||
Request.Builder builder = new Request.Builder()
|
||||
.url(url)
|
||||
.header("Authorization", "Bearer " + apiKey)
|
||||
.header("Content-Type", "application/json")
|
||||
.post(requestBody);
|
||||
for (Map.Entry<String, String> entry : extraHeaders.entrySet()) {
|
||||
builder.header(entry.getKey(), entry.getValue());
|
||||
}
|
||||
Request request = builder.build();
|
||||
return executeWithRetry(request, responseType);
|
||||
}
|
||||
|
||||
/**
|
||||
* Sends a PATCH request with JSON body.
|
||||
*/
|
||||
<T> T patch(String path, Object body, Class<T> responseType) {
|
||||
String url = baseUrl + path;
|
||||
String json;
|
||||
try {
|
||||
json = objectMapper.writeValueAsString(body);
|
||||
} catch (JsonProcessingException e) {
|
||||
throw new FirecrawlException("Failed to serialize request body", e);
|
||||
}
|
||||
RequestBody requestBody = RequestBody.create(json, JSON);
|
||||
Request request = new Request.Builder()
|
||||
.url(url)
|
||||
.header("Authorization", "Bearer " + apiKey)
|
||||
.header("Content-Type", "application/json")
|
||||
.patch(requestBody)
|
||||
.build();
|
||||
return executeWithRetry(request, responseType);
|
||||
}
|
||||
|
||||
/**
|
||||
* Sends a POST multipart/form-data request.
|
||||
*/
|
||||
<T> T postMultipart(
|
||||
String path,
|
||||
Map<String, String> fields,
|
||||
String fileFieldName,
|
||||
byte[] fileContent,
|
||||
String filename,
|
||||
String contentType,
|
||||
Class<T> responseType
|
||||
) {
|
||||
String url = baseUrl + path;
|
||||
MultipartBody.Builder multipart = new MultipartBody.Builder()
|
||||
.setType(MultipartBody.FORM);
|
||||
|
||||
for (Map.Entry<String, String> entry : fields.entrySet()) {
|
||||
multipart.addFormDataPart(entry.getKey(), entry.getValue());
|
||||
}
|
||||
|
||||
MediaType mediaType;
|
||||
if (contentType != null && !contentType.isBlank()) {
|
||||
try {
|
||||
mediaType = MediaType.get(contentType);
|
||||
} catch (IllegalArgumentException ignored) {
|
||||
mediaType = MediaType.get("application/octet-stream");
|
||||
}
|
||||
} else {
|
||||
mediaType = MediaType.get("application/octet-stream");
|
||||
}
|
||||
RequestBody fileBody = RequestBody.create(fileContent, mediaType);
|
||||
multipart.addFormDataPart(fileFieldName, filename, fileBody);
|
||||
|
||||
Request request = new Request.Builder()
|
||||
.url(url)
|
||||
.header("Authorization", "Bearer " + apiKey)
|
||||
.post(multipart.build())
|
||||
.build();
|
||||
|
||||
return executeWithRetry(request, responseType);
|
||||
}
|
||||
|
||||
/**
|
||||
* Sends a GET request.
|
||||
*/
|
||||
<T> T get(String path, Class<T> responseType) {
|
||||
String url = baseUrl + path;
|
||||
Request request = new Request.Builder()
|
||||
.url(url)
|
||||
.header("Authorization", "Bearer " + apiKey)
|
||||
.get()
|
||||
.build();
|
||||
return executeWithRetry(request, responseType);
|
||||
}
|
||||
|
||||
/**
|
||||
* Sends a GET request with full URL (for following next-page cursors).
|
||||
*/
|
||||
<T> T getAbsolute(String absoluteUrl, Class<T> responseType) {
|
||||
Request request = new Request.Builder()
|
||||
.url(absoluteUrl)
|
||||
.header("Authorization", "Bearer " + apiKey)
|
||||
.get()
|
||||
.build();
|
||||
return executeWithRetry(request, responseType);
|
||||
}
|
||||
|
||||
/**
|
||||
* Sends a DELETE request.
|
||||
*/
|
||||
<T> T delete(String path, Class<T> responseType) {
|
||||
String url = baseUrl + path;
|
||||
Request request = new Request.Builder()
|
||||
.url(url)
|
||||
.header("Authorization", "Bearer " + apiKey)
|
||||
.delete()
|
||||
.build();
|
||||
return executeWithRetry(request, responseType);
|
||||
}
|
||||
|
||||
/**
|
||||
* Sends a raw GET request and returns the response body as a parsed Map.
|
||||
*/
|
||||
@SuppressWarnings("unchecked")
|
||||
Map<String, Object> getRaw(String path) {
|
||||
return get(path, Map.class);
|
||||
}
|
||||
|
||||
private <T> T executeWithRetry(Request request, Class<T> responseType) {
|
||||
int attempt = 0;
|
||||
while (true) {
|
||||
try {
|
||||
try (Response response = httpClient.newCall(request).execute()) {
|
||||
ResponseBody responseBody = response.body();
|
||||
String bodyStr = responseBody != null ? responseBody.string() : "";
|
||||
|
||||
if (response.isSuccessful()) {
|
||||
if (responseType == Void.class || responseType == void.class) {
|
||||
return null;
|
||||
}
|
||||
return objectMapper.readValue(bodyStr, responseType);
|
||||
}
|
||||
|
||||
int code = response.code();
|
||||
|
||||
// Parse error details from response
|
||||
String errorMessage = extractErrorMessage(bodyStr, code);
|
||||
String errorCode = extractErrorCode(bodyStr);
|
||||
|
||||
// Non-retryable client errors
|
||||
if (code == 401) {
|
||||
throw new AuthenticationException(errorMessage, errorCode, null);
|
||||
}
|
||||
if (code == 429) {
|
||||
throw new RateLimitException(errorMessage, errorCode, null);
|
||||
}
|
||||
if (code >= 400 && code < 500 && code != 408 && code != 409) {
|
||||
throw new FirecrawlException(errorMessage, code, errorCode, null);
|
||||
}
|
||||
|
||||
// Retryable errors: 408, 409, 502, 5xx
|
||||
if (attempt < maxRetries) {
|
||||
attempt++;
|
||||
sleepWithBackoff(attempt);
|
||||
continue;
|
||||
}
|
||||
|
||||
throw new FirecrawlException(errorMessage, code, errorCode, null);
|
||||
}
|
||||
} catch (FirecrawlException e) {
|
||||
throw e;
|
||||
} catch (IOException e) {
|
||||
if (attempt < maxRetries) {
|
||||
attempt++;
|
||||
sleepWithBackoff(attempt);
|
||||
continue;
|
||||
}
|
||||
throw new FirecrawlException("Request failed: " + e.getMessage(), e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
private String extractErrorMessage(String body, int statusCode) {
|
||||
try {
|
||||
Map<String, Object> parsed = objectMapper.readValue(body, Map.class);
|
||||
if (parsed.containsKey("error")) {
|
||||
return String.valueOf(parsed.get("error"));
|
||||
}
|
||||
if (parsed.containsKey("message")) {
|
||||
return String.valueOf(parsed.get("message"));
|
||||
}
|
||||
} catch (Exception ignored) {
|
||||
}
|
||||
return "HTTP " + statusCode + " error";
|
||||
}
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
private String extractErrorCode(String body) {
|
||||
try {
|
||||
Map<String, Object> parsed = objectMapper.readValue(body, Map.class);
|
||||
Object code = parsed.get("code");
|
||||
return code != null ? String.valueOf(code) : null;
|
||||
} catch (Exception ignored) {
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private void sleepWithBackoff(int attempt) {
|
||||
long delayMs = (long) (backoffFactor * 1000 * Math.pow(2, attempt - 1));
|
||||
try {
|
||||
Thread.sleep(delayMs);
|
||||
} catch (InterruptedException e) {
|
||||
Thread.currentThread().interrupt();
|
||||
throw new FirecrawlException("Request interrupted during retry backoff", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
package com.firecrawl.errors;
|
||||
|
||||
/**
|
||||
* Thrown when the API returns a 401 Unauthorized response.
|
||||
*/
|
||||
public class AuthenticationException extends FirecrawlException {
|
||||
|
||||
public AuthenticationException(String message) {
|
||||
super(message, 401);
|
||||
}
|
||||
|
||||
public AuthenticationException(String message, String errorCode, Object details) {
|
||||
super(message, 401, errorCode, details);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
package com.firecrawl.errors;
|
||||
|
||||
/**
|
||||
* Base exception for all Firecrawl SDK errors.
|
||||
*/
|
||||
public class FirecrawlException extends RuntimeException {
|
||||
|
||||
private final int statusCode;
|
||||
private final String errorCode;
|
||||
private final Object details;
|
||||
|
||||
public FirecrawlException(String message) {
|
||||
this(message, 0, null, null);
|
||||
}
|
||||
|
||||
public FirecrawlException(String message, int statusCode) {
|
||||
this(message, statusCode, null, null);
|
||||
}
|
||||
|
||||
public FirecrawlException(String message, int statusCode, String errorCode, Object details) {
|
||||
super(message);
|
||||
this.statusCode = statusCode;
|
||||
this.errorCode = errorCode;
|
||||
this.details = details;
|
||||
}
|
||||
|
||||
public FirecrawlException(String message, Throwable cause) {
|
||||
super(message, cause);
|
||||
this.statusCode = 0;
|
||||
this.errorCode = null;
|
||||
this.details = null;
|
||||
}
|
||||
|
||||
/** HTTP status code (0 if not an HTTP error). */
|
||||
public int getStatusCode() { return statusCode; }
|
||||
|
||||
/** Error code from the API response, if any. */
|
||||
public String getErrorCode() { return errorCode; }
|
||||
|
||||
/** Additional error details from the API response, if any. */
|
||||
public Object getDetails() { return details; }
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
package com.firecrawl.errors;
|
||||
|
||||
/**
|
||||
* Thrown when an async job (crawl, batch, agent) does not complete within the specified timeout.
|
||||
*/
|
||||
public class JobTimeoutException extends FirecrawlException {
|
||||
|
||||
private final String jobId;
|
||||
private final int timeoutSeconds;
|
||||
|
||||
public JobTimeoutException(String jobId, int timeoutSeconds, String jobType) {
|
||||
super(jobType + " job " + jobId + " did not complete within " + timeoutSeconds + " seconds");
|
||||
this.jobId = jobId;
|
||||
this.timeoutSeconds = timeoutSeconds;
|
||||
}
|
||||
|
||||
/** The ID of the timed-out job. */
|
||||
public String getJobId() { return jobId; }
|
||||
|
||||
/** The timeout in seconds that was exceeded. */
|
||||
public int getTimeoutSeconds() { return timeoutSeconds; }
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
package com.firecrawl.errors;
|
||||
|
||||
/**
|
||||
* Thrown when the API returns a 429 Too Many Requests response.
|
||||
*/
|
||||
public class RateLimitException extends FirecrawlException {
|
||||
|
||||
public RateLimitException(String message) {
|
||||
super(message, 429);
|
||||
}
|
||||
|
||||
public RateLimitException(String message, String errorCode, Object details) {
|
||||
super(message, 429, errorCode, details);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,80 @@
|
||||
package com.firecrawl.models;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonInclude;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* Options for starting an agent task.
|
||||
*/
|
||||
@JsonInclude(JsonInclude.Include.NON_NULL)
|
||||
public class AgentOptions {
|
||||
|
||||
private List<String> urls;
|
||||
private String prompt;
|
||||
private Map<String, Object> schema;
|
||||
private String integration;
|
||||
private Integer maxCredits;
|
||||
private Boolean strictConstrainToURLs;
|
||||
private String model;
|
||||
private WebhookConfig webhook;
|
||||
|
||||
private AgentOptions() {}
|
||||
|
||||
public List<String> getUrls() { return urls; }
|
||||
public String getPrompt() { return prompt; }
|
||||
public Map<String, Object> getSchema() { return schema; }
|
||||
public String getIntegration() { return integration; }
|
||||
public Integer getMaxCredits() { return maxCredits; }
|
||||
public Boolean getStrictConstrainToURLs() { return strictConstrainToURLs; }
|
||||
public String getModel() { return model; }
|
||||
public WebhookConfig getWebhook() { return webhook; }
|
||||
|
||||
public static Builder builder() { return new Builder(); }
|
||||
|
||||
public static final class Builder {
|
||||
private List<String> urls;
|
||||
private String prompt;
|
||||
private Map<String, Object> schema;
|
||||
private String integration;
|
||||
private Integer maxCredits;
|
||||
private Boolean strictConstrainToURLs;
|
||||
private String model;
|
||||
private WebhookConfig webhook;
|
||||
|
||||
private Builder() {}
|
||||
|
||||
/** Optional URLs to constrain the agent to. */
|
||||
public Builder urls(List<String> urls) { this.urls = urls; return this; }
|
||||
/** Natural language prompt describing what data to find. */
|
||||
public Builder prompt(String prompt) { this.prompt = prompt; return this; }
|
||||
/** JSON Schema for structured output. */
|
||||
public Builder schema(Map<String, Object> schema) { this.schema = schema; return this; }
|
||||
/** Integration identifier. */
|
||||
public Builder integration(String integration) { this.integration = integration; return this; }
|
||||
/** Maximum credits to spend. */
|
||||
public Builder maxCredits(Integer maxCredits) { this.maxCredits = maxCredits; return this; }
|
||||
/** Don't navigate outside provided URLs. */
|
||||
public Builder strictConstrainToURLs(Boolean strictConstrainToURLs) { this.strictConstrainToURLs = strictConstrainToURLs; return this; }
|
||||
/** Agent model: "spark-1-pro" or "spark-1-mini". */
|
||||
public Builder model(String model) { this.model = model; return this; }
|
||||
/** Webhook configuration. */
|
||||
public Builder webhook(WebhookConfig webhook) { this.webhook = webhook; return this; }
|
||||
|
||||
public AgentOptions build() {
|
||||
if (prompt == null || prompt.isEmpty()) {
|
||||
throw new IllegalArgumentException("Agent prompt is required");
|
||||
}
|
||||
AgentOptions o = new AgentOptions();
|
||||
o.urls = this.urls;
|
||||
o.prompt = this.prompt;
|
||||
o.schema = this.schema;
|
||||
o.integration = this.integration;
|
||||
o.maxCredits = this.maxCredits;
|
||||
o.strictConstrainToURLs = this.strictConstrainToURLs;
|
||||
o.model = this.model;
|
||||
o.webhook = this.webhook;
|
||||
return o;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
package com.firecrawl.models;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
|
||||
|
||||
/**
|
||||
* Response from starting an agent task.
|
||||
*/
|
||||
@JsonIgnoreProperties(ignoreUnknown = true)
|
||||
public class AgentResponse {
|
||||
|
||||
private boolean success;
|
||||
private String id;
|
||||
private String error;
|
||||
|
||||
public boolean isSuccess() { return success; }
|
||||
public String getId() { return id; }
|
||||
public String getError() { return error; }
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "AgentResponse{success=" + success + ", id=" + id + "}";
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
package com.firecrawl.models;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
|
||||
|
||||
/**
|
||||
* Status response for an agent task.
|
||||
*/
|
||||
@JsonIgnoreProperties(ignoreUnknown = true)
|
||||
public class AgentStatusResponse {
|
||||
|
||||
private boolean success;
|
||||
private String status;
|
||||
private String error;
|
||||
private Object data;
|
||||
private String model;
|
||||
private String expiresAt;
|
||||
private Integer creditsUsed;
|
||||
|
||||
public boolean isSuccess() { return success; }
|
||||
public String getStatus() { return status; }
|
||||
public String getError() { return error; }
|
||||
public Object getData() { return data; }
|
||||
public String getModel() { return model; }
|
||||
public String getExpiresAt() { return expiresAt; }
|
||||
public Integer getCreditsUsed() { return creditsUsed; }
|
||||
|
||||
public boolean isDone() {
|
||||
return "completed".equals(status) || "failed".equals(status) || "cancelled".equals(status);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "AgentStatusResponse{status=" + status + ", model=" + model + "}";
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
package com.firecrawl.models;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* Status and results of a batch scrape job.
|
||||
*/
|
||||
@JsonIgnoreProperties(ignoreUnknown = true)
|
||||
public class BatchScrapeJob {
|
||||
|
||||
private String id;
|
||||
private String status;
|
||||
private int completed;
|
||||
private int total;
|
||||
private Integer creditsUsed;
|
||||
private String expiresAt;
|
||||
private String next;
|
||||
private List<Document> data;
|
||||
|
||||
public String getId() { return id; }
|
||||
public String getStatus() { return status; }
|
||||
public int getCompleted() { return completed; }
|
||||
public int getTotal() { return total; }
|
||||
public Integer getCreditsUsed() { return creditsUsed; }
|
||||
public String getExpiresAt() { return expiresAt; }
|
||||
public String getNext() { return next; }
|
||||
public List<Document> getData() { return data; }
|
||||
public void setData(List<Document> data) { this.data = data; }
|
||||
|
||||
public boolean isDone() {
|
||||
return "completed".equals(status) || "failed".equals(status) || "cancelled".equals(status);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "BatchScrapeJob{id=" + id + ", status=" + status + ", completed=" + completed + "/" + total + "}";
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,78 @@
|
||||
package com.firecrawl.models;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonIgnore;
|
||||
import com.fasterxml.jackson.annotation.JsonInclude;
|
||||
|
||||
/**
|
||||
* Options for a batch scrape job.
|
||||
*/
|
||||
@JsonInclude(JsonInclude.Include.NON_NULL)
|
||||
public class BatchScrapeOptions {
|
||||
|
||||
private ScrapeOptions options;
|
||||
private Object webhook;
|
||||
private String appendToId;
|
||||
private Boolean ignoreInvalidURLs;
|
||||
private Integer maxConcurrency;
|
||||
private Boolean zeroDataRetention;
|
||||
@JsonIgnore
|
||||
private String idempotencyKey;
|
||||
private String integration;
|
||||
|
||||
private BatchScrapeOptions() {}
|
||||
|
||||
public ScrapeOptions getOptions() { return options; }
|
||||
public Object getWebhook() { return webhook; }
|
||||
public String getAppendToId() { return appendToId; }
|
||||
public Boolean getIgnoreInvalidURLs() { return ignoreInvalidURLs; }
|
||||
public Integer getMaxConcurrency() { return maxConcurrency; }
|
||||
public Boolean getZeroDataRetention() { return zeroDataRetention; }
|
||||
@JsonIgnore
|
||||
public String getIdempotencyKey() { return idempotencyKey; }
|
||||
public String getIntegration() { return integration; }
|
||||
|
||||
public static Builder builder() { return new Builder(); }
|
||||
|
||||
public static final class Builder {
|
||||
private ScrapeOptions options;
|
||||
private Object webhook;
|
||||
private String appendToId;
|
||||
private Boolean ignoreInvalidURLs;
|
||||
private Integer maxConcurrency;
|
||||
private Boolean zeroDataRetention;
|
||||
private String idempotencyKey;
|
||||
private String integration;
|
||||
|
||||
private Builder() {}
|
||||
|
||||
/** Scrape options applied to each URL. */
|
||||
public Builder options(ScrapeOptions options) { this.options = options; return this; }
|
||||
/** Webhook URL string or {@link WebhookConfig} object. */
|
||||
public Builder webhook(Object webhook) { this.webhook = webhook; return this; }
|
||||
/** Append URLs to an existing batch job. */
|
||||
public Builder appendToId(String appendToId) { this.appendToId = appendToId; return this; }
|
||||
/** Ignore invalid URLs instead of failing. */
|
||||
public Builder ignoreInvalidURLs(Boolean ignoreInvalidURLs) { this.ignoreInvalidURLs = ignoreInvalidURLs; return this; }
|
||||
/** Max concurrent scrapes. */
|
||||
public Builder maxConcurrency(Integer maxConcurrency) { this.maxConcurrency = maxConcurrency; return this; }
|
||||
/** Do not store any data on Firecrawl servers. */
|
||||
public Builder zeroDataRetention(Boolean zeroDataRetention) { this.zeroDataRetention = zeroDataRetention; return this; }
|
||||
/** Idempotency key to prevent duplicate batch jobs. */
|
||||
public Builder idempotencyKey(String idempotencyKey) { this.idempotencyKey = idempotencyKey; return this; }
|
||||
/** Integration identifier. */
|
||||
public Builder integration(String integration) { this.integration = integration; return this; }
|
||||
|
||||
public BatchScrapeOptions build() {
|
||||
BatchScrapeOptions o = new BatchScrapeOptions();
|
||||
o.options = this.options;
|
||||
o.webhook = this.webhook;
|
||||
o.appendToId = this.appendToId;
|
||||
o.ignoreInvalidURLs = this.ignoreInvalidURLs;
|
||||
o.maxConcurrency = this.maxConcurrency;
|
||||
o.zeroDataRetention = this.zeroDataRetention;
|
||||
o.idempotencyKey = this.idempotencyKey;
|
||||
o.integration = this.integration;
|
||||
return o;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
package com.firecrawl.models;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* Response from starting an async batch scrape job.
|
||||
*/
|
||||
@JsonIgnoreProperties(ignoreUnknown = true)
|
||||
public class BatchScrapeResponse {
|
||||
|
||||
private String id;
|
||||
private String url;
|
||||
private List<String> invalidURLs;
|
||||
|
||||
public String getId() { return id; }
|
||||
public String getUrl() { return url; }
|
||||
public List<String> getInvalidURLs() { return invalidURLs; }
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "BatchScrapeResponse{id=" + id + ", url=" + url + "}";
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
package com.firecrawl.models;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
|
||||
|
||||
/**
|
||||
* Response from creating a new browser session.
|
||||
*/
|
||||
@JsonIgnoreProperties(ignoreUnknown = true)
|
||||
public class BrowserCreateResponse {
|
||||
|
||||
private boolean success;
|
||||
private String id;
|
||||
private String cdpUrl;
|
||||
private String liveViewUrl;
|
||||
private String expiresAt;
|
||||
private String error;
|
||||
|
||||
public boolean isSuccess() { return success; }
|
||||
public String getId() { return id; }
|
||||
public String getCdpUrl() { return cdpUrl; }
|
||||
public String getLiveViewUrl() { return liveViewUrl; }
|
||||
public String getExpiresAt() { return expiresAt; }
|
||||
public String getError() { return error; }
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "BrowserCreateResponse{id=" + id + ", success=" + success + "}";
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
package com.firecrawl.models;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
|
||||
|
||||
/**
|
||||
* Response from deleting a browser session.
|
||||
*/
|
||||
@JsonIgnoreProperties(ignoreUnknown = true)
|
||||
public class BrowserDeleteResponse {
|
||||
|
||||
private boolean success;
|
||||
private Long sessionDurationMs;
|
||||
private Integer creditsBilled;
|
||||
private String error;
|
||||
|
||||
public boolean isSuccess() { return success; }
|
||||
public Long getSessionDurationMs() { return sessionDurationMs; }
|
||||
public Integer getCreditsBilled() { return creditsBilled; }
|
||||
public String getError() { return error; }
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "BrowserDeleteResponse{success=" + success + ", creditsBilled=" + creditsBilled + "}";
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
package com.firecrawl.models;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
|
||||
|
||||
/**
|
||||
* Response from executing code in a browser session.
|
||||
*/
|
||||
@JsonIgnoreProperties(ignoreUnknown = true)
|
||||
public class BrowserExecuteResponse {
|
||||
|
||||
private boolean success;
|
||||
private String stdout;
|
||||
private String result;
|
||||
private String stderr;
|
||||
private Integer exitCode;
|
||||
private Boolean killed;
|
||||
private String error;
|
||||
|
||||
public boolean isSuccess() { return success; }
|
||||
public String getStdout() { return stdout; }
|
||||
public String getResult() { return result; }
|
||||
public String getStderr() { return stderr; }
|
||||
public Integer getExitCode() { return exitCode; }
|
||||
public Boolean getKilled() { return killed; }
|
||||
public String getError() { return error; }
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "BrowserExecuteResponse{success=" + success + ", exitCode=" + exitCode + "}";
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
package com.firecrawl.models;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* Response from listing browser sessions.
|
||||
*/
|
||||
@JsonIgnoreProperties(ignoreUnknown = true)
|
||||
public class BrowserListResponse {
|
||||
|
||||
private boolean success;
|
||||
private List<BrowserSession> sessions;
|
||||
private String error;
|
||||
|
||||
public boolean isSuccess() { return success; }
|
||||
public List<BrowserSession> getSessions() { return sessions; }
|
||||
public String getError() { return error; }
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
int count = sessions != null ? sessions.size() : 0;
|
||||
return "BrowserListResponse{success=" + success + ", sessions=" + count + "}";
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
package com.firecrawl.models;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
|
||||
|
||||
/**
|
||||
* Represents a browser session's metadata.
|
||||
*/
|
||||
@JsonIgnoreProperties(ignoreUnknown = true)
|
||||
public class BrowserSession {
|
||||
|
||||
private String id;
|
||||
private String status;
|
||||
private String cdpUrl;
|
||||
private String liveViewUrl;
|
||||
private boolean streamWebView;
|
||||
private String createdAt;
|
||||
private String lastActivity;
|
||||
|
||||
public String getId() { return id; }
|
||||
public String getStatus() { return status; }
|
||||
public String getCdpUrl() { return cdpUrl; }
|
||||
public String getLiveViewUrl() { return liveViewUrl; }
|
||||
public boolean isStreamWebView() { return streamWebView; }
|
||||
public String getCreatedAt() { return createdAt; }
|
||||
public String getLastActivity() { return lastActivity; }
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "BrowserSession{id=" + id + ", status=" + status + "}";
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
package com.firecrawl.models;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
|
||||
|
||||
/**
|
||||
* Current concurrency usage.
|
||||
*/
|
||||
@JsonIgnoreProperties(ignoreUnknown = true)
|
||||
public class ConcurrencyCheck {
|
||||
|
||||
private int concurrency;
|
||||
private int maxConcurrency;
|
||||
|
||||
public int getConcurrency() { return concurrency; }
|
||||
public int getMaxConcurrency() { return maxConcurrency; }
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "ConcurrencyCheck{concurrency=" + concurrency + "/" + maxConcurrency + "}";
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
package com.firecrawl.models;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* Status and results of a crawl job.
|
||||
*/
|
||||
@JsonIgnoreProperties(ignoreUnknown = true)
|
||||
public class CrawlJob {
|
||||
|
||||
private String id;
|
||||
private String status;
|
||||
private int total;
|
||||
private int completed;
|
||||
private Integer creditsUsed;
|
||||
private String expiresAt;
|
||||
private String next;
|
||||
private List<Document> data;
|
||||
|
||||
public String getId() { return id; }
|
||||
public String getStatus() { return status; }
|
||||
public int getTotal() { return total; }
|
||||
public int getCompleted() { return completed; }
|
||||
public Integer getCreditsUsed() { return creditsUsed; }
|
||||
public String getExpiresAt() { return expiresAt; }
|
||||
public String getNext() { return next; }
|
||||
public List<Document> getData() { return data; }
|
||||
public void setData(List<Document> data) { this.data = data; }
|
||||
|
||||
/** Returns true if the job has finished (completed, failed, or cancelled). */
|
||||
public boolean isDone() {
|
||||
return "completed".equals(status) || "failed".equals(status) || "cancelled".equals(status);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "CrawlJob{id=" + id + ", status=" + status + ", completed=" + completed + "/" + total + "}";
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,168 @@
|
||||
package com.firecrawl.models;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonInclude;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* Options for crawling a website.
|
||||
*/
|
||||
@JsonInclude(JsonInclude.Include.NON_NULL)
|
||||
public class CrawlOptions {
|
||||
|
||||
private String prompt;
|
||||
private List<String> excludePaths;
|
||||
private List<String> includePaths;
|
||||
private Integer maxDiscoveryDepth;
|
||||
private String sitemap;
|
||||
private Boolean ignoreQueryParameters;
|
||||
private Boolean deduplicateSimilarURLs;
|
||||
private Integer limit;
|
||||
private Boolean crawlEntireDomain;
|
||||
private Boolean allowExternalLinks;
|
||||
private Boolean allowSubdomains;
|
||||
private Boolean ignoreRobotsTxt;
|
||||
private String robotsUserAgent;
|
||||
private Integer delay;
|
||||
private Integer maxConcurrency;
|
||||
private Object webhook;
|
||||
private ScrapeOptions scrapeOptions;
|
||||
private Boolean regexOnFullURL;
|
||||
private Boolean zeroDataRetention;
|
||||
private String integration;
|
||||
|
||||
private CrawlOptions() {}
|
||||
|
||||
public String getPrompt() { return prompt; }
|
||||
public List<String> getExcludePaths() { return excludePaths; }
|
||||
public List<String> getIncludePaths() { return includePaths; }
|
||||
public Integer getMaxDiscoveryDepth() { return maxDiscoveryDepth; }
|
||||
public String getSitemap() { return sitemap; }
|
||||
public Boolean getIgnoreQueryParameters() { return ignoreQueryParameters; }
|
||||
public Boolean getDeduplicateSimilarURLs() { return deduplicateSimilarURLs; }
|
||||
public Integer getLimit() { return limit; }
|
||||
public Boolean getCrawlEntireDomain() { return crawlEntireDomain; }
|
||||
public Boolean getAllowExternalLinks() { return allowExternalLinks; }
|
||||
public Boolean getAllowSubdomains() { return allowSubdomains; }
|
||||
public Boolean getIgnoreRobotsTxt() { return ignoreRobotsTxt; }
|
||||
public String getRobotsUserAgent() { return robotsUserAgent; }
|
||||
public Integer getDelay() { return delay; }
|
||||
public Integer getMaxConcurrency() { return maxConcurrency; }
|
||||
public Object getWebhook() { return webhook; }
|
||||
public ScrapeOptions getScrapeOptions() { return scrapeOptions; }
|
||||
public Boolean getRegexOnFullURL() { return regexOnFullURL; }
|
||||
public Boolean getZeroDataRetention() { return zeroDataRetention; }
|
||||
public String getIntegration() { return integration; }
|
||||
|
||||
public static Builder builder() { return new Builder(); }
|
||||
|
||||
public static final class Builder {
|
||||
private String prompt;
|
||||
private List<String> excludePaths;
|
||||
private List<String> includePaths;
|
||||
private Integer maxDiscoveryDepth;
|
||||
private String sitemap;
|
||||
private Boolean ignoreQueryParameters;
|
||||
private Boolean deduplicateSimilarURLs;
|
||||
private Integer limit;
|
||||
private Boolean crawlEntireDomain;
|
||||
private Boolean allowExternalLinks;
|
||||
private Boolean allowSubdomains;
|
||||
private Boolean ignoreRobotsTxt;
|
||||
private String robotsUserAgent;
|
||||
private Integer delay;
|
||||
private Integer maxConcurrency;
|
||||
private Object webhook;
|
||||
private ScrapeOptions scrapeOptions;
|
||||
private Boolean regexOnFullURL;
|
||||
private Boolean zeroDataRetention;
|
||||
private String integration;
|
||||
|
||||
private Builder() {}
|
||||
|
||||
/** Natural language prompt to guide crawling. */
|
||||
public Builder prompt(String prompt) { this.prompt = prompt; return this; }
|
||||
|
||||
/** URL path patterns to exclude from crawling. */
|
||||
public Builder excludePaths(List<String> excludePaths) { this.excludePaths = excludePaths; return this; }
|
||||
|
||||
/** URL path patterns to include in crawling. */
|
||||
public Builder includePaths(List<String> includePaths) { this.includePaths = includePaths; return this; }
|
||||
|
||||
/** Maximum depth to discover links. */
|
||||
public Builder maxDiscoveryDepth(Integer maxDiscoveryDepth) { this.maxDiscoveryDepth = maxDiscoveryDepth; return this; }
|
||||
|
||||
/** Sitemap handling: "skip", "include", or "only". */
|
||||
public Builder sitemap(String sitemap) { this.sitemap = sitemap; return this; }
|
||||
|
||||
/** Ignore query parameters when deduplicating URLs. */
|
||||
public Builder ignoreQueryParameters(Boolean ignoreQueryParameters) { this.ignoreQueryParameters = ignoreQueryParameters; return this; }
|
||||
|
||||
/** Deduplicate URLs that are similar. */
|
||||
public Builder deduplicateSimilarURLs(Boolean deduplicateSimilarURLs) { this.deduplicateSimilarURLs = deduplicateSimilarURLs; return this; }
|
||||
|
||||
/** Maximum number of pages to crawl. */
|
||||
public Builder limit(Integer limit) { this.limit = limit; return this; }
|
||||
|
||||
/** Whether to crawl the entire domain. */
|
||||
public Builder crawlEntireDomain(Boolean crawlEntireDomain) { this.crawlEntireDomain = crawlEntireDomain; return this; }
|
||||
|
||||
/** Follow external links. */
|
||||
public Builder allowExternalLinks(Boolean allowExternalLinks) { this.allowExternalLinks = allowExternalLinks; return this; }
|
||||
|
||||
/** Follow subdomains. */
|
||||
public Builder allowSubdomains(Boolean allowSubdomains) { this.allowSubdomains = allowSubdomains; return this; }
|
||||
|
||||
/** Ignore the website's robots.txt rules. Enterprise only. */
|
||||
public Builder ignoreRobotsTxt(Boolean ignoreRobotsTxt) { this.ignoreRobotsTxt = ignoreRobotsTxt; return this; }
|
||||
|
||||
/** Custom User-Agent string for robots.txt evaluation. Enterprise only. */
|
||||
public Builder robotsUserAgent(String robotsUserAgent) { this.robotsUserAgent = robotsUserAgent; return this; }
|
||||
|
||||
/** Delay in milliseconds between requests. */
|
||||
public Builder delay(Integer delay) { this.delay = delay; return this; }
|
||||
|
||||
/** Maximum concurrent requests. */
|
||||
public Builder maxConcurrency(Integer maxConcurrency) { this.maxConcurrency = maxConcurrency; return this; }
|
||||
|
||||
/** Webhook URL string or {@link WebhookConfig} object. */
|
||||
public Builder webhook(Object webhook) { this.webhook = webhook; return this; }
|
||||
|
||||
/** Scrape options applied to each crawled page. */
|
||||
public Builder scrapeOptions(ScrapeOptions scrapeOptions) { this.scrapeOptions = scrapeOptions; return this; }
|
||||
|
||||
/** Apply regex patterns to the full URL, not just the path. */
|
||||
public Builder regexOnFullURL(Boolean regexOnFullURL) { this.regexOnFullURL = regexOnFullURL; return this; }
|
||||
|
||||
/** Do not store any scraped data on Firecrawl servers. */
|
||||
public Builder zeroDataRetention(Boolean zeroDataRetention) { this.zeroDataRetention = zeroDataRetention; return this; }
|
||||
|
||||
/** Integration identifier. */
|
||||
public Builder integration(String integration) { this.integration = integration; return this; }
|
||||
|
||||
public CrawlOptions build() {
|
||||
CrawlOptions o = new CrawlOptions();
|
||||
o.prompt = this.prompt;
|
||||
o.excludePaths = this.excludePaths;
|
||||
o.includePaths = this.includePaths;
|
||||
o.maxDiscoveryDepth = this.maxDiscoveryDepth;
|
||||
o.sitemap = this.sitemap;
|
||||
o.ignoreQueryParameters = this.ignoreQueryParameters;
|
||||
o.deduplicateSimilarURLs = this.deduplicateSimilarURLs;
|
||||
o.limit = this.limit;
|
||||
o.crawlEntireDomain = this.crawlEntireDomain;
|
||||
o.allowExternalLinks = this.allowExternalLinks;
|
||||
o.allowSubdomains = this.allowSubdomains;
|
||||
o.ignoreRobotsTxt = this.ignoreRobotsTxt;
|
||||
o.robotsUserAgent = this.robotsUserAgent;
|
||||
o.delay = this.delay;
|
||||
o.maxConcurrency = this.maxConcurrency;
|
||||
o.webhook = this.webhook;
|
||||
o.scrapeOptions = this.scrapeOptions;
|
||||
o.regexOnFullURL = this.regexOnFullURL;
|
||||
o.zeroDataRetention = this.zeroDataRetention;
|
||||
o.integration = this.integration;
|
||||
return o;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
package com.firecrawl.models;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
|
||||
|
||||
/**
|
||||
* Response from starting an async crawl job.
|
||||
*/
|
||||
@JsonIgnoreProperties(ignoreUnknown = true)
|
||||
public class CrawlResponse {
|
||||
|
||||
private String id;
|
||||
private String url;
|
||||
|
||||
public String getId() { return id; }
|
||||
public String getUrl() { return url; }
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "CrawlResponse{id=" + id + ", url=" + url + "}";
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
package com.firecrawl.models;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
|
||||
|
||||
/**
|
||||
* Current credit usage information.
|
||||
*/
|
||||
@JsonIgnoreProperties(ignoreUnknown = true)
|
||||
public class CreditUsage {
|
||||
|
||||
private int remainingCredits;
|
||||
private Integer planCredits;
|
||||
private String billingPeriodStart;
|
||||
private String billingPeriodEnd;
|
||||
|
||||
public int getRemainingCredits() { return remainingCredits; }
|
||||
public Integer getPlanCredits() { return planCredits; }
|
||||
public String getBillingPeriodStart() { return billingPeriodStart; }
|
||||
public String getBillingPeriodEnd() { return billingPeriodEnd; }
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "CreditUsage{remaining=" + remainingCredits + ", plan=" + planCredits + "}";
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
package com.firecrawl.models;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* A scraped document returned by scrape, crawl, and batch endpoints.
|
||||
*/
|
||||
@JsonIgnoreProperties(ignoreUnknown = true)
|
||||
public class Document {
|
||||
|
||||
private String markdown;
|
||||
private String html;
|
||||
private String rawHtml;
|
||||
private Object json;
|
||||
private String summary;
|
||||
private Map<String, Object> metadata;
|
||||
private List<String> links;
|
||||
private List<String> images;
|
||||
private String screenshot;
|
||||
private String audio;
|
||||
private List<Map<String, Object>> attributes;
|
||||
private Map<String, Object> actions;
|
||||
private String answer;
|
||||
private String highlights;
|
||||
private String warning;
|
||||
private Map<String, Object> changeTracking;
|
||||
private Map<String, Object> branding;
|
||||
|
||||
public String getMarkdown() { return markdown; }
|
||||
public String getHtml() { return html; }
|
||||
public String getRawHtml() { return rawHtml; }
|
||||
public Object getJson() { return json; }
|
||||
public String getSummary() { return summary; }
|
||||
public Map<String, Object> getMetadata() { return metadata; }
|
||||
public List<String> getLinks() { return links; }
|
||||
public List<String> getImages() { return images; }
|
||||
public String getScreenshot() { return screenshot; }
|
||||
public String getAudio() { return audio; }
|
||||
public List<Map<String, Object>> getAttributes() { return attributes; }
|
||||
public Map<String, Object> getActions() { return actions; }
|
||||
public String getAnswer() { return answer; }
|
||||
public String getHighlights() { return highlights; }
|
||||
public String getWarning() { return warning; }
|
||||
public Map<String, Object> getChangeTracking() { return changeTracking; }
|
||||
public Map<String, Object> getBranding() { return branding; }
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
String title = metadata != null ? String.valueOf(metadata.get("title")) : "untitled";
|
||||
String url = metadata != null ? String.valueOf(metadata.get("sourceURL")) : "unknown";
|
||||
return "Document{title=" + title + ", url=" + url + "}";
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
package com.firecrawl.models;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonInclude;
|
||||
|
||||
/**
|
||||
* Highlights format for extracting direct highlights from page content.
|
||||
*/
|
||||
@JsonInclude(JsonInclude.Include.NON_NULL)
|
||||
public class HighlightsFormat {
|
||||
|
||||
private final String type = "highlights";
|
||||
private String query;
|
||||
|
||||
private HighlightsFormat() {}
|
||||
|
||||
public String getType() { return type; }
|
||||
public String getQuery() { return query; }
|
||||
|
||||
public static Builder builder() { return new Builder(); }
|
||||
|
||||
public static final class Builder {
|
||||
private String query;
|
||||
|
||||
private Builder() {}
|
||||
|
||||
/** Query used to select highlights from the page content. */
|
||||
public Builder query(String query) { this.query = query; return this; }
|
||||
|
||||
public HighlightsFormat build() {
|
||||
HighlightsFormat f = new HighlightsFormat();
|
||||
f.query = this.query;
|
||||
return f;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,57 @@
|
||||
package com.firecrawl.models;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonInclude;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* JSON extraction format with optional schema and prompt.
|
||||
*
|
||||
* <p>Usage:
|
||||
* <pre>{@code
|
||||
* JsonFormat jsonFmt = JsonFormat.builder()
|
||||
* .prompt("Extract the product name and price")
|
||||
* .schema(Map.of(
|
||||
* "type", "object",
|
||||
* "properties", Map.of(
|
||||
* "name", Map.of("type", "string"),
|
||||
* "price", Map.of("type", "number")
|
||||
* )
|
||||
* ))
|
||||
* .build();
|
||||
* }</pre>
|
||||
*/
|
||||
@JsonInclude(JsonInclude.Include.NON_NULL)
|
||||
public class JsonFormat {
|
||||
|
||||
private final String type = "json";
|
||||
private String prompt;
|
||||
private Map<String, Object> schema;
|
||||
|
||||
private JsonFormat() {}
|
||||
|
||||
public String getType() { return type; }
|
||||
public String getPrompt() { return prompt; }
|
||||
public Map<String, Object> getSchema() { return schema; }
|
||||
|
||||
public static Builder builder() { return new Builder(); }
|
||||
|
||||
public static final class Builder {
|
||||
private String prompt;
|
||||
private Map<String, Object> schema;
|
||||
|
||||
private Builder() {}
|
||||
|
||||
/** LLM prompt for extraction. */
|
||||
public Builder prompt(String prompt) { this.prompt = prompt; return this; }
|
||||
|
||||
/** JSON Schema for structured extraction. */
|
||||
public Builder schema(Map<String, Object> schema) { this.schema = schema; return this; }
|
||||
|
||||
public JsonFormat build() {
|
||||
JsonFormat f = new JsonFormat();
|
||||
f.prompt = this.prompt;
|
||||
f.schema = this.schema;
|
||||
return f;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
package com.firecrawl.models;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonInclude;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* Geolocation configuration for requests.
|
||||
*/
|
||||
@JsonInclude(JsonInclude.Include.NON_NULL)
|
||||
public class LocationConfig {
|
||||
|
||||
private String country;
|
||||
private List<String> languages;
|
||||
|
||||
private LocationConfig() {}
|
||||
|
||||
public String getCountry() { return country; }
|
||||
public List<String> getLanguages() { return languages; }
|
||||
|
||||
public static Builder builder() { return new Builder(); }
|
||||
|
||||
public static final class Builder {
|
||||
private String country;
|
||||
private List<String> languages;
|
||||
|
||||
private Builder() {}
|
||||
|
||||
public Builder country(String country) { this.country = country; return this; }
|
||||
public Builder languages(List<String> languages) { this.languages = languages; return this; }
|
||||
|
||||
public LocationConfig build() {
|
||||
LocationConfig c = new LocationConfig();
|
||||
c.country = this.country;
|
||||
c.languages = this.languages;
|
||||
return c;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
package com.firecrawl.models;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* Result of a map operation containing discovered URLs.
|
||||
*
|
||||
* <p>The v2 API may return {@code links} as either plain URL strings or
|
||||
* objects with {@code url}, {@code title}, and {@code description} fields.
|
||||
* This class normalises both representations into a uniform
|
||||
* {@code List<Map<String, Object>>} where each entry always contains at
|
||||
* least a {@code "url"} key.
|
||||
*/
|
||||
@JsonIgnoreProperties(ignoreUnknown = true)
|
||||
public class MapData {
|
||||
|
||||
private List<Object> links;
|
||||
|
||||
/**
|
||||
* Returns the discovered links, normalised so that every entry is a
|
||||
* {@code Map<String, Object>} containing at least a {@code "url"} key.
|
||||
* Plain-string entries returned by the API are wrapped as
|
||||
* {@code {"url": "<value>"}}.
|
||||
*/
|
||||
@SuppressWarnings("unchecked")
|
||||
public List<Map<String, Object>> getLinks() {
|
||||
if (links == null) {
|
||||
return null;
|
||||
}
|
||||
List<Map<String, Object>> result = new ArrayList<>(links.size());
|
||||
for (Object item : links) {
|
||||
if (item instanceof Map) {
|
||||
result.add((Map<String, Object>) item);
|
||||
} else if (item instanceof String) {
|
||||
Map<String, Object> wrapped = new LinkedHashMap<>();
|
||||
wrapped.put("url", item);
|
||||
result.add(wrapped);
|
||||
}
|
||||
}
|
||||
return Collections.unmodifiableList(result);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
int count = links != null ? links.size() : 0;
|
||||
return "MapData{links=" + count + "}";
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,76 @@
|
||||
package com.firecrawl.models;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonInclude;
|
||||
|
||||
/**
|
||||
* Options for mapping (discovering URLs on) a website.
|
||||
*/
|
||||
@JsonInclude(JsonInclude.Include.NON_NULL)
|
||||
public class MapOptions {
|
||||
|
||||
private String search;
|
||||
private String sitemap;
|
||||
private Boolean includeSubdomains;
|
||||
private Boolean ignoreQueryParameters;
|
||||
private Integer limit;
|
||||
private Integer timeout;
|
||||
private String integration;
|
||||
private LocationConfig location;
|
||||
|
||||
private MapOptions() {}
|
||||
|
||||
public String getSearch() { return search; }
|
||||
/** Sitemap mode: "only", "include", or "skip". */
|
||||
public String getSitemap() { return sitemap; }
|
||||
public Boolean getIncludeSubdomains() { return includeSubdomains; }
|
||||
public Boolean getIgnoreQueryParameters() { return ignoreQueryParameters; }
|
||||
public Integer getLimit() { return limit; }
|
||||
public Integer getTimeout() { return timeout; }
|
||||
public String getIntegration() { return integration; }
|
||||
public LocationConfig getLocation() { return location; }
|
||||
|
||||
public static Builder builder() { return new Builder(); }
|
||||
|
||||
public static final class Builder {
|
||||
private String search;
|
||||
private String sitemap;
|
||||
private Boolean includeSubdomains;
|
||||
private Boolean ignoreQueryParameters;
|
||||
private Integer limit;
|
||||
private Integer timeout;
|
||||
private String integration;
|
||||
private LocationConfig location;
|
||||
|
||||
private Builder() {}
|
||||
|
||||
/** Filter discovered URLs by keyword. */
|
||||
public Builder search(String search) { this.search = search; return this; }
|
||||
/** Sitemap mode: "only", "include", or "skip". */
|
||||
public Builder sitemap(String sitemap) { this.sitemap = sitemap; return this; }
|
||||
/** Include subdomains. */
|
||||
public Builder includeSubdomains(Boolean includeSubdomains) { this.includeSubdomains = includeSubdomains; return this; }
|
||||
/** Ignore query parameters when deduplicating URLs. */
|
||||
public Builder ignoreQueryParameters(Boolean ignoreQueryParameters) { this.ignoreQueryParameters = ignoreQueryParameters; return this; }
|
||||
/** Maximum number of URLs to return. */
|
||||
public Builder limit(Integer limit) { this.limit = limit; return this; }
|
||||
/** Timeout in milliseconds. */
|
||||
public Builder timeout(Integer timeout) { this.timeout = timeout; return this; }
|
||||
/** Integration identifier. */
|
||||
public Builder integration(String integration) { this.integration = integration; return this; }
|
||||
/** Geolocation configuration. */
|
||||
public Builder location(LocationConfig location) { this.location = location; return this; }
|
||||
|
||||
public MapOptions build() {
|
||||
MapOptions o = new MapOptions();
|
||||
o.search = this.search;
|
||||
o.sitemap = this.sitemap;
|
||||
o.includeSubdomains = this.includeSubdomains;
|
||||
o.ignoreQueryParameters = this.ignoreQueryParameters;
|
||||
o.limit = this.limit;
|
||||
o.timeout = this.timeout;
|
||||
o.integration = this.integration;
|
||||
o.location = this.location;
|
||||
return o;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
package com.firecrawl.models;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
@JsonIgnoreProperties(ignoreUnknown = true)
|
||||
public class Monitor {
|
||||
private String id;
|
||||
private String name;
|
||||
private String status;
|
||||
private MonitorSchedule schedule;
|
||||
private String nextRunAt;
|
||||
private String lastRunAt;
|
||||
private String currentCheckId;
|
||||
private List<Map<String, Object>> targets;
|
||||
private Map<String, Object> webhook;
|
||||
private Map<String, Object> notification;
|
||||
private int retentionDays;
|
||||
private Integer estimatedCreditsPerMonth;
|
||||
private MonitorSummary lastCheckSummary;
|
||||
private String createdAt;
|
||||
private String updatedAt;
|
||||
|
||||
public String getId() { return id; }
|
||||
public String getName() { return name; }
|
||||
public String getStatus() { return status; }
|
||||
public MonitorSchedule getSchedule() { return schedule; }
|
||||
public String getNextRunAt() { return nextRunAt; }
|
||||
public String getLastRunAt() { return lastRunAt; }
|
||||
public String getCurrentCheckId() { return currentCheckId; }
|
||||
public List<Map<String, Object>> getTargets() { return targets; }
|
||||
public Map<String, Object> getWebhook() { return webhook; }
|
||||
public Map<String, Object> getNotification() { return notification; }
|
||||
public int getRetentionDays() { return retentionDays; }
|
||||
public Integer getEstimatedCreditsPerMonth() { return estimatedCreditsPerMonth; }
|
||||
public MonitorSummary getLastCheckSummary() { return lastCheckSummary; }
|
||||
public String getCreatedAt() { return createdAt; }
|
||||
public String getUpdatedAt() { return updatedAt; }
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
package com.firecrawl.models;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
|
||||
|
||||
@JsonIgnoreProperties(ignoreUnknown = true)
|
||||
public class MonitorCheck {
|
||||
private String id;
|
||||
private String monitorId;
|
||||
private String status;
|
||||
private String trigger;
|
||||
private String scheduledFor;
|
||||
private String startedAt;
|
||||
private String finishedAt;
|
||||
private Integer estimatedCredits;
|
||||
private Integer reservedCredits;
|
||||
private Integer actualCredits;
|
||||
private String billingStatus;
|
||||
private MonitorSummary summary;
|
||||
private Object targetResults;
|
||||
private Object notificationStatus;
|
||||
private String error;
|
||||
private String createdAt;
|
||||
private String updatedAt;
|
||||
|
||||
public String getId() { return id; }
|
||||
public String getMonitorId() { return monitorId; }
|
||||
public String getStatus() { return status; }
|
||||
public String getTrigger() { return trigger; }
|
||||
public String getScheduledFor() { return scheduledFor; }
|
||||
public String getStartedAt() { return startedAt; }
|
||||
public String getFinishedAt() { return finishedAt; }
|
||||
public Integer getEstimatedCredits() { return estimatedCredits; }
|
||||
public Integer getReservedCredits() { return reservedCredits; }
|
||||
public Integer getActualCredits() { return actualCredits; }
|
||||
public String getBillingStatus() { return billingStatus; }
|
||||
public MonitorSummary getSummary() { return summary; }
|
||||
public Object getTargetResults() { return targetResults; }
|
||||
public Object getNotificationStatus() { return notificationStatus; }
|
||||
public String getError() { return error; }
|
||||
public String getCreatedAt() { return createdAt; }
|
||||
public String getUpdatedAt() { return updatedAt; }
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
package com.firecrawl.models;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
|
||||
import java.util.List;
|
||||
|
||||
@JsonIgnoreProperties(ignoreUnknown = true)
|
||||
public class MonitorCheckDetail extends MonitorCheck {
|
||||
private List<MonitorCheckPage> pages;
|
||||
private String next;
|
||||
|
||||
public List<MonitorCheckPage> getPages() { return pages; }
|
||||
public void setPages(List<MonitorCheckPage> pages) { this.pages = pages; }
|
||||
public String getNext() { return next; }
|
||||
public void setNext(String next) { this.next = next; }
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
package com.firecrawl.models;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
|
||||
|
||||
@JsonIgnoreProperties(ignoreUnknown = true)
|
||||
public class MonitorCheckPage {
|
||||
private String id;
|
||||
private String targetId;
|
||||
private String url;
|
||||
private String status;
|
||||
private String previousScrapeId;
|
||||
private String currentScrapeId;
|
||||
private Integer statusCode;
|
||||
private String error;
|
||||
private Object metadata;
|
||||
private Object diff;
|
||||
private String createdAt;
|
||||
|
||||
public String getId() { return id; }
|
||||
public String getTargetId() { return targetId; }
|
||||
public String getUrl() { return url; }
|
||||
public String getStatus() { return status; }
|
||||
public String getPreviousScrapeId() { return previousScrapeId; }
|
||||
public String getCurrentScrapeId() { return currentScrapeId; }
|
||||
public Integer getStatusCode() { return statusCode; }
|
||||
public String getError() { return error; }
|
||||
public Object getMetadata() { return metadata; }
|
||||
public Object getDiff() { return diff; }
|
||||
public String getCreatedAt() { return createdAt; }
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
package com.firecrawl.models;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
|
||||
|
||||
@JsonIgnoreProperties(ignoreUnknown = true)
|
||||
public class MonitorSchedule {
|
||||
private String cron;
|
||||
private String timezone;
|
||||
|
||||
public String getCron() { return cron; }
|
||||
public String getTimezone() { return timezone; }
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
package com.firecrawl.models;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
|
||||
|
||||
@JsonIgnoreProperties(ignoreUnknown = true)
|
||||
public class MonitorSummary {
|
||||
private int totalPages;
|
||||
private int same;
|
||||
private int changed;
|
||||
private int newCount;
|
||||
private int removed;
|
||||
private int error;
|
||||
|
||||
public int getTotalPages() { return totalPages; }
|
||||
public int getSame() { return same; }
|
||||
public int getChanged() { return changed; }
|
||||
public int getNew() { return newCount; }
|
||||
public int getRemoved() { return removed; }
|
||||
public int getError() { return error; }
|
||||
|
||||
@com.fasterxml.jackson.annotation.JsonProperty("new")
|
||||
private void setNewCount(int value) {
|
||||
this.newCount = value;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,77 @@
|
||||
package com.firecrawl.models;
|
||||
|
||||
import java.util.Arrays;
|
||||
import java.util.Objects;
|
||||
|
||||
/**
|
||||
* Binary upload payload for the v2 parse endpoint.
|
||||
*/
|
||||
public class ParseFile {
|
||||
private final byte[] content;
|
||||
private final String filename;
|
||||
private final String contentType;
|
||||
|
||||
private ParseFile(byte[] content, String filename, String contentType) {
|
||||
this.content = content;
|
||||
this.filename = filename;
|
||||
this.contentType = contentType;
|
||||
}
|
||||
|
||||
public byte[] getContent() {
|
||||
return Arrays.copyOf(content, content.length);
|
||||
}
|
||||
|
||||
public String getFilename() {
|
||||
return filename;
|
||||
}
|
||||
|
||||
public String getContentType() {
|
||||
return contentType;
|
||||
}
|
||||
|
||||
public static Builder builder() {
|
||||
return new Builder();
|
||||
}
|
||||
|
||||
public static final class Builder {
|
||||
private byte[] content;
|
||||
private String filename;
|
||||
private String contentType;
|
||||
|
||||
private Builder() {}
|
||||
|
||||
/** Raw file content bytes. */
|
||||
public Builder content(byte[] content) {
|
||||
this.content = content != null ? Arrays.copyOf(content, content.length) : null;
|
||||
return this;
|
||||
}
|
||||
|
||||
/** Uploaded filename (e.g., "document.pdf"). */
|
||||
public Builder filename(String filename) {
|
||||
this.filename = filename;
|
||||
return this;
|
||||
}
|
||||
|
||||
/** Optional MIME type hint (e.g., "application/pdf"). */
|
||||
public Builder contentType(String contentType) {
|
||||
this.contentType = contentType;
|
||||
return this;
|
||||
}
|
||||
|
||||
public ParseFile build() {
|
||||
Objects.requireNonNull(content, "File content is required");
|
||||
if (content.length == 0) {
|
||||
throw new IllegalArgumentException("File content cannot be empty");
|
||||
}
|
||||
Objects.requireNonNull(filename, "Filename is required");
|
||||
if (filename.isBlank()) {
|
||||
throw new IllegalArgumentException("Filename cannot be blank");
|
||||
}
|
||||
return new ParseFile(
|
||||
Arrays.copyOf(content, content.length),
|
||||
filename.trim(),
|
||||
contentType
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,155 @@
|
||||
package com.firecrawl.models;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonInclude;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* Options for parsing uploaded files via /v2/parse.
|
||||
*
|
||||
* <p>Parse does not support browser-rendering formats/options such as
|
||||
* change tracking, screenshot, branding, actions, waitFor, location, or mobile.
|
||||
*/
|
||||
@JsonInclude(JsonInclude.Include.NON_NULL)
|
||||
public class ParseOptions {
|
||||
|
||||
private List<Object> formats;
|
||||
private Map<String, String> headers;
|
||||
private List<String> includeTags;
|
||||
private List<String> excludeTags;
|
||||
private Boolean onlyMainContent;
|
||||
private Integer timeout;
|
||||
private List<Object> parsers;
|
||||
private Boolean skipTlsVerification;
|
||||
private Boolean removeBase64Images;
|
||||
private Boolean blockAds;
|
||||
private String proxy;
|
||||
private String integration;
|
||||
|
||||
private ParseOptions() {}
|
||||
|
||||
public List<Object> getFormats() { return formats; }
|
||||
public Map<String, String> getHeaders() { return headers; }
|
||||
public List<String> getIncludeTags() { return includeTags; }
|
||||
public List<String> getExcludeTags() { return excludeTags; }
|
||||
public Boolean getOnlyMainContent() { return onlyMainContent; }
|
||||
public Integer getTimeout() { return timeout; }
|
||||
public List<Object> getParsers() { return parsers; }
|
||||
public Boolean getSkipTlsVerification() { return skipTlsVerification; }
|
||||
public Boolean getRemoveBase64Images() { return removeBase64Images; }
|
||||
public Boolean getBlockAds() { return blockAds; }
|
||||
public String getProxy() { return proxy; }
|
||||
public String getIntegration() { return integration; }
|
||||
|
||||
public static Builder builder() { return new Builder(); }
|
||||
|
||||
public Builder toBuilder() {
|
||||
Builder b = new Builder();
|
||||
b.formats = this.formats != null ? new ArrayList<>(this.formats) : null;
|
||||
b.headers = this.headers != null ? new HashMap<>(this.headers) : null;
|
||||
b.includeTags = this.includeTags != null ? new ArrayList<>(this.includeTags) : null;
|
||||
b.excludeTags = this.excludeTags != null ? new ArrayList<>(this.excludeTags) : null;
|
||||
b.onlyMainContent = this.onlyMainContent;
|
||||
b.timeout = this.timeout;
|
||||
b.parsers = this.parsers != null ? new ArrayList<>(this.parsers) : null;
|
||||
b.skipTlsVerification = this.skipTlsVerification;
|
||||
b.removeBase64Images = this.removeBase64Images;
|
||||
b.blockAds = this.blockAds;
|
||||
b.proxy = this.proxy;
|
||||
b.integration = this.integration;
|
||||
return b;
|
||||
}
|
||||
|
||||
private static String extractFormatType(Object fmt) {
|
||||
if (fmt instanceof String) return (String) fmt;
|
||||
if (fmt instanceof Map<?, ?>) {
|
||||
Map<?, ?> mapObj = (Map<?, ?>) fmt;
|
||||
Object type = mapObj.get("type");
|
||||
if (type instanceof String) return (String) type;
|
||||
}
|
||||
try {
|
||||
Object type = fmt.getClass().getMethod("getType").invoke(fmt);
|
||||
if (type instanceof String) return (String) type;
|
||||
} catch (ReflectiveOperationException ignored) {
|
||||
// Ignore: format object doesn't expose a getType() method.
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private static boolean isUnsupportedParseFormat(String formatType) {
|
||||
if (formatType == null) return false;
|
||||
String normalized = formatType.trim();
|
||||
return normalized.equals("changeTracking")
|
||||
|| normalized.equals("change_tracking")
|
||||
|| normalized.equals("screenshot")
|
||||
|| normalized.equals("screenshot@fullPage")
|
||||
|| normalized.equals("branding");
|
||||
}
|
||||
|
||||
public static final class Builder {
|
||||
private List<Object> formats;
|
||||
private Map<String, String> headers;
|
||||
private List<String> includeTags;
|
||||
private List<String> excludeTags;
|
||||
private Boolean onlyMainContent;
|
||||
private Integer timeout;
|
||||
private List<Object> parsers;
|
||||
private Boolean skipTlsVerification;
|
||||
private Boolean removeBase64Images;
|
||||
private Boolean blockAds;
|
||||
private String proxy;
|
||||
private String integration;
|
||||
|
||||
private Builder() {}
|
||||
|
||||
public Builder formats(List<Object> formats) { this.formats = formats; return this; }
|
||||
public Builder headers(Map<String, String> headers) { this.headers = headers; return this; }
|
||||
public Builder includeTags(List<String> includeTags) { this.includeTags = includeTags; return this; }
|
||||
public Builder excludeTags(List<String> excludeTags) { this.excludeTags = excludeTags; return this; }
|
||||
public Builder onlyMainContent(Boolean onlyMainContent) { this.onlyMainContent = onlyMainContent; return this; }
|
||||
public Builder timeout(Integer timeout) { this.timeout = timeout; return this; }
|
||||
public Builder parsers(List<Object> parsers) { this.parsers = parsers; return this; }
|
||||
public Builder skipTlsVerification(Boolean skipTlsVerification) { this.skipTlsVerification = skipTlsVerification; return this; }
|
||||
public Builder removeBase64Images(Boolean removeBase64Images) { this.removeBase64Images = removeBase64Images; return this; }
|
||||
public Builder blockAds(Boolean blockAds) { this.blockAds = blockAds; return this; }
|
||||
public Builder proxy(String proxy) { this.proxy = proxy; return this; }
|
||||
public Builder integration(String integration) { this.integration = integration; return this; }
|
||||
|
||||
public ParseOptions build() {
|
||||
if (timeout != null && timeout <= 0) {
|
||||
throw new IllegalArgumentException("timeout must be positive");
|
||||
}
|
||||
if (proxy != null && !proxy.isBlank()) {
|
||||
if (!proxy.equals("basic") && !proxy.equals("auto")) {
|
||||
throw new IllegalArgumentException("parse only supports proxy values 'basic' or 'auto'");
|
||||
}
|
||||
}
|
||||
if (formats != null) {
|
||||
for (Object fmt : formats) {
|
||||
String formatType = extractFormatType(fmt);
|
||||
if (isUnsupportedParseFormat(formatType)) {
|
||||
throw new IllegalArgumentException("parse does not support format: " + formatType);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
ParseOptions o = new ParseOptions();
|
||||
o.formats = this.formats != null ? Collections.unmodifiableList(new ArrayList<>(this.formats)) : null;
|
||||
o.headers = this.headers != null ? Collections.unmodifiableMap(new HashMap<>(this.headers)) : null;
|
||||
o.includeTags = this.includeTags != null ? Collections.unmodifiableList(new ArrayList<>(this.includeTags)) : null;
|
||||
o.excludeTags = this.excludeTags != null ? Collections.unmodifiableList(new ArrayList<>(this.excludeTags)) : null;
|
||||
o.onlyMainContent = this.onlyMainContent;
|
||||
o.timeout = this.timeout;
|
||||
o.parsers = this.parsers != null ? Collections.unmodifiableList(new ArrayList<>(this.parsers)) : null;
|
||||
o.skipTlsVerification = this.skipTlsVerification;
|
||||
o.removeBase64Images = this.removeBase64Images;
|
||||
o.blockAds = this.blockAds;
|
||||
o.proxy = this.proxy;
|
||||
o.integration = this.integration;
|
||||
return o;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,62 @@
|
||||
package com.firecrawl.models;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonInclude;
|
||||
import com.fasterxml.jackson.annotation.JsonValue;
|
||||
|
||||
/**
|
||||
* Deprecated query format for asking a question about page content.
|
||||
*
|
||||
* @deprecated Use {@link QuestionFormat} or {@link HighlightsFormat} instead.
|
||||
*/
|
||||
@Deprecated
|
||||
@JsonInclude(JsonInclude.Include.NON_NULL)
|
||||
public class QueryFormat {
|
||||
|
||||
public enum Mode {
|
||||
FREEFORM("freeform"),
|
||||
DIRECT_QUOTE("directQuote");
|
||||
|
||||
private final String value;
|
||||
|
||||
Mode(String value) {
|
||||
this.value = value;
|
||||
}
|
||||
|
||||
@JsonValue
|
||||
public String getValue() {
|
||||
return value;
|
||||
}
|
||||
}
|
||||
|
||||
private final String type = "query";
|
||||
private String prompt;
|
||||
private Mode mode;
|
||||
|
||||
private QueryFormat() {}
|
||||
|
||||
public String getType() { return type; }
|
||||
public String getPrompt() { return prompt; }
|
||||
public Mode getMode() { return mode; }
|
||||
|
||||
public static Builder builder() { return new Builder(); }
|
||||
|
||||
public static final class Builder {
|
||||
private String prompt;
|
||||
private Mode mode;
|
||||
|
||||
private Builder() {}
|
||||
|
||||
/** Question to answer from the page content. */
|
||||
public Builder prompt(String prompt) { this.prompt = prompt; return this; }
|
||||
|
||||
/** Query answer mode: freeform or direct quote. */
|
||||
public Builder mode(Mode mode) { this.mode = mode; return this; }
|
||||
|
||||
public QueryFormat build() {
|
||||
QueryFormat f = new QueryFormat();
|
||||
f.prompt = this.prompt;
|
||||
f.mode = this.mode;
|
||||
return f;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
package com.firecrawl.models;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonInclude;
|
||||
|
||||
/**
|
||||
* Question format for asking a question about page content.
|
||||
*/
|
||||
@JsonInclude(JsonInclude.Include.NON_NULL)
|
||||
public class QuestionFormat {
|
||||
|
||||
private final String type = "question";
|
||||
private String question;
|
||||
|
||||
private QuestionFormat() {}
|
||||
|
||||
public String getType() { return type; }
|
||||
public String getQuestion() { return question; }
|
||||
|
||||
public static Builder builder() { return new Builder(); }
|
||||
|
||||
public static final class Builder {
|
||||
private String question;
|
||||
|
||||
private Builder() {}
|
||||
|
||||
/** Question to answer from the page content. */
|
||||
public Builder question(String question) { this.question = question; return this; }
|
||||
|
||||
public QuestionFormat build() {
|
||||
QuestionFormat f = new QuestionFormat();
|
||||
f.question = this.question;
|
||||
return f;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,194 @@
|
||||
package com.firecrawl.models;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonInclude;
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* Options for scraping a single URL.
|
||||
*/
|
||||
@JsonInclude(JsonInclude.Include.NON_NULL)
|
||||
public class ScrapeOptions {
|
||||
|
||||
private List<Object> formats;
|
||||
private Map<String, String> headers;
|
||||
private List<String> includeTags;
|
||||
private List<String> excludeTags;
|
||||
private Boolean onlyMainContent;
|
||||
private Integer timeout;
|
||||
private Integer waitFor;
|
||||
private Boolean mobile;
|
||||
private List<Object> parsers;
|
||||
private List<Map<String, Object>> actions;
|
||||
private LocationConfig location;
|
||||
private Boolean skipTlsVerification;
|
||||
private Boolean removeBase64Images;
|
||||
private Boolean blockAds;
|
||||
private String proxy;
|
||||
@JsonProperty("maxAge")
|
||||
private Long maxAge;
|
||||
private Boolean storeInCache;
|
||||
private Boolean lockdown;
|
||||
private String integration;
|
||||
|
||||
private ScrapeOptions() {}
|
||||
|
||||
public List<Object> getFormats() { return formats; }
|
||||
public Map<String, String> getHeaders() { return headers; }
|
||||
public List<String> getIncludeTags() { return includeTags; }
|
||||
public List<String> getExcludeTags() { return excludeTags; }
|
||||
public Boolean getOnlyMainContent() { return onlyMainContent; }
|
||||
public Integer getTimeout() { return timeout; }
|
||||
public Integer getWaitFor() { return waitFor; }
|
||||
public Boolean getMobile() { return mobile; }
|
||||
public List<Object> getParsers() { return parsers; }
|
||||
public List<Map<String, Object>> getActions() { return actions; }
|
||||
public LocationConfig getLocation() { return location; }
|
||||
public Boolean getSkipTlsVerification() { return skipTlsVerification; }
|
||||
public Boolean getRemoveBase64Images() { return removeBase64Images; }
|
||||
public Boolean getBlockAds() { return blockAds; }
|
||||
public String getProxy() { return proxy; }
|
||||
public Long getMaxAge() { return maxAge; }
|
||||
public Boolean getStoreInCache() { return storeInCache; }
|
||||
public Boolean getLockdown() { return lockdown; }
|
||||
public String getIntegration() { return integration; }
|
||||
|
||||
public static Builder builder() { return new Builder(); }
|
||||
|
||||
public Builder toBuilder() {
|
||||
Builder b = new Builder();
|
||||
b.formats = this.formats != null ? new ArrayList<>(this.formats) : null;
|
||||
b.headers = this.headers != null ? new HashMap<>(this.headers) : null;
|
||||
b.includeTags = this.includeTags != null ? new ArrayList<>(this.includeTags) : null;
|
||||
b.excludeTags = this.excludeTags != null ? new ArrayList<>(this.excludeTags) : null;
|
||||
b.onlyMainContent = this.onlyMainContent;
|
||||
b.timeout = this.timeout;
|
||||
b.waitFor = this.waitFor;
|
||||
b.mobile = this.mobile;
|
||||
b.parsers = this.parsers != null ? new ArrayList<>(this.parsers) : null;
|
||||
b.actions = this.actions != null ? new ArrayList<>(this.actions) : null;
|
||||
b.location = this.location;
|
||||
b.skipTlsVerification = this.skipTlsVerification;
|
||||
b.removeBase64Images = this.removeBase64Images;
|
||||
b.blockAds = this.blockAds;
|
||||
b.proxy = this.proxy;
|
||||
b.maxAge = this.maxAge;
|
||||
b.storeInCache = this.storeInCache;
|
||||
b.lockdown = this.lockdown;
|
||||
b.integration = this.integration;
|
||||
return b;
|
||||
}
|
||||
|
||||
public static final class Builder {
|
||||
private List<Object> formats;
|
||||
private Map<String, String> headers;
|
||||
private List<String> includeTags;
|
||||
private List<String> excludeTags;
|
||||
private Boolean onlyMainContent;
|
||||
private Integer timeout;
|
||||
private Integer waitFor;
|
||||
private Boolean mobile;
|
||||
private List<Object> parsers;
|
||||
private List<Map<String, Object>> actions;
|
||||
private LocationConfig location;
|
||||
private Boolean skipTlsVerification;
|
||||
private Boolean removeBase64Images;
|
||||
private Boolean blockAds;
|
||||
private String proxy;
|
||||
private Long maxAge;
|
||||
private Boolean storeInCache;
|
||||
private Boolean lockdown;
|
||||
private String integration;
|
||||
|
||||
private Builder() {}
|
||||
|
||||
/**
|
||||
* Output formats to request. Accepts strings like "markdown", "html", "rawHtml",
|
||||
* "links", "screenshot", "json", "audio", etc., or format configuration maps/objects for
|
||||
* advanced formats (e.g., JsonFormat, QuestionFormat, HighlightsFormat).
|
||||
*/
|
||||
public Builder formats(List<Object> formats) { this.formats = formats; return this; }
|
||||
|
||||
/** Custom HTTP headers to send with the request. */
|
||||
public Builder headers(Map<String, String> headers) { this.headers = headers; return this; }
|
||||
|
||||
/** Only include content from these HTML tags. */
|
||||
public Builder includeTags(List<String> includeTags) { this.includeTags = includeTags; return this; }
|
||||
|
||||
/** Exclude content from these HTML tags. */
|
||||
public Builder excludeTags(List<String> excludeTags) { this.excludeTags = excludeTags; return this; }
|
||||
|
||||
/** Only return the main content of the page, excluding navbars/footers. */
|
||||
public Builder onlyMainContent(Boolean onlyMainContent) { this.onlyMainContent = onlyMainContent; return this; }
|
||||
|
||||
/** Timeout in milliseconds for the scrape request. */
|
||||
public Builder timeout(Integer timeout) { this.timeout = timeout; return this; }
|
||||
|
||||
/** Wait time in milliseconds before scraping (for JS rendering). */
|
||||
public Builder waitFor(Integer waitFor) { this.waitFor = waitFor; return this; }
|
||||
|
||||
/** Scrape as a mobile device. */
|
||||
public Builder mobile(Boolean mobile) { this.mobile = mobile; return this; }
|
||||
|
||||
/** Parsers to use (e.g., "pdf" or {"type": "pdf", "maxPages": 10}). */
|
||||
public Builder parsers(List<Object> parsers) { this.parsers = parsers; return this; }
|
||||
|
||||
/** Actions to execute before/during scraping. */
|
||||
public Builder actions(List<Map<String, Object>> actions) { this.actions = actions; return this; }
|
||||
|
||||
/** Geolocation configuration. */
|
||||
public Builder location(LocationConfig location) { this.location = location; return this; }
|
||||
|
||||
/** Skip TLS certificate verification. */
|
||||
public Builder skipTlsVerification(Boolean skipTlsVerification) { this.skipTlsVerification = skipTlsVerification; return this; }
|
||||
|
||||
/** Remove base64-encoded images from the response. */
|
||||
public Builder removeBase64Images(Boolean removeBase64Images) { this.removeBase64Images = removeBase64Images; return this; }
|
||||
|
||||
/** Block advertisements during scraping. */
|
||||
public Builder blockAds(Boolean blockAds) { this.blockAds = blockAds; return this; }
|
||||
|
||||
/** Proxy mode: "basic", "stealth", "enhanced", "auto", or a custom proxy URL. */
|
||||
public Builder proxy(String proxy) { this.proxy = proxy; return this; }
|
||||
|
||||
/** Use cached result if younger than this many milliseconds. */
|
||||
public Builder maxAge(Long maxAge) { this.maxAge = maxAge; return this; }
|
||||
|
||||
/** Whether to cache the result. */
|
||||
public Builder storeInCache(Boolean storeInCache) { this.storeInCache = storeInCache; return this; }
|
||||
|
||||
/** Lockdown mode: serve only previously cached results, never make outbound requests. */
|
||||
public Builder lockdown(Boolean lockdown) { this.lockdown = lockdown; return this; }
|
||||
|
||||
/** Integration identifier. */
|
||||
public Builder integration(String integration) { this.integration = integration; return this; }
|
||||
|
||||
public ScrapeOptions build() {
|
||||
ScrapeOptions o = new ScrapeOptions();
|
||||
o.formats = this.formats != null ? Collections.unmodifiableList(new ArrayList<>(this.formats)) : null;
|
||||
o.headers = this.headers != null ? Collections.unmodifiableMap(new HashMap<>(this.headers)) : null;
|
||||
o.includeTags = this.includeTags != null ? Collections.unmodifiableList(new ArrayList<>(this.includeTags)) : null;
|
||||
o.excludeTags = this.excludeTags != null ? Collections.unmodifiableList(new ArrayList<>(this.excludeTags)) : null;
|
||||
o.onlyMainContent = this.onlyMainContent;
|
||||
o.timeout = this.timeout;
|
||||
o.waitFor = this.waitFor;
|
||||
o.mobile = this.mobile;
|
||||
o.parsers = this.parsers != null ? Collections.unmodifiableList(new ArrayList<>(this.parsers)) : null;
|
||||
o.actions = this.actions != null ? Collections.unmodifiableList(new ArrayList<>(this.actions)) : null;
|
||||
o.location = this.location;
|
||||
o.skipTlsVerification = this.skipTlsVerification;
|
||||
o.removeBase64Images = this.removeBase64Images;
|
||||
o.blockAds = this.blockAds;
|
||||
o.proxy = this.proxy;
|
||||
o.maxAge = this.maxAge;
|
||||
o.storeInCache = this.storeInCache;
|
||||
o.lockdown = this.lockdown;
|
||||
o.integration = this.integration;
|
||||
return o;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
package com.firecrawl.models;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* Search results from the v2 search API.
|
||||
* The API returns an object with web, news, and images arrays.
|
||||
*/
|
||||
@JsonIgnoreProperties(ignoreUnknown = true)
|
||||
public class SearchData {
|
||||
|
||||
private List<Map<String, Object>> web;
|
||||
private List<Map<String, Object>> news;
|
||||
private List<Map<String, Object>> images;
|
||||
|
||||
/** Web search results. */
|
||||
public List<Map<String, Object>> getWeb() { return web; }
|
||||
public void setWeb(List<Map<String, Object>> web) { this.web = web; }
|
||||
|
||||
/** News search results. */
|
||||
public List<Map<String, Object>> getNews() { return news; }
|
||||
public void setNews(List<Map<String, Object>> news) { this.news = news; }
|
||||
|
||||
/** Image search results. */
|
||||
public List<Map<String, Object>> getImages() { return images; }
|
||||
public void setImages(List<Map<String, Object>> images) { this.images = images; }
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
int webCount = web != null ? web.size() : 0;
|
||||
int newsCount = news != null ? news.size() : 0;
|
||||
int imageCount = images != null ? images.size() : 0;
|
||||
return "SearchData{web=" + webCount + ", news=" + newsCount + ", images=" + imageCount + "}";
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,94 @@
|
||||
package com.firecrawl.models;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonInclude;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* Options for a web search request.
|
||||
*/
|
||||
@JsonInclude(JsonInclude.Include.NON_NULL)
|
||||
public class SearchOptions {
|
||||
|
||||
private List<Object> sources;
|
||||
private List<Object> categories;
|
||||
private List<String> includeDomains;
|
||||
private List<String> excludeDomains;
|
||||
private Integer limit;
|
||||
private String tbs;
|
||||
private String location;
|
||||
private Boolean ignoreInvalidURLs;
|
||||
private Integer timeout;
|
||||
private ScrapeOptions scrapeOptions;
|
||||
private String integration;
|
||||
|
||||
private SearchOptions() {}
|
||||
|
||||
public List<Object> getSources() { return sources; }
|
||||
public List<Object> getCategories() { return categories; }
|
||||
public List<String> getIncludeDomains() { return includeDomains; }
|
||||
public List<String> getExcludeDomains() { return excludeDomains; }
|
||||
public Integer getLimit() { return limit; }
|
||||
public String getTbs() { return tbs; }
|
||||
public String getLocation() { return location; }
|
||||
public Boolean getIgnoreInvalidURLs() { return ignoreInvalidURLs; }
|
||||
public Integer getTimeout() { return timeout; }
|
||||
public ScrapeOptions getScrapeOptions() { return scrapeOptions; }
|
||||
public String getIntegration() { return integration; }
|
||||
|
||||
public static Builder builder() { return new Builder(); }
|
||||
|
||||
public static final class Builder {
|
||||
private List<Object> sources;
|
||||
private List<Object> categories;
|
||||
private List<String> includeDomains;
|
||||
private List<String> excludeDomains;
|
||||
private Integer limit;
|
||||
private String tbs;
|
||||
private String location;
|
||||
private Boolean ignoreInvalidURLs;
|
||||
private Integer timeout;
|
||||
private ScrapeOptions scrapeOptions;
|
||||
private String integration;
|
||||
|
||||
private Builder() {}
|
||||
|
||||
/** Source types: "web", "news", "images" as strings or {type: "web"} maps. */
|
||||
public Builder sources(List<Object> sources) { this.sources = sources; return this; }
|
||||
/** Categories: "github", "research", "pdf". */
|
||||
public Builder categories(List<Object> categories) { this.categories = categories; return this; }
|
||||
/** Domains to include in search results. */
|
||||
public Builder includeDomains(List<String> includeDomains) { this.includeDomains = includeDomains; return this; }
|
||||
/** Domains to exclude from search results. */
|
||||
public Builder excludeDomains(List<String> excludeDomains) { this.excludeDomains = excludeDomains; return this; }
|
||||
/** Maximum number of results. */
|
||||
public Builder limit(Integer limit) { this.limit = limit; return this; }
|
||||
/** Time-based search filter (e.g., "qdr:d" for past day, "qdr:w" for past week). */
|
||||
public Builder tbs(String tbs) { this.tbs = tbs; return this; }
|
||||
/** Location for search results (e.g., "US"). */
|
||||
public Builder location(String location) { this.location = location; return this; }
|
||||
/** Ignore invalid URLs in results. */
|
||||
public Builder ignoreInvalidURLs(Boolean ignoreInvalidURLs) { this.ignoreInvalidURLs = ignoreInvalidURLs; return this; }
|
||||
/** Timeout in milliseconds. */
|
||||
public Builder timeout(Integer timeout) { this.timeout = timeout; return this; }
|
||||
/** Scrape options applied to search result pages. */
|
||||
public Builder scrapeOptions(ScrapeOptions scrapeOptions) { this.scrapeOptions = scrapeOptions; return this; }
|
||||
/** Integration identifier. */
|
||||
public Builder integration(String integration) { this.integration = integration; return this; }
|
||||
|
||||
public SearchOptions build() {
|
||||
SearchOptions o = new SearchOptions();
|
||||
o.sources = this.sources;
|
||||
o.categories = this.categories;
|
||||
o.includeDomains = this.includeDomains;
|
||||
o.excludeDomains = this.excludeDomains;
|
||||
o.limit = this.limit;
|
||||
o.tbs = this.tbs;
|
||||
o.location = this.location;
|
||||
o.ignoreInvalidURLs = this.ignoreInvalidURLs;
|
||||
o.timeout = this.timeout;
|
||||
o.scrapeOptions = this.scrapeOptions;
|
||||
o.integration = this.integration;
|
||||
return o;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,57 @@
|
||||
package com.firecrawl.models;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonInclude;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* Webhook configuration for async jobs.
|
||||
*/
|
||||
@JsonInclude(JsonInclude.Include.NON_NULL)
|
||||
public class WebhookConfig {
|
||||
|
||||
private String url;
|
||||
private Map<String, String> headers;
|
||||
private Map<String, String> metadata;
|
||||
private List<String> events;
|
||||
|
||||
private WebhookConfig() {}
|
||||
|
||||
public String getUrl() { return url; }
|
||||
public Map<String, String> getHeaders() { return headers; }
|
||||
public Map<String, String> getMetadata() { return metadata; }
|
||||
public List<String> getEvents() { return events; }
|
||||
|
||||
public static Builder builder() { return new Builder(); }
|
||||
|
||||
public static final class Builder {
|
||||
private String url;
|
||||
private Map<String, String> headers;
|
||||
private Map<String, String> metadata;
|
||||
private List<String> events;
|
||||
|
||||
private Builder() {}
|
||||
|
||||
public Builder url(String url) { this.url = url; return this; }
|
||||
public Builder headers(Map<String, String> headers) { this.headers = headers; return this; }
|
||||
public Builder metadata(Map<String, String> metadata) { this.metadata = metadata; return this; }
|
||||
|
||||
/**
|
||||
* Events to subscribe to. Crawl/batch events: "completed", "failed", "page", "started".
|
||||
* Agent events: "started", "action", "completed", "failed", "cancelled".
|
||||
*/
|
||||
public Builder events(List<String> events) { this.events = events; return this; }
|
||||
|
||||
public WebhookConfig build() {
|
||||
if (url == null || url.isEmpty()) {
|
||||
throw new IllegalArgumentException("Webhook URL is required");
|
||||
}
|
||||
WebhookConfig c = new WebhookConfig();
|
||||
c.url = this.url;
|
||||
c.headers = this.headers;
|
||||
c.metadata = this.metadata;
|
||||
c.events = this.events;
|
||||
return c;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
/**
|
||||
* Firecrawl Java SDK — a type-safe client for the Firecrawl v2 web scraping API.
|
||||
*
|
||||
* <p>Quick start:
|
||||
* <pre>{@code
|
||||
* import com.firecrawl.client.FirecrawlClient;
|
||||
* import com.firecrawl.models.*;
|
||||
*
|
||||
* FirecrawlClient client = FirecrawlClient.builder()
|
||||
* .apiKey("fc-your-api-key")
|
||||
* .build();
|
||||
*
|
||||
* Document doc = client.scrape("https://example.com",
|
||||
* ScrapeOptions.builder()
|
||||
* .formats(List.of("markdown"))
|
||||
* .build());
|
||||
*
|
||||
* System.out.println(doc.getMarkdown());
|
||||
* }</pre>
|
||||
*
|
||||
* @see com.firecrawl.client.FirecrawlClient
|
||||
*/
|
||||
package com.firecrawl;
|
||||
@@ -0,0 +1,285 @@
|
||||
package com.firecrawl;
|
||||
|
||||
import com.firecrawl.client.FirecrawlClient;
|
||||
import com.firecrawl.models.AgentOptions;
|
||||
import com.firecrawl.models.AgentResponse;
|
||||
import com.firecrawl.models.AgentStatusResponse;
|
||||
import org.junit.jupiter.api.BeforeAll;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.*;
|
||||
|
||||
/**
|
||||
* Comprehensive Agent Tests
|
||||
*
|
||||
* Tests the AI agent functionality with various configurations.
|
||||
* Based on Node.js SDK patterns and tested against live firecrawl.dev.
|
||||
*
|
||||
* Run with: FIRECRAWL_API_KEY=fc-xxx gradle test --tests "com.firecrawl.AgentTest"
|
||||
*/
|
||||
class AgentTest {
|
||||
|
||||
private static FirecrawlClient client;
|
||||
|
||||
@BeforeAll
|
||||
static void setup() {
|
||||
String apiKey = System.getenv("FIRECRAWL_API_KEY");
|
||||
if (apiKey != null && !apiKey.isBlank()) {
|
||||
client = FirecrawlClient.fromEnv();
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
@EnabledIfEnvironmentVariable(named = "FIRECRAWL_API_KEY", matches = ".*\\S.*")
|
||||
void testAgentWithPrompt() {
|
||||
System.out.println("\n=== Test: Agent with Prompt ===");
|
||||
|
||||
AgentStatusResponse result = client.agent(
|
||||
AgentOptions.builder()
|
||||
.prompt("Find information about Firecrawl's main features and pricing")
|
||||
.build());
|
||||
|
||||
assertNotNull(result, "Agent result should not be null");
|
||||
assertNotNull(result.getStatus(), "Status should not be null");
|
||||
assertTrue(List.of("completed", "failed").contains(result.getStatus()),
|
||||
"Status should be completed or failed: " + result.getStatus());
|
||||
|
||||
System.out.println("✓ Agent task completed");
|
||||
System.out.println(" Status: " + result.getStatus());
|
||||
if (result.getData() != null) {
|
||||
System.out.println(" Data returned: ✓");
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
@EnabledIfEnvironmentVariable(named = "FIRECRAWL_API_KEY", matches = ".*\\S.*")
|
||||
void testAgentWithURLs() {
|
||||
System.out.println("\n=== Test: Agent with Specific URLs ===");
|
||||
|
||||
AgentStatusResponse result = client.agent(
|
||||
AgentOptions.builder()
|
||||
.urls(List.of("https://firecrawl.dev", "https://docs.firecrawl.dev"))
|
||||
.prompt("What are the main features of Firecrawl?")
|
||||
.build());
|
||||
|
||||
assertNotNull(result, "Agent result should not be null");
|
||||
assertTrue(List.of("completed", "failed").contains(result.getStatus()),
|
||||
"Status should be completed or failed");
|
||||
|
||||
System.out.println("✓ Agent with URLs completed");
|
||||
System.out.println(" URLs provided: 2");
|
||||
System.out.println(" Status: " + result.getStatus());
|
||||
}
|
||||
|
||||
@Test
|
||||
@EnabledIfEnvironmentVariable(named = "FIRECRAWL_API_KEY", matches = ".*\\S.*")
|
||||
void testAgentWithSchema() {
|
||||
System.out.println("\n=== Test: Agent with Schema ===");
|
||||
|
||||
Map<String, Object> schema = Map.of(
|
||||
"type", "object",
|
||||
"properties", Map.of(
|
||||
"features", Map.of(
|
||||
"type", "array",
|
||||
"items", Map.of("type", "string")
|
||||
),
|
||||
"pricing", Map.of(
|
||||
"type", "object",
|
||||
"properties", Map.of(
|
||||
"plans", Map.of("type", "array")
|
||||
)
|
||||
)
|
||||
),
|
||||
"required", List.of("features")
|
||||
);
|
||||
|
||||
AgentStatusResponse result = client.agent(
|
||||
AgentOptions.builder()
|
||||
.urls(List.of("https://firecrawl.dev"))
|
||||
.prompt("Extract features and pricing information")
|
||||
.schema(schema)
|
||||
.build());
|
||||
|
||||
assertNotNull(result, "Agent result should not be null");
|
||||
assertTrue(List.of("completed", "failed").contains(result.getStatus()),
|
||||
"Status should be completed or failed");
|
||||
|
||||
System.out.println("✓ Agent with schema completed");
|
||||
System.out.println(" Schema provided: ✓");
|
||||
System.out.println(" Status: " + result.getStatus());
|
||||
}
|
||||
|
||||
@Test
|
||||
@EnabledIfEnvironmentVariable(named = "FIRECRAWL_API_KEY", matches = ".*\\S.*")
|
||||
void testStartAgent() {
|
||||
System.out.println("\n=== Test: Start Agent (Async) ===");
|
||||
|
||||
AgentResponse response = client.startAgent(
|
||||
AgentOptions.builder()
|
||||
.prompt("Research Firecrawl features")
|
||||
.build());
|
||||
|
||||
assertNotNull(response, "Agent response should not be null");
|
||||
assertNotNull(response.getId(), "Agent ID should not be null");
|
||||
assertTrue(response.isSuccess(), "Response should be successful");
|
||||
|
||||
System.out.println("✓ Agent started successfully");
|
||||
System.out.println(" Job ID: " + response.getId());
|
||||
System.out.println(" Success: " + response.isSuccess());
|
||||
}
|
||||
|
||||
@Test
|
||||
@EnabledIfEnvironmentVariable(named = "FIRECRAWL_API_KEY", matches = ".*\\S.*")
|
||||
void testAgentStatusCheck() {
|
||||
System.out.println("\n=== Test: Check Agent Status ===");
|
||||
|
||||
// Start an agent
|
||||
AgentResponse start = client.startAgent(
|
||||
AgentOptions.builder()
|
||||
.prompt("Find information about web scraping")
|
||||
.build());
|
||||
|
||||
// Check status
|
||||
AgentStatusResponse status = client.getAgentStatus(start.getId());
|
||||
|
||||
assertNotNull(status, "Status should not be null");
|
||||
assertNotNull(status.getStatus(), "Status field should not be null");
|
||||
assertTrue(List.of("scraping", "completed", "failed", "cancelled").contains(status.getStatus()),
|
||||
"Status should be valid: " + status.getStatus());
|
||||
|
||||
System.out.println("✓ Agent status retrieved");
|
||||
System.out.println(" Status: " + status.getStatus());
|
||||
System.out.println(" Job ID: " + start.getId());
|
||||
}
|
||||
|
||||
@Test
|
||||
@EnabledIfEnvironmentVariable(named = "FIRECRAWL_API_KEY", matches = ".*\\S.*")
|
||||
void testCancelAgent() {
|
||||
System.out.println("\n=== Test: Cancel Agent ===");
|
||||
|
||||
AgentResponse start = client.startAgent(
|
||||
AgentOptions.builder()
|
||||
.prompt("Long-running research task")
|
||||
.build());
|
||||
|
||||
Map<String, Object> result = client.cancelAgent(start.getId());
|
||||
|
||||
assertNotNull(result, "Cancel result should not be null");
|
||||
|
||||
System.out.println("✓ Agent cancelled successfully");
|
||||
System.out.println(" Job ID: " + start.getId());
|
||||
}
|
||||
|
||||
@Test
|
||||
@EnabledIfEnvironmentVariable(named = "FIRECRAWL_API_KEY", matches = ".*\\S.*")
|
||||
void testAgentWithStrictURLConstraints() {
|
||||
System.out.println("\n=== Test: Agent with Strict URL Constraints ===");
|
||||
|
||||
AgentStatusResponse result = client.agent(
|
||||
AgentOptions.builder()
|
||||
.urls(List.of("https://docs.firecrawl.dev"))
|
||||
.prompt("Extract API documentation structure")
|
||||
.strictConstrainToURLs(true)
|
||||
.build());
|
||||
|
||||
assertNotNull(result, "Agent result should not be null");
|
||||
assertTrue(List.of("completed", "failed").contains(result.getStatus()),
|
||||
"Status should be completed or failed");
|
||||
|
||||
System.out.println("✓ Agent with strict constraints completed");
|
||||
System.out.println(" Strict URL constraint: true");
|
||||
System.out.println(" Status: " + result.getStatus());
|
||||
}
|
||||
|
||||
@Test
|
||||
@EnabledIfEnvironmentVariable(named = "FIRECRAWL_API_KEY", matches = ".*\\S.*")
|
||||
void testAgentWithMaxCredits() {
|
||||
System.out.println("\n=== Test: Agent with Max Credits Limit ===");
|
||||
|
||||
AgentStatusResponse result = client.agent(
|
||||
AgentOptions.builder()
|
||||
.prompt("Quick research on Firecrawl")
|
||||
.maxCredits(10)
|
||||
.build());
|
||||
|
||||
assertNotNull(result, "Agent result should not be null");
|
||||
|
||||
System.out.println("✓ Agent with credit limit completed");
|
||||
System.out.println(" Max credits: 10");
|
||||
System.out.println(" Status: " + result.getStatus());
|
||||
}
|
||||
|
||||
@Test
|
||||
@EnabledIfEnvironmentVariable(named = "FIRECRAWL_API_KEY", matches = ".*\\S.*")
|
||||
void testAgentResearchTask() {
|
||||
System.out.println("\n=== Test: Agent Research - Firecrawl Features ===");
|
||||
|
||||
AgentStatusResponse result = client.agent(
|
||||
AgentOptions.builder()
|
||||
.urls(List.of("https://firecrawl.dev", "https://docs.firecrawl.dev"))
|
||||
.prompt("Research and summarize the key features of Firecrawl, including scraping, crawling, and extraction capabilities")
|
||||
.build());
|
||||
|
||||
assertNotNull(result, "Agent result should not be null");
|
||||
assertEquals("completed", result.getStatus(), "Agent should complete successfully");
|
||||
assertNotNull(result.getData(), "Agent should return data");
|
||||
|
||||
System.out.println("✓ Research task completed");
|
||||
System.out.println(" Status: " + result.getStatus());
|
||||
System.out.println(" Data collected: ✓");
|
||||
|
||||
if (result.getData() != null) {
|
||||
System.out.println(" Data summary: " +
|
||||
result.getData().toString().substring(0,
|
||||
Math.min(200, result.getData().toString().length())) + "...");
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
@EnabledIfEnvironmentVariable(named = "FIRECRAWL_API_KEY", matches = ".*\\S.*")
|
||||
void testAgentComprehensive() {
|
||||
System.out.println("\n=== Test: Agent with All Options ===");
|
||||
|
||||
Map<String, Object> schema = Map.of(
|
||||
"type", "object",
|
||||
"properties", Map.of(
|
||||
"product_name", Map.of("type", "string"),
|
||||
"features", Map.of(
|
||||
"type", "array",
|
||||
"items", Map.of("type", "string")
|
||||
),
|
||||
"pricing", Map.of("type", "string")
|
||||
),
|
||||
"required", List.of("product_name", "features")
|
||||
);
|
||||
|
||||
AgentStatusResponse result = client.agent(
|
||||
AgentOptions.builder()
|
||||
.urls(List.of("https://firecrawl.dev"))
|
||||
.prompt("Extract comprehensive product information including name, features, and pricing")
|
||||
.schema(schema)
|
||||
.maxCredits(20)
|
||||
.strictConstrainToURLs(true)
|
||||
.build());
|
||||
|
||||
assertNotNull(result, "Agent result should not be null");
|
||||
assertTrue(List.of("completed", "failed").contains(result.getStatus()),
|
||||
"Status should be completed or failed");
|
||||
|
||||
System.out.println("✓ Comprehensive agent task completed");
|
||||
System.out.println(" Configuration:");
|
||||
System.out.println(" - URLs: 1");
|
||||
System.out.println(" - Schema: ✓");
|
||||
System.out.println(" - Max credits: 20");
|
||||
System.out.println(" - Strict constraints: true");
|
||||
System.out.println(" Results:");
|
||||
System.out.println(" - Status: " + result.getStatus());
|
||||
if (result.getData() != null) {
|
||||
System.out.println(" - Data returned: ✓");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,310 @@
|
||||
package com.firecrawl;
|
||||
|
||||
import com.firecrawl.client.FirecrawlClient;
|
||||
import com.firecrawl.models.BrowserCreateResponse;
|
||||
import com.firecrawl.models.BrowserDeleteResponse;
|
||||
import com.firecrawl.models.BrowserExecuteResponse;
|
||||
import com.firecrawl.models.BrowserListResponse;
|
||||
import org.junit.jupiter.api.BeforeAll;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.*;
|
||||
|
||||
/**
|
||||
* Browser Sandbox Endpoint Tests
|
||||
*
|
||||
* Tests the browser session management functionality of the Firecrawl Java SDK.
|
||||
* These tests require FIRECRAWL_API_KEY environment variable to be set.
|
||||
*
|
||||
* Run with: FIRECRAWL_API_KEY=fc-xxx gradle test --tests "com.firecrawl.BrowserTest"
|
||||
*/
|
||||
class BrowserTest {
|
||||
|
||||
private static FirecrawlClient client;
|
||||
|
||||
@BeforeAll
|
||||
static void setup() {
|
||||
String apiKey = System.getenv("FIRECRAWL_API_KEY");
|
||||
if (apiKey != null && !apiKey.isBlank()) {
|
||||
client = FirecrawlClient.fromEnv();
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
@EnabledIfEnvironmentVariable(named = "FIRECRAWL_API_KEY", matches = ".*\\S.*")
|
||||
void testBrowserCreateAndDelete() {
|
||||
System.out.println("Testing browser session create and delete...");
|
||||
|
||||
// Create a browser session
|
||||
BrowserCreateResponse createRes = client.browser();
|
||||
assertNotNull(createRes, "Create response should not be null");
|
||||
assertTrue(createRes.isSuccess(), "Create should succeed");
|
||||
assertNotNull(createRes.getId(), "Session ID should not be null");
|
||||
|
||||
String sessionId = createRes.getId();
|
||||
System.out.println(" Created session: " + sessionId);
|
||||
|
||||
// Delete the browser session
|
||||
BrowserDeleteResponse deleteRes = client.deleteBrowser(sessionId);
|
||||
assertNotNull(deleteRes, "Delete response should not be null");
|
||||
assertTrue(deleteRes.isSuccess(), "Delete should succeed");
|
||||
|
||||
System.out.println("✓ Browser create and delete test passed");
|
||||
}
|
||||
|
||||
@Test
|
||||
@EnabledIfEnvironmentVariable(named = "FIRECRAWL_API_KEY", matches = ".*\\S.*")
|
||||
void testBrowserCreateWithOptions() {
|
||||
System.out.println("Testing browser session create with options...");
|
||||
|
||||
// Create a session with custom TTL and activity TTL
|
||||
BrowserCreateResponse createRes = client.browser(300, 120, true);
|
||||
assertNotNull(createRes, "Create response should not be null");
|
||||
assertTrue(createRes.isSuccess(), "Create should succeed");
|
||||
assertNotNull(createRes.getId(), "Session ID should not be null");
|
||||
|
||||
String sessionId = createRes.getId();
|
||||
System.out.println(" Created session with options: " + sessionId);
|
||||
|
||||
// Clean up
|
||||
client.deleteBrowser(sessionId);
|
||||
|
||||
System.out.println("✓ Browser create with options test passed");
|
||||
}
|
||||
|
||||
@Test
|
||||
@EnabledIfEnvironmentVariable(named = "FIRECRAWL_API_KEY", matches = ".*\\S.*")
|
||||
void testBrowserExecuteBash() {
|
||||
System.out.println("Testing browser execute with bash...");
|
||||
|
||||
// Create a session
|
||||
BrowserCreateResponse createRes = client.browser();
|
||||
assertTrue(createRes.isSuccess(), "Create should succeed");
|
||||
String sessionId = createRes.getId();
|
||||
|
||||
try {
|
||||
// Execute bash code
|
||||
BrowserExecuteResponse execRes = client.browserExecute(sessionId, "echo 'hello from java sdk'");
|
||||
assertNotNull(execRes, "Execute response should not be null");
|
||||
assertTrue(execRes.isSuccess(), "Execute should succeed");
|
||||
assertNotNull(execRes.getStdout(), "Stdout should not be null");
|
||||
assertTrue(execRes.getStdout().contains("hello from java sdk"),
|
||||
"Stdout should contain our echo output");
|
||||
|
||||
System.out.println(" Stdout: " + execRes.getStdout().trim());
|
||||
System.out.println(" Exit code: " + execRes.getExitCode());
|
||||
} finally {
|
||||
// Clean up
|
||||
client.deleteBrowser(sessionId);
|
||||
}
|
||||
|
||||
System.out.println("✓ Browser execute bash test passed");
|
||||
}
|
||||
|
||||
@Test
|
||||
@EnabledIfEnvironmentVariable(named = "FIRECRAWL_API_KEY", matches = ".*\\S.*")
|
||||
void testBrowserExecuteNode() {
|
||||
System.out.println("Testing browser execute with node...");
|
||||
|
||||
// Create a session
|
||||
BrowserCreateResponse createRes = client.browser();
|
||||
assertTrue(createRes.isSuccess(), "Create should succeed");
|
||||
String sessionId = createRes.getId();
|
||||
|
||||
try {
|
||||
// Execute node code
|
||||
BrowserExecuteResponse execRes = client.browserExecute(
|
||||
sessionId, "console.log(1 + 2)", "node", null);
|
||||
assertNotNull(execRes, "Execute response should not be null");
|
||||
assertTrue(execRes.isSuccess(), "Execute should succeed");
|
||||
|
||||
System.out.println(" Stdout: " + (execRes.getStdout() != null ? execRes.getStdout().trim() : "null"));
|
||||
} finally {
|
||||
client.deleteBrowser(sessionId);
|
||||
}
|
||||
|
||||
System.out.println("✓ Browser execute node test passed");
|
||||
}
|
||||
|
||||
@Test
|
||||
@EnabledIfEnvironmentVariable(named = "FIRECRAWL_API_KEY", matches = ".*\\S.*")
|
||||
void testBrowserExecutePython() {
|
||||
System.out.println("Testing browser execute with python...");
|
||||
|
||||
// Create a session
|
||||
BrowserCreateResponse createRes = client.browser();
|
||||
assertTrue(createRes.isSuccess(), "Create should succeed");
|
||||
String sessionId = createRes.getId();
|
||||
|
||||
try {
|
||||
// Execute python code
|
||||
BrowserExecuteResponse execRes = client.browserExecute(
|
||||
sessionId, "print('hello from python')", "python", null);
|
||||
assertNotNull(execRes, "Execute response should not be null");
|
||||
assertTrue(execRes.isSuccess(), "Execute should succeed");
|
||||
|
||||
System.out.println(" Stdout: " + (execRes.getStdout() != null ? execRes.getStdout().trim() : "null"));
|
||||
} finally {
|
||||
client.deleteBrowser(sessionId);
|
||||
}
|
||||
|
||||
System.out.println("✓ Browser execute python test passed");
|
||||
}
|
||||
|
||||
@Test
|
||||
@EnabledIfEnvironmentVariable(named = "FIRECRAWL_API_KEY", matches = ".*\\S.*")
|
||||
void testBrowserExecuteWithTimeout() {
|
||||
System.out.println("Testing browser execute with custom timeout...");
|
||||
|
||||
// Create a session
|
||||
BrowserCreateResponse createRes = client.browser();
|
||||
assertTrue(createRes.isSuccess(), "Create should succeed");
|
||||
String sessionId = createRes.getId();
|
||||
|
||||
try {
|
||||
// Execute with custom timeout (60 seconds)
|
||||
BrowserExecuteResponse execRes = client.browserExecute(
|
||||
sessionId, "echo 'timeout test'", "bash", 60);
|
||||
assertNotNull(execRes, "Execute response should not be null");
|
||||
assertTrue(execRes.isSuccess(), "Execute should succeed");
|
||||
|
||||
System.out.println(" Stdout: " + (execRes.getStdout() != null ? execRes.getStdout().trim() : "null"));
|
||||
} finally {
|
||||
client.deleteBrowser(sessionId);
|
||||
}
|
||||
|
||||
System.out.println("✓ Browser execute with timeout test passed");
|
||||
}
|
||||
|
||||
@Test
|
||||
@EnabledIfEnvironmentVariable(named = "FIRECRAWL_API_KEY", matches = ".*\\S.*")
|
||||
void testBrowserListSessions() {
|
||||
System.out.println("Testing list browser sessions...");
|
||||
|
||||
// List all sessions
|
||||
BrowserListResponse listRes = client.listBrowsers();
|
||||
assertNotNull(listRes, "List response should not be null");
|
||||
assertTrue(listRes.isSuccess(), "List should succeed");
|
||||
|
||||
System.out.println(" Total sessions: " + (listRes.getSessions() != null ? listRes.getSessions().size() : 0));
|
||||
|
||||
System.out.println("✓ List browser sessions test passed");
|
||||
}
|
||||
|
||||
@Test
|
||||
@EnabledIfEnvironmentVariable(named = "FIRECRAWL_API_KEY", matches = ".*\\S.*")
|
||||
void testBrowserListActiveFilter() {
|
||||
System.out.println("Testing list browser sessions with active filter...");
|
||||
|
||||
// Create a session so we have at least one active
|
||||
BrowserCreateResponse createRes = client.browser();
|
||||
assertTrue(createRes.isSuccess(), "Create should succeed");
|
||||
String sessionId = createRes.getId();
|
||||
|
||||
try {
|
||||
// List only active sessions
|
||||
BrowserListResponse listRes = client.listBrowsers("active");
|
||||
assertNotNull(listRes, "List response should not be null");
|
||||
assertTrue(listRes.isSuccess(), "List should succeed");
|
||||
assertNotNull(listRes.getSessions(), "Sessions list should not be null");
|
||||
assertFalse(listRes.getSessions().isEmpty(), "Should have at least one active session");
|
||||
|
||||
System.out.println(" Active sessions: " + listRes.getSessions().size());
|
||||
} finally {
|
||||
client.deleteBrowser(sessionId);
|
||||
}
|
||||
|
||||
System.out.println("✓ List active browser sessions test passed");
|
||||
}
|
||||
|
||||
@Test
|
||||
@EnabledIfEnvironmentVariable(named = "FIRECRAWL_API_KEY", matches = ".*\\S.*")
|
||||
void testBrowserFullLifecycle() {
|
||||
System.out.println("Testing full browser session lifecycle...");
|
||||
|
||||
// 1. Create session
|
||||
BrowserCreateResponse createRes = client.browser(300, 120, true);
|
||||
assertTrue(createRes.isSuccess(), "Create should succeed");
|
||||
assertNotNull(createRes.getId(), "Should have session ID");
|
||||
String sessionId = createRes.getId();
|
||||
System.out.println(" 1. Created session: " + sessionId);
|
||||
|
||||
// CDP URL and live view URL may be present
|
||||
if (createRes.getCdpUrl() != null) {
|
||||
System.out.println(" CDP URL present: true");
|
||||
}
|
||||
if (createRes.getLiveViewUrl() != null) {
|
||||
System.out.println(" Live View URL present: true");
|
||||
}
|
||||
|
||||
// 2. Navigate to a page
|
||||
BrowserExecuteResponse navRes = client.browserExecute(
|
||||
sessionId, "agent-browser open https://example.com", "bash", 30);
|
||||
assertTrue(navRes.isSuccess(), "Navigation should succeed");
|
||||
System.out.println(" 2. Navigated to example.com");
|
||||
|
||||
// 3. Take a snapshot
|
||||
BrowserExecuteResponse snapRes = client.browserExecute(
|
||||
sessionId, "agent-browser snapshot -i -c", "bash", 30);
|
||||
assertTrue(snapRes.isSuccess(), "Snapshot should succeed");
|
||||
System.out.println(" 3. Took snapshot");
|
||||
|
||||
// 4. Get page title
|
||||
BrowserExecuteResponse titleRes = client.browserExecute(
|
||||
sessionId, "agent-browser get title", "bash", 30);
|
||||
assertTrue(titleRes.isSuccess(), "Get title should succeed");
|
||||
System.out.println(" 4. Page title: " + (titleRes.getStdout() != null ? titleRes.getStdout().trim() : "null"));
|
||||
|
||||
// 5. Verify session is active
|
||||
BrowserListResponse listRes = client.listBrowsers("active");
|
||||
assertTrue(listRes.isSuccess(), "List should succeed");
|
||||
System.out.println(" 5. Active sessions: " + (listRes.getSessions() != null ? listRes.getSessions().size() : 0));
|
||||
|
||||
// 6. Delete session
|
||||
BrowserDeleteResponse deleteRes = client.deleteBrowser(sessionId);
|
||||
assertTrue(deleteRes.isSuccess(), "Delete should succeed");
|
||||
System.out.println(" 6. Deleted session");
|
||||
if (deleteRes.getSessionDurationMs() != null) {
|
||||
System.out.println(" Session duration: " + deleteRes.getSessionDurationMs() + "ms");
|
||||
}
|
||||
if (deleteRes.getCreditsBilled() != null) {
|
||||
System.out.println(" Credits billed: " + deleteRes.getCreditsBilled());
|
||||
}
|
||||
|
||||
System.out.println("✓ Full browser session lifecycle test passed");
|
||||
}
|
||||
|
||||
@Test
|
||||
void testBrowserExecuteRequiresSessionId() {
|
||||
FirecrawlClient testClient = FirecrawlClient.builder()
|
||||
.apiKey("fc-test-key")
|
||||
.build();
|
||||
|
||||
assertThrows(NullPointerException.class, () ->
|
||||
testClient.browserExecute(null, "echo test")
|
||||
);
|
||||
}
|
||||
|
||||
@Test
|
||||
void testBrowserExecuteRequiresCode() {
|
||||
FirecrawlClient testClient = FirecrawlClient.builder()
|
||||
.apiKey("fc-test-key")
|
||||
.build();
|
||||
|
||||
assertThrows(NullPointerException.class, () ->
|
||||
testClient.browserExecute("some-session-id", null)
|
||||
);
|
||||
}
|
||||
|
||||
@Test
|
||||
void testBrowserDeleteRequiresSessionId() {
|
||||
FirecrawlClient testClient = FirecrawlClient.builder()
|
||||
.apiKey("fc-test-key")
|
||||
.build();
|
||||
|
||||
assertThrows(NullPointerException.class, () ->
|
||||
testClient.deleteBrowser(null)
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,302 @@
|
||||
package com.firecrawl;
|
||||
|
||||
import com.firecrawl.client.FirecrawlClient;
|
||||
import com.firecrawl.errors.FirecrawlException;
|
||||
import com.firecrawl.models.*;
|
||||
import org.junit.jupiter.api.BeforeAll;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.*;
|
||||
|
||||
/**
|
||||
* Comprehensive Crawl Tests
|
||||
*
|
||||
* Tests the crawl functionality with various configurations.
|
||||
* Based on Node.js SDK patterns and tested against live firecrawl.dev.
|
||||
*
|
||||
* Run with: FIRECRAWL_API_KEY=fc-xxx gradle test --tests "com.firecrawl.CrawlTest"
|
||||
*/
|
||||
class CrawlTest {
|
||||
|
||||
private static FirecrawlClient client;
|
||||
|
||||
@BeforeAll
|
||||
static void setup() {
|
||||
String apiKey = System.getenv("FIRECRAWL_API_KEY");
|
||||
if (apiKey != null && !apiKey.isBlank()) {
|
||||
client = FirecrawlClient.fromEnv();
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
@EnabledIfEnvironmentVariable(named = "FIRECRAWL_API_KEY", matches = ".*\\S.*")
|
||||
void testStartCrawlMinimal() {
|
||||
System.out.println("\n=== Test: Start Crawl - Minimal Request ===");
|
||||
|
||||
CrawlResponse response = client.startCrawl("https://docs.firecrawl.dev",
|
||||
CrawlOptions.builder()
|
||||
.limit(3)
|
||||
.build());
|
||||
|
||||
assertNotNull(response, "Crawl response should not be null");
|
||||
assertNotNull(response.getId(), "Crawl ID should not be null");
|
||||
assertNotNull(response.getUrl(), "Crawl URL should not be null");
|
||||
|
||||
System.out.println("✓ Crawl started successfully");
|
||||
System.out.println(" Job ID: " + response.getId());
|
||||
System.out.println(" Status URL: " + response.getUrl());
|
||||
}
|
||||
|
||||
@Test
|
||||
@EnabledIfEnvironmentVariable(named = "FIRECRAWL_API_KEY", matches = ".*\\S.*")
|
||||
void testStartCrawlWithOptions() {
|
||||
System.out.println("\n=== Test: Start Crawl - With Options ===");
|
||||
|
||||
CrawlResponse response = client.startCrawl("https://docs.firecrawl.dev",
|
||||
CrawlOptions.builder()
|
||||
.limit(5)
|
||||
.maxDiscoveryDepth(2)
|
||||
.build());
|
||||
|
||||
assertNotNull(response.getId(), "Job ID should not be null");
|
||||
assertNotNull(response.getUrl(), "Status URL should not be null");
|
||||
|
||||
System.out.println("✓ Crawl with options started");
|
||||
System.out.println(" Limit: 5 pages");
|
||||
System.out.println(" Max depth: 2");
|
||||
System.out.println(" Job ID: " + response.getId());
|
||||
}
|
||||
|
||||
@Test
|
||||
@EnabledIfEnvironmentVariable(named = "FIRECRAWL_API_KEY", matches = ".*\\S.*")
|
||||
void testGetCrawlStatus() {
|
||||
System.out.println("\n=== Test: Get Crawl Status ===");
|
||||
|
||||
// Start a crawl
|
||||
CrawlResponse start = client.startCrawl("https://docs.firecrawl.dev",
|
||||
CrawlOptions.builder()
|
||||
.limit(3)
|
||||
.build());
|
||||
|
||||
System.out.println("CrawlResponse: " + start);
|
||||
System.out.println("ID: " + start.getId());
|
||||
assertNotNull(start, "CrawlResponse should not be null");
|
||||
assertNotNull(start.getId(), "Crawl ID should not be null");
|
||||
|
||||
// Get status
|
||||
CrawlJob status = client.getCrawlStatus(start.getId());
|
||||
|
||||
assertNotNull(status, "Status should not be null");
|
||||
assertNotNull(status.getStatus(), "Status should not be null");
|
||||
assertTrue(List.of("scraping", "completed", "failed", "cancelled").contains(status.getStatus()),
|
||||
"Status should be valid: " + status.getStatus());
|
||||
assertTrue(status.getCompleted() >= 0, "Completed count should be non-negative");
|
||||
// Data may be null while crawl is still in progress (status=scraping)
|
||||
if ("completed".equals(status.getStatus())) {
|
||||
assertNotNull(status.getData(), "Data should not be null when completed");
|
||||
}
|
||||
|
||||
System.out.println("✓ Status retrieved successfully");
|
||||
System.out.println(" Status: " + status.getStatus());
|
||||
System.out.println(" Completed: " + status.getCompleted() + "/" + status.getTotal());
|
||||
System.out.println(" Documents: " + (status.getData() != null ? status.getData().size() : 0));
|
||||
}
|
||||
|
||||
@Test
|
||||
@EnabledIfEnvironmentVariable(named = "FIRECRAWL_API_KEY", matches = ".*\\S.*")
|
||||
void testCancelCrawl() {
|
||||
System.out.println("\n=== Test: Cancel Crawl ===");
|
||||
|
||||
CrawlResponse start = client.startCrawl("https://docs.firecrawl.dev",
|
||||
CrawlOptions.builder()
|
||||
.limit(10)
|
||||
.build());
|
||||
|
||||
Map<String, Object> result = client.cancelCrawl(start.getId());
|
||||
|
||||
assertNotNull(result, "Cancel result should not be null");
|
||||
|
||||
System.out.println("✓ Crawl cancelled successfully");
|
||||
System.out.println(" Job ID: " + start.getId());
|
||||
}
|
||||
|
||||
@Test
|
||||
@EnabledIfEnvironmentVariable(named = "FIRECRAWL_API_KEY", matches = ".*\\S.*")
|
||||
void testCrawlWithWait() {
|
||||
System.out.println("\n=== Test: Crawl with Wait (Blocking) ===");
|
||||
|
||||
CrawlJob job = client.crawl("https://firecrawl.dev",
|
||||
CrawlOptions.builder()
|
||||
.limit(3)
|
||||
.maxDiscoveryDepth(1)
|
||||
.build(),
|
||||
2, // pollInterval in seconds
|
||||
120 // timeout in seconds
|
||||
);
|
||||
|
||||
assertNotNull(job, "Job should not be null");
|
||||
assertTrue(List.of("completed", "failed").contains(job.getStatus()),
|
||||
"Final status should be completed or failed: " + job.getStatus());
|
||||
assertTrue(job.getCompleted() >= 0, "Completed count should be non-negative");
|
||||
assertTrue(job.getTotal() >= 0, "Total count should be non-negative");
|
||||
assertNotNull(job.getData(), "Data should not be null");
|
||||
|
||||
System.out.println("✓ Crawl completed (with wait)");
|
||||
System.out.println(" Final status: " + job.getStatus());
|
||||
System.out.println(" Pages crawled: " + job.getCompleted() + "/" + job.getTotal());
|
||||
System.out.println(" Documents returned: " + job.getData().size());
|
||||
|
||||
if (!job.getData().isEmpty()) {
|
||||
Document firstDoc = job.getData().get(0);
|
||||
System.out.println(" Sample URL: " + firstDoc.getMetadata().get("sourceURL"));
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
@EnabledIfEnvironmentVariable(named = "FIRECRAWL_API_KEY", matches = ".*\\S.*")
|
||||
void testCrawlWithScrapeOptions() {
|
||||
System.out.println("\n=== Test: Crawl with Scrape Options ===");
|
||||
|
||||
CrawlResponse response = client.startCrawl("https://docs.firecrawl.dev",
|
||||
CrawlOptions.builder()
|
||||
.limit(2)
|
||||
.scrapeOptions(ScrapeOptions.builder()
|
||||
.formats(List.of("markdown", "links"))
|
||||
.onlyMainContent(true)
|
||||
.build())
|
||||
.build());
|
||||
|
||||
assertNotNull(response.getId(), "Job ID should not be null");
|
||||
|
||||
System.out.println("✓ Crawl with scrape options started");
|
||||
System.out.println(" Formats: markdown, links");
|
||||
System.out.println(" Only main content: true");
|
||||
System.out.println(" Job ID: " + response.getId());
|
||||
}
|
||||
|
||||
@Test
|
||||
@EnabledIfEnvironmentVariable(named = "FIRECRAWL_API_KEY", matches = ".*\\S.*")
|
||||
void testCrawlWithExcludePaths() {
|
||||
System.out.println("\n=== Test: Crawl with Exclude Paths ===");
|
||||
|
||||
CrawlResponse response = client.startCrawl("https://docs.firecrawl.dev",
|
||||
CrawlOptions.builder()
|
||||
.limit(5)
|
||||
.excludePaths(List.of("/blog/*", "/admin/*"))
|
||||
.build());
|
||||
|
||||
assertNotNull(response.getId(), "Job ID should not be null");
|
||||
|
||||
System.out.println("✓ Crawl with exclude paths started");
|
||||
System.out.println(" Excluding: /blog/*, /admin/*");
|
||||
System.out.println(" Job ID: " + response.getId());
|
||||
}
|
||||
|
||||
@Test
|
||||
@EnabledIfEnvironmentVariable(named = "FIRECRAWL_API_KEY", matches = ".*\\S.*")
|
||||
void testCrawlWithIncludePaths() {
|
||||
System.out.println("\n=== Test: Crawl with Include Paths ===");
|
||||
|
||||
CrawlResponse response = client.startCrawl("https://docs.firecrawl.dev",
|
||||
CrawlOptions.builder()
|
||||
.limit(5)
|
||||
.includePaths(List.of("/docs/*"))
|
||||
.build());
|
||||
|
||||
assertNotNull(response.getId(), "Job ID should not be null");
|
||||
|
||||
System.out.println("✓ Crawl with include paths started");
|
||||
System.out.println(" Including only: /docs/*");
|
||||
System.out.println(" Job ID: " + response.getId());
|
||||
}
|
||||
|
||||
@Test
|
||||
@EnabledIfEnvironmentVariable(named = "FIRECRAWL_API_KEY", matches = ".*\\S.*")
|
||||
void testCrawlWithAllowExternalLinks() {
|
||||
System.out.println("\n=== Test: Crawl with Allow External Links ===");
|
||||
|
||||
CrawlResponse response = client.startCrawl("https://docs.firecrawl.dev",
|
||||
CrawlOptions.builder()
|
||||
.limit(5)
|
||||
.allowExternalLinks(true)
|
||||
.build());
|
||||
|
||||
assertNotNull(response.getId(), "Job ID should not be null");
|
||||
|
||||
System.out.println("✓ Crawl with external links allowed");
|
||||
System.out.println(" Job ID: " + response.getId());
|
||||
}
|
||||
|
||||
@Test
|
||||
@EnabledIfEnvironmentVariable(named = "FIRECRAWL_API_KEY", matches = ".*\\S.*")
|
||||
void testCrawlWithWebhookConfig() {
|
||||
System.out.println("\n=== Test: Crawl with Webhook (if available) ===");
|
||||
|
||||
try {
|
||||
// Using a test webhook URL (requestbin, webhook.site, etc.)
|
||||
CrawlResponse response = client.startCrawl("https://firecrawl.dev",
|
||||
CrawlOptions.builder()
|
||||
.limit(2)
|
||||
.webhook(WebhookConfig.builder()
|
||||
.url("https://webhook.site/test")
|
||||
.build())
|
||||
.build());
|
||||
|
||||
assertNotNull(response.getId(), "Job ID should not be null");
|
||||
|
||||
System.out.println("✓ Crawl with webhook started");
|
||||
System.out.println(" Job ID: " + response.getId());
|
||||
} catch (Exception e) {
|
||||
System.out.println("⚠ Webhook test skipped or failed: " + e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
@EnabledIfEnvironmentVariable(named = "FIRECRAWL_API_KEY", matches = ".*\\S.*")
|
||||
void testCrawlFirecrawlHomepage() {
|
||||
System.out.println("\n=== Test: Crawl Firecrawl.dev Homepage ===");
|
||||
|
||||
CrawlJob job = client.crawl("https://firecrawl.dev",
|
||||
CrawlOptions.builder()
|
||||
.limit(5)
|
||||
.maxDiscoveryDepth(2)
|
||||
.scrapeOptions(ScrapeOptions.builder()
|
||||
.formats(List.of("markdown"))
|
||||
.onlyMainContent(true)
|
||||
.build())
|
||||
.build(),
|
||||
2,
|
||||
120
|
||||
);
|
||||
|
||||
assertNotNull(job, "Job should not be null");
|
||||
assertTrue(job.getData() != null && !job.getData().isEmpty(),
|
||||
"Should have crawled at least one page");
|
||||
|
||||
// Verify content from Firecrawl site
|
||||
boolean hasFirecrawlContent = job.getData().stream()
|
||||
.anyMatch(doc -> {
|
||||
String markdown = doc.getMarkdown();
|
||||
return markdown != null &&
|
||||
(markdown.toLowerCase().contains("firecrawl") ||
|
||||
markdown.toLowerCase().contains("scrape") ||
|
||||
markdown.toLowerCase().contains("crawl"));
|
||||
});
|
||||
|
||||
assertTrue(hasFirecrawlContent, "Should contain Firecrawl-related content");
|
||||
|
||||
System.out.println("✓ Successfully crawled Firecrawl homepage");
|
||||
System.out.println(" Pages crawled: " + job.getData().size());
|
||||
System.out.println(" Status: " + job.getStatus());
|
||||
|
||||
// Print sample URLs
|
||||
System.out.println(" Sample pages:");
|
||||
job.getData().stream()
|
||||
.limit(3)
|
||||
.forEach(doc -> System.out.println(" - " + doc.getMetadata().get("sourceURL")));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,330 @@
|
||||
package com.firecrawl;
|
||||
|
||||
import com.firecrawl.client.FirecrawlClient;
|
||||
import com.firecrawl.errors.FirecrawlException;
|
||||
import com.firecrawl.models.*;
|
||||
import okhttp3.OkHttpClient;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable;
|
||||
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.*;
|
||||
|
||||
/**
|
||||
* Integration tests for the Firecrawl Java SDK.
|
||||
*
|
||||
* <p>These tests require a valid FIRECRAWL_API_KEY environment variable.
|
||||
* Run with: FIRECRAWL_API_KEY=fc-xxx ./gradlew test
|
||||
*/
|
||||
class FirecrawlClientTest {
|
||||
|
||||
@Test
|
||||
void testBuilderRequiresApiKey() {
|
||||
assertThrows(FirecrawlException.class, () ->
|
||||
FirecrawlClient.builder().apiKey("").build()
|
||||
);
|
||||
}
|
||||
|
||||
@Test
|
||||
void testBuilderRejectsExplicitNullApiKey() {
|
||||
assertThrows(FirecrawlException.class, () ->
|
||||
FirecrawlClient.builder().apiKey(null).build()
|
||||
);
|
||||
}
|
||||
|
||||
@Test
|
||||
void testBuilderAcceptsApiKey() {
|
||||
// Should not throw — just validates construction
|
||||
FirecrawlClient client = FirecrawlClient.builder()
|
||||
.apiKey("fc-test-key")
|
||||
.build();
|
||||
assertNotNull(client);
|
||||
}
|
||||
|
||||
@Test
|
||||
void testBuilderAcceptsCustomHttpClient() {
|
||||
OkHttpClient custom = new OkHttpClient.Builder()
|
||||
.connectTimeout(10, TimeUnit.SECONDS)
|
||||
.readTimeout(10, TimeUnit.SECONDS)
|
||||
.build();
|
||||
|
||||
FirecrawlClient client = FirecrawlClient.builder()
|
||||
.apiKey("fc-test-key")
|
||||
.httpClient(custom)
|
||||
.build();
|
||||
assertNotNull(client);
|
||||
}
|
||||
|
||||
@Test
|
||||
void testScrapeOptionsBuilder() {
|
||||
QueryFormat queryFormat = QueryFormat.builder()
|
||||
.prompt("What is Firecrawl?")
|
||||
.mode(QueryFormat.Mode.DIRECT_QUOTE)
|
||||
.build();
|
||||
|
||||
ScrapeOptions options = ScrapeOptions.builder()
|
||||
.formats(List.of("markdown", "html", queryFormat))
|
||||
.onlyMainContent(true)
|
||||
.timeout(30000)
|
||||
.mobile(false)
|
||||
.build();
|
||||
|
||||
assertEquals(List.of("markdown", "html", queryFormat), options.getFormats());
|
||||
assertEquals("query", queryFormat.getType());
|
||||
assertEquals(QueryFormat.Mode.DIRECT_QUOTE, queryFormat.getMode());
|
||||
assertTrue(options.getOnlyMainContent());
|
||||
assertEquals(30000, options.getTimeout());
|
||||
assertFalse(options.getMobile());
|
||||
}
|
||||
|
||||
@Test
|
||||
void testQuestionAndHighlightsFormats() {
|
||||
QuestionFormat questionFormat = QuestionFormat.builder()
|
||||
.question("What is Firecrawl?")
|
||||
.build();
|
||||
HighlightsFormat highlightsFormat = HighlightsFormat.builder()
|
||||
.query("What is Firecrawl?")
|
||||
.build();
|
||||
|
||||
ScrapeOptions options = ScrapeOptions.builder()
|
||||
.formats(List.of(questionFormat, highlightsFormat))
|
||||
.build();
|
||||
|
||||
assertEquals(List.of(questionFormat, highlightsFormat), options.getFormats());
|
||||
assertEquals("question", questionFormat.getType());
|
||||
assertEquals("What is Firecrawl?", questionFormat.getQuestion());
|
||||
assertEquals("highlights", highlightsFormat.getType());
|
||||
assertEquals("What is Firecrawl?", highlightsFormat.getQuery());
|
||||
}
|
||||
|
||||
@Test
|
||||
void testCrawlOptionsBuilder() {
|
||||
CrawlOptions options = CrawlOptions.builder()
|
||||
.limit(100)
|
||||
.maxDiscoveryDepth(3)
|
||||
.sitemap("include")
|
||||
.excludePaths(List.of("/admin/*"))
|
||||
.build();
|
||||
|
||||
assertEquals(100, options.getLimit());
|
||||
assertEquals(3, options.getMaxDiscoveryDepth());
|
||||
assertEquals("include", options.getSitemap());
|
||||
assertEquals(List.of("/admin/*"), options.getExcludePaths());
|
||||
}
|
||||
|
||||
@Test
|
||||
void testAgentOptionsRequiresPrompt() {
|
||||
assertThrows(IllegalArgumentException.class, () ->
|
||||
AgentOptions.builder().build()
|
||||
);
|
||||
}
|
||||
|
||||
@Test
|
||||
void testWebhookConfigRequiresUrl() {
|
||||
assertThrows(IllegalArgumentException.class, () ->
|
||||
WebhookConfig.builder().build()
|
||||
);
|
||||
}
|
||||
|
||||
@Test
|
||||
void testScrapeOptionsToBuilder() {
|
||||
ScrapeOptions original = ScrapeOptions.builder()
|
||||
.formats(List.of("markdown"))
|
||||
.timeout(5000)
|
||||
.build();
|
||||
|
||||
ScrapeOptions modified = original.toBuilder()
|
||||
.timeout(10000)
|
||||
.build();
|
||||
|
||||
assertEquals(5000, original.getTimeout());
|
||||
assertEquals(10000, modified.getTimeout());
|
||||
assertEquals(List.of("markdown"), modified.getFormats());
|
||||
}
|
||||
|
||||
@Test
|
||||
void testBrowserExecuteRequiresSessionId() {
|
||||
FirecrawlClient client = FirecrawlClient.builder()
|
||||
.apiKey("fc-test-key")
|
||||
.build();
|
||||
assertThrows(NullPointerException.class, () ->
|
||||
client.browserExecute(null, "echo test")
|
||||
);
|
||||
}
|
||||
|
||||
@Test
|
||||
void testInteractRequiresJobId() {
|
||||
FirecrawlClient client = FirecrawlClient.builder()
|
||||
.apiKey("fc-test-key")
|
||||
.build();
|
||||
assertThrows(NullPointerException.class, () ->
|
||||
client.interact(null, "console.log('hi')")
|
||||
);
|
||||
}
|
||||
|
||||
@Test
|
||||
void testInteractRequiresCode() {
|
||||
FirecrawlClient client = FirecrawlClient.builder()
|
||||
.apiKey("fc-test-key")
|
||||
.build();
|
||||
assertThrows(NullPointerException.class, () ->
|
||||
client.interact("job-id", null)
|
||||
);
|
||||
}
|
||||
|
||||
@Test
|
||||
void testBrowserDeleteRequiresSessionId() {
|
||||
FirecrawlClient client = FirecrawlClient.builder()
|
||||
.apiKey("fc-test-key")
|
||||
.build();
|
||||
assertThrows(NullPointerException.class, () ->
|
||||
client.deleteBrowser(null)
|
||||
);
|
||||
}
|
||||
|
||||
@Test
|
||||
void testStopInteractiveBrowserRequiresJobId() {
|
||||
FirecrawlClient client = FirecrawlClient.builder()
|
||||
.apiKey("fc-test-key")
|
||||
.build();
|
||||
assertThrows(NullPointerException.class, () ->
|
||||
client.stopInteractiveBrowser(null)
|
||||
);
|
||||
}
|
||||
|
||||
@Test
|
||||
void testParseFileBuilder() {
|
||||
ParseFile file = ParseFile.builder()
|
||||
.filename("upload.html")
|
||||
.content("<html><body>hello</body></html>".getBytes(StandardCharsets.UTF_8))
|
||||
.contentType("text/html")
|
||||
.build();
|
||||
|
||||
assertEquals("upload.html", file.getFilename());
|
||||
assertEquals("text/html", file.getContentType());
|
||||
assertTrue(file.getContent().length > 0);
|
||||
byte[] firstRead = file.getContent();
|
||||
firstRead[0] = 'X';
|
||||
assertNotEquals(firstRead[0], file.getContent()[0]);
|
||||
}
|
||||
|
||||
@Test
|
||||
void testParseRequiresFile() {
|
||||
FirecrawlClient client = FirecrawlClient.builder()
|
||||
.apiKey("fc-test-key")
|
||||
.build();
|
||||
assertThrows(NullPointerException.class, () ->
|
||||
client.parse(null)
|
||||
);
|
||||
}
|
||||
|
||||
@Test
|
||||
void testParseOptionsRejectsChangeTrackingFormat() {
|
||||
assertThrows(IllegalArgumentException.class, () ->
|
||||
ParseOptions.builder()
|
||||
.formats(List.of("markdown", "changeTracking"))
|
||||
.build()
|
||||
);
|
||||
}
|
||||
|
||||
// ================================================================
|
||||
// E2E TESTS (require FIRECRAWL_API_KEY)
|
||||
// ================================================================
|
||||
|
||||
@Test
|
||||
@EnabledIfEnvironmentVariable(named = "FIRECRAWL_API_KEY", matches = ".*\\S.*")
|
||||
void testScrapeE2E() {
|
||||
FirecrawlClient client = FirecrawlClient.fromEnv();
|
||||
Document doc = client.scrape("https://example.com",
|
||||
ScrapeOptions.builder()
|
||||
.formats(List.of("markdown"))
|
||||
.build());
|
||||
|
||||
assertNotNull(doc);
|
||||
assertNotNull(doc.getMarkdown());
|
||||
assertFalse(doc.getMarkdown().isEmpty());
|
||||
}
|
||||
|
||||
@Test
|
||||
@EnabledIfEnvironmentVariable(named = "FIRECRAWL_API_KEY", matches = ".*\\S.*")
|
||||
void testMapE2E() {
|
||||
FirecrawlClient client = FirecrawlClient.fromEnv();
|
||||
MapData data = client.map("https://example.com",
|
||||
MapOptions.builder()
|
||||
.limit(10)
|
||||
.build());
|
||||
|
||||
assertNotNull(data);
|
||||
assertNotNull(data.getLinks());
|
||||
}
|
||||
|
||||
@Test
|
||||
@EnabledIfEnvironmentVariable(named = "FIRECRAWL_API_KEY", matches = ".*\\S.*")
|
||||
void testCrawlE2E() {
|
||||
FirecrawlClient client = FirecrawlClient.fromEnv();
|
||||
CrawlJob job = client.crawl("https://example.com",
|
||||
CrawlOptions.builder()
|
||||
.limit(3)
|
||||
.build(),
|
||||
2, 60);
|
||||
|
||||
assertNotNull(job);
|
||||
assertEquals("completed", job.getStatus());
|
||||
assertNotNull(job.getData());
|
||||
assertFalse(job.getData().isEmpty());
|
||||
}
|
||||
|
||||
@Test
|
||||
@EnabledIfEnvironmentVariable(named = "FIRECRAWL_API_KEY", matches = ".*\\S.*")
|
||||
void testSearchE2E() {
|
||||
FirecrawlClient client = FirecrawlClient.fromEnv();
|
||||
SearchData data = client.search("firecrawl web scraping",
|
||||
SearchOptions.builder()
|
||||
.limit(5)
|
||||
.build());
|
||||
|
||||
assertNotNull(data);
|
||||
}
|
||||
|
||||
@Test
|
||||
@EnabledIfEnvironmentVariable(named = "FIRECRAWL_API_KEY", matches = ".*\\S.*")
|
||||
void testConcurrencyE2E() {
|
||||
FirecrawlClient client = FirecrawlClient.fromEnv();
|
||||
ConcurrencyCheck check = client.getConcurrency();
|
||||
|
||||
assertNotNull(check);
|
||||
assertTrue(check.getMaxConcurrency() > 0);
|
||||
}
|
||||
|
||||
@Test
|
||||
@EnabledIfEnvironmentVariable(named = "FIRECRAWL_API_KEY", matches = ".*\\S.*")
|
||||
void testCreditUsageE2E() {
|
||||
FirecrawlClient client = FirecrawlClient.fromEnv();
|
||||
CreditUsage usage = client.getCreditUsage();
|
||||
|
||||
assertNotNull(usage);
|
||||
}
|
||||
|
||||
@Test
|
||||
@EnabledIfEnvironmentVariable(named = "FIRECRAWL_API_KEY", matches = ".*\\S.*")
|
||||
void testParseE2E() {
|
||||
FirecrawlClient client = FirecrawlClient.fromEnv();
|
||||
ParseFile file = ParseFile.builder()
|
||||
.filename("java-parse-e2e.html")
|
||||
.content("<!DOCTYPE html><html><body><h1>Java SDK Parse E2E</h1></body></html>".getBytes(StandardCharsets.UTF_8))
|
||||
.contentType("text/html")
|
||||
.build();
|
||||
|
||||
Document doc = client.parse(file, ParseOptions.builder()
|
||||
.formats(List.of("markdown"))
|
||||
.build());
|
||||
|
||||
assertNotNull(doc);
|
||||
assertNotNull(doc.getMarkdown());
|
||||
assertFalse(doc.getMarkdown().isEmpty());
|
||||
assertTrue(doc.getMarkdown().contains("Java SDK Parse E2E"));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,131 @@
|
||||
package com.firecrawl;
|
||||
|
||||
import com.firecrawl.client.FirecrawlClient;
|
||||
import com.firecrawl.models.Document;
|
||||
import com.firecrawl.models.ScrapeOptions;
|
||||
import org.junit.jupiter.api.BeforeAll;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.*;
|
||||
|
||||
/**
|
||||
* Live Site Test - Firecrawl.dev
|
||||
*
|
||||
* Tests the Java SDK against the actual Firecrawl production website.
|
||||
* This demonstrates real-world usage of the API against live content.
|
||||
*
|
||||
* Run with: FIRECRAWL_API_KEY=fc-xxx gradle test --tests "com.firecrawl.FirecrawlLiveSiteTest"
|
||||
*/
|
||||
class FirecrawlLiveSiteTest {
|
||||
|
||||
private static FirecrawlClient client;
|
||||
|
||||
@BeforeAll
|
||||
static void setup() {
|
||||
String apiKey = System.getenv("FIRECRAWL_API_KEY");
|
||||
if (apiKey != null && !apiKey.isBlank()) {
|
||||
client = FirecrawlClient.fromEnv();
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
@EnabledIfEnvironmentVariable(named = "FIRECRAWL_API_KEY", matches = ".*\\S.*")
|
||||
void testScrapeFirecrawlHomepage() {
|
||||
System.out.println("\n=== Testing against LIVE Firecrawl.dev website ===\n");
|
||||
System.out.println("Scraping: https://firecrawl.dev");
|
||||
|
||||
Document doc = client.scrape("https://firecrawl.dev",
|
||||
ScrapeOptions.builder()
|
||||
.formats(List.of("markdown", "html"))
|
||||
.onlyMainContent(true)
|
||||
.build());
|
||||
|
||||
// Assertions
|
||||
assertNotNull(doc, "Document should not be null");
|
||||
assertNotNull(doc.getMarkdown(), "Markdown content should not be null");
|
||||
assertNotNull(doc.getHtml(), "HTML content should not be null");
|
||||
assertNotNull(doc.getMetadata(), "Metadata should not be null");
|
||||
|
||||
// Verify it's actually the Firecrawl site
|
||||
String markdown = doc.getMarkdown().toLowerCase();
|
||||
assertTrue(markdown.contains("firecrawl") || markdown.contains("scrape") || markdown.contains("crawl"),
|
||||
"Content should mention Firecrawl features");
|
||||
|
||||
// Check metadata
|
||||
String sourceUrl = doc.getMetadata().get("sourceURL").toString();
|
||||
assertTrue(sourceUrl.contains("firecrawl.dev"), "Source URL should be firecrawl.dev");
|
||||
|
||||
// Display results
|
||||
System.out.println("\n✓ Successfully scraped Firecrawl.dev!");
|
||||
System.out.println("\nMetadata:");
|
||||
System.out.println(" Source URL: " + sourceUrl);
|
||||
if (doc.getMetadata().get("title") != null) {
|
||||
System.out.println(" Title: " + doc.getMetadata().get("title"));
|
||||
}
|
||||
System.out.println(" Status Code: " + doc.getMetadata().get("statusCode"));
|
||||
|
||||
System.out.println("\nContent Stats:");
|
||||
System.out.println(" Markdown length: " + doc.getMarkdown().length() + " characters");
|
||||
System.out.println(" HTML length: " + doc.getHtml().length() + " characters");
|
||||
|
||||
System.out.println("\nFirst 500 characters of markdown:");
|
||||
System.out.println(" " + doc.getMarkdown().substring(0, Math.min(500, doc.getMarkdown().length())).replace("\n", "\n "));
|
||||
|
||||
System.out.println("\n=== Live site test completed successfully! ===\n");
|
||||
}
|
||||
|
||||
@Test
|
||||
@EnabledIfEnvironmentVariable(named = "FIRECRAWL_API_KEY", matches = ".*\\S.*")
|
||||
void testScrapeFirecrawlPricing() {
|
||||
System.out.println("\n=== Testing Firecrawl Pricing Page ===\n");
|
||||
System.out.println("Scraping: https://firecrawl.dev/pricing");
|
||||
|
||||
Document doc = client.scrape("https://firecrawl.dev/pricing",
|
||||
ScrapeOptions.builder()
|
||||
.formats(List.of("markdown"))
|
||||
.build());
|
||||
|
||||
// Assertions
|
||||
assertNotNull(doc, "Document should not be null");
|
||||
assertNotNull(doc.getMarkdown(), "Markdown content should not be null");
|
||||
|
||||
String markdown = doc.getMarkdown().toLowerCase();
|
||||
assertTrue(markdown.contains("pricing") || markdown.contains("plan") || markdown.contains("price"),
|
||||
"Pricing page should contain pricing information");
|
||||
|
||||
System.out.println("✓ Successfully scraped pricing page!");
|
||||
System.out.println(" Content length: " + doc.getMarkdown().length() + " characters");
|
||||
System.out.println(" Source: " + doc.getMetadata().get("sourceURL"));
|
||||
}
|
||||
|
||||
@Test
|
||||
@EnabledIfEnvironmentVariable(named = "FIRECRAWL_API_KEY", matches = ".*\\S.*")
|
||||
void testScrapeFirecrawlDocs() {
|
||||
System.out.println("\n=== Testing Firecrawl Documentation ===\n");
|
||||
System.out.println("Scraping: https://docs.firecrawl.dev");
|
||||
|
||||
Document doc = client.scrape("https://docs.firecrawl.dev",
|
||||
ScrapeOptions.builder()
|
||||
.formats(List.of("markdown"))
|
||||
.waitFor(2000) // Wait for docs to load
|
||||
.build());
|
||||
|
||||
// Assertions
|
||||
assertNotNull(doc, "Document should not be null");
|
||||
assertNotNull(doc.getMarkdown(), "Markdown content should not be null");
|
||||
assertFalse(doc.getMarkdown().isEmpty(), "Markdown should not be empty");
|
||||
|
||||
String markdown = doc.getMarkdown().toLowerCase();
|
||||
assertTrue(markdown.contains("document") || markdown.contains("api") || markdown.contains("firecrawl"),
|
||||
"Docs should contain documentation content");
|
||||
|
||||
System.out.println("✓ Successfully scraped documentation!");
|
||||
System.out.println(" Content length: " + doc.getMarkdown().length() + " characters");
|
||||
System.out.println(" Source: " + doc.getMetadata().get("sourceURL"));
|
||||
|
||||
System.out.println("\n=== All Firecrawl.dev tests passed! ===\n");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,279 @@
|
||||
package com.firecrawl;
|
||||
|
||||
import com.firecrawl.client.FirecrawlClient;
|
||||
import com.firecrawl.models.MapData;
|
||||
import com.firecrawl.models.MapOptions;
|
||||
import org.junit.jupiter.api.BeforeAll;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable;
|
||||
|
||||
import java.util.Map;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.*;
|
||||
|
||||
/**
|
||||
* Comprehensive Map Tests
|
||||
*
|
||||
* Tests the map functionality with various configurations.
|
||||
* Based on Node.js SDK patterns and tested against live firecrawl.dev.
|
||||
*
|
||||
* Run with: FIRECRAWL_API_KEY=fc-xxx gradle test --tests "com.firecrawl.MapTest"
|
||||
*/
|
||||
class MapTest {
|
||||
|
||||
private static FirecrawlClient client;
|
||||
|
||||
@BeforeAll
|
||||
static void setup() {
|
||||
String apiKey = System.getenv("FIRECRAWL_API_KEY");
|
||||
if (apiKey != null && !apiKey.isBlank()) {
|
||||
client = FirecrawlClient.fromEnv();
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
@EnabledIfEnvironmentVariable(named = "FIRECRAWL_API_KEY", matches = ".*\\S.*")
|
||||
void testMapMinimal() {
|
||||
System.out.println("\n=== Test: Map - Minimal Request ===");
|
||||
|
||||
MapData data = client.map("https://docs.firecrawl.dev");
|
||||
|
||||
assertNotNull(data, "Map data should not be null");
|
||||
assertNotNull(data.getLinks(), "Links should not be null");
|
||||
assertTrue(!data.getLinks().isEmpty(), "Should have at least one link");
|
||||
|
||||
// Verify link structure (v2 links are MapDocument objects with url, title, description)
|
||||
Map<String, Object> firstLink = data.getLinks().get(0);
|
||||
assertNotNull(firstLink, "Link should not be null");
|
||||
assertNotNull(firstLink.get("url"), "Link should have url");
|
||||
assertTrue(firstLink.get("url").toString().startsWith("http"), "URL should start with http");
|
||||
|
||||
System.out.println("✓ Map completed successfully");
|
||||
System.out.println(" Total links found: " + data.getLinks().size());
|
||||
System.out.println(" Sample URL: " + firstLink.get("url"));
|
||||
if (firstLink.get("title") != null) {
|
||||
System.out.println(" Title: " + firstLink.get("title"));
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
@EnabledIfEnvironmentVariable(named = "FIRECRAWL_API_KEY", matches = ".*\\S.*")
|
||||
void testMapWithLimit() {
|
||||
System.out.println("\n=== Test: Map with Limit ===");
|
||||
|
||||
MapData data = client.map("https://docs.firecrawl.dev",
|
||||
MapOptions.builder()
|
||||
.limit(10)
|
||||
.build());
|
||||
|
||||
assertNotNull(data.getLinks(), "Links should not be null");
|
||||
assertTrue(data.getLinks().size() <= 10,
|
||||
"Should respect limit of 10: got " + data.getLinks().size());
|
||||
|
||||
System.out.println("✓ Map with limit completed");
|
||||
System.out.println(" Requested limit: 10");
|
||||
System.out.println(" Actual links: " + data.getLinks().size());
|
||||
}
|
||||
|
||||
@Test
|
||||
@EnabledIfEnvironmentVariable(named = "FIRECRAWL_API_KEY", matches = ".*\\S.*")
|
||||
void testMapWithSearch() {
|
||||
System.out.println("\n=== Test: Map with Search Filter ===");
|
||||
|
||||
MapData data = client.map("https://docs.firecrawl.dev",
|
||||
MapOptions.builder()
|
||||
.search("api")
|
||||
.limit(20)
|
||||
.build());
|
||||
|
||||
assertNotNull(data.getLinks(), "Links should not be null");
|
||||
|
||||
// Verify that filtered results contain the search term
|
||||
long matchingLinks = data.getLinks().stream()
|
||||
.filter(link -> {
|
||||
String url = link.get("url") != null ? link.get("url").toString().toLowerCase() : "";
|
||||
String title = link.get("title") != null ? link.get("title").toString().toLowerCase() : "";
|
||||
return url.contains("api") || title.contains("api");
|
||||
})
|
||||
.count();
|
||||
|
||||
System.out.println("✓ Map with search completed");
|
||||
System.out.println(" Total links: " + data.getLinks().size());
|
||||
System.out.println(" Links matching 'api': " + matchingLinks);
|
||||
}
|
||||
|
||||
@Test
|
||||
@EnabledIfEnvironmentVariable(named = "FIRECRAWL_API_KEY", matches = ".*\\S.*")
|
||||
void testMapWithSkipSitemap() {
|
||||
System.out.println("\n=== Test: Map with Sitemap Skip ===");
|
||||
|
||||
MapData data = client.map("https://firecrawl.dev",
|
||||
MapOptions.builder()
|
||||
.sitemap("skip")
|
||||
.limit(15)
|
||||
.build());
|
||||
|
||||
assertNotNull(data.getLinks(), "Links should not be null");
|
||||
assertTrue(data.getLinks().size() <= 15, "Should respect limit");
|
||||
|
||||
// Verify all links are valid HTTP(S) URLs
|
||||
boolean allValidUrls = data.getLinks().stream()
|
||||
.allMatch(link -> {
|
||||
String url = link.get("url") != null ? link.get("url").toString() : "";
|
||||
return url.startsWith("http://") || url.startsWith("https://");
|
||||
});
|
||||
|
||||
assertTrue(allValidUrls, "All URLs should be valid HTTP(S)");
|
||||
|
||||
System.out.println("✓ Map with sitemap=skip completed");
|
||||
System.out.println(" Links found: " + data.getLinks().size());
|
||||
}
|
||||
|
||||
@Test
|
||||
@EnabledIfEnvironmentVariable(named = "FIRECRAWL_API_KEY", matches = ".*\\S.*")
|
||||
void testMapWithSitemapOnly() {
|
||||
System.out.println("\n=== Test: Map with Sitemap Only ===");
|
||||
|
||||
MapData data = client.map("https://firecrawl.dev",
|
||||
MapOptions.builder()
|
||||
.sitemap("only")
|
||||
.limit(50)
|
||||
.build());
|
||||
|
||||
assertNotNull(data.getLinks(), "Links should not be null");
|
||||
// Note: sitemapOnly may not always respect the limit strictly
|
||||
|
||||
// Verify all links are valid HTTP(S) URLs
|
||||
boolean allValidUrls = data.getLinks().stream()
|
||||
.allMatch(link -> {
|
||||
String url = link.get("url") != null ? link.get("url").toString() : "";
|
||||
return url.startsWith("http://") || url.startsWith("https://");
|
||||
});
|
||||
|
||||
assertTrue(allValidUrls, "All URLs should be valid HTTP(S)");
|
||||
|
||||
System.out.println("✓ Map with sitemap=only completed");
|
||||
System.out.println(" Links found: " + data.getLinks().size());
|
||||
}
|
||||
|
||||
@Test
|
||||
@EnabledIfEnvironmentVariable(named = "FIRECRAWL_API_KEY", matches = ".*\\S.*")
|
||||
void testMapWithIncludeSubdomains() {
|
||||
System.out.println("\n=== Test: Map with Include Subdomains ===");
|
||||
|
||||
MapData data = client.map("https://firecrawl.dev",
|
||||
MapOptions.builder()
|
||||
.includeSubdomains(true)
|
||||
.limit(20)
|
||||
.build());
|
||||
|
||||
assertNotNull(data.getLinks(), "Links should not be null");
|
||||
|
||||
System.out.println("✓ Map with subdomains completed");
|
||||
System.out.println(" Total links: " + data.getLinks().size());
|
||||
|
||||
// Check if any subdomains were found
|
||||
boolean hasSubdomains = data.getLinks().stream()
|
||||
.anyMatch(link -> {
|
||||
String url = link.get("url") != null ? link.get("url").toString() : "";
|
||||
return url.contains("docs.firecrawl.dev") ||
|
||||
url.contains("api.firecrawl.dev") ||
|
||||
(url.contains(".firecrawl.dev") && !url.contains("www.firecrawl.dev"));
|
||||
});
|
||||
|
||||
if (hasSubdomains) {
|
||||
System.out.println(" ✓ Found subdomain links");
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
@EnabledIfEnvironmentVariable(named = "FIRECRAWL_API_KEY", matches = ".*\\S.*")
|
||||
void testMapFirecrawlDocs() {
|
||||
System.out.println("\n=== Test: Map Firecrawl Documentation ===");
|
||||
|
||||
MapData data = client.map("https://docs.firecrawl.dev",
|
||||
MapOptions.builder()
|
||||
.limit(50)
|
||||
.build());
|
||||
|
||||
assertNotNull(data.getLinks(), "Links should not be null");
|
||||
assertFalse(data.getLinks().isEmpty(), "Should find documentation links");
|
||||
|
||||
System.out.println("✓ Mapped Firecrawl documentation");
|
||||
System.out.println(" Total links: " + data.getLinks().size());
|
||||
|
||||
// Print sample links
|
||||
System.out.println(" Sample documentation pages:");
|
||||
data.getLinks().stream()
|
||||
.limit(5)
|
||||
.forEach(link -> System.out.println(" - " + link.get("url")));
|
||||
}
|
||||
|
||||
@Test
|
||||
@EnabledIfEnvironmentVariable(named = "FIRECRAWL_API_KEY", matches = ".*\\S.*")
|
||||
void testMapLinkStructure() {
|
||||
System.out.println("\n=== Test: Verify Map Link Structure ===");
|
||||
|
||||
MapData data = client.map("https://firecrawl.dev",
|
||||
MapOptions.builder()
|
||||
.limit(5)
|
||||
.build());
|
||||
|
||||
assertNotNull(data.getLinks(), "Links should not be null");
|
||||
assertFalse(data.getLinks().isEmpty(), "Should have links");
|
||||
|
||||
// Verify each link is a valid URL with expected fields
|
||||
for (Map<String, Object> link : data.getLinks()) {
|
||||
assertNotNull(link, "Link should not be null");
|
||||
assertNotNull(link.get("url"), "Link should have url field");
|
||||
assertTrue(link.get("url").toString().startsWith("http"), "URL should be valid: " + link.get("url"));
|
||||
}
|
||||
|
||||
System.out.println("✓ All links have correct structure");
|
||||
System.out.println(" Verified " + data.getLinks().size() + " links");
|
||||
}
|
||||
|
||||
@Test
|
||||
@EnabledIfEnvironmentVariable(named = "FIRECRAWL_API_KEY", matches = ".*\\S.*")
|
||||
void testMapWithTimeout() {
|
||||
System.out.println("\n=== Test: Map with Timeout ===");
|
||||
|
||||
MapData data = client.map("https://firecrawl.dev",
|
||||
MapOptions.builder()
|
||||
.timeout(15000) // 15 seconds
|
||||
.limit(10)
|
||||
.build());
|
||||
|
||||
assertNotNull(data.getLinks(), "Links should not be null");
|
||||
|
||||
System.out.println("✓ Map with timeout completed");
|
||||
System.out.println(" Timeout: 15000ms");
|
||||
System.out.println(" Links found: " + data.getLinks().size());
|
||||
}
|
||||
|
||||
@Test
|
||||
@EnabledIfEnvironmentVariable(named = "FIRECRAWL_API_KEY", matches = ".*\\S.*")
|
||||
void testMapComprehensive() {
|
||||
System.out.println("\n=== Test: Map with All Options ===");
|
||||
|
||||
MapData data = client.map("https://docs.firecrawl.dev",
|
||||
MapOptions.builder()
|
||||
.includeSubdomains(false)
|
||||
.limit(25)
|
||||
.sitemap("include")
|
||||
.timeout(20000)
|
||||
.build());
|
||||
|
||||
assertNotNull(data.getLinks(), "Links should not be null");
|
||||
assertTrue(data.getLinks().size() <= 25, "Should respect limit");
|
||||
|
||||
System.out.println("✓ Comprehensive map completed");
|
||||
System.out.println(" Configuration:");
|
||||
System.out.println(" - Include subdomains: false");
|
||||
System.out.println(" - Limit: 25");
|
||||
System.out.println(" - Ignore sitemap: false");
|
||||
System.out.println(" - Timeout: 20000ms");
|
||||
System.out.println(" Results:");
|
||||
System.out.println(" - Links found: " + data.getLinks().size());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,177 @@
|
||||
package com.firecrawl;
|
||||
|
||||
import com.firecrawl.client.FirecrawlClient;
|
||||
import com.firecrawl.errors.FirecrawlException;
|
||||
import com.firecrawl.models.Document;
|
||||
import com.firecrawl.models.ScrapeOptions;
|
||||
import org.junit.jupiter.api.BeforeAll;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.*;
|
||||
|
||||
/**
|
||||
* Scrape Endpoint Tests
|
||||
*
|
||||
* Tests the scrape functionality of the Firecrawl Java SDK.
|
||||
* These tests require FIRECRAWL_API_KEY environment variable to be set.
|
||||
*
|
||||
* Run with: FIRECRAWL_API_KEY=fc-xxx gradle test --tests "com.firecrawl.ScrapeTest"
|
||||
*/
|
||||
class ScrapeTest {
|
||||
|
||||
private static FirecrawlClient client;
|
||||
|
||||
@BeforeAll
|
||||
static void setup() {
|
||||
// Initialize client from environment variable
|
||||
String apiKey = System.getenv("FIRECRAWL_API_KEY");
|
||||
if (apiKey != null && !apiKey.isBlank()) {
|
||||
client = FirecrawlClient.fromEnv();
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
@EnabledIfEnvironmentVariable(named = "FIRECRAWL_API_KEY", matches = ".*\\S.*")
|
||||
void testScrapeBasic() {
|
||||
// Test basic scraping with markdown format
|
||||
System.out.println("Testing basic scrape with markdown format...");
|
||||
|
||||
Document doc = client.scrape("https://example.com",
|
||||
ScrapeOptions.builder()
|
||||
.formats(List.of("markdown"))
|
||||
.build());
|
||||
|
||||
// Assertions
|
||||
assertNotNull(doc, "Document should not be null");
|
||||
assertNotNull(doc.getMarkdown(), "Markdown content should not be null");
|
||||
assertFalse(doc.getMarkdown().isEmpty(), "Markdown content should not be empty");
|
||||
|
||||
System.out.println("✓ Basic scrape test passed");
|
||||
System.out.println(" Markdown length: " + doc.getMarkdown().length() + " characters");
|
||||
}
|
||||
|
||||
@Test
|
||||
@EnabledIfEnvironmentVariable(named = "FIRECRAWL_API_KEY", matches = ".*\\S.*")
|
||||
void testScrapeWithMultipleFormats() {
|
||||
// Test scraping with multiple formats
|
||||
System.out.println("Testing scrape with multiple formats (markdown + html)...");
|
||||
|
||||
Document doc = client.scrape("https://example.com",
|
||||
ScrapeOptions.builder()
|
||||
.formats(List.of("markdown", "html"))
|
||||
.build());
|
||||
|
||||
// Assertions
|
||||
assertNotNull(doc, "Document should not be null");
|
||||
assertNotNull(doc.getMarkdown(), "Markdown content should not be null");
|
||||
assertNotNull(doc.getHtml(), "HTML content should not be null");
|
||||
assertFalse(doc.getMarkdown().isEmpty(), "Markdown should not be empty");
|
||||
assertFalse(doc.getHtml().isEmpty(), "HTML should not be empty");
|
||||
|
||||
System.out.println("✓ Multiple formats test passed");
|
||||
System.out.println(" Markdown length: " + doc.getMarkdown().length());
|
||||
System.out.println(" HTML length: " + doc.getHtml().length());
|
||||
}
|
||||
|
||||
@Test
|
||||
@EnabledIfEnvironmentVariable(named = "FIRECRAWL_API_KEY", matches = ".*\\S.*")
|
||||
void testScrapeWithMetadata() {
|
||||
// Test that metadata is properly extracted
|
||||
System.out.println("Testing scrape with metadata extraction...");
|
||||
|
||||
Document doc = client.scrape("https://example.com",
|
||||
ScrapeOptions.builder()
|
||||
.formats(List.of("markdown"))
|
||||
.build());
|
||||
|
||||
// Assertions
|
||||
assertNotNull(doc.getMetadata(), "Metadata should not be null");
|
||||
assertNotNull(doc.getMetadata().get("sourceURL"), "Source URL should be in metadata");
|
||||
assertTrue(doc.getMetadata().get("sourceURL").toString().contains("example.com"),
|
||||
"Source URL should contain example.com");
|
||||
|
||||
System.out.println("✓ Metadata extraction test passed");
|
||||
System.out.println(" Source URL: " + doc.getMetadata().get("sourceURL"));
|
||||
if (doc.getMetadata().get("title") != null) {
|
||||
System.out.println(" Title: " + doc.getMetadata().get("title"));
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
@EnabledIfEnvironmentVariable(named = "FIRECRAWL_API_KEY", matches = ".*\\S.*")
|
||||
void testScrapeWithOnlyMainContent() {
|
||||
// Test scraping with onlyMainContent option
|
||||
System.out.println("Testing scrape with onlyMainContent option...");
|
||||
|
||||
Document doc = client.scrape("https://example.com",
|
||||
ScrapeOptions.builder()
|
||||
.formats(List.of("markdown"))
|
||||
.onlyMainContent(true)
|
||||
.build());
|
||||
|
||||
// Assertions
|
||||
assertNotNull(doc, "Document should not be null");
|
||||
assertNotNull(doc.getMarkdown(), "Markdown content should not be null");
|
||||
assertFalse(doc.getMarkdown().isEmpty(), "Markdown should not be empty");
|
||||
|
||||
System.out.println("✓ Only main content test passed");
|
||||
System.out.println(" Content length: " + doc.getMarkdown().length());
|
||||
}
|
||||
|
||||
@Test
|
||||
@EnabledIfEnvironmentVariable(named = "FIRECRAWL_API_KEY", matches = ".*\\S.*")
|
||||
void testScrapeWithTimeout() {
|
||||
// Test scraping with custom timeout
|
||||
System.out.println("Testing scrape with custom timeout...");
|
||||
|
||||
Document doc = client.scrape("https://example.com",
|
||||
ScrapeOptions.builder()
|
||||
.formats(List.of("markdown"))
|
||||
.timeout(10000) // 10 seconds
|
||||
.build());
|
||||
|
||||
// Assertions
|
||||
assertNotNull(doc, "Document should not be null");
|
||||
assertNotNull(doc.getMarkdown(), "Markdown should not be null");
|
||||
|
||||
System.out.println("✓ Timeout configuration test passed");
|
||||
}
|
||||
|
||||
@Test
|
||||
@EnabledIfEnvironmentVariable(named = "FIRECRAWL_API_KEY", matches = ".*\\S.*")
|
||||
void testScrapeInvalidUrl() {
|
||||
// Test that invalid URLs are handled properly
|
||||
System.out.println("Testing scrape with invalid URL...");
|
||||
|
||||
assertThrows(FirecrawlException.class, () -> {
|
||||
client.scrape("not-a-valid-url",
|
||||
ScrapeOptions.builder()
|
||||
.formats(List.of("markdown"))
|
||||
.build());
|
||||
}, "Should throw FirecrawlException for invalid URL");
|
||||
|
||||
System.out.println("✓ Invalid URL handling test passed");
|
||||
}
|
||||
|
||||
@Test
|
||||
@EnabledIfEnvironmentVariable(named = "FIRECRAWL_API_KEY", matches = ".*\\S.*")
|
||||
void testScrapeWithWaitFor() {
|
||||
// Test scraping with waitFor option (useful for dynamic content)
|
||||
System.out.println("Testing scrape with waitFor option...");
|
||||
|
||||
Document doc = client.scrape("https://example.com",
|
||||
ScrapeOptions.builder()
|
||||
.formats(List.of("markdown"))
|
||||
.waitFor(1000) // Wait 1 second for page to load
|
||||
.build());
|
||||
|
||||
// Assertions
|
||||
assertNotNull(doc, "Document should not be null");
|
||||
assertNotNull(doc.getMarkdown(), "Markdown should not be null");
|
||||
|
||||
System.out.println("✓ WaitFor option test passed");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,337 @@
|
||||
package com.firecrawl;
|
||||
|
||||
import com.firecrawl.client.FirecrawlClient;
|
||||
import com.firecrawl.models.*;
|
||||
import org.junit.jupiter.api.BeforeAll;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.*;
|
||||
|
||||
/**
|
||||
* Comprehensive Search Tests
|
||||
*
|
||||
* Tests the search functionality with various configurations.
|
||||
* Based on Node.js SDK patterns and tested against live firecrawl.dev.
|
||||
*
|
||||
* Run with: FIRECRAWL_API_KEY=fc-xxx gradle test --tests "com.firecrawl.SearchTest"
|
||||
*/
|
||||
class SearchTest {
|
||||
|
||||
private static FirecrawlClient client;
|
||||
|
||||
@BeforeAll
|
||||
static void setup() {
|
||||
String apiKey = System.getenv("FIRECRAWL_API_KEY");
|
||||
if (apiKey != null && !apiKey.isBlank()) {
|
||||
client = FirecrawlClient.fromEnv();
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
@EnabledIfEnvironmentVariable(named = "FIRECRAWL_API_KEY", matches = ".*\\S.*")
|
||||
void testSearchMinimal() {
|
||||
System.out.println("\n=== Test: Search - Minimal Request ===");
|
||||
|
||||
SearchData results = client.search("What is Firecrawl?");
|
||||
|
||||
assertNotNull(results, "Search results should not be null");
|
||||
assertNotNull(results.getWeb(), "Web results should not be null");
|
||||
assertTrue(!results.getWeb().isEmpty(), "Should have at least one web result");
|
||||
|
||||
// Verify result structure
|
||||
Map<String, Object> firstResult = results.getWeb().get(0);
|
||||
assertNotNull(firstResult.get("url"), "Result should have URL");
|
||||
assertTrue(firstResult.get("url").toString().startsWith("http"),
|
||||
"URL should be valid");
|
||||
|
||||
System.out.println("✓ Search completed successfully");
|
||||
System.out.println(" Web results: " + results.getWeb().size());
|
||||
System.out.println(" Sample result: " + firstResult.get("url"));
|
||||
if (firstResult.get("title") != null) {
|
||||
System.out.println(" Title: " + firstResult.get("title"));
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
@EnabledIfEnvironmentVariable(named = "FIRECRAWL_API_KEY", matches = ".*\\S.*")
|
||||
void testSearchWithLimit() {
|
||||
System.out.println("\n=== Test: Search with Limit ===");
|
||||
|
||||
SearchData results = client.search("artificial intelligence",
|
||||
SearchOptions.builder()
|
||||
.limit(5)
|
||||
.build());
|
||||
|
||||
assertNotNull(results.getWeb(), "Web results should not be null");
|
||||
assertTrue(results.getWeb().size() <= 5,
|
||||
"Should respect limit of 5: got " + results.getWeb().size());
|
||||
|
||||
System.out.println("✓ Search with limit completed");
|
||||
System.out.println(" Requested limit: 5");
|
||||
System.out.println(" Actual results: " + results.getWeb().size());
|
||||
}
|
||||
|
||||
@Test
|
||||
@EnabledIfEnvironmentVariable(named = "FIRECRAWL_API_KEY", matches = ".*\\S.*")
|
||||
void testSearchWithMultipleSources() {
|
||||
System.out.println("\n=== Test: Search with Multiple Sources ===");
|
||||
|
||||
SearchData results = client.search("Firecrawl web scraping",
|
||||
SearchOptions.builder()
|
||||
.sources(List.of("web", "news"))
|
||||
.limit(3)
|
||||
.build());
|
||||
|
||||
assertNotNull(results.getWeb(), "Web results should not be null");
|
||||
assertTrue(results.getWeb().size() <= 3, "Web results should respect limit");
|
||||
|
||||
System.out.println("✓ Multi-source search completed");
|
||||
System.out.println(" Web results: " + results.getWeb().size());
|
||||
if (results.getNews() != null) {
|
||||
System.out.println(" News results: " + results.getNews().size());
|
||||
} else {
|
||||
System.out.println(" News results: 0");
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
@EnabledIfEnvironmentVariable(named = "FIRECRAWL_API_KEY", matches = ".*\\S.*")
|
||||
void testSearchResultStructure() {
|
||||
System.out.println("\n=== Test: Verify Search Result Structure ===");
|
||||
|
||||
SearchData results = client.search("test query",
|
||||
SearchOptions.builder()
|
||||
.limit(1)
|
||||
.build());
|
||||
|
||||
assertNotNull(results.getWeb(), "Web results should not be null");
|
||||
|
||||
if (!results.getWeb().isEmpty()) {
|
||||
Map<String, Object> result = results.getWeb().get(0);
|
||||
|
||||
assertNotNull(result.get("url"), "Result must have URL");
|
||||
assertTrue(result.get("url") instanceof String, "URL should be string");
|
||||
assertTrue(result.get("url").toString().startsWith("http"),
|
||||
"URL should be valid");
|
||||
|
||||
// Title and description may be null but if present should be strings
|
||||
if (result.get("title") != null) {
|
||||
assertTrue(result.get("title") instanceof String,
|
||||
"Title should be string");
|
||||
}
|
||||
if (result.get("description") != null) {
|
||||
assertTrue(result.get("description") instanceof String,
|
||||
"Description should be string");
|
||||
}
|
||||
|
||||
System.out.println("✓ Result structure verified");
|
||||
System.out.println(" URL: ✓");
|
||||
System.out.println(" Title: " + (result.get("title") != null ? "✓" : "null"));
|
||||
System.out.println(" Description: " + (result.get("description") != null ? "✓" : "null"));
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
@EnabledIfEnvironmentVariable(named = "FIRECRAWL_API_KEY", matches = ".*\\S.*")
|
||||
void testSearchWithLocation() {
|
||||
System.out.println("\n=== Test: Search with Location ===");
|
||||
|
||||
SearchData results = client.search("restaurants near me",
|
||||
SearchOptions.builder()
|
||||
.location("US")
|
||||
.limit(5)
|
||||
.build());
|
||||
|
||||
assertNotNull(results.getWeb(), "Web results should not be null");
|
||||
|
||||
System.out.println("✓ Search with location completed");
|
||||
System.out.println(" Location: US");
|
||||
System.out.println(" Results: " + results.getWeb().size());
|
||||
}
|
||||
|
||||
@Test
|
||||
@EnabledIfEnvironmentVariable(named = "FIRECRAWL_API_KEY", matches = ".*\\S.*")
|
||||
void testSearchWithTimeFilter() {
|
||||
System.out.println("\n=== Test: Search with Time Filter ===");
|
||||
|
||||
SearchData results = client.search("latest AI news",
|
||||
SearchOptions.builder()
|
||||
.tbs("qdr:m") // Past month
|
||||
.limit(5)
|
||||
.build());
|
||||
|
||||
assertNotNull(results.getWeb(), "Web results should not be null");
|
||||
|
||||
System.out.println("✓ Search with time filter completed");
|
||||
System.out.println(" Time filter: Past month (qdr:m)");
|
||||
System.out.println(" Results: " + results.getWeb().size());
|
||||
}
|
||||
|
||||
@Test
|
||||
@EnabledIfEnvironmentVariable(named = "FIRECRAWL_API_KEY", matches = ".*\\S.*")
|
||||
void testSearchWithScrapeOptions() {
|
||||
System.out.println("\n=== Test: Search with Scrape Options ===");
|
||||
|
||||
SearchData results = client.search("Firecrawl documentation",
|
||||
SearchOptions.builder()
|
||||
.limit(2)
|
||||
.scrapeOptions(ScrapeOptions.builder()
|
||||
.formats(List.of("markdown"))
|
||||
.onlyMainContent(true)
|
||||
.build())
|
||||
.build());
|
||||
|
||||
assertNotNull(results.getWeb(), "Web results should not be null");
|
||||
|
||||
// When scrapeOptions with markdown format are provided, results should include markdown content
|
||||
if (!results.getWeb().isEmpty()) {
|
||||
Map<String, Object> first = results.getWeb().get(0);
|
||||
Object markdown = first.get("markdown");
|
||||
assertNotNull(markdown, "Scraped result should contain markdown content when formats=[markdown]");
|
||||
assertFalse(markdown.toString().isEmpty(), "Markdown content should not be empty");
|
||||
}
|
||||
|
||||
System.out.println("✓ Search with scrape options completed");
|
||||
System.out.println(" Results: " + results.getWeb().size());
|
||||
System.out.println(" Scrape formats: markdown");
|
||||
}
|
||||
|
||||
@Test
|
||||
@EnabledIfEnvironmentVariable(named = "FIRECRAWL_API_KEY", matches = ".*\\S.*")
|
||||
void testSearchFirecrawlSpecific() {
|
||||
System.out.println("\n=== Test: Search for Firecrawl ===");
|
||||
|
||||
SearchData results = client.search("Firecrawl web scraping API",
|
||||
SearchOptions.builder()
|
||||
.limit(10)
|
||||
.build());
|
||||
|
||||
assertNotNull(results.getWeb(), "Web results should not be null");
|
||||
assertFalse(results.getWeb().isEmpty(), "Should find Firecrawl results");
|
||||
|
||||
// Verify results contain Firecrawl-related content
|
||||
boolean hasFirecrawlContent = results.getWeb().stream()
|
||||
.anyMatch(result -> {
|
||||
String url = result.get("url").toString().toLowerCase();
|
||||
String title = result.get("title") != null ?
|
||||
result.get("title").toString().toLowerCase() : "";
|
||||
String desc = result.get("description") != null ?
|
||||
result.get("description").toString().toLowerCase() : "";
|
||||
|
||||
return url.contains("firecrawl") ||
|
||||
title.contains("firecrawl") ||
|
||||
desc.contains("firecrawl");
|
||||
});
|
||||
|
||||
assertTrue(hasFirecrawlContent, "Results should mention Firecrawl");
|
||||
|
||||
System.out.println("✓ Firecrawl search completed");
|
||||
System.out.println(" Total results: " + results.getWeb().size());
|
||||
System.out.println(" Results mentioning Firecrawl: ✓");
|
||||
|
||||
// Print sample results
|
||||
System.out.println(" Sample results:");
|
||||
results.getWeb().stream()
|
||||
.limit(3)
|
||||
.forEach(result -> {
|
||||
System.out.println(" - " + result.get("title"));
|
||||
System.out.println(" " + result.get("url"));
|
||||
});
|
||||
}
|
||||
|
||||
@Test
|
||||
@EnabledIfEnvironmentVariable(named = "FIRECRAWL_API_KEY", matches = ".*\\S.*")
|
||||
void testSearchComprehensive() {
|
||||
System.out.println("\n=== Test: Search with All Options ===");
|
||||
|
||||
SearchData results = client.search("web scraping tools",
|
||||
SearchOptions.builder()
|
||||
.sources(List.of("web"))
|
||||
.limit(5)
|
||||
.tbs("qdr:y") // Past year
|
||||
.location("US")
|
||||
.timeout(30000)
|
||||
.scrapeOptions(ScrapeOptions.builder()
|
||||
.formats(List.of("markdown"))
|
||||
.onlyMainContent(true)
|
||||
.waitFor(1000)
|
||||
.build())
|
||||
.build());
|
||||
|
||||
assertNotNull(results.getWeb(), "Web results should not be null");
|
||||
assertTrue(results.getWeb().size() <= 5, "Should respect limit");
|
||||
|
||||
System.out.println("✓ Comprehensive search completed");
|
||||
System.out.println(" Configuration:");
|
||||
System.out.println(" - Sources: web");
|
||||
System.out.println(" - Limit: 5");
|
||||
System.out.println(" - Time filter: Past year");
|
||||
System.out.println(" - Location: US");
|
||||
System.out.println(" - Timeout: 30000ms");
|
||||
System.out.println(" - Scrape: markdown, main content only");
|
||||
System.out.println(" Results:");
|
||||
System.out.println(" - Web results: " + results.getWeb().size());
|
||||
}
|
||||
|
||||
@Test
|
||||
@EnabledIfEnvironmentVariable(named = "FIRECRAWL_API_KEY", matches = ".*\\S.*")
|
||||
void testSearchContentVerification() {
|
||||
System.out.println("\n=== Test: Search Content Verification ===");
|
||||
|
||||
SearchData results = client.search("Python programming language",
|
||||
SearchOptions.builder()
|
||||
.limit(5)
|
||||
.build());
|
||||
|
||||
assertNotNull(results.getWeb(), "Web results should not be null");
|
||||
assertFalse(results.getWeb().isEmpty(), "Should have results");
|
||||
|
||||
// Verify results are relevant to the query
|
||||
boolean hasRelevantContent = results.getWeb().stream()
|
||||
.anyMatch(result -> {
|
||||
String text = String.format("%s %s %s",
|
||||
result.get("url"),
|
||||
result.get("title"),
|
||||
result.get("description")
|
||||
).toLowerCase();
|
||||
return text.contains("python");
|
||||
});
|
||||
|
||||
assertTrue(hasRelevantContent, "Results should be relevant to query");
|
||||
|
||||
System.out.println("✓ Content verification passed");
|
||||
System.out.println(" Query: Python programming language");
|
||||
System.out.println(" Relevant results found: ✓");
|
||||
}
|
||||
|
||||
@Test
|
||||
@EnabledIfEnvironmentVariable(named = "FIRECRAWL_API_KEY", matches = ".*\\S.*")
|
||||
void testSearchIgnoreInvalidURLs() {
|
||||
System.out.println("\n=== Test: Search with Ignore Invalid URLs ===");
|
||||
|
||||
SearchData results = client.search("technology news",
|
||||
SearchOptions.builder()
|
||||
.limit(5)
|
||||
.ignoreInvalidURLs(true)
|
||||
.build());
|
||||
|
||||
assertNotNull(results.getWeb(), "Web results should not be null");
|
||||
|
||||
// Verify all URLs are valid
|
||||
boolean allValidUrls = results.getWeb().stream()
|
||||
.allMatch(result -> {
|
||||
String url = result.get("url").toString();
|
||||
return url.startsWith("http://") || url.startsWith("https://");
|
||||
});
|
||||
|
||||
assertTrue(allValidUrls, "All URLs should be valid HTTP(S)");
|
||||
|
||||
System.out.println("✓ Search with URL validation completed");
|
||||
System.out.println(" Results: " + results.getWeb().size());
|
||||
System.out.println(" All URLs valid: ✓");
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user