참고소스 수정본
This commit is contained in:
LASTA_DEV01\lasta
@@ -0,0 +1,285 @@
|
||||
package com.firecrawl;
|
||||
|
||||
import com.firecrawl.client.FirecrawlClient;
|
||||
import com.firecrawl.models.AgentOptions;
|
||||
import com.firecrawl.models.AgentResponse;
|
||||
import com.firecrawl.models.AgentStatusResponse;
|
||||
import org.junit.jupiter.api.BeforeAll;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.*;
|
||||
|
||||
/**
|
||||
* Comprehensive Agent Tests
|
||||
*
|
||||
* Tests the AI agent functionality with various configurations.
|
||||
* Based on Node.js SDK patterns and tested against live firecrawl.dev.
|
||||
*
|
||||
* Run with: FIRECRAWL_API_KEY=fc-xxx gradle test --tests "com.firecrawl.AgentTest"
|
||||
*/
|
||||
class AgentTest {
|
||||
|
||||
private static FirecrawlClient client;
|
||||
|
||||
@BeforeAll
|
||||
static void setup() {
|
||||
String apiKey = System.getenv("FIRECRAWL_API_KEY");
|
||||
if (apiKey != null && !apiKey.isBlank()) {
|
||||
client = FirecrawlClient.fromEnv();
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
@EnabledIfEnvironmentVariable(named = "FIRECRAWL_API_KEY", matches = ".*\\S.*")
|
||||
void testAgentWithPrompt() {
|
||||
System.out.println("\n=== Test: Agent with Prompt ===");
|
||||
|
||||
AgentStatusResponse result = client.agent(
|
||||
AgentOptions.builder()
|
||||
.prompt("Find information about Firecrawl's main features and pricing")
|
||||
.build());
|
||||
|
||||
assertNotNull(result, "Agent result should not be null");
|
||||
assertNotNull(result.getStatus(), "Status should not be null");
|
||||
assertTrue(List.of("completed", "failed").contains(result.getStatus()),
|
||||
"Status should be completed or failed: " + result.getStatus());
|
||||
|
||||
System.out.println("✓ Agent task completed");
|
||||
System.out.println(" Status: " + result.getStatus());
|
||||
if (result.getData() != null) {
|
||||
System.out.println(" Data returned: ✓");
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
@EnabledIfEnvironmentVariable(named = "FIRECRAWL_API_KEY", matches = ".*\\S.*")
|
||||
void testAgentWithURLs() {
|
||||
System.out.println("\n=== Test: Agent with Specific URLs ===");
|
||||
|
||||
AgentStatusResponse result = client.agent(
|
||||
AgentOptions.builder()
|
||||
.urls(List.of("https://firecrawl.dev", "https://docs.firecrawl.dev"))
|
||||
.prompt("What are the main features of Firecrawl?")
|
||||
.build());
|
||||
|
||||
assertNotNull(result, "Agent result should not be null");
|
||||
assertTrue(List.of("completed", "failed").contains(result.getStatus()),
|
||||
"Status should be completed or failed");
|
||||
|
||||
System.out.println("✓ Agent with URLs completed");
|
||||
System.out.println(" URLs provided: 2");
|
||||
System.out.println(" Status: " + result.getStatus());
|
||||
}
|
||||
|
||||
@Test
|
||||
@EnabledIfEnvironmentVariable(named = "FIRECRAWL_API_KEY", matches = ".*\\S.*")
|
||||
void testAgentWithSchema() {
|
||||
System.out.println("\n=== Test: Agent with Schema ===");
|
||||
|
||||
Map<String, Object> schema = Map.of(
|
||||
"type", "object",
|
||||
"properties", Map.of(
|
||||
"features", Map.of(
|
||||
"type", "array",
|
||||
"items", Map.of("type", "string")
|
||||
),
|
||||
"pricing", Map.of(
|
||||
"type", "object",
|
||||
"properties", Map.of(
|
||||
"plans", Map.of("type", "array")
|
||||
)
|
||||
)
|
||||
),
|
||||
"required", List.of("features")
|
||||
);
|
||||
|
||||
AgentStatusResponse result = client.agent(
|
||||
AgentOptions.builder()
|
||||
.urls(List.of("https://firecrawl.dev"))
|
||||
.prompt("Extract features and pricing information")
|
||||
.schema(schema)
|
||||
.build());
|
||||
|
||||
assertNotNull(result, "Agent result should not be null");
|
||||
assertTrue(List.of("completed", "failed").contains(result.getStatus()),
|
||||
"Status should be completed or failed");
|
||||
|
||||
System.out.println("✓ Agent with schema completed");
|
||||
System.out.println(" Schema provided: ✓");
|
||||
System.out.println(" Status: " + result.getStatus());
|
||||
}
|
||||
|
||||
@Test
|
||||
@EnabledIfEnvironmentVariable(named = "FIRECRAWL_API_KEY", matches = ".*\\S.*")
|
||||
void testStartAgent() {
|
||||
System.out.println("\n=== Test: Start Agent (Async) ===");
|
||||
|
||||
AgentResponse response = client.startAgent(
|
||||
AgentOptions.builder()
|
||||
.prompt("Research Firecrawl features")
|
||||
.build());
|
||||
|
||||
assertNotNull(response, "Agent response should not be null");
|
||||
assertNotNull(response.getId(), "Agent ID should not be null");
|
||||
assertTrue(response.isSuccess(), "Response should be successful");
|
||||
|
||||
System.out.println("✓ Agent started successfully");
|
||||
System.out.println(" Job ID: " + response.getId());
|
||||
System.out.println(" Success: " + response.isSuccess());
|
||||
}
|
||||
|
||||
@Test
|
||||
@EnabledIfEnvironmentVariable(named = "FIRECRAWL_API_KEY", matches = ".*\\S.*")
|
||||
void testAgentStatusCheck() {
|
||||
System.out.println("\n=== Test: Check Agent Status ===");
|
||||
|
||||
// Start an agent
|
||||
AgentResponse start = client.startAgent(
|
||||
AgentOptions.builder()
|
||||
.prompt("Find information about web scraping")
|
||||
.build());
|
||||
|
||||
// Check status
|
||||
AgentStatusResponse status = client.getAgentStatus(start.getId());
|
||||
|
||||
assertNotNull(status, "Status should not be null");
|
||||
assertNotNull(status.getStatus(), "Status field should not be null");
|
||||
assertTrue(List.of("scraping", "completed", "failed", "cancelled").contains(status.getStatus()),
|
||||
"Status should be valid: " + status.getStatus());
|
||||
|
||||
System.out.println("✓ Agent status retrieved");
|
||||
System.out.println(" Status: " + status.getStatus());
|
||||
System.out.println(" Job ID: " + start.getId());
|
||||
}
|
||||
|
||||
@Test
|
||||
@EnabledIfEnvironmentVariable(named = "FIRECRAWL_API_KEY", matches = ".*\\S.*")
|
||||
void testCancelAgent() {
|
||||
System.out.println("\n=== Test: Cancel Agent ===");
|
||||
|
||||
AgentResponse start = client.startAgent(
|
||||
AgentOptions.builder()
|
||||
.prompt("Long-running research task")
|
||||
.build());
|
||||
|
||||
Map<String, Object> result = client.cancelAgent(start.getId());
|
||||
|
||||
assertNotNull(result, "Cancel result should not be null");
|
||||
|
||||
System.out.println("✓ Agent cancelled successfully");
|
||||
System.out.println(" Job ID: " + start.getId());
|
||||
}
|
||||
|
||||
@Test
|
||||
@EnabledIfEnvironmentVariable(named = "FIRECRAWL_API_KEY", matches = ".*\\S.*")
|
||||
void testAgentWithStrictURLConstraints() {
|
||||
System.out.println("\n=== Test: Agent with Strict URL Constraints ===");
|
||||
|
||||
AgentStatusResponse result = client.agent(
|
||||
AgentOptions.builder()
|
||||
.urls(List.of("https://docs.firecrawl.dev"))
|
||||
.prompt("Extract API documentation structure")
|
||||
.strictConstrainToURLs(true)
|
||||
.build());
|
||||
|
||||
assertNotNull(result, "Agent result should not be null");
|
||||
assertTrue(List.of("completed", "failed").contains(result.getStatus()),
|
||||
"Status should be completed or failed");
|
||||
|
||||
System.out.println("✓ Agent with strict constraints completed");
|
||||
System.out.println(" Strict URL constraint: true");
|
||||
System.out.println(" Status: " + result.getStatus());
|
||||
}
|
||||
|
||||
@Test
|
||||
@EnabledIfEnvironmentVariable(named = "FIRECRAWL_API_KEY", matches = ".*\\S.*")
|
||||
void testAgentWithMaxCredits() {
|
||||
System.out.println("\n=== Test: Agent with Max Credits Limit ===");
|
||||
|
||||
AgentStatusResponse result = client.agent(
|
||||
AgentOptions.builder()
|
||||
.prompt("Quick research on Firecrawl")
|
||||
.maxCredits(10)
|
||||
.build());
|
||||
|
||||
assertNotNull(result, "Agent result should not be null");
|
||||
|
||||
System.out.println("✓ Agent with credit limit completed");
|
||||
System.out.println(" Max credits: 10");
|
||||
System.out.println(" Status: " + result.getStatus());
|
||||
}
|
||||
|
||||
@Test
|
||||
@EnabledIfEnvironmentVariable(named = "FIRECRAWL_API_KEY", matches = ".*\\S.*")
|
||||
void testAgentResearchTask() {
|
||||
System.out.println("\n=== Test: Agent Research - Firecrawl Features ===");
|
||||
|
||||
AgentStatusResponse result = client.agent(
|
||||
AgentOptions.builder()
|
||||
.urls(List.of("https://firecrawl.dev", "https://docs.firecrawl.dev"))
|
||||
.prompt("Research and summarize the key features of Firecrawl, including scraping, crawling, and extraction capabilities")
|
||||
.build());
|
||||
|
||||
assertNotNull(result, "Agent result should not be null");
|
||||
assertEquals("completed", result.getStatus(), "Agent should complete successfully");
|
||||
assertNotNull(result.getData(), "Agent should return data");
|
||||
|
||||
System.out.println("✓ Research task completed");
|
||||
System.out.println(" Status: " + result.getStatus());
|
||||
System.out.println(" Data collected: ✓");
|
||||
|
||||
if (result.getData() != null) {
|
||||
System.out.println(" Data summary: " +
|
||||
result.getData().toString().substring(0,
|
||||
Math.min(200, result.getData().toString().length())) + "...");
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
@EnabledIfEnvironmentVariable(named = "FIRECRAWL_API_KEY", matches = ".*\\S.*")
|
||||
void testAgentComprehensive() {
|
||||
System.out.println("\n=== Test: Agent with All Options ===");
|
||||
|
||||
Map<String, Object> schema = Map.of(
|
||||
"type", "object",
|
||||
"properties", Map.of(
|
||||
"product_name", Map.of("type", "string"),
|
||||
"features", Map.of(
|
||||
"type", "array",
|
||||
"items", Map.of("type", "string")
|
||||
),
|
||||
"pricing", Map.of("type", "string")
|
||||
),
|
||||
"required", List.of("product_name", "features")
|
||||
);
|
||||
|
||||
AgentStatusResponse result = client.agent(
|
||||
AgentOptions.builder()
|
||||
.urls(List.of("https://firecrawl.dev"))
|
||||
.prompt("Extract comprehensive product information including name, features, and pricing")
|
||||
.schema(schema)
|
||||
.maxCredits(20)
|
||||
.strictConstrainToURLs(true)
|
||||
.build());
|
||||
|
||||
assertNotNull(result, "Agent result should not be null");
|
||||
assertTrue(List.of("completed", "failed").contains(result.getStatus()),
|
||||
"Status should be completed or failed");
|
||||
|
||||
System.out.println("✓ Comprehensive agent task completed");
|
||||
System.out.println(" Configuration:");
|
||||
System.out.println(" - URLs: 1");
|
||||
System.out.println(" - Schema: ✓");
|
||||
System.out.println(" - Max credits: 20");
|
||||
System.out.println(" - Strict constraints: true");
|
||||
System.out.println(" Results:");
|
||||
System.out.println(" - Status: " + result.getStatus());
|
||||
if (result.getData() != null) {
|
||||
System.out.println(" - Data returned: ✓");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,310 @@
|
||||
package com.firecrawl;
|
||||
|
||||
import com.firecrawl.client.FirecrawlClient;
|
||||
import com.firecrawl.models.BrowserCreateResponse;
|
||||
import com.firecrawl.models.BrowserDeleteResponse;
|
||||
import com.firecrawl.models.BrowserExecuteResponse;
|
||||
import com.firecrawl.models.BrowserListResponse;
|
||||
import org.junit.jupiter.api.BeforeAll;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.*;
|
||||
|
||||
/**
|
||||
* Browser Sandbox Endpoint Tests
|
||||
*
|
||||
* Tests the browser session management functionality of the Firecrawl Java SDK.
|
||||
* These tests require FIRECRAWL_API_KEY environment variable to be set.
|
||||
*
|
||||
* Run with: FIRECRAWL_API_KEY=fc-xxx gradle test --tests "com.firecrawl.BrowserTest"
|
||||
*/
|
||||
class BrowserTest {
|
||||
|
||||
private static FirecrawlClient client;
|
||||
|
||||
@BeforeAll
|
||||
static void setup() {
|
||||
String apiKey = System.getenv("FIRECRAWL_API_KEY");
|
||||
if (apiKey != null && !apiKey.isBlank()) {
|
||||
client = FirecrawlClient.fromEnv();
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
@EnabledIfEnvironmentVariable(named = "FIRECRAWL_API_KEY", matches = ".*\\S.*")
|
||||
void testBrowserCreateAndDelete() {
|
||||
System.out.println("Testing browser session create and delete...");
|
||||
|
||||
// Create a browser session
|
||||
BrowserCreateResponse createRes = client.browser();
|
||||
assertNotNull(createRes, "Create response should not be null");
|
||||
assertTrue(createRes.isSuccess(), "Create should succeed");
|
||||
assertNotNull(createRes.getId(), "Session ID should not be null");
|
||||
|
||||
String sessionId = createRes.getId();
|
||||
System.out.println(" Created session: " + sessionId);
|
||||
|
||||
// Delete the browser session
|
||||
BrowserDeleteResponse deleteRes = client.deleteBrowser(sessionId);
|
||||
assertNotNull(deleteRes, "Delete response should not be null");
|
||||
assertTrue(deleteRes.isSuccess(), "Delete should succeed");
|
||||
|
||||
System.out.println("✓ Browser create and delete test passed");
|
||||
}
|
||||
|
||||
@Test
|
||||
@EnabledIfEnvironmentVariable(named = "FIRECRAWL_API_KEY", matches = ".*\\S.*")
|
||||
void testBrowserCreateWithOptions() {
|
||||
System.out.println("Testing browser session create with options...");
|
||||
|
||||
// Create a session with custom TTL and activity TTL
|
||||
BrowserCreateResponse createRes = client.browser(300, 120, true);
|
||||
assertNotNull(createRes, "Create response should not be null");
|
||||
assertTrue(createRes.isSuccess(), "Create should succeed");
|
||||
assertNotNull(createRes.getId(), "Session ID should not be null");
|
||||
|
||||
String sessionId = createRes.getId();
|
||||
System.out.println(" Created session with options: " + sessionId);
|
||||
|
||||
// Clean up
|
||||
client.deleteBrowser(sessionId);
|
||||
|
||||
System.out.println("✓ Browser create with options test passed");
|
||||
}
|
||||
|
||||
@Test
|
||||
@EnabledIfEnvironmentVariable(named = "FIRECRAWL_API_KEY", matches = ".*\\S.*")
|
||||
void testBrowserExecuteBash() {
|
||||
System.out.println("Testing browser execute with bash...");
|
||||
|
||||
// Create a session
|
||||
BrowserCreateResponse createRes = client.browser();
|
||||
assertTrue(createRes.isSuccess(), "Create should succeed");
|
||||
String sessionId = createRes.getId();
|
||||
|
||||
try {
|
||||
// Execute bash code
|
||||
BrowserExecuteResponse execRes = client.browserExecute(sessionId, "echo 'hello from java sdk'");
|
||||
assertNotNull(execRes, "Execute response should not be null");
|
||||
assertTrue(execRes.isSuccess(), "Execute should succeed");
|
||||
assertNotNull(execRes.getStdout(), "Stdout should not be null");
|
||||
assertTrue(execRes.getStdout().contains("hello from java sdk"),
|
||||
"Stdout should contain our echo output");
|
||||
|
||||
System.out.println(" Stdout: " + execRes.getStdout().trim());
|
||||
System.out.println(" Exit code: " + execRes.getExitCode());
|
||||
} finally {
|
||||
// Clean up
|
||||
client.deleteBrowser(sessionId);
|
||||
}
|
||||
|
||||
System.out.println("✓ Browser execute bash test passed");
|
||||
}
|
||||
|
||||
@Test
|
||||
@EnabledIfEnvironmentVariable(named = "FIRECRAWL_API_KEY", matches = ".*\\S.*")
|
||||
void testBrowserExecuteNode() {
|
||||
System.out.println("Testing browser execute with node...");
|
||||
|
||||
// Create a session
|
||||
BrowserCreateResponse createRes = client.browser();
|
||||
assertTrue(createRes.isSuccess(), "Create should succeed");
|
||||
String sessionId = createRes.getId();
|
||||
|
||||
try {
|
||||
// Execute node code
|
||||
BrowserExecuteResponse execRes = client.browserExecute(
|
||||
sessionId, "console.log(1 + 2)", "node", null);
|
||||
assertNotNull(execRes, "Execute response should not be null");
|
||||
assertTrue(execRes.isSuccess(), "Execute should succeed");
|
||||
|
||||
System.out.println(" Stdout: " + (execRes.getStdout() != null ? execRes.getStdout().trim() : "null"));
|
||||
} finally {
|
||||
client.deleteBrowser(sessionId);
|
||||
}
|
||||
|
||||
System.out.println("✓ Browser execute node test passed");
|
||||
}
|
||||
|
||||
@Test
|
||||
@EnabledIfEnvironmentVariable(named = "FIRECRAWL_API_KEY", matches = ".*\\S.*")
|
||||
void testBrowserExecutePython() {
|
||||
System.out.println("Testing browser execute with python...");
|
||||
|
||||
// Create a session
|
||||
BrowserCreateResponse createRes = client.browser();
|
||||
assertTrue(createRes.isSuccess(), "Create should succeed");
|
||||
String sessionId = createRes.getId();
|
||||
|
||||
try {
|
||||
// Execute python code
|
||||
BrowserExecuteResponse execRes = client.browserExecute(
|
||||
sessionId, "print('hello from python')", "python", null);
|
||||
assertNotNull(execRes, "Execute response should not be null");
|
||||
assertTrue(execRes.isSuccess(), "Execute should succeed");
|
||||
|
||||
System.out.println(" Stdout: " + (execRes.getStdout() != null ? execRes.getStdout().trim() : "null"));
|
||||
} finally {
|
||||
client.deleteBrowser(sessionId);
|
||||
}
|
||||
|
||||
System.out.println("✓ Browser execute python test passed");
|
||||
}
|
||||
|
||||
@Test
|
||||
@EnabledIfEnvironmentVariable(named = "FIRECRAWL_API_KEY", matches = ".*\\S.*")
|
||||
void testBrowserExecuteWithTimeout() {
|
||||
System.out.println("Testing browser execute with custom timeout...");
|
||||
|
||||
// Create a session
|
||||
BrowserCreateResponse createRes = client.browser();
|
||||
assertTrue(createRes.isSuccess(), "Create should succeed");
|
||||
String sessionId = createRes.getId();
|
||||
|
||||
try {
|
||||
// Execute with custom timeout (60 seconds)
|
||||
BrowserExecuteResponse execRes = client.browserExecute(
|
||||
sessionId, "echo 'timeout test'", "bash", 60);
|
||||
assertNotNull(execRes, "Execute response should not be null");
|
||||
assertTrue(execRes.isSuccess(), "Execute should succeed");
|
||||
|
||||
System.out.println(" Stdout: " + (execRes.getStdout() != null ? execRes.getStdout().trim() : "null"));
|
||||
} finally {
|
||||
client.deleteBrowser(sessionId);
|
||||
}
|
||||
|
||||
System.out.println("✓ Browser execute with timeout test passed");
|
||||
}
|
||||
|
||||
@Test
|
||||
@EnabledIfEnvironmentVariable(named = "FIRECRAWL_API_KEY", matches = ".*\\S.*")
|
||||
void testBrowserListSessions() {
|
||||
System.out.println("Testing list browser sessions...");
|
||||
|
||||
// List all sessions
|
||||
BrowserListResponse listRes = client.listBrowsers();
|
||||
assertNotNull(listRes, "List response should not be null");
|
||||
assertTrue(listRes.isSuccess(), "List should succeed");
|
||||
|
||||
System.out.println(" Total sessions: " + (listRes.getSessions() != null ? listRes.getSessions().size() : 0));
|
||||
|
||||
System.out.println("✓ List browser sessions test passed");
|
||||
}
|
||||
|
||||
@Test
|
||||
@EnabledIfEnvironmentVariable(named = "FIRECRAWL_API_KEY", matches = ".*\\S.*")
|
||||
void testBrowserListActiveFilter() {
|
||||
System.out.println("Testing list browser sessions with active filter...");
|
||||
|
||||
// Create a session so we have at least one active
|
||||
BrowserCreateResponse createRes = client.browser();
|
||||
assertTrue(createRes.isSuccess(), "Create should succeed");
|
||||
String sessionId = createRes.getId();
|
||||
|
||||
try {
|
||||
// List only active sessions
|
||||
BrowserListResponse listRes = client.listBrowsers("active");
|
||||
assertNotNull(listRes, "List response should not be null");
|
||||
assertTrue(listRes.isSuccess(), "List should succeed");
|
||||
assertNotNull(listRes.getSessions(), "Sessions list should not be null");
|
||||
assertFalse(listRes.getSessions().isEmpty(), "Should have at least one active session");
|
||||
|
||||
System.out.println(" Active sessions: " + listRes.getSessions().size());
|
||||
} finally {
|
||||
client.deleteBrowser(sessionId);
|
||||
}
|
||||
|
||||
System.out.println("✓ List active browser sessions test passed");
|
||||
}
|
||||
|
||||
@Test
|
||||
@EnabledIfEnvironmentVariable(named = "FIRECRAWL_API_KEY", matches = ".*\\S.*")
|
||||
void testBrowserFullLifecycle() {
|
||||
System.out.println("Testing full browser session lifecycle...");
|
||||
|
||||
// 1. Create session
|
||||
BrowserCreateResponse createRes = client.browser(300, 120, true);
|
||||
assertTrue(createRes.isSuccess(), "Create should succeed");
|
||||
assertNotNull(createRes.getId(), "Should have session ID");
|
||||
String sessionId = createRes.getId();
|
||||
System.out.println(" 1. Created session: " + sessionId);
|
||||
|
||||
// CDP URL and live view URL may be present
|
||||
if (createRes.getCdpUrl() != null) {
|
||||
System.out.println(" CDP URL present: true");
|
||||
}
|
||||
if (createRes.getLiveViewUrl() != null) {
|
||||
System.out.println(" Live View URL present: true");
|
||||
}
|
||||
|
||||
// 2. Navigate to a page
|
||||
BrowserExecuteResponse navRes = client.browserExecute(
|
||||
sessionId, "agent-browser open https://example.com", "bash", 30);
|
||||
assertTrue(navRes.isSuccess(), "Navigation should succeed");
|
||||
System.out.println(" 2. Navigated to example.com");
|
||||
|
||||
// 3. Take a snapshot
|
||||
BrowserExecuteResponse snapRes = client.browserExecute(
|
||||
sessionId, "agent-browser snapshot -i -c", "bash", 30);
|
||||
assertTrue(snapRes.isSuccess(), "Snapshot should succeed");
|
||||
System.out.println(" 3. Took snapshot");
|
||||
|
||||
// 4. Get page title
|
||||
BrowserExecuteResponse titleRes = client.browserExecute(
|
||||
sessionId, "agent-browser get title", "bash", 30);
|
||||
assertTrue(titleRes.isSuccess(), "Get title should succeed");
|
||||
System.out.println(" 4. Page title: " + (titleRes.getStdout() != null ? titleRes.getStdout().trim() : "null"));
|
||||
|
||||
// 5. Verify session is active
|
||||
BrowserListResponse listRes = client.listBrowsers("active");
|
||||
assertTrue(listRes.isSuccess(), "List should succeed");
|
||||
System.out.println(" 5. Active sessions: " + (listRes.getSessions() != null ? listRes.getSessions().size() : 0));
|
||||
|
||||
// 6. Delete session
|
||||
BrowserDeleteResponse deleteRes = client.deleteBrowser(sessionId);
|
||||
assertTrue(deleteRes.isSuccess(), "Delete should succeed");
|
||||
System.out.println(" 6. Deleted session");
|
||||
if (deleteRes.getSessionDurationMs() != null) {
|
||||
System.out.println(" Session duration: " + deleteRes.getSessionDurationMs() + "ms");
|
||||
}
|
||||
if (deleteRes.getCreditsBilled() != null) {
|
||||
System.out.println(" Credits billed: " + deleteRes.getCreditsBilled());
|
||||
}
|
||||
|
||||
System.out.println("✓ Full browser session lifecycle test passed");
|
||||
}
|
||||
|
||||
@Test
|
||||
void testBrowserExecuteRequiresSessionId() {
|
||||
FirecrawlClient testClient = FirecrawlClient.builder()
|
||||
.apiKey("fc-test-key")
|
||||
.build();
|
||||
|
||||
assertThrows(NullPointerException.class, () ->
|
||||
testClient.browserExecute(null, "echo test")
|
||||
);
|
||||
}
|
||||
|
||||
@Test
|
||||
void testBrowserExecuteRequiresCode() {
|
||||
FirecrawlClient testClient = FirecrawlClient.builder()
|
||||
.apiKey("fc-test-key")
|
||||
.build();
|
||||
|
||||
assertThrows(NullPointerException.class, () ->
|
||||
testClient.browserExecute("some-session-id", null)
|
||||
);
|
||||
}
|
||||
|
||||
@Test
|
||||
void testBrowserDeleteRequiresSessionId() {
|
||||
FirecrawlClient testClient = FirecrawlClient.builder()
|
||||
.apiKey("fc-test-key")
|
||||
.build();
|
||||
|
||||
assertThrows(NullPointerException.class, () ->
|
||||
testClient.deleteBrowser(null)
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,302 @@
|
||||
package com.firecrawl;
|
||||
|
||||
import com.firecrawl.client.FirecrawlClient;
|
||||
import com.firecrawl.errors.FirecrawlException;
|
||||
import com.firecrawl.models.*;
|
||||
import org.junit.jupiter.api.BeforeAll;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.*;
|
||||
|
||||
/**
|
||||
* Comprehensive Crawl Tests
|
||||
*
|
||||
* Tests the crawl functionality with various configurations.
|
||||
* Based on Node.js SDK patterns and tested against live firecrawl.dev.
|
||||
*
|
||||
* Run with: FIRECRAWL_API_KEY=fc-xxx gradle test --tests "com.firecrawl.CrawlTest"
|
||||
*/
|
||||
class CrawlTest {
|
||||
|
||||
private static FirecrawlClient client;
|
||||
|
||||
@BeforeAll
|
||||
static void setup() {
|
||||
String apiKey = System.getenv("FIRECRAWL_API_KEY");
|
||||
if (apiKey != null && !apiKey.isBlank()) {
|
||||
client = FirecrawlClient.fromEnv();
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
@EnabledIfEnvironmentVariable(named = "FIRECRAWL_API_KEY", matches = ".*\\S.*")
|
||||
void testStartCrawlMinimal() {
|
||||
System.out.println("\n=== Test: Start Crawl - Minimal Request ===");
|
||||
|
||||
CrawlResponse response = client.startCrawl("https://docs.firecrawl.dev",
|
||||
CrawlOptions.builder()
|
||||
.limit(3)
|
||||
.build());
|
||||
|
||||
assertNotNull(response, "Crawl response should not be null");
|
||||
assertNotNull(response.getId(), "Crawl ID should not be null");
|
||||
assertNotNull(response.getUrl(), "Crawl URL should not be null");
|
||||
|
||||
System.out.println("✓ Crawl started successfully");
|
||||
System.out.println(" Job ID: " + response.getId());
|
||||
System.out.println(" Status URL: " + response.getUrl());
|
||||
}
|
||||
|
||||
@Test
|
||||
@EnabledIfEnvironmentVariable(named = "FIRECRAWL_API_KEY", matches = ".*\\S.*")
|
||||
void testStartCrawlWithOptions() {
|
||||
System.out.println("\n=== Test: Start Crawl - With Options ===");
|
||||
|
||||
CrawlResponse response = client.startCrawl("https://docs.firecrawl.dev",
|
||||
CrawlOptions.builder()
|
||||
.limit(5)
|
||||
.maxDiscoveryDepth(2)
|
||||
.build());
|
||||
|
||||
assertNotNull(response.getId(), "Job ID should not be null");
|
||||
assertNotNull(response.getUrl(), "Status URL should not be null");
|
||||
|
||||
System.out.println("✓ Crawl with options started");
|
||||
System.out.println(" Limit: 5 pages");
|
||||
System.out.println(" Max depth: 2");
|
||||
System.out.println(" Job ID: " + response.getId());
|
||||
}
|
||||
|
||||
@Test
|
||||
@EnabledIfEnvironmentVariable(named = "FIRECRAWL_API_KEY", matches = ".*\\S.*")
|
||||
void testGetCrawlStatus() {
|
||||
System.out.println("\n=== Test: Get Crawl Status ===");
|
||||
|
||||
// Start a crawl
|
||||
CrawlResponse start = client.startCrawl("https://docs.firecrawl.dev",
|
||||
CrawlOptions.builder()
|
||||
.limit(3)
|
||||
.build());
|
||||
|
||||
System.out.println("CrawlResponse: " + start);
|
||||
System.out.println("ID: " + start.getId());
|
||||
assertNotNull(start, "CrawlResponse should not be null");
|
||||
assertNotNull(start.getId(), "Crawl ID should not be null");
|
||||
|
||||
// Get status
|
||||
CrawlJob status = client.getCrawlStatus(start.getId());
|
||||
|
||||
assertNotNull(status, "Status should not be null");
|
||||
assertNotNull(status.getStatus(), "Status should not be null");
|
||||
assertTrue(List.of("scraping", "completed", "failed", "cancelled").contains(status.getStatus()),
|
||||
"Status should be valid: " + status.getStatus());
|
||||
assertTrue(status.getCompleted() >= 0, "Completed count should be non-negative");
|
||||
// Data may be null while crawl is still in progress (status=scraping)
|
||||
if ("completed".equals(status.getStatus())) {
|
||||
assertNotNull(status.getData(), "Data should not be null when completed");
|
||||
}
|
||||
|
||||
System.out.println("✓ Status retrieved successfully");
|
||||
System.out.println(" Status: " + status.getStatus());
|
||||
System.out.println(" Completed: " + status.getCompleted() + "/" + status.getTotal());
|
||||
System.out.println(" Documents: " + (status.getData() != null ? status.getData().size() : 0));
|
||||
}
|
||||
|
||||
@Test
|
||||
@EnabledIfEnvironmentVariable(named = "FIRECRAWL_API_KEY", matches = ".*\\S.*")
|
||||
void testCancelCrawl() {
|
||||
System.out.println("\n=== Test: Cancel Crawl ===");
|
||||
|
||||
CrawlResponse start = client.startCrawl("https://docs.firecrawl.dev",
|
||||
CrawlOptions.builder()
|
||||
.limit(10)
|
||||
.build());
|
||||
|
||||
Map<String, Object> result = client.cancelCrawl(start.getId());
|
||||
|
||||
assertNotNull(result, "Cancel result should not be null");
|
||||
|
||||
System.out.println("✓ Crawl cancelled successfully");
|
||||
System.out.println(" Job ID: " + start.getId());
|
||||
}
|
||||
|
||||
@Test
|
||||
@EnabledIfEnvironmentVariable(named = "FIRECRAWL_API_KEY", matches = ".*\\S.*")
|
||||
void testCrawlWithWait() {
|
||||
System.out.println("\n=== Test: Crawl with Wait (Blocking) ===");
|
||||
|
||||
CrawlJob job = client.crawl("https://firecrawl.dev",
|
||||
CrawlOptions.builder()
|
||||
.limit(3)
|
||||
.maxDiscoveryDepth(1)
|
||||
.build(),
|
||||
2, // pollInterval in seconds
|
||||
120 // timeout in seconds
|
||||
);
|
||||
|
||||
assertNotNull(job, "Job should not be null");
|
||||
assertTrue(List.of("completed", "failed").contains(job.getStatus()),
|
||||
"Final status should be completed or failed: " + job.getStatus());
|
||||
assertTrue(job.getCompleted() >= 0, "Completed count should be non-negative");
|
||||
assertTrue(job.getTotal() >= 0, "Total count should be non-negative");
|
||||
assertNotNull(job.getData(), "Data should not be null");
|
||||
|
||||
System.out.println("✓ Crawl completed (with wait)");
|
||||
System.out.println(" Final status: " + job.getStatus());
|
||||
System.out.println(" Pages crawled: " + job.getCompleted() + "/" + job.getTotal());
|
||||
System.out.println(" Documents returned: " + job.getData().size());
|
||||
|
||||
if (!job.getData().isEmpty()) {
|
||||
Document firstDoc = job.getData().get(0);
|
||||
System.out.println(" Sample URL: " + firstDoc.getMetadata().get("sourceURL"));
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
@EnabledIfEnvironmentVariable(named = "FIRECRAWL_API_KEY", matches = ".*\\S.*")
|
||||
void testCrawlWithScrapeOptions() {
|
||||
System.out.println("\n=== Test: Crawl with Scrape Options ===");
|
||||
|
||||
CrawlResponse response = client.startCrawl("https://docs.firecrawl.dev",
|
||||
CrawlOptions.builder()
|
||||
.limit(2)
|
||||
.scrapeOptions(ScrapeOptions.builder()
|
||||
.formats(List.of("markdown", "links"))
|
||||
.onlyMainContent(true)
|
||||
.build())
|
||||
.build());
|
||||
|
||||
assertNotNull(response.getId(), "Job ID should not be null");
|
||||
|
||||
System.out.println("✓ Crawl with scrape options started");
|
||||
System.out.println(" Formats: markdown, links");
|
||||
System.out.println(" Only main content: true");
|
||||
System.out.println(" Job ID: " + response.getId());
|
||||
}
|
||||
|
||||
@Test
|
||||
@EnabledIfEnvironmentVariable(named = "FIRECRAWL_API_KEY", matches = ".*\\S.*")
|
||||
void testCrawlWithExcludePaths() {
|
||||
System.out.println("\n=== Test: Crawl with Exclude Paths ===");
|
||||
|
||||
CrawlResponse response = client.startCrawl("https://docs.firecrawl.dev",
|
||||
CrawlOptions.builder()
|
||||
.limit(5)
|
||||
.excludePaths(List.of("/blog/*", "/admin/*"))
|
||||
.build());
|
||||
|
||||
assertNotNull(response.getId(), "Job ID should not be null");
|
||||
|
||||
System.out.println("✓ Crawl with exclude paths started");
|
||||
System.out.println(" Excluding: /blog/*, /admin/*");
|
||||
System.out.println(" Job ID: " + response.getId());
|
||||
}
|
||||
|
||||
@Test
|
||||
@EnabledIfEnvironmentVariable(named = "FIRECRAWL_API_KEY", matches = ".*\\S.*")
|
||||
void testCrawlWithIncludePaths() {
|
||||
System.out.println("\n=== Test: Crawl with Include Paths ===");
|
||||
|
||||
CrawlResponse response = client.startCrawl("https://docs.firecrawl.dev",
|
||||
CrawlOptions.builder()
|
||||
.limit(5)
|
||||
.includePaths(List.of("/docs/*"))
|
||||
.build());
|
||||
|
||||
assertNotNull(response.getId(), "Job ID should not be null");
|
||||
|
||||
System.out.println("✓ Crawl with include paths started");
|
||||
System.out.println(" Including only: /docs/*");
|
||||
System.out.println(" Job ID: " + response.getId());
|
||||
}
|
||||
|
||||
@Test
|
||||
@EnabledIfEnvironmentVariable(named = "FIRECRAWL_API_KEY", matches = ".*\\S.*")
|
||||
void testCrawlWithAllowExternalLinks() {
|
||||
System.out.println("\n=== Test: Crawl with Allow External Links ===");
|
||||
|
||||
CrawlResponse response = client.startCrawl("https://docs.firecrawl.dev",
|
||||
CrawlOptions.builder()
|
||||
.limit(5)
|
||||
.allowExternalLinks(true)
|
||||
.build());
|
||||
|
||||
assertNotNull(response.getId(), "Job ID should not be null");
|
||||
|
||||
System.out.println("✓ Crawl with external links allowed");
|
||||
System.out.println(" Job ID: " + response.getId());
|
||||
}
|
||||
|
||||
@Test
|
||||
@EnabledIfEnvironmentVariable(named = "FIRECRAWL_API_KEY", matches = ".*\\S.*")
|
||||
void testCrawlWithWebhookConfig() {
|
||||
System.out.println("\n=== Test: Crawl with Webhook (if available) ===");
|
||||
|
||||
try {
|
||||
// Using a test webhook URL (requestbin, webhook.site, etc.)
|
||||
CrawlResponse response = client.startCrawl("https://firecrawl.dev",
|
||||
CrawlOptions.builder()
|
||||
.limit(2)
|
||||
.webhook(WebhookConfig.builder()
|
||||
.url("https://webhook.site/test")
|
||||
.build())
|
||||
.build());
|
||||
|
||||
assertNotNull(response.getId(), "Job ID should not be null");
|
||||
|
||||
System.out.println("✓ Crawl with webhook started");
|
||||
System.out.println(" Job ID: " + response.getId());
|
||||
} catch (Exception e) {
|
||||
System.out.println("⚠ Webhook test skipped or failed: " + e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
@EnabledIfEnvironmentVariable(named = "FIRECRAWL_API_KEY", matches = ".*\\S.*")
|
||||
void testCrawlFirecrawlHomepage() {
|
||||
System.out.println("\n=== Test: Crawl Firecrawl.dev Homepage ===");
|
||||
|
||||
CrawlJob job = client.crawl("https://firecrawl.dev",
|
||||
CrawlOptions.builder()
|
||||
.limit(5)
|
||||
.maxDiscoveryDepth(2)
|
||||
.scrapeOptions(ScrapeOptions.builder()
|
||||
.formats(List.of("markdown"))
|
||||
.onlyMainContent(true)
|
||||
.build())
|
||||
.build(),
|
||||
2,
|
||||
120
|
||||
);
|
||||
|
||||
assertNotNull(job, "Job should not be null");
|
||||
assertTrue(job.getData() != null && !job.getData().isEmpty(),
|
||||
"Should have crawled at least one page");
|
||||
|
||||
// Verify content from Firecrawl site
|
||||
boolean hasFirecrawlContent = job.getData().stream()
|
||||
.anyMatch(doc -> {
|
||||
String markdown = doc.getMarkdown();
|
||||
return markdown != null &&
|
||||
(markdown.toLowerCase().contains("firecrawl") ||
|
||||
markdown.toLowerCase().contains("scrape") ||
|
||||
markdown.toLowerCase().contains("crawl"));
|
||||
});
|
||||
|
||||
assertTrue(hasFirecrawlContent, "Should contain Firecrawl-related content");
|
||||
|
||||
System.out.println("✓ Successfully crawled Firecrawl homepage");
|
||||
System.out.println(" Pages crawled: " + job.getData().size());
|
||||
System.out.println(" Status: " + job.getStatus());
|
||||
|
||||
// Print sample URLs
|
||||
System.out.println(" Sample pages:");
|
||||
job.getData().stream()
|
||||
.limit(3)
|
||||
.forEach(doc -> System.out.println(" - " + doc.getMetadata().get("sourceURL")));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,330 @@
|
||||
package com.firecrawl;
|
||||
|
||||
import com.firecrawl.client.FirecrawlClient;
|
||||
import com.firecrawl.errors.FirecrawlException;
|
||||
import com.firecrawl.models.*;
|
||||
import okhttp3.OkHttpClient;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable;
|
||||
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.*;
|
||||
|
||||
/**
|
||||
* Integration tests for the Firecrawl Java SDK.
|
||||
*
|
||||
* <p>These tests require a valid FIRECRAWL_API_KEY environment variable.
|
||||
* Run with: FIRECRAWL_API_KEY=fc-xxx ./gradlew test
|
||||
*/
|
||||
class FirecrawlClientTest {
|
||||
|
||||
@Test
|
||||
void testBuilderRequiresApiKey() {
|
||||
assertThrows(FirecrawlException.class, () ->
|
||||
FirecrawlClient.builder().apiKey("").build()
|
||||
);
|
||||
}
|
||||
|
||||
@Test
|
||||
void testBuilderRejectsExplicitNullApiKey() {
|
||||
assertThrows(FirecrawlException.class, () ->
|
||||
FirecrawlClient.builder().apiKey(null).build()
|
||||
);
|
||||
}
|
||||
|
||||
@Test
|
||||
void testBuilderAcceptsApiKey() {
|
||||
// Should not throw — just validates construction
|
||||
FirecrawlClient client = FirecrawlClient.builder()
|
||||
.apiKey("fc-test-key")
|
||||
.build();
|
||||
assertNotNull(client);
|
||||
}
|
||||
|
||||
@Test
|
||||
void testBuilderAcceptsCustomHttpClient() {
|
||||
OkHttpClient custom = new OkHttpClient.Builder()
|
||||
.connectTimeout(10, TimeUnit.SECONDS)
|
||||
.readTimeout(10, TimeUnit.SECONDS)
|
||||
.build();
|
||||
|
||||
FirecrawlClient client = FirecrawlClient.builder()
|
||||
.apiKey("fc-test-key")
|
||||
.httpClient(custom)
|
||||
.build();
|
||||
assertNotNull(client);
|
||||
}
|
||||
|
||||
@Test
|
||||
void testScrapeOptionsBuilder() {
|
||||
QueryFormat queryFormat = QueryFormat.builder()
|
||||
.prompt("What is Firecrawl?")
|
||||
.mode(QueryFormat.Mode.DIRECT_QUOTE)
|
||||
.build();
|
||||
|
||||
ScrapeOptions options = ScrapeOptions.builder()
|
||||
.formats(List.of("markdown", "html", queryFormat))
|
||||
.onlyMainContent(true)
|
||||
.timeout(30000)
|
||||
.mobile(false)
|
||||
.build();
|
||||
|
||||
assertEquals(List.of("markdown", "html", queryFormat), options.getFormats());
|
||||
assertEquals("query", queryFormat.getType());
|
||||
assertEquals(QueryFormat.Mode.DIRECT_QUOTE, queryFormat.getMode());
|
||||
assertTrue(options.getOnlyMainContent());
|
||||
assertEquals(30000, options.getTimeout());
|
||||
assertFalse(options.getMobile());
|
||||
}
|
||||
|
||||
@Test
|
||||
void testQuestionAndHighlightsFormats() {
|
||||
QuestionFormat questionFormat = QuestionFormat.builder()
|
||||
.question("What is Firecrawl?")
|
||||
.build();
|
||||
HighlightsFormat highlightsFormat = HighlightsFormat.builder()
|
||||
.query("What is Firecrawl?")
|
||||
.build();
|
||||
|
||||
ScrapeOptions options = ScrapeOptions.builder()
|
||||
.formats(List.of(questionFormat, highlightsFormat))
|
||||
.build();
|
||||
|
||||
assertEquals(List.of(questionFormat, highlightsFormat), options.getFormats());
|
||||
assertEquals("question", questionFormat.getType());
|
||||
assertEquals("What is Firecrawl?", questionFormat.getQuestion());
|
||||
assertEquals("highlights", highlightsFormat.getType());
|
||||
assertEquals("What is Firecrawl?", highlightsFormat.getQuery());
|
||||
}
|
||||
|
||||
@Test
|
||||
void testCrawlOptionsBuilder() {
|
||||
CrawlOptions options = CrawlOptions.builder()
|
||||
.limit(100)
|
||||
.maxDiscoveryDepth(3)
|
||||
.sitemap("include")
|
||||
.excludePaths(List.of("/admin/*"))
|
||||
.build();
|
||||
|
||||
assertEquals(100, options.getLimit());
|
||||
assertEquals(3, options.getMaxDiscoveryDepth());
|
||||
assertEquals("include", options.getSitemap());
|
||||
assertEquals(List.of("/admin/*"), options.getExcludePaths());
|
||||
}
|
||||
|
||||
@Test
|
||||
void testAgentOptionsRequiresPrompt() {
|
||||
assertThrows(IllegalArgumentException.class, () ->
|
||||
AgentOptions.builder().build()
|
||||
);
|
||||
}
|
||||
|
||||
@Test
|
||||
void testWebhookConfigRequiresUrl() {
|
||||
assertThrows(IllegalArgumentException.class, () ->
|
||||
WebhookConfig.builder().build()
|
||||
);
|
||||
}
|
||||
|
||||
@Test
|
||||
void testScrapeOptionsToBuilder() {
|
||||
ScrapeOptions original = ScrapeOptions.builder()
|
||||
.formats(List.of("markdown"))
|
||||
.timeout(5000)
|
||||
.build();
|
||||
|
||||
ScrapeOptions modified = original.toBuilder()
|
||||
.timeout(10000)
|
||||
.build();
|
||||
|
||||
assertEquals(5000, original.getTimeout());
|
||||
assertEquals(10000, modified.getTimeout());
|
||||
assertEquals(List.of("markdown"), modified.getFormats());
|
||||
}
|
||||
|
||||
@Test
|
||||
void testBrowserExecuteRequiresSessionId() {
|
||||
FirecrawlClient client = FirecrawlClient.builder()
|
||||
.apiKey("fc-test-key")
|
||||
.build();
|
||||
assertThrows(NullPointerException.class, () ->
|
||||
client.browserExecute(null, "echo test")
|
||||
);
|
||||
}
|
||||
|
||||
@Test
|
||||
void testInteractRequiresJobId() {
|
||||
FirecrawlClient client = FirecrawlClient.builder()
|
||||
.apiKey("fc-test-key")
|
||||
.build();
|
||||
assertThrows(NullPointerException.class, () ->
|
||||
client.interact(null, "console.log('hi')")
|
||||
);
|
||||
}
|
||||
|
||||
@Test
|
||||
void testInteractRequiresCode() {
|
||||
FirecrawlClient client = FirecrawlClient.builder()
|
||||
.apiKey("fc-test-key")
|
||||
.build();
|
||||
assertThrows(NullPointerException.class, () ->
|
||||
client.interact("job-id", null)
|
||||
);
|
||||
}
|
||||
|
||||
@Test
|
||||
void testBrowserDeleteRequiresSessionId() {
|
||||
FirecrawlClient client = FirecrawlClient.builder()
|
||||
.apiKey("fc-test-key")
|
||||
.build();
|
||||
assertThrows(NullPointerException.class, () ->
|
||||
client.deleteBrowser(null)
|
||||
);
|
||||
}
|
||||
|
||||
@Test
|
||||
void testStopInteractiveBrowserRequiresJobId() {
|
||||
FirecrawlClient client = FirecrawlClient.builder()
|
||||
.apiKey("fc-test-key")
|
||||
.build();
|
||||
assertThrows(NullPointerException.class, () ->
|
||||
client.stopInteractiveBrowser(null)
|
||||
);
|
||||
}
|
||||
|
||||
@Test
|
||||
void testParseFileBuilder() {
|
||||
ParseFile file = ParseFile.builder()
|
||||
.filename("upload.html")
|
||||
.content("<html><body>hello</body></html>".getBytes(StandardCharsets.UTF_8))
|
||||
.contentType("text/html")
|
||||
.build();
|
||||
|
||||
assertEquals("upload.html", file.getFilename());
|
||||
assertEquals("text/html", file.getContentType());
|
||||
assertTrue(file.getContent().length > 0);
|
||||
byte[] firstRead = file.getContent();
|
||||
firstRead[0] = 'X';
|
||||
assertNotEquals(firstRead[0], file.getContent()[0]);
|
||||
}
|
||||
|
||||
@Test
|
||||
void testParseRequiresFile() {
|
||||
FirecrawlClient client = FirecrawlClient.builder()
|
||||
.apiKey("fc-test-key")
|
||||
.build();
|
||||
assertThrows(NullPointerException.class, () ->
|
||||
client.parse(null)
|
||||
);
|
||||
}
|
||||
|
||||
@Test
|
||||
void testParseOptionsRejectsChangeTrackingFormat() {
|
||||
assertThrows(IllegalArgumentException.class, () ->
|
||||
ParseOptions.builder()
|
||||
.formats(List.of("markdown", "changeTracking"))
|
||||
.build()
|
||||
);
|
||||
}
|
||||
|
||||
// ================================================================
|
||||
// E2E TESTS (require FIRECRAWL_API_KEY)
|
||||
// ================================================================
|
||||
|
||||
@Test
|
||||
@EnabledIfEnvironmentVariable(named = "FIRECRAWL_API_KEY", matches = ".*\\S.*")
|
||||
void testScrapeE2E() {
|
||||
FirecrawlClient client = FirecrawlClient.fromEnv();
|
||||
Document doc = client.scrape("https://example.com",
|
||||
ScrapeOptions.builder()
|
||||
.formats(List.of("markdown"))
|
||||
.build());
|
||||
|
||||
assertNotNull(doc);
|
||||
assertNotNull(doc.getMarkdown());
|
||||
assertFalse(doc.getMarkdown().isEmpty());
|
||||
}
|
||||
|
||||
@Test
|
||||
@EnabledIfEnvironmentVariable(named = "FIRECRAWL_API_KEY", matches = ".*\\S.*")
|
||||
void testMapE2E() {
|
||||
FirecrawlClient client = FirecrawlClient.fromEnv();
|
||||
MapData data = client.map("https://example.com",
|
||||
MapOptions.builder()
|
||||
.limit(10)
|
||||
.build());
|
||||
|
||||
assertNotNull(data);
|
||||
assertNotNull(data.getLinks());
|
||||
}
|
||||
|
||||
@Test
|
||||
@EnabledIfEnvironmentVariable(named = "FIRECRAWL_API_KEY", matches = ".*\\S.*")
|
||||
void testCrawlE2E() {
|
||||
FirecrawlClient client = FirecrawlClient.fromEnv();
|
||||
CrawlJob job = client.crawl("https://example.com",
|
||||
CrawlOptions.builder()
|
||||
.limit(3)
|
||||
.build(),
|
||||
2, 60);
|
||||
|
||||
assertNotNull(job);
|
||||
assertEquals("completed", job.getStatus());
|
||||
assertNotNull(job.getData());
|
||||
assertFalse(job.getData().isEmpty());
|
||||
}
|
||||
|
||||
@Test
|
||||
@EnabledIfEnvironmentVariable(named = "FIRECRAWL_API_KEY", matches = ".*\\S.*")
|
||||
void testSearchE2E() {
|
||||
FirecrawlClient client = FirecrawlClient.fromEnv();
|
||||
SearchData data = client.search("firecrawl web scraping",
|
||||
SearchOptions.builder()
|
||||
.limit(5)
|
||||
.build());
|
||||
|
||||
assertNotNull(data);
|
||||
}
|
||||
|
||||
@Test
|
||||
@EnabledIfEnvironmentVariable(named = "FIRECRAWL_API_KEY", matches = ".*\\S.*")
|
||||
void testConcurrencyE2E() {
|
||||
FirecrawlClient client = FirecrawlClient.fromEnv();
|
||||
ConcurrencyCheck check = client.getConcurrency();
|
||||
|
||||
assertNotNull(check);
|
||||
assertTrue(check.getMaxConcurrency() > 0);
|
||||
}
|
||||
|
||||
@Test
|
||||
@EnabledIfEnvironmentVariable(named = "FIRECRAWL_API_KEY", matches = ".*\\S.*")
|
||||
void testCreditUsageE2E() {
|
||||
FirecrawlClient client = FirecrawlClient.fromEnv();
|
||||
CreditUsage usage = client.getCreditUsage();
|
||||
|
||||
assertNotNull(usage);
|
||||
}
|
||||
|
||||
@Test
|
||||
@EnabledIfEnvironmentVariable(named = "FIRECRAWL_API_KEY", matches = ".*\\S.*")
|
||||
void testParseE2E() {
|
||||
FirecrawlClient client = FirecrawlClient.fromEnv();
|
||||
ParseFile file = ParseFile.builder()
|
||||
.filename("java-parse-e2e.html")
|
||||
.content("<!DOCTYPE html><html><body><h1>Java SDK Parse E2E</h1></body></html>".getBytes(StandardCharsets.UTF_8))
|
||||
.contentType("text/html")
|
||||
.build();
|
||||
|
||||
Document doc = client.parse(file, ParseOptions.builder()
|
||||
.formats(List.of("markdown"))
|
||||
.build());
|
||||
|
||||
assertNotNull(doc);
|
||||
assertNotNull(doc.getMarkdown());
|
||||
assertFalse(doc.getMarkdown().isEmpty());
|
||||
assertTrue(doc.getMarkdown().contains("Java SDK Parse E2E"));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,131 @@
|
||||
package com.firecrawl;
|
||||
|
||||
import com.firecrawl.client.FirecrawlClient;
|
||||
import com.firecrawl.models.Document;
|
||||
import com.firecrawl.models.ScrapeOptions;
|
||||
import org.junit.jupiter.api.BeforeAll;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.*;
|
||||
|
||||
/**
|
||||
* Live Site Test - Firecrawl.dev
|
||||
*
|
||||
* Tests the Java SDK against the actual Firecrawl production website.
|
||||
* This demonstrates real-world usage of the API against live content.
|
||||
*
|
||||
* Run with: FIRECRAWL_API_KEY=fc-xxx gradle test --tests "com.firecrawl.FirecrawlLiveSiteTest"
|
||||
*/
|
||||
class FirecrawlLiveSiteTest {
|
||||
|
||||
private static FirecrawlClient client;
|
||||
|
||||
@BeforeAll
|
||||
static void setup() {
|
||||
String apiKey = System.getenv("FIRECRAWL_API_KEY");
|
||||
if (apiKey != null && !apiKey.isBlank()) {
|
||||
client = FirecrawlClient.fromEnv();
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
@EnabledIfEnvironmentVariable(named = "FIRECRAWL_API_KEY", matches = ".*\\S.*")
|
||||
void testScrapeFirecrawlHomepage() {
|
||||
System.out.println("\n=== Testing against LIVE Firecrawl.dev website ===\n");
|
||||
System.out.println("Scraping: https://firecrawl.dev");
|
||||
|
||||
Document doc = client.scrape("https://firecrawl.dev",
|
||||
ScrapeOptions.builder()
|
||||
.formats(List.of("markdown", "html"))
|
||||
.onlyMainContent(true)
|
||||
.build());
|
||||
|
||||
// Assertions
|
||||
assertNotNull(doc, "Document should not be null");
|
||||
assertNotNull(doc.getMarkdown(), "Markdown content should not be null");
|
||||
assertNotNull(doc.getHtml(), "HTML content should not be null");
|
||||
assertNotNull(doc.getMetadata(), "Metadata should not be null");
|
||||
|
||||
// Verify it's actually the Firecrawl site
|
||||
String markdown = doc.getMarkdown().toLowerCase();
|
||||
assertTrue(markdown.contains("firecrawl") || markdown.contains("scrape") || markdown.contains("crawl"),
|
||||
"Content should mention Firecrawl features");
|
||||
|
||||
// Check metadata
|
||||
String sourceUrl = doc.getMetadata().get("sourceURL").toString();
|
||||
assertTrue(sourceUrl.contains("firecrawl.dev"), "Source URL should be firecrawl.dev");
|
||||
|
||||
// Display results
|
||||
System.out.println("\n✓ Successfully scraped Firecrawl.dev!");
|
||||
System.out.println("\nMetadata:");
|
||||
System.out.println(" Source URL: " + sourceUrl);
|
||||
if (doc.getMetadata().get("title") != null) {
|
||||
System.out.println(" Title: " + doc.getMetadata().get("title"));
|
||||
}
|
||||
System.out.println(" Status Code: " + doc.getMetadata().get("statusCode"));
|
||||
|
||||
System.out.println("\nContent Stats:");
|
||||
System.out.println(" Markdown length: " + doc.getMarkdown().length() + " characters");
|
||||
System.out.println(" HTML length: " + doc.getHtml().length() + " characters");
|
||||
|
||||
System.out.println("\nFirst 500 characters of markdown:");
|
||||
System.out.println(" " + doc.getMarkdown().substring(0, Math.min(500, doc.getMarkdown().length())).replace("\n", "\n "));
|
||||
|
||||
System.out.println("\n=== Live site test completed successfully! ===\n");
|
||||
}
|
||||
|
||||
@Test
|
||||
@EnabledIfEnvironmentVariable(named = "FIRECRAWL_API_KEY", matches = ".*\\S.*")
|
||||
void testScrapeFirecrawlPricing() {
|
||||
System.out.println("\n=== Testing Firecrawl Pricing Page ===\n");
|
||||
System.out.println("Scraping: https://firecrawl.dev/pricing");
|
||||
|
||||
Document doc = client.scrape("https://firecrawl.dev/pricing",
|
||||
ScrapeOptions.builder()
|
||||
.formats(List.of("markdown"))
|
||||
.build());
|
||||
|
||||
// Assertions
|
||||
assertNotNull(doc, "Document should not be null");
|
||||
assertNotNull(doc.getMarkdown(), "Markdown content should not be null");
|
||||
|
||||
String markdown = doc.getMarkdown().toLowerCase();
|
||||
assertTrue(markdown.contains("pricing") || markdown.contains("plan") || markdown.contains("price"),
|
||||
"Pricing page should contain pricing information");
|
||||
|
||||
System.out.println("✓ Successfully scraped pricing page!");
|
||||
System.out.println(" Content length: " + doc.getMarkdown().length() + " characters");
|
||||
System.out.println(" Source: " + doc.getMetadata().get("sourceURL"));
|
||||
}
|
||||
|
||||
@Test
|
||||
@EnabledIfEnvironmentVariable(named = "FIRECRAWL_API_KEY", matches = ".*\\S.*")
|
||||
void testScrapeFirecrawlDocs() {
|
||||
System.out.println("\n=== Testing Firecrawl Documentation ===\n");
|
||||
System.out.println("Scraping: https://docs.firecrawl.dev");
|
||||
|
||||
Document doc = client.scrape("https://docs.firecrawl.dev",
|
||||
ScrapeOptions.builder()
|
||||
.formats(List.of("markdown"))
|
||||
.waitFor(2000) // Wait for docs to load
|
||||
.build());
|
||||
|
||||
// Assertions
|
||||
assertNotNull(doc, "Document should not be null");
|
||||
assertNotNull(doc.getMarkdown(), "Markdown content should not be null");
|
||||
assertFalse(doc.getMarkdown().isEmpty(), "Markdown should not be empty");
|
||||
|
||||
String markdown = doc.getMarkdown().toLowerCase();
|
||||
assertTrue(markdown.contains("document") || markdown.contains("api") || markdown.contains("firecrawl"),
|
||||
"Docs should contain documentation content");
|
||||
|
||||
System.out.println("✓ Successfully scraped documentation!");
|
||||
System.out.println(" Content length: " + doc.getMarkdown().length() + " characters");
|
||||
System.out.println(" Source: " + doc.getMetadata().get("sourceURL"));
|
||||
|
||||
System.out.println("\n=== All Firecrawl.dev tests passed! ===\n");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,279 @@
|
||||
package com.firecrawl;
|
||||
|
||||
import com.firecrawl.client.FirecrawlClient;
|
||||
import com.firecrawl.models.MapData;
|
||||
import com.firecrawl.models.MapOptions;
|
||||
import org.junit.jupiter.api.BeforeAll;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable;
|
||||
|
||||
import java.util.Map;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.*;
|
||||
|
||||
/**
|
||||
* Comprehensive Map Tests
|
||||
*
|
||||
* Tests the map functionality with various configurations.
|
||||
* Based on Node.js SDK patterns and tested against live firecrawl.dev.
|
||||
*
|
||||
* Run with: FIRECRAWL_API_KEY=fc-xxx gradle test --tests "com.firecrawl.MapTest"
|
||||
*/
|
||||
class MapTest {
|
||||
|
||||
private static FirecrawlClient client;
|
||||
|
||||
@BeforeAll
|
||||
static void setup() {
|
||||
String apiKey = System.getenv("FIRECRAWL_API_KEY");
|
||||
if (apiKey != null && !apiKey.isBlank()) {
|
||||
client = FirecrawlClient.fromEnv();
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
@EnabledIfEnvironmentVariable(named = "FIRECRAWL_API_KEY", matches = ".*\\S.*")
|
||||
void testMapMinimal() {
|
||||
System.out.println("\n=== Test: Map - Minimal Request ===");
|
||||
|
||||
MapData data = client.map("https://docs.firecrawl.dev");
|
||||
|
||||
assertNotNull(data, "Map data should not be null");
|
||||
assertNotNull(data.getLinks(), "Links should not be null");
|
||||
assertTrue(!data.getLinks().isEmpty(), "Should have at least one link");
|
||||
|
||||
// Verify link structure (v2 links are MapDocument objects with url, title, description)
|
||||
Map<String, Object> firstLink = data.getLinks().get(0);
|
||||
assertNotNull(firstLink, "Link should not be null");
|
||||
assertNotNull(firstLink.get("url"), "Link should have url");
|
||||
assertTrue(firstLink.get("url").toString().startsWith("http"), "URL should start with http");
|
||||
|
||||
System.out.println("✓ Map completed successfully");
|
||||
System.out.println(" Total links found: " + data.getLinks().size());
|
||||
System.out.println(" Sample URL: " + firstLink.get("url"));
|
||||
if (firstLink.get("title") != null) {
|
||||
System.out.println(" Title: " + firstLink.get("title"));
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
@EnabledIfEnvironmentVariable(named = "FIRECRAWL_API_KEY", matches = ".*\\S.*")
|
||||
void testMapWithLimit() {
|
||||
System.out.println("\n=== Test: Map with Limit ===");
|
||||
|
||||
MapData data = client.map("https://docs.firecrawl.dev",
|
||||
MapOptions.builder()
|
||||
.limit(10)
|
||||
.build());
|
||||
|
||||
assertNotNull(data.getLinks(), "Links should not be null");
|
||||
assertTrue(data.getLinks().size() <= 10,
|
||||
"Should respect limit of 10: got " + data.getLinks().size());
|
||||
|
||||
System.out.println("✓ Map with limit completed");
|
||||
System.out.println(" Requested limit: 10");
|
||||
System.out.println(" Actual links: " + data.getLinks().size());
|
||||
}
|
||||
|
||||
@Test
|
||||
@EnabledIfEnvironmentVariable(named = "FIRECRAWL_API_KEY", matches = ".*\\S.*")
|
||||
void testMapWithSearch() {
|
||||
System.out.println("\n=== Test: Map with Search Filter ===");
|
||||
|
||||
MapData data = client.map("https://docs.firecrawl.dev",
|
||||
MapOptions.builder()
|
||||
.search("api")
|
||||
.limit(20)
|
||||
.build());
|
||||
|
||||
assertNotNull(data.getLinks(), "Links should not be null");
|
||||
|
||||
// Verify that filtered results contain the search term
|
||||
long matchingLinks = data.getLinks().stream()
|
||||
.filter(link -> {
|
||||
String url = link.get("url") != null ? link.get("url").toString().toLowerCase() : "";
|
||||
String title = link.get("title") != null ? link.get("title").toString().toLowerCase() : "";
|
||||
return url.contains("api") || title.contains("api");
|
||||
})
|
||||
.count();
|
||||
|
||||
System.out.println("✓ Map with search completed");
|
||||
System.out.println(" Total links: " + data.getLinks().size());
|
||||
System.out.println(" Links matching 'api': " + matchingLinks);
|
||||
}
|
||||
|
||||
@Test
|
||||
@EnabledIfEnvironmentVariable(named = "FIRECRAWL_API_KEY", matches = ".*\\S.*")
|
||||
void testMapWithSkipSitemap() {
|
||||
System.out.println("\n=== Test: Map with Sitemap Skip ===");
|
||||
|
||||
MapData data = client.map("https://firecrawl.dev",
|
||||
MapOptions.builder()
|
||||
.sitemap("skip")
|
||||
.limit(15)
|
||||
.build());
|
||||
|
||||
assertNotNull(data.getLinks(), "Links should not be null");
|
||||
assertTrue(data.getLinks().size() <= 15, "Should respect limit");
|
||||
|
||||
// Verify all links are valid HTTP(S) URLs
|
||||
boolean allValidUrls = data.getLinks().stream()
|
||||
.allMatch(link -> {
|
||||
String url = link.get("url") != null ? link.get("url").toString() : "";
|
||||
return url.startsWith("http://") || url.startsWith("https://");
|
||||
});
|
||||
|
||||
assertTrue(allValidUrls, "All URLs should be valid HTTP(S)");
|
||||
|
||||
System.out.println("✓ Map with sitemap=skip completed");
|
||||
System.out.println(" Links found: " + data.getLinks().size());
|
||||
}
|
||||
|
||||
@Test
|
||||
@EnabledIfEnvironmentVariable(named = "FIRECRAWL_API_KEY", matches = ".*\\S.*")
|
||||
void testMapWithSitemapOnly() {
|
||||
System.out.println("\n=== Test: Map with Sitemap Only ===");
|
||||
|
||||
MapData data = client.map("https://firecrawl.dev",
|
||||
MapOptions.builder()
|
||||
.sitemap("only")
|
||||
.limit(50)
|
||||
.build());
|
||||
|
||||
assertNotNull(data.getLinks(), "Links should not be null");
|
||||
// Note: sitemapOnly may not always respect the limit strictly
|
||||
|
||||
// Verify all links are valid HTTP(S) URLs
|
||||
boolean allValidUrls = data.getLinks().stream()
|
||||
.allMatch(link -> {
|
||||
String url = link.get("url") != null ? link.get("url").toString() : "";
|
||||
return url.startsWith("http://") || url.startsWith("https://");
|
||||
});
|
||||
|
||||
assertTrue(allValidUrls, "All URLs should be valid HTTP(S)");
|
||||
|
||||
System.out.println("✓ Map with sitemap=only completed");
|
||||
System.out.println(" Links found: " + data.getLinks().size());
|
||||
}
|
||||
|
||||
@Test
|
||||
@EnabledIfEnvironmentVariable(named = "FIRECRAWL_API_KEY", matches = ".*\\S.*")
|
||||
void testMapWithIncludeSubdomains() {
|
||||
System.out.println("\n=== Test: Map with Include Subdomains ===");
|
||||
|
||||
MapData data = client.map("https://firecrawl.dev",
|
||||
MapOptions.builder()
|
||||
.includeSubdomains(true)
|
||||
.limit(20)
|
||||
.build());
|
||||
|
||||
assertNotNull(data.getLinks(), "Links should not be null");
|
||||
|
||||
System.out.println("✓ Map with subdomains completed");
|
||||
System.out.println(" Total links: " + data.getLinks().size());
|
||||
|
||||
// Check if any subdomains were found
|
||||
boolean hasSubdomains = data.getLinks().stream()
|
||||
.anyMatch(link -> {
|
||||
String url = link.get("url") != null ? link.get("url").toString() : "";
|
||||
return url.contains("docs.firecrawl.dev") ||
|
||||
url.contains("api.firecrawl.dev") ||
|
||||
(url.contains(".firecrawl.dev") && !url.contains("www.firecrawl.dev"));
|
||||
});
|
||||
|
||||
if (hasSubdomains) {
|
||||
System.out.println(" ✓ Found subdomain links");
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
@EnabledIfEnvironmentVariable(named = "FIRECRAWL_API_KEY", matches = ".*\\S.*")
|
||||
void testMapFirecrawlDocs() {
|
||||
System.out.println("\n=== Test: Map Firecrawl Documentation ===");
|
||||
|
||||
MapData data = client.map("https://docs.firecrawl.dev",
|
||||
MapOptions.builder()
|
||||
.limit(50)
|
||||
.build());
|
||||
|
||||
assertNotNull(data.getLinks(), "Links should not be null");
|
||||
assertFalse(data.getLinks().isEmpty(), "Should find documentation links");
|
||||
|
||||
System.out.println("✓ Mapped Firecrawl documentation");
|
||||
System.out.println(" Total links: " + data.getLinks().size());
|
||||
|
||||
// Print sample links
|
||||
System.out.println(" Sample documentation pages:");
|
||||
data.getLinks().stream()
|
||||
.limit(5)
|
||||
.forEach(link -> System.out.println(" - " + link.get("url")));
|
||||
}
|
||||
|
||||
@Test
|
||||
@EnabledIfEnvironmentVariable(named = "FIRECRAWL_API_KEY", matches = ".*\\S.*")
|
||||
void testMapLinkStructure() {
|
||||
System.out.println("\n=== Test: Verify Map Link Structure ===");
|
||||
|
||||
MapData data = client.map("https://firecrawl.dev",
|
||||
MapOptions.builder()
|
||||
.limit(5)
|
||||
.build());
|
||||
|
||||
assertNotNull(data.getLinks(), "Links should not be null");
|
||||
assertFalse(data.getLinks().isEmpty(), "Should have links");
|
||||
|
||||
// Verify each link is a valid URL with expected fields
|
||||
for (Map<String, Object> link : data.getLinks()) {
|
||||
assertNotNull(link, "Link should not be null");
|
||||
assertNotNull(link.get("url"), "Link should have url field");
|
||||
assertTrue(link.get("url").toString().startsWith("http"), "URL should be valid: " + link.get("url"));
|
||||
}
|
||||
|
||||
System.out.println("✓ All links have correct structure");
|
||||
System.out.println(" Verified " + data.getLinks().size() + " links");
|
||||
}
|
||||
|
||||
@Test
|
||||
@EnabledIfEnvironmentVariable(named = "FIRECRAWL_API_KEY", matches = ".*\\S.*")
|
||||
void testMapWithTimeout() {
|
||||
System.out.println("\n=== Test: Map with Timeout ===");
|
||||
|
||||
MapData data = client.map("https://firecrawl.dev",
|
||||
MapOptions.builder()
|
||||
.timeout(15000) // 15 seconds
|
||||
.limit(10)
|
||||
.build());
|
||||
|
||||
assertNotNull(data.getLinks(), "Links should not be null");
|
||||
|
||||
System.out.println("✓ Map with timeout completed");
|
||||
System.out.println(" Timeout: 15000ms");
|
||||
System.out.println(" Links found: " + data.getLinks().size());
|
||||
}
|
||||
|
||||
@Test
|
||||
@EnabledIfEnvironmentVariable(named = "FIRECRAWL_API_KEY", matches = ".*\\S.*")
|
||||
void testMapComprehensive() {
|
||||
System.out.println("\n=== Test: Map with All Options ===");
|
||||
|
||||
MapData data = client.map("https://docs.firecrawl.dev",
|
||||
MapOptions.builder()
|
||||
.includeSubdomains(false)
|
||||
.limit(25)
|
||||
.sitemap("include")
|
||||
.timeout(20000)
|
||||
.build());
|
||||
|
||||
assertNotNull(data.getLinks(), "Links should not be null");
|
||||
assertTrue(data.getLinks().size() <= 25, "Should respect limit");
|
||||
|
||||
System.out.println("✓ Comprehensive map completed");
|
||||
System.out.println(" Configuration:");
|
||||
System.out.println(" - Include subdomains: false");
|
||||
System.out.println(" - Limit: 25");
|
||||
System.out.println(" - Ignore sitemap: false");
|
||||
System.out.println(" - Timeout: 20000ms");
|
||||
System.out.println(" Results:");
|
||||
System.out.println(" - Links found: " + data.getLinks().size());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,177 @@
|
||||
package com.firecrawl;
|
||||
|
||||
import com.firecrawl.client.FirecrawlClient;
|
||||
import com.firecrawl.errors.FirecrawlException;
|
||||
import com.firecrawl.models.Document;
|
||||
import com.firecrawl.models.ScrapeOptions;
|
||||
import org.junit.jupiter.api.BeforeAll;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.*;
|
||||
|
||||
/**
|
||||
* Scrape Endpoint Tests
|
||||
*
|
||||
* Tests the scrape functionality of the Firecrawl Java SDK.
|
||||
* These tests require FIRECRAWL_API_KEY environment variable to be set.
|
||||
*
|
||||
* Run with: FIRECRAWL_API_KEY=fc-xxx gradle test --tests "com.firecrawl.ScrapeTest"
|
||||
*/
|
||||
class ScrapeTest {
|
||||
|
||||
private static FirecrawlClient client;
|
||||
|
||||
@BeforeAll
|
||||
static void setup() {
|
||||
// Initialize client from environment variable
|
||||
String apiKey = System.getenv("FIRECRAWL_API_KEY");
|
||||
if (apiKey != null && !apiKey.isBlank()) {
|
||||
client = FirecrawlClient.fromEnv();
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
@EnabledIfEnvironmentVariable(named = "FIRECRAWL_API_KEY", matches = ".*\\S.*")
|
||||
void testScrapeBasic() {
|
||||
// Test basic scraping with markdown format
|
||||
System.out.println("Testing basic scrape with markdown format...");
|
||||
|
||||
Document doc = client.scrape("https://example.com",
|
||||
ScrapeOptions.builder()
|
||||
.formats(List.of("markdown"))
|
||||
.build());
|
||||
|
||||
// Assertions
|
||||
assertNotNull(doc, "Document should not be null");
|
||||
assertNotNull(doc.getMarkdown(), "Markdown content should not be null");
|
||||
assertFalse(doc.getMarkdown().isEmpty(), "Markdown content should not be empty");
|
||||
|
||||
System.out.println("✓ Basic scrape test passed");
|
||||
System.out.println(" Markdown length: " + doc.getMarkdown().length() + " characters");
|
||||
}
|
||||
|
||||
@Test
|
||||
@EnabledIfEnvironmentVariable(named = "FIRECRAWL_API_KEY", matches = ".*\\S.*")
|
||||
void testScrapeWithMultipleFormats() {
|
||||
// Test scraping with multiple formats
|
||||
System.out.println("Testing scrape with multiple formats (markdown + html)...");
|
||||
|
||||
Document doc = client.scrape("https://example.com",
|
||||
ScrapeOptions.builder()
|
||||
.formats(List.of("markdown", "html"))
|
||||
.build());
|
||||
|
||||
// Assertions
|
||||
assertNotNull(doc, "Document should not be null");
|
||||
assertNotNull(doc.getMarkdown(), "Markdown content should not be null");
|
||||
assertNotNull(doc.getHtml(), "HTML content should not be null");
|
||||
assertFalse(doc.getMarkdown().isEmpty(), "Markdown should not be empty");
|
||||
assertFalse(doc.getHtml().isEmpty(), "HTML should not be empty");
|
||||
|
||||
System.out.println("✓ Multiple formats test passed");
|
||||
System.out.println(" Markdown length: " + doc.getMarkdown().length());
|
||||
System.out.println(" HTML length: " + doc.getHtml().length());
|
||||
}
|
||||
|
||||
@Test
|
||||
@EnabledIfEnvironmentVariable(named = "FIRECRAWL_API_KEY", matches = ".*\\S.*")
|
||||
void testScrapeWithMetadata() {
|
||||
// Test that metadata is properly extracted
|
||||
System.out.println("Testing scrape with metadata extraction...");
|
||||
|
||||
Document doc = client.scrape("https://example.com",
|
||||
ScrapeOptions.builder()
|
||||
.formats(List.of("markdown"))
|
||||
.build());
|
||||
|
||||
// Assertions
|
||||
assertNotNull(doc.getMetadata(), "Metadata should not be null");
|
||||
assertNotNull(doc.getMetadata().get("sourceURL"), "Source URL should be in metadata");
|
||||
assertTrue(doc.getMetadata().get("sourceURL").toString().contains("example.com"),
|
||||
"Source URL should contain example.com");
|
||||
|
||||
System.out.println("✓ Metadata extraction test passed");
|
||||
System.out.println(" Source URL: " + doc.getMetadata().get("sourceURL"));
|
||||
if (doc.getMetadata().get("title") != null) {
|
||||
System.out.println(" Title: " + doc.getMetadata().get("title"));
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
@EnabledIfEnvironmentVariable(named = "FIRECRAWL_API_KEY", matches = ".*\\S.*")
|
||||
void testScrapeWithOnlyMainContent() {
|
||||
// Test scraping with onlyMainContent option
|
||||
System.out.println("Testing scrape with onlyMainContent option...");
|
||||
|
||||
Document doc = client.scrape("https://example.com",
|
||||
ScrapeOptions.builder()
|
||||
.formats(List.of("markdown"))
|
||||
.onlyMainContent(true)
|
||||
.build());
|
||||
|
||||
// Assertions
|
||||
assertNotNull(doc, "Document should not be null");
|
||||
assertNotNull(doc.getMarkdown(), "Markdown content should not be null");
|
||||
assertFalse(doc.getMarkdown().isEmpty(), "Markdown should not be empty");
|
||||
|
||||
System.out.println("✓ Only main content test passed");
|
||||
System.out.println(" Content length: " + doc.getMarkdown().length());
|
||||
}
|
||||
|
||||
@Test
|
||||
@EnabledIfEnvironmentVariable(named = "FIRECRAWL_API_KEY", matches = ".*\\S.*")
|
||||
void testScrapeWithTimeout() {
|
||||
// Test scraping with custom timeout
|
||||
System.out.println("Testing scrape with custom timeout...");
|
||||
|
||||
Document doc = client.scrape("https://example.com",
|
||||
ScrapeOptions.builder()
|
||||
.formats(List.of("markdown"))
|
||||
.timeout(10000) // 10 seconds
|
||||
.build());
|
||||
|
||||
// Assertions
|
||||
assertNotNull(doc, "Document should not be null");
|
||||
assertNotNull(doc.getMarkdown(), "Markdown should not be null");
|
||||
|
||||
System.out.println("✓ Timeout configuration test passed");
|
||||
}
|
||||
|
||||
@Test
|
||||
@EnabledIfEnvironmentVariable(named = "FIRECRAWL_API_KEY", matches = ".*\\S.*")
|
||||
void testScrapeInvalidUrl() {
|
||||
// Test that invalid URLs are handled properly
|
||||
System.out.println("Testing scrape with invalid URL...");
|
||||
|
||||
assertThrows(FirecrawlException.class, () -> {
|
||||
client.scrape("not-a-valid-url",
|
||||
ScrapeOptions.builder()
|
||||
.formats(List.of("markdown"))
|
||||
.build());
|
||||
}, "Should throw FirecrawlException for invalid URL");
|
||||
|
||||
System.out.println("✓ Invalid URL handling test passed");
|
||||
}
|
||||
|
||||
@Test
|
||||
@EnabledIfEnvironmentVariable(named = "FIRECRAWL_API_KEY", matches = ".*\\S.*")
|
||||
void testScrapeWithWaitFor() {
|
||||
// Test scraping with waitFor option (useful for dynamic content)
|
||||
System.out.println("Testing scrape with waitFor option...");
|
||||
|
||||
Document doc = client.scrape("https://example.com",
|
||||
ScrapeOptions.builder()
|
||||
.formats(List.of("markdown"))
|
||||
.waitFor(1000) // Wait 1 second for page to load
|
||||
.build());
|
||||
|
||||
// Assertions
|
||||
assertNotNull(doc, "Document should not be null");
|
||||
assertNotNull(doc.getMarkdown(), "Markdown should not be null");
|
||||
|
||||
System.out.println("✓ WaitFor option test passed");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,337 @@
|
||||
package com.firecrawl;
|
||||
|
||||
import com.firecrawl.client.FirecrawlClient;
|
||||
import com.firecrawl.models.*;
|
||||
import org.junit.jupiter.api.BeforeAll;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.*;
|
||||
|
||||
/**
|
||||
* Comprehensive Search Tests
|
||||
*
|
||||
* Tests the search functionality with various configurations.
|
||||
* Based on Node.js SDK patterns and tested against live firecrawl.dev.
|
||||
*
|
||||
* Run with: FIRECRAWL_API_KEY=fc-xxx gradle test --tests "com.firecrawl.SearchTest"
|
||||
*/
|
||||
class SearchTest {
|
||||
|
||||
private static FirecrawlClient client;
|
||||
|
||||
@BeforeAll
|
||||
static void setup() {
|
||||
String apiKey = System.getenv("FIRECRAWL_API_KEY");
|
||||
if (apiKey != null && !apiKey.isBlank()) {
|
||||
client = FirecrawlClient.fromEnv();
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
@EnabledIfEnvironmentVariable(named = "FIRECRAWL_API_KEY", matches = ".*\\S.*")
|
||||
void testSearchMinimal() {
|
||||
System.out.println("\n=== Test: Search - Minimal Request ===");
|
||||
|
||||
SearchData results = client.search("What is Firecrawl?");
|
||||
|
||||
assertNotNull(results, "Search results should not be null");
|
||||
assertNotNull(results.getWeb(), "Web results should not be null");
|
||||
assertTrue(!results.getWeb().isEmpty(), "Should have at least one web result");
|
||||
|
||||
// Verify result structure
|
||||
Map<String, Object> firstResult = results.getWeb().get(0);
|
||||
assertNotNull(firstResult.get("url"), "Result should have URL");
|
||||
assertTrue(firstResult.get("url").toString().startsWith("http"),
|
||||
"URL should be valid");
|
||||
|
||||
System.out.println("✓ Search completed successfully");
|
||||
System.out.println(" Web results: " + results.getWeb().size());
|
||||
System.out.println(" Sample result: " + firstResult.get("url"));
|
||||
if (firstResult.get("title") != null) {
|
||||
System.out.println(" Title: " + firstResult.get("title"));
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
@EnabledIfEnvironmentVariable(named = "FIRECRAWL_API_KEY", matches = ".*\\S.*")
|
||||
void testSearchWithLimit() {
|
||||
System.out.println("\n=== Test: Search with Limit ===");
|
||||
|
||||
SearchData results = client.search("artificial intelligence",
|
||||
SearchOptions.builder()
|
||||
.limit(5)
|
||||
.build());
|
||||
|
||||
assertNotNull(results.getWeb(), "Web results should not be null");
|
||||
assertTrue(results.getWeb().size() <= 5,
|
||||
"Should respect limit of 5: got " + results.getWeb().size());
|
||||
|
||||
System.out.println("✓ Search with limit completed");
|
||||
System.out.println(" Requested limit: 5");
|
||||
System.out.println(" Actual results: " + results.getWeb().size());
|
||||
}
|
||||
|
||||
@Test
|
||||
@EnabledIfEnvironmentVariable(named = "FIRECRAWL_API_KEY", matches = ".*\\S.*")
|
||||
void testSearchWithMultipleSources() {
|
||||
System.out.println("\n=== Test: Search with Multiple Sources ===");
|
||||
|
||||
SearchData results = client.search("Firecrawl web scraping",
|
||||
SearchOptions.builder()
|
||||
.sources(List.of("web", "news"))
|
||||
.limit(3)
|
||||
.build());
|
||||
|
||||
assertNotNull(results.getWeb(), "Web results should not be null");
|
||||
assertTrue(results.getWeb().size() <= 3, "Web results should respect limit");
|
||||
|
||||
System.out.println("✓ Multi-source search completed");
|
||||
System.out.println(" Web results: " + results.getWeb().size());
|
||||
if (results.getNews() != null) {
|
||||
System.out.println(" News results: " + results.getNews().size());
|
||||
} else {
|
||||
System.out.println(" News results: 0");
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
@EnabledIfEnvironmentVariable(named = "FIRECRAWL_API_KEY", matches = ".*\\S.*")
|
||||
void testSearchResultStructure() {
|
||||
System.out.println("\n=== Test: Verify Search Result Structure ===");
|
||||
|
||||
SearchData results = client.search("test query",
|
||||
SearchOptions.builder()
|
||||
.limit(1)
|
||||
.build());
|
||||
|
||||
assertNotNull(results.getWeb(), "Web results should not be null");
|
||||
|
||||
if (!results.getWeb().isEmpty()) {
|
||||
Map<String, Object> result = results.getWeb().get(0);
|
||||
|
||||
assertNotNull(result.get("url"), "Result must have URL");
|
||||
assertTrue(result.get("url") instanceof String, "URL should be string");
|
||||
assertTrue(result.get("url").toString().startsWith("http"),
|
||||
"URL should be valid");
|
||||
|
||||
// Title and description may be null but if present should be strings
|
||||
if (result.get("title") != null) {
|
||||
assertTrue(result.get("title") instanceof String,
|
||||
"Title should be string");
|
||||
}
|
||||
if (result.get("description") != null) {
|
||||
assertTrue(result.get("description") instanceof String,
|
||||
"Description should be string");
|
||||
}
|
||||
|
||||
System.out.println("✓ Result structure verified");
|
||||
System.out.println(" URL: ✓");
|
||||
System.out.println(" Title: " + (result.get("title") != null ? "✓" : "null"));
|
||||
System.out.println(" Description: " + (result.get("description") != null ? "✓" : "null"));
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
@EnabledIfEnvironmentVariable(named = "FIRECRAWL_API_KEY", matches = ".*\\S.*")
|
||||
void testSearchWithLocation() {
|
||||
System.out.println("\n=== Test: Search with Location ===");
|
||||
|
||||
SearchData results = client.search("restaurants near me",
|
||||
SearchOptions.builder()
|
||||
.location("US")
|
||||
.limit(5)
|
||||
.build());
|
||||
|
||||
assertNotNull(results.getWeb(), "Web results should not be null");
|
||||
|
||||
System.out.println("✓ Search with location completed");
|
||||
System.out.println(" Location: US");
|
||||
System.out.println(" Results: " + results.getWeb().size());
|
||||
}
|
||||
|
||||
@Test
|
||||
@EnabledIfEnvironmentVariable(named = "FIRECRAWL_API_KEY", matches = ".*\\S.*")
|
||||
void testSearchWithTimeFilter() {
|
||||
System.out.println("\n=== Test: Search with Time Filter ===");
|
||||
|
||||
SearchData results = client.search("latest AI news",
|
||||
SearchOptions.builder()
|
||||
.tbs("qdr:m") // Past month
|
||||
.limit(5)
|
||||
.build());
|
||||
|
||||
assertNotNull(results.getWeb(), "Web results should not be null");
|
||||
|
||||
System.out.println("✓ Search with time filter completed");
|
||||
System.out.println(" Time filter: Past month (qdr:m)");
|
||||
System.out.println(" Results: " + results.getWeb().size());
|
||||
}
|
||||
|
||||
@Test
|
||||
@EnabledIfEnvironmentVariable(named = "FIRECRAWL_API_KEY", matches = ".*\\S.*")
|
||||
void testSearchWithScrapeOptions() {
|
||||
System.out.println("\n=== Test: Search with Scrape Options ===");
|
||||
|
||||
SearchData results = client.search("Firecrawl documentation",
|
||||
SearchOptions.builder()
|
||||
.limit(2)
|
||||
.scrapeOptions(ScrapeOptions.builder()
|
||||
.formats(List.of("markdown"))
|
||||
.onlyMainContent(true)
|
||||
.build())
|
||||
.build());
|
||||
|
||||
assertNotNull(results.getWeb(), "Web results should not be null");
|
||||
|
||||
// When scrapeOptions with markdown format are provided, results should include markdown content
|
||||
if (!results.getWeb().isEmpty()) {
|
||||
Map<String, Object> first = results.getWeb().get(0);
|
||||
Object markdown = first.get("markdown");
|
||||
assertNotNull(markdown, "Scraped result should contain markdown content when formats=[markdown]");
|
||||
assertFalse(markdown.toString().isEmpty(), "Markdown content should not be empty");
|
||||
}
|
||||
|
||||
System.out.println("✓ Search with scrape options completed");
|
||||
System.out.println(" Results: " + results.getWeb().size());
|
||||
System.out.println(" Scrape formats: markdown");
|
||||
}
|
||||
|
||||
@Test
|
||||
@EnabledIfEnvironmentVariable(named = "FIRECRAWL_API_KEY", matches = ".*\\S.*")
|
||||
void testSearchFirecrawlSpecific() {
|
||||
System.out.println("\n=== Test: Search for Firecrawl ===");
|
||||
|
||||
SearchData results = client.search("Firecrawl web scraping API",
|
||||
SearchOptions.builder()
|
||||
.limit(10)
|
||||
.build());
|
||||
|
||||
assertNotNull(results.getWeb(), "Web results should not be null");
|
||||
assertFalse(results.getWeb().isEmpty(), "Should find Firecrawl results");
|
||||
|
||||
// Verify results contain Firecrawl-related content
|
||||
boolean hasFirecrawlContent = results.getWeb().stream()
|
||||
.anyMatch(result -> {
|
||||
String url = result.get("url").toString().toLowerCase();
|
||||
String title = result.get("title") != null ?
|
||||
result.get("title").toString().toLowerCase() : "";
|
||||
String desc = result.get("description") != null ?
|
||||
result.get("description").toString().toLowerCase() : "";
|
||||
|
||||
return url.contains("firecrawl") ||
|
||||
title.contains("firecrawl") ||
|
||||
desc.contains("firecrawl");
|
||||
});
|
||||
|
||||
assertTrue(hasFirecrawlContent, "Results should mention Firecrawl");
|
||||
|
||||
System.out.println("✓ Firecrawl search completed");
|
||||
System.out.println(" Total results: " + results.getWeb().size());
|
||||
System.out.println(" Results mentioning Firecrawl: ✓");
|
||||
|
||||
// Print sample results
|
||||
System.out.println(" Sample results:");
|
||||
results.getWeb().stream()
|
||||
.limit(3)
|
||||
.forEach(result -> {
|
||||
System.out.println(" - " + result.get("title"));
|
||||
System.out.println(" " + result.get("url"));
|
||||
});
|
||||
}
|
||||
|
||||
@Test
|
||||
@EnabledIfEnvironmentVariable(named = "FIRECRAWL_API_KEY", matches = ".*\\S.*")
|
||||
void testSearchComprehensive() {
|
||||
System.out.println("\n=== Test: Search with All Options ===");
|
||||
|
||||
SearchData results = client.search("web scraping tools",
|
||||
SearchOptions.builder()
|
||||
.sources(List.of("web"))
|
||||
.limit(5)
|
||||
.tbs("qdr:y") // Past year
|
||||
.location("US")
|
||||
.timeout(30000)
|
||||
.scrapeOptions(ScrapeOptions.builder()
|
||||
.formats(List.of("markdown"))
|
||||
.onlyMainContent(true)
|
||||
.waitFor(1000)
|
||||
.build())
|
||||
.build());
|
||||
|
||||
assertNotNull(results.getWeb(), "Web results should not be null");
|
||||
assertTrue(results.getWeb().size() <= 5, "Should respect limit");
|
||||
|
||||
System.out.println("✓ Comprehensive search completed");
|
||||
System.out.println(" Configuration:");
|
||||
System.out.println(" - Sources: web");
|
||||
System.out.println(" - Limit: 5");
|
||||
System.out.println(" - Time filter: Past year");
|
||||
System.out.println(" - Location: US");
|
||||
System.out.println(" - Timeout: 30000ms");
|
||||
System.out.println(" - Scrape: markdown, main content only");
|
||||
System.out.println(" Results:");
|
||||
System.out.println(" - Web results: " + results.getWeb().size());
|
||||
}
|
||||
|
||||
@Test
|
||||
@EnabledIfEnvironmentVariable(named = "FIRECRAWL_API_KEY", matches = ".*\\S.*")
|
||||
void testSearchContentVerification() {
|
||||
System.out.println("\n=== Test: Search Content Verification ===");
|
||||
|
||||
SearchData results = client.search("Python programming language",
|
||||
SearchOptions.builder()
|
||||
.limit(5)
|
||||
.build());
|
||||
|
||||
assertNotNull(results.getWeb(), "Web results should not be null");
|
||||
assertFalse(results.getWeb().isEmpty(), "Should have results");
|
||||
|
||||
// Verify results are relevant to the query
|
||||
boolean hasRelevantContent = results.getWeb().stream()
|
||||
.anyMatch(result -> {
|
||||
String text = String.format("%s %s %s",
|
||||
result.get("url"),
|
||||
result.get("title"),
|
||||
result.get("description")
|
||||
).toLowerCase();
|
||||
return text.contains("python");
|
||||
});
|
||||
|
||||
assertTrue(hasRelevantContent, "Results should be relevant to query");
|
||||
|
||||
System.out.println("✓ Content verification passed");
|
||||
System.out.println(" Query: Python programming language");
|
||||
System.out.println(" Relevant results found: ✓");
|
||||
}
|
||||
|
||||
@Test
|
||||
@EnabledIfEnvironmentVariable(named = "FIRECRAWL_API_KEY", matches = ".*\\S.*")
|
||||
void testSearchIgnoreInvalidURLs() {
|
||||
System.out.println("\n=== Test: Search with Ignore Invalid URLs ===");
|
||||
|
||||
SearchData results = client.search("technology news",
|
||||
SearchOptions.builder()
|
||||
.limit(5)
|
||||
.ignoreInvalidURLs(true)
|
||||
.build());
|
||||
|
||||
assertNotNull(results.getWeb(), "Web results should not be null");
|
||||
|
||||
// Verify all URLs are valid
|
||||
boolean allValidUrls = results.getWeb().stream()
|
||||
.allMatch(result -> {
|
||||
String url = result.get("url").toString();
|
||||
return url.startsWith("http://") || url.startsWith("https://");
|
||||
});
|
||||
|
||||
assertTrue(allValidUrls, "All URLs should be valid HTTP(S)");
|
||||
|
||||
System.out.println("✓ Search with URL validation completed");
|
||||
System.out.println(" Results: " + results.getWeb().size());
|
||||
System.out.println(" All URLs valid: ✓");
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user