참고소스 수정본
This commit is contained in:
24
참고/firecrawl-main/apps/ui/ingestion-ui/.gitignore
vendored
Normal file
24
참고/firecrawl-main/apps/ui/ingestion-ui/.gitignore
vendored
Normal file
@@ -0,0 +1,24 @@
|
||||
# Logs
|
||||
logs
|
||||
*.log
|
||||
npm-debug.log*
|
||||
yarn-debug.log*
|
||||
yarn-error.log*
|
||||
pnpm-debug.log*
|
||||
lerna-debug.log*
|
||||
|
||||
node_modules
|
||||
dist
|
||||
dist-ssr
|
||||
*.local
|
||||
|
||||
# Editor directories and files
|
||||
.vscode/*
|
||||
!.vscode/extensions.json
|
||||
.idea
|
||||
.DS_Store
|
||||
*.suo
|
||||
*.ntvs*
|
||||
*.njsproj
|
||||
*.sln
|
||||
*.sw?
|
||||
21
참고/firecrawl-main/apps/ui/ingestion-ui/LICENSE
Normal file
21
참고/firecrawl-main/apps/ui/ingestion-ui/LICENSE
Normal file
@@ -0,0 +1,21 @@
|
||||
MIT License
|
||||
|
||||
Copyright (c) 2024 Sideguide Technologies Inc.
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
65
참고/firecrawl-main/apps/ui/ingestion-ui/README.md
Normal file
65
참고/firecrawl-main/apps/ui/ingestion-ui/README.md
Normal file
@@ -0,0 +1,65 @@
|
||||
# Firecrawl UI Template
|
||||
|
||||
This template provides an easy way to spin up a UI for Firecrawl using React. It includes a pre-built component that interacts with the Firecrawl API, allowing you to quickly set up a web crawling and scraping interface.
|
||||
|
||||
## ⚠️ Important Security Notice
|
||||
|
||||
**This template exposes Firecrawl API keys in the client-side code. For production use, it is strongly recommended to move API interactions to a server-side implementation to protect your API keys.**
|
||||
|
||||
## Prerequisites
|
||||
|
||||
- Node.js (v14 or later recommended)
|
||||
- npm
|
||||
|
||||
## Getting Started
|
||||
|
||||
1. Install dependencies:
|
||||
|
||||
```
|
||||
npm install
|
||||
```
|
||||
|
||||
2. Set up your Firecrawl API key:
|
||||
Open `src/components/ingestion.tsx` and replace the placeholder API key:
|
||||
|
||||
```typescript
|
||||
const FIRECRAWL_API_KEY = "your-api-key-here";
|
||||
```
|
||||
|
||||
3. Start the development server:
|
||||
|
||||
```
|
||||
npm run dev
|
||||
```
|
||||
|
||||
4. Open your browser and navigate to the port specified in your terminal
|
||||
|
||||
## Customization
|
||||
|
||||
The main Firecrawl component is located in `src/components/ingestion.tsx`. You can modify this file to customize the UI or add additional features.
|
||||
|
||||
## Security Considerations
|
||||
|
||||
For production use, consider the following security measures:
|
||||
|
||||
1. Move API interactions to a server-side implementation to protect your Firecrawl API key.
|
||||
2. Implement proper authentication and authorization for your application.
|
||||
3. Set up CORS policies to restrict access to your API endpoints.
|
||||
|
||||
## Learn More
|
||||
|
||||
For more information about Firecrawl and its API, visit the [Firecrawl documentation](https://docs.firecrawl.dev/).
|
||||
|
||||
## Contributing
|
||||
|
||||
Contributions are welcome! Please feel free to submit a Pull Request.
|
||||
|
||||
## License
|
||||
|
||||
The Firecrawl Ingestion UI Template is licensed under the MIT License. This means you are free to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the SDK, subject to the following conditions:
|
||||
|
||||
- The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
|
||||
Please note that while this SDK is MIT licensed, it is part of a larger project which may be under different licensing terms. Always refer to the license information in the root directory of the main project for overall licensing details.
|
||||
4
참고/firecrawl-main/apps/ui/ingestion-ui/audit-ci.jsonc
Normal file
4
참고/firecrawl-main/apps/ui/ingestion-ui/audit-ci.jsonc
Normal file
@@ -0,0 +1,4 @@
|
||||
{
|
||||
"$schema": "https://github.com/IBM/audit-ci/raw/main/docs/schema.json",
|
||||
"low": true
|
||||
}
|
||||
17
참고/firecrawl-main/apps/ui/ingestion-ui/components.json
Normal file
17
참고/firecrawl-main/apps/ui/ingestion-ui/components.json
Normal file
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"$schema": "https://ui.shadcn.com/schema.json",
|
||||
"style": "default",
|
||||
"rsc": false,
|
||||
"tsx": true,
|
||||
"tailwind": {
|
||||
"config": "tailwind.config.js",
|
||||
"css": "src/index.css",
|
||||
"baseColor": "slate",
|
||||
"cssVariables": true,
|
||||
"prefix": ""
|
||||
},
|
||||
"aliases": {
|
||||
"components": "@/components",
|
||||
"utils": "@/lib/utils"
|
||||
}
|
||||
}
|
||||
28
참고/firecrawl-main/apps/ui/ingestion-ui/eslint.config.js
Normal file
28
참고/firecrawl-main/apps/ui/ingestion-ui/eslint.config.js
Normal file
@@ -0,0 +1,28 @@
|
||||
import js from '@eslint/js'
|
||||
import globals from 'globals'
|
||||
import reactHooks from 'eslint-plugin-react-hooks'
|
||||
import reactRefresh from 'eslint-plugin-react-refresh'
|
||||
import tseslint from 'typescript-eslint'
|
||||
|
||||
export default tseslint.config(
|
||||
{ ignores: ['dist'] },
|
||||
{
|
||||
extends: [js.configs.recommended, ...tseslint.configs.recommended],
|
||||
files: ['**/*.{ts,tsx}'],
|
||||
languageOptions: {
|
||||
ecmaVersion: 2020,
|
||||
globals: globals.browser,
|
||||
},
|
||||
plugins: {
|
||||
'react-hooks': reactHooks,
|
||||
'react-refresh': reactRefresh,
|
||||
},
|
||||
rules: {
|
||||
...reactHooks.configs.recommended.rules,
|
||||
'react-refresh/only-export-components': [
|
||||
'warn',
|
||||
{ allowConstantExport: true, allowExportNames: ['buttonVariants'] },
|
||||
],
|
||||
},
|
||||
},
|
||||
)
|
||||
16
참고/firecrawl-main/apps/ui/ingestion-ui/index.html
Normal file
16
참고/firecrawl-main/apps/ui/ingestion-ui/index.html
Normal file
@@ -0,0 +1,16 @@
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
|
||||
<head>
|
||||
<meta charset="UTF-8" />
|
||||
<link rel="icon" type="image/x-icon" href="/favicon.ico" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>Firecrawl UI Template</title>
|
||||
</head>
|
||||
|
||||
<body>
|
||||
<div id="root"></div>
|
||||
<script type="module" src="/src/main.tsx"></script>
|
||||
</body>
|
||||
|
||||
</html>
|
||||
55
참고/firecrawl-main/apps/ui/ingestion-ui/package.json
Normal file
55
참고/firecrawl-main/apps/ui/ingestion-ui/package.json
Normal file
@@ -0,0 +1,55 @@
|
||||
{
|
||||
"name": "ingestion-ui",
|
||||
"private": true,
|
||||
"version": "0.0.0",
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
"dev": "vite",
|
||||
"build": "tsc -b && vite build",
|
||||
"lint": "eslint . --report-unused-disable-directives --max-warnings 0",
|
||||
"preview": "vite preview"
|
||||
},
|
||||
"dependencies": {
|
||||
"@radix-ui/react-checkbox": "^1.1.1",
|
||||
"@radix-ui/react-collapsible": "^1.1.0",
|
||||
"@radix-ui/react-label": "^2.1.0",
|
||||
"@radix-ui/react-radio-group": "^1.2.0",
|
||||
"@radix-ui/react-slot": "^1.1.0",
|
||||
"class-variance-authority": "^0.7.0",
|
||||
"clsx": "^2.1.1",
|
||||
"lucide-react": "^0.414.0",
|
||||
"react": "^18.3.1",
|
||||
"react-dom": "^18.3.1",
|
||||
"tailwind-merge": "^2.4.0",
|
||||
"tailwindcss-animate": "^1.0.7"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/node": "^20.14.12",
|
||||
"@types/react": "^18.3.3",
|
||||
"@types/react-dom": "^18.3.0",
|
||||
"@eslint/js": "^9.26.0",
|
||||
"@vitejs/plugin-react": "^4.3.1",
|
||||
"@vitejs/plugin-react-swc": "^3.5.0",
|
||||
"autoprefixer": "^10.4.19",
|
||||
"eslint": "^9.26.0",
|
||||
"eslint-plugin-react-hooks": "^5.1.0",
|
||||
"eslint-plugin-react-refresh": "^0.4.19",
|
||||
"globals": "^16.1.0",
|
||||
"typescript-eslint": "^8.32.0",
|
||||
"postcss": "^8.5.10",
|
||||
"tailwindcss": "^3.4.6",
|
||||
"typescript": "^5.2.2",
|
||||
"vite": "^7.3.2"
|
||||
},
|
||||
"pnpm": {
|
||||
"overrides": {
|
||||
"ajv": "^8.18.0",
|
||||
"fast-uri": ">=3.1.2 <4.0.0",
|
||||
"flatted@<3.4.2": ">=3.4.2",
|
||||
"minimatch@<10.2.3": ">=10.2.3",
|
||||
"rollup@<4.59.0": ">=4.59.0",
|
||||
"picomatch@<4.0.4": ">=4.0.4",
|
||||
"brace-expansion": ">=5.0.5"
|
||||
}
|
||||
}
|
||||
}
|
||||
3161
참고/firecrawl-main/apps/ui/ingestion-ui/pnpm-lock.yaml
generated
Normal file
3161
참고/firecrawl-main/apps/ui/ingestion-ui/pnpm-lock.yaml
generated
Normal file
File diff suppressed because it is too large
Load Diff
6
참고/firecrawl-main/apps/ui/ingestion-ui/postcss.config.js
Normal file
6
참고/firecrawl-main/apps/ui/ingestion-ui/postcss.config.js
Normal file
@@ -0,0 +1,6 @@
|
||||
export default {
|
||||
plugins: {
|
||||
tailwindcss: {},
|
||||
autoprefixer: {},
|
||||
},
|
||||
}
|
||||
BIN
참고/firecrawl-main/apps/ui/ingestion-ui/public/favicon.ico
Normal file
BIN
참고/firecrawl-main/apps/ui/ingestion-ui/public/favicon.ico
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 15 KiB |
1
참고/firecrawl-main/apps/ui/ingestion-ui/public/vite.svg
Normal file
1
참고/firecrawl-main/apps/ui/ingestion-ui/public/vite.svg
Normal file
@@ -0,0 +1 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" class="iconify iconify--logos" width="31.88" height="32" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 257"><defs><linearGradient id="IconifyId1813088fe1fbc01fb466" x1="-.828%" x2="57.636%" y1="7.652%" y2="78.411%"><stop offset="0%" stop-color="#41D1FF"></stop><stop offset="100%" stop-color="#BD34FE"></stop></linearGradient><linearGradient id="IconifyId1813088fe1fbc01fb467" x1="43.376%" x2="50.316%" y1="2.242%" y2="89.03%"><stop offset="0%" stop-color="#FFEA83"></stop><stop offset="8.333%" stop-color="#FFDD35"></stop><stop offset="100%" stop-color="#FFA800"></stop></linearGradient></defs><path fill="url(#IconifyId1813088fe1fbc01fb466)" d="M255.153 37.938L134.897 252.976c-2.483 4.44-8.862 4.466-11.382.048L.875 37.958c-2.746-4.814 1.371-10.646 6.827-9.67l120.385 21.517a6.537 6.537 0 0 0 2.322-.004l117.867-21.483c5.438-.991 9.574 4.796 6.877 9.62Z"></path><path fill="url(#IconifyId1813088fe1fbc01fb467)" d="M185.432.063L96.44 17.501a3.268 3.268 0 0 0-2.634 3.014l-5.474 92.456a3.268 3.268 0 0 0 3.997 3.378l24.777-5.718c2.318-.535 4.413 1.507 3.936 3.838l-7.361 36.047c-.495 2.426 1.782 4.5 4.151 3.78l15.304-4.649c2.372-.72 4.652 1.36 4.15 3.788l-11.698 56.621c-.732 3.542 3.979 5.473 5.943 2.437l1.313-2.028l72.516-144.72c1.215-2.423-.88-5.186-3.54-4.672l-25.505 4.922c-2.396.462-4.435-1.77-3.759-4.114l16.646-57.705c.677-2.35-1.37-4.583-3.769-4.113Z"></path></svg>
|
||||
|
After Width: | Height: | Size: 1.5 KiB |
37
참고/firecrawl-main/apps/ui/ingestion-ui/src/App.tsx
Normal file
37
참고/firecrawl-main/apps/ui/ingestion-ui/src/App.tsx
Normal file
@@ -0,0 +1,37 @@
|
||||
import { useState } from "react";
|
||||
import FirecrawlComponent from "./components/ingestion";
|
||||
import FirecrawlComponentV1 from "./components/ingestionV1";
|
||||
import { RadioGroup, RadioGroupItem } from "@/components/ui/radio-group";
|
||||
import { Label } from "@/components/ui/label";
|
||||
|
||||
function App() {
|
||||
const [selectedComponent, setSelectedComponent] = useState<"v0" | "v1">("v1");
|
||||
|
||||
return (
|
||||
<>
|
||||
<div className="flex justify-center items-center space-x-2 p-4">
|
||||
<RadioGroup
|
||||
defaultValue="v1"
|
||||
onValueChange={(value) => setSelectedComponent(value as "v0" | "v1")}
|
||||
className="flex space-x-6 mt-6"
|
||||
>
|
||||
<div className="flex items-center space-x-2 p-2">
|
||||
<RadioGroupItem value="v0" id="v0"></RadioGroupItem>
|
||||
<Label htmlFor="v0">Firecrawl Component V0</Label>
|
||||
</div>
|
||||
<div className="flex items-center space-x-2 p-2">
|
||||
<RadioGroupItem value="v1" id="v1"></RadioGroupItem>
|
||||
<Label htmlFor="v1">Firecrawl Component V1</Label>
|
||||
</div>
|
||||
</RadioGroup>
|
||||
</div>
|
||||
{selectedComponent === "v1" ? (
|
||||
<FirecrawlComponentV1 />
|
||||
) : (
|
||||
<FirecrawlComponent />
|
||||
)}
|
||||
</>
|
||||
);
|
||||
}
|
||||
|
||||
export default App;
|
||||
@@ -0,0 +1,656 @@
|
||||
import { useState, ChangeEvent, FormEvent, useEffect } from "react";
|
||||
import {
|
||||
Card,
|
||||
CardHeader,
|
||||
CardTitle,
|
||||
CardContent,
|
||||
CardFooter,
|
||||
} from "@/components/ui/card";
|
||||
import { Input } from "@/components/ui/input";
|
||||
import { Button } from "@/components/ui/button";
|
||||
import { Checkbox } from "@/components/ui/checkbox";
|
||||
import { Label } from "@/components/ui/label";
|
||||
import {
|
||||
Collapsible,
|
||||
CollapsibleContent,
|
||||
CollapsibleTrigger,
|
||||
} from "@/components/ui/collapsible";
|
||||
import { ChevronDown, ChevronLeft, ChevronRight } from "lucide-react";
|
||||
|
||||
//! Hardcoded values (not recommended for production)
|
||||
//! Highly recommended to move all Firecrawl API calls to the backend (e.g. Next.js API route)
|
||||
const FIRECRAWL_API_URL = "https://api.firecrawl.dev"; // Replace with your actual API URL whether it is local or using Firecrawl Cloud
|
||||
const FIRECRAWL_API_KEY = "fc-YOUR_API_KEY"; // Replace with your actual API key
|
||||
|
||||
interface FormData {
|
||||
url: string;
|
||||
crawlSubPages: boolean;
|
||||
limit: string;
|
||||
maxDepth: string;
|
||||
excludePaths: string;
|
||||
includePaths: string;
|
||||
extractMainContent: boolean;
|
||||
}
|
||||
|
||||
interface CrawlerOptions {
|
||||
includes?: string[];
|
||||
excludes?: string[];
|
||||
maxDepth?: number;
|
||||
limit?: number;
|
||||
returnOnlyUrls: boolean;
|
||||
}
|
||||
|
||||
interface PageOptions {
|
||||
onlyMainContent: boolean;
|
||||
}
|
||||
|
||||
interface RequestBody {
|
||||
url: string;
|
||||
crawlerOptions?: CrawlerOptions;
|
||||
pageOptions: PageOptions;
|
||||
}
|
||||
|
||||
interface ScrapeResultMetadata {
|
||||
title: string;
|
||||
description: string;
|
||||
language: string;
|
||||
sourceURL: string;
|
||||
pageStatusCode: number;
|
||||
pageError?: string;
|
||||
[key: string]: string | number | undefined;
|
||||
}
|
||||
|
||||
interface ScrapeResultData {
|
||||
markdown: string;
|
||||
content: string;
|
||||
html: string;
|
||||
rawHtml: string;
|
||||
metadata: ScrapeResultMetadata;
|
||||
llm_extraction: Record<string, unknown>;
|
||||
warning?: string;
|
||||
}
|
||||
|
||||
interface ScrapeResult {
|
||||
success: boolean;
|
||||
data: ScrapeResultData;
|
||||
}
|
||||
|
||||
export default function FirecrawlComponent() {
|
||||
const [formData, setFormData] = useState<FormData>({
|
||||
url: "",
|
||||
crawlSubPages: false,
|
||||
limit: "",
|
||||
maxDepth: "",
|
||||
excludePaths: "",
|
||||
includePaths: "",
|
||||
extractMainContent: false,
|
||||
});
|
||||
const [loading, setLoading] = useState<boolean>(false);
|
||||
const [scrapingSelectedLoading, setScrapingSelectedLoading] =
|
||||
useState<boolean>(false);
|
||||
const [crawledUrls, setCrawledUrls] = useState<string[]>([]);
|
||||
const [selectedUrls, setSelectedUrls] = useState<string[]>([]);
|
||||
const [scrapeResults, setScrapeResults] = useState<
|
||||
Record<string, ScrapeResult>
|
||||
>({});
|
||||
const [isCollapsibleOpen, setIsCollapsibleOpen] = useState(true);
|
||||
const [crawlStatus, setCrawlStatus] = useState<{
|
||||
current: number;
|
||||
total: number | null;
|
||||
}>({ current: 0, total: null });
|
||||
const [elapsedTime, setElapsedTime] = useState<number>(0);
|
||||
const [showCrawlStatus, setShowCrawlStatus] = useState<boolean>(false);
|
||||
const [isScraping, setIsScraping] = useState<boolean>(false);
|
||||
const [currentPage, setCurrentPage] = useState<number>(1);
|
||||
const urlsPerPage = 10;
|
||||
|
||||
useEffect(() => {
|
||||
let timer: NodeJS.Timeout;
|
||||
if (loading) {
|
||||
setShowCrawlStatus(true);
|
||||
timer = setInterval(() => {
|
||||
setElapsedTime((prevTime) => prevTime + 1);
|
||||
}, 1000);
|
||||
}
|
||||
return () => {
|
||||
if (timer) clearInterval(timer);
|
||||
};
|
||||
}, [loading]);
|
||||
|
||||
const handleChange = (e: ChangeEvent<HTMLInputElement>) => {
|
||||
const { name, value, type, checked } = e.target;
|
||||
setFormData((prevData) => ({
|
||||
...prevData,
|
||||
[name]: type === "checkbox" ? checked : value,
|
||||
}));
|
||||
};
|
||||
|
||||
const handleSubmit = async (e: FormEvent<HTMLFormElement>) => {
|
||||
e.preventDefault();
|
||||
setLoading(true);
|
||||
setIsCollapsibleOpen(false);
|
||||
setElapsedTime(0);
|
||||
setCrawlStatus({ current: 0, total: null });
|
||||
setIsScraping(!formData.crawlSubPages);
|
||||
setCrawledUrls([]);
|
||||
setSelectedUrls([]);
|
||||
setScrapeResults({});
|
||||
setScrapingSelectedLoading(false);
|
||||
setShowCrawlStatus(false);
|
||||
|
||||
try {
|
||||
const endpoint = `${FIRECRAWL_API_URL}/v0/${
|
||||
formData.crawlSubPages ? "crawl" : "scrape"
|
||||
}`;
|
||||
|
||||
const requestBody: RequestBody = formData.crawlSubPages
|
||||
? {
|
||||
url: formData.url,
|
||||
crawlerOptions: {
|
||||
includes: formData.includePaths
|
||||
? formData.includePaths.split(",").map((p) => p.trim())
|
||||
: undefined,
|
||||
excludes: formData.excludePaths
|
||||
? formData.excludePaths.split(",").map((p) => p.trim())
|
||||
: undefined,
|
||||
maxDepth: formData.maxDepth
|
||||
? parseInt(formData.maxDepth)
|
||||
: undefined,
|
||||
limit: formData.limit ? parseInt(formData.limit) : undefined,
|
||||
returnOnlyUrls: true,
|
||||
},
|
||||
pageOptions: {
|
||||
onlyMainContent: formData.extractMainContent,
|
||||
},
|
||||
}
|
||||
: {
|
||||
url: formData.url,
|
||||
pageOptions: {
|
||||
onlyMainContent: formData.extractMainContent,
|
||||
},
|
||||
};
|
||||
|
||||
const response = await fetch(endpoint, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
Authorization: `Bearer ${FIRECRAWL_API_KEY}`,
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
body: JSON.stringify(requestBody),
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`HTTP error! status: ${response.status}`);
|
||||
}
|
||||
|
||||
const data = await response.json();
|
||||
if (formData.crawlSubPages) {
|
||||
const jobId = data.jobId;
|
||||
if (jobId) {
|
||||
const statusEndpoint = `${FIRECRAWL_API_URL}/v0/crawl/status/${jobId}`;
|
||||
let statusData: {
|
||||
status: string;
|
||||
data?: { url: string }[];
|
||||
current?: number;
|
||||
total?: number;
|
||||
};
|
||||
do {
|
||||
const statusResponse = await fetch(statusEndpoint, {
|
||||
headers: {
|
||||
Authorization: `Bearer ${FIRECRAWL_API_KEY}`,
|
||||
},
|
||||
});
|
||||
if (statusResponse.ok) {
|
||||
statusData = await statusResponse.json();
|
||||
|
||||
const urls = statusData.data
|
||||
? statusData.data.map((urlObj) => urlObj.url)
|
||||
: [];
|
||||
setCrawledUrls(urls);
|
||||
setSelectedUrls(urls);
|
||||
setCrawlStatus({
|
||||
current: urls.length || 0,
|
||||
total: urls.length || null,
|
||||
});
|
||||
if (statusData.status !== "completed") {
|
||||
// Wait for 1 second before polling again
|
||||
await new Promise((resolve) => setTimeout(resolve, 1000));
|
||||
console.log("Polling again...");
|
||||
console.log(statusData);
|
||||
} else {
|
||||
console.log("Crawl completed with status:", statusData.status);
|
||||
console.log(statusData);
|
||||
}
|
||||
} else {
|
||||
console.error("Failed to fetch crawl status");
|
||||
break;
|
||||
}
|
||||
} while (statusData.status !== "completed");
|
||||
} else {
|
||||
console.error("No jobId received from crawl request");
|
||||
}
|
||||
} else {
|
||||
setScrapeResults({ [formData.url]: data });
|
||||
setCrawlStatus({ current: 1, total: 1 });
|
||||
}
|
||||
} catch (error) {
|
||||
console.error("Error:", error);
|
||||
setScrapeResults({
|
||||
error: {
|
||||
success: false,
|
||||
data: {
|
||||
metadata: {
|
||||
pageError: "Error occurred while fetching data",
|
||||
title: "",
|
||||
description: "",
|
||||
language: "",
|
||||
sourceURL: "",
|
||||
pageStatusCode: 0,
|
||||
},
|
||||
markdown: "",
|
||||
content: "",
|
||||
html: "",
|
||||
rawHtml: "",
|
||||
llm_extraction: {},
|
||||
},
|
||||
},
|
||||
});
|
||||
} finally {
|
||||
setLoading(false);
|
||||
}
|
||||
};
|
||||
|
||||
const handleScrapeSelected = async () => {
|
||||
setLoading(true);
|
||||
setElapsedTime(0);
|
||||
setCrawlStatus({ current: 0, total: selectedUrls.length });
|
||||
setIsScraping(true);
|
||||
setScrapingSelectedLoading(true);
|
||||
const newScrapeResults: Record<string, ScrapeResult> = {};
|
||||
|
||||
for (const [index, url] of selectedUrls.entries()) {
|
||||
try {
|
||||
const response = await fetch(`${FIRECRAWL_API_URL}/v0/scrape`, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
Authorization: `Bearer ${FIRECRAWL_API_KEY}`,
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
body: JSON.stringify({
|
||||
url: url,
|
||||
pageOptions: {
|
||||
onlyMainContent: formData.extractMainContent,
|
||||
},
|
||||
}),
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`HTTP error! status: ${response.status}`);
|
||||
}
|
||||
|
||||
const data: ScrapeResult = await response.json();
|
||||
newScrapeResults[url] = data;
|
||||
setCrawlStatus((prev) => ({ ...prev, current: index + 1 }));
|
||||
setScrapeResults({ ...scrapeResults, ...newScrapeResults });
|
||||
} catch (error) {
|
||||
console.error(`Error scraping ${url}:`, error);
|
||||
newScrapeResults[url] = {
|
||||
success: false,
|
||||
data: {
|
||||
markdown: "",
|
||||
content: "",
|
||||
html: "",
|
||||
rawHtml: "",
|
||||
metadata: {
|
||||
title: "",
|
||||
description: "",
|
||||
language: "",
|
||||
sourceURL: url,
|
||||
pageStatusCode: 0,
|
||||
pageError: (error as Error).message,
|
||||
},
|
||||
llm_extraction: {},
|
||||
},
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
setLoading(false);
|
||||
setIsScraping(false);
|
||||
};
|
||||
|
||||
const handlePageChange = (newPage: number) => {
|
||||
setCurrentPage(newPage);
|
||||
};
|
||||
|
||||
const paginatedUrls = crawledUrls.slice(
|
||||
(currentPage - 1) * urlsPerPage,
|
||||
currentPage * urlsPerPage
|
||||
);
|
||||
|
||||
return (
|
||||
<div className="max-w-2xl mx-auto p-4">
|
||||
<Card>
|
||||
<CardHeader className="flex items-start justify-between mb-0 pb-4">
|
||||
<CardTitle className="flex items-center justify-between w-full space-x-2">
|
||||
<span className="text-base">Extract web content</span>
|
||||
<a
|
||||
href="https://www.firecrawl.dev"
|
||||
className="text-xs text-gray-500 font-normal px-3 py-1 bg-zinc-100 rounded-xl hover:bg-zinc-200 transition-colors"
|
||||
>
|
||||
Powered by Firecrawl 🔥
|
||||
</a>
|
||||
</CardTitle>
|
||||
<div className="text-sm text-gray-500 w-11/12 items-center">
|
||||
Use this component to quickly give your users the ability to connect
|
||||
their AI apps to web data with Firecrawl. Learn more on the{" "}
|
||||
<a
|
||||
href="https://docs.firecrawl.dev/"
|
||||
className="text-sm text-blue-500"
|
||||
>
|
||||
Firecrawl docs!
|
||||
</a>
|
||||
</div>
|
||||
</CardHeader>
|
||||
<CardContent className="space-y-4">
|
||||
<form onSubmit={handleSubmit}>
|
||||
<div className="flex items-center space-x-2">
|
||||
<Input
|
||||
placeholder="https://www.firecrawl.dev/"
|
||||
className="flex-grow"
|
||||
name="url"
|
||||
value={formData.url}
|
||||
onChange={handleChange}
|
||||
/>
|
||||
<Button type="submit" variant="default" disabled={loading}>
|
||||
{loading ? (
|
||||
<div
|
||||
role="status"
|
||||
className="flex items-center justify-between space-x-2"
|
||||
>
|
||||
<svg
|
||||
className="animate-spin h-4 w-4 text-white"
|
||||
xmlns="http://www.w3.org/2000/svg"
|
||||
fill="none"
|
||||
viewBox="0 0 24 24"
|
||||
>
|
||||
<circle
|
||||
className="opacity-25"
|
||||
cx="12"
|
||||
cy="12"
|
||||
r="10"
|
||||
stroke="currentColor"
|
||||
strokeWidth="4"
|
||||
></circle>
|
||||
<path
|
||||
className="opacity-75"
|
||||
fill="currentColor"
|
||||
d="M4 12a8 8 0 018-8V0C5.373 0 0 5.373 0 12h4zm2 5.291A7.962 7.962 0 014 12H0c0 3.042 1.135 5.824 3 7.938l3-2.647z"
|
||||
></path>
|
||||
</svg>
|
||||
<span className="sr-only">Loading...</span>
|
||||
</div>
|
||||
) : (
|
||||
"Run"
|
||||
)}
|
||||
</Button>
|
||||
</div>
|
||||
<Collapsible
|
||||
open={isCollapsibleOpen}
|
||||
onOpenChange={setIsCollapsibleOpen}
|
||||
className="mt-2"
|
||||
>
|
||||
<CollapsibleTrigger asChild>
|
||||
<Button variant="ghost" className="w-full justify-between pl-2">
|
||||
Advanced Options
|
||||
<ChevronDown className="h-4 w-4 opacity-50" />
|
||||
</Button>
|
||||
</CollapsibleTrigger>
|
||||
<CollapsibleContent className="space-y-4 mt-4 px-2">
|
||||
<div className="flex items-center space-x-2">
|
||||
<Checkbox
|
||||
id="crawlSubPages"
|
||||
name="crawlSubPages"
|
||||
checked={formData.crawlSubPages}
|
||||
onCheckedChange={(checked: boolean) =>
|
||||
setFormData((prev) => ({
|
||||
...prev,
|
||||
crawlSubPages: checked,
|
||||
}))
|
||||
}
|
||||
/>
|
||||
<label htmlFor="crawlSubPages" className="text-sm">
|
||||
Crawl sub-pages
|
||||
</label>
|
||||
</div>
|
||||
<div className="grid grid-cols-2 gap-4">
|
||||
<div>
|
||||
<Label
|
||||
htmlFor="limit"
|
||||
className="block text-left w-full pb-2"
|
||||
>
|
||||
Limit *
|
||||
</Label>
|
||||
<Input
|
||||
id="limit"
|
||||
name="limit"
|
||||
placeholder="10"
|
||||
value={formData.limit}
|
||||
onChange={handleChange}
|
||||
/>
|
||||
</div>
|
||||
<div>
|
||||
<Label
|
||||
htmlFor="maxDepth"
|
||||
className="block text-left w-full pb-2"
|
||||
>
|
||||
Max depth
|
||||
</Label>
|
||||
<Input
|
||||
id="maxDepth"
|
||||
name="maxDepth"
|
||||
placeholder="5"
|
||||
value={formData.maxDepth}
|
||||
onChange={handleChange}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
<div className="grid grid-cols-2 gap-4">
|
||||
<div>
|
||||
<Label
|
||||
htmlFor="excludePaths"
|
||||
className="block text-left w-full pb-2"
|
||||
>
|
||||
Exclude paths
|
||||
</Label>
|
||||
<Input
|
||||
id="excludePaths"
|
||||
name="excludePaths"
|
||||
placeholder="blog/, /about/"
|
||||
value={formData.excludePaths}
|
||||
onChange={handleChange}
|
||||
/>
|
||||
</div>
|
||||
<div>
|
||||
<Label
|
||||
htmlFor="includePaths"
|
||||
className="block text-left w-full pb-2"
|
||||
>
|
||||
Include only paths
|
||||
</Label>
|
||||
<Input
|
||||
id="includePaths"
|
||||
name="includePaths"
|
||||
placeholder="articles/"
|
||||
value={formData.includePaths}
|
||||
onChange={handleChange}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
<div className="flex items-center space-x-2">
|
||||
<Checkbox
|
||||
id="extractMainContent"
|
||||
name="extractMainContent"
|
||||
checked={formData.extractMainContent}
|
||||
onCheckedChange={(checked: boolean) =>
|
||||
setFormData((prev) => ({
|
||||
...prev,
|
||||
extractMainContent: checked,
|
||||
}))
|
||||
}
|
||||
/>
|
||||
<label htmlFor="extractMainContent" className="text-sm">
|
||||
Extract only main content (no headers, navs, footers, etc.)
|
||||
</label>
|
||||
</div>
|
||||
</CollapsibleContent>
|
||||
</Collapsible>
|
||||
</form>
|
||||
{showCrawlStatus && (
|
||||
<div className="flex items-center justify-between mb-2 space-x-2 bg-gray-100 p-2 rounded-md">
|
||||
<div className="flex items-center space-x-2">
|
||||
{!isScraping &&
|
||||
crawledUrls.length > 0 &&
|
||||
!scrapingSelectedLoading && (
|
||||
<>
|
||||
<Checkbox
|
||||
id="selectAll"
|
||||
checked={selectedUrls.length === crawledUrls.length}
|
||||
onCheckedChange={(checked) => {
|
||||
if (checked) {
|
||||
setSelectedUrls([...crawledUrls]);
|
||||
} else {
|
||||
setSelectedUrls([]);
|
||||
}
|
||||
}}
|
||||
/>
|
||||
<label
|
||||
htmlFor="selectAll"
|
||||
className="text-sm cursor-pointer"
|
||||
>
|
||||
{selectedUrls.length === crawledUrls.length
|
||||
? `Unselect All (${selectedUrls.length})`
|
||||
: `Select All (${selectedUrls.length})`}
|
||||
</label>
|
||||
</>
|
||||
)}
|
||||
</div>
|
||||
<div className="text-sm text-gray-600">
|
||||
{isScraping
|
||||
? `Scraped ${crawlStatus.current} page(s) in ${elapsedTime}s`
|
||||
: `Crawled ${crawlStatus.current} pages in ${elapsedTime}s`}
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{crawledUrls.length > 0 &&
|
||||
!scrapingSelectedLoading &&
|
||||
!isScraping && (
|
||||
<>
|
||||
<ul className="pl-2">
|
||||
{paginatedUrls.map((url, index) => (
|
||||
<li
|
||||
key={index}
|
||||
className="flex items-center space-x-2 my-2 text-sm"
|
||||
>
|
||||
<Checkbox
|
||||
checked={selectedUrls.includes(url)}
|
||||
onCheckedChange={() =>
|
||||
setSelectedUrls((prev) =>
|
||||
prev.includes(url)
|
||||
? prev.filter((u) => u !== url)
|
||||
: [...prev, url]
|
||||
)
|
||||
}
|
||||
/>
|
||||
<span className="flex items-center max-w-lg">
|
||||
{url.length > 70 ? `${url.slice(0, 70)}...` : url}
|
||||
</span>
|
||||
</li>
|
||||
))}
|
||||
</ul>
|
||||
<div className="flex items-center justify-between mt-4">
|
||||
<Button
|
||||
variant="outline"
|
||||
className="px-2"
|
||||
onClick={() => handlePageChange(currentPage - 1)}
|
||||
disabled={currentPage === 1}
|
||||
>
|
||||
<ChevronLeft className="h-5 w-5" />
|
||||
</Button>
|
||||
<span className="text-sm text-gray-500">
|
||||
Page {currentPage} of{" "}
|
||||
{Math.ceil(crawledUrls.length / urlsPerPage)}
|
||||
</span>
|
||||
<Button
|
||||
variant="outline"
|
||||
className="px-2"
|
||||
onClick={() => handlePageChange(currentPage + 1)}
|
||||
disabled={currentPage * urlsPerPage >= crawledUrls.length}
|
||||
>
|
||||
<ChevronRight className="h-5 w-5 " />
|
||||
</Button>
|
||||
</div>
|
||||
</>
|
||||
)}
|
||||
</CardContent>
|
||||
<CardFooter className="w-full flex justify-center">
|
||||
{crawledUrls.length > 0 && !scrapingSelectedLoading && (
|
||||
<Button
|
||||
variant="default"
|
||||
className="w-full"
|
||||
onClick={handleScrapeSelected}
|
||||
disabled={loading || selectedUrls.length === 0}
|
||||
>
|
||||
Scrape Selected URLs
|
||||
</Button>
|
||||
)}
|
||||
</CardFooter>
|
||||
</Card>
|
||||
|
||||
{Object.keys(scrapeResults).length > 0 && (
|
||||
<div className="mt-4">
|
||||
<h2 className="text-base font-bold ">Scrape Results</h2>
|
||||
<p className="text-sm text-gray-500">
|
||||
You can do whatever you want with the scrape results. Here is a
|
||||
basic showcase of the markdown.
|
||||
</p>
|
||||
<div className="flex flex-col gap-4 mt-4 w-full">
|
||||
{Object.entries(scrapeResults).map(([url, result]) => (
|
||||
<Card key={url} className="relative p-4 w-full">
|
||||
<CardTitle className="text-sm font-normal flex flex-col">
|
||||
<span>{result.data.metadata.title}</span>
|
||||
<span className="text-xs text-gray-500">
|
||||
{url
|
||||
.replace(/^(https?:\/\/)?(www\.)?/, "")
|
||||
.replace(/\/$/, "")}
|
||||
</span>
|
||||
</CardTitle>
|
||||
<CardContent className="relative px-0 pt-2 !text-xs w-full">
|
||||
<div className=" overflow-y-auto h-32 bg-zinc-100 rounded-md p-2 w-full">
|
||||
{result.success ? (
|
||||
<>
|
||||
<pre className="text-xs whitespace-pre-wrap">
|
||||
{result.data.markdown.trim()}
|
||||
</pre>
|
||||
</>
|
||||
) : (
|
||||
<>
|
||||
<p className="text-red-500">
|
||||
Failed to scrape this URL
|
||||
</p>
|
||||
<p className="text-zinc-500 font-mono">
|
||||
{result.toString()}
|
||||
</p>
|
||||
</>
|
||||
)}
|
||||
</div>
|
||||
</CardContent>
|
||||
</Card>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,603 @@
|
||||
import { useState, ChangeEvent, FormEvent, useEffect } from "react";
|
||||
import {
|
||||
Card,
|
||||
CardHeader,
|
||||
CardTitle,
|
||||
CardContent,
|
||||
CardFooter,
|
||||
} from "@/components/ui/card";
|
||||
import { Input } from "@/components/ui/input";
|
||||
import { Button } from "@/components/ui/button";
|
||||
import { Checkbox } from "@/components/ui/checkbox";
|
||||
import { Label } from "@/components/ui/label";
|
||||
import {
|
||||
Collapsible,
|
||||
CollapsibleContent,
|
||||
CollapsibleTrigger,
|
||||
} from "@/components/ui/collapsible";
|
||||
import { ChevronDown, ChevronLeft, ChevronRight } from "lucide-react";
|
||||
|
||||
//! Hardcoded values (not recommended for production)
|
||||
//! Highly recommended to move all Firecrawl API calls to the backend (e.g. Next.js API route)
|
||||
const FIRECRAWL_API_URL = "https://api.firecrawl.dev"; // Replace with your actual API URL whether it is local or using Firecrawl Cloud
|
||||
const FIRECRAWL_API_KEY = "fc-YOUR_API_KEY"; // Replace with your actual API key
|
||||
|
||||
interface FormData {
|
||||
url: string;
|
||||
crawlSubPages: boolean;
|
||||
search: string;
|
||||
limit: string;
|
||||
maxDepth: string;
|
||||
excludePaths: string;
|
||||
includePaths: string;
|
||||
extractMainContent: boolean;
|
||||
}
|
||||
|
||||
interface CrawlerOptions {
|
||||
includes?: string[];
|
||||
excludes?: string[];
|
||||
maxDepth?: number;
|
||||
limit?: number;
|
||||
returnOnlyUrls: boolean;
|
||||
}
|
||||
|
||||
interface ScrapeOptions {
|
||||
formats?: string[];
|
||||
onlyMainContent?: boolean;
|
||||
}
|
||||
|
||||
interface PageOptions {
|
||||
onlyMainContent: boolean;
|
||||
}
|
||||
|
||||
interface RequestBody {
|
||||
url: string;
|
||||
crawlerOptions?: CrawlerOptions;
|
||||
pageOptions?: PageOptions;
|
||||
search?: string;
|
||||
excludePaths?: string[];
|
||||
includePaths?: string[];
|
||||
maxDepth?: number;
|
||||
limit?: number;
|
||||
scrapeOptions?: ScrapeOptions;
|
||||
formats?: string[];
|
||||
}
|
||||
|
||||
interface ScrapeResultMetadata {
|
||||
title: string;
|
||||
description: string;
|
||||
language: string;
|
||||
sourceURL: string;
|
||||
pageStatusCode: number;
|
||||
pageError?: string;
|
||||
[key: string]: string | number | undefined;
|
||||
}
|
||||
|
||||
interface ScrapeResultData {
|
||||
markdown: string;
|
||||
content: string;
|
||||
html: string;
|
||||
rawHtml: string;
|
||||
metadata: ScrapeResultMetadata;
|
||||
llm_extraction: Record<string, unknown>;
|
||||
warning?: string;
|
||||
}
|
||||
|
||||
interface ScrapeResult {
|
||||
success: boolean;
|
||||
data: ScrapeResultData;
|
||||
}
|
||||
|
||||
export default function FirecrawlComponentV1() {
|
||||
const [formData, setFormData] = useState<FormData>({
|
||||
url: "",
|
||||
crawlSubPages: false,
|
||||
search: "",
|
||||
limit: "",
|
||||
maxDepth: "",
|
||||
excludePaths: "",
|
||||
includePaths: "",
|
||||
extractMainContent: false,
|
||||
});
|
||||
const [loading, setLoading] = useState<boolean>(false);
|
||||
const [scrapingSelectedLoading, setScrapingSelectedLoading] =
|
||||
useState<boolean>(false);
|
||||
const [crawledUrls, setCrawledUrls] = useState<string[]>([]);
|
||||
const [selectedUrls, setSelectedUrls] = useState<string[]>([]);
|
||||
const [scrapeResults, setScrapeResults] = useState<
|
||||
Record<string, ScrapeResult>
|
||||
>({});
|
||||
const [isCollapsibleOpen, setIsCollapsibleOpen] = useState(true);
|
||||
const [crawlStatus, setCrawlStatus] = useState<{
|
||||
current: number;
|
||||
total: number | null;
|
||||
}>({ current: 0, total: null });
|
||||
const [elapsedTime, setElapsedTime] = useState<number>(0);
|
||||
const [showCrawlStatus, setShowCrawlStatus] = useState<boolean>(false);
|
||||
const [isScraping, setIsScraping] = useState<boolean>(false);
|
||||
const [currentPage, setCurrentPage] = useState<number>(1);
|
||||
const urlsPerPage = 10;
|
||||
|
||||
useEffect(() => {
|
||||
let timer: NodeJS.Timeout;
|
||||
if (loading) {
|
||||
setShowCrawlStatus(true);
|
||||
timer = setInterval(() => {
|
||||
setElapsedTime((prevTime) => prevTime + 1);
|
||||
}, 1000);
|
||||
}
|
||||
return () => {
|
||||
if (timer) clearInterval(timer);
|
||||
};
|
||||
}, [loading]);
|
||||
|
||||
const handleChange = (e: ChangeEvent<HTMLInputElement>) => {
|
||||
const { name, value, type, checked } = e.target;
|
||||
setFormData((prevData) => {
|
||||
const newData = {
|
||||
...prevData,
|
||||
[name]: type === "checkbox" ? checked : value,
|
||||
};
|
||||
|
||||
// Automatically check "Crawl Sub-pages" if limit or search have content
|
||||
if (name === "limit" || name === "search") {
|
||||
newData.crawlSubPages = !!value || !!newData.limit || !!newData.search;
|
||||
}
|
||||
|
||||
return newData;
|
||||
});
|
||||
};
|
||||
|
||||
const handleSubmit = async (e: FormEvent<HTMLFormElement>) => {
|
||||
e.preventDefault();
|
||||
setLoading(true);
|
||||
setIsCollapsibleOpen(false);
|
||||
setElapsedTime(0);
|
||||
setCrawlStatus({ current: 0, total: null });
|
||||
setIsScraping(!formData.crawlSubPages);
|
||||
setCrawledUrls([]);
|
||||
setSelectedUrls([]);
|
||||
setScrapeResults({});
|
||||
setScrapingSelectedLoading(false);
|
||||
setShowCrawlStatus(false);
|
||||
|
||||
try {
|
||||
const endpoint = `${FIRECRAWL_API_URL}/v1/${
|
||||
formData.crawlSubPages ? "map" : "scrape"
|
||||
}`;
|
||||
|
||||
const requestBody: RequestBody = formData.crawlSubPages
|
||||
? {
|
||||
url: formData.url,
|
||||
search: formData.search || undefined,
|
||||
limit: formData.limit ? parseInt(formData.limit) : undefined,
|
||||
}
|
||||
: {
|
||||
url: formData.url,
|
||||
formats: ["markdown"],
|
||||
};
|
||||
|
||||
const response = await fetch(endpoint, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
Authorization: `Bearer ${FIRECRAWL_API_KEY}`,
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
body: JSON.stringify(requestBody),
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`HTTP error! status: ${response.status}`);
|
||||
}
|
||||
|
||||
const data = await response.json();
|
||||
if (formData.crawlSubPages) {
|
||||
if (data.success === true && Array.isArray(data.links)) {
|
||||
setCrawledUrls(data.links);
|
||||
setSelectedUrls(data.links);
|
||||
setCrawlStatus({
|
||||
current: data.links.length,
|
||||
total: data.links.length,
|
||||
});
|
||||
|
||||
// Set scrape results with the links
|
||||
const linkResults: Record<string, ScrapeResult> = {};
|
||||
data.links.forEach((link: string) => {
|
||||
linkResults[link] = {
|
||||
success: true,
|
||||
data: {
|
||||
metadata: {
|
||||
sourceURL: link,
|
||||
title: "",
|
||||
description: "",
|
||||
language: "",
|
||||
pageStatusCode: 200,
|
||||
},
|
||||
markdown: "",
|
||||
content: "",
|
||||
html: "",
|
||||
rawHtml: "",
|
||||
llm_extraction: {},
|
||||
},
|
||||
};
|
||||
});
|
||||
} else {
|
||||
console.error("Unexpected response format from map endpoint");
|
||||
console.log(data);
|
||||
}
|
||||
} else {
|
||||
setScrapeResults({ [formData.url]: data });
|
||||
setCrawlStatus({ current: 1, total: 1 });
|
||||
}
|
||||
} catch (error) {
|
||||
console.error("Error:", error);
|
||||
setScrapeResults({
|
||||
error: {
|
||||
success: false,
|
||||
data: {
|
||||
metadata: {
|
||||
pageError: "Error occurred while fetching data",
|
||||
title: "",
|
||||
description: "",
|
||||
language: "",
|
||||
sourceURL: "",
|
||||
pageStatusCode: 0,
|
||||
},
|
||||
markdown: "",
|
||||
content: "",
|
||||
html: "",
|
||||
rawHtml: "",
|
||||
llm_extraction: {},
|
||||
},
|
||||
},
|
||||
});
|
||||
} finally {
|
||||
setLoading(false);
|
||||
}
|
||||
};
|
||||
|
||||
const handleScrapeSelected = async () => {
|
||||
setLoading(true);
|
||||
setElapsedTime(0);
|
||||
setCrawlStatus({ current: 0, total: selectedUrls.length });
|
||||
setIsScraping(true);
|
||||
setScrapingSelectedLoading(true);
|
||||
const newScrapeResults: Record<string, ScrapeResult> = {};
|
||||
|
||||
for (const [index, url] of selectedUrls.entries()) {
|
||||
try {
|
||||
const response = await fetch(`${FIRECRAWL_API_URL}/v1/scrape`, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
Authorization: `Bearer ${FIRECRAWL_API_KEY}`,
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
body: JSON.stringify({
|
||||
url: url,
|
||||
formats: ["markdown"],
|
||||
}),
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`HTTP error! status: ${response.status}`);
|
||||
}
|
||||
|
||||
const data: ScrapeResult = await response.json();
|
||||
newScrapeResults[url] = data;
|
||||
setCrawlStatus((prev) => ({ ...prev, current: index + 1 }));
|
||||
setScrapeResults({ ...scrapeResults, ...newScrapeResults });
|
||||
} catch (error) {
|
||||
console.error(`Error scraping ${url}:`, error);
|
||||
newScrapeResults[url] = {
|
||||
success: false,
|
||||
data: {
|
||||
markdown: "",
|
||||
content: "",
|
||||
html: "",
|
||||
rawHtml: "",
|
||||
metadata: {
|
||||
title: "",
|
||||
description: "",
|
||||
language: "",
|
||||
sourceURL: url,
|
||||
pageStatusCode: 0,
|
||||
pageError: (error as Error).message,
|
||||
},
|
||||
llm_extraction: {},
|
||||
},
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
setLoading(false);
|
||||
setIsScraping(false);
|
||||
};
|
||||
|
||||
const handlePageChange = (newPage: number) => {
|
||||
setCurrentPage(newPage);
|
||||
};
|
||||
|
||||
const paginatedUrls = crawledUrls.slice(
|
||||
(currentPage - 1) * urlsPerPage,
|
||||
currentPage * urlsPerPage
|
||||
);
|
||||
|
||||
return (
|
||||
<div className="max-w-2xl mx-auto p-4">
|
||||
<Card>
|
||||
<CardHeader className="flex items-start justify-between mb-0 pb-4">
|
||||
<CardTitle className="flex items-center justify-between w-full space-x-2">
|
||||
<span className="text-base">Extract web content (V1)</span>
|
||||
<a
|
||||
href="https://www.firecrawl.dev"
|
||||
className="text-xs text-gray-500 font-normal px-3 py-1 bg-zinc-100 rounded-xl hover:bg-zinc-200 transition-colors"
|
||||
>
|
||||
Powered by Firecrawl 🔥
|
||||
</a>
|
||||
</CardTitle>
|
||||
<div className="text-sm text-gray-500 w-11/12 items-center">
|
||||
Use this component to quickly give your users the ability to connect
|
||||
their AI apps to web data with Firecrawl. Learn more on the{" "}
|
||||
<a
|
||||
href="https://docs.firecrawl.dev/"
|
||||
className="text-sm text-blue-500"
|
||||
>
|
||||
Firecrawl docs!
|
||||
</a>
|
||||
</div>
|
||||
</CardHeader>
|
||||
<CardContent className="space-y-4">
|
||||
<form onSubmit={handleSubmit}>
|
||||
<div className="flex items-center space-x-2">
|
||||
<Input
|
||||
placeholder="https://www.firecrawl.dev/"
|
||||
className="flex-grow"
|
||||
name="url"
|
||||
value={formData.url}
|
||||
onChange={handleChange}
|
||||
/>
|
||||
<Button type="submit" variant="default" disabled={loading}>
|
||||
{loading ? (
|
||||
<div
|
||||
role="status"
|
||||
className="flex items-center justify-between space-x-2"
|
||||
>
|
||||
<svg
|
||||
className="animate-spin h-4 w-4 text-white"
|
||||
xmlns="http://www.w3.org/2000/svg"
|
||||
fill="none"
|
||||
viewBox="0 0 24 24"
|
||||
>
|
||||
<circle
|
||||
className="opacity-25"
|
||||
cx="12"
|
||||
cy="12"
|
||||
r="10"
|
||||
stroke="currentColor"
|
||||
strokeWidth="4"
|
||||
></circle>
|
||||
<path
|
||||
className="opacity-75"
|
||||
fill="currentColor"
|
||||
d="M4 12a8 8 0 018-8V0C5.373 0 0 5.373 0 12h4zm2 5.291A7.962 7.962 0 014 12H0c0 3.042 1.135 5.824 3 7.938l3-2.647z"
|
||||
></path>
|
||||
</svg>
|
||||
<span className="sr-only">Loading...</span>
|
||||
</div>
|
||||
) : (
|
||||
"Run"
|
||||
)}
|
||||
</Button>
|
||||
</div>
|
||||
<Collapsible
|
||||
open={isCollapsibleOpen}
|
||||
onOpenChange={setIsCollapsibleOpen}
|
||||
className="mt-2"
|
||||
>
|
||||
<CollapsibleTrigger asChild>
|
||||
<Button variant="ghost" className="w-full justify-between pl-2">
|
||||
Advanced Options
|
||||
<ChevronDown className="h-4 w-4 opacity-50" />
|
||||
</Button>
|
||||
</CollapsibleTrigger>
|
||||
<CollapsibleContent className="space-y-4 mt-4 px-2">
|
||||
<div className="flex items-center space-x-2">
|
||||
<Checkbox
|
||||
id="crawlSubPages"
|
||||
name="crawlSubPages"
|
||||
checked={formData.crawlSubPages}
|
||||
onCheckedChange={(checked: boolean) =>
|
||||
setFormData((prev) => ({
|
||||
...prev,
|
||||
crawlSubPages: checked,
|
||||
}))
|
||||
}
|
||||
/>
|
||||
<label htmlFor="crawlSubPages" className="text-sm">
|
||||
Crawl Sub-pages
|
||||
</label>
|
||||
</div>
|
||||
|
||||
<div className="grid grid-cols-2 gap-4">
|
||||
<div>
|
||||
<Label
|
||||
htmlFor="search"
|
||||
className="block text-left w-full pb-2"
|
||||
>
|
||||
Search for specific pages in crawl *
|
||||
</Label>
|
||||
<Input
|
||||
id="search"
|
||||
name="search"
|
||||
placeholder="python sdk"
|
||||
value={formData.search}
|
||||
onChange={handleChange}
|
||||
/>
|
||||
</div>
|
||||
<div>
|
||||
<Label
|
||||
htmlFor="limit"
|
||||
className="block text-left w-full pb-2"
|
||||
>
|
||||
Limit *
|
||||
</Label>
|
||||
<Input
|
||||
id="limit"
|
||||
name="limit"
|
||||
placeholder="10"
|
||||
value={formData.limit}
|
||||
onChange={handleChange}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
</CollapsibleContent>
|
||||
</Collapsible>
|
||||
</form>
|
||||
{showCrawlStatus && (
|
||||
<div className="flex items-center justify-between mb-2 space-x-2 bg-gray-100 p-2 rounded-md">
|
||||
<div className="flex items-center space-x-2">
|
||||
{!isScraping &&
|
||||
crawledUrls.length > 0 &&
|
||||
!scrapingSelectedLoading && (
|
||||
<>
|
||||
<Checkbox
|
||||
id="selectAll"
|
||||
checked={selectedUrls.length === crawledUrls.length}
|
||||
onCheckedChange={(checked) => {
|
||||
if (checked) {
|
||||
setSelectedUrls([...crawledUrls]);
|
||||
} else {
|
||||
setSelectedUrls([]);
|
||||
}
|
||||
}}
|
||||
/>
|
||||
<label
|
||||
htmlFor="selectAll"
|
||||
className="text-sm cursor-pointer"
|
||||
>
|
||||
{selectedUrls.length === crawledUrls.length
|
||||
? `Unselect All (${selectedUrls.length})`
|
||||
: `Select All (${selectedUrls.length})`}
|
||||
</label>
|
||||
</>
|
||||
)}
|
||||
</div>
|
||||
<div className="text-sm text-gray-600">
|
||||
{isScraping
|
||||
? `Scraped ${crawlStatus.current} page(s) in ${elapsedTime}s`
|
||||
: `Crawled ${crawlStatus.current} pages in ${elapsedTime}s`}
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{crawledUrls.length > 0 &&
|
||||
!scrapingSelectedLoading &&
|
||||
!isScraping && (
|
||||
<>
|
||||
<ul className="pl-2">
|
||||
{paginatedUrls.map((url, index) => (
|
||||
<li
|
||||
key={index}
|
||||
className="flex items-center space-x-2 my-2 text-sm"
|
||||
>
|
||||
<Checkbox
|
||||
checked={selectedUrls.includes(url)}
|
||||
onCheckedChange={() =>
|
||||
setSelectedUrls((prev) =>
|
||||
prev.includes(url)
|
||||
? prev.filter((u) => u !== url)
|
||||
: [...prev, url]
|
||||
)
|
||||
}
|
||||
/>
|
||||
<span className="flex items-center max-w-lg">
|
||||
{url.length > 70 ? `${url.slice(0, 70)}...` : url}
|
||||
</span>
|
||||
</li>
|
||||
))}
|
||||
</ul>
|
||||
<div className="flex items-center justify-between mt-4">
|
||||
<Button
|
||||
variant="outline"
|
||||
className="px-2"
|
||||
onClick={() => handlePageChange(currentPage - 1)}
|
||||
disabled={currentPage === 1}
|
||||
>
|
||||
<ChevronLeft className="h-5 w-5" />
|
||||
</Button>
|
||||
<span className="text-sm text-gray-500">
|
||||
Page {currentPage} of{" "}
|
||||
{Math.ceil(crawledUrls.length / urlsPerPage)}
|
||||
</span>
|
||||
<Button
|
||||
variant="outline"
|
||||
className="px-2"
|
||||
onClick={() => handlePageChange(currentPage + 1)}
|
||||
disabled={currentPage * urlsPerPage >= crawledUrls.length}
|
||||
>
|
||||
<ChevronRight className="h-5 w-5 " />
|
||||
</Button>
|
||||
</div>
|
||||
</>
|
||||
)}
|
||||
</CardContent>
|
||||
<CardFooter className="w-full flex justify-center">
|
||||
{crawledUrls.length > 0 && !scrapingSelectedLoading && (
|
||||
<Button
|
||||
variant="default"
|
||||
className="w-full"
|
||||
onClick={handleScrapeSelected}
|
||||
disabled={loading || selectedUrls.length === 0}
|
||||
>
|
||||
Scrape Selected URLs
|
||||
</Button>
|
||||
)}
|
||||
</CardFooter>
|
||||
</Card>
|
||||
|
||||
{Object.keys(scrapeResults).length > 0 && (
|
||||
<div className="mt-4">
|
||||
<h2 className="text-base font-bold ">Scrape Results</h2>
|
||||
<p className="text-sm text-gray-500">
|
||||
You can do whatever you want with the scrape results. Here is a
|
||||
basic showcase of the markdown.
|
||||
</p>
|
||||
<div className="flex flex-col gap-4 mt-4 w-full">
|
||||
{Object.entries(scrapeResults).map(([url, result]) => (
|
||||
<Card key={url} className="relative p-4 w-full">
|
||||
<CardTitle className="text-sm font-normal flex flex-col">
|
||||
<span>{result.data.metadata.title}</span>
|
||||
<span className="text-xs text-gray-500">
|
||||
{url
|
||||
.replace(/^(https?:\/\/)?(www\.)?/, "")
|
||||
.replace(/\/$/, "")}
|
||||
</span>
|
||||
</CardTitle>
|
||||
<CardContent className="relative px-0 pt-2 !text-xs w-full">
|
||||
<div className=" overflow-y-auto h-32 bg-zinc-100 rounded-md p-2 w-full">
|
||||
{result.success ? (
|
||||
<>
|
||||
<pre className="text-xs whitespace-pre-wrap">
|
||||
{result.data.markdown.trim()}
|
||||
</pre>
|
||||
</>
|
||||
) : (
|
||||
<>
|
||||
<p className="text-red-500">
|
||||
Failed to scrape this URL
|
||||
</p>
|
||||
<p className="text-zinc-500 font-mono">
|
||||
{result.toString()}
|
||||
</p>
|
||||
</>
|
||||
)}
|
||||
</div>
|
||||
</CardContent>
|
||||
</Card>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,56 @@
|
||||
import * as React from "react"
|
||||
import { Slot } from "@radix-ui/react-slot"
|
||||
import { cva, type VariantProps } from "class-variance-authority"
|
||||
|
||||
import { cn } from "@/lib/utils"
|
||||
|
||||
const buttonVariants = cva(
|
||||
"inline-flex items-center justify-center whitespace-nowrap rounded-md text-sm font-medium ring-offset-background transition-colors focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-2 disabled:pointer-events-none disabled:opacity-50",
|
||||
{
|
||||
variants: {
|
||||
variant: {
|
||||
default: "bg-primary text-primary-foreground hover:bg-primary/90",
|
||||
destructive:
|
||||
"bg-destructive text-destructive-foreground hover:bg-destructive/90",
|
||||
outline:
|
||||
"border border-input bg-background hover:bg-accent hover:text-accent-foreground",
|
||||
secondary:
|
||||
"bg-secondary text-secondary-foreground hover:bg-secondary/80",
|
||||
ghost: "hover:bg-accent hover:text-accent-foreground",
|
||||
link: "text-primary underline-offset-4 hover:underline",
|
||||
},
|
||||
size: {
|
||||
default: "h-10 px-4 py-2",
|
||||
sm: "h-9 rounded-md px-3",
|
||||
lg: "h-11 rounded-md px-8",
|
||||
icon: "h-10 w-10",
|
||||
},
|
||||
},
|
||||
defaultVariants: {
|
||||
variant: "default",
|
||||
size: "default",
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
export interface ButtonProps
|
||||
extends React.ButtonHTMLAttributes<HTMLButtonElement>,
|
||||
VariantProps<typeof buttonVariants> {
|
||||
asChild?: boolean
|
||||
}
|
||||
|
||||
const Button = React.forwardRef<HTMLButtonElement, ButtonProps>(
|
||||
({ className, variant, size, asChild = false, ...props }, ref) => {
|
||||
const Comp = asChild ? Slot : "button"
|
||||
return (
|
||||
<Comp
|
||||
className={cn(buttonVariants({ variant, size, className }))}
|
||||
ref={ref}
|
||||
{...props}
|
||||
/>
|
||||
)
|
||||
}
|
||||
)
|
||||
Button.displayName = "Button"
|
||||
|
||||
export { Button, buttonVariants }
|
||||
@@ -0,0 +1,79 @@
|
||||
import * as React from "react"
|
||||
|
||||
import { cn } from "@/lib/utils"
|
||||
|
||||
const Card = React.forwardRef<
|
||||
HTMLDivElement,
|
||||
React.HTMLAttributes<HTMLDivElement>
|
||||
>(({ className, ...props }, ref) => (
|
||||
<div
|
||||
ref={ref}
|
||||
className={cn(
|
||||
"rounded-lg border bg-card text-card-foreground shadow-sm",
|
||||
className
|
||||
)}
|
||||
{...props}
|
||||
/>
|
||||
))
|
||||
Card.displayName = "Card"
|
||||
|
||||
const CardHeader = React.forwardRef<
|
||||
HTMLDivElement,
|
||||
React.HTMLAttributes<HTMLDivElement>
|
||||
>(({ className, ...props }, ref) => (
|
||||
<div
|
||||
ref={ref}
|
||||
className={cn("flex flex-col space-y-1.5 p-6", className)}
|
||||
{...props}
|
||||
/>
|
||||
))
|
||||
CardHeader.displayName = "CardHeader"
|
||||
|
||||
const CardTitle = React.forwardRef<
|
||||
HTMLParagraphElement,
|
||||
React.HTMLAttributes<HTMLHeadingElement>
|
||||
>(({ className, ...props }, ref) => (
|
||||
<h3
|
||||
ref={ref}
|
||||
className={cn(
|
||||
"text-2xl font-semibold leading-none tracking-tight",
|
||||
className
|
||||
)}
|
||||
{...props}
|
||||
/>
|
||||
))
|
||||
CardTitle.displayName = "CardTitle"
|
||||
|
||||
const CardDescription = React.forwardRef<
|
||||
HTMLParagraphElement,
|
||||
React.HTMLAttributes<HTMLParagraphElement>
|
||||
>(({ className, ...props }, ref) => (
|
||||
<p
|
||||
ref={ref}
|
||||
className={cn("text-sm text-muted-foreground", className)}
|
||||
{...props}
|
||||
/>
|
||||
))
|
||||
CardDescription.displayName = "CardDescription"
|
||||
|
||||
const CardContent = React.forwardRef<
|
||||
HTMLDivElement,
|
||||
React.HTMLAttributes<HTMLDivElement>
|
||||
>(({ className, ...props }, ref) => (
|
||||
<div ref={ref} className={cn("p-6 pt-0", className)} {...props} />
|
||||
))
|
||||
CardContent.displayName = "CardContent"
|
||||
|
||||
const CardFooter = React.forwardRef<
|
||||
HTMLDivElement,
|
||||
React.HTMLAttributes<HTMLDivElement>
|
||||
>(({ className, ...props }, ref) => (
|
||||
<div
|
||||
ref={ref}
|
||||
className={cn("flex items-center p-6 pt-0", className)}
|
||||
{...props}
|
||||
/>
|
||||
))
|
||||
CardFooter.displayName = "CardFooter"
|
||||
|
||||
export { Card, CardHeader, CardFooter, CardTitle, CardDescription, CardContent }
|
||||
@@ -0,0 +1,28 @@
|
||||
import * as React from "react"
|
||||
import * as CheckboxPrimitive from "@radix-ui/react-checkbox"
|
||||
import { Check } from "lucide-react"
|
||||
|
||||
import { cn } from "@/lib/utils"
|
||||
|
||||
const Checkbox = React.forwardRef<
|
||||
React.ElementRef<typeof CheckboxPrimitive.Root>,
|
||||
React.ComponentPropsWithoutRef<typeof CheckboxPrimitive.Root>
|
||||
>(({ className, ...props }, ref) => (
|
||||
<CheckboxPrimitive.Root
|
||||
ref={ref}
|
||||
className={cn(
|
||||
"peer h-4 w-4 shrink-0 rounded-sm border border-primary ring-offset-background focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-2 disabled:cursor-not-allowed disabled:opacity-50 data-[state=checked]:bg-primary data-[state=checked]:text-primary-foreground",
|
||||
className
|
||||
)}
|
||||
{...props}
|
||||
>
|
||||
<CheckboxPrimitive.Indicator
|
||||
className={cn("flex items-center justify-center text-current")}
|
||||
>
|
||||
<Check className="h-4 w-4" />
|
||||
</CheckboxPrimitive.Indicator>
|
||||
</CheckboxPrimitive.Root>
|
||||
))
|
||||
Checkbox.displayName = CheckboxPrimitive.Root.displayName
|
||||
|
||||
export { Checkbox }
|
||||
@@ -0,0 +1,9 @@
|
||||
import * as CollapsiblePrimitive from "@radix-ui/react-collapsible"
|
||||
|
||||
const Collapsible = CollapsiblePrimitive.Root
|
||||
|
||||
const CollapsibleTrigger = CollapsiblePrimitive.CollapsibleTrigger
|
||||
|
||||
const CollapsibleContent = CollapsiblePrimitive.CollapsibleContent
|
||||
|
||||
export { Collapsible, CollapsibleTrigger, CollapsibleContent }
|
||||
@@ -0,0 +1,24 @@
|
||||
import * as React from "react"
|
||||
|
||||
import { cn } from "@/lib/utils"
|
||||
|
||||
export type InputProps = React.InputHTMLAttributes<HTMLInputElement>
|
||||
|
||||
const Input = React.forwardRef<HTMLInputElement, InputProps>(
|
||||
({ className, type, ...props }, ref) => {
|
||||
return (
|
||||
<input
|
||||
type={type}
|
||||
className={cn(
|
||||
"flex h-10 w-full rounded-md border border-input bg-background px-3 py-2 text-sm ring-offset-background file:border-0 file:bg-transparent file:text-sm file:font-medium placeholder:text-muted-foreground focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-2 disabled:cursor-not-allowed disabled:opacity-50",
|
||||
className
|
||||
)}
|
||||
ref={ref}
|
||||
{...props}
|
||||
/>
|
||||
)
|
||||
}
|
||||
)
|
||||
Input.displayName = "Input"
|
||||
|
||||
export { Input }
|
||||
@@ -0,0 +1,24 @@
|
||||
import * as React from "react"
|
||||
import * as LabelPrimitive from "@radix-ui/react-label"
|
||||
import { cva, type VariantProps } from "class-variance-authority"
|
||||
|
||||
import { cn } from "@/lib/utils"
|
||||
|
||||
const labelVariants = cva(
|
||||
"text-sm font-medium leading-none peer-disabled:cursor-not-allowed peer-disabled:opacity-70"
|
||||
)
|
||||
|
||||
const Label = React.forwardRef<
|
||||
React.ElementRef<typeof LabelPrimitive.Root>,
|
||||
React.ComponentPropsWithoutRef<typeof LabelPrimitive.Root> &
|
||||
VariantProps<typeof labelVariants>
|
||||
>(({ className, ...props }, ref) => (
|
||||
<LabelPrimitive.Root
|
||||
ref={ref}
|
||||
className={cn(labelVariants(), className)}
|
||||
{...props}
|
||||
/>
|
||||
))
|
||||
Label.displayName = LabelPrimitive.Root.displayName
|
||||
|
||||
export { Label }
|
||||
@@ -0,0 +1,42 @@
|
||||
import * as React from "react"
|
||||
import * as RadioGroupPrimitive from "@radix-ui/react-radio-group"
|
||||
import { Circle } from "lucide-react"
|
||||
|
||||
import { cn } from "@/lib/utils"
|
||||
|
||||
const RadioGroup = React.forwardRef<
|
||||
React.ElementRef<typeof RadioGroupPrimitive.Root>,
|
||||
React.ComponentPropsWithoutRef<typeof RadioGroupPrimitive.Root>
|
||||
>(({ className, ...props }, ref) => {
|
||||
return (
|
||||
<RadioGroupPrimitive.Root
|
||||
className={cn("grid gap-2", className)}
|
||||
{...props}
|
||||
ref={ref}
|
||||
/>
|
||||
)
|
||||
})
|
||||
RadioGroup.displayName = RadioGroupPrimitive.Root.displayName
|
||||
|
||||
const RadioGroupItem = React.forwardRef<
|
||||
React.ElementRef<typeof RadioGroupPrimitive.Item>,
|
||||
React.ComponentPropsWithoutRef<typeof RadioGroupPrimitive.Item>
|
||||
>(({ className, ...props }, ref) => {
|
||||
return (
|
||||
<RadioGroupPrimitive.Item
|
||||
ref={ref}
|
||||
className={cn(
|
||||
"aspect-square h-4 w-4 rounded-full border border-primary text-primary ring-offset-background focus:outline-none focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-2 disabled:cursor-not-allowed disabled:opacity-50",
|
||||
className
|
||||
)}
|
||||
{...props}
|
||||
>
|
||||
<RadioGroupPrimitive.Indicator className="flex items-center justify-center">
|
||||
<Circle className="h-2.5 w-2.5 fill-current text-current" />
|
||||
</RadioGroupPrimitive.Indicator>
|
||||
</RadioGroupPrimitive.Item>
|
||||
)
|
||||
})
|
||||
RadioGroupItem.displayName = RadioGroupPrimitive.Item.displayName
|
||||
|
||||
export { RadioGroup, RadioGroupItem }
|
||||
69
참고/firecrawl-main/apps/ui/ingestion-ui/src/index.css
Normal file
69
참고/firecrawl-main/apps/ui/ingestion-ui/src/index.css
Normal file
@@ -0,0 +1,69 @@
|
||||
@tailwind base;
|
||||
@tailwind components;
|
||||
@tailwind utilities;
|
||||
|
||||
@layer base {
|
||||
:root {
|
||||
--background: 0 0% 100%;
|
||||
--foreground: 222.2 84% 4.9%;
|
||||
--card: 0 0% 100%;
|
||||
--card-foreground: 222.2 84% 4.9%;
|
||||
--popover: 0 0% 100%;
|
||||
--popover-foreground: 222.2 84% 4.9%;
|
||||
--primary: 222.2 47.4% 11.2%;
|
||||
--primary-foreground: 210 40% 98%;
|
||||
--secondary: 210 40% 96.1%;
|
||||
--secondary-foreground: 222.2 47.4% 11.2%;
|
||||
--muted: 210 40% 96.1%;
|
||||
--muted-foreground: 215.4 16.3% 46.9%;
|
||||
--accent: 210 40% 96.1%;
|
||||
--accent-foreground: 222.2 47.4% 11.2%;
|
||||
--destructive: 0 84.2% 60.2%;
|
||||
--destructive-foreground: 210 40% 98%;
|
||||
--border: 214.3 31.8% 91.4%;
|
||||
--input: 214.3 31.8% 91.4%;
|
||||
--ring: 222.2 84% 4.9%;
|
||||
--radius: 0.5rem;
|
||||
--chart-1: 12 76% 61%;
|
||||
--chart-2: 173 58% 39%;
|
||||
--chart-3: 197 37% 24%;
|
||||
--chart-4: 43 74% 66%;
|
||||
--chart-5: 27 87% 67%;
|
||||
}
|
||||
|
||||
.dark {
|
||||
--background: 222.2 84% 4.9%;
|
||||
--foreground: 210 40% 98%;
|
||||
--card: 222.2 84% 4.9%;
|
||||
--card-foreground: 210 40% 98%;
|
||||
--popover: 222.2 84% 4.9%;
|
||||
--popover-foreground: 210 40% 98%;
|
||||
--primary: 210 40% 98%;
|
||||
--primary-foreground: 222.2 47.4% 11.2%;
|
||||
--secondary: 217.2 32.6% 17.5%;
|
||||
--secondary-foreground: 210 40% 98%;
|
||||
--muted: 217.2 32.6% 17.5%;
|
||||
--muted-foreground: 215 20.2% 65.1%;
|
||||
--accent: 217.2 32.6% 17.5%;
|
||||
--accent-foreground: 210 40% 98%;
|
||||
--destructive: 0 62.8% 30.6%;
|
||||
--destructive-foreground: 210 40% 98%;
|
||||
--border: 217.2 32.6% 17.5%;
|
||||
--input: 217.2 32.6% 17.5%;
|
||||
--ring: 212.7 26.8% 83.9%;
|
||||
--chart-1: 220 70% 50%;
|
||||
--chart-2: 160 60% 45%;
|
||||
--chart-3: 30 80% 55%;
|
||||
--chart-4: 280 65% 60%;
|
||||
--chart-5: 340 75% 55%;
|
||||
}
|
||||
}
|
||||
|
||||
@layer base {
|
||||
* {
|
||||
@apply border-border;
|
||||
}
|
||||
body {
|
||||
@apply bg-background text-foreground;
|
||||
}
|
||||
}
|
||||
6
참고/firecrawl-main/apps/ui/ingestion-ui/src/lib/utils.ts
Normal file
6
참고/firecrawl-main/apps/ui/ingestion-ui/src/lib/utils.ts
Normal file
@@ -0,0 +1,6 @@
|
||||
import { type ClassValue, clsx } from "clsx"
|
||||
import { twMerge } from "tailwind-merge"
|
||||
|
||||
export function cn(...inputs: ClassValue[]) {
|
||||
return twMerge(clsx(inputs))
|
||||
}
|
||||
10
참고/firecrawl-main/apps/ui/ingestion-ui/src/main.tsx
Normal file
10
참고/firecrawl-main/apps/ui/ingestion-ui/src/main.tsx
Normal file
@@ -0,0 +1,10 @@
|
||||
import React from 'react'
|
||||
import ReactDOM from 'react-dom/client'
|
||||
import App from './App.tsx'
|
||||
import './index.css'
|
||||
|
||||
ReactDOM.createRoot(document.getElementById('root')!).render(
|
||||
<React.StrictMode>
|
||||
<App />
|
||||
</React.StrictMode>,
|
||||
)
|
||||
1
참고/firecrawl-main/apps/ui/ingestion-ui/src/vite-env.d.ts
vendored
Normal file
1
참고/firecrawl-main/apps/ui/ingestion-ui/src/vite-env.d.ts
vendored
Normal file
@@ -0,0 +1 @@
|
||||
/// <reference types="vite/client" />
|
||||
77
참고/firecrawl-main/apps/ui/ingestion-ui/tailwind.config.js
Normal file
77
참고/firecrawl-main/apps/ui/ingestion-ui/tailwind.config.js
Normal file
@@ -0,0 +1,77 @@
|
||||
/** @type {import('tailwindcss').Config} */
|
||||
module.exports = {
|
||||
darkMode: ["class"],
|
||||
content: [
|
||||
'./pages/**/*.{ts,tsx}',
|
||||
'./components/**/*.{ts,tsx}',
|
||||
'./app/**/*.{ts,tsx}',
|
||||
'./src/**/*.{ts,tsx}',
|
||||
],
|
||||
prefix: "",
|
||||
theme: {
|
||||
container: {
|
||||
center: true,
|
||||
padding: "2rem",
|
||||
screens: {
|
||||
"2xl": "1400px",
|
||||
},
|
||||
},
|
||||
extend: {
|
||||
colors: {
|
||||
border: "hsl(var(--border))",
|
||||
input: "hsl(var(--input))",
|
||||
ring: "hsl(var(--ring))",
|
||||
background: "hsl(var(--background))",
|
||||
foreground: "hsl(var(--foreground))",
|
||||
primary: {
|
||||
DEFAULT: "hsl(var(--primary))",
|
||||
foreground: "hsl(var(--primary-foreground))",
|
||||
},
|
||||
secondary: {
|
||||
DEFAULT: "hsl(var(--secondary))",
|
||||
foreground: "hsl(var(--secondary-foreground))",
|
||||
},
|
||||
destructive: {
|
||||
DEFAULT: "hsl(var(--destructive))",
|
||||
foreground: "hsl(var(--destructive-foreground))",
|
||||
},
|
||||
muted: {
|
||||
DEFAULT: "hsl(var(--muted))",
|
||||
foreground: "hsl(var(--muted-foreground))",
|
||||
},
|
||||
accent: {
|
||||
DEFAULT: "hsl(var(--accent))",
|
||||
foreground: "hsl(var(--accent-foreground))",
|
||||
},
|
||||
popover: {
|
||||
DEFAULT: "hsl(var(--popover))",
|
||||
foreground: "hsl(var(--popover-foreground))",
|
||||
},
|
||||
card: {
|
||||
DEFAULT: "hsl(var(--card))",
|
||||
foreground: "hsl(var(--card-foreground))",
|
||||
},
|
||||
},
|
||||
borderRadius: {
|
||||
lg: "var(--radius)",
|
||||
md: "calc(var(--radius) - 2px)",
|
||||
sm: "calc(var(--radius) - 4px)",
|
||||
},
|
||||
keyframes: {
|
||||
"accordion-down": {
|
||||
from: { height: "0" },
|
||||
to: { height: "var(--radix-accordion-content-height)" },
|
||||
},
|
||||
"accordion-up": {
|
||||
from: { height: "var(--radix-accordion-content-height)" },
|
||||
to: { height: "0" },
|
||||
},
|
||||
},
|
||||
animation: {
|
||||
"accordion-down": "accordion-down 0.2s ease-out",
|
||||
"accordion-up": "accordion-up 0.2s ease-out",
|
||||
},
|
||||
},
|
||||
},
|
||||
plugins: [require("tailwindcss-animate")],
|
||||
}
|
||||
33
참고/firecrawl-main/apps/ui/ingestion-ui/tsconfig.app.json
Normal file
33
참고/firecrawl-main/apps/ui/ingestion-ui/tsconfig.app.json
Normal file
@@ -0,0 +1,33 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"composite": true,
|
||||
"tsBuildInfoFile": "./node_modules/.tmp/tsconfig.app.tsbuildinfo",
|
||||
"target": "ES2020",
|
||||
"useDefineForClassFields": true,
|
||||
"lib": ["ES2020", "DOM", "DOM.Iterable"],
|
||||
"module": "ESNext",
|
||||
"skipLibCheck": true,
|
||||
|
||||
/* Bundler mode */
|
||||
"moduleResolution": "bundler",
|
||||
"allowImportingTsExtensions": true,
|
||||
"resolveJsonModule": true,
|
||||
"isolatedModules": true,
|
||||
"moduleDetection": "force",
|
||||
"noEmit": true,
|
||||
"jsx": "react-jsx",
|
||||
|
||||
/* Linting */
|
||||
"strict": true,
|
||||
"noUnusedLocals": true,
|
||||
"noUnusedParameters": true,
|
||||
"noFallthroughCasesInSwitch": true,
|
||||
"baseUrl": ".",
|
||||
"paths": {
|
||||
"@/*": [
|
||||
"./src/*"
|
||||
]
|
||||
}
|
||||
},
|
||||
"include": ["src"]
|
||||
}
|
||||
17
참고/firecrawl-main/apps/ui/ingestion-ui/tsconfig.json
Normal file
17
참고/firecrawl-main/apps/ui/ingestion-ui/tsconfig.json
Normal file
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"files": [],
|
||||
"references": [
|
||||
{
|
||||
"path": "./tsconfig.app.json"
|
||||
},
|
||||
{
|
||||
"path": "./tsconfig.node.json"
|
||||
}
|
||||
],
|
||||
"compilerOptions": {
|
||||
"baseUrl": ".",
|
||||
"paths": {
|
||||
"@/*": ["./src/*"]
|
||||
}
|
||||
}
|
||||
}
|
||||
13
참고/firecrawl-main/apps/ui/ingestion-ui/tsconfig.node.json
Normal file
13
참고/firecrawl-main/apps/ui/ingestion-ui/tsconfig.node.json
Normal file
@@ -0,0 +1,13 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"composite": true,
|
||||
"tsBuildInfoFile": "./node_modules/.tmp/tsconfig.node.tsbuildinfo",
|
||||
"skipLibCheck": true,
|
||||
"module": "ESNext",
|
||||
"moduleResolution": "bundler",
|
||||
"allowSyntheticDefaultImports": true,
|
||||
"strict": true,
|
||||
"noEmit": true
|
||||
},
|
||||
"include": ["vite.config.ts"]
|
||||
}
|
||||
12
참고/firecrawl-main/apps/ui/ingestion-ui/vite.config.ts
Normal file
12
참고/firecrawl-main/apps/ui/ingestion-ui/vite.config.ts
Normal file
@@ -0,0 +1,12 @@
|
||||
import path from "path"
|
||||
import react from "@vitejs/plugin-react"
|
||||
import { defineConfig } from "vite"
|
||||
|
||||
export default defineConfig({
|
||||
plugins: [react()],
|
||||
resolve: {
|
||||
alias: {
|
||||
"@": path.resolve(__dirname, "./src"),
|
||||
},
|
||||
},
|
||||
})
|
||||
Reference in New Issue
Block a user