참고소스 수정본
This commit is contained in:
LASTA_DEV01\lasta
@@ -0,0 +1,39 @@
|
||||
# Install Firecrawl on a Kubernetes Cluster (Simple Version)
|
||||
# Before installing
|
||||
1. Set [secret.yaml](secret.yaml) and [configmap.yaml](configmap.yaml) and do not check in secrets
|
||||
- **Note**: If `REDIS_PASSWORD` is configured in the secret, please modify the ConfigMap to reflect the following format for `REDIS_URL` and `REDIS_RATE_LIMIT_URL`:
|
||||
```yaml
|
||||
REDIS_URL: "redis://:password@host:port"
|
||||
REDIS_RATE_LIMIT_URL: "redis://:password@host:port"
|
||||
```
|
||||
Replace `password`, `host`, and `port` with the appropriate values.
|
||||
|
||||
## Install
|
||||
```bash
|
||||
kubectl apply -f configmap.yaml
|
||||
kubectl apply -f secret.yaml
|
||||
kubectl apply -f playwright-service.yaml
|
||||
kubectl apply -f api.yaml
|
||||
kubectl apply -f worker.yaml
|
||||
kubectl apply -f nuq-worker.yaml
|
||||
kubectl apply -f nuq-postgres.yaml
|
||||
kubectl apply -f redis.yaml
|
||||
```
|
||||
|
||||
|
||||
# Port Forwarding for Testing
|
||||
```bash
|
||||
kubectl port-forward svc/api 3002:3002 -n dev
|
||||
```
|
||||
|
||||
# Delete Firecrawl
|
||||
```bash
|
||||
kubectl delete -f configmap.yaml
|
||||
kubectl delete -f secret.yaml
|
||||
kubectl delete -f playwright-service.yaml
|
||||
kubectl delete -f api.yaml
|
||||
kubectl delete -f worker.yaml
|
||||
kubectl delete -f nuq-worker.yaml
|
||||
kubectl delete -f nuq-postgres.yaml
|
||||
kubectl delete -f redis.yaml
|
||||
```
|
||||
@@ -0,0 +1,72 @@
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: api
|
||||
spec:
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app: api
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: api
|
||||
spec:
|
||||
terminationGracePeriodSeconds: 180
|
||||
imagePullSecrets:
|
||||
- name: docker-registry-secret
|
||||
containers:
|
||||
- name: api
|
||||
image: ghcr.io/firecrawl/firecrawl:latest
|
||||
imagePullPolicy: Always
|
||||
command: [ "node" ]
|
||||
args: [ "--max-old-space-size=6144", "dist/src/index.js" ]
|
||||
ports:
|
||||
- containerPort: 3002
|
||||
env:
|
||||
- name: FLY_PROCESS_GROUP
|
||||
value: "app"
|
||||
- name: PORT
|
||||
value: "3002"
|
||||
envFrom:
|
||||
- configMapRef:
|
||||
name: firecrawl-config
|
||||
- secretRef:
|
||||
name: firecrawl-secret
|
||||
resources:
|
||||
requests:
|
||||
memory: "4G"
|
||||
cpu: "2000m"
|
||||
limits:
|
||||
memory: "6G"
|
||||
cpu: "2000m"
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /v0/health/liveness
|
||||
port: 3002
|
||||
initialDelaySeconds: 30
|
||||
periodSeconds: 30
|
||||
timeoutSeconds: 5
|
||||
successThreshold: 1
|
||||
failureThreshold: 3
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /v0/health/readiness
|
||||
port: 3002
|
||||
initialDelaySeconds: 30
|
||||
periodSeconds: 30
|
||||
timeoutSeconds: 5
|
||||
successThreshold: 1
|
||||
failureThreshold: 3
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: api
|
||||
spec:
|
||||
selector:
|
||||
app: api
|
||||
ports:
|
||||
- protocol: TCP
|
||||
port: 3002
|
||||
targetPort: 3002
|
||||
@@ -0,0 +1,15 @@
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: firecrawl-config
|
||||
data:
|
||||
HOST: "0.0.0.0"
|
||||
REDIS_URL: "redis://redis:6379"
|
||||
REDIS_RATE_LIMIT_URL: "redis://redis:6379"
|
||||
PLAYWRIGHT_MICROSERVICE_URL: "http://playwright-service:3000"
|
||||
USE_DB_AUTHENTICATION: "false"
|
||||
SENTRY_ENVIRONMENT: "production"
|
||||
ENV: "production"
|
||||
LOGGING_LEVEL: "DEBUG"
|
||||
IS_KUBERNETES: "true"
|
||||
NUQ_DATABASE_URL: "postgresql://postgres:password@nuq-postgres:5432/postgres"
|
||||
@@ -0,0 +1,53 @@
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: nuq-postgres
|
||||
spec:
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app: nuq-postgres
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: nuq-postgres
|
||||
spec:
|
||||
containers:
|
||||
- name: nuq-postgres
|
||||
image: ghcr.io/firecrawl/nuq-postgres:latest
|
||||
imagePullPolicy: Always
|
||||
env:
|
||||
- name: POSTGRES_USER
|
||||
value: "postgres"
|
||||
- name: POSTGRES_PASSWORD
|
||||
value: "password"
|
||||
- name: POSTGRES_DB
|
||||
value: "postgres"
|
||||
ports:
|
||||
- containerPort: 5432
|
||||
volumeMounts:
|
||||
- name: postgres-storage
|
||||
mountPath: /var/lib/postgresql/data
|
||||
resources:
|
||||
requests:
|
||||
memory: "512Mi"
|
||||
cpu: "250m"
|
||||
limits:
|
||||
memory: "1Gi"
|
||||
cpu: "500m"
|
||||
volumes:
|
||||
- name: postgres-storage
|
||||
emptyDir: {}
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: nuq-postgres
|
||||
spec:
|
||||
selector:
|
||||
app: nuq-postgres
|
||||
ports:
|
||||
- protocol: TCP
|
||||
port: 5432
|
||||
targetPort: 5432
|
||||
type: ClusterIP
|
||||
@@ -0,0 +1,58 @@
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: nuq-worker
|
||||
spec:
|
||||
replicas: 5
|
||||
selector:
|
||||
matchLabels:
|
||||
app: nuq-worker
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: nuq-worker
|
||||
spec:
|
||||
imagePullSecrets:
|
||||
- name: docker-registry-secret
|
||||
terminationGracePeriodSeconds: 60
|
||||
containers:
|
||||
- name: nuq-worker
|
||||
image: ghcr.io/firecrawl/firecrawl:latest
|
||||
imagePullPolicy: Always
|
||||
command: [ "node" ]
|
||||
args: [ "--max-old-space-size=3072", "dist/src/services/worker/nuq-worker.js" ]
|
||||
env:
|
||||
- name: FLY_PROCESS_GROUP
|
||||
value: "nuq-worker"
|
||||
- name: NUQ_WORKER_PORT
|
||||
value: "3006"
|
||||
envFrom:
|
||||
- configMapRef:
|
||||
name: firecrawl-config
|
||||
- secretRef:
|
||||
name: firecrawl-secret
|
||||
resources:
|
||||
requests:
|
||||
memory: "3G"
|
||||
cpu: "1000m"
|
||||
limits:
|
||||
memory: "4G"
|
||||
cpu: "1000m"
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /health
|
||||
port: 3006
|
||||
initialDelaySeconds: 5
|
||||
periodSeconds: 5
|
||||
timeoutSeconds: 5
|
||||
successThreshold: 1
|
||||
failureThreshold: 3
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /health
|
||||
port: 3006
|
||||
initialDelaySeconds: 5
|
||||
periodSeconds: 5
|
||||
timeoutSeconds: 5
|
||||
successThreshold: 1
|
||||
failureThreshold: 3
|
||||
@@ -0,0 +1,54 @@
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: playwright-service-config
|
||||
data:
|
||||
PORT: "3000"
|
||||
ALLOW_LOCAL_WEBHOOKS: "false"
|
||||
---
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: playwright-service
|
||||
spec:
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app: playwright-service
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: playwright-service
|
||||
spec:
|
||||
imagePullSecrets:
|
||||
- name: docker-registry-secret
|
||||
containers:
|
||||
- name: playwright-service
|
||||
image: ghcr.io/firecrawl/playwright-service:latest
|
||||
imagePullPolicy: Always
|
||||
ports:
|
||||
- containerPort: 3000
|
||||
envFrom:
|
||||
- configMapRef:
|
||||
name: playwright-service-config
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /health
|
||||
port: 3000
|
||||
initialDelaySeconds: 30
|
||||
periodSeconds: 30
|
||||
timeoutSeconds: 5
|
||||
successThreshold: 1
|
||||
failureThreshold: 3
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: playwright-service
|
||||
spec:
|
||||
selector:
|
||||
app: playwright-service
|
||||
ports:
|
||||
- protocol: TCP
|
||||
port: 3000
|
||||
targetPort: 3000
|
||||
@@ -0,0 +1,45 @@
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: redis
|
||||
spec:
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app: redis
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: redis
|
||||
spec:
|
||||
containers:
|
||||
- name: redis
|
||||
image: redis:alpine
|
||||
command: [ "/bin/sh", "-c" ] # Run a shell script as entrypoint
|
||||
args:
|
||||
- |
|
||||
if [ -n "$REDIS_PASSWORD" ]; then
|
||||
echo "Starting Redis with authentication"
|
||||
exec redis-server --bind 0.0.0.0 --requirepass "$REDIS_PASSWORD"
|
||||
else
|
||||
echo "Starting Redis without authentication"
|
||||
exec redis-server --bind 0.0.0.0
|
||||
fi
|
||||
env:
|
||||
- name: REDIS_PASSWORD
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: firecrawl-secret
|
||||
key: REDIS_PASSWORD
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: redis
|
||||
spec:
|
||||
selector:
|
||||
app: redis
|
||||
ports:
|
||||
- protocol: TCP
|
||||
port: 6379
|
||||
targetPort: 6379
|
||||
@@ -0,0 +1,15 @@
|
||||
apiVersion: v1
|
||||
kind: Secret
|
||||
metadata:
|
||||
name: firecrawl-secret
|
||||
type: Opaque
|
||||
data:
|
||||
OPENAI_API_KEY: ""
|
||||
SLACK_WEBHOOK_URL: ""
|
||||
LLAMAPARSE_API_KEY: ""
|
||||
BULL_AUTH_KEY: ""
|
||||
TEST_API_KEY: ""
|
||||
STRIPE_PRICE_ID_STANDARD: ""
|
||||
STRIPE_PRICE_ID_SCALE: ""
|
||||
FIRE_ENGINE_BETA_URL: ""
|
||||
REDIS_PASSWORD: ""
|
||||
@@ -0,0 +1,49 @@
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: worker
|
||||
spec:
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app: worker
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: worker
|
||||
spec:
|
||||
imagePullSecrets:
|
||||
- name: docker-registry-secret
|
||||
terminationGracePeriodSeconds: 60
|
||||
containers:
|
||||
- name: worker
|
||||
image: ghcr.io/firecrawl/firecrawl:latest
|
||||
imagePullPolicy: Always
|
||||
command: [ "node" ]
|
||||
args: [ "--max-old-space-size=3072", "dist/src/services/queue-worker.js" ]
|
||||
env:
|
||||
- name: FLY_PROCESS_GROUP
|
||||
value: "worker"
|
||||
- name: PORT
|
||||
value: "3005"
|
||||
envFrom:
|
||||
- configMapRef:
|
||||
name: firecrawl-config
|
||||
- secretRef:
|
||||
name: firecrawl-secret
|
||||
resources:
|
||||
requests:
|
||||
memory: "3G"
|
||||
cpu: "1000m"
|
||||
limits:
|
||||
memory: "4G"
|
||||
cpu: "1000m"
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /liveness
|
||||
port: 3005
|
||||
initialDelaySeconds: 5
|
||||
periodSeconds: 5
|
||||
timeoutSeconds: 5
|
||||
successThreshold: 1
|
||||
failureThreshold: 3
|
||||
Reference in New Issue
Block a user