참고소스 수정본
This commit is contained in:
LASTA_DEV01\lasta
@@ -0,0 +1,23 @@
|
||||
# Patterns to ignore when building packages.
|
||||
# This supports shell glob matching, relative path matching, and
|
||||
# negation (prefixed with !). Only one pattern per line.
|
||||
.DS_Store
|
||||
# Common VCS dirs
|
||||
.git/
|
||||
.gitignore
|
||||
.bzr/
|
||||
.bzrignore
|
||||
.hg/
|
||||
.hgignore
|
||||
.svn/
|
||||
# Common backup files
|
||||
*.swp
|
||||
*.bak
|
||||
*.tmp
|
||||
*.orig
|
||||
*~
|
||||
# Various IDEs
|
||||
.project
|
||||
.idea/
|
||||
*.tmproj
|
||||
.vscode/
|
||||
@@ -0,0 +1,5 @@
|
||||
apiVersion: v2
|
||||
name: firecrawl
|
||||
description: A Helm chart for deploying the Firecrawl application
|
||||
type: application
|
||||
version: 0.2.0
|
||||
120
참고/firecrawl-main/examples/kubernetes/firecrawl-helm/README.md
Normal file
120
참고/firecrawl-main/examples/kubernetes/firecrawl-helm/README.md
Normal file
@@ -0,0 +1,120 @@
|
||||
# Firecrawl Helm Chart
|
||||
|
||||
This chart deploys Firecrawl on Kubernetes with:
|
||||
- `api`
|
||||
- `worker` (queue-worker)
|
||||
- `extract-worker`
|
||||
- `nuq-worker`
|
||||
- `nuq-prefetch-worker`
|
||||
- `playwright-service`
|
||||
- `redis`
|
||||
- `nuq-postgres`
|
||||
- `rabbitmq`
|
||||
|
||||
## Image Strategy
|
||||
|
||||
- **x86-only cluster**: use official Firecrawl images from GHCR (`ghcr.io/firecrawl/...`).
|
||||
- **ARM or mixed ARM+x86 cluster**: use your multi-arch `winkkgmbh` images.
|
||||
|
||||
Official Firecrawl images are fine for x86. Use winkk images only when ARM support is needed.
|
||||
|
||||
## Configure Values
|
||||
|
||||
Use `values.yaml` plus one overlay.
|
||||
|
||||
Important fields:
|
||||
- `secret.*` for API keys and sensitive values.
|
||||
- `config.extra` / `secret.extra` for custom env vars.
|
||||
- `image.dockerSecretEnabled` and `imagePullSecrets` for private registries.
|
||||
- `resources.enabled` enables/disables all container resource requests/limits.
|
||||
Default: `false`.
|
||||
- `rabbitmq.enabled`, `extractWorker.enabled`, `nuqPrefetchWorker.enabled` to toggle components.
|
||||
|
||||
## Deploy
|
||||
|
||||
Render:
|
||||
|
||||
```bash
|
||||
HELM_NO_PLUGINS=1 helm template firecrawl . \
|
||||
-f values.yaml \
|
||||
-f overlays/prod/values.yaml \
|
||||
-n firecrawl
|
||||
```
|
||||
|
||||
Install/upgrade:
|
||||
|
||||
```bash
|
||||
HELM_NO_PLUGINS=1 helm upgrade firecrawl . \
|
||||
-f values.yaml \
|
||||
-f overlays/prod/values.yaml \
|
||||
-n firecrawl \
|
||||
--install \
|
||||
--create-namespace
|
||||
```
|
||||
|
||||
### Use Official Firecrawl Images (x86-only)
|
||||
|
||||
If your cluster is x86-only and you want official images, override repositories:
|
||||
|
||||
```bash
|
||||
HELM_NO_PLUGINS=1 helm upgrade firecrawl . \
|
||||
-f values.yaml \
|
||||
-f overlays/prod/values.yaml \
|
||||
--set image.repository=ghcr.io/firecrawl/firecrawl \
|
||||
--set playwright.repository=ghcr.io/firecrawl/playwright-service \
|
||||
--set nuqPostgres.image.repository=ghcr.io/firecrawl/nuq-postgres \
|
||||
-n firecrawl \
|
||||
--install \
|
||||
--create-namespace
|
||||
```
|
||||
|
||||
## Build and Push Multi-Arch Containers (ARM+x86)
|
||||
|
||||
Run from `examples/kubernetes/firecrawl-helm`:
|
||||
|
||||
```bash
|
||||
docker buildx create --name multiarch --use --bootstrap
|
||||
```
|
||||
|
||||
```bash
|
||||
docker buildx build --platform linux/amd64,linux/arm64 --push \
|
||||
-t docker.io/winkkgmbh/firecrawl:latest \
|
||||
../../../apps/api
|
||||
|
||||
docker buildx build --platform linux/amd64,linux/arm64 --push \
|
||||
-t docker.io/winkkgmbh/firecrawl-playwright:latest \
|
||||
../../../apps/playwright-service-ts
|
||||
|
||||
docker buildx build --platform linux/amd64,linux/arm64 --push \
|
||||
-t docker.io/winkkgmbh/nuq-postgres:latest \
|
||||
../../../apps/nuq-postgres
|
||||
```
|
||||
|
||||
## Package and Push Helm Chart (OCI)
|
||||
|
||||
```bash
|
||||
HELM_NO_PLUGINS=1 helm package . --destination /tmp/helm-packages
|
||||
HELM_NO_PLUGINS=1 helm push /tmp/helm-packages/firecrawl-0.2.0.tgz oci://registry-1.docker.io/winkkgmbh
|
||||
```
|
||||
|
||||
Install from OCI:
|
||||
|
||||
```bash
|
||||
HELM_NO_PLUGINS=1 helm upgrade --install firecrawl oci://registry-1.docker.io/winkkgmbh/firecrawl \
|
||||
--version 0.2.0 \
|
||||
-n firecrawl --create-namespace \
|
||||
-f values.yaml \
|
||||
-f overlays/prod/values.yaml
|
||||
```
|
||||
|
||||
## Test
|
||||
|
||||
```bash
|
||||
kubectl port-forward svc/firecrawl-firecrawl-api 3002:3002 -n firecrawl
|
||||
```
|
||||
|
||||
## Cleanup
|
||||
|
||||
```bash
|
||||
helm uninstall firecrawl -n firecrawl
|
||||
```
|
||||
@@ -0,0 +1 @@
|
||||
# Override the default values for the dev environment
|
||||
@@ -0,0 +1 @@
|
||||
# Override the default values for the prod environment
|
||||
@@ -0,0 +1,18 @@
|
||||
{{/*
|
||||
Return the name of the chart.
|
||||
*/}}
|
||||
{{- define "firecrawl.name" -}}
|
||||
{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}}
|
||||
{{- end -}}
|
||||
|
||||
{{/*
|
||||
Return the fully qualified name of the chart.
|
||||
*/}}
|
||||
{{- define "firecrawl.fullname" -}}
|
||||
{{- $name := default .Chart.Name .Values.nameOverride -}}
|
||||
{{- if .Values.fullnameOverride -}}
|
||||
{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}}
|
||||
{{- else -}}
|
||||
{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
@@ -0,0 +1,50 @@
|
||||
{{- $fullname := include "firecrawl.fullname" . -}}
|
||||
{{- $redisDefault := printf "redis://%s-redis:%v" $fullname .Values.service.redis.port -}}
|
||||
{{- $playwrightDefault := printf "http://%s-playwright:%v/scrape" $fullname .Values.service.playwright.port -}}
|
||||
{{- $pgUser := default "postgres" .Values.nuqPostgres.auth.username -}}
|
||||
{{- $pgPassword := default "password" .Values.nuqPostgres.auth.password -}}
|
||||
{{- $pgDb := default "postgres" .Values.nuqPostgres.auth.database -}}
|
||||
{{- $nuqDbDefault := printf "postgresql://%s:%s@%s-nuq-postgres:5432/%s" $pgUser $pgPassword $fullname $pgDb -}}
|
||||
{{- $nuqDbUrl := default $nuqDbDefault .Values.config.NUQ_DATABASE_URL -}}
|
||||
{{- $rabbitMqDefault := printf "amqp://%s-rabbitmq:%v" $fullname .Values.service.rabbitmq.port -}}
|
||||
{{- $apiHostDefault := printf "%s-api" $fullname -}}
|
||||
{{- $apiPortDefault := printf "%v" .Values.service.api.port -}}
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: {{ include "firecrawl.fullname" . }}-config
|
||||
data:
|
||||
NUM_WORKERS_PER_QUEUE: {{ .Values.config.NUM_WORKERS_PER_QUEUE | quote }}
|
||||
PORT: {{ default (printf "%v" .Values.service.api.port) .Values.config.PORT | quote }}
|
||||
HOST: {{ .Values.config.HOST | quote }}
|
||||
WORKER_PORT: {{ default (printf "%v" .Values.worker.port) .Values.config.WORKER_PORT | quote }}
|
||||
EXTRACT_WORKER_PORT: {{ default (printf "%v" .Values.extractWorker.port) .Values.config.EXTRACT_WORKER_PORT | quote }}
|
||||
NUQ_WORKER_PORT: {{ default (printf "%v" .Values.nuqWorker.port) .Values.config.NUQ_WORKER_PORT | quote }}
|
||||
NUQ_PREFETCH_WORKER_PORT: {{ default (printf "%v" .Values.nuqPrefetchWorker.port) .Values.config.NUQ_PREFETCH_WORKER_PORT | quote }}
|
||||
NUQ_WORKER_COUNT: {{ default (printf "%v" .Values.nuqWorker.replicaCount) .Values.config.NUQ_WORKER_COUNT | quote }}
|
||||
REDIS_URL: {{ default $redisDefault .Values.config.REDIS_URL | quote }}
|
||||
REDIS_RATE_LIMIT_URL: {{ default $redisDefault .Values.config.REDIS_RATE_LIMIT_URL | quote }}
|
||||
PLAYWRIGHT_MICROSERVICE_URL: {{ default $playwrightDefault .Values.config.PLAYWRIGHT_MICROSERVICE_URL | quote }}
|
||||
NUQ_DATABASE_URL: {{ $nuqDbUrl | quote }}
|
||||
NUQ_DATABASE_URL_LISTEN: {{ default $nuqDbUrl .Values.config.NUQ_DATABASE_URL_LISTEN | quote }}
|
||||
NUQ_RABBITMQ_URL: {{ default $rabbitMqDefault .Values.config.NUQ_RABBITMQ_URL | quote }}
|
||||
USE_DB_AUTHENTICATION: {{ .Values.config.USE_DB_AUTHENTICATION | quote }}
|
||||
IS_KUBERNETES: {{ .Values.config.IS_KUBERNETES | quote }}
|
||||
ENV: {{ .Values.config.ENV | quote }}
|
||||
LOGGING_LEVEL: {{ .Values.config.LOGGING_LEVEL | quote }}
|
||||
FIRECRAWL_APP_SCHEME: {{ .Values.config.FIRECRAWL_APP_SCHEME | quote }}
|
||||
FIRECRAWL_APP_HOST: {{ default $apiHostDefault .Values.config.FIRECRAWL_APP_HOST | quote }}
|
||||
FIRECRAWL_APP_PORT: {{ default $apiPortDefault .Values.config.FIRECRAWL_APP_PORT | quote }}
|
||||
OPENAI_BASE_URL: {{ .Values.config.OPENAI_BASE_URL | quote }}
|
||||
MODEL_NAME: {{ .Values.config.MODEL_NAME | quote }}
|
||||
MODEL_EMBEDDING_NAME: {{ .Values.config.MODEL_EMBEDDING_NAME | quote }}
|
||||
OLLAMA_BASE_URL: {{ .Values.config.OLLAMA_BASE_URL | quote }}
|
||||
PROXY_SERVER: {{ .Values.config.PROXY_SERVER | quote }}
|
||||
PROXY_USERNAME: {{ .Values.config.PROXY_USERNAME | quote }}
|
||||
SEARXNG_ENDPOINT: {{ .Values.config.SEARXNG_ENDPOINT | quote }}
|
||||
SEARXNG_ENGINES: {{ .Values.config.SEARXNG_ENGINES | quote }}
|
||||
SEARXNG_CATEGORIES: {{ .Values.config.SEARXNG_CATEGORIES | quote }}
|
||||
SELF_HOSTED_WEBHOOK_URL: {{ .Values.config.SELF_HOSTED_WEBHOOK_URL | quote }}
|
||||
{{- range $k, $v := .Values.config.extra }}
|
||||
{{ $k }}: {{ $v | quote }}
|
||||
{{- end }}
|
||||
@@ -0,0 +1,63 @@
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: {{ include "firecrawl.fullname" . }}-api
|
||||
labels:
|
||||
app: {{ include "firecrawl.name" . }}-api
|
||||
spec:
|
||||
replicas: {{ .Values.api.replicaCount | default .Values.replicaCount }}
|
||||
selector:
|
||||
matchLabels:
|
||||
app: {{ include "firecrawl.name" . }}-api
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: {{ include "firecrawl.name" . }}-api
|
||||
spec:
|
||||
{{- if .Values.image.dockerSecretEnabled }}
|
||||
imagePullSecrets:
|
||||
{{- toYaml .Values.imagePullSecrets | nindent 8 }}
|
||||
{{- end }}
|
||||
terminationGracePeriodSeconds: 180
|
||||
containers:
|
||||
- name: api
|
||||
image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
|
||||
imagePullPolicy: {{ .Values.image.pullPolicy }}
|
||||
command: [ "node" ]
|
||||
args: [ "--max-old-space-size=6144", "dist/src/index.js" ]
|
||||
ports:
|
||||
- containerPort: {{ .Values.service.api.port }}
|
||||
env:
|
||||
- name: FLY_PROCESS_GROUP
|
||||
value: "app"
|
||||
- name: NUQ_POD_NAME
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
fieldPath: metadata.name
|
||||
envFrom:
|
||||
- configMapRef:
|
||||
name: {{ include "firecrawl.fullname" . }}-config
|
||||
- secretRef:
|
||||
name: {{ include "firecrawl.fullname" . }}-secret
|
||||
{{- if and .Values.resources.enabled .Values.api.resources }}
|
||||
resources:
|
||||
{{- toYaml .Values.api.resources | nindent 12 }}
|
||||
{{- end }}
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /v0/health/liveness
|
||||
port: {{ .Values.service.api.port }}
|
||||
initialDelaySeconds: 30
|
||||
periodSeconds: 30
|
||||
timeoutSeconds: 5
|
||||
successThreshold: 1
|
||||
failureThreshold: 3
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /v0/health/readiness
|
||||
port: {{ .Values.service.api.port }}
|
||||
initialDelaySeconds: 30
|
||||
periodSeconds: 30
|
||||
timeoutSeconds: 5
|
||||
successThreshold: 1
|
||||
failureThreshold: 3
|
||||
@@ -0,0 +1,67 @@
|
||||
{{- if .Values.extractWorker.enabled }}
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: {{ include "firecrawl.fullname" . }}-extract-worker
|
||||
labels:
|
||||
app: {{ include "firecrawl.name" . }}-extract-worker
|
||||
spec:
|
||||
replicas: {{ .Values.extractWorker.replicaCount | default 1 }}
|
||||
selector:
|
||||
matchLabels:
|
||||
app: {{ include "firecrawl.name" . }}-extract-worker
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: {{ include "firecrawl.name" . }}-extract-worker
|
||||
spec:
|
||||
{{- if .Values.image.dockerSecretEnabled }}
|
||||
imagePullSecrets:
|
||||
{{- toYaml .Values.imagePullSecrets | nindent 8 }}
|
||||
{{- end }}
|
||||
terminationGracePeriodSeconds: 60
|
||||
containers:
|
||||
- name: extract-worker
|
||||
image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
|
||||
imagePullPolicy: {{ .Values.image.pullPolicy }}
|
||||
command: [ "node" ]
|
||||
args: [ "--max-old-space-size=3072", "dist/src/services/extract-worker.js" ]
|
||||
ports:
|
||||
- containerPort: {{ .Values.extractWorker.port | int }}
|
||||
env:
|
||||
- name: FLY_PROCESS_GROUP
|
||||
value: "extract-worker"
|
||||
- name: EXTRACT_WORKER_PORT
|
||||
value: {{ .Values.extractWorker.port | quote }}
|
||||
- name: NUQ_POD_NAME
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
fieldPath: metadata.name
|
||||
envFrom:
|
||||
- configMapRef:
|
||||
name: {{ include "firecrawl.fullname" . }}-config
|
||||
- secretRef:
|
||||
name: {{ include "firecrawl.fullname" . }}-secret
|
||||
{{- if and .Values.resources.enabled .Values.extractWorker.resources }}
|
||||
resources:
|
||||
{{- toYaml .Values.extractWorker.resources | nindent 12 }}
|
||||
{{- end }}
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /liveness
|
||||
port: {{ .Values.extractWorker.port | int }}
|
||||
initialDelaySeconds: 10
|
||||
periodSeconds: 10
|
||||
timeoutSeconds: 5
|
||||
successThreshold: 1
|
||||
failureThreshold: 3
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /health
|
||||
port: {{ .Values.extractWorker.port | int }}
|
||||
initialDelaySeconds: 10
|
||||
periodSeconds: 10
|
||||
timeoutSeconds: 5
|
||||
successThreshold: 1
|
||||
failureThreshold: 3
|
||||
{{- end }}
|
||||
@@ -0,0 +1,73 @@
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: {{ include "firecrawl.fullname" . }}-nuq-postgres
|
||||
labels:
|
||||
app: {{ include "firecrawl.name" . }}-nuq-postgres
|
||||
spec:
|
||||
replicas: {{ .Values.nuqPostgres.replicaCount | default 1 }}
|
||||
selector:
|
||||
matchLabels:
|
||||
app: {{ include "firecrawl.name" . }}-nuq-postgres
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: {{ include "firecrawl.name" . }}-nuq-postgres
|
||||
spec:
|
||||
{{- if .Values.image.dockerSecretEnabled }}
|
||||
imagePullSecrets:
|
||||
{{- toYaml .Values.imagePullSecrets | nindent 8 }}
|
||||
{{- end }}
|
||||
containers:
|
||||
- name: nuq-postgres
|
||||
image: "{{ .Values.nuqPostgres.image.repository }}:{{ .Values.nuqPostgres.image.tag | default "latest" }}"
|
||||
imagePullPolicy: {{ .Values.nuqPostgres.image.pullPolicy | default "Always" }}
|
||||
env:
|
||||
- name: POSTGRES_USER
|
||||
value: "{{ .Values.nuqPostgres.auth.username | default "postgres" }}"
|
||||
- name: POSTGRES_PASSWORD
|
||||
value: "{{ .Values.nuqPostgres.auth.password | default "password" }}"
|
||||
- name: POSTGRES_DB
|
||||
value: "{{ .Values.nuqPostgres.auth.database | default "postgres" }}"
|
||||
ports:
|
||||
- containerPort: 5432
|
||||
volumeMounts:
|
||||
- name: postgres-storage
|
||||
mountPath: /var/lib/postgresql/data
|
||||
{{- if .Values.resources.enabled }}
|
||||
{{- if .Values.nuqPostgres.resources }}
|
||||
resources:
|
||||
{{- toYaml .Values.nuqPostgres.resources | nindent 12 }}
|
||||
{{- else }}
|
||||
resources:
|
||||
requests:
|
||||
memory: "512Mi"
|
||||
cpu: "250m"
|
||||
limits:
|
||||
memory: "1Gi"
|
||||
cpu: "500m"
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
volumes:
|
||||
- name: postgres-storage
|
||||
{{- if .Values.nuqPostgres.persistence.enabled }}
|
||||
persistentVolumeClaim:
|
||||
claimName: {{ include "firecrawl.fullname" . }}-nuq-postgres-pvc
|
||||
{{- else }}
|
||||
emptyDir: {}
|
||||
{{- end }}
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: {{ include "firecrawl.fullname" . }}-nuq-postgres
|
||||
labels:
|
||||
app: {{ include "firecrawl.name" . }}-nuq-postgres
|
||||
spec:
|
||||
selector:
|
||||
app: {{ include "firecrawl.name" . }}-nuq-postgres
|
||||
ports:
|
||||
- protocol: TCP
|
||||
port: 5432
|
||||
targetPort: 5432
|
||||
type: ClusterIP
|
||||
@@ -0,0 +1,17 @@
|
||||
{{- if .Values.nuqPostgres.persistence.enabled }}
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: {{ include "firecrawl.fullname" . }}-nuq-postgres-pvc
|
||||
labels:
|
||||
app: {{ include "firecrawl.name" . }}-nuq-postgres
|
||||
spec:
|
||||
accessModes:
|
||||
- ReadWriteOnce
|
||||
resources:
|
||||
requests:
|
||||
storage: {{ .Values.nuqPostgres.persistence.size | quote }}
|
||||
{{- if .Values.nuqPostgres.persistence.storageClass }}
|
||||
storageClassName: {{ .Values.nuqPostgres.persistence.storageClass | quote }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
@@ -0,0 +1,69 @@
|
||||
{{- if .Values.nuqPrefetchWorker.enabled }}
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: {{ include "firecrawl.fullname" . }}-nuq-prefetch-worker
|
||||
labels:
|
||||
app: {{ include "firecrawl.name" . }}-nuq-prefetch-worker
|
||||
spec:
|
||||
replicas: {{ .Values.nuqPrefetchWorker.replicaCount | default 1 }}
|
||||
selector:
|
||||
matchLabels:
|
||||
app: {{ include "firecrawl.name" . }}-nuq-prefetch-worker
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: {{ include "firecrawl.name" . }}-nuq-prefetch-worker
|
||||
spec:
|
||||
{{- if .Values.image.dockerSecretEnabled }}
|
||||
imagePullSecrets:
|
||||
{{- toYaml .Values.imagePullSecrets | nindent 8 }}
|
||||
{{- end }}
|
||||
terminationGracePeriodSeconds: 60
|
||||
containers:
|
||||
- name: nuq-prefetch-worker
|
||||
image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
|
||||
imagePullPolicy: {{ .Values.image.pullPolicy }}
|
||||
command: [ "node" ]
|
||||
args: [ "--max-old-space-size=2048", "dist/src/services/worker/nuq-prefetch-worker.js" ]
|
||||
ports:
|
||||
- containerPort: {{ .Values.nuqPrefetchWorker.port | int }}
|
||||
env:
|
||||
- name: FLY_PROCESS_GROUP
|
||||
value: "nuq-prefetch-worker"
|
||||
- name: NUQ_PREFETCH_WORKER_PORT
|
||||
value: {{ .Values.nuqPrefetchWorker.port | quote }}
|
||||
- name: NUQ_PREFETCH_REPLICAS
|
||||
value: {{ .Values.nuqPrefetchWorker.replicaCount | quote }}
|
||||
- name: NUQ_POD_NAME
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
fieldPath: metadata.name
|
||||
envFrom:
|
||||
- configMapRef:
|
||||
name: {{ include "firecrawl.fullname" . }}-config
|
||||
- secretRef:
|
||||
name: {{ include "firecrawl.fullname" . }}-secret
|
||||
{{- if and .Values.resources.enabled .Values.nuqPrefetchWorker.resources }}
|
||||
resources:
|
||||
{{- toYaml .Values.nuqPrefetchWorker.resources | nindent 12 }}
|
||||
{{- end }}
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /health
|
||||
port: {{ .Values.nuqPrefetchWorker.port | int }}
|
||||
initialDelaySeconds: 10
|
||||
periodSeconds: 10
|
||||
timeoutSeconds: 5
|
||||
successThreshold: 1
|
||||
failureThreshold: 3
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /health
|
||||
port: {{ .Values.nuqPrefetchWorker.port | int }}
|
||||
initialDelaySeconds: 10
|
||||
periodSeconds: 10
|
||||
timeoutSeconds: 5
|
||||
successThreshold: 1
|
||||
failureThreshold: 3
|
||||
{{- end }}
|
||||
@@ -0,0 +1,75 @@
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: {{ include "firecrawl.fullname" . }}-nuq-worker
|
||||
labels:
|
||||
app: {{ include "firecrawl.name" . }}-nuq-worker
|
||||
spec:
|
||||
replicas: {{ .Values.nuqWorker.replicaCount | default 5 }}
|
||||
selector:
|
||||
matchLabels:
|
||||
app: {{ include "firecrawl.name" . }}-nuq-worker
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: {{ include "firecrawl.name" . }}-nuq-worker
|
||||
spec:
|
||||
{{- if .Values.image.dockerSecretEnabled }}
|
||||
imagePullSecrets:
|
||||
{{- toYaml .Values.imagePullSecrets | nindent 8 }}
|
||||
{{- end }}
|
||||
terminationGracePeriodSeconds: 60
|
||||
containers:
|
||||
- name: nuq-worker
|
||||
image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
|
||||
imagePullPolicy: {{ .Values.image.pullPolicy }}
|
||||
command: [ "node" ]
|
||||
args: [ "--max-old-space-size=3072", "dist/src/services/worker/nuq-worker.js" ]
|
||||
ports:
|
||||
- containerPort: {{ .Values.nuqWorker.port | int }}
|
||||
env:
|
||||
- name: FLY_PROCESS_GROUP
|
||||
value: "nuq-worker"
|
||||
- name: NUQ_WORKER_PORT
|
||||
value: {{ .Values.nuqWorker.port | quote }}
|
||||
- name: NUQ_POD_NAME
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
fieldPath: metadata.name
|
||||
envFrom:
|
||||
- configMapRef:
|
||||
name: {{ include "firecrawl.fullname" . }}-config
|
||||
- secretRef:
|
||||
name: {{ include "firecrawl.fullname" . }}-secret
|
||||
{{- if .Values.resources.enabled }}
|
||||
{{- if .Values.nuqWorker.resources }}
|
||||
resources:
|
||||
{{- toYaml .Values.nuqWorker.resources | nindent 12 }}
|
||||
{{- else }}
|
||||
resources:
|
||||
requests:
|
||||
memory: "3G"
|
||||
cpu: "1000m"
|
||||
limits:
|
||||
memory: "4G"
|
||||
cpu: "1000m"
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /health
|
||||
port: {{ .Values.nuqWorker.port | int }}
|
||||
initialDelaySeconds: 5
|
||||
periodSeconds: 5
|
||||
timeoutSeconds: 5
|
||||
successThreshold: 1
|
||||
failureThreshold: 3
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /health
|
||||
port: {{ .Values.nuqWorker.port | int }}
|
||||
initialDelaySeconds: 5
|
||||
periodSeconds: 5
|
||||
timeoutSeconds: 5
|
||||
successThreshold: 1
|
||||
failureThreshold: 3
|
||||
@@ -0,0 +1,9 @@
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: {{ include "firecrawl.fullname" . }}-playwright-config
|
||||
data:
|
||||
PORT: {{ default (printf "%v" .Values.service.playwright.port) .Values.playwrightConfig.PORT | quote }}
|
||||
ALLOW_LOCAL_WEBHOOKS: {{ .Values.playwrightConfig.ALLOW_LOCAL_WEBHOOKS | quote }}
|
||||
BLOCK_MEDIA: {{ .Values.playwrightConfig.BLOCK_MEDIA | quote }}
|
||||
MAX_CONCURRENT_PAGES: {{ .Values.playwrightConfig.MAX_CONCURRENT_PAGES | quote }}
|
||||
@@ -0,0 +1,51 @@
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: {{ include "firecrawl.fullname" . }}-playwright
|
||||
labels:
|
||||
app: {{ include "firecrawl.name" . }}-playwright
|
||||
spec:
|
||||
replicas: {{ .Values.playwright.replicaCount | default .Values.replicaCount }}
|
||||
selector:
|
||||
matchLabels:
|
||||
app: {{ include "firecrawl.name" . }}-playwright
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: {{ include "firecrawl.name" . }}-playwright
|
||||
spec:
|
||||
{{- if .Values.image.dockerSecretEnabled }}
|
||||
imagePullSecrets:
|
||||
{{- toYaml .Values.imagePullSecrets | nindent 8 }}
|
||||
{{- end }}
|
||||
containers:
|
||||
- name: playwright-service
|
||||
image: "{{ .Values.playwright.repository }}:{{ .Values.playwright.tag }}"
|
||||
imagePullPolicy: {{ .Values.playwright.pullPolicy }}
|
||||
ports:
|
||||
- containerPort: {{ .Values.service.playwright.port }}
|
||||
envFrom:
|
||||
- configMapRef:
|
||||
name: {{ include "firecrawl.fullname" . }}-config
|
||||
- secretRef:
|
||||
name: {{ include "firecrawl.fullname" . }}-secret
|
||||
- configMapRef:
|
||||
name: {{ include "firecrawl.fullname" . }}-playwright-config
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /health
|
||||
port: {{ .Values.service.playwright.port }}
|
||||
initialDelaySeconds: 30
|
||||
periodSeconds: 30
|
||||
timeoutSeconds: 5
|
||||
successThreshold: 1
|
||||
failureThreshold: 3
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /health
|
||||
port: {{ .Values.service.playwright.port }}
|
||||
initialDelaySeconds: 30
|
||||
periodSeconds: 30
|
||||
timeoutSeconds: 5
|
||||
successThreshold: 1
|
||||
failureThreshold: 3
|
||||
@@ -0,0 +1,12 @@
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: {{ include "firecrawl.fullname" . }}-playwright
|
||||
spec:
|
||||
type: {{ .Values.service.playwright.type }}
|
||||
selector:
|
||||
app: {{ include "firecrawl.name" . }}-playwright
|
||||
ports:
|
||||
- protocol: TCP
|
||||
port: {{ .Values.service.playwright.port }}
|
||||
targetPort: {{ .Values.service.playwright.port }}
|
||||
@@ -0,0 +1,60 @@
|
||||
{{- if .Values.rabbitmq.enabled }}
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: {{ include "firecrawl.fullname" . }}-rabbitmq
|
||||
labels:
|
||||
app: {{ include "firecrawl.name" . }}-rabbitmq
|
||||
spec:
|
||||
replicas: {{ .Values.rabbitmq.replicaCount | default 1 }}
|
||||
selector:
|
||||
matchLabels:
|
||||
app: {{ include "firecrawl.name" . }}-rabbitmq
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: {{ include "firecrawl.name" . }}-rabbitmq
|
||||
spec:
|
||||
containers:
|
||||
- name: rabbitmq
|
||||
image: {{ .Values.rabbitmq.image }}
|
||||
ports:
|
||||
- containerPort: {{ .Values.service.rabbitmq.port }}
|
||||
- containerPort: {{ .Values.service.rabbitmq.managementPort }}
|
||||
{{- if and .Values.resources.enabled .Values.rabbitmq.resources }}
|
||||
resources:
|
||||
{{- toYaml .Values.rabbitmq.resources | nindent 12 }}
|
||||
{{- end }}
|
||||
livenessProbe:
|
||||
tcpSocket:
|
||||
port: {{ .Values.service.rabbitmq.port }}
|
||||
initialDelaySeconds: 20
|
||||
periodSeconds: 10
|
||||
timeoutSeconds: 5
|
||||
failureThreshold: 6
|
||||
readinessProbe:
|
||||
tcpSocket:
|
||||
port: {{ .Values.service.rabbitmq.port }}
|
||||
initialDelaySeconds: 10
|
||||
periodSeconds: 5
|
||||
timeoutSeconds: 3
|
||||
failureThreshold: 6
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: {{ include "firecrawl.fullname" . }}-rabbitmq
|
||||
spec:
|
||||
type: {{ .Values.service.rabbitmq.type }}
|
||||
selector:
|
||||
app: {{ include "firecrawl.name" . }}-rabbitmq
|
||||
ports:
|
||||
- name: amqp
|
||||
protocol: TCP
|
||||
port: {{ .Values.service.rabbitmq.port }}
|
||||
targetPort: {{ .Values.service.rabbitmq.port }}
|
||||
- name: management
|
||||
protocol: TCP
|
||||
port: {{ .Values.service.rabbitmq.managementPort }}
|
||||
targetPort: {{ .Values.service.rabbitmq.managementPort }}
|
||||
{{- end }}
|
||||
@@ -0,0 +1,35 @@
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: {{ include "firecrawl.fullname" . }}-redis
|
||||
labels:
|
||||
app: {{ include "firecrawl.name" . }}-redis
|
||||
spec:
|
||||
replicas: {{ .Values.redis.replicaCount }}
|
||||
selector:
|
||||
matchLabels:
|
||||
app: {{ include "firecrawl.name" . }}-redis
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: {{ include "firecrawl.name" . }}-redis
|
||||
spec:
|
||||
containers:
|
||||
- name: redis
|
||||
image: {{ .Values.redis.image }}
|
||||
command: [ "/bin/sh", "-c" ]
|
||||
args:
|
||||
- |
|
||||
if [ -n "$REDIS_PASSWORD" ]; then
|
||||
echo "Starting Redis with authentication"
|
||||
exec redis-server --bind 0.0.0.0 --requirepass "$REDIS_PASSWORD"
|
||||
else
|
||||
echo "Starting Redis without authentication"
|
||||
exec redis-server --bind 0.0.0.0
|
||||
fi
|
||||
env:
|
||||
- name: REDIS_PASSWORD
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: {{ include "firecrawl.fullname" . }}-secret
|
||||
key: REDIS_PASSWORD
|
||||
@@ -0,0 +1,12 @@
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: {{ include "firecrawl.fullname" . }}-redis
|
||||
spec:
|
||||
type: {{ .Values.service.redis.type }}
|
||||
selector:
|
||||
app: {{ include "firecrawl.name" . }}-redis
|
||||
ports:
|
||||
- protocol: TCP
|
||||
port: {{ .Values.service.redis.port }}
|
||||
targetPort: {{ .Values.service.redis.port }}
|
||||
@@ -0,0 +1,21 @@
|
||||
apiVersion: v1
|
||||
kind: Secret
|
||||
metadata:
|
||||
name: {{ include "firecrawl.fullname" . }}-secret
|
||||
type: Opaque
|
||||
data:
|
||||
OPENAI_API_KEY: {{ .Values.secret.OPENAI_API_KEY | b64enc | quote }}
|
||||
SLACK_WEBHOOK_URL: {{ .Values.secret.SLACK_WEBHOOK_URL | b64enc | quote }}
|
||||
LLAMAPARSE_API_KEY: {{ .Values.secret.LLAMAPARSE_API_KEY | b64enc | quote }}
|
||||
BULL_AUTH_KEY: {{ .Values.secret.BULL_AUTH_KEY | b64enc | quote }}
|
||||
TEST_API_KEY: {{ .Values.secret.TEST_API_KEY | b64enc | quote }}
|
||||
FIRE_ENGINE_BETA_URL: {{ .Values.secret.FIRE_ENGINE_BETA_URL | b64enc | quote }}
|
||||
SUPABASE_ANON_TOKEN: {{ .Values.secret.SUPABASE_ANON_TOKEN | b64enc | quote }}
|
||||
SUPABASE_URL: {{ .Values.secret.SUPABASE_URL | b64enc | quote }}
|
||||
SUPABASE_SERVICE_TOKEN: {{ .Values.secret.SUPABASE_SERVICE_TOKEN | b64enc | quote }}
|
||||
PROXY_PASSWORD: {{ .Values.secret.PROXY_PASSWORD | b64enc | quote }}
|
||||
SELF_HOSTED_WEBHOOK_HMAC_SECRET: {{ .Values.secret.SELF_HOSTED_WEBHOOK_HMAC_SECRET | b64enc | quote }}
|
||||
REDIS_PASSWORD: {{ .Values.secret.REDIS_PASSWORD | b64enc | quote }}
|
||||
{{- range $k, $v := .Values.secret.extra }}
|
||||
{{ $k }}: {{ $v | b64enc | quote }}
|
||||
{{- end }}
|
||||
@@ -0,0 +1,12 @@
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: {{ include "firecrawl.fullname" . }}-api
|
||||
spec:
|
||||
type: {{ .Values.service.api.type }}
|
||||
selector:
|
||||
app: {{ include "firecrawl.name" . }}-api
|
||||
ports:
|
||||
- protocol: TCP
|
||||
port: {{ .Values.service.api.port }}
|
||||
targetPort: {{ .Values.service.api.port }}
|
||||
@@ -0,0 +1,56 @@
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: {{ include "firecrawl.fullname" . }}-worker
|
||||
labels:
|
||||
app: {{ include "firecrawl.name" . }}-worker
|
||||
spec:
|
||||
replicas: {{ .Values.worker.replicaCount | default .Values.replicaCount }}
|
||||
selector:
|
||||
matchLabels:
|
||||
app: {{ include "firecrawl.name" . }}-worker
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: {{ include "firecrawl.name" . }}-worker
|
||||
spec:
|
||||
{{- if .Values.image.dockerSecretEnabled }}
|
||||
imagePullSecrets:
|
||||
{{- toYaml .Values.imagePullSecrets | nindent 8 }}
|
||||
{{- end }}
|
||||
terminationGracePeriodSeconds: 60
|
||||
containers:
|
||||
- name: worker
|
||||
image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
|
||||
imagePullPolicy: {{ .Values.image.pullPolicy }}
|
||||
command: [ "node" ]
|
||||
args: [ "--max-old-space-size=3072", "dist/src/services/queue-worker.js" ]
|
||||
ports:
|
||||
- containerPort: {{ .Values.worker.port | int }}
|
||||
env:
|
||||
- name: FLY_PROCESS_GROUP
|
||||
value: "worker"
|
||||
- name: WORKER_PORT
|
||||
value: {{ .Values.worker.port | quote }}
|
||||
- name: NUQ_POD_NAME
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
fieldPath: metadata.name
|
||||
envFrom:
|
||||
- configMapRef:
|
||||
name: {{ include "firecrawl.fullname" . }}-config
|
||||
- secretRef:
|
||||
name: {{ include "firecrawl.fullname" . }}-secret
|
||||
{{- if and .Values.resources.enabled .Values.worker.resources }}
|
||||
resources:
|
||||
{{- toYaml .Values.worker.resources | nindent 12 }}
|
||||
{{- end }}
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /liveness
|
||||
port: {{ .Values.worker.port | int }}
|
||||
initialDelaySeconds: 10
|
||||
periodSeconds: 10
|
||||
timeoutSeconds: 5
|
||||
successThreshold: 1
|
||||
failureThreshold: 3
|
||||
183
참고/firecrawl-main/examples/kubernetes/firecrawl-helm/values.yaml
Normal file
183
참고/firecrawl-main/examples/kubernetes/firecrawl-helm/values.yaml
Normal file
@@ -0,0 +1,183 @@
|
||||
replicaCount: 1
|
||||
|
||||
resources:
|
||||
enabled: false
|
||||
|
||||
api:
|
||||
replicaCount: 1
|
||||
resources:
|
||||
requests:
|
||||
memory: "4G"
|
||||
cpu: "2000m"
|
||||
limits:
|
||||
memory: "6G"
|
||||
cpu: "2000m"
|
||||
|
||||
worker:
|
||||
replicaCount: 1
|
||||
port: 3005
|
||||
resources:
|
||||
requests:
|
||||
memory: "3G"
|
||||
cpu: "1000m"
|
||||
limits:
|
||||
memory: "4G"
|
||||
cpu: "1000m"
|
||||
|
||||
extractWorker:
|
||||
enabled: true
|
||||
replicaCount: 1
|
||||
port: 3004
|
||||
resources:
|
||||
requests:
|
||||
memory: "2G"
|
||||
cpu: "1000m"
|
||||
limits:
|
||||
memory: "3G"
|
||||
cpu: "1000m"
|
||||
|
||||
nuqWorker:
|
||||
replicaCount: 5
|
||||
port: 3006
|
||||
resources:
|
||||
requests:
|
||||
memory: "3G"
|
||||
cpu: "1000m"
|
||||
limits:
|
||||
memory: "4G"
|
||||
cpu: "1000m"
|
||||
|
||||
nuqPrefetchWorker:
|
||||
enabled: true
|
||||
replicaCount: 1
|
||||
port: 3011
|
||||
resources:
|
||||
requests:
|
||||
memory: "1G"
|
||||
cpu: "500m"
|
||||
limits:
|
||||
memory: "2G"
|
||||
cpu: "1000m"
|
||||
|
||||
nuqPostgres:
|
||||
replicaCount: 1
|
||||
image:
|
||||
repository: docker.io/winkkgmbh/nuq-postgres
|
||||
tag: latest
|
||||
pullPolicy: Always
|
||||
auth:
|
||||
username: postgres
|
||||
password: password
|
||||
database: postgres
|
||||
resources:
|
||||
requests:
|
||||
memory: "512Mi"
|
||||
cpu: "250m"
|
||||
limits:
|
||||
memory: "1Gi"
|
||||
cpu: "500m"
|
||||
persistence:
|
||||
enabled: false
|
||||
size: 10Gi
|
||||
storageClass: ""
|
||||
|
||||
rabbitmq:
|
||||
enabled: true
|
||||
image: rabbitmq:3-management
|
||||
replicaCount: 1
|
||||
resources:
|
||||
requests:
|
||||
memory: "512Mi"
|
||||
cpu: "250m"
|
||||
limits:
|
||||
memory: "1Gi"
|
||||
cpu: "500m"
|
||||
|
||||
image:
|
||||
repository: docker.io/winkkgmbh/firecrawl
|
||||
tag: "latest"
|
||||
pullPolicy: Always
|
||||
dockerSecretEnabled: false
|
||||
|
||||
playwright:
|
||||
repository: docker.io/winkkgmbh/firecrawl-playwright
|
||||
tag: "latest"
|
||||
pullPolicy: Always
|
||||
replicaCount: 1
|
||||
|
||||
redis:
|
||||
image: redis:alpine
|
||||
replicaCount: 1
|
||||
|
||||
service:
|
||||
api:
|
||||
type: ClusterIP
|
||||
port: 3002
|
||||
playwright:
|
||||
type: ClusterIP
|
||||
port: 3000
|
||||
redis:
|
||||
type: ClusterIP
|
||||
port: 6379
|
||||
rabbitmq:
|
||||
type: ClusterIP
|
||||
port: 5672
|
||||
managementPort: 15672
|
||||
|
||||
config:
|
||||
NUM_WORKERS_PER_QUEUE: "8"
|
||||
PORT: "3002"
|
||||
HOST: "0.0.0.0"
|
||||
WORKER_PORT: "3005"
|
||||
EXTRACT_WORKER_PORT: "3004"
|
||||
NUQ_WORKER_PORT: "3006"
|
||||
NUQ_PREFETCH_WORKER_PORT: "3011"
|
||||
NUQ_WORKER_COUNT: "5"
|
||||
USE_DB_AUTHENTICATION: "false"
|
||||
IS_KUBERNETES: "true"
|
||||
ENV: "production"
|
||||
LOGGING_LEVEL: "INFO"
|
||||
FIRECRAWL_APP_SCHEME: "http"
|
||||
FIRECRAWL_APP_HOST: ""
|
||||
FIRECRAWL_APP_PORT: ""
|
||||
REDIS_URL: ""
|
||||
REDIS_RATE_LIMIT_URL: ""
|
||||
PLAYWRIGHT_MICROSERVICE_URL: ""
|
||||
NUQ_DATABASE_URL: ""
|
||||
NUQ_DATABASE_URL_LISTEN: ""
|
||||
NUQ_RABBITMQ_URL: ""
|
||||
OPENAI_BASE_URL: ""
|
||||
MODEL_NAME: ""
|
||||
MODEL_EMBEDDING_NAME: ""
|
||||
OLLAMA_BASE_URL: ""
|
||||
PROXY_SERVER: ""
|
||||
PROXY_USERNAME: ""
|
||||
SEARXNG_ENDPOINT: ""
|
||||
SEARXNG_ENGINES: ""
|
||||
SEARXNG_CATEGORIES: ""
|
||||
SELF_HOSTED_WEBHOOK_URL: ""
|
||||
extra: {}
|
||||
|
||||
playwrightConfig:
|
||||
PORT: "3000"
|
||||
ALLOW_LOCAL_WEBHOOKS: ""
|
||||
BLOCK_MEDIA: ""
|
||||
MAX_CONCURRENT_PAGES: "10"
|
||||
|
||||
secret:
|
||||
OPENAI_API_KEY: ""
|
||||
SLACK_WEBHOOK_URL: ""
|
||||
LLAMAPARSE_API_KEY: ""
|
||||
BULL_AUTH_KEY: ""
|
||||
TEST_API_KEY: ""
|
||||
FIRE_ENGINE_BETA_URL: ""
|
||||
SUPABASE_ANON_TOKEN: ""
|
||||
SUPABASE_URL: ""
|
||||
SUPABASE_SERVICE_TOKEN: ""
|
||||
PROXY_PASSWORD: ""
|
||||
SELF_HOSTED_WEBHOOK_HMAC_SECRET: ""
|
||||
REDIS_PASSWORD: ""
|
||||
extra: {}
|
||||
|
||||
imagePullSecrets:
|
||||
- name: docker-registry-secret
|
||||
Reference in New Issue
Block a user