51 lines
831 B
INI
Vendored
51 lines
831 B
INI
Vendored
# Defines settings for trafilatura (https://github.com/adbar/trafilatura)
|
|
|
|
[DEFAULT]
|
|
|
|
# Download
|
|
DOWNLOAD_TIMEOUT = 10
|
|
MAX_FILE_SIZE = 20000000
|
|
MIN_FILE_SIZE = 10
|
|
|
|
# sleep between requests
|
|
SLEEP_TIME = 0.25
|
|
|
|
# one line per user-agent
|
|
USER_AGENTS =
|
|
Firefox
|
|
Chrome
|
|
|
|
# cookie for HTTP requests
|
|
COOKIE = yummy_cookie=choco; tasty_cookie=strawberry
|
|
|
|
# maximum number of redirects that we will follow
|
|
MAX_REDIRECTS = 2
|
|
|
|
|
|
# Extraction
|
|
MIN_EXTRACTED_SIZE = 0
|
|
MIN_EXTRACTED_COMM_SIZE = 0
|
|
MIN_OUTPUT_SIZE = 0
|
|
MIN_OUTPUT_COMM_SIZE = 0
|
|
|
|
|
|
# discard documents with too many elements
|
|
MAX_TREE_SIZE = 100
|
|
|
|
|
|
# Set to 0 to disable signal
|
|
EXTRACTION_TIMEOUT = 0
|
|
|
|
|
|
# Deduplication
|
|
MIN_DUPLCHECK_SIZE = 10
|
|
MAX_REPETITIONS = 3
|
|
|
|
|
|
# Extraction option for Htmldate
|
|
EXTENSIVE_DATE_SEARCH = off
|
|
|
|
|
|
# URLs in feeds and sitemaps
|
|
EXTERNAL_URLS = off
|