48 lines
783 B
INI
48 lines
783 B
INI
|
# Defines settings for trafilatura (https://github.com/adbar/trafilatura)
|
||
|
|
|
||
|
|
[DEFAULT]
|
||
|
|
|
||
|
|
# Download
|
||
|
|
DOWNLOAD_TIMEOUT = 10
|
||
|
|
MAX_FILE_SIZE = 20000000
|
||
|
|
MIN_FILE_SIZE = 10
|
||
|
|
|
||
|
|
# sleep between requests
|
||
|
|
SLEEP_TIME = 0.25
|
||
|
|
|
||
|
|
# one line per user-agent
|
||
|
|
USER_AGENTS =
|
||
|
|
Firefox
|
||
|
|
Chrome
|
||
|
|
|
||
|
|
# cookie for HTTP requests
|
||
|
|
|
||
|
|
COOKIE = yummy_cookie=choco; tasty_cookie=strawberry
|
||
|
|
|
||
|
|
# maximum number of redirects that we will follow
|
||
|
|
MAX_REDIRECTS = 2
|
||
|
|
|
||
|
|
|
||
|
|
# Extraction
|
||
|
|
MIN_EXTRACTED_SIZE = 10000
|
||
|
|
MIN_EXTRACTED_COMM_SIZE = 10000
|
||
|
|
MIN_OUTPUT_SIZE = 10000
|
||
|
|
MIN_OUTPUT_COMM_SIZE = 10000
|
||
|
|
|
||
|
|
|
||
|
|
# Set to 0 to disable signal
|
||
|
|
EXTRACTION_TIMEOUT = 0
|
||
|
|
|
||
|
|
|
||
|
|
# Deduplication
|
||
|
|
MIN_DUPLCHECK_SIZE = 10
|
||
|
|
MAX_REPETITIONS = 3
|
||
|
|
|
||
|
|
|
||
|
|
# Extraction option for Htmldate
|
||
|
|
EXTENSIVE_DATE_SEARCH = off
|
||
|
|
|
||
|
|
|
||
|
|
# URLs in feeds and sitemaps
|
||
|
|
EXTERNAL_URLS = off
|