51 lines
830 B
INI
51 lines
830 B
INI
|
# Defines settings for trafilatura (https://trafilatura.readthedocs.io/en/latest/settings.html)
|
||
|
|
|
||
|
|
[DEFAULT]
|
||
|
|
|
||
|
|
# Download
|
||
|
|
DOWNLOAD_TIMEOUT = 30
|
||
|
|
MAX_FILE_SIZE = 20000000
|
||
|
|
MIN_FILE_SIZE = 10
|
||
|
|
|
||
|
|
# sleep between requests
|
||
|
|
SLEEP_TIME = 5.0
|
||
|
|
|
||
|
|
# one line per user-agent
|
||
|
|
USER_AGENTS =
|
||
|
|
# "agent1"
|
||
|
|
# "agent2"
|
||
|
|
|
||
|
|
# cookie for HTTP requests
|
||
|
|
COOKIE =
|
||
|
|
|
||
|
|
# maximum number of redirects that we will follow
|
||
|
|
MAX_REDIRECTS = 2
|
||
|
|
|
||
|
|
|
||
|
|
# Extraction
|
||
|
|
MIN_EXTRACTED_SIZE = 250
|
||
|
|
MIN_EXTRACTED_COMM_SIZE = 1
|
||
|
|
MIN_OUTPUT_SIZE = 1
|
||
|
|
MIN_OUTPUT_COMM_SIZE = 1
|
||
|
|
|
||
|
|
|
||
|
|
# discard documents with too many elements
|
||
|
|
MAX_TREE_SIZE =
|
||
|
|
|
||
|
|
|
||
|
|
# CLI file processing only, set to 0 to disable
|
||
|
|
EXTRACTION_TIMEOUT = 30
|
||
|
|
|
||
|
|
|
||
|
|
# Deduplication
|
||
|
|
MIN_DUPLCHECK_SIZE = 100
|
||
|
|
MAX_REPETITIONS = 2
|
||
|
|
|
||
|
|
|
||
|
|
# Extraction option for Htmldate
|
||
|
|
EXTENSIVE_DATE_SEARCH = on
|
||
|
|
|
||
|
|
|
||
|
|
# URLs in feeds and sitemaps
|
||
|
|
EXTERNAL_URLS = off
|