"""
Unit tests for feeds reading and parsing.
"""
import logging
import os
import sys
from unittest.mock import patch
from courlan import get_hostinfo
from trafilatura.cli import main
from trafilatura.feeds import (
FeedParameters,
determine_feed,
extract_links,
find_links,
find_feed_urls,
handle_link_list,
probe_gnews,
)
logging.basicConfig(stream=sys.stdout, level=logging.DEBUG)
TEST_DIR = os.path.abspath(os.path.dirname(__file__))
RESOURCES_DIR = os.path.join(TEST_DIR, "resources")
XMLDECL = '\n'
def test_atom_extraction():
"""Test link extraction from an Atom feed"""
params = FeedParameters("https://example.org", "example.org", "")
assert not extract_links(None, params)
assert len(extract_links("", params)) == 0
filepath = os.path.join(RESOURCES_DIR, "feed1.atom")
with open(filepath, "r", encoding="utf-8") as f:
teststring = f.read()
assert len(extract_links(teststring, params)) > 0
params = FeedParameters("https://www.dwds.de", "dwds.de", "")
assert (
len(
extract_links(
f'{XMLDECL}',
params,
)
)
== 0
)
params = FeedParameters("http://example.org", "example.org", "http://example.org")
assert (
len(
extract_links(
f'{XMLDECL}',
params,
)
)
== 0
)
params = FeedParameters("https://example.org", "example.org", "")
assert (
len(
extract_links(
f'{XMLDECL}',
params,
)
)
== 0
)
params = FeedParameters("http://example.org/", "example.org", "http://example.org")
assert extract_links(
f'{XMLDECL}', params
) == [
"http://example.org/article1/"
] # TODO: remove slash?
def test_rss_extraction():
"""Test link extraction from a RSS feed"""
params = FeedParameters("http://example.org/", "example.org", "")
assert (
len(
extract_links(f"{XMLDECL}http://example.org/article1/", params)
)
== 1
)
# CDATA
assert extract_links(
f"{XMLDECL}", params
) == [
"http://example.org/article1/"
] # TODO: remove slash?
# spaces
params = FeedParameters("https://www.ak-kurier.de/", "ak-kurier.de", "")
assert (
len(
extract_links(
XMLDECL
+ "\r\n https://www.ak-kurier.de/akkurier/www/artikel/108815-sinfonisches-blasorchester-spielt-1500-euro-fuer-kinder-in-drk-krankenhaus-kirchen-ein ",
params,
)
)
== 1
)
params = FeedParameters("http://example.org", "example.org", "http://example.org")
assert len(extract_links(f"{XMLDECL}http://example.org/", params)) == 0
params = FeedParameters("http://example.org", "example.org", "")
assert len(extract_links(f"{XMLDECL}https://example.org", params)) == 0
params = FeedParameters("https://www.dwds.de", "dwds.de", "https://www.dwds.de")
assert extract_links(
f"{XMLDECL}/api/feed/themenglossar/Corona", params
) == ["https://www.dwds.de/api/feed/themenglossar/Corona"]
params = FeedParameters("https://example.org", "example.org", "")
filepath = os.path.join(RESOURCES_DIR, "feed2.rss")
with open(filepath, "r", encoding="utf-8") as f:
teststring = f.read()
assert len(extract_links(teststring, params)) > 0
def test_json_extraction():
"""Test link extraction from a JSON feed"""
# find link
params = FeedParameters("https://www.jsonfeed.org", "jsonfeed.org", "")
assert (
len(
determine_feed(
'>',
params,
)
)
== 1
)
# extract data
assert not extract_links("{/}", params)
filepath = os.path.join(RESOURCES_DIR, "feed.json")
with open(filepath, "r", encoding="utf-8") as f:
teststring = f.read()
params = FeedParameters("https://npr.org", "npr.org", "")
links = extract_links(teststring, params)
assert len(links) == 25
# id as a backup
params = FeedParameters("https://example.org", "example.org", "")
links = extract_links(
r'{"version":"https:\/\/jsonfeed.org\/version\/1","items":[{"id":"https://www.example.org/1","title":"Test"}]}',
params,
)
assert len(links) == 1
def test_feeds_helpers():
"""Test helper functions for feed extraction"""
params = FeedParameters("https://example.org", "example.org", "https://example.org")
domainname, baseurl = get_hostinfo("https://example.org")
assert domainname == params.domain and baseurl == params.base
# empty page
assert find_links("