""" Unit tests for the metadata parts. """ import logging import sys from lxml import html from lxml.etree import XPath from trafilatura.json_metadata import normalize_authors, normalize_json from trafilatura.metadata import check_authors, extract_metadata, extract_metainfo, extract_url, normalize_tags logging.basicConfig(stream=sys.stdout, level=logging.DEBUG) def test_titles(): '''Test the extraction of titles''' tests = [ ('

T

', None), ('Test Title

First

', 'First'), ('Test Title

First

', 'First'), ('Test Title', 'Test Title'), ('

First

Second

', 'First'), ('

Test Title
', 'Test Title'), ('

Main menu

Test Title

', 'Test Title'), ('

First

Second

', 'Second'), ('

First

Second

', 'First'), ('', None) ] for doc, expected_title in tests: metadata = extract_metadata(doc) assert metadata.title == expected_title metadata = extract_metadata(r''' ''') assert metadata.title == 'Semantic satiation' metadata = extract_metadata(' - Home') assert metadata.title == '- Home' metadata = extract_metadata('My Title » My Website') assert metadata.title == "My Title" # TODO: and metadata.sitename == "My Website" def test_authors(): '''Test the extraction of author names''' # normalization assert normalize_authors(None, 'abc') == 'Abc' assert normalize_authors(None, 'Steve Steve 123') == 'Steve Steve' assert normalize_authors(None, 'By Steve Steve') == 'Steve Steve' assert normalize_json('Test \\nthis') == 'Test this' assert normalize_json('\\uD800\\uDC00Hello\\uDBFF\\uDFFF') == 'Hello' assert normalize_json('Test \u3010ABC\u3011') == 'Test 【ABC】' assert normalize_json("Seán Federico O'Murchú") == "Seán Federico O'Murchú" # blacklist metadata = extract_metadata('', author_blacklist={'Jenny Smith'}) assert metadata.author is None # extraction begin, end = '', '' htmldocs = [ f'{begin}{end}', f'{begin}{end}', f'{begin}{end}', f'{begin}{end}', f'{begin}{end}', f'{begin}{end}', f'{begin}{end}', f'{begin}{end}', f'{begin}{end}', f'{begin}{end}', f'{begin}{end}', f'{begin}{end}', ] expected_authors = [ 'Jenny Smith', 'Jenny Smith; John Smith', 'Jenny Smith; John Smith', 'Jenny Smith; John Smith', 'Jenny Smith; John Smith', 'Jenny Smith', 'Hank O\'Hop', 'Jenny Smith', 'Jenny Smith; John Smith', 'Jenny Smith; John Smith', 'Jenny Smith; John Smith', 'Jenny Smith', ] for doc, expected_author in zip(htmldocs, expected_authors): metadata = extract_metadata(doc) assert metadata.author == expected_author begin, end = '', '' htmldocs = [ f'{begin}{end}', f'{begin}{end}', f'{begin}Jenny Smithf{end}', f'{begin}

Jenny Smith

f{end}', f'{begin}

Jenny Smith — Trafilatura

f{end}', f'{begin}Jenny Smithf{end}', f'{begin}Jenny Smithf{end}', f'{begin}
Jenny Smith
f{end}', f'{begin}
Jenny Smith
f{end}', f'{begin}

My quote here

— Jenny Smith

f{end}', f'{begin}Jenny Smith and John Smithf{end}', f'{begin}Jenny Smithf{end}', f'{begin}Jenny Smith
Editor
f{end}', f'{begin}Jenny Smith from Trafilaturaf{end}', f'{begin}Jenny Smith from Trafilaturaf{end}', f'{begin}Jenny Smithf{end}', f'{begin}
Jenny Smith
f{end}', f'{begin}
Jenny Smith
f{end}', f'{begin}f{end}', f'{begin}f{end}', f'{begin}
Jenny Smith
f{end}', f'{begin}Jenny Smithf{end}', f'{begin}Jenny Smith – The Moonf{end}', f'{begin}Jenny_Smithf{end}', ] expected_authors = [ 'Jenny Smith', 'Jenny Smith', 'Jenny Smith', 'Jenny Smith', 'Jenny Smith', 'Jenny Smith', 'Jenny Smith', None, None, None, 'Jenny Smith; John Smith', 'Jenny Smith', 'Jenny Smith', 'Jenny Smith', 'Jenny Smith', 'Jenny Smith', 'Jenny Smith', 'Jenny Smith', 'Jenny Smith', 'Jenny Smith', 'Jenny Smith', 'Jenny Smith', 'Jenny Smith', 'Jenny Smith', ] for doc, expected_author in zip(htmldocs, expected_authors): metadata = extract_metadata(doc) assert metadata.author == expected_author htmldocs = [ f'{begin}Shannon Deery, Mitch Clarke, Susie O’Brien, Laura Placella, Kara Irving, Jordy Atkinson, Suzan Delibasicf{end}', f'{begin}
Jenny Smith
f{end}', f'{begin}Jenny Smithf{end}', 'f{end}', f'{begin}
Jenny Smith 11:57AM
f{end}', f'{begin}

By Jenny Smith For Daily Mail Australia

f{end}', f'{begin}
By:Ian Shive
f{end}', f'{begin}
By Ben Coxworth
f{end}', f'{begin}
AUTHOR NAME
f{end}' ] expected_authors = [ 'Shannon Deery; Mitch Clarke; Susie O’Brien; Laura Placella; Kara Irving; Jordy Atkinson; Suzan Delibasic', 'Jenny Smith', 'Jenny Smith', "Amir Vera; Seán Federico O'Murchú; Tara Subramaniam; Adam Renton; CNN", 'Jenny Smith', 'Jenny Smith', 'Ian Shive', 'Ben Coxworth', 'AUTHOR NAME' ] for doc, expected_author in zip(htmldocs, expected_authors): metadata = extract_metadata(doc) assert metadata.author == expected_author # check authors string blacklist = {"A", "b"} assert check_authors("a; B; c; d", blacklist) == "c; d" assert check_authors("a;B;c;d", blacklist) == "c; d" def test_url(): '''Test URL extraction''' htmldocs = [ '', '', '', '', '' '', ] default_urls = [None, None, None, None, "https://example.org", None] expected_url = 'https://example.org' for doc, default_url in zip(htmldocs, default_urls): metadata = extract_metadata(doc, default_url) assert metadata.url == expected_url # test on partial URLs url = extract_url(html.fromstring('')) assert url == 'https://example.org/article/medical-record' def test_description(): '''Test the extraction of descriptions''' metadata = extract_metadata('') assert metadata.description == 'Description' metadata = extract_metadata('') assert metadata.description == 'A Northern Territory action plan, which includes plans to support development and employment on Aboriginal land, has received an update. ...' def test_dates(): '''Simple tests for date extraction (most of the tests are carried out externally for htmldate module)''' tests = [ ('', '2017-09-01', False), ('', '2017-09-01', False), ('', '2017-09-01', False), ('

Veröffentlicht am 1.9.17

', '2017-09-01', True), ('

Veröffentlicht am 1.9.17

', '2017-09-01', False), ] for doc, expected, extensive in tests: metadata = extract_metadata(doc, extensive=extensive) assert metadata.date == expected def test_sitename(): '''Test extraction of site name''' tests = [ ('', None), ('', 'The Newspaper'), ('', 'The Newspaper'), ('sitemaps.org - Home', 'sitemaps.org'), ] for doc, expected in tests: metadata = extract_metadata(doc) assert metadata.sitename == expected def test_meta(): '''Test extraction out of meta-elements''' doc = html.fromstring("

a

b

cde

") assert extract_metainfo(doc, [XPath(".//p[@class]")]) is None assert extract_metainfo(doc, [XPath(".//p[@type]")]) == "cde" metadata = extract_metadata('Creative Commons') assert metadata.pagetype == 'Open Graph Type' assert metadata.title == 'Open Graph Title' assert metadata.author == 'Jenny Smith' assert metadata.description == 'This is an Open Graph description' assert metadata.sitename == 'My first site' assert metadata.url == 'https://example.org/test' assert metadata.license == 'Creative Commons' metadata = extract_metadata('') assert metadata.title == 'Open Graph Title' assert metadata.author == 'Jenny Smith' assert metadata.description == 'This is an Open Graph description' metadata = extract_metadata('') assert metadata.title == 'Title' # catch errors metadata = extract_metadata('') target_slots = set(metadata.__slots__) - {"body", "commentsbody"} assert all(getattr(metadata, a) is None for a in target_slots) metadata = extract_metadata('') assert metadata.sitename is None metadata = extract_metadata('' + 'AAA'*10000 + '') assert metadata.title.endswith('…') and len(metadata.title) == 10000 assert extract_metadata('') is not None assert extract_metadata('') is not None def test_catstags(): '''Test extraction of categories and tags''' assert normalize_tags(" ") == "" assert normalize_tags(" 1 & 2 ") == "1 & 2" htmldocs = [ '

Cat1, Cat2

', '
Cat1
', '

Tag1, Tag2

', '', ] attrs = ['categories', 'categories', 'tags', 'tags'] expected = [ ['Cat1', 'Cat2'], ['Cat1'], ['Tag1', 'Tag2'], ['sodium, salt, paracetamol, blood, pressure, high, heart, intake, warning, study, medicine, dissolvable, cardiovascular'], ] for doc, attr, expected_value in zip(htmldocs, attrs, expected): metadata = extract_metadata(doc) assert getattr(metadata, attr) == expected_value def test_license(): '''Test extraction of CC licenses''' # a rel metadata = extract_metadata('

CC BY-SA

') assert metadata.license == 'CC BY-SA 4.0' metadata = extract_metadata('

Unknown

') assert metadata.license == 'Unknown' # footer metadata = extract_metadata('') assert metadata.license == 'CC BY-SA 4.0' # footer: netzpolitik.org metadata = extract_metadata('''

Lizenz: Die von uns verfassten Inhalte stehen, soweit nicht anders vermerkt, unter der Lizenz Creative Commons BY-NC-SA 4.0.

''') assert metadata.license == 'CC BY-NC-SA 4.0' # this is not a license metadata = extract_metadata('''''') assert metadata.license is None # this is a license metadata = extract_metadata('''''') assert metadata.license == 'CC BY-NC' def test_images(): '''Image extraction from meta SEO tags''' htmldocs = [ '', '', '', '', '', '', ] expected_images = [ 'https://example.org/example.jpg', 'example.jpg', 'https://example.org/example-opengraph.jpg', 'https://example.org/example-twitter.jpg', 'example-twitter.jpg', None, ] for doc, expected_image in zip(htmldocs, expected_images): metadata = extract_metadata(doc) assert metadata.image == expected_image def test_document_as_dict(): """Tests that the dict serialization works and preserves data.""" htmldoc = """ Test Title

Cat1, Cat2

CC BY-SA

""" document = extract_metadata(htmldoc) dict_ = document.as_dict() assert dict_["title"] == "Test Title" assert dict_["author"] == "Jenny Smith" assert dict_["url"] == "https://example.org" assert dict_["description"] == "Description" assert dict_["sitename"] == "The Newspaper" assert dict_["date"] == "2017-09-01" assert dict_["categories"] == ["Cat1", "Cat2"] assert dict_["license"] == "CC BY-SA 4.0" assert dict_["image"] == "https://example.org/example.jpg" if __name__ == '__main__': test_titles() test_authors() test_dates() test_meta() test_url() test_description() test_catstags() test_sitename() test_license() test_images() test_document_as_dict()