""" Unit tests for JSON metadata extraction. """ import logging import sys from lxml import html from trafilatura.metadata import Document, extract_meta_json, extract_metadata, normalize_authors logging.basicConfig(stream=sys.stdout, level=logging.DEBUG) def test_json_extraction(): ## JSON extraction metadata = extract_metadata('''''') assert metadata.author == 'Maggie Haberman; Shane Goldmacher; Michael Crowley' assert metadata.pagetype == 'newsarticle' metadata = extract_metadata('''''') assert metadata.sitename == 'Safety Insurance Group, Inc.' metadata = extract_metadata('''Jenny Smith''') assert metadata.author == 'Jenny Smith' assert metadata.pagetype == 'newsarticle' metadata = extract_metadata( '''''') assert metadata.author == "Amir Vera; Seán Federico O'Murchú; Tara Subramaniam; Adam Renton; CNN" assert metadata.pagetype == 'newsarticle' metadata = extract_metadata( '''''') assert metadata.author == "Deborah O'Donoghue" assert metadata.pagetype == 'website' metadata = extract_metadata( '''''') assert metadata.author == "Sam McPhee; Tara Cosoleto" assert metadata.pagetype == 'newsarticle' metadata = Document() metadata = extract_meta_json(html.fromstring(''''''), metadata) assert metadata.author == "Jean Sévillia" assert metadata.pagetype == 'webpage' metadata = Document() metadata = extract_meta_json(html.fromstring( ''''''), metadata) assert metadata.author == "John Doe" assert metadata.pagetype == 'article' assert metadata.title == 'Example Article' metadata = Document() metadata = extract_meta_json(html.fromstring( ''''''), metadata) assert metadata.author == "John Doe" assert metadata.pagetype == 'blogposting' assert metadata.title == 'Breaking News: Example Article' metadata = Document() metadata = extract_meta_json(html.fromstring( ''''''), metadata) assert metadata.sitename == 'Example Webpage' ### Test for potential errors metadata = Document() metadata = extract_meta_json(html.fromstring(''' '''), metadata) assert metadata is not None and metadata.title == 'Apple Spring Forward Event Live Blog' assert metadata is not None and metadata.pagetype == 'liveblogposting' ### Test for potential errors metadata = Document() metadata = extract_meta_json(html.fromstring(''' '''), metadata) assert metadata is not None and metadata.title == 'Apple Spring Forward Event Live Blog' assert metadata is not None and metadata.pagetype == 'liveblogposting' ### Test for potential errors - Missing content on live blog metadata = Document() metadata = extract_meta_json(html.fromstring(''' '''), metadata) assert metadata is not None and metadata.title == 'Apple Spring Forward Event Live Blog' assert metadata is not None and metadata.pagetype == 'liveblogposting' metadata = Document() metadata = extract_meta_json(html.fromstring(''' '''), metadata) assert metadata is not None and metadata.author == 'Douglas Noel Adams' assert metadata is not None and metadata.pagetype == 'socialmediaposting' metadata = Document() metadata = extract_meta_json(html.fromstring(''' '''), metadata) assert metadata is not None and len(metadata.categories) == 0 assert metadata is not None and metadata.pagetype == 'article' metadata = Document() metadata = extract_meta_json(html.fromstring(''' '''), metadata) assert metadata is not None and metadata.title == "Mickelson comments hurt new league: Norman" and metadata.sitename == "7NEWS" and metadata.author == "Digital Staff" and "Golf" in metadata.categories and metadata.pagetype == 'newsarticle' metadata = Document() metadata = extract_meta_json(html.fromstring(''' '''), metadata) assert metadata is not None and metadata.title == "Australians stuck in Shanghai's COVID lockdown beg consular officials to help them flee" and metadata.author == "Bill Birtles" and metadata.sitename == "ABC News" and metadata.pagetype == 'newsarticle' metadata = Document() metadata = extract_meta_json(html.fromstring(''' '''), metadata) assert metadata is not None and metadata.title == "New York City Enters Higher Coronavirus Risk Level as Case Numbers Rise" and metadata.author == "Sharon Otterman; Emma G Fitzsimmons" and metadata.sitename == "The New York Times" and metadata.pagetype == 'newsarticle' metadata = Document() metadata = extract_meta_json(html.fromstring(''' '''), metadata) assert metadata is not None and metadata.title == "Decreto permite que consumidor cancele serviços de empresas via WhatsApp" and metadata.author == "Caio Mello" and metadata.sitename == "UOL" metadata = Document() metadata = extract_meta_json(html.fromstring(''' '''), metadata) assert metadata is not None and metadata.title == "12 words and phrases you need to survive in Hamburg" and metadata.author == "Alexander Johnstone" and metadata.sitename == "The Local" and metadata.pagetype == 'newsarticle' metadata = Document() metadata = extract_meta_json(html.fromstring(''' '''), metadata) assert metadata is not None and metadata.author is None and metadata.sitename == "Andreessen Horowitz" and metadata.pagetype == 'website' metadata = Document() metadata = extract_meta_json(html.fromstring(''' '''), metadata) assert metadata is not None and metadata.author is None and metadata.sitename is None and metadata.pagetype is None metadata = Document() metadata = extract_meta_json(html.fromstring(''' '''), metadata) assert metadata is not None and metadata.author is None and metadata.sitename is None and metadata.pagetype == 'liveblogposting' metadata = Document() metadata = extract_meta_json(html.fromstring(''' '''), metadata) assert metadata is not None and metadata.title == 'Apple Spring Forward Event Live Blog' and metadata.pagetype == 'liveblogposting' metadata = Document() metadata = extract_meta_json(html.fromstring(r''' '''), metadata) assert metadata is not None and metadata.title == "EastEnders' June Brown leaves soap 'for good'" and metadata.sitename == "BBC News" and metadata.pagetype == 'reportagenewsarticle' metadata = Document() metadata.sitename = "https://bbcnews.com" metadata = extract_meta_json(html.fromstring(r''' '''), metadata) assert metadata is not None and metadata.sitename == "BBC News" and metadata.pagetype == 'reportagenewsarticle' metadata = Document() metadata = extract_meta_json(html.fromstring(''' '''), metadata) assert metadata is not None and metadata.author == "John Doe" and metadata.title == "How to Tie a Reef Knot" and metadata.pagetype == 'article' metadata = Document() metadata = extract_meta_json(html.fromstring(''' '''), metadata) assert metadata is not None and metadata.author == "Bill Birtles; John Smith" and metadata.pagetype == 'newsarticle' metadata = Document() metadata = extract_meta_json(html.fromstring(''' '''), metadata) assert metadata is not None and metadata.title is None and metadata.sitename is None and metadata.pagetype is None metadata = Document() metadata = extract_meta_json(html.fromstring(''' '''), metadata) assert metadata is not None and metadata.title == "Find perfection in these places where land meets water." and metadata.sitename == "National Geographic" and metadata.author == "Kimberley Lovato" and metadata.pagetype == 'article' # tests that "@type": [] in the JSON doesn't cause an exception metadata = Document() metadata = extract_meta_json(html.fromstring( r""" """), metadata) assert metadata is not None # tests if the JSON-LD with html entities is parsed correctly metadata = Document() metadata = extract_meta_json(html.fromstring( """ """), metadata) assert metadata.pagetype == 'website' def test_normalize_authors(): "Test if author names are normalized correctly." assert normalize_authors(None, "John Doe") == "John Doe" assert normalize_authors("Alice; Bob", "John Doe") == "Alice; Bob; John Doe" assert normalize_authors("Alice; Bob", "john.doe@example.com") == "Alice; Bob" assert normalize_authors(None, "\\u00e9tienne") == "Étienne" assert normalize_authors(None, "étienne") == "Étienne" assert normalize_authors(None, "Alice & Bob") == "Alice; Bob" assert normalize_authors(None, "John Doe") == "John Doe" assert normalize_authors(None, "John 😊 Doe") == "John Doe" assert normalize_authors(None, "words by John Doe") == "John Doe" assert normalize_authors(None, "John Doe123") == "John Doe" assert normalize_authors(None, "John_Doe") == "John Doe" assert normalize_authors(None, "John Doe* ") == "John Doe" assert normalize_authors(None, "John Doe of John Doe") == "John Doe" assert normalize_authors(None, "John Doe — John Doe") == "John Doe" assert normalize_authors(None, 'John "The King" Doe') == "John Doe" if __name__ == '__main__': test_json_extraction() test_normalize_authors()