# pylint:disable-msg=I1101,W1401 """ Unit tests for the trafilatura library. """ import logging import sys import time from copy import copy from os import path import pytest from lxml import etree, html try: from cchardet import detect except ImportError: from charset_normalizer import detect import trafilatura.htmlprocessing from trafilatura import bare_extraction, extract, extract_with_metadata, xml from trafilatura.core import Extractor from trafilatura.external import sanitize_tree, try_justext, try_readability from trafilatura.main_extractor import (handle_formatting, handle_image, handle_lists, handle_paragraphs, handle_quotes, handle_table, handle_textelem) from trafilatura.meta import reset_caches from trafilatura.metadata import Document from trafilatura.readability_lxml import is_probably_readerable from trafilatura.settings import DEFAULT_CONFIG, TAG_CATALOG, use_config from trafilatura.utils import (LANGID_FLAG, detect_encoding, is_dubious_html, is_image_file, language_classifier, load_html, normalize_unicode, repair_faulty_html, sanitize, textfilter, trim) logging.basicConfig(stream=sys.stdout, level=logging.DEBUG) TEST_DIR = path.abspath(path.dirname(__file__)) RESOURCES_DIR = path.join(TEST_DIR, 'resources') SAMPLE_META = Document() ZERO_CONFIG = DEFAULT_CONFIG ZERO_CONFIG['DEFAULT']['MIN_OUTPUT_SIZE'] = '0' ZERO_CONFIG['DEFAULT']['MIN_EXTRACTED_SIZE'] = '0' NEW_CONFIG = use_config(filename=path.join(RESOURCES_DIR, 'newsettings.cfg')) MOCK_PAGES = { 'http://exotic_tags': 'exotic_tags.html', } DEFAULT_OPTIONS = Extractor() def load_mock_page(url, xml_flag=False, langcheck=None, tei_output=False): '''load mock page from samples''' try: with open(path.join(TEST_DIR, "resources", MOCK_PAGES[url]), "r", encoding="utf-8") as inputf: htmlstring = inputf.read() # encoding/windows fix for the tests except UnicodeDecodeError: # read as binary with open(path.join(TEST_DIR, "resources", MOCK_PAGES[url]), "rb") as inputf: htmlbinary = inputf.read() guessed_encoding = detect(htmlbinary)['encoding'] if guessed_encoding is not None: try: htmlstring = htmlbinary.decode(guessed_encoding) except UnicodeDecodeError: htmlstring = htmlbinary else: print('Encoding error') if xml_flag: output_format = 'xml' elif tei_output: output_format = 'xmltei' else: output_format = 'txt' return extract( htmlstring, url, record_id='0000', output_format=output_format, target_language=langcheck ) def test_trim(): '''test string trimming''' assert trim(' Test ') == 'Test' assert trim('\t\tTest Test\r\n') == 'Test Test' my_elem = etree.Element('body') my_elem.text = 'Test Text' assert textfilter(my_elem) is False # my_elem.text = 'Tags: Arbeit, Urlaub' my_elem.text = 'Instagram' assert textfilter(my_elem) is True my_elem.text = '\t\t' assert textfilter(my_elem) is True # sanitize logic assert sanitize(None) is None # non-breaking spaces print(sanitize('Test Text')) assert sanitize('Test Text') == 'Test Text' # clear cache # reset caches: examine_date_elements used above old_values = trim.cache_info() reset_caches() assert trim.cache_info() != old_values def test_input(): '''test if loaded strings/trees are handled properly''' teststring = "高山云雾出好茶".encode("utf-8") assert detect_encoding(teststring) == ["utf-8"] teststring = "高山云雾出好茶".encode("gb18030") assert "gb18030" in detect_encoding(teststring) assert "gb18030" in detect_encoding(teststring*1000) assert is_dubious_html("This is a string.") is True htmlstring = "\n" beginning = htmlstring[:50].lower() assert repair_faulty_html(htmlstring, beginning) == "\n" htmlstring = "\n" beginning = htmlstring[:50].lower() assert repair_faulty_html(htmlstring, beginning) == htmlstring htmlstring = "\n" beginning = htmlstring[:50].lower() assert repair_faulty_html(htmlstring, beginning) == "\n" htmlstring = '\n\n\n\n' beginning = htmlstring[:50].lower() assert ( repair_faulty_html(htmlstring, beginning) == '\n\n\n\n' ) htmlstring = 'Foo
Bar' beginning = htmlstring[:50].lower() assert ( repair_faulty_html(htmlstring, beginning) == 'Foo
Bar\n' ) with pytest.raises(TypeError) as err: assert load_html(123) is None assert 'incompatible' in str(err.value) assert load_html('ÄÖÜ') is not None assert load_html(b'\x2f\x2e\x9f') is not None assert load_html('\x2f\x2e\x9f'.encode('latin-1')) is not None #assert load_html(b'0'*int(10e3)) is None # old: with pytest.raises(TypeError) as err: assert extract(None, 'url', '0000', target_language=None) is None # GZip with open(path.join(RESOURCES_DIR, 'webpage.html.gz'), 'rb') as gzfile: myinput = gzfile.read() assert 'Long story short,' in extract(myinput) # unicode normalization assert normalize_unicode('A\u0308ffin') != 'A\u0308ffin' testresult = extract('

A\u0308ffin

', config=ZERO_CONFIG) assert testresult != 'A\u0308ffin' and testresult == 'Äffin' options = Extractor(source="test\udcc3this") assert options.source == "test?this" # output format assert extract('

ABC

', output_format="xml") is not None with pytest.raises(AttributeError): assert extract('

ABC

', output_format="xyz") is not None assert bare_extraction('

ABC

', output_format="python") is not None with pytest.raises(AttributeError): assert bare_extraction('

ABC

', output_format="xyz") is not None # text elements elem = etree.Element("p") elem.text = "text" assert handle_textelem(elem, [], DEFAULT_OPTIONS) is not None elem = etree.Element("unexpected") elem.text = "text" assert handle_textelem(elem, [], DEFAULT_OPTIONS) is None def test_xmltocsv(): doc = Document() doc.body = etree.fromstring('') doc.commentsbody = etree.fromstring('') assert xml.xmltocsv(doc, False) == 'null\tnull\tnull\tnull\tnull\tnull\tnull\tnull\tnull\tnull\tnull\r\n' doc.title = 'Test title' doc.url = 'https://example.org' doc.hostname = 'example.org' doc.id = '1' doc.license = 'CC BY-SA' doc.image = 'https://example.org/image.jpg' doc.pagetype = 'article' text = 'Test text' comments = 'Test comment' doc.body = etree.fromstring(f'

{text}

') doc.commentsbody = etree.fromstring(f'

{comments}

') target = 'https://example.org\t1\tnull\texample.org\tTest title\thttps://example.org/image.jpg\tnull\tTest text\tTest comment\tCC BY-SA\tarticle\r\n' assert xml.xmltocsv(doc, False) == target mystring = '

ÄÄÄÄÄÄÄÄÄÄÄÄÄÄ

' assert extract(mystring, output_format='csv', config=ZERO_CONFIG) is not None assert extract(mystring, output_format='csv', include_comments=False, config=ZERO_CONFIG).endswith('\tnull\r\n') def test_tojson(): # test json mystring = '

ÄÄÄÄÄÄÄÄÄÄÄÄÄÄ

' result = extract(mystring, output_format='json', config=ZERO_CONFIG) assert "Ä" in result and result.endswith('}') result = extract(mystring, output_format='json', config=ZERO_CONFIG, with_metadata=True) assert result.endswith('}') and '"fingerprint":' in result and '"language":' in result assert extract(mystring, output_format='json', include_comments=False, config=ZERO_CONFIG).endswith('}') def test_python_output(): # bare extraction for python mystring = '

ÄÄÄÄÄÄÄÄÄÄÄÄÄÄ

' result = bare_extraction(mystring, config=ZERO_CONFIG) dict_result = result.as_dict() assert isinstance(dict_result, dict) and len(dict_result) == 21 def test_exotic_tags(xmloutput=False): options = DEFAULT_OPTIONS options._add_config(ZERO_CONFIG) # cover some edge cases with a specially crafted file result = load_mock_page('http://exotic_tags', xml_flag=xmloutput, tei_output=True) assert 'Teletype text' in result and 'My new car is silver.' in result filepath = path.join(TEST_DIR, 'resources', 'exotic_tags_tei.html') with open(filepath, "r", encoding="utf-8") as f: content = etree.fromstring(f.read()) res = xml.check_tei(content, 'http://dummy') assert etree.tostring(res).startswith(b'\n\n\n
\n\nHello\n

Teletype text

') # misformed HTML declaration htmlstring = '

ABC

' # outputs '012"http://www.w3.org/TR/html4/loose.dtd">\nABC' assert 'ABC' in extract(htmlstring, config=ZERO_CONFIG) # quotes assert handle_quotes(etree.Element('quote'), options) is None assert handle_table(etree.Element('table'), TAG_CATALOG, options) is None # p within p element, second = etree.Element('p'), etree.Element('p') element.text, second.text = '1st part.', '2nd part.' element.append(second) # delete last element.append(etree.Element('lb')) converted = handle_paragraphs(element, ['p'], options) assert etree.tostring(converted) == b'

1st part. 2nd part.

' # naked div with assert '1.\n2.\n3.' in extract('
1.
2.
3.
', fast=True, config=ZERO_CONFIG) # HTML5:
htmlstring = '
Epcot Center

Epcot is a theme park at Walt Disney World Resort featuring exciting attractions, international pavilions, award-winning fireworks and seasonal special events.

' my_result = extract(htmlstring, fast=True, config=ZERO_CONFIG) assert 'Epcot Center' in my_result and 'award-winning fireworks' in my_result my_result = extract(htmlstring, fast=False, config=ZERO_CONFIG) assert 'Epcot Center' in my_result and 'award-winning fireworks' in my_result # edge cases htmlstring = ''' A weird bug

Lorem ipsum dolor sit amet, consectetur adipiscing elit.

Sed et interdum lectus.

Quisque molestie nunc eu arcu condimentum fringilla.

Aliquam eget interdum elit, id posuere ipsum.

Phasellus lectus erat, hendrerit sed tortor ac, dignissim vehicula metus.

''' assert extract(htmlstring, include_formatting=True, include_links=True, include_images=True) is not None htmlstring = ''' A weird bug

A header

Very specific bug so odd

Nested header

Some "hyphenated-word quote" followed by a bit more text line.

em improperly wrapping p here

Text here

More articles

''' common = {"include_formatting": True, "include_links": True, "include_images": True} params = [ common, {**common, "favor_precision": True}, {**common, "favor_recall": True} ] for p in params: result = extract(htmlstring, **p) assert "em improperly wrapping p here" in result and result.endswith("Text here") # comments assert extract('

text

comment

', include_comments=True, fast=True, config=ZERO_CONFIG).endswith("\ncomment") def test_formatting(): '''Test HTML formatting conversion and extraction''' options = DEFAULT_OPTIONS # trailing my_document = html.fromstring('

This here is the text.

') my_result = extract(my_document, output_format='xml', config=ZERO_CONFIG) assert 'lb' not in my_result # simple formatting my_document = html.fromstring('

This here is in bold font.

') my_result = extract(my_document, output_format='xml', include_formatting=True, config=ZERO_CONFIG) assert 'This here is in bold font.' in my_result # titles as markdown my_string = '

Title

This here is in bold font.Non-bold here

' my_document = html.fromstring(my_string) my_result = extract(my_document, output_format='txt', include_formatting=True, config=ZERO_CONFIG) assert my_result == '### Title\n\n**This here is in bold font.**Non-bold here' assert extract(my_string, output_format='markdown', config=ZERO_CONFIG) == my_result assert '' in etree.tostring(bare_extraction(my_string, output_format='markdown', config=ZERO_CONFIG).body, encoding="unicode") meta_string = 'Test

ABC.

' meta_result = extract(meta_string, output_format='markdown', config=ZERO_CONFIG, with_metadata=True) assert " ".join(meta_result.split()) == "--- title: Test --- ABC." # space between paragraphs my_document = html.fromstring('

Title

Paragraph 1

Paragraph 2

') my_result = extract(my_document, output_format='txt', include_formatting=True, config=ZERO_CONFIG) assert my_result.endswith('Paragraph 1\n\nParagraph 2') # code sections my_document = html.fromstring('

Title

Here is a code sample:

import trafilatura

') my_result = extract(my_document, output_format='txt', include_formatting=True, config=ZERO_CONFIG) assert my_result == """### Title Here is a code sample: `import trafilatura`""" my_document = html.fromstring('

Title

Here is a code sample:

import something
something.run("somewhere")

Sometimes code is wrapped using pre and code:

import trafilatura\ntrafilatura.extract("")

Less often code is wrapped using just pre:

\ntrafilatura.extract("")
') my_result = extract(my_document, output_format='txt', include_formatting=True, config=ZERO_CONFIG) print(my_result) assert my_result == """### Title Here is a code sample: ``` import something something.run("somewhere") ``` Sometimes code is wrapped using `pre` and `code`: ``` import trafilatura trafilatura.extract("") ``` Less often code is wrapped using just `pre`: ``` trafilatura.extract("") ```""" # nested my_document = html.fromstring('

This here is in bold and italic font.

') my_result = extract(my_document, output_format='xml', include_formatting=True, config=ZERO_CONFIG) assert 'This here is in bold and italic font.' in my_result # empty my_document = html.fromstring('

') my_result = extract(my_document, output_format='xml', include_formatting=True, config=ZERO_CONFIG) assert '
' in my_result # wild div my_document = html.fromstring('
Wild text
') my_result = extract(my_document, output_format='xml', include_formatting=True, config=ZERO_CONFIG) assert '

' in my_result and 'Wild text' in my_result # no rend so far my_document = html.fromstring('

Wild text
') my_result = extract(my_document, config=ZERO_CONFIG) assert my_result == 'Wild text' # links doc = html.fromstring('

Link text

') my_result = extract(doc, config=ZERO_CONFIG) assert my_result == 'Link text' # line-breaks doc = html.fromstring('


') my_result = extract(doc, config=ZERO_CONFIG) assert my_result == '' doc = html.fromstring('


Here is the text.

') my_result = extract(doc, config=ZERO_CONFIG) assert my_result == 'Here is the text.' # handle formatting tails element = etree.Element("hi") element.text = 'Here is the text.' element.tail = 'And a tail.' options._add_config(ZERO_CONFIG) converted = handle_formatting(element, options) assert etree.tostring(converted) == b'

Here is the text.And a tail.

' # empty elements my_document = html.fromstring('
\t\n
There is text here.
') my_result = extract(my_document, output_format='xml', config=ZERO_CONFIG) assert '
\n

There is text here.

\n
' in my_result # lists with links my_document = html.fromstring('
  • Number 1
  • Number 2
  • Number 3

    • Test

') my_result = extract(my_document, output_format='xml', include_links=True, config=ZERO_CONFIG) assert 'Number 2' in my_result my_document = html.fromstring("""
  • Number 0
  • Number 1
  • Number 2 n2
  • Number 3

  • Number 4

    n4
Test
""") my_result = extract(my_document, output_format='markdown', include_links=True, config=ZERO_CONFIG) assert my_result == '- Number 0\n- Number [1](test.html)\n- [Number 2](test.html)n2\n- Number 3\n- Number 4 n4\n\nTest' # XML and Markdown formatting within

-tag my_document = html.fromstring('

bold, italics, tt, deleted, underlined, link and additional text to bypass detection.

') my_result = extract(copy(my_document), fast=True, include_formatting=False, config=ZERO_CONFIG) assert my_result == 'bold, italics, tt, deleted, underlined, link and additional text to bypass detection.' my_result = extract(copy(my_document), fast=True, include_formatting=True, config=ZERO_CONFIG) assert my_result == '**bold**, *italics*, `tt`, ~~deleted~~, __underlined__, link and additional text to bypass detection.' my_result = extract(copy(my_document), fast=True, include_links=True, include_formatting=True, config=ZERO_CONFIG) assert my_result == '**bold**, *italics*, `tt`, ~~deleted~~, __underlined__, [link](test.html) and additional text to bypass detection.' my_result = extract(copy(my_document), output_format='xml', fast=True, include_formatting=True, config=ZERO_CONFIG) assert '

bold, italics, tt, deleted, underlined, link and additional text to bypass detection.

' in my_result assert 'rend="#b"' in my_result and 'rend="#i"' in my_result and 'rend="#t"' in my_result and 'rend="#u"' in my_result and '' in my_result my_result = extract(copy(my_document), output_format='xml', include_formatting=True, include_links=True, fast=True, config=ZERO_CONFIG) assert '

bold, italics, tt, deleted, underlined, link and additional text to bypass detection.

' in my_result my_result = extract(my_document, output_format='txt', fast=True, include_formatting=True, config=ZERO_CONFIG) assert my_result == '**bold**, *italics*, `tt`, ~~deleted~~, __underlined__, link and additional text to bypass detection.' # double

-elems # could be solved by keeping the elements instead of reconstructing them my_document = html.fromstring('

AAA,

BBB

, CCC.

') my_result = extract(my_document, output_format='xml', include_formatting=True, include_links=True, fast=True, config=ZERO_CONFIG) assert 'AAA' in my_result and 'BBB' in my_result and 'CCC' in my_result # line-break following formatting my_document = html.fromstring('

Staff Review of the Financial Situation
Domestic financial conditions remained accommodative over the intermeeting period.

') my_result = extract(my_document, output_format='txt', fast=True, config=ZERO_CONFIG) assert my_result == 'Staff Review of the Financial Situation\nDomestic financial conditions remained accommodative over the intermeeting period.' # title with formatting my_document = html.fromstring('

1) The in Operator

The easiest way to check if a Python string contains a substring is to use the in operator. The in operator is used to check data structures for membership in Python. It returns a Boolean (either True or False) and can be used as follows:

') my_result = extract(my_document, output_format='xml', fast=True, include_formatting=True, config=ZERO_CONFIG) assert '1) The in Operator' in my_result and '

The easiest way to check if a Python string contains a substring is to use the in operator. The in operator is used to check data structures for membership in Python. It returns a Boolean (either True or False) and can be used as follows:

' in my_result my_document = html.fromstring("""
python code below: 

def test:
    print('hello')
    print('world')
""") my_result = extract(my_document, output_format='markdown', include_formatting=True) assert "python code below:\n```\ndef test:\n print('hello')\n print('world')\n \n```" == my_result my_result = extract(my_document, output_format='markdown', include_formatting=True) assert """python code below: ``` def test: print('hello') print('world') ```""" == my_result def test_extract_with_metadata(): '''Test extract_with_metadata method''' url = 'http://aa.bb/cc.html' my_document = html.fromstring("""

AAA,

BBB

, CCC.

""") parsed_doc = extract_with_metadata(my_document, output_format='txt', include_formatting=True, fast=True, url=url) content = parsed_doc.text assert 'AAA' in content and 'BBB' in content and 'CCC' in content assert url == parsed_doc.url and parsed_doc.date is None and parsed_doc.title is None my_document = html.fromstring(""" title
May 24, 2021

AAA,

BBB

, CCC.

""") parsed_doc = extract_with_metadata(my_document, output_format='txt', include_formatting=True, fast=True, url=url) content = parsed_doc.text assert 'AAA' in content and 'BBB' in content and 'CCC' in content assert url == parsed_doc.url and '2021-05-24' == parsed_doc.date and 'title' == parsed_doc.title parsed_doc = extract_with_metadata(my_document, output_format='xml') assert 'AAA, BBB , CCC.' == parsed_doc.raw_text and 'ee7d2fb6fcf2837d' == parsed_doc.fingerprint content = parsed_doc.text assert 'AAA' in content and 'BBB' in content and 'CCC' in content my_document = html.fromstring("""

AAA,

BBB

, CCC.

""") parsed_doc = extract_with_metadata(my_document, target_language='en', fast=True) assert parsed_doc is None with pytest.raises(ValueError) as err: extract_with_metadata(my_document, output_format="python") def test_external(): '''Test external components''' options = DEFAULT_OPTIONS options.tables = True # remove unwanted elements mydoc = html.fromstring('
Test text
') _, _, mylen = sanitize_tree(mydoc, options) assert mylen == 0 mydoc = html.fromstring('
Test text
Test
') _, _, mylen = sanitize_tree(mydoc, options) assert mylen > 0 # strip fancy tags while including links and images mydoc = html.fromstring('

Text here Test textwith a link.

') mytree, _, _ = sanitize_tree(mydoc, options) assert len(mytree) == 1 mydoc = html.fromstring('

Text here Test textwith a link.

') options.links, options.images = True, True mytree, _, _ = sanitize_tree(mydoc, options) myelems = {element.tag for element in set(mytree.iter())} assert 'graphic' in myelems and 'ref' in myelems # test langid if LANGID_FLAG is True: doc = html.fromstring('' + '

Non è inglese.

'*20 + '') assert extract(doc, fast=False, target_language='en', deduplicate=False) is None # no tables with open(path.join(RESOURCES_DIR, "apache.html"), "r", encoding="utf-8") as f: teststring = f.read() assert 'localhost:80' in extract(teststring, fast=False, include_tables=True) assert 'localhost:80' not in extract(teststring, fast=False, include_tables=False) with open(path.join(RESOURCES_DIR, "scam.html"), "r", encoding="utf-8") as f: teststring = f.read() assert extract(teststring, fast=True, include_tables=False) == '' assert extract(teststring, fast=False, include_tables=False) == '' # invalid XML attributes: namespace colon in attribute key (issue #375). Those attributes should be stripped bad_xml = 'Testing
    Features:
  • Saves the cost of two dedicated phone lines.
    • al station using Internet or cellular technology.
  • Requires no change to the existing Fire Alarm Control Panel configuration. The IPGSM-4G connects directly to the primary and secondary telephone ports.
    • ' res = extract(bad_xml, output_format='xml') assert "Features" in res def test_images(): '''Test image extraction function''' # file type assert is_image_file(None) is False assert is_image_file('') is False assert is_image_file('test.jpg') is True assert is_image_file('test.txt') is False assert is_image_file('test.jpg'*2000) is False # length threshold # tag with attributes assert handle_image(None) is None assert handle_image(html.fromstring('')) is not None assert handle_image(html.fromstring('text')) is not None assert handle_image(html.fromstring('')) is None # HTML conversion assert handle_textelem(etree.Element('graphic'), [], DEFAULT_OPTIONS) is None with open(path.join(RESOURCES_DIR, "http_sample.html"), "r", encoding="utf-8") as f: teststring = f.read() assert '![Example image](test.jpg)' not in extract(teststring) assert '![Example image](test.jpg)' in extract(teststring, include_images=True, fast=True) assert '' in extract(teststring, include_images=True, fast=True, output_format='xml', config=ZERO_CONFIG) assert extract('
    text
    ', include_images=True, fast=True) == '![a title text](test.jpg)' assert extract('

    text

    ', include_images=True, fast=True) == '![a title text](test.jpg)' assert extract('

    text

    ', include_images=True, fast=True) == '' assert extract('

    text

    ', include_images=True, fast=True) == '![a title text](test.jpg)' assert extract('

    text

    ', include_images=True, fast=True) == '![a title text](test.jpg)' assert extract('

    text

    ', include_images=True, fast=True) == '![a title text](https://a.b/test.jpg)' url = 'http://a.b/c/d.html' assert extract('

    text

    ', url=url, include_images=True, fast=True) == '![a title text](http://a.b/test.jpg)' assert extract('

    text

    ', url=url, include_images=True, fast=True) == '![a title text](http://a.b/a.b/test.jpg)' assert extract('

    text

    ', url=url, include_images=True, fast=True) == '![a title text](http://a.b/c/a.b/test.jpg)' assert extract('

    text

    ', url=url, include_images=True, fast=True) == '![a title text](http://a.b/a.b/test.jpg)' assert handle_image(html.fromstring('text')) is None # CNN example mydoc = html.fromstring('Harry and Meghan last March, in their final royal engagement.') myimage = handle_image(mydoc) assert myimage is not None and 'alt' in myimage.attrib and 'src' in myimage.attrib # modified CNN example mydoc = html.fromstring('Harry and Meghan last March, in their final royal engagement.') myimage = handle_image(mydoc) assert myimage is not None and 'alt' in myimage.attrib and 'src' in myimage.attrib and myimage.get('src').startswith('http') def test_links(): '''Test link extraction function''' options = DEFAULT_OPTIONS options._add_config(ZERO_CONFIG) assert handle_textelem(etree.Element('ref'), [], options) is None assert handle_formatting(html.fromstring('Test link text.'), options) is not None # empty link mydoc = html.fromstring('

    Some text.

    ') assert extract(mydoc) is not None # link with target mydoc = html.fromstring('

    Test link text. This part of the text has to be long enough.

    ') assert 'testlink.html' not in extract(copy(mydoc)) assert '[Test link text.](testlink.html) This part of the text has to be long enough.' in extract(copy(mydoc), include_links=True, fast=True, config=ZERO_CONFIG) # relative link conversion assert '[Test link text.](https://www.example.com/testlink.html) This part of the text has to be long enough.' in extract(copy(mydoc), url='https://www.example.com/', include_links=True, fast=True, config=ZERO_CONFIG) # link without target mydoc = html.fromstring('

    Test link text. This part of the text has to be long enough.

    ') assert '[Test link text.] This part of the text has to be long enough.' in extract(copy(mydoc), include_links=True, fast=True, config=ZERO_CONFIG) mydoc = html.fromstring('') result = extract(copy(mydoc), output_format='xml', include_links=True, fast=True, config=ZERO_CONFIG) assert '1' in result and '2' in result and '3' in result with open(path.join(RESOURCES_DIR, "http_sample.html"), "r", encoding="utf-8") as f: teststring = f.read() assert 'testlink.html' not in extract(teststring, config=ZERO_CONFIG) assert '[link](testlink.html)' in extract(teststring, include_links=True, fast=True, config=ZERO_CONFIG) assert 'link' in extract(teststring, include_links=True, fast=True, output_format='xml', config=ZERO_CONFIG) # test license link mydoc = html.fromstring('

    Test text under CC BY-SA license.

    ') assert 'license="CC BY-SA license"' in extract(mydoc, include_links=True, fast=True, output_format='xml', config=ZERO_CONFIG, with_metadata=True) # link in p, length threshold mydoc = html.fromstring(f'') assert "abc" in extract(copy(mydoc), fast=True, config=ZERO_CONFIG, favor_precision=False) assert extract(mydoc, fast=True, config=ZERO_CONFIG, favor_precision=True) == "" def test_tei(): '''test TEI-related functions''' # open local resources to avoid redownloading at each run with open(path.join(RESOURCES_DIR, "httpbin_sample.html"), "r", encoding="utf-8") as f: teststring = f.read() # download, parse and validate simple html file result1 = extract(teststring, "mocked", fast=True, output_format='xmltei', tei_validation=False) result2 = extract(teststring, "mocked", fast=True, output_format='xmltei', tei_validation=True) assert result1 is not None and result1 == result2 assert xml.validate_tei(etree.fromstring(result1)) is True assert xml.validate_tei(etree.fromstring(teststring)) is False # test with another file with open(path.join(RESOURCES_DIR, "http_sample.html"), "r", encoding="utf-8") as f: teststring = f.read() # download, parse and validate simple html file result = extract(teststring, "mocked", fast=True, include_comments=True, output_format='xmltei', tei_validation=False) assert result is not None # and '

    license

    ' in result assert xml.validate_tei(etree.fromstring(result)) is True result = extract(teststring, "mocked", fast=True, include_comments=False, output_format='xmltei', tei_validation=False) assert result is not None # and '

    license

    ' in result assert xml.validate_tei(etree.fromstring(result)) is True # include ID in metadata result = extract(teststring, "mocked", fast=True, output_format='xmltei', tei_validation=False, record_id='0001') assert result is not None assert xml.validate_tei(etree.fromstring(result)) is True # test header + metadata tei = etree.Element('TEI', xmlns='http://www.tei-c.org/ns/1.0') header = etree.SubElement(tei, 'teiHeader') docmeta = Document() docmeta.categories, docmeta.tags = [], [] docmeta.title = 'Title' assert xml.write_fullheader(header, docmeta) is not None docmeta.sitename = 'Site Name' docmeta.date = '2021-01-01' assert xml.write_fullheader(header, docmeta) is not None docmeta.date = None assert xml.write_fullheader(header, docmeta) is not None docmeta.hostname = 'hostname' assert xml.write_fullheader(header, docmeta) is not None docmeta.sitename = None docmeta.license = 'CC BY-SA' docmeta.url = 'https://test.org/' docmeta.categories = ['cat1', 'cat2'] assert xml.write_fullheader(header, docmeta) is not None docmeta.date = '2021-01-01' assert xml.write_fullheader(header, docmeta) is not None docmeta.title, docmeta.sitename = None, None assert xml.write_fullheader(header, docmeta) is not None xml_doc = etree.fromstring("
    text
    ") cleaned = xml.check_tei(xml_doc, "fake_url") result = [(elem.tag, elem.text) for elem in cleaned.find(".//div").iter()] expected = [("div", None), ("p", "text")] assert result == expected xml_doc = etree.fromstring("
    text1

    text2

    ") cleaned = xml.check_tei(xml_doc, "fake_url") result = [(elem.tag, elem.text) for elem in cleaned.find(".//div").iter()] expected = [("div", None), ("div", None), ("p", "text1 text2")] assert result == expected xml_doc = etree.fromstring("
    text1text2
    ") cleaned = xml.check_tei(xml_doc, "fake_url") result = [(elem.tag, elem.text) for elem in cleaned.find(".//div").iter()] expected = [("div", None), ("div", None), ("p", "text1"), ("ab", "text2")] assert result == expected xml_doc = etree.fromstring("
    text1

    text2

    has to be there
    ") cleaned = xml.check_tei(xml_doc, "fake_url") result = [(elem.tag, elem.text, elem.tail) for elem in cleaned.find(".//div/div").iter()] expected = [("div", None, None), ("p", "text1 text2 has to be there", None)] assert result == expected xml_doc = etree.fromstring("
    text1text2
    has to be there
    ") cleaned = xml.check_tei(xml_doc, "fake_url") result = [(elem.tag, elem.text, elem.tail) for elem in cleaned.find(".//div/div").iter()] expected = [("div", None, None), ("p", "text1", None), ("quote", "text2", None), ("p", "has to be there", None)] assert result == expected xml_doc = etree.fromstring("
    text1

    text2

    has to be there
    ") cleaned = xml.check_tei(xml_doc, "fake_url") result = [(elem.tag, elem.text, elem.tail) for elem in cleaned.find(".//div/div").iter()] expected = [("div", None, None), ("p", "text1 text2 has to be there", None)] assert result == expected htmlstring = html.fromstring("

    text

    ") extracted = extract(htmlstring, url='mocked', fast=True, output_format="xmltei") assert xml.validate_tei(etree.fromstring(extracted)) is True htmlstring = html.fromstring("

    title

    subtitle

    text

    ") extracted = extract(htmlstring, url="mocked", fast=True, output_format="xmltei") assert 'title' in extracted assert 'subtitle' in extracted htmlstring = html.fromstring( """

    content

    • text1
    • text2

    """ ) extracted = extract(htmlstring, url="mocked", fast=True, output_format="xmltei") assert 'contenttext1' in extracted.replace("\n", "") # merge double elements tree = html.fromstring( """

    content

    """ ) tree = xml.remove_empty_elements(xml.strip_double_tags(tree)) result = sanitize(etree.tostring(tree, encoding="unicode")).replace("\n", "") assert result == "

    content

    " tree = html.fromstring( """

    text

    """ ) xml.strip_double_tags(tree) assert tree.find(".//div/div") is not None and tree.find(".//p/p") is None tree = etree.XML( """

    text1text2

    text3

    text4

    text5

    text6

    """ ) xml.strip_double_tags(tree) assert tree.find(".//p/p") is None tree = etree.XML( """

    text1text2

    text3

    text4

    text5

    text6

    text7

    """ ) xml.strip_double_tags(tree) assert tree.find(".//p/p") is None assert "text7" in etree.tostring(tree, encoding="unicode") # nested elements with same tag not merged tree = html.fromstring( """

    text

    text1

    text2

    text3

    text4

    """ ) xml.strip_double_tags(tree) for parent_tag in ["item", "cell", "quote", "note", "figure"]: assert tree.find(f".//{parent_tag}/p") is not None def test_htmlprocessing(): '''test html-related functions''' assert xml.xmltotxt(None, include_formatting=False) == "" options = DEFAULT_OPTIONS options.tables = True assert trafilatura.htmlprocessing.tree_cleaning(etree.Element('html'), options) is not None assert trafilatura.htmlprocessing.prune_html(etree.Element('unwanted')) is not None mydoc = html.fromstring('Link
    UnderlinedTrue TypeTextText') options.formatting, options.images, options.links = True, True, True myconverted = trafilatura.htmlprocessing.convert_tags(mydoc, options) assert myconverted.xpath('.//ref') and myconverted.xpath('.//graphic') and myconverted.xpath('.//hi[@rend="#t"]') and myconverted.xpath('.//table') options.images, options.tables = True, False myconverted = trafilatura.htmlprocessing.tree_cleaning(mydoc, options) assert myconverted.xpath('.//graphic') and not myconverted.xpath('.//table') mydoc = html.fromstring('

    Test headline

    Test

    ') assert 'Test headline' in extract(copy(mydoc), output_format='xml', config=ZERO_CONFIG, fast=True) assert 'Test headline' in extract(copy(mydoc), output_format='xmltei', config=ZERO_CONFIG, fast=True) # merge with parent function element = etree.Element('test') xml.delete_element(element) assert etree.tostring(element) == b'' element = etree.Element('test') xml.merge_with_parent(element) assert etree.tostring(element) == b'' mydoc = html.fromstring('

    ABC

    ') for element in mydoc.iter('span'): xml.merge_with_parent(element) assert b'

    A B C

    ' in etree.tostring(mydoc) mydoc = html.fromstring('

    AB tailC

    ') for element in mydoc.iter('span'): xml.merge_with_parent(element) assert b'

    A B tail C

    ' in etree.tostring(mydoc) # paywalls my_html = '

    1

    2

    3

    ' assert extract(my_html, config=ZERO_CONFIG, fast=True) == '1\n3' assert extract(my_html, config=ZERO_CONFIG, fast=False) == '1\n3' # test tail of node deleted if set as text node = etree.fromstring("

    tail
    ")[0] trafilatura.htmlprocessing.process_node(node, options) assert node.text == 'tail' assert node.tail is None node = etree.fromstring("text in tail")[0] trafilatura.htmlprocessing.process_node(node, options) assert node.text == "text in tail" assert node.tail is None line_break = etree.fromstring("

    tail

    ")[0] trafilatura.htmlprocessing.process_node(line_break, options) assert line_break.text is None assert line_break.tail == "tail" node = etree.fromstring("

    some text

    tail
    ")[0] trafilatura.htmlprocessing.process_node(node, options) assert node.text == "some text" assert node.tail == "tail" node = etree.fromstring("

    boldinnerouter

    ")[0] processed = trafilatura.htmlprocessing.handle_textnode(node, options) assert processed.tail == "outer" node = etree.fromstring("

    texttail

    ")[0] processed = trafilatura.htmlprocessing.handle_textnode(node, options) assert processed.tail == "tail" and processed.text == "text" node = etree.fromstring("

    tail

    ")[0] processed = trafilatura.htmlprocessing.handle_textnode(node, options) assert processed.tail == "" and processed.text == "tail" node = etree.fromstring("

    textboldtail

    ")[0] processed = trafilatura.htmlprocessing.handle_textnode(node, options) assert processed.tail == "tail" and processed.text == "text" # fix for bug 807 node = html.fragment_fromstring("

    span span tail

    p tail
    ") assert node.text_content() == "span span tail p tail " prune = etree.XPath(".//span") processed = trafilatura.htmlprocessing.prune_unwanted_nodes(node, [prune]) assert node.text_content() == " span tail p tail " def test_extraction_options(): '''Test the different parameters available in extract() and bare_extraction()''' my_html = '

    Text.

' with pytest.raises(ValueError) as err: extract(my_html, output_format="python") assert extract(my_html, config=NEW_CONFIG) is None assert extract(my_html, config=ZERO_CONFIG) is not None assert extract(my_html, only_with_metadata=False, output_format='xml', config=ZERO_CONFIG) is not None assert extract(my_html, only_with_metadata=True, output_format='xml', config=ZERO_CONFIG) is None assert extract(my_html, target_language='de', config=ZERO_CONFIG) is None assert extract(my_html, target_language='de', fast=True, config=ZERO_CONFIG) is None # justext hardening assert etree.tostring(try_justext(html.fromstring(my_html), None, 'de')) == b'' assert etree.tostring(try_justext(None, None, 'de')) == b'' # assert extract(my_html) is None # readability my_html = '

' + 'Text. '*10 + '

' result = etree.tostring(try_readability(html.fromstring(my_html))) assert len(result) > 10 and b'Text' in result my_html = '

' + 'Text. '*10 + 'Test

' result = etree.tostring(try_readability(html.fromstring(my_html))) assert b'Test' not in result my_html = '' + '

ABC def ghi jkl.

'*1000 + '

Posted on 1st Dec 2019<.

' assert bare_extraction(my_html, config=ZERO_CONFIG, with_metadata=True).date is not None assert bare_extraction(my_html, config=NEW_CONFIG, with_metadata=True).date is None assert bare_extraction(my_html, config=NEW_CONFIG, with_metadata=False).date is None def test_precision_recall(): '''test precision- and recall-oriented settings''' # the test cases could be better my_document = html.fromstring('

This here is the text.

') assert extract(copy(my_document), favor_precision=True, config=ZERO_CONFIG, fast=True) is not None assert extract(copy(my_document), favor_recall=True, config=ZERO_CONFIG, fast=True) is not None my_document = html.fromstring('

This here is a teaser text.

This here is the text.

') assert 'teaser text' in extract(copy(my_document), favor_recall=True, config=ZERO_CONFIG, fast=True) assert 'teaser text' not in extract(copy(my_document), config=ZERO_CONFIG, fast=True) assert 'teaser text' not in extract(copy(my_document), favor_precision=True, config=ZERO_CONFIG, fast=True) my_document = html.fromstring('') result = extract(copy(my_document), favor_recall=True, config=ZERO_CONFIG, fast=True) assert '1' not in result result = extract(copy(my_document), favor_precision=True, config=ZERO_CONFIG, fast=True) assert '1' not in result my_document = html.fromstring('

content

Test

') result = extract(copy(my_document), favor_precision=False, config=ZERO_CONFIG, fast=True) assert 'content' in result and 'Test' in result result = extract(copy(my_document), favor_precision=True, config=ZERO_CONFIG, fast=True) assert 'content' in result and 'Test' not in result my_document = html.fromstring('
') result = extract(copy(my_document), favor_recall=False, config=ZERO_CONFIG, fast=True) assert result != "Here is the text." result = extract(copy(my_document), favor_recall=True, config=ZERO_CONFIG, fast=True) assert result == "Here is the text." my_document = html.fromstring('

Title

Text.
') result = extract(copy(my_document), favor_recall=True, config=ZERO_CONFIG, fast=False) assert len(result) > 0 my_document = html.fromstring('
Text.
') assert extract(copy(my_document), favor_precision=True, fast=True) == "" assert extract(copy(my_document), favor_recall=True, fast=True) == "Text." def test_table_processing(): options = DEFAULT_OPTIONS table_simple_cell = html.fromstring( "
cell1cell2
cell3cell4
" ) processed_table = handle_table(table_simple_cell, TAG_CATALOG, options) result = [(child.tag, child.text) for child in processed_table.iter()] assert result == [ ("table", None), ("row", None), ("cell", "cell1"), ("cell", "cell2"), ("row", None), ("cell", "cell3"), ("cell", "cell4"), ] # if a cell contains 'exotic' tags, they are cleaned during the extraction # process and the content is merged with the parent e.g. table_cell_with_children = html.fromstring( "

text

more text

" ) processed_table = handle_table(table_cell_with_children, TAG_CATALOG, options) assert ( etree.tostring(processed_table, encoding="unicode") == "

text

more text

" ) # complex table that hasn't been cleaned yet htmlstring = html.fromstring( """ """ ) processed = extract( htmlstring, fast=True, output_format='xml', config=DEFAULT_CONFIG, include_links=True ) result = processed.replace('\n', '').replace(' ', '') assert """textmore_text
""" in result table_cell_w_text_and_child = html.fromstring( "
text

more text

" ) processed_table = handle_table( table_cell_w_text_and_child, TAG_CATALOG, options ) assert ( etree.tostring(processed_table, encoding="unicode") == "text

more text

" ) table_cell_with_link = html.fromstring( "
link
" ) processed_table = handle_table(table_cell_with_link, TAG_CATALOG, options) result = [child.tag for child in processed_table.find(".//cell").iterdescendants()] assert result == ["p"] table_with_head = html.fromstring( """
Month Days
January 31
February 28
""" ) processed_table = handle_table( table_with_head, TAG_CATALOG, options ) first_row = processed_table[0] assert len(processed_table) == 3 assert [ (child.tag, child.attrib, child.text) for child in first_row.iterdescendants() ] == [("cell", {"role": "head"}, "Month"), ("cell", {"role": "head"}, "Days")] table_with_head_spanning_two_cols = html.fromstring( """
Name Adress Phone
Jane Doe test@example.com phone 1 phone 2
""" ) processed_table = handle_table( table_with_head_spanning_two_cols, TAG_CATALOG, options, ) first_row = processed_table[0] assert len(first_row) == 3 assert {child.tag for child in first_row.iterdescendants()} == {"cell"} table_cell_with_hi = html.fromstring( "
highlighted text
" ) processed_table = handle_table(table_cell_with_hi, TAG_CATALOG, options) result = etree.tostring(processed_table.find(".//cell"), encoding="unicode") assert result == "highlighted text" table_cell_with_span = html.fromstring( "
span text
" ) processed_table = handle_table(table_cell_with_span, TAG_CATALOG, options) result = etree.tostring(processed_table.find(".//cell"), encoding="unicode") assert result == "

" # tables with nested elements htmlstring = '''

Present Tense I buy you buy he/she/it buys we buy you buy they buy
''' my_result = extract(htmlstring, fast=True, output_format='xml', include_formatting=True, config=ZERO_CONFIG) assert ''' Present Tense I buy you buy he/she/it buys we buy you buy they buy ''' in my_result assert extract(htmlstring, fast=True, output_format='txt').startswith("| Present Tense | I buy | you buy |") # table with links # todo: further tests and adjustments htmlstring = '' result = extract(htmlstring, fast=True, output_format='xml', config=ZERO_CONFIG, include_tables=True, include_links=True) assert 'ABCD' not in result # nested table htmlstring = '
1
2
' result = extract(htmlstring, fast=True, output_format='xml', config=ZERO_CONFIG, include_tables=True) # todo: all elements are there, but output not nested assert '1' in result and '2' in result nested_table = html.fromstring( """
1
""" ) processed_table = handle_table(nested_table, TAG_CATALOG, options) result = [ (el.tag, el.text) if el.text is not None and el.text.strip() else el.tag for el in processed_table.iter() ] #assert result == ["table", "row", "cell", "table", "row", ("cell", "1")] assert result == ["table", "row", "cell", ("cell", "1")] complex_nested_table = html.fromstring( """
1
 text1
text2
""" ) processed_table = handle_table(complex_nested_table, TAG_CATALOG, options) result = [ (el.tag, el.text) if el.text is not None and el.text.strip() else el.tag for el in processed_table.iter() ] #assert ( # result # == ["table", "row", "cell", "table", "row", ("cell", "1"), ("cell", "text1"), "row", ("cell", "text2")] #) assert result == ['table', 'row', 'cell', ('cell', '1'), ('cell', 'text1'), 'row', ('cell', 'text2')] table_with_list = html.fromstring( """

a list

one two
""") processed_table = handle_table(copy(table_with_list), TAG_CATALOG, options) result = [ (el.tag, el.text) if el.text is not None and el.text.strip() else el.tag for el in processed_table.iter() ] assert result == ['table', 'row', 'cell', ('p', 'a list'), 'list'] options.focus = "recall" processed_table = handle_table(copy(table_with_list), TAG_CATALOG, options) result = [ (el.tag, el.text) if el.text is not None and el.text.strip() else el.tag for el in processed_table.iter() ] assert result == ["table", "row", "cell", ("p", "a list"), 'list', ("item", "one"), ("item", "two"),] broken_table = html.fromstring("
cell1
cell2
") processed_table = handle_table(broken_table, TAG_CATALOG, options) result = [el.tag for el in processed_table.iter()] assert result == ['table', 'row', 'cell', 'row', 'cell'] broken_table = html.fromstring("

text

cell
") processed_table = handle_table(broken_table, TAG_CATALOG, options) result = [el.tag for el in processed_table.iter()] assert result == ["table", "row", "cell", ] # table nested in figure https://github.com/adbar/trafilatura/issues/301 htmlstring = '
1
2
' result = extract(htmlstring, fast=True, output_format='xml', config=ZERO_CONFIG, include_tables=True) assert "1" in result and "2" in result # table headers in non-XML formats htmlstring = '
head 1head 2
12
' assert "|---|---|" in extract(htmlstring, fast=True, output_format='txt', config=ZERO_CONFIG, include_tables=True) # remove new lines in table cells in text format htmlstring = '
cell
1		cell

2

' result = extract(htmlstring, fast=True, output_format='txt', config=ZERO_CONFIG, include_tables=True) assert "| cell 1 | cell 2 |" in result # only one header row is allowed in text format htmlstring = '
ab
cd
' result = extract(htmlstring, fast=True, output_format='txt', config=ZERO_CONFIG, include_tables=True) assert result.count("---|") == 2 # handle colspan by appending columns in text format htmlstring = '
ab
cde
' result = extract(htmlstring, fast=True, output_format='txt', config=ZERO_CONFIG, include_tables=True) assert "| a | b | |" in result htmlstring = '
ab
cde
' result = extract(htmlstring, fast=True, output_format='txt', config=ZERO_CONFIG, include_tables=True) assert "| a | b | |" in result htmlstring = '
ab
cde
' result = extract(htmlstring, fast=True, output_format='txt', config=ZERO_CONFIG, include_tables=True) assert "| a | b | |" in result # MemoryError: https://github.com/adbar/trafilatura/issues/657 htmlstring = '
ab
cde
' result = extract(htmlstring, fast=True, output_format='txt', config=ZERO_CONFIG, include_tables=True) assert result is not None htmlstring = '
ab
cde
' result = extract(htmlstring, fast=True, output_format='txt', config=ZERO_CONFIG, include_tables=True) assert result is not None # wrong span info htmlstring = '
ab
cde
' result = extract(htmlstring, fast=True, output_format='txt', config=ZERO_CONFIG, include_tables=True) assert "| a | b | |" in result htmlstring = '
ab
cde
' result = extract(htmlstring, fast=True, output_format='txt', config=ZERO_CONFIG, include_tables=True) assert "| a | b | |" in result # links: this gets through (for now) htmlstring = '' result = extract(htmlstring, fast=True, output_format='txt', config=ZERO_CONFIG, include_tables=True) assert result == "| a |" # link: this is filtered out htmlstring = f'' result = extract(htmlstring, fast=True, output_format='txt', config=ZERO_CONFIG, include_tables=True) assert result == "" htmlstring = f'' result = extract(htmlstring, fast=True, output_format='txt', config=ZERO_CONFIG, include_tables=True) assert result == "" htmlstring = """
abc
a

b

c

""" result = extract(htmlstring, fast=True, output_format='txt', config=ZERO_CONFIG, include_tables=True) assert result == "| a | b | c | \n| a | b c | |" htmlstring = """
abc
a

b

c

a

b

c

""" result = extract(htmlstring, fast=True, output_format='txt', config=ZERO_CONFIG, include_tables=True) assert result == "| a | b | c | \n| a | b c | |\n| a | b c | |" htmlstring = """
abc
aimga

b

c

 d
""" result = extract(htmlstring, fast=True, output_format='markdown', config=ZERO_CONFIG, include_images=True, include_tables=True) assert result == "| a | b | c | \n| a ![img](http://aa.bb/c.jpg) a | b c | d |" htmlstring = """
abc
imga

b

c

 d
""" result = extract(htmlstring, fast=True, output_format='markdown', config=ZERO_CONFIG, include_images=True, include_tables=True) assert result == "| a | b | c | \n| ![img](http://aa.bb/c.jpg) a | b c | d |" htmlstring = """
abc
imga

b

c

 d
""" result = extract(htmlstring, fast=True, output_format='markdown', config=ZERO_CONFIG, include_images=True, include_tables=True) assert result == "| a | b | c | \n| ![img](http://aa.bb/c.jpg) a | b c | d |" htmlstring = """
abc
img1aimg2

b

c

 d
""" result = extract(htmlstring, fast=True, output_format='markdown', config=ZERO_CONFIG, include_images=True, include_tables=True) assert result == "| a | b | c | \n| ![img1](http://aa.bb/c.jpg) a ![img2](http://aa.bb/c.jpg) | b c | d |" def test_list_processing(): options = DEFAULT_OPTIONS # basic lists my_doc = "

P 1

  • Item 1
  • Item 2

P 2

" my_result = extract(my_doc, fast=True, output_format='txt', config=ZERO_CONFIG) assert my_result == "P 1\n- Item 1\n- Item 2\nP 2" # malformed lists (common error) result = etree.tostring(handle_lists(etree.fromstring('Description of the list:List item 1List item 2List item 3'), options)) assert result.count(b'List item') == 3 assert b"Description" in result # nested list htmlstring = '''
  • Coffee
  • Tea
    • Black tea
    • Green tea
  • Milk
''' my_result = extract(htmlstring, fast=True, output_format='xml', config=ZERO_CONFIG) expected = ''' Coffee Tea Black tea Green tea Milk '''.replace("\n", "").replace(" ", "") assert expected in my_result.replace("\n", "").replace(" ", "") # description list htmlstring = '''
Coffee
Black hot drink
Milk
White cold drink
''' my_result = extract(htmlstring, fast=True, output_format='xml', config=ZERO_CONFIG) assert ''' Coffee Black hot drink Milk White cold drink ''' in my_result list_item_with_child = html.fromstring("

text

") processed_list = handle_lists(list_item_with_child, options) result = [(child.tag, child.text) if child.text is not None else child.tag for child in processed_list.iter()] assert result == ["list", "item", ("p", "text")] list_item_with_text_and_child = html.fromstring("text1

text2

") processed_list = handle_lists(list_item_with_text_and_child, options) result = [(child.tag, child.text) if child.text is not None else child.tag for child in processed_list.iter()] assert result == ["list", ("item", "text1"), ("p", "text2")] list_item_with_lb = html.fromstring("textmore text") processed_list = handle_lists(list_item_with_lb, options) result = [(child.tag, child.text) if child.text is not None else child.tag for child in processed_list.iter()] assert result == ["list", ("item", "text"), "lb"] list_with_text_outside_item = html.fromstring("headertext") processed_list = handle_lists(list_with_text_outside_item, options) result = [(child.tag, child.text) if child.text is not None else child.tag for child in processed_list.iter()] assert result == ["list", ("item", "header"), ("item", "text")] empty_list = html.fromstring(" text") processed_list = handle_lists(empty_list, options) assert len(processed_list) == 1 list_item_with_tail = html.fromstring("texttail") processed_list = handle_lists(list_item_with_tail, options) assert processed_list[0].text == "text tail" list_item_with_child_and_tail = html.fromstring("

text

tail") processed_list = handle_lists(list_item_with_child_and_tail, options) item_element = processed_list[0] assert item_element.tail is not True assert item_element[0].tail == "tail" list_item_with_child_and_tail = html.fromstring("

text

tail1tail") processed_list = handle_lists(list_item_with_child_and_tail, options) item_element = processed_list[0] assert item_element.tail is not True assert item_element[0].tail == "tail1 tail" list_item_with_child_and_tail = html.fromstring("

text

\ntail") processed_list = handle_lists(list_item_with_child_and_tail, options) item_element = processed_list[0] assert item_element.tail is not True assert item_element[0].tail == "tail" list_item_with_tail_and_nested_list = html.fromstring("texttail") processed_list = handle_lists(list_item_with_tail_and_nested_list, options) target_element = processed_list.find(".//item/list") assert target_element.tail == 'tail' def test_code_blocks(): highlightjs = '''

Code:

code\n
highlighted more code
''' testresult = extract(highlightjs, config=ZERO_CONFIG, output_format='xml') assert 'code\n\nhighlighted more code\n' in testresult and 'quote' not in testresult github = '''
$ pip install PyGithub
''' testresult = extract(github, config=ZERO_CONFIG, output_format='xml') assert '$ pip install PyGithub' in testresult and 'quote' not in testresult inline_code = '

paragraph

here is some code

' testresult = extract(inline_code, config=ZERO_CONFIG, output_format='xml') assert 'some' in testresult and 'quote' not in testresult w3schools = '''

Example

Create a class named Person, use the __init__() function to assign values for name and age:

class Person:
  def __init__(self, name, age):
    self.name = name
    self.age = age

p1 = Person("John", 36)

print(p1.name)
print(p1.age)
''' testresult = extract(w3schools, config=ZERO_CONFIG, output_format='xml') expected = ''' class Person:\xa0 def __init__(self, name, age):\xa0\xa0\xa0 self.name = name\xa0\xa0\xa0 self.age = agep1 = Person("John", 36) print(p1.name)print(p1.age) ''' assert expected in testresult and 'quote' not in testresult pip = '''

Code:

import openai
from openai_function_call import openai_function
''' expected = '''import openai from openai_function_call import openai_function''' testresult = extract(pip, config=ZERO_CONFIG, output_format='xml') assert expected in testresult and 'quote' not in testresult medium_js = '''

Code:

import openai_function

@openai_function
''' expected = '''import openai_function@openai_function''' testresult = extract(medium_js, config=ZERO_CONFIG, output_format='xml') assert expected in testresult and 'quote' not in testresult medium_ssr = '''

Code:

import openai_function

@openai_function
def sum(a:int, b:int):
  """Sum description adds a + b"""
''' expected = '''import openai_function@openai_functiondef sum(a:int, b:int): """Sum description adds a + b"""''' testresult = extract(medium_ssr, config=ZERO_CONFIG, output_format='xml') assert expected in testresult and 'quote' not in testresult code_el = '''

Code:

my code
''' expected = '''my code''' testresult = extract(code_el, config=ZERO_CONFIG, output_format='xml') assert expected in testresult and 'quote' not in testresult def test_mixed_content_extraction(): """ Test extraction from HTML with mixed content. """ html_content = '

Text here