"""
Unit tests for baseline functions of the trafilatura library.
"""
from lxml import html
from trafilatura import baseline, html2txt
def test_baseline():
# Empty input
result = baseline(b'')
assert isinstance(result, tuple) and len(result) == 3
assert result[0].tag == 'body'
assert result[1] == ''
assert result[2] == 0
result = baseline('')
assert isinstance(result, tuple) and len(result) == 3
assert result[0].tag == 'body'
assert result[1] == ''
assert result[2] == 0
# Invalid HTML
_, result, _ = baseline(b'')
assert result == ''
tests = [
'' + 'The article consists of this text.'*10 + '',
'The article consists of this text.',
'This is only a quote but it is better than nothing.
',
]
for doc in tests:
_, result, _ = baseline(doc)
assert result is not None
# Invalid JSON
filecontent = b'''
'''
_, result, _ = baseline(filecontent)
assert result == ''
# JSON OK
filecontent = b'''
'''
_, result, _ = baseline(filecontent)
assert len(result) > 100
# JSON malformed
filecontent = br'''
'''
_, result, _ = baseline(filecontent)
assert result == ''
# Real-world examples
my_document = r''
_, result, _ = baseline(my_document)
assert result.startswith('In letzter Zeit kam man') and result.endswith('erst mal überlegen.')
my_document = " Document body...
"
_, result, _ = baseline(my_document)
assert result == 'Document body...'
def test_html2txt():
mydoc = "Here is the body text"
assert html2txt(mydoc) == "Here is the body text"
assert html2txt(html.fromstring(mydoc)) == "Here is the body text"
assert html2txt("") == ""
assert html2txt("123") == ""
assert html2txt("") == ""
assert html2txt("") == ""
assert html2txt("ABC
") == "ABC"
if __name__ == '__main__':
test_baseline()
test_html2txt()