참고소스 수정본

2026-05-12 19:40:31 +09:00
parent 0f34a451fc
commit 2e9204243d
8708 changed files with 3259488 additions and 869 deletions
--- a/참고/trafilatura-master/trafilatura/baseline.py
+++ b/참고/trafilatura-master/trafilatura/baseline.py
@@ -0,0 +1,123 @@
+"""
+Module regrouping baseline and basic extraction functions.
+"""
+# pylint:disable-msg=E0611
+
+import json
+
+from typing import Any, Tuple
+
+from lxml.etree import _Element, Element, SubElement
+from lxml.html import HtmlElement
+
+from .settings import BASIC_CLEAN_XPATH
+from .utils import load_html, trim
+from .xml import delete_element
+
+
+def basic_cleaning(tree: HtmlElement) -> HtmlElement:
+    "Remove a few section types from the document."
+    for elem in BASIC_CLEAN_XPATH(tree):
+        delete_element(elem)
+    return tree
+
+
+def baseline(filecontent: Any) -> Tuple[_Element, str, int]:
+    """Use baseline extraction function targeting text paragraphs and/or JSON metadata.
+
+    Args:
+        filecontent: HTML code as binary string or string.
+
+    Returns:
+        A LXML <body> element containing the extracted paragraphs,
+        the main text as string, and its length as integer.
+
+    """
+    tree = load_html(filecontent)
+    postbody = Element('body')
+    if tree is None:
+        return postbody, '', 0
+
+    # scrape from json text
+    temp_text = ""
+    for elem in tree.iterfind('.//script[@type="application/ld+json"]'):
+        if elem.text and 'articleBody' in elem.text:
+            try:
+                json_body = json.loads(elem.text).get("articleBody", "")
+            except Exception:  # JSONDecodeError or 'list' object has no attribute 'get'
+                json_body = ""
+            if json_body:
+                if "<p>" in json_body:
+                    parsed = load_html(json_body)
+                    text = trim(parsed.text_content()) if parsed is not None else ""
+                else:
+                    text = trim(json_body)
+                SubElement(postbody, 'p').text = text
+                temp_text += " " + text if temp_text else text
+                # return postbody, elem.text, len(elem.text)
+    if len(temp_text) > 100:
+        return postbody, temp_text, len(temp_text)
+
+    tree = basic_cleaning(tree)
+
+    # scrape from article tag
+    temp_text = ""
+    for article_elem in tree.iterfind('.//article'):
+        text = trim(article_elem.text_content())
+        if len(text) > 100:
+            SubElement(postbody, 'p').text = text
+            temp_text += " " + text if temp_text else text
+    if len(postbody) > 0:
+        # temp_text = trim('\n'.join(postbody.itertext()))
+        return postbody, temp_text, len(temp_text)
+
+    # scrape from text paragraphs
+    results = set()
+    temp_text = ""
+    # postbody = Element('body')
+    for element in tree.iter('blockquote', 'code', 'p', 'pre', 'q', 'quote'):
+        entry = trim(element.text_content())
+        if entry not in results:
+            SubElement(postbody, 'p').text = entry
+            temp_text += " " + entry if temp_text else entry
+            results.add(entry)
+    # temp_text = trim('\n'.join(postbody.itertext()))
+    if len(temp_text) > 100:
+        return postbody, temp_text, len(temp_text)
+
+    # default strategy: clean the tree and take everything
+    postbody = Element('body')
+    body_elem = tree.find('.//body')
+    if body_elem is not None:
+        p_elem = SubElement(postbody, 'p')
+        # todo: sanitize?
+        text_elems = [trim(e) for e in body_elem.itertext()]
+        p_elem.text = '\n'.join([e for e in text_elems if e])
+        return postbody, p_elem.text, len(p_elem.text)
+
+    # new fallback
+    text = html2txt(tree, clean=False)
+    SubElement(postbody, 'p').text = text
+    return postbody, text, len(text)
+
+
+def html2txt(content: Any, clean: bool = True) -> str:
+    """Run basic html2txt on a document.
+
+    Args:
+        content: HTML document as string or LXML element.
+        clean: remove potentially undesirable elements.
+
+    Returns:
+        The extracted text in the form of a string or an empty string.
+
+    """
+    tree = load_html(content)
+    if tree is None:
+        return ""
+    body = tree.find(".//body")
+    if body is None:
+        return ""
+    if clean:
+        body = basic_cleaning(body)
+    return " ".join(body.text_content().split()).strip()