참고소스 수정본

2026-05-12 19:40:31 +09:00
parent 0f34a451fc
commit 2e9204243d
8708 changed files with 3259488 additions and 869 deletions
--- a/참고/trafilatura-master/pyproject.toml
+++ b/참고/trafilatura-master/pyproject.toml
@@ -0,0 +1,116 @@
+# https://pip.pypa.io/en/stable/reference/build-system/pyproject-toml/
+[build-system]
+requires = ["setuptools>=61.0"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "trafilatura"
+description = "Python & Command-line tool to gather text and metadata on the Web: Crawling, scraping, extraction, output as CSV, JSON, HTML, MD, TXT, XML."
+readme = "README.md"
+license = { text = "Apache 2.0" }
+dynamic = ["version"]
+requires-python = ">=3.8"
+authors = [
+  {name = "Adrien Barbaresi", email = "barbaresi@bbaw.de"}
+]
+keywords=[
+    "corpus",
+    "html2text",
+    "news-crawler",
+    "natural-language-processing",
+    "scraper",
+    "tei-xml",
+    "text-extraction",
+    "webscraping",
+    "web-scraping",
+]
+classifiers = [
+    # https://pypi.python.org/pypi?%3Aaction=list_classifiers
+    "Development Status :: 5 - Production/Stable",
+    #'Development Status :: 6 - Mature',
+    "Environment :: Console",
+    "Intended Audience :: Developers",
+    "Intended Audience :: Education",
+    "Intended Audience :: Information Technology",
+    "Intended Audience :: Science/Research",
+    "License :: OSI Approved :: Apache Software License",
+    "Operating System :: MacOS",
+    "Operating System :: Microsoft",
+    "Operating System :: POSIX",
+    "Programming Language :: Python",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.8",
+    "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+    "Topic :: Internet :: WWW/HTTP",
+    "Topic :: Scientific/Engineering :: Information Analysis",
+    "Topic :: Security",
+    "Topic :: Text Editors :: Text Processing",
+    "Topic :: Text Processing :: Linguistic",
+    "Topic :: Text Processing :: Markup :: HTML",
+    "Topic :: Text Processing :: Markup :: Markdown",
+    "Topic :: Text Processing :: Markup :: XML",
+    "Topic :: Utilities",
+]
+dependencies = [
+    "certifi",
+    "charset_normalizer >= 3.4.0",
+    "courlan >= 1.3.2",
+    "htmldate >= 1.9.2",
+    "justext >= 3.0.1",
+    # see tests on Github Actions
+    "lxml == 4.9.2 ; platform_system == 'Darwin' and python_version <= '3.8'",
+    "lxml >= 5.3.0 ; platform_system != 'Darwin' or python_version > '3.8'",
+    "urllib3 >= 1.26, < 3",
+]
+
+# https://setuptools.pypa.io/en/latest/userguide/pyproject_config.html
+[tool.setuptools]
+packages = ["trafilatura"]
+
+# https://packaging.python.org/en/latest/guides/single-sourcing-package-version/
+[tool.setuptools.dynamic]
+version = {attr = "trafilatura.__version__"}
+
+# https://setuptools.pypa.io/en/stable/userguide/datafiles.html
+[tool.setuptools.package-data]
+trafilatura = [
+    "data/tei_corpus.dtd",
+    "settings.cfg",
+]
+
+[project.scripts]
+trafilatura = "trafilatura.cli:main"
+
+[project.urls]
+"Homepage" = "https://trafilatura.readthedocs.io"
+"Source" = "https://github.com/adbar/trafilatura"
+"Blog" = "https://adrien.barbaresi.eu/blog/tag/trafilatura.html"
+"Tracker" = "https://github.com/adbar/trafilatura/issues"
+
+# Development extras
+[project.optional-dependencies]
+dev = [
+    "flake8",
+    "mypy",
+    "pytest",
+    "pytest-cov",
+    "types-lxml",
+    "types-urllib3",
+]
+all = [
+    "brotli",
+    "cchardet >= 2.1.7; python_version < '3.11'",  # build issue
+    "faust-cchardet >= 2.1.19; python_version >= '3.11'",
+    "htmldate[speed] >= 1.9.2",
+    "py3langid >= 0.3.0",
+    "pycurl >= 7.45.3",
+    "urllib3[socks]",
+    "zstandard >= 0.23.0",
+]
+
+[tool.pytest.ini_options]
+testpaths = "tests/*test*.py"