ontology

2026-05-13 19:57:34 +09:00
parent 2e9204243d
commit 9e88f4c7ad
4310 changed files with 48538 additions and 905279 deletions
--- a/crawler_platform/app/config/loader.py
+++ b/crawler_platform/app/config/loader.py
@@ -16,7 +16,7 @@ class SourceConfig:
    parser: str = "generic"
    fetcher: str = "requests"
    rate_limit_per_minute: int = 30
-    respect_robots_txt: bool = True
+    respect_robots_txt: bool = False


@dataclass(slots=True)