ontology

crawler_platform/app/web/frontend/src/i18n.js

@@ -26,6 +26,7 @@ const dict = {
     "sidebar.max_depth": "최대 깊이",
     "sidebar.max_pages": "최대 페이지",
     "sidebar.same_domain": "같은 도메인만",
+    "sidebar.respect_robots": "robots.txt 준수",
     "sidebar.crawl_site": "시드부터 사이트 크롤",
     "sidebar.stop_crawl": "현재 크롤 중지",
 
@@ -262,6 +263,7 @@ const dict = {
     "sidebar.max_depth": "Max depth",
     "sidebar.max_pages": "Max pages",
     "sidebar.same_domain": "Same domain",
+    "sidebar.respect_robots": "Respect robots.txt",
     "sidebar.crawl_site": "Crawl site from seed",
     "sidebar.stop_crawl": "Stop current crawl",
 

crawler_platform/app/web/frontend/src/sidebar.js

@@ -85,6 +85,10 @@ function sidebarHtml() {
           <input id="sameDomainOnly" type="checkbox" checked />
           <span data-i18n="sidebar.same_domain"></span>
         </label>
+        <label class="check-row">
+          <input id="respectRobotsTxt" type="checkbox" />
+          <span data-i18n="sidebar.respect_robots"></span>
+        </label>
       </div>
       <button id="siteCrawlBtn" class="primary full" data-i18n="sidebar.crawl_site"></button>
       <button id="stopSiteCrawlBtn" class="full" disabled data-i18n="sidebar.stop_crawl"></button>
@@ -102,6 +106,7 @@ export function requestBase() {
     extractor_provider: $("extractorProvider").value,
     extractor_model: $("extractorModel").value.trim() || null,
     extractor_base_url: $("extractorBaseUrl").value.trim() || null,
+    check_robots_txt: $("respectRobotsTxt")?.checked ?? false,
   };
 }
 
@@ -331,6 +336,7 @@ async function discover() {
         source_name: $("sourceSelect").value,
         url: $("crawlUrl").value.trim(),
         limit: 30,
+        check_robots_txt: $("respectRobotsTxt")?.checked ?? false,
       }),
     });
     if (!result.ok) {