lasta/AI
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

ontology

Browse Source
This commit is contained in:
LASTA_DEV01\lasta
2026-05-13 19:57:34 +09:00
parent 2e9204243d
commit 9e88f4c7ad
4310 changed files with 48538 additions and 905279 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
crawler_platform/app/core/research/graph_research_loop.py
View File

@@ -200,8 +200,9 @@ class GraphResearchLoop:
url = item.target
if same_domain_only and seed_host and normalized_host(url) != seed_host:
return {"status": "skipped", "reason": "outside same-domain research boundary"}
if not self.robots_policy.allowed(url, source_config.respect_robots_txt):
return {"status": "skipped", "reason": f"robots.txt blocked {url}"}
robots_decision = self.robots_policy.check(url, source_config.respect_robots_txt)
if not robots_decision.allowed:
return {"status": "skipped", "reason": f"{robots_decision.reason}: {url}"}
fetch_result = fetcher.fetch(url)
parser_result = parser.parse(fetch_result.analysis_html, fetch_result.final_url or url)
@@ -229,6 +230,8 @@ class GraphResearchLoop:
"final_url": fetch_result.final_url,
"crawl_status": fetch_result.crawl_status,
"page_type": page_type,
"robots_status": robots_decision.status,
"robots_reason": robots_decision.reason,
"raw_text_length": len(parser_result.raw_text or ""),
"clean_text_length": len(parser_result.text or ""),
"main_content_preview": (parser_result.main_content or parser_result.text)[:800],