참고소스 수정본

참고/knowledge_agent-main/sub_agents/advisor.py

@@ -0,0 +1,56 @@
+# advisor.py
+from langchain.agents import create_openai_tools_agent, AgentExecutor
+from langchain_core.prompts import ChatPromptTemplate
+from langchain_core.tools import tool, ToolException
+from langchain_core.messages import AIMessage
+import json
+import os
+import re
+from state import AgentState
+from db_utils import load_latest_report, extract_and_clean_json
+from terminal_utils import print_colorful_break
+
+async def advisor_agent_node(state: AgentState):
+    print_colorful_break("ADVISOR")
+    logger = state['logger']
+    logger.info("--- Running Advisor Agent ---")
+    all_tools = state['mcp_tools']
+    model = state['model']
+    timestamp = state['timestamp']
+    
+    advisor_tools = [t for t in all_tools if t.name in ["list_allowed_directories", "list_directory", "search_files", "read_text_file"]] + [load_latest_report]
+    advisor_prompt = '''Your goal is to provide recommendations for systemic improvements.
+1.  **Analyze Reports**: Load and analyze `auditor_report.json` and `fixer_report.json`.
+2.  **Generate Recommendations**: Based on recurring patterns, generate actionable recommendations for ingestion prompts or server configuration.
+3.  **Compile Report**: Save a final report with your top 3-5 suggestions to `advisor_report.json`.'''
+
+    prompt = ChatPromptTemplate.from_template(advisor_prompt)
+    agent_executor = create_openai_tools_agent(model, advisor_tools, prompt)
+    
+    task_input = "Your task is to provide recommendations based on the latest audit and fix reports. Begin now."
+
+    result = await agent_executor.ainvoke({"input": task_input, "timestamp": timestamp})
+
+    logger.info(f"Advisor Agent finished with output: {result['output']}")
+    return {"messages": state['messages'] + [AIMessage(content=result['output'])]}
+
+def save_advisor_report_node(state: AgentState):
+    """Saves the final report from the last AI message."""
+    logger = state['logger']
+    final_message_from_agent = state['messages'][-1]
+
+    status = f"--- Saving Advisor Report ---\n{final_message_from_agent.content}"
+    logger.info(status)
+
+    try:
+        report_json = extract_and_clean_json(final_message_from_agent.content)
+        if 'report_id' not in report_json:
+            report_json['report_id'] = state.get('advisor_report_id', 'unknown_id')
+        save_advisor_report({"advisor_report": json.dumps(report_json)})
+        status = f"Successfully saved advisor report with ID {report_json.get('report_id')}"
+        logger.info(status)
+    except (ValueError, KeyError) as e:
+        status = f"Error processing or saving advisor report: {e}"
+        logger.error(status, exc_info=True)
+
+    return {"messages": state['messages'] + [AIMessage(content=status)]}

참고/knowledge_agent-main/sub_agents/analyst.py

@@ -0,0 +1,78 @@
+# sub_agents/analyst.py
+from langchain.agents import create_openai_tools_agent, AgentExecutor
+from langchain_core.prompts import ChatPromptTemplate
+from state import AgentState
+from db_utils import save_analyst_report, extract_and_clean_json
+from terminal_utils import print_colorful_break
+
+async def analyst_agent_node(state: AgentState):
+    """Runs the analyst agent and returns its raw output and the new report ID."""
+    print_colorful_break("ANALYST")
+    logger = state['logger']
+    timestamp = state['timestamp']
+    report_id = f"ana_{timestamp.replace('-', '').replace(':', '').replace('T', '_').split('.')[0]}"
+    status = f"Initialized analyst report with ID: {report_id}"
+    logger.info(status)
+
+    with open("prompts/analyst_prompt.txt", "r") as f:
+        analyst_prompt_template = f.read()
+        
+    analyst_prompt = ChatPromptTemplate.from_template(analyst_prompt_template)    
+    analyst_tools = [t for t in state['mcp_tools'] if t.name in ["query", "graphs_get", "graph_labels", "google_search", "fetch"]]
+
+    status = f"Attempting to invoke analyst agent executor with tools: {analyst_tools}"
+    logger.info(status)
+    try:
+        agent_runnable = create_openai_tools_agent(state['model'], analyst_tools, analyst_prompt)
+        executor = AgentExecutor(agent=agent_runnable, tools=analyst_tools, verbose=True)
+    except Exception as e:
+        status = f"Failed to create agent executor: {e}"
+        logger.error(status, exc_info=True)
+        return {"status": status}
+
+
+    task = state['messages'][0].content
+    status = f"Attempting to run agent executor with input: {task}"
+    logger.info(status)
+    try:
+        analyst_result = await executor.ainvoke({
+            "input": task,
+            "analyst_report_id": report_id
+        })
+        raw_report = analyst_result.get('output', '')
+        status = f"Analyst agent completed.\nRaw output: {raw_report}"
+        logger.info(status)
+
+    except Exception as e:
+        status = f"Analyst agent failed: {e}"
+        logger.error(status, exc_info=True)
+        raw_report = f"Error in Analyst Agent: {e}"
+
+    return {
+        "analyst_report_id": report_id,
+        "analyst_report": raw_report,
+        "status": status
+    }
+
+def save_analyst_report_node(state: AgentState):
+    """Saves the final report and updates the main status field."""
+    logger = state['logger']
+    raw_report_content = state.get("analyst_report")
+
+    report_id = state.get("analyst_report_id")
+    status = f"--- Saving Analyst Report: {report_id} ---"
+    logger.info(status)
+
+    try:
+        report_json = extract_and_clean_json(raw_report_content)
+        if 'report_id' not in report_json:
+            report_json['report_id'] = report_id
+        
+        save_analyst_report(report_json)
+        status = f"Successfully saved analyst report with ID {report_json.get('report_id')}"
+        logger.info(status)
+    except (ValueError, KeyError) as e:
+        status = f"Error processing or saving analyst report: {e}"
+        logger.error(status)
+    
+    return {"status": status}

참고/knowledge_agent-main/sub_agents/auditor.py

@@ -0,0 +1,56 @@
+# auditor.py
+from langchain.agents import create_openai_tools_agent, AgentExecutor
+from langchain_core.prompts import ChatPromptTemplate
+from langchain_core.tools import tool, ToolException
+from langchain_core.messages import AIMessage
+import json
+import os
+import re
+from state import AgentState
+from db_utils import extract_and_clean_json
+from terminal_utils import print_colorful_break
+
+async def auditor_agent_node(state: AgentState):
+    print_colorful_break("AUDITOR")
+    logger = state['logger']
+    logger.info("--- Running Auditor Agent ---")
+    all_tools = state['mcp_tools']
+    model = state['model']
+    timestamp = state['timestamp']
+    
+    auditor_tools = [t for t in all_tools if t.name in ["graphs_get", "query"]]
+    auditor_prompt = '''Your goal is to review the LightRAG knowledge base for data quality issues.
+1.  **Identify issues**: Scan the graph for duplicates, irregular normalization, etc.
+2.  **Generate Report**: Create a report of your findings.
+3.  **Save Report**: Use `save_report` to save the findings to `auditor_report.json`.'''
+
+    prompt = ChatPromptTemplate.from_template(auditor_prompt)
+    agent_executor = create_openai_tools_agent(model, auditor_tools, prompt)
+
+    task_input = "Your task is to audit the knowledge base. Begin now."
+
+    result = await agent_executor.ainvoke({"input": task_input, "timestamp": timestamp})
+    
+    logger.info(f"Auditor Agent finished with output: {result['output']}")
+    return {"messages": state['messages'] + [AIMessage(content=result['output'])]}
+
+def save_auditor_report_node(state: AgentState):
+    """Saves the final report from the last AI message."""
+    logger = state['logger']
+    final_message_from_agent = state['messages'][-1]
+
+    status = f"--- Saving Auditor Report ---\n{final_message_from_agent.content}"
+    logger.info(status)
+
+    try:
+        report_json = extract_and_clean_json(final_message_from_agent.content)
+        if 'report_id' not in report_json:
+            report_json['report_id'] = state.get('auditor_report_id', 'unknown_id')
+        save_auditor_report({"auditor_report": json.dumps(report_json)})
+        status = f"Successfully saved auditor report with ID {report_json.get('report_id')}"
+        logger.info(status)
+    except (ValueError, KeyError) as e:
+        status = f"Error processing or saving auditor report: {e}"
+        logger.error(status, exc_info=True)
+
+    return {"messages": state['messages'] + [AIMessage(content=status)]}

참고/knowledge_agent-main/sub_agents/curator.py

@@ -0,0 +1,177 @@
+# sub_agents/curator.py
+from langchain.agents import create_openai_tools_agent, AgentExecutor
+from langchain_core.prompts import ChatPromptTemplate
+from state import AgentState
+from db_utils import initialize_curator, update_curator_report, extract_and_clean_json
+from terminal_utils import print_colorful_break
+
+async def curator_agent_node(state: AgentState):
+    """Orchestrates the curation process."""
+    print_colorful_break("CURATOR")
+    logger = state['logger']
+
+    report_id = state.get("curator_report_id")
+    searches_todo = []
+    curator_urls_for_ingestion = []
+    curator_url_ingestion_status = []
+    final_status = None
+
+    # One-time initialization of the curator state
+    if not report_id:
+        try:
+            init_result = initialize_curator(state['timestamp'])
+            report_id = init_result.get("curator_report_id")
+            searches_todo = init_result.get("curator_searches_todo")
+            status = f"--- Initialized curator state: {init_result} ---"
+            logger.info(status)
+        except Exception as e:
+            status = f"Failed to initialize curator: {e}"
+            logger.error(status, exc_info=True)
+            return {"status": status}
+
+    # Create the specialized agent for search ranking
+    search_ranker_prompt = ChatPromptTemplate.from_template(open("prompts/search_ranker_prompt.txt", "r").read())
+    search_ranker_tools = [t for t in state['mcp_tools'] if t.name in ["google_search", "fetch"]]
+    status = f"Attempting to invoke search ranker agent executor with tools: {search_ranker_tools}"
+    logger.info(status)
+    try:
+        agent_runnable = create_openai_tools_agent(state['model'], search_ranker_tools, search_ranker_prompt)
+        executor = AgentExecutor(agent=agent_runnable, tools=search_ranker_tools, verbose=True)
+    except Exception as e:
+        status = f"Failed to create search ranker agent executor: {e}"
+        logger.error(status, exc_info=True)
+        return {"status": status}
+
+    # Main control loop for search ranking
+    if isinstance(searches_todo, list) and searches_todo:
+        for item in searches_todo:
+            current_search = item.get("search", {})
+            research_topic = item.get("research_topic", {})
+            
+            search_id = current_search.get("search_id", "unknown_search")
+            search_rationale = current_search.get('rationale', '')
+            search_results = current_search.get('results', [])
+
+            status = f"Processing search: {search_id}"
+            logger.info(status)
+            try:
+                search_ranker_result = await executor.ainvoke({
+                    "input": {
+                        "research_topic": research_topic,
+                        "search_results": search_results,
+                        "search_rationale": search_rationale
+                    }
+                })
+                raw_search_ranker_result = search_ranker_result.get('output', '')
+                status = f"Curator agent for search {search_id} completed. Raw output: {raw_search_ranker_result}"
+                logger.info(status)
+                
+                try:
+                    json_output = extract_and_clean_json(raw_search_ranker_result)
+                    ranked_urls = json_output.get("ranked_urls", [])
+                    approved_urls = [url['url'] for url in ranked_urls if url.get('status') == 'approved']
+                    curator_urls_for_ingestion.extend(approved_urls)
+                    status = f"Successfully parsed ranked URLs for search {search_id}: {len(approved_urls)} approved."
+                    logger.info(status)
+                except Exception as e:
+                    status = f"Failed to parse ranked URLs for search {search_id}: {e}"
+                    logger.error(status, exc_info=True)
+                    continue
+
+                status = f"Preparing to update report for search {search_id} with {len(approved_urls)} URLs."
+                logger.info(status)
+                try:
+                    tool_input = {
+                        "curator_report_id": report_id,
+                        "job": "urls_for_ingestion",
+                        "results": approved_urls
+                    }
+                    update_curator_report(tool_input)
+                    status = f"Updated report for search {search_id} with {len(approved_urls)} URLs."
+                    logger.info(status)
+                except Exception as e:
+                    status = f"Failed to update report for search {search_id}: {e}"
+                    logger.error(status, exc_info=True)
+                    continue
+
+                status = f"Successfully updated report for search {search_id} with {len(approved_urls)} URLs."
+                logger.info(status)
+
+            except Exception as e:
+                status = f"Curator agent search ranking for search {search_id} failed: {e}"
+                logger.error(status, exc_info=True)
+                continue
+
+    status = f"Successfully completed ranking of all searches for report: {report_id}"
+    logger.info(status)
+
+    # Create the specialized agent for url ingestion
+    ingester_prompt = ChatPromptTemplate.from_template(open("prompts/ingester_prompt.txt", "r").read())
+    ingester_tools = [t for t in state['mcp_tools'] if t.name in ["fetch", "documents_upload_file", "documents_upload_files", "documents_insert_text", "documents_pipeline_status"]]
+    status = f"Attempting to invoke url ingestion agent executor with tools: {ingester_tools}"
+    logger.info(status)
+    try:
+        agent_runnable = create_openai_tools_agent(state['model'], ingester_tools, ingester_prompt)
+        executor = AgentExecutor(agent=agent_runnable, tools=ingester_tools, verbose=True)
+    except Exception as e:
+        status = f"Failed to create url ingestion agent executor: {e}"
+        logger.error(status, exc_info=True)
+        return {"status": status}
+
+   # Main control for url ingestion
+    task = "Ingest the URLs in `{urls_for_ingestion}` and return the ingestion status for each URL."
+    status = f"Attempting to run agent executor for url ingestion"
+    logger.info(status)
+    try:
+        # The executor handles the entire loop of tool calls and reasoning.
+        ingestion_result = await executor.ainvoke({
+            "input": task,
+            "urls_for_ingestion": curator_urls_for_ingestion
+        })
+        raw_ingestion_result = ingestion_result.get('output', '')
+        status = f"Curator agent ingestion for report {report_id} completed.\nRaw output: {raw_ingestion_result}"
+        logger.info(status)
+        
+        try:
+            json_output = extract_and_clean_json(raw_ingestion_result)
+            curator_url_ingestion_status = json_output.get("url_ingestion_status", [])
+            status = f"Successfully parsed URL ingestion status: {curator_url_ingestion_status}."
+            logger.info(status)
+        except Exception as e:
+            status = f"Failed to parse URL ingestion status: {e}"
+            logger.error(status, exc_info=True)
+            return {"status": status}
+
+        status = f"Preparing to update report: {report_id} with ingestion status for {len(curator_url_ingestion_status)} URLs."
+        logger.info(status)
+        try:
+            tool_input = {
+                "curator_report_id": report_id,
+                "job": "url_ingestion_status",
+                "results": curator_url_ingestion_status
+            }
+            update_curator_report(tool_input)
+            status = f"Updated report {report_id} with ingestions status for {len(curator_url_ingestion_status)} URLs."
+            logger.info(status)
+        except Exception as e:
+            status = f"Failed to update report {report_id}: {e}"
+            logger.error(status, exc_info=True)
+            return {"status": status}
+
+        status = f"Successfully updated report {report_id} with {len(curator_url_ingestion_status)} URLs."
+        logger.info(status)
+
+    except Exception as e:
+        final_status = f"Curator agent failed to run ingestion of ranked URLs: {e}"
+        logger.error(final_status, exc_info=True)
+    
+    if not final_status:
+        final_status = f"Curator successfully ranked and ingested URLs, generating curator report summary written to file `state/curator_report.json`"
+        logger.info(status)
+
+    return {
+        "status": final_status,
+        "curator_report_id": report_id,
+        "curator_urls_for_ingestion": curator_urls_for_ingestion,
+        "curator_url_ingestion_status": curator_url_ingestion_status
+    }

참고/knowledge_agent-main/sub_agents/fixer.py

@@ -0,0 +1,60 @@
+# sub_agents.py
+from langchain.agents import create_openai_tools_agent, AgentExecutor
+from langchain_core.prompts import ChatPromptTemplate
+from langchain_core.tools import tool, ToolException
+from langchain_core.messages import AIMessage
+import json
+import os
+import re
+from state import AgentState
+from tools import human_approval
+from db_utils import load_latest_report, extract_and_clean_json
+from terminal_utils import print_colorful_break
+
+
+async def fixer_agent_node(state: AgentState):
+    print_colorful_break("FIXER")
+    logger = state['logger']
+    logger.info("--- Running Fixer Agent ---")
+    all_tools = state['mcp_tools']
+    model = state['model']
+    timestamp = state['timestamp']
+    
+    fixer_tools = [t for t in all_tools if t.name in ["graph_update_entity", "documents_delete_entity", "graph_update_relation", "documents_delete_relation", "graph_entity_exists"]] + [load_latest_report, human_approval]
+    fixer_prompt = '''Your goal is to correct data quality issues.
+1.  **Load Auditor's Report**: Load `auditor_report.json`.
+2.  **Create a Plan**: Create a step-by-step plan to correct the issues.
+3.  **Get Human Approval**: Use `human_approval` to get your plan approved by calling the tool with the plan.
+4.  **Execute**: Execute the approved plan.
+5.  **Save Report**: Save a report of your actions to `fixer_report.json`.'''
+
+    prompt = ChatPromptTemplate.from_template(fixer_prompt)
+    agent_executor = create_openai_tools_agent(model, fixer_tools, prompt)
+
+    task_input = "Your task is to fix issues from the auditor's report. Begin now."
+
+    result = await agent_executor.ainvoke({"input": task_input, "timestamp": timestamp})
+
+    logger.info(f"Fixer Agent finished with output: {result['output']}")
+    return {"messages": state['messages'] + [AIMessage(content=result['output'])]}
+
+def save_fixer_report_node(state: AgentState):
+    """Saves the final report from the last AI message."""
+    logger = state['logger']
+    final_message_from_agent = state['messages'][-1]
+
+    status = f"--- Saving Fixer Report ---\n{final_message_from_agent.content}"
+    logger.info(status)
+
+    try:
+        report_json = extract_and_clean_json(final_message_from_agent.content)
+        if 'report_id' not in report_json:
+            report_json['report_id'] = state.get('fixer_report_id', 'unknown_id')
+        save_fixer_report({"fixer_report": json.dumps(report_json)})
+        status = f"Successfully saved fixer report with ID {report_json.get('report_id')}"
+        logger.info(status)
+    except (ValueError, KeyError) as e:
+        status = f"Error processing or saving fixer report: {e}"
+        logger.error(status, exc_info=True)
+
+    return {"messages": state['messages'] + [AIMessage(content=status)]}

참고/knowledge_agent-main/sub_agents/researcher.py

@@ -0,0 +1,296 @@
+# sub_agents/researcher.py
+from langchain.agents import create_openai_tools_agent, AgentExecutor
+from langchain_core.prompts import ChatPromptTemplate
+from state import AgentState
+from db_utils import initialize_researcher, update_researcher_report, extract_and_clean_json, get_document_object, update_document_object
+from tools import process_url
+from utils import filter_content_for_summarization
+from terminal_utils import print_colorful_break
+
+async def researcher_agent_node(state: AgentState):
+    """The main node for the researcher workflow."""
+    print_colorful_break("RESEARCHER")
+    logger = state['logger']
+    report_id = state.get("researcher_report_id")
+    gaps_todo = state.get("researcher_gaps_todo", [])
+    gaps_complete = state.get("researcher_gaps_complete", [])
+    final_status = None
+
+    if not report_id:
+        try:
+            init_result = initialize_researcher(state['timestamp'])
+            report_id = init_result.get("researcher_report_id")
+            gaps_todo = init_result.get("researcher_gaps_todo")
+            gaps_complete = []
+            status = f"--- Initialized researcher state: {init_result} ---"
+            logger.info(status)
+        except Exception as e:
+            status = f"Failed to initialize researcher state: {e}"
+            logger.error(status)
+            return {"status": status}
+
+    # Define Planner Agent
+    with open("prompts/planner_prompt.txt", "r") as f:
+        planner_prompt_template = f.read()
+    planner_prompt = ChatPromptTemplate.from_template(planner_prompt_template)
+    try:
+        planner_agent_runnable = create_openai_tools_agent(state['model'], [], planner_prompt)
+        planner_executor = AgentExecutor(agent=planner_agent_runnable, tools=[], verbose=True)
+    except Exception as e:
+        status = f"Failed to create planner agent executor: {e}"
+        logger.error(status)
+        return {"status": status}
+
+    # Define Refiner Agent
+    with open("prompts/refiner_prompt.txt", "r") as f:
+        refiner_prompt_template = f.read()
+    refiner_prompt = ChatPromptTemplate.from_template(refiner_prompt_template)
+    try:
+        refiner_agent_runnable = create_openai_tools_agent(state['model'], [], refiner_prompt)
+        refiner_executor = AgentExecutor(agent=refiner_agent_runnable, tools=[], verbose=True)
+    except Exception as e:
+        status = f"Failed to create refiner agent executor: {e}"
+        logger.error(status)
+        return {"status": status}
+
+    # Define Summarizer Agent
+    with open("prompts/summarizer_prompt.txt", "r") as f:
+        summarizer_prompt_template = f.read()
+    summarizer_prompt = ChatPromptTemplate.from_template(summarizer_prompt_template)
+    try:
+        summarizer_agent_runnable = create_openai_tools_agent(state['model'], [], summarizer_prompt)
+        summarizer_executor = AgentExecutor(agent=summarizer_agent_runnable, tools=[], verbose=True)
+    except Exception as e:
+        status = f"Failed to create summarizer agent executor: {e}"
+        logger.error(status)
+        return {"status": status}
+
+    # Get the tools
+    google_search_tool = next((tool for tool in state['mcp_tools'] if tool.name == 'google_search'), None)
+    if not google_search_tool:
+        status = "google_search tool not found."
+        logger.error(status)
+        return {"status": status}
+    
+
+    # Main control loop
+    try:
+        if isinstance(gaps_todo, list) and gaps_todo:
+            for current_gap in list(gaps_todo):
+                gap_id = current_gap['gap_id']
+                research_topic = current_gap['research_topic']
+                all_searches_for_gap = []
+
+                research_topic_title = research_topic.get('title', 'No Title')
+                status = f"Starting research for gap: {gap_id}, research topic: {research_topic_title}"
+                logger.info(status)
+
+                try:
+                    # 1. Planning Step
+                    status = f"Invoking planner for gap {gap_id}."
+                    logger.info(status)
+                    planner_result = await planner_executor.ainvoke({"input": research_topic})
+                    planner_output = extract_and_clean_json(planner_result.get("output", ""))
+                    planned_searches = planner_output.get("searches", [])
+                    status = f"Planner for gap {gap_id} returned {len(planned_searches)} searches."
+                    logger.info(status)
+
+                    # 2. Initial Execution Step
+                    for planned_search in planned_searches:
+                        query = planned_search.get("query")
+                        rationale = planned_search.get("rationale")
+                        parameters = planned_search.get("parameters", {})
+                        search_id = planned_search.get("search_id")
+                        if not query:
+                            continue
+                        
+                        if 'query' not in parameters:
+                            parameters['query'] = query
+
+                        try:
+                            status = f"Executing search for gap {gap_id}, with parameters: {parameters}"
+                            logger.info(status)
+                            
+                            raw_search_results = await google_search_tool.arun(parameters)
+                            search_results = extract_and_clean_json(raw_search_results)
+
+                            for i, result in enumerate(search_results):
+                                url = result.get('url')
+                                if url:
+                                    logger.info(f"Attempting document store initialzation for URL: {url}")
+                                    try:
+                                        url_id, url_status = await process_url(url, logger)                                        
+                                    except Exception as e:
+                                        status = f"Error processing URL {url}: {e}"
+                                        logger.error(status, exc_info=True)
+                                        continue                                    
+                                    search_results[i]['url_id'] = url_id
+                                    if url_status == "new":
+                                        status=f"New URL {url} added to the database with ID: {url_id}."
+                                    else:
+                                        status = f"URL {url} already exists in the database with ID: {url_id}."
+                                    logger.info(status)
+                                    
+                            search_object = {
+                                "search_id": search_id,
+                                "rationale": rationale,
+                                "parameters": parameters,
+                                "results": search_results
+                            }
+                            all_searches_for_gap.append(search_object)
+                            
+                            status = f"Search for gap {gap_id} finished for query: '{query}'"
+                            logger.info(status)
+                            
+                        except Exception as e:
+                            status = f"Search for gap {gap_id}, query '{query}' failed: {e}"
+                            logger.error(status)
+                            continue
+
+                    # 3. Refinement Step
+                    status = f"Invoking refiner for gap {gap_id}."
+                    logger.info(status)
+                    refiner_input = {"research_topic": research_topic, "search_results": all_searches_for_gap}
+                    refiner_result = await refiner_executor.ainvoke({"input": refiner_input})
+                    refiner_output = extract_and_clean_json(refiner_result.get("output", ""))
+                    
+                    status_check = ""
+                    if isinstance(refiner_output, dict):
+                        status_check = refiner_output.get("status", "").lower()
+                    elif isinstance(refiner_output, str):
+                        if "insufficient" in refiner_output.lower():
+                            status_check = "insufficient"
+
+                    if status_check == "insufficient":
+                        refined_searches = []
+                        if isinstance(refiner_output, dict):
+                            refined_searches = refiner_output.get("searches", [])
+                        status = f"Refiner for gap {gap_id} returned {len(refined_searches)} new searches."
+                        logger.info(status)
+
+                        # 4. Refined Execution Step
+                        for refined_search in refined_searches:
+                            query = refined_search.get("query")
+                            rationale = refined_search.get("rationale")
+                            parameters = refined_search.get("parameters", {})
+                            search_id = refined_search.get("search_id")
+                            if not query:
+                                continue
+
+                            if 'query' not in parameters:
+                                parameters['query'] = query
+                            
+                            try:
+                                status = f"Executing refined search for gap {gap_id}, with parameters: {parameters}"
+                                logger.info(status)
+
+                                raw_search_results = await google_search_tool.arun(parameters)
+                                search_results = extract_and_clean_json(raw_search_results)
+
+                                for i, result in enumerate(search_results):
+                                    url = result.get('url')
+                                    if url:
+                                        logger.info(f"Attempting document store initialzation for URL: {url}")
+                                        try:
+                                            url_id, url_status = await process_url(url, logger)                                        
+                                        except Exception as e:
+                                            status = f"Error processing URL {url}: {e}"
+                                            logger.error(status, exc_info=True)
+                                            continue                                    
+                                        search_results[i]['url_id'] = url_id
+                                        if url_status == "new":
+                                            status=f"New URL {url} added to the database with ID: {url_id}."
+                                        else:
+                                            status = f"URL {url} already exists in the database with ID: {url_id}."
+                                        logger.info(status)
+
+                                search_object = {
+                                    "search_id": search_id,
+                                    "rationale": rationale,
+                                    "parameters": parameters,
+                                    "results": search_results
+                                }
+                                all_searches_for_gap.append(search_object)
+
+                                status = f"Refined search for gap {gap_id} finished for query: '{query}'"
+                                logger.info(status)
+                            except Exception as e:
+                                status = f"Refined search for gap {gap_id}, query '{query}' failed: {e}"
+                                logger.error(status)
+                                continue
+                    else:
+                        status = f"Refiner for gap {gap_id} deemed results sufficient."
+                        logger.info(status)
+
+                    # 5. Summarization Step
+                    status = f"Starting summarization for gap {gap_id}."
+                    logger.info(status)
+                    for search in all_searches_for_gap:
+                        for i, result in enumerate(search.get('results', [])):
+                            url = result.get('url')
+                            url_id = result.get('url_id')
+                            url_summary = get_document_object(url_id, type="summary")
+                            if url_summary:
+                                logger.info(f"Skipping summarization for url_id: {url_id}, summary already exists.")
+                                continue
+                            else:
+                                markdown_content = get_document_object(url_id, type="markdown_content")
+                                if not markdown_content or markdown_content.startswith("[MARKDOWN_GENERATION_FAILED"):
+                                    logger.info(f"Skipping summarization for url_id: {url_id}, no valid markdown content available.")
+                                    continue
+
+                                logger.info(f"Attempting summary for url_id: {url_id}")                               
+                                try:
+                                    filtered_content = filter_content_for_summarization(markdown_content)
+                                    summarizer_result = await summarizer_executor.ainvoke({"input": filtered_content})
+                                    summary_output = extract_and_clean_json(summarizer_result.get("output", ""))
+                                    
+                                    if isinstance(summary_output, dict):
+                                        summary = summary_output.get('summary')
+                                    else:
+                                        summary = str(summary_output)
+
+                                    update_document_object(url_id, type="summary", object=summary)
+                                    status = f"Successfully summarized and updated document for url_id: {url_id}"
+                                    logger.info(status)
+                                except Exception as e:
+                                    status = f"Error summarizing url_id {url_id}: {e}"
+                                    logger.error(status, exc_info=True)
+                                    continue
+
+                    # 6. Update Step
+                    status = f"Preparing to update report for gap {gap_id} with {len(all_searches_for_gap)} searches."
+                    logger.info(status)
+                    try:
+                        update_researcher_report(report_id, gap_id, all_searches_for_gap)
+                        gaps_complete.append(gap_id)
+                        gaps_todo = [g for g in gaps_todo if g.get("gap_id") != gap_id]
+                        status = f"Updated researcher report for gap: {gap_id}"
+                        logger.info(status)
+                    except Exception as e:
+                        status = f"Error updating report for gap {gap_id}: {e}"
+                        logger.error(status, exc_info=True)
+                        continue
+
+                    status = f"--- Successfully completed research and report writing for gap: {gap_id} ---"
+                    logger.info(status)
+
+                except Exception as e:
+                    status = f"An unexpected error occurred while processing gap {gap_id}: {e}"
+                    logger.error(status, exc_info=True)
+                    continue
+    
+    except Exception as e:
+        final_status = f"Main loop failed: {e}"
+        logger.error(final_status, exc_info=True)
+
+    if not final_status:
+        final_status = f"Successfully and incrementally completed researcher report with ID {report_id} and wrote report to DB."
+        logger.info(final_status)
+
+    return {
+        "status": final_status,
+        "researcher_report_id": report_id,
+        "researcher_gaps_todo": gaps_todo,
+        "researcher_gaps_complete": gaps_complete
+    }