{ "nbformat": 4, "nbformat_minor": 0, "metadata": { "colab": { "provenance": [], "authorship_tag": "ABX9TyOe5BsaH0aplNCjknkFtnjg", "include_colab_link": true }, "kernelspec": { "name": "python3", "display_name": "Python 3" }, "language_info": { "name": "python" } }, "cells": [ { "cell_type": "markdown", "metadata": { "id": "view-in-github", "colab_type": "text" }, "source": [ "\"Open" ] }, { "cell_type": "code", "source": [ "!pip install nest_asyncio\n", "import nest_asyncio\n", "nest_asyncio.apply()" ], "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "y7cTpP9rDZW-", "outputId": "5a443ad2-7a8d-4fef-f315-12108c28f1a2" }, "execution_count": null, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "Requirement already satisfied: nest_asyncio in /usr/local/lib/python3.11/dist-packages (1.6.0)\n" ] } ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "GJTo96a7DGUz" }, "outputs": [], "source": [ "import asyncio\n", "import aiohttp\n", "import json\n", "\n", "# =======================\n", "# Configuration Constants\n", "# =======================\n", "OPENROUTER_API_KEY = \"REDACTED\" # Replace with your OpenRouter API key\n", "SERPAPI_API_KEY = \"REDACTED\" # Replace with your SERPAPI API key\n", "JINA_API_KEY = \"REDACTED\" # Replace with your JINA API key\n", "\n", "# Endpoints\n", "OPENROUTER_URL = \"https://openrouter.ai/api/v1/chat/completions\"\n", "SERPAPI_URL = \"https://serpapi.com/search\"\n", "JINA_BASE_URL = \"https://r.jina.ai/\"\n", "\n", "# Default LLM model (can be changed if desired)\n", "DEFAULT_MODEL = \"anthropic/claude-3.5-haiku\"\n", "\n", "\n", "# ============================\n", "# Asynchronous Helper Functions\n", "# ============================\n", "\n", "async def call_openrouter_async(session, messages, model=DEFAULT_MODEL):\n", " \"\"\"\n", " Asynchronously call the OpenRouter chat completion API with the provided messages.\n", " Returns the content of the assistant’s reply.\n", " \"\"\"\n", " headers = {\n", " \"Authorization\": f\"Bearer {OPENROUTER_API_KEY}\",\n", " \"X-Title\": \"OpenDeepResearcher, by Matt Shumer\",\n", " \"Content-Type\": \"application/json\"\n", " }\n", " payload = {\n", " \"model\": model,\n", " \"messages\": messages\n", " }\n", " try:\n", " async with session.post(OPENROUTER_URL, headers=headers, json=payload) as resp:\n", " if resp.status == 200:\n", " result = await resp.json()\n", " try:\n", " return result['choices'][0]['message']['content']\n", " except (KeyError, IndexError) as e:\n", " print(\"Unexpected OpenRouter response structure:\", result)\n", " return None\n", " else:\n", " text = await resp.text()\n", " print(f\"OpenRouter API error: {resp.status} - {text}\")\n", " return None\n", " except Exception as e:\n", " print(\"Error calling OpenRouter:\", e)\n", " return None\n", "\n", "\n", "async def generate_search_queries_async(session, user_query):\n", " \"\"\"\n", " Ask the LLM to produce up to four precise search queries (in Python list format)\n", " based on the user’s query.\n", " \"\"\"\n", " prompt = (\n", " \"You are an expert research assistant. Given the user's query, generate up to four distinct, \"\n", " \"precise search queries that would help gather comprehensive information on the topic. \"\n", " \"Return only a Python list of strings, for example: ['query1', 'query2', 'query3'].\"\n", " )\n", " messages = [\n", " {\"role\": \"system\", \"content\": \"You are a helpful and precise research assistant.\"},\n", " {\"role\": \"user\", \"content\": f\"User Query: {user_query}\\n\\n{prompt}\"}\n", " ]\n", " response = await call_openrouter_async(session, messages)\n", " if response:\n", " try:\n", " # Expect exactly a Python list (e.g., \"['query1', 'query2']\")\n", " search_queries = eval(response)\n", " if isinstance(search_queries, list):\n", " return search_queries\n", " else:\n", " print(\"LLM did not return a list. Response:\", response)\n", " return []\n", " except Exception as e:\n", " print(\"Error parsing search queries:\", e, \"\\nResponse:\", response)\n", " return []\n", " return []\n", "\n", "\n", "async def perform_search_async(session, query):\n", " \"\"\"\n", " Asynchronously perform a Google search using SERPAPI for the given query.\n", " Returns a list of result URLs.\n", " \"\"\"\n", " params = {\n", " \"q\": query,\n", " \"api_key\": SERPAPI_API_KEY,\n", " \"engine\": \"google\"\n", " }\n", " try:\n", " async with session.get(SERPAPI_URL, params=params) as resp:\n", " if resp.status == 200:\n", " results = await resp.json()\n", " if \"organic_results\" in results:\n", " links = [item.get(\"link\") for item in results[\"organic_results\"] if \"link\" in item]\n", " return links\n", " else:\n", " print(\"No organic results in SERPAPI response.\")\n", " return []\n", " else:\n", " text = await resp.text()\n", " print(f\"SERPAPI error: {resp.status} - {text}\")\n", " return []\n", " except Exception as e:\n", " print(\"Error performing SERPAPI search:\", e)\n", " return []\n", "\n", "\n", "async def fetch_webpage_text_async(session, url):\n", " \"\"\"\n", " Asynchronously retrieve the text content of a webpage using Jina.\n", " The URL is appended to the Jina endpoint.\n", " \"\"\"\n", " full_url = f\"{JINA_BASE_URL}{url}\"\n", " headers = {\n", " \"Authorization\": f\"Bearer {JINA_API_KEY}\"\n", " }\n", " try:\n", " async with session.get(full_url, headers=headers) as resp:\n", " if resp.status == 200:\n", " return await resp.text()\n", " else:\n", " text = await resp.text()\n", " print(f\"Jina fetch error for {url}: {resp.status} - {text}\")\n", " return \"\"\n", " except Exception as e:\n", " print(\"Error fetching webpage text with Jina:\", e)\n", " return \"\"\n", "\n", "\n", "async def is_page_useful_async(session, user_query, page_text):\n", " \"\"\"\n", " Ask the LLM if the provided webpage content is useful for answering the user's query.\n", " The LLM must reply with exactly \"Yes\" or \"No\".\n", " \"\"\"\n", " prompt = (\n", " \"You are a critical research evaluator. Given the user's query and the content of a webpage, \"\n", " \"determine if the webpage contains information relevant and useful for addressing the query. \"\n", " \"Respond with exactly one word: 'Yes' if the page is useful, or 'No' if it is not. Do not include any extra text.\"\n", " )\n", " messages = [\n", " {\"role\": \"system\", \"content\": \"You are a strict and concise evaluator of research relevance.\"},\n", " {\"role\": \"user\", \"content\": f\"User Query: {user_query}\\n\\nWebpage Content (first 20000 characters):\\n{page_text[:20000]}\\n\\n{prompt}\"}\n", " ]\n", " response = await call_openrouter_async(session, messages)\n", " if response:\n", " answer = response.strip()\n", " if answer in [\"Yes\", \"No\"]:\n", " return answer\n", " else:\n", " # Fallback: try to extract Yes/No from the response.\n", " if \"Yes\" in answer:\n", " return \"Yes\"\n", " elif \"No\" in answer:\n", " return \"No\"\n", " return \"No\"\n", "\n", "\n", "async def extract_relevant_context_async(session, user_query, search_query, page_text):\n", " \"\"\"\n", " Given the original query, the search query used, and the page content,\n", " have the LLM extract all information relevant for answering the query.\n", " \"\"\"\n", " prompt = (\n", " \"You are an expert information extractor. Given the user's query, the search query that led to this page, \"\n", " \"and the webpage content, extract all pieces of information that are relevant to answering the user's query. \"\n", " \"Return only the relevant context as plain text without commentary.\"\n", " )\n", " messages = [\n", " {\"role\": \"system\", \"content\": \"You are an expert in extracting and summarizing relevant information.\"},\n", " {\"role\": \"user\", \"content\": f\"User Query: {user_query}\\nSearch Query: {search_query}\\n\\nWebpage Content (first 20000 characters):\\n{page_text[:20000]}\\n\\n{prompt}\"}\n", " ]\n", " response = await call_openrouter_async(session, messages)\n", " if response:\n", " return response.strip()\n", " return \"\"\n", "\n", "\n", "async def get_new_search_queries_async(session, user_query, previous_search_queries, all_contexts):\n", " \"\"\"\n", " Based on the original query, the previously used search queries, and all the extracted contexts,\n", " ask the LLM whether additional search queries are needed. If yes, return a Python list of up to four queries;\n", " if the LLM thinks research is complete, it should return \"\".\n", " \"\"\"\n", " context_combined = \"\\n\".join(all_contexts)\n", " prompt = (\n", " \"You are an analytical research assistant. Based on the original query, the search queries performed so far, \"\n", " \"and the extracted contexts from webpages, determine if further research is needed. \"\n", " \"If further research is needed, provide up to four new search queries as a Python list (for example, \"\n", " \"['new query1', 'new query2']). If you believe no further research is needed, respond with exactly .\"\n", " \"\\nOutput only a Python list or the token without any additional text.\"\n", " )\n", " messages = [\n", " {\"role\": \"system\", \"content\": \"You are a systematic research planner.\"},\n", " {\"role\": \"user\", \"content\": f\"User Query: {user_query}\\nPrevious Search Queries: {previous_search_queries}\\n\\nExtracted Relevant Contexts:\\n{context_combined}\\n\\n{prompt}\"}\n", " ]\n", " response = await call_openrouter_async(session, messages)\n", " if response:\n", " cleaned = response.strip()\n", " if cleaned == \"\":\n", " return \"\"\n", " try:\n", " new_queries = eval(cleaned)\n", " if isinstance(new_queries, list):\n", " return new_queries\n", " else:\n", " print(\"LLM did not return a list for new search queries. Response:\", response)\n", " return []\n", " except Exception as e:\n", " print(\"Error parsing new search queries:\", e, \"\\nResponse:\", response)\n", " return []\n", " return []\n", "\n", "\n", "async def generate_final_report_async(session, user_query, all_contexts):\n", " \"\"\"\n", " Generate the final comprehensive report using all gathered contexts.\n", " \"\"\"\n", " context_combined = \"\\n\".join(all_contexts)\n", " prompt = (\n", " \"You are an expert researcher and report writer. Based on the gathered contexts below and the original query, \"\n", " \"write a comprehensive, well-structured, and detailed report that addresses the query thoroughly. \"\n", " \"Include all relevant insights and conclusions without extraneous commentary.\"\n", " )\n", " messages = [\n", " {\"role\": \"system\", \"content\": \"You are a skilled report writer.\"},\n", " {\"role\": \"user\", \"content\": f\"User Query: {user_query}\\n\\nGathered Relevant Contexts:\\n{context_combined}\\n\\n{prompt}\"}\n", " ]\n", " report = await call_openrouter_async(session, messages)\n", " return report\n", "\n", "\n", "async def process_link(session, link, user_query, search_query):\n", " \"\"\"\n", " Process a single link: fetch its content, judge its usefulness, and if useful, extract the relevant context.\n", " \"\"\"\n", " print(f\"Fetching content from: {link}\")\n", " page_text = await fetch_webpage_text_async(session, link)\n", " if not page_text:\n", " return None\n", " usefulness = await is_page_useful_async(session, user_query, page_text)\n", " print(f\"Page usefulness for {link}: {usefulness}\")\n", " if usefulness == \"Yes\":\n", " context = await extract_relevant_context_async(session, user_query, search_query, page_text)\n", " if context:\n", " print(f\"Extracted context from {link} (first 200 chars): {context[:200]}\")\n", " return context\n", " return None\n", "\n", "\n", "# =========================\n", "# Main Asynchronous Routine\n", "# =========================\n", "\n", "async def async_main():\n", " user_query = input(\"Enter your research query/topic: \").strip()\n", " iter_limit_input = input(\"Enter maximum number of iterations (default 10): \").strip()\n", " iteration_limit = int(iter_limit_input) if iter_limit_input.isdigit() else 10\n", "\n", " aggregated_contexts = [] # All useful contexts from every iteration\n", " all_search_queries = [] # Every search query used across iterations\n", " iteration = 0\n", "\n", " async with aiohttp.ClientSession() as session:\n", " # ----- INITIAL SEARCH QUERIES -----\n", " new_search_queries = await generate_search_queries_async(session, user_query)\n", " if not new_search_queries:\n", " print(\"No search queries were generated by the LLM. Exiting.\")\n", " return\n", " all_search_queries.extend(new_search_queries)\n", "\n", " # ----- ITERATIVE RESEARCH LOOP -----\n", " while iteration < iteration_limit:\n", " print(f\"\\n=== Iteration {iteration + 1} ===\")\n", " iteration_contexts = []\n", "\n", " # For each search query, perform SERPAPI searches concurrently.\n", " search_tasks = [perform_search_async(session, query) for query in new_search_queries]\n", " search_results = await asyncio.gather(*search_tasks)\n", "\n", " # Aggregate all unique links from all search queries of this iteration.\n", " # Map each unique link to the search query that produced it.\n", " unique_links = {}\n", " for idx, links in enumerate(search_results):\n", " query = new_search_queries[idx]\n", " for link in links:\n", " if link not in unique_links:\n", " unique_links[link] = query\n", "\n", " print(f\"Aggregated {len(unique_links)} unique links from this iteration.\")\n", "\n", " # Process each link concurrently: fetch, judge, and extract context.\n", " link_tasks = [\n", " process_link(session, link, user_query, unique_links[link])\n", " for link in unique_links\n", " ]\n", " link_results = await asyncio.gather(*link_tasks)\n", "\n", " # Collect non-None contexts.\n", " for res in link_results:\n", " if res:\n", " iteration_contexts.append(res)\n", "\n", " if iteration_contexts:\n", " aggregated_contexts.extend(iteration_contexts)\n", " else:\n", " print(\"No useful contexts were found in this iteration.\")\n", "\n", " # ----- ASK THE LLM IF MORE SEARCHES ARE NEEDED -----\n", " new_search_queries = await get_new_search_queries_async(session, user_query, all_search_queries, aggregated_contexts)\n", " if new_search_queries == \"\":\n", " print(\"LLM indicated that no further research is needed.\")\n", " break\n", " elif new_search_queries:\n", " print(\"LLM provided new search queries:\", new_search_queries)\n", " all_search_queries.extend(new_search_queries)\n", " else:\n", " print(\"LLM did not provide any new search queries. Ending the loop.\")\n", " break\n", "\n", " iteration += 1\n", "\n", " # ----- FINAL REPORT -----\n", " print(\"\\nGenerating final report...\")\n", " final_report = await generate_final_report_async(session, user_query, aggregated_contexts)\n", " print(\"\\n==== FINAL REPORT ====\\n\")\n", " print(final_report)\n", "\n", "\n", "def main():\n", " asyncio.run(async_main())\n", "\n", "\n", "if __name__ == \"__main__\":\n", " main()\n" ] }, { "cell_type": "code", "source": [], "metadata": { "id": "46Q5XpapDJZT" }, "execution_count": null, "outputs": [] } ] }