참고소스 수정본
This commit is contained in:
LASTA_DEV01\lasta
10
참고/knowledge_agent-main/.env.example
Normal file
10
참고/knowledge_agent-main/.env.example
Normal file
@@ -0,0 +1,10 @@
|
||||
# .env
|
||||
# This should match the configuration in your lightrag_mcp server
|
||||
LIGHTRAG_BASE_URL=http://localhost:9621
|
||||
|
||||
# PostgreSQL configuration
|
||||
DATABASE_URL="postgresql://user:password@localhost:5432/knowledge_agent"
|
||||
|
||||
# Google API credentials
|
||||
GOOGLE_API_KEY=<YOUR_API_KEY>
|
||||
GOOGLE_CSE_ID=<YOUR_CSE_ID>
|
||||
219
참고/knowledge_agent-main/.gitignore
vendored
Normal file
219
참고/knowledge_agent-main/.gitignore
vendored
Normal file
@@ -0,0 +1,219 @@
|
||||
# Byte-compiled / optimized / DLL files
|
||||
__pycache__/
|
||||
*.py[codz]
|
||||
*$py.class
|
||||
|
||||
# C extensions
|
||||
*.so
|
||||
|
||||
# Distribution / packaging
|
||||
.Python
|
||||
build/
|
||||
develop-eggs/
|
||||
dist/
|
||||
downloads/
|
||||
eggs/
|
||||
.eggs/
|
||||
lib/
|
||||
lib64/
|
||||
parts/
|
||||
sdist/
|
||||
var/
|
||||
wheels/
|
||||
share/python-wheels/
|
||||
*.egg-info/
|
||||
.installed.cfg
|
||||
*.egg
|
||||
MANIFEST
|
||||
|
||||
# PyInstaller
|
||||
# Usually these files are written by a python script from a template
|
||||
# before PyInstaller builds the exe, so as to inject date/other infos into it.
|
||||
*.manifest
|
||||
*.spec
|
||||
|
||||
# Installer logs
|
||||
pip-log.txt
|
||||
pip-delete-this-directory.txt
|
||||
|
||||
# Unit test / coverage reports
|
||||
htmlcov/
|
||||
.tox/
|
||||
.nox/
|
||||
.coverage
|
||||
.coverage.*
|
||||
.cache
|
||||
nosetests.xml
|
||||
coverage.xml
|
||||
*.cover
|
||||
*.py.cover
|
||||
.hypothesis/
|
||||
.pytest_cache/
|
||||
cover/
|
||||
|
||||
# Translations
|
||||
*.mo
|
||||
*.pot
|
||||
|
||||
# Django stuff:
|
||||
*.log
|
||||
local_settings.py
|
||||
db.sqlite3
|
||||
db.sqlite3-journal
|
||||
|
||||
# Flask stuff:
|
||||
instance/
|
||||
.webassets-cache
|
||||
|
||||
# Scrapy stuff:
|
||||
.scrapy
|
||||
|
||||
# Sphinx documentation
|
||||
docs/_build/
|
||||
|
||||
# PyBuilder
|
||||
.pybuilder/
|
||||
target/
|
||||
|
||||
# Jupyter Notebook
|
||||
.ipynb_checkpoints
|
||||
|
||||
# IPython
|
||||
profile_default/
|
||||
ipython_config.py
|
||||
|
||||
# pyenv
|
||||
# For a library or package, you might want to ignore these files since the code is
|
||||
# intended to run in multiple environments; otherwise, check them in:
|
||||
# .python-version
|
||||
|
||||
# pipenv
|
||||
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
|
||||
# However, in case of collaboration, if having platform-specific dependencies or dependencies
|
||||
# having no cross-platform support, pipenv may install dependencies that don't work, or not
|
||||
# install all needed dependencies.
|
||||
#Pipfile.lock
|
||||
|
||||
# UV
|
||||
# Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
|
||||
# This is especially recommended for binary packages to ensure reproducibility, and is more
|
||||
# commonly ignored for libraries.
|
||||
uv.lock
|
||||
|
||||
# poetry
|
||||
# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
|
||||
# This is especially recommended for binary packages to ensure reproducibility, and is more
|
||||
# commonly ignored for libraries.
|
||||
# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
|
||||
#poetry.lock
|
||||
#poetry.toml
|
||||
|
||||
# pdm
|
||||
# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
|
||||
# pdm recommends including project-wide configuration in pdm.toml, but excluding .pdm-python.
|
||||
# https://pdm-project.org/en/latest/usage/project/#working-with-version-control
|
||||
#pdm.lock
|
||||
#pdm.toml
|
||||
.pdm-python
|
||||
.pdm-build/
|
||||
|
||||
# pixi
|
||||
# Similar to Pipfile.lock, it is generally recommended to include pixi.lock in version control.
|
||||
#pixi.lock
|
||||
# Pixi creates a virtual environment in the .pixi directory, just like venv module creates one
|
||||
# in the .venv directory. It is recommended not to include this directory in version control.
|
||||
.pixi
|
||||
|
||||
# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
|
||||
__pypackages__/
|
||||
|
||||
# Celery stuff
|
||||
celerybeat-schedule
|
||||
celerybeat.pid
|
||||
|
||||
# SageMath parsed files
|
||||
*.sage.py
|
||||
|
||||
# Environments
|
||||
.env
|
||||
.envrc
|
||||
.venv
|
||||
env/
|
||||
venv/
|
||||
ENV/
|
||||
env.bak/
|
||||
venv.bak/
|
||||
|
||||
# Spyder project settings
|
||||
.spyderproject
|
||||
.spyproject
|
||||
|
||||
# Rope project settings
|
||||
.ropeproject
|
||||
|
||||
# mkdocs documentation
|
||||
/site
|
||||
|
||||
# mypy
|
||||
.mypy_cache/
|
||||
.dmypy.json
|
||||
dmypy.json
|
||||
|
||||
# Pyre type checker
|
||||
.pyre/
|
||||
|
||||
# pytype static type analyzer
|
||||
.pytype/
|
||||
|
||||
# Cython debug symbols
|
||||
cython_debug/
|
||||
|
||||
# PyCharm
|
||||
# JetBrains specific template is maintained in a separate JetBrains.gitignore that can
|
||||
# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
|
||||
# and can be added to the global gitignore or merged into this file. For a more nuclear
|
||||
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
|
||||
#.idea/
|
||||
|
||||
# Abstra
|
||||
# Abstra is an AI-powered process automation framework.
|
||||
# Ignore directories containing user credentials, local state, and settings.
|
||||
# Learn more at https://abstra.io/docs
|
||||
.abstra/
|
||||
|
||||
# Visual Studio Code
|
||||
# Visual Studio Code specific template is maintained in a separate VisualStudioCode.gitignore
|
||||
# that can be found at https://github.com/github/gitignore/blob/main/Global/VisualStudioCode.gitignore
|
||||
# and can be added to the global gitignore or merged into this file. However, if you prefer,
|
||||
# you could uncomment the following to ignore the entire vscode folder
|
||||
.vscode/
|
||||
|
||||
# Ruff stuff:
|
||||
.ruff_cache/
|
||||
|
||||
# PyPI configuration file
|
||||
.pypirc
|
||||
|
||||
# Cursor
|
||||
# Cursor is an AI-powered code editor. `.cursorignore` specifies files/directories to
|
||||
# exclude from AI features like autocomplete and code analysis. Recommended for sensitive data
|
||||
# refer to https://docs.cursor.com/context/ignore-files
|
||||
.cursorignore
|
||||
.cursorindexingignore
|
||||
|
||||
# Marimo
|
||||
marimo/_static/
|
||||
marimo/_lsp/
|
||||
__marimo__/
|
||||
|
||||
# trace/log files
|
||||
trace*
|
||||
verbose_log*
|
||||
terminal_output*
|
||||
logs/
|
||||
|
||||
# Gemini code assist
|
||||
.gemini/
|
||||
gemini/
|
||||
gemini*
|
||||
GEMINI*
|
||||
1
참고/knowledge_agent-main/.python-version
Normal file
1
참고/knowledge_agent-main/.python-version
Normal file
@@ -0,0 +1 @@
|
||||
3.12
|
||||
255
참고/knowledge_agent-main/README.md
Normal file
255
참고/knowledge_agent-main/README.md
Normal file
@@ -0,0 +1,255 @@
|
||||
# Knowledge Agent
|
||||
|
||||
An autonomous AI agent for intelligently updating, maintaining, and curating a [LightRAG](https://github.com/HKUDS/LightRAG) knowledge base.
|
||||
|
||||
## Table of Contents
|
||||
|
||||
- [About The Project](#about-the-project)
|
||||
- [Architecture](#architecture)
|
||||
- [Frameworks and Libraries](#frameworks-and-libraries)
|
||||
- [Agent Roles](#agent-roles)
|
||||
- [Getting Started](#getting-started)
|
||||
- [Prerequisites](#prerequisites)
|
||||
- [Installation](#installation)
|
||||
- [Usage](#usage)
|
||||
- [Workflows](#workflows)
|
||||
- [Configuration](#configuration)
|
||||
- [Prompts](#prompts)
|
||||
- [Logging](#logging)
|
||||
- [Database](#database)
|
||||
- [Workflow Details](#workflow-details)
|
||||
|
||||
## About The Project
|
||||
|
||||
This project provides a sophisticated, autonomous AI agent—the "Knowledge Agent"—that proactively maintains, expands, and curates a local LightRAG knowledge base. It transforms the knowledge base from a static repository into a living, self-improving intelligence system, ensuring the information it contains is always accurate, relevant, and up-to-date.
|
||||
|
||||
The Knowledge Agent is designed to solve the challenges of maintaining a static knowledge base:
|
||||
|
||||
- **Staleness:** Information quickly becomes outdated without a process for continuous updates.
|
||||
- **Incompleteness:** The knowledge base is limited to manually selected documents, creating information silos and knowledge gaps.
|
||||
- **High Maintenance Overhead:** The manual effort required to find new sources, ingest them, and fix data quality issues is significant and does not scale.
|
||||
- **Data Quality Degradation:** As more data is added, inconsistencies and duplicates can accumulate, reducing the reliability of RAG outputs and polluting the knowledge graph.
|
||||
|
||||
The agent now features a **robust document processing pipeline** that can fetch raw web content (including PDFs and HTML), generate clean markdown, and store it in a structured database for further analysis and summarization.
|
||||
|
||||
## Architecture
|
||||
|
||||
The Knowledge Agent uses a multi-agent architecture, where a primary **Orchestrator Agent** manages the overall workflow by delegating tasks to a team of specialized sub-agents.
|
||||
|
||||
```
|
||||
+---------------------+
|
||||
| Orchestrator Agent |
|
||||
+----------+----------+
|
||||
|
|
||||
v
|
||||
+----------+----------+
|
||||
| Sub-Agents |
|
||||
+----------+----------+
|
||||
|
|
||||
v
|
||||
+----------+----------+
|
||||
| MCP Servers |
|
||||
+---------------------+
|
||||
```
|
||||
|
||||
- **Orchestrator (`Knowledge Agent`)**: The project manager. It holds the high-level plan and the overall state. It invokes the appropriate sub-agent for each task and handles the flow of information between them.
|
||||
- **Sub-Agents**: A team of specialized agents, each with a specific role in the knowledge management lifecycle.
|
||||
- **MCP Servers**: All agents interact with the outside world and the knowledge base exclusively through tools provided by MCP servers.
|
||||
|
||||
### Frameworks and Libraries
|
||||
|
||||
- **[LangChain](https://www.langchain.com/)**: A framework for developing applications powered by language models.
|
||||
- **[LangGraph](https://langchain-ai.github.io/langgraph/)**: A library for building stateful, multi-agent applications with LLMs.
|
||||
- **[langchain-mcp-adapters](https://github.com/intelligent-soft-works/langchain-mcp-adapters)**: Used for connecting to and using tools from MCP servers.
|
||||
- **[ChatOpenAI](https://python.langchain.com/docs/integrations/chat/openai)**: The language model used for the agents.
|
||||
- **[pydantic](https://pydantic-docs.helpmanual.io/)**: Used for data validation and settings management.
|
||||
- **[psycopg2-binary](https://pypi.org/project/psycopg2-binary/)**: A PostgreSQL adapter for Python.
|
||||
- **[python-dotenv](https://pypi.org/project/python-dotenv/)**: A library for managing environment variables.
|
||||
- **[json-repair](https://pypi.org/project/json-repair/)**: A library for repairing malformed JSON.
|
||||
- **[requests](https://pypi.org/project/requests/)**: A library for making HTTP requests to download web content.
|
||||
- **[pdfplumber](https://pypi.org/project/pdfplumber/)**: A library for extracting text from PDF documents.
|
||||
- **[Trafilatura](https://trafilatura.readthedocs.io/)**: A tool for fast and accurate extraction of main content from HTML.
|
||||
- **[Playwright](https://playwright.dev/)**: A library for browser automation, used as a fallback for complex websites.
|
||||
- **[beautifulsoup4](https://pypi.org/project/beautifulsoup4/)**: A library for parsing HTML content.
|
||||
- **[html2text](https://pypi.org/project/html2text/)**: A library for converting HTML to markdown.
|
||||
- **[tiktoken](https://github.com/openai/tiktoken)**: A tool for counting tokens to ensure content fits within the LLM's context window.
|
||||
|
||||
### Agent Roles
|
||||
|
||||
- **Analyst**: Identifies knowledge gaps and stale information in the knowledge base by analyzing its content and structure.
|
||||
- **Researcher**: Acts as the primary research arm of the agent. It breaks down research tasks and manages the entire content acquisition pipeline:
|
||||
- **Planner**: Creates a strategic, diversified search plan using advanced search operators.
|
||||
- **Content Processor**: Uses a hybrid strategy to extract clean, reader-mode content. It first tries the fast and accurate `trafilatura` library, and if that fails to return quality content, it falls back to a full browser rendering with `Playwright` to handle complex, JavaScript-heavy sites.
|
||||
- **Refiner**: If the initial search plan is unsuccessful, the refiner adjusts the strategy to find the missing information.
|
||||
- **Summarizer**: Generates a concise summary from the clean markdown content. Before summarizing, the content is passed through a filter that truncates it to a safe token limit (16k) to ensure efficiency and prevent context window errors.
|
||||
- **Curator**: Takes the URLs from the Researcher and decides which ones are relevant, then carries out ingestion of approved content into the knowledge base.
|
||||
- **Auditor**: Scans the knowledge graph for data quality issues like duplicate entities, inconsistent naming, and messy relationships.
|
||||
- **Fixer**: Corrects the data quality issues identified by the Auditor, with a human approval step for destructive operations.
|
||||
- **Advisor**: Analyzes recurring error patterns and suggests improvements to the LightRAG system's configuration to prevent future issues.
|
||||
|
||||
## Getting Started
|
||||
|
||||
### Prerequisites
|
||||
|
||||
- Python 3.12+
|
||||
- [uv](https://github.com/astral-sh/uv) package manager
|
||||
- A running [LightRAG](https://github.com/HKUDS/LightRAG) instance
|
||||
- Running MCP servers for tools (e.g., Google Search)
|
||||
- PostgreSQL database
|
||||
|
||||
### Installation
|
||||
|
||||
1. Clone the repository:
|
||||
|
||||
```sh
|
||||
git clone https://github.com/fvanevski/knowledge-agent.git
|
||||
cd knowledge-agent
|
||||
```
|
||||
|
||||
2. Install the dependencies using uv:
|
||||
|
||||
```sh
|
||||
uv sync
|
||||
```
|
||||
|
||||
3. Set up the environment variables by creating a `.env` file in the root directory. You can use the `.env.example` file as a template.
|
||||
|
||||
4. Install Playwright's browser binaries:
|
||||
|
||||
```sh
|
||||
uv run python -m playwright install
|
||||
```
|
||||
|
||||
## Usage
|
||||
|
||||
The Knowledge Agent is executed via the `run.py` script. You can specify different workflows using command-line arguments.
|
||||
|
||||
### Workflows
|
||||
|
||||
- **Full Maintenance (`--maintenance`)**: This is the default workflow and runs all the sub-agents in sequence to perform a full maintenance cycle on the knowledge base.
|
||||
|
||||
```sh
|
||||
uv run python run.py --maintenance
|
||||
```
|
||||
|
||||
- **Analyze (`--analyze`)**: Identifies knowledge gaps and stale information.
|
||||
|
||||
```sh
|
||||
uv run python run.py --analyze
|
||||
```
|
||||
|
||||
- **Research (`--research`)**: Finds new sources for the topics identified by the Analyst.
|
||||
|
||||
```sh
|
||||
uv run python run.py --research
|
||||
```
|
||||
|
||||
- **Curate (`--curate`)**: Ranks search results and ingests approved new content into the knowledge base.
|
||||
|
||||
```sh
|
||||
uv run python run.py --curate
|
||||
```
|
||||
|
||||
- **Audit (`--audit`)**: Reviews the knowledge base for data quality issues.
|
||||
|
||||
```sh
|
||||
uv run python run.py --audit
|
||||
```
|
||||
|
||||
- **Fix (`--fix`)**: Corrects the data quality issues found by the Auditor.
|
||||
|
||||
```sh
|
||||
uv run python run.py --fix
|
||||
```
|
||||
|
||||
- **Advise (`--advise`)**: Provides recommendations for systemic improvements.
|
||||
|
||||
```sh
|
||||
uv run python run.py --advise
|
||||
```
|
||||
|
||||
## Configuration
|
||||
|
||||
The Knowledge Agent requires a `mcp.json` file in the root directory to configure the connection to the MCP tool servers. This file should contain the server configurations, for example:
|
||||
|
||||
```json
|
||||
{
|
||||
"google_search": {
|
||||
"command": "uv",
|
||||
"args": ["run", "python", "google_search_mcp.py"],
|
||||
"cwd": "/workspace/mcp_servers/google_search_mcp",
|
||||
"transport": "stdio"
|
||||
},
|
||||
"lightrag": {
|
||||
"command": "uv",
|
||||
"args": ["run", "python", "lightrag_mcp.py"],
|
||||
"cwd": "/workspace/mcp_servers/lightrag_mcp",
|
||||
"transport": "stdio"
|
||||
},
|
||||
"file_tools": {
|
||||
"command": "npx",
|
||||
"args": ["-y", "@modelcontextprotocol/server-filesystem", "/workspace/knowledge_agent", "/workspace/LightRAG"],
|
||||
"transport": "stdio"
|
||||
},
|
||||
"deepwiki": {
|
||||
"url": "https://mcp.deepwiki.com/sse",
|
||||
"transport": "sse"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Prompts
|
||||
|
||||
The behavior of each sub-agent is guided by a system prompt located in the `prompts/` directory. These prompts define the agent's persona, goals, and expected output format.
|
||||
|
||||
- **`analyst_prompt.txt`**: Guides the Analyst in identifying knowledge gaps.
|
||||
- **`planner_prompt.txt`**: Guides the Researcher's Planner in creating a search strategy.
|
||||
- **`refiner_prompt.txt`**: Guides the Researcher's Refiner in adjusting the search strategy.
|
||||
- **`summarizer_prompt.txt`**: Guides the Researcher's Summarizer in creating a concise summary.
|
||||
- **`search_ranker_prompt.txt`**: Guides the Curator in ranking search results for ingestion.
|
||||
- **`ingester_prompt.txt`**: Guides the Curator in ingesting new sources.
|
||||
- **`auditor_prompt.txt`**: Guides the Auditor in identifying data quality issues.
|
||||
- **`fixer_prompt.txt`**: Guides the Fixer in correcting data quality issues.
|
||||
- **`advisor_prompt.txt`**: Guides the Advisor in providing recommendations.
|
||||
|
||||
## Logging
|
||||
|
||||
The agent's operations are logged to a file in the `logs/` directory. The logs are in JSON format and include the timestamp, log level, agent name, and the message, providing a detailed record of the agent's activity.
|
||||
|
||||
## Database
|
||||
|
||||
The Knowledge Agent uses a PostgreSQL database to store the reports generated by the sub-agents and to cache processed web content. The `db_utils.py` file contains the functions for creating the tables and interacting with the database.
|
||||
|
||||
The database schema consists of tables for each agent's reports and a central `documents` table:
|
||||
|
||||
- `analyst_reports`
|
||||
- `researcher_reports`
|
||||
- `curator_reports`
|
||||
- `auditor_reports`
|
||||
- `fixer_reports`
|
||||
- `advisor_reports`
|
||||
- `documents`
|
||||
|
||||
The `documents` table stores processed web content and has the following structure:
|
||||
|
||||
- `id`: Primary key (integer)
|
||||
- `url`: The unique URL of the source document (text)
|
||||
- `raw_document`: The raw binary content of the document (BYTEA)
|
||||
- `markdown_content`: The processed, clean markdown version of the content (text)
|
||||
- `summary`: A concise summary of the document (text)
|
||||
- `created_at`: Timestamp of when the document was first added
|
||||
|
||||
## Workflow Details
|
||||
|
||||
The `maintenance` workflow is the most comprehensive, executing the full lifecycle of knowledge management. Here is a step-by-step breakdown of the process:
|
||||
|
||||
1. **Analysis**: The **Analyst** examines the knowledge base to identify areas that are outdated or incomplete. It generates a report detailing these knowledge gaps.
|
||||
2. **Research**: The **Researcher** takes the Analyst's report and executes the entire content acquisition pipeline:
|
||||
- The **Planner** develops a set of targeted, diversified search queries.
|
||||
- The agent executes these searches. For each resulting URL, it uses the **hybrid content processor** (Trafilatura with a Playwright fallback) to extract clean, main content and generate high-quality markdown.
|
||||
- All artifacts (raw document, markdown, and summary) are stored in the `documents` table in the database.
|
||||
- If the initial searches are insufficient, the **Refiner** adjusts the plan and tries again.
|
||||
3. **Curation**: The **Curator** ranks the URLs from the Researcher and decides which ones to ingest into the knowledge base, and then proceeds to ingest approved content.
|
||||
4. **Audit**: The **Auditor** scans the knowledge graph for inconsistencies, duplicates, and other data quality issues, producing a report of its findings.
|
||||
5. **Fix**: The **Fixer** takes the Auditor's report and attempts to correct the identified issues. For any destructive changes (e.g., deleting an entity), it will require human approval.
|
||||
6. **Advise**: Finally, the **Advisor** analyzes the reports from all the other agents, identifies recurring problems, and suggests systemic improvements to the LightRAG configuration or the agent's own processes.
|
||||
11
참고/knowledge_agent-main/blocklist.md
Normal file
11
참고/knowledge_agent-main/blocklist.md
Normal file
@@ -0,0 +1,11 @@
|
||||
2. Problematic Websites
|
||||
|
||||
The following websites were identified as problematic and should be added to a blocklist to prevent the agent from attempting to access them in the future:
|
||||
|
||||
federalregister.gov (Blocks programmatic access)
|
||||
congress.gov (Returns 403 Forbidden error)
|
||||
jsis.washington.edu (SSL certificate issue)
|
||||
gao.gov (Returns 403 Forbidden error)
|
||||
consilium.europa.eu (Returns 403 Forbidden error)
|
||||
wilmerhale.com (Returns 403 Forbidden error)
|
||||
|
||||
282
참고/knowledge_agent-main/db_utils.py
Normal file
282
참고/knowledge_agent-main/db_utils.py
Normal file
@@ -0,0 +1,282 @@
|
||||
# db_utils.py
|
||||
import os
|
||||
import psycopg2
|
||||
import json
|
||||
from contextlib import contextmanager
|
||||
import json_repair
|
||||
|
||||
# --- Database Connection ---
|
||||
@contextmanager
|
||||
def get_db_connection():
|
||||
"""Provides a database connection using a context manager."""
|
||||
conn = psycopg2.connect(os.environ["DATABASE_URL"])
|
||||
try:
|
||||
yield conn
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
# --- Table Creation ---
|
||||
def create_tables():
|
||||
"""Creates all necessary tables in the database if they don't exist."""
|
||||
commands = (
|
||||
"""CREATE TABLE IF NOT EXISTS analyst_reports (id SERIAL PRIMARY KEY, report_id VARCHAR(255) UNIQUE NOT NULL, report JSONB, created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP);""",
|
||||
"""CREATE TABLE IF NOT EXISTS researcher_reports (id SERIAL PRIMARY KEY, report_id VARCHAR(255) UNIQUE NOT NULL, report JSONB, created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP);""",
|
||||
"""CREATE TABLE IF NOT EXISTS curator_reports (id SERIAL PRIMARY KEY, report_id VARCHAR(255) UNIQUE NOT NULL, report JSONB, created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP);""",
|
||||
"""CREATE TABLE IF NOT EXISTS auditor_reports (id SERIAL PRIMARY KEY, report_id VARCHAR(255) UNIQUE NOT NULL, report JSONB, created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP);""",
|
||||
"""CREATE TABLE IF NOT EXISTS fixer_reports (id SERIAL PRIMARY KEY, report_id VARCHAR(255) UNIQUE NOT NULL, report JSONB, created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP);""",
|
||||
"""CREATE TABLE IF NOT EXISTS advisor_reports (id SERIAL PRIMARY KEY, report_id VARCHAR(255) UNIQUE NOT NULL, report JSONB, created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP);""",
|
||||
"""CREATE TABLE IF NOT EXISTS documents (id SERIAL PRIMARY KEY, url TEXT UNIQUE NOT NULL, raw_document BYTEA, markdown_content TEXT, summary TEXT, created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP);"""
|
||||
)
|
||||
with get_db_connection() as conn:
|
||||
with conn.cursor() as cur:
|
||||
for command in commands:
|
||||
cur.execute(command)
|
||||
conn.commit()
|
||||
|
||||
# --- Document Handling Functions ---
|
||||
def add_url_or_get_id(url: str) -> list:
|
||||
"""Adds a URL to the documents table if it doesn't exist, or returns the existing id."""
|
||||
with get_db_connection() as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute("SELECT id FROM documents WHERE url = %s;", (url,))
|
||||
result = cur.fetchone()
|
||||
if result:
|
||||
return result[0], "existing"
|
||||
else:
|
||||
cur.execute("INSERT INTO documents (url) VALUES (%s) RETURNING id;", (url,))
|
||||
new_id = cur.fetchone()[0]
|
||||
conn.commit()
|
||||
return new_id, "new"
|
||||
|
||||
def update_document_content(url_id: int, raw_document: bytes, markdown_content: str):
|
||||
"""Updates the raw_document and markdown_content for a given url_id."""
|
||||
# Clean the markdown_content to remove any null characters
|
||||
cleaned_markdown_content = markdown_content.replace('\x00', '')
|
||||
|
||||
with get_db_connection() as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
"UPDATE documents SET raw_document = %s, markdown_content = %s WHERE id = %s;",
|
||||
(raw_document, cleaned_markdown_content, url_id)
|
||||
)
|
||||
conn.commit()
|
||||
|
||||
def get_document_object(url_id: int, type: str):
|
||||
"""Gets the object for a given url_id in the documents table."""
|
||||
allowed_types = ["raw_document", "markdown_content", "summary"]
|
||||
if type not in allowed_types:
|
||||
raise ValueError(f"Invalid type specified: {type}")
|
||||
|
||||
with get_db_connection() as conn:
|
||||
with conn.cursor() as cur:
|
||||
query = f"SELECT {type} FROM documents WHERE id = %s;"
|
||||
cur.execute(query, (url_id,))
|
||||
result = cur.fetchone()
|
||||
if result:
|
||||
return result[0]
|
||||
else:
|
||||
return None
|
||||
|
||||
def update_document_object(url_id: int, type: str, object: str | bytes):
|
||||
"""Updates the object for a given url_id in the documents table."""
|
||||
allowed_types = ["raw_document", "markdown_content", "summary"]
|
||||
if type not in allowed_types:
|
||||
raise ValueError(f"Invalid type specified: {type}")
|
||||
|
||||
with get_db_connection() as conn:
|
||||
with conn.cursor() as cur:
|
||||
query = f"UPDATE documents SET {type} = %s WHERE id = %s;"
|
||||
cur.execute(query, (object, url_id))
|
||||
conn.commit()
|
||||
|
||||
def get_document(url_id: int) -> dict:
|
||||
"""Retrieves a document from the documents table."""
|
||||
with get_db_connection() as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute("SELECT url, raw_document, markdown_content, summary FROM documents WHERE id = %s;", (url_id,))
|
||||
result = cur.fetchone()
|
||||
if result:
|
||||
return {
|
||||
"url": result[0],
|
||||
"raw_document": result[1],
|
||||
"markdown_content": result[2],
|
||||
"summary": result[3]
|
||||
}
|
||||
else:
|
||||
return None
|
||||
|
||||
# --- Utility Functions ---
|
||||
|
||||
def extract_and_clean_json(llm_output: str) -> dict:
|
||||
|
||||
# Use json_repair.loads() directly as a robust, drop-in replacement for json_repair.loads()
|
||||
try:
|
||||
return json_repair.loads(llm_output)
|
||||
except Exception as e:
|
||||
raise ValueError(f"Failed to repair or parse JSON: {e}")
|
||||
|
||||
# --- Report Handling Functions ---
|
||||
def _save_report(table_name: str, report_data: dict):
|
||||
"""Generic function to save a report to a specified table."""
|
||||
report_id = report_data.get("report_id")
|
||||
if not report_id:
|
||||
raise ValueError("Report data must include a 'report_id'")
|
||||
|
||||
report_json_string = json.dumps(report_data)
|
||||
|
||||
with get_db_connection() as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
f"INSERT INTO {table_name} (report_id, report) VALUES (%s, %s);",
|
||||
(report_id, report_json_string)
|
||||
)
|
||||
conn.commit()
|
||||
|
||||
def save_analyst_report(report_data: dict):
|
||||
_save_report("analyst_reports", report_data)
|
||||
|
||||
def save_auditor_report(report_data: dict):
|
||||
_save_report("auditor_reports", report_data)
|
||||
|
||||
def save_fixer_report(report_data: dict):
|
||||
_save_report("fixer_reports", report_data)
|
||||
|
||||
def save_advisor_report(report_data: dict):
|
||||
_save_report("advisor_reports", report_data)
|
||||
|
||||
def initialize_researcher(timestamp: str) -> dict:
|
||||
"""Initializes the researcher's report in the database."""
|
||||
analyst_report_str = load_latest_report('analyst')
|
||||
analyst_report = json.loads(analyst_report_str)
|
||||
|
||||
report_id = f"res_{timestamp.replace('-', '').replace(':', '').replace('T', '_').split('.')[0]}"
|
||||
|
||||
gaps_to_do = [
|
||||
{"gap_id": gap["gap_id"], "description": gap["description"], "research_topic": gap["research_topic"], "searches": [" "]}
|
||||
for gap in analyst_report.get("identified_gaps", [])
|
||||
]
|
||||
|
||||
new_report = {"report_id": report_id, "gaps": gaps_to_do}
|
||||
report_json_string = json.dumps(new_report)
|
||||
|
||||
with get_db_connection() as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
"INSERT INTO researcher_reports (report_id, report) VALUES (%s, %s);",
|
||||
(report_id, report_json_string)
|
||||
)
|
||||
conn.commit()
|
||||
|
||||
return {"researcher_report_id": report_id, "researcher_gaps_todo": gaps_to_do}
|
||||
|
||||
def update_researcher_report(report_id: str, gap_id: str, searches: list):
|
||||
"""Updates a researcher report in the database with search results."""
|
||||
with get_db_connection() as conn:
|
||||
with conn.cursor() as cur:
|
||||
# Read the existing report
|
||||
cur.execute("SELECT report FROM researcher_reports WHERE report_id = %s FOR UPDATE;", (report_id,))
|
||||
result = cur.fetchone()
|
||||
if not result:
|
||||
raise ValueError(f"No researcher report found with id {report_id}")
|
||||
|
||||
report_data = result[0]
|
||||
|
||||
# Modify the report in Python
|
||||
gaps = report_data.get("gaps", [])
|
||||
gap_found = False
|
||||
for gap in gaps:
|
||||
if gap.get("gap_id") == gap_id:
|
||||
gap["searches"] = searches
|
||||
gap_found = True
|
||||
break
|
||||
|
||||
if not gap_found:
|
||||
# This case should ideally not be reached if initialization is correct
|
||||
raise ValueError(f"Gap with id {gap_id} not found in report {report_id}")
|
||||
|
||||
# Write the modified report back
|
||||
report_json_string = json.dumps(report_data)
|
||||
cur.execute(
|
||||
"UPDATE researcher_reports SET report = %s WHERE report_id = %s;",
|
||||
(report_json_string, report_id)
|
||||
)
|
||||
conn.commit()
|
||||
|
||||
def initialize_curator(timestamp: str) -> dict:
|
||||
"""Initializes the curator's report in the database."""
|
||||
researcher_report_str = load_latest_report('researcher')
|
||||
researcher_report = json.loads(researcher_report_str)
|
||||
|
||||
report_id = f"cur_{timestamp.replace('-', '').replace(':', '').replace('T', '_').split('.')[0]}"
|
||||
|
||||
searches_todo = []
|
||||
for gap in researcher_report.get("gaps", []):
|
||||
research_topic = gap.get("research_topic", {})
|
||||
for search in gap.get("searches", []):
|
||||
searches_todo.append({
|
||||
"search": search,
|
||||
"research_topic": research_topic
|
||||
})
|
||||
|
||||
new_report = {
|
||||
"report_id": report_id,
|
||||
"urls_for_ingestion": [],
|
||||
"url_ingestion_status": []
|
||||
}
|
||||
report_json_string = json.dumps(new_report)
|
||||
|
||||
with get_db_connection() as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
"INSERT INTO curator_reports (report_id, report) VALUES (%s, %s);",
|
||||
(report_id, report_json_string)
|
||||
)
|
||||
conn.commit()
|
||||
|
||||
return {"curator_report_id": report_id, "curator_searches_todo": searches_todo}
|
||||
|
||||
def update_curator_report(report_id: str, job: str, results: list):
|
||||
"""Appends results to a job list in a curator report."""
|
||||
with get_db_connection() as conn:
|
||||
with conn.cursor() as cur:
|
||||
# Read the existing report
|
||||
cur.execute("SELECT report FROM curator_reports WHERE report_id = %s FOR UPDATE;", (report_id,))
|
||||
result = cur.fetchone()
|
||||
if not result:
|
||||
raise ValueError(f"No curator report found with id {report_id}")
|
||||
|
||||
report_data = result[0]
|
||||
|
||||
# Modify the report in Python
|
||||
if job not in report_data:
|
||||
report_data[job] = []
|
||||
|
||||
report_data[job].extend(results)
|
||||
|
||||
# Write the modified report back
|
||||
report_json_string = json.dumps(report_data)
|
||||
cur.execute(
|
||||
"UPDATE curator_reports SET report = %s WHERE report_id = %s;",
|
||||
(report_json_string, report_id)
|
||||
)
|
||||
conn.commit()
|
||||
|
||||
def load_latest_report(report_type: str) -> str:
|
||||
"""Loads the most recent report from the database."""
|
||||
table_map = {
|
||||
"analyst": "analyst_reports",
|
||||
"researcher": "researcher_reports",
|
||||
"curator": "curator_reports"
|
||||
}
|
||||
table_name = table_map.get(report_type)
|
||||
if not table_name:
|
||||
raise ValueError(f"Invalid report_type '{report_type}'.")
|
||||
|
||||
with get_db_connection() as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(f"SELECT report FROM {table_name} ORDER BY created_at DESC LIMIT 1;")
|
||||
result = cur.fetchone()
|
||||
if result:
|
||||
return json.dumps(result[0])
|
||||
else:
|
||||
raise FileNotFoundError(f"No reports found in table {table_name}")
|
||||
103
참고/knowledge_agent-main/knowledge_agent.py
Normal file
103
참고/knowledge_agent-main/knowledge_agent.py
Normal file
@@ -0,0 +1,103 @@
|
||||
# knowledge_agent.py
|
||||
|
||||
import json
|
||||
from langchain_mcp_adapters.client import MultiServerMCPClient
|
||||
from langgraph.graph import StateGraph, END
|
||||
from sub_agents.analyst import analyst_agent_node, save_analyst_report_node
|
||||
from sub_agents.researcher import researcher_agent_node
|
||||
from sub_agents.curator import curator_agent_node
|
||||
from sub_agents.auditor import auditor_agent_node, save_auditor_report_node
|
||||
from sub_agents.fixer import fixer_agent_node, save_fixer_report_node
|
||||
from sub_agents.advisor import advisor_agent_node, save_advisor_report_node
|
||||
|
||||
from state import AgentState
|
||||
|
||||
async def get_mcp_tools():
|
||||
"""Initializes the MCP client and fetches the available tools."""
|
||||
with open('mcp.json', 'r') as f:
|
||||
mcp_server_config = json.load(f)
|
||||
|
||||
mcp_client = MultiServerMCPClient(mcp_server_config)
|
||||
tools = await mcp_client.get_tools()
|
||||
print(f"Successfully loaded {len(tools)} tools from MCP server.")
|
||||
return tools
|
||||
|
||||
def create_knowledge_agent_graph(task: str, all_tools: list):
|
||||
"""Creates the Knowledge Agent as a LangGraph StateGraph."""
|
||||
|
||||
workflow = StateGraph(AgentState)
|
||||
|
||||
# Define the workflow based on the task
|
||||
|
||||
if task == "maintenance":
|
||||
# Full workflow with loops for each agent
|
||||
workflow.add_node("analyst", analyst_agent_node)
|
||||
workflow.add_node("save_analyst_report", save_analyst_report_node)
|
||||
workflow.add_node("researcher", researcher_agent_node)
|
||||
workflow.add_node("curator", curator_agent_node)
|
||||
workflow.add_node("auditor", auditor_agent_node)
|
||||
workflow.add_node("save_auditor_report", save_auditor_report_node)
|
||||
workflow.add_node("fixer", fixer_agent_node)
|
||||
workflow.add_node("save_fixer_report", save_fixer_report_node)
|
||||
workflow.add_node("advisor", advisor_agent_node)
|
||||
workflow.add_node("save_advisor_report", save_advisor_report_node)
|
||||
|
||||
workflow.set_entry_point("analyst")
|
||||
workflow.add_edge("analyst", "save_analyst_report")
|
||||
workflow.add_edge("save_analyst_report", "researcher")
|
||||
workflow.add_edge("researcher", "curator")
|
||||
workflow.add_edge("curator", "auditor")
|
||||
workflow.add_edge("auditor", "save_auditor_report")
|
||||
workflow.add_edge("save_auditor_report", "fixer")
|
||||
workflow.add_edge("fixer", "save_fixer_report")
|
||||
workflow.add_edge("save_fixer_report", "advisor")
|
||||
workflow.add_edge("advisor", "save_advisor_report")
|
||||
workflow.add_edge("save_advisor_report", END)
|
||||
|
||||
elif task == "analyze":
|
||||
workflow.add_node("analyst", analyst_agent_node)
|
||||
workflow.add_node("save_analyst_report", save_analyst_report_node)
|
||||
|
||||
workflow.set_entry_point("analyst")
|
||||
workflow.add_edge("analyst", "save_analyst_report")
|
||||
workflow.add_edge("save_analyst_report", END)
|
||||
|
||||
elif task == "research":
|
||||
workflow.add_node("researcher", researcher_agent_node)
|
||||
|
||||
workflow.set_entry_point("researcher")
|
||||
workflow.add_edge("researcher", END)
|
||||
|
||||
elif task == "curate":
|
||||
workflow.add_node("curator", curator_agent_node)
|
||||
|
||||
workflow.set_entry_point("curator")
|
||||
workflow.add_edge("curator", END)
|
||||
|
||||
elif task == "audit":
|
||||
workflow.add_node("auditor", auditor_agent_node)
|
||||
workflow.add_node("save_auditor_report", save_auditor_report_node)
|
||||
|
||||
workflow.set_entry_point("auditor")
|
||||
workflow.add_edge("auditor", "save_auditor_report")
|
||||
workflow.add_edge("save_auditor_report", END)
|
||||
|
||||
elif task == "fix":
|
||||
workflow.add_node("fixer", fixer_agent_node)
|
||||
workflow.add_node("save_fixer_report", save_fixer_report_node)
|
||||
|
||||
workflow.set_entry_point("fixer")
|
||||
workflow.add_edge("fixer", "save_fixer_report")
|
||||
workflow.add_edge("save_fixer_report", END)
|
||||
|
||||
elif task == "advise":
|
||||
workflow.add_node("advisor", advisor_agent_node)
|
||||
workflow.add_node("save_advisor_report", save_advisor_report_node)
|
||||
|
||||
workflow.set_entry_point("advisor")
|
||||
workflow.add_edge("advisor", "save_advisor_report")
|
||||
workflow.add_edge("save_advisor_report", END)
|
||||
|
||||
# Compile the graph
|
||||
app = workflow.compile()
|
||||
return app
|
||||
509
참고/knowledge_agent-main/lightrag/prompt.py
Normal file
509
참고/knowledge_agent-main/lightrag/prompt.py
Normal file
@@ -0,0 +1,509 @@
|
||||
from __future__ import annotations
|
||||
import json
|
||||
from typing import Any
|
||||
|
||||
PROMPTS: dict[str, Any] = {}
|
||||
|
||||
# --- DEPRECATED DELIMITERS ---
|
||||
# Kept for reference for any old cached items that might be reprocessed.
|
||||
# PROMPTS["DEFAULT_TUPLE_DELIMITER"] = "<|>"
|
||||
# PROMPTS["DEFAULT_RECORD_DELIMITER"] = "##"
|
||||
# PROMPTS["DEFAULT_COMPLETION_DELIMITER"] = "<|COMPLETE|>"
|
||||
|
||||
PROMPTS["DEFAULT_USER_PROMPT"] = "n/a"
|
||||
|
||||
# --- JSON Schema for Entity Definitions ---
|
||||
# Structuring the definitions as a dictionary makes them easy to manage and convert to a JSON string.
|
||||
PROMPTS["ENTITY_DEFINITIONS"] = {
|
||||
"organization/institution": {
|
||||
"description": "A named corporate, governmental, or non-profit entity.",
|
||||
"examples": ["The Heritage Foundation", "U.S. Supreme Court", "U.S. Department of Agriculture (USDA)"]
|
||||
},
|
||||
"person": {
|
||||
"description": "A specific, named individual.",
|
||||
"examples": ["Elon Musk", "Earl Warren"]
|
||||
},
|
||||
"location/geo": {
|
||||
"description": "A specific geographical place.",
|
||||
"examples": ["United States", "California", "Texas"]
|
||||
},
|
||||
"event": {
|
||||
"description": "A specific, named occurrence at a point in time.",
|
||||
"examples": ["October 7, 2023 Hamas Attack", "2024 G7 Summit"]
|
||||
},
|
||||
"policy/proposal": {
|
||||
"description": "A specific policy, proposal, or named formal plan originating from a person or organization.",
|
||||
"examples": ["Tariff Policy", "Police Militarization Plan", "In-State Tuition Policy"]
|
||||
},
|
||||
"law/regulation": {
|
||||
"description": "A named law or regulation.",
|
||||
"examples": ["California Indian Child Welfare Act", "Personal Responsibility and Work Opportunity Reconciliation Act of 1996"]
|
||||
},
|
||||
"tax/fiscal_instrument": {
|
||||
"description": "A specific tax, fee, or financial mechanism for revenue or distribution.",
|
||||
"examples": ["Sales and Excise Taxes", "Property Taxes", "Payroll Deductions"]
|
||||
},
|
||||
"narrative": {
|
||||
"description": "A specific, socially constructed story (not objective facts) used to shape public perception, justify social policy, and influence behavior by defining a social group, issue, or event.",
|
||||
"examples": ["Welfare Queen", "Climate Hoax", "Meritocracy"]
|
||||
},
|
||||
"misinformation/disinformation": {
|
||||
"description": "False or inaccurate information, such as fabricated content, manipulated content, imposter content, false context, and unsupported conspiracy theories.",
|
||||
"examples": ["Pizzagate", "Birtherism", "Plandemic", "Altered Pelosi Video"]
|
||||
},
|
||||
"digital_asset": {
|
||||
"description": "A specific digital item or platform.",
|
||||
"examples": ["X (formerly Twitter)", "$TRUMP Memecoin"]
|
||||
},
|
||||
"concept/idea": {
|
||||
"description": "An abstract idea, theory, or social construct.",
|
||||
"examples": ["Free Speech Absolutism", "Racial Segregation", "Unreimbursed Hospital Costs"]
|
||||
},
|
||||
"metric/score": {
|
||||
"description": "A specific, quantifiable measure or statistic, often including numbers and units.",
|
||||
"examples": ["Freedom on the Net Score", "$89.8 billion", "11,862 hate crime incidents"]
|
||||
},
|
||||
"publication/article": {
|
||||
"description": "A named report, book, or article.",
|
||||
"examples": ["Mandate for Leadership", "State Report on Undocumented Immigrant Hospital Costs"]
|
||||
},
|
||||
"political_group": {
|
||||
"description": "A group defined by political affiliation or ideology, not a formal organization.",
|
||||
"examples": ["White Supremacist Groups"]
|
||||
},
|
||||
"scenario/situation": {
|
||||
"description": "A described situation or context without a formal name.",
|
||||
"examples": ["Wealth dynamics between households"]
|
||||
},
|
||||
"demographic/population": {
|
||||
"description": "A group of people defined by shared characteristics.",
|
||||
"examples": ["Black Americans", "LGBTQ+ Individuals", "Undocumented Residents"]
|
||||
},
|
||||
"publisher/outlet": {
|
||||
"description": "A named publisher or media outlet.",
|
||||
"examples": ["The Washington Post", "The Texas Tribune", "Journal of Law"]
|
||||
},
|
||||
"time_period/era": {
|
||||
"description": "A specific period of time.",
|
||||
"examples": ["2025", "2020–2024", "The Civil Rights Era"]
|
||||
}
|
||||
}
|
||||
|
||||
# Convert the dictionary to a formatted JSON string to be inserted into the prompt
|
||||
PROMPTS["ENTITY_DEFINITIONS_JSON"] = json.dumps(PROMPTS["ENTITY_DEFINITIONS"], indent=2)
|
||||
|
||||
|
||||
PROMPTS["entity_extraction"] = """---Goal---
|
||||
From the text snippet provided, extract key entities and their relationships. The text is a small chunk of a larger document. Identify both direct and implicit relationships (e.g., causal links, memberships, hierarchies) when the connection is clear from the context.
|
||||
|
||||
---Output Format---
|
||||
Your output MUST be a single, valid JSON object and nothing else. Do not include any explanatory text, markdown code fences (like ```json), or any other text before or after the JSON. The JSON object will contain two keys: "entities" and "relationships".
|
||||
|
||||
---Entity Definitions---
|
||||
Extract entities that belong to one of the types defined in the following JSON object. Use these definitions and examples to guide your classification. The `type` you assign in your output must be one of the keys from this JSON object.
|
||||
```json
|
||||
{entity_definitions_json}
|
||||
```
|
||||
|
||||
---Relationship Definitions---
|
||||
For each relationship, select the most appropriate `relationship_type` from this fixed list:
|
||||
[TARGETS, EVALUATES, PRODUCES, CAUSES, IS_A, IS_PART_OF, IS_LOCATED_IN, INFLUENCES, PUBLISHED_BY, LED_BY, CRITICIZES, SUPPORTS, USES, INVOLVES, ESTIMATES, AFFIRMED_BY, PAYS_INTO, REIMBURSES]
|
||||
|
||||
---Instructions & Rules---
|
||||
1. **Entities**:
|
||||
* `name`: Standardize the name. Use the full, formal name (e.g., "American Civil Liberties Union" instead of "ACLU"). Use Title Case.
|
||||
* `type`: Assign one of the specified entity types, using the exact lowercase format from the keys in the Entity Definitions JSON.
|
||||
* `description`: Briefly describe the entity using only information from the text. If no description is available, state "Not specified in text."
|
||||
|
||||
2. **Relationships**:
|
||||
* `source` / `target`: Use the standardized entity names.
|
||||
* `description`: Describe the relationship in one concise sentence based on the text.
|
||||
* `type`: Choose exactly one type from the predefined list above.
|
||||
* `strength`: Score from 1-10 indicating how explicit the relationship is (1=implied, 10=directly stated).
|
||||
|
||||
3. **CRITICAL RULES**:
|
||||
* Your output must be a clean, valid JSON object. Do not include any non-JSON text, comments, or markdown formatting.
|
||||
* You are strictly forbidden from using 'UNKNOWN' as an entity type. If an entity does not clearly fit any defined type, you MUST classify it as `concept/idea`.
|
||||
|
||||
---Examples---
|
||||
{examples}
|
||||
|
||||
---Real Data---
|
||||
Text:
|
||||
{input_text}
|
||||
|
||||
---Output---
|
||||
"""
|
||||
|
||||
PROMPTS["entity_extraction_examples"] = [
|
||||
"""------Example 1------
|
||||
|
||||
Text:
|
||||
```
|
||||
Project 2025, an initiative led by the Heritage Foundation, proposes sweeping changes to the federal government. A key part of the plan is the reintroduction of "Schedule F," an executive order that would strip job protections from thousands of federal employees, making them easier to fire. Critics, such as the ACLU, argue this policy undermines the principle of a non-partisan civil service. The plan is detailed in their publication, "Mandate for Leadership."
|
||||
```
|
||||
|
||||
Output:
|
||||
{
|
||||
"entities": [
|
||||
{
|
||||
"name": "Project 2025",
|
||||
"type": "policy/proposal",
|
||||
"description": "An initiative proposing major changes to the federal government, led by the Heritage Foundation."
|
||||
},
|
||||
{
|
||||
"name": "The Heritage Foundation",
|
||||
"type": "organization/institution",
|
||||
"description": "The organization leading the Project 2025 initiative."
|
||||
},
|
||||
{
|
||||
"name": "Schedule F",
|
||||
"type": "policy/proposal",
|
||||
"description": "An executive order designed to remove job protections from federal employees, proposed as part of Project 2025."
|
||||
},
|
||||
{
|
||||
"name": "American Civil Liberties Union",
|
||||
"type": "organization/institution",
|
||||
"description": "An organization that criticizes the Schedule F policy."
|
||||
},
|
||||
{
|
||||
"name": "Mandate for Leadership",
|
||||
"type": "publication/article",
|
||||
"description": "The publication where the Project 2025 plan is detailed."
|
||||
},
|
||||
{
|
||||
"name": "Non-Partisan Civil Service",
|
||||
"type": "concept/idea",
|
||||
"description": "A principle that critics argue is undermined by the Schedule F policy."
|
||||
}
|
||||
],
|
||||
"relationships": [
|
||||
{
|
||||
"source": "The Heritage Foundation",
|
||||
"target": "Project 2025",
|
||||
"description": "The Heritage Foundation leads the Project 2025 initiative.",
|
||||
"type": "LED_BY",
|
||||
"strength": 10
|
||||
},
|
||||
{
|
||||
"source": "Project 2025",
|
||||
"target": "Schedule F",
|
||||
"description": "Project 2025 includes the reintroduction of the Schedule F policy.",
|
||||
"type": "INVOLVES",
|
||||
"strength": 9
|
||||
},
|
||||
{
|
||||
"source": "American Civil Liberties Union",
|
||||
"target": "Schedule F",
|
||||
"description": "The ACLU argues that the Schedule F policy undermines civil service principles.",
|
||||
"type": "CRITICIZES",
|
||||
"strength": 8
|
||||
},
|
||||
{
|
||||
"source": "Mandate for Leadership",
|
||||
"target": "Project 2025",
|
||||
"description": "The details of Project 2025 are found in the 'Mandate for Leadership' publication.",
|
||||
"type": "PUBLISHED_BY",
|
||||
"strength": 10
|
||||
}
|
||||
]
|
||||
}
|
||||
""",
|
||||
"""------Example 2------
|
||||
|
||||
Text:
|
||||
```
|
||||
Following his acquisition of X (formerly Twitter), Elon Musk reinstated several controversial accounts, citing a commitment to "free speech absolutism." A study by the Center for Strategic and International Studies (CSIS) analyzed the impact of these changes on the platform's "Freedom on the Net Score," which dropped by 5 points in the subsequent year. The study also noted an increase in the prevalence of the $TRUMP memecoin, a digital asset often promoted by the reinstated accounts.
|
||||
```
|
||||
|
||||
Output:
|
||||
{
|
||||
"entities": [
|
||||
{
|
||||
"name": "Elon Musk",
|
||||
"type": "person",
|
||||
"description": "Acquired X (formerly Twitter) and reinstated controversial accounts."
|
||||
},
|
||||
{
|
||||
"name": "X",
|
||||
"type": "organization/institution",
|
||||
"description": "A social media platform, formerly known as Twitter, acquired by Elon Musk."
|
||||
},
|
||||
{
|
||||
"name": "Free Speech Absolutism",
|
||||
"type": "concept/idea",
|
||||
"description": "The principle cited by Elon Musk for reinstating controversial accounts on X."
|
||||
},
|
||||
{
|
||||
"name": "Center for Strategic and International Studies",
|
||||
"type": "organization/institution",
|
||||
"description": "An organization that studied the impact of policy changes on X."
|
||||
},
|
||||
{
|
||||
"name": "Freedom on the Net Score",
|
||||
"type": "metric/score",
|
||||
"description": "A metric that decreased by 5 points for the X platform after the acquisition."
|
||||
},
|
||||
{
|
||||
"name": "$TRUMP Memecoin",
|
||||
"type": "digital_asset",
|
||||
"description": "A digital asset that saw increased promotion on X after account reinstatements."
|
||||
}
|
||||
],
|
||||
"relationships": [
|
||||
{
|
||||
"source": "Elon Musk",
|
||||
"target": "X",
|
||||
"description": "Elon Musk acquired the platform X.",
|
||||
"type": "IS_PART_OF",
|
||||
"strength": 10
|
||||
},
|
||||
{
|
||||
"source": "Elon Musk",
|
||||
"target": "Free Speech Absolutism",
|
||||
"description": "Elon Musk's actions were justified by his stated commitment to free speech absolutism.",
|
||||
"type": "SUPPORTS",
|
||||
"strength": 9
|
||||
},
|
||||
{
|
||||
"source": "Center for Strategic and International Studies",
|
||||
"target": "Freedom on the Net Score",
|
||||
"description": "The CSIS study analyzed the platform's Freedom on the Net Score.",
|
||||
"type": "EVALUATES",
|
||||
"strength": 8
|
||||
},
|
||||
{
|
||||
"source": "X",
|
||||
"target": "$TRUMP Memecoin",
|
||||
"description": "The $TRUMP memecoin was increasingly promoted on the X platform following the acquisition.",
|
||||
"type": "INVOLVES",
|
||||
"strength": 7
|
||||
}
|
||||
]
|
||||
}
|
||||
""",
|
||||
"""------Example 3------
|
||||
|
||||
Text:
|
||||
```
|
||||
Estimates from the Institute on Taxation and Economic Policy (ITEP) indicate that undocumented immigrants paid nearly $97 billion in federal, state, and local taxes in 2022. This figure includes over $54 billion in payments to the federal government.
|
||||
```
|
||||
|
||||
Output:
|
||||
{
|
||||
"entities": [
|
||||
{
|
||||
"name": "Institute on Taxation and Economic Policy",
|
||||
"type": "organization/institution",
|
||||
"description": "An organization that provides estimates on tax contributions from undocumented immigrants."
|
||||
},
|
||||
{
|
||||
"name": "$97 Billion",
|
||||
"type": "metric/score",
|
||||
"description": "The estimated total tax contribution from undocumented immigrants in 2022."
|
||||
},
|
||||
{
|
||||
"name": "Undocumented Immigrants",
|
||||
"type": "demographic/population",
|
||||
"description": "The population group whose tax contributions were estimated."
|
||||
}
|
||||
],
|
||||
"relationships": [
|
||||
{
|
||||
"source": "Institute on Taxation and Economic Policy",
|
||||
"target": "$97 Billion",
|
||||
"description": "The Institute on Taxation and Economic Policy (ITEP) estimated the total tax contribution to be nearly $97 billion.",
|
||||
"type": "ESTIMATES",
|
||||
"strength": 10
|
||||
},
|
||||
{
|
||||
"source": "Undocumented Immigrants",
|
||||
"target": "$97 Billion",
|
||||
"description": "Undocumented immigrants were estimated to have paid nearly $97 billion in taxes.",
|
||||
"type": "PAYS_INTO",
|
||||
"strength": 9
|
||||
}
|
||||
]
|
||||
}
|
||||
"""
|
||||
]
|
||||
|
||||
PROMPTS["summarize_entity_descriptions"] = """---Role---
|
||||
You are a Knowledge Graph Specialist responsible for data curation and synthesis.
|
||||
|
||||
---Task---
|
||||
Your task is to synthesize a list of descriptions of a given entity or relation into a single, comprehensive, and cohesive summary.
|
||||
|
||||
---Instructions---
|
||||
1. **Comprehensiveness:** The summary must integrate key information from all provided descriptions. Do not omit important facts.
|
||||
2. **Context:** The summary must explicitly mention the name of the entity or relation for full context.
|
||||
3. **Style:** The output must be written from an objective, third-person perspective.
|
||||
4. **Length:** Maintain depth and completeness while ensuring the summary's length not exceed {summary_length} tokens.
|
||||
5. **Language:** The entire output must be written in {language}.
|
||||
|
||||
---Data---
|
||||
{description_type} Name: {description_name}
|
||||
Description List:
|
||||
{description_list}
|
||||
|
||||
---Output---
|
||||
Output:
|
||||
"""
|
||||
|
||||
PROMPTS["entity_continue_extraction"] = """
|
||||
It seems some entities or relationships were missed. Please review the previous text and extract ONLY the missing items.
|
||||
|
||||
---Remember Rules---
|
||||
1. **Output Format**: Your output MUST be a valid JSON object containing "entities" and "relationships" keys. Add only new items to these lists.
|
||||
2. **Entity Schema**: `{"name": "...", "type": "...", "description": "..."}`
|
||||
3. **Relationship Schema**: `{"source": "...", "target": "...", "description": "...", "type": "...", "strength": ...}`
|
||||
4. **CRITICAL**: If nothing was missed, output an empty JSON object: `{"entities": [], "relationships": []}`.
|
||||
|
||||
---Output---
|
||||
Add only new entities and relations below.
|
||||
"""
|
||||
|
||||
PROMPTS["entity_if_loop_extraction"] = """
|
||||
It appears some entities may have still been missed.
|
||||
|
||||
---Output---
|
||||
Output:
|
||||
"""
|
||||
|
||||
PROMPTS["fail_response"] = (
|
||||
"Sorry, I'm not able to provide an answer to that question.[no-context]"
|
||||
)
|
||||
|
||||
PROMPTS["rag_response"] = """---Role---
|
||||
|
||||
You are a helpful assistant responding to user query about Knowledge Graph and Document Chunks provided in JSON format below.
|
||||
|
||||
|
||||
---Goal---
|
||||
|
||||
Generate a concise response based on Knowledge Base and follow Response Rules, considering both current query and the conversation history if provided. Summarize all information in the provided Knowledge Base, and incorporating general knowledge relevant to the Knowledge Base. Do not include information not provided by Knowledge Base.
|
||||
|
||||
---Conversation History---
|
||||
{history}
|
||||
|
||||
---Knowledge Graph and Document Chunks---
|
||||
{context_data}
|
||||
|
||||
---Response Guidelines---
|
||||
**1. Content & Adherence:**
|
||||
- Strictly adhere to the provided context from the Knowledge Base. Do not invent, assume, or include any information not present in the source data.
|
||||
- If the answer cannot be found in the provided context, state that you do not have enough information to answer.
|
||||
- Ensure the response maintains continuity with the conversation history.
|
||||
|
||||
**2. Formatting & Language:**
|
||||
- Format the response using markdown with appropriate section headings.
|
||||
- The response language must in the same language as the user's question.
|
||||
- Target format and length: {response_type}
|
||||
|
||||
**3. Citations / References:**
|
||||
- At the end of the response, under a "References" section, each citation must clearly indicate its origin (KG or DC).
|
||||
- The maximum number of citations is 5, including both KG and DC.
|
||||
- Use the following formats for citations:
|
||||
- For a Knowledge Graph Entity: `[KG] <entity_name>`
|
||||
- For a Knowledge Graph Relationship: `[KG] <entity1_name> - <entity2_name>`
|
||||
- For a Document Chunk: `[DC] <file_path_or_document_name>`
|
||||
|
||||
---USER CONTEXT---
|
||||
- Additional user prompt: {user_prompt}
|
||||
|
||||
---Response---
|
||||
Output:"""
|
||||
|
||||
PROMPTS["keywords_extraction"] = """---Role---
|
||||
You are an expert keyword extractor, specializing in analyzing user queries for a Retrieval-Augmented Generation (RAG) system. Your purpose is to identify both high-level and low-level keywords in the user's query that will be used for effective document retrieval.
|
||||
|
||||
---Goal---
|
||||
Given a user query, your task is to extract two distinct types of keywords:
|
||||
1. **high_level_keywords**: for overarching concepts or themes, capturing user's core intent, the subject area, or the type of question being asked.
|
||||
2. **low_level_keywords**: for specific entities or details, identifying the specific entities, proper nouns, technical jargon, product names, or concrete items.
|
||||
|
||||
---Instructions & Constraints---
|
||||
1. **Output Format**: Your output MUST be a valid JSON object and nothing else. Do not include any explanatory text, markdown code fences (like ```json), or any other text before or after the JSON. It will be parsed directly by a JSON parser.
|
||||
2. **Source of Truth**: All keywords must be explicitly derived from the user query, with both high-level and low-level keyword categories required to contain content.
|
||||
3. **Concise & Meaningful**: Keywords should be concise words or meaningful phrases. Prioritize multi-word phrases when they represent a single concept. For example, from "latest financial report of Apple Inc.", you should extract "latest financial report" and "Apple Inc." rather than "latest", "financial", "report", and "Apple".
|
||||
4. **Handle Edge Cases**: For queries that are too simple, vague, or nonsensical (e.g., "hello", "ok", "asdfghjkl"), you must return a JSON object with empty lists for both keyword types.
|
||||
|
||||
---Examples---
|
||||
{examples}
|
||||
|
||||
---Real Data---
|
||||
User Query: {query}
|
||||
|
||||
---Output---
|
||||
Output:"""
|
||||
|
||||
PROMPTS["keywords_extraction_examples"] = [
|
||||
"""Example 1:
|
||||
|
||||
Query: "How does international trade influence global economic stability?"
|
||||
|
||||
Output:
|
||||
{
|
||||
"high_level_keywords": ["International trade", "Global economic stability", "Economic impact"],
|
||||
"low_level_keywords": ["Trade agreements", "Tariffs", "Currency exchange", "Imports", "Exports"]
|
||||
}
|
||||
|
||||
""",
|
||||
"""Example 2:
|
||||
|
||||
Query: "What are the environmental consequences of deforestation on biodiversity?"
|
||||
|
||||
Output:
|
||||
{
|
||||
"high_level_keywords": ["Environmental consequences", "Deforestation", "Biodiversity loss"],
|
||||
"low_level_keywords": ["Species extinction", "Habitat destruction", "Carbon emissions", "Rainforest", "Ecosystem"]
|
||||
}
|
||||
|
||||
""",
|
||||
"""Example 3:
|
||||
|
||||
Query: "What is the role of education in reducing poverty?"
|
||||
|
||||
Output:
|
||||
{
|
||||
"high_level_keywords": ["Education", "Poverty reduction", "Socioeconomic development"],
|
||||
"low_level_keywords": ["School access", "Literacy rates", "Job training", "Income inequality"]
|
||||
}
|
||||
|
||||
""",
|
||||
]
|
||||
|
||||
PROMPTS["naive_rag_response"] = """---Role---
|
||||
|
||||
You are a helpful assistant responding to user query about Document Chunks provided provided in JSON format below.
|
||||
|
||||
---Goal---
|
||||
|
||||
Generate a concise response based on Document Chunks and follow Response Rules, considering both the conversation history and the current query. Summarize all information in the provided Document Chunks, and incorporating general knowledge relevant to the Document Chunks. Do not include information not provided by Document Chunks.
|
||||
|
||||
---Conversation History---
|
||||
{history}
|
||||
|
||||
---Document Chunks(DC)---
|
||||
{content_data}
|
||||
|
||||
---RESPONSE GUIDELINES---
|
||||
**1. Content & Adherence:**
|
||||
- Strictly adhere to the provided context from the Knowledge Base. Do not invent, assume, or include any information not present in the source data.
|
||||
- If the answer cannot be found in the provided context, state that you do not have enough information to answer.
|
||||
- Ensure the response maintains continuity with the conversation history.
|
||||
|
||||
**2. Formatting & Language:**
|
||||
- Format the response using markdown with appropriate section headings.
|
||||
- The response language must match the user's question language.
|
||||
- Target format and length: {response_type}
|
||||
|
||||
**3. Citations / References:**
|
||||
- At the end of the response, under a "References" section, cite a maximum of 5 most relevant sources used.
|
||||
- Use the following formats for citations: `[DC] <file_path_or_document_name>`
|
||||
|
||||
---USER CONTEXT---
|
||||
- Additional user prompt: {user_prompt}
|
||||
|
||||
---Response---
|
||||
Output:"""
|
||||
33
참고/knowledge_agent-main/log_analysis_report.md
Normal file
33
참고/knowledge_agent-main/log_analysis_report.md
Normal file
@@ -0,0 +1,33 @@
|
||||
### Analysis of Knowledge Agent Run (2025-08-27_000158)
|
||||
|
||||
**1. Summary of Findings**
|
||||
|
||||
The analysis of the log file for the research task revealed several issues that impact the agent's efficiency and effectiveness. The most significant problem is the agent's inability to handle websites that block programmatic access or have SSL certificate issues. This leads to wasted time and resources, as the agent repeatedly attempts to access and process content from these sites without success. Additionally, the agent's current workflow includes some inefficiencies, such as attempting to summarize documents that have no content.
|
||||
|
||||
**2. Problematic Websites**
|
||||
|
||||
The following websites were identified as problematic and should be added to a blocklist to prevent the agent from attempting to access them in the future:
|
||||
|
||||
* `federalregister.gov` (Blocks programmatic access)
|
||||
* `congress.gov` (Returns 403 Forbidden error)
|
||||
* `jsis.washington.edu` (SSL certificate issue)
|
||||
* `gao.gov` (Returns 403 Forbidden error)
|
||||
* `consilium.europa.eu` (Returns 403 Forbidden error)
|
||||
* `wilmerhale.com` (Returns 403 Forbidden error)
|
||||
|
||||
**3. Other Identified Issues**
|
||||
|
||||
* **Inefficient Summarization:** The agent attempts to summarize documents even when markdown generation has failed. This is a waste of resources and should be prevented.
|
||||
* **Lack of Fallback for 403 Errors:** The current implementation of `fetch_and_generate_markdown` doesn't have a specific fallback mechanism for 403 errors. It just logs the error and moves on. This could be improved by adding a retry mechanism or a different content extraction strategy for these cases.
|
||||
* **Noisy Logs:** The logs are quite verbose, making it difficult to spot important errors. The logging level could be adjusted to be more concise, or a more structured logging format could be used to make the logs easier to parse and analyze.
|
||||
|
||||
**4. Recommendations for Improvement**
|
||||
|
||||
Based on the findings above, I recommend the following actions:
|
||||
|
||||
1. **Create a blocklist of problematic websites:** As suggested by the user, a blocklist of problematic websites should be created and stored in the database. The agent's code should be modified to check this blocklist before attempting to access any URL.
|
||||
2. **Improve the `fetch_and_generate_markdown` function:** The `fetch_and_generate_markdown` function should be improved to handle errors more gracefully. Specifically, it should:
|
||||
* Check if markdown generation was successful before attempting to summarize a document.
|
||||
* Implement a fallback mechanism for 403 errors, such as using a different user-agent string or a proxy service.
|
||||
* Handle SSL certificate issues more gracefully, for example by allowing the user to specify whether to ignore SSL errors.
|
||||
3. **Improve logging:** The logging configuration should be reviewed to make the logs less noisy and easier to parse. This could involve adjusting the logging level, using a more structured logging format, or both.
|
||||
28
참고/knowledge_agent-main/mcp.json
Normal file
28
참고/knowledge_agent-main/mcp.json
Normal file
@@ -0,0 +1,28 @@
|
||||
{
|
||||
"google_search": {
|
||||
"command": "uv",
|
||||
"args": ["run", "python", "google_search_mcp.py"],
|
||||
"cwd": "/workspace/mcp_servers/google_search_mcp",
|
||||
"transport": "stdio"
|
||||
},
|
||||
"lightrag": {
|
||||
"command": "uv",
|
||||
"args": ["run", "python", "lightrag_mcp.py"],
|
||||
"cwd": "/workspace/mcp_servers/lightrag_mcp",
|
||||
"transport": "stdio"
|
||||
},
|
||||
"fetch": {
|
||||
"command": "uvx",
|
||||
"args": ["mcp-server-fetch"],
|
||||
"transport": "stdio"
|
||||
},
|
||||
"file_tools": {
|
||||
"command": "npx",
|
||||
"args": ["-y", "@modelcontextprotocol/server-filesystem", "/workspace/knowledge_agent", "/workspace/LightRAG"],
|
||||
"transport": "stdio"
|
||||
},
|
||||
"deepwiki": {
|
||||
"url": "https://mcp.deepwiki.com/sse",
|
||||
"transport": "sse"
|
||||
}
|
||||
}
|
||||
0
참고/knowledge_agent-main/prompts/advisor_prompt.txt
Normal file
0
참고/knowledge_agent-main/prompts/advisor_prompt.txt
Normal file
68
참고/knowledge_agent-main/prompts/analyst_prompt.txt
Normal file
68
참고/knowledge_agent-main/prompts/analyst_prompt.txt
Normal file
@@ -0,0 +1,68 @@
|
||||
System: You are an expert AI assistant tasked with analyzing a LightRAG knowledge base.
|
||||
|
||||
**Your Goal:** To perform a comprehensive analysis of the LightRAG knowledge base, identify high-value knowledge gaps, and generate a single, valid JSON report with detailed research topics.
|
||||
|
||||
**Workflow:**
|
||||
|
||||
1. **Discover Existing Topics:** Start by using your tools (`query`, `graphs_get`, `graph_labels`) to get a high-level overview of the knowledge base's content and structure.
|
||||
2. **Create a Thematic Summary:** Synthesize the topics into 5-10 high-level themes.
|
||||
3. **Survey Thematic Landscapes:** Use your search tools (`google_search` and `fetch`) to expand on the thematic summary. When using `google_search`, remember to use advanced parameters to get more targeted results.
|
||||
4. **Identify knowledge base gaps:** Based on your summary, identify temporal and/or logical gaps.
|
||||
5. **Formulate Detailed Research Topics:** For each gap, formulate a detailed, structured research topic. This topic will serve as a comprehensive briefing for the Researcher agent.
|
||||
6. **Produce a Report.** Consolidate your findings into a structured final report in proper and correct JSON format to be passed to a Researcher.
|
||||
7. **Final Output:** Your final and ONLY output must be a single, valid JSON object that strictly follows the schema below. Do not include any other text, explanations, or markdown formatting.
|
||||
|
||||
**JSON Output Schema:**
|
||||
{{
|
||||
"report_id": "{analyst_report_id}",
|
||||
"knowledge_base_summary": {{
|
||||
"summary": "A brief overview of the knowledge base's contents.",
|
||||
"themes": [
|
||||
{{
|
||||
"theme_id": "T1",
|
||||
"description": "Description of theme 1."
|
||||
}}
|
||||
]
|
||||
}},
|
||||
"identified_gaps": [
|
||||
{{
|
||||
"gap_id": "G1",
|
||||
"description": "A description of the knowledge gap.",
|
||||
"research_topic": {{
|
||||
"title": "A concise title for the research task.",
|
||||
"summary": "A brief summary of the knowledge gap and why it's important to fill.",
|
||||
"key_questions": [
|
||||
"A list of specific questions that the research should answer.",
|
||||
"What are the key developments on this topic since YYYY-MM-DD?",
|
||||
"Who are the key actors and what are their positions?"
|
||||
],
|
||||
"keywords": [
|
||||
"A list of important keywords, entities, and concepts related to the topic.",
|
||||
"keyword1",
|
||||
"entity2"
|
||||
],
|
||||
"sources_to_consult": [
|
||||
"A list of suggested websites or types of sources to consult (e.g., 'sec.gov', 'gao.gov', 'academic journals', 'major news outlets').",
|
||||
"example.com",
|
||||
"organization.org"
|
||||
],
|
||||
"sources_to_avoid": [
|
||||
"A list of sources that are likely to be irrelevant or low-quality.",
|
||||
"unreliable-source.com"
|
||||
]
|
||||
}}
|
||||
}}
|
||||
]
|
||||
}}
|
||||
|
||||
**Key field requirements:**
|
||||
* `report_id`: Use the `{analyst_report_id}` from your state.
|
||||
* `knowledge_base_summary`: A brief overview of the knowledge base's contents.
|
||||
* `themes`: A list of major themes discovered.
|
||||
* `identified_gaps`: A list of specific, high-value knowledge gaps. For each gap, provide a clear `description` and a detailed, structured `research_topic`.
|
||||
|
||||
**You must base your analysis exclusively on the output of your tools. Do not use your general knowledge.**
|
||||
|
||||
User: {input}
|
||||
|
||||
{agent_scratchpad}
|
||||
0
참고/knowledge_agent-main/prompts/auditor_prompt.txt
Normal file
0
참고/knowledge_agent-main/prompts/auditor_prompt.txt
Normal file
0
참고/knowledge_agent-main/prompts/fixer_prompt.txt
Normal file
0
참고/knowledge_agent-main/prompts/fixer_prompt.txt
Normal file
32
참고/knowledge_agent-main/prompts/ingester_prompt.txt
Normal file
32
참고/knowledge_agent-main/prompts/ingester_prompt.txt
Normal file
@@ -0,0 +1,32 @@
|
||||
System: You are an expert AI assistant tasked with ingesting the contents of URLs into a LightRAG knowledge base.
|
||||
|
||||
**Your Goal:** To process URLs for ingestion into a LightRAG knowledge base.
|
||||
|
||||
**Workflow:**
|
||||
|
||||
1. **Process URLs**: For each URL in {urls_for_ingestion}, fetch the content.
|
||||
|
||||
2. **Ingest Sources**: Ingest the successfully fetched content and monitor the progress of ingestion using the tools available to you.
|
||||
|
||||
3. **Final Output:** After you have finished ingesting the URLs and have a list of ingestion status for each URL, your final and ONLY output must be a single, valid JSON object that strictly follows the schema below. Do not include any other text, explanations, or markdown formatting.
|
||||
|
||||
**JSON Output Schema:**
|
||||
{{
|
||||
"url_ingestion_status": [
|
||||
{{
|
||||
"url": "url_1",
|
||||
"status": "status_1"
|
||||
}},
|
||||
{{
|
||||
"url": "url_2",
|
||||
"status": "status_2"
|
||||
}},
|
||||
...
|
||||
]
|
||||
}}
|
||||
|
||||
**You must base your analysis exclusively on the output of your tools. Do not use your general knowledge.**
|
||||
|
||||
User: {input}
|
||||
|
||||
{agent_scratchpad}
|
||||
87
참고/knowledge_agent-main/prompts/planner_prompt.txt
Normal file
87
참고/knowledge_agent-main/prompts/planner_prompt.txt
Normal file
@@ -0,0 +1,87 @@
|
||||
System: You are an expert AI research assistant tasked with planning a research strategy.
|
||||
|
||||
**Your Goal:** To generate a list of 5 effective Google search queries based on a detailed research topic brief.
|
||||
|
||||
**Workflow:**
|
||||
|
||||
1. **Analyze the Research Topic Brief:** You will be given a structured `research_topic` object that contains a `title`, `summary`, `key_questions`, `keywords`, `sources_to_consult`, and `sources_to_avoid`. Analyze this information carefully to build a comprehensive understanding of the research goal.
|
||||
|
||||
2. **Design Effective Searches:**
|
||||
* Based on the research topic brief, create a list of 5 Google search queries.
|
||||
* Your search plan should be designed to answer the `key_questions` and cover the `keywords`.
|
||||
* Prioritize the `sources_to_consult` and actively avoid the `sources_to_avoid`.
|
||||
* For each search, provide a unique `search_id` (using the format `S_P1`, `S_P2`, etc.), a `query`, a `rationale`, and a `parameters` object.
|
||||
* **Leverage Google's advanced search operators directly within the `query` string** (e.g., `AND`, `OR`, `""` for exact phrases, `-` for exclusion, `site:` for valid root domains or subdomains (e.g., example.com, sub.example.com, but NOT example.com/path), `filetype:` for document types). This allows for powerful and flexible combinations.
|
||||
* **Diversify your searches.** Do not exclusively search for one file type (e.g., `filetype:pdf`). Include searches for general web pages (HTML) to find articles, landing pages, and general information, unless the research topic specifically calls for formal reports.
|
||||
|
||||
3. **Parameter Usage Guide:**
|
||||
* **`query` field (for core search logic and advanced operators):**
|
||||
* Use `AND`, `OR`, `""`, `-`, `site:`, `filetype:` directly in the `query` string for complex logical combinations.
|
||||
* Only use `filetype:` when you have a strong reason to believe the best information will be in a specific format (e.g., searching for official government reports).
|
||||
* Example: "renewable energy" AND (site:gov OR site:edu)
|
||||
* **`parameters` object (for API-specific controls):**
|
||||
* Use `dateRestrict`: To narrow your search to a specific time period (e.g., `d7`, `m6`, `y1`).
|
||||
* Use `sort`: To sort results. Can be `date` for estimated page date, or `_TYPE_-_NAME_` for structured data attributes (e.g., `metatags-pubdate`).
|
||||
* Direction: `:a` (ascending) or `:d` (descending). Default is descending.
|
||||
* Bias: `:s` (strong) or `:w` (weak) bias towards values. Example: `sort=date:d:s` (strong bias towards newer dates).
|
||||
* Range: `:r:_LOWER_:_UPPER_` for numerical ranges (e.g., `sort=review-rating:r:3.0:5.0`). Dates should be `YYYYMMDD` without dashes (e.g., `sort=date:r:20230101:20231231`).
|
||||
* Multiple sorts/biases/ranges can be combined with commas (e.g., `sort=review-rating:d:s,release-date:r:20230101:20231231`).
|
||||
* Use `num`: To specify the number of results (1-10).
|
||||
* Use `gl`, `lr`, `searchType` for other API-specific controls. For 'safe', use 'active' or 'off'.
|
||||
* **Avoid duplicating filters:** If you use `site:` or `filetype:` in the `query` string, do not also use `siteSearch` or `fileType` in the `parameters` object for the same search.
|
||||
* **`orTerms`**: This parameter takes a string of space-separated words. At least one of these words must be present in the search results. It is useful for synonyms or related concepts (e.g., `orTerms`: "EV electric car").
|
||||
|
||||
4. **Final Output:**
|
||||
* Your final and ONLY output must be a single, valid JSON object with a single key "searches" that contains a list of 5 search objects.
|
||||
|
||||
**JSON Output Schema:**
|
||||
{{
|
||||
"searches": [
|
||||
{{
|
||||
"search_id": "S_P1",
|
||||
"query": "initial broad query based on title and summary",
|
||||
"rationale": "A broad search to get an overview of the topic, based on the title and summary from the research brief.",
|
||||
"parameters": {{
|
||||
"sort": "date"
|
||||
}}
|
||||
}},
|
||||
{{
|
||||
"search_id": "S_P2",
|
||||
"query": "\"cryptocurrency regulation\" AND (\"SEC enforcement\" OR \"CFTC guidance\") (site:sec.gov OR site:cftc.gov) -blog -news",
|
||||
"rationale": "A highly targeted search combining exact phrases, boolean operators, exclusion terms, and direct site/filetype filters within the query string to find official regulatory documents.",
|
||||
"parameters": {{
|
||||
"dateRestrict": "y1",
|
||||
"sort": "date"
|
||||
}}
|
||||
}},
|
||||
{{
|
||||
"search_id": "S_P3",
|
||||
"query": "(\"policy initiative\" OR \"regulatory framework\") AND \"impact assessment\" (site:epa.gov OR site:energy.gov)",
|
||||
"rationale": "A complex query using boolean OR for concepts and targeting multiple authoritative government domains directly within the query string.",
|
||||
"parameters": {{
|
||||
"sort": "date"
|
||||
}}
|
||||
}},
|
||||
{{
|
||||
"search_id": "S_P4",
|
||||
"query": "latest news \"research topic title\" -blog -opinion",
|
||||
"rationale": "A search for recent news, excluding common low-quality sources, to capture the most up-to-date developments.",
|
||||
"parameters": {{
|
||||
"dateRestrict": "m3",
|
||||
"sort": "date"
|
||||
}}
|
||||
}},
|
||||
{{
|
||||
"search_id": "S_P5",
|
||||
"query": "(\"synonym A\" OR \"synonym B\") \"key concept\" -site:unreliable.com",
|
||||
"rationale": "A search using alternative terminology and explicitly avoiding identified unreliable sources to broaden coverage while maintaining quality.",
|
||||
"parameters": {{
|
||||
"orTerms": "alternative_keyword1 alternative_keyword2"
|
||||
}}
|
||||
}}
|
||||
]
|
||||
}}
|
||||
|
||||
User: {input}
|
||||
|
||||
{agent_scratchpad}
|
||||
69
참고/knowledge_agent-main/prompts/refiner_prompt.txt
Normal file
69
참고/knowledge_agent-main/prompts/refiner_prompt.txt
Normal file
@@ -0,0 +1,69 @@
|
||||
System: You are an expert AI research assistant tasked with refining a research strategy.
|
||||
|
||||
**Your Goal:** To review a research topic brief and the results of an initial set of Google searches, and to decide if more research is needed.
|
||||
|
||||
**Workflow:**
|
||||
|
||||
1. **Analyze the Topic and Initial Results:**
|
||||
* You will be given a structured `research_topic` object (with `title`, `summary`, `key_questions`, etc.) and the `search_results` from an initial set of searches.
|
||||
* Review the `search_results` in the context of the `research_topic` brief. Do the results adequately answer the `key_questions`? Do they come from the `sources_to_consult`?
|
||||
|
||||
2. **Assess Sufficiency and Provide Rationale:**
|
||||
* Based on your analysis, decide if the provided search results are "sufficient" or "insufficient".
|
||||
* **You must provide a rationale for your decision.**
|
||||
* Your default assumption should be that the initial searches are **sufficient** unless you identify a *specific, critical, and easily articulated* gap in the information (e.g., a key question that is completely unanswered).
|
||||
|
||||
3. **Design Refined Searches (if necessary):**
|
||||
* If, and only if, the initial results are clearly "insufficient", you must:
|
||||
1. Clearly state the single most important piece of missing information in your rationale (e.g., "The results did not answer the key question about X.").
|
||||
2. Design a maximum of **two** new, highly targeted search queries to find that specific information.
|
||||
* **Do not simply repeat the initial queries.** Use the context from the research brief to create a new, more effective search strategy.
|
||||
* **Diversify your new searches.** Do not default to a single file type (e.g., `filetype:pdf`). If the initial results were all one type, try searching for other types (like web pages) to find different kinds of information.
|
||||
|
||||
4. **Parameter Usage Guide:**
|
||||
* **`query` field (for core search logic and advanced operators):**
|
||||
* Use `AND`, `OR`, `""`, `-`, `site:` for valid root domains or subdomains (e.g., example.com, sub.example.com, but NOT example.com/path), `filetype:` for document types).
|
||||
* **`parameters` object (for API-specific controls):**
|
||||
* Use `dateRestrict`: To narrow your search to a specific time period (e.g., `d7`, `m6`, `y1`).
|
||||
* Use `sort`: To sort results. Can be `date` for estimated page date, or `_TYPE_-_NAME_` for structured data attributes (e.g., `metatags-pubdate`).
|
||||
* Direction: `:a` (ascending) or `:d` (descending). Default is descending.
|
||||
* Bias: `:s` (strong) or `:w` (weak) bias towards values. Example: `sort=date:d:s` (strong bias towards newer dates).
|
||||
* Range: `:r:_LOWER_:_UPPER_` for numerical ranges (e.g., `sort=review-rating:r:3.0:5.0`). Dates should be `YYYYMMDD` without dashes (e.g., `sort=date:r:20230101:20231231`).
|
||||
* Multiple sorts/biases/ranges can be combined with commas (e.g., `sort=review-rating:d:s,release-date:r:20230101:20231231`).
|
||||
* Use `num`: To specify the number of results (1-10).
|
||||
* Use `gl`, `lr`, `searchType` for other API-specific controls. For 'safe', use 'active' or 'off'.
|
||||
* **Avoid duplicating filters:** If you use `site:` or `filetype:` in the `query` string, do not also use `siteSearch` or `fileType` in the `parameters` object for the same search.
|
||||
* **`orTerms`**: This parameter takes a string of space-separated words. At least one of these words must be present in the search results. It is useful for synonyms or related concepts (e.g., `orTerms`: "EV electric car").
|
||||
|
||||
5. **Final Output:**
|
||||
* Your final and ONLY output must be a single, valid JSON object.
|
||||
* **ALWAYS** wrap your response in a single JSON object.
|
||||
* **NEVER** output a raw string.
|
||||
* **DO NOT** include any extra text, explanations, apologies, or markdown formatting before or after the JSON object.
|
||||
|
||||
**JSON Output Schema (Sufficient):**
|
||||
{{
|
||||
"status": "sufficient",
|
||||
"rationale": "The initial searches provided several recent, authoritative reports that directly answer the key questions in the research brief."
|
||||
}}
|
||||
|
||||
**JSON Output Schema (Insufficient):**
|
||||
{{
|
||||
"status": "insufficient",
|
||||
"rationale": "The initial results did not answer the key question about the project's budget. The new search will target government sites to find this information.",
|
||||
"searches": [
|
||||
{{
|
||||
"search_id": "S_R1",
|
||||
"query": "\"project budget\" AND (\"fiscal year 2024\" OR \"FY24\") (site:gao.gov OR site:whitehouse.gov/omb)",
|
||||
"rationale": "A highly targeted search combining exact phrases, boolean operators, and specific government domains to find official budget documents.",
|
||||
"parameters": {{
|
||||
"dateRestrict": "y1",
|
||||
"sort": "date"
|
||||
}}
|
||||
}}
|
||||
]
|
||||
}}
|
||||
|
||||
User: {input}
|
||||
|
||||
{agent_scratchpad}
|
||||
44
참고/knowledge_agent-main/prompts/search_ranker_prompt.txt
Normal file
44
참고/knowledge_agent-main/prompts/search_ranker_prompt.txt
Normal file
@@ -0,0 +1,44 @@
|
||||
System: You are an expert AI assistant tasked with analyzing search results and deciding what to ingest into a LightRAG knowledge base.
|
||||
|
||||
**Your Goal:** To review a set of search results in the context of a detailed research brief and decide which URLs should be ingested into the knowledge base.
|
||||
|
||||
**Workflow:**
|
||||
|
||||
1. **Analyze the Context:** You will be given a `research_topic` object (with `title`, `summary`, `key_questions`, etc.) and the `search_results` from a specific search. Use the `research_topic` to understand the overall goal of the research.
|
||||
|
||||
2. **Rank the Search Results:** For each result in `search_results`, evaluate it against the `research_topic` and the `search_rationale`. Use the following criteria:
|
||||
* **Relevance:** Does the URL directly help to answer one of the `key_questions` in the research brief?
|
||||
* **Authority:** Is the source reputable and trustworthy (e.g., one of the `sources_to_consult`)?
|
||||
* **Quality:** Is the information well-written and well-researched?
|
||||
* **Novelty:** Does the URL provide new information that is not already present in the other search results?
|
||||
|
||||
3. **Produce a Ranked List:** Create a ranked list of all the URLs from the search results. For each URL, provide a `status` ("approved" or "denied") and a brief `rationale` for your decision.
|
||||
|
||||
4. **Final Output:** Your final and ONLY output must be a single, valid JSON object that strictly follows the schema below. Do not include any other text, explanations, or markdown formatting.
|
||||
|
||||
**JSON Output Schema:**
|
||||
{{
|
||||
"ranked_urls": [
|
||||
{{
|
||||
"url": "url_1",
|
||||
"status": "approved",
|
||||
"rationale": "This article directly answers a key question and comes from an authoritative source."
|
||||
}},
|
||||
{{
|
||||
"url": "url_2",
|
||||
"status": "denied",
|
||||
"rationale": "This article is a low-quality blog post with no new information."
|
||||
}},
|
||||
{{
|
||||
"url": "url_3",
|
||||
"status": "approved",
|
||||
"rationale": "This report provides valuable statistics related to the research topic."
|
||||
}}
|
||||
]
|
||||
}}
|
||||
|
||||
**You must base your analysis exclusively on the output of your tools. Do not use your general knowledge.**
|
||||
|
||||
User: {input}
|
||||
|
||||
{agent_scratchpad}
|
||||
28
참고/knowledge_agent-main/prompts/summarizer_prompt.txt
Normal file
28
참고/knowledge_agent-main/prompts/summarizer_prompt.txt
Normal file
@@ -0,0 +1,28 @@
|
||||
System: You are an expert AI assistant tasked with summarizing web content. Your primary function is to return a concise, well-written summary formatted as a JSON object.
|
||||
|
||||
**Your Goal:** To receive a document and return a single, valid JSON object containing a summary.
|
||||
|
||||
**Instructions:**
|
||||
|
||||
1. **Analyze the Document:** You will be given a document in markdown format.
|
||||
2. **Summarize:** Based on your analysis, write a summary of the document that is between 2 to 4 sentences in length.
|
||||
3. **Format Output:** Your final and ONLY output MUST be a single, valid JSON object. It must strictly follow the schema: `{{"summary": "A 2 to 4 sentence summary of the document."}}`
|
||||
|
||||
**Crucial Rules for Output:**
|
||||
- **ALWAYS** wrap your response in a single JSON object.
|
||||
- **NEVER** output a raw string.
|
||||
- **DO NOT** include any extra text, explanations, apologies, or markdown formatting before or after the JSON object.
|
||||
|
||||
**Correct Output: JSON Schema**
|
||||
{{
|
||||
"summary": "This document discusses the impact of climate change on global supply chains, highlighting the increased risks of disruption and the need for more resilient systems. It recommends diversification and investment in predictive analytics."
|
||||
}}
|
||||
|
||||
**Incorrect Output: Raw String**
|
||||
Here is the summary you requested:
|
||||
The document is about climate change. It is very important.
|
||||
|
||||
|
||||
User: {input}
|
||||
|
||||
{agent_scratchpad}
|
||||
23
참고/knowledge_agent-main/pyproject.toml
Normal file
23
참고/knowledge_agent-main/pyproject.toml
Normal file
@@ -0,0 +1,23 @@
|
||||
[project]
|
||||
name = "knowledge-agent"
|
||||
version = "0.1.0"
|
||||
description = "AI agent for intelligently updating and maintaining LightRAG knowledge base"
|
||||
readme = "README.md"
|
||||
requires-python = ">=3.12"
|
||||
dependencies = [
|
||||
"json-repair",
|
||||
"langchain-mcp-adapters",
|
||||
"langchain-openai",
|
||||
"pydantic",
|
||||
"psycopg2-binary",
|
||||
"python-dotenv",
|
||||
"langchain",
|
||||
"langgraph",
|
||||
"requests",
|
||||
"pdfplumber",
|
||||
"beautifulsoup4",
|
||||
"html2text",
|
||||
"tiktoken",
|
||||
"playwright",
|
||||
"trafilatura",
|
||||
]
|
||||
123
참고/knowledge_agent-main/run.py
Normal file
123
참고/knowledge_agent-main/run.py
Normal file
@@ -0,0 +1,123 @@
|
||||
# run.py
|
||||
import asyncio
|
||||
import logging
|
||||
import json
|
||||
from datetime import datetime
|
||||
from zoneinfo import ZoneInfo
|
||||
import os
|
||||
import argparse
|
||||
from langchain_openai.chat_models import ChatOpenAI
|
||||
from langchain_core.messages import HumanMessage
|
||||
from knowledge_agent import get_mcp_tools, create_knowledge_agent_graph
|
||||
from dotenv import load_dotenv
|
||||
from db_utils import create_tables
|
||||
from terminal_utils import print_colorful_break
|
||||
|
||||
# Load environment variables from .env file
|
||||
load_dotenv()
|
||||
|
||||
# Create the tables if they don't exist
|
||||
create_tables()
|
||||
|
||||
# Create a custom JSON formatter
|
||||
class JsonFormatter(logging.Formatter):
|
||||
def format(self, record):
|
||||
log_record = {
|
||||
"timestamp": self.formatTime(record, self.datefmt),
|
||||
"level": record.levelname,
|
||||
"name": record.name,
|
||||
"message": record.getMessage(),
|
||||
"module": record.module,
|
||||
"funcName": record.funcName,
|
||||
"lineno": record.lineno
|
||||
}
|
||||
return json.dumps(log_record)
|
||||
|
||||
# Create a logs directory if it doesn't exist
|
||||
if not os.path.exists('logs'):
|
||||
os.makedirs('logs')
|
||||
|
||||
# Configure the logger
|
||||
log_file = f"logs/knowledge_agent_run_{datetime.now().strftime('%Y%m%d_%H%M%S')}.log"
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
|
||||
handlers=[
|
||||
logging.FileHandler(log_file, mode="w"),
|
||||
logging.StreamHandler()
|
||||
]
|
||||
)
|
||||
|
||||
# Get a specific logger for our application's own messages
|
||||
logger = logging.getLogger('KnowledgeAgent')
|
||||
|
||||
|
||||
async def main():
|
||||
print_colorful_break("KNOWLEDGE AGENT INITIALIZING")
|
||||
parser = argparse.ArgumentParser(description="Run the Knowledge Agent with a specific workflow.")
|
||||
parser.add_argument("--maintenance", action="store_true", help="Run the full maintenance workflow.")
|
||||
parser.add_argument("--analyze", action="store_true", help="Run the analysis workflow.")
|
||||
parser.add_argument("--research", action="store_true", help="Run the research workflow.")
|
||||
parser.add_argument("--curate", action="store_true", help="Run the curation workflow.")
|
||||
parser.add_argument("--audit", action="store_true", help="Run the audit workflow.")
|
||||
parser.add_argument("--fix", action="store_true", help="Run the fix workflow.")
|
||||
parser.add_argument("--advise", action="store_true", help="Run the advise workflow.")
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
task = "maintenance" # Default task
|
||||
if args.analyze:
|
||||
task = "analyze"
|
||||
elif args.research:
|
||||
task = "research"
|
||||
elif args.curate:
|
||||
task = "curate"
|
||||
elif args.audit:
|
||||
task = "audit"
|
||||
elif args.fix:
|
||||
task = "fix"
|
||||
elif args.advise:
|
||||
task = "advise"
|
||||
|
||||
logger.info(f"Initializing Knowledge Agent for task: {task}...")
|
||||
|
||||
try:
|
||||
# 1. Load tools asynchronously
|
||||
mcp_tools = await get_mcp_tools()
|
||||
|
||||
model = ChatOpenAI(
|
||||
model=os.environ.get("OPENAI_MODEL_NAME", "chat"),
|
||||
base_url=os.environ.get("OPENAI_BASE_URL", "http://localhost:8001/v1"),
|
||||
temperature=0.6,
|
||||
top_p=0.6,
|
||||
)
|
||||
|
||||
# 2. Pass tools into the graph creation function
|
||||
app = create_knowledge_agent_graph(task, mcp_tools)
|
||||
|
||||
run_timestamp = datetime.now(ZoneInfo("America/Los_Angeles")).isoformat()
|
||||
|
||||
# 3. Initialize the state with the messages list and the first message
|
||||
initial_state = {
|
||||
"messages": [HumanMessage(content="Your task is to identify knowledge gaps in the LightRAG knowledge base. Begin now.")],
|
||||
"task": task,
|
||||
"status": f"Starting '{task}' workflow.",
|
||||
"timestamp": run_timestamp,
|
||||
"mcp_tools": mcp_tools,
|
||||
"model": model,
|
||||
"logger": logger
|
||||
}
|
||||
|
||||
logger.info(f"--- Invoking graph for task: {task} ---")
|
||||
|
||||
final_state = await app.ainvoke(initial_state)
|
||||
|
||||
logger.info("--- Workflow Complete ---")
|
||||
logger.info(f"Final Status: {final_state['status']}")
|
||||
print_colorful_break("KNOWLEDGE AGENT RUN COMPLETE")
|
||||
|
||||
finally:
|
||||
logging.shutdown()
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
45
참고/knowledge_agent-main/state.py
Normal file
45
참고/knowledge_agent-main/state.py
Normal file
@@ -0,0 +1,45 @@
|
||||
# state.py
|
||||
from typing import TypedDict, List, Optional, Annotated
|
||||
from langchain_core.messages import BaseMessage
|
||||
import operator
|
||||
|
||||
class AgentState(TypedDict):
|
||||
"""
|
||||
Represents the state of the agent graph.
|
||||
"""
|
||||
messages: Annotated[List[BaseMessage], operator.add]
|
||||
task: str
|
||||
status: str
|
||||
timestamp: str
|
||||
mcp_tools: List[any]
|
||||
model: any
|
||||
logger: any
|
||||
|
||||
# Fields for the analyst agent's stateful workflow
|
||||
analyst_report_id: Optional[str]
|
||||
analyst_report: Optional[str]
|
||||
|
||||
# Fields for the researcher agent's stateful workflow
|
||||
researcher_report_id: Optional[str]
|
||||
researcher_gaps_todo: Optional[List[dict]]
|
||||
researcher_gaps_complete: Optional[List[str]]
|
||||
researcher_report: Optional[str]
|
||||
|
||||
|
||||
# Fields for the curator agent's stateful workflow
|
||||
curator_report_id: Optional[str]
|
||||
curator_urls_for_ingestion: Optional[List[str]]
|
||||
curator_url_ingestion_status: Optional[List[dict]]
|
||||
curator_report: Optional[str]
|
||||
|
||||
# Fields for the auditor agent's stateful workflow
|
||||
auditor_report_id: Optional[str]
|
||||
auditor_report: Optional[str]
|
||||
|
||||
# Fields for the fixer agent's stateful workflow
|
||||
fixer_report_id: Optional[str]
|
||||
fixer_report: Optional[str]
|
||||
|
||||
# Fields for the advisor agent's stateful workflow
|
||||
advisor_report_id: Optional[str]
|
||||
advisor_report: Optional[str]
|
||||
0
참고/knowledge_agent-main/sub_agents/__init__.py
Normal file
0
참고/knowledge_agent-main/sub_agents/__init__.py
Normal file
56
참고/knowledge_agent-main/sub_agents/advisor.py
Normal file
56
참고/knowledge_agent-main/sub_agents/advisor.py
Normal file
@@ -0,0 +1,56 @@
|
||||
# advisor.py
|
||||
from langchain.agents import create_openai_tools_agent, AgentExecutor
|
||||
from langchain_core.prompts import ChatPromptTemplate
|
||||
from langchain_core.tools import tool, ToolException
|
||||
from langchain_core.messages import AIMessage
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
from state import AgentState
|
||||
from db_utils import load_latest_report, extract_and_clean_json
|
||||
from terminal_utils import print_colorful_break
|
||||
|
||||
async def advisor_agent_node(state: AgentState):
|
||||
print_colorful_break("ADVISOR")
|
||||
logger = state['logger']
|
||||
logger.info("--- Running Advisor Agent ---")
|
||||
all_tools = state['mcp_tools']
|
||||
model = state['model']
|
||||
timestamp = state['timestamp']
|
||||
|
||||
advisor_tools = [t for t in all_tools if t.name in ["list_allowed_directories", "list_directory", "search_files", "read_text_file"]] + [load_latest_report]
|
||||
advisor_prompt = '''Your goal is to provide recommendations for systemic improvements.
|
||||
1. **Analyze Reports**: Load and analyze `auditor_report.json` and `fixer_report.json`.
|
||||
2. **Generate Recommendations**: Based on recurring patterns, generate actionable recommendations for ingestion prompts or server configuration.
|
||||
3. **Compile Report**: Save a final report with your top 3-5 suggestions to `advisor_report.json`.'''
|
||||
|
||||
prompt = ChatPromptTemplate.from_template(advisor_prompt)
|
||||
agent_executor = create_openai_tools_agent(model, advisor_tools, prompt)
|
||||
|
||||
task_input = "Your task is to provide recommendations based on the latest audit and fix reports. Begin now."
|
||||
|
||||
result = await agent_executor.ainvoke({"input": task_input, "timestamp": timestamp})
|
||||
|
||||
logger.info(f"Advisor Agent finished with output: {result['output']}")
|
||||
return {"messages": state['messages'] + [AIMessage(content=result['output'])]}
|
||||
|
||||
def save_advisor_report_node(state: AgentState):
|
||||
"""Saves the final report from the last AI message."""
|
||||
logger = state['logger']
|
||||
final_message_from_agent = state['messages'][-1]
|
||||
|
||||
status = f"--- Saving Advisor Report ---\n{final_message_from_agent.content}"
|
||||
logger.info(status)
|
||||
|
||||
try:
|
||||
report_json = extract_and_clean_json(final_message_from_agent.content)
|
||||
if 'report_id' not in report_json:
|
||||
report_json['report_id'] = state.get('advisor_report_id', 'unknown_id')
|
||||
save_advisor_report({"advisor_report": json.dumps(report_json)})
|
||||
status = f"Successfully saved advisor report with ID {report_json.get('report_id')}"
|
||||
logger.info(status)
|
||||
except (ValueError, KeyError) as e:
|
||||
status = f"Error processing or saving advisor report: {e}"
|
||||
logger.error(status, exc_info=True)
|
||||
|
||||
return {"messages": state['messages'] + [AIMessage(content=status)]}
|
||||
78
참고/knowledge_agent-main/sub_agents/analyst.py
Normal file
78
참고/knowledge_agent-main/sub_agents/analyst.py
Normal file
@@ -0,0 +1,78 @@
|
||||
# sub_agents/analyst.py
|
||||
from langchain.agents import create_openai_tools_agent, AgentExecutor
|
||||
from langchain_core.prompts import ChatPromptTemplate
|
||||
from state import AgentState
|
||||
from db_utils import save_analyst_report, extract_and_clean_json
|
||||
from terminal_utils import print_colorful_break
|
||||
|
||||
async def analyst_agent_node(state: AgentState):
|
||||
"""Runs the analyst agent and returns its raw output and the new report ID."""
|
||||
print_colorful_break("ANALYST")
|
||||
logger = state['logger']
|
||||
timestamp = state['timestamp']
|
||||
report_id = f"ana_{timestamp.replace('-', '').replace(':', '').replace('T', '_').split('.')[0]}"
|
||||
status = f"Initialized analyst report with ID: {report_id}"
|
||||
logger.info(status)
|
||||
|
||||
with open("prompts/analyst_prompt.txt", "r") as f:
|
||||
analyst_prompt_template = f.read()
|
||||
|
||||
analyst_prompt = ChatPromptTemplate.from_template(analyst_prompt_template)
|
||||
analyst_tools = [t for t in state['mcp_tools'] if t.name in ["query", "graphs_get", "graph_labels", "google_search", "fetch"]]
|
||||
|
||||
status = f"Attempting to invoke analyst agent executor with tools: {analyst_tools}"
|
||||
logger.info(status)
|
||||
try:
|
||||
agent_runnable = create_openai_tools_agent(state['model'], analyst_tools, analyst_prompt)
|
||||
executor = AgentExecutor(agent=agent_runnable, tools=analyst_tools, verbose=True)
|
||||
except Exception as e:
|
||||
status = f"Failed to create agent executor: {e}"
|
||||
logger.error(status, exc_info=True)
|
||||
return {"status": status}
|
||||
|
||||
|
||||
task = state['messages'][0].content
|
||||
status = f"Attempting to run agent executor with input: {task}"
|
||||
logger.info(status)
|
||||
try:
|
||||
analyst_result = await executor.ainvoke({
|
||||
"input": task,
|
||||
"analyst_report_id": report_id
|
||||
})
|
||||
raw_report = analyst_result.get('output', '')
|
||||
status = f"Analyst agent completed.\nRaw output: {raw_report}"
|
||||
logger.info(status)
|
||||
|
||||
except Exception as e:
|
||||
status = f"Analyst agent failed: {e}"
|
||||
logger.error(status, exc_info=True)
|
||||
raw_report = f"Error in Analyst Agent: {e}"
|
||||
|
||||
return {
|
||||
"analyst_report_id": report_id,
|
||||
"analyst_report": raw_report,
|
||||
"status": status
|
||||
}
|
||||
|
||||
def save_analyst_report_node(state: AgentState):
|
||||
"""Saves the final report and updates the main status field."""
|
||||
logger = state['logger']
|
||||
raw_report_content = state.get("analyst_report")
|
||||
|
||||
report_id = state.get("analyst_report_id")
|
||||
status = f"--- Saving Analyst Report: {report_id} ---"
|
||||
logger.info(status)
|
||||
|
||||
try:
|
||||
report_json = extract_and_clean_json(raw_report_content)
|
||||
if 'report_id' not in report_json:
|
||||
report_json['report_id'] = report_id
|
||||
|
||||
save_analyst_report(report_json)
|
||||
status = f"Successfully saved analyst report with ID {report_json.get('report_id')}"
|
||||
logger.info(status)
|
||||
except (ValueError, KeyError) as e:
|
||||
status = f"Error processing or saving analyst report: {e}"
|
||||
logger.error(status)
|
||||
|
||||
return {"status": status}
|
||||
56
참고/knowledge_agent-main/sub_agents/auditor.py
Normal file
56
참고/knowledge_agent-main/sub_agents/auditor.py
Normal file
@@ -0,0 +1,56 @@
|
||||
# auditor.py
|
||||
from langchain.agents import create_openai_tools_agent, AgentExecutor
|
||||
from langchain_core.prompts import ChatPromptTemplate
|
||||
from langchain_core.tools import tool, ToolException
|
||||
from langchain_core.messages import AIMessage
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
from state import AgentState
|
||||
from db_utils import extract_and_clean_json
|
||||
from terminal_utils import print_colorful_break
|
||||
|
||||
async def auditor_agent_node(state: AgentState):
|
||||
print_colorful_break("AUDITOR")
|
||||
logger = state['logger']
|
||||
logger.info("--- Running Auditor Agent ---")
|
||||
all_tools = state['mcp_tools']
|
||||
model = state['model']
|
||||
timestamp = state['timestamp']
|
||||
|
||||
auditor_tools = [t for t in all_tools if t.name in ["graphs_get", "query"]]
|
||||
auditor_prompt = '''Your goal is to review the LightRAG knowledge base for data quality issues.
|
||||
1. **Identify issues**: Scan the graph for duplicates, irregular normalization, etc.
|
||||
2. **Generate Report**: Create a report of your findings.
|
||||
3. **Save Report**: Use `save_report` to save the findings to `auditor_report.json`.'''
|
||||
|
||||
prompt = ChatPromptTemplate.from_template(auditor_prompt)
|
||||
agent_executor = create_openai_tools_agent(model, auditor_tools, prompt)
|
||||
|
||||
task_input = "Your task is to audit the knowledge base. Begin now."
|
||||
|
||||
result = await agent_executor.ainvoke({"input": task_input, "timestamp": timestamp})
|
||||
|
||||
logger.info(f"Auditor Agent finished with output: {result['output']}")
|
||||
return {"messages": state['messages'] + [AIMessage(content=result['output'])]}
|
||||
|
||||
def save_auditor_report_node(state: AgentState):
|
||||
"""Saves the final report from the last AI message."""
|
||||
logger = state['logger']
|
||||
final_message_from_agent = state['messages'][-1]
|
||||
|
||||
status = f"--- Saving Auditor Report ---\n{final_message_from_agent.content}"
|
||||
logger.info(status)
|
||||
|
||||
try:
|
||||
report_json = extract_and_clean_json(final_message_from_agent.content)
|
||||
if 'report_id' not in report_json:
|
||||
report_json['report_id'] = state.get('auditor_report_id', 'unknown_id')
|
||||
save_auditor_report({"auditor_report": json.dumps(report_json)})
|
||||
status = f"Successfully saved auditor report with ID {report_json.get('report_id')}"
|
||||
logger.info(status)
|
||||
except (ValueError, KeyError) as e:
|
||||
status = f"Error processing or saving auditor report: {e}"
|
||||
logger.error(status, exc_info=True)
|
||||
|
||||
return {"messages": state['messages'] + [AIMessage(content=status)]}
|
||||
177
참고/knowledge_agent-main/sub_agents/curator.py
Normal file
177
참고/knowledge_agent-main/sub_agents/curator.py
Normal file
@@ -0,0 +1,177 @@
|
||||
# sub_agents/curator.py
|
||||
from langchain.agents import create_openai_tools_agent, AgentExecutor
|
||||
from langchain_core.prompts import ChatPromptTemplate
|
||||
from state import AgentState
|
||||
from db_utils import initialize_curator, update_curator_report, extract_and_clean_json
|
||||
from terminal_utils import print_colorful_break
|
||||
|
||||
async def curator_agent_node(state: AgentState):
|
||||
"""Orchestrates the curation process."""
|
||||
print_colorful_break("CURATOR")
|
||||
logger = state['logger']
|
||||
|
||||
report_id = state.get("curator_report_id")
|
||||
searches_todo = []
|
||||
curator_urls_for_ingestion = []
|
||||
curator_url_ingestion_status = []
|
||||
final_status = None
|
||||
|
||||
# One-time initialization of the curator state
|
||||
if not report_id:
|
||||
try:
|
||||
init_result = initialize_curator(state['timestamp'])
|
||||
report_id = init_result.get("curator_report_id")
|
||||
searches_todo = init_result.get("curator_searches_todo")
|
||||
status = f"--- Initialized curator state: {init_result} ---"
|
||||
logger.info(status)
|
||||
except Exception as e:
|
||||
status = f"Failed to initialize curator: {e}"
|
||||
logger.error(status, exc_info=True)
|
||||
return {"status": status}
|
||||
|
||||
# Create the specialized agent for search ranking
|
||||
search_ranker_prompt = ChatPromptTemplate.from_template(open("prompts/search_ranker_prompt.txt", "r").read())
|
||||
search_ranker_tools = [t for t in state['mcp_tools'] if t.name in ["google_search", "fetch"]]
|
||||
status = f"Attempting to invoke search ranker agent executor with tools: {search_ranker_tools}"
|
||||
logger.info(status)
|
||||
try:
|
||||
agent_runnable = create_openai_tools_agent(state['model'], search_ranker_tools, search_ranker_prompt)
|
||||
executor = AgentExecutor(agent=agent_runnable, tools=search_ranker_tools, verbose=True)
|
||||
except Exception as e:
|
||||
status = f"Failed to create search ranker agent executor: {e}"
|
||||
logger.error(status, exc_info=True)
|
||||
return {"status": status}
|
||||
|
||||
# Main control loop for search ranking
|
||||
if isinstance(searches_todo, list) and searches_todo:
|
||||
for item in searches_todo:
|
||||
current_search = item.get("search", {})
|
||||
research_topic = item.get("research_topic", {})
|
||||
|
||||
search_id = current_search.get("search_id", "unknown_search")
|
||||
search_rationale = current_search.get('rationale', '')
|
||||
search_results = current_search.get('results', [])
|
||||
|
||||
status = f"Processing search: {search_id}"
|
||||
logger.info(status)
|
||||
try:
|
||||
search_ranker_result = await executor.ainvoke({
|
||||
"input": {
|
||||
"research_topic": research_topic,
|
||||
"search_results": search_results,
|
||||
"search_rationale": search_rationale
|
||||
}
|
||||
})
|
||||
raw_search_ranker_result = search_ranker_result.get('output', '')
|
||||
status = f"Curator agent for search {search_id} completed. Raw output: {raw_search_ranker_result}"
|
||||
logger.info(status)
|
||||
|
||||
try:
|
||||
json_output = extract_and_clean_json(raw_search_ranker_result)
|
||||
ranked_urls = json_output.get("ranked_urls", [])
|
||||
approved_urls = [url['url'] for url in ranked_urls if url.get('status') == 'approved']
|
||||
curator_urls_for_ingestion.extend(approved_urls)
|
||||
status = f"Successfully parsed ranked URLs for search {search_id}: {len(approved_urls)} approved."
|
||||
logger.info(status)
|
||||
except Exception as e:
|
||||
status = f"Failed to parse ranked URLs for search {search_id}: {e}"
|
||||
logger.error(status, exc_info=True)
|
||||
continue
|
||||
|
||||
status = f"Preparing to update report for search {search_id} with {len(approved_urls)} URLs."
|
||||
logger.info(status)
|
||||
try:
|
||||
tool_input = {
|
||||
"curator_report_id": report_id,
|
||||
"job": "urls_for_ingestion",
|
||||
"results": approved_urls
|
||||
}
|
||||
update_curator_report(tool_input)
|
||||
status = f"Updated report for search {search_id} with {len(approved_urls)} URLs."
|
||||
logger.info(status)
|
||||
except Exception as e:
|
||||
status = f"Failed to update report for search {search_id}: {e}"
|
||||
logger.error(status, exc_info=True)
|
||||
continue
|
||||
|
||||
status = f"Successfully updated report for search {search_id} with {len(approved_urls)} URLs."
|
||||
logger.info(status)
|
||||
|
||||
except Exception as e:
|
||||
status = f"Curator agent search ranking for search {search_id} failed: {e}"
|
||||
logger.error(status, exc_info=True)
|
||||
continue
|
||||
|
||||
status = f"Successfully completed ranking of all searches for report: {report_id}"
|
||||
logger.info(status)
|
||||
|
||||
# Create the specialized agent for url ingestion
|
||||
ingester_prompt = ChatPromptTemplate.from_template(open("prompts/ingester_prompt.txt", "r").read())
|
||||
ingester_tools = [t for t in state['mcp_tools'] if t.name in ["fetch", "documents_upload_file", "documents_upload_files", "documents_insert_text", "documents_pipeline_status"]]
|
||||
status = f"Attempting to invoke url ingestion agent executor with tools: {ingester_tools}"
|
||||
logger.info(status)
|
||||
try:
|
||||
agent_runnable = create_openai_tools_agent(state['model'], ingester_tools, ingester_prompt)
|
||||
executor = AgentExecutor(agent=agent_runnable, tools=ingester_tools, verbose=True)
|
||||
except Exception as e:
|
||||
status = f"Failed to create url ingestion agent executor: {e}"
|
||||
logger.error(status, exc_info=True)
|
||||
return {"status": status}
|
||||
|
||||
# Main control for url ingestion
|
||||
task = "Ingest the URLs in `{urls_for_ingestion}` and return the ingestion status for each URL."
|
||||
status = f"Attempting to run agent executor for url ingestion"
|
||||
logger.info(status)
|
||||
try:
|
||||
# The executor handles the entire loop of tool calls and reasoning.
|
||||
ingestion_result = await executor.ainvoke({
|
||||
"input": task,
|
||||
"urls_for_ingestion": curator_urls_for_ingestion
|
||||
})
|
||||
raw_ingestion_result = ingestion_result.get('output', '')
|
||||
status = f"Curator agent ingestion for report {report_id} completed.\nRaw output: {raw_ingestion_result}"
|
||||
logger.info(status)
|
||||
|
||||
try:
|
||||
json_output = extract_and_clean_json(raw_ingestion_result)
|
||||
curator_url_ingestion_status = json_output.get("url_ingestion_status", [])
|
||||
status = f"Successfully parsed URL ingestion status: {curator_url_ingestion_status}."
|
||||
logger.info(status)
|
||||
except Exception as e:
|
||||
status = f"Failed to parse URL ingestion status: {e}"
|
||||
logger.error(status, exc_info=True)
|
||||
return {"status": status}
|
||||
|
||||
status = f"Preparing to update report: {report_id} with ingestion status for {len(curator_url_ingestion_status)} URLs."
|
||||
logger.info(status)
|
||||
try:
|
||||
tool_input = {
|
||||
"curator_report_id": report_id,
|
||||
"job": "url_ingestion_status",
|
||||
"results": curator_url_ingestion_status
|
||||
}
|
||||
update_curator_report(tool_input)
|
||||
status = f"Updated report {report_id} with ingestions status for {len(curator_url_ingestion_status)} URLs."
|
||||
logger.info(status)
|
||||
except Exception as e:
|
||||
status = f"Failed to update report {report_id}: {e}"
|
||||
logger.error(status, exc_info=True)
|
||||
return {"status": status}
|
||||
|
||||
status = f"Successfully updated report {report_id} with {len(curator_url_ingestion_status)} URLs."
|
||||
logger.info(status)
|
||||
|
||||
except Exception as e:
|
||||
final_status = f"Curator agent failed to run ingestion of ranked URLs: {e}"
|
||||
logger.error(final_status, exc_info=True)
|
||||
|
||||
if not final_status:
|
||||
final_status = f"Curator successfully ranked and ingested URLs, generating curator report summary written to file `state/curator_report.json`"
|
||||
logger.info(status)
|
||||
|
||||
return {
|
||||
"status": final_status,
|
||||
"curator_report_id": report_id,
|
||||
"curator_urls_for_ingestion": curator_urls_for_ingestion,
|
||||
"curator_url_ingestion_status": curator_url_ingestion_status
|
||||
}
|
||||
60
참고/knowledge_agent-main/sub_agents/fixer.py
Normal file
60
참고/knowledge_agent-main/sub_agents/fixer.py
Normal file
@@ -0,0 +1,60 @@
|
||||
# sub_agents.py
|
||||
from langchain.agents import create_openai_tools_agent, AgentExecutor
|
||||
from langchain_core.prompts import ChatPromptTemplate
|
||||
from langchain_core.tools import tool, ToolException
|
||||
from langchain_core.messages import AIMessage
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
from state import AgentState
|
||||
from tools import human_approval
|
||||
from db_utils import load_latest_report, extract_and_clean_json
|
||||
from terminal_utils import print_colorful_break
|
||||
|
||||
|
||||
async def fixer_agent_node(state: AgentState):
|
||||
print_colorful_break("FIXER")
|
||||
logger = state['logger']
|
||||
logger.info("--- Running Fixer Agent ---")
|
||||
all_tools = state['mcp_tools']
|
||||
model = state['model']
|
||||
timestamp = state['timestamp']
|
||||
|
||||
fixer_tools = [t for t in all_tools if t.name in ["graph_update_entity", "documents_delete_entity", "graph_update_relation", "documents_delete_relation", "graph_entity_exists"]] + [load_latest_report, human_approval]
|
||||
fixer_prompt = '''Your goal is to correct data quality issues.
|
||||
1. **Load Auditor's Report**: Load `auditor_report.json`.
|
||||
2. **Create a Plan**: Create a step-by-step plan to correct the issues.
|
||||
3. **Get Human Approval**: Use `human_approval` to get your plan approved by calling the tool with the plan.
|
||||
4. **Execute**: Execute the approved plan.
|
||||
5. **Save Report**: Save a report of your actions to `fixer_report.json`.'''
|
||||
|
||||
prompt = ChatPromptTemplate.from_template(fixer_prompt)
|
||||
agent_executor = create_openai_tools_agent(model, fixer_tools, prompt)
|
||||
|
||||
task_input = "Your task is to fix issues from the auditor's report. Begin now."
|
||||
|
||||
result = await agent_executor.ainvoke({"input": task_input, "timestamp": timestamp})
|
||||
|
||||
logger.info(f"Fixer Agent finished with output: {result['output']}")
|
||||
return {"messages": state['messages'] + [AIMessage(content=result['output'])]}
|
||||
|
||||
def save_fixer_report_node(state: AgentState):
|
||||
"""Saves the final report from the last AI message."""
|
||||
logger = state['logger']
|
||||
final_message_from_agent = state['messages'][-1]
|
||||
|
||||
status = f"--- Saving Fixer Report ---\n{final_message_from_agent.content}"
|
||||
logger.info(status)
|
||||
|
||||
try:
|
||||
report_json = extract_and_clean_json(final_message_from_agent.content)
|
||||
if 'report_id' not in report_json:
|
||||
report_json['report_id'] = state.get('fixer_report_id', 'unknown_id')
|
||||
save_fixer_report({"fixer_report": json.dumps(report_json)})
|
||||
status = f"Successfully saved fixer report with ID {report_json.get('report_id')}"
|
||||
logger.info(status)
|
||||
except (ValueError, KeyError) as e:
|
||||
status = f"Error processing or saving fixer report: {e}"
|
||||
logger.error(status, exc_info=True)
|
||||
|
||||
return {"messages": state['messages'] + [AIMessage(content=status)]}
|
||||
296
참고/knowledge_agent-main/sub_agents/researcher.py
Normal file
296
참고/knowledge_agent-main/sub_agents/researcher.py
Normal file
@@ -0,0 +1,296 @@
|
||||
# sub_agents/researcher.py
|
||||
from langchain.agents import create_openai_tools_agent, AgentExecutor
|
||||
from langchain_core.prompts import ChatPromptTemplate
|
||||
from state import AgentState
|
||||
from db_utils import initialize_researcher, update_researcher_report, extract_and_clean_json, get_document_object, update_document_object
|
||||
from tools import process_url
|
||||
from utils import filter_content_for_summarization
|
||||
from terminal_utils import print_colorful_break
|
||||
|
||||
async def researcher_agent_node(state: AgentState):
|
||||
"""The main node for the researcher workflow."""
|
||||
print_colorful_break("RESEARCHER")
|
||||
logger = state['logger']
|
||||
report_id = state.get("researcher_report_id")
|
||||
gaps_todo = state.get("researcher_gaps_todo", [])
|
||||
gaps_complete = state.get("researcher_gaps_complete", [])
|
||||
final_status = None
|
||||
|
||||
if not report_id:
|
||||
try:
|
||||
init_result = initialize_researcher(state['timestamp'])
|
||||
report_id = init_result.get("researcher_report_id")
|
||||
gaps_todo = init_result.get("researcher_gaps_todo")
|
||||
gaps_complete = []
|
||||
status = f"--- Initialized researcher state: {init_result} ---"
|
||||
logger.info(status)
|
||||
except Exception as e:
|
||||
status = f"Failed to initialize researcher state: {e}"
|
||||
logger.error(status)
|
||||
return {"status": status}
|
||||
|
||||
# Define Planner Agent
|
||||
with open("prompts/planner_prompt.txt", "r") as f:
|
||||
planner_prompt_template = f.read()
|
||||
planner_prompt = ChatPromptTemplate.from_template(planner_prompt_template)
|
||||
try:
|
||||
planner_agent_runnable = create_openai_tools_agent(state['model'], [], planner_prompt)
|
||||
planner_executor = AgentExecutor(agent=planner_agent_runnable, tools=[], verbose=True)
|
||||
except Exception as e:
|
||||
status = f"Failed to create planner agent executor: {e}"
|
||||
logger.error(status)
|
||||
return {"status": status}
|
||||
|
||||
# Define Refiner Agent
|
||||
with open("prompts/refiner_prompt.txt", "r") as f:
|
||||
refiner_prompt_template = f.read()
|
||||
refiner_prompt = ChatPromptTemplate.from_template(refiner_prompt_template)
|
||||
try:
|
||||
refiner_agent_runnable = create_openai_tools_agent(state['model'], [], refiner_prompt)
|
||||
refiner_executor = AgentExecutor(agent=refiner_agent_runnable, tools=[], verbose=True)
|
||||
except Exception as e:
|
||||
status = f"Failed to create refiner agent executor: {e}"
|
||||
logger.error(status)
|
||||
return {"status": status}
|
||||
|
||||
# Define Summarizer Agent
|
||||
with open("prompts/summarizer_prompt.txt", "r") as f:
|
||||
summarizer_prompt_template = f.read()
|
||||
summarizer_prompt = ChatPromptTemplate.from_template(summarizer_prompt_template)
|
||||
try:
|
||||
summarizer_agent_runnable = create_openai_tools_agent(state['model'], [], summarizer_prompt)
|
||||
summarizer_executor = AgentExecutor(agent=summarizer_agent_runnable, tools=[], verbose=True)
|
||||
except Exception as e:
|
||||
status = f"Failed to create summarizer agent executor: {e}"
|
||||
logger.error(status)
|
||||
return {"status": status}
|
||||
|
||||
# Get the tools
|
||||
google_search_tool = next((tool for tool in state['mcp_tools'] if tool.name == 'google_search'), None)
|
||||
if not google_search_tool:
|
||||
status = "google_search tool not found."
|
||||
logger.error(status)
|
||||
return {"status": status}
|
||||
|
||||
|
||||
# Main control loop
|
||||
try:
|
||||
if isinstance(gaps_todo, list) and gaps_todo:
|
||||
for current_gap in list(gaps_todo):
|
||||
gap_id = current_gap['gap_id']
|
||||
research_topic = current_gap['research_topic']
|
||||
all_searches_for_gap = []
|
||||
|
||||
research_topic_title = research_topic.get('title', 'No Title')
|
||||
status = f"Starting research for gap: {gap_id}, research topic: {research_topic_title}"
|
||||
logger.info(status)
|
||||
|
||||
try:
|
||||
# 1. Planning Step
|
||||
status = f"Invoking planner for gap {gap_id}."
|
||||
logger.info(status)
|
||||
planner_result = await planner_executor.ainvoke({"input": research_topic})
|
||||
planner_output = extract_and_clean_json(planner_result.get("output", ""))
|
||||
planned_searches = planner_output.get("searches", [])
|
||||
status = f"Planner for gap {gap_id} returned {len(planned_searches)} searches."
|
||||
logger.info(status)
|
||||
|
||||
# 2. Initial Execution Step
|
||||
for planned_search in planned_searches:
|
||||
query = planned_search.get("query")
|
||||
rationale = planned_search.get("rationale")
|
||||
parameters = planned_search.get("parameters", {})
|
||||
search_id = planned_search.get("search_id")
|
||||
if not query:
|
||||
continue
|
||||
|
||||
if 'query' not in parameters:
|
||||
parameters['query'] = query
|
||||
|
||||
try:
|
||||
status = f"Executing search for gap {gap_id}, with parameters: {parameters}"
|
||||
logger.info(status)
|
||||
|
||||
raw_search_results = await google_search_tool.arun(parameters)
|
||||
search_results = extract_and_clean_json(raw_search_results)
|
||||
|
||||
for i, result in enumerate(search_results):
|
||||
url = result.get('url')
|
||||
if url:
|
||||
logger.info(f"Attempting document store initialzation for URL: {url}")
|
||||
try:
|
||||
url_id, url_status = await process_url(url, logger)
|
||||
except Exception as e:
|
||||
status = f"Error processing URL {url}: {e}"
|
||||
logger.error(status, exc_info=True)
|
||||
continue
|
||||
search_results[i]['url_id'] = url_id
|
||||
if url_status == "new":
|
||||
status=f"New URL {url} added to the database with ID: {url_id}."
|
||||
else:
|
||||
status = f"URL {url} already exists in the database with ID: {url_id}."
|
||||
logger.info(status)
|
||||
|
||||
search_object = {
|
||||
"search_id": search_id,
|
||||
"rationale": rationale,
|
||||
"parameters": parameters,
|
||||
"results": search_results
|
||||
}
|
||||
all_searches_for_gap.append(search_object)
|
||||
|
||||
status = f"Search for gap {gap_id} finished for query: '{query}'"
|
||||
logger.info(status)
|
||||
|
||||
except Exception as e:
|
||||
status = f"Search for gap {gap_id}, query '{query}' failed: {e}"
|
||||
logger.error(status)
|
||||
continue
|
||||
|
||||
# 3. Refinement Step
|
||||
status = f"Invoking refiner for gap {gap_id}."
|
||||
logger.info(status)
|
||||
refiner_input = {"research_topic": research_topic, "search_results": all_searches_for_gap}
|
||||
refiner_result = await refiner_executor.ainvoke({"input": refiner_input})
|
||||
refiner_output = extract_and_clean_json(refiner_result.get("output", ""))
|
||||
|
||||
status_check = ""
|
||||
if isinstance(refiner_output, dict):
|
||||
status_check = refiner_output.get("status", "").lower()
|
||||
elif isinstance(refiner_output, str):
|
||||
if "insufficient" in refiner_output.lower():
|
||||
status_check = "insufficient"
|
||||
|
||||
if status_check == "insufficient":
|
||||
refined_searches = []
|
||||
if isinstance(refiner_output, dict):
|
||||
refined_searches = refiner_output.get("searches", [])
|
||||
status = f"Refiner for gap {gap_id} returned {len(refined_searches)} new searches."
|
||||
logger.info(status)
|
||||
|
||||
# 4. Refined Execution Step
|
||||
for refined_search in refined_searches:
|
||||
query = refined_search.get("query")
|
||||
rationale = refined_search.get("rationale")
|
||||
parameters = refined_search.get("parameters", {})
|
||||
search_id = refined_search.get("search_id")
|
||||
if not query:
|
||||
continue
|
||||
|
||||
if 'query' not in parameters:
|
||||
parameters['query'] = query
|
||||
|
||||
try:
|
||||
status = f"Executing refined search for gap {gap_id}, with parameters: {parameters}"
|
||||
logger.info(status)
|
||||
|
||||
raw_search_results = await google_search_tool.arun(parameters)
|
||||
search_results = extract_and_clean_json(raw_search_results)
|
||||
|
||||
for i, result in enumerate(search_results):
|
||||
url = result.get('url')
|
||||
if url:
|
||||
logger.info(f"Attempting document store initialzation for URL: {url}")
|
||||
try:
|
||||
url_id, url_status = await process_url(url, logger)
|
||||
except Exception as e:
|
||||
status = f"Error processing URL {url}: {e}"
|
||||
logger.error(status, exc_info=True)
|
||||
continue
|
||||
search_results[i]['url_id'] = url_id
|
||||
if url_status == "new":
|
||||
status=f"New URL {url} added to the database with ID: {url_id}."
|
||||
else:
|
||||
status = f"URL {url} already exists in the database with ID: {url_id}."
|
||||
logger.info(status)
|
||||
|
||||
search_object = {
|
||||
"search_id": search_id,
|
||||
"rationale": rationale,
|
||||
"parameters": parameters,
|
||||
"results": search_results
|
||||
}
|
||||
all_searches_for_gap.append(search_object)
|
||||
|
||||
status = f"Refined search for gap {gap_id} finished for query: '{query}'"
|
||||
logger.info(status)
|
||||
except Exception as e:
|
||||
status = f"Refined search for gap {gap_id}, query '{query}' failed: {e}"
|
||||
logger.error(status)
|
||||
continue
|
||||
else:
|
||||
status = f"Refiner for gap {gap_id} deemed results sufficient."
|
||||
logger.info(status)
|
||||
|
||||
# 5. Summarization Step
|
||||
status = f"Starting summarization for gap {gap_id}."
|
||||
logger.info(status)
|
||||
for search in all_searches_for_gap:
|
||||
for i, result in enumerate(search.get('results', [])):
|
||||
url = result.get('url')
|
||||
url_id = result.get('url_id')
|
||||
url_summary = get_document_object(url_id, type="summary")
|
||||
if url_summary:
|
||||
logger.info(f"Skipping summarization for url_id: {url_id}, summary already exists.")
|
||||
continue
|
||||
else:
|
||||
markdown_content = get_document_object(url_id, type="markdown_content")
|
||||
if not markdown_content or markdown_content.startswith("[MARKDOWN_GENERATION_FAILED"):
|
||||
logger.info(f"Skipping summarization for url_id: {url_id}, no valid markdown content available.")
|
||||
continue
|
||||
|
||||
logger.info(f"Attempting summary for url_id: {url_id}")
|
||||
try:
|
||||
filtered_content = filter_content_for_summarization(markdown_content)
|
||||
summarizer_result = await summarizer_executor.ainvoke({"input": filtered_content})
|
||||
summary_output = extract_and_clean_json(summarizer_result.get("output", ""))
|
||||
|
||||
if isinstance(summary_output, dict):
|
||||
summary = summary_output.get('summary')
|
||||
else:
|
||||
summary = str(summary_output)
|
||||
|
||||
update_document_object(url_id, type="summary", object=summary)
|
||||
status = f"Successfully summarized and updated document for url_id: {url_id}"
|
||||
logger.info(status)
|
||||
except Exception as e:
|
||||
status = f"Error summarizing url_id {url_id}: {e}"
|
||||
logger.error(status, exc_info=True)
|
||||
continue
|
||||
|
||||
# 6. Update Step
|
||||
status = f"Preparing to update report for gap {gap_id} with {len(all_searches_for_gap)} searches."
|
||||
logger.info(status)
|
||||
try:
|
||||
update_researcher_report(report_id, gap_id, all_searches_for_gap)
|
||||
gaps_complete.append(gap_id)
|
||||
gaps_todo = [g for g in gaps_todo if g.get("gap_id") != gap_id]
|
||||
status = f"Updated researcher report for gap: {gap_id}"
|
||||
logger.info(status)
|
||||
except Exception as e:
|
||||
status = f"Error updating report for gap {gap_id}: {e}"
|
||||
logger.error(status, exc_info=True)
|
||||
continue
|
||||
|
||||
status = f"--- Successfully completed research and report writing for gap: {gap_id} ---"
|
||||
logger.info(status)
|
||||
|
||||
except Exception as e:
|
||||
status = f"An unexpected error occurred while processing gap {gap_id}: {e}"
|
||||
logger.error(status, exc_info=True)
|
||||
continue
|
||||
|
||||
except Exception as e:
|
||||
final_status = f"Main loop failed: {e}"
|
||||
logger.error(final_status, exc_info=True)
|
||||
|
||||
if not final_status:
|
||||
final_status = f"Successfully and incrementally completed researcher report with ID {report_id} and wrote report to DB."
|
||||
logger.info(final_status)
|
||||
|
||||
return {
|
||||
"status": final_status,
|
||||
"researcher_report_id": report_id,
|
||||
"researcher_gaps_todo": gaps_todo,
|
||||
"researcher_gaps_complete": gaps_complete
|
||||
}
|
||||
22
참고/knowledge_agent-main/terminal_utils.py
Normal file
22
참고/knowledge_agent-main/terminal_utils.py
Normal file
@@ -0,0 +1,22 @@
|
||||
# terminal_utils.py
|
||||
import random
|
||||
|
||||
def print_colorful_break(text: str):
|
||||
"""Prints a colorful horizontal break with a title."""
|
||||
colors = [
|
||||
"\033[91m", # Red
|
||||
"\033[92m", # Green
|
||||
"\033[93m", # Yellow
|
||||
"\033[94m", # Blue
|
||||
"\033[95m", # Magenta
|
||||
"\033[96m", # Cyan
|
||||
]
|
||||
reset_color = "\033[0m"
|
||||
|
||||
color = random.choice(colors)
|
||||
|
||||
width = 80
|
||||
padding = (width - len(text) - 2)
|
||||
|
||||
print(f"\n{color}{'=' * (padding // 2)} {text} {'=' * (padding - (padding // 2))}{reset_color}\n")
|
||||
|
||||
123
참고/knowledge_agent-main/tools.py
Normal file
123
참고/knowledge_agent-main/tools.py
Normal file
@@ -0,0 +1,123 @@
|
||||
# tools.py
|
||||
from langchain_core.tools import tool, ToolException
|
||||
from db_utils import add_url_or_get_id, update_document_content
|
||||
from utils import format_bytes
|
||||
import requests
|
||||
import io
|
||||
import pdfplumber
|
||||
from playwright.async_api import async_playwright
|
||||
import trafilatura
|
||||
from trafilatura.settings import use_config
|
||||
|
||||
|
||||
@tool
|
||||
def human_approval(plan: str) -> str:
|
||||
"""
|
||||
Asks for human approval for a given plan.
|
||||
The plan is a string that describes the actions to be taken.
|
||||
Returns 'approved' or 'denied'.
|
||||
"""
|
||||
import logging
|
||||
logger = logging.getLogger('KnowledgeAgent')
|
||||
|
||||
status = f"PROPOSED PLAN:\n{plan}"
|
||||
print(f"\n[INFO] {status}")
|
||||
logger.info(f"{status}")
|
||||
try:
|
||||
response = input("Do you approve this plan? (y/n): ").lower()
|
||||
if response == 'y':
|
||||
status = "Approved."
|
||||
print(status)
|
||||
logger.info(status)
|
||||
return status
|
||||
status = "Denied."
|
||||
print(status)
|
||||
logger.info(status)
|
||||
return status
|
||||
except Exception as e:
|
||||
status = f"Error in human_approval: {e}"
|
||||
print(status)
|
||||
logger.error(status)
|
||||
raise ToolException(status)
|
||||
|
||||
async def fetch_and_generate_markdown(url: str, logger):
|
||||
"""Fetches raw content from a URL and generates markdown using a hybrid approach."""
|
||||
raw_document = b''
|
||||
markdown_content = ""
|
||||
MIN_CONTENT_LENGTH = 200 # Minimum character length to be considered valid content
|
||||
|
||||
try:
|
||||
# Use a HEAD request to check the content type first
|
||||
head_response = requests.head(url, timeout=10, headers={'User-Agent': 'Mozilla/5.0'})
|
||||
head_response.raise_for_status()
|
||||
content_type = head_response.headers.get("Content-Type", "")
|
||||
|
||||
if "text/html" in content_type:
|
||||
# 1. Try Trafilatura first
|
||||
logger.info(f"Attempting to extract content with Trafilatura from: {url}")
|
||||
config = use_config()
|
||||
config.set("DEFAULT", "EXTRACTION_TIMEOUT", "0")
|
||||
downloaded_html = trafilatura.fetch_url(url)
|
||||
if downloaded_html:
|
||||
raw_document = downloaded_html.encode('utf-8')
|
||||
markdown_content = trafilatura.extract(
|
||||
downloaded_html,
|
||||
config=config,
|
||||
include_comments=False,
|
||||
include_tables=True,
|
||||
)
|
||||
|
||||
# 2. Validate output and fallback to Playwright if necessary
|
||||
if not markdown_content or len(markdown_content) < MIN_CONTENT_LENGTH:
|
||||
logger.warning(f"Trafilatura extraction failed or content too short. Falling back to Playwright for: {url}")
|
||||
async with async_playwright() as p:
|
||||
browser = await p.chromium.launch()
|
||||
page = await browser.new_page()
|
||||
await page.goto(url, wait_until="networkidle", timeout=15000)
|
||||
raw_document = (await page.content()).encode('utf-8')
|
||||
# Use a robust JS evaluation to get main content text
|
||||
markdown_content = await page.evaluate('''() => {
|
||||
const main = document.querySelector('main, #main, #content, [role="main"]');
|
||||
return main ? main.innerText : document.body.innerText;
|
||||
}''')
|
||||
await browser.close()
|
||||
logger.info(f"Successfully fetched content with Playwright for url: {url}")
|
||||
else:
|
||||
logger.info(f"Successfully extracted content with Trafilatura for url: {url}")
|
||||
|
||||
elif "application/pdf" in content_type:
|
||||
logger.info(f"Downloading PDF content from: {url}")
|
||||
response = requests.get(url, timeout=10, headers={'User-Agent': 'Mozilla/5.0'})
|
||||
response.raise_for_status()
|
||||
raw_document = response.content
|
||||
with pdfplumber.open(io.BytesIO(raw_document)) as pdf:
|
||||
markdown_content = "\n".join(page.extract_text() for page in pdf.pages if page.extract_text())
|
||||
logger.info(f"Successfully processed PDF for url: {url}")
|
||||
|
||||
else:
|
||||
markdown_content = f"[MARKDOWN_GENERATION_FAILED: Unsupported content type '{content_type}']"
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"An unexpected error occurred while processing {url}: {e}", exc_info=True)
|
||||
markdown_content = f"[MARKDOWN_GENERATION_FAILED: {e}]"
|
||||
|
||||
return raw_document, markdown_content
|
||||
|
||||
async def process_url(url: str, logger):
|
||||
"""Downloads, processes, and stores content from a URL."""
|
||||
url_id, url_status = add_url_or_get_id(url)
|
||||
if url_status == "existing":
|
||||
# Optionally, we could check here if the content is missing and re-process if needed
|
||||
return url_id, url_status
|
||||
|
||||
raw_document, markdown_content = await fetch_and_generate_markdown(url, logger)
|
||||
|
||||
if raw_document or markdown_content:
|
||||
logger.info(f"Updating document content for url_id: {url_id}")
|
||||
try:
|
||||
update_document_content(url_id, raw_document, markdown_content)
|
||||
logger.info(f"Successfully updated document content for url_id: {url_id}")
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to update document content for url_id {url_id}: {e}", exc_info=True)
|
||||
|
||||
return url_id, url_status
|
||||
43
참고/knowledge_agent-main/utils.py
Normal file
43
참고/knowledge_agent-main/utils.py
Normal file
@@ -0,0 +1,43 @@
|
||||
# utils.py
|
||||
import tiktoken
|
||||
|
||||
def format_bytes(byte_count):
|
||||
"""
|
||||
Formats an integer of bytes into a human-readable string in B, KB, or MB.
|
||||
|
||||
Args:
|
||||
byte_count: An integer representing the number of bytes.
|
||||
|
||||
Returns:
|
||||
A string formatted as B, KB, or MB with commas and no decimal places.
|
||||
"""
|
||||
if not isinstance(byte_count, int):
|
||||
raise TypeError("Input must be an integer.")
|
||||
|
||||
if byte_count < 1024:
|
||||
# Format as Bytes if less than 1 KB
|
||||
return f"{byte_count:,} B"
|
||||
elif byte_count < 1024 * 1024:
|
||||
# Format as Kilobytes if less than 1 MB
|
||||
kb_value = round(byte_count / 1024)
|
||||
return f"{kb_value:,} KB"
|
||||
else:
|
||||
# Format as Megabytes for 1 MB or more
|
||||
mb_value = round(byte_count / (1024 * 1024))
|
||||
return f"{mb_value:,} MB"
|
||||
|
||||
def filter_content_for_summarization(content: str) -> str:
|
||||
"""Truncates content to a safe number of tokens for the summarization model."""
|
||||
MAX_TOKENS = 16384 # Cap content for summarization at 16k tokens for efficiency
|
||||
try:
|
||||
encoding = tiktoken.get_encoding("cl100k_base")
|
||||
tokens = encoding.encode(content)
|
||||
if len(tokens) > MAX_TOKENS:
|
||||
truncated_tokens = tokens[:MAX_TOKENS]
|
||||
return encoding.decode(truncated_tokens)
|
||||
else:
|
||||
return content
|
||||
except Exception as e:
|
||||
# Fallback to simple character truncation if tokenization fails
|
||||
print(f"Token-based filtering failed: {e}. Falling back to character-based truncation.")
|
||||
return content[:MAX_TOKENS * 4] # Rough approximation
|
||||
Reference in New Issue
Block a user