참고소스 수정본
This commit is contained in:
@@ -0,0 +1,21 @@
|
||||
import asyncio
|
||||
|
||||
import neo4j
|
||||
from neo4j_graphrag.experimental.components.neo4j_reader import Neo4jChunkReader
|
||||
from neo4j_graphrag.experimental.components.types import LexicalGraphConfig, TextChunks
|
||||
|
||||
|
||||
async def main(driver: neo4j.Driver) -> TextChunks:
|
||||
config = LexicalGraphConfig( # only needed to overwrite the default values
|
||||
chunk_node_label="TextPart",
|
||||
)
|
||||
reader = Neo4jChunkReader(driver)
|
||||
result = await reader.run(lexical_graph_config=config)
|
||||
return result
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
with neo4j.GraphDatabase.driver(
|
||||
"bolt://localhost:7687", auth=("neo4j", "password")
|
||||
) as driver:
|
||||
print(asyncio.run(main(driver)))
|
||||
@@ -0,0 +1,66 @@
|
||||
"""This examples shows how to create a custom component
|
||||
that can be added to a Pipeline with:
|
||||
|
||||
c = MyComponent(min_value=0, max_value=10)
|
||||
pipe = Pipeline()
|
||||
pipe.add_component(c, name="my_component")
|
||||
"""
|
||||
|
||||
import random
|
||||
|
||||
from neo4j_graphrag.experimental.pipeline import Component, DataModel
|
||||
from pydantic import BaseModel, validate_call
|
||||
|
||||
|
||||
class ComponentInputModel(BaseModel):
|
||||
"""A class to model the component inputs.
|
||||
This is not required, inputs can also be passed individually.
|
||||
|
||||
Note: can also inherit from DataModel.
|
||||
"""
|
||||
|
||||
text: str
|
||||
|
||||
|
||||
class ComponentResultModel(DataModel):
|
||||
"""A class to model the component outputs.
|
||||
Each component must have such a description of the output,
|
||||
so that the parameter mapping can be validated before the
|
||||
pipeline run starts.
|
||||
"""
|
||||
|
||||
value: int
|
||||
text: str
|
||||
|
||||
|
||||
class MyComponent(Component):
|
||||
"""Multiplies an input text by a random number
|
||||
between `min_value` and `max_value`
|
||||
"""
|
||||
|
||||
def __init__(self, min_value: int, max_value: int) -> None:
|
||||
self.min_value = min_value
|
||||
self.max_value = max_value
|
||||
|
||||
# this decorator is required when a Pydantic model is used in the inputs
|
||||
@validate_call
|
||||
async def run(self, inputs: ComponentInputModel) -> ComponentResultModel:
|
||||
# logic here
|
||||
random_value = random.randint(self.min_value, self.max_value)
|
||||
return ComponentResultModel(
|
||||
value=random_value,
|
||||
text=inputs.text * random_value,
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import asyncio
|
||||
|
||||
c = MyComponent(min_value=0, max_value=10)
|
||||
print(
|
||||
asyncio.run(
|
||||
c.run(
|
||||
inputs={"text": "Hello"} # type: ignore
|
||||
)
|
||||
)
|
||||
)
|
||||
@@ -0,0 +1,41 @@
|
||||
from typing import Any, Optional
|
||||
|
||||
from neo4j_graphrag.experimental.components.entity_relation_extractor import (
|
||||
EntityRelationExtractor,
|
||||
OnError,
|
||||
)
|
||||
from neo4j_graphrag.experimental.components.types import (
|
||||
DocumentInfo,
|
||||
LexicalGraphConfig,
|
||||
Neo4jGraph,
|
||||
TextChunks,
|
||||
)
|
||||
|
||||
|
||||
class MyExtractor(EntityRelationExtractor):
|
||||
def __init__(
|
||||
self,
|
||||
*args: Any,
|
||||
on_error: OnError = OnError.IGNORE,
|
||||
create_lexical_graph: bool = True,
|
||||
**kwargs: Any,
|
||||
) -> None:
|
||||
super().__init__(
|
||||
*args,
|
||||
on_error=on_error,
|
||||
create_lexical_graph=create_lexical_graph,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
async def run(
|
||||
self,
|
||||
chunks: TextChunks,
|
||||
document_info: Optional[DocumentInfo] = None,
|
||||
lexical_graph_config: Optional[LexicalGraphConfig] = None,
|
||||
**kwargs: Any,
|
||||
) -> Neo4jGraph:
|
||||
# Implement your logic here
|
||||
# you can loop over all text chunks with:
|
||||
for chunk in chunks.chunks:
|
||||
pass
|
||||
return Neo4jGraph(nodes=[], relationships=[])
|
||||
@@ -0,0 +1,33 @@
|
||||
from neo4j_graphrag.experimental.components.entity_relation_extractor import (
|
||||
LLMEntityRelationExtractor,
|
||||
)
|
||||
from neo4j_graphrag.experimental.components.types import (
|
||||
Neo4jGraph,
|
||||
TextChunk,
|
||||
TextChunks,
|
||||
)
|
||||
from neo4j_graphrag.llm import LLMInterface
|
||||
|
||||
|
||||
async def main(llm: LLMInterface) -> Neo4jGraph:
|
||||
"""
|
||||
|
||||
Args:
|
||||
llm (LLMInterface): Any LLM implemented in neo4j_graphrag.llm or from LangChain chat models.
|
||||
"""
|
||||
extractor = LLMEntityRelationExtractor(
|
||||
llm=llm,
|
||||
# optional: customize the prompt used for entity and relation extraction
|
||||
# prompt_template="",
|
||||
# optional: disable the creation of the lexical graph (Document and Chunk nodes)
|
||||
# create_lexical_graph=False,
|
||||
# optional: if an LLM error happens, ignore the chunk and continue process with the next ones
|
||||
# default value is OnError.RAISE which will end the process
|
||||
# on_error=OnError.IGNORE,
|
||||
# optional: tune the max_concurrency parameter to optimize speed
|
||||
# max_concurrency=5,
|
||||
)
|
||||
graph = await extractor.run(
|
||||
chunks=TextChunks(chunks=[TextChunk(text="....", index=0)])
|
||||
)
|
||||
return graph
|
||||
@@ -0,0 +1,33 @@
|
||||
from neo4j_graphrag.experimental.components.entity_relation_extractor import (
|
||||
LLMEntityRelationExtractor,
|
||||
)
|
||||
from neo4j_graphrag.experimental.components.types import (
|
||||
Neo4jGraph,
|
||||
TextChunk,
|
||||
TextChunks,
|
||||
)
|
||||
from neo4j_graphrag.llm import LLMInterface
|
||||
|
||||
|
||||
async def main(llm: LLMInterface) -> Neo4jGraph:
|
||||
"""
|
||||
|
||||
Args:
|
||||
llm (LLMInterface): Any LLM implemented in neo4j_graphrag.llm or from LangChain chat models.
|
||||
"""
|
||||
extractor = LLMEntityRelationExtractor(
|
||||
llm=llm,
|
||||
# optional: customize the prompt used for entity and relation extraction
|
||||
# prompt_template="",
|
||||
# optional: disable the creation of the lexical graph (Document and Chunk nodes)
|
||||
# create_lexical_graph=False,
|
||||
# optional: if an LLM error happens, ignore the chunk and continue process with the next ones
|
||||
# default value is OnError.RAISE which will end the process
|
||||
# on_error=OnError.IGNORE,
|
||||
# optional: tune the max_concurrency parameter to optimize speed
|
||||
# max_concurrency=5,
|
||||
)
|
||||
graph = await extractor.run(
|
||||
chunks=TextChunks(chunks=[TextChunk(text="....", index=0)])
|
||||
)
|
||||
return graph
|
||||
@@ -0,0 +1,74 @@
|
||||
"""
|
||||
Simple example demonstrating structured output with LLMEntityRelationExtractor.
|
||||
|
||||
This example shows how to use structured output for more reliable entity and
|
||||
relationship extraction with automatic schema validation.
|
||||
|
||||
The Neo4jGraph schema is now compatible with both OpenAI and VertexAI structured
|
||||
output APIs, with strict schema validation (additionalProperties: false) and
|
||||
proper required field definitions.
|
||||
|
||||
Prerequisites:
|
||||
- Google Cloud credentials configured for VertexAI
|
||||
- Or OpenAI API key set in OPENAI_API_KEY environment variable
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
from dotenv import load_dotenv
|
||||
|
||||
from neo4j_graphrag.experimental.components.entity_relation_extractor import (
|
||||
LLMEntityRelationExtractor,
|
||||
)
|
||||
from neo4j_graphrag.experimental.components.types import (
|
||||
Neo4jGraph,
|
||||
TextChunk,
|
||||
TextChunks,
|
||||
)
|
||||
from neo4j_graphrag.llm import VertexAILLM
|
||||
|
||||
|
||||
async def main() -> Neo4jGraph:
|
||||
"""
|
||||
Demonstrates entity and relation extraction with structured output.
|
||||
|
||||
With use_structured_output=True:
|
||||
- Uses LLMInterfaceV2 (list of messages)
|
||||
- Passes Neo4jGraph Pydantic model as response_format to invoke()
|
||||
- Ensures response conforms to expected graph structure
|
||||
- Provides automatic type validation
|
||||
- Reduces need for JSON repair and error handling
|
||||
"""
|
||||
load_dotenv()
|
||||
# Initialize LLM - no response_format in constructor!
|
||||
llm = VertexAILLM(model_name="gemini-2.5-flash")
|
||||
|
||||
# llm = OpenAILLM(
|
||||
# model_name="gpt-5-mini",
|
||||
# model_params={"temperature": 0}
|
||||
# )
|
||||
|
||||
# Enable structured output for reliable extraction
|
||||
extractor = LLMEntityRelationExtractor(
|
||||
llm=llm,
|
||||
use_structured_output=True, # This is the key parameter!
|
||||
)
|
||||
|
||||
# Sample text about a person and organization
|
||||
sample_text = """
|
||||
Albert Einstein was a theoretical physicist who developed the theory of relativity.
|
||||
He worked at the Institute for Advanced Study in Princeton from 1933 until his death in 1955.
|
||||
"""
|
||||
|
||||
# Extract entities and relationships
|
||||
graph = await extractor.run(
|
||||
chunks=TextChunks(chunks=[TextChunk(text=sample_text, index=0)])
|
||||
)
|
||||
|
||||
return graph
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
# Run extraction
|
||||
graph = asyncio.run(main())
|
||||
|
||||
print(graph)
|
||||
@@ -0,0 +1,30 @@
|
||||
from neo4j_graphrag.experimental.components.lexical_graph import (
|
||||
LexicalGraphBuilder,
|
||||
)
|
||||
from neo4j_graphrag.experimental.components.types import (
|
||||
GraphResult,
|
||||
LexicalGraphConfig,
|
||||
TextChunk,
|
||||
TextChunks,
|
||||
)
|
||||
|
||||
|
||||
async def main() -> GraphResult:
|
||||
# optionally, define a LexicalGraphConfig object
|
||||
# shown below with default values
|
||||
config = LexicalGraphConfig(
|
||||
chunk_node_label="Chunk",
|
||||
document_node_label="Document",
|
||||
chunk_to_document_relationship_type="PART_OF_DOCUMENT",
|
||||
next_chunk_relationship_type="NEXT_CHUNK",
|
||||
node_to_chunk_relationship_type="PART_OF_CHUNK",
|
||||
chunk_embedding_property="embeddings",
|
||||
)
|
||||
builder = LexicalGraphBuilder(
|
||||
config=config, # optional
|
||||
)
|
||||
graph_result = await builder.run(
|
||||
text_chunks=TextChunks(chunks=[TextChunk(text="....", index=0)]),
|
||||
# document_info={"path": "example"}, # uncomment to create a "Document" node
|
||||
)
|
||||
return graph_result
|
||||
@@ -0,0 +1,27 @@
|
||||
"""Create a custom data loader to transform content into text."""
|
||||
|
||||
from pathlib import Path
|
||||
from typing import Dict, Optional, Union
|
||||
|
||||
from fsspec import AbstractFileSystem
|
||||
|
||||
from neo4j_graphrag.experimental.components.data_loader import DataLoader
|
||||
from neo4j_graphrag.experimental.components.types import DocumentInfo, LoadedDocument
|
||||
|
||||
|
||||
class MyLoader(DataLoader):
|
||||
async def run(
|
||||
self,
|
||||
filepath: Union[str, Path],
|
||||
metadata: Optional[Dict[str, str]] = None,
|
||||
fs: Optional[Union[AbstractFileSystem, str]] = None,
|
||||
) -> LoadedDocument:
|
||||
# Implement logic here; use ``fs`` when reading from non-local storage.
|
||||
_ = fs
|
||||
return LoadedDocument(
|
||||
text="<extracted text>",
|
||||
document_info=DocumentInfo(
|
||||
path=str(filepath),
|
||||
metadata=metadata,
|
||||
),
|
||||
)
|
||||
@@ -0,0 +1,20 @@
|
||||
"""Use the PdfLoader component to extract text from a PDF file."""
|
||||
|
||||
import asyncio
|
||||
from pathlib import Path
|
||||
|
||||
from neo4j_graphrag.experimental.components.data_loader import PdfLoader
|
||||
|
||||
root_dir = Path(__file__).parents[4]
|
||||
file_path = root_dir / "data" / "Harry Potter and the Chamber of Secrets Summary.pdf"
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
loader = PdfLoader()
|
||||
document = await loader.run(filepath=file_path)
|
||||
print(document.text[:200])
|
||||
print(document.document_info)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,17 @@
|
||||
"""Use the PdfLoader component to extract text from a remote PDF file."""
|
||||
|
||||
import asyncio
|
||||
|
||||
from neo4j_graphrag.experimental.components.data_loader import PdfLoader
|
||||
|
||||
url = "https://raw.githubusercontent.com/neo4j/neo4j-graphrag-python/c166afc4d5abc56a5686f3da46a97ed7c07da19d/examples/data/Harry%20Potter%20and%20the%20Chamber%20of%20Secrets%20Summary.pdf"
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
loader = PdfLoader()
|
||||
document = await loader.run(filepath=url, fs="http")
|
||||
print(document.text[:100])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,161 @@
|
||||
"""This example demonstrates how to use the GraphPruning component."""
|
||||
|
||||
import asyncio
|
||||
|
||||
from neo4j_graphrag.experimental.components.graph_pruning import GraphPruning
|
||||
from neo4j_graphrag.experimental.components.schema import (
|
||||
ConstraintType,
|
||||
GraphConstraintType,
|
||||
GraphSchema,
|
||||
NodeType,
|
||||
Pattern,
|
||||
PropertyType,
|
||||
RelationshipType,
|
||||
)
|
||||
from neo4j_graphrag.experimental.components.types import (
|
||||
Neo4jGraph,
|
||||
Neo4jNode,
|
||||
Neo4jRelationship,
|
||||
)
|
||||
|
||||
graph = Neo4jGraph(
|
||||
nodes=[
|
||||
Neo4jNode(
|
||||
id="Person/John",
|
||||
label="Person",
|
||||
properties={
|
||||
"firstName": "John",
|
||||
"lastName": "Doe",
|
||||
"occupation": "employee",
|
||||
},
|
||||
),
|
||||
Neo4jNode(
|
||||
id="Person/Jane",
|
||||
label="Person",
|
||||
properties={
|
||||
"firstName": "Jane",
|
||||
},
|
||||
),
|
||||
Neo4jNode(
|
||||
id="Person/Jack",
|
||||
label="Person",
|
||||
properties={"firstName": "Jack", "lastName": "Dae"},
|
||||
),
|
||||
Neo4jNode(
|
||||
id="Organization/Corp1",
|
||||
label="Organization",
|
||||
properties={"name": "Corp1"},
|
||||
),
|
||||
],
|
||||
relationships=[
|
||||
Neo4jRelationship(
|
||||
start_node_id="Person/John",
|
||||
end_node_id="Person/Jack",
|
||||
type="KNOWS",
|
||||
),
|
||||
Neo4jRelationship(
|
||||
start_node_id="Organization/Corp2",
|
||||
end_node_id="Person/Jack",
|
||||
type="WORKS_FOR",
|
||||
),
|
||||
Neo4jRelationship(
|
||||
start_node_id="Person/John",
|
||||
end_node_id="Person/Jack",
|
||||
type="PARENT_OF",
|
||||
),
|
||||
],
|
||||
)
|
||||
|
||||
schema = GraphSchema(
|
||||
node_types=(
|
||||
NodeType(
|
||||
label="Person",
|
||||
properties=[
|
||||
PropertyType(name="firstName", type="STRING"),
|
||||
PropertyType(name="lastName", type="STRING"),
|
||||
PropertyType(name="age", type="INTEGER"),
|
||||
],
|
||||
additional_properties=False,
|
||||
),
|
||||
NodeType(
|
||||
label="Organization",
|
||||
properties=[
|
||||
PropertyType(name="name", type="STRING"),
|
||||
PropertyType(name="address", type="STRING"),
|
||||
],
|
||||
additional_properties=True,
|
||||
),
|
||||
),
|
||||
relationship_types=(
|
||||
RelationshipType(
|
||||
label="WORKS_FOR",
|
||||
properties=[PropertyType(name="since", type="LOCAL_DATETIME")],
|
||||
additional_properties=True,
|
||||
),
|
||||
RelationshipType(
|
||||
label="KNOWS",
|
||||
),
|
||||
),
|
||||
patterns=(
|
||||
Pattern(source="Person", relationship="KNOWS", target="Person"),
|
||||
Pattern(source="Person", relationship="WORKS_FOR", target="Organization"),
|
||||
),
|
||||
constraints=(
|
||||
ConstraintType(
|
||||
type=GraphConstraintType.EXISTENCE,
|
||||
node_type="Person",
|
||||
property_names=("firstName",),
|
||||
relationship_type=None,
|
||||
),
|
||||
ConstraintType(
|
||||
type=GraphConstraintType.EXISTENCE,
|
||||
node_type="Person",
|
||||
property_names=("lastName",),
|
||||
relationship_type=None,
|
||||
),
|
||||
ConstraintType(
|
||||
type=GraphConstraintType.EXISTENCE,
|
||||
node_type="Organization",
|
||||
property_names=("name",),
|
||||
relationship_type=None,
|
||||
),
|
||||
),
|
||||
additional_node_types=False,
|
||||
additional_relationship_types=False,
|
||||
additional_patterns=False,
|
||||
)
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
pruner = GraphPruning()
|
||||
res = await pruner.run(graph, schema)
|
||||
print("=" * 20, "FINAL CLEANED GRAPH:", "=" * 20)
|
||||
print(res.graph)
|
||||
print("=" * 20, "PRUNED ITEM:", "=" * 20)
|
||||
print(res.pruning_stats)
|
||||
print("-" * 10, "PRUNED NODES:")
|
||||
for node in res.pruning_stats.pruned_nodes:
|
||||
print(
|
||||
node.item.label,
|
||||
"with properties",
|
||||
node.item.properties,
|
||||
"pruned because",
|
||||
node.pruned_reason,
|
||||
node.metadata,
|
||||
)
|
||||
print("-" * 10, "PRUNED RELATIONSHIPS:")
|
||||
for rel in res.pruning_stats.pruned_relationships:
|
||||
print(rel.item.type, "pruned because", rel.pruned_reason)
|
||||
print("-" * 10, "PRUNED PROPERTIES:")
|
||||
for prop in res.pruning_stats.pruned_properties:
|
||||
print(
|
||||
prop.item,
|
||||
"from node label",
|
||||
prop.label,
|
||||
"pruned because",
|
||||
prop.pruned_reason,
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,25 @@
|
||||
"""The base EntityResolver class does not enforce
|
||||
a specific signature for the run method, which makes it very flexible.
|
||||
"""
|
||||
|
||||
from typing import Any, Optional
|
||||
|
||||
import neo4j
|
||||
from neo4j_graphrag.experimental.components.resolver import EntityResolver
|
||||
from neo4j_graphrag.experimental.components.types import ResolutionStats
|
||||
|
||||
|
||||
class MyEntityResolver(EntityResolver):
|
||||
def __init__(
|
||||
self,
|
||||
driver: neo4j.Driver,
|
||||
filter_query: Optional[str] = None,
|
||||
) -> None:
|
||||
super().__init__(driver, filter_query)
|
||||
|
||||
async def run(self, *args: Any, **kwargs: Any) -> ResolutionStats:
|
||||
# logic here
|
||||
return ResolutionStats(
|
||||
number_of_nodes_to_resolve=0,
|
||||
number_of_created_nodes=0,
|
||||
)
|
||||
@@ -0,0 +1,36 @@
|
||||
"""The FuzzyMatchResolver merges nodes with same label
|
||||
and similar textual properties (by default using the "name" property) based on RapidFuzz
|
||||
for string matching.
|
||||
|
||||
If the resolution is intended to be applied only on some nodes, for instance nodes that
|
||||
belong to a specific document, a "WHERE" query can be added. The only variable in the
|
||||
query scope is "entity".
|
||||
|
||||
WARNING: this process is destructive, initial nodes are deleted and replaced
|
||||
by the resolved ones, but all relationships are kept.
|
||||
See apoc.refactor.mergeNodes documentation for more details.
|
||||
"""
|
||||
|
||||
from neo4j_graphrag.experimental.components.resolver import (
|
||||
FuzzyMatchResolver,
|
||||
)
|
||||
from neo4j_graphrag.experimental.components.types import ResolutionStats
|
||||
|
||||
import neo4j
|
||||
|
||||
|
||||
async def main(driver: neo4j.Driver) -> None:
|
||||
resolver = FuzzyMatchResolver(
|
||||
driver,
|
||||
# let's filter all entities that belong to a certain docId
|
||||
filter_query="WHERE (entity)-[:FROM_CHUNK]->(:Chunk)-[:FROM_DOCUMENT]->(doc:"
|
||||
"Document {id = 'docId'}",
|
||||
# optionally, change the properties used for resolution (default is "name")
|
||||
# resolve_properties=["name", "ssn"],
|
||||
# the similarity threshold (default is 0.8)
|
||||
# similarity_threshold=0.9
|
||||
# and the neo4j database where data is updated
|
||||
# neo4j_database="neo4j",
|
||||
)
|
||||
res: ResolutionStats = await resolver.run()
|
||||
print(res)
|
||||
@@ -0,0 +1,36 @@
|
||||
"""The SinglePropertyExactMatchResolver merge nodes with same label
|
||||
and exact same property value (by default using the "name" property).
|
||||
|
||||
If some nodes need to be excluded from the resolution, for instance nodes
|
||||
created from a previous run, a "WHERE" query can be added. The only variable
|
||||
in the query scope is "entity".
|
||||
|
||||
WARNING: this process is destructive, initial nodes are deleted and replaced
|
||||
by the resolved ones, but all relationships are kept.
|
||||
See apoc.refactor.mergeNodes documentation for more details.
|
||||
"""
|
||||
|
||||
import neo4j
|
||||
from neo4j_graphrag.experimental.components.resolver import (
|
||||
SinglePropertyExactMatchResolver,
|
||||
)
|
||||
from neo4j_graphrag.experimental.components.types import ResolutionStats
|
||||
|
||||
|
||||
async def main(driver: neo4j.Driver) -> None:
|
||||
resolver = SinglePropertyExactMatchResolver(
|
||||
driver,
|
||||
# let's filter out some entities assuming the EntityToExclude label
|
||||
# was manually added to nodes in the db
|
||||
filter_query="WHERE NOT entity:EntityToExclude",
|
||||
# another example: in some cases, we do not want to merge
|
||||
# entities whose name is John Doe because we don't know if it
|
||||
# corresponds to the same real person
|
||||
# filter_query="WHERE entity.name <> 'John Doe'",
|
||||
# optionally, change the property used for resolution (default is "name")
|
||||
# resolve_property="name",
|
||||
# and the neo4j database where data is updated
|
||||
# neo4j_database="neo4j",
|
||||
)
|
||||
res: ResolutionStats = await resolver.run()
|
||||
print(res)
|
||||
@@ -0,0 +1,37 @@
|
||||
"""The SpaCySemanticMatchResolver merges nodes with same label
|
||||
and similar textual properties (by default using the "name" property) based on spaCy
|
||||
embeddings and cosine similarities of embedding vectors.
|
||||
|
||||
If the resolution is intended to be applied only on some nodes, for instance nodes that
|
||||
belong to a specific document, a "WHERE" query can be added. The only variable in the
|
||||
query scope is "entity".
|
||||
|
||||
WARNING: this process is destructive, initial nodes are deleted and replaced
|
||||
by the resolved ones, but all relationships are kept.
|
||||
See apoc.refactor.mergeNodes documentation for more details.
|
||||
"""
|
||||
|
||||
import neo4j
|
||||
from neo4j_graphrag.experimental.components.resolver import (
|
||||
SpaCySemanticMatchResolver,
|
||||
)
|
||||
from neo4j_graphrag.experimental.components.types import ResolutionStats
|
||||
|
||||
|
||||
async def main(driver: neo4j.Driver) -> None:
|
||||
resolver = SpaCySemanticMatchResolver(
|
||||
driver,
|
||||
# let's filter all entities that belong to a certain docId
|
||||
filter_query="WHERE (entity)-[:FROM_CHUNK]->(:Chunk)-[:FROM_DOCUMENT]->(doc:"
|
||||
"Document {id = 'docId'}",
|
||||
# optionally, change the properties used for resolution (default is "name")
|
||||
# resolve_properties=["name", "ssn"],
|
||||
# the similarity threshold (default is 0.8)
|
||||
# similarity_threshold=0.9
|
||||
# the spaCy trained model (default is "en_core_web_lg")
|
||||
# spacy_model="en_core_web_sm"
|
||||
# and the neo4j database where data is updated
|
||||
# neo4j_database="neo4j",
|
||||
)
|
||||
res: ResolutionStats = await resolver.run()
|
||||
print(res)
|
||||
@@ -0,0 +1,57 @@
|
||||
# Copyright (c) "Neo4j"
|
||||
# Neo4j Sweden AB [https://neo4j.com]
|
||||
# #
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
# #
|
||||
# https://www.apache.org/licenses/LICENSE-2.0
|
||||
# #
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
from neo4j_graphrag.experimental.components.schema import (
|
||||
SchemaBuilder,
|
||||
NodeType,
|
||||
PropertyType,
|
||||
RelationshipType,
|
||||
)
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
schema_builder = SchemaBuilder()
|
||||
|
||||
result = await schema_builder.run(
|
||||
node_types=[
|
||||
NodeType(
|
||||
label="Person",
|
||||
properties=[
|
||||
PropertyType(name="name", type="STRING"),
|
||||
PropertyType(name="place_of_birth", type="STRING"),
|
||||
PropertyType(name="date_of_birth", type="DATE"),
|
||||
],
|
||||
),
|
||||
NodeType(
|
||||
label="Organization",
|
||||
properties=[
|
||||
PropertyType(name="name", type="STRING"),
|
||||
PropertyType(name="country", type="STRING"),
|
||||
],
|
||||
),
|
||||
],
|
||||
relationship_types=[
|
||||
RelationshipType(
|
||||
label="WORKED_ON",
|
||||
),
|
||||
RelationshipType(
|
||||
label="WORKED_FOR",
|
||||
),
|
||||
],
|
||||
patterns=[
|
||||
("Person", "WORKED_ON", "Field"),
|
||||
("Person", "WORKED_FOR", "Organization"),
|
||||
],
|
||||
)
|
||||
print(result)
|
||||
@@ -0,0 +1,43 @@
|
||||
"""This example demonstrates how to use the SchemaFromExistingGraphExtractor component
|
||||
to automatically extract a schema from an existing Neo4j database.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
from pprint import pprint
|
||||
|
||||
import neo4j
|
||||
|
||||
from neo4j_graphrag.experimental.components.schema import (
|
||||
SchemaFromExistingGraphExtractor,
|
||||
GraphSchema,
|
||||
)
|
||||
|
||||
|
||||
URI = "neo4j+s://demo.neo4jlabs.com"
|
||||
AUTH = ("recommendations", "recommendations")
|
||||
DATABASE = "recommendations"
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
"""Run the example."""
|
||||
|
||||
with neo4j.GraphDatabase.driver(
|
||||
URI,
|
||||
auth=AUTH,
|
||||
) as driver:
|
||||
extractor = SchemaFromExistingGraphExtractor(
|
||||
driver,
|
||||
# optional:
|
||||
neo4j_database=DATABASE,
|
||||
additional_patterns=True,
|
||||
additional_node_types=True,
|
||||
additional_relationship_types=True,
|
||||
additional_properties=True,
|
||||
)
|
||||
schema: GraphSchema = await extractor.run()
|
||||
# schema.store_as_json("my_schema.json")
|
||||
pprint(schema.model_dump())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,131 @@
|
||||
"""This example demonstrates how to use the SchemaFromTextExtractor component
|
||||
to automatically extract a schema from text and save it to JSON and YAML files.
|
||||
|
||||
The SchemaFromTextExtractor component uses an LLM to analyze the text and identify entities,
|
||||
relations, and their properties.
|
||||
|
||||
Note: This example requires an OpenAI API key to be set in the .env file.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from pathlib import Path
|
||||
from dotenv import load_dotenv
|
||||
|
||||
from neo4j_graphrag.experimental.components.schema import (
|
||||
SchemaFromTextExtractor,
|
||||
GraphSchema,
|
||||
)
|
||||
from neo4j_graphrag.llm import OpenAILLM
|
||||
|
||||
# Load environment variables from .env file
|
||||
load_dotenv()
|
||||
|
||||
# Configure logging
|
||||
logging.basicConfig()
|
||||
logging.getLogger("neo4j_graphrag").setLevel(logging.INFO)
|
||||
|
||||
# Sample text to extract schema from - it's about a company and its employees
|
||||
TEXT = """
|
||||
Acme Corporation was founded in 1985 by John Smith in New York City.
|
||||
The company specializes in manufacturing high-quality widgets and gadgets
|
||||
for the consumer electronics industry.
|
||||
|
||||
Sarah Johnson joined Acme in 2010 as a Senior Engineer and was promoted to
|
||||
Engineering Director in 2015. She oversees a team of 12 engineers working on
|
||||
next-generation products. Sarah holds a PhD in Electrical Engineering from MIT
|
||||
and has filed 5 patents during her time at Acme.
|
||||
|
||||
The company expanded to international markets in 2012, opening offices in London,
|
||||
Tokyo, and Berlin. Each office is managed by a regional director who reports
|
||||
directly to the CEO, Michael Brown, who took over leadership in 2008.
|
||||
|
||||
Acme's most successful product, the SuperWidget X1, was launched in 2018 and
|
||||
has sold over 2 million units worldwide. The product was developed by a team led
|
||||
by Robert Chen, who joined the company in 2016 after working at TechGiant for 8 years.
|
||||
|
||||
The company currently employs 250 people across its 4 locations and had a revenue
|
||||
of $75 million in the last fiscal year. Acme is planning to go public in 2024
|
||||
with an estimated valuation of $500 million.
|
||||
"""
|
||||
|
||||
# Define the file paths for saving the schema
|
||||
root_dir = Path(__file__).parents[4]
|
||||
OUTPUT_DIR = str(root_dir / "data")
|
||||
JSON_FILE_PATH = str(root_dir / "data" / "extracted_schema.json")
|
||||
YAML_FILE_PATH = str(root_dir / "data" / "extracted_schema.yaml")
|
||||
|
||||
|
||||
async def extract_and_save_schema() -> None:
|
||||
"""Extract schema from text and save it to JSON and YAML files."""
|
||||
|
||||
# Define LLM parameters
|
||||
llm_model_params = {
|
||||
"max_completion_tokens": 2000,
|
||||
"response_format": {"type": "json_object"},
|
||||
"temperature": 0, # Lower temperature for more consistent output
|
||||
}
|
||||
|
||||
# Create the LLM instance
|
||||
llm = OpenAILLM(
|
||||
model_name="gpt-5",
|
||||
model_params=llm_model_params,
|
||||
)
|
||||
|
||||
try:
|
||||
# Create a SchemaFromTextExtractor component with the default template
|
||||
schema_extractor = SchemaFromTextExtractor(llm=llm)
|
||||
|
||||
print("Extracting schema from text...")
|
||||
# Extract schema from text
|
||||
inferred_schema = await schema_extractor.run(text=TEXT)
|
||||
|
||||
# Ensure the output directory exists
|
||||
Path(OUTPUT_DIR).mkdir(exist_ok=True)
|
||||
|
||||
print(f"Saving schema to JSON file: {JSON_FILE_PATH}")
|
||||
# Save the schema to JSON file
|
||||
inferred_schema.save(JSON_FILE_PATH, overwrite=True)
|
||||
|
||||
print(f"Saving schema to YAML file: {YAML_FILE_PATH}")
|
||||
# Save the schema to YAML file
|
||||
inferred_schema.save(YAML_FILE_PATH, overwrite=True)
|
||||
|
||||
print("\nExtracted Schema Summary:")
|
||||
print(f"Node types: {list(inferred_schema.node_types)}")
|
||||
print(
|
||||
f"Relationship types: {list(inferred_schema.relationship_types if inferred_schema.relationship_types else [])}"
|
||||
)
|
||||
|
||||
if inferred_schema.patterns:
|
||||
print("\nPatterns:")
|
||||
for entity1, relation, entity2 in inferred_schema.patterns:
|
||||
print(f" {entity1} --[{relation}]--> {entity2}")
|
||||
|
||||
finally:
|
||||
# Close the LLM client
|
||||
if hasattr(llm, "aclose"):
|
||||
await llm.aclose()
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
"""Run the example."""
|
||||
|
||||
# extract schema and save to files
|
||||
await extract_and_save_schema()
|
||||
|
||||
print("\nSchema files have been saved to:")
|
||||
print(f" - JSON: {JSON_FILE_PATH}")
|
||||
print(f" - YAML: {YAML_FILE_PATH}")
|
||||
|
||||
# load schema from files
|
||||
print("\nLoading schemas from saved files:")
|
||||
schema_from_json = GraphSchema.from_file(JSON_FILE_PATH)
|
||||
schema_from_yaml = GraphSchema.from_file(YAML_FILE_PATH)
|
||||
|
||||
print(f"Node types in JSON schema: {list(schema_from_json.node_types)}")
|
||||
print(f"Node types in YAML schema: {list(schema_from_yaml.node_types)}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,205 @@
|
||||
"""
|
||||
Simple example demonstrating structured output with SchemaFromTextExtractor.
|
||||
|
||||
This example shows how to use structured output for more reliable schema extraction
|
||||
with automatic validation against the GraphSchema Pydantic model.
|
||||
|
||||
The GraphSchema is now compatible with both OpenAI and VertexAI structured output APIs,
|
||||
with strict validation and proper field definitions. With structured output enabled:
|
||||
- Uses LLMInterfaceV2 (list of messages)
|
||||
- Passes GraphSchema Pydantic model as response_format to ainvoke()
|
||||
- Ensures response conforms to expected schema structure
|
||||
- Provides automatic type validation
|
||||
- Reduces need for JSON repair and error handling
|
||||
- Enforces min_length=1 on node properties (nodes must have at least one property)
|
||||
|
||||
Prerequisites:
|
||||
- Google Cloud credentials configured for VertexAI
|
||||
- Or OpenAI API key set in OPENAI_API_KEY environment variable
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
from dotenv import load_dotenv
|
||||
|
||||
from neo4j_graphrag.experimental.components.schema import (
|
||||
SchemaFromTextExtractor,
|
||||
GraphSchema,
|
||||
)
|
||||
from neo4j_graphrag.llm import OpenAILLM
|
||||
|
||||
|
||||
# Sample text to extract schema from
|
||||
SAMPLE_TEXT = """
|
||||
Acme Corporation was founded in 1985 by John Smith in New York City.
|
||||
The company specializes in manufacturing high-quality widgets and gadgets
|
||||
for the consumer electronics industry.
|
||||
|
||||
Sarah Johnson joined Acme in 2010 as a Senior Engineer and was promoted to
|
||||
Engineering Director in 2015. She oversees a team of 12 engineers working on
|
||||
next-generation products. Sarah holds a PhD in Electrical Engineering from MIT
|
||||
and has filed 5 patents during her time at Acme.
|
||||
|
||||
The company expanded to international markets in 2012, opening offices in London,
|
||||
Tokyo, and Berlin. Each office is managed by a regional director who reports
|
||||
directly to the CEO, Michael Brown, who took over leadership in 2008.
|
||||
|
||||
Acme's most successful product, the SuperWidget X1, was launched in 2018 and
|
||||
has sold over 2 million units worldwide. The product was developed by a team led
|
||||
by Robert Chen, who joined the company in 2016 after working at TechGiant for 8 years.
|
||||
"""
|
||||
|
||||
|
||||
def print_schema_summary(schema: GraphSchema, title: str) -> None:
|
||||
"""Print a formatted summary of the extracted schema."""
|
||||
print(f"\n{'='*60}")
|
||||
print(f"{title}")
|
||||
print(f"{'='*60}")
|
||||
|
||||
print(f"\nNode Types ({len(schema.node_types)}):")
|
||||
for node in schema.node_types:
|
||||
props = [f"{p.name} ({p.type})" for p in node.properties]
|
||||
print(f" - {node.label}")
|
||||
if props:
|
||||
print(f" Properties: {', '.join(props)}")
|
||||
if node.description:
|
||||
print(f" Description: {node.description}")
|
||||
|
||||
if schema.relationship_types:
|
||||
print(f"\nRelationship Types ({len(schema.relationship_types)}):")
|
||||
for rel in schema.relationship_types:
|
||||
props = [f"{p.name} ({p.type})" for p in rel.properties]
|
||||
print(f" - {rel.label}")
|
||||
if props:
|
||||
print(f" Properties: {', '.join(props)}")
|
||||
|
||||
if schema.patterns:
|
||||
print(f"\nPatterns ({len(schema.patterns)}):")
|
||||
for source, relationship, target in schema.patterns:
|
||||
print(f" {source} --[{relationship}]--> {target}")
|
||||
|
||||
if schema.constraints:
|
||||
print(f"\nConstraints ({len(schema.constraints)}):")
|
||||
for constraint in schema.constraints:
|
||||
print(
|
||||
f" - {constraint.type} on {constraint.node_type}.{list(constraint.property_names)}"
|
||||
)
|
||||
|
||||
|
||||
async def test_v1_without_structured_output() -> GraphSchema:
|
||||
"""
|
||||
Test V1 approach (default): Prompt-based JSON extraction with manual cleanup.
|
||||
|
||||
With use_structured_output=False (default):
|
||||
- Uses LLMInterface V1 (plain string prompts)
|
||||
- LLM returns JSON string that needs parsing and cleanup
|
||||
- Extensive filtering and validation applied manually
|
||||
- More forgiving of LLM errors
|
||||
- Works with all LLM providers
|
||||
"""
|
||||
print("\n" + "=" * 60)
|
||||
print("Testing V1: Prompt-based JSON extraction (default)")
|
||||
print("=" * 60)
|
||||
|
||||
# Initialize LLM with response_format for JSON mode (V1 approach)
|
||||
llm = OpenAILLM(
|
||||
model_name="gpt-5-mini",
|
||||
model_params={
|
||||
"temperature": 0,
|
||||
"response_format": {"type": "json_object"}, # JSON mode for V1
|
||||
},
|
||||
)
|
||||
|
||||
# For VertexAI V1, use:
|
||||
# llm = VertexAILLM(
|
||||
# model_name="gemini-2.5-flash",
|
||||
# model_params={"temperature": 0}
|
||||
# )
|
||||
|
||||
# Create extractor WITHOUT structured output (V1 default)
|
||||
extractor = SchemaFromTextExtractor(
|
||||
llm=llm,
|
||||
use_structured_output=False, # Default, can be omitted
|
||||
)
|
||||
|
||||
# Extract schema
|
||||
schema = await extractor.run(text=SAMPLE_TEXT)
|
||||
|
||||
print_schema_summary(schema, "V1 Result (Prompt-based)")
|
||||
|
||||
return schema
|
||||
|
||||
|
||||
async def test_v2_with_structured_output() -> GraphSchema:
|
||||
"""
|
||||
Test V2 approach: Structured output with GraphSchema validation.
|
||||
|
||||
With use_structured_output=True:
|
||||
- Uses LLMInterfaceV2 (list of messages)
|
||||
- Passes GraphSchema as response_format to ainvoke()
|
||||
- LLM returns properly structured data conforming to GraphSchema
|
||||
- Automatic validation via Pydantic
|
||||
- Less manual cleanup needed
|
||||
- Only works with OpenAI and VertexAI
|
||||
- Enforces min_length=1 on node properties
|
||||
"""
|
||||
print("\n" + "=" * 60)
|
||||
print("Testing V2: Structured output with GraphSchema")
|
||||
print("=" * 60)
|
||||
|
||||
# Initialize LLM - NO response_format in constructor for V2!
|
||||
llm = OpenAILLM(model_name="gpt-5-mini", model_params={"temperature": 0})
|
||||
|
||||
# For VertexAI V2, use:
|
||||
# llm = VertexAILLM(
|
||||
# model_name="gemini-2.5-flash",
|
||||
# model_params={"temperature": 0}
|
||||
# )
|
||||
|
||||
# Create extractor WITH structured output (V2)
|
||||
extractor = SchemaFromTextExtractor(
|
||||
llm=llm,
|
||||
use_structured_output=True, # This is the key parameter!
|
||||
)
|
||||
|
||||
# Extract schema
|
||||
schema = await extractor.run(text=SAMPLE_TEXT)
|
||||
|
||||
print_schema_summary(schema, "V2 Result (Structured Output)")
|
||||
|
||||
return schema
|
||||
|
||||
|
||||
async def compare_approaches() -> None:
|
||||
"""Run both approaches and compare results."""
|
||||
load_dotenv()
|
||||
|
||||
# Test V1 (default)
|
||||
schema_v1 = await test_v1_without_structured_output()
|
||||
|
||||
# Test V2 (structured output)
|
||||
schema_v2 = await test_v2_with_structured_output()
|
||||
|
||||
# Comparison
|
||||
print("\n" + "=" * 60)
|
||||
print("COMPARISON")
|
||||
print("=" * 60)
|
||||
print("V1 (Prompt-based):")
|
||||
print(f" - Node types: {len(schema_v1.node_types)}")
|
||||
print(f" - Relationship types: {len(schema_v1.relationship_types)}")
|
||||
print(f" - Patterns: {len(schema_v1.patterns)}")
|
||||
print(
|
||||
f" - Total properties: {sum(len(n.properties) for n in schema_v1.node_types)}"
|
||||
)
|
||||
|
||||
print("\nV2 (Structured Output):")
|
||||
print(f" - Node types: {len(schema_v2.node_types)}")
|
||||
print(f" - Relationship types: {len(schema_v2.relationship_types)}")
|
||||
print(f" - Patterns: {len(schema_v2.patterns)}")
|
||||
print(
|
||||
f" - Total properties: {sum(len(n.properties) for n in schema_v2.node_types)}"
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
# Run comparison between V1 and V2
|
||||
asyncio.run(compare_approaches())
|
||||
@@ -0,0 +1,14 @@
|
||||
from neo4j_graphrag.experimental.components.text_splitters.base import TextSplitter
|
||||
from neo4j_graphrag.experimental.components.types import TextChunk, TextChunks
|
||||
|
||||
|
||||
class MySplitter(TextSplitter):
|
||||
async def run(self, text: str) -> TextChunks:
|
||||
# your logic here
|
||||
return TextChunks(
|
||||
chunks=[
|
||||
TextChunk(text="", index=0),
|
||||
# optional metadata
|
||||
TextChunk(text="", index=1, metadata={"key": "value"}),
|
||||
]
|
||||
)
|
||||
@@ -0,0 +1,15 @@
|
||||
from neo4j_graphrag.experimental.components.text_splitters.fixed_size_splitter import (
|
||||
FixedSizeSplitter,
|
||||
)
|
||||
from neo4j_graphrag.experimental.components.types import TextChunks
|
||||
|
||||
|
||||
async def main() -> TextChunks:
|
||||
splitter = FixedSizeSplitter(
|
||||
# optionally, configure chunk_size, chunk_overlap, and approximate flag
|
||||
# chunk_size=4000,
|
||||
# chunk_overlap=200,
|
||||
# approximate = False
|
||||
)
|
||||
chunks = await splitter.run(text="text to split")
|
||||
return chunks
|
||||
@@ -0,0 +1,26 @@
|
||||
"""Implement a custom writer to save the results, for instance by using
|
||||
custom Cypher queries.
|
||||
"""
|
||||
|
||||
import neo4j
|
||||
from neo4j_graphrag.experimental.components.kg_writer import KGWriter, KGWriterModel
|
||||
from neo4j_graphrag.experimental.components.types import LexicalGraphConfig, Neo4jGraph
|
||||
from pydantic import validate_call
|
||||
from neo4j_graphrag.utils import driver_config
|
||||
|
||||
|
||||
class MyWriter(KGWriter):
|
||||
def __init__(self, driver: neo4j.Driver) -> None:
|
||||
self.driver = driver_config.override_user_agent(driver)
|
||||
|
||||
@validate_call
|
||||
async def run(
|
||||
self,
|
||||
graph: Neo4jGraph,
|
||||
lexical_graph_config: LexicalGraphConfig = LexicalGraphConfig(),
|
||||
) -> KGWriterModel:
|
||||
try:
|
||||
self.driver.execute_query("my query")
|
||||
return KGWriterModel(status="SUCCESS")
|
||||
except Exception:
|
||||
return KGWriterModel(status="FAILURE")
|
||||
@@ -0,0 +1,19 @@
|
||||
import neo4j
|
||||
from neo4j_graphrag.experimental.components.kg_writer import (
|
||||
KGWriterModel,
|
||||
Neo4jWriter,
|
||||
)
|
||||
from neo4j_graphrag.experimental.components.types import Neo4jGraph
|
||||
|
||||
|
||||
async def main(driver: neo4j.Driver, graph: Neo4jGraph) -> KGWriterModel:
|
||||
writer = Neo4jWriter(
|
||||
driver,
|
||||
# optionally, configure the neo4j database
|
||||
# neo4j_database="neo4j",
|
||||
# you can tune batch_size to
|
||||
# improve speed
|
||||
# batch_size=1000,
|
||||
)
|
||||
result = await writer.run(graph=graph)
|
||||
return result
|
||||
@@ -0,0 +1,68 @@
|
||||
{
|
||||
"version_": "1",
|
||||
"template_": "none",
|
||||
"name": "",
|
||||
"neo4j_config": {
|
||||
"params_": {
|
||||
"uri": {
|
||||
"resolver_": "ENV",
|
||||
"var_": "NEO4J_URI"
|
||||
},
|
||||
"user": {
|
||||
"resolver_": "ENV",
|
||||
"var_": "NEO4J_USER"
|
||||
},
|
||||
"password": {
|
||||
"resolver_": "ENV",
|
||||
"var_": "NEO4J_PASSWORD"
|
||||
}
|
||||
}
|
||||
},
|
||||
"extras": {
|
||||
"database": "neo4j"
|
||||
},
|
||||
"component_config": {
|
||||
"splitter": {
|
||||
"class_": "text_splitters.fixed_size_splitter.FixedSizeSplitter"
|
||||
},
|
||||
"builder": {
|
||||
"class_": "lexical_graph.LexicalGraphBuilder",
|
||||
"params_": {
|
||||
"config": {
|
||||
"chunk_node_label": "TextPart"
|
||||
}
|
||||
}
|
||||
},
|
||||
"writer": {
|
||||
"name_": "writer",
|
||||
"class_": "kg_writer.Neo4jWriter",
|
||||
"params_": {
|
||||
"driver": {
|
||||
"resolver_": "CONFIG_KEY",
|
||||
"key_": "neo4j_config.default"
|
||||
},
|
||||
"neo4j_database": {
|
||||
"resolver_": "CONFIG_KEY",
|
||||
"key_": "extras.database"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"connection_config": [
|
||||
{
|
||||
"start": "splitter",
|
||||
"end": "builder",
|
||||
"input_config": {
|
||||
"text_chunks": "splitter"
|
||||
}
|
||||
},
|
||||
{
|
||||
"start": "builder",
|
||||
"end": "writer",
|
||||
"input_config": {
|
||||
"graph": "builder.graph",
|
||||
"lexical_graph_config": "builder.config"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
version_: "1"
|
||||
template_: none
|
||||
neo4j_config:
|
||||
params_:
|
||||
uri:
|
||||
resolver_: ENV
|
||||
var_: NEO4J_URI
|
||||
user:
|
||||
resolver_: ENV
|
||||
var_: NEO4J_USER
|
||||
password:
|
||||
resolver_: ENV
|
||||
var_: NEO4J_PASSWORD
|
||||
extras:
|
||||
database: neo4j
|
||||
component_config:
|
||||
splitter:
|
||||
class_: text_splitters.fixed_size_splitter.FixedSizeSplitter
|
||||
params_:
|
||||
chunk_size: 100
|
||||
chunk_overlap: 10
|
||||
builder:
|
||||
class_: lexical_graph.LexicalGraphBuilder
|
||||
params_:
|
||||
config:
|
||||
chunk_node_label: TextPart
|
||||
writer:
|
||||
class_: kg_writer.Neo4jWriter
|
||||
params_:
|
||||
driver:
|
||||
resolver_: CONFIG_KEY
|
||||
key_: neo4j_config.default
|
||||
neo4j_database:
|
||||
resolver_: CONFIG_KEY
|
||||
key_: extras.database
|
||||
connection_config:
|
||||
- start: splitter
|
||||
end: builder
|
||||
input_config:
|
||||
text_chunks: splitter
|
||||
- start: builder
|
||||
end: writer
|
||||
input_config:
|
||||
graph: builder.graph
|
||||
lexical_graph_config: builder.config
|
||||
@@ -0,0 +1,40 @@
|
||||
"""In this example, the pipeline is defined in a JSON file 'pipeline_config.json'.
|
||||
According to the configuration file, some parameters will be read from the env vars
|
||||
(Neo4j credentials and the OpenAI API key).
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
|
||||
## If env vars are in a .env file, uncomment:
|
||||
## (requires pip install python-dotenv)
|
||||
# from dotenv import load_dotenv
|
||||
# load_dotenv()
|
||||
# env vars manually set for testing:
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
from neo4j_graphrag.experimental.pipeline.config.runner import PipelineRunner
|
||||
from neo4j_graphrag.experimental.pipeline.pipeline import PipelineResult
|
||||
|
||||
os.environ["NEO4J_URI"] = "bolt://localhost:7687"
|
||||
os.environ["NEO4J_USER"] = "neo4j"
|
||||
os.environ["NEO4J_PASSWORD"] = "password"
|
||||
# os.environ["OPENAI_API_KEY"] = "sk-..."
|
||||
|
||||
|
||||
root_dir = Path(__file__).parent
|
||||
# file_path = root_dir / "pipeline_config.json"
|
||||
file_path = root_dir / "pipeline_config.yaml"
|
||||
|
||||
# Text to process
|
||||
TEXT = """The son of Duke Leto Atreides and the Lady Jessica, Paul is the heir of House Atreides,
|
||||
an aristocratic family that rules the planet Caladan, the rainy planet, since 10191."""
|
||||
|
||||
|
||||
async def main() -> PipelineResult:
|
||||
pipeline = PipelineRunner.from_config_file(file_path)
|
||||
return await pipeline.run({"splitter": {"text": TEXT}})
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
print(asyncio.run(main()))
|
||||
@@ -0,0 +1,147 @@
|
||||
# Copyright (c) "Neo4j"
|
||||
# Neo4j Sweden AB [https://neo4j.com]
|
||||
# #
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
# #
|
||||
# https://www.apache.org/licenses/LICENSE-2.0
|
||||
# #
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
|
||||
from neo4j_graphrag.experimental.components.entity_relation_extractor import (
|
||||
LLMEntityRelationExtractor,
|
||||
OnError,
|
||||
)
|
||||
from neo4j_graphrag.experimental.components.kg_writer import Neo4jWriter
|
||||
from neo4j_graphrag.experimental.components.data_loader import PdfLoader
|
||||
from neo4j_graphrag.experimental.components.schema import (
|
||||
SchemaBuilder,
|
||||
NodeType,
|
||||
RelationshipType,
|
||||
)
|
||||
from neo4j_graphrag.experimental.components.text_splitters.fixed_size_splitter import (
|
||||
FixedSizeSplitter,
|
||||
)
|
||||
from neo4j_graphrag.experimental.pipeline.pipeline import PipelineResult
|
||||
from neo4j_graphrag.llm import LLMInterface, OpenAILLM
|
||||
|
||||
import neo4j
|
||||
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
|
||||
|
||||
async def define_and_run_pipeline(
|
||||
neo4j_driver: neo4j.Driver, llm: LLMInterface
|
||||
) -> PipelineResult:
|
||||
from neo4j_graphrag.experimental.pipeline import Pipeline
|
||||
|
||||
# Instantiate Entity and Relation objects
|
||||
node_types = [
|
||||
NodeType(label="PERSON", description="An individual human being."),
|
||||
NodeType(
|
||||
label="ORGANIZATION",
|
||||
description="A structured group of people with a common purpose.",
|
||||
),
|
||||
NodeType(label="LOCATION", description="A location or place."),
|
||||
NodeType(
|
||||
label="HORCRUX",
|
||||
description="A magical item in the Harry Potter universe.",
|
||||
),
|
||||
]
|
||||
relationship_types = [
|
||||
RelationshipType(
|
||||
label="SITUATED_AT", description="Indicates the location of a person."
|
||||
),
|
||||
RelationshipType(
|
||||
label="LED_BY",
|
||||
description="Indicates the leader of an organization.",
|
||||
),
|
||||
RelationshipType(
|
||||
label="OWNS",
|
||||
description="Indicates the ownership of an item such as a Horcrux.",
|
||||
),
|
||||
RelationshipType(
|
||||
label="INTERACTS", description="The interaction between two people."
|
||||
),
|
||||
]
|
||||
patterns = [
|
||||
("PERSON", "SITUATED_AT", "LOCATION"),
|
||||
("PERSON", "INTERACTS", "PERSON"),
|
||||
("PERSON", "OWNS", "HORCRUX"),
|
||||
("ORGANIZATION", "LED_BY", "PERSON"),
|
||||
]
|
||||
|
||||
# Set up the pipeline
|
||||
pipe = Pipeline()
|
||||
pipe.add_component(PdfLoader(), "pdf_loader")
|
||||
pipe.add_component(
|
||||
FixedSizeSplitter(chunk_size=4000, chunk_overlap=200, approximate=False),
|
||||
"splitter",
|
||||
)
|
||||
pipe.add_component(SchemaBuilder(), "schema")
|
||||
pipe.add_component(
|
||||
LLMEntityRelationExtractor(
|
||||
llm=llm,
|
||||
on_error=OnError.RAISE,
|
||||
use_structured_output=True,
|
||||
),
|
||||
"extractor",
|
||||
)
|
||||
pipe.add_component(Neo4jWriter(neo4j_driver), "writer")
|
||||
pipe.connect("pdf_loader", "splitter", input_config={"text": "pdf_loader.text"})
|
||||
pipe.connect("splitter", "extractor", input_config={"chunks": "splitter"})
|
||||
pipe.connect(
|
||||
"schema",
|
||||
"extractor",
|
||||
input_config={
|
||||
"schema": "schema",
|
||||
"document_info": "pdf_loader.document_info",
|
||||
},
|
||||
)
|
||||
pipe.connect(
|
||||
"extractor",
|
||||
"writer",
|
||||
input_config={"graph": "extractor"},
|
||||
)
|
||||
|
||||
pipe_inputs = {
|
||||
"pdf_loader": {
|
||||
"filepath": "examples/data/Harry Potter and the Death Hallows Summary.pdf"
|
||||
},
|
||||
"schema": {
|
||||
"node_types": node_types,
|
||||
"relationship_types": relationship_types,
|
||||
"patterns": patterns,
|
||||
},
|
||||
}
|
||||
return await pipe.run(pipe_inputs)
|
||||
|
||||
|
||||
async def main() -> PipelineResult:
|
||||
res = None
|
||||
try:
|
||||
llm = OpenAILLM(
|
||||
model_name="gpt-5",
|
||||
)
|
||||
driver = neo4j.GraphDatabase.driver(
|
||||
"bolt://localhost:7687", auth=("neo4j", "password")
|
||||
)
|
||||
res = await define_and_run_pipeline(driver, llm)
|
||||
finally:
|
||||
driver.close()
|
||||
await llm.aclose()
|
||||
return res
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
res = asyncio.run(main())
|
||||
print(res)
|
||||
@@ -0,0 +1,163 @@
|
||||
# Copyright (c) "Neo4j"
|
||||
# Neo4j Sweden AB [https://neo4j.com]
|
||||
# #
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
# #
|
||||
# https://www.apache.org/licenses/LICENSE-2.0
|
||||
# #
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
|
||||
from neo4j_graphrag.embeddings.openai import OpenAIEmbeddings
|
||||
from neo4j_graphrag.experimental.components.embedder import TextChunkEmbedder
|
||||
from neo4j_graphrag.experimental.components.entity_relation_extractor import (
|
||||
LLMEntityRelationExtractor,
|
||||
OnError,
|
||||
)
|
||||
from neo4j_graphrag.experimental.components.kg_writer import Neo4jWriter
|
||||
from neo4j_graphrag.experimental.components.schema import (
|
||||
SchemaBuilder,
|
||||
NodeType,
|
||||
PropertyType,
|
||||
RelationshipType,
|
||||
)
|
||||
from neo4j_graphrag.experimental.components.text_splitters.fixed_size_splitter import (
|
||||
FixedSizeSplitter,
|
||||
)
|
||||
from neo4j_graphrag.experimental.pipeline import Pipeline
|
||||
from neo4j_graphrag.experimental.pipeline.pipeline import PipelineResult
|
||||
from neo4j_graphrag.llm import LLMInterface, OpenAILLM
|
||||
|
||||
import neo4j
|
||||
|
||||
|
||||
async def define_and_run_pipeline(
|
||||
neo4j_driver: neo4j.Driver, llm: LLMInterface
|
||||
) -> PipelineResult:
|
||||
"""This is where we define and run the KG builder pipeline, instantiating a few
|
||||
components:
|
||||
- Text Splitter: in this example we use the fixed size text splitter
|
||||
- Chunk Embedder: to embed the chunks' text
|
||||
- Schema Builder: this component takes a list of entities, relationships and
|
||||
possible triplets as inputs, validate them and return a schema ready to use
|
||||
for the rest of the pipeline
|
||||
- LLM Entity Relation Extractor is an LLM-based entity and relation extractor:
|
||||
based on the provided schema, the LLM will do its best to identity these
|
||||
entities and their relations within the provided text
|
||||
- KG writer: once entities and relations are extracted, they can be writen
|
||||
to a Neo4j database
|
||||
"""
|
||||
pipe = Pipeline()
|
||||
# define the components
|
||||
pipe.add_component(
|
||||
# chunk_size=50 for the sake of this demo
|
||||
FixedSizeSplitter(chunk_size=4000, chunk_overlap=200, approximate=False),
|
||||
"splitter",
|
||||
)
|
||||
pipe.add_component(TextChunkEmbedder(embedder=OpenAIEmbeddings()), "chunk_embedder")
|
||||
pipe.add_component(SchemaBuilder(), "schema")
|
||||
pipe.add_component(
|
||||
LLMEntityRelationExtractor(
|
||||
llm=llm,
|
||||
on_error=OnError.RAISE,
|
||||
use_structured_output=True,
|
||||
),
|
||||
"extractor",
|
||||
)
|
||||
pipe.add_component(Neo4jWriter(neo4j_driver), "writer")
|
||||
# define the execution order of component
|
||||
# and how the output of previous components must be used
|
||||
pipe.connect("splitter", "chunk_embedder", input_config={"text_chunks": "splitter"})
|
||||
pipe.connect("schema", "extractor", input_config={"schema": "schema"})
|
||||
pipe.connect(
|
||||
"chunk_embedder", "extractor", input_config={"chunks": "chunk_embedder"}
|
||||
)
|
||||
pipe.connect(
|
||||
"extractor",
|
||||
"writer",
|
||||
input_config={"graph": "extractor"},
|
||||
)
|
||||
# user input:
|
||||
# the initial text
|
||||
# and the list of entities and relations we are looking for
|
||||
pipe_inputs = {
|
||||
"splitter": {
|
||||
"text": """Albert Einstein was a German physicist born in 1879 who
|
||||
wrote many groundbreaking papers especially about general relativity
|
||||
and quantum mechanics. He worked for many different institutions, including
|
||||
the University of Bern in Switzerland and the University of Oxford."""
|
||||
},
|
||||
"schema": {
|
||||
"node_types": [
|
||||
NodeType(
|
||||
label="Person",
|
||||
properties=[
|
||||
PropertyType(name="name", type="STRING"),
|
||||
PropertyType(name="place_of_birth", type="STRING"),
|
||||
PropertyType(name="date_of_birth", type="DATE"),
|
||||
],
|
||||
),
|
||||
NodeType(
|
||||
label="Organization",
|
||||
properties=[
|
||||
PropertyType(name="name", type="STRING"),
|
||||
PropertyType(name="country", type="STRING"),
|
||||
],
|
||||
),
|
||||
NodeType(
|
||||
label="Field",
|
||||
properties=[
|
||||
PropertyType(name="name", type="STRING"),
|
||||
],
|
||||
),
|
||||
],
|
||||
"relationship_types": [
|
||||
RelationshipType(
|
||||
label="WORKED_ON",
|
||||
),
|
||||
RelationshipType(
|
||||
label="WORKED_FOR",
|
||||
),
|
||||
],
|
||||
"patterns": [
|
||||
("Person", "WORKED_ON", "Field"),
|
||||
("Person", "WORKED_FOR", "Organization"),
|
||||
],
|
||||
},
|
||||
"extractor": {
|
||||
"document_info": {
|
||||
"path": "my text",
|
||||
}
|
||||
},
|
||||
}
|
||||
# run the pipeline
|
||||
return await pipe.run(pipe_inputs)
|
||||
|
||||
|
||||
async def main() -> PipelineResult:
|
||||
res = None
|
||||
try:
|
||||
llm = OpenAILLM(
|
||||
model_name="gpt-5",
|
||||
)
|
||||
driver = neo4j.GraphDatabase.driver(
|
||||
"bolt://localhost:7687", auth=("neo4j", "password")
|
||||
)
|
||||
res = await define_and_run_pipeline(driver, llm)
|
||||
finally:
|
||||
driver.close()
|
||||
await llm.aclose()
|
||||
return res
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
res = asyncio.run(main())
|
||||
print(res)
|
||||
@@ -0,0 +1,156 @@
|
||||
# Copyright (c) "Neo4j"
|
||||
# Neo4j Sweden AB [https://neo4j.com]
|
||||
# #
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
# #
|
||||
# https://www.apache.org/licenses/LICENSE-2.0
|
||||
# #
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
|
||||
import neo4j
|
||||
from neo4j_graphrag.experimental.components.entity_relation_extractor import (
|
||||
LLMEntityRelationExtractor,
|
||||
OnError,
|
||||
)
|
||||
from neo4j_graphrag.experimental.components.kg_writer import Neo4jWriter
|
||||
from neo4j_graphrag.experimental.components.data_loader import PdfLoader
|
||||
from neo4j_graphrag.experimental.components.resolver import (
|
||||
SinglePropertyExactMatchResolver,
|
||||
)
|
||||
from neo4j_graphrag.experimental.components.schema import (
|
||||
SchemaBuilder,
|
||||
NodeType,
|
||||
PropertyType,
|
||||
RelationshipType,
|
||||
)
|
||||
from neo4j_graphrag.experimental.components.text_splitters.fixed_size_splitter import (
|
||||
FixedSizeSplitter,
|
||||
)
|
||||
from neo4j_graphrag.experimental.pipeline import Pipeline
|
||||
from neo4j_graphrag.llm import LLMInterface, OpenAILLM
|
||||
|
||||
|
||||
async def define_and_run_pipeline(
|
||||
neo4j_driver: neo4j.Driver, llm: LLMInterface
|
||||
) -> None:
|
||||
"""This is where we define and run the KG builder pipeline, instantiating a few
|
||||
components:
|
||||
- Text Splitter: in this example we use the fixed size text splitter
|
||||
- Schema Builder: this component takes a list of entities, relationships and
|
||||
possible triplets as inputs, validate them and return a schema ready to use
|
||||
for the rest of the pipeline
|
||||
- LLM Entity Relation Extractor is an LLM-based entity and relation extractor:
|
||||
based on the provided schema, the LLM will do its best to identity these
|
||||
entities and their relations within the provided text
|
||||
- KG writer: once entities and relations are extracted, they can be writen
|
||||
to a Neo4j database
|
||||
"""
|
||||
pipe = Pipeline()
|
||||
# define the components
|
||||
pipe.add_component(PdfLoader(), "loader")
|
||||
pipe.add_component(
|
||||
FixedSizeSplitter(),
|
||||
"splitter",
|
||||
)
|
||||
pipe.add_component(SchemaBuilder(), "schema")
|
||||
pipe.add_component(
|
||||
LLMEntityRelationExtractor(
|
||||
llm=llm,
|
||||
on_error=OnError.IGNORE,
|
||||
use_structured_output=True,
|
||||
),
|
||||
"extractor",
|
||||
)
|
||||
pipe.add_component(Neo4jWriter(neo4j_driver), "writer")
|
||||
pipe.add_component(SinglePropertyExactMatchResolver(neo4j_driver), "resolver")
|
||||
# define the execution order of component
|
||||
# and how the output of previous components must be used
|
||||
pipe.connect("loader", "splitter", {"text": "loader.text"})
|
||||
pipe.connect("splitter", "extractor", input_config={"chunks": "splitter"})
|
||||
pipe.connect(
|
||||
"schema",
|
||||
"extractor",
|
||||
input_config={"schema": "schema", "document_info": "loader.document_info"},
|
||||
)
|
||||
pipe.connect(
|
||||
"extractor",
|
||||
"writer",
|
||||
input_config={"graph": "extractor"},
|
||||
)
|
||||
pipe.connect("writer", "resolver", {})
|
||||
# user input:
|
||||
# the initial text
|
||||
# and the list of entities and relations we are looking for
|
||||
pipe_inputs = {
|
||||
"loader": {},
|
||||
"schema": {
|
||||
"node_types": [
|
||||
NodeType(
|
||||
label="Person",
|
||||
properties=[
|
||||
PropertyType(name="name", type="STRING"),
|
||||
PropertyType(name="place_of_birth", type="STRING"),
|
||||
PropertyType(name="date_of_birth", type="DATE"),
|
||||
],
|
||||
),
|
||||
NodeType(
|
||||
label="Organization",
|
||||
properties=[
|
||||
PropertyType(name="name", type="STRING"),
|
||||
PropertyType(name="country", type="STRING"),
|
||||
],
|
||||
),
|
||||
],
|
||||
"relationship_types": [
|
||||
RelationshipType(
|
||||
label="WORKED_FOR",
|
||||
),
|
||||
RelationshipType(
|
||||
label="FRIEND",
|
||||
),
|
||||
RelationshipType(
|
||||
label="ENEMY",
|
||||
),
|
||||
],
|
||||
"patterns": [
|
||||
("Person", "WORKED_FOR", "Organization"),
|
||||
("Person", "FRIEND", "Person"),
|
||||
("Person", "ENEMY", "Person"),
|
||||
],
|
||||
},
|
||||
}
|
||||
# run the pipeline for each documents
|
||||
for document in [
|
||||
"examples/data/Harry Potter and the Chamber of Secrets Summary.pdf",
|
||||
"examples/data/Harry Potter and the Death Hallows Summary.pdf",
|
||||
]:
|
||||
pipe_inputs["loader"]["filepath"] = document
|
||||
await pipe.run(pipe_inputs)
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
llm = OpenAILLM(
|
||||
model_name="gpt-5",
|
||||
model_params={
|
||||
"max_completion_tokens": 1000,
|
||||
},
|
||||
)
|
||||
driver = neo4j.GraphDatabase.driver(
|
||||
"bolt://localhost:7687", auth=("neo4j", "password")
|
||||
)
|
||||
await define_and_run_pipeline(driver, llm)
|
||||
driver.close()
|
||||
await llm.aclose()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,85 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
|
||||
from neo4j_graphrag.embeddings.openai import OpenAIEmbeddings
|
||||
from neo4j_graphrag.experimental.components.embedder import TextChunkEmbedder
|
||||
from neo4j_graphrag.experimental.components.kg_writer import Neo4jWriter
|
||||
from neo4j_graphrag.experimental.components.lexical_graph import LexicalGraphBuilder
|
||||
from neo4j_graphrag.experimental.components.text_splitters.fixed_size_splitter import (
|
||||
FixedSizeSplitter,
|
||||
)
|
||||
from neo4j_graphrag.experimental.components.types import LexicalGraphConfig
|
||||
from neo4j_graphrag.experimental.pipeline import Pipeline
|
||||
from neo4j_graphrag.experimental.pipeline.pipeline import PipelineResult
|
||||
|
||||
import neo4j
|
||||
|
||||
|
||||
async def main(neo4j_driver: neo4j.Driver) -> PipelineResult:
|
||||
"""This is where we define and run the Lexical Graph builder pipeline, instantiating
|
||||
a few components:
|
||||
|
||||
- Text Splitter: to split the text into manageable chunks of fixed size
|
||||
- Chunk Embedder: to embed the chunks' text
|
||||
- Lexical Graph Builder: to build the lexical graph, ie creating the chunk nodes and relationships between them
|
||||
- KG writer: save the lexical graph to Neo4j
|
||||
"""
|
||||
pipe = Pipeline()
|
||||
# define the components
|
||||
pipe.add_component(
|
||||
FixedSizeSplitter(chunk_size=20, chunk_overlap=1, approximate=False),
|
||||
"splitter",
|
||||
)
|
||||
pipe.add_component(TextChunkEmbedder(embedder=OpenAIEmbeddings()), "chunk_embedder")
|
||||
# optional: define some custom node labels for the lexical graph:
|
||||
lexical_graph_config = LexicalGraphConfig(
|
||||
chunk_node_label="TextPart",
|
||||
)
|
||||
pipe.add_component(
|
||||
LexicalGraphBuilder(lexical_graph_config),
|
||||
"lexical_graph_builder",
|
||||
)
|
||||
pipe.add_component(Neo4jWriter(neo4j_driver), "writer")
|
||||
# define the execution order of component
|
||||
# and how the output of previous components must be used
|
||||
pipe.connect("splitter", "chunk_embedder", input_config={"text_chunks": "splitter"})
|
||||
pipe.connect(
|
||||
"chunk_embedder",
|
||||
"lexical_graph_builder",
|
||||
input_config={"text_chunks": "chunk_embedder"},
|
||||
)
|
||||
pipe.connect(
|
||||
"lexical_graph_builder",
|
||||
"writer",
|
||||
input_config={
|
||||
"graph": "lexical_graph_builder.graph",
|
||||
"lexical_graph_config": "lexical_graph_builder.config",
|
||||
},
|
||||
)
|
||||
# user input:
|
||||
# the initial text
|
||||
# and the list of entities and relations we are looking for
|
||||
pipe_inputs = {
|
||||
"splitter": {
|
||||
"text": """Albert Einstein was a German physicist born in 1879 who
|
||||
wrote many groundbreaking papers especially about general relativity
|
||||
and quantum mechanics. He worked for many different institutions, including
|
||||
the University of Bern in Switzerland and the University of Oxford."""
|
||||
},
|
||||
"lexical_graph_builder": {
|
||||
"document_info": {
|
||||
# 'path' can be anything
|
||||
"path": "example/lexical_graph_from_text.py"
|
||||
},
|
||||
},
|
||||
}
|
||||
# run the pipeline
|
||||
return await pipe.run(pipe_inputs)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
with neo4j.GraphDatabase.driver(
|
||||
"bolt://localhost:7687", auth=("neo4j", "password")
|
||||
) as driver:
|
||||
print(asyncio.run(main(driver)))
|
||||
@@ -0,0 +1,88 @@
|
||||
import asyncio
|
||||
|
||||
from neo4j_graphrag.experimental.pipeline.component import Component, DataModel
|
||||
from neo4j_graphrag.experimental.pipeline.pipeline import Pipeline
|
||||
from neo4j_graphrag.experimental.pipeline.notification import EventType, Event
|
||||
from neo4j_graphrag.experimental.pipeline.types.context import RunContext
|
||||
|
||||
|
||||
# Define some example components with progress notifications
|
||||
class OutputModel(DataModel):
|
||||
result: int
|
||||
|
||||
|
||||
class SlowAdder(Component):
|
||||
"""A component that slowly adds numbers and reports progress"""
|
||||
|
||||
def __init__(self, number: int) -> None:
|
||||
self.number = number
|
||||
|
||||
async def run_with_context(self, context_: RunContext, value: int) -> OutputModel:
|
||||
# Simulate work with progress updates
|
||||
for i in range(value):
|
||||
await asyncio.sleep(0.5) # Simulate work
|
||||
await context_.notify(
|
||||
message=f"Added {i + 1}/{value}",
|
||||
data={"current": i + 1, "total": value},
|
||||
)
|
||||
return OutputModel(result=value + self.number)
|
||||
|
||||
|
||||
class SlowMultiplier(Component):
|
||||
"""A component that slowly multiplies numbers and reports progress"""
|
||||
|
||||
def __init__(self, multiplier: int) -> None:
|
||||
self.multiplier = multiplier
|
||||
|
||||
async def run_with_context(self, context_: RunContext, value: int) -> OutputModel:
|
||||
# Simulate work with progress updates
|
||||
for i in range(3): # Always do 3 steps
|
||||
await asyncio.sleep(0.7) # Simulate work
|
||||
await context_.notify(
|
||||
message=f"Multiplication step {i + 1}/3",
|
||||
data={"step": i + 1, "total": 3},
|
||||
)
|
||||
return OutputModel(result=value * self.multiplier)
|
||||
|
||||
|
||||
async def callback(event: Event) -> None:
|
||||
await asyncio.sleep(0.1)
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
# Create pipeline
|
||||
pipeline = Pipeline(callback=callback)
|
||||
|
||||
# Add components
|
||||
pipeline.add_component(SlowAdder(number=3), "adder")
|
||||
pipeline.add_component(SlowMultiplier(multiplier=2), "multiplier")
|
||||
|
||||
# Connect components
|
||||
pipeline.connect("adder", "multiplier", {"value": "adder.result"})
|
||||
|
||||
print("\n=== Running pipeline with streaming ===")
|
||||
# Run pipeline with streaming - see events as they happen
|
||||
async for event in pipeline.stream(
|
||||
{"adder": {"value": 2}},
|
||||
raise_exception=False, # default is True
|
||||
):
|
||||
if event.event_type == EventType.PIPELINE_STARTED:
|
||||
print("Stream: Pipeline started!")
|
||||
elif event.event_type == EventType.PIPELINE_FINISHED:
|
||||
print(f"Stream: Pipeline finished! Final results: {event.payload}")
|
||||
elif event.event_type == EventType.PIPELINE_FAILED:
|
||||
print(f"Stream: Pipeline failed with message: {event.message}")
|
||||
elif event.event_type == EventType.TASK_STARTED:
|
||||
print(
|
||||
f"Stream: Task {event.task_name} started with inputs: {event.payload}" # type: ignore
|
||||
)
|
||||
elif event.event_type == EventType.TASK_PROGRESS:
|
||||
print(f"Stream: Task {event.task_name} progress - {event.message}") # type: ignore
|
||||
elif event.event_type == EventType.TASK_FINISHED:
|
||||
print(
|
||||
f"Stream: Task {event.task_name} finished with result: {event.payload}" # type: ignore
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,109 @@
|
||||
"""This example demonstrates how to use event callback to receive notifications
|
||||
about the component progress.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from typing import Any
|
||||
|
||||
from neo4j_graphrag.experimental.pipeline import Pipeline, Component, DataModel
|
||||
from neo4j_graphrag.experimental.pipeline.notification import Event, EventType
|
||||
from neo4j_graphrag.experimental.pipeline.types.context import RunContext
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
logging.basicConfig()
|
||||
logger.setLevel(logging.INFO)
|
||||
|
||||
|
||||
class MultiplyComponentResult(DataModel):
|
||||
result: list[int]
|
||||
|
||||
|
||||
class MultiplicationComponent(Component):
|
||||
def __init__(self, f: int) -> None:
|
||||
self.f = f
|
||||
|
||||
async def multiply_number(
|
||||
self,
|
||||
context_: RunContext,
|
||||
number: int,
|
||||
) -> int:
|
||||
await context_.notify(
|
||||
message=f"Processing number {number}",
|
||||
data={"number_processed": number},
|
||||
)
|
||||
return self.f * number
|
||||
|
||||
# implementing `run_with_context` to get access to
|
||||
# the pipeline's RunContext:
|
||||
async def run_with_context(
|
||||
self,
|
||||
context_: RunContext,
|
||||
numbers: list[int],
|
||||
**kwargs: Any,
|
||||
) -> MultiplyComponentResult:
|
||||
result = await asyncio.gather(
|
||||
*[
|
||||
self.multiply_number(
|
||||
context_,
|
||||
number,
|
||||
)
|
||||
for number in numbers
|
||||
]
|
||||
)
|
||||
return MultiplyComponentResult(result=result)
|
||||
|
||||
|
||||
async def event_handler(event: Event) -> None:
|
||||
"""Function can do anything about the event,
|
||||
here we're just logging it if it's a pipeline-level event.
|
||||
"""
|
||||
if event.event_type == EventType.TASK_PROGRESS:
|
||||
logger.warning(event)
|
||||
else:
|
||||
logger.info(event)
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
""" """
|
||||
pipe = Pipeline(
|
||||
callback=event_handler,
|
||||
)
|
||||
# define the components
|
||||
pipe.add_component(
|
||||
MultiplicationComponent(f=2),
|
||||
"multiply_by_2",
|
||||
)
|
||||
pipe.add_component(
|
||||
MultiplicationComponent(f=10),
|
||||
"multiply_by_10",
|
||||
)
|
||||
# define the execution order of component
|
||||
# and how the output of previous components must be used
|
||||
pipe.connect(
|
||||
"multiply_by_2",
|
||||
"multiply_by_10",
|
||||
input_config={"numbers": "multiply_by_2.result"},
|
||||
)
|
||||
# user input:
|
||||
pipe_inputs_1 = {
|
||||
"multiply_by_2": {
|
||||
"numbers": [1, 2, 5, 4],
|
||||
},
|
||||
}
|
||||
pipe_inputs_2 = {
|
||||
"multiply_by_2": {
|
||||
"numbers": [3, 10, 1],
|
||||
}
|
||||
}
|
||||
# run the pipeline
|
||||
await asyncio.gather(
|
||||
pipe.run(pipe_inputs_1),
|
||||
pipe.run(pipe_inputs_2),
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,93 @@
|
||||
"""This example demonstrates how to use event callback to receive notifications
|
||||
about the pipeline progress.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
|
||||
import neo4j
|
||||
from neo4j_graphrag.experimental.components.kg_writer import Neo4jWriter
|
||||
from neo4j_graphrag.experimental.components.lexical_graph import LexicalGraphBuilder
|
||||
from neo4j_graphrag.experimental.components.text_splitters.fixed_size_splitter import (
|
||||
FixedSizeSplitter,
|
||||
)
|
||||
from neo4j_graphrag.experimental.pipeline import Pipeline
|
||||
from neo4j_graphrag.experimental.pipeline.pipeline import PipelineResult
|
||||
from neo4j_graphrag.experimental.pipeline.notification import Event
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
logging.basicConfig()
|
||||
logger.setLevel(logging.WARNING)
|
||||
|
||||
|
||||
async def event_handler(event: Event) -> None:
|
||||
"""Function can do anything about the event,
|
||||
here we're just logging it if it's a pipeline-level event.
|
||||
"""
|
||||
if event.event_type.is_pipeline_event:
|
||||
logger.warning(event)
|
||||
|
||||
|
||||
async def main(neo4j_driver: neo4j.Driver) -> PipelineResult:
|
||||
"""This is where we define and run the Lexical Graph builder pipeline, instantiating
|
||||
a few components:
|
||||
|
||||
- Text Splitter: to split the text into manageable chunks of fixed size
|
||||
- Chunk Embedder: to embed the chunks' text
|
||||
- Lexical Graph Builder: to build the lexical graph, ie creating the chunk nodes and relationships between them
|
||||
- KG writer: save the lexical graph to Neo4j
|
||||
"""
|
||||
pipe = Pipeline(
|
||||
callback=event_handler,
|
||||
)
|
||||
# define the components
|
||||
pipe.add_component(
|
||||
FixedSizeSplitter(chunk_size=300, chunk_overlap=10, approximate=False),
|
||||
"splitter",
|
||||
)
|
||||
pipe.add_component(
|
||||
LexicalGraphBuilder(),
|
||||
"lexical_graph_builder",
|
||||
)
|
||||
pipe.add_component(Neo4jWriter(neo4j_driver), "writer")
|
||||
# define the execution order of component
|
||||
# and how the output of previous components must be used
|
||||
pipe.connect(
|
||||
"splitter", "lexical_graph_builder", input_config={"text_chunks": "splitter"}
|
||||
)
|
||||
pipe.connect(
|
||||
"lexical_graph_builder",
|
||||
"writer",
|
||||
input_config={
|
||||
"graph": "lexical_graph_builder.graph",
|
||||
"lexical_graph_config": "lexical_graph_builder.config",
|
||||
},
|
||||
)
|
||||
# user input:
|
||||
# the initial text
|
||||
# and the list of entities and relations we are looking for
|
||||
pipe_inputs = {
|
||||
"splitter": {
|
||||
"text": """Albert Einstein was a German physicist born in 1879 who
|
||||
wrote many groundbreaking papers especially about general relativity
|
||||
and quantum mechanics. He worked for many different institutions, including
|
||||
the University of Bern in Switzerland and the University of Oxford."""
|
||||
},
|
||||
"lexical_graph_builder": {
|
||||
"document_info": {
|
||||
# 'path' can be anything
|
||||
"path": "example/pipeline_with_notifications"
|
||||
},
|
||||
},
|
||||
}
|
||||
# run the pipeline
|
||||
return await pipe.run(pipe_inputs)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
with neo4j.GraphDatabase.driver(
|
||||
"bolt://localhost:7687", auth=("neo4j", "password")
|
||||
) as driver:
|
||||
print(asyncio.run(main(driver)))
|
||||
@@ -0,0 +1,196 @@
|
||||
"""In this example, we set up a single pipeline with two Neo4j writers:
|
||||
one for creating the lexical graph (Document and Chunks)
|
||||
and another for creating the entity graph (entities and relations derived from the text).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
|
||||
from neo4j_graphrag.embeddings.openai import OpenAIEmbeddings
|
||||
from neo4j_graphrag.experimental.components.embedder import TextChunkEmbedder
|
||||
from neo4j_graphrag.experimental.components.entity_relation_extractor import (
|
||||
LLMEntityRelationExtractor,
|
||||
)
|
||||
from neo4j_graphrag.experimental.components.kg_writer import Neo4jWriter
|
||||
from neo4j_graphrag.experimental.components.lexical_graph import LexicalGraphBuilder
|
||||
from neo4j_graphrag.experimental.components.schema import (
|
||||
SchemaBuilder,
|
||||
NodeType,
|
||||
PropertyType,
|
||||
RelationshipType,
|
||||
)
|
||||
from neo4j_graphrag.experimental.components.text_splitters.fixed_size_splitter import (
|
||||
FixedSizeSplitter,
|
||||
)
|
||||
from neo4j_graphrag.experimental.components.types import LexicalGraphConfig
|
||||
from neo4j_graphrag.experimental.pipeline import Pipeline
|
||||
from neo4j_graphrag.experimental.pipeline.pipeline import PipelineResult
|
||||
from neo4j_graphrag.llm import LLMInterface, OpenAILLM
|
||||
|
||||
import neo4j
|
||||
|
||||
|
||||
async def define_and_run_pipeline(
|
||||
neo4j_driver: neo4j.Driver,
|
||||
llm: LLMInterface,
|
||||
lexical_graph_config: LexicalGraphConfig,
|
||||
text: str,
|
||||
) -> PipelineResult:
|
||||
"""Define and run the pipeline with the following components:
|
||||
|
||||
- Text Splitter: to split the text into manageable chunks of fixed size
|
||||
- Chunk Embedder: to embed the chunks' text
|
||||
- Lexical Graph Builder: to build the lexical graph, ie creating the chunk nodes and relationships between them
|
||||
- LG KG writer: save the lexical graph to Neo4j
|
||||
|
||||
- Schema Builder: this component takes a list of entities, relationships and
|
||||
possible triplets as inputs, validate them and return a schema ready to use
|
||||
for the rest of the pipeline
|
||||
- LLM Entity Relation Extractor is an LLM-based entity and relation extractor:
|
||||
based on the provided schema, the LLM will do its best to identity these
|
||||
entities and their relations within the provided text
|
||||
- EG KG writer: once entities and relations are extracted, they can be writen
|
||||
to a Neo4j database
|
||||
|
||||
"""
|
||||
pipe = Pipeline()
|
||||
# define the components
|
||||
pipe.add_component(
|
||||
FixedSizeSplitter(chunk_size=200, chunk_overlap=50, approximate=False),
|
||||
"splitter",
|
||||
)
|
||||
pipe.add_component(TextChunkEmbedder(embedder=OpenAIEmbeddings()), "chunk_embedder")
|
||||
pipe.add_component(
|
||||
LexicalGraphBuilder(lexical_graph_config),
|
||||
"lexical_graph_builder",
|
||||
)
|
||||
pipe.add_component(Neo4jWriter(neo4j_driver), "lg_writer")
|
||||
pipe.add_component(SchemaBuilder(), "schema")
|
||||
pipe.add_component(
|
||||
LLMEntityRelationExtractor(
|
||||
llm=llm,
|
||||
create_lexical_graph=False,
|
||||
),
|
||||
"extractor",
|
||||
)
|
||||
pipe.add_component(Neo4jWriter(neo4j_driver), "eg_writer")
|
||||
# define the execution order of component
|
||||
# and how the output of previous components must be used
|
||||
pipe.connect("splitter", "chunk_embedder", input_config={"text_chunks": "splitter"})
|
||||
pipe.connect(
|
||||
"chunk_embedder",
|
||||
"lexical_graph_builder",
|
||||
input_config={"text_chunks": "chunk_embedder"},
|
||||
)
|
||||
pipe.connect(
|
||||
"lexical_graph_builder",
|
||||
"lg_writer",
|
||||
input_config={
|
||||
"graph": "lexical_graph_builder.graph",
|
||||
"lexical_graph_config": "lexical_graph_builder.config",
|
||||
},
|
||||
)
|
||||
# define the execution order of component
|
||||
# and how the output of previous components must be used
|
||||
pipe.connect(
|
||||
"chunk_embedder", "extractor", input_config={"chunks": "chunk_embedder"}
|
||||
)
|
||||
pipe.connect("schema", "extractor", input_config={"schema": "schema"})
|
||||
pipe.connect(
|
||||
"extractor",
|
||||
"eg_writer",
|
||||
input_config={"graph": "extractor"},
|
||||
)
|
||||
# make sure the lexical graph is created before creating the entity graph:
|
||||
pipe.connect("lg_writer", "eg_writer", {})
|
||||
# user input:
|
||||
# the initial text
|
||||
# and the list of entities and relations we are looking for
|
||||
pipe_inputs = {
|
||||
"splitter": {
|
||||
"text": text,
|
||||
},
|
||||
"lexical_graph_builder": {
|
||||
"document_info": {
|
||||
# 'path' can be anything
|
||||
"path": "example/lexical_graph_from_text.py"
|
||||
},
|
||||
},
|
||||
"schema": {
|
||||
"node_types": [
|
||||
NodeType(
|
||||
label="Person",
|
||||
properties=[
|
||||
PropertyType(name="name", type="STRING"),
|
||||
PropertyType(name="place_of_birth", type="STRING"),
|
||||
PropertyType(name="date_of_birth", type="DATE"),
|
||||
],
|
||||
),
|
||||
NodeType(
|
||||
label="Organization",
|
||||
properties=[
|
||||
PropertyType(name="name", type="STRING"),
|
||||
PropertyType(name="country", type="STRING"),
|
||||
],
|
||||
),
|
||||
NodeType(
|
||||
label="Field",
|
||||
properties=[
|
||||
PropertyType(name="name", type="STRING"),
|
||||
],
|
||||
),
|
||||
],
|
||||
"relationship_types": [
|
||||
RelationshipType(
|
||||
label="WORKED_ON",
|
||||
),
|
||||
RelationshipType(
|
||||
label="WORKED_FOR",
|
||||
),
|
||||
],
|
||||
"patterns": [
|
||||
("Person", "WORKED_ON", "Field"),
|
||||
("Person", "WORKED_FOR", "Organization"),
|
||||
],
|
||||
},
|
||||
"extractor": {
|
||||
"lexical_graph_config": lexical_graph_config,
|
||||
},
|
||||
}
|
||||
# run the pipeline
|
||||
return await pipe.run(pipe_inputs)
|
||||
|
||||
|
||||
async def main(driver: neo4j.Driver) -> PipelineResult:
|
||||
# optional: define some custom node labels for the lexical graph:
|
||||
lexical_graph_config = LexicalGraphConfig(
|
||||
chunk_node_label="TextPart",
|
||||
document_node_label="Text",
|
||||
)
|
||||
text = """Albert Einstein was a German physicist born in 1879 who
|
||||
wrote many groundbreaking papers especially about general relativity
|
||||
and quantum mechanics. He worked for many different institutions, including
|
||||
the University of Bern in Switzerland and the University of Oxford."""
|
||||
llm = OpenAILLM(
|
||||
model_name="gpt-5",
|
||||
model_params={
|
||||
"max_tokens": 1000,
|
||||
"response_format": {"type": "json_object"},
|
||||
},
|
||||
)
|
||||
res = await define_and_run_pipeline(
|
||||
driver,
|
||||
llm,
|
||||
lexical_graph_config,
|
||||
text,
|
||||
)
|
||||
await llm.aclose()
|
||||
return res
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
with neo4j.GraphDatabase.driver(
|
||||
"bolt://localhost:7687", auth=("neo4j", "password")
|
||||
) as driver:
|
||||
print(asyncio.run(main(driver)))
|
||||
@@ -0,0 +1,215 @@
|
||||
"""In this example, we implement two pipelines:
|
||||
|
||||
1. A first pipeline reads a text, chunks it and save the chunks into Neo4j (the lexical graph)
|
||||
2. A second pipeline reads the chunks from the database, performs entity and relation extraction and save the extracted entities into the database.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
|
||||
from neo4j_graphrag.embeddings.openai import OpenAIEmbeddings
|
||||
from neo4j_graphrag.experimental.components.embedder import TextChunkEmbedder
|
||||
from neo4j_graphrag.experimental.components.entity_relation_extractor import (
|
||||
LLMEntityRelationExtractor,
|
||||
)
|
||||
from neo4j_graphrag.experimental.components.kg_writer import Neo4jWriter
|
||||
from neo4j_graphrag.experimental.components.lexical_graph import LexicalGraphBuilder
|
||||
from neo4j_graphrag.experimental.components.neo4j_reader import Neo4jChunkReader
|
||||
from neo4j_graphrag.experimental.components.schema import (
|
||||
SchemaBuilder,
|
||||
NodeType,
|
||||
PropertyType,
|
||||
RelationshipType,
|
||||
)
|
||||
from neo4j_graphrag.experimental.components.text_splitters.fixed_size_splitter import (
|
||||
FixedSizeSplitter,
|
||||
)
|
||||
from neo4j_graphrag.experimental.components.types import LexicalGraphConfig
|
||||
from neo4j_graphrag.experimental.pipeline import Pipeline
|
||||
from neo4j_graphrag.experimental.pipeline.pipeline import PipelineResult
|
||||
from neo4j_graphrag.llm import LLMInterface, OpenAILLM
|
||||
|
||||
import neo4j
|
||||
|
||||
|
||||
async def build_lexical_graph(
|
||||
neo4j_driver: neo4j.Driver,
|
||||
lexical_graph_config: LexicalGraphConfig,
|
||||
text: str,
|
||||
) -> PipelineResult:
|
||||
"""Define and run the pipeline with the following components:
|
||||
|
||||
- Text Splitter: to split the text into manageable chunks of fixed size
|
||||
- Chunk Embedder: to embed the chunks' text
|
||||
- Lexical Graph Builder: to build the lexical graph, ie creating the chunk nodes and relationships between them
|
||||
- KG writer: save the lexical graph to Neo4j
|
||||
"""
|
||||
pipe = Pipeline()
|
||||
# define the components
|
||||
pipe.add_component(
|
||||
FixedSizeSplitter(chunk_size=200, chunk_overlap=50, approximate=False),
|
||||
"splitter",
|
||||
)
|
||||
pipe.add_component(TextChunkEmbedder(embedder=OpenAIEmbeddings()), "chunk_embedder")
|
||||
pipe.add_component(
|
||||
LexicalGraphBuilder(lexical_graph_config),
|
||||
"lexical_graph_builder",
|
||||
)
|
||||
pipe.add_component(Neo4jWriter(neo4j_driver), "writer")
|
||||
# define the execution order of component
|
||||
# and how the output of previous components must be used
|
||||
pipe.connect("splitter", "chunk_embedder", input_config={"text_chunks": "splitter"})
|
||||
pipe.connect(
|
||||
"chunk_embedder",
|
||||
"lexical_graph_builder",
|
||||
input_config={"text_chunks": "chunk_embedder"},
|
||||
)
|
||||
pipe.connect(
|
||||
"lexical_graph_builder",
|
||||
"writer",
|
||||
input_config={
|
||||
"graph": "lexical_graph_builder.graph",
|
||||
"lexical_graph_config": "lexical_graph_builder.config",
|
||||
},
|
||||
)
|
||||
# user input:
|
||||
# the initial text
|
||||
# and the list of entities and relations we are looking for
|
||||
pipe_inputs = {
|
||||
"splitter": {
|
||||
"text": text,
|
||||
},
|
||||
"lexical_graph_builder": {
|
||||
"document_info": {
|
||||
# 'path' can be anything
|
||||
"path": "example/lexical_graph_from_text.py"
|
||||
},
|
||||
},
|
||||
}
|
||||
# run the pipeline
|
||||
return await pipe.run(pipe_inputs)
|
||||
|
||||
|
||||
async def read_chunk_and_perform_entity_extraction(
|
||||
neo4j_driver: neo4j.Driver,
|
||||
llm: LLMInterface,
|
||||
lexical_graph_config: LexicalGraphConfig,
|
||||
) -> PipelineResult:
|
||||
"""This is where we define and run the KG builder pipeline, instantiating a few
|
||||
components:
|
||||
|
||||
- Neo4j Chunk Reader: to embed the chunks' text
|
||||
- Schema Builder: this component takes a list of entities, relationships and
|
||||
possible triplets as inputs, validate them and return a schema ready to use
|
||||
for the rest of the pipeline
|
||||
- LLM Entity Relation Extractor is an LLM-based entity and relation extractor:
|
||||
based on the provided schema, the LLM will do its best to identity these
|
||||
entities and their relations within the provided text
|
||||
- KG writer: once entities and relations are extracted, they can be writen
|
||||
to a Neo4j database
|
||||
"""
|
||||
pipe = Pipeline()
|
||||
# define the components
|
||||
pipe.add_component(Neo4jChunkReader(neo4j_driver), "reader")
|
||||
pipe.add_component(SchemaBuilder(), "schema")
|
||||
pipe.add_component(
|
||||
LLMEntityRelationExtractor(
|
||||
llm=llm,
|
||||
create_lexical_graph=False,
|
||||
),
|
||||
"extractor",
|
||||
)
|
||||
pipe.add_component(Neo4jWriter(neo4j_driver), "writer")
|
||||
# define the execution order of component
|
||||
# and how the output of previous components must be used
|
||||
pipe.connect("reader", "extractor", input_config={"chunks": "reader"})
|
||||
pipe.connect("schema", "extractor", input_config={"schema": "schema"})
|
||||
pipe.connect(
|
||||
"extractor",
|
||||
"writer",
|
||||
input_config={"graph": "extractor"},
|
||||
)
|
||||
# user input:
|
||||
# the initial text
|
||||
# and the list of entities and relations we are looking for
|
||||
pipe_inputs = {
|
||||
"reader": {
|
||||
"lexical_graph_config": lexical_graph_config,
|
||||
},
|
||||
"schema": {
|
||||
"node_types": [
|
||||
NodeType(
|
||||
label="Person",
|
||||
properties=[
|
||||
PropertyType(name="name", type="STRING"),
|
||||
PropertyType(name="place_of_birth", type="STRING"),
|
||||
PropertyType(name="date_of_birth", type="DATE"),
|
||||
],
|
||||
),
|
||||
NodeType(
|
||||
label="Organization",
|
||||
properties=[
|
||||
PropertyType(name="name", type="STRING"),
|
||||
PropertyType(name="country", type="STRING"),
|
||||
],
|
||||
),
|
||||
NodeType(
|
||||
label="Field",
|
||||
properties=[
|
||||
PropertyType(name="name", type="STRING"),
|
||||
],
|
||||
),
|
||||
],
|
||||
"relationship_types": [
|
||||
RelationshipType(
|
||||
label="WORKED_ON",
|
||||
),
|
||||
RelationshipType(
|
||||
label="WORKED_FOR",
|
||||
),
|
||||
],
|
||||
"patterns": [
|
||||
("Person", "WORKED_ON", "Field"),
|
||||
("Person", "WORKED_FOR", "Organization"),
|
||||
],
|
||||
},
|
||||
"extractor": {
|
||||
"lexical_graph_config": lexical_graph_config,
|
||||
},
|
||||
}
|
||||
# run the pipeline
|
||||
return await pipe.run(pipe_inputs)
|
||||
|
||||
|
||||
async def main(driver: neo4j.Driver) -> PipelineResult:
|
||||
# optional: define some custom node labels for the lexical graph:
|
||||
lexical_graph_config = LexicalGraphConfig(
|
||||
document_node_label="Book", # default: "Document"
|
||||
chunk_node_label="Chapter", # default "Chunk"
|
||||
chunk_text_property="content", # default: "text"
|
||||
)
|
||||
text = """Albert Einstein was a German physicist born in 1879 who
|
||||
wrote many groundbreaking papers especially about general relativity
|
||||
and quantum mechanics. He worked for many different institutions, including
|
||||
the University of Bern in Switzerland and the University of Oxford."""
|
||||
llm = OpenAILLM(
|
||||
model_name="gpt-5",
|
||||
model_params={
|
||||
"max_tokens": 1000,
|
||||
"response_format": {"type": "json_object"},
|
||||
},
|
||||
)
|
||||
await build_lexical_graph(driver, lexical_graph_config, text=text)
|
||||
res = await read_chunk_and_perform_entity_extraction(
|
||||
driver, llm, lexical_graph_config
|
||||
)
|
||||
await llm.aclose()
|
||||
return res
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
with neo4j.GraphDatabase.driver(
|
||||
"bolt://localhost:7687", auth=("neo4j", "password")
|
||||
) as driver:
|
||||
print(asyncio.run(main(driver)))
|
||||
@@ -0,0 +1,58 @@
|
||||
# Copyright (c) "Neo4j"
|
||||
# Neo4j Sweden AB [https://neo4j.com]
|
||||
# #
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
# #
|
||||
# https://www.apache.org/licenses/LICENSE-2.0
|
||||
# #
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""This example illustrates how to visualize a Pipeline"""
|
||||
|
||||
from neo4j_graphrag.experimental.pipeline import Component, Pipeline
|
||||
from neo4j_graphrag.experimental.pipeline.component import DataModel
|
||||
from pydantic import validate_call
|
||||
|
||||
|
||||
class IntDataModel(DataModel):
|
||||
value: int
|
||||
message: str
|
||||
|
||||
|
||||
class Addition(Component):
|
||||
async def run(self, a: int, b: int) -> IntDataModel:
|
||||
return IntDataModel(value=a + b, message="addition complete")
|
||||
|
||||
|
||||
class Duplicate(Component):
|
||||
def __init__(self, factor: int = 2) -> None:
|
||||
self.factor = factor
|
||||
|
||||
async def run(self, number: int) -> IntDataModel:
|
||||
return IntDataModel(
|
||||
value=number * self.factor, message=f"multiplication by {self.factor} done"
|
||||
)
|
||||
|
||||
|
||||
class Save(Component):
|
||||
@validate_call
|
||||
async def run(self, number: IntDataModel) -> IntDataModel:
|
||||
return IntDataModel(value=number.value, message="saved")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
pipe = Pipeline()
|
||||
pipe.add_component(Duplicate(), "times_two")
|
||||
pipe.add_component(Duplicate(factor=10), "times_ten")
|
||||
pipe.add_component(Addition(), "addition")
|
||||
pipe.add_component(Save(), "save")
|
||||
pipe.connect("times_two", "addition", {"a": "times_two.value"})
|
||||
pipe.connect("times_ten", "addition", {"b": "times_ten.value"})
|
||||
pipe.connect("addition", "save", {"number": "addition"})
|
||||
pipe.draw("graph.html")
|
||||
pipe.draw("graph_full.html", hide_unused_outputs=False)
|
||||
Reference in New Issue
Block a user