참고소스 수정본
This commit is contained in:
51
참고/ontocast-main/ontocast/tool/triple_manager/__init__.py
Normal file
51
참고/ontocast-main/ontocast/tool/triple_manager/__init__.py
Normal file
@@ -0,0 +1,51 @@
|
||||
"""Triple store management package for OntoCast.
|
||||
|
||||
This package provides a unified interface for managing RDF triple stores
|
||||
across different backends. It includes abstract base classes and concrete
|
||||
implementations for various triple store technologies.
|
||||
|
||||
The package supports:
|
||||
- Abstract interfaces for triple store operations
|
||||
- Neo4j implementation using the n10s plugin
|
||||
- Fuseki implementation using Apache Fuseki
|
||||
- Filesystem implementation for local storage
|
||||
|
||||
All implementations support:
|
||||
- Fetching and storing ontologies
|
||||
- Serializing and retrieving facts
|
||||
- Authentication and connection management
|
||||
- Error handling and logging
|
||||
|
||||
Example:
|
||||
>>> from ontocast.tool.triple_manager import Neo4jTripleStoreManager
|
||||
>>> manager = Neo4jTripleStoreManager(uri="bolt://localhost:7687")
|
||||
>>> ontologies = manager.fetch_ontologies()
|
||||
"""
|
||||
|
||||
from .core import (
|
||||
TripleStoreManager,
|
||||
)
|
||||
from .filesystem_manager import (
|
||||
FilesystemTripleStoreManager,
|
||||
)
|
||||
from .fuseki import (
|
||||
FusekiTripleStoreManager,
|
||||
)
|
||||
from .mock import (
|
||||
MockFusekiTripleStoreManager,
|
||||
MockNeo4jTripleStoreManager,
|
||||
MockTripleStoreManager,
|
||||
)
|
||||
from .neo4j import (
|
||||
Neo4jTripleStoreManager,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"TripleStoreManager",
|
||||
"Neo4jTripleStoreManager",
|
||||
"FusekiTripleStoreManager",
|
||||
"FilesystemTripleStoreManager",
|
||||
"MockTripleStoreManager",
|
||||
"MockFusekiTripleStoreManager",
|
||||
"MockNeo4jTripleStoreManager",
|
||||
]
|
||||
159
참고/ontocast-main/ontocast/tool/triple_manager/core.py
Normal file
159
참고/ontocast-main/ontocast/tool/triple_manager/core.py
Normal file
@@ -0,0 +1,159 @@
|
||||
"""Triple store management tools for OntoCast.
|
||||
|
||||
This module provides functionality for managing RDF triple stores, including
|
||||
abstract interfaces and concrete implementations for different triple store backends.
|
||||
"""
|
||||
|
||||
import abc
|
||||
import os
|
||||
|
||||
from pydantic import Field
|
||||
from rdflib import Graph
|
||||
|
||||
from ontocast.onto.ontology import Ontology
|
||||
from ontocast.onto.rdfgraph import RDFGraph
|
||||
from ontocast.tool import Tool
|
||||
|
||||
|
||||
class TripleStoreManager(Tool):
|
||||
"""Base class for managing RDF triple stores.
|
||||
|
||||
This class defines the interface for triple store management operations,
|
||||
including fetching and storing ontologies and their graphs. All concrete
|
||||
triple store implementations should inherit from this class.
|
||||
|
||||
This is an abstract base class that must be implemented by specific
|
||||
triple store backends (e.g., Neo4j, Fuseki, Filesystem).
|
||||
"""
|
||||
|
||||
def __init__(self, **kwargs):
|
||||
"""Initialize the triple store manager.
|
||||
|
||||
Args:
|
||||
**kwargs: Additional keyword arguments passed to the parent class.
|
||||
"""
|
||||
super().__init__(**kwargs)
|
||||
|
||||
@abc.abstractmethod
|
||||
def fetch_ontologies(self) -> list[Ontology]:
|
||||
"""Fetch all available ontologies from the triple store.
|
||||
|
||||
This method should retrieve all ontologies stored in the triple store
|
||||
and return them as Ontology objects with their associated RDF graphs.
|
||||
|
||||
Returns:
|
||||
list[Ontology]: List of available ontologies with their graphs.
|
||||
"""
|
||||
return []
|
||||
|
||||
@abc.abstractmethod
|
||||
def serialize_graph(self, graph: Graph, **kwargs) -> bool | None:
|
||||
"""Store an RDF graph in the triple store.
|
||||
|
||||
This method should store the given RDF graph in the triple store.
|
||||
The implementation may choose how to organize the storage (e.g., as named graphs,
|
||||
in specific collections, etc.).
|
||||
|
||||
Args:
|
||||
graph: The RDF graph to store.
|
||||
**kwargs: Implementation-specific arguments (e.g., fname for filesystem, graph_uri for Fuseki).
|
||||
|
||||
Returns:
|
||||
bool | None: Implementation-specific return value (bool for Fuseki, summary for Neo4j, None for Filesystem).
|
||||
"""
|
||||
pass
|
||||
|
||||
@abc.abstractmethod
|
||||
def serialize(self, o: Ontology | RDFGraph, **kwargs) -> bool | None: # type: ignore[override]
|
||||
"""Store an RDF graph in the triple store.
|
||||
|
||||
This method should store the given RDF graph in the triple store.
|
||||
The implementation may choose how to organize the storage (e.g., as named graphs,
|
||||
in specific collections, etc.).
|
||||
|
||||
Args:
|
||||
o: RDF graph or Ontology object to store.
|
||||
**kwargs: Implementation-specific arguments (e.g., graph_uri for Fuseki).
|
||||
|
||||
Returns:
|
||||
bool | None: Implementation-specific return value (bool for Fuseki, summary for Neo4j, None for Filesystem).
|
||||
"""
|
||||
pass
|
||||
|
||||
@abc.abstractmethod
|
||||
async def clean(self, dataset: str | None = None) -> None:
|
||||
"""Clean/flush data from the triple store.
|
||||
|
||||
This method removes data from the triple store. For Fuseki, the optional
|
||||
dataset parameter allows cleaning a specific dataset, or all datasets if None.
|
||||
For Neo4j and Filesystem, the dataset parameter is ignored.
|
||||
|
||||
Args:
|
||||
dataset: Optional dataset name to clean (Fuseki only). If None, cleans
|
||||
all data. For other stores, this parameter is ignored.
|
||||
|
||||
Warning: This operation is irreversible and will delete all data.
|
||||
|
||||
Raises:
|
||||
NotImplementedError: If the triple store doesn't support cleaning.
|
||||
"""
|
||||
raise NotImplementedError("clean() method must be implemented by subclasses")
|
||||
|
||||
|
||||
class TripleStoreManagerWithAuth(TripleStoreManager):
|
||||
"""Base class for triple store managers that require authentication.
|
||||
|
||||
This class provides common functionality for triple store managers that
|
||||
need URI and authentication credentials. It handles environment variable
|
||||
loading and credential parsing.
|
||||
|
||||
Attributes:
|
||||
uri: The connection URI for the triple store.
|
||||
auth: Authentication tuple (username, password) for the triple store.
|
||||
"""
|
||||
|
||||
uri: str | None = Field(default=None, description="Triple store connection URI")
|
||||
auth: tuple | None = Field(
|
||||
default=None, description="Triple store authentication tuple (user, password)"
|
||||
)
|
||||
|
||||
def __init__(self, uri=None, auth=None, env_uri=None, env_auth=None, **kwargs):
|
||||
"""Initialize the triple store manager with authentication.
|
||||
|
||||
This method handles loading URI and authentication credentials from
|
||||
either direct parameters or environment variables. It also parses
|
||||
authentication strings in the format "user/password".
|
||||
|
||||
Args:
|
||||
uri: Direct URI for the triple store connection.
|
||||
auth: Direct authentication tuple or string in "user/password" format.
|
||||
env_uri: Environment variable name for the URI (e.g., "NEO4J_URI").
|
||||
env_auth: Environment variable name for authentication (e.g., "NEO4J_AUTH").
|
||||
**kwargs: Additional keyword arguments passed to the parent class.
|
||||
|
||||
Raises:
|
||||
ValueError: If authentication string is not in "user/password" format.
|
||||
|
||||
Example:
|
||||
>>> manager = TripleStoreManagerWithAuth(
|
||||
... env_uri="NEO4J_URI",
|
||||
... env_auth="NEO4J_AUTH"
|
||||
... )
|
||||
"""
|
||||
# Use env vars if not provided
|
||||
uri = uri or (os.getenv(env_uri) if env_uri else None)
|
||||
auth_env = auth or (os.getenv(env_auth) if env_auth else None)
|
||||
|
||||
if auth_env and not isinstance(auth_env, tuple):
|
||||
if "/" in auth_env:
|
||||
user, password = auth_env.split("/", 1)
|
||||
auth = (user, password)
|
||||
else:
|
||||
raise ValueError(
|
||||
f"{env_auth or 'TRIPLESTORE_AUTH'} must be in 'user/password' format"
|
||||
)
|
||||
elif isinstance(auth_env, tuple):
|
||||
auth = auth_env
|
||||
# else: auth remains None
|
||||
|
||||
super().__init__(uri=uri, auth=auth, **kwargs)
|
||||
@@ -0,0 +1,151 @@
|
||||
"""Filesystem triple store management for OntoCast.
|
||||
|
||||
This module provides a concrete implementation of triple store management
|
||||
using the local filesystem for storage. It supports reading and writing
|
||||
ontologies and facts as Turtle files.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import pathlib
|
||||
|
||||
from rdflib import Graph
|
||||
|
||||
from ontocast.onto.ontology import Ontology
|
||||
from ontocast.onto.rdfgraph import RDFGraph
|
||||
from ontocast.tool.triple_manager.core import TripleStoreManager
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class FilesystemTripleStoreManager(TripleStoreManager):
|
||||
"""Filesystem-based implementation of triple store management.
|
||||
|
||||
This class provides a concrete implementation of triple store management
|
||||
using the local filesystem for storage. It reads and writes ontologies
|
||||
and facts as Turtle (.ttl) files in specified directories.
|
||||
|
||||
The manager supports:
|
||||
- Loading ontologies from a dedicated ontology directory
|
||||
- Storing ontologies with versioned filenames
|
||||
- Storing facts with customizable filenames based on specifications
|
||||
- Error handling for file operations
|
||||
|
||||
Attributes:
|
||||
working_directory: Path to the working directory for storing data.
|
||||
ontology_path: Optional path to the ontology directory for loading ontologies.
|
||||
"""
|
||||
|
||||
working_directory: pathlib.Path | None
|
||||
ontology_path: pathlib.Path | None
|
||||
|
||||
def __init__(self, **kwargs):
|
||||
"""Initialize the filesystem triple store manager.
|
||||
|
||||
This method sets up the filesystem manager with the specified
|
||||
working and ontology directories.
|
||||
|
||||
Args:
|
||||
**kwargs: Additional keyword arguments passed to the parent class.
|
||||
working_directory: Path to the working directory for storing data.
|
||||
ontology_path: Path to the ontology directory for loading ontologies.
|
||||
|
||||
Example:
|
||||
>>> manager = FilesystemTripleStoreManager(
|
||||
... working_directory="/path/to/work",
|
||||
... ontology_path="/path/to/ontologies"
|
||||
... )
|
||||
"""
|
||||
super().__init__(**kwargs)
|
||||
|
||||
def fetch_ontologies(self) -> list[Ontology]:
|
||||
"""Fetch all available ontologies from the filesystem.
|
||||
|
||||
This method scans the ontology directory for Turtle (.ttl) files
|
||||
and loads each one as an Ontology object. Files are processed
|
||||
in sorted order for consistent results.
|
||||
|
||||
Returns:
|
||||
list[Ontology]: List of all ontologies found in the ontology directory.
|
||||
|
||||
Example:
|
||||
>>> ontologies = manager.fetch_ontologies()
|
||||
>>> for onto in ontologies:
|
||||
... print(f"Loaded ontology: {onto.ontology_id}")
|
||||
"""
|
||||
ontologies = []
|
||||
if self.ontology_path is not None:
|
||||
sorted_files = sorted(self.ontology_path.glob("*.ttl"))
|
||||
for fname in sorted_files:
|
||||
try:
|
||||
ontology = Ontology.from_file(fname)
|
||||
ontologies.append(ontology)
|
||||
logger.debug(f"Successfully loaded ontology from {fname}")
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to load ontology {fname}: {str(e)}")
|
||||
return ontologies
|
||||
|
||||
def serialize_graph(self, graph: Graph, **kwargs) -> bool | None:
|
||||
"""Store an RDF graph in the filesystem.
|
||||
|
||||
This method stores the given RDF graph as a Turtle file in the
|
||||
working directory. The filename is generated based on the graph_uri
|
||||
parameter or defaults to "current.ttl".
|
||||
|
||||
Args:
|
||||
graph: The RDF graph to store.
|
||||
fname: str
|
||||
|
||||
Example:
|
||||
>>> graph = RDFGraph()
|
||||
>>> manager.serialize_graph(graph)
|
||||
# Creates: working_directory/current.ttl
|
||||
|
||||
>>> manager.serialize_graph(graph, fname="facts_abc.ttl")
|
||||
"""
|
||||
if self.working_directory is None:
|
||||
return
|
||||
|
||||
fname: str = kwargs.pop("fname")
|
||||
output_path = self.working_directory / fname
|
||||
graph.serialize(format="turtle", destination=output_path)
|
||||
logger.info(f"Graph saved to {output_path}")
|
||||
|
||||
def serialize(self, o: Ontology | RDFGraph, graph_uri: str | None = None): # type: ignore[override]
|
||||
if isinstance(o, Ontology):
|
||||
graph = o.graph
|
||||
fname = f"ontology_{o.ontology_id}_{o.version}.ttl"
|
||||
elif isinstance(o, RDFGraph):
|
||||
graph = o
|
||||
if graph_uri:
|
||||
s = graph_uri.split("/")[-2:]
|
||||
s = "_".join([x for x in s if x])
|
||||
fname = f"facts_{s}.ttl"
|
||||
else:
|
||||
fname = "facts_default.ttl"
|
||||
else:
|
||||
raise TypeError(f"unsupported obj of type {type(o)} received")
|
||||
|
||||
self.serialize_graph(graph=graph, fname=fname)
|
||||
|
||||
async def clean(self, dataset: str | None = None) -> None:
|
||||
"""Clean/flush all data from the filesystem triple store.
|
||||
|
||||
This method deletes all Turtle (.ttl) files from both the working
|
||||
directory and the ontology directory.
|
||||
|
||||
Args:
|
||||
dataset: Optional dataset parameter (ignored for Filesystem, which doesn't
|
||||
support datasets). Included for interface compatibility.
|
||||
|
||||
Warning: This operation is irreversible and will delete all data.
|
||||
|
||||
Raises:
|
||||
Exception: If the cleanup operation fails.
|
||||
"""
|
||||
if dataset is not None:
|
||||
logger.warning(
|
||||
f"Dataset parameter '{dataset}' ignored for Filesystem (datasets not supported)"
|
||||
)
|
||||
logger.warning(
|
||||
"clean method not implemented for FilesystemTripleStoreManager"
|
||||
)
|
||||
811
참고/ontocast-main/ontocast/tool/triple_manager/fuseki.py
Normal file
811
참고/ontocast-main/ontocast/tool/triple_manager/fuseki.py
Normal file
@@ -0,0 +1,811 @@
|
||||
"""Fuseki triple store management for OntoCast.
|
||||
|
||||
This module provides a concrete implementation of triple store management
|
||||
using Apache Fuseki as the backend. It supports named graphs for ontologies
|
||||
and facts, with proper authentication and dataset management.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import re
|
||||
from collections import defaultdict
|
||||
from urllib.parse import quote
|
||||
|
||||
import httpx
|
||||
from pydantic import Field
|
||||
from rdflib import Graph
|
||||
from rdflib.namespace import OWL, RDF
|
||||
|
||||
from ontocast.onto.constants import DEFAULT_DATASET, DEFAULT_ONTOLOGIES_DATASET
|
||||
from ontocast.onto.ontology import Ontology
|
||||
from ontocast.onto.rdfgraph import RDFGraph
|
||||
from ontocast.tool.triple_manager.core import TripleStoreManagerWithAuth
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def deterministic_turtle_serialization(graph: Graph) -> str:
|
||||
"""Create a deterministic Turtle serialization of an RDF graph.
|
||||
|
||||
This function ensures that the same graph content will always produce
|
||||
the same Turtle output, regardless of the order triples were added or
|
||||
how they're stored in Fuseki. This is crucial for caching to work
|
||||
correctly.
|
||||
|
||||
Args:
|
||||
graph: The RDF graph to serialize.
|
||||
|
||||
Returns:
|
||||
str: Deterministically serialized Turtle string.
|
||||
"""
|
||||
# Capture and sort namespaces
|
||||
prefix_lines = [
|
||||
f"@prefix {p}: <{ns}> ."
|
||||
for p, ns in sorted(graph.namespace_manager.namespaces())
|
||||
]
|
||||
|
||||
# Sort triples by their string representation
|
||||
triples_sorted = sorted(graph, key=lambda t: (str(t[0]), str(t[1]), str(t[2])))
|
||||
|
||||
# Serialize triples using n3 format to get proper Turtle syntax
|
||||
triple_lines = [
|
||||
f"{s.n3(graph.namespace_manager)} {p.n3(graph.namespace_manager)} {o.n3(graph.namespace_manager)} ."
|
||||
for s, p, o in triples_sorted
|
||||
]
|
||||
|
||||
# Return sorted prefixes followed by sorted triples
|
||||
return "\n".join(prefix_lines + [""] + triple_lines)
|
||||
|
||||
|
||||
def _compare_versions(ver1: str, ver2: str) -> int:
|
||||
"""Compare two semantic version strings.
|
||||
|
||||
Args:
|
||||
ver1: First version string (e.g., "1.2.3")
|
||||
ver2: Second version string (e.g., "1.3.0")
|
||||
|
||||
Returns:
|
||||
int: Negative if ver1 < ver2, 0 if equal, positive if ver1 > ver2
|
||||
"""
|
||||
|
||||
def _parse_version(v: str) -> tuple:
|
||||
# Simple version parser - splits by dots and converts to int
|
||||
parts = v.split(".")
|
||||
result = []
|
||||
for part in parts:
|
||||
# Remove any non-numeric suffix
|
||||
numeric_part = re.sub(r"[^0-9].*$", "", part)
|
||||
result.append(int(numeric_part) if numeric_part else 0)
|
||||
# Pad to 3 components
|
||||
while len(result) < 3:
|
||||
result.append(0)
|
||||
return tuple(result)
|
||||
|
||||
try:
|
||||
v1_parts = _parse_version(ver1)
|
||||
v2_parts = _parse_version(ver2)
|
||||
if v1_parts < v2_parts:
|
||||
return -1
|
||||
elif v1_parts > v2_parts:
|
||||
return 1
|
||||
return 0
|
||||
except Exception:
|
||||
# If parsing fails, use string comparison
|
||||
return 1 if ver1 > ver2 else (-1 if ver1 < ver2 else 0)
|
||||
|
||||
|
||||
class FusekiTripleStoreManager(TripleStoreManagerWithAuth):
|
||||
"""Fuseki-based triple store manager.
|
||||
|
||||
This class provides a concrete implementation of triple store management
|
||||
using Apache Fuseki. It stores ontologies as named graphs using their
|
||||
URIs as graph names, and supports dataset creation and cleanup.
|
||||
|
||||
The manager uses Fuseki's REST API for all operations, including:
|
||||
- Dataset creation and management
|
||||
- Named graph operations for ontologies
|
||||
- SPARQL queries for ontology discovery
|
||||
- Graph-level data operations
|
||||
|
||||
Attributes:
|
||||
dataset: The Fuseki dataset name to use for storage.
|
||||
clean: Whether to clean the dataset on initialization.
|
||||
"""
|
||||
|
||||
dataset: str | None = Field(default=None, description="Fuseki dataset name")
|
||||
ontologies_dataset: str = Field(
|
||||
default=DEFAULT_ONTOLOGIES_DATASET,
|
||||
description="Fuseki dataset name for ontologies",
|
||||
)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
uri=None,
|
||||
auth=None,
|
||||
dataset=None,
|
||||
ontologies_dataset=None,
|
||||
**kwargs,
|
||||
):
|
||||
"""Initialize the Fuseki triple store manager.
|
||||
|
||||
This method sets up the connection to Fuseki and creates the dataset
|
||||
if it doesn't exist. The dataset is NOT cleaned on initialization.
|
||||
|
||||
Args:
|
||||
uri: Fuseki server URI (e.g., "http://localhost:3030").
|
||||
auth: Authentication tuple (username, password) or string in "user/password" format.
|
||||
dataset: Dataset name to use for storage.
|
||||
ontologies_dataset: Dataset name for ontologies (defaults to separate dataset).
|
||||
**kwargs: Additional keyword arguments passed to the parent class.
|
||||
|
||||
Raises:
|
||||
ValueError: If dataset is not specified in URI or as argument.
|
||||
|
||||
Example:
|
||||
>>> manager = FusekiTripleStoreManager(
|
||||
... uri="http://localhost:3030",
|
||||
... dataset="test"
|
||||
... )
|
||||
>>> # To clean the dataset, use the clean() method explicitly:
|
||||
>>> await manager.clean()
|
||||
"""
|
||||
super().__init__(
|
||||
uri=uri, auth=auth, env_uri="FUSEKI_URI", env_auth="FUSEKI_AUTH", **kwargs
|
||||
)
|
||||
if dataset is None:
|
||||
self.dataset = DEFAULT_DATASET
|
||||
else:
|
||||
self.dataset = dataset
|
||||
self.ontologies_dataset = ontologies_dataset or DEFAULT_ONTOLOGIES_DATASET
|
||||
|
||||
# Initialize httpx client for async operations
|
||||
self._client: httpx.AsyncClient | None = None
|
||||
|
||||
# Initialize datasets synchronously (for backward compatibility)
|
||||
# In async contexts, use async_init() instead
|
||||
asyncio.run(self._async_init_with_cleanup())
|
||||
|
||||
async def _async_init_with_cleanup(self):
|
||||
"""Wrapper for async_init that ensures proper cleanup when using asyncio.run().
|
||||
|
||||
This method creates a temporary client and ensures it's properly closed
|
||||
before returning, preventing "Event loop is closed" errors.
|
||||
"""
|
||||
async with httpx.AsyncClient(
|
||||
auth=self._prepare_auth(), timeout=30.0
|
||||
) as temp_client:
|
||||
# Temporarily replace the client
|
||||
original_client = self._client
|
||||
self._client = temp_client
|
||||
try:
|
||||
await self._async_init()
|
||||
finally:
|
||||
# Restore original client
|
||||
self._client = original_client
|
||||
|
||||
async def _async_init(self):
|
||||
"""Async initialization of datasets."""
|
||||
await self.init_dataset(self.dataset)
|
||||
if self.ontologies_dataset != self.dataset:
|
||||
await self.init_dataset(self.ontologies_dataset)
|
||||
|
||||
def _prepare_auth(self) -> httpx.BasicAuth | None:
|
||||
"""Prepare httpx BasicAuth from self.auth.
|
||||
|
||||
Returns:
|
||||
httpx.BasicAuth instance or None if no auth is configured.
|
||||
"""
|
||||
if self.auth:
|
||||
if isinstance(self.auth, tuple):
|
||||
return httpx.BasicAuth(*self.auth)
|
||||
elif isinstance(self.auth, str) and "/" in self.auth:
|
||||
parts = self.auth.split("/", 1)
|
||||
if len(parts) == 2:
|
||||
username, password = parts[0], parts[1]
|
||||
return httpx.BasicAuth(username, password)
|
||||
return None
|
||||
|
||||
async def _get_client(self) -> httpx.AsyncClient:
|
||||
"""Get or create the httpx async client."""
|
||||
if self._client is None:
|
||||
auth = self._prepare_auth()
|
||||
self._client = httpx.AsyncClient(auth=auth, timeout=30.0)
|
||||
return self._client
|
||||
|
||||
async def close(self):
|
||||
"""Close the httpx client."""
|
||||
if self._client is not None:
|
||||
await self._client.aclose()
|
||||
self._client = None
|
||||
|
||||
async def update_dataset(self, new_dataset: str) -> None:
|
||||
"""Update the dataset name for this manager.
|
||||
|
||||
This method allows changing the dataset without recreating the entire
|
||||
manager, which is useful for API requests that specify different datasets.
|
||||
|
||||
Args:
|
||||
new_dataset: The new dataset name to use.
|
||||
"""
|
||||
if not new_dataset:
|
||||
raise ValueError("Dataset name cannot be empty")
|
||||
|
||||
self.dataset = new_dataset
|
||||
await self.init_dataset(self.dataset)
|
||||
logger.info(f"Updated Fuseki dataset to: {self.dataset}")
|
||||
|
||||
async def clean(self, dataset: str | None = None) -> None:
|
||||
"""Clean/flush data from Fuseki dataset(s).
|
||||
|
||||
This method removes all named graphs and clears the default graph
|
||||
from the specified dataset, or all datasets if no dataset is specified.
|
||||
|
||||
Args:
|
||||
dataset: Optional dataset name to clean. If None, cleans both the main
|
||||
dataset and the ontologies dataset. If specified, cleans only that dataset.
|
||||
|
||||
Warning: This operation is irreversible and will delete all data
|
||||
from the specified dataset(s).
|
||||
|
||||
The method handles errors gracefully and logs the results of
|
||||
each cleanup operation.
|
||||
|
||||
Example:
|
||||
>>> # Clean all datasets
|
||||
>>> await manager.clean()
|
||||
>>> # Clean specific dataset
|
||||
>>> await manager.clean(dataset="my_dataset")
|
||||
"""
|
||||
if dataset is None:
|
||||
# Clean all datasets (main and ontologies)
|
||||
# self.dataset is guaranteed to be a string (set to DEFAULT_DATASET if None in __init__)
|
||||
assert self.dataset is not None, "Dataset should never be None"
|
||||
await self._clean_dataset_by_name(self.dataset)
|
||||
logger.info(f"Fuseki dataset '{self.dataset}' cleaned (all data deleted)")
|
||||
|
||||
# Also clean the ontologies dataset if it's different
|
||||
if self.ontologies_dataset != self.dataset:
|
||||
await self._clean_dataset_by_name(self.ontologies_dataset)
|
||||
logger.info(
|
||||
f"Fuseki ontologies dataset '{self.ontologies_dataset}' cleaned (all data deleted)"
|
||||
)
|
||||
else:
|
||||
# Clean only the specified dataset
|
||||
await self._clean_dataset_by_name(dataset)
|
||||
logger.info(f"Fuseki dataset '{dataset}' cleaned (all data deleted)")
|
||||
|
||||
async def _clean_dataset_by_name(self, dataset_name: str) -> None:
|
||||
"""Clean a specific dataset by name.
|
||||
|
||||
This is a helper method that performs the actual cleaning of a single dataset.
|
||||
It deletes all named graphs and clears the default graph.
|
||||
|
||||
Uses a temporary client to avoid event loop cleanup issues when called
|
||||
from different async contexts.
|
||||
|
||||
Args:
|
||||
dataset_name: Name of the dataset to clean.
|
||||
|
||||
Raises:
|
||||
Exception: If the cleanup operation fails.
|
||||
"""
|
||||
# Use a temporary client to avoid event loop cleanup issues
|
||||
async with httpx.AsyncClient(auth=self._prepare_auth(), timeout=30.0) as client:
|
||||
try:
|
||||
dataset_url = f"{self.uri}/{dataset_name}"
|
||||
sparql_update_url = f"{dataset_url}/update"
|
||||
sparql_url = f"{dataset_url}/sparql"
|
||||
|
||||
# Delete all named graphs
|
||||
query = """
|
||||
SELECT DISTINCT ?g WHERE {
|
||||
GRAPH ?g { ?s ?p ?o }
|
||||
}
|
||||
"""
|
||||
response = await client.post(
|
||||
sparql_url,
|
||||
data={"query": query, "format": "application/sparql-results+json"},
|
||||
)
|
||||
|
||||
if response.status_code == 200:
|
||||
results = response.json()
|
||||
tasks = []
|
||||
for binding in results.get("results", {}).get("bindings", []):
|
||||
graph_uri = binding["g"]["value"]
|
||||
# Delete the named graph using SPARQL UPDATE
|
||||
drop_query = f"DROP GRAPH <{graph_uri}>"
|
||||
tasks.append(
|
||||
client.post(
|
||||
sparql_update_url,
|
||||
data={"update": drop_query},
|
||||
)
|
||||
)
|
||||
|
||||
# Execute all deletions in parallel
|
||||
delete_responses = await asyncio.gather(
|
||||
*tasks, return_exceptions=True
|
||||
)
|
||||
for i, delete_response in enumerate(delete_responses):
|
||||
graph_uri = results["results"]["bindings"][i]["g"]["value"]
|
||||
if isinstance(delete_response, Exception):
|
||||
logger.warning(
|
||||
f"Failed to delete graph {graph_uri}: {delete_response}"
|
||||
)
|
||||
elif isinstance(delete_response, httpx.Response):
|
||||
if delete_response.status_code in (200, 204):
|
||||
logger.debug(f"Deleted named graph: {graph_uri}")
|
||||
else:
|
||||
logger.warning(
|
||||
f"Failed to delete graph {graph_uri}: {delete_response.status_code}"
|
||||
)
|
||||
|
||||
# Clear the default graph using SPARQL UPDATE
|
||||
clear_query = "CLEAR DEFAULT"
|
||||
clear_response = await client.post(
|
||||
sparql_update_url,
|
||||
data={"update": clear_query},
|
||||
)
|
||||
if clear_response.status_code in (200, 204):
|
||||
logger.debug(f"Cleared default graph in dataset '{dataset_name}'")
|
||||
else:
|
||||
logger.warning(
|
||||
f"Failed to clear default graph in dataset '{dataset_name}': {clear_response.status_code}"
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to clean dataset '{dataset_name}': {e}")
|
||||
raise
|
||||
|
||||
async def init_dataset(self, dataset_name):
|
||||
"""Initialize a Fuseki dataset.
|
||||
|
||||
This method creates a new dataset in Fuseki if it doesn't already exist.
|
||||
It uses Fuseki's admin API to create the dataset with TDB2 storage.
|
||||
|
||||
Uses a temporary client to avoid event loop cleanup issues when called
|
||||
from different async contexts.
|
||||
|
||||
Args:
|
||||
dataset_name: Name of the dataset to create.
|
||||
|
||||
Note:
|
||||
This method will not fail if the dataset already exists.
|
||||
"""
|
||||
# Use a temporary client to avoid event loop cleanup issues
|
||||
async with httpx.AsyncClient(auth=self._prepare_auth(), timeout=30.0) as client:
|
||||
fuseki_admin_url = f"{self.uri}/$/datasets"
|
||||
|
||||
payload = {"dbName": dataset_name, "dbType": "tdb2"}
|
||||
|
||||
headers = {"Content-Type": "application/x-www-form-urlencoded"}
|
||||
|
||||
response = await client.post(
|
||||
fuseki_admin_url, data=payload, headers=headers
|
||||
)
|
||||
|
||||
if response.status_code == 200 or response.status_code == 201:
|
||||
logger.info(f"Fuseki dataset '{dataset_name}' created successfully.")
|
||||
elif response.status_code == 409:
|
||||
logger.info(
|
||||
f"Fuseki status code: {response.status_code}; {response.text.strip()}"
|
||||
)
|
||||
else:
|
||||
logger.error(
|
||||
f"Failed to create dataset {dataset_name}. Status code: {response.status_code}"
|
||||
)
|
||||
logger.error(f"Response: {response.text.strip()}")
|
||||
|
||||
def _get_dataset_url(self):
|
||||
"""Get the full URL for the dataset.
|
||||
|
||||
Returns:
|
||||
str: The complete URL for the dataset endpoint.
|
||||
"""
|
||||
return f"{self.uri}/{self.dataset}"
|
||||
|
||||
def _get_ontologies_dataset_url(self):
|
||||
"""Get the full URL for the ontologies dataset.
|
||||
|
||||
Returns:
|
||||
str: The complete URL for the ontologies dataset endpoint.
|
||||
"""
|
||||
return f"{self.uri}/{self.ontologies_dataset}"
|
||||
|
||||
def fetch_ontologies(self) -> list[Ontology]:
|
||||
"""Synchronous wrapper for fetch_ontologies.
|
||||
|
||||
For async usage, use afetch_ontologies() instead.
|
||||
"""
|
||||
# Use a temporary client for this operation to avoid event loop cleanup issues
|
||||
return asyncio.run(self._fetch_ontologies_with_cleanup())
|
||||
|
||||
async def afetch_ontologies(self) -> list[Ontology]:
|
||||
"""Async version of fetch_ontologies.
|
||||
|
||||
This is the preferred method when running in an async context.
|
||||
"""
|
||||
return await self._fetch_ontologies_async()
|
||||
|
||||
async def _fetch_ontologies_with_cleanup(self) -> list[Ontology]:
|
||||
"""Wrapper that ensures proper cleanup when using asyncio.run().
|
||||
|
||||
This method creates a temporary client and ensures it's properly closed
|
||||
before returning, preventing "Event loop is closed" errors.
|
||||
"""
|
||||
async with httpx.AsyncClient(
|
||||
auth=self._prepare_auth(), timeout=30.0
|
||||
) as temp_client:
|
||||
# Temporarily replace the client
|
||||
original_client = self._client
|
||||
self._client = temp_client
|
||||
try:
|
||||
return await self._fetch_ontologies_async()
|
||||
finally:
|
||||
# Restore original client
|
||||
self._client = original_client
|
||||
|
||||
async def _fetch_ontologies_async(self) -> list[Ontology]:
|
||||
"""Fetch all ontologies from their corresponding named graphs.
|
||||
|
||||
This method discovers all ontologies in the Fuseki ontologies dataset and
|
||||
fetches each one from its corresponding named graph. For versioned ontologies,
|
||||
it returns only the latest version for each unique ontology IRI.
|
||||
|
||||
1. Discovery: List all named graphs (which may be versioned URIs)
|
||||
2. Fetching: Retrieve each ontology from its named graph (in parallel)
|
||||
3. Deduplication: For versioned ontologies, keep only the latest version
|
||||
|
||||
Returns:
|
||||
list[Ontology]: List of the latest version of each ontology found.
|
||||
|
||||
Example:
|
||||
>>> ontologies = await manager.fetch_ontologies()
|
||||
>>> for onto in ontologies:
|
||||
... print(f"Found ontology: {onto.iri} v{onto.version}")
|
||||
"""
|
||||
client = await self._get_client()
|
||||
sparql_url = f"{self._get_ontologies_dataset_url()}/sparql"
|
||||
|
||||
# Step 1: List all named graphs
|
||||
list_query = """
|
||||
SELECT DISTINCT ?g WHERE {
|
||||
GRAPH ?g { ?s ?p ?o }
|
||||
}
|
||||
"""
|
||||
response = await client.post(
|
||||
sparql_url,
|
||||
data={"query": list_query, "format": "application/sparql-results+json"},
|
||||
)
|
||||
if response.status_code != 200:
|
||||
logger.error(f"Failed to list graphs from Fuseki: {response.text}")
|
||||
return []
|
||||
|
||||
results = response.json()
|
||||
graph_uris = []
|
||||
for binding in results.get("results", {}).get("bindings", []):
|
||||
graph_uri = binding["g"]["value"]
|
||||
graph_uris.append(graph_uri)
|
||||
|
||||
logger.debug(f"Found {len(graph_uris)} named graphs: {graph_uris}")
|
||||
|
||||
# Step 2: Fetch each ontology from its corresponding named graph (in parallel)
|
||||
async def fetch_single_ontology(graph_uri: str) -> Ontology | None:
|
||||
"""Fetch a single ontology from a graph URI."""
|
||||
try:
|
||||
graph = RDFGraph()
|
||||
# URL encode the graph URI to handle special characters like #
|
||||
encoded_graph_uri = quote(str(graph_uri), safe="/:")
|
||||
export_url = f"{self._get_ontologies_dataset_url()}/get?graph={encoded_graph_uri}"
|
||||
export_resp = await client.get(
|
||||
export_url, headers={"Accept": "text/turtle"}
|
||||
)
|
||||
|
||||
if export_resp.status_code == 200:
|
||||
graph.parse(data=export_resp.text, format="turtle")
|
||||
|
||||
# Re-serialize deterministically to ensure consistent cache keys
|
||||
# This sorts both namespaces and triples alphabetically
|
||||
deterministic_turtle = deterministic_turtle_serialization(graph)
|
||||
|
||||
# Re-parse from deterministic serialization to ensure we have RDFGraph
|
||||
deterministic_graph = RDFGraph()
|
||||
deterministic_graph.parse(
|
||||
data=deterministic_turtle, format="turtle"
|
||||
)
|
||||
|
||||
# Copy namespace bindings from original graph
|
||||
for prefix, namespace in graph.namespaces():
|
||||
if prefix:
|
||||
deterministic_graph.bind(prefix, namespace)
|
||||
|
||||
graph = deterministic_graph
|
||||
|
||||
# Find the ontology IRI in the graph
|
||||
for onto_subj, _, obj in graph.triples(
|
||||
(None, RDF.type, OWL.Ontology)
|
||||
):
|
||||
onto_iri = str(onto_subj)
|
||||
# Extract base IRI if graph_uri is versioned
|
||||
# Handle both hash fragments (#19193944...) and semantic versions (#v1.2.3)
|
||||
if "#" in graph_uri:
|
||||
base_iri = graph_uri.split("#")[0]
|
||||
# Use base IRI from graph_uri (named graph identifier)
|
||||
# The graph content should have simplified IRI, but use graph_uri as source of truth
|
||||
onto_iri = base_iri
|
||||
|
||||
ontology = Ontology(
|
||||
graph=graph,
|
||||
iri=onto_iri,
|
||||
)
|
||||
# Load properties from graph (will strip any hash fragments if present)
|
||||
ontology.sync_properties_from_graph()
|
||||
logger.debug(
|
||||
f"Successfully loaded ontology: {onto_iri} version: {ontology.version}"
|
||||
)
|
||||
return ontology
|
||||
else:
|
||||
logger.warning(
|
||||
f"Failed to fetch graph {graph_uri}: {export_resp.status_code}"
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"Error fetching ontology from {graph_uri}: {e}")
|
||||
return None
|
||||
|
||||
# Fetch all ontologies in parallel
|
||||
all_ontologies_results = await asyncio.gather(
|
||||
*[fetch_single_ontology(uri) for uri in graph_uris], return_exceptions=True
|
||||
)
|
||||
|
||||
# Filter out None and exceptions
|
||||
all_ontologies = []
|
||||
for result in all_ontologies_results:
|
||||
if isinstance(result, Exception):
|
||||
logger.warning(f"Exception fetching ontology: {result}")
|
||||
elif result is not None:
|
||||
all_ontologies.append(result)
|
||||
|
||||
# Step 3: Deduplicate and keep latest terminal versions
|
||||
ontology_dict = defaultdict(list)
|
||||
|
||||
for onto in all_ontologies:
|
||||
ontology_dict[onto.iri].append(onto)
|
||||
|
||||
# Build set of all parent hashes to identify terminal ontologies
|
||||
# A terminal ontology is one that is not a parent for any other ontology
|
||||
all_parent_hashes = set()
|
||||
|
||||
for onto in all_ontologies:
|
||||
if onto.hash:
|
||||
# Collect all parent hashes
|
||||
for parent_hash in onto.parent_hashes:
|
||||
all_parent_hashes.add(parent_hash)
|
||||
|
||||
# For each unique IRI, select the latest terminal ontology
|
||||
ontologies = []
|
||||
|
||||
for iri, versions in ontology_dict.items():
|
||||
if len(versions) == 1:
|
||||
ontologies.append(versions[0])
|
||||
else:
|
||||
# Multiple versions - find terminal ontologies (not parents)
|
||||
terminal_versions = [
|
||||
v for v in versions if v.hash and v.hash not in all_parent_hashes
|
||||
]
|
||||
|
||||
if not terminal_versions:
|
||||
# No terminal ontologies found - all are parents
|
||||
# Fall back to non-terminal versions
|
||||
logger.warning(
|
||||
f"No terminal ontologies found for {iri}, "
|
||||
f"using all versions for selection"
|
||||
)
|
||||
terminal_versions = versions
|
||||
|
||||
# Select latest by created_at among terminal ontologies
|
||||
try:
|
||||
versions_with_created = [
|
||||
v for v in terminal_versions if v.created_at is not None
|
||||
]
|
||||
|
||||
if versions_with_created:
|
||||
# Sort by created_at (most recent first)
|
||||
versions_with_created.sort(
|
||||
key=lambda x: x.created_at, reverse=True
|
||||
)
|
||||
selected = versions_with_created[0]
|
||||
hash_str = (
|
||||
f"{selected.hash[:16]}..." if selected.hash else "no hash"
|
||||
)
|
||||
logger.debug(
|
||||
f"Selected terminal ontology for {iri} "
|
||||
f"by created_at: {selected.created_at} "
|
||||
f"(hash: {hash_str})"
|
||||
)
|
||||
ontologies.append(selected)
|
||||
else:
|
||||
# No created_at available - fall back to version-based sorting
|
||||
versions_with_ver = [v for v in terminal_versions if v.version]
|
||||
if versions_with_ver:
|
||||
versions_with_ver.sort(
|
||||
key=lambda x: str(x.version), reverse=False
|
||||
)
|
||||
selected = versions_with_ver[-1]
|
||||
logger.debug(
|
||||
f"Selected terminal ontology for {iri} "
|
||||
f"by version: {selected.version} "
|
||||
f"(no created_at available)"
|
||||
)
|
||||
ontologies.append(selected)
|
||||
else:
|
||||
# No version info either - use first terminal ontology
|
||||
selected = terminal_versions[0]
|
||||
logger.debug(
|
||||
f"Selected first terminal ontology for {iri} "
|
||||
f"(no created_at or version available)"
|
||||
)
|
||||
ontologies.append(selected)
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
f"Could not select terminal ontology for {iri}: {e}, "
|
||||
f"using first version"
|
||||
)
|
||||
ontologies.append(terminal_versions[0])
|
||||
|
||||
logger.info(
|
||||
f"Successfully loaded {len(ontologies)} unique ontologies from Fuseki "
|
||||
)
|
||||
return ontologies
|
||||
|
||||
def serialize_graph(self, graph: Graph, **kwargs) -> bool | None:
|
||||
"""Synchronous wrapper for serialize_graph.
|
||||
|
||||
For async usage, use aserialize_graph() instead.
|
||||
"""
|
||||
return asyncio.run(self._serialize_graph_with_cleanup(graph, **kwargs))
|
||||
|
||||
async def aserialize_graph(self, graph: Graph, **kwargs) -> bool | None:
|
||||
"""Async version of serialize_graph.
|
||||
|
||||
This is the preferred method when running in an async context.
|
||||
"""
|
||||
return await self._serialize_graph_async(graph, **kwargs)
|
||||
|
||||
async def _serialize_graph_with_cleanup(
|
||||
self, graph: Graph, **kwargs
|
||||
) -> bool | None:
|
||||
"""Wrapper that ensures proper cleanup when using asyncio.run().
|
||||
|
||||
This method creates a temporary client and ensures it's properly closed
|
||||
before returning, preventing "Event loop is closed" errors.
|
||||
"""
|
||||
async with httpx.AsyncClient(
|
||||
auth=self._prepare_auth(), timeout=30.0
|
||||
) as temp_client:
|
||||
# Temporarily replace the client
|
||||
original_client = self._client
|
||||
self._client = temp_client
|
||||
try:
|
||||
return await self._serialize_graph_async(graph, **kwargs)
|
||||
finally:
|
||||
# Restore original client
|
||||
self._client = original_client
|
||||
|
||||
async def _serialize_graph_async(self, graph: Graph, **kwargs) -> bool | None:
|
||||
"""Store an RDF graph as a named graph in a specific Fuseki dataset.
|
||||
|
||||
This is a private helper method that handles the common logic for storing
|
||||
graphs in Fuseki datasets.
|
||||
|
||||
Args:
|
||||
graph: The RDF graph to store.
|
||||
**kwargs: Additional parameters including graph_uri, dataset_url, default_graph_uri, log_prefix.
|
||||
|
||||
Returns:
|
||||
bool: True if the graph was successfully stored, False otherwise.
|
||||
"""
|
||||
client = await self._get_client()
|
||||
graph_uri = kwargs.get("graph_uri")
|
||||
dataset_url = kwargs.get("dataset_url")
|
||||
default_graph_uri = kwargs.get("default_graph_uri")
|
||||
log_prefix = kwargs.get("log_prefix")
|
||||
|
||||
turtle_data = graph.serialize(format="turtle")
|
||||
if graph_uri is None:
|
||||
graph_uri = default_graph_uri
|
||||
|
||||
# URL encode the graph URI to handle special characters like #
|
||||
encoded_graph_uri = quote(str(graph_uri), safe="/:")
|
||||
url = f"{dataset_url}/data?graph={encoded_graph_uri}"
|
||||
headers = {"Content-Type": "text/turtle;charset=utf-8"}
|
||||
response = await client.put(url, headers=headers, content=turtle_data)
|
||||
if response.status_code in (200, 201, 204):
|
||||
logger.info(
|
||||
f"{log_prefix} graph {graph_uri} uploaded to Fuseki as named graph."
|
||||
)
|
||||
return True
|
||||
else:
|
||||
logger.error(
|
||||
f"Failed to upload {log_prefix.lower() if log_prefix else 'unknown'} graph {graph_uri}. Status code: {response.status_code}"
|
||||
)
|
||||
logger.error(f"Response: {response.text}")
|
||||
return False
|
||||
|
||||
def serialize(self, o: Ontology | RDFGraph, **kwargs) -> bool | None:
|
||||
"""Synchronous wrapper for serialize.
|
||||
|
||||
For async usage, use aserialize() instead.
|
||||
"""
|
||||
return asyncio.run(self._serialize_with_cleanup(o, **kwargs))
|
||||
|
||||
async def aserialize(self, o: Ontology | RDFGraph, **kwargs) -> bool | None:
|
||||
"""Async version of serialize.
|
||||
|
||||
This is the preferred method when running in an async context.
|
||||
"""
|
||||
return await self._serialize_async(o, **kwargs)
|
||||
|
||||
async def _serialize_with_cleanup(
|
||||
self, o: Ontology | RDFGraph, **kwargs
|
||||
) -> bool | None:
|
||||
"""Wrapper that ensures proper cleanup when using asyncio.run().
|
||||
|
||||
This method creates a temporary client and ensures it's properly closed
|
||||
before returning, preventing "Event loop is closed" errors.
|
||||
"""
|
||||
async with httpx.AsyncClient(
|
||||
auth=self._prepare_auth(), timeout=30.0
|
||||
) as temp_client:
|
||||
# Temporarily replace the client
|
||||
original_client = self._client
|
||||
self._client = temp_client
|
||||
try:
|
||||
return await self._serialize_async(o, **kwargs)
|
||||
finally:
|
||||
# Restore original client
|
||||
self._client = original_client
|
||||
|
||||
async def _serialize_async(self, o: Ontology | RDFGraph, **kwargs) -> bool | None:
|
||||
"""Store an RDF graph as a named graph in Fuseki.
|
||||
|
||||
This method stores the given RDF graph as a named graph in Fuseki.
|
||||
The graph name is taken from the graph_uri parameter or defaults to
|
||||
"urn:data:default".
|
||||
|
||||
Args:
|
||||
o: RDF graph or Ontology object.
|
||||
**kwargs: Additional parameters including graph_uri.
|
||||
|
||||
Returns:
|
||||
bool: True if the graph was successfully stored, False otherwise.
|
||||
|
||||
Example:
|
||||
>>> graph = RDFGraph()
|
||||
>>> success = await manager.serialize(graph)
|
||||
|
||||
>>> success = await manager.serialize(graph, graph_uri="http://example.org/chunk1")
|
||||
"""
|
||||
graph_uri = kwargs.get("graph_uri")
|
||||
|
||||
if isinstance(o, Ontology):
|
||||
graph = o.graph
|
||||
# Use versioned IRI for storage to enable multiple versions to coexist
|
||||
graph_uri = o.versioned_iri
|
||||
default_graph_uri = "urn:ontology:default"
|
||||
log_prefix = "Ontology"
|
||||
# Use ontologies dataset for ontology storage
|
||||
dataset_url = self._get_ontologies_dataset_url()
|
||||
elif isinstance(o, RDFGraph):
|
||||
graph = o
|
||||
default_graph_uri = "urn:data:default"
|
||||
log_prefix = "Graph"
|
||||
# Use regular dataset for facts storage
|
||||
dataset_url = self._get_dataset_url()
|
||||
else:
|
||||
raise TypeError(f"unsupported obj of type {type(o)} received")
|
||||
|
||||
return await self._serialize_graph_async(
|
||||
graph=graph,
|
||||
graph_uri=graph_uri,
|
||||
dataset_url=dataset_url,
|
||||
default_graph_uri=default_graph_uri,
|
||||
log_prefix=log_prefix,
|
||||
)
|
||||
502
참고/ontocast-main/ontocast/tool/triple_manager/mock.py
Normal file
502
참고/ontocast-main/ontocast/tool/triple_manager/mock.py
Normal file
@@ -0,0 +1,502 @@
|
||||
"""Mock triple store implementations for testing.
|
||||
|
||||
This module provides mock implementations of triple store managers that simulate
|
||||
the behavior of real triple stores (Fuseki, Neo4j) without requiring external
|
||||
services. These mocks are useful for testing and development.
|
||||
|
||||
The mocks maintain in-memory storage and provide the same interface as the
|
||||
real implementations, allowing tests to run without external dependencies.
|
||||
"""
|
||||
|
||||
import logging
|
||||
from typing import Any, Dict, List
|
||||
|
||||
from pydantic import Field
|
||||
from rdflib import Graph, URIRef
|
||||
from rdflib.namespace import OWL, RDF
|
||||
|
||||
from ontocast.onto.ontology import Ontology
|
||||
from ontocast.onto.rdfgraph import RDFGraph
|
||||
from ontocast.onto.util import derive_ontology_id
|
||||
from ontocast.tool.triple_manager.core import (
|
||||
TripleStoreManager,
|
||||
TripleStoreManagerWithAuth,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class MockTripleStoreManager(TripleStoreManager):
|
||||
"""Mock triple store manager for testing.
|
||||
|
||||
This class provides an in-memory implementation of triple store operations
|
||||
that simulates the behavior of real triple stores without requiring external
|
||||
services. It stores ontologies and graphs in memory and provides the same
|
||||
interface as concrete implementations.
|
||||
|
||||
Attributes:
|
||||
ontologies: In-memory storage for ontologies.
|
||||
graphs: In-memory storage for RDF graphs.
|
||||
"""
|
||||
|
||||
model_config = {"arbitrary_types_allowed": True}
|
||||
|
||||
ontologies: List[Ontology] = Field(
|
||||
default_factory=list, description="In-memory storage for ontologies"
|
||||
)
|
||||
graphs: Dict[str, Graph] = Field(
|
||||
default_factory=dict, description="In-memory storage for RDF graphs"
|
||||
)
|
||||
|
||||
def __init__(self, **kwargs):
|
||||
"""Initialize the mock triple store manager.
|
||||
|
||||
Args:
|
||||
**kwargs: Additional keyword arguments passed to the parent class.
|
||||
"""
|
||||
super().__init__(**kwargs)
|
||||
|
||||
def fetch_ontologies(self) -> List[Ontology]:
|
||||
"""Fetch all available ontologies from the mock store.
|
||||
|
||||
Returns:
|
||||
List[Ontology]: List of available ontologies with their graphs.
|
||||
"""
|
||||
return self.ontologies.copy()
|
||||
|
||||
def serialize_graph(self, graph: Graph, **kwargs) -> bool | None:
|
||||
"""Store an RDF graph in the mock store.
|
||||
|
||||
Args:
|
||||
graph: The RDF graph to store.
|
||||
**kwargs: Optional keyword arguments including graph_uri.
|
||||
|
||||
Returns:
|
||||
bool: True if the graph was stored successfully.
|
||||
"""
|
||||
graph_uri = kwargs.get("graph_uri")
|
||||
# Create a new Graph and copy all triples
|
||||
new_graph = Graph()
|
||||
for triple in graph:
|
||||
new_graph.add(triple)
|
||||
|
||||
if graph_uri:
|
||||
self.graphs[graph_uri] = new_graph
|
||||
else:
|
||||
# Generate a default URI based on graph content
|
||||
graph_uri = f"mock://graph/{len(self.graphs)}"
|
||||
self.graphs[graph_uri] = new_graph
|
||||
|
||||
# Try to extract ontology information from the graph
|
||||
ontology_id = self._extract_ontology_id(graph)
|
||||
if ontology_id:
|
||||
ontology = Ontology(
|
||||
ontology_id=ontology_id,
|
||||
title=f"Mock Ontology {ontology_id}",
|
||||
description="Mock ontology for testing",
|
||||
version="1.0.0",
|
||||
iri=graph_uri,
|
||||
graph=self._create_rdf_graph_from_graph(graph),
|
||||
)
|
||||
# Update existing ontology or add new one
|
||||
existing = next(
|
||||
(o for o in self.ontologies if o.ontology_id == ontology_id), None
|
||||
)
|
||||
if existing:
|
||||
existing.graph = self._create_rdf_graph_from_graph(graph)
|
||||
existing.iri = graph_uri
|
||||
else:
|
||||
self.ontologies.append(ontology)
|
||||
|
||||
return True
|
||||
|
||||
def serialize(self, o: Ontology | RDFGraph, **kwargs) -> bool | None:
|
||||
"""Store an Ontology or RDFGraph in the mock store.
|
||||
|
||||
Args:
|
||||
o: Ontology or RDFGraph object to store.
|
||||
**kwargs: Additional keyword arguments.
|
||||
|
||||
Returns:
|
||||
bool: True if the object was stored successfully.
|
||||
"""
|
||||
if isinstance(o, Ontology):
|
||||
graph = o.graph
|
||||
graph_uri = o.iri
|
||||
elif isinstance(o, RDFGraph):
|
||||
graph = o
|
||||
graph_uri = kwargs.get("graph_uri")
|
||||
else:
|
||||
raise TypeError(f"unsupported obj of type {type(o)} received")
|
||||
|
||||
return self.serialize_graph(graph, graph_uri=graph_uri)
|
||||
|
||||
def _extract_ontology_id(self, graph: Graph) -> str | None:
|
||||
"""Extract ontology ID from graph content.
|
||||
|
||||
Args:
|
||||
graph: The RDF graph to analyze.
|
||||
|
||||
Returns:
|
||||
str | None: The extracted ontology ID, or None if not found.
|
||||
"""
|
||||
# Look for owl:Ontology declarations
|
||||
for s, p, o in graph.triples((None, RDF.type, OWL.Ontology)):
|
||||
if isinstance(s, URIRef):
|
||||
return derive_ontology_id(str(s))
|
||||
return None
|
||||
|
||||
def clear(self):
|
||||
"""Clear all stored data."""
|
||||
self.ontologies.clear()
|
||||
self.graphs.clear()
|
||||
|
||||
async def clean(self, dataset: str | None = None) -> None:
|
||||
"""Clean/flush data from the mock triple store.
|
||||
|
||||
Args:
|
||||
dataset: Optional dataset name (ignored for mock, kept for interface compatibility).
|
||||
"""
|
||||
self.clear()
|
||||
|
||||
def _create_rdf_graph_from_graph(self, graph: Graph) -> RDFGraph:
|
||||
"""Create an RDFGraph from a regular Graph by copying all triples.
|
||||
|
||||
Args:
|
||||
graph: The source graph to copy from.
|
||||
|
||||
Returns:
|
||||
RDFGraph: A new RDFGraph with all triples copied.
|
||||
"""
|
||||
rdf_graph = RDFGraph()
|
||||
for triple in graph:
|
||||
rdf_graph.add(triple)
|
||||
return rdf_graph
|
||||
|
||||
|
||||
class MockFusekiTripleStoreManager(TripleStoreManagerWithAuth):
|
||||
"""Mock Fuseki triple store manager for testing.
|
||||
|
||||
This class simulates the behavior of FusekiTripleStoreManager without
|
||||
requiring an actual Fuseki server. It maintains in-memory storage and
|
||||
provides the same interface as the real implementation.
|
||||
|
||||
Attributes:
|
||||
dataset: The mock dataset name.
|
||||
ontologies_dataset: The mock ontologies dataset name.
|
||||
ontologies: In-memory storage for ontologies.
|
||||
graphs: In-memory storage for RDF graphs.
|
||||
"""
|
||||
|
||||
model_config = {"arbitrary_types_allowed": True}
|
||||
|
||||
dataset: str | None = None
|
||||
ontologies_dataset: str = "ontologies"
|
||||
ontologies: List[Ontology] = Field(
|
||||
default_factory=list, description="In-memory storage for ontologies"
|
||||
)
|
||||
graphs: Dict[str, Graph] = Field(
|
||||
default_factory=dict, description="In-memory storage for RDF graphs"
|
||||
)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
uri=None,
|
||||
auth=None,
|
||||
dataset=None,
|
||||
ontologies_dataset=None,
|
||||
clean=False,
|
||||
**kwargs,
|
||||
):
|
||||
"""Initialize the mock Fuseki triple store manager.
|
||||
|
||||
Args:
|
||||
uri: Mock URI (ignored but kept for interface compatibility).
|
||||
auth: Mock authentication (ignored but kept for interface compatibility).
|
||||
dataset: Mock dataset name.
|
||||
ontologies_dataset: Mock ontologies dataset name.
|
||||
clean: Whether to clean the store on initialization.
|
||||
**kwargs: Additional keyword arguments.
|
||||
"""
|
||||
super().__init__(uri=uri, auth=auth, **kwargs)
|
||||
self.dataset = dataset or "test"
|
||||
self.ontologies_dataset = ontologies_dataset or "ontologies"
|
||||
|
||||
if clean:
|
||||
self.clear()
|
||||
|
||||
def fetch_ontologies(self) -> List[Ontology]:
|
||||
"""Fetch all available ontologies from the mock store.
|
||||
|
||||
Returns:
|
||||
List[Ontology]: List of available ontologies with their graphs.
|
||||
"""
|
||||
return self.ontologies.copy()
|
||||
|
||||
def serialize_graph(self, graph: Graph, **kwargs) -> bool | None:
|
||||
"""Store an RDF graph in the mock store.
|
||||
|
||||
Args:
|
||||
graph: The RDF graph to store.
|
||||
**kwargs: Optional keyword arguments including graph_uri.
|
||||
|
||||
Returns:
|
||||
bool: True if the graph was stored successfully.
|
||||
"""
|
||||
graph_uri = kwargs.get("graph_uri")
|
||||
# Create a new Graph and copy all triples
|
||||
new_graph = Graph()
|
||||
for triple in graph:
|
||||
new_graph.add(triple)
|
||||
|
||||
if graph_uri:
|
||||
self.graphs[graph_uri] = new_graph
|
||||
else:
|
||||
# Generate a default URI based on graph content
|
||||
graph_uri = f"mock://{self.dataset}/graph/{len(self.graphs)}"
|
||||
self.graphs[graph_uri] = new_graph
|
||||
|
||||
# Try to extract ontology information from the graph
|
||||
ontology_id = self._extract_ontology_id(graph)
|
||||
if ontology_id:
|
||||
ontology = Ontology(
|
||||
ontology_id=ontology_id,
|
||||
title=f"Mock Ontology {ontology_id}",
|
||||
description="Mock ontology for testing",
|
||||
version="1.0.0",
|
||||
iri=graph_uri,
|
||||
graph=self._create_rdf_graph_from_graph(graph),
|
||||
)
|
||||
# Update existing ontology or add new one
|
||||
existing = next(
|
||||
(o for o in self.ontologies if o.ontology_id == ontology_id), None
|
||||
)
|
||||
if existing:
|
||||
existing.graph = self._create_rdf_graph_from_graph(graph)
|
||||
existing.iri = graph_uri
|
||||
else:
|
||||
self.ontologies.append(ontology)
|
||||
|
||||
return True
|
||||
|
||||
def serialize(self, o: Ontology | RDFGraph, **kwargs) -> bool | None:
|
||||
"""Store an Ontology or RDFGraph in the mock store.
|
||||
|
||||
Args:
|
||||
o: Ontology or RDFGraph object to store.
|
||||
**kwargs: Additional keyword arguments.
|
||||
|
||||
Returns:
|
||||
bool: True if the object was stored successfully.
|
||||
"""
|
||||
if isinstance(o, Ontology):
|
||||
graph = o.graph
|
||||
graph_uri = o.iri
|
||||
elif isinstance(o, RDFGraph):
|
||||
graph = o
|
||||
graph_uri = kwargs.get("graph_uri")
|
||||
else:
|
||||
raise TypeError(f"unsupported obj of type {type(o)} received")
|
||||
|
||||
return self.serialize_graph(graph, graph_uri=graph_uri)
|
||||
|
||||
def _extract_ontology_id(self, graph: Graph) -> str | None:
|
||||
"""Extract ontology ID from graph content.
|
||||
|
||||
Args:
|
||||
graph: The RDF graph to analyze.
|
||||
|
||||
Returns:
|
||||
str | None: The extracted ontology ID, or None if not found.
|
||||
"""
|
||||
# Look for owl:Ontology declarations
|
||||
for s, p, o in graph.triples((None, RDF.type, OWL.Ontology)):
|
||||
if isinstance(s, URIRef):
|
||||
return derive_ontology_id(str(s))
|
||||
return None
|
||||
|
||||
def clear(self):
|
||||
"""Clear all stored data."""
|
||||
self.ontologies.clear()
|
||||
self.graphs.clear()
|
||||
|
||||
async def clean(self, dataset: str | None = None) -> None:
|
||||
"""Clean/flush data from the mock Fuseki triple store.
|
||||
|
||||
Args:
|
||||
dataset: Optional dataset name (ignored for mock, kept for interface compatibility).
|
||||
"""
|
||||
self.clear()
|
||||
|
||||
def _create_rdf_graph_from_graph(self, graph: Graph) -> RDFGraph:
|
||||
"""Create an RDFGraph from a regular Graph by copying all triples.
|
||||
|
||||
Args:
|
||||
graph: The source graph to copy from.
|
||||
|
||||
Returns:
|
||||
RDFGraph: A new RDFGraph with all triples copied.
|
||||
"""
|
||||
rdf_graph = RDFGraph()
|
||||
for triple in graph:
|
||||
rdf_graph.add(triple)
|
||||
return rdf_graph
|
||||
|
||||
|
||||
class MockNeo4jTripleStoreManager(TripleStoreManagerWithAuth):
|
||||
"""Mock Neo4j triple store manager for testing.
|
||||
|
||||
This class simulates the behavior of Neo4jTripleStoreManager without
|
||||
requiring an actual Neo4j server. It maintains in-memory storage and
|
||||
provides the same interface as the real implementation.
|
||||
|
||||
Attributes:
|
||||
ontologies: In-memory storage for ontologies.
|
||||
graphs: In-memory storage for RDF graphs.
|
||||
"""
|
||||
|
||||
model_config = {"arbitrary_types_allowed": True}
|
||||
|
||||
ontologies: List[Ontology] = Field(
|
||||
default_factory=list, description="In-memory storage for ontologies"
|
||||
)
|
||||
graphs: Dict[str, Graph] = Field(
|
||||
default_factory=dict, description="In-memory storage for RDF graphs"
|
||||
)
|
||||
|
||||
def __init__(self, uri=None, auth=None, clean=False, **kwargs):
|
||||
"""Initialize the mock Neo4j triple store manager.
|
||||
|
||||
Args:
|
||||
uri: Mock URI (ignored but kept for interface compatibility).
|
||||
auth: Mock authentication (ignored but kept for interface compatibility).
|
||||
clean: Whether to clean the store on initialization.
|
||||
**kwargs: Additional keyword arguments.
|
||||
"""
|
||||
super().__init__(uri=uri, auth=auth, **kwargs)
|
||||
|
||||
if clean:
|
||||
self.clear()
|
||||
|
||||
def fetch_ontologies(self) -> List[Ontology]:
|
||||
"""Fetch all available ontologies from the mock store.
|
||||
|
||||
Returns:
|
||||
List[Ontology]: List of available ontologies with their graphs.
|
||||
"""
|
||||
return self.ontologies.copy()
|
||||
|
||||
def serialize_graph(self, graph: Graph, **kwargs) -> Dict[str, Any] | None: # type: ignore[override]
|
||||
"""Store an RDF graph in the mock store.
|
||||
|
||||
Args:
|
||||
graph: The RDF graph to store.
|
||||
**kwargs: Optional keyword arguments including graph_uri.
|
||||
|
||||
Returns:
|
||||
Dict[str, Any]: Mock summary of the operation.
|
||||
"""
|
||||
graph_uri = kwargs.get("graph_uri")
|
||||
# Create a new Graph and copy all triples
|
||||
new_graph = Graph()
|
||||
for triple in graph:
|
||||
new_graph.add(triple)
|
||||
|
||||
if graph_uri:
|
||||
self.graphs[graph_uri] = new_graph
|
||||
else:
|
||||
# Generate a default URI based on graph content
|
||||
graph_uri = f"mock://neo4j/graph/{len(self.graphs)}"
|
||||
self.graphs[graph_uri] = new_graph
|
||||
|
||||
# Try to extract ontology information from the graph
|
||||
ontology_id = self._extract_ontology_id(graph)
|
||||
if ontology_id:
|
||||
ontology = Ontology(
|
||||
ontology_id=ontology_id,
|
||||
title=f"Mock Ontology {ontology_id}",
|
||||
description="Mock ontology for testing",
|
||||
version="1.0.0",
|
||||
iri=graph_uri,
|
||||
graph=self._create_rdf_graph_from_graph(graph),
|
||||
)
|
||||
# Update existing ontology or add new one
|
||||
existing = next(
|
||||
(o for o in self.ontologies if o.ontology_id == ontology_id), None
|
||||
)
|
||||
if existing:
|
||||
existing.graph = self._create_rdf_graph_from_graph(graph)
|
||||
existing.iri = graph_uri
|
||||
else:
|
||||
self.ontologies.append(ontology)
|
||||
|
||||
# Return mock summary similar to Neo4j
|
||||
return {
|
||||
"nodes_created": len(graph),
|
||||
"relationships_created": 0,
|
||||
"properties_set": len(graph),
|
||||
"labels_added": 1,
|
||||
}
|
||||
|
||||
def serialize(self, o: Ontology | RDFGraph, **kwargs) -> Dict[str, Any] | None: # type: ignore[override]
|
||||
"""Store an Ontology or RDFGraph in the mock store.
|
||||
|
||||
Args:
|
||||
o: Ontology or RDFGraph object to store.
|
||||
**kwargs: Additional keyword arguments.
|
||||
|
||||
Returns:
|
||||
Dict[str, Any]: Mock summary of the operation.
|
||||
"""
|
||||
if isinstance(o, Ontology):
|
||||
graph = o.graph
|
||||
graph_uri = o.iri
|
||||
elif isinstance(o, RDFGraph):
|
||||
graph = o
|
||||
graph_uri = kwargs.get("graph_uri")
|
||||
else:
|
||||
raise TypeError(f"unsupported obj of type {type(o)} received")
|
||||
|
||||
return self.serialize_graph(graph, graph_uri=graph_uri)
|
||||
|
||||
def _extract_ontology_id(self, graph: Graph) -> str | None:
|
||||
"""Extract ontology ID from graph content.
|
||||
|
||||
Args:
|
||||
graph: The RDF graph to analyze.
|
||||
|
||||
Returns:
|
||||
str | None: The extracted ontology ID, or None if not found.
|
||||
"""
|
||||
# Look for owl:Ontology declarations
|
||||
for s, p, o in graph.triples((None, RDF.type, OWL.Ontology)):
|
||||
if isinstance(s, URIRef):
|
||||
return derive_ontology_id(str(s))
|
||||
return None
|
||||
|
||||
def clear(self):
|
||||
"""Clear all stored data."""
|
||||
self.ontologies.clear()
|
||||
self.graphs.clear()
|
||||
|
||||
async def clean(self, dataset: str | None = None) -> None:
|
||||
"""Clean/flush data from the mock Neo4j triple store.
|
||||
|
||||
Args:
|
||||
dataset: Optional dataset name (ignored for Neo4j mock, kept for interface compatibility).
|
||||
"""
|
||||
self.clear()
|
||||
|
||||
def _create_rdf_graph_from_graph(self, graph: Graph) -> RDFGraph:
|
||||
"""Create an RDFGraph from a regular Graph by copying all triples.
|
||||
|
||||
Args:
|
||||
graph: The source graph to copy from.
|
||||
|
||||
Returns:
|
||||
RDFGraph: A new RDFGraph with all triples copied.
|
||||
"""
|
||||
rdf_graph = RDFGraph()
|
||||
for triple in graph:
|
||||
rdf_graph.add(triple)
|
||||
return rdf_graph
|
||||
476
참고/ontocast-main/ontocast/tool/triple_manager/neo4j.py
Normal file
476
참고/ontocast-main/ontocast/tool/triple_manager/neo4j.py
Normal file
@@ -0,0 +1,476 @@
|
||||
"""Neo4j triple store management for OntoCast.
|
||||
|
||||
This module provides a concrete implementation of triple store management
|
||||
using Neo4j with the n10s (neosemantics) plugin. It handles RDF data
|
||||
faithfully by using both n10s property graph representation and raw RDF
|
||||
triple storage for accurate reconstruction.
|
||||
"""
|
||||
|
||||
import logging
|
||||
from typing import Any
|
||||
|
||||
from neo4j import GraphDatabase
|
||||
from rdflib import Graph
|
||||
from rdflib.namespace import OWL, RDF
|
||||
|
||||
from ontocast.onto.ontology import Ontology
|
||||
from ontocast.onto.rdfgraph import RDFGraph
|
||||
from ontocast.onto.util import derive_ontology_id
|
||||
from ontocast.tool.triple_manager.core import TripleStoreManagerWithAuth
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class Neo4jTripleStoreManager(TripleStoreManagerWithAuth):
|
||||
"""Neo4j-based triple store manager using n10s (neosemantics) plugin.
|
||||
|
||||
This implementation handles RDF data more faithfully by using both the n10s
|
||||
property graph representation and raw RDF triple storage for accurate reconstruction.
|
||||
It provides comprehensive ontology management with namespace-based organization.
|
||||
|
||||
The manager uses Neo4j's n10s plugin for RDF operations, including:
|
||||
- RDF import and export via n10s
|
||||
- Ontology metadata storage and retrieval
|
||||
- Namespace-based ontology organization
|
||||
- Faithful RDF graph reconstruction
|
||||
|
||||
Attributes:
|
||||
_driver: Private Neo4j driver instance.
|
||||
"""
|
||||
|
||||
_driver: Any = None # private attribute, not a pydantic field
|
||||
|
||||
def __init__(self, uri=None, auth=None, **kwargs):
|
||||
"""Initialize the Neo4j triple store manager.
|
||||
|
||||
This method sets up the connection to Neo4j, initializes the n10s
|
||||
plugin configuration, and creates necessary constraints and indexes.
|
||||
The database is NOT cleaned on initialization.
|
||||
|
||||
Args:
|
||||
uri: Neo4j connection URI (e.g., "bolt://localhost:7687").
|
||||
auth: Authentication tuple (username, password) or string in "user/password" format.
|
||||
**kwargs: Additional keyword arguments passed to the parent class.
|
||||
|
||||
Raises:
|
||||
ImportError: If the neo4j Python driver is not installed.
|
||||
|
||||
Example:
|
||||
>>> manager = Neo4jTripleStoreManager(
|
||||
... uri="bolt://localhost:7687",
|
||||
... auth="neo4j/password"
|
||||
... )
|
||||
>>> # To clean the database, use the clean() method explicitly:
|
||||
>>> await manager.clean()
|
||||
"""
|
||||
super().__init__(
|
||||
uri=uri, auth=auth, env_uri="NEO4J_URI", env_auth="NEO4J_AUTH", **kwargs
|
||||
)
|
||||
if GraphDatabase is None:
|
||||
raise ImportError("neo4j Python driver is not installed.")
|
||||
if self.uri is None:
|
||||
raise ValueError("Neo4j URI is required but not provided.")
|
||||
self._driver = GraphDatabase.driver(self.uri, auth=self.auth)
|
||||
|
||||
# Type assertion: we know _driver is not None after initialization
|
||||
assert self._driver is not None
|
||||
|
||||
with self._driver.session() as session:
|
||||
# Initialize n10s configuration
|
||||
self._init_n10s_config(session)
|
||||
|
||||
# Create constraints and indexes
|
||||
self._create_constraints_and_indexes(session)
|
||||
|
||||
async def clean(self, dataset: str | None = None) -> None:
|
||||
"""Clean/flush all data from the Neo4j database.
|
||||
|
||||
This method deletes all nodes and relationships from the Neo4j database,
|
||||
effectively clearing all stored data.
|
||||
|
||||
Args:
|
||||
dataset: Optional dataset parameter (ignored for Neo4j, which doesn't
|
||||
support datasets). Included for interface compatibility.
|
||||
|
||||
Warning: This operation is irreversible and will delete all data.
|
||||
|
||||
Raises:
|
||||
Exception: If the cleanup operation fails.
|
||||
"""
|
||||
if dataset is not None:
|
||||
logger.warning(
|
||||
f"Dataset parameter '{dataset}' ignored for Neo4j (datasets not supported)"
|
||||
)
|
||||
|
||||
if self._driver is None:
|
||||
raise ValueError("Neo4j driver is not initialized")
|
||||
|
||||
with self._driver.session() as session:
|
||||
try:
|
||||
session.run("MATCH (n) DETACH DELETE n")
|
||||
logger.info("Neo4j database cleaned (all nodes deleted)")
|
||||
except Exception as e:
|
||||
logger.error(f"Neo4j cleanup failed: {e}")
|
||||
raise
|
||||
|
||||
def _init_n10s_config(self, session):
|
||||
"""Initialize n10s configuration with better RDF handling.
|
||||
|
||||
This method configures the n10s plugin for optimal RDF handling.
|
||||
It sets up the configuration to preserve vocabulary URIs, handle
|
||||
multivalued properties, and maintain RDF types as nodes.
|
||||
|
||||
Args:
|
||||
session: Neo4j session for executing configuration commands.
|
||||
"""
|
||||
try:
|
||||
# Check if already configured
|
||||
result = session.run("CALL n10s.graphconfig.show()")
|
||||
if result.single():
|
||||
logger.debug("n10s already configured")
|
||||
except:
|
||||
pass
|
||||
|
||||
try:
|
||||
session.run("""
|
||||
CALL n10s.graphconfig.init({
|
||||
handleVocabUris: "KEEP",
|
||||
handleMultival: "OVERWRITE",
|
||||
typesToLabels: false,
|
||||
keepLangTag: false,
|
||||
keepCustomDataTypes: true,
|
||||
handleRDFTypes: "NODES"
|
||||
})
|
||||
""")
|
||||
logger.debug("n10s configuration initialized")
|
||||
except Exception as e:
|
||||
logger.warning(f"n10s configuration failed: {e}")
|
||||
|
||||
def _create_constraints_and_indexes(self, session):
|
||||
"""Create necessary constraints and indexes for optimal performance.
|
||||
|
||||
This method creates Neo4j constraints and indexes that are needed
|
||||
for efficient ontology operations and data integrity.
|
||||
|
||||
Args:
|
||||
session: Neo4j session for executing constraint/index creation commands.
|
||||
"""
|
||||
constraints = [
|
||||
"CREATE CONSTRAINT n10s_unique_uri IF NOT EXISTS FOR (r:Resource) REQUIRE r.uri IS UNIQUE",
|
||||
"CREATE CONSTRAINT ontology_iri_unique IF NOT EXISTS FOR (o:Ontology) REQUIRE o.uri IS UNIQUE",
|
||||
"CREATE INDEX namespace_prefix IF NOT EXISTS FOR (ns:Namespace) ON (ns.prefix)",
|
||||
]
|
||||
|
||||
for constraint in constraints:
|
||||
try:
|
||||
session.run(constraint)
|
||||
logger.debug(f"Created constraint/index: {constraint.split()[-1]}")
|
||||
except Exception as e:
|
||||
logger.debug(f"Constraint/index creation (might already exist): {e}")
|
||||
|
||||
def _extract_namespace_prefix(self, uri: str) -> tuple[str, str]:
|
||||
"""Extract namespace and local name from URI.
|
||||
|
||||
This method parses a URI to extract the namespace and local name
|
||||
using common separators (#, /, :).
|
||||
|
||||
Args:
|
||||
uri: The URI to parse.
|
||||
|
||||
Returns:
|
||||
tuple[str, str]: A tuple of (namespace, local_name).
|
||||
|
||||
Example:
|
||||
>>> manager._extract_namespace_prefix("http://example.org/onto#Class")
|
||||
("http://example.org/onto#", "Class")
|
||||
"""
|
||||
common_separators = ["#", "/", ":"]
|
||||
for sep in common_separators:
|
||||
if sep in uri:
|
||||
parts = uri.rsplit(sep, 1)
|
||||
if len(parts) == 2:
|
||||
return parts[0] + sep, parts[1]
|
||||
return uri, ""
|
||||
|
||||
def _get_ontology_namespaces(self, session) -> dict:
|
||||
"""Get all known ontology namespaces from the database.
|
||||
|
||||
This method queries the Neo4j database to retrieve all known
|
||||
namespace prefixes and their corresponding URIs.
|
||||
|
||||
Args:
|
||||
session: Neo4j session for executing queries.
|
||||
|
||||
Returns:
|
||||
dict: Dictionary mapping namespace prefixes to URIs.
|
||||
"""
|
||||
result = session.run("""
|
||||
MATCH (ns:Namespace)
|
||||
RETURN ns.prefix as prefix, ns.uri as uri
|
||||
UNION
|
||||
MATCH (o:Ontology)
|
||||
RETURN null as prefix, o.uri as uri
|
||||
""")
|
||||
|
||||
namespaces = {}
|
||||
for record in result:
|
||||
uri = record.get("uri")
|
||||
prefix = record.get("prefix")
|
||||
if uri:
|
||||
if prefix:
|
||||
namespaces[prefix] = uri
|
||||
else:
|
||||
# Extract potential namespace from ontology URI
|
||||
ns, _ = self._extract_namespace_prefix(uri)
|
||||
if ns != uri: # Only if we actually found a namespace
|
||||
namespaces[ns] = ns
|
||||
|
||||
return namespaces
|
||||
|
||||
def fetch_ontologies(self) -> list[Ontology]:
|
||||
"""Fetch ontologies from Neo4j with faithful RDF reconstruction.
|
||||
|
||||
This method retrieves all ontologies from Neo4j and reconstructs
|
||||
their RDF graphs faithfully. It uses a multi-step process:
|
||||
|
||||
1. Identifies distinct ontologies by their namespace URIs
|
||||
2. Fetches all entities belonging to each ontology
|
||||
3. Reconstructs the RDF graph faithfully using stored triples when available
|
||||
4. Falls back to n10s property graph conversion when needed
|
||||
|
||||
Returns:
|
||||
list[Ontology]: List of all ontologies found in the database.
|
||||
|
||||
Example:
|
||||
>>> ontologies = manager.fetch_ontologies()
|
||||
>>> for onto in ontologies:
|
||||
... print(f"Found ontology: {onto.iri}")
|
||||
"""
|
||||
ontologies = []
|
||||
|
||||
# Type assertion: we know _driver is not None after initialization
|
||||
assert self._driver is not None
|
||||
with self._driver.session() as session:
|
||||
try:
|
||||
# First, try to get explicitly stored ontology metadata
|
||||
ontology_iris = self._fetch_ontology_iris(session)
|
||||
|
||||
if ontology_iris:
|
||||
for ont_iri in ontology_iris:
|
||||
ontology = self._reconstruct_ontology_from_metadata(
|
||||
session, ont_iri
|
||||
)
|
||||
if ontology:
|
||||
ontologies.append(ontology)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error in fetch_ontologies: {e}")
|
||||
|
||||
logger.info(f"Successfully loaded {len(ontologies)} ontologies")
|
||||
return ontologies
|
||||
|
||||
def _fetch_ontology_iris(self, session) -> list[str]:
|
||||
"""Fetch explicit ontology metadata from Neo4j.
|
||||
|
||||
This method queries Neo4j to find all entities that are explicitly
|
||||
typed as owl:Ontology.
|
||||
|
||||
Args:
|
||||
session: Neo4j session for executing queries.
|
||||
|
||||
Returns:
|
||||
list[str]: List of ontology IRIs found in the database.
|
||||
"""
|
||||
result = session.run(f"""
|
||||
MATCH (o)-[:`{str(RDF.type)}`]->(t:Resource {{ uri: "{str(OWL.Ontology)}" }})
|
||||
WHERE o.uri IS NOT NULL
|
||||
RETURN
|
||||
o.uri AS iri
|
||||
""")
|
||||
|
||||
iris = []
|
||||
for record in result:
|
||||
iri = record.get("iri", None)
|
||||
iris += [iri]
|
||||
iris = [iri for iri in iris if iri is not None]
|
||||
return iris
|
||||
|
||||
def _reconstruct_ontology_from_metadata(self, session, iri) -> Ontology | None:
|
||||
"""Reconstruct an ontology from its metadata and related entities.
|
||||
|
||||
This method takes an ontology IRI and reconstructs the complete
|
||||
ontology by fetching all related entities from the namespace.
|
||||
|
||||
Args:
|
||||
session: Neo4j session for executing queries.
|
||||
iri: The ontology IRI to reconstruct.
|
||||
|
||||
Returns:
|
||||
Ontology | None: The reconstructed ontology, or None if failed.
|
||||
"""
|
||||
namespace_uri, _ = self._extract_namespace_prefix(iri)
|
||||
|
||||
logger.debug(f"Reconstructing ontology: {iri} with namespace: {namespace_uri}")
|
||||
|
||||
# Fallback to n10s export for this namespace
|
||||
graph = self._export_namespace_via_n10s(session, namespace_uri)
|
||||
if graph and len(graph) > 0:
|
||||
return self._create_ontology_object(iri, iri, graph)
|
||||
|
||||
def _export_namespace_via_n10s(
|
||||
self, session, namespace_uri: str
|
||||
) -> RDFGraph | None:
|
||||
"""Export entities belonging to a namespace using n10s.
|
||||
|
||||
This method uses Neo4j's n10s plugin to export all entities
|
||||
belonging to a specific namespace as RDF triples.
|
||||
|
||||
Args:
|
||||
session: Neo4j session for executing queries.
|
||||
namespace_uri: The namespace URI to export.
|
||||
|
||||
Returns:
|
||||
RDFGraph | None: The exported RDF graph, or None if failed.
|
||||
"""
|
||||
try:
|
||||
result = session.run(
|
||||
f"""
|
||||
CALL n10s.rdf.export.cypher(
|
||||
'MATCH (n)-[r]->(m) WHERE n.uri STARTS WITH "{namespace_uri}" RETURN n,r,m',
|
||||
{{format: 'Turtle'}}
|
||||
)
|
||||
YIELD subject, predicate, object, isLiteral, literalType, literalLang
|
||||
RETURN subject, predicate, object, isLiteral, literalType, literalLang
|
||||
"""
|
||||
)
|
||||
|
||||
# Process into Turtle format
|
||||
turtle_lines = []
|
||||
|
||||
for record in result:
|
||||
subj = record["subject"]
|
||||
pred = record["predicate"]
|
||||
obj = record["object"]
|
||||
is_literal = record["isLiteral"]
|
||||
literal_type = record["literalType"]
|
||||
literal_lang = record["literalLang"]
|
||||
|
||||
# Format object
|
||||
if is_literal:
|
||||
# Escape special characters in literals
|
||||
obj = obj.replace('"', r"\"")
|
||||
obj_str = f'"{obj}"'
|
||||
|
||||
# Add datatype or language tag if present
|
||||
if literal_lang:
|
||||
obj_str += f"@{literal_lang}"
|
||||
elif literal_type:
|
||||
obj_str += f"^^<{literal_type}>"
|
||||
else:
|
||||
obj_str = f"<{obj}>"
|
||||
|
||||
# Format triple
|
||||
turtle_lines.append(f"<{subj}> <{pred}> {obj_str} .")
|
||||
|
||||
# Combine into single string
|
||||
turtle_string = "\n".join(turtle_lines)
|
||||
|
||||
if turtle_string.strip():
|
||||
graph = RDFGraph()
|
||||
graph.parse(data=turtle_string, format="turtle")
|
||||
logger.debug(
|
||||
f"Exported {len(graph)} triples via n10s for namespace {namespace_uri}"
|
||||
)
|
||||
return graph
|
||||
return None
|
||||
|
||||
except Exception as e:
|
||||
logger.debug(
|
||||
f"Failed to export via n10s for namespace {namespace_uri}: {e}"
|
||||
)
|
||||
|
||||
return None
|
||||
|
||||
def _create_ontology_object(
|
||||
self, iri: str, metadata: dict, graph: RDFGraph
|
||||
) -> Ontology:
|
||||
"""Create an Ontology object from IRI, metadata, and graph.
|
||||
|
||||
Args:
|
||||
iri: The ontology IRI.
|
||||
metadata: Metadata dictionary (currently unused, kept for compatibility).
|
||||
graph: The RDF graph containing the ontology data.
|
||||
|
||||
Returns:
|
||||
Ontology: The created ontology object.
|
||||
"""
|
||||
ontology_id = derive_ontology_id(iri)
|
||||
return Ontology(graph=graph, iri=iri, ontology_id=ontology_id)
|
||||
|
||||
def serialize_graph(self, graph: Graph, **kwargs) -> bool | None:
|
||||
"""Serialize an RDF graph to Neo4j with both n10s and raw triple storage.
|
||||
|
||||
This method stores the given RDF graph in Neo4j using the n10s plugin
|
||||
for RDF import. The data is stored as RDF triples that can be faithfully
|
||||
reconstructed later.
|
||||
|
||||
Args:
|
||||
graph: The RDF graph to store.
|
||||
**kwargs: Additional parameters (not used by Neo4j implementation).
|
||||
|
||||
Returns:
|
||||
Any: The result summary from n10s import operation.
|
||||
"""
|
||||
# Convert to RDFGraph if needed
|
||||
if not isinstance(graph, RDFGraph):
|
||||
rdf_graph = RDFGraph()
|
||||
for triple in graph:
|
||||
rdf_graph.add(triple)
|
||||
for prefix, namespace in graph.namespaces():
|
||||
rdf_graph.bind(prefix, namespace)
|
||||
graph = rdf_graph
|
||||
|
||||
turtle_data = graph.serialize(format="turtle")
|
||||
|
||||
# Type assertion: we know _driver is not None after initialization
|
||||
assert self._driver is not None
|
||||
with self._driver.session() as session:
|
||||
# Store via n10s for graph queries
|
||||
result = session.run(
|
||||
"CALL n10s.rdf.import.inline($ttl, 'Turtle')", ttl=turtle_data
|
||||
)
|
||||
summary = result.single()
|
||||
|
||||
return summary
|
||||
|
||||
def serialize(self, o: Ontology | RDFGraph, **kwargs) -> bool | None:
|
||||
"""Serialize an Ontology or RDFGraph to Neo4j with both n10s and raw triple storage.
|
||||
|
||||
This method stores the given Ontology or RDFGraph in Neo4j using the n10s plugin
|
||||
for RDF import. The data is stored as RDF triples that can be faithfully
|
||||
reconstructed later.
|
||||
|
||||
Args:
|
||||
o: Ontology or RDFGraph object to store.
|
||||
**kwargs: Additional keyword arguments (not used by Neo4j implementation).
|
||||
|
||||
Returns:
|
||||
Any: The result summary from n10s import operation.
|
||||
"""
|
||||
if isinstance(o, Ontology):
|
||||
graph = o.graph
|
||||
elif isinstance(o, RDFGraph):
|
||||
graph = o
|
||||
else:
|
||||
raise TypeError(f"unsupported obj of type {type(o)} received")
|
||||
|
||||
return self.serialize_graph(graph)
|
||||
|
||||
def close(self):
|
||||
"""Close the Neo4j driver connection.
|
||||
|
||||
This method should be called when the manager is no longer needed
|
||||
to properly close the database connection and free resources.
|
||||
"""
|
||||
if self._driver:
|
||||
self._driver.close()
|
||||
Reference in New Issue
Block a user