# TODO: # - [ ] Rename this to just validator.py 0.5.x # - [ ] Maintain validator_base.py for exports but deprecate them # - [ ] Remove validator_base.py in 0.6.x import asyncio from contextvars import Context, ContextVar from functools import partial import inspect import importlib import logging from collections import defaultdict from dataclasses import dataclass import re from string import Template from typing import Any, Callable, Dict, List, Optional, Type, TypeVar, Union from warnings import warn import requests from guardrails.hub.registry import get_registry from langchain_core.runnables import Runnable from guardrails.settings import settings from guardrails_ai.types import ErrorSpan, PassResult, FailResult, ValidationResult # noqa from guardrails.constants import hub from guardrails.hub_token.token import VALIDATOR_HUB_SERVICE, get_jwt_token from guardrails.logger import logger from guardrails.remote_inference import remote_inference from guardrails.hub_telemetry.hub_tracing import trace from guardrails.types.on_fail import OnFailAction from guardrails.utils.safe_get import safe_get from guardrails.utils.hub_telemetry_utils import HubTelemetry from guardrails.utils.tokenization_utils import ( postproc_splits, ) ### functions to get chunks ### def split_sentence_str(chunk: str): """A naive sentence splitter that splits on periods.""" if "." not in chunk: return [] fragments = chunk.split(".") return [fragments[0] + ".", ".".join(fragments[1:])] def split_sentence_word_tokenizers_jl_separator( chunk: str, separator: str = "abcdsentenceseperatordcba" ): """Use a sentence tokenizer to detect if at least one sentence is present in the chunk. We return the first sentence and the remaining chunks without the first sentence. We perform the first step of WordTokenizers.jl's split_sentences function to detect possible sentence boundaries before calling the sentence tokenizer. Args: chunk (str): The text to split into sentences. Returns: List[str]: A list of two strings. The first string is the first sentence in the chunk. The second string is the remaining text in the chunk. """ # using the sentence tokenizer is expensive # we check for a . to avoid wastefully calling the tokenizer # check at least 3 characters have been accumulated before splitting third_chunk = safe_get(chunk, 2) is_minimum_length = third_chunk is not None # check for potential line endings, which is what split_sentences does chunk_with_potential_line_endings, count = re.subn( r"([?!.])(?=\s|$)", rf"\1{separator}", chunk ) any_potential_line_endings = count > 0 if not is_minimum_length or not any_potential_line_endings: return [] sentences = postproc_splits(chunk_with_potential_line_endings, separator) sentences = re.split(rf"\n?{separator} ?\n?", sentences) # if not more than one sentence, we haven't accumulated enough for a validation if len(sentences) <= 1: return [] # return the sentence # then the remaining chunks that aren't finished accumulating return [sentences[0], "".join(sentences[1:])] # TODO: Can we remove dataclass? It was originally added to support pydantic 1.* @dataclass # type: ignore class Validator: """Base class for validators.""" rail_alias: str = "" run_in_separate_process = False override_value_on_pass = False required_metadata_keys = [] _metadata = {} def __init__( self, on_fail: Optional[Union[Callable[[Any, FailResult], Any], OnFailAction]] = None, **kwargs, ): self._disable_telemetry = settings.rc.enable_metrics is not True if not self._disable_telemetry: self._hub_telemetry = HubTelemetry(enabled=settings.rc.enable_metrics) self.use_local = kwargs.get("use_local", None) self.validation_endpoint = kwargs.get("validation_endpoint", None) # NOTE: I think this is an evergreen check # We should test w/o an rc file, # and if this doesn't raise then we should remove this. if not settings.rc: raise ValueError( "No .guardrailsrc file found." " Please run `guardrails configure` and try again." ) # If use_local is not set, we can fall back to the setting determined in CLI if self.use_local is None: self.use_local = not remote_inference.get_use_remote_inference(settings.rc) if not self.validation_endpoint: validator_id = self.rail_alias.split("/")[-1] submission_url = ( f"{VALIDATOR_HUB_SERVICE}/validator/{validator_id}/inference" ) self.validation_endpoint = submission_url self.on_fail_descriptor: Union[str, OnFailAction] = "custom" # chunking function returns empty list or list of 2 chunks # first chunk is the chunk to validate # second chunk is incomplete chunk that needs further accumulation self.accumulated_chunks: List[str] = [] if on_fail is None: on_fail = OnFailAction.EXCEPTION if isinstance(on_fail, OnFailAction): self.on_fail_descriptor = on_fail self.on_fail_method = None elif ( isinstance(on_fail, str) and OnFailAction.__members__.get(on_fail.upper()) is not None ): self.on_fail_descriptor = ( OnFailAction.__members__.get(on_fail.upper()) or "" # this default isn't needed, it's just for pyright ) self.on_fail_method = None else: self.on_fail_descriptor = OnFailAction.CUSTOM self._set_on_fail_method(on_fail) # Store the kwargs for the validator. self._kwargs = kwargs assert self.rail_alias in validators_registry, ( f"Validator {self.__class__.__name__} is not registered. " ) def _set_on_fail_method(self, on_fail: Callable[[Any, FailResult], Any]): """Set the on_fail method for the validator.""" on_fail_args = inspect.getfullargspec(on_fail) second_arg = safe_get(on_fail_args.args, 1) if second_arg is None: raise ValueError( "The on_fail method must take two arguments: " "the value being validated and the FailResult." ) self.on_fail_method = on_fail def _validate(self, value: Any, metadata: Dict[str, Any]) -> ValidationResult: """User implementable function. Validates a value and return a validation result. This method should call _inference() in the implementation to perform inference on some input value. """ raise NotImplementedError def _inference_local(self, model_input: Any) -> Any: """User implementable function. Runs a machine learning pipeline on some input on the local machine. This function should receive the expected input to the ML model, and output the results from the ml model. """ raise NotImplementedError def _inference_remote(self, model_input: Any) -> Any: """User implementable function. Runs a machine learning pipeline on some input on a remote machine. This function should receive the expected input to the ML model, and output the results from the ml model. Can call _hub_inference_request() if request is routed through the hub. """ raise NotImplementedError def validate(self, value: Any, metadata: Dict[str, Any]) -> ValidationResult: """Do not override this function, instead implement _validate(). External facing validate function. This function acts as a wrapper for _validate() and is intended to apply any meta- validation requirements, logic, or pre/post processing. """ validation_result = self._validate(value, metadata) return validation_result async def async_validate( self, value: Any, metadata: Dict[str, Any] ) -> ValidationResult: """Use this function if your validation logic requires asyncio. Guaranteed to work with AsyncGuard May not work with synchronous Guards if they are used within an async context due to lack of available event loops. """ loop = asyncio.get_event_loop() return await loop.run_in_executor(None, self.validate, value, metadata) @trace(name="/validator_inference", origin="Validator._inference") def _inference(self, model_input: Any) -> Any: """Calls either a local or remote inference engine for use in the validation call. Args: model_input (Any): Receives the input to be passed to your ML model. Returns: Any: Returns the output from the ML model inference. """ # Only use if both are set, otherwise fall back to local inference if self.use_local: logger.debug(f" ==> {self.rail_alias} is using local inference.") return self._inference_local(model_input) if not self.use_local and self.validation_endpoint: logger.debug(f" ==> {self.rail_alias} is using remote inference.") return self._inference_remote(model_input) raise RuntimeError( "No inference endpoint set, but use_local was false. " "Please set either use_local=True or " "set an validation_endpoint to perform inference in the validator." ) def _chunking_function(self, chunk: str) -> List[str]: """The strategy used for chunking accumulated text input into validation sets. Args: chunk (str): The text to chunk into some subset. Returns: list[str]: The text chunked into some subset. """ return split_sentence_word_tokenizers_jl_separator(chunk) def validate_stream( self, chunk: Any, metadata: Dict[str, Any], *, property_path: Optional[str] = "$", context_vars: Optional[ContextVar[Dict[str, ContextVar[List[str]]]]] = None, context: Optional[Context] = None, **kwargs, ) -> Optional[ValidationResult]: """Validates a chunk emitted by an LLM. If the LLM chunk is smaller than the validator's chunking strategy, it will be accumulated until it reaches the desired size. In the meantime, the validator will return None. If the LLM chunk is larger than the validator's chunking strategy, it will split it into validator-sized chunks and validate each one, returning an array of validation results. Otherwise, the validator will validate the chunk and return the result. """ # combine accumulated chunks and new [:-1]chunk accumulated_chunks = self.accumulated_chunks # if context_vars is passed, use it to get the accumulated chunks context_var: Optional[ContextVar[List[str]]] = None ctx_var_map: Optional[Dict[str, ContextVar[List[str]]]] = None context_key = f"{property_path}_{self.rail_alias}" if context_vars and context: ctx_var_map = context.run(context_vars.get) context_var = ctx_var_map.get(context_key) if context_var: accumulated_chunks = context.run(context_var.get) accumulated_chunks.append(chunk) accumulated_text = "".join(accumulated_chunks) # check if enough chunks have accumulated for validation split_contents = self._chunking_function(accumulated_text) # if remainder kwargs is passed, validate remainder regardless remainder = kwargs.get("remainder", False) if remainder: split_contents = [accumulated_text, ""] # if no chunks are returned, we haven't accumulated enough if len(split_contents) == 0: if context_vars and context_var and context and ctx_var_map: context.run(context_var.set, accumulated_chunks) ctx_var_map[context_key] = context_var context.run(context_vars.set, ctx_var_map) else: self.accumulated_chunks = accumulated_chunks return None [chunk_to_validate, new_accumulated_chunks] = split_contents if context_vars and context_var and context and ctx_var_map: context.run(context_var.set, [new_accumulated_chunks]) ctx_var_map[context_key] = context_var context.run(context_vars.set, ctx_var_map) else: self.accumulated_chunks = [new_accumulated_chunks] # exclude last chunk, because it may not be a complete chunk validation_result = self.validate(chunk_to_validate, metadata) # if validate doesn't set validated chunk, we set it if validation_result.validated_chunk is None: validation_result.validated_chunk = chunk_to_validate if isinstance(validation_result, FailResult): if validation_result.error_spans is None: validation_result.error_spans = [ ErrorSpan( start=0, end=len(chunk_to_validate), reason="The input failed validation.", ) ] return validation_result async def async_validate_stream( self, chunk: Any, metadata: Dict[str, Any], **kwargs ) -> Optional[ValidationResult]: loop = asyncio.get_event_loop() validate_stream_partial = partial( self.validate_stream, chunk, metadata, **kwargs ) return await loop.run_in_executor(None, validate_stream_partial) def _hub_inference_request( self, request_body: Union[dict, str], validation_endpoint: str ) -> Any: """Makes a request to the Validator Hub to run a ML based validation model. This request is authed through the hub and rerouted to a hosted ML model. The reply from the hosted endpoint is returned and sent to this client. Args: request_body (dict): A dictionary containing the required info for the final validation_endpoint (str): The url to request as an endpoint inference endpoint to run. Raises: HttpError: If the received reply was not ok. Returns: Any: Post request response from the ML based validation model. """ hub_jwt_token = get_jwt_token(settings.rc) headers = { "Authorization": f"Bearer {hub_jwt_token}", "Content-Type": "application/json", } req = requests.post(validation_endpoint, data=request_body, headers=headers) if not req.ok: if req.status_code == 401: raise Exception( "401: Remote Inference Unauthorized. Please run " "`guardrails configure`. You can find a new" " token at https://guardrailsai.com/hub/keys" ) else: logging.error(req.status_code) return req.json() def to_prompt(self, with_keywords: bool = True) -> str: """Convert the validator to a prompt. E.g. ValidLength(5, 10) -> "length: 5 10" when with_keywords is False. ValidLength(5, 10) -> "length: min=5 max=10" when with_keywords is True. Args: with_keywords: Whether to include the keyword arguments in the prompt. Returns: A string representation of the validator. """ if not len(self._kwargs): return self.rail_alias kwargs = self._kwargs.copy() for k, v in kwargs.items(): if not isinstance(v, str): kwargs[k] = str(v) params = " ".join(list(kwargs.values())) if with_keywords: params = " ".join([f"{k}={v}" for k, v in kwargs.items()]) return f"{self.rail_alias}: {params}" # TODO: Is this still used anywhere? def to_xml_attrib(self): """Convert the validator to an XML attribute.""" if not len(self._kwargs): return self.rail_alias validator_args = [] init_args = inspect.getfullargspec(self.__init__) for arg in init_args.args[1:]: if arg not in ("on_fail", "args", "kwargs"): arg_value = self._kwargs.get(arg) str_arg = str(arg_value) if str_arg is not None: str_arg = "{" + str_arg + "}" if " " in str_arg else str_arg validator_args.append(str_arg) params = " ".join(validator_args) return f"{self.rail_alias}: {params}" def get_args(self): """Get the arguments for the validator.""" return self._kwargs def __call__(self, value): result = self.validate(value, {}) if isinstance(result, FailResult): from guardrails.validator_service.validator_service_base import ( ValidatorServiceBase, ) validator_service = ValidatorServiceBase() return validator_service.perform_correction(result, value, self) return value def __eq__(self, other): if not isinstance(other, Validator): return False return self.to_prompt() == other.to_prompt() # TODO: Make this a generic method on an abstract class def __stringify__(self): template = Template( """ ${class_name} { rail_alias: ${rail_alias}, on_fail: ${on_fail_descriptor}, run_in_separate_process: ${run_in_separate_process}, override_value_on_pass: ${override_value_on_pass}, required_metadata_keys: ${required_metadata_keys}, kwargs: ${kwargs} }""" ) return template.safe_substitute( { "class_name": self.__class__.__name__, "rail_alias": self.rail_alias, "on_fail_descriptor": self.on_fail_descriptor, "run_in_separate_process": self.run_in_separate_process, "override_value_on_pass": self.override_value_on_pass, "required_metadata_keys": self.required_metadata_keys, "kwargs": self._kwargs, } ) """ This method allows the user to provide metadata to validators used in an LCEL chain. This is necessary because they can't pass metadata directly to `validate` in a chain because is called internally during `invoke`. Usage --- my_validator = Validator(args).with_metadata({ "key": "value" }) chain = prompt | model | my_validator | output_parser chain.invoke({...}) When called multiple times on the same validator instance, the metadata value will be override. This allows the user to change the metadata programmatically for different chains or calls. """ def with_metadata(self, metadata: Dict[str, Any]): """Assigns metadata to this validator to use during validation.""" self._metadata = metadata return self def to_runnable(self) -> Runnable: from guardrails.integrations.langchain.validator_runnable import ( ValidatorRunnable, ) return ValidatorRunnable(self) V = TypeVar("V", bound=Validator, covariant=True) validators_registry: Dict[str, Type[Validator]] = {} types_to_validators = defaultdict(list) def validator_factory(name: str, validate: Callable) -> Type[Validator]: def validate_wrapper(self, *args, **kwargs): return validate(*args, **kwargs) validator = type( name, (Validator,), {"validate": validate_wrapper, "rail_alias": name}, ) return validator def register_validator( name: str, data_type: Union[str, List[str]], has_guardrails_endpoint: bool = False ) -> Callable[[Union[Type[V], Callable]], Union[Type[V], Type[Validator]]]: """Register a validator for a data type.""" from guardrails.datatypes import types_registry if isinstance(data_type, str): data_type = types_registry if data_type == "all" else [data_type] # Make sure that the data type string exists in the data types registry. for dt in data_type: if dt not in types_registry: raise ValueError(f"Data type {dt} is not registered.") types_to_validators[dt].append(name) def decorator( cls_or_func: Union[Type[V], Callable], ) -> Union[Type[V], Type[Validator]]: """Register a validator for a data type.""" if isinstance(cls_or_func, type) and issubclass(cls_or_func, Validator): cls = cls_or_func cls.rail_alias = name elif callable(cls_or_func) and not isinstance(cls_or_func, type): func = cls_or_func func.rail_alias = name # type: ignore # ensure function takes two args if not func.__code__.co_argcount == 2: raise ValueError( f"Validator function {func.__name__} must take two arguments." ) # dynamically create Validator subclass with `validate` method as `func` cls = validator_factory(name, func) else: raise ValueError( "Only functions and Validator subclasses " "can be registered as validators." ) validators_registry[name] = cls return cls return decorator def try_to_import_from_hub(validator_key: str): try: hub_registry = get_registry() validator_entry = hub_registry.validators.get(validator_key) if validator_entry and validator_entry.import_path: importlib.import_module(validator_entry.import_path) except (ImportError, KeyError): logger.error("Could not import from hub. Validators may not work properly.") # TODO: Move this to validator_utils.py def get_validator_class(name: Optional[str]) -> Optional[Type[Validator]]: if not name: return None is_hub_validator = name.startswith(hub) validator_key = name.replace(hub, "") if is_hub_validator else name registration = validators_registry.get(validator_key) if not registration: try_to_import_from_hub(validator_key) registration = validators_registry.get(validator_key) if not registration: warn(f"Validator with id {name} was not found in the registry! Ignoring...") return None return registration