AI/참고/guardrails-main/guardrails/validator_base.py

# TODO:
#   - [ ] Rename this to just validator.py 0.5.x
#   - [ ] Maintain validator_base.py for exports but deprecate them
#   - [ ] Remove validator_base.py in 0.6.x

import asyncio
from contextvars import Context, ContextVar
from functools import partial
import inspect
import importlib
import logging
from collections import defaultdict
from dataclasses import dataclass
import re
from string import Template
from typing import Any, Callable, Dict, List, Optional, Type, TypeVar, Union
from warnings import warn

import requests
from guardrails.hub.registry import get_registry
from langchain_core.runnables import Runnable

from guardrails.settings import settings
from guardrails_ai.types import ErrorSpan, PassResult, FailResult, ValidationResult  # noqa
from guardrails.constants import hub
from guardrails.hub_token.token import VALIDATOR_HUB_SERVICE, get_jwt_token
from guardrails.logger import logger
from guardrails.remote_inference import remote_inference
from guardrails.hub_telemetry.hub_tracing import trace
from guardrails.types.on_fail import OnFailAction
from guardrails.utils.safe_get import safe_get
from guardrails.utils.hub_telemetry_utils import HubTelemetry
from guardrails.utils.tokenization_utils import (
    postproc_splits,
)


### functions to get chunks ###
def split_sentence_str(chunk: str):
    """A naive sentence splitter that splits on periods."""
    if "." not in chunk:
        return []
    fragments = chunk.split(".")
    return [fragments[0] + ".", ".".join(fragments[1:])]


def split_sentence_word_tokenizers_jl_separator(
    chunk: str, separator: str = "abcdsentenceseperatordcba"
):
    """Use a sentence tokenizer to detect if at least one sentence is present
    in the chunk. We return the first sentence and the remaining chunks without
    the first sentence.

    We perform the first step of WordTokenizers.jl's split_sentences function to
    detect possible sentence boundaries before calling the sentence tokenizer.

    Args:
        chunk (str): The text to split into sentences.

    Returns:
        List[str]: A list of two strings. The first string is the first sentence
            in the chunk. The second string is the remaining text in the chunk.
    """
    # using the sentence tokenizer is expensive
    # we check for a . to avoid wastefully calling the tokenizer

    # check at least 3 characters have been accumulated before splitting
    third_chunk = safe_get(chunk, 2)
    is_minimum_length = third_chunk is not None

    # check for potential line endings, which is what split_sentences does
    chunk_with_potential_line_endings, count = re.subn(
        r"([?!.])(?=\s|$)", rf"\1{separator}", chunk
    )
    any_potential_line_endings = count > 0
    if not is_minimum_length or not any_potential_line_endings:
        return []

    sentences = postproc_splits(chunk_with_potential_line_endings, separator)
    sentences = re.split(rf"\n?{separator} ?\n?", sentences)
    # if not more than one sentence, we haven't accumulated enough for a validation
    if len(sentences) <= 1:
        return []

    # return the sentence
    # then the remaining chunks that aren't finished accumulating
    return [sentences[0], "".join(sentences[1:])]


# TODO: Can we remove dataclass? It was originally added to support pydantic 1.*
@dataclass  # type: ignore
class Validator:
    """Base class for validators."""

    rail_alias: str = ""

    run_in_separate_process = False
    override_value_on_pass = False
    required_metadata_keys = []
    _metadata = {}

    def __init__(
        self,
        on_fail: Optional[Union[Callable[[Any, FailResult], Any], OnFailAction]] = None,
        **kwargs,
    ):
        self._disable_telemetry = settings.rc.enable_metrics is not True
        if not self._disable_telemetry:
            self._hub_telemetry = HubTelemetry(enabled=settings.rc.enable_metrics)

        self.use_local = kwargs.get("use_local", None)
        self.validation_endpoint = kwargs.get("validation_endpoint", None)
        # NOTE: I think this is an evergreen check
        # We should test w/o an rc file,
        #   and if this doesn't raise then we should remove this.
        if not settings.rc:
            raise ValueError(
                "No .guardrailsrc file found."
                " Please run `guardrails configure` and try again."
            )

        # If use_local is not set, we can fall back to the setting determined in CLI
        if self.use_local is None:
            self.use_local = not remote_inference.get_use_remote_inference(settings.rc)

        if not self.validation_endpoint:
            validator_id = self.rail_alias.split("/")[-1]
            submission_url = (
                f"{VALIDATOR_HUB_SERVICE}/validator/{validator_id}/inference"
            )
            self.validation_endpoint = submission_url
        self.on_fail_descriptor: Union[str, OnFailAction] = "custom"

        # chunking function returns empty list or list of 2 chunks
        # first chunk is the chunk to validate
        # second chunk is incomplete chunk that needs further accumulation
        self.accumulated_chunks: List[str] = []

        if on_fail is None:
            on_fail = OnFailAction.EXCEPTION
        if isinstance(on_fail, OnFailAction):
            self.on_fail_descriptor = on_fail
            self.on_fail_method = None
        elif (
            isinstance(on_fail, str)
            and OnFailAction.__members__.get(on_fail.upper()) is not None
        ):
            self.on_fail_descriptor = (
                OnFailAction.__members__.get(on_fail.upper())
                or ""  # this default isn't needed, it's just for pyright
            )
            self.on_fail_method = None
        else:
            self.on_fail_descriptor = OnFailAction.CUSTOM
            self._set_on_fail_method(on_fail)

        # Store the kwargs for the validator.
        self._kwargs = kwargs

        assert self.rail_alias in validators_registry, (
            f"Validator {self.__class__.__name__} is not registered. "
        )

    def _set_on_fail_method(self, on_fail: Callable[[Any, FailResult], Any]):
        """Set the on_fail method for the validator."""
        on_fail_args = inspect.getfullargspec(on_fail)
        second_arg = safe_get(on_fail_args.args, 1)
        if second_arg is None:
            raise ValueError(
                "The on_fail method must take two arguments: "
                "the value being validated and the FailResult."
            )

        self.on_fail_method = on_fail

    def _validate(self, value: Any, metadata: Dict[str, Any]) -> ValidationResult:
        """User implementable function.

        Validates a value and return a validation result. This method
        should call _inference() in the implementation to perform
        inference on some input value.
        """
        raise NotImplementedError

    def _inference_local(self, model_input: Any) -> Any:
        """User implementable function.

        Runs a machine learning pipeline on some input on the local
        machine. This function should receive the expected input to the
        ML model, and output the results from the ml model.
        """
        raise NotImplementedError

    def _inference_remote(self, model_input: Any) -> Any:
        """User implementable function.

        Runs a machine learning pipeline on some input on a remote
        machine. This function should receive the expected input to the
        ML model, and output the results from the ml model.

        Can call _hub_inference_request() if request is routed through
        the hub.
        """
        raise NotImplementedError

    def validate(self, value: Any, metadata: Dict[str, Any]) -> ValidationResult:
        """Do not override this function, instead implement _validate().

        External facing validate function. This function acts as a
        wrapper for _validate() and is intended to apply any meta-
        validation requirements, logic, or pre/post processing.
        """
        validation_result = self._validate(value, metadata)
        return validation_result

    async def async_validate(
        self, value: Any, metadata: Dict[str, Any]
    ) -> ValidationResult:
        """Use this function if your validation logic requires asyncio.

        Guaranteed to work with AsyncGuard

        May not work with synchronous Guards if they are used within an
        async context     due to lack of available event loops.
        """
        loop = asyncio.get_event_loop()
        return await loop.run_in_executor(None, self.validate, value, metadata)

    @trace(name="/validator_inference", origin="Validator._inference")
    def _inference(self, model_input: Any) -> Any:
        """Calls either a local or remote inference engine for use in the
        validation call.

        Args:
            model_input (Any): Receives the input to be passed to your ML model.

        Returns:
            Any: Returns the output from the ML model inference.
        """
        # Only use if both are set, otherwise fall back to local inference
        if self.use_local:
            logger.debug(f" ==> {self.rail_alias} is using local inference.")
            return self._inference_local(model_input)
        if not self.use_local and self.validation_endpoint:
            logger.debug(f" ==> {self.rail_alias} is using remote inference.")
            return self._inference_remote(model_input)

        raise RuntimeError(
            "No inference endpoint set, but use_local was false. "
            "Please set either use_local=True or "
            "set an validation_endpoint to perform inference in the validator."
        )

    def _chunking_function(self, chunk: str) -> List[str]:
        """The strategy used for chunking accumulated text input into
        validation sets.

        Args:
            chunk (str): The text to chunk into some subset.

        Returns:
            list[str]: The text chunked into some subset.
        """
        return split_sentence_word_tokenizers_jl_separator(chunk)

    def validate_stream(
        self,
        chunk: Any,
        metadata: Dict[str, Any],
        *,
        property_path: Optional[str] = "$",
        context_vars: Optional[ContextVar[Dict[str, ContextVar[List[str]]]]] = None,
        context: Optional[Context] = None,
        **kwargs,
    ) -> Optional[ValidationResult]:
        """Validates a chunk emitted by an LLM. If the LLM chunk is smaller
        than the validator's chunking strategy, it will be accumulated until it
        reaches the desired size. In the meantime, the validator will return
        None.

        If the LLM chunk is larger than the validator's chunking
        strategy, it will split it into validator-sized chunks and
        validate each one, returning an array of validation results.

        Otherwise, the validator will validate the chunk and return the
        result.
        """
        # combine accumulated chunks and new [:-1]chunk
        accumulated_chunks = self.accumulated_chunks

        # if context_vars is passed, use it to get the accumulated chunks
        context_var: Optional[ContextVar[List[str]]] = None
        ctx_var_map: Optional[Dict[str, ContextVar[List[str]]]] = None
        context_key = f"{property_path}_{self.rail_alias}"
        if context_vars and context:
            ctx_var_map = context.run(context_vars.get)
            context_var = ctx_var_map.get(context_key)
            if context_var:
                accumulated_chunks = context.run(context_var.get)

        accumulated_chunks.append(chunk)
        accumulated_text = "".join(accumulated_chunks)
        # check if enough chunks have accumulated for validation
        split_contents = self._chunking_function(accumulated_text)

        # if remainder kwargs is passed, validate remainder regardless
        remainder = kwargs.get("remainder", False)
        if remainder:
            split_contents = [accumulated_text, ""]
        # if no chunks are returned, we haven't accumulated enough
        if len(split_contents) == 0:
            if context_vars and context_var and context and ctx_var_map:
                context.run(context_var.set, accumulated_chunks)
                ctx_var_map[context_key] = context_var
                context.run(context_vars.set, ctx_var_map)
            else:
                self.accumulated_chunks = accumulated_chunks
            return None
        [chunk_to_validate, new_accumulated_chunks] = split_contents
        if context_vars and context_var and context and ctx_var_map:
            context.run(context_var.set, [new_accumulated_chunks])
            ctx_var_map[context_key] = context_var
            context.run(context_vars.set, ctx_var_map)
        else:
            self.accumulated_chunks = [new_accumulated_chunks]
        # exclude last chunk, because it may not be a complete chunk
        validation_result = self.validate(chunk_to_validate, metadata)
        # if validate doesn't set validated chunk, we set it
        if validation_result.validated_chunk is None:
            validation_result.validated_chunk = chunk_to_validate
        if isinstance(validation_result, FailResult):
            if validation_result.error_spans is None:
                validation_result.error_spans = [
                    ErrorSpan(
                        start=0,
                        end=len(chunk_to_validate),
                        reason="The input failed validation.",
                    )
                ]

        return validation_result

    async def async_validate_stream(
        self, chunk: Any, metadata: Dict[str, Any], **kwargs
    ) -> Optional[ValidationResult]:
        loop = asyncio.get_event_loop()
        validate_stream_partial = partial(
            self.validate_stream, chunk, metadata, **kwargs
        )
        return await loop.run_in_executor(None, validate_stream_partial)

    def _hub_inference_request(
        self, request_body: Union[dict, str], validation_endpoint: str
    ) -> Any:
        """Makes a request to the Validator Hub to run a ML based validation
        model. This request is authed through the hub and rerouted to a hosted
        ML model. The reply from the hosted endpoint is returned and sent to
        this client.

        Args:
            request_body (dict): A dictionary containing the required info for the final
            validation_endpoint (str): The url to request as an endpoint
            inference endpoint to run.

        Raises:
            HttpError: If the received reply was not ok.

        Returns:
            Any: Post request response from the ML based validation model.
        """
        hub_jwt_token = get_jwt_token(settings.rc)
        headers = {
            "Authorization": f"Bearer {hub_jwt_token}",
            "Content-Type": "application/json",
        }
        req = requests.post(validation_endpoint, data=request_body, headers=headers)
        if not req.ok:
            if req.status_code == 401:
                raise Exception(
                    "401: Remote Inference Unauthorized. Please run "
                    "`guardrails configure`. You can find a new"
                    " token at https://guardrailsai.com/hub/keys"
                )
            else:
                logging.error(req.status_code)

        return req.json()

    def to_prompt(self, with_keywords: bool = True) -> str:
        """Convert the validator to a prompt.

        E.g. ValidLength(5, 10) -> "length: 5 10" when with_keywords is False.
        ValidLength(5, 10) -> "length: min=5 max=10" when with_keywords is True.

        Args:
            with_keywords: Whether to include the keyword arguments in the prompt.

        Returns:
            A string representation of the validator.
        """
        if not len(self._kwargs):
            return self.rail_alias

        kwargs = self._kwargs.copy()
        for k, v in kwargs.items():
            if not isinstance(v, str):
                kwargs[k] = str(v)

        params = " ".join(list(kwargs.values()))
        if with_keywords:
            params = " ".join([f"{k}={v}" for k, v in kwargs.items()])
        return f"{self.rail_alias}: {params}"

    # TODO: Is this still used anywhere?
    def to_xml_attrib(self):
        """Convert the validator to an XML attribute."""

        if not len(self._kwargs):
            return self.rail_alias

        validator_args = []
        init_args = inspect.getfullargspec(self.__init__)
        for arg in init_args.args[1:]:
            if arg not in ("on_fail", "args", "kwargs"):
                arg_value = self._kwargs.get(arg)
                str_arg = str(arg_value)
                if str_arg is not None:
                    str_arg = "{" + str_arg + "}" if " " in str_arg else str_arg
                    validator_args.append(str_arg)

        params = " ".join(validator_args)
        return f"{self.rail_alias}: {params}"

    def get_args(self):
        """Get the arguments for the validator."""
        return self._kwargs

    def __call__(self, value):
        result = self.validate(value, {})
        if isinstance(result, FailResult):
            from guardrails.validator_service.validator_service_base import (
                ValidatorServiceBase,
            )

            validator_service = ValidatorServiceBase()
            return validator_service.perform_correction(result, value, self)
        return value

    def __eq__(self, other):
        if not isinstance(other, Validator):
            return False
        return self.to_prompt() == other.to_prompt()

    # TODO: Make this a generic method on an abstract class
    def __stringify__(self):
        template = Template(
            """
            ${class_name} {
                rail_alias: ${rail_alias},
                on_fail: ${on_fail_descriptor},
                run_in_separate_process: ${run_in_separate_process},
                override_value_on_pass: ${override_value_on_pass},
                required_metadata_keys: ${required_metadata_keys},
                kwargs: ${kwargs}
            }"""
        )
        return template.safe_substitute(
            {
                "class_name": self.__class__.__name__,
                "rail_alias": self.rail_alias,
                "on_fail_descriptor": self.on_fail_descriptor,
                "run_in_separate_process": self.run_in_separate_process,
                "override_value_on_pass": self.override_value_on_pass,
                "required_metadata_keys": self.required_metadata_keys,
                "kwargs": self._kwargs,
            }
        )

    """
    This method allows the user to provide metadata to validators used in an LCEL chain.
    This is necessary because they can't pass metadata directly to `validate` in a chain
        because is called internally during `invoke`.

    Usage
    ---
    my_validator = Validator(args).with_metadata({ "key": "value" })

    chain = prompt | model | my_validator | output_parser
    chain.invoke({...})

    When called multiple times on the same validator instance,
        the metadata value will be override.
    This allows the user to change the metadata programmatically
        for different chains or calls.
    """

    def with_metadata(self, metadata: Dict[str, Any]):
        """Assigns metadata to this validator to use during validation."""
        self._metadata = metadata
        return self

    def to_runnable(self) -> Runnable:
        from guardrails.integrations.langchain.validator_runnable import (
            ValidatorRunnable,
        )

        return ValidatorRunnable(self)


V = TypeVar("V", bound=Validator, covariant=True)
validators_registry: Dict[str, Type[Validator]] = {}
types_to_validators = defaultdict(list)


def validator_factory(name: str, validate: Callable) -> Type[Validator]:
    def validate_wrapper(self, *args, **kwargs):
        return validate(*args, **kwargs)

    validator = type(
        name,
        (Validator,),
        {"validate": validate_wrapper, "rail_alias": name},
    )
    return validator


def register_validator(
    name: str, data_type: Union[str, List[str]], has_guardrails_endpoint: bool = False
) -> Callable[[Union[Type[V], Callable]], Union[Type[V], Type[Validator]]]:
    """Register a validator for a data type."""
    from guardrails.datatypes import types_registry

    if isinstance(data_type, str):
        data_type = types_registry if data_type == "all" else [data_type]
    # Make sure that the data type string exists in the data types registry.
    for dt in data_type:
        if dt not in types_registry:
            raise ValueError(f"Data type {dt} is not registered.")

        types_to_validators[dt].append(name)

    def decorator(
        cls_or_func: Union[Type[V], Callable],
    ) -> Union[Type[V], Type[Validator]]:
        """Register a validator for a data type."""
        if isinstance(cls_or_func, type) and issubclass(cls_or_func, Validator):
            cls = cls_or_func
            cls.rail_alias = name
        elif callable(cls_or_func) and not isinstance(cls_or_func, type):
            func = cls_or_func
            func.rail_alias = name  # type: ignore
            # ensure function takes two args
            if not func.__code__.co_argcount == 2:
                raise ValueError(
                    f"Validator function {func.__name__} must take two arguments."
                )
            # dynamically create Validator subclass with `validate` method as `func`
            cls = validator_factory(name, func)
        else:
            raise ValueError(
                "Only functions and Validator subclasses "
                "can be registered as validators."
            )
        validators_registry[name] = cls
        return cls

    return decorator


def try_to_import_from_hub(validator_key: str):
    try:
        hub_registry = get_registry()
        validator_entry = hub_registry.validators.get(validator_key)
        if validator_entry and validator_entry.import_path:
            importlib.import_module(validator_entry.import_path)
    except (ImportError, KeyError):
        logger.error("Could not import from hub. Validators may not work properly.")


# TODO: Move this to validator_utils.py
def get_validator_class(name: Optional[str]) -> Optional[Type[Validator]]:
    if not name:
        return None
    is_hub_validator = name.startswith(hub)
    validator_key = name.replace(hub, "") if is_hub_validator else name

    registration = validators_registry.get(validator_key)
    if not registration:
        try_to_import_from_hub(validator_key)
        registration = validators_registry.get(validator_key)

    if not registration:
        warn(f"Validator with id {name} was not found in the registry!  Ignoring...")
        return None

    return registration