{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "# Create Custom Validators\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "To make the implementation of custom validators easier, we have provided [an interface in the OSS](https://github.com/guardrails-ai/guardrails/blob/main/guardrails/validator_base.py).\n", "\n", "There are a few key steps to get up and running with a custom validator:\n", "\n", "1. Implementing the validator\n", "2. Conforming to the required interface\n", "3. Running Locally/Submitting to the Validator Hub\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "First, let's build out a simple custom validator. This validator will check if all text in the input is lowercase.\n" ] }, { "cell_type": "code", "execution_count": 31, "metadata": {}, "outputs": [], "source": [ "from typing import Any, Dict\n", "import requests\n", "from guardrails.validator_base import (\n", " FailResult,\n", " PassResult,\n", " ValidationResult,\n", " Validator,\n", " register_validator,\n", " ErrorSpan,\n", ")\n", "from typing import Optional, Callable\n", "from rich import print\n", "\n", "\n", "@register_validator(name=\"guardrails/lower_case\", data_type=\"string\")\n", "class LowercaseValidator(Validator):\n", " def __init__(\n", " self,\n", " on_fail: Optional[Callable] = None,\n", " **kwargs,\n", " ):\n", " super().__init__(on_fail=on_fail, **kwargs)\n", " self.rail_alias = \"lowercase\"\n", "\n", " def _validate(self, value: Any, metadata: Dict[str, Any]) -> ValidationResult:\n", " if not isinstance(value, str):\n", " return FailResult(\n", " metadata=metadata,\n", " error_message=\"Input must be a string.\",\n", " fix_value=None,\n", " )\n", "\n", " inference_result = self._inference(value)\n", "\n", " if inference_result:\n", " return PassResult()\n", " else:\n", " return FailResult(\n", " metadata=metadata,\n", " error_message=\"Input must be lowercase.\",\n", " fix_value=value.lower(),\n", " )\n", "\n", " def _inference_local(self, model_input: str) -> bool:\n", " \"\"\"Implement a function to perform inference on a local machine.\"\"\"\n", " return model_input.islower()\n", "\n", " def _inference_remote(self, model_input: str) -> bool:\n", " \"\"\"Implement a function that will build a request and perform inference on a\n", " remote machine. This is not required if you will always use local mode.\n", " \"\"\"\n", " response = requests.post(self.validation_endpoint, json={\"inputs\": model_input})\n", " if response.status_code != 200:\n", " raise Exception(\n", " f\"Remote inference failed with status code {response.status_code}\"\n", " )\n", "\n", " return response.json().get(\"is_lowercase\", False)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Simple usage running locally\n", "\n", "If this validator was stored locally in your codebase, you would want to import it\n", "\n", "```bash\n", "from lowercase_validator import LowercaseValidator\n", "```\n" ] }, { "cell_type": "code", "execution_count": 32, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
PassResult(\n",
       "    outcome='pass',\n",
       "    value_override=<class 'guardrails.classes.validation.validation_result.PassResult.ValueOverrideSentinel'>,\n",
       "    metadata=None,\n",
       "    validated_chunk=None\n",
       ")\n",
       "
\n" ], "text/plain": [ "\u001b[1;35mPassResult\u001b[0m\u001b[1m(\u001b[0m\n", " \u001b[33moutcome\u001b[0m=\u001b[32m'pass'\u001b[0m,\n", " \u001b[33mvalue_override\u001b[0m=\u001b[1m<\u001b[0m\u001b[1;95mclass\u001b[0m\u001b[39m \u001b[0m\u001b[32m'guardrails.classes.validation.validation_result.PassResult.ValueOverrideSentinel'\u001b[0m\u001b[1m>\u001b[0m,\n", " \u001b[33mmetadata\u001b[0m=\u001b[3;35mNone\u001b[0m,\n", " \u001b[33mvalidated_chunk\u001b[0m=\u001b[3;35mNone\u001b[0m\n", "\u001b[1m)\u001b[0m\n" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/html": [ "
FailResult(\n",
       "    outcome='fail',\n",
       "    error_message='Input must be lowercase.',\n",
       "    fix_value='hello world',\n",
       "    error_spans=None,\n",
       "    metadata={},\n",
       "    validated_chunk=None\n",
       ")\n",
       "
\n" ], "text/plain": [ "\u001b[1;35mFailResult\u001b[0m\u001b[1m(\u001b[0m\n", " \u001b[33moutcome\u001b[0m=\u001b[32m'fail'\u001b[0m,\n", " \u001b[33merror_message\u001b[0m=\u001b[32m'Input must be lowercase.'\u001b[0m,\n", " \u001b[33mfix_value\u001b[0m=\u001b[32m'hello world'\u001b[0m,\n", " \u001b[33merror_spans\u001b[0m=\u001b[3;35mNone\u001b[0m,\n", " \u001b[33mmetadata\u001b[0m=\u001b[1m{\u001b[0m\u001b[1m}\u001b[0m,\n", " \u001b[33mvalidated_chunk\u001b[0m=\u001b[3;35mNone\u001b[0m\n", "\u001b[1m)\u001b[0m\n" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/html": [ "
FailResult(\n",
       "    outcome='fail',\n",
       "    error_message='Input must be lowercase.',\n",
       "    fix_value='123',\n",
       "    error_spans=None,\n",
       "    metadata={},\n",
       "    validated_chunk=None\n",
       ")\n",
       "
\n" ], "text/plain": [ "\u001b[1;35mFailResult\u001b[0m\u001b[1m(\u001b[0m\n", " \u001b[33moutcome\u001b[0m=\u001b[32m'fail'\u001b[0m,\n", " \u001b[33merror_message\u001b[0m=\u001b[32m'Input must be lowercase.'\u001b[0m,\n", " \u001b[33mfix_value\u001b[0m=\u001b[32m'123'\u001b[0m,\n", " \u001b[33merror_spans\u001b[0m=\u001b[3;35mNone\u001b[0m,\n", " \u001b[33mmetadata\u001b[0m=\u001b[1m{\u001b[0m\u001b[1m}\u001b[0m,\n", " \u001b[33mvalidated_chunk\u001b[0m=\u001b[3;35mNone\u001b[0m\n", "\u001b[1m)\u001b[0m\n" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/html": [ "
FailResult(\n",
       "    outcome='fail',\n",
       "    error_message='Input must be a string.',\n",
       "    fix_value=None,\n",
       "    error_spans=None,\n",
       "    metadata={},\n",
       "    validated_chunk=None\n",
       ")\n",
       "
\n" ], "text/plain": [ "\u001b[1;35mFailResult\u001b[0m\u001b[1m(\u001b[0m\n", " \u001b[33moutcome\u001b[0m=\u001b[32m'fail'\u001b[0m,\n", " \u001b[33merror_message\u001b[0m=\u001b[32m'Input must be a string.'\u001b[0m,\n", " \u001b[33mfix_value\u001b[0m=\u001b[3;35mNone\u001b[0m,\n", " \u001b[33merror_spans\u001b[0m=\u001b[3;35mNone\u001b[0m,\n", " \u001b[33mmetadata\u001b[0m=\u001b[1m{\u001b[0m\u001b[1m}\u001b[0m,\n", " \u001b[33mvalidated_chunk\u001b[0m=\u001b[3;35mNone\u001b[0m\n", "\u001b[1m)\u001b[0m\n" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "# Usage example:\n", "lowercase_validator = LowercaseValidator(use_local=True)\n", "\n", "\n", "# Test cases for local validator\n", "print(lowercase_validator.validate(value=\"hello world\", metadata={})) # PassResult\n", "print(lowercase_validator.validate(\"Hello World\", {})) # FailResult\n", "print(\n", " lowercase_validator.validate(\"123\", {})\n", ") # PassResult (numbers are considered lowercase)\n", "print(lowercase_validator.validate(123, {})) # FailResult (not a string)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "The best practice would be to use the validator in a Guardrails.Guard()\n" ] }, { "cell_type": "code", "execution_count": 33, "metadata": {}, "outputs": [], "source": [ "# Make some imports\n", "from rich import print\n", "import guardrails as gd\n", "import litellm" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "prompt = \"\"\"Tell me a short snippet about the company Apple.\n", "Make sure the output is all in lowercase. Don't use any capital letters.\"\"\"\n", "\n", "guard = gd.Guard().use(LowercaseValidator(use_local=True))" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Instantiate a guard object\n" ] }, { "cell_type": "code", "execution_count": 35, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
ValidationOutcome(\n",
       "    call_id='12637511488',\n",
       "    raw_llm_output=\"apple inc. is a multinational technology company headquartered in cupertino, california. it \n",
       "designs, manufactures, and markets consumer electronics, software, and online services. some of its most well-known\n",
       "products include the iphone, ipad, mac computers, apple watch, and apple tv. the company is also known for its \n",
       "innovative software platforms like ios, macos, watchos, and tvos, as well as services such as the app store, apple \n",
       "music, and icloud. founded in 1976 by steve jobs, steve wozniak, and ronald wayne, apple has grown to become one of\n",
       "the world's most valuable and influential companies.\",\n",
       "    validated_output=\"apple inc. is a multinational technology company headquartered in cupertino, california. it \n",
       "designs, manufactures, and markets consumer electronics, software, and online services. some of its most well-known\n",
       "products include the iphone, ipad, mac computers, apple watch, and apple tv. the company is also known for its \n",
       "innovative software platforms like ios, macos, watchos, and tvos, as well as services such as the app store, apple \n",
       "music, and icloud. founded in 1976 by steve jobs, steve wozniak, and ronald wayne, apple has grown to become one of\n",
       "the world's most valuable and influential companies.\",\n",
       "    reask=None,\n",
       "    validation_passed=True,\n",
       "    error=None\n",
       ")\n",
       "
\n" ], "text/plain": [ "\u001b[1;35mValidationOutcome\u001b[0m\u001b[1m(\u001b[0m\n", " \u001b[33mcall_id\u001b[0m=\u001b[32m'12637511488'\u001b[0m,\n", " \u001b[33mraw_llm_output\u001b[0m=\u001b[32m\"apple\u001b[0m\u001b[32m inc. is a multinational technology company headquartered in cupertino, california. it \u001b[0m\n", "\u001b[32mdesigns, manufactures, and markets consumer electronics, software, and online services. some of its most well-known\u001b[0m\n", "\u001b[32mproducts include the iphone, ipad, mac computers, apple watch, and apple tv. the company is also known for its \u001b[0m\n", "\u001b[32minnovative software platforms like ios, macos, watchos, and tvos, as well as services such as the app store, apple \u001b[0m\n", "\u001b[32mmusic, and icloud. founded in 1976 by steve jobs, steve wozniak, and ronald wayne, apple has grown to become one of\u001b[0m\n", "\u001b[32mthe world's most valuable and influential companies.\"\u001b[0m,\n", " \u001b[33mvalidated_output\u001b[0m=\u001b[32m\"apple\u001b[0m\u001b[32m inc. is a multinational technology company headquartered in cupertino, california. it \u001b[0m\n", "\u001b[32mdesigns, manufactures, and markets consumer electronics, software, and online services. some of its most well-known\u001b[0m\n", "\u001b[32mproducts include the iphone, ipad, mac computers, apple watch, and apple tv. the company is also known for its \u001b[0m\n", "\u001b[32minnovative software platforms like ios, macos, watchos, and tvos, as well as services such as the app store, apple \u001b[0m\n", "\u001b[32mmusic, and icloud. founded in 1976 by steve jobs, steve wozniak, and ronald wayne, apple has grown to become one of\u001b[0m\n", "\u001b[32mthe world's most valuable and influential companies.\"\u001b[0m,\n", " \u001b[33mreask\u001b[0m=\u001b[3;35mNone\u001b[0m,\n", " \u001b[33mvalidation_passed\u001b[0m=\u001b[3;92mTrue\u001b[0m,\n", " \u001b[33merror\u001b[0m=\u001b[3;35mNone\u001b[0m\n", "\u001b[1m)\u001b[0m\n" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "result = fragment_generator = guard(\n", " litellm.completion,\n", " model=\"gpt-4o\",\n", " messages=[\n", " {\"role\": \"system\", \"content\": \"You are a helpful assistant.\"},\n", " {\"role\": \"user\", \"content\": prompt},\n", " ],\n", " max_tokens=1024,\n", " temperature=0,\n", ")\n", "\n", "print(result)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "If you would like an improved implementation, you can implement the new ErrorSpan feature. This class provides a way to define the span of the error in the input.\n" ] }, { "cell_type": "code", "execution_count": 36, "metadata": {}, "outputs": [], "source": [ "@register_validator(name=\"guardrails/lower_case\", data_type=\"string\")\n", "class LowercaseValidator(Validator):\n", " def __init__(\n", " self,\n", " on_fail: Optional[Callable] = None,\n", " **kwargs,\n", " ):\n", " super().__init__(on_fail=on_fail, **kwargs)\n", " self.rail_alias = \"lowercase\"\n", "\n", " def _validate(self, value: Any, metadata: Dict[str, Any]) -> ValidationResult:\n", " if not isinstance(value, str):\n", " return FailResult(error_message=\"Input must be a string.\", fix_value=None)\n", "\n", " inference_result = self._inference(value)\n", " error_spans = []\n", " if inference_result:\n", " for result in inference_result:\n", " error_spans.append(\n", " ErrorSpan(\n", " start=result[0],\n", " end=result[1],\n", " reason=\"Input must be lowercase.\",\n", " )\n", " )\n", " return FailResult(\n", " error_message=\"Inputs must be lowercase.\",\n", " fix_value=value.lower(),\n", " error_spans=error_spans,\n", " )\n", " return PassResult()\n", "\n", " def _inference_local(self, model_input: str) -> bool:\n", " \"\"\"Implement a function to perform inference on a local machine.\"\"\"\n", " error_spans = []\n", " start = None\n", "\n", " for i, char in enumerate(model_input):\n", " if char.isupper():\n", " if start is None:\n", " start = i\n", " elif start is not None:\n", " error_spans.append((start, i - 1))\n", " start = None\n", "\n", " if start is not None:\n", " error_spans.append((start, len(model_input) - 1))\n", " return error_spans\n", "\n", " def _inference_remote(self, model_input: str) -> bool:\n", " \"\"\"Implement a function that will build a request and perform inference on a\n", " remote machine. This is not required if you will always use local mode.\n", " \"\"\"\n", " response = requests.post(self.validation_endpoint, json={\"inputs\": model_input})\n", " if response.status_code != 200:\n", " raise Exception(\n", " f\"Remote inference failed with status code {response.status_code}\"\n", " )\n", "\n", " return response.json().get(\"is_lowercase\", False)" ] }, { "cell_type": "code", "execution_count": 37, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
ValidationOutcome(\n",
       "    call_id='12637505888',\n",
       "    raw_llm_output=\"APPLE INC. IS A MULTINATIONAL TECHNOLOGY COMPANY HEADQUARTERED IN CUPERTINO, CALIFORNIA. \n",
       "FOUNDED BY STEVE JOBS, STEVE WOZNIAK, AND RONALD WAYNE IN 1976, APPLE IS RENOWNED FOR ITS INNOVATIVE PRODUCTS SUCH \n",
       "AS THE IPHONE, IPAD, MAC COMPUTERS, APPLE WATCH, AND APPLE TV. THE COMPANY IS ALSO KNOWN FOR ITS SOFTWARE \n",
       "ECOSYSTEM, INCLUDING IOS, MACOS, WATCHOS, AND SERVICES LIKE THE APP STORE, APPLE MUSIC, AND ICLOUD. APPLE IS ONE OF\n",
       "THE WORLD'S MOST VALUABLE COMPANIES AND A LEADER IN THE TECHNOLOGY INDUSTRY.\",\n",
       "    validated_output=\"APPLE INC. IS A MULTINATIONAL TECHNOLOGY COMPANY HEADQUARTERED IN CUPERTINO, CALIFORNIA. \n",
       "FOUNDED BY STEVE JOBS, STEVE WOZNIAK, AND RONALD WAYNE IN 1976, APPLE IS RENOWNED FOR ITS INNOVATIVE PRODUCTS SUCH \n",
       "AS THE IPHONE, IPAD, MAC COMPUTERS, APPLE WATCH, AND APPLE TV. THE COMPANY IS ALSO KNOWN FOR ITS SOFTWARE \n",
       "ECOSYSTEM, INCLUDING IOS, MACOS, WATCHOS, AND SERVICES LIKE THE APP STORE, APPLE MUSIC, AND ICLOUD. APPLE IS ONE OF\n",
       "THE WORLD'S MOST VALUABLE COMPANIES AND A LEADER IN THE TECHNOLOGY INDUSTRY.\",\n",
       "    reask=None,\n",
       "    validation_passed=False,\n",
       "    error=None\n",
       ")\n",
       "
\n" ], "text/plain": [ "\u001b[1;35mValidationOutcome\u001b[0m\u001b[1m(\u001b[0m\n", " \u001b[33mcall_id\u001b[0m=\u001b[32m'12637505888'\u001b[0m,\n", " \u001b[33mraw_llm_output\u001b[0m=\u001b[32m\"APPLE\u001b[0m\u001b[32m INC. IS A MULTINATIONAL TECHNOLOGY COMPANY HEADQUARTERED IN CUPERTINO, CALIFORNIA. \u001b[0m\n", "\u001b[32mFOUNDED BY STEVE JOBS, STEVE WOZNIAK, AND RONALD WAYNE IN 1976, APPLE IS RENOWNED FOR ITS INNOVATIVE PRODUCTS SUCH \u001b[0m\n", "\u001b[32mAS THE IPHONE, IPAD, MAC COMPUTERS, APPLE WATCH, AND APPLE TV. THE COMPANY IS ALSO KNOWN FOR ITS SOFTWARE \u001b[0m\n", "\u001b[32mECOSYSTEM, INCLUDING IOS, MACOS, WATCHOS, AND SERVICES LIKE THE APP STORE, APPLE MUSIC, AND ICLOUD. APPLE IS ONE OF\u001b[0m\n", "\u001b[32mTHE WORLD'S MOST VALUABLE COMPANIES AND A LEADER IN THE TECHNOLOGY INDUSTRY.\"\u001b[0m,\n", " \u001b[33mvalidated_output\u001b[0m=\u001b[32m\"APPLE\u001b[0m\u001b[32m INC. IS A MULTINATIONAL TECHNOLOGY COMPANY HEADQUARTERED IN CUPERTINO, CALIFORNIA. \u001b[0m\n", "\u001b[32mFOUNDED BY STEVE JOBS, STEVE WOZNIAK, AND RONALD WAYNE IN 1976, APPLE IS RENOWNED FOR ITS INNOVATIVE PRODUCTS SUCH \u001b[0m\n", "\u001b[32mAS THE IPHONE, IPAD, MAC COMPUTERS, APPLE WATCH, AND APPLE TV. THE COMPANY IS ALSO KNOWN FOR ITS SOFTWARE \u001b[0m\n", "\u001b[32mECOSYSTEM, INCLUDING IOS, MACOS, WATCHOS, AND SERVICES LIKE THE APP STORE, APPLE MUSIC, AND ICLOUD. APPLE IS ONE OF\u001b[0m\n", "\u001b[32mTHE WORLD'S MOST VALUABLE COMPANIES AND A LEADER IN THE TECHNOLOGY INDUSTRY.\"\u001b[0m,\n", " \u001b[33mreask\u001b[0m=\u001b[3;35mNone\u001b[0m,\n", " \u001b[33mvalidation_passed\u001b[0m=\u001b[3;91mFalse\u001b[0m,\n", " \u001b[33merror\u001b[0m=\u001b[3;35mNone\u001b[0m\n", "\u001b[1m)\u001b[0m\n" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "guard = gd.Guard().use(LowercaseValidator, use_local=True)\n", "prompt = \"\"\"Tell me a short snippet about the company Apple. Make it all uppercase\"\"\"\n", "\n", "result = guard(\n", " litellm.completion,\n", " prompt=prompt,\n", " model=\"gpt-4o\",\n", " messages=[\n", " {\"role\": \"system\", \"content\": \"You are a helpful assistant.\"},\n", " {\"role\": \"user\", \"content\": prompt},\n", " ],\n", " max_tokens=1024,\n", " temperature=0,\n", ")\n", "print(result)" ] } ], "metadata": { "kernelspec": { "display_name": ".venv", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.11.8" } }, "nbformat": 4, "nbformat_minor": 2 }