504 lines
29 KiB
Plaintext
504 lines
29 KiB
Plaintext
{
|
|
"cells": [
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"# Create Custom Validators\n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"To make the implementation of custom validators easier, we have provided [an interface in the OSS](https://github.com/guardrails-ai/guardrails/blob/main/guardrails/validator_base.py).\n",
|
|
"\n",
|
|
"There are a few key steps to get up and running with a custom validator:\n",
|
|
"\n",
|
|
"1. Implementing the validator\n",
|
|
"2. Conforming to the required interface\n",
|
|
"3. Running Locally/Submitting to the Validator Hub\n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"First, let's build out a simple custom validator. This validator will check if all text in the input is lowercase.\n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 31,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"from typing import Any, Dict\n",
|
|
"import requests\n",
|
|
"from guardrails.validator_base import (\n",
|
|
" FailResult,\n",
|
|
" PassResult,\n",
|
|
" ValidationResult,\n",
|
|
" Validator,\n",
|
|
" register_validator,\n",
|
|
" ErrorSpan,\n",
|
|
")\n",
|
|
"from typing import Optional, Callable\n",
|
|
"from rich import print\n",
|
|
"\n",
|
|
"\n",
|
|
"@register_validator(name=\"guardrails/lower_case\", data_type=\"string\")\n",
|
|
"class LowercaseValidator(Validator):\n",
|
|
" def __init__(\n",
|
|
" self,\n",
|
|
" on_fail: Optional[Callable] = None,\n",
|
|
" **kwargs,\n",
|
|
" ):\n",
|
|
" super().__init__(on_fail=on_fail, **kwargs)\n",
|
|
" self.rail_alias = \"lowercase\"\n",
|
|
"\n",
|
|
" def _validate(self, value: Any, metadata: Dict[str, Any]) -> ValidationResult:\n",
|
|
" if not isinstance(value, str):\n",
|
|
" return FailResult(\n",
|
|
" metadata=metadata,\n",
|
|
" error_message=\"Input must be a string.\",\n",
|
|
" fix_value=None,\n",
|
|
" )\n",
|
|
"\n",
|
|
" inference_result = self._inference(value)\n",
|
|
"\n",
|
|
" if inference_result:\n",
|
|
" return PassResult()\n",
|
|
" else:\n",
|
|
" return FailResult(\n",
|
|
" metadata=metadata,\n",
|
|
" error_message=\"Input must be lowercase.\",\n",
|
|
" fix_value=value.lower(),\n",
|
|
" )\n",
|
|
"\n",
|
|
" def _inference_local(self, model_input: str) -> bool:\n",
|
|
" \"\"\"Implement a function to perform inference on a local machine.\"\"\"\n",
|
|
" return model_input.islower()\n",
|
|
"\n",
|
|
" def _inference_remote(self, model_input: str) -> bool:\n",
|
|
" \"\"\"Implement a function that will build a request and perform inference on a\n",
|
|
" remote machine. This is not required if you will always use local mode.\n",
|
|
" \"\"\"\n",
|
|
" response = requests.post(self.validation_endpoint, json={\"inputs\": model_input})\n",
|
|
" if response.status_code != 200:\n",
|
|
" raise Exception(\n",
|
|
" f\"Remote inference failed with status code {response.status_code}\"\n",
|
|
" )\n",
|
|
"\n",
|
|
" return response.json().get(\"is_lowercase\", False)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"# Simple usage running locally\n",
|
|
"\n",
|
|
"If this validator was stored locally in your codebase, you would want to import it\n",
|
|
"\n",
|
|
"```bash\n",
|
|
"from lowercase_validator import LowercaseValidator\n",
|
|
"```\n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 32,
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"text/html": [
|
|
"<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">PassResult</span><span style=\"font-weight: bold\">(</span>\n",
|
|
" <span style=\"color: #808000; text-decoration-color: #808000\">outcome</span>=<span style=\"color: #008000; text-decoration-color: #008000\">'pass'</span>,\n",
|
|
" <span style=\"color: #808000; text-decoration-color: #808000\">value_override</span>=<span style=\"font-weight: bold\"><</span><span style=\"color: #ff00ff; text-decoration-color: #ff00ff; font-weight: bold\">class</span><span style=\"color: #000000; text-decoration-color: #000000\"> </span><span style=\"color: #008000; text-decoration-color: #008000\">'guardrails.classes.validation.validation_result.PassResult.ValueOverrideSentinel'</span><span style=\"font-weight: bold\">></span>,\n",
|
|
" <span style=\"color: #808000; text-decoration-color: #808000\">metadata</span>=<span style=\"color: #800080; text-decoration-color: #800080; font-style: italic\">None</span>,\n",
|
|
" <span style=\"color: #808000; text-decoration-color: #808000\">validated_chunk</span>=<span style=\"color: #800080; text-decoration-color: #800080; font-style: italic\">None</span>\n",
|
|
"<span style=\"font-weight: bold\">)</span>\n",
|
|
"</pre>\n"
|
|
],
|
|
"text/plain": [
|
|
"\u001b[1;35mPassResult\u001b[0m\u001b[1m(\u001b[0m\n",
|
|
" \u001b[33moutcome\u001b[0m=\u001b[32m'pass'\u001b[0m,\n",
|
|
" \u001b[33mvalue_override\u001b[0m=\u001b[1m<\u001b[0m\u001b[1;95mclass\u001b[0m\u001b[39m \u001b[0m\u001b[32m'guardrails.classes.validation.validation_result.PassResult.ValueOverrideSentinel'\u001b[0m\u001b[1m>\u001b[0m,\n",
|
|
" \u001b[33mmetadata\u001b[0m=\u001b[3;35mNone\u001b[0m,\n",
|
|
" \u001b[33mvalidated_chunk\u001b[0m=\u001b[3;35mNone\u001b[0m\n",
|
|
"\u001b[1m)\u001b[0m\n"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"text/html": [
|
|
"<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">FailResult</span><span style=\"font-weight: bold\">(</span>\n",
|
|
" <span style=\"color: #808000; text-decoration-color: #808000\">outcome</span>=<span style=\"color: #008000; text-decoration-color: #008000\">'fail'</span>,\n",
|
|
" <span style=\"color: #808000; text-decoration-color: #808000\">error_message</span>=<span style=\"color: #008000; text-decoration-color: #008000\">'Input must be lowercase.'</span>,\n",
|
|
" <span style=\"color: #808000; text-decoration-color: #808000\">fix_value</span>=<span style=\"color: #008000; text-decoration-color: #008000\">'hello world'</span>,\n",
|
|
" <span style=\"color: #808000; text-decoration-color: #808000\">error_spans</span>=<span style=\"color: #800080; text-decoration-color: #800080; font-style: italic\">None</span>,\n",
|
|
" <span style=\"color: #808000; text-decoration-color: #808000\">metadata</span>=<span style=\"font-weight: bold\">{}</span>,\n",
|
|
" <span style=\"color: #808000; text-decoration-color: #808000\">validated_chunk</span>=<span style=\"color: #800080; text-decoration-color: #800080; font-style: italic\">None</span>\n",
|
|
"<span style=\"font-weight: bold\">)</span>\n",
|
|
"</pre>\n"
|
|
],
|
|
"text/plain": [
|
|
"\u001b[1;35mFailResult\u001b[0m\u001b[1m(\u001b[0m\n",
|
|
" \u001b[33moutcome\u001b[0m=\u001b[32m'fail'\u001b[0m,\n",
|
|
" \u001b[33merror_message\u001b[0m=\u001b[32m'Input must be lowercase.'\u001b[0m,\n",
|
|
" \u001b[33mfix_value\u001b[0m=\u001b[32m'hello world'\u001b[0m,\n",
|
|
" \u001b[33merror_spans\u001b[0m=\u001b[3;35mNone\u001b[0m,\n",
|
|
" \u001b[33mmetadata\u001b[0m=\u001b[1m{\u001b[0m\u001b[1m}\u001b[0m,\n",
|
|
" \u001b[33mvalidated_chunk\u001b[0m=\u001b[3;35mNone\u001b[0m\n",
|
|
"\u001b[1m)\u001b[0m\n"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"text/html": [
|
|
"<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">FailResult</span><span style=\"font-weight: bold\">(</span>\n",
|
|
" <span style=\"color: #808000; text-decoration-color: #808000\">outcome</span>=<span style=\"color: #008000; text-decoration-color: #008000\">'fail'</span>,\n",
|
|
" <span style=\"color: #808000; text-decoration-color: #808000\">error_message</span>=<span style=\"color: #008000; text-decoration-color: #008000\">'Input must be lowercase.'</span>,\n",
|
|
" <span style=\"color: #808000; text-decoration-color: #808000\">fix_value</span>=<span style=\"color: #008000; text-decoration-color: #008000\">'123'</span>,\n",
|
|
" <span style=\"color: #808000; text-decoration-color: #808000\">error_spans</span>=<span style=\"color: #800080; text-decoration-color: #800080; font-style: italic\">None</span>,\n",
|
|
" <span style=\"color: #808000; text-decoration-color: #808000\">metadata</span>=<span style=\"font-weight: bold\">{}</span>,\n",
|
|
" <span style=\"color: #808000; text-decoration-color: #808000\">validated_chunk</span>=<span style=\"color: #800080; text-decoration-color: #800080; font-style: italic\">None</span>\n",
|
|
"<span style=\"font-weight: bold\">)</span>\n",
|
|
"</pre>\n"
|
|
],
|
|
"text/plain": [
|
|
"\u001b[1;35mFailResult\u001b[0m\u001b[1m(\u001b[0m\n",
|
|
" \u001b[33moutcome\u001b[0m=\u001b[32m'fail'\u001b[0m,\n",
|
|
" \u001b[33merror_message\u001b[0m=\u001b[32m'Input must be lowercase.'\u001b[0m,\n",
|
|
" \u001b[33mfix_value\u001b[0m=\u001b[32m'123'\u001b[0m,\n",
|
|
" \u001b[33merror_spans\u001b[0m=\u001b[3;35mNone\u001b[0m,\n",
|
|
" \u001b[33mmetadata\u001b[0m=\u001b[1m{\u001b[0m\u001b[1m}\u001b[0m,\n",
|
|
" \u001b[33mvalidated_chunk\u001b[0m=\u001b[3;35mNone\u001b[0m\n",
|
|
"\u001b[1m)\u001b[0m\n"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"text/html": [
|
|
"<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">FailResult</span><span style=\"font-weight: bold\">(</span>\n",
|
|
" <span style=\"color: #808000; text-decoration-color: #808000\">outcome</span>=<span style=\"color: #008000; text-decoration-color: #008000\">'fail'</span>,\n",
|
|
" <span style=\"color: #808000; text-decoration-color: #808000\">error_message</span>=<span style=\"color: #008000; text-decoration-color: #008000\">'Input must be a string.'</span>,\n",
|
|
" <span style=\"color: #808000; text-decoration-color: #808000\">fix_value</span>=<span style=\"color: #800080; text-decoration-color: #800080; font-style: italic\">None</span>,\n",
|
|
" <span style=\"color: #808000; text-decoration-color: #808000\">error_spans</span>=<span style=\"color: #800080; text-decoration-color: #800080; font-style: italic\">None</span>,\n",
|
|
" <span style=\"color: #808000; text-decoration-color: #808000\">metadata</span>=<span style=\"font-weight: bold\">{}</span>,\n",
|
|
" <span style=\"color: #808000; text-decoration-color: #808000\">validated_chunk</span>=<span style=\"color: #800080; text-decoration-color: #800080; font-style: italic\">None</span>\n",
|
|
"<span style=\"font-weight: bold\">)</span>\n",
|
|
"</pre>\n"
|
|
],
|
|
"text/plain": [
|
|
"\u001b[1;35mFailResult\u001b[0m\u001b[1m(\u001b[0m\n",
|
|
" \u001b[33moutcome\u001b[0m=\u001b[32m'fail'\u001b[0m,\n",
|
|
" \u001b[33merror_message\u001b[0m=\u001b[32m'Input must be a string.'\u001b[0m,\n",
|
|
" \u001b[33mfix_value\u001b[0m=\u001b[3;35mNone\u001b[0m,\n",
|
|
" \u001b[33merror_spans\u001b[0m=\u001b[3;35mNone\u001b[0m,\n",
|
|
" \u001b[33mmetadata\u001b[0m=\u001b[1m{\u001b[0m\u001b[1m}\u001b[0m,\n",
|
|
" \u001b[33mvalidated_chunk\u001b[0m=\u001b[3;35mNone\u001b[0m\n",
|
|
"\u001b[1m)\u001b[0m\n"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
}
|
|
],
|
|
"source": [
|
|
"# Usage example:\n",
|
|
"lowercase_validator = LowercaseValidator(use_local=True)\n",
|
|
"\n",
|
|
"\n",
|
|
"# Test cases for local validator\n",
|
|
"print(lowercase_validator.validate(value=\"hello world\", metadata={})) # PassResult\n",
|
|
"print(lowercase_validator.validate(\"Hello World\", {})) # FailResult\n",
|
|
"print(\n",
|
|
" lowercase_validator.validate(\"123\", {})\n",
|
|
") # PassResult (numbers are considered lowercase)\n",
|
|
"print(lowercase_validator.validate(123, {})) # FailResult (not a string)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"The best practice would be to use the validator in a Guardrails.Guard()\n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 33,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"# Make some imports\n",
|
|
"from rich import print\n",
|
|
"import guardrails as gd\n",
|
|
"import litellm"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"prompt = \"\"\"Tell me a short snippet about the company Apple.\n",
|
|
"Make sure the output is all in lowercase. Don't use any capital letters.\"\"\"\n",
|
|
"\n",
|
|
"guard = gd.Guard().use(LowercaseValidator(use_local=True))"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"Instantiate a guard object\n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 35,
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"text/html": [
|
|
"<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">ValidationOutcome</span><span style=\"font-weight: bold\">(</span>\n",
|
|
" <span style=\"color: #808000; text-decoration-color: #808000\">call_id</span>=<span style=\"color: #008000; text-decoration-color: #008000\">'12637511488'</span>,\n",
|
|
" <span style=\"color: #808000; text-decoration-color: #808000\">raw_llm_output</span>=<span style=\"color: #008000; text-decoration-color: #008000\">\"apple inc. is a multinational technology company headquartered in cupertino, california. it </span>\n",
|
|
"<span style=\"color: #008000; text-decoration-color: #008000\">designs, manufactures, and markets consumer electronics, software, and online services. some of its most well-known</span>\n",
|
|
"<span style=\"color: #008000; text-decoration-color: #008000\">products include the iphone, ipad, mac computers, apple watch, and apple tv. the company is also known for its </span>\n",
|
|
"<span style=\"color: #008000; text-decoration-color: #008000\">innovative software platforms like ios, macos, watchos, and tvos, as well as services such as the app store, apple </span>\n",
|
|
"<span style=\"color: #008000; text-decoration-color: #008000\">music, and icloud. founded in 1976 by steve jobs, steve wozniak, and ronald wayne, apple has grown to become one of</span>\n",
|
|
"<span style=\"color: #008000; text-decoration-color: #008000\">the world's most valuable and influential companies.\"</span>,\n",
|
|
" <span style=\"color: #808000; text-decoration-color: #808000\">validated_output</span>=<span style=\"color: #008000; text-decoration-color: #008000\">\"apple inc. is a multinational technology company headquartered in cupertino, california. it </span>\n",
|
|
"<span style=\"color: #008000; text-decoration-color: #008000\">designs, manufactures, and markets consumer electronics, software, and online services. some of its most well-known</span>\n",
|
|
"<span style=\"color: #008000; text-decoration-color: #008000\">products include the iphone, ipad, mac computers, apple watch, and apple tv. the company is also known for its </span>\n",
|
|
"<span style=\"color: #008000; text-decoration-color: #008000\">innovative software platforms like ios, macos, watchos, and tvos, as well as services such as the app store, apple </span>\n",
|
|
"<span style=\"color: #008000; text-decoration-color: #008000\">music, and icloud. founded in 1976 by steve jobs, steve wozniak, and ronald wayne, apple has grown to become one of</span>\n",
|
|
"<span style=\"color: #008000; text-decoration-color: #008000\">the world's most valuable and influential companies.\"</span>,\n",
|
|
" <span style=\"color: #808000; text-decoration-color: #808000\">reask</span>=<span style=\"color: #800080; text-decoration-color: #800080; font-style: italic\">None</span>,\n",
|
|
" <span style=\"color: #808000; text-decoration-color: #808000\">validation_passed</span>=<span style=\"color: #00ff00; text-decoration-color: #00ff00; font-style: italic\">True</span>,\n",
|
|
" <span style=\"color: #808000; text-decoration-color: #808000\">error</span>=<span style=\"color: #800080; text-decoration-color: #800080; font-style: italic\">None</span>\n",
|
|
"<span style=\"font-weight: bold\">)</span>\n",
|
|
"</pre>\n"
|
|
],
|
|
"text/plain": [
|
|
"\u001b[1;35mValidationOutcome\u001b[0m\u001b[1m(\u001b[0m\n",
|
|
" \u001b[33mcall_id\u001b[0m=\u001b[32m'12637511488'\u001b[0m,\n",
|
|
" \u001b[33mraw_llm_output\u001b[0m=\u001b[32m\"apple\u001b[0m\u001b[32m inc. is a multinational technology company headquartered in cupertino, california. it \u001b[0m\n",
|
|
"\u001b[32mdesigns, manufactures, and markets consumer electronics, software, and online services. some of its most well-known\u001b[0m\n",
|
|
"\u001b[32mproducts include the iphone, ipad, mac computers, apple watch, and apple tv. the company is also known for its \u001b[0m\n",
|
|
"\u001b[32minnovative software platforms like ios, macos, watchos, and tvos, as well as services such as the app store, apple \u001b[0m\n",
|
|
"\u001b[32mmusic, and icloud. founded in 1976 by steve jobs, steve wozniak, and ronald wayne, apple has grown to become one of\u001b[0m\n",
|
|
"\u001b[32mthe world's most valuable and influential companies.\"\u001b[0m,\n",
|
|
" \u001b[33mvalidated_output\u001b[0m=\u001b[32m\"apple\u001b[0m\u001b[32m inc. is a multinational technology company headquartered in cupertino, california. it \u001b[0m\n",
|
|
"\u001b[32mdesigns, manufactures, and markets consumer electronics, software, and online services. some of its most well-known\u001b[0m\n",
|
|
"\u001b[32mproducts include the iphone, ipad, mac computers, apple watch, and apple tv. the company is also known for its \u001b[0m\n",
|
|
"\u001b[32minnovative software platforms like ios, macos, watchos, and tvos, as well as services such as the app store, apple \u001b[0m\n",
|
|
"\u001b[32mmusic, and icloud. founded in 1976 by steve jobs, steve wozniak, and ronald wayne, apple has grown to become one of\u001b[0m\n",
|
|
"\u001b[32mthe world's most valuable and influential companies.\"\u001b[0m,\n",
|
|
" \u001b[33mreask\u001b[0m=\u001b[3;35mNone\u001b[0m,\n",
|
|
" \u001b[33mvalidation_passed\u001b[0m=\u001b[3;92mTrue\u001b[0m,\n",
|
|
" \u001b[33merror\u001b[0m=\u001b[3;35mNone\u001b[0m\n",
|
|
"\u001b[1m)\u001b[0m\n"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
}
|
|
],
|
|
"source": [
|
|
"result = fragment_generator = guard(\n",
|
|
" litellm.completion,\n",
|
|
" model=\"gpt-4o\",\n",
|
|
" messages=[\n",
|
|
" {\"role\": \"system\", \"content\": \"You are a helpful assistant.\"},\n",
|
|
" {\"role\": \"user\", \"content\": prompt},\n",
|
|
" ],\n",
|
|
" max_tokens=1024,\n",
|
|
" temperature=0,\n",
|
|
")\n",
|
|
"\n",
|
|
"print(result)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"If you would like an improved implementation, you can implement the new ErrorSpan feature. This class provides a way to define the span of the error in the input.\n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 36,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"@register_validator(name=\"guardrails/lower_case\", data_type=\"string\")\n",
|
|
"class LowercaseValidator(Validator):\n",
|
|
" def __init__(\n",
|
|
" self,\n",
|
|
" on_fail: Optional[Callable] = None,\n",
|
|
" **kwargs,\n",
|
|
" ):\n",
|
|
" super().__init__(on_fail=on_fail, **kwargs)\n",
|
|
" self.rail_alias = \"lowercase\"\n",
|
|
"\n",
|
|
" def _validate(self, value: Any, metadata: Dict[str, Any]) -> ValidationResult:\n",
|
|
" if not isinstance(value, str):\n",
|
|
" return FailResult(error_message=\"Input must be a string.\", fix_value=None)\n",
|
|
"\n",
|
|
" inference_result = self._inference(value)\n",
|
|
" error_spans = []\n",
|
|
" if inference_result:\n",
|
|
" for result in inference_result:\n",
|
|
" error_spans.append(\n",
|
|
" ErrorSpan(\n",
|
|
" start=result[0],\n",
|
|
" end=result[1],\n",
|
|
" reason=\"Input must be lowercase.\",\n",
|
|
" )\n",
|
|
" )\n",
|
|
" return FailResult(\n",
|
|
" error_message=\"Inputs must be lowercase.\",\n",
|
|
" fix_value=value.lower(),\n",
|
|
" error_spans=error_spans,\n",
|
|
" )\n",
|
|
" return PassResult()\n",
|
|
"\n",
|
|
" def _inference_local(self, model_input: str) -> bool:\n",
|
|
" \"\"\"Implement a function to perform inference on a local machine.\"\"\"\n",
|
|
" error_spans = []\n",
|
|
" start = None\n",
|
|
"\n",
|
|
" for i, char in enumerate(model_input):\n",
|
|
" if char.isupper():\n",
|
|
" if start is None:\n",
|
|
" start = i\n",
|
|
" elif start is not None:\n",
|
|
" error_spans.append((start, i - 1))\n",
|
|
" start = None\n",
|
|
"\n",
|
|
" if start is not None:\n",
|
|
" error_spans.append((start, len(model_input) - 1))\n",
|
|
" return error_spans\n",
|
|
"\n",
|
|
" def _inference_remote(self, model_input: str) -> bool:\n",
|
|
" \"\"\"Implement a function that will build a request and perform inference on a\n",
|
|
" remote machine. This is not required if you will always use local mode.\n",
|
|
" \"\"\"\n",
|
|
" response = requests.post(self.validation_endpoint, json={\"inputs\": model_input})\n",
|
|
" if response.status_code != 200:\n",
|
|
" raise Exception(\n",
|
|
" f\"Remote inference failed with status code {response.status_code}\"\n",
|
|
" )\n",
|
|
"\n",
|
|
" return response.json().get(\"is_lowercase\", False)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 37,
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"text/html": [
|
|
"<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">ValidationOutcome</span><span style=\"font-weight: bold\">(</span>\n",
|
|
" <span style=\"color: #808000; text-decoration-color: #808000\">call_id</span>=<span style=\"color: #008000; text-decoration-color: #008000\">'12637505888'</span>,\n",
|
|
" <span style=\"color: #808000; text-decoration-color: #808000\">raw_llm_output</span>=<span style=\"color: #008000; text-decoration-color: #008000\">\"APPLE INC. IS A MULTINATIONAL TECHNOLOGY COMPANY HEADQUARTERED IN CUPERTINO, CALIFORNIA. </span>\n",
|
|
"<span style=\"color: #008000; text-decoration-color: #008000\">FOUNDED BY STEVE JOBS, STEVE WOZNIAK, AND RONALD WAYNE IN 1976, APPLE IS RENOWNED FOR ITS INNOVATIVE PRODUCTS SUCH </span>\n",
|
|
"<span style=\"color: #008000; text-decoration-color: #008000\">AS THE IPHONE, IPAD, MAC COMPUTERS, APPLE WATCH, AND APPLE TV. THE COMPANY IS ALSO KNOWN FOR ITS SOFTWARE </span>\n",
|
|
"<span style=\"color: #008000; text-decoration-color: #008000\">ECOSYSTEM, INCLUDING IOS, MACOS, WATCHOS, AND SERVICES LIKE THE APP STORE, APPLE MUSIC, AND ICLOUD. APPLE IS ONE OF</span>\n",
|
|
"<span style=\"color: #008000; text-decoration-color: #008000\">THE WORLD'S MOST VALUABLE COMPANIES AND A LEADER IN THE TECHNOLOGY INDUSTRY.\"</span>,\n",
|
|
" <span style=\"color: #808000; text-decoration-color: #808000\">validated_output</span>=<span style=\"color: #008000; text-decoration-color: #008000\">\"APPLE INC. IS A MULTINATIONAL TECHNOLOGY COMPANY HEADQUARTERED IN CUPERTINO, CALIFORNIA. </span>\n",
|
|
"<span style=\"color: #008000; text-decoration-color: #008000\">FOUNDED BY STEVE JOBS, STEVE WOZNIAK, AND RONALD WAYNE IN 1976, APPLE IS RENOWNED FOR ITS INNOVATIVE PRODUCTS SUCH </span>\n",
|
|
"<span style=\"color: #008000; text-decoration-color: #008000\">AS THE IPHONE, IPAD, MAC COMPUTERS, APPLE WATCH, AND APPLE TV. THE COMPANY IS ALSO KNOWN FOR ITS SOFTWARE </span>\n",
|
|
"<span style=\"color: #008000; text-decoration-color: #008000\">ECOSYSTEM, INCLUDING IOS, MACOS, WATCHOS, AND SERVICES LIKE THE APP STORE, APPLE MUSIC, AND ICLOUD. APPLE IS ONE OF</span>\n",
|
|
"<span style=\"color: #008000; text-decoration-color: #008000\">THE WORLD'S MOST VALUABLE COMPANIES AND A LEADER IN THE TECHNOLOGY INDUSTRY.\"</span>,\n",
|
|
" <span style=\"color: #808000; text-decoration-color: #808000\">reask</span>=<span style=\"color: #800080; text-decoration-color: #800080; font-style: italic\">None</span>,\n",
|
|
" <span style=\"color: #808000; text-decoration-color: #808000\">validation_passed</span>=<span style=\"color: #ff0000; text-decoration-color: #ff0000; font-style: italic\">False</span>,\n",
|
|
" <span style=\"color: #808000; text-decoration-color: #808000\">error</span>=<span style=\"color: #800080; text-decoration-color: #800080; font-style: italic\">None</span>\n",
|
|
"<span style=\"font-weight: bold\">)</span>\n",
|
|
"</pre>\n"
|
|
],
|
|
"text/plain": [
|
|
"\u001b[1;35mValidationOutcome\u001b[0m\u001b[1m(\u001b[0m\n",
|
|
" \u001b[33mcall_id\u001b[0m=\u001b[32m'12637505888'\u001b[0m,\n",
|
|
" \u001b[33mraw_llm_output\u001b[0m=\u001b[32m\"APPLE\u001b[0m\u001b[32m INC. IS A MULTINATIONAL TECHNOLOGY COMPANY HEADQUARTERED IN CUPERTINO, CALIFORNIA. \u001b[0m\n",
|
|
"\u001b[32mFOUNDED BY STEVE JOBS, STEVE WOZNIAK, AND RONALD WAYNE IN 1976, APPLE IS RENOWNED FOR ITS INNOVATIVE PRODUCTS SUCH \u001b[0m\n",
|
|
"\u001b[32mAS THE IPHONE, IPAD, MAC COMPUTERS, APPLE WATCH, AND APPLE TV. THE COMPANY IS ALSO KNOWN FOR ITS SOFTWARE \u001b[0m\n",
|
|
"\u001b[32mECOSYSTEM, INCLUDING IOS, MACOS, WATCHOS, AND SERVICES LIKE THE APP STORE, APPLE MUSIC, AND ICLOUD. APPLE IS ONE OF\u001b[0m\n",
|
|
"\u001b[32mTHE WORLD'S MOST VALUABLE COMPANIES AND A LEADER IN THE TECHNOLOGY INDUSTRY.\"\u001b[0m,\n",
|
|
" \u001b[33mvalidated_output\u001b[0m=\u001b[32m\"APPLE\u001b[0m\u001b[32m INC. IS A MULTINATIONAL TECHNOLOGY COMPANY HEADQUARTERED IN CUPERTINO, CALIFORNIA. \u001b[0m\n",
|
|
"\u001b[32mFOUNDED BY STEVE JOBS, STEVE WOZNIAK, AND RONALD WAYNE IN 1976, APPLE IS RENOWNED FOR ITS INNOVATIVE PRODUCTS SUCH \u001b[0m\n",
|
|
"\u001b[32mAS THE IPHONE, IPAD, MAC COMPUTERS, APPLE WATCH, AND APPLE TV. THE COMPANY IS ALSO KNOWN FOR ITS SOFTWARE \u001b[0m\n",
|
|
"\u001b[32mECOSYSTEM, INCLUDING IOS, MACOS, WATCHOS, AND SERVICES LIKE THE APP STORE, APPLE MUSIC, AND ICLOUD. APPLE IS ONE OF\u001b[0m\n",
|
|
"\u001b[32mTHE WORLD'S MOST VALUABLE COMPANIES AND A LEADER IN THE TECHNOLOGY INDUSTRY.\"\u001b[0m,\n",
|
|
" \u001b[33mreask\u001b[0m=\u001b[3;35mNone\u001b[0m,\n",
|
|
" \u001b[33mvalidation_passed\u001b[0m=\u001b[3;91mFalse\u001b[0m,\n",
|
|
" \u001b[33merror\u001b[0m=\u001b[3;35mNone\u001b[0m\n",
|
|
"\u001b[1m)\u001b[0m\n"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
}
|
|
],
|
|
"source": [
|
|
"guard = gd.Guard().use(LowercaseValidator, use_local=True)\n",
|
|
"prompt = \"\"\"Tell me a short snippet about the company Apple. Make it all uppercase\"\"\"\n",
|
|
"\n",
|
|
"result = guard(\n",
|
|
" litellm.completion,\n",
|
|
" prompt=prompt,\n",
|
|
" model=\"gpt-4o\",\n",
|
|
" messages=[\n",
|
|
" {\"role\": \"system\", \"content\": \"You are a helpful assistant.\"},\n",
|
|
" {\"role\": \"user\", \"content\": prompt},\n",
|
|
" ],\n",
|
|
" max_tokens=1024,\n",
|
|
" temperature=0,\n",
|
|
")\n",
|
|
"print(result)"
|
|
]
|
|
}
|
|
],
|
|
"metadata": {
|
|
"kernelspec": {
|
|
"display_name": ".venv",
|
|
"language": "python",
|
|
"name": "python3"
|
|
},
|
|
"language_info": {
|
|
"codemirror_mode": {
|
|
"name": "ipython",
|
|
"version": 3
|
|
},
|
|
"file_extension": ".py",
|
|
"mimetype": "text/x-python",
|
|
"name": "python",
|
|
"nbconvert_exporter": "python",
|
|
"pygments_lexer": "ipython3",
|
|
"version": "3.11.8"
|
|
}
|
|
},
|
|
"nbformat": 4,
|
|
"nbformat_minor": 2
|
|
}
|