119 lines
3.6 KiB
Python
119 lines
3.6 KiB
Python
|
from typing import Callable
|
||
|
|
|
||
|
|
from openai import OpenAI
|
||
|
|
|
||
|
|
from ..processing.validators import Validator
|
||
|
|
from ..core.client import Instructor
|
||
|
|
|
||
|
|
|
||
|
|
def llm_validator(
|
||
|
|
statement: str,
|
||
|
|
client: Instructor,
|
||
|
|
allow_override: bool = False,
|
||
|
|
model: str = "gpt-3.5-turbo",
|
||
|
|
temperature: float = 0,
|
||
|
|
) -> Callable[[str], str]:
|
||
|
|
"""
|
||
|
|
Create a validator that uses the LLM to validate an attribute
|
||
|
|
|
||
|
|
## Usage
|
||
|
|
|
||
|
|
```python
|
||
|
|
from instructor import llm_validator
|
||
|
|
from pydantic import BaseModel, Field, field_validator
|
||
|
|
|
||
|
|
class User(BaseModel):
|
||
|
|
name: str = Annotated[str, llm_validator("The name must be a full name all lowercase")
|
||
|
|
age: int = Field(description="The age of the person")
|
||
|
|
|
||
|
|
try:
|
||
|
|
user = User(name="Jason Liu", age=20)
|
||
|
|
except ValidationError as e:
|
||
|
|
print(e)
|
||
|
|
```
|
||
|
|
|
||
|
|
```
|
||
|
|
1 validation error for User
|
||
|
|
name
|
||
|
|
The name is valid but not all lowercase (type=value_error.llm_validator)
|
||
|
|
```
|
||
|
|
|
||
|
|
Note that there, the error message is written by the LLM, and the error type is `value_error.llm_validator`.
|
||
|
|
|
||
|
|
Parameters:
|
||
|
|
statement (str): The statement to validate
|
||
|
|
model (str): The LLM to use for validation (default: "gpt-4o-mini")
|
||
|
|
temperature (float): The temperature to use for the LLM (default: 0)
|
||
|
|
client (OpenAI): The OpenAI client to use (default: None)
|
||
|
|
"""
|
||
|
|
|
||
|
|
def llm(v: str) -> str:
|
||
|
|
resp = client.chat.completions.create(
|
||
|
|
response_model=Validator,
|
||
|
|
messages=[
|
||
|
|
{
|
||
|
|
"role": "system",
|
||
|
|
"content": "You are a world class validation model. Capable to determine if the following value is valid for the statement, if it is not, explain why and suggest a new value.",
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"role": "user",
|
||
|
|
"content": f"Does `{v}` follow the rules: {statement}",
|
||
|
|
},
|
||
|
|
],
|
||
|
|
model=model,
|
||
|
|
temperature=temperature,
|
||
|
|
)
|
||
|
|
|
||
|
|
# If the value is not valid but we allow overrides and the LLM
|
||
|
|
# suggested a corrected value, return the fixed value instead of
|
||
|
|
# raising an assertion error.
|
||
|
|
if not resp.is_valid:
|
||
|
|
if allow_override and resp.fixed_value is not None:
|
||
|
|
return resp.fixed_value
|
||
|
|
assert resp.is_valid, resp.reason
|
||
|
|
|
||
|
|
return v
|
||
|
|
|
||
|
|
return llm
|
||
|
|
|
||
|
|
|
||
|
|
def openai_moderation(client: OpenAI) -> Callable[[str], str]:
|
||
|
|
"""
|
||
|
|
Validates a message using OpenAI moderation model.
|
||
|
|
|
||
|
|
Should only be used for monitoring inputs and outputs of OpenAI APIs
|
||
|
|
Other use cases are disallowed as per:
|
||
|
|
https://platform.openai.com/docs/guides/moderation/overview
|
||
|
|
|
||
|
|
Example:
|
||
|
|
```python
|
||
|
|
from instructor import OpenAIModeration
|
||
|
|
|
||
|
|
class Response(BaseModel):
|
||
|
|
message: Annotated[str, AfterValidator(OpenAIModeration(openai_client=client))]
|
||
|
|
|
||
|
|
Response(message="I hate you")
|
||
|
|
```
|
||
|
|
|
||
|
|
```
|
||
|
|
ValidationError: 1 validation error for Response
|
||
|
|
message
|
||
|
|
Value error, `I hate you.` was flagged for ['harassment'] [type=value_error, input_value='I hate you.', input_type=str]
|
||
|
|
```
|
||
|
|
|
||
|
|
client (OpenAI): The OpenAI client to use, must be sync (default: None)
|
||
|
|
"""
|
||
|
|
|
||
|
|
def validate_message_with_openai_mod(v: str) -> str:
|
||
|
|
response = client.moderations.create(input=v)
|
||
|
|
out = response.results[0]
|
||
|
|
cats = out.categories.model_dump()
|
||
|
|
if out.flagged:
|
||
|
|
raise ValueError(
|
||
|
|
f"`{v}` was flagged for {', '.join(cat for cat in cats if cats[cat])}"
|
||
|
|
)
|
||
|
|
|
||
|
|
return v
|
||
|
|
|
||
|
|
return validate_message_with_openai_mod
|