AI/참고/guardrails-main/tests/integration_tests/test_python_rail.py

import json
from datetime import date, time
from typing import List, Literal, Union

import pytest
from pydantic import BaseModel, Field, field_validator, model_validator

import guardrails as gd
from guardrails import Validator, register_validator
from guardrails.classes.llm.llm_response import LLMResponse
from guardrails.types import OnFailAction
from guardrails_ai.types import (
    FailResult,
    PassResult,
    ValidationResult,
)
from tests.integration_tests.test_assets.validators import ValidLength, TwoWords

from .test_assets import python_rail, string


@register_validator(name="is-valid-director", data_type="string")
class IsValidDirector(Validator):
    def validate(self, value, metadata) -> ValidationResult:
        valid_names = [
            "Christopher Nolan",
            "Steven Spielberg",
            "Martin Scorsese",
            "Quentin Tarantino",
            "James Cameron",
        ]
        if value not in valid_names:
            return FailResult(
                error_message=f"Value {value} is not a valid director name. "
                f"Valid choices are {valid_names}.",
            )
        return PassResult()


class BoxOfficeRevenue(BaseModel):
    revenue_type: Literal["box_office"]
    gross: float
    opening_weekend: float

    # Field-level validation using Pydantic (not Guardrails)
    @field_validator("gross")
    def validate_gross(cls, gross):
        if gross <= 0:
            raise ValueError("Gross revenue must be a positive value")
        return gross


class StreamingRevenue(BaseModel):
    revenue_type: Literal["streaming"]
    subscriptions: int
    subscription_fee: float


class Details(BaseModel):
    release_date: date
    duration: time
    budget: float
    is_sequel: bool = Field(default=False)
    website: str = Field(
        json_schema_extra={
            "validators": [ValidLength(min=9, max=100, on_fail=OnFailAction.REASK)]
        }
    )

    # Root-level validation using Pydantic (Not in Guardrails)
    @model_validator(mode="before")
    def validate_budget_and_gross(cls, values):
        budget = values.get("budget")
        revenue = values.get("revenue")
        if revenue["revenue_type"] == "box_office":
            gross = revenue["gross"]
            if budget >= gross:
                raise ValueError("Budget must be less than gross revenue")
        return values

    contact_email: str
    revenue: Union[BoxOfficeRevenue, StreamingRevenue] = Field(
        ..., discriminator="revenue_type"
    )


class Movie(BaseModel):
    rank: int
    title: str
    details: Details


class Director(BaseModel):
    name: str = Field(validators=[IsValidDirector()])
    movies: List[Movie]


def test_python_rail(mocker):
    mock_invoke_llm = mocker.patch(
        "guardrails.llm_providers.LiteLLMCallable._invoke_llm"
    )
    mock_invoke_llm.side_effect = [
        LLMResponse(
            output=python_rail.LLM_OUTPUT_1_FAIL_GUARDRAILS_VALIDATION,
            prompt_token_count=123,
            response_token_count=1234,
        ),
        LLMResponse(
            output=python_rail.LLM_OUTPUT_2_SUCCEED_GUARDRAILS_BUT_FAIL_PYDANTIC_VALIDATION,
            prompt_token_count=123,
            response_token_count=1234,
        ),
    ]

    guard = gd.Guard.for_pydantic(
        output_class=Director,
        messages=[
            {
                "role": "system",
                "content": "\nYou are a helpful assistant"
                " only capable of communicating"
                " with valid JSON, and no other"
                " text.\n${gr.xml_suffix_prompt_examples}",
            },
            {
                "role": "user",
                "content": "Provide detailed information"
                " about the top 5 grossing movies from"
                " ${director} including release date, duration, budget, whether "
                "it's a sequel, website, and contact email.\n"
                "${gr.xml_suffix_without_examples}",
            },
        ],
    )

    # Guardrails runs validation and fixes the first failing output through reasking
    final_output = guard(
        model="gpt-3.5-turbo",
        prompt_params={"director": "Christopher Nolan"},
        num_reasks=2,
        full_schema_reask=False,
    )

    # Assertions are made on the guard state object.
    expected_gd_output = json.loads(
        python_rail.LLM_OUTPUT_2_SUCCEED_GUARDRAILS_BUT_FAIL_PYDANTIC_VALIDATION
    )
    assert final_output.validated_output == expected_gd_output

    call = guard.history.first

    # Check that the guard state object has the correct number of re-asks.
    assert call.iterations.length == 2

    assert (
        call.compiled_messages[1]["content"]
        == python_rail.COMPILED_PROMPT_1_PYDANTIC_2_WITHOUT_INSTRUCTIONS
    )

    assert (
        call.iterations.first.raw_output
        == python_rail.LLM_OUTPUT_1_FAIL_GUARDRAILS_VALIDATION
    )

    assert call.iterations.last.inputs.messages[1]["content"] == gd.Prompt(
        python_rail.COMPILED_PROMPT_2_WITHOUT_INSTRUCTIONS
    )
    # Same as above
    assert (
        call.reask_messages[0][1]["content"]
        == python_rail.COMPILED_PROMPT_2_WITHOUT_INSTRUCTIONS
    )
    assert (
        call.raw_outputs.last
        == python_rail.LLM_OUTPUT_2_SUCCEED_GUARDRAILS_BUT_FAIL_PYDANTIC_VALIDATION
    )

    with pytest.raises(ValueError):
        Director.model_validate_json(
            python_rail.LLM_OUTPUT_2_SUCCEED_GUARDRAILS_BUT_FAIL_PYDANTIC_VALIDATION
        )
    Director.model_validate_json(
        python_rail.LLM_OUTPUT_3_SUCCEED_GUARDRAILS_AND_PYDANTIC
    )


def test_python_string(mocker):
    """Test single string (non-JSON) generation via pydantic with re-asking."""
    # mocker.patch("guardrails.llm_providers.LiteLLMCallable", new=MockLiteLLMCallable)
    mock_invoke_llm = mocker.patch(
        "guardrails.llm_providers.LiteLLMCallable._invoke_llm",
    )
    mock_invoke_llm.side_effect = [
        LLMResponse(
            output=string.LLM_OUTPUT,
            prompt_token_count=123,
            response_token_count=1234,
        ),
        LLMResponse(
            output=string.LLM_OUTPUT_REASK,
            prompt_token_count=123,
            response_token_count=1234,
        ),
    ]
    validators = [TwoWords(on_fail=OnFailAction.REASK)]
    description = "Name for the pizza"
    instructions = """
You are a helpful assistant, and you are helping me come up with a name for a pizza.

${gr.complete_string_suffix}
"""

    prompt = """
Given the following ingredients, what would you call this pizza?

${ingredients}
"""

    guard = gd.Guard.for_string(
        validators,
        string_description=description,
        messages=[
            {"role": "system", "content": instructions},
            {"role": "user", "content": prompt},
        ],
    )
    final_output = guard(
        model="gpt-3.5-turbo",
        prompt_params={"ingredients": "tomato, cheese, sour cream"},
        num_reasks=1,
        max_tokens=100,
    )

    assert final_output.validated_output == string.LLM_OUTPUT_REASK

    call = guard.history.first

    # Check that the guard state object has the correct number of re-asks.
    assert call.iterations.length == 2

    # For orginal prompt and output
    assert call.compiled_messages[0]["content"] == string.COMPILED_INSTRUCTIONS
    assert call.compiled_messages[1]["content"] == string.COMPILED_PROMPT
    assert call.iterations.first.raw_output == string.LLM_OUTPUT
    assert call.iterations.first.validation_response == string.VALIDATED_OUTPUT_REASK

    # For re-asked prompt and output
    assert (
        call.iterations.last.inputs.messages[1]["content"]
        == string.COMPILED_PROMPT_REASK
    )
    # Same as above
    assert call.reask_messages.last[-1]["content"] == string.COMPILED_PROMPT_REASK
    assert call.raw_outputs.last == string.LLM_OUTPUT_REASK
    assert call.guarded_output == string.LLM_OUTPUT_REASK