참고소스 수정본

참고/guardrails-main/tests/unit_tests/classes/history/test_call.py

@@ -0,0 +1,207 @@
+from guardrails.classes.generic.stack import Stack
+from guardrails.classes.history.call import Call
+from guardrails.classes.history.call_inputs import CallInputs
+from guardrails.classes.history.inputs import Inputs
+from guardrails.classes.history.iteration import Iteration
+from guardrails.classes.history.outputs import Outputs
+from guardrails.constants import not_run_status, pass_status
+from guardrails.llm_providers import ArbitraryCallable
+from guardrails.classes.llm.llm_response import LLMResponse
+from guardrails.classes.validation.validator_logs import ValidatorLogs
+from guardrails.actions.reask import ReAsk
+from guardrails.validator_base import FailResult, PassResult
+
+
+def test_empty_initialization():
+    call = Call()
+
+    assert call.iterations == Stack()
+    assert call.inputs == CallInputs()
+    assert call.messages is None
+    assert call.prompt_params is None
+    assert call.compiled_messages is None
+    assert call.reask_messages == []
+    assert call.logs == Stack()
+    assert call.tokens_consumed is None
+    assert call.prompt_tokens_consumed is None
+    assert call.completion_tokens_consumed is None
+    assert call.raw_outputs == Stack()
+    assert call.parsed_outputs == Stack()
+    assert call.validation_response is None
+    assert call.fixed_output is None
+    assert call.guarded_output is None
+    assert call.reasks == Stack()
+    assert call.validator_logs == Stack()
+    assert call.error is None
+    assert call.failed_validations == Stack()
+    assert call.status == not_run_status
+    # FIXME: how to do shallow comparison?
+    # assert call.tree == Tree("Logs")
+    assert call.tree is not None
+
+
+def test_non_empty_initialization():
+    # Call input
+    def custom_llm(messages, *args, **kwargs):
+        return "Hello there!"
+
+    llm_api = custom_llm
+    messages = [
+        {
+            "role": "system",
+            "content": "You are a greeting bot.",
+        },
+        {
+            "role": "user",
+            "content": "Respond with a {greeting_type} greeting.",
+        },
+    ]
+    args = ["arg1"]
+    kwargs = {"kwarg1": 1}
+    prompt_params = {"greeting_type": "friendly"}
+
+    call_inputs = CallInputs(
+        llm_api=llm_api,
+        messages=messages,
+        prompt_params=prompt_params,
+        args=args,
+        kwargs=kwargs,
+    )
+
+    # First Iteration Inputs
+    iter_llm_api = ArbitraryCallable(llm_api=llm_api)
+    llm_output = "Hello there!"
+    num_reasks = 0
+    metadata = {"some_meta_data": "doesn't actually matter"}
+    full_schema_reask = False
+
+    inputs = Inputs(
+        llm_api=iter_llm_api,
+        llm_output=llm_output,
+        messages=messages,
+        prompt_params=prompt_params,
+        num_reasks=num_reasks,
+        metadata=metadata,
+        full_schema_reask=full_schema_reask,
+    )
+
+    # Outputs
+    first_validation_result = FailResult(
+        outcome="fail",
+        error_message="Should not include punctuation",
+        fix_value="Hello there",
+    )
+    first_llm_response_info = LLMResponse(
+        output="Hello there!", prompt_token_count=10, response_token_count=3
+    )
+    first_parsed_output = "Hello there!"
+    first_validated_output = "Hello there"
+    first_reasks = [
+        ReAsk(incorrect_value="Hello there!", fail_results=[first_validation_result])
+    ]
+    first_validator_log = ValidatorLogs(
+        registered_name="no-punctuation",
+        validator_name="no-punctuation",
+        value_before_validation="Hello there!",
+        validation_result=first_validation_result,
+        value_after_validation="Hello there",
+        property_path="$",
+    )
+    first_validator_logs = [first_validator_log]
+    first_outputs = Outputs(
+        llm_response_info=first_llm_response_info,
+        parsed_output=first_parsed_output,
+        validated_output=first_validated_output,
+        reasks=first_reasks,
+        validator_logs=first_validator_logs,
+    )
+
+    first_iteration = Iteration(
+        call_id="mock-call", index=0, inputs=inputs, outputs=first_outputs
+    )
+
+    second_iter_messages = [
+        {
+            "role": "user",
+            "content": "That wasn't quite right. Try again.",
+        }
+    ]
+    second_inputs = Inputs(
+        llm_api=iter_llm_api,
+        llm_output=llm_output,
+        messages=second_iter_messages,
+        num_reasks=num_reasks,
+        metadata=metadata,
+        full_schema_reask=full_schema_reask,
+    )
+
+    second_llm_response_info = LLMResponse(
+        output="Hello there", prompt_token_count=10, response_token_count=3
+    )
+    second_parsed_output = "Hello there"
+    second_validated_output = "Hello there"
+    second_reasks = []
+    second_validator_log = ValidatorLogs(
+        registered_name="no-punctuation",
+        validator_name="no-punctuation",
+        value_before_validation="Hello there",
+        validation_result=PassResult(),
+        value_after_validation="Hello there",
+        property_path="$",
+    )
+    second_validator_logs = [second_validator_log]
+    second_outputs = Outputs(
+        llm_response_info=second_llm_response_info,
+        parsed_output=second_parsed_output,
+        validation_response="Hello there",
+        validated_output=second_validated_output,
+        reasks=second_reasks,
+        validator_logs=second_validator_logs,
+    )
+
+    second_iteration = Iteration(
+        call_id="mock-call", index=0, inputs=second_inputs, outputs=second_outputs
+    )
+
+    iterations: Stack[Iteration] = Stack(first_iteration, second_iteration)
+
+    call = Call(inputs=call_inputs, iterations=iterations)
+
+    assert call.iterations == iterations
+    assert isinstance(call.iterations, Stack) is True
+    assert call.inputs == call_inputs
+
+    assert call.messages == messages
+    assert call.prompt_params == prompt_params
+    assert call.compiled_messages == [
+        {
+            "role": "system",
+            "content": "You are a greeting bot.",
+        },
+        {
+            "role": "user",
+            "content": "Respond with a friendly greeting.",
+        },
+    ]
+    assert call.reask_messages == Stack(second_iter_messages)
+
+    # TODO: Test this in the integration tests
+    assert call.logs == []
+
+    assert call.tokens_consumed == 26
+    assert call.prompt_tokens_consumed == 20
+    assert call.completion_tokens_consumed == 6
+
+    assert call.raw_outputs == Stack("Hello there!", "Hello there")
+    assert call.parsed_outputs == Stack("Hello there!", "Hello there")
+    assert call.validation_response == "Hello there"
+    assert call.fixed_output == "Hello there"
+    assert call.guarded_output == "Hello there"
+    assert call.reasks == Stack()
+    assert call.validator_logs == Stack(first_validator_log, second_validator_log)
+    assert call.error is None
+    assert call.failed_validations == Stack(first_validator_log)
+    assert call.status == pass_status
+    # TODO: How to do shallow comparison
+    # assert call.tree == "something"
+    assert call.tree is not None

참고/guardrails-main/tests/unit_tests/classes/history/test_call_inputs.py

@@ -0,0 +1,46 @@
+from guardrails.classes.history.call_inputs import CallInputs
+
+
+def test_empty_initialization():
+    call_inputs = CallInputs()
+
+    # Overrides and additional properties
+    assert call_inputs.llm_api is None
+    assert call_inputs.messages is None
+    assert call_inputs.args == []
+    assert call_inputs.kwargs == {}
+
+    # Inherited properties
+    assert call_inputs.llm_output is None
+    assert call_inputs.prompt_params is None
+    assert call_inputs.num_reasks is None
+    assert call_inputs.metadata is None
+    assert call_inputs.full_schema_reask is None
+
+
+def test_non_empty_initialization():
+    def custom_llm():
+        return "Hello there!"
+
+    llm_api = custom_llm
+    messages = [
+        {"role": "system", "content": "You are a greeting bot."},
+        {"role": "user", "content": "Respond with a friendly greeting."},
+    ]
+
+    args = ["arg1"]
+    kwargs = {"kwarg1": 1}
+
+    call_inputs = CallInputs(
+        llm_api=llm_api,
+        messages=messages,
+        args=args,
+        kwargs=kwargs,
+    )
+
+    # We only care about overrides and additional props
+    # because the others were tested in test_inputs.py
+    assert call_inputs.llm_api == llm_api
+    assert call_inputs.messages == messages
+    assert call_inputs.args == args
+    assert call_inputs.kwargs == kwargs

참고/guardrails-main/tests/unit_tests/classes/history/test_inputs.py

@@ -0,0 +1,59 @@
+from guardrails.classes.history.inputs import Inputs
+from guardrails.llm_providers import LiteLLMCallable
+
+
+# Guard against regressions in pydantic BaseModel
+def test_empty_initialization():
+    inputs = Inputs()
+
+    assert inputs.llm_api is None
+    assert inputs.llm_output is None
+    assert inputs.messages is None
+    assert inputs.prompt_params is None
+    assert inputs.num_reasks is None
+    assert inputs.metadata is None
+    assert inputs.full_schema_reask is None
+
+
+def test_non_empty_initialization():
+    llm_api = LiteLLMCallable(text="Respond with a greeting.")
+    llm_output = "Hello there!"
+    messages = [
+        {
+            "role": "system",
+            "content": "You are a greeting bot.",
+        },
+        {
+            "role": "user",
+            "content": "Respond with a ${greeting_type} greeting.",
+        },
+    ]
+    prompt_params = {"greeting_type": "friendly"}
+    num_reasks = 0
+    metadata = {"some_meta_data": "doesn't actually matter"}
+    full_schema_reask = False
+
+    inputs = Inputs(
+        llm_api=llm_api,
+        llm_output=llm_output,
+        messages=messages,
+        prompt_params=prompt_params,
+        num_reasks=num_reasks,
+        metadata=metadata,
+        full_schema_reask=full_schema_reask,
+    )
+
+    assert inputs.llm_api is not None
+    assert inputs.llm_api == llm_api
+    assert inputs.llm_output is not None
+    assert inputs.llm_output == llm_output
+    assert inputs.messages is not None
+    assert inputs.messages == messages
+    assert inputs.prompt_params is not None
+    assert inputs.prompt_params == prompt_params
+    assert inputs.num_reasks is not None
+    assert inputs.num_reasks == num_reasks
+    assert inputs.metadata is not None
+    assert inputs.metadata == metadata
+    assert inputs.full_schema_reask is not None
+    assert inputs.full_schema_reask == full_schema_reask

참고/guardrails-main/tests/unit_tests/classes/history/test_iteration.py

@@ -0,0 +1,117 @@
+from guardrails.classes.generic.stack import Stack
+from guardrails.classes.history.inputs import Inputs
+from guardrails.classes.history.iteration import Iteration
+from guardrails.classes.history.outputs import Outputs
+from guardrails.constants import error_status, not_run_status
+from guardrails.llm_providers import LiteLLMCallable
+from guardrails.classes.llm.llm_response import LLMResponse
+from guardrails.classes.validation.validator_logs import ValidatorLogs
+from guardrails.actions.reask import FieldReAsk
+from guardrails.validator_base import FailResult
+
+
+def test_empty_initialization():
+    iteration = Iteration(
+        call_id="mock-call",
+        index=0,
+    )
+
+    assert iteration.inputs == Inputs()
+    assert iteration.outputs == Outputs()
+    assert iteration.logs == Stack()
+    assert iteration.tokens_consumed is None
+    assert iteration.prompt_tokens_consumed is None
+    assert iteration.completion_tokens_consumed is None
+    assert iteration.raw_output is None
+    assert iteration.parsed_output is None
+    assert iteration.validation_response is None
+    assert iteration.guarded_output is None
+    assert iteration.reasks == []
+    assert iteration.validator_logs == []
+    assert iteration.error is None
+    assert iteration.failed_validations == []
+    assert iteration.status == not_run_status
+    assert iteration.rich_group is not None
+
+
+def test_non_empty_initialization():
+    # Inputs
+    llm_api = LiteLLMCallable(text="Respond with a greeting.")
+    llm_output = "Hello there!"
+    messages = [
+        {
+            "role": "system",
+            "content": "You are a greeting bot.",
+        },
+        {
+            "role": "user",
+            "content": "Respond with a ${greeting_type} greeting.",
+        },
+    ]
+    prompt_params = {"greeting_type": "friendly"}
+    num_reasks = 0
+    metadata = {"some_meta_data": "doesn't actually matter"}
+    full_schema_reask = False
+
+    inputs = Inputs(
+        llm_api=llm_api,
+        llm_output=llm_output,
+        messages=messages,
+        prompt_params=prompt_params,
+        num_reasks=num_reasks,
+        metadata=metadata,
+        full_schema_reask=full_schema_reask,
+    )
+
+    # Outputs
+    validation_result = FailResult(
+        outcome="fail",
+        error_message="Should not include punctuation",
+        fix_value="Hello there",
+    )
+    llm_response_info = LLMResponse(
+        output="Hello there!", prompt_token_count=10, response_token_count=3
+    )
+    parsed_output = "Hello there!"
+    guarded_output = "Hello there"
+    reask = FieldReAsk(
+        incorrect_value="Hello there!", fail_results=[validation_result], path=[]
+    )
+    reasks = [reask]
+    validator_logs = [
+        ValidatorLogs(
+            registered_name="no-punctuation",
+            validator_name="no-punctuation",
+            value_before_validation="Hello there!",
+            validation_result=validation_result,
+            value_after_validation="Hello there",
+            property_path="$",
+        )
+    ]
+    error = "Validation Failed!"
+    outputs = Outputs(
+        llm_response_info=llm_response_info,
+        parsed_output=parsed_output,
+        validation_response=reask,
+        guarded_output=guarded_output,
+        reasks=reasks,
+        validator_logs=validator_logs,
+        error=error,
+    )
+
+    iteration = Iteration(call_id="mock-call", index=0, inputs=inputs, outputs=outputs)
+
+    assert iteration.inputs == inputs
+    assert iteration.outputs == outputs
+    assert iteration.logs == Stack()
+    assert iteration.tokens_consumed == 13
+    assert iteration.prompt_tokens_consumed == 10
+    assert iteration.completion_tokens_consumed == 3
+    assert iteration.raw_output == "Hello there!"
+    assert iteration.parsed_output == "Hello there!"
+    assert iteration.validation_response == reask
+    assert iteration.guarded_output == "Hello there"
+    assert iteration.reasks == reasks
+    assert iteration.validator_logs == validator_logs
+    assert iteration.error == error
+    assert iteration.status == error_status

참고/guardrails-main/tests/unit_tests/classes/history/test_outputs.py

@@ -0,0 +1,196 @@
+import pytest
+
+from guardrails.classes.history.outputs import Outputs
+from guardrails.constants import error_status, fail_status, not_run_status, pass_status
+from guardrails.classes.llm.llm_response import LLMResponse
+from guardrails.classes.validation.validator_logs import ValidatorLogs
+from guardrails.actions.reask import ReAsk
+from guardrails.validator_base import FailResult, PassResult
+
+
+def test_empty_initialization():
+    empty_outputs = Outputs()
+
+    assert empty_outputs.llm_response_info is None
+    assert empty_outputs.parsed_output is None
+    assert empty_outputs.validation_response is None
+    assert empty_outputs.guarded_output is None
+    assert empty_outputs.reasks == []
+    assert empty_outputs.validator_logs == []
+    assert empty_outputs.error is None
+    assert empty_outputs.failed_validations == []
+    assert empty_outputs.status == not_run_status
+
+
+def test_non_empty_initialization():
+    validation_result = FailResult(
+        outcome="fail",
+        error_message="Should not include punctuation",
+        fix_value="Hello there",
+    )
+    llm_response_info = LLMResponse(
+        output="Hello there!", prompt_token_count=10, response_token_count=3
+    )
+    parsed_output = "Hello there!"
+    guarded_output = "Hello there"
+    reasks = [ReAsk(incorrect_value="Hello there!", fail_results=[validation_result])]
+    validator_logs = [
+        ValidatorLogs(
+            registered_name="no-punctuation",
+            validator_name="no-punctuation",
+            value_before_validation="Hello there!",
+            validation_result=validation_result,
+            value_after_validation="Hello there",
+            property_path="$",
+        )
+    ]
+    error = "Validation Failed!"
+    non_empty_outputs = Outputs(
+        llm_response_info=llm_response_info,
+        parsed_output=parsed_output,
+        guarded_output=guarded_output,
+        reasks=reasks,
+        validator_logs=validator_logs,
+        error=error,
+    )
+
+    assert non_empty_outputs.llm_response_info is not None
+    assert non_empty_outputs.llm_response_info == llm_response_info
+    assert non_empty_outputs.parsed_output is not None
+    assert non_empty_outputs.parsed_output == parsed_output
+    assert non_empty_outputs.guarded_output is not None
+    assert non_empty_outputs.guarded_output == guarded_output
+    assert non_empty_outputs.reasks != []
+    assert non_empty_outputs.reasks == reasks
+    assert non_empty_outputs.validator_logs != []
+    assert non_empty_outputs.validator_logs == validator_logs
+    assert non_empty_outputs.error is not None
+    assert non_empty_outputs.error == error
+    assert non_empty_outputs.failed_validations == validator_logs
+    assert non_empty_outputs.status == error_status
+
+
+fixable_fail_result = FailResult(
+    outcome="fail",
+    error_message="Should not include punctuation",
+    fix_value="Hello there",
+)
+non_fixable_fail_result = FailResult(
+    outcome="fail",
+    error_message="Should not include punctuation",
+)
+
+
+@pytest.mark.parametrize(
+    "outputs,expected_result",
+    [
+        (Outputs(), True),
+        (Outputs(llm_response_info=LLMResponse(output="Hello there!")), False),
+        (Outputs(parsed_output="Hello there!"), False),
+        (Outputs(parsed_output="Hello there!"), False),
+        (Outputs(guarded_output="Hello there"), False),
+        (
+            Outputs(
+                reasks=[
+                    ReAsk(
+                        incorrect_value="Hello there!",
+                        fail_results=[fixable_fail_result],
+                    )
+                ]
+            ),
+            False,
+        ),
+        (
+            Outputs(
+                validator_logs=[
+                    ValidatorLogs(
+                        registered_name="no-punctuation",
+                        validator_name="no-punctuation",
+                        value_before_validation="Hello there!",
+                        validation_result=fixable_fail_result,
+                        value_after_validation="Hello there",
+                        property_path="$",
+                    )
+                ]
+            ),
+            False,
+        ),
+        (Outputs(error="Validation Failed!"), False),
+    ],
+)
+def test__all_empty(outputs: Outputs, expected_result: bool):
+    are_outputs_empty = outputs._all_empty()
+
+    assert are_outputs_empty == expected_result
+
+
+def test_failed_validations():
+    validator_logs = [
+        ValidatorLogs(
+            registered_name="no-punctuation",
+            validator_name="no-punctuation",
+            value_before_validation="Hello there!",
+            validation_result=fixable_fail_result,
+            value_after_validation="Hello there",
+            property_path="$",
+        ),
+        ValidatorLogs(
+            registered_name="length",
+            validator_name="length",
+            value_before_validation="Hello there!",
+            validation_result=PassResult(),
+            value_after_validation="Hello there!",
+            property_path="$",
+        ),
+    ]
+    outputs = Outputs(validator_logs=validator_logs)
+
+    assert outputs.failed_validations == [validator_logs[0]]
+
+
+@pytest.mark.parametrize(
+    "outputs,expected_status",
+    [
+        (Outputs(), not_run_status),
+        (Outputs(error="Validations Failed!"), error_status),
+        (
+            Outputs(
+                validator_logs=[
+                    ValidatorLogs(
+                        registered_name="no-punctuation",
+                        validator_name="no-punctuation",
+                        value_before_validation="Hello there!",
+                        validation_result=non_fixable_fail_result,
+                        value_after_validation="Hello there",
+                        property_path="$",
+                    )
+                ],
+                reasks=[
+                    ReAsk(
+                        incorrect_value="Hello there!",
+                        fail_results=[non_fixable_fail_result],
+                    )
+                ],
+            ),
+            fail_status,
+        ),
+        (Outputs(validator_logs=[], guarded_output="Hello there!"), pass_status),
+    ],
+)
+def test_status(outputs: Outputs, expected_status: str):
+    status = outputs.status
+
+    assert status == expected_status
+
+
+def test_status_reask():
+    outputs = Outputs(
+        validation_response=ReAsk(
+            incorrect_value="Hello there!",
+            fail_results=[non_fixable_fail_result],
+        ),
+    )
+
+    status = outputs.status
+
+    assert status == fail_status