참고소스 수정본

참고/instructor-main/tests/llm/test_writer/evals/test_classification_enums.py

@@ -0,0 +1,113 @@
+import enum
+from itertools import product
+from writerai import Writer
+
+import pytest
+import instructor
+
+from pydantic import BaseModel
+
+from instructor.mode import Mode
+from ..util import models, modes
+
+
+class Labels(str, enum.Enum):
+    SPAM = "spam"
+    NOT_SPAM = "not_spam"
+
+
+class SinglePrediction(BaseModel):
+    """
+    Correct class label for the given text
+    """
+
+    class_label: Labels
+
+
+data = [
+    (
+        "I am a spammer",
+        Labels.SPAM,
+    ),
+    (
+        "I am not a spammer",
+        Labels.NOT_SPAM,
+    ),
+]
+
+
+@pytest.mark.parametrize("model, data, mode", product(models, data, modes))
+def test_writer_classification(
+    model: str, data: list[tuple[str, Labels]], mode: instructor.Mode
+):
+    client = instructor.from_writer(client=Writer(), mode=mode)
+
+    input, expected = data
+    resp = client.chat.completions.create(
+        model=model,
+        response_model=SinglePrediction,
+        messages=[
+            {
+                "role": "user",
+                "content": f"Classify the following text: {input}. "
+                f"Apply this or another class only in cases when "
+                f"when you are 100% sure.",
+            },
+        ],
+    )
+    assert resp.class_label == expected
+
+
+class MultiLabels(str, enum.Enum):
+    BILLING = "billing"
+    GENERAL_QUERY = "general_query"
+    HARDWARE = "hardware"
+
+
+class MultiClassPrediction(BaseModel):
+    predicted_labels: list[MultiLabels]
+
+
+data = [
+    (
+        "I am having trouble with my billing",
+        [MultiLabels.BILLING],
+    ),
+    (
+        "I am having trouble with my hardware",
+        [MultiLabels.HARDWARE],
+    ),
+    (
+        "I have a general query and a billing issue",
+        [MultiLabels.GENERAL_QUERY, MultiLabels.BILLING],
+    ),
+]
+
+
+@pytest.mark.parametrize("model, data, mode", product(models, data, modes))
+def test_writer_multi_classify(
+    model: str, data: list[tuple[str, list[MultiLabels]]], mode: instructor.Mode
+):
+    client = instructor.from_writer(client=Writer(), mode=mode)
+
+    if (mode, model) in {
+        (Mode.JSON, "gpt-3.5-turbo"),
+        (Mode.JSON, "gpt-4"),
+    }:
+        pytest.skip(f"{mode} mode is not supported for {model}, skipping test")
+
+    input, expected = data
+
+    resp = client.chat.completions.create(
+        model=model,
+        response_model=MultiClassPrediction,
+        messages=[
+            {
+                "role": "user",
+                "content": f"Classify the following support ticket: {input} "
+                f"Apply this or another class only in cases when "
+                f"when you are 100% sure.",
+            },
+        ],
+    )
+    assert set(resp.predicted_labels) == set(expected)

참고/instructor-main/tests/llm/test_writer/evals/test_classification_literals.py

@@ -0,0 +1,93 @@
+from itertools import product
+from typing import Literal
+from writerai import AsyncWriter
+
+import pytest
+import instructor
+
+from pydantic import BaseModel
+
+from ..util import models, modes
+
+
+class SinglePrediction(BaseModel):
+    """
+    Correct class label for the given text
+    """
+
+    class_label: Literal["spam", "not_spam"]
+
+
+data = [
+    ("I am a spammer", "spam"),
+    ("I am not a spammer", "not_spam"),
+]
+
+
+@pytest.mark.parametrize("model, data, mode", product(models, data, modes))
+@pytest.mark.asyncio
+async def test_classification(
+    model: str,
+    data: list[tuple[str, Literal["spam", "not_spam"]]],
+    mode: instructor.Mode,
+):
+    client = instructor.from_writer(client=AsyncWriter(), mode=mode)
+
+    input, expected = data
+    resp = await client.chat.completions.create(
+        model=model,
+        response_model=SinglePrediction,
+        messages=[
+            {
+                "role": "user",
+                "content": f"Classify the following text: {input}",
+            },
+        ],
+    )
+    assert resp.class_label == expected
+
+
+class MultiClassPrediction(BaseModel):
+    predicted_labels: list[Literal["billing", "general_query", "hardware"]]
+
+
+data = [
+    (
+        "I am having trouble with my billing",
+        ["billing"],
+    ),
+    (
+        "I am having trouble with my hardware",
+        ["hardware"],
+    ),
+    (
+        "I have a general query and a billing issue",
+        ["general_query", "billing"],
+    ),
+]
+
+
+@pytest.mark.parametrize("model, data, mode", product(models, data, modes))
+@pytest.mark.asyncio
+async def test_writer_multi_classify(
+    model: str,
+    data: list[tuple[str, list[Literal["billing", "general_query", "hardware"]]]],
+    mode: instructor.Mode,
+):
+    client = instructor.from_writer(client=AsyncWriter(), mode=mode)
+
+    input, expected = data
+
+    resp = await client.chat.completions.create(
+        model=model,
+        response_model=MultiClassPrediction,
+        messages=[
+            {
+                "role": "user",
+                "content": f"Classify the following support ticket: {input}. "
+                f"Apply this or another class only in cases when "
+                f"you sure by 100%.",
+            },
+        ],
+    )
+    assert set(resp.predicted_labels) == set(expected)

참고/instructor-main/tests/llm/test_writer/evals/test_entities.py

@@ -0,0 +1,95 @@
+from itertools import product
+from writerai import Writer
+
+from pydantic import BaseModel, Field
+import pytest
+
+import instructor
+from instructor import Instructor
+
+from ..util import models, modes
+
+
+class Property(BaseModel):
+    key: str
+    value: str
+    resolved_absolute_value: str
+
+
+class Entity(BaseModel):
+    id: int = Field(
+        ...,
+        description="Unique identifier for the entity, used for deduplication, design a scheme allows multiple entities",
+    )
+    subquote_string: list[str] = Field(
+        ...,
+        description="Correctly resolved value of the entity, if the entity is a reference to another entity, this should be the id of the referenced entity, include a few more words before and after the value to allow for some context to be used in the resolution",
+    )
+    entity_title: str
+    properties: list[Property] = Field(
+        ..., description="List of properties of the entity"
+    )
+    dependencies: list[int] = Field(
+        ...,
+        description="List of entity ids that this entity depends  or relies on to resolve it",
+    )
+
+
+class DocumentExtraction(BaseModel):
+    entities: list[Entity] = Field(
+        ...,
+        description="Body of the answer, each fact should be its separate object with a body and a list of sources",
+    )
+
+
+def ask_ai(content: str, model: str, client: Instructor) -> DocumentExtraction:
+    resp: DocumentExtraction = client.chat.completions.create(
+        model=model,
+        response_model=DocumentExtraction,
+        messages=[
+            {
+                "role": "system",
+                "content": "You are a perfect entity resolution system that extracts facts from the document. Extract and resolve a list of entities from the following document:",
+            },
+            {
+                "role": "user",
+                "content": content,
+            },
+        ],
+        max_retries=4,
+    )  # type: ignore
+    return resp
+
+
+content = """
+Sample Legal Contract
+Agreement Contract
+
+This Agreement is made and entered into on 2020-01-01 by and between Company A ("the Client") and Company B ("the Service Provider").
+
+Article 1: Scope of Work
+
+The Service Provider will deliver the software product to the Client 30 days after the agreement date.
+
+Article 2: Payment Terms
+
+The total payment for the service is $50,000.
+An initial payment of $10,000 will be made within 7 days of the the signed date.
+The final payment will be due 45 days after [SignDate].
+
+Article 3: Confidentiality
+
+The parties agree not to disclose any confidential information received from the other party for 3 months after the final payment date.
+
+Article 4: Termination
+
+The contract can be terminated with a 30-day notice, unless there are outstanding obligations that must be fulfilled after the [DeliveryDate].
+"""
+
+
+@pytest.mark.parametrize("model, mode", product(models, modes))
+def test_writer_extract(model: str, mode: instructor.Mode):
+    client = instructor.from_writer(client=Writer(), mode=mode)
+
+    extract = ask_ai(content=content, model=model, client=client)
+    assert len(extract.entities) > 0

참고/instructor-main/tests/llm/test_writer/evals/test_extract_users.py

@@ -0,0 +1,42 @@
+import pytest
+from itertools import product
+from pydantic import BaseModel
+from writerai import Writer
+import instructor
+from ..util import models, modes
+
+
+class UserDetails(BaseModel):
+    first_name: str
+    age: int
+
+
+test_data = [
+    ("Jason is 10", "Jason", 10),
+    ("Alice is 25", "Alice", 25),
+    ("Bob is 35", "Bob", 35),
+]
+
+
+@pytest.mark.parametrize("model, data, mode", product(models, test_data, modes))
+def test_writer_extract(
+    model: str, data: list[tuple[str, str, int]], mode: instructor.Mode
+):
+    client = instructor.from_writer(client=Writer(), mode=mode)
+
+    sample_data, expected_name, expected_age = data
+
+    response = client.chat.completions.create(
+        model=model,
+        response_model=UserDetails,
+        messages=[
+            {"role": "user", "content": sample_data},
+        ],
+    )
+
+    assert response.first_name == expected_name, (
+        f"Expected name {expected_name}, got {response.first_name}"
+    )
+    assert response.age == expected_age, (
+        f"Expected age {expected_age}, got {response.age}"
+    )

참고/instructor-main/tests/llm/test_writer/evals/test_sentiment_analysis.py

@@ -0,0 +1,57 @@
+import enum
+from itertools import product
+
+from pydantic import BaseModel
+from writerai import Writer
+import pytest
+import instructor
+from ..util import models, modes
+
+
+class Sentiment(str, enum.Enum):
+    POSITIVE = "positive"
+    NEGATIVE = "negative"
+    NEUTRAL = "neutral"
+
+
+class SentimentAnalysis(BaseModel):
+    sentiment: Sentiment
+
+
+test_data = [
+    (
+        "I absolutely love this product! It has exceeded all my expectations.",
+        Sentiment.POSITIVE,
+    ),
+    (
+        "The service was terrible. I will never use this company again.",
+        Sentiment.NEGATIVE,
+    ),
+    (
+        "The movie was okay. It had some good moments but overall it was average.",
+        Sentiment.NEUTRAL,
+    ),
+]
+
+
+@pytest.mark.parametrize("model, data, mode", product(models, test_data, modes))
+def test_writer_sentiment_analysis(
+    model: str, data: list[tuple[str, Sentiment]], mode: instructor.Mode
+):
+    client = instructor.from_writer(client=Writer(), mode=mode)
+
+    sample_data, expected_sentiment = data
+
+    response = client.chat.completions.create(
+        model=model,
+        response_model=SentimentAnalysis,
+        messages=[
+            {
+                "role": "system",
+                "content": "You are a sentiment analysis model. Analyze the sentiment of the given text and provide the sentiment (positive, negative, or neutral).",
+            },
+            {"role": "user", "content": sample_data},
+        ],
+    )
+
+    assert response.sentiment == expected_sentiment