참고소스 수정본

참고/instructor-main/examples/partial_streaming/benchmark.py

@@ -0,0 +1,103 @@
+# Part of this code is adapted from the following examples from OpenAI Cookbook:
+# https://cookbook.openai.com/examples/how_to_stream_completions
+# https://github.com/openai/openai-cookbook/blob/main/examples/How_to_count_tokens_with_tiktoken.ipynb
+import time
+import tiktoken
+import instructor
+from openai import OpenAI
+from pydantic import BaseModel
+
+client = instructor.from_openai(OpenAI(), mode=instructor.Mode.MD_JSON)
+
+
+def num_tokens_from_string(string: str, model_name: str) -> int:
+    """Returns the number of tokens in a text string."""
+    encoding = tiktoken.encoding_for_model(model_name)
+
+    num_tokens = len(encoding.encode(string))
+    num_tokens += 3  # every reply is primed with <|start|>assistant<|message|>
+
+    return num_tokens
+
+
+class User(BaseModel):
+    name: str
+    role: str
+    age: int
+
+
+def benchmark_raw_stream(model="gpt-4"):
+    content = f"""Respond only in JSON that would validate to this schema and include nothing extra.
+    Otherwise something bad will happen:\n {User.model_json_schema()}"""
+
+    start_time = time.time()
+    extraction_stream = client.chat.completions.create_fn(
+        model=model,
+        messages=[
+            {"role": "system", "content": content},
+            {
+                "role": "user",
+                "content": "give me a harry pottery character in json, name, role, age",
+            },
+        ],
+        stream=True,
+    )
+
+    collected_messages = [chunk.choices[0].delta.content for chunk in extraction_stream]
+    collected_messages = [m for m in collected_messages if m is not None]
+    collected_messages = "".join(collected_messages)
+    User.model_validate_json(collected_messages)
+    end_time = time.time() - start_time
+
+    output_tokens = num_tokens_from_string(collected_messages, model)
+    char_per_sec = output_tokens / end_time
+    return char_per_sec
+
+
+def benchmark_partial_streaming(model="gpt-4"):
+    start_time = time.time()
+    extraction_stream = client.chat.completions.create_partial(
+        model=model,
+        response_model=User,
+        messages=[
+            {
+                "role": "user",
+                "content": "give me a harry pottery character in json, name, role, age",
+            }
+        ],
+        stream=True,
+    )
+
+    for chunk in extraction_stream:  # noqa: B007
+        pass
+    end_time = time.time() - start_time
+
+    output_tokens = num_tokens_from_string(chunk.model_dump_json(), model)
+    char_per_sec = output_tokens / end_time
+    return char_per_sec
+
+
+if __name__ == "__main__":
+    partial_times = [
+        benchmark_partial_streaming(model="gpt-3.5-turbo-1106") for _ in range(10)
+    ]
+    avg_partial_time = sum(partial_times) / len(partial_times)
+
+    raw_times = [benchmark_raw_stream(model="gpt-3.5-turbo") for _ in range(10)]
+    avg_raw_time = sum(raw_times) / len(raw_times)
+    print(f"Raw streaming: {avg_raw_time:.2f} tokens/sec")
+
+    print(f"Partial streaming: {avg_partial_time:.2f} token/sec")
+    print(f"Overhead: {avg_partial_time / avg_raw_time:.2f}x")
+
+    """OLD IMPLEMENTATION
+    Raw streaming: 35.73 tokens/sec
+    Partial streaming: 24.42 token/sec
+    Overhead: 0.68x
+    """
+
+    """NEW IMPLEMENTATION
+    Raw streaming: 35.77 tokens/sec
+    Partial streaming: 31.58 token/sec
+    Overhead: 0.88x
+    """

참고/instructor-main/examples/partial_streaming/run.py

@@ -0,0 +1,28 @@
+# Part of this code is adapted from the following examples from OpenAI Cookbook:
+# https://cookbook.openai.com/examples/how_to_stream_completions
+# https://github.com/openai/openai-cookbook/blob/main/examples/How_to_count_tokens_with_tiktoken.ipynb
+import instructor
+from openai import OpenAI
+from pydantic import BaseModel
+
+client = instructor.from_openai(OpenAI(), mode=instructor.Mode.TOOLS)
+
+
+class User(BaseModel):
+    name: str
+    role: str
+
+
+extraction_stream = client.chat.completions.create_partial(
+    model="gpt-4",
+    response_model=User,
+    messages=[
+        {
+            "role": "user",
+            "content": "give me a harry pottery character in json, name, role, age",
+        }
+    ],
+)
+
+for chunk in extraction_stream:
+    print(chunk)