From 137b9e92f62cea3df7921190dedcb57c657f50f8 Mon Sep 17 00:00:00 2001 From: almogbaku Date: Sat, 16 Mar 2024 00:51:08 +0200 Subject: [PATCH 1/7] feat: #1 add supports complex struct parsing with streaming --- .github/workflows/publish.yaml | 2 +- .github/workflows/test.yaml | 2 +- .gitignore | 1 + openai_streaming/struct/__init__.py | 1 + openai_streaming/struct/handler.py | 119 +++++++++++++++++++++++++ openai_streaming/struct/yaml_parser.py | 27 ++++++ pyproject.toml | 3 + requirements.txt | 3 +- tests/struct_example.py | 67 ++++++++++++++ 9 files changed, 222 insertions(+), 3 deletions(-) create mode 100644 openai_streaming/struct/__init__.py create mode 100644 openai_streaming/struct/handler.py create mode 100644 openai_streaming/struct/yaml_parser.py create mode 100644 tests/struct_example.py diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml index d1e7667..8b78493 100644 --- a/.github/workflows/publish.yaml +++ b/.github/workflows/publish.yaml @@ -29,7 +29,7 @@ jobs: if [ -f requirements.txt ]; then pip install -r requirements.txt; fi - name: Test with pytest run: | - pytest --ignore=tests/example.py --doctest-modules --junitxml=junit/test-results.xml + pytest --ignore=tests/example.py --ignore=tests/struct_example.py --doctest-modules --junitxml=junit/test-results.xml version: runs-on: ubuntu-latest outputs: diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 15d554e..459e743 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -27,4 +27,4 @@ jobs: if [ -f requirements.txt ]; then pip install -r requirements.txt; fi - name: Test with pytest run: | - pytest --ignore=tests/example.py --doctest-modules --junitxml=junit/test-results.xml \ No newline at end of file + pytest --ignore=tests/example.py --ignore=tests/struct_example.py --doctest-modules --junitxml=junit/test-results.xml \ No newline at end of file diff --git a/.gitignore b/.gitignore index 138ebdf..7bdacb8 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,4 @@ dist openai_streaming.egg-info/ .benchmarks junit +.venv \ No newline at end of file diff --git a/openai_streaming/struct/__init__.py b/openai_streaming/struct/__init__.py new file mode 100644 index 0000000..a53de96 --- /dev/null +++ b/openai_streaming/struct/__init__.py @@ -0,0 +1 @@ +from handler import process_struct_response, Terminate, BaseHandler \ No newline at end of file diff --git a/openai_streaming/struct/handler.py b/openai_streaming/struct/handler.py new file mode 100644 index 0000000..42c0de0 --- /dev/null +++ b/openai_streaming/struct/handler.py @@ -0,0 +1,119 @@ +from typing import Protocol, Literal, AsyncGenerator, Optional, Type, TypeVar, Union + +from pydantic import BaseModel + +from json_streamer import Parser, JsonParser +from .yaml_parser import YamlParser +from ..stream_processing import OAIResponse, process_response + +TModel = TypeVar('TModel', bound=BaseModel) + + +class Terminate: + pass + + +class BaseHandler(Protocol[TModel]): + def model(self) -> Type[TModel]: + """ + The Pydantic Data Model that we parse + :return: type of the Pydantic model + """ + pass + + async def handle_partially_parsed(self, data: TModel) -> Optional[Terminate]: + """ + Handle partially parsed model + :param data: The partially parsed object + :return: None or Terminate if we want to terminate the parsing + """ + pass + + async def terminated(self): + """ + Called when the parsing was terminated + """ + + +OutputSerialization = Literal["json", "yaml"] + + +class _ContentHandler: + parser: Parser = None + _last_resp: Optional[Union[TModel, Terminate]] = None + + def __init__(self, handler: BaseHandler, output_serialization: OutputSerialization = "yaml"): + self.handler = handler + if output_serialization == "json": + self.parser = JsonParser() + elif output_serialization == "yaml": + self.parser = YamlParser() + + async def handle_content(self, content: AsyncGenerator[str, None]): + """ + Handle the content of the response from OpenAI. + :param content: A generator that yields the content of the response from OpenAI + :return: None + """ + + loader = self.parser() # create a Streaming loader + next(loader) + + last_resp = None + + async for token in content: + parsed = loader.send(token) # send the token to the JSON loader + while parsed: # loop until through the parsed parts as the loader yields them + last_resp = await self._handle_parsed(parsed[1]) # handle the parsed dict of the response + if isinstance(last_resp, Terminate): + break + try: + parsed = next(loader) + except StopIteration: + break + if isinstance(last_resp, Terminate): + break + + if not last_resp: + return + if isinstance(last_resp, Terminate): + await self.handler.terminated() + + self._last_resp = last_resp + + async def _handle_parsed(self, part) -> Optional[Union[TModel, Terminate]]: + """ + Handle a parsed part of the response from OpenAI. + It parses the "parsed dictionary" as a type of `TModel` object and processes it with the handler. + + :param part: A dictionary containing the parsed part of the response + :return: The parsed part of the response as an `TModel` object, `Terminate` to terminate the handling, + or `None` if the part is not valid + """ + try: + parsed = self.handler.model()(**part) + except (TypeError, ValueError): + return + + ret = await self.handler.handle_partially_parsed(parsed) + return ret if ret else parsed + + def get_last_response(self) -> Optional[Union[TModel, Terminate]]: + """ + Get the last response from OpenAI. + :return: The last response from OpenAI + """ + return self._last_resp + + +async def process_struct_response( + response: OAIResponse, + handler: BaseHandler, + output_serialization: OutputSerialization = "json" +): + handler = _ContentHandler(handler, output_serialization) + _, result = await process_response(response, handler.handle_content, self=handler) + if not handler.get_last_response(): + raise ValueError("Probably invalid response from OpenAI") + + return result diff --git a/openai_streaming/struct/yaml_parser.py b/openai_streaming/struct/yaml_parser.py new file mode 100644 index 0000000..da10597 --- /dev/null +++ b/openai_streaming/struct/yaml_parser.py @@ -0,0 +1,27 @@ +from typing import List, Dict, Tuple, Generator, Optional +from json_streamer import Parser, ParseState + + +class YamlParser(Parser): + """ + Parse partial YAML + """ + + @staticmethod + def opening_symbols() -> List[chr]: + return ['{', '['] + + def raw_decode(self, s: str) -> Tuple[Dict, int]: + try: + from yaml import safe_load + except ImportError: + raise ImportError("You must install PyYAML to use the YamlParser: pip install PyYAML") + return safe_load(s), -1 + + def parse_part(self, part: str) -> Generator[Tuple[ParseState, dict], None, None]: + for y in super().parse_part(part): + yield ParseState.UNKNOWN, y[1] + + +def loads(s: Optional[Generator[chr, None, None]] = None) -> Generator[Tuple[ParseState, dict], Optional[str], None]: + return YamlParser()(s) diff --git a/pyproject.toml b/pyproject.toml index a59d3e9..d61ae91 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,6 +18,9 @@ dependencies= [ ] requires-python = ">=3.9" +[project.optional-dependencies] +yaml = ["PyYAML>6.0.0<7.0.0"] + [project.urls] "Homepage" = "https://github.com/AlmogBaku/openai-streaming" "Bug Reports" = "https://github.com/AlmogBaku/openai-streaming/issues" diff --git a/requirements.txt b/requirements.txt index 8a4b728..34d0d2e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ openai==1.14.0 json-streamer==0.1.0 pydantic==2.6.4 -docstring-parser==0.15 \ No newline at end of file +docstring-parser==0.15 +PyYAML==6.0.1 \ No newline at end of file diff --git a/tests/struct_example.py b/tests/struct_example.py new file mode 100644 index 0000000..055aa74 --- /dev/null +++ b/tests/struct_example.py @@ -0,0 +1,67 @@ +import os +from time import sleep + +from openai import AsyncOpenAI +import asyncio + +from pydantic import BaseModel + +from typing import Optional +from openai_streaming.struct import BaseHandler, process_struct_response, Terminate + +# Initialize OpenAI Client +client = AsyncOpenAI( + api_key=os.environ.get("OPENAI_API_KEY"), +) + + +class Letter(BaseModel): + title: str + to: Optional[str] = None + content: Optional[str] = None + + +# Define handler +class Handler(BaseHandler): + def model(self): + return Letter + + last_content = "" + + async def handle_partially_parsed(self, data: Letter) -> Optional[Terminate]: + if data.to and data.to.lower() != "larry": + print("You can only write a letter to Larry") + return Terminate() + if data.content: + # here we mingle with the content a bit for the sake of the animation + data.content = data.content[len(self.last_content):] + self.last_content = self.last_content + data.content + print(data.content, end="") + sleep(0.1) + + async def terminated(self): + print("Terminated") + + +# Invoke Function in a streaming request +async def main(): + # Request and process stream + resp = await client.chat.completions.create( + model="gpt-3.5-turbo", + messages=[{ + "role": "system", + "content": + "You are a letter writer able to communicate only with VALID YAML. " + "You must include only these fields: title, to, content." + "ONLY write the YAML, without any other text or wrapping it in a code block." + }, {"role": "user", "content": + "Write a SHORT letter to my friend Larry congratulating him for his newborn baby Lily." + "It should be funny and rhythmic. It MUST be very short!" + }], + stream=True + ) + await process_struct_response(resp, Handler(), 'yaml') + + +# Start the script asynchronously +asyncio.run(main()) From e687a13ba06ba971fcfc1a60176c460bc4941a60 Mon Sep 17 00:00:00 2001 From: almogbaku Date: Sat, 16 Mar 2024 00:51:08 +0200 Subject: [PATCH 2/7] feat: #1 add supports complex struct parsing with streaming --- openai_streaming/struct/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openai_streaming/struct/__init__.py b/openai_streaming/struct/__init__.py index a53de96..699f327 100644 --- a/openai_streaming/struct/__init__.py +++ b/openai_streaming/struct/__init__.py @@ -1 +1 @@ -from handler import process_struct_response, Terminate, BaseHandler \ No newline at end of file +from .handler import process_struct_response, Terminate, BaseHandler \ No newline at end of file From 1791aca00a4e7fdbb7f73637b21b38cb0cf23b91 Mon Sep 17 00:00:00 2001 From: almogbaku Date: Sat, 16 Mar 2024 08:32:24 +0200 Subject: [PATCH 3/7] build: reuse tests --- .github/workflows/publish.yaml | 16 ++-------------- .github/workflows/test.yaml | 2 +- 2 files changed, 3 insertions(+), 15 deletions(-) diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml index 8b78493..5ad732f 100644 --- a/.github/workflows/publish.yaml +++ b/.github/workflows/publish.yaml @@ -16,20 +16,8 @@ jobs: name: "Run tests" runs-on: ubuntu-latest steps: - - name: Checkout repository - uses: actions/checkout@v4 - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: '3.9' - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install ruff pytest - if [ -f requirements.txt ]; then pip install -r requirements.txt; fi - - name: Test with pytest - run: | - pytest --ignore=tests/example.py --ignore=tests/struct_example.py --doctest-modules --junitxml=junit/test-results.xml + - name: Test + uses: ./.github/workflows/test.yaml version: runs-on: ubuntu-latest outputs: diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 459e743..a4f1b5c 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -27,4 +27,4 @@ jobs: if [ -f requirements.txt ]; then pip install -r requirements.txt; fi - name: Test with pytest run: | - pytest --ignore=tests/example.py --ignore=tests/struct_example.py --doctest-modules --junitxml=junit/test-results.xml \ No newline at end of file + pytest --ignore=tests/example.py --ignore=tests/example_struct.py --doctest-modules --junitxml=junit/test-results.xml \ No newline at end of file From edd3c8eefbb947074c7708f9fce7d9cde6fa8a03 Mon Sep 17 00:00:00 2001 From: almogbaku Date: Sat, 16 Mar 2024 08:52:02 +0200 Subject: [PATCH 4/7] fix[struct]: add a bit more docs and tests --- openai_streaming/struct/handler.py | 21 +- openai_streaming/struct/yaml_parser.py | 2 +- tests/example_struct.py | 61 ++ tests/mock_response_struct.json | 742 +++++++++++++++++++++++++ tests/mock_response_tools.json | 64 +-- tests/struct_example.py | 67 --- tests/test_with_struct.py | 102 ++++ 7 files changed, 956 insertions(+), 103 deletions(-) create mode 100644 tests/example_struct.py create mode 100644 tests/mock_response_struct.json delete mode 100644 tests/struct_example.py create mode 100644 tests/test_with_struct.py diff --git a/openai_streaming/struct/handler.py b/openai_streaming/struct/handler.py index 42c0de0..b427b14 100644 --- a/openai_streaming/struct/handler.py +++ b/openai_streaming/struct/handler.py @@ -1,4 +1,4 @@ -from typing import Protocol, Literal, AsyncGenerator, Optional, Type, TypeVar, Union +from typing import Protocol, Literal, AsyncGenerator, Optional, Type, TypeVar, Union, Dict, Any, Tuple from pydantic import BaseModel @@ -14,6 +14,10 @@ class Terminate: class BaseHandler(Protocol[TModel]): + """ + The base handler for the structured response from OpenAI. + """ + def model(self) -> Type[TModel]: """ The Pydantic Data Model that we parse @@ -110,10 +114,21 @@ async def process_struct_response( response: OAIResponse, handler: BaseHandler, output_serialization: OutputSerialization = "json" -): +) -> Tuple[Optional[Union[TModel, Terminate]], Dict[str, Any]]: + """ + Process the structured response from OpenAI. + This is useful when we want to parse a structured response from OpenAI in streaming mode. For example: our response + contains reasoning, and content - but we want to stream only the content to the user. + + :param response: The response from OpenAI + :param handler: The handler for the response. It should be a subclass of `BaseHandler` + :param output_serialization: The output serialization of the response. It should be either "json" or "yaml" + :return: A tuple of the last parsed response, and a dictionary containing the OpenAI response + """ + handler = _ContentHandler(handler, output_serialization) _, result = await process_response(response, handler.handle_content, self=handler) if not handler.get_last_response(): raise ValueError("Probably invalid response from OpenAI") - return result + return handler.get_last_response(), result diff --git a/openai_streaming/struct/yaml_parser.py b/openai_streaming/struct/yaml_parser.py index da10597..4e37ad4 100644 --- a/openai_streaming/struct/yaml_parser.py +++ b/openai_streaming/struct/yaml_parser.py @@ -9,7 +9,7 @@ class YamlParser(Parser): @staticmethod def opening_symbols() -> List[chr]: - return ['{', '['] + return ['{', '[', '"'] def raw_decode(self, s: str) -> Tuple[Dict, int]: try: diff --git a/tests/example_struct.py b/tests/example_struct.py new file mode 100644 index 0000000..3ed5bee --- /dev/null +++ b/tests/example_struct.py @@ -0,0 +1,61 @@ +import os +from time import sleep + +from openai import AsyncOpenAI +import asyncio + +from pydantic import BaseModel + +from typing import Optional, List +from openai_streaming.struct import BaseHandler, process_struct_response, Terminate + +# Initialize OpenAI Client +client = AsyncOpenAI( + api_key=os.environ.get("OPENAI_API_KEY"), +) + + +class MathProblem(BaseModel): + steps: List[str] + answer: Optional[int] = None + + +# Define handler +class Handler(BaseHandler): + def model(self): + return MathProblem + + async def handle_partially_parsed(self, data: MathProblem) -> Optional[Terminate]: + if len(data.steps) == 0 and data.answer: + return Terminate() + + print(f"Steps: {', '.join(data.steps)}", end="\r") + sleep(0.1) + if data.answer: + print(f"\nAnswer: {data.answer}") + + async def terminated(self): + print("Terminated") + + +# Invoke OpenAI request +async def main(): + resp = await client.chat.completions.create( + model="gpt-3.5-turbo", + messages=[{ + "role": "system", + "content": + "For every question asked, you must first state the steps, and then the answer." + "Your response should be in the following format: \n" + " steps: List[str]\n" + " answer: int\n" + "ONLY write the YAML, without any other text or wrapping it in a code block." + "YAML should be VALID, and strings must be in double quotes." + }, {"role": "user", "content": "1+3*2"}], + stream=True + ) + await process_struct_response(resp, Handler(), 'yaml') + + +# Start the script asynchronously +asyncio.run(main()) diff --git a/tests/mock_response_struct.json b/tests/mock_response_struct.json new file mode 100644 index 0000000..129f791 --- /dev/null +++ b/tests/mock_response_struct.json @@ -0,0 +1,742 @@ +[ + { + "id": "chatcmpl-test-struct", + "choices": [ + { + "delta": { + "content": "", + "function_call": null, + "role": "assistant", + "tool_calls": null + }, + "finish_reason": null, + "index": 0, + "logprobs": null + } + ], + "created": 1, + "model": "gpt-3.5-turbo-0125", + "object": "chat.completion.chunk", + "system_fingerprint": null + }, + { + "id": "chatcmpl-test-struct", + "choices": [ + { + "delta": { + "content": "steps", + "function_call": null, + "role": null, + "tool_calls": null + }, + "finish_reason": null, + "index": 0, + "logprobs": null + } + ], + "created": 1, + "model": "gpt-3.5-turbo-0125", + "object": "chat.completion.chunk", + "system_fingerprint": null + }, + { + "id": "chatcmpl-test-struct", + "choices": [ + { + "delta": { + "content": ":\n", + "function_call": null, + "role": null, + "tool_calls": null + }, + "finish_reason": null, + "index": 0, + "logprobs": null + } + ], + "created": 1, + "model": "gpt-3.5-turbo-0125", + "object": "chat.completion.chunk", + "system_fingerprint": null + }, + { + "id": "chatcmpl-test-struct", + "choices": [ + { + "delta": { + "content": " ", + "function_call": null, + "role": null, + "tool_calls": null + }, + "finish_reason": null, + "index": 0, + "logprobs": null + } + ], + "created": 1, + "model": "gpt-3.5-turbo-0125", + "object": "chat.completion.chunk", + "system_fingerprint": null + }, + { + "id": "chatcmpl-test-struct", + "choices": [ + { + "delta": { + "content": " -", + "function_call": null, + "role": null, + "tool_calls": null + }, + "finish_reason": null, + "index": 0, + "logprobs": null + } + ], + "created": 1, + "model": "gpt-3.5-turbo-0125", + "object": "chat.completion.chunk", + "system_fingerprint": null + }, + { + "id": "chatcmpl-test-struct", + "choices": [ + { + "delta": { + "content": " \"", + "function_call": null, + "role": null, + "tool_calls": null + }, + "finish_reason": null, + "index": 0, + "logprobs": null + } + ], + "created": 1, + "model": "gpt-3.5-turbo-0125", + "object": "chat.completion.chunk", + "system_fingerprint": null + }, + { + "id": "chatcmpl-test-struct", + "choices": [ + { + "delta": { + "content": "Multiply", + "function_call": null, + "role": null, + "tool_calls": null + }, + "finish_reason": null, + "index": 0, + "logprobs": null + } + ], + "created": 1, + "model": "gpt-3.5-turbo-0125", + "object": "chat.completion.chunk", + "system_fingerprint": null + }, + { + "id": "chatcmpl-test-struct", + "choices": [ + { + "delta": { + "content": " ", + "function_call": null, + "role": null, + "tool_calls": null + }, + "finish_reason": null, + "index": 0, + "logprobs": null + } + ], + "created": 1, + "model": "gpt-3.5-turbo-0125", + "object": "chat.completion.chunk", + "system_fingerprint": null + }, + { + "id": "chatcmpl-test-struct", + "choices": [ + { + "delta": { + "content": "3", + "function_call": null, + "role": null, + "tool_calls": null + }, + "finish_reason": null, + "index": 0, + "logprobs": null + } + ], + "created": 1, + "model": "gpt-3.5-turbo-0125", + "object": "chat.completion.chunk", + "system_fingerprint": null + }, + { + "id": "chatcmpl-test-struct", + "choices": [ + { + "delta": { + "content": " by", + "function_call": null, + "role": null, + "tool_calls": null + }, + "finish_reason": null, + "index": 0, + "logprobs": null + } + ], + "created": 1, + "model": "gpt-3.5-turbo-0125", + "object": "chat.completion.chunk", + "system_fingerprint": null + }, + { + "id": "chatcmpl-test-struct", + "choices": [ + { + "delta": { + "content": " ", + "function_call": null, + "role": null, + "tool_calls": null + }, + "finish_reason": null, + "index": 0, + "logprobs": null + } + ], + "created": 1, + "model": "gpt-3.5-turbo-0125", + "object": "chat.completion.chunk", + "system_fingerprint": null + }, + { + "id": "chatcmpl-test-struct", + "choices": [ + { + "delta": { + "content": "2", + "function_call": null, + "role": null, + "tool_calls": null + }, + "finish_reason": null, + "index": 0, + "logprobs": null + } + ], + "created": 1, + "model": "gpt-3.5-turbo-0125", + "object": "chat.completion.chunk", + "system_fingerprint": null + }, + { + "id": "chatcmpl-test-struct", + "choices": [ + { + "delta": { + "content": " to", + "function_call": null, + "role": null, + "tool_calls": null + }, + "finish_reason": null, + "index": 0, + "logprobs": null + } + ], + "created": 1, + "model": "gpt-3.5-turbo-0125", + "object": "chat.completion.chunk", + "system_fingerprint": null + }, + { + "id": "chatcmpl-test-struct", + "choices": [ + { + "delta": { + "content": " get", + "function_call": null, + "role": null, + "tool_calls": null + }, + "finish_reason": null, + "index": 0, + "logprobs": null + } + ], + "created": 1, + "model": "gpt-3.5-turbo-0125", + "object": "chat.completion.chunk", + "system_fingerprint": null + }, + { + "id": "chatcmpl-test-struct", + "choices": [ + { + "delta": { + "content": " ", + "function_call": null, + "role": null, + "tool_calls": null + }, + "finish_reason": null, + "index": 0, + "logprobs": null + } + ], + "created": 1, + "model": "gpt-3.5-turbo-0125", + "object": "chat.completion.chunk", + "system_fingerprint": null + }, + { + "id": "chatcmpl-test-struct", + "choices": [ + { + "delta": { + "content": "6", + "function_call": null, + "role": null, + "tool_calls": null + }, + "finish_reason": null, + "index": 0, + "logprobs": null + } + ], + "created": 1, + "model": "gpt-3.5-turbo-0125", + "object": "chat.completion.chunk", + "system_fingerprint": null + }, + { + "id": "chatcmpl-test-struct", + "choices": [ + { + "delta": { + "content": "\"\n", + "function_call": null, + "role": null, + "tool_calls": null + }, + "finish_reason": null, + "index": 0, + "logprobs": null + } + ], + "created": 1, + "model": "gpt-3.5-turbo-0125", + "object": "chat.completion.chunk", + "system_fingerprint": null + }, + { + "id": "chatcmpl-test-struct", + "choices": [ + { + "delta": { + "content": " ", + "function_call": null, + "role": null, + "tool_calls": null + }, + "finish_reason": null, + "index": 0, + "logprobs": null + } + ], + "created": 1, + "model": "gpt-3.5-turbo-0125", + "object": "chat.completion.chunk", + "system_fingerprint": null + }, + { + "id": "chatcmpl-test-struct", + "choices": [ + { + "delta": { + "content": " -", + "function_call": null, + "role": null, + "tool_calls": null + }, + "finish_reason": null, + "index": 0, + "logprobs": null + } + ], + "created": 1, + "model": "gpt-3.5-turbo-0125", + "object": "chat.completion.chunk", + "system_fingerprint": null + }, + { + "id": "chatcmpl-test-struct", + "choices": [ + { + "delta": { + "content": " \"", + "function_call": null, + "role": null, + "tool_calls": null + }, + "finish_reason": null, + "index": 0, + "logprobs": null + } + ], + "created": 1, + "model": "gpt-3.5-turbo-0125", + "object": "chat.completion.chunk", + "system_fingerprint": null + }, + { + "id": "chatcmpl-test-struct", + "choices": [ + { + "delta": { + "content": "Add", + "function_call": null, + "role": null, + "tool_calls": null + }, + "finish_reason": null, + "index": 0, + "logprobs": null + } + ], + "created": 1, + "model": "gpt-3.5-turbo-0125", + "object": "chat.completion.chunk", + "system_fingerprint": null + }, + { + "id": "chatcmpl-test-struct", + "choices": [ + { + "delta": { + "content": " ", + "function_call": null, + "role": null, + "tool_calls": null + }, + "finish_reason": null, + "index": 0, + "logprobs": null + } + ], + "created": 1, + "model": "gpt-3.5-turbo-0125", + "object": "chat.completion.chunk", + "system_fingerprint": null + }, + { + "id": "chatcmpl-test-struct", + "choices": [ + { + "delta": { + "content": "1", + "function_call": null, + "role": null, + "tool_calls": null + }, + "finish_reason": null, + "index": 0, + "logprobs": null + } + ], + "created": 1, + "model": "gpt-3.5-turbo-0125", + "object": "chat.completion.chunk", + "system_fingerprint": null + }, + { + "id": "chatcmpl-test-struct", + "choices": [ + { + "delta": { + "content": " to", + "function_call": null, + "role": null, + "tool_calls": null + }, + "finish_reason": null, + "index": 0, + "logprobs": null + } + ], + "created": 1, + "model": "gpt-3.5-turbo-0125", + "object": "chat.completion.chunk", + "system_fingerprint": null + }, + { + "id": "chatcmpl-test-struct", + "choices": [ + { + "delta": { + "content": " ", + "function_call": null, + "role": null, + "tool_calls": null + }, + "finish_reason": null, + "index": 0, + "logprobs": null + } + ], + "created": 1, + "model": "gpt-3.5-turbo-0125", + "object": "chat.completion.chunk", + "system_fingerprint": null + }, + { + "id": "chatcmpl-test-struct", + "choices": [ + { + "delta": { + "content": "6", + "function_call": null, + "role": null, + "tool_calls": null + }, + "finish_reason": null, + "index": 0, + "logprobs": null + } + ], + "created": 1, + "model": "gpt-3.5-turbo-0125", + "object": "chat.completion.chunk", + "system_fingerprint": null + }, + { + "id": "chatcmpl-test-struct", + "choices": [ + { + "delta": { + "content": " to", + "function_call": null, + "role": null, + "tool_calls": null + }, + "finish_reason": null, + "index": 0, + "logprobs": null + } + ], + "created": 1, + "model": "gpt-3.5-turbo-0125", + "object": "chat.completion.chunk", + "system_fingerprint": null + }, + { + "id": "chatcmpl-test-struct", + "choices": [ + { + "delta": { + "content": " get", + "function_call": null, + "role": null, + "tool_calls": null + }, + "finish_reason": null, + "index": 0, + "logprobs": null + } + ], + "created": 1, + "model": "gpt-3.5-turbo-0125", + "object": "chat.completion.chunk", + "system_fingerprint": null + }, + { + "id": "chatcmpl-test-struct", + "choices": [ + { + "delta": { + "content": " the", + "function_call": null, + "role": null, + "tool_calls": null + }, + "finish_reason": null, + "index": 0, + "logprobs": null + } + ], + "created": 1, + "model": "gpt-3.5-turbo-0125", + "object": "chat.completion.chunk", + "system_fingerprint": null + }, + { + "id": "chatcmpl-test-struct", + "choices": [ + { + "delta": { + "content": " final", + "function_call": null, + "role": null, + "tool_calls": null + }, + "finish_reason": null, + "index": 0, + "logprobs": null + } + ], + "created": 1, + "model": "gpt-3.5-turbo-0125", + "object": "chat.completion.chunk", + "system_fingerprint": null + }, + { + "id": "chatcmpl-test-struct", + "choices": [ + { + "delta": { + "content": " result", + "function_call": null, + "role": null, + "tool_calls": null + }, + "finish_reason": null, + "index": 0, + "logprobs": null + } + ], + "created": 1, + "model": "gpt-3.5-turbo-0125", + "object": "chat.completion.chunk", + "system_fingerprint": null + }, + { + "id": "chatcmpl-test-struct", + "choices": [ + { + "delta": { + "content": "\"\n", + "function_call": null, + "role": null, + "tool_calls": null + }, + "finish_reason": null, + "index": 0, + "logprobs": null + } + ], + "created": 1, + "model": "gpt-3.5-turbo-0125", + "object": "chat.completion.chunk", + "system_fingerprint": null + }, + { + "id": "chatcmpl-test-struct", + "choices": [ + { + "delta": { + "content": "answer", + "function_call": null, + "role": null, + "tool_calls": null + }, + "finish_reason": null, + "index": 0, + "logprobs": null + } + ], + "created": 1, + "model": "gpt-3.5-turbo-0125", + "object": "chat.completion.chunk", + "system_fingerprint": null + }, + { + "id": "chatcmpl-test-struct", + "choices": [ + { + "delta": { + "content": ":", + "function_call": null, + "role": null, + "tool_calls": null + }, + "finish_reason": null, + "index": 0, + "logprobs": null + } + ], + "created": 1, + "model": "gpt-3.5-turbo-0125", + "object": "chat.completion.chunk", + "system_fingerprint": null + }, + { + "id": "chatcmpl-test-struct", + "choices": [ + { + "delta": { + "content": " ", + "function_call": null, + "role": null, + "tool_calls": null + }, + "finish_reason": null, + "index": 0, + "logprobs": null + } + ], + "created": 1, + "model": "gpt-3.5-turbo-0125", + "object": "chat.completion.chunk", + "system_fingerprint": null + }, + { + "id": "chatcmpl-test-struct", + "choices": [ + { + "delta": { + "content": "7", + "function_call": null, + "role": null, + "tool_calls": null + }, + "finish_reason": null, + "index": 0, + "logprobs": null + } + ], + "created": 1, + "model": "gpt-3.5-turbo-0125", + "object": "chat.completion.chunk", + "system_fingerprint": null + }, + { + "id": "chatcmpl-test-struct", + "choices": [ + { + "delta": { + "content": null, + "function_call": null, + "role": null, + "tool_calls": null + }, + "finish_reason": "stop", + "index": 0, + "logprobs": null + } + ], + "created": 1, + "model": "gpt-3.5-turbo-0125", + "object": "chat.completion.chunk", + "system_fingerprint": null + } +] \ No newline at end of file diff --git a/tests/mock_response_tools.json b/tests/mock_response_tools.json index 7aa642b..4172916 100644 --- a/tests/mock_response_tools.json +++ b/tests/mock_response_tools.json @@ -24,7 +24,7 @@ "logprobs": null } ], - "created": 1704213673, + "created": 1, "model": "gpt-3.5-turbo-0613", "object": "chat.completion.chunk", "system_fingerprint": null @@ -54,7 +54,7 @@ "logprobs": null } ], - "created": 1704213673, + "created": 1, "model": "gpt-3.5-turbo-0613", "object": "chat.completion.chunk", "system_fingerprint": null @@ -84,7 +84,7 @@ "logprobs": null } ], - "created": 1704213673, + "created": 1, "model": "gpt-3.5-turbo-0613", "object": "chat.completion.chunk", "system_fingerprint": null @@ -114,7 +114,7 @@ "logprobs": null } ], - "created": 1704213673, + "created": 1, "model": "gpt-3.5-turbo-0613", "object": "chat.completion.chunk", "system_fingerprint": null @@ -144,7 +144,7 @@ "logprobs": null } ], - "created": 1704213673, + "created": 1, "model": "gpt-3.5-turbo-0613", "object": "chat.completion.chunk", "system_fingerprint": null @@ -174,7 +174,7 @@ "logprobs": null } ], - "created": 1704213673, + "created": 1, "model": "gpt-3.5-turbo-0613", "object": "chat.completion.chunk", "system_fingerprint": null @@ -204,7 +204,7 @@ "logprobs": null } ], - "created": 1704213673, + "created": 1, "model": "gpt-3.5-turbo-0613", "object": "chat.completion.chunk", "system_fingerprint": null @@ -234,7 +234,7 @@ "logprobs": null } ], - "created": 1704213673, + "created": 1, "model": "gpt-3.5-turbo-0613", "object": "chat.completion.chunk", "system_fingerprint": null @@ -264,7 +264,7 @@ "logprobs": null } ], - "created": 1704213673, + "created": 1, "model": "gpt-3.5-turbo-0613", "object": "chat.completion.chunk", "system_fingerprint": null @@ -294,7 +294,7 @@ "logprobs": null } ], - "created": 1704213673, + "created": 1, "model": "gpt-3.5-turbo-0613", "object": "chat.completion.chunk", "system_fingerprint": null @@ -324,7 +324,7 @@ "logprobs": null } ], - "created": 1704213673, + "created": 1, "model": "gpt-3.5-turbo-0613", "object": "chat.completion.chunk", "system_fingerprint": null @@ -354,7 +354,7 @@ "logprobs": null } ], - "created": 1704213673, + "created": 1, "model": "gpt-3.5-turbo-0613", "object": "chat.completion.chunk", "system_fingerprint": null @@ -384,7 +384,7 @@ "logprobs": null } ], - "created": 1704213673, + "created": 1, "model": "gpt-3.5-turbo-0613", "object": "chat.completion.chunk", "system_fingerprint": null @@ -414,7 +414,7 @@ "logprobs": null } ], - "created": 1704213673, + "created": 1, "model": "gpt-3.5-turbo-0613", "object": "chat.completion.chunk", "system_fingerprint": null @@ -444,7 +444,7 @@ "logprobs": null } ], - "created": 1704213673, + "created": 1, "model": "gpt-3.5-turbo-0613", "object": "chat.completion.chunk", "system_fingerprint": null @@ -474,7 +474,7 @@ "logprobs": null } ], - "created": 1704213673, + "created": 1, "model": "gpt-3.5-turbo-0613", "object": "chat.completion.chunk", "system_fingerprint": null @@ -504,7 +504,7 @@ "logprobs": null } ], - "created": 1704213673, + "created": 1, "model": "gpt-3.5-turbo-0613", "object": "chat.completion.chunk", "system_fingerprint": null @@ -534,7 +534,7 @@ "logprobs": null } ], - "created": 1704213673, + "created": 1, "model": "gpt-3.5-turbo-0613", "object": "chat.completion.chunk", "system_fingerprint": null @@ -564,7 +564,7 @@ "logprobs": null } ], - "created": 1704213673, + "created": 1, "model": "gpt-3.5-turbo-0613", "object": "chat.completion.chunk", "system_fingerprint": null @@ -594,7 +594,7 @@ "logprobs": null } ], - "created": 1704213673, + "created": 1, "model": "gpt-3.5-turbo-0613", "object": "chat.completion.chunk", "system_fingerprint": null @@ -624,7 +624,7 @@ "logprobs": null } ], - "created": 1704213673, + "created": 1, "model": "gpt-3.5-turbo-0613", "object": "chat.completion.chunk", "system_fingerprint": null @@ -654,7 +654,7 @@ "logprobs": null } ], - "created": 1704213673, + "created": 1, "model": "gpt-3.5-turbo-0613", "object": "chat.completion.chunk", "system_fingerprint": null @@ -684,7 +684,7 @@ "logprobs": null } ], - "created": 1704213673, + "created": 1, "model": "gpt-3.5-turbo-0613", "object": "chat.completion.chunk", "system_fingerprint": null @@ -714,7 +714,7 @@ "logprobs": null } ], - "created": 1704213673, + "created": 1, "model": "gpt-3.5-turbo-0613", "object": "chat.completion.chunk", "system_fingerprint": null @@ -744,7 +744,7 @@ "logprobs": null } ], - "created": 1704213673, + "created": 1, "model": "gpt-3.5-turbo-0613", "object": "chat.completion.chunk", "system_fingerprint": null @@ -774,7 +774,7 @@ "logprobs": null } ], - "created": 1704213673, + "created": 1, "model": "gpt-3.5-turbo-0613", "object": "chat.completion.chunk", "system_fingerprint": null @@ -804,7 +804,7 @@ "logprobs": null } ], - "created": 1704213673, + "created": 1, "model": "gpt-3.5-turbo-0613", "object": "chat.completion.chunk", "system_fingerprint": null @@ -834,7 +834,7 @@ "logprobs": null } ], - "created": 1704213673, + "created": 1, "model": "gpt-3.5-turbo-0613", "object": "chat.completion.chunk", "system_fingerprint": null @@ -864,7 +864,7 @@ "logprobs": null } ], - "created": 1704213673, + "created": 1, "model": "gpt-3.5-turbo-0613", "object": "chat.completion.chunk", "system_fingerprint": null @@ -894,7 +894,7 @@ "logprobs": null } ], - "created": 1704213673, + "created": 1, "model": "gpt-3.5-turbo-0613", "object": "chat.completion.chunk", "system_fingerprint": null @@ -924,7 +924,7 @@ "logprobs": null } ], - "created": 1704213673, + "created": 1, "model": "gpt-3.5-turbo-0613", "object": "chat.completion.chunk", "system_fingerprint": null @@ -944,7 +944,7 @@ "logprobs": null } ], - "created": 1704213673, + "created": 1, "model": "gpt-3.5-turbo-0613", "object": "chat.completion.chunk", "system_fingerprint": null diff --git a/tests/struct_example.py b/tests/struct_example.py deleted file mode 100644 index 055aa74..0000000 --- a/tests/struct_example.py +++ /dev/null @@ -1,67 +0,0 @@ -import os -from time import sleep - -from openai import AsyncOpenAI -import asyncio - -from pydantic import BaseModel - -from typing import Optional -from openai_streaming.struct import BaseHandler, process_struct_response, Terminate - -# Initialize OpenAI Client -client = AsyncOpenAI( - api_key=os.environ.get("OPENAI_API_KEY"), -) - - -class Letter(BaseModel): - title: str - to: Optional[str] = None - content: Optional[str] = None - - -# Define handler -class Handler(BaseHandler): - def model(self): - return Letter - - last_content = "" - - async def handle_partially_parsed(self, data: Letter) -> Optional[Terminate]: - if data.to and data.to.lower() != "larry": - print("You can only write a letter to Larry") - return Terminate() - if data.content: - # here we mingle with the content a bit for the sake of the animation - data.content = data.content[len(self.last_content):] - self.last_content = self.last_content + data.content - print(data.content, end="") - sleep(0.1) - - async def terminated(self): - print("Terminated") - - -# Invoke Function in a streaming request -async def main(): - # Request and process stream - resp = await client.chat.completions.create( - model="gpt-3.5-turbo", - messages=[{ - "role": "system", - "content": - "You are a letter writer able to communicate only with VALID YAML. " - "You must include only these fields: title, to, content." - "ONLY write the YAML, without any other text or wrapping it in a code block." - }, {"role": "user", "content": - "Write a SHORT letter to my friend Larry congratulating him for his newborn baby Lily." - "It should be funny and rhythmic. It MUST be very short!" - }], - stream=True - ) - await process_struct_response(resp, Handler(), 'yaml') - - -# Start the script asynchronously -asyncio.run(main()) diff --git a/tests/test_with_struct.py b/tests/test_with_struct.py new file mode 100644 index 0000000..dd84a71 --- /dev/null +++ b/tests/test_with_struct.py @@ -0,0 +1,102 @@ +import json +import unittest +from os.path import dirname + +import openai +from unittest.mock import patch, AsyncMock + +from openai import BaseModel +from openai.types.chat import ChatCompletionChunk + +from typing import Dict, Generator, Optional, List + +from openai_streaming.struct import Terminate, BaseHandler, process_struct_response + +openai.api_key = '...' + + +class MathProblem(BaseModel): + steps: List[str] + answer: Optional[int] = None + + +# Define handler +class Handler(BaseHandler): + def model(self): + return MathProblem + + async def handle_partially_parsed(self, data: MathProblem) -> Optional[Terminate]: + pass + + async def terminated(self): + pass + + +class Handler2(BaseHandler): + def model(self): + return MathProblem + + async def handle_partially_parsed(self, data: MathProblem) -> Optional[Terminate]: + return Terminate() + + async def terminated(self): + pass + + +class TestOpenAIChatCompletion(unittest.IsolatedAsyncioTestCase): + _mock_response = None + _mock_response_tools = None + + def setUp(self): + if not self._mock_response: + with open(f"{dirname(__file__)}/mock_response_struct.json", 'r') as f: + self.mock_response = json.load(f) + + def mock_chat_completion(self, *args, **kwargs) -> Generator[Dict, None, None]: + for item in self.mock_response: + yield ChatCompletionChunk(**item) + + async def test_struct(self): + with patch('openai.chat.completions.create', new=AsyncMock(side_effect=self.mock_chat_completion)): + resp = await openai.chat.completions.create( + model="gpt-3.5-turbo", + messages=[{ + "role": "system", + "content": + "For every question asked, you must first state the steps, and then the answer." + "Your response should be in the following format: \n" + " steps: List[str]\n" + " answer: int\n" + "ONLY write the YAML, without any other text or wrapping it in a code block." + "YAML should be VALID, and strings must be in double quotes." + }, {"role": "user", "content": "1+3*2"}], + stream=True, + ) + last_resp, _ = await process_struct_response(resp, Handler(), 'yaml') + + wanted = MathProblem(steps=['Multiply 3 by 2 to get 6', 'Add 1 to 6 to get the final result'], answer=7) + self.assertEqual(last_resp, wanted) + + async def test_struct_terminate(self): + with patch('openai.chat.completions.create', new=AsyncMock(side_effect=self.mock_chat_completion)): + resp = await openai.chat.completions.create( + model="gpt-3.5-turbo", + messages=[{ + "role": "system", + "content": + "For every question asked, you must first state the steps, and then the answer." + "Your response should be in the following format: \n" + " steps: List[str]\n" + " answer: int\n" + "ONLY write the YAML, without any other text or wrapping it in a code block." + "YAML should be VALID, and strings must be in double quotes." + }, {"role": "user", "content": "1+3*2"}], + stream=True, + ) + last_resp, _ = await process_struct_response(resp, Handler2(), 'yaml') + + self.assertIsInstance(last_resp, Terminate) + + +if __name__ == '__main__': + unittest.main() From 7e128714698aed3e1c461ffa7b93f52a13421d0c Mon Sep 17 00:00:00 2001 From: almogbaku Date: Sat, 16 Mar 2024 09:12:50 +0200 Subject: [PATCH 5/7] docs[struct]: add to the reference --- docs/reference.md | 115 ++++++++++++++++++++++++++++++++++++++++++++- pydoc-markdown.yml | 1 + 2 files changed, 115 insertions(+), 1 deletion(-) diff --git a/docs/reference.md b/docs/reference.md index aff2544..5a3e892 100644 --- a/docs/reference.md +++ b/docs/reference.md @@ -2,12 +2,28 @@ # decorator + + +## OpenAIStreamingFunction Objects + +```python +class OpenAIStreamingFunction(Protocol) +``` + +A Protocol that represents a function that can be used with OpenAI Streaming. + + + +#### openai\_schema + +The OpenAI Schema for the function. + #### openai\_streaming\_function ```python -def openai_streaming_function(func: FunctionType) -> Any +def openai_streaming_function(func: F) -> OpenAIStreamingFunction ``` Decorator that creates an OpenAI Schema for your function, while support using Generators for Streaming. @@ -189,3 +205,100 @@ dictionary of arguments A set of function names that were invoked + + +# struct.handler + + + +## BaseHandler Objects + +```python +class BaseHandler(Protocol[TModel]) +``` + +The base handler for the structured response from OpenAI. + + + +#### model + +```python +def model() -> Type[TModel] +``` + +The Pydantic Data Model that we parse + +**Returns**: + +type of the Pydantic model + + + +#### handle\_partially\_parsed + +```python +async def handle_partially_parsed(data: TModel) -> Optional[Terminate] +``` + +Handle partially parsed model + +**Arguments**: + +- `data`: The partially parsed object + +**Returns**: + +None or Terminate if we want to terminate the parsing + + + +#### terminated + +```python +async def terminated() +``` + +Called when the parsing was terminated + + + +#### process\_struct\_response + +```python +async def process_struct_response( + response: OAIResponse, + handler: BaseHandler, + output_serialization: OutputSerialization = "json" +) -> Tuple[Optional[Union[TModel, Terminate]], Dict[str, Any]] +``` + +Process the structured response from OpenAI. + +This is useful when we want to parse a structured response from OpenAI in streaming mode. For example: our response +contains reasoning, and content - but we want to stream only the content to the user. + +**Arguments**: + +- `response`: The response from OpenAI +- `handler`: The handler for the response. It should be a subclass of `BaseHandler` +- `output_serialization`: The output serialization of the response. It should be either "json" or "yaml" + +**Returns**: + +A tuple of the last parsed response, and a dictionary containing the OpenAI response + + + +# struct.yaml\_parser + + + +## YamlParser Objects + +```python +class YamlParser(Parser) +``` + +Parse partial YAML + diff --git a/pydoc-markdown.yml b/pydoc-markdown.yml index 0a6ca32..5486e7c 100644 --- a/pydoc-markdown.yml +++ b/pydoc-markdown.yml @@ -1,3 +1,4 @@ +# build with: pipx run pydoc-markdown loaders: - type: python search_path: From c40e1dbb055d011296d50e8bf5274ba5dca56b89 Mon Sep 17 00:00:00 2001 From: almogbaku Date: Fri, 10 May 2024 19:33:59 +0300 Subject: [PATCH 6/7] build: pr triage --- .github/workflows/pr-triage.yaml | 55 ++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) create mode 100644 .github/workflows/pr-triage.yaml diff --git a/.github/workflows/pr-triage.yaml b/.github/workflows/pr-triage.yaml new file mode 100644 index 0000000..991c229 --- /dev/null +++ b/.github/workflows/pr-triage.yaml @@ -0,0 +1,55 @@ +name: "Pull Request Triage" +on: + # NB: using `pull_request_target` runs this in the context of + # the base repository, so it has permission to upload to the checks API. + # This means changes won't kick in to this file until merged onto the + # main branch. + pull_request_target: + types: [ opened, edited, reopened, synchronize ] + +permissions: + contents: read + pull-requests: write + issues: write + +jobs: + triage: + name: "Triage Pull Request" + runs-on: ubuntu-latest + steps: + - uses: actions/labeler@v4 + with: + repo-token: "${{ secrets.GITHUB_TOKEN }}" + - uses: codelytv/pr-size-labeler@v1 + with: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + xs_label: 'size/xs' + xs_max_size: '15' + s_label: 'size/s' + s_max_size: '100' + m_label: 'size/m' + m_max_size: '500' + l_label: 'size/l' + l_max_size: '1000' + xl_label: 'size/xl' + fail_if_xl: 'false' + message_if_xl: > + This PR exceeds the recommended size of 1000 lines. + Please make sure you are NOT addressing multiple issues with one PR. + Note this PR might be rejected due to its size. + files_to_ignore: '' +# - name: "Check for PR body length" +# shell: bash +# env: +# PR_BODY: ${{ github.event.pull_request.body }} +# run: | +# if [ ${#PR_BODY} -lt 80 ]; then +# echo "::error title=PR body is too short::Your PR is probably isn't descriptive enough.\nYou should give a description that highlights both what you're doing it and *why* you're doing it. Someone reading the PR description without clicking any issue links should be able to roughly understand what's going on." +# exit 1 +# fi + - uses: amannn/action-semantic-pull-request@v5 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + disallowScopes: | + release \ No newline at end of file From 54c82df9b84929ca0c609e9e145568238225dc53 Mon Sep 17 00:00:00 2001 From: almogbaku Date: Sat, 11 May 2024 17:52:01 +0300 Subject: [PATCH 7/7] build: remove auto labeling --- .github/workflows/pr-triage.yaml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/pr-triage.yaml b/.github/workflows/pr-triage.yaml index 991c229..2a6aa4f 100644 --- a/.github/workflows/pr-triage.yaml +++ b/.github/workflows/pr-triage.yaml @@ -17,9 +17,6 @@ jobs: name: "Triage Pull Request" runs-on: ubuntu-latest steps: - - uses: actions/labeler@v4 - with: - repo-token: "${{ secrets.GITHUB_TOKEN }}" - uses: codelytv/pr-size-labeler@v1 with: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}