Make cache store typed, and improve docs

python/packages/autogen-agentchat/tests/test_group_chat.py

@@ -34,10 +34,10 @@
 from autogen_agentchat.teams._group_chat._swarm_group_chat import SwarmGroupChatManager
 from autogen_agentchat.ui import Console
 from autogen_core import AgentId, CancellationToken
-from autogen_core.models import ReplayChatCompletionClient
 from autogen_core.tools import FunctionTool
 from autogen_ext.code_executors.local import LocalCommandLineCodeExecutor
 from autogen_ext.models.openai import OpenAIChatCompletionClient
+from autogen_ext.models.replay import ReplayChatCompletionClient
 from openai.resources.chat.completions import AsyncCompletions
 from openai.types.chat.chat_completion import ChatCompletion, Choice
 from openai.types.chat.chat_completion_chunk import ChatCompletionChunk

python/packages/autogen-agentchat/tests/test_magentic_one_group_chat.py

@@ -18,7 +18,7 @@
 )
 from autogen_agentchat.teams._group_chat._magentic_one._magentic_one_orchestrator import MagenticOneOrchestrator
 from autogen_core import AgentId, CancellationToken
-from autogen_core.models import ReplayChatCompletionClient
+from autogen_ext.models.replay import ReplayChatCompletionClient
 from utils import FileLogHandler
 
 logger = logging.getLogger(EVENT_LOGGER_NAME)

python/packages/autogen-core/docs/src/reference/index.md

@@ -49,6 +49,7 @@ python/autogen_ext.agents.video_surfer.tools
 python/autogen_ext.auth.azure
 python/autogen_ext.teams.magentic_one
 python/autogen_ext.models.openai
+python/autogen_ext.models.replay
 python/autogen_ext.tools.langchain
 python/autogen_ext.tools.graphrag
 python/autogen_ext.tools.code_execution

python/packages/autogen-core/docs/src/reference/python/autogen_ext.cache_store.diskcache.rst

@@ -0,0 +1,8 @@
+autogen\_ext.cache_store.diskcache
+==================================
+
+
+.. automodule:: autogen_ext.cache_store.diskcache
+   :members:
+   :undoc-members:
+   :show-inheritance:

python/packages/autogen-core/docs/src/reference/python/autogen_ext.cache_store.redis.rst

@@ -0,0 +1,8 @@
+autogen\_ext.cache_store.redis
+==============================
+
+
+.. automodule:: autogen_ext.cache_store.redis
+   :members:
+   :undoc-members:
+   :show-inheritance:

python/packages/autogen-core/docs/src/reference/python/autogen_ext.models.cache.rst

@@ -0,0 +1,8 @@
+autogen\_ext.models.cache
+=========================
+
+
+.. automodule:: autogen_ext.models.cache
+   :members:
+   :undoc-members:
+   :show-inheritance:

python/packages/autogen-core/docs/src/reference/python/autogen_ext.models.replay.rst

@@ -0,0 +1,8 @@
+autogen\_ext.models.replay
+==========================
+
+
+.. automodule:: autogen_ext.models.replay
+   :members:
+   :undoc-members:
+   :show-inheritance:

python/packages/autogen-core/docs/src/user-guide/agentchat-user-guide/tutorial/models.ipynb

@@ -8,7 +8,9 @@
                 "\n",
                 "In many cases, agents need access to LLM model services such as OpenAI, Azure OpenAI, or local models. Since there are many different providers with different APIs, `autogen-core` implements a protocol for [model clients](../../core-user-guide/framework/model-clients.ipynb) and `autogen-ext` implements a set of model clients for popular model services. AgentChat can use these model clients to interact with model services. \n",
                 "\n",
-                "**NOTE:** See {py:class}`~autogen_core.models.ChatCompletionCache` for a caching wrapper to use with the following clients."
+                "```{note}\n",
+                "See {py:class}`~autogen_ext.models.cache.ChatCompletionCache` for a caching wrapper to use with the following clients.\n",
+                "```"
             ]
         },
         {

python/packages/autogen-core/docs/src/user-guide/core-user-guide/framework/model-clients.ipynb

@@ -327,9 +327,20 @@
    "source": [
     "## Caching Wrapper\n",
     "\n",
-    "`autogen_core` implements a {py:class}`~autogen_core.models.ChatCompletionCache` that can wrap any {py:class}`~autogen_core.models.ChatCompletionClient`. Using this wrapper avoids incurring token usage when querying the underlying client with the same prompt multiple times. \n",
+    "`autogen_ext` implements {py:class}`~autogen_ext.models.cache.ChatCompletionCache` that can wrap any {py:class}`~autogen_core.models.ChatCompletionClient`. Using this wrapper avoids incurring token usage when querying the underlying client with the same prompt multiple times.\n",
     "\n",
-    "{py:class}`~autogen_core.models.ChatCompletionCache` uses a {py:class}`~autogen_core.CacheStore` protocol to allow duck-typing any storage object that has a pair of `get` & `set` methods (such as `redis.Redis` or `diskcache.Cache`). Here's an example of using `diskcache` for local caching:"
+    "{py:class}`~autogen_core.models.ChatCompletionCache` uses a {py:class}`~autogen_core.CacheStore` protocol. We have implemented some useful variants of {py:class}`~autogen_core.CacheStore` including {py:class}`~autogen_ext.cache_store.diskcache.DiskCacheStore` and {py:class}`~autogen_ext.cache_store.redis.RedisStore`.\n",
+    "\n",
+    "Here's an example of using `diskcache` for local caching:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# pip install -U \"autogen-ext[openai, diskcache]\""
    ]
   },
   {
@@ -346,18 +357,37 @@
     }
    ],
    "source": [
-    "from typing import Any, Dict, Optional\n",
+    "import asyncio\n",
+    "import tempfile\n",
     "\n",
-    "from autogen_core.models import ChatCompletionCache\n",
+    "from autogen_core.models import UserMessage\n",
+    "from autogen_ext.cache_store.diskcache import DiskCacheStore\n",
+    "from autogen_ext.models.cache import CHAT_CACHE_VALUE_TYPE, ChatCompletionCache\n",
+    "from autogen_ext.models.openai import OpenAIChatCompletionClient\n",
     "from diskcache import Cache\n",
     "\n",
-    "diskcache_client = Cache(\"/tmp/diskcache\")\n",
     "\n",
-    "cached_client = ChatCompletionCache(model_client, diskcache_client)\n",
-    "response = await cached_client.create(messages=messages)\n",
+    "async def main() -> None:\n",
+    "    with tempfile.TemporaryDirectory() as tmpdirname:\n",
+    "        # Initialize the original client\n",
+    "        openai_model_client = OpenAIChatCompletionClient(model=\"gpt-4o\")\n",
+    "\n",
+    "        # Then initialize the CacheStore, in this case with diskcache.Cache.\n",
+    "        # You can also use redis like:\n",
+    "        # from autogen_ext.cache_store.redis import RedisStore\n",
+    "        # import redis\n",
+    "        # redis_instance = redis.Redis()\n",
+    "        # cache_store = RedisCacheStore[CHAT_CACHE_VALUE_TYPE](redis_instance)\n",
+    "        cache_store = DiskCacheStore[CHAT_CACHE_VALUE_TYPE](Cache(tmpdirname))\n",
+    "        cache_client = ChatCompletionCache(openai_model_client, cache_store)\n",
+    "\n",
+    "        response = await cache_client.create([UserMessage(content=\"Hello, how are you?\", source=\"user\")])\n",
+    "        print(response)  # Should print response from OpenAI\n",
+    "        response = await cache_client.create([UserMessage(content=\"Hello, how are you?\", source=\"user\")])\n",
+    "        print(response)  # Should print cached response\n",
+    "\n",
     "\n",
-    "cached_response = await cached_client.create(messages=messages)\n",
-    "print(cached_response.cached)"
+    "asyncio.run(main())"
    ]
   },
   {

python/packages/autogen-core/src/autogen_core/__init__.py

@@ -10,7 +10,7 @@
 from ._agent_runtime import AgentRuntime
 from ._agent_type import AgentType
 from ._base_agent import BaseAgent
-from ._cache_store import CacheStore
+from ._cache_store import CacheStore, InMemoryStore
 from ._cancellation_token import CancellationToken
 from ._closure_agent import ClosureAgent, ClosureContext
 from ._component_config import (
@@ -87,6 +87,7 @@
     "AgentRuntime",
     "BaseAgent",
     "CacheStore",
+    "InMemoryStore",
     "CancellationToken",
     "AgentInstantiationContext",
     "TopicId",

python/packages/autogen-core/src/autogen_core/_cache_store.py

@@ -1,15 +1,16 @@
-from typing import Any, Optional, Protocol
+from typing import Any, Dict, Generic, Optional, Protocol, TypeVar, cast
 
+T = TypeVar("T")
 
-class CacheStore(Protocol):
+
+class CacheStore(Protocol, Generic[T]):
     """
     This protocol defines the basic interface for store/cache operations.
 
-    Allows duck-typing with any object that implements the get and set methods,
-    such as redis or diskcache interfaces.
+    Sub-classes should handle the lifecycle of underlying storage.
     """
 
-    def get(self, key: str, default: Optional[Any] = None) -> Optional[Any]:
+    def get(self, key: str, default: Optional[T] = None) -> Optional[T]:
         """
         Retrieve an item from the store.
 
@@ -23,7 +24,7 @@ def get(self, key: str, default: Optional[Any] = None) -> Optional[Any]:
         """
         ...
 
-    def set(self, key: str, value: Any) -> Optional[Any]:
+    def set(self, key: str, value: T) -> None:
         """
         Set an item in the store.
 
@@ -32,3 +33,14 @@ def set(self, key: str, value: Any) -> Optional[Any]:
             value: The value to be stored in the store.
         """
         ...
+
+
+class InMemoryStore(CacheStore[T]):
+    def __init__(self) -> None:
+        self.store: Dict[str, Any] = {}
+
+    def get(self, key: str, default: Optional[T] = None) -> Optional[T]:
+        return cast(Optional[T], self.store.get(key, default))
+
+    def set(self, key: str, value: T) -> None:
+        self.store[key] = value

python/packages/autogen-core/src/autogen_core/models/__init__.py

@@ -1,6 +1,4 @@
-from ._cache import ChatCompletionCache
 from ._model_client import ChatCompletionClient, ModelCapabilities, ModelFamily, ModelInfo  # type: ignore
-from ._replay_chat_completion_client import ReplayChatCompletionClient
 from ._types import (
     AssistantMessage,
     ChatCompletionTokenLogprob,
@@ -17,7 +15,6 @@
 
 __all__ = [
     "ModelCapabilities",
-    "ChatCompletionCache",
     "ChatCompletionClient",
     "SystemMessage",
     "UserMessage",
@@ -32,5 +29,4 @@
     "ChatCompletionTokenLogprob",
     "ModelFamily",
     "ModelInfo",
-    "ReplayChatCompletionClient",
 ]

python/packages/autogen-core/tests/test_cache_store.py

@@ -1,6 +1,6 @@
 from unittest.mock import Mock
 
-from autogen_core import CacheStore
+from autogen_core import CacheStore, InMemoryStore
 
 
 def test_set_and_get_object_key_value() -> None:
@@ -30,3 +30,19 @@ def test_set_overwrite_existing_key() -> None:
     mock_store.get.return_value = new_value
     mock_store.set.assert_called_with(key, new_value)
     assert mock_store.get(key) == new_value
+
+
+def test_inmemory_store() -> None:
+    store = InMemoryStore[int]()
+    test_key = "test_key"
+    test_value = 42
+    store.set(test_key, test_value)
+    assert store.get(test_key) == test_value
+
+    new_value = 2
+    store.set(test_key, new_value)
+    assert store.get(test_key) == new_value
+
+    key = "non_existent_key"
+    default_value = 99
+    assert store.get(key, default_value) == default_value

python/packages/autogen-ext/pyproject.toml

@@ -46,6 +46,12 @@ video-surfer = [
     "ffmpeg-python",
     "openai-whisper",
 ]
+diskcache = [
+    "diskcache>=5.6.3"
+]
+redis = [
+    "redis>=5.2.1"
+]
 
 grpc = [
     "grpcio~=1.62.0", # TODO: update this once we have a stable version.

python/packages/autogen-ext/src/autogen_ext/cache_store/diskcache.py

@@ -0,0 +1,26 @@
+from typing import Any, Optional, TypeVar, cast
+
+import diskcache
+from autogen_core import CacheStore
+
+T = TypeVar("T")
+
+
+class DiskCacheStore(CacheStore[T]):
+    """
+    A typed CacheStore implementation that uses diskcache as the underlying storage.
+    See :class:`~autogen_ext.models.cache.ChatCompletionCache` for an example of usage.
+
+    Args:
+        cache_instance: An instance of diskcache.Cache.
+                        The user is responsible for managing the DiskCache instance's lifetime.
+    """
+
+    def __init__(self, cache_instance: diskcache.Cache):  # type: ignore[no-any-unimported]
+        self.cache = cache_instance
+
+    def get(self, key: str, default: Optional[T] = None) -> Optional[T]:
+        return cast(Optional[T], self.cache.get(key, default))  # type: ignore[reportUnknownMemberType]
+
+    def set(self, key: str, value: T) -> None:
+        self.cache.set(key, cast(Any, value))  # type: ignore[reportUnknownMemberType]

python/packages/autogen-ext/src/autogen_ext/cache_store/redis.py

@@ -0,0 +1,29 @@
+from typing import Any, Optional, TypeVar, cast
+
+import redis
+from autogen_core import CacheStore
+
+T = TypeVar("T")
+
+
+class RedisStore(CacheStore[T]):
+    """
+    A typed CacheStore implementation that uses redis as the underlying storage.
+    See :class:`~autogen_ext.models.cache.ChatCompletionCache` for an example of usage.
+
+    Args:
+        cache_instance: An instance of `redis.Redis`.
+                        The user is responsible for managing the Redis instance's lifetime.
+    """
+
+    def __init__(self, redis_instance: redis.Redis):
+        self.cache = redis_instance
+
+    def get(self, key: str, default: Optional[T] = None) -> Optional[T]:
+        value = cast(Optional[T], self.cache.get(key))
+        if value is None:
+            return default
+        return value
+
+    def set(self, key: str, value: T) -> None:
+        self.cache.set(key, cast(Any, value))

python/packages/autogen-ext/src/autogen_ext/models/cache/__init__.py

@@ -0,0 +1,6 @@
+from ._chat_completion_cache import CHAT_CACHE_VALUE_TYPE, ChatCompletionCache
+
+__all__ = [
+    "CHAT_CACHE_VALUE_TYPE",
+    "ChatCompletionCache",
+]