auto-gen 0.4.0dev13 WebSurfer Agent Timeout Error with ACA Dynamic Session

[viswanadula8828]

What happened?

I ran a websurfer agent using MagenticOne Group chat with the below code:

async def initialize_agents():
    # OpenAI Assistant Agent
    client = OpenAIChatCompletionClient(
        model="gpt-4o-2024-08-06",
        api_key=os.getenv("OPENAI_API_KEY")
    )
    openai_client = AsyncClient(api_key=os.getenv("OPENAI_API_KEY"))
    assistant_agent = OpenAIAssistantAgent(
        client=openai_client,
        name="assistant_agent",
        description="Helps with retreiving information from the the attached vector store.",
        model="gpt-4o",
        instructions="You are a retrieval assistant.",
        assistant_id="asst_ZUSoZeLzx24NHiB3dNGAwWRH",
        tools=["file_search"]
    )

    # Web Surfer Agent
    web_surfer_agent = MultimodalWebSurfer(
        name="web_surfer_agent",
        model_client=client
    )

    aca_pool_management_endpoint = os.getenv("POOL_MANAGEMENT_ENDPOINT")
    aca_sessions_executor = ACADynamicSessionsCodeExecutor(aca_pool_management_endpoint, credential = DefaultAzureCredential())
    # Code Executor Agent with Docker
    
    code_executor_agent = CodeExecutorAgent(
        name="code_executor_agent",
        code_executor=aca_sessions_executor
    )

    # Coder Agent
    coder_agent = OpenAIAssistantAgent(
        name="coder_agent",
        description="Helps with Python programming",
        client=openai_client,
        model="gpt-4o",
        instructions="You are a helpful Python programming assistant.",
        tools=["code_interpreter"],
    )

    # Group Chat with RoundRobin strategy
    agent_team = MagenticOneGroupChat(
        [assistant_agent, web_surfer_agent, code_executor_agent, coder_agent], model_client=client)

    return agent_team

async def get_agent_team():
    return await initialize_agents()

@app.get("/")
async def root():
    return RedirectResponse("/docs")


@app.get("/chat")
async def chat(message: str, agent_team= Depends(get_agent_team)):
    try:
        # Initiating chat between CodeExecutor and CodeWriter with the provided message
        chat_result = await agent_team.run(task=message)
        print(chat_result)
        task_result_dict = task_result_to_dict(chat_result)
        result_json = json.dumps(task_result_dict, indent=2)
        print(result_json)
        result_dict = json.loads(result_json)
        messages = result_dict.get("messages", [])

        # Find the last TextMessage
        last_text_message = next(
            (message for message in reversed(messages) if message.get("type") == "TextMessage"),
            None
        )

        # Print the last TextMessage or handle if not found
        if last_text_message:
            last_content = last_text_message.get("content", "")
            print("Last TextMessage:", last_content)
        else:
            print("No TextMessage found.")
        # Find the last message containing 'Code output:' from assistant
        # messages = [message.content for message in chat_result.messages]
        return JSONResponse(content={"chat_history": result_json, "output": last_content})
    except Exception as e:
        raise HTTPException(status_code=500, detail=str(e))

Error Log

"Web surfing error:\n\nTraceback (most recent call last):\n File \"/Users/vlsk/Documents/AI Work/fantastic-chainsaw/.venv/lib/python3.11/site-packages/autogen_ext/agents/web_surfer/_multimodal_web_surfer.py\", line 377, in on_messages_stream\n content = await self._generate_reply(cancellation_token=cancellation_token)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/Users/vlsk/Documents/AI Work/fantastic-chainsaw/.venv/lib/python3.11/site-packages/autogen_ext/agents/web_surfer/_multimodal_web_surfer.py\", line 515, in _generate_reply\n return await self._execute_tool(message, rects, tool_names, cancellation_token=cancellation_token)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/Users/vlsk/Documents/AI Work/fantastic-chainsaw/.venv/lib/python3.11/site-packages/autogen_ext/agents/web_surfer/_multimodal_web_surfer.py\", line 662, in _execute_tool\n await self._page.wait_for_load_state()\n File \"/Users/vlsk/Documents/AI Work/fantastic-chainsaw/.venv/lib/python3.11/site-packages/playwright/async_api/_generated.py\", line 9069, in wait_for_load_state\n await self._impl_obj.wait_for_load_state(state=state, timeout=timeout)\n File \"/Users/vlsk/Documents/AI Work/fantastic-chainsaw/.venv/lib/python3.11/site-packages/playwright/_impl/_page.py\", line 567, in wait_for_load_state\n return await self._main_frame.wait_for_load_state(**locals_to_params(locals()))\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/Users/vlsk/Documents/AI Work/fantastic-chainsaw/.venv/lib/python3.11/site-packages/playwright/_impl/_frame.py\", line 243, in wait_for_load_state\n return await self._wait_for_load_state_impl(state, timeout)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/Users/vlsk/Documents/AI Work/fantastic-chainsaw/.venv/lib/python3.11/site-packages/playwright/_impl/_frame.py\", line 271, in _wait_for_load_state_impl\n await waiter.result()\nplaywright._impl._errors.TimeoutError: Timeout 30000ms exceeded.\

What did you expect to happen?

No errors

How can we reproduce it (as minimally and precisely as possible)?

User WebSurfer Agent within a MagenticOneChat

AutoGen version

0.4.0dev13

Which package was this bug in

Extensions

Model used

gpt-4o

Python version

3.11

Operating system

MacOS

Any additional info you think would be helpful for fixing this bug

No response

[ekzhu]

@viswanadula8828 It is a time out error from the web browser. What's the log? It may be hard to avoid time out in case some websites may become inaccessible.

Perhaps we can catch the timeout error in web surfer @husseinmozannar @afourney

[husseinmozannar]

A question: does the run crash or does it continue as usual but printing the web surfer error?

This web surfer error should already be caught, and this error message is eventually what the web surfer returns. There is no fix for this as some websites sometimes don't load due to either an unstable connection or errors from the website itself. However, the agent and team is not interrupted by such errors

[viswanadula8828]

I get a response from other agents but prints the websurfer error as shown below. This will be an issue when the agent has to get a response only from websurfer

"Web surfing error:\n\nTraceback (most recent call last):\n File \"/Users/vlsk/Documents/AI Work/fantastic-chainsaw/.venv/lib/python3.11/site-packages/autogen_ext/agents/web_surfer/_multimodal_web_surfer.py\", line 377, in on_messages_stream\n content = await self._generate_reply(cancellation_token=cancellation_token)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/Users/vlsk/Documents/AI Work/fantastic-chainsaw/.venv/lib/python3.11/site-packages/autogen_ext/agents/web_surfer/_multimodal_web_surfer.py\", line 515, in _generate_reply\n return await self._execute_tool(message, rects, tool_names, cancellation_token=cancellation_token)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/Users/vlsk/Documents/AI Work/fantastic-chainsaw/.venv/lib/python3.11/site-packages/autogen_ext/agents/web_surfer/_multimodal_web_surfer.py\", line 662, in _execute_tool\n await self._page.wait_for_load_state()\n File \"/Users/vlsk/Documents/AI Work/fantastic-chainsaw/.venv/lib/python3.11/site-packages/playwright/async_api/_generated.py\", line 9069, in wait_for_load_state\n await self._impl_obj.wait_for_load_state(state=state, timeout=timeout)\n File \"/Users/vlsk/Documents/AI Work/fantastic-chainsaw/.venv/lib/python3.11/site-packages/playwright/_impl/_page.py\", line 567, in wait_for_load_state\n return await self._main_frame.wait_for_load_state(**locals_to_params(locals()))\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/Users/vlsk/Documents/AI Work/fantastic-chainsaw/.venv/lib/python3.11/site-packages/playwright/_impl/_frame.py\", line 243, in wait_for_load_state\n return await self._wait_for_load_state_impl(state, timeout)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/Users/vlsk/Documents/AI Work/fantastic-chainsaw/.venv/lib/python3.11/site-packages/playwright/_impl/_frame.py\", line 271, in _wait_for_load_state_impl\n await waiter.result()\nplaywright._impl._errors.TimeoutError: Timeout 30000ms exceeded.\

[ekzhu]

This will be an issue when the agent has to get a response only from websurfer

What's your desired behavior in your case when loading the website times out?

[viswanadula8828]

may be good to have a fallback mechanism and reason for the timeout ?