refactor: Together AI using beta client for structured output & support llama4 (camel-ai#2168)

Wendong-Fan · web-flow · commit b7369dc33f7e · 2025-04-11T09:31:37.000+08:00
diff --git a/camel/models/togetherai_model.py b/camel/models/togetherai_model.py
@@ -96,70 +96,145 @@ def __init__(
             base_url=self._url,
         )
 
-    async def _arun(
+    @property
+    def token_counter(self) -> BaseTokenCounter:
+        r"""Initialize the token counter for the model backend.
+
+        Returns:
+            BaseTokenCounter: The token counter following the model's
+                tokenization style.
+        """
+        if not self._token_counter:
+            self._token_counter = OpenAITokenCounter(ModelType.GPT_4O_MINI)
+        return self._token_counter
+
+    def _run(
         self,
         messages: List[OpenAIMessage],
         response_format: Optional[Type[BaseModel]] = None,
         tools: Optional[List[Dict[str, Any]]] = None,
-    ) -> Union[ChatCompletion, AsyncStream[ChatCompletionChunk]]:
+    ) -> Union[ChatCompletion, Stream[ChatCompletionChunk]]:
         r"""Runs inference of OpenAI chat completion.
 
         Args:
             messages (List[OpenAIMessage]): Message list with the chat history
                 in OpenAI API format.
+            response_format (Optional[Type[BaseModel]]): The format of the
+                response.
+            tools (Optional[List[Dict[str, Any]]]): The schema of the tools to
+                use for the request.
 
         Returns:
-            Union[ChatCompletion, AsyncStream[ChatCompletionChunk]]:
+            Union[ChatCompletion, Stream[ChatCompletionChunk]]:
                 `ChatCompletion` in the non-stream mode, or
-                `AsyncStream[ChatCompletionChunk]` in the stream mode.
+                `Stream[ChatCompletionChunk]` in the stream mode.
         """
-        # Use OpenAI client as interface call Together AI
-        # Reference: https://docs.together.ai/docs/openai-api-compatibility
-        response = await self._async_client.chat.completions.create(
-            messages=messages,
-            model=self.model_type,
-            **self.model_config_dict,
+        response_format = response_format or self.model_config_dict.get(
+            "response_format", None
         )
-        return response
+        if response_format:
+            return self._request_parse(messages, response_format, tools)
+        else:
+            return self._request_chat_completion(messages, tools)
 
-    def _run(
+    async def _arun(
         self,
         messages: List[OpenAIMessage],
         response_format: Optional[Type[BaseModel]] = None,
         tools: Optional[List[Dict[str, Any]]] = None,
-    ) -> Union[ChatCompletion, Stream[ChatCompletionChunk]]:
-        r"""Runs inference of OpenAI chat completion.
+    ) -> Union[ChatCompletion, AsyncStream[ChatCompletionChunk]]:
+        r"""Runs inference of OpenAI chat completion in async mode.
 
         Args:
             messages (List[OpenAIMessage]): Message list with the chat history
                 in OpenAI API format.
+            response_format (Optional[Type[BaseModel]]): The format of the
+                response.
+            tools (Optional[List[Dict[str, Any]]]): The schema of the tools to
+                use for the request.
 
         Returns:
-            Union[ChatCompletion, Stream[ChatCompletionChunk]]:
+            Union[ChatCompletion, AsyncStream[ChatCompletionChunk]]:
                 `ChatCompletion` in the non-stream mode, or
-                `Stream[ChatCompletionChunk]` in the stream mode.
+                `AsyncStream[ChatCompletionChunk]` in the stream mode.
         """
-        # Use OpenAI client as interface call Together AI
-        # Reference: https://docs.together.ai/docs/openai-api-compatibility
-        response = self._client.chat.completions.create(
+        response_format = response_format or self.model_config_dict.get(
+            "response_format", None
+        )
+        if response_format:
+            return await self._arequest_parse(messages, response_format, tools)
+        else:
+            return await self._arequest_chat_completion(messages, tools)
+
+    def _request_chat_completion(
+        self,
+        messages: List[OpenAIMessage],
+        tools: Optional[List[Dict[str, Any]]] = None,
+    ) -> Union[ChatCompletion, Stream[ChatCompletionChunk]]:
+        request_config = self.model_config_dict.copy()
+
+        if tools:
+            request_config["tools"] = tools
+
+        return self._client.chat.completions.create(
             messages=messages,
             model=self.model_type,
-            **self.model_config_dict,
+            **request_config,
         )
-        return response
 
-    @property
-    def token_counter(self) -> BaseTokenCounter:
-        r"""Initialize the token counter for the model backend.
+    async def _arequest_chat_completion(
+        self,
+        messages: List[OpenAIMessage],
+        tools: Optional[List[Dict[str, Any]]] = None,
+    ) -> Union[ChatCompletion, AsyncStream[ChatCompletionChunk]]:
+        request_config = self.model_config_dict.copy()
 
-        Returns:
-            OpenAITokenCounter: The token counter following the model's
-                tokenization style.
-        """
+        if tools:
+            request_config["tools"] = tools
 
-        if not self._token_counter:
-            self._token_counter = OpenAITokenCounter(ModelType.GPT_4O_MINI)
-        return self._token_counter
+        return await self._async_client.chat.completions.create(
+            messages=messages,
+            model=self.model_type,
+            **request_config,
+        )
+
+    def _request_parse(
+        self,
+        messages: List[OpenAIMessage],
+        response_format: Type[BaseModel],
+        tools: Optional[List[Dict[str, Any]]] = None,
+    ) -> ChatCompletion:
+        request_config = self.model_config_dict.copy()
+
+        request_config["response_format"] = response_format
+
+        if tools is not None:
+            request_config["tools"] = tools
+
+        return self._client.beta.chat.completions.parse(
+            messages=messages,
+            model=self.model_type,
+            **request_config,
+        )
+
+    async def _arequest_parse(
+        self,
+        messages: List[OpenAIMessage],
+        response_format: Type[BaseModel],
+        tools: Optional[List[Dict[str, Any]]] = None,
+    ) -> ChatCompletion:
+        request_config = self.model_config_dict.copy()
+
+        request_config["response_format"] = response_format
+
+        if tools is not None:
+            request_config["tools"] = tools
+
+        return await self._async_client.beta.chat.completions.parse(
+            messages=messages,
+            model=self.model_type,
+            **request_config,
+        )
 
     def check_model_config(self):
         r"""Check whether the model configuration contains any
diff --git a/camel/toolkits/browser_toolkit.py b/camel/toolkits/browser_toolkit.py
@@ -1175,7 +1175,7 @@ def _observe(
         message = BaseMessage.make_user_message(
             role_name='user', content=observe_prompt, image_list=[img]
         )
-        r""" Reset the history message of web_agent. Important history information is already included in the history field in observe_prompt."""
+        # Reset the history message of web_agent.
         self.web_agent.reset()
         resp = self.web_agent.step(message)
 
@@ -1406,7 +1406,7 @@ def _task_replanning(
 - `if_need_replan`: bool, A boolean value indicating whether the task needs to be fundamentally replanned.
 - `replanned_schema`: str, The replanned schema for the task, which should not be changed too much compared with the original one. If the task does not need to be replanned, the value should be an empty string. 
 """
-        r""" Reset the history message of planning_agent. Important history information is already included in the history field in replanning_prompt."""
+        # Reset the history message of planning_agent.
         self.planning_agent.reset()
         resp = self.planning_agent.step(replanning_prompt)
         resp_dict = _parse_json_output(resp.msgs[0].content)
diff --git a/camel/types/enums.py b/camel/types/enums.py
@@ -79,6 +79,10 @@ class ModelType(UnifiedModelType, Enum):
     TOGETHER_LLAMA_3_3_70B = "meta-llama/Llama-3.3-70B-Instruct-Turbo"
     TOGETHER_MIXTRAL_8_7B = "mistralai/Mixtral-8x7B-Instruct-v0.1"
     TOGETHER_MISTRAL_7B = "mistralai/Mistral-7B-Instruct-v0.1"
+    TOGETHER_LLAMA_4_MAVERICK = (
+        "meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8"
+    )
+    TOGETHER_LLAMA_4_SCOUT = "meta-llama/Llama-4-Scout-17B-16E-Instruct"
 
     # PPIO platform models support tool calling
     PPIO_DEEPSEEK_R1_TURBO = "deepseek/deepseek-r1-turbo"
@@ -851,11 +855,13 @@ def token_limit(self) -> int:
             ModelType.GEMINI_2_0_FLASH_LITE_PREVIEW,
             ModelType.GEMINI_2_0_PRO_EXP,  # Not given in doc, assume the same
             ModelType.GLM_4_LONG,
+            ModelType.TOGETHER_LLAMA_4_MAVERICK,
+            ModelType.OPENROUTER_LLAMA_4_MAVERICK,
         }:
             return 1_048_576
         elif self in {
             ModelType.QWEN_LONG,
-            ModelType.OPENROUTER_LLAMA_4_MAVERICK,
+            ModelType.TOGETHER_LLAMA_4_SCOUT,
         }:
             return 10_000_000
         else:
diff --git a/examples/models/togetherai_model_example.py b/examples/models/togetherai_model_example.py
@@ -14,19 +14,19 @@
 from camel.agents import ChatAgent
 from camel.configs import TogetherAIConfig
 from camel.models import ModelFactory
-from camel.types import ModelPlatformType
+from camel.types import ModelPlatformType, ModelType
 
 model = ModelFactory.create(
     model_platform=ModelPlatformType.TOGETHER,
-    model_type="meta-llama/Llama-3-8b-chat-hf",
+    model_type=ModelType.TOGETHER_LLAMA_4_MAVERICK,
     model_config_dict=TogetherAIConfig(temperature=0.2).as_dict(),
 )
 
 # Define system message
 sys_msg = "You are a helpful assistant."
 
 # Set agent
-camel_agent = ChatAgent(system_message=sys_msg, model=model, token_limit=500)
+camel_agent = ChatAgent(system_message=sys_msg, model=model)
 
 user_msg = """Say hi to CAMEL AI, one open-source community dedicated to the 
     study of autonomous and communicative agents."""
@@ -36,14 +36,38 @@
 print(response.msgs[0].content)
 '''
 ===============================================================================
-Hello CAMEL AI community!
+Hello CAMEL AI! It's great to connect with an open-source community that's 
+pushing the boundaries of autonomous and communicative agents. I'm excited to 
+learn more about the innovative work being done here. What are some of the 
+most interesting projects or research areas that CAMEL AI is currently 
+exploring?
+===============================================================================
+'''
+
+model = ModelFactory.create(
+    model_platform=ModelPlatformType.TOGETHER,
+    model_type=ModelType.TOGETHER_LLAMA_3_3_70B,
+    model_config_dict=TogetherAIConfig(temperature=0.2).as_dict(),
+)
 
-I'm thrilled to be here and assist you with any questions or topics related to 
-autonomous and communicative agents. As an open-source community, I'm excited 
-to see the innovative projects and research being developed by your members.
+# Define system message
+sys_msg = "You are a helpful assistant."
 
-What's on your mind? Do you have a specific question, project, or topic you'd 
-like to discuss? I'm here to help and provide any assistance I can. Let's get 
-started!
+# Set agent
+camel_agent = ChatAgent(system_message=sys_msg, model=model)
+
+user_msg = """Say hi to CAMEL AI, one open-source community dedicated to the 
+    study of autonomous and communicative agents."""
+
+# Get response information
+response = camel_agent.step(user_msg)
+print(response.msgs[0].content)
+'''
+===============================================================================
+Hello CAMEL AI community. It's great to connect with a group of like-minded 
+individuals dedicated to advancing the field of autonomous and communicative 
+agents. Your open-source approach to sharing knowledge and resources is truly 
+commendable, and I'm excited to see the innovative projects and research that 
+come out of your community. How can I assist or contribute to your endeavors?
 ===============================================================================
 '''
