From acd4767ab87adb9b537cc946d0bec7c2c3c765af Mon Sep 17 00:00:00 2001 From: samolego <34912839+samolego@users.noreply.github.com> Date: Wed, 7 May 2025 19:05:45 +0200 Subject: [PATCH 1/9] feat(server): add basic js tool call support --- tools/server/webui/package.json | 1 + tools/server/webui/src/Config.ts | 5 +- .../webui/src/components/SettingDialog.tsx | 15 ++ tools/server/webui/src/utils/app.context.tsx | 138 +++++++++++++++--- tools/server/webui/src/utils/js_tool_call.ts | 18 +++ tools/server/webui/src/utils/types.ts | 9 ++ 6 files changed, 161 insertions(+), 25 deletions(-) create mode 100644 tools/server/webui/src/utils/js_tool_call.ts diff --git a/tools/server/webui/package.json b/tools/server/webui/package.json index 6ac06b1a49bd3..153ae54f46c9f 100644 --- a/tools/server/webui/package.json +++ b/tools/server/webui/package.json @@ -33,6 +33,7 @@ "remark-math": "^6.0.0", "tailwindcss": "^4.1.1", "textlinestream": "^1.1.1", + "unist-util-visit": "^5.0.0", "vite-plugin-singlefile": "^2.0.3" }, "devDependencies": { diff --git a/tools/server/webui/src/Config.ts b/tools/server/webui/src/Config.ts index dd1cc0e100a2e..2f5a742c2b6c2 100644 --- a/tools/server/webui/src/Config.ts +++ b/tools/server/webui/src/Config.ts @@ -4,9 +4,7 @@ import { isNumeric } from './utils/misc'; export const isDev = import.meta.env.MODE === 'development'; // constants -export const BASE_URL = new URL('.', document.baseURI).href - .toString() - .replace(/\/$/, ''); +export const BASE_URL = 'http://127.0.0.1:8080'; export const CONFIG_DEFAULT = { // Note: in order not to introduce breaking changes, please keep the same data type (number, string, etc) if you want to change the default value. Do not use null or undefined for default value. @@ -39,6 +37,7 @@ export const CONFIG_DEFAULT = { custom: '', // custom json-stringified object // experimental features pyIntepreterEnabled: false, + jsInterpreterToolUse: false, }; export const CONFIG_INFO: Record = { apiKey: 'Set the API Key if you are using --api-key option for the server.', diff --git a/tools/server/webui/src/components/SettingDialog.tsx b/tools/server/webui/src/components/SettingDialog.tsx index b0044d25403b5..7baf17dd27a23 100644 --- a/tools/server/webui/src/components/SettingDialog.tsx +++ b/tools/server/webui/src/components/SettingDialog.tsx @@ -254,6 +254,21 @@ const SETTING_SECTIONS: SettingSection[] = [ ), key: 'pyIntepreterEnabled', }, + { + type: SettingInputType.CHECKBOX, + label: ( + <> + Enable JavaScript tool use +
+ + This alows LLM to use browser your browser console as tool. If + model supports function calling, it can use the console to do e.g. + data analysis etc. by itself. + + + ), + key: 'jsInterpreterToolUse', + }, ], }, ]; diff --git a/tools/server/webui/src/utils/app.context.tsx b/tools/server/webui/src/utils/app.context.tsx index 54bb65b6e3cb2..de306d85055e5 100644 --- a/tools/server/webui/src/utils/app.context.tsx +++ b/tools/server/webui/src/utils/app.context.tsx @@ -5,6 +5,7 @@ import { Conversation, Message, PendingMessage, + ToolCall, ViewingChat, } from './types'; import StorageUtils from './storage'; @@ -15,6 +16,7 @@ import { } from './misc'; import { BASE_URL, CONFIG_DEFAULT, isDev } from '../Config'; import { matchPath, useLocation, useNavigate } from 'react-router'; +import { JS_TOOL_CALL_SPEC } from './js_tool_call'; interface AppContextValue { // conversations and messages @@ -181,10 +183,13 @@ export const AppContextProvider = ({ } if (isDev) console.log({ messages }); + // stream does not support tool-use + const streamResponse = !config.jsInterpreterToolUse; + // prepare params const params = { messages, - stream: true, + stream: streamResponse, cache_prompt: true, samplers: config.samplers, temperature: config.temperature, @@ -206,6 +211,7 @@ export const AppContextProvider = ({ dry_penalty_last_n: config.dry_penalty_last_n, max_tokens: config.max_tokens, timings_per_token: !!config.showTokensPerSecond, + tools: config.jsInterpreterToolUse ? [JS_TOOL_CALL_SPEC] : undefined, ...(config.custom.length ? JSON.parse(config.custom) : {}), }; @@ -221,36 +227,124 @@ export const AppContextProvider = ({ body: JSON.stringify(params), signal: abortController.signal, }); + if (fetchResponse.status !== 200) { const body = await fetchResponse.json(); throw new Error(body?.error?.message || 'Unknown error'); } - const chunks = getSSEStreamAsync(fetchResponse); - for await (const chunk of chunks) { - // const stop = chunk.stop; - if (chunk.error) { - throw new Error(chunk.error?.message || 'Unknown error'); + + if (streamResponse) { + const chunks = getSSEStreamAsync(fetchResponse); + for await (const chunk of chunks) { + // const stop = chunk.stop; + if (chunk.error) { + throw new Error(chunk.error?.message || 'Unknown error'); + } + const addedContent = chunk.choices[0].delta.content; + const lastContent = pendingMsg.content || ''; + if (addedContent) { + pendingMsg = { + ...pendingMsg, + content: lastContent + addedContent, + }; + } + const timings = chunk.timings; + if (timings && config.showTokensPerSecond) { + // only extract what's really needed, to save some space + pendingMsg.timings = { + prompt_n: timings.prompt_n, + prompt_ms: timings.prompt_ms, + predicted_n: timings.predicted_n, + predicted_ms: timings.predicted_ms, + }; + } + setPending(convId, pendingMsg); + onChunk(); // don't need to switch node for pending message } - const addedContent = chunk.choices[0].delta.content; - const lastContent = pendingMsg.content || ''; - if (addedContent) { + } else { + const responseData = await fetchResponse.json(); + if (isDev) console.log({ responseData }); + + const choice = responseData.choices?.[0]; + if (choice) { + const messageFromAPI = choice.message; + let newContent = ''; + + if (messageFromAPI.content) { + newContent = messageFromAPI.content; + } + + if ( + messageFromAPI.tool_calls && + messageFromAPI.tool_calls.length > 0 + ) { + console.log(messageFromAPI.tool_calls[0]); + for (let i = 0; i < messageFromAPI.tool_calls.length; i++) { + console.log('Tool use #' + i); + const tc = messageFromAPI.tool_calls[i] as ToolCall; + console.log(tc); + + if (tc) { + if ( + tc.function.name === 'javascript_interpreter' && + config.jsInterpreterToolUse + ) { + // Execute code provided + const args = JSON.parse(tc.function.arguments); + console.log('Arguments for tool call:'); + console.log(args); + const result = eval(args.code); + console.log(result); + + newContent += `${result}`; + } + } + } + + const toolCallsInfo = messageFromAPI.tool_calls + .map( + ( + tc: any // Use 'any' for tc temporarily if type is not imported/defined here + ) => + `Tool Call Invoked: ${tc.function.name}\nArguments: ${tc.function.arguments}` + ) + .join('\n\n'); + + if (newContent.length > 0) { + newContent += `\n\n${toolCallsInfo}`; + } else { + newContent = toolCallsInfo; + } + // TODO: Ideally, store structured tool_calls in pendingMsg if its type supports it. + // pendingMsg.tool_calls = messageFromAPI.tool_calls; + } + pendingMsg = { ...pendingMsg, - content: lastContent + addedContent, - }; - } - const timings = chunk.timings; - if (timings && config.showTokensPerSecond) { - // only extract what's really needed, to save some space - pendingMsg.timings = { - prompt_n: timings.prompt_n, - prompt_ms: timings.prompt_ms, - predicted_n: timings.predicted_n, - predicted_ms: timings.predicted_ms, + content: newContent, }; + + // Handle timings from the non-streaming response + // The exact location of 'timings' in responseData might vary by API. + // Assuming responseData.timings similar to streaming chunk for now. + const apiTimings = responseData.timings; + if (apiTimings && config.showTokensPerSecond) { + pendingMsg.timings = { + prompt_n: apiTimings.prompt_n, + prompt_ms: apiTimings.prompt_ms, + predicted_n: apiTimings.predicted_n, + predicted_ms: apiTimings.predicted_ms, + }; + } + setPending(convId, pendingMsg); + onChunk(); // Update UI to show the processed message + } else { + console.error( + 'API response missing choices or message:', + responseData + ); + throw new Error('Invalid API response structure'); } - setPending(convId, pendingMsg); - onChunk(); // don't need to switch node for pending message } } catch (err) { setPending(convId, null); diff --git a/tools/server/webui/src/utils/js_tool_call.ts b/tools/server/webui/src/utils/js_tool_call.ts new file mode 100644 index 0000000000000..a4dc1822ea81d --- /dev/null +++ b/tools/server/webui/src/utils/js_tool_call.ts @@ -0,0 +1,18 @@ +export const JS_TOOL_CALL_SPEC = { + type: 'function', + function: { + name: 'javascript_interpreter', + description: + 'Executes JavaScript code in the browser console and returns the output or error. The code should be self-contained. Use JSON.stringify for complex return objects.', + parameters: { + type: 'object', + properties: { + code: { + type: 'string', + description: 'The JavaScript code to execute.', + }, + }, + required: ['code'], + }, + }, +}; diff --git a/tools/server/webui/src/utils/types.ts b/tools/server/webui/src/utils/types.ts index 0eb774001ecc5..be8ec10d2bbd4 100644 --- a/tools/server/webui/src/utils/types.ts +++ b/tools/server/webui/src/utils/types.ts @@ -89,3 +89,12 @@ export interface CanvasPyInterpreter { } export type CanvasData = CanvasPyInterpreter; + +export interface ToolCall { + id: string; + type: 'function'; + function: { + name: string; + arguments: string; // JSON string of arguments + }; +} From 623691885c6067c694e93a4d39cfc5fe5c08295c Mon Sep 17 00:00:00 2001 From: samolego <34912839+samolego@users.noreply.github.com> Date: Thu, 8 May 2025 14:13:45 +0200 Subject: [PATCH 2/9] code abstraction for tool calling --- .../webui/src/components/ChatMessage.tsx | 107 ++++++++++++++---- tools/server/webui/src/utils/app.context.tsx | 51 +++++---- tools/server/webui/src/utils/js_tool_call.ts | 18 --- .../src/utils/tool_calling/available_tools.ts | 53 +++++++++ .../src/utils/tool_calling/js_tool_call.ts | 84 ++++++++++++++ tools/server/webui/src/utils/types.ts | 20 ++++ 6 files changed, 271 insertions(+), 62 deletions(-) delete mode 100644 tools/server/webui/src/utils/js_tool_call.ts create mode 100644 tools/server/webui/src/utils/tool_calling/available_tools.ts create mode 100644 tools/server/webui/src/utils/tool_calling/js_tool_call.ts diff --git a/tools/server/webui/src/components/ChatMessage.tsx b/tools/server/webui/src/components/ChatMessage.tsx index 40ea74711f349..86f4bc9cfcd95 100644 --- a/tools/server/webui/src/components/ChatMessage.tsx +++ b/tools/server/webui/src/components/ChatMessage.tsx @@ -9,6 +9,8 @@ interface SplitMessage { content: PendingMessage['content']; thought?: string; isThinking?: boolean; + toolOutput?: string; + toolTitle?: string; } export default function ChatMessage({ @@ -48,32 +50,71 @@ export default function ChatMessage({ const nextSibling = siblingLeafNodeIds[siblingCurrIdx + 1]; const prevSibling = siblingLeafNodeIds[siblingCurrIdx - 1]; - // for reasoning model, we split the message into content and thought - // TODO: implement this as remark/rehype plugin in the future - const { content, thought, isThinking }: SplitMessage = useMemo(() => { - if (msg.content === null || msg.role !== 'assistant') { - return { content: msg.content }; - } - let actualContent = ''; - let thought = ''; - let isThinking = false; - let thinkSplit = msg.content.split('', 2); - actualContent += thinkSplit[0]; - while (thinkSplit[1] !== undefined) { - // tag found - thinkSplit = thinkSplit[1].split('', 2); - thought += thinkSplit[0]; - isThinking = true; - if (thinkSplit[1] !== undefined) { - // closing tag found - isThinking = false; - thinkSplit = thinkSplit[1].split('', 2); - actualContent += thinkSplit[0]; + // for reasoning model, we split the message into content, thought, and tool output + const { content, thought, isThinking, toolOutput, toolTitle }: SplitMessage = + useMemo(() => { + if (msg.content === null || msg.role !== 'assistant') { + return { content: msg.content }; } - } - return { content: actualContent, thought, isThinking }; - }, [msg]); + let currentContent = msg.content; + let extractedThought: string | undefined = undefined; + let isCurrentlyThinking = false; + let extractedToolOutput: string | undefined = undefined; + let extractedToolTitle: string | undefined = 'Tool Output'; + // Process tags + const thinkParts = currentContent.split(''); + currentContent = thinkParts[0]; + if (thinkParts.length > 1) { + isCurrentlyThinking = true; + const tempThoughtArray: string[] = []; + for (let i = 1; i < thinkParts.length; i++) { + const thinkSegment = thinkParts[i].split(''); + tempThoughtArray.push(thinkSegment[0]); + if (thinkSegment.length > 1) { + isCurrentlyThinking = false; // Closing tag found + currentContent += thinkSegment[1]; + } + } + extractedThought = tempThoughtArray.join('\n'); + } + + // Process tags (after thoughts are processed) + const toolParts = currentContent.split(''); + console.log(toolParts); + currentContent = toolParts[0]; + if (toolParts.length > 1) { + const tempToolOutputArray: string[] = []; + for (let i = 1; i < toolParts.length; i++) { + const toolSegment = toolParts[i].split(''); + const toolContent = toolSegment[0].trim(); + + const firstLineEnd = toolContent.indexOf('\n'); + if (firstLineEnd !== -1) { + extractedToolTitle = toolContent.substring(0, firstLineEnd); + tempToolOutputArray.push( + toolContent.substring(firstLineEnd + 1).trim() + ); + } else { + // If no newline, extractedToolTitle keeps its default; toolContent is pushed as is. + tempToolOutputArray.push(toolContent); + } + + if (toolSegment.length > 1) { + currentContent += toolSegment[1]; + } + } + extractedToolOutput = tempToolOutputArray.join('\n\n'); + } + + return { + content: currentContent.trim(), + thought: extractedThought, + isThinking: isCurrentlyThinking, + toolOutput: extractedToolOutput, + toolTitle: extractedToolTitle, + }; + }, [msg]); if (!viewingChat) return null; return ( @@ -192,6 +233,24 @@ export default function ChatMessage({ content={content} isGenerating={isPending} /> + + {toolOutput && ( +
+ + {toolTitle || 'Tool Output'} + +
+ +
+
+ )} )} diff --git a/tools/server/webui/src/utils/app.context.tsx b/tools/server/webui/src/utils/app.context.tsx index de306d85055e5..886466d56a62d 100644 --- a/tools/server/webui/src/utils/app.context.tsx +++ b/tools/server/webui/src/utils/app.context.tsx @@ -16,7 +16,7 @@ import { } from './misc'; import { BASE_URL, CONFIG_DEFAULT, isDev } from '../Config'; import { matchPath, useLocation, useNavigate } from 'react-router'; -import { JS_TOOL_CALL_SPEC } from './js_tool_call'; +import { AVAILABLE_TOOLS } from './tool_calling/available_tools'; interface AppContextValue { // conversations and messages @@ -211,7 +211,11 @@ export const AppContextProvider = ({ dry_penalty_last_n: config.dry_penalty_last_n, max_tokens: config.max_tokens, timings_per_token: !!config.showTokensPerSecond, - tools: config.jsInterpreterToolUse ? [JS_TOOL_CALL_SPEC] : undefined, + tools: config.jsInterpreterToolUse + ? Array.from(AVAILABLE_TOOLS, ([_name, tool], _index) => tool).filter( + (tool) => tool.enabled() + ) + : undefined, ...(config.custom.length ? JSON.parse(config.custom) : {}), }; @@ -278,35 +282,37 @@ export const AppContextProvider = ({ messageFromAPI.tool_calls && messageFromAPI.tool_calls.length > 0 ) { - console.log(messageFromAPI.tool_calls[0]); + console.log(messageFromAPI.tool_calls); for (let i = 0; i < messageFromAPI.tool_calls.length; i++) { console.log('Tool use #' + i); const tc = messageFromAPI.tool_calls[i] as ToolCall; console.log(tc); if (tc) { - if ( - tc.function.name === 'javascript_interpreter' && - config.jsInterpreterToolUse - ) { - // Execute code provided - const args = JSON.parse(tc.function.arguments); - console.log('Arguments for tool call:'); - console.log(args); - const result = eval(args.code); - console.log(result); - - newContent += `${result}`; + // Set up call id + tc.call_id ??= `call_${i}`; + + // Process tool call + const toolResult = AVAILABLE_TOOLS.get( + tc.function.name + )?.processCall(tc); + + if (toolResult) { + newContent += `Tool use: ${tc.function.name}\n\n`; + newContent += toolResult.output; + newContent += ''; + } else { + newContent += `Tool use: ${tc.function.name}\n\nError: invalid tool call!\n`; } } } - const toolCallsInfo = messageFromAPI.tool_calls + /*const toolCallsInfo = messageFromAPI.tool_calls .map( ( tc: any // Use 'any' for tc temporarily if type is not imported/defined here ) => - `Tool Call Invoked: ${tc.function.name}\nArguments: ${tc.function.arguments}` + `Tool invoked: ${tc.function.name}\nArguments: ${tc.function.arguments}` ) .join('\n\n'); @@ -314,9 +320,7 @@ export const AppContextProvider = ({ newContent += `\n\n${toolCallsInfo}`; } else { newContent = toolCallsInfo; - } - // TODO: Ideally, store structured tool_calls in pendingMsg if its type supports it. - // pendingMsg.tool_calls = messageFromAPI.tool_calls; + }*/ } pendingMsg = { @@ -338,6 +342,13 @@ export const AppContextProvider = ({ } setPending(convId, pendingMsg); onChunk(); // Update UI to show the processed message + + // if message ended due to "finish_reason": "tool_calls" + // resend it to assistant to process the result. + if (choice.finish_reason === 'tool_calls') { + console.log('Ended due to tool call. Interpreting results ...'); + generateMessage(convId, pendingId, onChunk); + } } else { console.error( 'API response missing choices or message:', diff --git a/tools/server/webui/src/utils/js_tool_call.ts b/tools/server/webui/src/utils/js_tool_call.ts deleted file mode 100644 index a4dc1822ea81d..0000000000000 --- a/tools/server/webui/src/utils/js_tool_call.ts +++ /dev/null @@ -1,18 +0,0 @@ -export const JS_TOOL_CALL_SPEC = { - type: 'function', - function: { - name: 'javascript_interpreter', - description: - 'Executes JavaScript code in the browser console and returns the output or error. The code should be self-contained. Use JSON.stringify for complex return objects.', - parameters: { - type: 'object', - properties: { - code: { - type: 'string', - description: 'The JavaScript code to execute.', - }, - }, - required: ['code'], - }, - }, -}; diff --git a/tools/server/webui/src/utils/tool_calling/available_tools.ts b/tools/server/webui/src/utils/tool_calling/available_tools.ts new file mode 100644 index 0000000000000..af5cf2152d4eb --- /dev/null +++ b/tools/server/webui/src/utils/tool_calling/available_tools.ts @@ -0,0 +1,53 @@ +import { ToolCall, ToolCallOutput, ToolCallSpec } from '../types'; + +/** + * Map of available tools for function calling. + * Note that these tools are not necessarily enabled by the user. + */ +export const AVAILABLE_TOOLS = new Map(); + +export abstract class AgentTool { + id: string; + isEnabled: () => boolean; + + constructor(id: string, enabled: () => boolean) { + this.id = id; + this.isEnabled = enabled; + AVAILABLE_TOOLS.set(id, this); + } + + /** + * "Public" wrapper for the tool call processing logic. + * @param call The tool call object from the API response. + * @returns The tool call output or undefined if the tool is not enabled. + */ + public processCall(call: ToolCall): ToolCallOutput | undefined { + if (this.enabled()) { + return this._process(call); + } + + return undefined; + } + + /** + * Whether calling this tool is enabled. + * User can toggle the status from the settings panel. + * @returns enabled status. + */ + public enabled(): boolean { + return this.isEnabled(); + } + + /** + * Specifications for the tool call. + * https://github.com/ggml-org/llama.cpp/blob/master/docs/function-calling.md + * https://platform.openai.com/docs/guides/function-calling?api-mode=responses#defining-functions + */ + public abstract specs(): ToolCallSpec; + + /** + * The actual tool call processing logic. + * @param call: The tool call object from the API response. + */ + protected abstract _process(call: ToolCall): ToolCallOutput; +} diff --git a/tools/server/webui/src/utils/tool_calling/js_tool_call.ts b/tools/server/webui/src/utils/tool_calling/js_tool_call.ts new file mode 100644 index 0000000000000..b00dec94e8927 --- /dev/null +++ b/tools/server/webui/src/utils/tool_calling/js_tool_call.ts @@ -0,0 +1,84 @@ +import StorageUtils from '../storage'; +import { ToolCall, ToolCallOutput, ToolCallSpec } from '../types'; +import { AgentTool } from './available_tools'; + +class JSReplAgentTool extends AgentTool { + private static readonly id = 'javascript_interpreter'; + private fakeLogger: FakeConsoleLog; + + constructor() { + super( + JSReplAgentTool.id, + () => StorageUtils.getConfig().jsInterpreterToolUse + ); + this.fakeLogger = new FakeConsoleLog(); + } + + _process(tc: ToolCall): ToolCallOutput { + const args = JSON.parse(tc.function.arguments); + console.log('Arguments for tool call:'); + console.log(args); + + // Redirect console.log which agent will use to + // the fake logger so that later we can get the content + const originalConsoleLog = console.log; + console.log = this.fakeLogger.log; + + let result = ''; + try { + // Evaluate the provided agent code + result = eval(args.code); + } catch (err) { + result = String(err); + } finally { + // Ensure original console.log is restored even if eval throws + console.log = originalConsoleLog; + } + + result = this.fakeLogger.content + result; + + this.fakeLogger.clear(); + + return { call_id: tc.call_id, output: result } as ToolCallOutput; + } + + public specs(): ToolCallSpec { + return { + type: 'function', + function: { + name: this.id, + description: + 'Executes JavaScript code in the browser console. The code should be self-contained valid javascript. You can use console.log(variable) to print out intermediate values..', + parameters: { + type: 'object', + properties: { + code: { + type: 'string', + description: 'Valid JavaScript code to execute.', + }, + }, + required: ['code'], + }, + }, + }; + } +} +export const jsAgentTool = new JSReplAgentTool(); + +class FakeConsoleLog { + private _content: string = ''; + + public get content(): string { + return this._content; + } + + // Use an arrow function for log to correctly bind 'this' + public log = (...args: any[]): void => { + // Convert arguments to strings and join them. + this._content += args.map((arg) => String(arg)).join(' ') + '\n'; + }; + + public clear = (): void => { + this._content = ''; + }; +} diff --git a/tools/server/webui/src/utils/types.ts b/tools/server/webui/src/utils/types.ts index be8ec10d2bbd4..16280924c18b8 100644 --- a/tools/server/webui/src/utils/types.ts +++ b/tools/server/webui/src/utils/types.ts @@ -93,8 +93,28 @@ export type CanvasData = CanvasPyInterpreter; export interface ToolCall { id: string; type: 'function'; + call_id: string; function: { name: string; arguments: string; // JSON string of arguments }; } + +export interface ToolCallSpec { + type: 'function'; + function: { + name: string; + description: string; + parameters: { + type: 'object'; + properties: object; + required: string[]; + }; + }; +} + +export interface ToolCallOutput { + type: 'function_call_output'; + call_id: string; + output: string; +} From e84e819bae9ad29266f27bad7a07f872949ede04 Mon Sep 17 00:00:00 2001 From: samolego <34912839+samolego@users.noreply.github.com> Date: Thu, 8 May 2025 14:57:49 +0200 Subject: [PATCH 3/9] minor changes, renames --- .../webui/src/components/ChatMessage.tsx | 1 - tools/server/webui/src/utils/app.context.tsx | 8 ++-- tools/server/webui/src/utils/misc.ts | 1 - .../{available_tools.ts => agent_tool.ts} | 36 ++++++++++---- .../{js_tool_call.ts => js_repl_tool.ts} | 47 +++++++------------ .../src/utils/tool_calling/register_tools.ts | 21 +++++++++ tools/server/webui/src/utils/types.ts | 14 +++--- 7 files changed, 75 insertions(+), 53 deletions(-) rename tools/server/webui/src/utils/tool_calling/{available_tools.ts => agent_tool.ts} (66%) rename tools/server/webui/src/utils/tool_calling/{js_tool_call.ts => js_repl_tool.ts} (55%) create mode 100644 tools/server/webui/src/utils/tool_calling/register_tools.ts diff --git a/tools/server/webui/src/components/ChatMessage.tsx b/tools/server/webui/src/components/ChatMessage.tsx index 86f4bc9cfcd95..2419703482699 100644 --- a/tools/server/webui/src/components/ChatMessage.tsx +++ b/tools/server/webui/src/components/ChatMessage.tsx @@ -81,7 +81,6 @@ export default function ChatMessage({ // Process tags (after thoughts are processed) const toolParts = currentContent.split(''); - console.log(toolParts); currentContent = toolParts[0]; if (toolParts.length > 1) { const tempToolOutputArray: string[] = []; diff --git a/tools/server/webui/src/utils/app.context.tsx b/tools/server/webui/src/utils/app.context.tsx index 886466d56a62d..1b459e2beba3b 100644 --- a/tools/server/webui/src/utils/app.context.tsx +++ b/tools/server/webui/src/utils/app.context.tsx @@ -16,7 +16,7 @@ import { } from './misc'; import { BASE_URL, CONFIG_DEFAULT, isDev } from '../Config'; import { matchPath, useLocation, useNavigate } from 'react-router'; -import { AVAILABLE_TOOLS } from './tool_calling/available_tools'; +import { AVAILABLE_TOOLS } from './tool_calling/register_tools'; interface AppContextValue { // conversations and messages @@ -212,9 +212,9 @@ export const AppContextProvider = ({ max_tokens: config.max_tokens, timings_per_token: !!config.showTokensPerSecond, tools: config.jsInterpreterToolUse - ? Array.from(AVAILABLE_TOOLS, ([_name, tool], _index) => tool).filter( - (tool) => tool.enabled() - ) + ? Array.from(AVAILABLE_TOOLS, ([_name, tool], _index) => tool) + .filter((tool) => tool.enabled()) + .map((tool) => tool.specs()) : undefined, ...(config.custom.length ? JSON.parse(config.custom) : {}), }; diff --git a/tools/server/webui/src/utils/misc.ts b/tools/server/webui/src/utils/misc.ts index 87f55b2af95c2..a47720fa6f773 100644 --- a/tools/server/webui/src/utils/misc.ts +++ b/tools/server/webui/src/utils/misc.ts @@ -22,7 +22,6 @@ export async function* getSSEStreamAsync(fetchResponse: Response) { .pipeThrough(new TextLineStream()); // @ts-expect-error asyncIterator complains about type, but it should work for await (const line of asyncIterator(lines)) { - //if (isDev) console.log({ line }); if (line.startsWith('data:') && !line.endsWith('[DONE]')) { const data = JSON.parse(line.slice(5)); yield data; diff --git a/tools/server/webui/src/utils/tool_calling/available_tools.ts b/tools/server/webui/src/utils/tool_calling/agent_tool.ts similarity index 66% rename from tools/server/webui/src/utils/tool_calling/available_tools.ts rename to tools/server/webui/src/utils/tool_calling/agent_tool.ts index af5cf2152d4eb..f6f24691a0aed 100644 --- a/tools/server/webui/src/utils/tool_calling/available_tools.ts +++ b/tools/server/webui/src/utils/tool_calling/agent_tool.ts @@ -1,19 +1,26 @@ -import { ToolCall, ToolCallOutput, ToolCallSpec } from '../types'; - -/** - * Map of available tools for function calling. - * Note that these tools are not necessarily enabled by the user. - */ -export const AVAILABLE_TOOLS = new Map(); +import { + ToolCall, + ToolCallOutput, + ToolCallParameters, + ToolCallSpec, +} from '../types'; export abstract class AgentTool { id: string; isEnabled: () => boolean; + toolDescription: string; + parameters: ToolCallParameters; - constructor(id: string, enabled: () => boolean) { + constructor( + id: string, + enabled: () => boolean, + toolDescription: string, + parameters: ToolCallParameters + ) { this.id = id; this.isEnabled = enabled; - AVAILABLE_TOOLS.set(id, this); + this.toolDescription = toolDescription; + this.parameters = parameters; } /** @@ -43,7 +50,16 @@ export abstract class AgentTool { * https://github.com/ggml-org/llama.cpp/blob/master/docs/function-calling.md * https://platform.openai.com/docs/guides/function-calling?api-mode=responses#defining-functions */ - public abstract specs(): ToolCallSpec; + public specs(): ToolCallSpec { + return { + type: 'function', + function: { + name: this.id, + description: this.toolDescription, + parameters: this.parameters, + }, + }; + } /** * The actual tool call processing logic. diff --git a/tools/server/webui/src/utils/tool_calling/js_tool_call.ts b/tools/server/webui/src/utils/tool_calling/js_repl_tool.ts similarity index 55% rename from tools/server/webui/src/utils/tool_calling/js_tool_call.ts rename to tools/server/webui/src/utils/tool_calling/js_repl_tool.ts index b00dec94e8927..da3b09185d721 100644 --- a/tools/server/webui/src/utils/tool_calling/js_tool_call.ts +++ b/tools/server/webui/src/utils/tool_calling/js_repl_tool.ts @@ -1,23 +1,32 @@ import StorageUtils from '../storage'; -import { ToolCall, ToolCallOutput, ToolCallSpec } from '../types'; -import { AgentTool } from './available_tools'; +import { ToolCall, ToolCallOutput, ToolCallParameters } from '../types'; +import { AgentTool } from './agent_tool'; -class JSReplAgentTool extends AgentTool { +export class JSReplAgentTool extends AgentTool { private static readonly id = 'javascript_interpreter'; private fakeLogger: FakeConsoleLog; constructor() { super( JSReplAgentTool.id, - () => StorageUtils.getConfig().jsInterpreterToolUse + () => StorageUtils.getConfig().jsInterpreterToolUse, + 'Executes JavaScript code in the browser console. The code should be self-contained valid javascript. You can use console.log(variable) to print out intermediate values..', + { + type: 'object', + properties: { + code: { + type: 'string', + description: 'Valid JavaScript code to execute.', + }, + }, + required: ['code'], + } as ToolCallParameters ); this.fakeLogger = new FakeConsoleLog(); } _process(tc: ToolCall): ToolCallOutput { const args = JSON.parse(tc.function.arguments); - console.log('Arguments for tool call:'); - console.log(args); // Redirect console.log which agent will use to // the fake logger so that later we can get the content @@ -30,40 +39,16 @@ class JSReplAgentTool extends AgentTool { result = eval(args.code); } catch (err) { result = String(err); - } finally { - // Ensure original console.log is restored even if eval throws - console.log = originalConsoleLog; } + console.log = originalConsoleLog; result = this.fakeLogger.content + result; this.fakeLogger.clear(); return { call_id: tc.call_id, output: result } as ToolCallOutput; } - - public specs(): ToolCallSpec { - return { - type: 'function', - function: { - name: this.id, - description: - 'Executes JavaScript code in the browser console. The code should be self-contained valid javascript. You can use console.log(variable) to print out intermediate values..', - parameters: { - type: 'object', - properties: { - code: { - type: 'string', - description: 'Valid JavaScript code to execute.', - }, - }, - required: ['code'], - }, - }, - }; - } } -export const jsAgentTool = new JSReplAgentTool(); class FakeConsoleLog { private _content: string = ''; diff --git a/tools/server/webui/src/utils/tool_calling/register_tools.ts b/tools/server/webui/src/utils/tool_calling/register_tools.ts new file mode 100644 index 0000000000000..ba132b788b23d --- /dev/null +++ b/tools/server/webui/src/utils/tool_calling/register_tools.ts @@ -0,0 +1,21 @@ +import { isDev } from '../../Config'; +import { AgentTool } from './agent_tool'; +import { JSReplAgentTool } from './js_repl_tool'; + +/** + * Map of available tools for function calling. + * Note that these tools are not necessarily enabled by the user. + */ +export const AVAILABLE_TOOLS = new Map(); + +function registerTool(tool: AgentTool): AgentTool { + AVAILABLE_TOOLS.set(tool.id, tool); + if (isDev) + console.log( + `Successfully registered tool: ${tool.id}, enabled: ${tool.isEnabled()}` + ); + return tool; +} + +// Available agent tools +export const jsReplTool = registerTool(new JSReplAgentTool()); diff --git a/tools/server/webui/src/utils/types.ts b/tools/server/webui/src/utils/types.ts index 16280924c18b8..021dd84591cba 100644 --- a/tools/server/webui/src/utils/types.ts +++ b/tools/server/webui/src/utils/types.ts @@ -39,7 +39,7 @@ export interface Message { convId: string; type: 'text' | 'root'; timestamp: number; // timestamp from Date.now() - role: 'user' | 'assistant' | 'system'; + role: 'user' | 'assistant' | 'system' | 'tool'; content: string; timings?: TimingReport; extra?: MessageExtra[]; @@ -105,14 +105,16 @@ export interface ToolCallSpec { function: { name: string; description: string; - parameters: { - type: 'object'; - properties: object; - required: string[]; - }; + parameters: ToolCallParameters; }; } +export interface ToolCallParameters { + type: 'object'; + properties: object; + required: string[]; +} + export interface ToolCallOutput { type: 'function_call_output'; call_id: string; From f6b138606850f1887b0e41c36afca190148c4210 Mon Sep 17 00:00:00 2001 From: samolego <34912839+samolego@users.noreply.github.com> Date: Mon, 12 May 2025 17:39:03 +0200 Subject: [PATCH 4/9] add tool call fields --- tools/server/webui/src/utils/tool_calling/js_repl_tool.ts | 2 +- tools/server/webui/src/utils/types.ts | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/tools/server/webui/src/utils/tool_calling/js_repl_tool.ts b/tools/server/webui/src/utils/tool_calling/js_repl_tool.ts index da3b09185d721..965badbb8c0dc 100644 --- a/tools/server/webui/src/utils/tool_calling/js_repl_tool.ts +++ b/tools/server/webui/src/utils/tool_calling/js_repl_tool.ts @@ -10,7 +10,7 @@ export class JSReplAgentTool extends AgentTool { super( JSReplAgentTool.id, () => StorageUtils.getConfig().jsInterpreterToolUse, - 'Executes JavaScript code in the browser console. The code should be self-contained valid javascript. You can use console.log(variable) to print out intermediate values..', + 'Executes JavaScript code in the browser console. The code should be self-contained valid javascript. You can use console.log(variable) to print out intermediate values.', { type: 'object', properties: { diff --git a/tools/server/webui/src/utils/types.ts b/tools/server/webui/src/utils/types.ts index 021dd84591cba..5d66fc663559e 100644 --- a/tools/server/webui/src/utils/types.ts +++ b/tools/server/webui/src/utils/types.ts @@ -43,6 +43,7 @@ export interface Message { content: string; timings?: TimingReport; extra?: MessageExtra[]; + tool_calls?: ToolCall[]; // node based system for branching parent: Message['id']; children: Message['id'][]; @@ -77,6 +78,7 @@ export interface ViewingChat { export type PendingMessage = Omit & { content: string | null; + tool_calls?: ToolCall[]; }; export enum CanvasType { From f2175cbc30b6a9e90fbbf3125ee330a61deaa34f Mon Sep 17 00:00:00 2001 From: samolego <34912839+samolego@users.noreply.github.com> Date: Tue, 13 May 2025 11:07:58 +0200 Subject: [PATCH 5/9] fix: Use structured `tool_calls` for tool handling --- .../webui/src/components/ChatMessage.tsx | 136 ++++++------- tools/server/webui/src/utils/app.context.tsx | 186 ++++++++++-------- tools/server/webui/src/utils/storage.ts | 114 ++++++++--- tools/server/webui/src/utils/types.ts | 1 - 4 files changed, 239 insertions(+), 198 deletions(-) diff --git a/tools/server/webui/src/components/ChatMessage.tsx b/tools/server/webui/src/components/ChatMessage.tsx index 2419703482699..86308ef50feff 100644 --- a/tools/server/webui/src/components/ChatMessage.tsx +++ b/tools/server/webui/src/components/ChatMessage.tsx @@ -9,8 +9,6 @@ interface SplitMessage { content: PendingMessage['content']; thought?: string; isThinking?: boolean; - toolOutput?: string; - toolTitle?: string; } export default function ChatMessage({ @@ -51,71 +49,40 @@ export default function ChatMessage({ const prevSibling = siblingLeafNodeIds[siblingCurrIdx - 1]; // for reasoning model, we split the message into content, thought, and tool output - const { content, thought, isThinking, toolOutput, toolTitle }: SplitMessage = - useMemo(() => { - if (msg.content === null || msg.role !== 'assistant') { - return { content: msg.content }; - } - let currentContent = msg.content; - let extractedThought: string | undefined = undefined; - let isCurrentlyThinking = false; - let extractedToolOutput: string | undefined = undefined; - let extractedToolTitle: string | undefined = 'Tool Output'; - - // Process tags - const thinkParts = currentContent.split(''); - currentContent = thinkParts[0]; - if (thinkParts.length > 1) { - isCurrentlyThinking = true; - const tempThoughtArray: string[] = []; - for (let i = 1; i < thinkParts.length; i++) { - const thinkSegment = thinkParts[i].split(''); - tempThoughtArray.push(thinkSegment[0]); - if (thinkSegment.length > 1) { - isCurrentlyThinking = false; // Closing tag found - currentContent += thinkSegment[1]; - } - } - extractedThought = tempThoughtArray.join('\n'); - } + const { content, thought, isThinking }: SplitMessage = useMemo(() => { + if ( + msg.content === null || + (msg.role !== 'assistant' && msg.role !== 'tool') + ) { + return { content: msg.content }; + } - // Process tags (after thoughts are processed) - const toolParts = currentContent.split(''); - currentContent = toolParts[0]; - if (toolParts.length > 1) { - const tempToolOutputArray: string[] = []; - for (let i = 1; i < toolParts.length; i++) { - const toolSegment = toolParts[i].split(''); - const toolContent = toolSegment[0].trim(); + let actualContent = ''; + let thought = ''; + let isThinking = false; + let thinkSplit = msg.content.split('', 2); - const firstLineEnd = toolContent.indexOf('\n'); - if (firstLineEnd !== -1) { - extractedToolTitle = toolContent.substring(0, firstLineEnd); - tempToolOutputArray.push( - toolContent.substring(firstLineEnd + 1).trim() - ); - } else { - // If no newline, extractedToolTitle keeps its default; toolContent is pushed as is. - tempToolOutputArray.push(toolContent); - } + actualContent += thinkSplit[0]; - if (toolSegment.length > 1) { - currentContent += toolSegment[1]; - } - } - extractedToolOutput = tempToolOutputArray.join('\n\n'); + while (thinkSplit[1] !== undefined) { + // tag found + thinkSplit = thinkSplit[1].split('', 2); + thought += thinkSplit[0]; + isThinking = true; + if (thinkSplit[1] !== undefined) { + // closing tag found + isThinking = false; + thinkSplit = thinkSplit[1].split('', 2); + actualContent += thinkSplit[0]; } + } - return { - content: currentContent.trim(), - thought: extractedThought, - isThinking: isCurrentlyThinking, - toolOutput: extractedToolOutput, - toolTitle: extractedToolTitle, - }; - }, [msg]); + return { content: actualContent, thought, isThinking }; + }, [msg]); if (!viewingChat) return null; + const toolCalls = msg.tool_calls ?? null; + return (
{content === null ? ( <> - {/* show loading dots for pending message */} - + {toolCalls ? null : ( + <> + {/* show loading dots for pending message */} + + + )} ) : ( <> @@ -232,27 +203,32 @@ export default function ChatMessage({ content={content} isGenerating={isPending} /> - - {toolOutput && ( -
- - {toolTitle || 'Tool Output'} - -
- -
-
- )}
)} + {toolCalls && + toolCalls.map((toolCall, i) => ( +
+ + Tool call: {toolCall.function.name} + + +
+
Arguments:
+ 
+                        {JSON.stringify(
+                          JSON.parse(toolCall.function.arguments),
+                          null,
+                          2
+                        )}
+
+
+
+ ))} {/* render timings if enabled */} {timings && config.showTokensPerSecond && (
diff --git a/tools/server/webui/src/utils/app.context.tsx b/tools/server/webui/src/utils/app.context.tsx index 1b459e2beba3b..b5dae2bb5216b 100644 --- a/tools/server/webui/src/utils/app.context.tsx +++ b/tools/server/webui/src/utils/app.context.tsx @@ -183,8 +183,16 @@ export const AppContextProvider = ({ } if (isDev) console.log({ messages }); - // stream does not support tool-use - const streamResponse = !config.jsInterpreterToolUse; + // tool calling from clientside + const enabledTools = Array.from( + AVAILABLE_TOOLS, + ([_name, tool], _index) => tool + ) + .filter((tool) => tool.enabled()) + .map((tool) => tool.specs()); + + // stream does not support tool-use (yet?) + const streamResponse = enabledTools.length === 0; // prepare params const params = { @@ -211,11 +219,7 @@ export const AppContextProvider = ({ dry_penalty_last_n: config.dry_penalty_last_n, max_tokens: config.max_tokens, timings_per_token: !!config.showTokensPerSecond, - tools: config.jsInterpreterToolUse - ? Array.from(AVAILABLE_TOOLS, ([_name, tool], _index) => tool) - .filter((tool) => tool.enabled()) - .map((tool) => tool.specs()) - : undefined, + tools: enabledTools.length > 0 ? enabledTools : undefined, ...(config.custom.length ? JSON.parse(config.custom) : {}), }; @@ -237,6 +241,11 @@ export const AppContextProvider = ({ throw new Error(body?.error?.message || 'Unknown error'); } + // Tool calls results we will process later + const pendingMessages: PendingMessage[] = []; + let lastMsgId = pendingMsg.id; + let shouldContinueChain = false; + if (streamResponse) { const chunks = getSSEStreamAsync(fetchResponse); for await (const chunk of chunks) { @@ -268,93 +277,101 @@ export const AppContextProvider = ({ } else { const responseData = await fetchResponse.json(); if (isDev) console.log({ responseData }); + if (responseData.error) { + throw new Error(responseData.error?.message || 'Unknown error'); + } - const choice = responseData.choices?.[0]; - if (choice) { - const messageFromAPI = choice.message; - let newContent = ''; + const choice = responseData.choices[0]; + const messageFromAPI = choice.message; + console.log({ messageFromAPI }); + let newContent = ''; - if (messageFromAPI.content) { - newContent = messageFromAPI.content; - } + if (messageFromAPI.content) { + newContent = messageFromAPI.content; + console.log(newContent); + } - if ( - messageFromAPI.tool_calls && - messageFromAPI.tool_calls.length > 0 - ) { - console.log(messageFromAPI.tool_calls); - for (let i = 0; i < messageFromAPI.tool_calls.length; i++) { - console.log('Tool use #' + i); - const tc = messageFromAPI.tool_calls[i] as ToolCall; - console.log(tc); - - if (tc) { - // Set up call id - tc.call_id ??= `call_${i}`; - - // Process tool call - const toolResult = AVAILABLE_TOOLS.get( - tc.function.name - )?.processCall(tc); - - if (toolResult) { - newContent += `Tool use: ${tc.function.name}\n\n`; - newContent += toolResult.output; - newContent += ''; - } else { - newContent += `Tool use: ${tc.function.name}\n\nError: invalid tool call!\n`; - } - } - } + // Process tool calls + if (messageFromAPI.tool_calls && messageFromAPI.tool_calls.length > 0) { + // Store the raw tool calls in the pendingMsg + pendingMsg = { + ...pendingMsg, + tool_calls: messageFromAPI.tool_calls as ToolCall[], + }; - /*const toolCallsInfo = messageFromAPI.tool_calls - .map( - ( - tc: any // Use 'any' for tc temporarily if type is not imported/defined here - ) => - `Tool invoked: ${tc.function.name}\nArguments: ${tc.function.arguments}` - ) - .join('\n\n'); - - if (newContent.length > 0) { - newContent += `\n\n${toolCallsInfo}`; - } else { - newContent = toolCallsInfo; - }*/ + for (let i = 0; i < messageFromAPI.tool_calls.length; i++) { + const tc = messageFromAPI.tool_calls[i] as ToolCall; + if (tc) { + // Set up call id + tc.call_id ??= `call_${i}`; + + if (isDev) console.log({ tc }); + + // Process tool call + const toolResult = AVAILABLE_TOOLS.get( + tc.function.name + )?.processCall(tc); + + const toolMsg: PendingMessage = { + id: lastMsgId + 1, + type: 'text', + convId: convId, + content: toolResult?.output ?? 'Error: invalid tool call!', + timestamp: Date.now(), + role: 'tool', + parent: lastMsgId, + children: [], + }; + pendingMessages.push(toolMsg); + lastMsgId += 1; + } } + } + if (newContent !== '') { pendingMsg = { ...pendingMsg, content: newContent, }; + } - // Handle timings from the non-streaming response - // The exact location of 'timings' in responseData might vary by API. - // Assuming responseData.timings similar to streaming chunk for now. - const apiTimings = responseData.timings; - if (apiTimings && config.showTokensPerSecond) { - pendingMsg.timings = { - prompt_n: apiTimings.prompt_n, - prompt_ms: apiTimings.prompt_ms, - predicted_n: apiTimings.predicted_n, - predicted_ms: apiTimings.predicted_ms, - }; - } - setPending(convId, pendingMsg); - onChunk(); // Update UI to show the processed message + // Handle timings from the non-streaming response + // The exact location of 'timings' in responseData might vary by API. + // Assuming responseData.timings similar to streaming chunk for now. + const apiTimings = responseData.timings; + if (apiTimings && config.showTokensPerSecond) { + pendingMsg.timings = { + prompt_n: apiTimings.prompt_n, + prompt_ms: apiTimings.prompt_ms, + predicted_n: apiTimings.predicted_n, + predicted_ms: apiTimings.predicted_ms, + }; + } - // if message ended due to "finish_reason": "tool_calls" - // resend it to assistant to process the result. - if (choice.finish_reason === 'tool_calls') { - console.log('Ended due to tool call. Interpreting results ...'); - generateMessage(convId, pendingId, onChunk); - } - } else { - console.error( - 'API response missing choices or message:', - responseData - ); - throw new Error('Invalid API response structure'); + for (const pendMsg of pendingMessages) { + console.log('Setting pending message', pendMsg.id); + setPending(convId, pendMsg); + } + + onChunk(); // Update UI to show the processed message + + shouldContinueChain = choice.finish_reason === 'tool_calls'; + } + + pendingMessages.unshift(pendingMsg); + if ( + pendingMsg.content !== null || + (pendingMsg.tool_calls?.length ?? 0) > 0 + ) { + await StorageUtils.appendMsgChain( + pendingMessages as Message[], + leafNodeId + ); + + // if message ended due to "finish_reason": "tool_calls" + // resend it to assistant to process the result. + if (shouldContinueChain) { + await generateMessage(convId, lastMsgId, onChunk); } } } catch (err) { @@ -370,9 +387,6 @@ export const AppContextProvider = ({ } } - if (pendingMsg.content !== null) { - await StorageUtils.appendMsg(pendingMsg as Message, leafNodeId); - } setPending(convId, null); onChunk(pendingId); // trigger scroll to bottom and switch to the last node }; @@ -398,7 +412,7 @@ export const AppContextProvider = ({ const now = Date.now(); const currMsgId = now; - StorageUtils.appendMsg( + await StorageUtils.appendMsg( { id: currMsgId, timestamp: now, diff --git a/tools/server/webui/src/utils/storage.ts b/tools/server/webui/src/utils/storage.ts index 1dfc9d9799311..87ee769cba2c3 100644 --- a/tools/server/webui/src/utils/storage.ts +++ b/tools/server/webui/src/utils/storage.ts @@ -1,7 +1,7 @@ // coversations is stored in localStorage // format: { [convId]: { id: string, lastModified: number, messages: [...] } } -import { CONFIG_DEFAULT } from '../Config'; +import { CONFIG_DEFAULT, isDev } from '../Config'; import { Conversation, Message, TimingReport } from './types'; import Dexie, { Table } from 'dexie'; @@ -123,39 +123,91 @@ const StorageUtils = { msg: Exclude, parentNodeId: Message['id'] ): Promise { - if (msg.content === null) return; - const { convId } = msg; - await db.transaction('rw', db.conversations, db.messages, async () => { - const conv = await StorageUtils.getOneConversation(convId); - const parentMsg = await db.messages - .where({ convId, id: parentNodeId }) - .first(); - // update the currNode of conversation - if (!conv) { - throw new Error(`Conversation ${convId} does not exist`); - } - if (!parentMsg) { - throw new Error( - `Parent message ID ${parentNodeId} does not exist in conversation ${convId}` - ); - } - await db.conversations.update(convId, { - lastModified: Date.now(), - currNode: msg.id, - }); - // update parent - await db.messages.update(parentNodeId, { - children: [...parentMsg.children, msg.id], - }); - // create message - await db.messages.add({ - ...msg, - parent: parentNodeId, - children: [], + await this.appendMsgChain([msg], parentNodeId); + }, + + /** + * Adds chain of messages to the DB, usually + * produced by tool calling. + */ + async appendMsgChain( + messages: Exclude[], + parentNodeId: Message['id'] + ): Promise { + if (messages.length === 0) return; + + console.log('Saving messges! ' + JSON.stringify(messages)); + + const { convId } = messages[0]; + + // Verify conversation exists + const conv = await this.getOneConversation(convId); + if (!conv) { + throw new Error(`Conversation ${convId} does not exist`); + } + + // Verify starting parent exists + const startParent = await db.messages + .where({ convId, id: parentNodeId }) + .first(); + if (!startParent) { + throw new Error( + `Starting parent message ${parentNodeId} does not exist in conversation ${convId}` + ); + } + + // Get the last message ID for updating the conversation + const lastMsgId = messages[messages.length - 1].id; + + try { + // Process all messages in a single transaction + await db.transaction('rw', db.messages, db.conversations, () => { + // First message connects to startParentId + let parentId = parentNodeId; + const parentChildren = [...startParent.children]; + + for (let i = 0; i < messages.length; i++) { + const msg = messages[i]; + + // Add this message to its parent's children + if (i === 0) { + // First message - update the starting parent + parentChildren.push(msg.id); + db.messages.update(parentId, { children: parentChildren }); + } else { + // Other messages - previous message is the parent + db.messages.update(parentId, { children: [msg.id] }); + } + + // Add the message + db.messages.add({ + ...msg, + parent: parentId, + children: [], // Will be updated if this message has children + }); + + // Next message's parent is this message + parentId = msg.id; + } + + // Update the conversation + db.conversations.update(convId, { + lastModified: Date.now(), + currNode: lastMsgId, + }); }); - }); + + console.log(`Successfully saved chain of ${messages.length} messages`); + } catch (error) { + console.error('Error saving message chain:', error); + throw error; + } + + if (isDev) console.log('Updated conversation:', convId); + dispatchConversationChange(convId); }, + /** * remove conversation by id */ diff --git a/tools/server/webui/src/utils/types.ts b/tools/server/webui/src/utils/types.ts index 5d66fc663559e..1bc3a306ba566 100644 --- a/tools/server/webui/src/utils/types.ts +++ b/tools/server/webui/src/utils/types.ts @@ -78,7 +78,6 @@ export interface ViewingChat { export type PendingMessage = Omit & { content: string | null; - tool_calls?: ToolCall[]; }; export enum CanvasType { From 4698b6675a8ebb263472d16afcb9db01fa8afdf2 Mon Sep 17 00:00:00 2001 From: samolego <34912839+samolego@users.noreply.github.com> Date: Tue, 13 May 2025 11:42:53 +0200 Subject: [PATCH 6/9] fix: forward tool call info back to api --- tools/server/webui/src/utils/misc.ts | 20 ++++++++++++++++---- tools/server/webui/src/utils/storage.ts | 6 ------ tools/server/webui/src/utils/types.ts | 2 +- 3 files changed, 17 insertions(+), 11 deletions(-) diff --git a/tools/server/webui/src/utils/misc.ts b/tools/server/webui/src/utils/misc.ts index a47720fa6f773..0fb82717373e8 100644 --- a/tools/server/webui/src/utils/misc.ts +++ b/tools/server/webui/src/utils/misc.ts @@ -66,10 +66,16 @@ export function normalizeMsgsForAPI(messages: Readonly) { newContent += msg.content; - return { + const apiMsg = { role: msg.role, content: newContent, - }; + } as APIMessage; + + if (msg.tool_calls && msg.tool_calls.length > 0) { + apiMsg.tool_calls = msg.tool_calls; + } + + return apiMsg; }) as APIMessage[]; } @@ -78,13 +84,19 @@ export function normalizeMsgsForAPI(messages: Readonly) { */ export function filterThoughtFromMsgs(messages: APIMessage[]) { return messages.map((msg) => { - return { + const filteredMessage: APIMessage = { role: msg.role, content: msg.role === 'assistant' ? msg.content.split('').at(-1)!.trim() : msg.content, - } as APIMessage; + }; + + if (msg.tool_calls && msg.tool_calls.length > 0) { + filteredMessage.tool_calls = msg.tool_calls; + } + + return filteredMessage; }); } diff --git a/tools/server/webui/src/utils/storage.ts b/tools/server/webui/src/utils/storage.ts index 87ee769cba2c3..cfbba2f9d4f9f 100644 --- a/tools/server/webui/src/utils/storage.ts +++ b/tools/server/webui/src/utils/storage.ts @@ -136,8 +136,6 @@ const StorageUtils = { ): Promise { if (messages.length === 0) return; - console.log('Saving messges! ' + JSON.stringify(messages)); - const { convId } = messages[0]; // Verify conversation exists @@ -196,15 +194,11 @@ const StorageUtils = { currNode: lastMsgId, }); }); - - console.log(`Successfully saved chain of ${messages.length} messages`); } catch (error) { console.error('Error saving message chain:', error); throw error; } - if (isDev) console.log('Updated conversation:', convId); - dispatchConversationChange(convId); }, diff --git a/tools/server/webui/src/utils/types.ts b/tools/server/webui/src/utils/types.ts index 1bc3a306ba566..eb334c0ccba14 100644 --- a/tools/server/webui/src/utils/types.ts +++ b/tools/server/webui/src/utils/types.ts @@ -62,7 +62,7 @@ export interface MessageExtraContext { content: string; } -export type APIMessage = Pick; +export type APIMessage = Pick; export interface Conversation { id: string; // format: `conv-{timestamp}` From 69e7119cb51af491ee2b9754429fd766cff5d8fd Mon Sep 17 00:00:00 2001 From: samolego <34912839+samolego@users.noreply.github.com> Date: Tue, 13 May 2025 11:59:22 +0200 Subject: [PATCH 7/9] provide tool call response in a dropdown --- .../webui/src/components/ChatMessage.tsx | 25 ++++++++++++++++--- .../src/utils/tool_calling/js_repl_tool.ts | 3 +++ 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/tools/server/webui/src/components/ChatMessage.tsx b/tools/server/webui/src/components/ChatMessage.tsx index 86308ef50feff..8cc6fc0df1424 100644 --- a/tools/server/webui/src/components/ChatMessage.tsx +++ b/tools/server/webui/src/components/ChatMessage.tsx @@ -199,10 +199,27 @@ export default function ChatMessage({ )} - + {msg.role === 'tool' ? ( +
+ + Tool call result + +
+ +
+
+ ) : ( + + )}
)} diff --git a/tools/server/webui/src/utils/tool_calling/js_repl_tool.ts b/tools/server/webui/src/utils/tool_calling/js_repl_tool.ts index 965badbb8c0dc..8fe614f030c76 100644 --- a/tools/server/webui/src/utils/tool_calling/js_repl_tool.ts +++ b/tools/server/webui/src/utils/tool_calling/js_repl_tool.ts @@ -37,6 +37,9 @@ export class JSReplAgentTool extends AgentTool { try { // Evaluate the provided agent code result = eval(args.code); + if (!result) { + result = ''; + } } catch (err) { result = String(err); } From 75fd25e741b012b7812b355bdd7dc1707fcf6663 Mon Sep 17 00:00:00 2001 From: samolego <34912839+samolego@users.noreply.github.com> Date: Tue, 13 May 2025 18:11:55 +0200 Subject: [PATCH 8/9] Fix UI updates after tool call chains --- tools/server/webui/src/utils/app.context.tsx | 42 ++++++++++++------- tools/server/webui/src/utils/storage.ts | 2 +- .../src/utils/tool_calling/agent_tool.ts | 34 ++++++--------- .../src/utils/tool_calling/js_repl_tool.ts | 12 +++--- .../src/utils/tool_calling/register_tools.ts | 7 ++-- tools/server/webui/src/utils/types.ts | 4 +- 6 files changed, 52 insertions(+), 49 deletions(-) diff --git a/tools/server/webui/src/utils/app.context.tsx b/tools/server/webui/src/utils/app.context.tsx index b5dae2bb5216b..97381066ed93d 100644 --- a/tools/server/webui/src/utils/app.context.tsx +++ b/tools/server/webui/src/utils/app.context.tsx @@ -136,8 +136,8 @@ export const AppContextProvider = ({ convId: string, leafNodeId: Message['id'], onChunk: CallbackGeneratedChunk - ) => { - if (isGenerating(convId)) return; + ): Promise => { + if (isGenerating(convId)) return leafNodeId; const config = StorageUtils.getConfig(); const currConversation = await StorageUtils.getOneConversation(convId); @@ -188,8 +188,8 @@ export const AppContextProvider = ({ AVAILABLE_TOOLS, ([_name, tool], _index) => tool ) - .filter((tool) => tool.enabled()) - .map((tool) => tool.specs()); + .filter((tool) => tool.enabled) + .map((tool) => tool.specs); // stream does not support tool-use (yet?) const streamResponse = enabledTools.length === 0; @@ -336,8 +336,6 @@ export const AppContextProvider = ({ } // Handle timings from the non-streaming response - // The exact location of 'timings' in responseData might vary by API. - // Assuming responseData.timings similar to streaming chunk for now. const apiTimings = responseData.timings; if (apiTimings && config.showTokensPerSecond) { pendingMsg.timings = { @@ -349,12 +347,10 @@ export const AppContextProvider = ({ } for (const pendMsg of pendingMessages) { - console.log('Setting pending message', pendMsg.id); setPending(convId, pendMsg); + onChunk(pendMsg.id); // Update UI to show the processed message } - onChunk(); // Update UI to show the processed message - shouldContinueChain = choice.finish_reason === 'tool_calls'; } @@ -367,13 +363,28 @@ export const AppContextProvider = ({ pendingMessages as Message[], leafNodeId ); + } - // if message ended due to "finish_reason": "tool_calls" - // resend it to assistant to process the result. - if (shouldContinueChain) { - await generateMessage(convId, lastMsgId, onChunk); - } + // if message ended due to "finish_reason": "tool_calls" + // resend it to assistant to process the result. + if (shouldContinueChain) { + console.log('Generating followup message!'); + lastMsgId = await generateMessage(convId, lastMsgId, onChunk); + console.log('Generating - done!'); + + // Fetch messages from DB for debug + const savedMsgs = await StorageUtils.getMessages(convId); + console.log({ savedMsgs }); } + + setPending(convId, null); + onChunk(lastMsgId); // trigger scroll to bottom and switch to the last node + + // Fetch messages from DB + const savedMsgs = await StorageUtils.getMessages(convId); + console.log({ savedMsgs }); + + return lastMsgId; } catch (err) { setPending(convId, null); if ((err as Error).name === 'AbortError') { @@ -387,8 +398,7 @@ export const AppContextProvider = ({ } } - setPending(convId, null); - onChunk(pendingId); // trigger scroll to bottom and switch to the last node + return pendingId; }; const sendMessage = async ( diff --git a/tools/server/webui/src/utils/storage.ts b/tools/server/webui/src/utils/storage.ts index cfbba2f9d4f9f..4133cf84edf16 100644 --- a/tools/server/webui/src/utils/storage.ts +++ b/tools/server/webui/src/utils/storage.ts @@ -1,7 +1,7 @@ // coversations is stored in localStorage // format: { [convId]: { id: string, lastModified: number, messages: [...] } } -import { CONFIG_DEFAULT, isDev } from '../Config'; +import { CONFIG_DEFAULT } from '../Config'; import { Conversation, Message, TimingReport } from './types'; import Dexie, { Table } from 'dexie'; diff --git a/tools/server/webui/src/utils/tool_calling/agent_tool.ts b/tools/server/webui/src/utils/tool_calling/agent_tool.ts index f6f24691a0aed..d66ec2bebe4d8 100644 --- a/tools/server/webui/src/utils/tool_calling/agent_tool.ts +++ b/tools/server/webui/src/utils/tool_calling/agent_tool.ts @@ -1,35 +1,25 @@ import { - ToolCall, + ToolCallRequest, ToolCallOutput, ToolCallParameters, ToolCallSpec, } from '../types'; export abstract class AgentTool { - id: string; - isEnabled: () => boolean; - toolDescription: string; - parameters: ToolCallParameters; - constructor( - id: string, - enabled: () => boolean, - toolDescription: string, - parameters: ToolCallParameters - ) { - this.id = id; - this.isEnabled = enabled; - this.toolDescription = toolDescription; - this.parameters = parameters; - } + public readonly id: string, + private readonly isEnabledCallback: () => boolean, + public readonly toolDescription: string, + public readonly parameters: ToolCallParameters + ) {} /** * "Public" wrapper for the tool call processing logic. * @param call The tool call object from the API response. * @returns The tool call output or undefined if the tool is not enabled. */ - public processCall(call: ToolCall): ToolCallOutput | undefined { - if (this.enabled()) { + public processCall(call: ToolCallRequest): ToolCallOutput | undefined { + if (this.enabled) { return this._process(call); } @@ -41,8 +31,8 @@ export abstract class AgentTool { * User can toggle the status from the settings panel. * @returns enabled status. */ - public enabled(): boolean { - return this.isEnabled(); + public get enabled(): boolean { + return this.isEnabledCallback(); } /** @@ -50,7 +40,7 @@ export abstract class AgentTool { * https://github.com/ggml-org/llama.cpp/blob/master/docs/function-calling.md * https://platform.openai.com/docs/guides/function-calling?api-mode=responses#defining-functions */ - public specs(): ToolCallSpec { + public get specs(): ToolCallSpec { return { type: 'function', function: { @@ -65,5 +55,5 @@ export abstract class AgentTool { * The actual tool call processing logic. * @param call: The tool call object from the API response. */ - protected abstract _process(call: ToolCall): ToolCallOutput; + protected abstract _process(call: ToolCallRequest): ToolCallOutput; } diff --git a/tools/server/webui/src/utils/tool_calling/js_repl_tool.ts b/tools/server/webui/src/utils/tool_calling/js_repl_tool.ts index 8fe614f030c76..25a1d8e23f5f0 100644 --- a/tools/server/webui/src/utils/tool_calling/js_repl_tool.ts +++ b/tools/server/webui/src/utils/tool_calling/js_repl_tool.ts @@ -1,14 +1,14 @@ import StorageUtils from '../storage'; -import { ToolCall, ToolCallOutput, ToolCallParameters } from '../types'; +import { ToolCallRequest, ToolCallOutput, ToolCallParameters } from '../types'; import { AgentTool } from './agent_tool'; export class JSReplAgentTool extends AgentTool { - private static readonly id = 'javascript_interpreter'; + private static readonly ID = 'javascript_interpreter'; private fakeLogger: FakeConsoleLog; constructor() { super( - JSReplAgentTool.id, + JSReplAgentTool.ID, () => StorageUtils.getConfig().jsInterpreterToolUse, 'Executes JavaScript code in the browser console. The code should be self-contained valid javascript. You can use console.log(variable) to print out intermediate values.', { @@ -25,7 +25,7 @@ export class JSReplAgentTool extends AgentTool { this.fakeLogger = new FakeConsoleLog(); } - _process(tc: ToolCall): ToolCallOutput { + _process(tc: ToolCallRequest): ToolCallOutput { const args = JSON.parse(tc.function.arguments); // Redirect console.log which agent will use to @@ -37,7 +37,9 @@ export class JSReplAgentTool extends AgentTool { try { // Evaluate the provided agent code result = eval(args.code); - if (!result) { + if (result) { + result = JSON.stringify(result, null, 2); + } else { result = ''; } } catch (err) { diff --git a/tools/server/webui/src/utils/tool_calling/register_tools.ts b/tools/server/webui/src/utils/tool_calling/register_tools.ts index ba132b788b23d..0ba08ac214822 100644 --- a/tools/server/webui/src/utils/tool_calling/register_tools.ts +++ b/tools/server/webui/src/utils/tool_calling/register_tools.ts @@ -8,12 +8,13 @@ import { JSReplAgentTool } from './js_repl_tool'; */ export const AVAILABLE_TOOLS = new Map(); -function registerTool(tool: AgentTool): AgentTool { +function registerTool(tool: T): T { AVAILABLE_TOOLS.set(tool.id, tool); - if (isDev) + if (isDev) { console.log( - `Successfully registered tool: ${tool.id}, enabled: ${tool.isEnabled()}` + `Successfully registered tool: ${tool.id}, enabled: ${tool.enabled}` ); + } return tool; } diff --git a/tools/server/webui/src/utils/types.ts b/tools/server/webui/src/utils/types.ts index eb334c0ccba14..6ad02a6780357 100644 --- a/tools/server/webui/src/utils/types.ts +++ b/tools/server/webui/src/utils/types.ts @@ -43,7 +43,7 @@ export interface Message { content: string; timings?: TimingReport; extra?: MessageExtra[]; - tool_calls?: ToolCall[]; + tool_calls?: ToolCallRequest[]; // node based system for branching parent: Message['id']; children: Message['id'][]; @@ -91,7 +91,7 @@ export interface CanvasPyInterpreter { export type CanvasData = CanvasPyInterpreter; -export interface ToolCall { +export interface ToolCallRequest { id: string; type: 'function'; call_id: string; From ae32a9a8d45238e128ead4f90efc4f999999623e Mon Sep 17 00:00:00 2001 From: samolego <34912839+samolego@users.noreply.github.com> Date: Tue, 13 May 2025 19:44:17 +0200 Subject: [PATCH 9/9] move js evaluation to sandboxed iframe, remove debug logs --- .../webui/src/assets/iframe_sandbox.html | 78 ++++++++ tools/server/webui/src/utils/app.context.tsx | 26 +-- .../src/utils/tool_calling/agent_tool.ts | 18 +- .../src/utils/tool_calling/js_repl_tool.ts | 176 ++++++++++++++---- 4 files changed, 239 insertions(+), 59 deletions(-) create mode 100644 tools/server/webui/src/assets/iframe_sandbox.html diff --git a/tools/server/webui/src/assets/iframe_sandbox.html b/tools/server/webui/src/assets/iframe_sandbox.html new file mode 100644 index 0000000000000..b98d8adf0c7fb --- /dev/null +++ b/tools/server/webui/src/assets/iframe_sandbox.html @@ -0,0 +1,78 @@ + + + + JS Sandbox + + + +

JavaScript Execution Sandbox

+ + diff --git a/tools/server/webui/src/utils/app.context.tsx b/tools/server/webui/src/utils/app.context.tsx index 97381066ed93d..7f8d005d88f23 100644 --- a/tools/server/webui/src/utils/app.context.tsx +++ b/tools/server/webui/src/utils/app.context.tsx @@ -5,7 +5,7 @@ import { Conversation, Message, PendingMessage, - ToolCall, + ToolCallRequest, ViewingChat, } from './types'; import StorageUtils from './storage'; @@ -276,14 +276,12 @@ export const AppContextProvider = ({ } } else { const responseData = await fetchResponse.json(); - if (isDev) console.log({ responseData }); if (responseData.error) { throw new Error(responseData.error?.message || 'Unknown error'); } const choice = responseData.choices[0]; const messageFromAPI = choice.message; - console.log({ messageFromAPI }); let newContent = ''; if (messageFromAPI.content) { @@ -296,21 +294,21 @@ export const AppContextProvider = ({ // Store the raw tool calls in the pendingMsg pendingMsg = { ...pendingMsg, - tool_calls: messageFromAPI.tool_calls as ToolCall[], + tool_calls: messageFromAPI.tool_calls as ToolCallRequest[], }; for (let i = 0; i < messageFromAPI.tool_calls.length; i++) { - const tc = messageFromAPI.tool_calls[i] as ToolCall; - if (tc) { + const toolCall = messageFromAPI.tool_calls[i] as ToolCallRequest; + if (toolCall) { // Set up call id - tc.call_id ??= `call_${i}`; + toolCall.call_id ??= `call_${i}`; - if (isDev) console.log({ tc }); + if (isDev) console.log({ tc: toolCall }); // Process tool call - const toolResult = AVAILABLE_TOOLS.get( - tc.function.name - )?.processCall(tc); + const toolResult = await AVAILABLE_TOOLS.get( + toolCall.function.name + )?.processCall(toolCall); const toolMsg: PendingMessage = { id: lastMsgId + 1, @@ -368,13 +366,7 @@ export const AppContextProvider = ({ // if message ended due to "finish_reason": "tool_calls" // resend it to assistant to process the result. if (shouldContinueChain) { - console.log('Generating followup message!'); lastMsgId = await generateMessage(convId, lastMsgId, onChunk); - console.log('Generating - done!'); - - // Fetch messages from DB for debug - const savedMsgs = await StorageUtils.getMessages(convId); - console.log({ savedMsgs }); } setPending(convId, null); diff --git a/tools/server/webui/src/utils/tool_calling/agent_tool.ts b/tools/server/webui/src/utils/tool_calling/agent_tool.ts index d66ec2bebe4d8..18234ca0668fc 100644 --- a/tools/server/webui/src/utils/tool_calling/agent_tool.ts +++ b/tools/server/webui/src/utils/tool_calling/agent_tool.ts @@ -18,11 +18,21 @@ export abstract class AgentTool { * @param call The tool call object from the API response. * @returns The tool call output or undefined if the tool is not enabled. */ - public processCall(call: ToolCallRequest): ToolCallOutput | undefined { + public async processCall( + call: ToolCallRequest + ): Promise { if (this.enabled) { - return this._process(call); + try { + return await this._process(call); + } catch (error) { + console.error(`Error processing tool call for ${this.id}:`, error); + return { + type: 'function_call_output', + call_id: call.call_id, + output: `Error during tool execution: ${(error as Error).message}`, + } as ToolCallOutput; + } } - return undefined; } @@ -55,5 +65,5 @@ export abstract class AgentTool { * The actual tool call processing logic. * @param call: The tool call object from the API response. */ - protected abstract _process(call: ToolCallRequest): ToolCallOutput; + protected abstract _process(call: ToolCallRequest): Promise; } diff --git a/tools/server/webui/src/utils/tool_calling/js_repl_tool.ts b/tools/server/webui/src/utils/tool_calling/js_repl_tool.ts index 25a1d8e23f5f0..37bbf4450817d 100644 --- a/tools/server/webui/src/utils/tool_calling/js_repl_tool.ts +++ b/tools/server/webui/src/utils/tool_calling/js_repl_tool.ts @@ -2,15 +2,34 @@ import StorageUtils from '../storage'; import { ToolCallRequest, ToolCallOutput, ToolCallParameters } from '../types'; import { AgentTool } from './agent_tool'; +// Import the HTML content as a raw string +import iframeHTMLContent from '../../assets/iframe_sandbox.html?raw'; + +interface IframeMessage { + call_id: string; + output?: string; + error?: string; + command?: 'executeCode' | 'iframeReady'; + code?: string; +} + export class JSReplAgentTool extends AgentTool { private static readonly ID = 'javascript_interpreter'; - private fakeLogger: FakeConsoleLog; + private iframe: HTMLIFrameElement | null = null; + private iframeReadyPromise: Promise | null = null; + private resolveIframeReady: (() => void) | null = null; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + private rejectIframeReady: ((reason?: any) => void) | null = null; + private pendingCalls = new Map void>(); + private messageHandler: + | ((event: MessageEvent) => void) + | null = null; constructor() { super( JSReplAgentTool.ID, () => StorageUtils.getConfig().jsInterpreterToolUse, - 'Executes JavaScript code in the browser console. The code should be self-contained valid javascript. You can use console.log(variable) to print out intermediate values.', + 'Executes JavaScript code in a sandboxed iframe. The code should be self-contained valid javascript. You can use console.log(variable) to print out intermediate values, which will be captured.', { type: 'object', properties: { @@ -22,53 +41,134 @@ export class JSReplAgentTool extends AgentTool { required: ['code'], } as ToolCallParameters ); - this.fakeLogger = new FakeConsoleLog(); + this.initIframe(); } - _process(tc: ToolCallRequest): ToolCallOutput { - const args = JSON.parse(tc.function.arguments); + private initIframe(): void { + if (typeof window === 'undefined' || typeof document === 'undefined') { + console.warn( + 'JSReplAgentTool: Not in a browser environment, iframe will not be created.' + ); + return; + } - // Redirect console.log which agent will use to - // the fake logger so that later we can get the content - const originalConsoleLog = console.log; - console.log = this.fakeLogger.log; + this.iframeReadyPromise = new Promise((resolve, reject) => { + this.resolveIframeReady = resolve; + this.rejectIframeReady = reject; + }); - let result = ''; - try { - // Evaluate the provided agent code - result = eval(args.code); - if (result) { - result = JSON.stringify(result, null, 2); - } else { - result = ''; + this.messageHandler = (event: MessageEvent) => { + if ( + !event.data || + !this.iframe || + !this.iframe.contentWindow || + event.source !== this.iframe.contentWindow + ) { + return; } - } catch (err) { - result = String(err); - } - console.log = originalConsoleLog; - result = this.fakeLogger.content + result; + const { command, call_id, output, error } = event.data; + if (command === 'iframeReady' && call_id === 'initial_ready') { + if (this.resolveIframeReady) { + this.resolveIframeReady(); + this.resolveIframeReady = null; + this.rejectIframeReady = null; + } + return; + } + if (typeof call_id !== 'string') { + return; + } + if (this.pendingCalls.has(call_id)) { + const callback = this.pendingCalls.get(call_id)!; + callback({ + type: 'function_call_output', + call_id: call_id, + output: error ? `Error: ${error}` : (output ?? ''), + } as ToolCallOutput); + this.pendingCalls.delete(call_id); + } + }; + window.addEventListener('message', this.messageHandler); - this.fakeLogger.clear(); + this.iframe = document.createElement('iframe'); + this.iframe.style.display = 'none'; + this.iframe.sandbox.add('allow-scripts'); - return { call_id: tc.call_id, output: result } as ToolCallOutput; - } -} + // Use srcdoc with the imported HTML content + this.iframe.srcdoc = iframeHTMLContent; -class FakeConsoleLog { - private _content: string = ''; + document.body.appendChild(this.iframe); - public get content(): string { - return this._content; + setTimeout(() => { + if (this.rejectIframeReady) { + this.rejectIframeReady(new Error('Iframe readiness timeout')); + this.resolveIframeReady = null; + this.rejectIframeReady = null; + } + }, 5000); } - // Use an arrow function for log to correctly bind 'this' - public log = (...args: any[]): void => { - // Convert arguments to strings and join them. - this._content += args.map((arg) => String(arg)).join(' ') + '\n'; - }; + async _process(tc: ToolCallRequest): Promise { + let error = null; + if ( + typeof window === 'undefined' || + !this.iframe || + !this.iframe.contentWindow || + !this.iframeReadyPromise + ) { + error = + 'Error: JavaScript interpreter is not available or iframe not ready.'; + } + + try { + await this.iframeReadyPromise; + } catch (e) { + error = `Error: Iframe for JavaScript interpreter failed to initialize. ${(e as Error).message}`; + } + + let args; + try { + args = JSON.parse(tc.function.arguments); + } catch (e) { + error = `Error: Could not parse arguments for tool call. ${(e as Error).message}`; + } + + const codeToExecute = args.code; + if (typeof codeToExecute !== 'string') { + error = 'Error: "code" argument must be a string.'; + } + + if (error) { + return { + type: 'function_call_output', + call_id: tc.call_id, + output: error, + } as ToolCallOutput; + } + + return new Promise((resolve) => { + this.pendingCalls.set(tc.call_id, resolve); + const message: IframeMessage = { + command: 'executeCode', + code: codeToExecute, + call_id: tc.call_id, + }; + this.iframe!.contentWindow!.postMessage(message, '*'); + }); + } - public clear = (): void => { - this._content = ''; - }; + // public dispose(): void { + // if (this.iframe) { + // document.body.removeChild(this.iframe); + // this.iframe = null; + // } + // if (this.messageHandler) { + // window.removeEventListener('message', this.messageHandler); + // this.messageHandler = null; + // } + // this.pendingCalls.clear(); + // this.resolveIframeReady = null; + // this.rejectIframeReady = null; + // } }