diff --git a/tools/server/webui/package.json b/tools/server/webui/package.json index 6ac06b1a49bd3..153ae54f46c9f 100644 --- a/tools/server/webui/package.json +++ b/tools/server/webui/package.json @@ -33,6 +33,7 @@ "remark-math": "^6.0.0", "tailwindcss": "^4.1.1", "textlinestream": "^1.1.1", + "unist-util-visit": "^5.0.0", "vite-plugin-singlefile": "^2.0.3" }, "devDependencies": { diff --git a/tools/server/webui/src/Config.ts b/tools/server/webui/src/Config.ts index dd1cc0e100a2e..2f5a742c2b6c2 100644 --- a/tools/server/webui/src/Config.ts +++ b/tools/server/webui/src/Config.ts @@ -4,9 +4,7 @@ import { isNumeric } from './utils/misc'; export const isDev = import.meta.env.MODE === 'development'; // constants -export const BASE_URL = new URL('.', document.baseURI).href - .toString() - .replace(/\/$/, ''); +export const BASE_URL = 'http://127.0.0.1:8080'; export const CONFIG_DEFAULT = { // Note: in order not to introduce breaking changes, please keep the same data type (number, string, etc) if you want to change the default value. Do not use null or undefined for default value. @@ -39,6 +37,7 @@ export const CONFIG_DEFAULT = { custom: '', // custom json-stringified object // experimental features pyIntepreterEnabled: false, + jsInterpreterToolUse: false, }; export const CONFIG_INFO: Record = { apiKey: 'Set the API Key if you are using --api-key option for the server.', diff --git a/tools/server/webui/src/assets/iframe_sandbox.html b/tools/server/webui/src/assets/iframe_sandbox.html new file mode 100644 index 0000000000000..b98d8adf0c7fb --- /dev/null +++ b/tools/server/webui/src/assets/iframe_sandbox.html @@ -0,0 +1,78 @@ + + + + JS Sandbox + + + +

JavaScript Execution Sandbox

+ + diff --git a/tools/server/webui/src/components/ChatMessage.tsx b/tools/server/webui/src/components/ChatMessage.tsx index 40ea74711f349..8cc6fc0df1424 100644 --- a/tools/server/webui/src/components/ChatMessage.tsx +++ b/tools/server/webui/src/components/ChatMessage.tsx @@ -48,17 +48,22 @@ export default function ChatMessage({ const nextSibling = siblingLeafNodeIds[siblingCurrIdx + 1]; const prevSibling = siblingLeafNodeIds[siblingCurrIdx - 1]; - // for reasoning model, we split the message into content and thought - // TODO: implement this as remark/rehype plugin in the future + // for reasoning model, we split the message into content, thought, and tool output const { content, thought, isThinking }: SplitMessage = useMemo(() => { - if (msg.content === null || msg.role !== 'assistant') { + if ( + msg.content === null || + (msg.role !== 'assistant' && msg.role !== 'tool') + ) { return { content: msg.content }; } + let actualContent = ''; let thought = ''; let isThinking = false; let thinkSplit = msg.content.split('', 2); + actualContent += thinkSplit[0]; + while (thinkSplit[1] !== undefined) { // tag found thinkSplit = thinkSplit[1].split('', 2); @@ -71,11 +76,13 @@ export default function ChatMessage({ actualContent += thinkSplit[0]; } } + return { content: actualContent, thought, isThinking }; }, [msg]); - if (!viewingChat) return null; + const toolCalls = msg.tool_calls ?? null; + return (
{content === null ? ( <> - {/* show loading dots for pending message */} - + {toolCalls ? null : ( + <> + {/* show loading dots for pending message */} + + + )} ) : ( <> @@ -188,13 +199,53 @@ export default function ChatMessage({ )} - + {msg.role === 'tool' ? ( +
+ + Tool call result + +
+ +
+
+ ) : ( + + )}
)} + {toolCalls && + toolCalls.map((toolCall, i) => ( +
+ + Tool call: {toolCall.function.name} + + +
+
Arguments:
+ 
+                        {JSON.stringify(
+                          JSON.parse(toolCall.function.arguments),
+                          null,
+                          2
+                        )}
+
+
+
+ ))} {/* render timings if enabled */} {timings && config.showTokensPerSecond && (
diff --git a/tools/server/webui/src/components/SettingDialog.tsx b/tools/server/webui/src/components/SettingDialog.tsx index b0044d25403b5..7baf17dd27a23 100644 --- a/tools/server/webui/src/components/SettingDialog.tsx +++ b/tools/server/webui/src/components/SettingDialog.tsx @@ -254,6 +254,21 @@ const SETTING_SECTIONS: SettingSection[] = [ ), key: 'pyIntepreterEnabled', }, + { + type: SettingInputType.CHECKBOX, + label: ( + <> + Enable JavaScript tool use +
+ + This alows LLM to use browser your browser console as tool. If + model supports function calling, it can use the console to do e.g. + data analysis etc. by itself. + + + ), + key: 'jsInterpreterToolUse', + }, ], }, ]; diff --git a/tools/server/webui/src/utils/app.context.tsx b/tools/server/webui/src/utils/app.context.tsx index 54bb65b6e3cb2..7f8d005d88f23 100644 --- a/tools/server/webui/src/utils/app.context.tsx +++ b/tools/server/webui/src/utils/app.context.tsx @@ -5,6 +5,7 @@ import { Conversation, Message, PendingMessage, + ToolCallRequest, ViewingChat, } from './types'; import StorageUtils from './storage'; @@ -15,6 +16,7 @@ import { } from './misc'; import { BASE_URL, CONFIG_DEFAULT, isDev } from '../Config'; import { matchPath, useLocation, useNavigate } from 'react-router'; +import { AVAILABLE_TOOLS } from './tool_calling/register_tools'; interface AppContextValue { // conversations and messages @@ -134,8 +136,8 @@ export const AppContextProvider = ({ convId: string, leafNodeId: Message['id'], onChunk: CallbackGeneratedChunk - ) => { - if (isGenerating(convId)) return; + ): Promise => { + if (isGenerating(convId)) return leafNodeId; const config = StorageUtils.getConfig(); const currConversation = await StorageUtils.getOneConversation(convId); @@ -181,10 +183,21 @@ export const AppContextProvider = ({ } if (isDev) console.log({ messages }); + // tool calling from clientside + const enabledTools = Array.from( + AVAILABLE_TOOLS, + ([_name, tool], _index) => tool + ) + .filter((tool) => tool.enabled) + .map((tool) => tool.specs); + + // stream does not support tool-use (yet?) + const streamResponse = enabledTools.length === 0; + // prepare params const params = { messages, - stream: true, + stream: streamResponse, cache_prompt: true, samplers: config.samplers, temperature: config.temperature, @@ -206,6 +219,7 @@ export const AppContextProvider = ({ dry_penalty_last_n: config.dry_penalty_last_n, max_tokens: config.max_tokens, timings_per_token: !!config.showTokensPerSecond, + tools: enabledTools.length > 0 ? enabledTools : undefined, ...(config.custom.length ? JSON.parse(config.custom) : {}), }; @@ -221,37 +235,148 @@ export const AppContextProvider = ({ body: JSON.stringify(params), signal: abortController.signal, }); + if (fetchResponse.status !== 200) { const body = await fetchResponse.json(); throw new Error(body?.error?.message || 'Unknown error'); } - const chunks = getSSEStreamAsync(fetchResponse); - for await (const chunk of chunks) { - // const stop = chunk.stop; - if (chunk.error) { - throw new Error(chunk.error?.message || 'Unknown error'); + + // Tool calls results we will process later + const pendingMessages: PendingMessage[] = []; + let lastMsgId = pendingMsg.id; + let shouldContinueChain = false; + + if (streamResponse) { + const chunks = getSSEStreamAsync(fetchResponse); + for await (const chunk of chunks) { + // const stop = chunk.stop; + if (chunk.error) { + throw new Error(chunk.error?.message || 'Unknown error'); + } + const addedContent = chunk.choices[0].delta.content; + const lastContent = pendingMsg.content || ''; + if (addedContent) { + pendingMsg = { + ...pendingMsg, + content: lastContent + addedContent, + }; + } + const timings = chunk.timings; + if (timings && config.showTokensPerSecond) { + // only extract what's really needed, to save some space + pendingMsg.timings = { + prompt_n: timings.prompt_n, + prompt_ms: timings.prompt_ms, + predicted_n: timings.predicted_n, + predicted_ms: timings.predicted_ms, + }; + } + setPending(convId, pendingMsg); + onChunk(); // don't need to switch node for pending message + } + } else { + const responseData = await fetchResponse.json(); + if (responseData.error) { + throw new Error(responseData.error?.message || 'Unknown error'); + } + + const choice = responseData.choices[0]; + const messageFromAPI = choice.message; + let newContent = ''; + + if (messageFromAPI.content) { + newContent = messageFromAPI.content; + console.log(newContent); + } + + // Process tool calls + if (messageFromAPI.tool_calls && messageFromAPI.tool_calls.length > 0) { + // Store the raw tool calls in the pendingMsg + pendingMsg = { + ...pendingMsg, + tool_calls: messageFromAPI.tool_calls as ToolCallRequest[], + }; + + for (let i = 0; i < messageFromAPI.tool_calls.length; i++) { + const toolCall = messageFromAPI.tool_calls[i] as ToolCallRequest; + if (toolCall) { + // Set up call id + toolCall.call_id ??= `call_${i}`; + + if (isDev) console.log({ tc: toolCall }); + + // Process tool call + const toolResult = await AVAILABLE_TOOLS.get( + toolCall.function.name + )?.processCall(toolCall); + + const toolMsg: PendingMessage = { + id: lastMsgId + 1, + type: 'text', + convId: convId, + content: toolResult?.output ?? 'Error: invalid tool call!', + timestamp: Date.now(), + role: 'tool', + parent: lastMsgId, + children: [], + }; + pendingMessages.push(toolMsg); + lastMsgId += 1; + } + } } - const addedContent = chunk.choices[0].delta.content; - const lastContent = pendingMsg.content || ''; - if (addedContent) { + + if (newContent !== '') { pendingMsg = { ...pendingMsg, - content: lastContent + addedContent, + content: newContent, }; } - const timings = chunk.timings; - if (timings && config.showTokensPerSecond) { - // only extract what's really needed, to save some space + + // Handle timings from the non-streaming response + const apiTimings = responseData.timings; + if (apiTimings && config.showTokensPerSecond) { pendingMsg.timings = { - prompt_n: timings.prompt_n, - prompt_ms: timings.prompt_ms, - predicted_n: timings.predicted_n, - predicted_ms: timings.predicted_ms, + prompt_n: apiTimings.prompt_n, + prompt_ms: apiTimings.prompt_ms, + predicted_n: apiTimings.predicted_n, + predicted_ms: apiTimings.predicted_ms, }; } - setPending(convId, pendingMsg); - onChunk(); // don't need to switch node for pending message + + for (const pendMsg of pendingMessages) { + setPending(convId, pendMsg); + onChunk(pendMsg.id); // Update UI to show the processed message + } + + shouldContinueChain = choice.finish_reason === 'tool_calls'; + } + + pendingMessages.unshift(pendingMsg); + if ( + pendingMsg.content !== null || + (pendingMsg.tool_calls?.length ?? 0) > 0 + ) { + await StorageUtils.appendMsgChain( + pendingMessages as Message[], + leafNodeId + ); } + + // if message ended due to "finish_reason": "tool_calls" + // resend it to assistant to process the result. + if (shouldContinueChain) { + lastMsgId = await generateMessage(convId, lastMsgId, onChunk); + } + + setPending(convId, null); + onChunk(lastMsgId); // trigger scroll to bottom and switch to the last node + + // Fetch messages from DB + const savedMsgs = await StorageUtils.getMessages(convId); + console.log({ savedMsgs }); + + return lastMsgId; } catch (err) { setPending(convId, null); if ((err as Error).name === 'AbortError') { @@ -265,11 +390,7 @@ export const AppContextProvider = ({ } } - if (pendingMsg.content !== null) { - await StorageUtils.appendMsg(pendingMsg as Message, leafNodeId); - } - setPending(convId, null); - onChunk(pendingId); // trigger scroll to bottom and switch to the last node + return pendingId; }; const sendMessage = async ( @@ -293,7 +414,7 @@ export const AppContextProvider = ({ const now = Date.now(); const currMsgId = now; - StorageUtils.appendMsg( + await StorageUtils.appendMsg( { id: currMsgId, timestamp: now, diff --git a/tools/server/webui/src/utils/misc.ts b/tools/server/webui/src/utils/misc.ts index 87f55b2af95c2..0fb82717373e8 100644 --- a/tools/server/webui/src/utils/misc.ts +++ b/tools/server/webui/src/utils/misc.ts @@ -22,7 +22,6 @@ export async function* getSSEStreamAsync(fetchResponse: Response) { .pipeThrough(new TextLineStream()); // @ts-expect-error asyncIterator complains about type, but it should work for await (const line of asyncIterator(lines)) { - //if (isDev) console.log({ line }); if (line.startsWith('data:') && !line.endsWith('[DONE]')) { const data = JSON.parse(line.slice(5)); yield data; @@ -67,10 +66,16 @@ export function normalizeMsgsForAPI(messages: Readonly) { newContent += msg.content; - return { + const apiMsg = { role: msg.role, content: newContent, - }; + } as APIMessage; + + if (msg.tool_calls && msg.tool_calls.length > 0) { + apiMsg.tool_calls = msg.tool_calls; + } + + return apiMsg; }) as APIMessage[]; } @@ -79,13 +84,19 @@ export function normalizeMsgsForAPI(messages: Readonly) { */ export function filterThoughtFromMsgs(messages: APIMessage[]) { return messages.map((msg) => { - return { + const filteredMessage: APIMessage = { role: msg.role, content: msg.role === 'assistant' ? msg.content.split('').at(-1)!.trim() : msg.content, - } as APIMessage; + }; + + if (msg.tool_calls && msg.tool_calls.length > 0) { + filteredMessage.tool_calls = msg.tool_calls; + } + + return filteredMessage; }); } diff --git a/tools/server/webui/src/utils/storage.ts b/tools/server/webui/src/utils/storage.ts index 1dfc9d9799311..4133cf84edf16 100644 --- a/tools/server/webui/src/utils/storage.ts +++ b/tools/server/webui/src/utils/storage.ts @@ -123,39 +123,85 @@ const StorageUtils = { msg: Exclude, parentNodeId: Message['id'] ): Promise { - if (msg.content === null) return; - const { convId } = msg; - await db.transaction('rw', db.conversations, db.messages, async () => { - const conv = await StorageUtils.getOneConversation(convId); - const parentMsg = await db.messages - .where({ convId, id: parentNodeId }) - .first(); - // update the currNode of conversation - if (!conv) { - throw new Error(`Conversation ${convId} does not exist`); - } - if (!parentMsg) { - throw new Error( - `Parent message ID ${parentNodeId} does not exist in conversation ${convId}` - ); - } - await db.conversations.update(convId, { - lastModified: Date.now(), - currNode: msg.id, - }); - // update parent - await db.messages.update(parentNodeId, { - children: [...parentMsg.children, msg.id], - }); - // create message - await db.messages.add({ - ...msg, - parent: parentNodeId, - children: [], + await this.appendMsgChain([msg], parentNodeId); + }, + + /** + * Adds chain of messages to the DB, usually + * produced by tool calling. + */ + async appendMsgChain( + messages: Exclude[], + parentNodeId: Message['id'] + ): Promise { + if (messages.length === 0) return; + + const { convId } = messages[0]; + + // Verify conversation exists + const conv = await this.getOneConversation(convId); + if (!conv) { + throw new Error(`Conversation ${convId} does not exist`); + } + + // Verify starting parent exists + const startParent = await db.messages + .where({ convId, id: parentNodeId }) + .first(); + if (!startParent) { + throw new Error( + `Starting parent message ${parentNodeId} does not exist in conversation ${convId}` + ); + } + + // Get the last message ID for updating the conversation + const lastMsgId = messages[messages.length - 1].id; + + try { + // Process all messages in a single transaction + await db.transaction('rw', db.messages, db.conversations, () => { + // First message connects to startParentId + let parentId = parentNodeId; + const parentChildren = [...startParent.children]; + + for (let i = 0; i < messages.length; i++) { + const msg = messages[i]; + + // Add this message to its parent's children + if (i === 0) { + // First message - update the starting parent + parentChildren.push(msg.id); + db.messages.update(parentId, { children: parentChildren }); + } else { + // Other messages - previous message is the parent + db.messages.update(parentId, { children: [msg.id] }); + } + + // Add the message + db.messages.add({ + ...msg, + parent: parentId, + children: [], // Will be updated if this message has children + }); + + // Next message's parent is this message + parentId = msg.id; + } + + // Update the conversation + db.conversations.update(convId, { + lastModified: Date.now(), + currNode: lastMsgId, + }); }); - }); + } catch (error) { + console.error('Error saving message chain:', error); + throw error; + } + dispatchConversationChange(convId); }, + /** * remove conversation by id */ diff --git a/tools/server/webui/src/utils/tool_calling/agent_tool.ts b/tools/server/webui/src/utils/tool_calling/agent_tool.ts new file mode 100644 index 0000000000000..18234ca0668fc --- /dev/null +++ b/tools/server/webui/src/utils/tool_calling/agent_tool.ts @@ -0,0 +1,69 @@ +import { + ToolCallRequest, + ToolCallOutput, + ToolCallParameters, + ToolCallSpec, +} from '../types'; + +export abstract class AgentTool { + constructor( + public readonly id: string, + private readonly isEnabledCallback: () => boolean, + public readonly toolDescription: string, + public readonly parameters: ToolCallParameters + ) {} + + /** + * "Public" wrapper for the tool call processing logic. + * @param call The tool call object from the API response. + * @returns The tool call output or undefined if the tool is not enabled. + */ + public async processCall( + call: ToolCallRequest + ): Promise { + if (this.enabled) { + try { + return await this._process(call); + } catch (error) { + console.error(`Error processing tool call for ${this.id}:`, error); + return { + type: 'function_call_output', + call_id: call.call_id, + output: `Error during tool execution: ${(error as Error).message}`, + } as ToolCallOutput; + } + } + return undefined; + } + + /** + * Whether calling this tool is enabled. + * User can toggle the status from the settings panel. + * @returns enabled status. + */ + public get enabled(): boolean { + return this.isEnabledCallback(); + } + + /** + * Specifications for the tool call. + * https://github.com/ggml-org/llama.cpp/blob/master/docs/function-calling.md + * https://platform.openai.com/docs/guides/function-calling?api-mode=responses#defining-functions + */ + public get specs(): ToolCallSpec { + return { + type: 'function', + function: { + name: this.id, + description: this.toolDescription, + parameters: this.parameters, + }, + }; + } + + /** + * The actual tool call processing logic. + * @param call: The tool call object from the API response. + */ + protected abstract _process(call: ToolCallRequest): Promise; +} diff --git a/tools/server/webui/src/utils/tool_calling/js_repl_tool.ts b/tools/server/webui/src/utils/tool_calling/js_repl_tool.ts new file mode 100644 index 0000000000000..37bbf4450817d --- /dev/null +++ b/tools/server/webui/src/utils/tool_calling/js_repl_tool.ts @@ -0,0 +1,174 @@ +import StorageUtils from '../storage'; +import { ToolCallRequest, ToolCallOutput, ToolCallParameters } from '../types'; +import { AgentTool } from './agent_tool'; + +// Import the HTML content as a raw string +import iframeHTMLContent from '../../assets/iframe_sandbox.html?raw'; + +interface IframeMessage { + call_id: string; + output?: string; + error?: string; + command?: 'executeCode' | 'iframeReady'; + code?: string; +} + +export class JSReplAgentTool extends AgentTool { + private static readonly ID = 'javascript_interpreter'; + private iframe: HTMLIFrameElement | null = null; + private iframeReadyPromise: Promise | null = null; + private resolveIframeReady: (() => void) | null = null; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + private rejectIframeReady: ((reason?: any) => void) | null = null; + private pendingCalls = new Map void>(); + private messageHandler: + | ((event: MessageEvent) => void) + | null = null; + + constructor() { + super( + JSReplAgentTool.ID, + () => StorageUtils.getConfig().jsInterpreterToolUse, + 'Executes JavaScript code in a sandboxed iframe. The code should be self-contained valid javascript. You can use console.log(variable) to print out intermediate values, which will be captured.', + { + type: 'object', + properties: { + code: { + type: 'string', + description: 'Valid JavaScript code to execute.', + }, + }, + required: ['code'], + } as ToolCallParameters + ); + this.initIframe(); + } + + private initIframe(): void { + if (typeof window === 'undefined' || typeof document === 'undefined') { + console.warn( + 'JSReplAgentTool: Not in a browser environment, iframe will not be created.' + ); + return; + } + + this.iframeReadyPromise = new Promise((resolve, reject) => { + this.resolveIframeReady = resolve; + this.rejectIframeReady = reject; + }); + + this.messageHandler = (event: MessageEvent) => { + if ( + !event.data || + !this.iframe || + !this.iframe.contentWindow || + event.source !== this.iframe.contentWindow + ) { + return; + } + + const { command, call_id, output, error } = event.data; + if (command === 'iframeReady' && call_id === 'initial_ready') { + if (this.resolveIframeReady) { + this.resolveIframeReady(); + this.resolveIframeReady = null; + this.rejectIframeReady = null; + } + return; + } + if (typeof call_id !== 'string') { + return; + } + if (this.pendingCalls.has(call_id)) { + const callback = this.pendingCalls.get(call_id)!; + callback({ + type: 'function_call_output', + call_id: call_id, + output: error ? `Error: ${error}` : (output ?? ''), + } as ToolCallOutput); + this.pendingCalls.delete(call_id); + } + }; + window.addEventListener('message', this.messageHandler); + + this.iframe = document.createElement('iframe'); + this.iframe.style.display = 'none'; + this.iframe.sandbox.add('allow-scripts'); + + // Use srcdoc with the imported HTML content + this.iframe.srcdoc = iframeHTMLContent; + + document.body.appendChild(this.iframe); + + setTimeout(() => { + if (this.rejectIframeReady) { + this.rejectIframeReady(new Error('Iframe readiness timeout')); + this.resolveIframeReady = null; + this.rejectIframeReady = null; + } + }, 5000); + } + + async _process(tc: ToolCallRequest): Promise { + let error = null; + if ( + typeof window === 'undefined' || + !this.iframe || + !this.iframe.contentWindow || + !this.iframeReadyPromise + ) { + error = + 'Error: JavaScript interpreter is not available or iframe not ready.'; + } + + try { + await this.iframeReadyPromise; + } catch (e) { + error = `Error: Iframe for JavaScript interpreter failed to initialize. ${(e as Error).message}`; + } + + let args; + try { + args = JSON.parse(tc.function.arguments); + } catch (e) { + error = `Error: Could not parse arguments for tool call. ${(e as Error).message}`; + } + + const codeToExecute = args.code; + if (typeof codeToExecute !== 'string') { + error = 'Error: "code" argument must be a string.'; + } + + if (error) { + return { + type: 'function_call_output', + call_id: tc.call_id, + output: error, + } as ToolCallOutput; + } + + return new Promise((resolve) => { + this.pendingCalls.set(tc.call_id, resolve); + const message: IframeMessage = { + command: 'executeCode', + code: codeToExecute, + call_id: tc.call_id, + }; + this.iframe!.contentWindow!.postMessage(message, '*'); + }); + } + + // public dispose(): void { + // if (this.iframe) { + // document.body.removeChild(this.iframe); + // this.iframe = null; + // } + // if (this.messageHandler) { + // window.removeEventListener('message', this.messageHandler); + // this.messageHandler = null; + // } + // this.pendingCalls.clear(); + // this.resolveIframeReady = null; + // this.rejectIframeReady = null; + // } +} diff --git a/tools/server/webui/src/utils/tool_calling/register_tools.ts b/tools/server/webui/src/utils/tool_calling/register_tools.ts new file mode 100644 index 0000000000000..0ba08ac214822 --- /dev/null +++ b/tools/server/webui/src/utils/tool_calling/register_tools.ts @@ -0,0 +1,22 @@ +import { isDev } from '../../Config'; +import { AgentTool } from './agent_tool'; +import { JSReplAgentTool } from './js_repl_tool'; + +/** + * Map of available tools for function calling. + * Note that these tools are not necessarily enabled by the user. + */ +export const AVAILABLE_TOOLS = new Map(); + +function registerTool(tool: T): T { + AVAILABLE_TOOLS.set(tool.id, tool); + if (isDev) { + console.log( + `Successfully registered tool: ${tool.id}, enabled: ${tool.enabled}` + ); + } + return tool; +} + +// Available agent tools +export const jsReplTool = registerTool(new JSReplAgentTool()); diff --git a/tools/server/webui/src/utils/types.ts b/tools/server/webui/src/utils/types.ts index 0eb774001ecc5..6ad02a6780357 100644 --- a/tools/server/webui/src/utils/types.ts +++ b/tools/server/webui/src/utils/types.ts @@ -39,10 +39,11 @@ export interface Message { convId: string; type: 'text' | 'root'; timestamp: number; // timestamp from Date.now() - role: 'user' | 'assistant' | 'system'; + role: 'user' | 'assistant' | 'system' | 'tool'; content: string; timings?: TimingReport; extra?: MessageExtra[]; + tool_calls?: ToolCallRequest[]; // node based system for branching parent: Message['id']; children: Message['id'][]; @@ -61,7 +62,7 @@ export interface MessageExtraContext { content: string; } -export type APIMessage = Pick; +export type APIMessage = Pick; export interface Conversation { id: string; // format: `conv-{timestamp}` @@ -89,3 +90,34 @@ export interface CanvasPyInterpreter { } export type CanvasData = CanvasPyInterpreter; + +export interface ToolCallRequest { + id: string; + type: 'function'; + call_id: string; + function: { + name: string; + arguments: string; // JSON string of arguments + }; +} + +export interface ToolCallSpec { + type: 'function'; + function: { + name: string; + description: string; + parameters: ToolCallParameters; + }; +} + +export interface ToolCallParameters { + type: 'object'; + properties: object; + required: string[]; +} + +export interface ToolCallOutput { + type: 'function_call_output'; + call_id: string; + output: string; +}