process selected file

common/chat.cpp

@@ -125,7 +125,9 @@ std::vector<common_chat_msg> common_chat_msgs_parse_oaicompat(const json & messa
             msgs.push_back(msg);
         }
     } catch (const std::exception & e) {
-        throw std::runtime_error("Failed to parse messages: " + std::string(e.what()) + "; messages = " + messages.dump(2));
+        // @ngxson : disable otherwise it's bloating the API response
+        // printf("%s\n", std::string("; messages = ") + messages.dump(2));
+        throw std::runtime_error("Failed to parse messages: " + std::string(e.what()));
     }
 
     return msgs;

tools/server/webui/src/components/ChatInputExtraContextItem.tsx

@@ -0,0 +1,92 @@
+import { DocumentTextIcon, XMarkIcon } from '@heroicons/react/24/outline';
+import { MessageExtra } from '../utils/types';
+import { useState } from 'react';
+import { classNames } from '../utils/misc';
+
+export default function ChatInputExtraContextItem({
+  items,
+  removeItem,
+  clickToShow,
+}: {
+  items?: MessageExtra[];
+  removeItem?: (index: number) => void;
+  clickToShow?: boolean;
+}) {
+  const [show, setShow] = useState(-1);
+  const showingItem = show >= 0 ? items?.[show] : undefined;
+
+  if (!items) return null;
+
+  return (
+    <div className="flex flex-row gap-4 overflow-x-auto py-2 px-1 mb-1">
+      {items.map((item, i) => (
+        <div
+          className="indicator"
+          key={i}
+          onClick={() => clickToShow && setShow(i)}
+        >
+          {removeItem && (
+            <div className="indicator-item indicator-top">
+              <button
+                className="btn btn-neutral btn-sm w-4 h-4 p-0 rounded-full"
+                onClick={() => removeItem(i)}
+              >
+                <XMarkIcon className="h-3 w-3" />
+              </button>
+            </div>
+          )}
+
+          <div
+            className={classNames({
+              'flex flex-row rounded-md shadow-sm items-center m-0 p-0': true,
+              'cursor-pointer hover:shadow-md': !!clickToShow,
+            })}
+          >
+            {item.type === 'imageFile' ? (
+              <>
+                <img
+                  src={item.base64Url}
+                  alt={item.name}
+                  className="w-14 h-14 object-cover rounded-md"
+                />
+              </>
+            ) : (
+              <>
+                <div className="w-14 h-14 flex items-center justify-center">
+                  <DocumentTextIcon className="h-8 w-14 text-base-content/50" />
+                </div>
+
+                <div className="text-xs pr-4">
+                  <b>{item.name}</b>
+                </div>
+              </>
+            )}
+          </div>
+        </div>
+      ))}
+
+      {showingItem && (
+        <dialog className="modal modal-open">
+          <div className="modal-box">
+            <div className="flex justify-between items-center mb-4">
+              <b>{showingItem.name}</b>
+              <button className="btn btn-ghost btn-sm">
+                <XMarkIcon className="h-5 w-5" onClick={() => setShow(-1)} />
+              </button>
+            </div>
+            {showingItem.type === 'imageFile' ? (
+              <img src={showingItem.base64Url} alt={showingItem.name} />
+            ) : (
+              <div className="overflow-x-auto">
+                <pre className="whitespace-pre-wrap break-words text-sm">
+                  {showingItem.content}
+                </pre>
+              </div>
+            )}
+          </div>
+          <div className="modal-backdrop" onClick={() => setShow(-1)}></div>
+        </dialog>
+      )}
+    </div>
+  );
+}

tools/server/webui/src/components/ChatMessage.tsx

@@ -4,6 +4,7 @@ import { Message, PendingMessage } from '../utils/types';
 import { classNames } from '../utils/misc';
 import MarkdownDisplay, { CopyButton } from './MarkdownDisplay';
 import { ChevronLeftIcon, ChevronRightIcon } from '@heroicons/react/24/outline';
+import ChatInputExtraContextItem from './ChatInputExtraContextItem';
 
 interface SplitMessage {
   content: PendingMessage['content'];
@@ -85,6 +86,10 @@ export default function ChatMessage({
           'chat-end': msg.role === 'user',
         })}
       >
+        {msg.extra && msg.extra.length > 0 && (
+          <ChatInputExtraContextItem items={msg.extra} clickToShow />
+        )}
+
         <div
           className={classNames({
             'chat-bubble markdown': true,
@@ -160,34 +165,6 @@ export default function ChatMessage({
                       </details>
                     )}
 
-                    {msg.extra && msg.extra.length > 0 && (
-                      <details
-                        className={classNames({
-                          'collapse collapse-arrow mb-4 bg-base-200': true,
-                          'bg-opacity-10': msg.role !== 'assistant',
-                        })}
-                      >
-                        <summary className="collapse-title">
-                          Extra content
-                        </summary>
-                        <div className="collapse-content">
-                          {msg.extra.map(
-                            (extra, i) =>
-                              extra.type === 'textFile' ? (
-                                <div key={extra.name}>
-                                  <b>{extra.name}</b>
-                                  <pre>{extra.content}</pre>
-                                </div>
-                              ) : extra.type === 'context' ? (
-                                <div key={i}>
-                                  <pre>{extra.content}</pre>
-                                </div>
-                              ) : null // TODO: support other extra types
-                          )}
-                        </div>
-                      </details>
-                    )}
-
                     <MarkdownDisplay
                       content={content}
                       isGenerating={isPending}

tools/server/webui/src/components/ChatScreen.tsx

@@ -1,7 +1,7 @@
 import { useEffect, useMemo, useState } from 'react';
 import { CallbackGeneratedChunk, useAppContext } from '../utils/app.context';
 import ChatMessage from './ChatMessage';
-import { CanvasType, Message, MessageExtraContext, PendingMessage } from '../utils/types';
+import { CanvasType, Message, PendingMessage } from '../utils/types';
 import { classNames, cleanCurrentUrl, throttle } from '../utils/misc';
 import CanvasPyInterpreter from './CanvasPyInterpreter';
 import StorageUtils from '../utils/storage';
@@ -12,12 +12,15 @@ import {
   StopIcon,
   PaperClipIcon,
   DocumentTextIcon,
+  XMarkIcon,
 } from '@heroicons/react/24/solid';
 import {
   ChatExtraContextApi,
   useChatExtraContext,
 } from './useChatExtraContext.tsx';
 import Dropzone from 'react-dropzone';
+import toast from 'react-hot-toast';
+import ChatInputExtraContextItem from './ChatInputExtraContextItem.tsx';
 
 /**
  * A message display is a message node with additional information for rendering.
@@ -143,8 +146,10 @@ export default function ChatScreen() {
 
   const sendNewMessage = async () => {
     const lastInpMsg = textarea.value();
-    if (lastInpMsg.trim().length === 0 || isGenerating(currConvId ?? ''))
+    if (lastInpMsg.trim().length === 0 || isGenerating(currConvId ?? '')) {
+      toast.error('Please enter a message');
       return;
+    }
     textarea.setValue('');
     scrollToBottom(false);
     setCurrNodeId(-1);
@@ -312,7 +317,16 @@ function ChatInput({
         multiple={true}
       >
         {({ getRootProps, getInputProps }) => (
-          <div className="flex rounded-xl border-1 border-base-content/30 p-3 w-full" {...getRootProps()}>
+          <div
+            className="flex flex-col rounded-xl border-1 border-base-content/30 p-3 w-full"
+            {...getRootProps()}
+          >
+            <ChatInputExtraContextItem
+              items={extraContext.items}
+              removeItem={extraContext.removeItem}
+            />
+
+            <div className="flex flex-row w-full">
               <textarea
                 // Default (mobile): Enable vertical resize, overflow auto for scrolling if needed
                 // Large screens (lg:): Disable manual resize, apply max-height for autosize limit
@@ -367,15 +381,9 @@ function ChatInput({
                 )}
               </div>
             </div>
+          </div>
         )}
       </Dropzone>
     </div>
   );
 }
-
-function ChatInputExtraContextItem({}: {
-  idx: number,
-  item: MessageExtraContext,
-}) {
-
-}

tools/server/webui/src/components/Sidebar.tsx

@@ -56,7 +56,7 @@ export default function Sidebar() {
           <div
             className={classNames({
               'btn btn-ghost justify-start': true,
-              'btn-active': !currConv,
+              'btn-soft': !currConv,
             })}
             onClick={() => navigate('/')}
           >
@@ -67,7 +67,7 @@ export default function Sidebar() {
               key={conv.id}
               className={classNames({
                 'btn btn-ghost justify-start font-normal': true,
-                'btn-active': conv.id === currConv?.id,
+                'btn-soft': conv.id === currConv?.id,
               })}
               onClick={() => navigate(`/chat/${conv.id}`)}
               dir="auto"

tools/server/webui/src/utils/app.context.tsx

@@ -15,6 +15,7 @@ import {
 } from './misc';
 import { BASE_URL, CONFIG_DEFAULT, isDev } from '../Config';
 import { matchPath, useLocation, useNavigate } from 'react-router';
+import toast from 'react-hot-toast';
 
 interface AppContextValue {
   // conversations and messages
@@ -260,7 +261,7 @@ export const AppContextProvider = ({
       } else {
         console.error(err);
         // eslint-disable-next-line @typescript-eslint/no-explicit-any
-        alert((err as any)?.message ?? 'Unknown error');
+        toast.error((err as any)?.message ?? 'Unknown error');
         throw err; // rethrow
       }
     }

tools/server/webui/src/utils/llama-vscode.ts

@@ -31,6 +31,7 @@ export const useVSCodeContext = (
           extraContext.addItems([
             {
               type: 'context',
+              name: 'Extra context',
               content: data.context,
             },
           ]);

tools/server/webui/src/utils/misc.ts

@@ -1,6 +1,6 @@
 // @ts-expect-error this package does not have typing
 import TextLineStream from 'textlinestream';
-import { APIMessage, Message } from './types';
+import { APIMessage, APIMessageContentPart, Message } from './types';
 
 // ponyfill for missing ReadableStream asyncIterator on Safari
 import { asyncIterator } from '@sec-ant/readable-stream/ponyfill/asyncIterator';
@@ -57,19 +57,44 @@ export const copyStr = (textToCopy: string) => {
  */
 export function normalizeMsgsForAPI(messages: Readonly<Message[]>) {
   return messages.map((msg) => {
-    let newContent = '';
+    if (msg.role !== 'user' || !msg.extra) {
+      return {
+        role: msg.role,
+        content: msg.content,
+      } as APIMessage;
+    }
+
+    const contentArr: APIMessageContentPart[] = [
+      {
+        type: 'text',
+        text: msg.content,
+      },
+    ];
 
     for (const extra of msg.extra ?? []) {
       if (extra.type === 'context') {
-        newContent += `${extra.content}\n\n`;
+        contentArr.push({
+          type: 'text',
+          text: extra.content,
+        });
+      } else if (extra.type === 'textFile') {
+        contentArr.push({
+          type: 'text',
+          text: `File: ${extra.name}\nContent:\n\n${extra.content}`,
+        });
+      } else if (extra.type === 'imageFile') {
+        contentArr.push({
+          type: 'image_url',
+          image_url: { url: extra.base64Url },
+        });
+      } else {
+        throw new Error('Unknown extra type');
       }
     }
 
-    newContent += msg.content;
-
     return {
       role: msg.role,
-      content: newContent,
+      content: contentArr,
     };
   }) as APIMessage[];
 }
@@ -78,13 +103,19 @@ export function normalizeMsgsForAPI(messages: Readonly<Message[]>) {
  * recommended for DeepsSeek-R1, filter out content between <think> and </think> tags
  */
 export function filterThoughtFromMsgs(messages: APIMessage[]) {
+  console.debug({ messages });
   return messages.map((msg) => {
+    if (msg.role !== 'assistant') {
+      return msg;
+    }
+    // assistant message is always a string
+    const contentStr = msg.content as string;
     return {
       role: msg.role,
       content:
         msg.role === 'assistant'
-          ? msg.content.split('</think>').at(-1)!.trim()
-          : msg.content,
+          ? contentStr.split('</think>').at(-1)!.trim()
+          : contentStr,
     } as APIMessage;
   });
 }

tools/server/webui/src/utils/types.ts

@@ -48,7 +48,10 @@ export interface Message {
   children: Message['id'][];
 }
 
-export type MessageExtra = MessageExtraTextFile | MessageExtraImageFile | MessageExtraContext;
+export type MessageExtra =
+  | MessageExtraTextFile
+  | MessageExtraImageFile
+  | MessageExtraContext;
 
 export interface MessageExtraTextFile {
   type: 'textFile';
@@ -64,10 +67,24 @@ export interface MessageExtraImageFile {
 
 export interface MessageExtraContext {
   type: 'context';
+  name: string;
   content: string;
 }
 
-export type APIMessage = Pick<Message, 'role' | 'content'>;
+export type APIMessageContentPart =
+  | {
+      type: 'text';
+      text: string;
+    }
+  | {
+      type: 'image_url';
+      image_url: { url: string };
+    };
+
+export type APIMessage = {
+  role: Message['role'];
+  content: string | APIMessageContentPart[];
+};
 
 export interface Conversation {
   id: string; // format: `conv-{timestamp}`