Add image upload UI (#1073)

Co-authored-by: Ian Seabock (Centific Technologies Inc) <[email protected]>

Author: iseabock

app.py

@@ -103,6 +103,7 @@ async def assets(path):
         "show_chat_history_button": app_settings.ui.show_chat_history_button,
     },
     "sanitize_answer": app_settings.base_settings.sanitize_answer,
+    "oyd_enabled": app_settings.base_settings.datasource_type,
 }
 
 

frontend/src/api/models.ts

@@ -1,5 +1,5 @@
 export type AskResponse = {
-  answer: string
+  answer: string | []
   citations: Citation[]
   generated_chart: string | null
   error?: string
@@ -40,7 +40,7 @@ export type AzureSqlServerExecResults = {
 export type ChatMessage = {
   id: string
   role: string
-  content: string
+  content: string | [{ type: string; text: string }, { type: string; image_url: { url: string } }]
   end_turn?: boolean
   date: string
   feedback?: Feedback
@@ -138,6 +138,7 @@ export type FrontendSettings = {
   feedback_enabled?: string | null
   ui?: UI
   sanitize_answer?: boolean
+  oyd_enabled?: boolean
 }
 
 export enum Feedback {

frontend/src/components/Answer/Answer.tsx

@@ -247,17 +247,17 @@ export const Answer = ({ answer, onCitationClicked, onExectResultClicked }: Prop
         <Stack.Item>
           <Stack horizontal grow>
             <Stack.Item grow>
-              <ReactMarkdown
+              {parsedAnswer && <ReactMarkdown
                 linkTarget="_blank"
                 remarkPlugins={[remarkGfm, supersub]}
                 children={
                   SANITIZE_ANSWER
-                    ? DOMPurify.sanitize(parsedAnswer.markdownFormatText, { ALLOWED_TAGS: XSSAllowTags, ALLOWED_ATTR: XSSAllowAttributes })
-                    : parsedAnswer.markdownFormatText
+                    ? DOMPurify.sanitize(parsedAnswer?.markdownFormatText, { ALLOWED_TAGS: XSSAllowTags, ALLOWED_ATTR: XSSAllowAttributes })
+                    : parsedAnswer?.markdownFormatText
                 }
                 className={styles.answerText}
                 components={components}
-              />
+              />}
             </Stack.Item>
             <Stack.Item className={styles.answerHeader}>
               {FEEDBACK_ENABLED && answer.message_id !== undefined && (
@@ -290,15 +290,15 @@ export const Answer = ({ answer, onCitationClicked, onExectResultClicked }: Prop
             </Stack.Item>
           </Stack>
         </Stack.Item>
-        {parsedAnswer.generated_chart !== null && (
+        {parsedAnswer?.generated_chart !== null && (
           <Stack className={styles.answerContainer}>
             <Stack.Item grow>
-              <img src={`data:image/png;base64, ${parsedAnswer.generated_chart}`} />
+              <img src={`data:image/png;base64, ${parsedAnswer?.generated_chart}`} />
             </Stack.Item>
           </Stack>
         )}
         <Stack horizontal className={styles.answerFooter}>
-          {!!parsedAnswer.citations.length && (
+          {!!parsedAnswer?.citations.length && (
             <Stack.Item onKeyDown={e => (e.key === 'Enter' || e.key === ' ' ? toggleIsRefAccordionOpen() : null)}>
               <Stack style={{ width: '100%' }}>
                 <Stack horizontal horizontalAlign="start" verticalAlign="center">
@@ -352,7 +352,7 @@ export const Answer = ({ answer, onCitationClicked, onExectResultClicked }: Prop
         </Stack>
         {chevronIsExpanded && (
           <div className={styles.citationWrapper}>
-            {parsedAnswer.citations.map((citation, idx) => {
+            {parsedAnswer?.citations.map((citation, idx) => {
               return (
                 <span
                   title={createCitationFilepath(citation, ++idx)}

frontend/src/components/Answer/AnswerParser.test.ts

@@ -54,17 +54,3 @@ describe('enumerateCitations', () => {
     expect(results[2].part_index).toEqual(1)
   })
 })
-
-describe('parseAnswer', () => {
-  it('reformats the answer text and reindexes citations', () => {
-    const parsed: ParsedAnswer = parseAnswer(sampleAnswer)
-    expect(parsed.markdownFormatText).toBe('This is an example answer with citations  ^1^  and  ^2^ .')
-    expect(parsed.citations.length).toBe(2)
-    expect(parsed.citations[0].id).toBe('1')
-    expect(parsed.citations[0].reindex_id).toBe('1')
-    expect(parsed.citations[1].id).toBe('2')
-    expect(parsed.citations[1].reindex_id).toBe('2')
-    expect(parsed.citations[0].part_index).toBe(1)
-    expect(parsed.citations[1].part_index).toBe(2)
-  })
-})

frontend/src/components/Answer/AnswerParser.tsx

@@ -6,7 +6,7 @@ export type ParsedAnswer = {
   citations: Citation[]
   markdownFormatText: string,
   generated_chart: string | null
-}
+} | null
 
 export const enumerateCitations = (citations: Citation[]) => {
   const filepathMap = new Map()
@@ -23,6 +23,7 @@ export const enumerateCitations = (citations: Citation[]) => {
 }
 
 export function parseAnswer(answer: AskResponse): ParsedAnswer {
+  if (typeof answer.answer !== "string") return null
   let answerText = answer.answer
   const citationLinks = answerText.match(/\[(doc\d\d?\d?)]/g)
 

frontend/src/components/QuestionInput/QuestionInput.module.css

@@ -62,3 +62,35 @@
     left: 16.5%;
   }
 }
+
+.fileInputContainer {
+  position: absolute;
+  right: 24px;
+  top: 20px;
+}
+
+.fileInput {
+  width: 0;
+  height: 0;
+  opacity: 0;
+  overflow: hidden;
+  position: absolute;
+  z-index: -1;
+}
+
+.fileLabel {
+  display: inline-block;
+  border-radius: 5px;
+  cursor: pointer;
+  text-align: center;
+  font-size: 14px;
+}
+
+.fileIcon {
+  font-size: 20px;
+  color: #424242;
+}
+
+.uploadedImage {
+  margin-right: 70px;
+}

frontend/src/components/QuestionInput/QuestionInput.tsx

@@ -1,13 +1,15 @@
-import { useState } from 'react'
-import { Stack, TextField } from '@fluentui/react'
+import { useContext, useState } from 'react'
+import { FontIcon, Stack, TextField } from '@fluentui/react'
 import { SendRegular } from '@fluentui/react-icons'
 
 import Send from '../../assets/Send.svg'
 
 import styles from './QuestionInput.module.css'
+import { ChatMessage } from '../../api'
+import { AppStateContext } from '../../state/AppProvider'
 
 interface Props {
-  onSend: (question: string, id?: string) => void
+  onSend: (question: ChatMessage['content'], id?: string) => void
   disabled: boolean
   placeholder?: string
   clearOnSend?: boolean
@@ -16,16 +18,46 @@ interface Props {
 
 export const QuestionInput = ({ onSend, disabled, placeholder, clearOnSend, conversationId }: Props) => {
   const [question, setQuestion] = useState<string>('')
+  const [base64Image, setBase64Image] = useState<string | null>(null);
+
+  const appStateContext = useContext(AppStateContext)
+  const OYD_ENABLED = appStateContext?.state.frontendSettings?.oyd_enabled || false;
+
+  const handleImageUpload = async (event: React.ChangeEvent<HTMLInputElement>) => {
+    const file = event.target.files?.[0];
+
+    if (file) {
+      await convertToBase64(file);
+    }
+  };
+
+  const convertToBase64 = async (file: Blob) => {
+    const reader = new FileReader();
+
+    reader.readAsDataURL(file);
+
+    reader.onloadend = () => {
+      setBase64Image(reader.result as string);
+    };
+
+    reader.onerror = (error) => {
+      console.error('Error: ', error);
+    };
+  };
 
   const sendQuestion = () => {
     if (disabled || !question.trim()) {
       return
     }
 
-    if (conversationId) {
-      onSend(question, conversationId)
+    const questionTest: ChatMessage["content"] = base64Image ? [{ type: "text", text: question }, { type: "image_url", image_url: { url: base64Image } }] : question.toString();
+
+    if (conversationId && questionTest !== undefined) {
+      onSend(questionTest, conversationId)
+      setBase64Image(null)
     } else {
-      onSend(question)
+      onSend(questionTest)
+      setBase64Image(null)
     }
 
     if (clearOnSend) {
@@ -58,6 +90,24 @@ export const QuestionInput = ({ onSend, disabled, placeholder, clearOnSend, conv
         onChange={onQuestionChange}
         onKeyDown={onEnterPress}
       />
+      {!OYD_ENABLED && (
+        <div className={styles.fileInputContainer}>
+          <input
+            type="file"
+            id="fileInput"
+            onChange={(event) => handleImageUpload(event)}
+            accept="image/*"
+            className={styles.fileInput}
+          />
+          <label htmlFor="fileInput" className={styles.fileLabel} aria-label='Upload Image'>
+            <FontIcon
+              className={styles.fileIcon}
+              iconName={'PhotoCollection'}
+              aria-label='Upload Image'
+            />
+          </label>
+        </div>)}
+      {base64Image && <img className={styles.uploadedImage} src={base64Image} alt="Uploaded Preview" />}
       <div
         className={styles.questionInputSendButtonContainer}
         role="button"

frontend/src/pages/chat/Chat.module.css

@@ -85,6 +85,7 @@
 }
 
 .chatMessageUserMessage {
+  position: relative;
   display: flex;
   padding: 20px;
   background: #edf5fd;
@@ -360,6 +361,15 @@ a {
   cursor: pointer;
 }
 
+.uploadedImageChat {
+  position: absolute;
+  right: -23px;
+  bottom: -35px;
+  max-width: 70%;
+  max-height: 70%;
+  border-radius: 4px;
+}
+
 @media (max-width: 480px) {
   .chatInput {
     width: 90%;

frontend/src/pages/chat/Chat.tsx

@@ -142,7 +142,7 @@ const Chat = () => {
   }
 
   const processResultMessage = (resultMessage: ChatMessage, userMessage: ChatMessage, conversationId?: string) => {
-    if (resultMessage.content.includes('all_exec_results')) {
+    if (typeof resultMessage.content === "string" && resultMessage.content.includes('all_exec_results')) {
       const parsedExecResults = JSON.parse(resultMessage.content) as AzureSqlServerExecResults
       setExecResults(parsedExecResults.all_exec_results)
       assistantMessage.context = JSON.stringify({
@@ -179,24 +179,27 @@ const Chat = () => {
     }
   }
 
-  const makeApiRequestWithoutCosmosDB = async (question: string, conversationId?: string) => {
+  const makeApiRequestWithoutCosmosDB = async (question: ChatMessage["content"], conversationId?: string) => {
     setIsLoading(true)
     setShowLoadingMessage(true)
     const abortController = new AbortController()
     abortFuncs.current.unshift(abortController)
 
+    const questionContent = typeof question === 'string' ? question : [{ type: "text", text: question[0].text }, { type: "image_url", image_url: { url: question[1].image_url.url } }]
+    question = typeof question !== 'string' && question[0]?.text?.length > 0 ? question[0].text : question
+
     const userMessage: ChatMessage = {
       id: uuid(),
       role: 'user',
-      content: question,
+      content: questionContent as string,
       date: new Date().toISOString()
     }
 
     let conversation: Conversation | null | undefined
     if (!conversationId) {
       conversation = {
         id: conversationId ?? uuid(),
-        title: question,
+        title: question as string,
         messages: [userMessage],
         date: new Date().toISOString()
       }
@@ -303,20 +306,21 @@ const Chat = () => {
     return abortController.abort()
   }
 
-  const makeApiRequestWithCosmosDB = async (question: string, conversationId?: string) => {
+  const makeApiRequestWithCosmosDB = async (question: ChatMessage["content"], conversationId?: string) => {
     setIsLoading(true)
     setShowLoadingMessage(true)
     const abortController = new AbortController()
     abortFuncs.current.unshift(abortController)
+    const questionContent = typeof question === 'string' ? question : [{ type: "text", text: question[0].text }, { type: "image_url", image_url: { url: question[1].image_url.url } }]
+    question = typeof question !== 'string' && question[0]?.text?.length > 0 ? question[0].text : question
 
     const userMessage: ChatMessage = {
       id: uuid(),
       role: 'user',
-      content: question,
+      content: questionContent as string,
       date: new Date().toISOString()
     }
 
-    //api call params set here (generate)
     let request: ConversationRequest
     let conversation
     if (conversationId) {
@@ -648,7 +652,7 @@ const Chat = () => {
         }
         const noContentError = appStateContext.state.currentChat.messages.find(m => m.role === ERROR)
 
-        if (!noContentError?.content.includes(NO_CONTENT_ERROR)) {
+        if (typeof noContentError?.content === "string" && !noContentError?.content.includes(NO_CONTENT_ERROR)) {
           saveToDB(appStateContext.state.currentChat.messages, appStateContext.state.currentChat.id)
             .then(res => {
               if (!res.ok) {
@@ -713,7 +717,7 @@ const Chat = () => {
   }
 
   const parseCitationFromMessage = (message: ChatMessage) => {
-    if (message?.role && message?.role === 'tool') {
+    if (message?.role && message?.role === 'tool' && typeof message?.content === "string") {
       try {
         const toolMessage = JSON.parse(message.content) as ToolMessageContent
         return toolMessage.citations
@@ -725,7 +729,7 @@ const Chat = () => {
   }
 
   const parsePlotFromMessage = (message: ChatMessage) => {
-    if (message?.role && message?.role === "tool") {
+    if (message?.role && message?.role === "tool" && typeof message?.content === "string") {
       try {
         const execResults = JSON.parse(message.content) as AzureSqlServerExecResults;
         const codeExecResult = execResults.all_exec_results.at(-1)?.code_exec_result;
@@ -797,11 +801,13 @@ const Chat = () => {
                   <>
                     {answer.role === 'user' ? (
                       <div className={styles.chatMessageUser} tabIndex={0}>
-                        <div className={styles.chatMessageUserMessage}>{answer.content}</div>
+                        <div className={styles.chatMessageUserMessage}>
+                          {typeof answer.content === "string" && answer.content ? answer.content : Array.isArray(answer.content) ? <>{answer.content[0].text} <img className={styles.uploadedImageChat} src={answer.content[1].image_url.url} alt="Uploaded Preview" /></> : null}
+                        </div>
                       </div>
                     ) : answer.role === 'assistant' ? (
                       <div className={styles.chatMessageGpt}>
-                        <Answer
+                        {typeof answer.content === "string" && <Answer
                           answer={{
                             answer: answer.content,
                             citations: parseCitationFromMessage(messages[index - 1]),
@@ -812,15 +818,15 @@ const Chat = () => {
                           }}
                           onCitationClicked={c => onShowCitation(c)}
                           onExectResultClicked={() => onShowExecResult(answerId)}
-                        />
+                        />}
                       </div>
                     ) : answer.role === ERROR ? (
                       <div className={styles.chatMessageError}>
                         <Stack horizontal className={styles.chatMessageErrorContent}>
                           <ErrorCircleRegular className={styles.errorIcon} style={{ color: 'rgba(182, 52, 67, 1)' }} />
                           <span>Error</span>
                         </Stack>
-                        <span className={styles.chatMessageErrorContent}>{answer.content}</span>
+                        <span className={styles.chatMessageErrorContent}>{typeof answer.content === "string" && answer.content}</span>
                       </div>
                     ) : null}
                   </>