Skip to content

Commit 2c4ecd0

Browse files
xingyaowwopenhands-agentamanape
authored
feat(frontend): add user feedback Likert scale for agent performance rating (only on OH Cloud) (OpenHands#8992)
Co-authored-by: openhands <[email protected]> Co-authored-by: sp.wack <[email protected]>
1 parent 2fd1fdc commit 2c4ecd0

File tree

14 files changed

+708
-98
lines changed

14 files changed

+708
-98
lines changed

‎frontend/src/api/open-hands.ts‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -111,6 +111,59 @@ class OpenHands {
111111
return data;
112112
}
113113

114+
/**
115+
* Submit conversation feedback with rating
116+
* @param conversationId The conversation ID
117+
* @param rating The rating (1-5)
118+
* @param eventId Optional event ID this feedback corresponds to
119+
* @param reason Optional reason for the rating
120+
* @returns Response from the feedback endpoint
121+
*/
122+
static async submitConversationFeedback(
123+
conversationId: string,
124+
rating: number,
125+
eventId?: number,
126+
reason?: string,
127+
): Promise<{ status: string; message: string }> {
128+
const url = `/feedback/conversation`;
129+
const payload = {
130+
conversation_id: conversationId,
131+
event_id: eventId,
132+
rating,
133+
reason,
134+
metadata: { source: "likert-scale" },
135+
};
136+
const { data } = await openHands.post<{ status: string; message: string }>(
137+
url,
138+
payload,
139+
);
140+
return data;
141+
}
142+
143+
/**
144+
* Check if feedback exists for a specific conversation and event
145+
* @param conversationId The conversation ID
146+
* @param eventId The event ID to check
147+
* @returns Feedback data including existence, rating, and reason
148+
*/
149+
static async checkFeedbackExists(
150+
conversationId: string,
151+
eventId: number,
152+
): Promise<{ exists: boolean; rating?: number; reason?: string }> {
153+
try {
154+
const url = `/feedback/conversation/${conversationId}/${eventId}`;
155+
const { data } = await openHands.get<{
156+
exists: boolean;
157+
rating?: number;
158+
reason?: string;
159+
}>(url);
160+
return data;
161+
} catch (error) {
162+
// Error checking if feedback exists
163+
return { exists: false };
164+
}
165+
}
166+
114167
/**
115168
* Authenticate with GitHub token
116169
* @returns Response with authentication status and user info if successful

‎frontend/src/components/features/chat/chat-interface.tsx‎

Lines changed: 90 additions & 64 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ import { useWsClient } from "#/context/ws-client-provider";
1818
import { Messages } from "./messages";
1919
import { ChatSuggestions } from "./chat-suggestions";
2020
import { ActionSuggestions } from "./action-suggestions";
21+
import { ScrollProvider } from "#/context/scroll-context";
2122

2223
import { ScrollToBottomButton } from "#/components/shared/buttons/scroll-to-bottom-button";
2324
import { LoadingSpinner } from "#/components/shared/loading-spinner";
@@ -28,6 +29,7 @@ import { useOptimisticUserMessage } from "#/hooks/use-optimistic-user-message";
2829
import { useWSErrorMessage } from "#/hooks/use-ws-error-message";
2930
import { ErrorMessageBanner } from "./error-message-banner";
3031
import { shouldRenderEvent } from "./event-content-helpers/should-render-event";
32+
import { useConfig } from "#/hooks/query/use-config";
3133

3234
function getEntryPoint(
3335
hasRepository: boolean | null,
@@ -45,8 +47,15 @@ export function ChatInterface() {
4547
useOptimisticUserMessage();
4648
const { t } = useTranslation();
4749
const scrollRef = React.useRef<HTMLDivElement>(null);
48-
const { scrollDomToBottom, onChatBodyScroll, hitBottom } =
49-
useScrollToBottom(scrollRef);
50+
const {
51+
scrollDomToBottom,
52+
onChatBodyScroll,
53+
hitBottom,
54+
autoScroll,
55+
setAutoScroll,
56+
setHitBottom,
57+
} = useScrollToBottom(scrollRef);
58+
const { data: config } = useConfig();
5059

5160
const { curAgentState } = useSelector((state: RootState) => state.agent);
5261

@@ -126,80 +135,97 @@ export function ChatInterface() {
126135
curAgentState === AgentState.AWAITING_USER_INPUT ||
127136
curAgentState === AgentState.FINISHED;
128137

138+
// Create a ScrollProvider with the scroll hook values
139+
const scrollProviderValue = {
140+
scrollRef,
141+
autoScroll,
142+
setAutoScroll,
143+
scrollDomToBottom,
144+
hitBottom,
145+
setHitBottom,
146+
onChatBodyScroll,
147+
};
148+
129149
return (
130-
<div className="h-full flex flex-col justify-between">
131-
{events.length === 0 && !optimisticUserMessage && (
132-
<ChatSuggestions onSuggestionsClick={setMessageToSend} />
133-
)}
134-
135-
<div
136-
ref={scrollRef}
137-
onScroll={(e) => onChatBodyScroll(e.currentTarget)}
138-
className="scrollbar scrollbar-thin scrollbar-thumb-gray-400 scrollbar-thumb-rounded-full scrollbar-track-gray-800 hover:scrollbar-thumb-gray-300 flex flex-col grow overflow-y-auto overflow-x-hidden px-4 pt-4 gap-2 fast-smooth-scroll"
139-
>
140-
{isLoadingMessages && (
141-
<div className="flex justify-center">
142-
<LoadingSpinner size="small" />
143-
</div>
150+
<ScrollProvider value={scrollProviderValue}>
151+
<div className="h-full flex flex-col justify-between">
152+
{events.length === 0 && !optimisticUserMessage && (
153+
<ChatSuggestions onSuggestionsClick={setMessageToSend} />
144154
)}
145155

146-
{!isLoadingMessages && (
147-
<Messages
148-
messages={events}
149-
isAwaitingUserConfirmation={
150-
curAgentState === AgentState.AWAITING_USER_CONFIRMATION
151-
}
152-
/>
153-
)}
156+
<div
157+
ref={scrollRef}
158+
onScroll={(e) => onChatBodyScroll(e.currentTarget)}
159+
className="scrollbar scrollbar-thin scrollbar-thumb-gray-400 scrollbar-thumb-rounded-full scrollbar-track-gray-800 hover:scrollbar-thumb-gray-300 flex flex-col grow overflow-y-auto overflow-x-hidden px-4 pt-4 gap-2 fast-smooth-scroll"
160+
>
161+
{isLoadingMessages && (
162+
<div className="flex justify-center">
163+
<LoadingSpinner size="small" />
164+
</div>
165+
)}
154166

155-
{isWaitingForUserInput &&
156-
events.length > 0 &&
157-
!optimisticUserMessage && (
158-
<ActionSuggestions
159-
onSuggestionsClick={(value) => handleSendMessage(value, [])}
167+
{!isLoadingMessages && (
168+
<Messages
169+
messages={events}
170+
isAwaitingUserConfirmation={
171+
curAgentState === AgentState.AWAITING_USER_CONFIRMATION
172+
}
160173
/>
161174
)}
162-
</div>
163175

164-
<div className="flex flex-col gap-[6px] px-4 pb-4">
165-
<div className="flex justify-between relative">
166-
<TrajectoryActions
167-
onPositiveFeedback={() =>
168-
onClickShareFeedbackActionButton("positive")
169-
}
170-
onNegativeFeedback={() =>
171-
onClickShareFeedbackActionButton("negative")
172-
}
173-
onExportTrajectory={() => onClickExportTrajectoryButton()}
174-
/>
176+
{isWaitingForUserInput &&
177+
events.length > 0 &&
178+
!optimisticUserMessage && (
179+
<ActionSuggestions
180+
onSuggestionsClick={(value) => handleSendMessage(value, [])}
181+
/>
182+
)}
183+
</div>
175184

176-
<div className="absolute left-1/2 transform -translate-x-1/2 bottom-0">
177-
{curAgentState === AgentState.RUNNING && <TypingIndicator />}
185+
<div className="flex flex-col gap-[6px] px-4 pb-4">
186+
<div className="flex justify-between relative">
187+
{config?.APP_MODE !== "saas" && (
188+
<TrajectoryActions
189+
onPositiveFeedback={() =>
190+
onClickShareFeedbackActionButton("positive")
191+
}
192+
onNegativeFeedback={() =>
193+
onClickShareFeedbackActionButton("negative")
194+
}
195+
onExportTrajectory={() => onClickExportTrajectoryButton()}
196+
/>
197+
)}
198+
199+
<div className="absolute left-1/2 transform -translate-x-1/2 bottom-0">
200+
{curAgentState === AgentState.RUNNING && <TypingIndicator />}
201+
</div>
202+
203+
{!hitBottom && <ScrollToBottomButton onClick={scrollDomToBottom} />}
178204
</div>
179205

180-
{!hitBottom && <ScrollToBottomButton onClick={scrollDomToBottom} />}
206+
{errorMessage && <ErrorMessageBanner message={errorMessage} />}
207+
208+
<InteractiveChatBox
209+
onSubmit={handleSendMessage}
210+
onStop={handleStop}
211+
isDisabled={
212+
curAgentState === AgentState.LOADING ||
213+
curAgentState === AgentState.AWAITING_USER_CONFIRMATION
214+
}
215+
mode={curAgentState === AgentState.RUNNING ? "stop" : "submit"}
216+
value={messageToSend ?? undefined}
217+
onChange={setMessageToSend}
218+
/>
181219
</div>
182220

183-
{errorMessage && <ErrorMessageBanner message={errorMessage} />}
184-
185-
<InteractiveChatBox
186-
onSubmit={handleSendMessage}
187-
onStop={handleStop}
188-
isDisabled={
189-
curAgentState === AgentState.LOADING ||
190-
curAgentState === AgentState.AWAITING_USER_CONFIRMATION
191-
}
192-
mode={curAgentState === AgentState.RUNNING ? "stop" : "submit"}
193-
value={messageToSend ?? undefined}
194-
onChange={setMessageToSend}
195-
/>
221+
{config?.APP_MODE !== "saas" && (
222+
<FeedbackModal
223+
isOpen={feedbackModalIsOpen}
224+
onClose={() => setFeedbackModalIsOpen(false)}
225+
polarity={feedbackPolarity}
226+
/>
227+
)}
196228
</div>
197-
198-
<FeedbackModal
199-
isOpen={feedbackModalIsOpen}
200-
onClose={() => setFeedbackModalIsOpen(false)}
201-
polarity={feedbackPolarity}
202-
/>
203-
</div>
229+
</ScrollProvider>
204230
);
205231
}

‎frontend/src/components/features/chat/event-message.tsx‎

Lines changed: 30 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
import React from "react";
12
import { ConfirmationButtons } from "#/components/shared/buttons/confirmation-buttons";
23
import { OpenHandsAction } from "#/types/core/actions";
34
import {
@@ -18,6 +19,10 @@ import { MCPObservationContent } from "./mcp-observation-content";
1819
import { getObservationResult } from "./event-content-helpers/get-observation-result";
1920
import { getEventContent } from "./event-content-helpers/get-event-content";
2021
import { GenericEventMessage } from "./generic-event-message";
22+
import { LikertScale } from "../feedback/likert-scale";
23+
24+
import { useConfig } from "#/hooks/query/use-config";
25+
import { useFeedbackExists } from "#/hooks/query/use-feedback-exists";
2126

2227
const hasThoughtProperty = (
2328
obj: Record<string, unknown>,
@@ -39,6 +44,14 @@ export function EventMessage({
3944
const shouldShowConfirmationButtons =
4045
isLastMessage && event.source === "agent" && isAwaitingUserConfirmation;
4146

47+
const { data: config } = useConfig();
48+
49+
// Use our query hook to check if feedback exists and get rating/reason
50+
const {
51+
data: feedbackData = { exists: false },
52+
isLoading: isCheckingFeedback,
53+
} = useFeedbackExists(isFinishAction(event) ? event.id : undefined);
54+
4255
if (isErrorObservation(event)) {
4356
return (
4457
<ErrorMessage
@@ -55,9 +68,25 @@ export function EventMessage({
5568
return null;
5669
}
5770

71+
const showLikertScale =
72+
config?.APP_MODE === "saas" &&
73+
isFinishAction(event) &&
74+
isLastMessage &&
75+
!isCheckingFeedback;
76+
5877
if (isFinishAction(event)) {
5978
return (
60-
<ChatMessage type="agent" message={getEventContent(event).details} />
79+
<>
80+
<ChatMessage type="agent" message={getEventContent(event).details} />
81+
{showLikertScale && (
82+
<LikertScale
83+
eventId={event.id}
84+
initiallySubmitted={feedbackData.exists}
85+
initialRating={feedbackData.rating}
86+
initialReason={feedbackData.reason}
87+
/>
88+
)}
89+
</>
6190
);
6291
}
6392

0 commit comments

Comments
 (0)