From 397b75a453fef24e62356642fe78a35098f536c7 Mon Sep 17 00:00:00 2001
From: Morgan Wowk <morgan.wowk@shopify.com>
Date: Thu, 18 Jun 2026 19:28:27 -0700
Subject: [PATCH] Add analytics tracking and thumbs feedback to AI sidekick (V2
 editor)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Tracks three events on the editor AI assistant:
- ai_assistant.message.submitted — fires on send with prompt + thread_message_count
- ai_assistant.response.received — fires after response completes with prompt, response, and thread_message_count
- ai_assistant.response.feedback — fires when user rates a response via thumbs up/down with rating, prompt, and response

Adds a ResponseFeedback component below each assistant message: outline thumbs icons that highlight to `text-info` on hover, then on click record the feedback event and fade (200ms out → 300ms in) to "Thank you for sharing your feedback".
---
 .../components/AiChat/AiChatContent.tsx       | 19 ++++-
 .../shared/components/AiChat/agentThread.ts   |  1 +
 .../AiChat/components/ChatMessage.tsx         | 10 ++-
 .../AiChat/components/ResponseFeedback.tsx    | 74 +++++++++++++++++++
 .../v2/shared/components/AiChat/types.ts      | 14 +++-
 5 files changed, 111 insertions(+), 7 deletions(-)
 create mode 100644 src/routes/v2/shared/components/AiChat/components/ResponseFeedback.tsx

diff --git a/src/routes/v2/shared/components/AiChat/AiChatContent.tsx b/src/routes/v2/shared/components/AiChat/AiChatContent.tsx
index 34223658d..5362b4842 100644
--- a/src/routes/v2/shared/components/AiChat/AiChatContent.tsx
+++ b/src/routes/v2/shared/components/AiChat/AiChatContent.tsx
@@ -10,6 +10,7 @@ import { Icon } from "@/components/ui/icon";
 import { BlockStack, InlineStack } from "@/components/ui/layout";
 import { useAiProviderSettings } from "@/hooks/useAiProviderSettings";
 import useToastNotification from "@/hooks/useToastNotification";
+import { useAnalytics } from "@/providers/AnalyticsProvider";
 import { useBackend } from "@/providers/BackendProvider";
 import { useSharedStores } from "@/routes/v2/shared/store/SharedStoreContext";
 import { fetchPipelineRuns } from "@/services/pipelineRunService";
@@ -50,6 +51,7 @@ export const AiChatContent = observer(function AiChatContent({
 }: AiChatContentProps) {
   const aiChat = useAiChatStore();
   const notify = useToastNotification();
+  const { track } = useAnalytics();
   const { navigation } = useSharedStores();
   const { backendUrl } = useBackend();
   const authStorage = useAuthLocalStorage();
@@ -103,17 +105,30 @@ export const AiChatContent = observer(function AiChatContent({
 
   const thread = aiChat.activeThread;
 
-  function handleSend(prompt: string) {
+  async function handleSend(prompt: string) {
     if (!thread) return;
     const recentRuns = recentRunsData
       ? projectRecentRuns(recentRunsData)
       : undefined;
-    thread.sendMessage(prompt, {
+
+    track("ai_assistant.message.submitted", {
+      prompt,
+      thread_message_count: thread.messages.length,
+    });
+
+    await thread.sendMessage(prompt, {
       onError: (msg) => notify(msg, "error"),
       bridge,
       aiConfig,
       ...(recentRuns && { recentRuns }),
     });
+
+    if (thread.messages[thread.messages.length - 1]?.role === "assistant") {
+      track("ai_assistant.response.received", {
+        prompt,
+        thread_message_count: thread.messages.length,
+      });
+    }
   }
 
   if (!isAiConfigured) {
diff --git a/src/routes/v2/shared/components/AiChat/agentThread.ts b/src/routes/v2/shared/components/AiChat/agentThread.ts
index 8b18419e6..da1eaeb57 100644
--- a/src/routes/v2/shared/components/AiChat/agentThread.ts
+++ b/src/routes/v2/shared/components/AiChat/agentThread.ts
@@ -104,6 +104,7 @@ export class AgentThread {
             id: generateMessageId(),
             role: "assistant",
             content: response.answer,
+            prompt,
             ...(Object.keys(componentReferences).length > 0
               ? { componentReferences }
               : {}),
diff --git a/src/routes/v2/shared/components/AiChat/components/ChatMessage.tsx b/src/routes/v2/shared/components/AiChat/components/ChatMessage.tsx
index b32383cec..9376107c6 100644
--- a/src/routes/v2/shared/components/AiChat/components/ChatMessage.tsx
+++ b/src/routes/v2/shared/components/AiChat/components/ChatMessage.tsx
@@ -3,6 +3,7 @@ import type { ChatMessage as ChatMessageType } from "@/routes/v2/shared/componen
 
 import { MessageBubble } from "./MessageBubble";
 import { renderMarkdown } from "./renderMarkdown";
+import { ResponseFeedback } from "./ResponseFeedback";
 
 interface ChatMessageProps {
   message: ChatMessageType;
@@ -18,9 +19,12 @@ export function ChatMessage({ message }: ChatMessageProps) {
           {message.content}
         </Text>
       ) : (
-        <div className="text-sm break-words min-w-0 overflow-x-auto">
-          {renderMarkdown(message.content, message.componentReferences)}
-        </div>
+        <>
+          <div className="text-sm break-words min-w-0 overflow-x-auto">
+            {renderMarkdown(message.content, message.componentReferences)}
+          </div>
+          <ResponseFeedback prompt={message.prompt} response={message.content} />
+        </>
       )}
     </MessageBubble>
   );
diff --git a/src/routes/v2/shared/components/AiChat/components/ResponseFeedback.tsx b/src/routes/v2/shared/components/AiChat/components/ResponseFeedback.tsx
new file mode 100644
index 000000000..8071044d4
--- /dev/null
+++ b/src/routes/v2/shared/components/AiChat/components/ResponseFeedback.tsx
@@ -0,0 +1,74 @@
+import { useState } from "react";
+
+import { Button } from "@/components/ui/button";
+import { Icon } from "@/components/ui/icon";
+import { InlineStack } from "@/components/ui/layout";
+import { Text } from "@/components/ui/typography";
+import { cn } from "@/lib/utils";
+import { useAnalytics } from "@/providers/AnalyticsProvider";
+
+interface ResponseFeedbackProps {
+  prompt: string;
+  response: string;
+}
+
+type FeedbackState = "idle" | "submitted";
+
+export function ResponseFeedback({ prompt, response }: ResponseFeedbackProps) {
+  const { track } = useAnalytics();
+  const [state, setState] = useState<FeedbackState>("idle");
+  const [showThanks, setShowThanks] = useState(false);
+
+  function handleFeedback(rating: "positive" | "negative") {
+    if (state !== "idle") return;
+    setState("submitted");
+    track("ai_assistant.response.feedback", { rating, prompt, response });
+    setTimeout(() => setShowThanks(true), 200);
+  }
+
+  return (
+    <div className="relative h-5 mt-1.5">
+      <InlineStack
+        gap="0"
+        className={cn(
+          "transition-opacity duration-200",
+          state === "submitted" ? "opacity-0" : "opacity-100",
+        )}
+        aria-hidden={state === "submitted"}
+      >
+        <Button
+          type="button"
+          variant="ghost"
+          size="min"
+          onClick={() => handleFeedback("positive")}
+          disabled={state !== "idle"}
+          aria-label="Helpful response"
+          className="text-muted-foreground/30 hover:text-info transition-colors duration-150"
+        >
+          <Icon name="ThumbsUp" size="xs" />
+        </Button>
+        <Button
+          type="button"
+          variant="ghost"
+          size="min"
+          onClick={() => handleFeedback("negative")}
+          disabled={state !== "idle"}
+          aria-label="Not helpful response"
+          className="text-muted-foreground/30 hover:text-info transition-colors duration-150"
+        >
+          <Icon name="ThumbsDown" size="xs" />
+        </Button>
+      </InlineStack>
+      <Text
+        size="xs"
+        tone="subdued"
+        className={cn(
+          "absolute left-0 top-0 transition-opacity duration-300",
+          showThanks ? "opacity-100" : "opacity-0 pointer-events-none",
+        )}
+      >
+        Thank you for sharing your feedback
+      </Text>
+    </div>
+  );
+}
diff --git a/src/routes/v2/shared/components/AiChat/types.ts b/src/routes/v2/shared/components/AiChat/types.ts
index 6d5ac751e..b0ae86989 100644
--- a/src/routes/v2/shared/components/AiChat/types.ts
+++ b/src/routes/v2/shared/components/AiChat/types.ts
@@ -3,9 +3,19 @@ export interface ComponentRefData {
   yamlText: string;
 }
 
-export interface ChatMessage {
+interface BaseChatMessage {
   id: string;
-  role: "user" | "assistant";
   content: string;
   componentReferences?: Record<string, ComponentRefData>;
 }
+
+export interface UserChatMessage extends BaseChatMessage {
+  role: "user";
+}
+
+export interface AssistantChatMessage extends BaseChatMessage {
+  role: "assistant";
+  prompt: string;
+}
+
+export type ChatMessage = UserChatMessage | AssistantChatMessage;