From a3b5f47acd583f9203982caee370041281b66308 Mon Sep 17 00:00:00 2001 From: frankkluijtmans Date: Thu, 23 Jul 2026 17:23:25 +0200 Subject: [PATCH] feat(apollo-vertex): add typed telemetry seam and action events to solution-tests --- .../app/templates/solution-tests/page.mdx | 41 +++++++++++++ .../registry/solution-tests/config.ts | 58 +++++++++++++++++++ .../solution-tests/expanded-agents.tsx | 9 ++- .../solution-tests/expanded-run-tests.tsx | 9 ++- .../registry/solution-tests/index.ts | 3 + .../registry/solution-tests/run-details.tsx | 7 ++- .../solution-tests/solution-tests-view.tsx | 9 +++ .../solution-tests/use-baseline-jobs.ts | 9 ++- .../registry/solution-tests/use-force-stop.ts | 7 ++- .../solution-tests/use-run-results.ts | 11 +++- .../solution-tests/use-solution-tests.ts | 13 ++++- 11 files changed, 166 insertions(+), 10 deletions(-) diff --git a/apps/apollo-vertex/app/templates/solution-tests/page.mdx b/apps/apollo-vertex/app/templates/solution-tests/page.mdx index ae3f1266d..f88511d82 100644 --- a/apps/apollo-vertex/app/templates/solution-tests/page.mdx +++ b/apps/apollo-vertex/app/templates/solution-tests/page.mdx @@ -85,6 +85,8 @@ interface SolutionTestsConfig { passThreshold?: number; /** Show the Expected/Actual input panels in run-result details. Defaults to false (outputs only). */ showInputs?: boolean; + /** Typed telemetry callback. Fires a named event when each action is triggered; a no-op unless supplied. */ + track?: TrackSolutionTestEvent; } ``` @@ -110,6 +112,42 @@ render the dumb `SolutionTestsView`, which takes all data and callbacks via props. It must still be wrapped in `SolutionTestsProvider` (the view reads `config` from context via `useSolutionTestsConfig`), as the preview above does. +## Telemetry + +The action hooks emit a typed telemetry event when the user triggers each action +(on intent, not on success). The template only *names* the events. It never ships an analytics backend. Pass a +`track` callback through `config` to route them wherever you collect analytics. +Omit it and tracking is a no-op. Because the events fire from the mutation hooks +(`useRunTests`, `useCreateTest`, and friends), only the smart container reports +them. A bare `SolutionTestsView` wired to your own handlers does not. + +```tsx +const config: SolutionTestsConfig = { + subjectColumns, + track: (event, properties) => analytics.track(event, properties), +}; +``` + +`SolutionTestEventMap` is the single source of truth for the event contract, so +the callback is fully typed: `properties` is narrowed to the payload of the +`event` you pass. Compose the map into your own app's event map to keep the +wiring type-safe end to end. + +| Event | Payload | Fired when | +| ------------------------------------ | -------------------------------------------------- | --------------------------------- | +| `VS.SolutionTest.Run` | `{ mode: "all" \| "test" \| "selected", testCount }` | A test run is started | +| `VS.SolutionTest.Created` | `{ subjectId }` | A test is created from a subject | +| `VS.SolutionTest.ActiveToggled` | `{ testId, isActive }` | A test is enabled or disabled | +| `VS.SolutionTest.Deleted` | `{ testId }` | A test is deleted | +| `VS.SolutionTest.BatchForceStopped` | `{ batchId }` | A batch run is force-stopped | +| `VS.SolutionTest.RunForceStopped` | `{ runId }` | A single run is force-stopped | +| `VS.SolutionTest.JobAdopted` | `{ resultId }` | A run result is adopted as baseline | +| `VS.SolutionTest.BaselineUpdated` | `{ resultId }` | A baseline is updated from a result | +| `VS.SolutionTest.BaselineRemoved` | `{ baselineId }` | A baseline is removed | + +`SolutionTestEventMap`, `SolutionTestEventName`, and `TrackSolutionTestEvent` +are all exported from the package barrel. + ## Saving a subject as a test Alongside the full-page view, the package exports a standalone `SaveAsTestButton` @@ -227,6 +265,9 @@ you prefer to supply your own data plumbing or render the view against a mock - **Fixed setup** — evaluator labels, status labels, and the poll interval are hard-coded in `constants.ts`; edit there to retarget a deployment. The pass threshold defaults to `0.9` but is overridable via `config.passThreshold`. +- **Telemetry** — `config.track` receives a typed event when each action is + triggered; wire it to your analytics backend (see [Telemetry](#telemetry)). + Omit it for no tracking. - **i18n** — framework strings use `react-i18next`; wrap your app in `ApolloShell` (which initializes i18n via `LocaleProvider`) or provide your own `I18nextProvider`. diff --git a/apps/apollo-vertex/registry/solution-tests/config.ts b/apps/apollo-vertex/registry/solution-tests/config.ts index ca9c27238..e436a253d 100644 --- a/apps/apollo-vertex/registry/solution-tests/config.ts +++ b/apps/apollo-vertex/registry/solution-tests/config.ts @@ -4,6 +4,59 @@ import type { EvaluatorRenderers } from "./evaluators/registry"; import type { ProcessOutputRenderers } from "./outputs/registry"; import type { SolutionTest } from "./types"; +/** + * Telemetry events the Solution Tests actions emit, with payloads. The template + * names the events; the host supplies `track` (below) to route them. Compose + * into a host event map with `interface … extends SolutionTestEventMap {}` (not + * an intersection) so a generic tracker's `Map[K]` indexing keeps working. + */ +export interface SolutionTestEventMap { + "VS.SolutionTest.Run": { + mode: "all" | "test" | "selected"; + testCount: number; + }; + "VS.SolutionTest.Created": { subjectId: string }; + "VS.SolutionTest.ActiveToggled": { testId: string; isActive: boolean }; + "VS.SolutionTest.Deleted": { testId: string }; + "VS.SolutionTest.BatchForceStopped": { batchId: string }; + "VS.SolutionTest.RunForceStopped": { runId: string }; + "VS.SolutionTest.JobAdopted": { resultId: string }; + "VS.SolutionTest.BaselineUpdated": { resultId: string }; + "VS.SolutionTest.BaselineRemoved": { baselineId: string }; + + // UI interactions (no server request) — user navigation and output inspection. + "VS.SolutionTest.TabViewed": { tab: "cases" | "runs" }; + "VS.SolutionTest.TestExpanded": { testId: string }; + "VS.SolutionTest.BatchExpanded": { batchId: string }; + "VS.SolutionTest.RunDetailsOpened": { runId: string }; + "VS.SolutionTest.ResultInspected": { resultId: string }; + "VS.SolutionTest.RawOutputViewed": { processName: string }; +} + +export type SolutionTestEventName = keyof SolutionTestEventMap; + +type UnionToIntersection = ( + U extends unknown + ? (arg: U) => void + : never +) extends (arg: infer I) => void + ? I + : never; + +/** + * Typed telemetry callback the host injects via config. An intersection of + * per-event signatures (not one generic signature) so a host tracker generic + * over a wider event map assigns to it cast-free. + */ +export type TrackSolutionTestEvent = UnionToIntersection< + { + [K in SolutionTestEventName]: ( + event: K, + properties: SolutionTestEventMap[K], + ) => void; + }[SolutionTestEventName] +>; + /** Per-vertical presentation config; everything else is hard-coded in `constants`. */ export interface SolutionTestsConfig { /** Columns inserted between the Test Name and Version columns. */ @@ -19,6 +72,8 @@ export interface SolutionTestsConfig { evaluatorRenderers?: EvaluatorRenderers; /** Keyed by stable agent id and/or process name; unmatched outputs render as raw JSON. */ outputRenderers?: ProcessOutputRenderers; + /** Emit a telemetry event for each action. No-op unless the host supplies it. */ + track?: TrackSolutionTestEvent; } /** Config with defaults applied — what components read from context. */ @@ -30,6 +85,8 @@ export interface ResolvedSolutionTestsConfig { showDebug: boolean; evaluatorRenderers: EvaluatorRenderers; outputRenderers: ProcessOutputRenderers; + /** Optional — call sites use `track?.(…)`, so no tracking happens if absent. */ + track?: TrackSolutionTestEvent; } export function resolveConfig( @@ -43,5 +100,6 @@ export function resolveConfig( showDebug: config.showDebug ?? false, evaluatorRenderers: config.evaluatorRenderers ?? {}, outputRenderers: config.outputRenderers ?? {}, + track: config.track, }; } diff --git a/apps/apollo-vertex/registry/solution-tests/expanded-agents.tsx b/apps/apollo-vertex/registry/solution-tests/expanded-agents.tsx index db900c4ab..655adf816 100644 --- a/apps/apollo-vertex/registry/solution-tests/expanded-agents.tsx +++ b/apps/apollo-vertex/registry/solution-tests/expanded-agents.tsx @@ -3,6 +3,7 @@ import { useState } from "react"; import { useQuery } from "@tanstack/react-query"; import type { SolutionTest, SolutionTestJob } from "./types"; +import { useSolutionTestsConfig } from "./context"; import { useBaselineJobs, useJobExpectedOutput } from "./hooks"; import { ExpandedAgentsView } from "./expanded-agents-view"; @@ -17,6 +18,7 @@ interface ExpandedAgentsProps { export const ExpandedAgents = ({ test }: ExpandedAgentsProps) => { const { jobs: baselines, isLoading } = useBaselineJobs(test.Id); const expectedOutput = useJobExpectedOutput(); + const { track } = useSolutionTestsConfig(); const [openJob, setOpenJob] = useState(null); @@ -42,7 +44,12 @@ export const ExpandedAgents = ({ test }: ExpandedAgentsProps) => { isLoading={isLoading} noOutputJobIds={noOutputJobIds} viewing={viewing} - onViewExpected={(job) => setOpenJob(job)} + onViewExpected={(job) => { + track?.("VS.SolutionTest.RawOutputViewed", { + processName: job.ProcessName, + }); + setOpenJob(job); + }} onCloseViewer={() => setOpenJob(null)} /> ); diff --git a/apps/apollo-vertex/registry/solution-tests/expanded-run-tests.tsx b/apps/apollo-vertex/registry/solution-tests/expanded-run-tests.tsx index 8e8bc3590..be7dfbd76 100644 --- a/apps/apollo-vertex/registry/solution-tests/expanded-run-tests.tsx +++ b/apps/apollo-vertex/registry/solution-tests/expanded-run-tests.tsx @@ -4,6 +4,7 @@ import { useNavigate } from "@tanstack/react-router"; import { useTranslation } from "react-i18next"; import { toast } from "sonner"; import type { SolutionTest, SolutionTestRun } from "./types"; +import { useSolutionTestsConfig } from "./context"; import { useForceStopRun } from "./hooks"; import { ExpandedRunTestsView } from "./expanded-run-tests-view"; @@ -16,6 +17,7 @@ interface ExpandedRunTestsProps { export const ExpandedRunTests = ({ runs, tests }: ExpandedRunTestsProps) => { const { t } = useTranslation(); const navigate = useNavigate(); + const { track } = useSolutionTestsConfig(); const forceStopRun = useForceStopRun(); const stoppingRunId = forceStopRun.isPending @@ -27,15 +29,16 @@ export const ExpandedRunTests = ({ runs, tests }: ExpandedRunTestsProps) => { runs={runs} tests={tests} stoppingRunId={stoppingRunId} - onOpenDetails={(run) => + onOpenDetails={(run) => { + track?.("VS.SolutionTest.RunDetailsOpened", { runId: run.Id }); void navigate({ to: ".", search: (prev: Record) => ({ ...prev, run: run.Id, }), - }) - } + }); + }} onForceStop={(runId) => forceStopRun.mutate(runId, { onSuccess: () => toast.success(t("force_stop_initiated")), diff --git a/apps/apollo-vertex/registry/solution-tests/index.ts b/apps/apollo-vertex/registry/solution-tests/index.ts index 4763593fb..28a3e5871 100644 --- a/apps/apollo-vertex/registry/solution-tests/index.ts +++ b/apps/apollo-vertex/registry/solution-tests/index.ts @@ -30,6 +30,9 @@ export { export type { SolutionTestsConfig, ResolvedSolutionTestsConfig, + SolutionTestEventMap, + SolutionTestEventName, + TrackSolutionTestEvent, } from "./config"; export { makeRenderer, diff --git a/apps/apollo-vertex/registry/solution-tests/run-details.tsx b/apps/apollo-vertex/registry/solution-tests/run-details.tsx index 4c8250351..d76423f8c 100644 --- a/apps/apollo-vertex/registry/solution-tests/run-details.tsx +++ b/apps/apollo-vertex/registry/solution-tests/run-details.tsx @@ -90,7 +90,7 @@ interface RunDetailsProps { * fetch + baseline write actions, driving the full-page run-details view. */ export const RunDetails = ({ run, subjectId, onBack }: RunDetailsProps) => { const { t } = useTranslation(); - const { showDebug } = useSolutionTestsConfig(); + const { showDebug, track } = useSolutionTestsConfig(); const { results, isLoading } = useRunResults(run.Id); const { jobs: baselines } = useBaselineJobs(run.SolutionTestId); @@ -166,7 +166,10 @@ export const RunDetails = ({ run, subjectId, onBack }: RunDetailsProps) => { updatingResultId={updatingResultId} removingBaselineId={removingBaselineId} onBack={onBack} - onSelect={setSelectedResultId} + onSelect={(id) => { + track?.("VS.SolutionTest.ResultInspected", { resultId: id }); + setSelectedResultId(id); + }} onAdopt={(id) => adopt.mutate(id, { onSuccess: () => toast.success(t("agent_adopted_successfully")), diff --git a/apps/apollo-vertex/registry/solution-tests/solution-tests-view.tsx b/apps/apollo-vertex/registry/solution-tests/solution-tests-view.tsx index 236fa7b5f..d62537cf9 100644 --- a/apps/apollo-vertex/registry/solution-tests/solution-tests-view.tsx +++ b/apps/apollo-vertex/registry/solution-tests/solution-tests-view.tsx @@ -163,6 +163,7 @@ export const SolutionTestsView = ({ const activeTab = activeTabProp ?? internalTab; const handleTabChange = (tab: TabValue) => { // When controlled, `activeTab` resolves to the prop so this is ignored. + config.track?.("VS.SolutionTest.TabViewed", { tab }); setInternalTab(tab); onTabChange?.(tab); }; @@ -215,6 +216,10 @@ export const SolutionTestsView = ({ aria-label={row.getIsExpanded() ? t("collapse") : t("expand")} onClick={(e) => { e.stopPropagation(); + if (!row.getIsExpanded()) + config.track?.("VS.SolutionTest.TestExpanded", { + testId: row.original.Id, + }); row.toggleExpanded(); }} > @@ -379,6 +384,10 @@ export const SolutionTestsView = ({ aria-label={row.getIsExpanded() ? t("collapse") : t("expand")} onClick={(e) => { e.stopPropagation(); + if (!row.getIsExpanded()) + config.track?.("VS.SolutionTest.BatchExpanded", { + batchId: row.original.Id, + }); row.toggleExpanded(); }} > diff --git a/apps/apollo-vertex/registry/solution-tests/use-baseline-jobs.ts b/apps/apollo-vertex/registry/solution-tests/use-baseline-jobs.ts index eb0e995b6..a70abea4a 100644 --- a/apps/apollo-vertex/registry/solution-tests/use-baseline-jobs.ts +++ b/apps/apollo-vertex/registry/solution-tests/use-baseline-jobs.ts @@ -12,7 +12,11 @@ import { useMutation } from "@tanstack/react-query"; import { useSolution } from "@uipath/vs-core"; import { fetchAttachment } from "./attachments"; import { ENTITY } from "./constants"; -import { useSolutionTestsActions, useSolutionTestsContext } from "./context"; +import { + useSolutionTestsActions, + useSolutionTestsConfig, + useSolutionTestsContext, +} from "./context"; import type { AttachmentFetcher, MutationHook } from "./mutations"; import { JobRole } from "./types"; import type { SolutionTestJob } from "./types"; @@ -39,12 +43,15 @@ export function useBaselineJobs(testId: string): UseBaselineJobsResult { /** Remove a job from the test's expected (baseline) results. */ export function useRemoveJobBaseline(): MutationHook { const actions = useSolutionTestsActions(); + const { track } = useSolutionTestsConfig(); const jobsCollection = useSolutionTestCollection(ENTITY.jobs); return useMutation({ mutationFn: async (baselineId: string) => { await actions.removeJobBaseline(baselineId); await jobsCollection.utils.refetch(); }, + onMutate: (baselineId) => + track?.("VS.SolutionTest.BaselineRemoved", { baselineId }), }); } diff --git a/apps/apollo-vertex/registry/solution-tests/use-force-stop.ts b/apps/apollo-vertex/registry/solution-tests/use-force-stop.ts index 5234ee7ad..65bc81ef7 100644 --- a/apps/apollo-vertex/registry/solution-tests/use-force-stop.ts +++ b/apps/apollo-vertex/registry/solution-tests/use-force-stop.ts @@ -7,21 +7,26 @@ */ import { useMutation } from "@tanstack/react-query"; -import { useSolutionTestsActions } from "./context"; +import { useSolutionTestsActions, useSolutionTestsConfig } from "./context"; import type { MutationHook } from "./mutations"; /** Force-stop a whole batch run. */ export function useForceStopBatch(): MutationHook { const actions = useSolutionTestsActions(); + const { track } = useSolutionTestsConfig(); return useMutation({ mutationFn: (batchId: string) => actions.forceStopBatch(batchId), + onMutate: (batchId) => + track?.("VS.SolutionTest.BatchForceStopped", { batchId }), }); } /** Force-stop a single run. */ export function useForceStopRun(): MutationHook { const actions = useSolutionTestsActions(); + const { track } = useSolutionTestsConfig(); return useMutation({ mutationFn: (runId: string) => actions.forceStopRun(runId), + onMutate: (runId) => track?.("VS.SolutionTest.RunForceStopped", { runId }), }); } diff --git a/apps/apollo-vertex/registry/solution-tests/use-run-results.ts b/apps/apollo-vertex/registry/solution-tests/use-run-results.ts index f5af8f6d1..08f5b1119 100644 --- a/apps/apollo-vertex/registry/solution-tests/use-run-results.ts +++ b/apps/apollo-vertex/registry/solution-tests/use-run-results.ts @@ -12,7 +12,11 @@ import { useMutation } from "@tanstack/react-query"; import { useSolution } from "@uipath/vs-core"; import { fetchAttachment } from "./attachments"; import { ENTITY } from "./constants"; -import { useSolutionTestsActions, useSolutionTestsContext } from "./context"; +import { + useSolutionTestsActions, + useSolutionTestsConfig, + useSolutionTestsContext, +} from "./context"; import type { AttachmentFetcher, MutationHook } from "./mutations"; import { JobRole } from "./types"; import type { @@ -70,16 +74,21 @@ export function useRunResults(runId: string): UseRunResultsResult { /** Adopt a run result's job as the test baseline. */ export function useAdoptJob(): MutationHook { const actions = useSolutionTestsActions(); + const { track } = useSolutionTestsConfig(); return useMutation({ mutationFn: (resultId: string) => actions.adoptJob(resultId), + onMutate: (resultId) => track?.("VS.SolutionTest.JobAdopted", { resultId }), }); } /** Update the test baseline from a run result. */ export function useUpdateBaseline(): MutationHook { const actions = useSolutionTestsActions(); + const { track } = useSolutionTestsConfig(); return useMutation({ mutationFn: (resultId: string) => actions.updateBaseline(resultId), + onMutate: (resultId) => + track?.("VS.SolutionTest.BaselineUpdated", { resultId }), }); } diff --git a/apps/apollo-vertex/registry/solution-tests/use-solution-tests.ts b/apps/apollo-vertex/registry/solution-tests/use-solution-tests.ts index 8c70e5735..2a7bdf29a 100644 --- a/apps/apollo-vertex/registry/solution-tests/use-solution-tests.ts +++ b/apps/apollo-vertex/registry/solution-tests/use-solution-tests.ts @@ -10,7 +10,7 @@ import { useLiveQuery } from "@tanstack/react-db"; import { useMutation } from "@tanstack/react-query"; import { useSolution } from "@uipath/vs-core"; import { ENTITY } from "./constants"; -import { useSolutionTestsActions } from "./context"; +import { useSolutionTestsActions, useSolutionTestsConfig } from "./context"; import type { MutationHook } from "./mutations"; import type { SolutionTest } from "./types"; import { useSolutionTestCollection } from "./use-solution-test-collection"; @@ -40,21 +40,27 @@ export function useRunTests(): MutationHook<{ mode: RunTestsMode; }> { const actions = useSolutionTestsActions(); + const { track } = useSolutionTestsConfig(); return useMutation({ mutationFn: ({ testIds }: { testIds?: string[]; mode: RunTestsMode }) => actions.runTests(testIds), + // Track the user action on trigger (not success) — we want intent, not outcome. + onMutate: ({ testIds, mode }) => + track?.("VS.SolutionTest.Run", { mode, testCount: testIds?.length ?? 0 }), }); } /** Create a solution test from a subject id (e.g. a loan). */ export function useCreateTest(): MutationHook { const actions = useSolutionTestsActions(); + const { track } = useSolutionTestsConfig(); const testsCollection = useSolutionTestCollection(ENTITY.tests); return useMutation({ mutationFn: async (subjectId: string) => { await actions.createTest(subjectId); await testsCollection.utils.refetch(); }, + onMutate: (subjectId) => track?.("VS.SolutionTest.Created", { subjectId }), }); } @@ -69,6 +75,7 @@ export function useToggleTestActive(): MutationHook<{ isActive: boolean; }> { const solution = useSolution(); + const { track } = useSolutionTestsConfig(); return useMutation({ mutationFn: async ({ testId, @@ -86,12 +93,15 @@ export function useToggleTestActive(): MutationHook<{ }); await transaction.isPersisted.promise; }, + onMutate: ({ testId, isActive }) => + track?.("VS.SolutionTest.ActiveToggled", { testId, isActive }), }); } /** Delete a test. */ export function useDeleteTest(): MutationHook { const actions = useSolutionTestsActions(); + const { track } = useSolutionTestsConfig(); const testsCollection = useSolutionTestCollection(ENTITY.tests); const jobsCollection = useSolutionTestCollection(ENTITY.jobs); return useMutation({ @@ -102,5 +112,6 @@ export function useDeleteTest(): MutationHook { jobsCollection.utils.refetch(), ]); }, + onMutate: (testId) => track?.("VS.SolutionTest.Deleted", { testId }), }); }