diff --git a/.github/workflows/manual-e2e.yml b/.github/workflows/manual-e2e.yml new file mode 100644 index 00000000..ab14a93b --- /dev/null +++ b/.github/workflows/manual-e2e.yml @@ -0,0 +1,106 @@ +name: Manual E2E Validation + +on: + workflow_dispatch: + inputs: + sdk_branch: + description: 'SDK branch to test' + required: true + default: 'main' + type: string + ui_branch: + description: 'agents-ui branch to test' + required: true + default: 'staging' + type: string + +jobs: + e2e-validation: + runs-on: ${{ github.event.inputs.ui_branch == 'prod' && 'ubuntu-latest' || 'aws-medium' }} + timeout-minutes: 30 + environment: + name: ${{ github.event.inputs.ui_branch == 'prod' && 'prod' || 'staging' }} + env: + ENV: ${{ github.event.inputs.ui_branch == 'prod' && 'prod' || 'staging' }} + + steps: + - name: Checkout SDK branch + uses: actions/checkout@v4 + with: + path: agents-sdk + ref: ${{ github.event.inputs.sdk_branch }} + + - name: Setup Node.js for SDK + uses: actions/setup-node@v4 + with: + node-version: 20 + cache-dependency-path: agents-sdk/yarn.lock + + - name: Install Yarn + run: npm install -g yarn + + - name: Install SDK dependencies + working-directory: agents-sdk + run: yarn install --frozen-lockfile + + - name: Build SDK + working-directory: agents-sdk + run: yarn build + + - name: Pack SDK for testing + working-directory: agents-sdk + run: | + npm pack + echo "SDK_PACKAGE=$(ls *.tgz)" >> $GITHUB_ENV + + - name: Checkout agents-ui branch + uses: actions/checkout@v4 + with: + repository: de-id/agents-ui + ref: ${{ github.event.inputs.ui_branch }} + path: agents-ui + token: ${{ secrets.DEVOPS_TOKEN }} + + - name: Set github environment variables + uses: rlespinasse/github-slug-action@v4 + + - name: Setup Node.js for agents-ui + uses: actions/setup-node@v4 + with: + node-version: 20 + + - name: Render .npmrc for agents-ui + working-directory: agents-ui + run: | + if [ -f .npmrc.template ]; then + sed "s/\$NPM_AUTH_TOKEN/${{ secrets.NPM_TOKEN }}/g" .npmrc.template > .npmrc + fi + + - name: Install local SDK build in agents-ui + working-directory: agents-ui + run: | + yarn remove @d-id/client-sdk || true + yarn add file:../agents-sdk/${{ env.SDK_PACKAGE }} + yarn install --frozen-lockfile + + - name: Install Playwright Chrome + working-directory: agents-ui + run: yarn playwright install chrome + + - name: Run E2E tests + working-directory: agents-ui + env: + E2E_USER_APIKEY: ${{ secrets.E2E_USER_APIKEY }} + VITE_CLIENT_KEY: ${{ secrets.VITE_CLIENT_KEY }} + ASSERT_CHAT_RESTART: 'false' + run: yarn test:${{ github.event.inputs.ui_branch == 'prod' && 'prod' || 'staging' }} + + - name: Upload test results + if: always() + uses: actions/upload-artifact@v4 + with: + name: e2e-test-results-manual-${{ github.event.inputs.sdk_branch }}-${{ github.event.inputs.ui_branch }} + path: | + agents-ui/playwright-report/ + agents-ui/test-results/ + retention-days: 30 diff --git a/.github/workflows/pr-main-e2e.yml b/.github/workflows/pr-main-e2e.yml new file mode 100644 index 00000000..5867c17f --- /dev/null +++ b/.github/workflows/pr-main-e2e.yml @@ -0,0 +1,101 @@ +name: UI prod e2e with local sdk build + +on: + pull_request: + branches: [main] + types: [opened, synchronize, reopened] + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }} + cancel-in-progress: true + +jobs: + e2e-validation: + runs-on: ubuntu-latest + timeout-minutes: 30 + environment: + name: prod + env: + ENV: prod + + steps: + - name: Checkout SDK branch + uses: actions/checkout@v4 + with: + path: agents-sdk + ref: ${{ github.head_ref || github.ref_name }} + + - name: Setup Node.js for SDK + uses: actions/setup-node@v4 + with: + node-version: 20 + cache-dependency-path: agents-sdk/yarn.lock + + - name: Install Yarn + run: npm install -g yarn + + - name: Install SDK dependencies + working-directory: agents-sdk + run: yarn install --frozen-lockfile + + - name: Build SDK + working-directory: agents-sdk + run: yarn build + + - name: Pack SDK for testing + working-directory: agents-sdk + run: | + npm pack + echo "SDK_PACKAGE=$(ls *.tgz)" >> $GITHUB_ENV + + - name: Checkout agents-ui production branch + uses: actions/checkout@v4 + with: + repository: de-id/agents-ui + ref: prod + path: agents-ui + token: ${{ secrets.DEVOPS_TOKEN }} + + - name: Set github environment variables + uses: rlespinasse/github-slug-action@v4 + + - name: Setup Node.js for agents-ui + uses: actions/setup-node@v4 + with: + node-version: 20 + + - name: Render .npmrc for agents-ui + working-directory: agents-ui + run: | + if [ -f .npmrc.template ]; then + sed "s/\$NPM_AUTH_TOKEN/${{ secrets.NPM_TOKEN }}/g" .npmrc.template > .npmrc + fi + + - name: Install local SDK build in agents-ui + working-directory: agents-ui + run: | + yarn remove @d-id/client-sdk || true + yarn add file:../agents-sdk/${{ env.SDK_PACKAGE }} + yarn install --frozen-lockfile + + - name: Install Playwright Chrome + working-directory: agents-ui + run: yarn playwright install chrome + + - name: Run E2E tests against production + working-directory: agents-ui + env: + E2E_USER_APIKEY: ${{ secrets.E2E_USER_APIKEY }} + VITE_CLIENT_KEY: ${{ secrets.VITE_CLIENT_KEY }} + ASSERT_CHAT_RESTART: 'false' + run: yarn test:prod + + - name: Upload test results + if: always() + uses: actions/upload-artifact@v4 + with: + name: e2e-test-results-main-pr-${{ github.event.number }} + path: | + agents-ui/playwright-report/ + agents-ui/test-results/ + retention-days: 30 diff --git a/.github/workflows/pr-prod-e2e.yml b/.github/workflows/pr-prod-e2e.yml new file mode 100644 index 00000000..885c9848 --- /dev/null +++ b/.github/workflows/pr-prod-e2e.yml @@ -0,0 +1,71 @@ +name: UI prod e2e with staging sdk build + +on: + pull_request: + branches: [prod] + types: [opened, synchronize, reopened] + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }} + cancel-in-progress: true + +jobs: + e2e-validation: + runs-on: ubuntu-latest + timeout-minutes: 30 + environment: + name: prod + env: + ENV: prod + + steps: + - name: Checkout agents-ui production branch + uses: actions/checkout@v4 + with: + repository: de-id/agents-ui + ref: prod + path: agents-ui + fetch-depth: 0 + lfs: true + token: ${{ secrets.DEVOPS_TOKEN }} + + - name: Set github environment variables + uses: rlespinasse/github-slug-action@v4 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: 20 + + - name: Render .npmrc for agents-ui + working-directory: agents-ui + run: | + if [ -f .npmrc.template ]; then + envsubst < .npmrc.template > .npmrc + fi + env: + NPM_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} + + - name: Install staging SDK version + working-directory: agents-ui + run: | + yarn remove @d-id/client-sdk || true + yarn add @d-id/client-sdk@staging + npm install -g yarn && yarn + + - name: Run E2E tests against production environment + working-directory: agents-ui + env: + E2E_USER_APIKEY: ${{ secrets.E2E_USER_APIKEY }} + VITE_CLIENT_KEY: ${{ secrets.VITE_CLIENT_KEY }} + run: yarn test:prod + + - name: Upload test results + if: always() + uses: actions/upload-artifact@v4 + with: + name: e2e-test-results-prod-pr-${{ github.event.number }} + path: | + agents-ui/playwright-report/ + agents-ui/test-results/ + retention-days: 30 diff --git a/.github/workflows/publish-on-merge.yml b/.github/workflows/publish-on-merge.yml new file mode 100644 index 00000000..a60990d3 --- /dev/null +++ b/.github/workflows/publish-on-merge.yml @@ -0,0 +1,210 @@ +name: Auto Publish SDK + +on: + push: + branches: [main, prod] + workflow_dispatch: + inputs: + dry_run: + description: 'Run in dry-run mode (no actual publishing)' + required: false + default: false + type: boolean + +jobs: + publish: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + token: ${{ secrets.GITHUB_TOKEN }} + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: 20 + registry-url: 'https://registry.npmjs.org' + cache: 'yarn' + + - name: Install dependencies + run: yarn install --frozen-lockfile + + - name: Build package + run: yarn build + + - name: Determine version and tag + id: version + run: | + if [ "${{ github.ref_name }}" = "main" ]; then + # For main branch - staging versions with run number + BASE_VERSION=$(jq -r '.version' package.json) + if [ "$BASE_VERSION" = "null" ]; then + echo "Error: Could not read version from package.json" + exit 1 + fi + CLEAN_VERSION=$(echo "$BASE_VERSION" | sed 's/-.*$//') + STAGING_VERSION="${CLEAN_VERSION}-staging.${{ github.run_number }}" + echo "version=$STAGING_VERSION" >> $GITHUB_OUTPUT + echo "tag=staging" >> $GITHUB_OUTPUT + echo "description=Staging release from main branch" >> $GITHUB_OUTPUT + echo "should_sync=false" >> $GITHUB_OUTPUT + else + # For prod branch - production versions + # Use npm version command (official npm way to bump versions) + NEW_VERSION=$(npm version patch --no-git-tag-version --silent) + NEW_VERSION=${NEW_VERSION#v} # Remove 'v' prefix if present + echo "version=$NEW_VERSION" >> $GITHUB_OUTPUT + echo "tag=latest" >> $GITHUB_OUTPUT + echo "description=Production release" >> $GITHUB_OUTPUT + echo "should_sync=true" >> $GITHUB_OUTPUT + fi + + - name: Update package.json version + run: | + jq --arg version "${{ steps.version.outputs.version }}" '.version = $version' package.json > package.json.tmp + + if ! jq empty package.json.tmp 2>/dev/null; then + echo "Error: Generated invalid JSON" + rm -f package.json.tmp + exit 1 + fi + + mv package.json.tmp package.json + + echo "Updated package.json version to: $(jq -r '.version' package.json)" + + - name: Publish to NPM + env: + NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} + run: | + if [ "${{ github.event.inputs.dry_run }}" = "true" ]; then + echo "🔍 DRY RUN MODE: Would publish version ${{ steps.version.outputs.version }} with tag ${{ steps.version.outputs.tag }}" + echo "📦 Package would be published to: https://www.npmjs.com/package/@d-id/client-sdk/v/${{ steps.version.outputs.version }}" + echo "🏷️ NPM tag would be: ${{ steps.version.outputs.tag }}" + echo "✅ Dry run completed successfully - no actual publishing occurred" + else + echo "🚀 Publishing version ${{ steps.version.outputs.version }} with tag ${{ steps.version.outputs.tag }}" + npm publish --access public --tag ${{ steps.version.outputs.tag }} + echo "✅ Successfully published to NPM" + fi + + - name: Create Git tag for production + if: github.ref_name == 'prod' && github.event.inputs.dry_run != 'true' + run: | + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + git tag "v${{ steps.version.outputs.version }}" + git push origin "v${{ steps.version.outputs.version }}" + + - name: Commit version bump (prod only) + if: github.ref_name == 'prod' && github.event.inputs.dry_run != 'true' + run: | + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + git add package.json + git commit -m "chore: bump version to ${{ steps.version.outputs.version }} [skip ci]" || echo "No changes to commit" + git push origin prod + + - name: Sync version back to main (prod only) + if: github.ref_name == 'prod' && github.event.inputs.dry_run != 'true' + run: | + # Fetch latest main + git fetch origin main + git checkout main + git pull origin main + + jq --arg version "${{ steps.version.outputs.version }}" '.version = $version' package.json > package.json.tmp + + if ! jq empty package.json.tmp 2>/dev/null; then + echo "Error: Generated invalid JSON" + rm -f package.json.tmp + exit 1 + fi + + mv package.json.tmp package.json + + if git diff --quiet package.json; then + echo "No version changes needed" + else + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + git add package.json + git commit -m "chore: sync version ${{ steps.version.outputs.version }} from prod [skip ci]" + git push origin main + fi + + - name: Create GitHub Release (prod only) + if: github.ref_name == 'prod' && github.event.inputs.dry_run != 'true' + uses: actions/create-release@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + tag_name: v${{ steps.version.outputs.version }} + release_name: Release v${{ steps.version.outputs.version }} + body: | + ## Release v${{ steps.version.outputs.version }} + + **Published to NPM:** [@d-id/client-sdk@${{ steps.version.outputs.version }}](https://www.npmjs.com/package/@d-id/client-sdk/v/${{ steps.version.outputs.version }}) + + ### Installation + ```bash + npm install @d-id/client-sdk@${{ steps.version.outputs.version }} + ``` + + ${{ steps.version.outputs.description }} + draft: false + prerelease: false + + - name: Checkout agents-ui repository + if: github.ref_name == 'prod' && github.event.inputs.dry_run != 'true' + uses: actions/checkout@v4 + with: + repository: de-id/agents-ui + token: ${{ secrets.GITHUB_TOKEN }} + path: agents-ui + + - name: Update SDK version and create PR + if: github.ref_name == 'prod' && github.event.inputs.dry_run != 'true' + run: | + cd agents-ui + + jq --arg version "${{ steps.version.outputs.version }}" '.dependencies."@d-id/client-sdk" = $version' package.json > package.json.tmp + mv package.json.tmp package.json + + if git diff --quiet package.json; then + echo "No version changes needed in agents-ui" + exit 0 + fi + + git checkout -b "chore/bump-sdk-version-${{ steps.version.outputs.version }}" + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + git add package.json + git commit -m "chore: bump @d-id/client-sdk to v${{ steps.version.outputs.version }}" + git push origin "chore/bump-sdk-version-${{ steps.version.outputs.version }}" + + gh pr create \ + --repo de-id/agents-ui \ + --title "chore: bump @d-id/client-sdk to v${{ steps.version.outputs.version }}" \ + --body "## SDK Version Update + + This PR updates the @d-id/client-sdk dependency to version ${{ steps.version.outputs.version }}. + + ### Changes + - Updated @d-id/client-sdk from previous version to v${{ steps.version.outputs.version }} + + ### Related + - SDK Release: [v${{ steps.version.outputs.version }}](https://github.com/d-id/agents-sdk/releases/tag/v${{ steps.version.outputs.version }}) + - NPM Package: [@d-id/client-sdk@${{ steps.version.outputs.version }}](https://www.npmjs.com/package/@d-id/client-sdk/v/${{ steps.version.outputs.version }}) + + ### Next Steps + - [ ] Review the changes + - [ ] Run tests to ensure compatibility + - [ ] Merge when ready" \ + --base main \ + --head "chore/bump-sdk-version-${{ steps.version.outputs.version }}" \ + --label "dependencies" \ + --label "automated" + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/demo/app.tsx b/demo/app.tsx index cc0f99e0..8fbe657f 100644 --- a/demo/app.tsx +++ b/demo/app.tsx @@ -8,7 +8,7 @@ import { useAgentManager } from './hooks/useAgentManager'; export function App() { const [warmup, setWarmup] = useState(true); const [text, setText] = useState( - 'oded bobobobo sagi mamamama . bla raga ode ovem. lol cha cha cha cha cha . bobobobo. cha cha cha cha. bobobobo cha cha cha cha bobobobo. ssssssss cha cha cha cha cha bobobobo . cha cha cha cha bobobobo . cha cha cha cha. bobobobo ssssssss' + 'Ben bobobobo sagi mamamama . bla raga ode ovem. lol cha cha cha cha cha . bobobobo. cha cha cha cha. bobobobo cha cha cha cha bobobobo. ssssssss cha cha cha cha cha bobobobo . cha cha cha cha bobobobo . cha cha cha cha. bobobobo ssssssss' ); const [mode, setMode] = useState(ChatMode.Functional); const [sessionTimeout, setSessionTimeout] = useState(); @@ -17,20 +17,21 @@ export function App() { const videoRef = useRef(null); - const { srcObject, connectionState, messages, isSpeaking, connect, disconnect, speak, chat } = useAgentManager({ - agentId, - baseURL: didApiUrl, - wsURL: didSocketApiUrl, - mode, - enableAnalytics: false, - auth: { type: 'key', clientKey }, - streamOptions: { - streamWarmup: warmup, - sessionTimeout, - compatibilityMode, - fluent, - }, - }); + const { srcObject, connectionState, messages, isSpeaking, connect, disconnect, speak, chat, interrupt } = + useAgentManager({ + agentId, + baseURL: didApiUrl, + wsURL: didSocketApiUrl, + mode, + enableAnalytics: false, + auth: { type: 'key', clientKey }, + streamOptions: { + streamWarmup: warmup, + sessionTimeout, + compatibilityMode, + fluent, + }, + }); async function onClick() { if (connectionState === ConnectionState.New || connectionState === ConnectionState.Fail) { @@ -81,6 +82,10 @@ export function App() { Send to Chat + + diff --git a/demo/hooks/useAgentManager.ts b/demo/hooks/useAgentManager.ts index 2c966bf3..82e04223 100644 --- a/demo/hooks/useAgentManager.ts +++ b/demo/hooks/useAgentManager.ts @@ -142,6 +142,10 @@ export function useAgentManager(props: UseAgentManagerOptions) { try { await agentManager.chat(userMessage.trim()); } catch (e) { + if (e instanceof Error && e.message?.includes('User stream has reached pending requests limit')) { + console.log('User stream has reached pending requests limit'); + return; + } setConnectionState(ConnectionState.Fail); throw e; @@ -150,6 +154,17 @@ export function useAgentManager(props: UseAgentManagerOptions) { [agentManager, connectionState] ); + const interrupt = useCallback(async () => { + if (!agentManager || connectionState !== ConnectionState.Connected) return; + + try { + agentManager.interrupt({ type: 'click' }); + } catch (e) { + console.error('Error interrupting:', e); + throw e; + } + }, [agentManager, connectionState]); + return { connectionState, messages, @@ -159,5 +174,6 @@ export function useAgentManager(props: UseAgentManagerOptions) { disconnect, speak, chat, + interrupt, }; } diff --git a/package.json b/package.json index 61567e77..6f4f26bc 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "@d-id/client-sdk", "private": false, - "version": "1.1.0-beta.5", + "version": "1.1.1", "type": "module", "description": "d-id client sdk", "repository": { @@ -34,7 +34,7 @@ "devDependencies": { "@preact/preset-vite": "^2.8.1", "@trivago/prettier-plugin-sort-imports": "^4.3.0", - "@types/node": "^20.11.24", + "@types/node": "^22.15.0", "commander": "^11.1.0", "glob": "^10.3.10", "preact": "^10.19.6", @@ -45,4 +45,4 @@ "vite": "^5.1.4", "vite-plugin-dts": "^3.7.3" } -} \ No newline at end of file +} diff --git a/src/auth/get-auth-header.ts b/src/auth/get-auth-header.ts index 1fdfd48b..bb6bf268 100644 --- a/src/auth/get-auth-header.ts +++ b/src/auth/get-auth-header.ts @@ -6,8 +6,9 @@ export function getExternalId() { let key = window.localStorage.getItem('did_external_key_id'); if (!key) { - key = Math.random().toString(16).slice(2); - window.localStorage.setItem('did_external_key_id', key); + let newKey = getRandom() + window.localStorage.setItem('did_external_key_id', newKey); + key = newKey } return key; diff --git a/src/services/agent-manager/connect-to-manager.ts b/src/services/agent-manager/connect-to-manager.ts index e28c7759..f7a7fd04 100644 --- a/src/services/agent-manager/connect-to-manager.ts +++ b/src/services/agent-manager/connect-to-manager.ts @@ -15,7 +15,7 @@ import { mapVideoType, } from '$/types'; import { Analytics } from '../analytics/mixpanel'; -import { timestampTracker } from '../analytics/timestamp-tracker'; +import { interruptTimestampTracker, latencyTimestampTracker } from '../analytics/timestamp-tracker'; import { createChat } from '../chat'; function getAgentStreamArgs(agent: Agent, options?: AgentManagerOptions): CreateStreamOptions { @@ -31,44 +31,114 @@ function getAgentStreamArgs(agent: Agent, options?: AgentManagerOptions): Create }; } -function trackStreamingStateAnalytics( +function trackVideoStateChangeAnalytics( state: StreamingState, agent: Agent, statsReport: any, analytics: Analytics, streamType: StreamType ) { - if (timestampTracker.get() > 0) { - if (state === StreamingState.Start) { - analytics.linkTrack( - 'agent-video', - { event: 'start', latency: timestampTracker.get(true), 'stream-type': streamType }, - 'start', - [StreamEvents.StreamVideoCreated] - ); - } else if (state === StreamingState.Stop) { - analytics.linkTrack( - 'agent-video', - { - event: 'stop', - is_greenscreen: agent.presenter.type === 'clip' && agent.presenter.is_greenscreen, - background: agent.presenter.type === 'clip' && agent.presenter.background, - 'stream-type': streamType, - ...statsReport, - }, - 'done', - [StreamEvents.StreamVideoDone] - ); - } + if (streamType === StreamType.Fluent) { + trackVideoStreamAnalytics(state, agent, statsReport, analytics, streamType); + } else { + trackLegacyVideoAnalytics(state, agent, statsReport, analytics, streamType); + } +} + +function trackVideoStreamAnalytics( + state: StreamingState, + agent: Agent, + statsReport: any, + analytics: Analytics, + streamType: StreamType +) { + if (state === StreamingState.Start) { + analytics.track('stream-session', { event: 'start', 'stream-type': streamType }); + } else if (state === StreamingState.Stop) { + analytics.track('stream-session', { + event: 'stop', + is_greenscreen: agent.presenter.type === 'clip' && agent.presenter.is_greenscreen, + background: agent.presenter.type === 'clip' && agent.presenter.background, + 'stream-type': streamType, + ...statsReport, + }); + } +} + +function trackAgentActivityAnalytics( + state: StreamingState, + agent: Agent, + analytics: Analytics, + streamType: StreamType +) { + if (latencyTimestampTracker.get() <= 0) return; + + if (state === StreamingState.Start) { + analytics.linkTrack( + 'agent-video', + { event: 'start', latency: latencyTimestampTracker.get(true), 'stream-type': streamType }, + 'start', + [StreamEvents.StreamVideoCreated] + ); + } else if (state === StreamingState.Stop) { + analytics.linkTrack( + 'agent-video', + { + event: 'stop', + is_greenscreen: agent.presenter.type === 'clip' && agent.presenter.is_greenscreen, + background: agent.presenter.type === 'clip' && agent.presenter.background, + 'stream-type': streamType, + }, + 'done', + [StreamEvents.StreamVideoDone] + ); } } +function trackLegacyVideoAnalytics( + state: StreamingState, + agent: Agent, + statsReport: any, + analytics: Analytics, + streamType: StreamType +) { + if (latencyTimestampTracker.get() <= 0) return; + + if (state === StreamingState.Start) { + analytics.linkTrack( + 'agent-video', + { event: 'start', latency: latencyTimestampTracker.get(true), 'stream-type': streamType }, + 'start', + [StreamEvents.StreamVideoCreated] + ); + } else if (state === StreamingState.Stop) { + analytics.linkTrack( + 'agent-video', + { + event: 'stop', + is_greenscreen: agent.presenter.type === 'clip' && agent.presenter.is_greenscreen, + background: agent.presenter.type === 'clip' && agent.presenter.background, + 'stream-type': streamType, + ...statsReport, + }, + 'done', + [StreamEvents.StreamVideoDone] + ); + } +} + +type ConnectToManagerOptions = AgentManagerOptions & { + callbacks: AgentManagerOptions['callbacks'] & { + onVideoIdChange?: (videoId: string | null) => void; + }; +}; + function connectToManager( agent: Agent, - options: AgentManagerOptions, + options: ConnectToManagerOptions, analytics: Analytics ): Promise> { - timestampTracker.reset(); + latencyTimestampTracker.reset(); return new Promise(async (resolve, reject) => { try { @@ -86,14 +156,27 @@ function connectToManager( }, onVideoStateChange: (state: StreamingState, statsReport?: any) => { options.callbacks.onVideoStateChange?.(state); - trackStreamingStateAnalytics(state, agent, statsReport, analytics, streamingManager.streamType); + + trackVideoStateChangeAnalytics( + state, + agent, + statsReport, + analytics, + streamingManager.streamType + ); }, onAgentActivityStateChange: (state: AgentActivityState) => { options.callbacks.onAgentActivityStateChange?.(state); - trackStreamingStateAnalytics( + + if (state === AgentActivityState.Talking) { + interruptTimestampTracker.update(); + } else { + interruptTimestampTracker.reset(); + } + + trackAgentActivityAnalytics( state === AgentActivityState.Talking ? StreamingState.Start : StreamingState.Stop, agent, - undefined, analytics, streamingManager.streamType ); @@ -108,7 +191,7 @@ function connectToManager( export async function initializeStreamAndChat( agent: Agent, - options: AgentManagerOptions, + options: ConnectToManagerOptions, agentsApi: AgentsAPI, analytics: Analytics, chat?: Chat diff --git a/src/services/agent-manager/index.ts b/src/services/agent-manager/index.ts index 65149b07..944fa2c2 100644 --- a/src/services/agent-manager/index.ts +++ b/src/services/agent-manager/index.ts @@ -7,6 +7,7 @@ import { ChatMode, ConnectionState, CreateStreamOptions, + Interrupt, Message, StreamScript, SupportedStreamScript, @@ -16,13 +17,15 @@ import { CONNECTION_RETRY_TIMEOUT_MS } from '$/config/consts'; import { didApiUrl, didSocketApiUrl, mixpanelKey } from '$/config/environment'; import { ChatCreationFailed, ValidationError } from '$/errors'; import { getRandom } from '$/utils'; +import { isChatModeWithoutChat, isTextualChat } from '$/utils/chat'; import { createAgentsApi } from '../../api/agents'; -import { getAnalyticsInfo } from '../../utils/analytics'; +import { getAgentInfo, getAnalyticsInfo } from '../../utils/analytics'; import { retryOperation } from '../../utils/retry-operation'; import { initializeAnalytics } from '../analytics/mixpanel'; -import { timestampTracker } from '../analytics/timestamp-tracker'; +import { interruptTimestampTracker, latencyTimestampTracker } from '../analytics/timestamp-tracker'; import { createChat, getRequestHeaders } from '../chat'; import { getInitialMessages } from '../chat/intial-messages'; +import { sendInterrupt, validateInterrupt } from '../interrupt'; import { SocketManager, createSocketManager } from '../socket-manager'; import { createMessageEventQueue } from '../socket-manager/message-queue'; import { StreamingManager } from '../streaming-manager'; @@ -50,20 +53,28 @@ export interface AgentManagerItems { */ export async function createAgentManager(agent: string, options: AgentManagerOptions): Promise { let firstConnection = true; + let videoId: string | null = null; const mxKey = options.mixpanelKey || mixpanelKey; const wsURL = options.wsURL || didSocketApiUrl; const baseURL = options.baseURL || didApiUrl; - const items: AgentManagerItems = { messages: [], chatMode: options.mode || ChatMode.Functional }; - const agentsApi = createAgentsApi(options.auth, baseURL, options.callbacks.onError); - const agentEntity = await agentsApi.getById(agent); + const items: AgentManagerItems = { + messages: [], + chatMode: options.mode || ChatMode.Functional, + }; const analytics = initializeAnalytics({ token: mxKey, - agent: agentEntity, + agentId: agent, isEnabled: options.enableAnalitics, distinctId: options.distinctId, }); + analytics.track('agent-sdk', { event: 'init' }); + const agentsApi = createAgentsApi(options.auth, baseURL, options.callbacks.onError); + + const agentEntity = await agentsApi.getById(agent); + analytics.enrich(getAgentInfo(agentEntity)); + const { onMessage, clearQueue } = createMessageEventQueue(analytics, items, options, agentEntity, () => items.socketManager?.disconnect() ); @@ -72,12 +83,16 @@ export async function createAgentManager(agent: string, options: AgentManagerOpt options.callbacks.onNewMessage?.([...items.messages], 'answer'); + const updateVideoId = (newVideoId: string | null) => { + videoId = newVideoId; + }; + analytics.track('agent-sdk', { event: 'loaded', ...getAnalyticsInfo(agentEntity) }); async function connect(newChat: boolean) { options.callbacks.onConnectionStateChange?.(ConnectionState.Connecting); - timestampTracker.reset(); + latencyTimestampTracker.reset(); if (newChat && !firstConnection) { delete items.chat; @@ -92,7 +107,13 @@ export async function createAgentManager(agent: string, options: AgentManagerOpt const initPromise = retryOperation( () => { - return initializeStreamAndChat(agentEntity, options, agentsApi, analytics, items.chat); + return initializeStreamAndChat( + agentEntity, + { ...options, callbacks: { ...options.callbacks, onVideoIdChange: updateVideoId } }, + agentsApi, + analytics, + items.chat + ); }, { limit: 3, @@ -149,6 +170,7 @@ export async function createAgentManager(agent: string, options: AgentManagerOpt return { agent: agentEntity, getStreamType: () => items.streamingManager?.streamType, + getIsInterruptAvailable: () => items.streamingManager?.interruptAvailable ?? false, starterMessages: agentEntity.knowledge?.starter_message || [], getSTTToken: () => agentsApi.getSTTToken(agentEntity.id), changeMode, @@ -186,8 +208,8 @@ export async function createAgentManager(agent: string, options: AgentManagerOpt }, async chat(userMessage: string) { const validateChatRequest = () => { - if (options.mode === ChatMode.DirectPlayback) { - throw new ValidationError('Direct playback is enabled, chat is disabled'); + if (isChatModeWithoutChat(options.mode)) { + throw new ValidationError(`${options.mode} is enabled, chat is disabled`); } else if (userMessage.length >= 800) { throw new ValidationError('Message cannot be more than 800 characters'); } else if (userMessage.length === 0) { @@ -271,7 +293,7 @@ export async function createAgentManager(agent: string, options: AgentManagerOpt id: getRandom(), role: 'user', content: userMessage, - created_at: new Date(timestampTracker.update()).toISOString(), + created_at: new Date(latencyTimestampTracker.update()).toISOString(), }); options.callbacks.onNewMessage?.([...items.messages], 'user'); @@ -298,7 +320,7 @@ export async function createAgentManager(agent: string, options: AgentManagerOpt options.callbacks.onNewMessage?.([...items.messages], 'answer'); analytics.track('agent-message-received', { - latency: timestampTracker.get(true), + latency: latencyTimestampTracker.get(true), mode: items.chatMode, messages: items.messages.length, }); @@ -364,11 +386,7 @@ export async function createAgentManager(agent: string, options: AgentManagerOpt return agentsApi.deleteRating(agentEntity.id, items.chat.id, id); }, - speak(payload: string | SupportedStreamScript) { - if (!items.streamingManager) { - throw new Error('Please connect to the agent first'); - } - + async speak(payload: string | SupportedStreamScript) { function getScript(): StreamScript { if (typeof payload === 'string') { if (!agentEntity.presenter.voice) { @@ -401,23 +419,57 @@ export async function createAgentManager(agent: string, options: AgentManagerOpt const script = getScript(); analytics.track('agent-speak', script); - timestampTracker.update(); + latencyTimestampTracker.update(); - if (items.chat?.id && script.type === 'text') { + if (items.messages && script.type === 'text') { items.messages.push({ id: getRandom(), role: 'assistant', content: script.input, - created_at: new Date(timestampTracker.get(true)).toISOString(), + created_at: new Date(latencyTimestampTracker.get(true)).toISOString(), }); options.callbacks.onNewMessage?.([...items.messages], 'answer'); } + const isTextual = isTextualChat(items.chatMode); + + // If the current chat is textual, we shouldn't activate the TTS. + if (isTextual) { + return { + duration: 0, + video_id: '', + status: 'success', + }; + } + + if (!items.streamingManager) { + throw new Error('Please connect to the agent first'); + } + return items.streamingManager.speak({ script, metadata: { chat_id: items.chat?.id, agent_id: agentEntity.id }, }); }, + async interrupt({ type }: Interrupt) { + validateInterrupt(items.streamingManager, items.streamingManager?.streamType, videoId); + const lastMessage = items.messages[items.messages.length - 1]; + + analytics.track('agent-video-interrupt', { + type: type || 'click', + stream_id: items.streamingManager?.streamId, + agent_id: agentEntity.id, + owner_id: agentEntity.owner_id, + video_duration_to_interrupt: interruptTimestampTracker.get(true), + message_duration_to_interrupt: latencyTimestampTracker.get(true), + chat_id: items.chat?.id, + mode: items.chatMode, + }); + + lastMessage.interrupted = true; + options.callbacks.onNewMessage?.([...items.messages], 'answer'); + sendInterrupt(items.streamingManager!, videoId!); + }, }; } diff --git a/src/services/analytics/mixpanel.ts b/src/services/analytics/mixpanel.ts index b06fc5de..c2a993a1 100644 --- a/src/services/analytics/mixpanel.ts +++ b/src/services/analytics/mixpanel.ts @@ -1,10 +1,9 @@ import { getExternalId } from '$/auth/get-auth-header'; -import { Agent } from '$/types'; -import { getAgentType } from '$/utils/agent'; +import { getRandom } from '$/utils'; export interface AnalyticsOptions { token: string; - agent: Agent; + agentId: string; isEnabled?: boolean; distinctId?: string; } @@ -15,7 +14,7 @@ export interface Analytics { isEnabled: boolean; chatId?: string; agentId: string; - owner_id: string; + owner_id?: string; getRandom(): string; track(event: string, props?: Record): Promise; linkTrack(mixpanelEvent: string, props: Record, event: string, dependencies: string[]): any; @@ -39,46 +38,15 @@ const mixpanelUrl = 'https://api-js.mixpanel.com/track/?verbose=1&ip=1'; export function initializeAnalytics(config: AnalyticsOptions): Analytics { const source = window?.hasOwnProperty('DID_AGENTS_API') ? 'agents-ui' : 'agents-sdk'; - const presenter = config.agent.presenter; - const promptCustomization = config.agent.llm?.prompt_customization; - const analyticProps = { + return { token: config.token || 'testKey', distinct_id: config.distinctId || getExternalId(), - agentId: config.agent.id, - agentType: getAgentType(presenter), - owner_id: config.agent.owner_id ?? '', - promptVersion: config.agent.llm?.prompt_version, - behavior: { - role: promptCustomization?.role, - personality: promptCustomization?.personality, - instructions: config.agent.llm?.instructions, - }, - temperature: config.agent.llm?.temperature, - knowledgeSource: promptCustomization?.knowledge_source, - starterQuestionsCount: config.agent.knowledge?.starter_message?.length, - topicsToAvoid: promptCustomization?.topics_to_avoid, - maxResponseLength: promptCustomization?.max_response_length, - }; - - return { - ...analyticProps, + agentId: config.agentId, additionalProperties: {}, isEnabled: config.isEnabled ?? true, - getRandom: () => Math.random().toString(16).slice(2), - enrich(properties: Record) { - const props = {}; - - if (properties && typeof properties !== 'object') { - throw new Error('properties must be a flat json object'); - } - - for (let prop in properties) { - if (typeof properties[prop] === 'string' || typeof properties[prop] === 'number') { - props[prop] = properties[prop]; - } - } - + getRandom, + enrich(props: Record) { this.additionalProperties = { ...this.additionalProperties, ...props }; }, async track(event: string, props?: Record) { @@ -101,7 +69,7 @@ export function initializeAnalytics(config: AnalyticsOptions): Analytics { properties: { ...this.additionalProperties, ...sendProps, - ...analyticProps, + agentId: this.agentId, source, time: Date.now(), $insert_id: this.getRandom(), diff --git a/src/services/analytics/timestamp-tracker.ts b/src/services/analytics/timestamp-tracker.ts index 5d282d0d..c3b217f5 100644 --- a/src/services/analytics/timestamp-tracker.ts +++ b/src/services/analytics/timestamp-tracker.ts @@ -8,4 +8,5 @@ function createTimestampTracker() { }; } -export const timestampTracker = createTimestampTracker(); +export const latencyTimestampTracker = createTimestampTracker(); +export const interruptTimestampTracker = createTimestampTracker(); diff --git a/src/services/chat/index.ts b/src/services/chat/index.ts index c4992d23..d0bce303 100644 --- a/src/services/chat/index.ts +++ b/src/services/chat/index.ts @@ -1,5 +1,7 @@ import { PLAYGROUND_HEADER } from '$/config/consts'; -import { Agent, AgentsAPI, Chat, ChatMode } from '$/types'; +import type { Agent, AgentsAPI, Chat } from '$/types'; +import { ChatMode } from '$/types'; +import { isChatModeWithoutChat } from '$/utils/chat'; import { Analytics } from '../analytics/mixpanel'; export function getRequestHeaders(chatMode?: ChatMode): Record> { @@ -15,7 +17,7 @@ export async function createChat( chat?: Chat ) { try { - if (!chat && chatMode !== ChatMode.DirectPlayback) { + if (!chat && !isChatModeWithoutChat(chatMode)) { chat = await agentsApi.newChat(agent.id, { persist }, getRequestHeaders(chatMode)); analytics.track('agent-chat', { diff --git a/src/services/interrupt/index.ts b/src/services/interrupt/index.ts new file mode 100644 index 00000000..95d196c3 --- /dev/null +++ b/src/services/interrupt/index.ts @@ -0,0 +1,37 @@ +import { CreateStreamOptions, StreamEvents, StreamInterruptPayload, StreamType } from '$/types'; +import { StreamingManager } from '../streaming-manager'; + +export function validateInterrupt( + streamingManager: StreamingManager | undefined, + streamType: StreamType | undefined, + videoId: string | null +): void { + if (!streamingManager) { + throw new Error('Please connect to the agent first'); + } + + if (!streamingManager.interruptAvailable) { + throw new Error('Interrupt is not enabled for this stream'); + } + + if (streamType !== StreamType.Fluent) { + throw new Error('Interrupt only available for Fluent streams'); + } + + if (!videoId) { + throw new Error('No active video to interrupt'); + } +} + +export async function sendInterrupt( + streamingManager: StreamingManager, + videoId: string +): Promise { + const payload: StreamInterruptPayload = { + type: StreamEvents.StreamInterrupt, + videoId, + timestamp: Date.now(), + }; + + streamingManager.sendDataChannelMessage(JSON.stringify(payload)); +} diff --git a/src/services/socket-manager/message-queue.ts b/src/services/socket-manager/message-queue.ts index d4e97e65..6400e03e 100644 --- a/src/services/socket-manager/message-queue.ts +++ b/src/services/socket-manager/message-queue.ts @@ -87,7 +87,7 @@ export function createMessageEventQueue( } else if (completedEvents.includes(event)) { // Stream video event const streamEvent = event.split('/')[1]; - + if (failedEvents.includes(event)) { // Dont depend on video state change if stream failed analytics.track('agent-video', { ...props, event: streamEvent }); diff --git a/src/services/streaming-manager/index.ts b/src/services/streaming-manager/index.ts index 0bf841f0..478a3769 100644 --- a/src/services/streaming-manager/index.ts +++ b/src/services/streaming-manager/index.ts @@ -4,14 +4,13 @@ import { AgentActivityState, ConnectionState, CreateStreamOptions, - DataChannelSignalMap, PayloadType, + StreamEvents, StreamType, StreamingManagerOptions, StreamingState, VideoType, } from '$/types/index'; -import { ConnectivityState } from '$/types/stream/stream'; import { pollStats } from './stats/poll'; import { VideoRTCStatsReport } from './stats/report'; @@ -23,6 +22,9 @@ const actualRTCPC = ( (window as any).mozRTCPeerConnection ).bind(window); +type DataChannelPayload = string | Record; +type DataChannelMessageHandler = (subject: S, payload?: DataChannelPayload) => void; + function mapConnectionState(state: RTCIceConnectionState): ConnectionState { switch (state) { case 'connected': @@ -44,6 +46,18 @@ function mapConnectionState(state: RTCIceConnectionState): ConnectionState { } } +function parseDataChannelMessage(message: string): { subject: StreamEvents; data: DataChannelPayload } { + const [subject, rawData = ''] = message.split(/:(.+)/); + try { + const data = JSON.parse(rawData); + log('parsed data channel message', { subject, data }); + return { subject: subject as StreamEvents, data }; + } catch (e) { + log('Failed to parse data channel message, returning data as string', { subject, rawData, error: e }); + return { subject: subject as StreamEvents, data: rawData }; + } +} + function handleLegacyStreamState({ statsSignal, dataChannelSignal, @@ -119,7 +133,7 @@ function handleStreamState({ export async function createStreamingManager( agentId: string, agent: T, - { debug = false, callbacks, auth, baseURL = didApiUrl }: StreamingManagerOptions + { debug = false, callbacks, auth, baseURL = didApiUrl, analytics }: StreamingManagerOptions ) { _debug = debug; let srcObject: MediaStream | null = null; @@ -127,14 +141,21 @@ export async function createStreamingManager( let isDatachannelOpen = false; let dataChannelSignal: StreamingState = StreamingState.Stop; let statsSignal: StreamingState = StreamingState.Stop; - let connectivityState: ConnectivityState = ConnectivityState.Unknown; const { startConnection, sendStreamRequest, close, createStream, addIceCandidate } = agent.videoType === VideoType.Clip ? createClipApi(auth, baseURL, agentId, callbacks.onError) : createTalkApi(auth, baseURL, agentId, callbacks.onError); - const { id: streamIdFromServer, offer, ice_servers, session_id, fluent } = await createStream(agent); + const { + id: streamIdFromServer, + offer, + ice_servers, + session_id, + fluent, + interrupt_enabled: interruptAvailable, + } = await createStream(agent); + callbacks.onStreamCreated?.({ stream_id: streamIdFromServer, session_id: session_id as string, agent_id: agentId }); const peerConnection = new actualRTCPC({ iceServers: ice_servers }); const pcDataChannel = peerConnection.createDataChannel('JanusDataChannel'); @@ -143,6 +164,11 @@ export async function createStreamingManager( } const streamType = fluent ? StreamType.Fluent : StreamType.Legacy; + + analytics.enrich({ + 'stream-type': streamType, + }); + const warmup = agent.stream_warmup && !fluent; const getIsConnected = () => isConnected; @@ -167,7 +193,7 @@ export async function createStreamingManager( report, streamType, }), - state => callbacks.onConnectivityStateChange?.(connectivityState), + state => callbacks.onConnectivityStateChange?.(state), warmup ); @@ -201,18 +227,49 @@ export async function createStreamingManager( } }; - pcDataChannel.onmessage = (event: MessageEvent) => { - if (event.data in DataChannelSignalMap) { - dataChannelSignal = DataChannelSignalMap[event.data]; + const handleStreamVideoIdChange = (videoId: string | null) => { + callbacks.onVideoIdChange?.(videoId); + }; - handleStreamState({ - statsSignal: streamType === StreamType.Legacy ? statsSignal : undefined, - dataChannelSignal, - onVideoStateChange: callbacks.onVideoStateChange, - onAgentActivityStateChange: callbacks.onAgentActivityStateChange, - streamType, - }); + function handleStreamVideoEvent( + subject: StreamEvents.StreamStarted | StreamEvents.StreamDone, + payload?: DataChannelPayload + ) { + if (subject === StreamEvents.StreamStarted && typeof payload === 'object' && 'metadata' in payload) { + const metadata = payload.metadata as { videoId: string }; + handleStreamVideoIdChange(metadata.videoId); } + + if (subject === StreamEvents.StreamDone) { + handleStreamVideoIdChange(null); + } + + dataChannelSignal = subject === StreamEvents.StreamStarted ? StreamingState.Start : StreamingState.Stop; + + handleStreamState({ + statsSignal: streamType === StreamType.Legacy ? statsSignal : undefined, + dataChannelSignal, + onVideoStateChange: callbacks.onVideoStateChange, + onAgentActivityStateChange: callbacks.onAgentActivityStateChange, + streamType, + }); + } + + function handleStreamReadyEvent(_subject: StreamEvents.StreamReady, payload?: DataChannelPayload) { + const streamMetadata = typeof payload === 'string' ? payload : payload?.metadata; + streamMetadata && analytics.enrich({ streamMetadata }); + analytics.track('agent-chat', { event: 'ready' }); + } + + const dataChannelHandlers = { + [StreamEvents.StreamStarted]: handleStreamVideoEvent, + [StreamEvents.StreamDone]: handleStreamVideoEvent, + [StreamEvents.StreamReady]: handleStreamReadyEvent, + } satisfies Partial<{ [K in StreamEvents]: DataChannelMessageHandler }>; + + pcDataChannel.onmessage = (event: MessageEvent) => { + const { subject, data } = parseDataChannelMessage(event.data); + dataChannelHandlers[subject]?.(subject, data); }; peerConnection.oniceconnectionstatechange = () => { @@ -265,7 +322,6 @@ export async function createStreamingManager( if (peerConnection) { if (state === ConnectionState.New) { // Connection already closed - callbacks.onVideoStateChange?.(StreamingState.Stop); clearInterval(videoStatsInterval); return; } @@ -285,11 +341,29 @@ export async function createStreamingManager( log('Error on close stream connection', e); } - callbacks.onVideoStateChange?.(StreamingState.Stop); callbacks.onAgentActivityStateChange?.(AgentActivityState.Idle); clearInterval(videoStatsInterval); } }, + /** + * Method to send data channel messages to the server + */ + sendDataChannelMessage(payload: string) { + if (!isConnected || pcDataChannel.readyState !== 'open') { + log('Data channel is not ready for sending messages'); + callbacks.onError?.(new Error('Data channel is not ready for sending messages'), { + streamId: streamIdFromServer, + }); + return; + } + + try { + pcDataChannel.send(payload); + } catch (e: any) { + log('Error sending data channel message', e); + callbacks.onError?.(e, { streamId: streamIdFromServer }); + } + }, /** * Session identifier information, should be returned in the body of all streaming requests */ @@ -300,6 +374,7 @@ export async function createStreamingManager( streamId: streamIdFromServer, streamType, + interruptAvailable, }; } diff --git a/src/services/streaming-manager/stats/poll.ts b/src/services/streaming-manager/stats/poll.ts index 9cdb7416..ab907ec9 100644 --- a/src/services/streaming-manager/stats/poll.ts +++ b/src/services/streaming-manager/stats/poll.ts @@ -72,8 +72,8 @@ export function pollStats( avgJitterDelayInInterval < LOW_JITTER_TRESHOLD ? ConnectivityState.Strong : avgJitterDelayInInterval > HIGH_JITTER_TRESHOLD && currFreezeCount > 1 - ? ConnectivityState.Weak - : prevLowConnState; + ? ConnectivityState.Weak + : prevLowConnState; if (currLowConnState !== prevLowConnState) { onConnectivityStateChange?.(currLowConnState); @@ -104,6 +104,7 @@ export function pollStats( onConnected(); } + prevFreezeCount = freezeCount isStreaming = false; } } diff --git a/src/services/streaming-manager/stats/report.ts b/src/services/streaming-manager/stats/report.ts index abdc309e..220ac820 100644 --- a/src/services/streaming-manager/stats/report.ts +++ b/src/services/streaming-manager/stats/report.ts @@ -5,6 +5,9 @@ export interface VideoRTCStatsReport { webRTCStats: { anomalies: AnalyticsRTCStatsReport[]; aggregateReport: AnalyticsRTCStatsReport; + minRtt: number; + maxRtt: number; + avgRtt: number; minJitterDelayInInterval: number; maxJitterDelayInInterval: number; avgJitterDelayInInterval: number; @@ -71,13 +74,18 @@ function extractAnomalies(stats: AnalyticsRTCStatsReport[]): AnalyticsRTCStatsRe export function formatStats(stats: RTCStatsReport): SlimRTCStatsReport { let codec = ''; + let currRtt: number = 0; for (const report of stats.values()) { if (report && report.type === 'codec' && report.mimeType.startsWith('video')) { codec = report.mimeType.split('/')[1]; } + if (report && report.type === 'candidate-pair') { + currRtt = report.currentRoundTripTime; + } if (report && report.type === 'inbound-rtp' && report.kind === 'video') { return { codec, + rtt: currRtt, timestamp: report.timestamp, bytesReceived: report.bytesReceived, packetsReceived: report.packetsReceived, @@ -109,6 +117,7 @@ export function createVideoStatsReport( if (!previousStats) { return { timestamp: report.timestamp, + rtt: report.rtt, duration: 0, bytesReceived: report.bytesReceived, bitrate: (report.bytesReceived * 8) / (interval / 1000), @@ -129,6 +138,7 @@ export function createVideoStatsReport( return { timestamp: report.timestamp, duration: 0, + rtt: report.rtt, bytesReceived: report.bytesReceived - previousStats.bytesReceived, bitrate: ((report.bytesReceived - previousStats.bytesReceived) * 8) / (interval / 1000), packetsReceived: report.packetsReceived - previousStats.packetsReceived, @@ -150,6 +160,7 @@ export function createVideoStatsReport( return { timestamp: report.timestamp, duration: (interval * index) / 1000, + rtt: report.rtt, bytesReceived: report.bytesReceived - stats[index - 1].bytesReceived, bitrate: ((report.bytesReceived - stats[index - 1].bytesReceived) * 8) / (interval / 1000), packetsReceived: report.packetsReceived - stats[index - 1].packetsReceived, @@ -169,11 +180,15 @@ export function createVideoStatsReport( }); const anomalies = extractAnomalies(differentialReport); const lowFpsCount = anomalies.reduce((acc, report) => acc + (report.causes!.includes('low fps') ? 1 : 0), 0); - const avgJittersSamples = differentialReport.map(stat => stat.avgJitterDelayInInterval); + const avgJittersSamples = differentialReport.filter(stat => !!stat.avgJitterDelayInInterval).map(stat => stat.avgJitterDelayInInterval); + const avgRttSamples = differentialReport.filter(stat => !!stat.rtt).map(stat => stat.rtt); return { webRTCStats: { anomalies: anomalies, + minRtt: Math.min(...avgRttSamples), + avgRtt: average(avgRttSamples), + maxRtt: Math.max(...avgRttSamples), aggregateReport: createAggregateReport(stats[0], stats[stats.length - 1], lowFpsCount), minJitterDelayInInterval: Math.min(...avgJittersSamples), maxJitterDelayInInterval: Math.max(...avgJittersSamples), diff --git a/src/types/entities/agents/chat.ts b/src/types/entities/agents/chat.ts index f2b0c287..c626e636 100644 --- a/src/types/entities/agents/chat.ts +++ b/src/types/entities/agents/chat.ts @@ -31,6 +31,8 @@ export interface Message { created_at?: string; matches?: ChatResponse['matches']; context?: string; + videoId?: string; + interrupted?: boolean; } export interface ChatPayload { @@ -55,6 +57,7 @@ export enum ChatMode { Maintenance = 'Maintenance', Playground = 'Playground', DirectPlayback = 'DirectPlayback', + Off = 'Off', } export interface ChatResponse { @@ -63,6 +66,7 @@ export interface ChatResponse { matches?: IRetrivalMetadata[]; chatMode?: ChatMode; context?: string; + videoId?: string; } export interface Chat { @@ -76,3 +80,7 @@ export interface Chat { agent_id__modified_at: string; chat_mode?: ChatMode; } + +export interface Interrupt { + type: 'text' | 'audio' | 'click' | 'manual'; +} diff --git a/src/types/entities/agents/manager.ts b/src/types/entities/agents/manager.ts index 4142a168..b4204175 100644 --- a/src/types/entities/agents/manager.ts +++ b/src/types/entities/agents/manager.ts @@ -11,8 +11,9 @@ import { StreamingState, } from '$/types/stream'; import { SupportedStreamScript } from '$/types/stream-script'; +import type { ManagerCallbacks as StreamManagerCallbacks } from '../../stream/stream'; import { Agent } from './agent'; -import { ChatMode, ChatResponse, Message, RatingEntity } from './chat'; +import { ChatMode, ChatResponse, Interrupt, Message, RatingEntity } from './chat'; /** * Types of events provided in Chat Progress Callback @@ -74,13 +75,11 @@ interface ManagerCallbacks { * @param chatId - id of the new chat */ onNewChat?(chatId: string): void; - /** * Optional callback function that will be triggered each time the chat mode changes * @param mode - ChatMode */ onModeChange?(mode: ChatMode): void; - /** * Optional callback function that will be triggered each time the user internet connectivity state change by realtime estimated bitrate * @param state - ConnectivityState @@ -90,12 +89,16 @@ interface ManagerCallbacks { * Optional callback function that will be triggered on fetch request errors */ onError?: (error: Error, errorData?: object) => void; - /** * Optional callback function that will be triggered each time the agent activity state changes * @param state - AgentActivityState */ onAgentActivityStateChange?(state: AgentActivityState): void; + /** + * Optional callback function that will be triggered each time a new stream is created + * @param stream - object containing stream_id, session_id and agent_id + */ + onStreamCreated?: StreamManagerCallbacks['onStreamCreated']; } interface StreamOptions { @@ -166,6 +169,12 @@ export interface AgentManager { * Get the current stream type of the agent */ getStreamType: () => StreamType | undefined; + + /** + * Get if the stream supports interrupt + */ + getIsInterruptAvailable: () => boolean; + /** * Array of starter messages that will be sent to the agent when the chat starts */ @@ -220,4 +229,10 @@ export interface AgentManager { * @param properties flat json object with properties that will be added to analytics events fired from the sdk */ enrichAnalytics: (properties: Record) => void; + + /** + * Method to interrupt the current video stream + * Only available for Fluent streams and when there's an active video to interrupt + */ + interrupt: (interrupt: Interrupt) => void; } diff --git a/src/types/index.ts b/src/types/index.ts index 37e40862..a84963f8 100644 --- a/src/types/index.ts +++ b/src/types/index.ts @@ -1,6 +1,6 @@ -export * from './stream-script'; export * from './auth'; export * from './entities'; export * from './stream'; +export * from './stream-script'; export * from './voice/stt'; export * from './voice/tts'; diff --git a/src/types/stream/rtc.ts b/src/types/stream/rtc.ts index 879ef2cf..e74db024 100644 --- a/src/types/stream/rtc.ts +++ b/src/types/stream/rtc.ts @@ -38,6 +38,7 @@ export interface ICreateStreamRequestResponse extends StickyRequest { offer: any; ice_servers: IceServer[]; fluent?: boolean; + interrupt_enabled?: boolean; } export interface IceCandidate { @@ -67,4 +68,5 @@ export interface Status { export interface SendStreamPayloadResponse extends Status, StickyRequest { duration: number; + video_id: string; } diff --git a/src/types/stream/stream.ts b/src/types/stream/stream.ts index 7a169c51..e135f13b 100644 --- a/src/types/stream/stream.ts +++ b/src/types/stream/stream.ts @@ -23,11 +23,6 @@ export enum AgentActivityState { Talking = 'TALKING', } -export const DataChannelSignalMap: Record = { - 'stream/started': StreamingState.Start, - 'stream/done': StreamingState.Stop, -}; - export enum StreamEvents { ChatAnswer = 'chat/answer', ChatPartial = 'chat/partial', @@ -36,6 +31,7 @@ export enum StreamEvents { StreamFailed = 'stream/error', StreamReady = 'stream/ready', StreamCreated = 'stream/created', + StreamInterrupt = 'stream/interrupt', StreamVideoCreated = 'stream-video/started', StreamVideoDone = 'stream-video/done', StreamVideoError = 'stream-video/error', @@ -65,6 +61,8 @@ export interface ManagerCallbacks { onError?: (error: Error, errorData: object) => void; onConnectivityStateChange?: (state: ConnectivityState) => void; onAgentActivityStateChange?: (state: AgentActivityState) => void; + onVideoIdChange?: (videoId: string | null) => void; + onStreamCreated?: (stream: { stream_id: string; session_id: string; agent_id: string }) => void; } export type ManagerCallbackKeys = keyof ManagerCallbacks; @@ -109,6 +107,7 @@ export interface StreamingManagerOptions { export interface SlimRTCStatsReport { index: number; codec: string; + rtt: number; duration?: number; bitrate?: number; timestamp: any; @@ -147,3 +146,9 @@ export interface AnalyticsRTCStatsReport { lowFpsCount?: number; causes?: string[]; } + +export interface StreamInterruptPayload { + type: StreamEvents.StreamInterrupt; + videoId: string; + timestamp: number; +} diff --git a/src/utils/analytics.ts b/src/utils/analytics.ts index acd30637..d90a4449 100644 --- a/src/utils/analytics.ts +++ b/src/utils/analytics.ts @@ -33,6 +33,25 @@ export function getAnalyticsInfo(agent: Agent) { }; } +export function getAgentInfo(agent: Agent) { + const promptCustomization = agent.llm?.prompt_customization; + + return { + agentType: getAgentType(agent.presenter), + owner_id: agent.owner_id ?? '', + promptVersion: agent.llm?.prompt_version, + behavior: { + role: promptCustomization?.role, + personality: promptCustomization?.personality, + instructions: agent.llm?.instructions, + }, + temperature: agent.llm?.temperature, + knowledgeSource: promptCustomization?.knowledge_source, + starterQuestionsCount: agent.knowledge?.starter_message?.length, + topicsToAvoid: promptCustomization?.topics_to_avoid, + maxResponseLength: promptCustomization?.max_response_length, + }; +} export const sumFunc = (numbers: number[]) => numbers.reduce((total, aNumber) => total + aNumber, 0); export const average = (numbers: number[]) => sumFunc(numbers) / numbers.length; diff --git a/src/utils/chat.ts b/src/utils/chat.ts new file mode 100644 index 00000000..cb722025 --- /dev/null +++ b/src/utils/chat.ts @@ -0,0 +1,7 @@ +import { ChatMode } from '$/types'; + +export const isTextualChat = (chatMode: ChatMode) => + [ChatMode.TextOnly, ChatMode.Playground, ChatMode.Maintenance].includes(chatMode); + +export const isChatModeWithoutChat = (chatMode: ChatMode | undefined) => + chatMode && [ChatMode.DirectPlayback, ChatMode.Off].includes(chatMode); diff --git a/src/utils/index.ts b/src/utils/index.ts index a8bbcfc6..3bc1dc9a 100644 --- a/src/utils/index.ts +++ b/src/utils/index.ts @@ -1,2 +1,7 @@ export const sleep = (ms: number) => new Promise(resolve => setTimeout(resolve, ms)); -export const getRandom = () => Math.random().toString(16).slice(2); +export const getRandom = (length: number = 16) => { + const arr = new Uint8Array(length); + window.crypto.getRandomValues(arr); + return Array.from(arr, byte => byte.toString(16).padStart(2, '0')).join('').slice(0,13); +} + diff --git a/yarn.lock b/yarn.lock index 056a5ad0..4cfabeb9 100644 --- a/yarn.lock +++ b/yarn.lock @@ -657,12 +657,12 @@ resolved "https://registry.npmjs.org/@types/estree/-/estree-1.0.5.tgz#a6ce3e556e00fd9895dd872dd172ad0d4bd687f4" integrity sha512-/kYRxGDLWzHOB7q+wtSUQlFrtcdUccpfy+X+9iMBpHK8QLLhx2wIPYuS5DYtR9Wa/YlZAbIovy7qVdB1Aq6Lyw== -"@types/node@^20.11.24": - version "20.11.25" - resolved "https://registry.npmjs.org/@types/node/-/node-20.11.25.tgz#0f50d62f274e54dd7a49f7704cc16bfbcccaf49f" - integrity sha512-TBHyJxk2b7HceLVGFcpAUjsa5zIdsPWlR6XHfyGzd0SFu+/NFgQgMAl96MSDZgQDvJAvV6BKsFOrt6zIL09JDw== +"@types/node@^22.15.0": + version "22.15.31" + resolved "https://registry.yarnpkg.com/@types/node/-/node-22.15.31.tgz#454f11e2061150135c8353d7f3b3b1823fca9f3f" + integrity sha512-jnVe5ULKl6tijxUhvQeNbQG/84fHfg+yMak02cT8QVhBx/F05rAVxCGBYYTh2EKz22D6JF5ktXuNwdx7b9iEGw== dependencies: - undici-types "~5.26.4" + undici-types "~6.21.0" "@volar/language-core@1.11.1", "@volar/language-core@~1.11.1": version "1.11.1" @@ -1438,16 +1438,8 @@ string-argv@~0.3.1: resolved "https://registry.npmjs.org/string-argv/-/string-argv-0.3.2.tgz#2b6d0ef24b656274d957d54e0a4bbf6153dc02b6" integrity sha512-aqD2Q0144Z+/RqG52NeHEkZauTAUWJO8c6yTftGJKO3Tja5tUgIfmIl6kExvhtxSDP7fXB6DvzkfMpCd/F3G+Q== -"string-width-cjs@npm:string-width@^4.2.0": - version "4.2.3" - resolved "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010" - integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g== - dependencies: - emoji-regex "^8.0.0" - is-fullwidth-code-point "^3.0.0" - strip-ansi "^6.0.1" - -string-width@^4.1.0: +"string-width-cjs@npm:string-width@^4.2.0", string-width@^4.1.0: + name string-width-cjs version "4.2.3" resolved "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010" integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g== @@ -1465,14 +1457,8 @@ string-width@^5.0.1, string-width@^5.1.2: emoji-regex "^9.2.2" strip-ansi "^7.0.1" -"strip-ansi-cjs@npm:strip-ansi@^6.0.1": - version "6.0.1" - resolved "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9" - integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A== - dependencies: - ansi-regex "^5.0.1" - -strip-ansi@^6.0.0, strip-ansi@^6.0.1: +"strip-ansi-cjs@npm:strip-ansi@^6.0.1", strip-ansi@^6.0.0, strip-ansi@^6.0.1: + name strip-ansi-cjs version "6.0.1" resolved "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9" integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A== @@ -1528,10 +1514,10 @@ typescript@^5.3.3: resolved "https://registry.npmjs.org/typescript/-/typescript-5.4.2.tgz#0ae9cebcfae970718474fe0da2c090cad6577372" integrity sha512-+2/g0Fds1ERlP6JsakQQDXjZdZMM+rqpamFZJEKh4kwTIn3iDkgKtby0CeNd5ATNZ4Ry1ax15TMx0W2V+miizQ== -undici-types@~5.26.4: - version "5.26.5" - resolved "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz#bcd539893d00b56e964fd2657a4866b221a65617" - integrity sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA== +undici-types@~6.21.0: + version "6.21.0" + resolved "https://registry.yarnpkg.com/undici-types/-/undici-types-6.21.0.tgz#691d00af3909be93a7faa13be61b3a5b50ef12cb" + integrity sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ== universalify@^0.1.0: version "0.1.2"