Feat support 360 cam

.changeset/stale-games-float.md

@@ -0,0 +1,5 @@
+---
+'@viamrobotics/test-widgets': minor
+---
+
+support 360 camera images with special XMP metadata

src/lib/components/widgets/camera/__tests__/get-xmp-json-from-image.spec.ts

@@ -0,0 +1,51 @@
+import { describe, expect, it } from 'vitest'
+
+import { getXmpJsonFromImageBytes } from '../get-xmp-json-from-image'
+
+const buildJpegWithXmp = (xmpXml: string): Uint8Array => {
+	const identifier = new TextEncoder().encode('http://ns.adobe.com/xap/1.0/\0')
+	const xmpBytes = new TextEncoder().encode(xmpXml)
+	const app1Length = 2 + identifier.length + xmpBytes.length
+	const app1Segment = new Uint8Array(2 + app1Length)
+
+	app1Segment[0] = 0xff
+	app1Segment[1] = 0xe1
+	app1Segment[2] = (app1Length >> 8) & 0xff
+	app1Segment[3] = app1Length & 0xff
+	app1Segment.set(identifier, 4)
+	app1Segment.set(xmpBytes, 4 + identifier.length)
+
+	return new Uint8Array([0xff, 0xd8, ...app1Segment, 0xff, 0xd9])
+}
+
+describe('getXmpJsonFromImageBytes', () => {
+	it('returns null when the image has no XMP segment', () => {
+		const jpeg = new Uint8Array([0xff, 0xd8, 0xff, 0xd9])
+
+		expect(getXmpJsonFromImageBytes(jpeg)).toBeNull()
+	})
+
+	it('parses viam:is360 from a JPEG XMP packet', () => {
+		const xmpXml = `<?xpacket begin="" id="W5M0MpCehiHzreSzNTczkc9d"?>
+<x:xmpmeta xmlns:x="adobe:ns:meta/">
+  <rdf:RDF xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#">
+    <rdf:Description
+      rdf:about=""
+      xmlns:viam="https://www.viam.com/"
+      viam:is360="true"
+    />
+  </rdf:RDF>
+</x:xmpmeta>
+<?xpacket end="w"?>`
+
+		const jpeg = buildJpegWithXmp(xmpXml)
+		const xmpJson = getXmpJsonFromImageBytes(jpeg, 'image/jpeg')
+
+		expect(xmpJson).toEqual({
+			'viam:is360': 'true',
+			'xmlns:rdf': 'http://www.w3.org/1999/02/22-rdf-syntax-ns#',
+			'xmlns:viam': 'https://www.viam.com/',
+			'xmlns:x': 'adobe:ns:meta/',
+		})
+	})
+})

src/lib/components/widgets/camera/camera.svelte

@@ -1,4 +1,5 @@
 <script lang="ts">
+	import { Canvas } from '@threlte/core'
 	import { Button, Label, Select, Switch, ToggleButtons } from '@viamrobotics/prime-core'
 	import { CameraClient } from '@viamrobotics/sdk'
 	import { createResourceClient, createResourceQuery } from '@viamrobotics/svelte-sdk'
@@ -15,8 +16,10 @@
 	import PCDWidget from '../pcd/pcd-widget.svelte'
 	import ExportScreenshot from './export-screenshot.svelte'
 	import { getSourceNames } from './get-source-names'
+	import { getXmpJsonFromImageBytes, type XmpJson } from './get-xmp-json-from-image'
 	import LiveOrPollingVideo from './live-or-polling-video.svelte'
 	import PictureInPictureButton from './picture-in-picture-button.svelte'
+	import ThreeSixtyCameraView from './three-sixty-camera-view.svelte'
 
 	interface Props {
 		partID: string
@@ -41,6 +44,7 @@
 	let isShowingPointcloud = $state(false)
 	let selectedSource = $state('')
 	let sourceNames = $state<string[]>([])
+	let displayAs360 = $state(false)
 
 	const { addImageToDataset } = useAddImageToDataset()
 	const setIsShowingPointcloud = (event: CustomEvent<boolean>) => {
@@ -73,6 +77,20 @@
 		}
 	})
 
+	const xmpJson = $derived.by((): XmpJson | null => {
+		const imageRecord = imageQuery.data?.images?.[0]
+		const image = imageRecord?.image
+		if (!image) {
+			return null
+		}
+
+		return getXmpJsonFromImageBytes(new Uint8Array(image), imageRecord.mimeType)
+	})
+
+	const is360EnabledImage = $derived.by((): boolean => {
+		return xmpJson?.['viam:is360'] === 'true'
+	})
+
 	const pointcloudQuery = createResourceQuery(client, 'getPointCloud', () => ({
 		enabled: isShowingPointcloud,
 		refetchInterval:
@@ -135,22 +153,41 @@
 						</Select>
 					</Label>
 				{/if}
+				{#if is360EnabledImage}
+					<Label>
+						Display as 360°
+						<ToggleButtons
+							slot="input"
+							options={['On', 'Off']}
+							selected={displayAs360 ? 'On' : 'Off'}
+							on:input={(event) => {
+								displayAs360 = event.detail === 'On'
+							}}
+						/>
+					</Label>
+				{/if}
 			</div>
 		{/if}
 		<div class="flex h-full w-full gap-4 p-4">
 			<div class="grow">
 				{#if isPlaying}
-					<LiveOrPollingVideo
-						{partID}
-						{resourceName}
-						showResolutionOptions
-						isLive={refetchInterval.current === RefetchIntervals.LIVE}
-						data={imageQuery.data}
-						error={imageQuery.error}
-						isLoading={imageQuery.isLoading}
-						refetch={imageQuery.refetch}
-						showMousePositionTooltip={mousePostionTooltip === 'On'}
-					/>
+					{#if displayAs360}
+						<Canvas>
+							<ThreeSixtyCameraView data={imageQuery.data} />
+						</Canvas>
+					{:else}
+						<LiveOrPollingVideo
+							{partID}
+							{resourceName}
+							showResolutionOptions
+							isLive={refetchInterval.current === RefetchIntervals.LIVE}
+							data={imageQuery.data}
+							error={imageQuery.error}
+							isLoading={imageQuery.isLoading}
+							refetch={imageQuery.refetch}
+							showMousePositionTooltip={mousePostionTooltip === 'On'}
+						/>
+					{/if}
 				{:else}
 					<div class="bg-medium flex h-64 w-80 items-center justify-center">
 						<Button

src/lib/components/widgets/camera/get-xmp-json-from-image.ts

@@ -0,0 +1,175 @@
+const JPEG_SOI = 0xffd8
+const JPEG_MARKER_PREFIX = 0xff
+const JPEG_APP1_MARKER = 0xe1
+const JPEG_EOI_MARKER = 0xd9
+
+const XMP_IDENTIFIER = 'http://ns.adobe.com/xap/1.0/\0'
+
+export type XmpJson = Record<string, unknown>
+
+/** Extract XMP metadata from image bytes and return it as a plain object. */
+export const getXmpJsonFromImageBytes = (image: Uint8Array, mimeType?: string): XmpJson | null => {
+	if (mimeType?.includes('png')) {
+		return getXmpJsonFromPng(image)
+	}
+
+	return getXmpJsonFromJpeg(image)
+}
+
+const getXmpJsonFromJpeg = (image: Uint8Array): XmpJson | null => {
+	if (image.length < 4 || readUint16(image, 0) !== JPEG_SOI) {
+		return null
+	}
+
+	const xmpXml = readXmpXmlFromJpeg(image)
+	if (!xmpXml) {
+		return null
+	}
+
+	return xmpXmlToJson(xmpXml)
+}
+
+const readXmpXmlFromJpeg = (image: Uint8Array): string | null => {
+	const identifier = new TextEncoder().encode(XMP_IDENTIFIER)
+	let offset = 2
+
+	while (offset + 4 < image.length) {
+		if (image[offset] !== JPEG_MARKER_PREFIX) {
+			break
+		}
+
+		const marker = image[offset + 1]
+		if (marker === undefined) {
+			break
+		}
+
+		if (marker === JPEG_EOI_MARKER) {
+			break
+		}
+
+		const segmentLength = readUint16(image, offset + 2)
+		if (segmentLength < 2 || offset + 2 + segmentLength > image.length) {
+			break
+		}
+
+		if (marker === JPEG_APP1_MARKER) {
+			const segmentData = image.subarray(offset + 4, offset + 2 + segmentLength)
+			if (startsWith(segmentData, identifier)) {
+				const xmpBytes = segmentData.subarray(identifier.length)
+				return new TextDecoder('utf-8').decode(xmpBytes)
+			}
+		}
+
+		offset += 2 + segmentLength
+	}
+
+	return null
+}
+
+const getXmpJsonFromPng = (image: Uint8Array): XmpJson | null => {
+	const signature = [0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]
+	if (image.length < signature.length || !startsWith(image, new Uint8Array(signature))) {
+		return null
+	}
+
+	let offset = signature.length
+
+	while (offset + 12 <= image.length) {
+		const chunkLength = readUint32(image, offset)
+		const chunkType = new TextDecoder('ascii').decode(image.subarray(offset + 4, offset + 8))
+
+		if (chunkType === 'iTXt') {
+			const chunkData = image.subarray(offset + 8, offset + 8 + chunkLength)
+			const xmpXml = readXmpXmlFromPngITXtChunk(chunkData)
+			if (xmpXml) {
+				return xmpXmlToJson(xmpXml)
+			}
+		}
+
+		offset += 12 + chunkLength
+	}
+
+	return null
+}
+
+const readXmpXmlFromPngITXtChunk = (chunkData: Uint8Array): string | null => {
+	let index = 0
+
+	const readNullTerminated = (): string | null => {
+		const start = index
+		while (index < chunkData.length && chunkData[index] !== 0) {
+			index += 1
+		}
+		if (index >= chunkData.length) {
+			return null
+		}
+		const value = new TextDecoder('utf-8').decode(chunkData.subarray(start, index))
+		index += 1
+		return value
+	}
+
+	const keyword = readNullTerminated()
+	if (keyword !== 'XML:com.adobe.xmp') {
+		return null
+	}
+
+	readNullTerminated() // compression flag
+	readNullTerminated() // compression method
+	readNullTerminated() // language tag
+	readNullTerminated() // translated keyword
+
+	if (index >= chunkData.length) {
+		return null
+	}
+
+	return new TextDecoder('utf-8').decode(chunkData.subarray(index))
+}
+
+const xmpXmlToJson = (xmpXml: string): XmpJson | null => {
+	const doc = new DOMParser().parseFromString(xmpXml, 'application/xml')
+	if (doc.querySelector('parsererror')) {
+		return null
+	}
+
+	const json: XmpJson = {}
+
+	for (const element of doc.querySelectorAll('*')) {
+		for (const attribute of element.attributes) {
+			if (attribute.localName === 'about' || attribute.name === 'rdf:about') {
+				continue
+			}
+
+			json[attribute.name] = attribute.value
+		}
+
+		if (element.childElementCount === 0 && element.textContent?.trim()) {
+			const key = element.prefix ? `${element.prefix}:${element.localName}` : element.localName
+			json[key] = element.textContent.trim()
+		}
+	}
+
+	return Object.keys(json).length > 0 ? json : null
+}
+
+const readUint16 = (bytes: Uint8Array, offset: number): number =>
+	(bytes[offset]! << 8) | bytes[offset + 1]!
+
+const readUint32 = (bytes: Uint8Array, offset: number): number =>
+	(bytes[offset]! << 24) |
+	(bytes[offset + 1]! << 16) |
+	(bytes[offset + 2]! << 8) |
+	bytes[offset + 3]!
+
+const startsWith = (bytes: Uint8Array, prefix: Uint8Array): boolean => {
+	if (bytes.length < prefix.length) {
+		return false
+	}
+
+	for (let index = 0; index < prefix.length; index += 1) {
+		if (bytes[index] !== prefix[index]) {
+			return false
+		}
+	}
+
+	return true
+}