From f2b2b456a4ca68cd730b0d9a2fe793fa0d1786a8 Mon Sep 17 00:00:00 2001 From: KarlS Date: Wed, 8 Apr 2026 18:25:39 -0700 Subject: [PATCH 1/3] Add support for image source/sink nodes --- api/datatype.proto | 21 +++++++++++++++++++++ api/device.proto | 1 + api/node.proto | 8 ++++++++ api/nodes/image_sink.proto | 14 ++++++++++++++ api/nodes/image_source.proto | 18 ++++++++++++++++++ 5 files changed, 62 insertions(+) create mode 100644 api/nodes/image_sink.proto create mode 100644 api/nodes/image_source.proto diff --git a/api/datatype.proto b/api/datatype.proto index 4619d4a..2f41b94 100644 --- a/api/datatype.proto +++ b/api/datatype.proto @@ -107,3 +107,24 @@ message AnnotatedTensor { // e.g. pipeline latencies map metadata = 2; } + +enum PixelFormat { + kPixelFormatUnknown = 0; + kYUV420_888 = 1; + kRGB888 = 2; + kRGBA8888 = 3; + kGrayscale8 = 4; + kRAW10 = 5; + kRAW16 = 6; + kNV12 = 7; + kNV21 = 8; +} + +message ImageFrame { + uint32 width = 1; + uint32 height = 2; + PixelFormat format = 3; + uint64 sequence_number = 4; + uint64 timestamp_ns = 5; + bytes data = 6; +} diff --git a/api/device.proto b/api/device.proto index 8883fe2..477b509 100644 --- a/api/device.proto +++ b/api/device.proto @@ -13,6 +13,7 @@ message Peripheral { kElectricalStimulation = 2; kOpticalStimulation = 3; kSpikeSource = 4; + kCamera = 5; } string name = 1; string vendor = 2; diff --git a/api/node.proto b/api/node.proto index f3369c6..ec5cab8 100644 --- a/api/node.proto +++ b/api/node.proto @@ -11,6 +11,8 @@ import "api/nodes/disk_writer.proto"; import "api/nodes/spike_source.proto"; import "api/nodes/spike_binner.proto"; import "api/nodes/application.proto"; +import "api/nodes/image_source.proto"; +import "api/nodes/image_sink.proto"; enum NodeType { kNodeTypeUnknown = 0; @@ -27,6 +29,8 @@ enum NodeType { kDiskWriter = 9; kSpikeBinner = 10; kApplication = 11; + kImageSource = 12; + kImageSink = 13; } message NodeConfig { @@ -47,6 +51,8 @@ message NodeConfig { SpikeSourceConfig spike_source = 12; SpikeBinnerConfig spike_binner = 13; ApplicationNodeConfig application = 14; + ImageSourceConfig image_source = 15; + ImageSinkConfig image_sink = 16; } } @@ -63,6 +69,8 @@ message NodeStatus { ApplicationNodeStatus application = 7; OpticalStimulationStatus optical_stimulation = 8; DiskWriterStatus disk_writer = 10; + ImageSourceStatus image_source = 11; + ImageSinkStatus image_sink = 12; } } diff --git a/api/nodes/image_sink.proto b/api/nodes/image_sink.proto new file mode 100644 index 0000000..470185a --- /dev/null +++ b/api/nodes/image_sink.proto @@ -0,0 +1,14 @@ +syntax = "proto3"; + +package synapse; + +message ImageSinkConfig { + uint32 peripheral_id = 1; + uint32 frame_rate_hz = 2; +} + +message ImageSinkStatus { + uint64 frames_exported = 1; + float actual_rate_hz = 2; + float latency_ms = 3; +} diff --git a/api/nodes/image_source.proto b/api/nodes/image_source.proto new file mode 100644 index 0000000..ab02fb0 --- /dev/null +++ b/api/nodes/image_source.proto @@ -0,0 +1,18 @@ +syntax = "proto3"; + +package synapse; + +import "api/datatype.proto"; + +message ImageSourceConfig { + uint32 peripheral_id = 1; + uint32 width = 2; + uint32 height = 3; + PixelFormat format = 4; + uint32 frame_rate_hz = 5; +} + +message ImageSourceStatus { + uint64 frames_produced = 1; + uint64 frames_dropped = 2; +} From 32a6d22c351690f592022f76c5200554428762eb Mon Sep 17 00:00:00 2001 From: KarlS Date: Mon, 20 Apr 2026 15:18:46 -0700 Subject: [PATCH 2/3] Rename ImageSource node to Camera and remove ImageSink Display peripherals should be handled by user-defined ApplicationNode logic rather than a dedicated Synapse sink node. --- api/node.proto | 12 ++++-------- api/nodes/{image_source.proto => camera.proto} | 4 ++-- api/nodes/image_sink.proto | 14 -------------- 3 files changed, 6 insertions(+), 24 deletions(-) rename api/nodes/{image_source.proto => camera.proto} (81%) delete mode 100644 api/nodes/image_sink.proto diff --git a/api/node.proto b/api/node.proto index ec5cab8..2eefb95 100644 --- a/api/node.proto +++ b/api/node.proto @@ -11,8 +11,7 @@ import "api/nodes/disk_writer.proto"; import "api/nodes/spike_source.proto"; import "api/nodes/spike_binner.proto"; import "api/nodes/application.proto"; -import "api/nodes/image_source.proto"; -import "api/nodes/image_sink.proto"; +import "api/nodes/camera.proto"; enum NodeType { kNodeTypeUnknown = 0; @@ -29,8 +28,7 @@ enum NodeType { kDiskWriter = 9; kSpikeBinner = 10; kApplication = 11; - kImageSource = 12; - kImageSink = 13; + kCamera = 12; } message NodeConfig { @@ -51,8 +49,7 @@ message NodeConfig { SpikeSourceConfig spike_source = 12; SpikeBinnerConfig spike_binner = 13; ApplicationNodeConfig application = 14; - ImageSourceConfig image_source = 15; - ImageSinkConfig image_sink = 16; + CameraConfig camera = 15; } } @@ -69,8 +66,7 @@ message NodeStatus { ApplicationNodeStatus application = 7; OpticalStimulationStatus optical_stimulation = 8; DiskWriterStatus disk_writer = 10; - ImageSourceStatus image_source = 11; - ImageSinkStatus image_sink = 12; + CameraStatus camera = 11; } } diff --git a/api/nodes/image_source.proto b/api/nodes/camera.proto similarity index 81% rename from api/nodes/image_source.proto rename to api/nodes/camera.proto index ab02fb0..42ba9c4 100644 --- a/api/nodes/image_source.proto +++ b/api/nodes/camera.proto @@ -4,7 +4,7 @@ package synapse; import "api/datatype.proto"; -message ImageSourceConfig { +message CameraConfig { uint32 peripheral_id = 1; uint32 width = 2; uint32 height = 3; @@ -12,7 +12,7 @@ message ImageSourceConfig { uint32 frame_rate_hz = 5; } -message ImageSourceStatus { +message CameraStatus { uint64 frames_produced = 1; uint64 frames_dropped = 2; } diff --git a/api/nodes/image_sink.proto b/api/nodes/image_sink.proto deleted file mode 100644 index 470185a..0000000 --- a/api/nodes/image_sink.proto +++ /dev/null @@ -1,14 +0,0 @@ -syntax = "proto3"; - -package synapse; - -message ImageSinkConfig { - uint32 peripheral_id = 1; - uint32 frame_rate_hz = 2; -} - -message ImageSinkStatus { - uint64 frames_exported = 1; - float actual_rate_hz = 2; - float latency_ms = 3; -} From 8a09b997a269d948c747cdeb1bc63ce6a5b3457a Mon Sep 17 00:00:00 2001 From: KarlS Date: Tue, 28 Apr 2026 12:26:56 -0700 Subject: [PATCH 3/3] added frame_rate_hz and unix timestamp into ImageFrame message --- api/datatype.proto | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/api/datatype.proto b/api/datatype.proto index 2f41b94..782913e 100644 --- a/api/datatype.proto +++ b/api/datatype.proto @@ -124,7 +124,9 @@ message ImageFrame { uint32 width = 1; uint32 height = 2; PixelFormat format = 3; - uint64 sequence_number = 4; - uint64 timestamp_ns = 5; - bytes data = 6; + uint32 sample_rate_hz = 4; + uint64 sequence_number = 5; + uint64 timestamp_ns = 6; + uint64 unix_timestamp_ns = 7; + bytes data = 8; }