diff --git a/setup.py b/setup.py index 88324ed..ffe30fc 100644 --- a/setup.py +++ b/setup.py @@ -26,7 +26,7 @@ setup( name="science-synapse", - version="2.2.10", + version="2.2.12", description="Client library and CLI for the Synapse API", author="Science Team", author_email="team@science.xyz", diff --git a/synapse-api b/synapse-api index 60bdb2f..de75a2c 160000 --- a/synapse-api +++ b/synapse-api @@ -1 +1 @@ -Subproject commit 60bdb2fdc8af14f19cef3b324232f1125ce33ead +Subproject commit de75a2cb1d9159994bf9a0f2d2c601757c400b09 diff --git a/synapse/cli/offline_hdf5_plotter.py b/synapse/cli/offline_hdf5_plotter.py index 7bba75d..a1db238 100644 --- a/synapse/cli/offline_hdf5_plotter.py +++ b/synapse/cli/offline_hdf5_plotter.py @@ -117,11 +117,60 @@ def load_h5_data(data_file, console, time_range=None): # Get frame data info frame_data = f["/acquisition/ElectricalSeries"] total_samples = len(frame_data) - samples_per_channel = total_samples // number_of_channels - console.print( - f"Total duration: {samples_per_channel / sample_rate:.2f} seconds" - ) + timestamps_ns = f["/acquisition/timestamp_ns"] + total_duration = timestamps_ns[-1] - timestamps_ns[0] + console.print(f"Total duration: {total_duration / 1e9:.2f} seconds") + + sequence_number = f["/acquisition/sequence_number"] + + # Check for gaps in sequence numbers with progress bar + gaps_found = [] + + with Progress(console=console) as progress: + task = progress.add_task( + "Checking sequence numbers...", total=len(sequence_number) + ) + + # Convert to numpy array for faster processing + seq_array = sequence_number[:] + + # Start with the first sequence number as the expected value + expected_sequence = seq_array[0] + + for i in range(len(seq_array)): + current_seq = seq_array[i] + if current_seq != expected_sequence: + gaps_found.append( + { + "index": i, + "expected": expected_sequence, + "actual": current_seq, + "gap_size": current_seq - expected_sequence, + } + ) + expected_sequence = current_seq + 1 + progress.update(task, advance=1) + + if gaps_found: + console.print( + f"[red]⚠️ Found {len(gaps_found)} gaps in sequence numbers:[/red]" + ) + for gap in gaps_found: + console.print( + f"[red] Index {gap['index']}: Expected {gap['expected']}, got {gap['actual']} (gap of {gap['gap_size']})[/red]" + ) + else: + console.print( + f"[green]✓ Sequence numbers are continuous ({seq_array[0]} to {seq_array[-1]})[/green]" + ) + + # Print first few and last few sequence numbers for reference + if len(seq_array) > 10: + console.print(f"First 5 sequence numbers: {seq_array[:5].tolist()}") + console.print(f"Last 5 sequence numbers: {seq_array[-5:].tolist()}") + else: + console.print(f"All sequence numbers: {seq_array[:].tolist()}") # Determine time range to load start_index = 0 diff --git a/synapse/cli/streaming.py b/synapse/cli/streaming.py index d1e51ec..ad3df6d 100644 --- a/synapse/cli/streaming.py +++ b/synapse/cli/streaming.py @@ -161,7 +161,16 @@ def __init__(self, output_dir: str): # Create datasets with chunking and compression for better performance self.timestamp_dataset = self.file.create_dataset( - "/acquisition/timestamp", + "/acquisition/timestamp_ns", + shape=(0,), + maxshape=(None,), + dtype="uint64", + chunks=True, + compression="gzip", + compression_opts=1, # Fast compression + ) + self.unix_timestamp_dataset = self.file.create_dataset( + "/acquisition/unix_timestamp_ns", shape=(0,), maxshape=(None,), dtype="uint64", @@ -354,6 +363,7 @@ def _write_buffer(self, frame_buffer: list): new_frame_size = current_frame_size + (num_frames * samples_per_frame) self.timestamp_dataset.resize(new_timestamp_size, axis=0) + self.unix_timestamp_dataset.resize(new_timestamp_size, axis=0) self.sequence_dataset.resize(new_timestamp_size, axis=0) self.frame_data_dataset.resize(new_frame_size, axis=0) @@ -361,16 +371,21 @@ def _write_buffer(self, frame_buffer: list): timestamps = [] sequences = [] all_frame_data = [] + unix_timestamps = [] for frame in frame_buffer: timestamps.append(frame.timestamp_ns) sequences.append(frame.sequence_number) all_frame_data.extend(frame.frame_data) + unix_timestamps.append(frame.unix_timestamp_ns) # Write all data at once (more efficient) self.timestamp_dataset[current_timestamp_size:new_timestamp_size] = ( timestamps ) + self.unix_timestamp_dataset[current_timestamp_size:new_timestamp_size] = ( + unix_timestamps + ) self.sequence_dataset[current_timestamp_size:new_timestamp_size] = sequences self.frame_data_dataset[current_frame_size:new_frame_size] = all_frame_data