diff --git a/neo/io/neuralynxio.py b/neo/io/neuralynxio.py index a836cf3ab..3be4ee077 100644 --- a/neo/io/neuralynxio.py +++ b/neo/io/neuralynxio.py @@ -36,6 +36,8 @@ def __init__( include_filenames=None, exclude_filenames=None, keep_original_times=False, + gap_tolerance_ms=None, + strict_gap_mode=None, filename=None, exclude_filename=None, ): @@ -67,6 +69,13 @@ def __init__( Preserve original time stamps as in data files. By default datasets are shifted to begin at t_start = 0*pq.second. Default: False + gap_tolerance_ms : float | None, default: None + Controls how timestamp gaps in NCS files are handled. + If None (default), a ValueError is raised when gaps are detected, with a + detailed gap report. If a float value is provided, gaps smaller than this + threshold (in milliseconds) are ignored, and gaps larger create new segments. + strict_gap_mode : bool | None, default: None + Deprecated. Use gap_tolerance_ms instead. """ if filename is not None: @@ -83,6 +92,8 @@ def __init__( include_filenames=include_filenames, exclude_filenames=exclude_filenames, keep_original_times=keep_original_times, + gap_tolerance_ms=gap_tolerance_ms, + strict_gap_mode=strict_gap_mode, use_cache=use_cache, cache_path=cache_path, ) diff --git a/neo/rawio/neuralynxrawio/ncssections.py b/neo/rawio/neuralynxrawio/ncssections.py index d67fff25d..26e0df2f9 100644 --- a/neo/rawio/neuralynxrawio/ncssections.py +++ b/neo/rawio/neuralynxrawio/ncssections.py @@ -35,6 +35,7 @@ """ import math +import warnings import numpy as np from enum import IntEnum, auto @@ -67,6 +68,7 @@ def __init__(self): self.sects = [] self.sampFreqUsed = 0 # actual sampling frequency of samples self.microsPerSampUsed = 0 # microseconds per sample + self.detected_gaps = [] # list of (record_index, gap_size_us) for all detected gaps def __eq__(self, other): samp_eq = self.sampFreqUsed == other.sampFreqUsed @@ -216,6 +218,8 @@ def _buildNcsSections(ncsMemMap, sampFreq, gapTolerance=0): n_samples=n_samples, ) ncsSects.sects.append(section0) + # No gaps detected in fast path + ncsSects.detected_gaps = [] else: # need to parse all data block to detect gaps @@ -223,6 +227,16 @@ def _buildNcsSections(ncsMemMap, sampFreq, gapTolerance=0): delta = (ncsMemMap["timestamp"][1:] - ncsMemMap["timestamp"][:-1]).astype(np.int64) delta_prediction = ((ncsMemMap["nb_valid"][:-1] / sampFreq) * 1e6).astype(np.int64) + # Always detect all gaps using the strict threshold for reporting + strict_tolerance = round(NcsSectionsFactory._maxGapSampFrac * 1e6 / sampFreq) + all_gap_inds = np.flatnonzero(np.abs(delta - delta_prediction) > strict_tolerance) + gap_sizes_us = (delta - delta_prediction)[all_gap_inds] + ncsSects.detected_gaps = [ + (int(record_index + 1), int(gap_size)) + for record_index, gap_size in zip(all_gap_inds, gap_sizes_us) + ] + + # Use user-provided tolerance for actual segmentation gap_inds = np.flatnonzero(np.abs(delta - delta_prediction) > gapTolerance) gap_inds += 1 @@ -245,7 +259,7 @@ def _buildNcsSections(ncsMemMap, sampFreq, gapTolerance=0): return ncsSects @staticmethod - def build_for_ncs_file(ncsMemMap, nlxHdr, gapTolerance=None, strict_gap_mode=True): + def build_for_ncs_file(ncsMemMap, nlxHdr, gap_tolerance_us=None, **kwargs): """ Build an NcsSections object for an NcsFile, given as a memmap and NlxHeader, handling gap detection appropriately given the file type as specified by the header. @@ -256,11 +270,37 @@ def build_for_ncs_file(ncsMemMap, nlxHdr, gapTolerance=None, strict_gap_mode=Tru memory map of file nlxHdr: NlxHeader from corresponding file. + gap_tolerance_us : float | None, default: None + Gap tolerance in microseconds for segmentation. Returns ------- An NcsSections corresponding to the provided ncsMemMap and nlxHdr """ + # Handle deprecated parameters + gapTolerance = kwargs.pop("gapTolerance", None) + strict_gap_mode = kwargs.pop("strict_gap_mode", None) + if kwargs: + raise TypeError(f"Unexpected keyword arguments: {list(kwargs.keys())}") + + if gapTolerance is not None: + warnings.warn( + "The `gapTolerance` parameter is deprecated and will be removed in version 0.16. " + "Use `gap_tolerance_us` instead.", + DeprecationWarning, + stacklevel=2, + ) + if gap_tolerance_us is None: + gap_tolerance_us = gapTolerance + + if strict_gap_mode is not None: + warnings.warn( + "The `strict_gap_mode` parameter is deprecated and will be removed in version 0.16. " + "Use `gap_tolerance_us` instead.", + DeprecationWarning, + stacklevel=2, + ) + acqType = nlxHdr.type_of_recording() freq = nlxHdr["sampling_rate"] @@ -269,14 +309,13 @@ def build_for_ncs_file(ncsMemMap, nlxHdr, gapTolerance=None, strict_gap_mode=Tru # restriction arose from the sampling being based on a master 1 MHz clock. microsPerSampUsed = math.floor(NcsSectionsFactory.get_micros_per_samp_for_freq(freq)) sampFreqUsed = NcsSectionsFactory.get_freq_for_micros_per_samp(microsPerSampUsed) - if gapTolerance is None: - if strict_gap_mode: - # this is the old behavior, maybe we could put 0.9 sample interval no ? - gapTolerance = 0 + if gap_tolerance_us is None: + if strict_gap_mode is not None and not strict_gap_mode: + gap_tolerance_us = 0 else: - gapTolerance = 0 + gap_tolerance_us = 0 - ncsSects = NcsSectionsFactory._buildNcsSections(ncsMemMap, sampFreqUsed, gapTolerance=gapTolerance) + ncsSects = NcsSectionsFactory._buildNcsSections(ncsMemMap, sampFreqUsed, gapTolerance=gap_tolerance_us) ncsSects.sampFreqUsed = sampFreqUsed ncsSects.microsPerSampUsed = microsPerSampUsed @@ -288,17 +327,16 @@ def build_for_ncs_file(ncsMemMap, nlxHdr, gapTolerance=None, strict_gap_mode=Tru AcqType.RAWDATAFILE, ]: # digital lynx style with fractional frequency and micros per samp determined from block times - if gapTolerance is None: - if strict_gap_mode: - # this is the old behavior - gapTolerance = round(NcsSectionsFactory._maxGapSampFrac * 1e6 / freq) + if gap_tolerance_us is None: + if strict_gap_mode is not None and not strict_gap_mode: + # quarter of packet size is tolerated + gap_tolerance_us = round(0.25 * NcsSection._RECORD_SIZE * 1e6 / freq) else: - # quarter of paquet size is tolerate - gapTolerance = round(0.25 * NcsSection._RECORD_SIZE * 1e6 / freq) - ncsSects = NcsSectionsFactory._buildNcsSections(ncsMemMap, freq, gapTolerance=gapTolerance) + # default: strict detection (0.2 of a sample interval) + gap_tolerance_us = round(NcsSectionsFactory._maxGapSampFrac * 1e6 / freq) + ncsSects = NcsSectionsFactory._buildNcsSections(ncsMemMap, freq, gapTolerance=gap_tolerance_us) - # take longer data block to compute reaal sampling rate - # ind_max = np.argmax([section.n_samples for section in ncsSects.sects]) + # take longer data block to compute real sampling rate ind_max = np.argmax([section.endRec - section.startRec for section in ncsSects.sects]) section = ncsSects.sects[ind_max] if section.endRec != section.startRec: @@ -315,13 +353,13 @@ def build_for_ncs_file(ncsMemMap, nlxHdr, gapTolerance=None, strict_gap_mode=Tru elif acqType == AcqType.BML or acqType == AcqType.ATLAS: # BML & ATLAS style with fractional frequency and micros per samp - if strict_gap_mode: - # this is the old behavior, maybe we could put 0.9 sample interval no ? - gapTolerance = 0 - else: - # quarter of paquet size is tolerate - gapTolerance = round(0.25 * NcsSection._RECORD_SIZE * 1e6 / freq) - ncsSects = NcsSectionsFactory._buildNcsSections(ncsMemMap, freq, gapTolerance=gapTolerance) + if gap_tolerance_us is None: + if strict_gap_mode is not None and not strict_gap_mode: + # quarter of packet size is tolerated + gap_tolerance_us = round(0.25 * NcsSection._RECORD_SIZE * 1e6 / freq) + else: + gap_tolerance_us = 0 + ncsSects = NcsSectionsFactory._buildNcsSections(ncsMemMap, freq, gapTolerance=gap_tolerance_us) ncsSects.sampFreqUsed = freq ncsSects.microsPerSampUsed = NcsSectionsFactory.get_micros_per_samp_for_freq(freq) diff --git a/neo/rawio/neuralynxrawio/neuralynxrawio.py b/neo/rawio/neuralynxrawio/neuralynxrawio.py index 9dcc6c846..a00abe4c5 100644 --- a/neo/rawio/neuralynxrawio/neuralynxrawio.py +++ b/neo/rawio/neuralynxrawio/neuralynxrawio.py @@ -91,12 +91,19 @@ class NeuralynxRawIO(BaseRawIO): keep_original_times: bool, default: False If True, keep original start time as in files, Otherwise set 0 of time to first time in dataset - strict_gap_mode: bool, default: True - Detect gaps using strict mode or not. - * strict_gap_mode = True then a gap is consider when timstamp difference between two - consequtive data packet is more than one sample interval. - * strict_gap_mode = False then a gap has an increased tolerance. Some new system with different clock need this option - otherwise, too many gaps are detected + gap_tolerance_ms : float | None, default: None + Controls how timestamp gaps in NCS files are handled. + If None (default), a ValueError is raised when gaps are detected, with a + detailed gap report showing the number, size, and location of each gap. + If a float value is provided, gaps smaller than this threshold (in milliseconds) + are ignored, and gaps larger than this threshold create new segments. + Use gap_tolerance_ms=0.0 to segment on all detected gaps. + strict_gap_mode : bool | None, default: None + .. deprecated:: + Use ``gap_tolerance_ms`` instead. Will be removed in version 0.16. + If explicitly set, uses legacy gap detection behavior: + strict_gap_mode=True uses tight tolerance (0.2 sample intervals), + strict_gap_mode=False uses loose tolerance (quarter of 512-sample packet). Notes ----- @@ -157,7 +164,8 @@ def __init__( include_filenames=None, exclude_filenames=None, keep_original_times=False, - strict_gap_mode=True, + gap_tolerance_ms=None, + strict_gap_mode=None, filename=None, exclude_filename=None, **kargs, @@ -196,11 +204,32 @@ def __init__( else: self.rawmode = "one-dir" + # Handle gap_tolerance_ms and deprecated strict_gap_mode + if strict_gap_mode is not None: + warnings.warn( + "`strict_gap_mode` is deprecated and will be removed in version 0.16. " + "Use `gap_tolerance_ms` instead to control gap handling. " + "See issue #1773 for details.", + DeprecationWarning, + stacklevel=2, + ) + if gap_tolerance_ms is not None: + warnings.warn( + "Both `gap_tolerance_ms` and `strict_gap_mode` were provided. " + "`gap_tolerance_ms` takes precedence.", + UserWarning, + stacklevel=2, + ) + self._use_legacy_gap_mode = gap_tolerance_ms is None + else: + self._use_legacy_gap_mode = False + self.dirname = dirname self.include_filenames = include_filenames self.exclude_filenames = exclude_filenames self.keep_original_times = keep_original_times - self.strict_gap_mode = strict_gap_mode + self.gap_tolerance_ms = gap_tolerance_ms + self.strict_gap_mode = strict_gap_mode if strict_gap_mode is not None else True BaseRawIO.__init__(self, **kargs) def _source_name(self): @@ -878,6 +907,76 @@ def _rescale_event_timestamp(self, event_timestamps, dtype, event_channel_index) event_times -= self.global_t_start return event_times + def _format_gap_report(self, detected_gaps, sampling_frequency, filename): + """ + Format a detailed gap report showing where timestamp discontinuities occur. + + Parameters + ---------- + detected_gaps : list of (int, int) + List of (record_index, gap_size_us) tuples from NcsSections.detected_gaps. + sampling_frequency : float + Sampling frequency in Hz used for the file. + filename : str + Path to the NCS file for the report header. + + Returns + ------- + str + Formatted gap report with table. + """ + gap_durations_ms = [abs(gap_size) / 1000.0 for _, gap_size in detected_gaps] + gap_positions_seconds = [ + record_index * NcsSection._RECORD_SIZE / sampling_frequency + for record_index, _ in detected_gaps + ] + + gap_detail_lines = [ + f"| {record_index:>15,} | {pos:>21.6f} | {dur:>21.3f} |\n" + for (record_index, _), pos, dur in zip(detected_gaps, gap_positions_seconds, gap_durations_ms) + ] + + return ( + f"Gap Report for {os.path.basename(filename)}:\n" + f"Found {len(detected_gaps)} timestamp gaps " + f"(detection threshold: {NcsSectionsFactory._maxGapSampFrac} sample intervals)\n\n" + "Gap Details:\n" + "+-----------------+-----------------------+-----------------------+\n" + "| Record Index | Record at (Seconds) | Gap Size (ms) |\n" + "+-----------------+-----------------------+-----------------------+\n" + + "".join(gap_detail_lines) + + "+-----------------+-----------------------+-----------------------+\n" + ) + + def _get_neuralynx_timestamps(self, block_index, seg_index, stream_index): + """ + Return original NCS record timestamps for the first channel in a stream/segment. + + These are the raw hardware timestamps from the NCS file records, one timestamp + per 512-sample record, in microseconds. + + Parameters + ---------- + block_index : int + Block index (always 0 for Neuralynx). + seg_index : int + Segment index. + stream_index : int + Stream index. + + Returns + ------- + np.ndarray + Timestamps in microseconds from the NCS records, one per 512-sample record. + """ + stream_id = self.header["signal_streams"][stream_index]["id"] + stream_mask = self.header["signal_channels"]["stream_id"] == stream_id + channel = self.header["signal_channels"][stream_mask][0] + chan_uid = (channel["name"], channel["id"]) + + data = self._sigs_memmaps[seg_index][chan_uid] + return data["timestamp"].copy() + def scan_stream_ncs_files(self, ncs_filenames): """ Given a list of ncs files, read their basic structure. @@ -906,6 +1005,17 @@ def scan_stream_ncs_files(self, ncs_filenames): if len(ncs_filenames) == 0: return None, None, None + # Determine gap tolerance in microseconds for NcsSectionsFactory + if self._use_legacy_gap_mode: + # Legacy mode: let NcsSectionsFactory use strict_gap_mode defaults + factory_kwargs = {"strict_gap_mode": self.strict_gap_mode} + elif self.gap_tolerance_ms is not None: + # New API: convert ms to us + factory_kwargs = {"gap_tolerance_us": self.gap_tolerance_ms * 1000.0} + else: + # New default: detect all gaps strictly, we'll error later if gaps found + factory_kwargs = {} + # Build dictionary of chan_uid to associated NcsSections, memmap and NlxHeaders. Only # construct new NcsSections when it is different from that for the preceding file. chanSectMap = dict() @@ -918,9 +1028,36 @@ def scan_stream_ncs_files(self, ncs_filenames): verify_sec_struct = NcsSectionsFactory._verifySectionsStructure if not chanSectMap or (not verify_sec_struct(data, chan_ncs_sections)): chan_ncs_sections = NcsSectionsFactory.build_for_ncs_file( - data, nlxHeader, strict_gap_mode=self.strict_gap_mode + data, nlxHeader, **factory_kwargs ) + # Check for gaps and handle according to gap_tolerance_ms + if not self._use_legacy_gap_mode and chan_ncs_sections.detected_gaps: + if self.gap_tolerance_ms is None: + # Default mode: only error on gaps >= 1 sample period. + # Sub-sample deviations (from timestamp rounding, clock jitter, etc.) + # are not real gaps and should not produce false positives. + one_sample_us = 1e6 / chan_ncs_sections.sampFreqUsed + significant_gaps = [ + (record_index, gap_us) + for record_index, gap_us in chan_ncs_sections.detected_gaps + if abs(gap_us) >= one_sample_us + ] + if significant_gaps: + gap_report = self._format_gap_report( + significant_gaps, + chan_ncs_sections.sampFreqUsed, + ncs_filename, + ) + raise ValueError( + f"Detected {len(significant_gaps)} timestamp gaps " + f"in {os.path.basename(ncs_filename)}.\n" + f"{gap_report}\n" + f"To load this data, provide the gap_tolerance_ms parameter to " + f"automatically segment at gaps larger than the specified tolerance.\n" + f"Example: NeuralynxRawIO(dirname=..., gap_tolerance_ms=1.0)" + ) + # register file section structure for all contained channels for chan_uid in zip(nlxHeader["channel_names"], np.asarray(nlxHeader["channel_ids"], dtype=str)): chanSectMap[chan_uid] = [chan_ncs_sections, nlxHeader, ncs_filename] diff --git a/neo/test/iotest/common_io_test.py b/neo/test/iotest/common_io_test.py index 7607e58e1..71aa1ed23 100644 --- a/neo/test/iotest/common_io_test.py +++ b/neo/test/iotest/common_io_test.py @@ -80,6 +80,7 @@ class BaseTestIO: entities_to_test = [] # list of files to test compliance entities_to_download = [] # when files are at gin + io_kwargs = {} # extra kwargs passed to ioclass constructor # when reading then writing produces files with identical hashes hash_conserved_when_write_read = False @@ -190,7 +191,12 @@ def generic_io_object(cls, filename=None, return_path=False, clean=False): before creating the io object. Default is False. """ return create_generic_io_object( - ioclass=cls.ioclass, filename=filename, directory=cls.local_test_dir, return_path=return_path, clean=clean + ioclass=cls.ioclass, + filename=filename, + directory=cls.local_test_dir, + return_path=return_path, + clean=clean, + io_kwargs=cls.io_kwargs, ) def read_file(self, filename=None, return_path=False, clean=False, target=None, readall=False, lazy=False): @@ -267,6 +273,7 @@ def iter_io_objects(self, return_path=False, clean=False): directory=self.local_test_dir, return_path=return_path, clean=clean, + io_kwargs=self.io_kwargs, ) def iter_readers(self, target=None, readall=False, return_path=False, return_ioobj=False, clean=False): @@ -297,6 +304,7 @@ def iter_readers(self, target=None, readall=False, return_path=False, return_ioo target=target, clean=clean, readall=readall, + io_kwargs=self.io_kwargs, ) def iter_objects( @@ -349,6 +357,7 @@ def iter_objects( clean=clean, readall=readall, lazy=lazy, + io_kwargs=self.io_kwargs, ) @classmethod @@ -510,10 +519,11 @@ def test__handle_pathlib_filename(self): ) pathlib_filename = pathlib.Path(filename) + kwargs = {**self.default_keyword_arguments, **self.io_kwargs} if self.ioclass.mode == "file": - self.ioclass(filename=pathlib_filename, *self.default_arguments, **self.default_keyword_arguments) + self.ioclass(filename=pathlib_filename, *self.default_arguments, **kwargs) elif self.ioclass.mode == "dir": - self.ioclass(dirname=pathlib_filename, *self.default_arguments, **self.default_keyword_arguments) + self.ioclass(dirname=pathlib_filename, *self.default_arguments, **kwargs) def test_list_candidate_ios(self): for entity in self.entities_to_test: diff --git a/neo/test/iotest/test_neuralynxio.py b/neo/test/iotest/test_neuralynxio.py index ebfbde4f0..031241f0e 100644 --- a/neo/test/iotest/test_neuralynxio.py +++ b/neo/test/iotest/test_neuralynxio.py @@ -26,6 +26,9 @@ class CommonNeuralynxIOTest( ioclass = NeuralynxIO entities_to_download = TestNeuralynxRawIO.entities_to_download entities_to_test = TestNeuralynxRawIO.entities_to_test + # Some test datasets have real gaps (pause/resume). Pass gap_tolerance_ms + # so the test infrastructure can load them without erroring. + io_kwargs = {"gap_tolerance_ms": 0.01} class TestCheetah_Neuraview(CommonNeuralynxIOTest, unittest.TestCase): @@ -47,7 +50,7 @@ class TestCheetah_v551(CommonNeuralynxIOTest, unittest.TestCase): def test_read_block(self): """Read data in a certain time range into one block""" dirname = self.get_local_path("neuralynx/Cheetah_v5.5.1/original_data") - nio = NeuralynxIO(dirname=dirname, use_cache=False) + nio = NeuralynxIO(dirname=dirname, use_cache=False, gap_tolerance_ms=0.01) block = nio.read_block() @@ -77,7 +80,7 @@ def test_read_block(self): def test_read_segment(self): dirname = self.get_local_path("neuralynx/Cheetah_v5.5.1/original_data") - nio = NeuralynxIO(dirname=dirname, use_cache=False) + nio = NeuralynxIO(dirname=dirname, use_cache=False, gap_tolerance_ms=0.01) # read first segment entirely seg = nio.read_segment(seg_index=0, time_slice=None) @@ -101,7 +104,7 @@ class TestCheetah_v563(CommonNeuralynxIOTest, unittest.TestCase): def test_read_block(self): """Read data in a certain time range into one block""" dirname = self.get_local_path("neuralynx/Cheetah_v5.6.3/original_data") - nio = NeuralynxIO(dirname=dirname, use_cache=False) + nio = NeuralynxIO(dirname=dirname, use_cache=False, gap_tolerance_ms=0.01) block = nio.read_block() @@ -133,7 +136,7 @@ def test_read_block(self): def test_read_segment(self): dirname = self.get_local_path("neuralynx/Cheetah_v5.5.1/original_data") - nio = NeuralynxIO(dirname=dirname, use_cache=False) + nio = NeuralynxIO(dirname=dirname, use_cache=False, gap_tolerance_ms=0.01) # read first segment entirely seg = nio.read_segment(seg_index=0, time_slice=None) @@ -156,7 +159,7 @@ class TestCheetah_v574(CommonNeuralynxIOTest, unittest.TestCase): def test_read_block(self): dirname = self.get_local_path("neuralynx/Cheetah_v5.7.4/original_data") - nio = NeuralynxIO(dirname=dirname, use_cache=False) + nio = NeuralynxIO(dirname=dirname, use_cache=False, gap_tolerance_ms=0.01) block = nio.read_block() @@ -183,7 +186,7 @@ def test_read_block(self): def test_include_filenames(self): filename = self.get_local_path("neuralynx/Cheetah_v5.7.4/original_data/CSC1.ncs") dirname, filename = os.path.split(filename) - nio = NeuralynxIO(dirname=dirname, include_filenames=filename, use_cache=False) + nio = NeuralynxIO(dirname=dirname, include_filenames=filename, use_cache=False, gap_tolerance_ms=0.01) block = nio.read_block() self.assertTrue(len(block.segments[0].analogsignals) > 0) self.assertTrue((len(block.segments[0].spiketrains)) == 0) @@ -194,7 +197,7 @@ def test_exclude_filenames(self): dname = self.get_local_path("neuralynx/Cheetah_v5.7.4/original_data/") # exclude a single file - nio = NeuralynxIO(dirname=dname, exclude_filenames="CSC1.ncs", use_cache=False) + nio = NeuralynxIO(dirname=dname, exclude_filenames="CSC1.ncs", use_cache=False, gap_tolerance_ms=0.01) block = nio.read_block() self.assertTrue(len(block.segments[0].analogsignals) > 0) self.assertTrue((len(block.segments[0].spiketrains)) >= 0) @@ -203,7 +206,7 @@ def test_exclude_filenames(self): # exclude all ncs files from session exclude_files = [f"CSC{i}.ncs" for i in range(6)] - nio = NeuralynxIO(dirname=dname, exclude_filenames=exclude_files, use_cache=False) + nio = NeuralynxIO(dirname=dname, exclude_filenames=exclude_files, use_cache=False, gap_tolerance_ms=0.01) block = nio.read_block() self.assertTrue(len(block.segments[0].analogsignals) == 0) self.assertTrue((len(block.segments[0].spiketrains)) >= 0) @@ -217,7 +220,7 @@ class TestPegasus_v211(CommonNeuralynxIOTest, unittest.TestCase): def test_read_block(self): dirname = self.get_local_path("neuralynx/Pegasus_v2.1.1") - nio = NeuralynxIO(dirname=dirname, use_cache=False) + nio = NeuralynxIO(dirname=dirname, use_cache=False, gap_tolerance_ms=0.01) block = nio.read_block() @@ -268,7 +271,7 @@ def _load_plaindata(self, filename, numSamps): def test_ncs(self): for session in self.files_to_test: dirname = self.get_local_path(session) - nio = NeuralynxIO(dirname=dirname, use_cache=False) + nio = NeuralynxIO(dirname=dirname, use_cache=False, gap_tolerance_ms=0.01) block = nio.read_block() # check that data agrees in first segment first channel only @@ -298,7 +301,7 @@ def test_ncs(self): def test_keep_original_spike_times(self): for session in self.files_to_test: dirname = self.get_local_path(session) - nio = NeuralynxIO(dirname=dirname, keep_original_times=True) + nio = NeuralynxIO(dirname=dirname, keep_original_times=True, gap_tolerance_ms=0.01) block = nio.read_block() for st in block.segments[0].spiketrains: @@ -322,7 +325,7 @@ def test_keep_original_spike_times(self): class TestIncompleteBlocks(CommonNeuralynxIOTest, unittest.TestCase): def test_incomplete_block_handling_v632(self): dirname = self.get_local_path("neuralynx/Cheetah_v6.3.2/incomplete_blocks") - nio = NeuralynxIO(dirname=dirname, use_cache=False) + nio = NeuralynxIO(dirname=dirname, use_cache=False, gap_tolerance_ms=0.01) block = nio.read_block() @@ -346,7 +349,7 @@ def test_incomplete_block_handling_v632(self): class TestGaps(CommonNeuralynxIOTest, unittest.TestCase): def test_gap_handling_v551(self): dirname = self.get_local_path("neuralynx/Cheetah_v5.5.1/original_data") - nio = NeuralynxIO(dirname=dirname, use_cache=False) + nio = NeuralynxIO(dirname=dirname, use_cache=False, gap_tolerance_ms=0.01) block = nio.read_block() @@ -359,7 +362,7 @@ def test_gap_handling_v551(self): def test_gap_handling_v563(self): dirname = self.get_local_path("neuralynx/Cheetah_v5.6.3/original_data") - nio = NeuralynxIO(dirname=dirname, use_cache=False) + nio = NeuralynxIO(dirname=dirname, use_cache=False, gap_tolerance_ms=0.01) block = nio.read_block() # known gap values @@ -373,7 +376,7 @@ def test_gap_handling_v563(self): class TestMultiSamplingRates(CommonNeuralynxIOTest, unittest.TestCase): def test_multi_sampling_rates(self): # Test Cheetah 6.4.1, with different sampling rates across ncs files. - nio = NeuralynxIO(self.get_local_path("neuralynx/Cheetah_v6.4.1dev/original_data")) + nio = NeuralynxIO(self.get_local_path("neuralynx/Cheetah_v6.4.1dev/original_data"), gap_tolerance_ms=0.01) block = nio.read_block() self.assertEqual(len(block.segments), 1) diff --git a/neo/test/iotest/tools.py b/neo/test/iotest/tools.py index e57094294..7289eec96 100644 --- a/neo/test/iotest/tools.py +++ b/neo/test/iotest/tools.py @@ -82,7 +82,7 @@ def get_test_file_full_path(ioclass, filename=None, directory=None, clean=False) get_test_file_full_path.__test__ = False -def create_generic_io_object(ioclass, filename=None, directory=None, return_path=False, clean=False): +def create_generic_io_object(ioclass, filename=None, directory=None, return_path=False, clean=False, io_kwargs=None): """ Create an io object in a generic way that can work with both file-based and directory-based io objects @@ -99,14 +99,19 @@ def create_generic_io_object(ioclass, filename=None, directory=None, return_path If clean is True, try to delete existing versions of the file before creating the io object. Default is False. + + If io_kwargs is not None, pass them as extra keyword arguments to + the io class constructor. """ + if io_kwargs is None: + io_kwargs = {} filename = get_test_file_full_path(ioclass, filename=filename, directory=directory, clean=clean) try: # actually create the object if ioclass.mode == "file": - ioobj = ioclass(filename=filename) + ioobj = ioclass(filename=filename, **io_kwargs) elif ioclass.mode == "dir": - ioobj = ioclass(dirname=filename) + ioobj = ioclass(dirname=filename, **io_kwargs) else: ioobj = None except: @@ -119,7 +124,7 @@ def create_generic_io_object(ioclass, filename=None, directory=None, return_path return ioobj -def iter_generic_io_objects(ioclass, filenames, directory=None, return_path=False, clean=False): +def iter_generic_io_objects(ioclass, filenames, directory=None, return_path=False, clean=False, io_kwargs=None): """ Return an iterable over the io objects created from a list of filenames. @@ -136,7 +141,7 @@ def iter_generic_io_objects(ioclass, filenames, directory=None, return_path=Fals """ for filename in filenames: ioobj, path = create_generic_io_object( - ioclass, filename=filename, directory=directory, return_path=True, clean=clean + ioclass, filename=filename, directory=directory, return_path=True, clean=clean, io_kwargs=io_kwargs ) if ioobj is None: @@ -183,7 +188,15 @@ def create_generic_reader(ioobj, target=None, readall=False): def iter_generic_readers( - ioclass, filenames, directory=None, target=None, return_path=False, return_ioobj=False, clean=False, readall=False + ioclass, + filenames, + directory=None, + target=None, + return_path=False, + return_ioobj=False, + clean=False, + readall=False, + io_kwargs=None, ): """ Iterate over functions that can read the target object from a list of @@ -214,7 +227,7 @@ def iter_generic_readers( Default is False. """ for ioobj, path in iter_generic_io_objects( - ioclass=ioclass, filenames=filenames, directory=directory, return_path=True, clean=clean + ioclass=ioclass, filenames=filenames, directory=directory, return_path=True, clean=clean, io_kwargs=io_kwargs ): res = create_generic_reader(ioobj, target=target, readall=readall) if not return_path and not return_ioobj: @@ -289,6 +302,7 @@ def iter_read_objects( clean=False, readall=False, lazy=False, + io_kwargs=None, ): """ Iterate over objects read from a list of filenames. @@ -331,6 +345,7 @@ def iter_read_objects( return_ioobj=True, clean=clean, readall=readall, + io_kwargs=io_kwargs, ): obj = obj_reader(lazy=lazy) if not return_path and not return_ioobj and not return_reader: diff --git a/neo/test/rawiotest/common_rawio_test.py b/neo/test/rawiotest/common_rawio_test.py index 0297194b7..8dec0c4fc 100644 --- a/neo/test/rawiotest/common_rawio_test.py +++ b/neo/test/rawiotest/common_rawio_test.py @@ -61,6 +61,7 @@ class BaseTestRawIO: entities_to_test = [] # list of files to test compliances entities_to_download = [] # when files are at gin + rawio_kwargs = {} # extra kwargs passed to rawioclass constructor # allow environment to tell avoid using network use_network = can_use_network() @@ -101,9 +102,9 @@ def test_read_all(self): local_path = self.get_local_path(entity_name) if self.rawioclass.rawmode.endswith("-file"): - reader = self.rawioclass(filename=local_path) + reader = self.rawioclass(filename=local_path, **self.rawio_kwargs) elif self.rawioclass.rawmode.endswith("-dir"): - reader = self.rawioclass(dirname=local_path) + reader = self.rawioclass(dirname=local_path, **self.rawio_kwargs) txt = reader.__repr__() assert "nb_block" not in txt, "Before parser_header() nb_block should be NOT known" diff --git a/neo/test/rawiotest/test_neuralynxrawio.py b/neo/test/rawiotest/test_neuralynxrawio.py index 0a1c90980..144b04930 100644 --- a/neo/test/rawiotest/test_neuralynxrawio.py +++ b/neo/test/rawiotest/test_neuralynxrawio.py @@ -33,13 +33,16 @@ class TestNeuralynxRawIO( "neuralynx/Cheetah_v6.3.2/incomplete_blocks", "neuralynx/two_streams_different_header_encoding", ] + # Some test datasets have real gaps (pause/resume). Pass gap_tolerance_ms + # so the base test_read_all can load them without erroring. + rawio_kwargs = {"gap_tolerance_ms": 0.01} def test_scan_ncs_files(self): # Test BML style of Ncs files, similar to PRE4 but with fractional frequency # in the header and fractional microsPerSamp, which is then rounded as appropriate # in each record. - rawio = NeuralynxRawIO(self.get_local_path("neuralynx/BML/original_data")) + rawio = NeuralynxRawIO(self.get_local_path("neuralynx/BML/original_data"), gap_tolerance_ms=0.0) rawio.parse_header() # test values here from direct inspection of .ncs files self.assertEqual(rawio._nb_segment, 1) @@ -51,7 +54,7 @@ def test_scan_ncs_files(self): # Test Cheetah 4.0.2, which is PRE4 type with frequency in header and # no microsPerSamp. Number of microseconds per sample in file is inverse of # sampling frequency in header trucated to microseconds. - rawio = NeuralynxRawIO(self.get_local_path("neuralynx/Cheetah_v4.0.2/original_data")) + rawio = NeuralynxRawIO(self.get_local_path("neuralynx/Cheetah_v4.0.2/original_data"), gap_tolerance_ms=0.0) rawio.parse_header() # test values here from direct inspection of .ncs files self.assertEqual(rawio._nb_segment, 1) @@ -63,7 +66,7 @@ def test_scan_ncs_files(self): # Test Cheetah 5.5.1, which is DigitalLynxSX and has two blocks of records # with a fairly large gap. - rawio = NeuralynxRawIO(self.get_local_path("neuralynx/Cheetah_v5.5.1/original_data")) + rawio = NeuralynxRawIO(self.get_local_path("neuralynx/Cheetah_v5.5.1/original_data"), gap_tolerance_ms=0.0) rawio.parse_header() # test values here from direct inspection of .ncs files self.assertEqual(rawio._nb_segment, 2) @@ -76,7 +79,9 @@ def test_scan_ncs_files(self): # Test Cheetah 6.3.2, the incomplete_blocks test. This is a DigitalLynxSX with # three blocks of records. Gaps are on the order of 60 microseconds or so. - rawio = NeuralynxRawIO(self.get_local_path("neuralynx/Cheetah_v6.3.2/incomplete_blocks")) + # Use tolerance of 0.01 ms (10 us) to match old strict_gap_mode=True behavior + # which used 0.2 * sample_interval (~6.25 us at 32kHz) + rawio = NeuralynxRawIO(self.get_local_path("neuralynx/Cheetah_v6.3.2/incomplete_blocks"), gap_tolerance_ms=0.01) rawio.parse_header() # test values here from direct inspection of .ncs file, except for 3rd block # t_stop, which is extended due to events past the last block of ncs records. @@ -94,7 +99,7 @@ def test_scan_ncs_files(self): self.assertEqual(len(rawio._sigs_memmaps), 3) # check that there are only 3 memmaps # Test Cheetah 6.4.1, with different sampling rates across ncs files. - rawio = NeuralynxRawIO(self.get_local_path("neuralynx/Cheetah_v6.4.1dev/original_data")) + rawio = NeuralynxRawIO(self.get_local_path("neuralynx/Cheetah_v6.4.1dev/original_data"), gap_tolerance_ms=0.0) rawio.parse_header() self.assertEqual(rawio._nb_segment, 1) @@ -117,7 +122,7 @@ def test_include_filenames(self): # test single analog signal channel fname = self.get_local_path("neuralynx/Cheetah_v5.6.3/original_data/CSC1.ncs") dirname, filename = os.path.split(fname) - rawio = NeuralynxRawIO(dirname=dirname, include_filenames=filename) + rawio = NeuralynxRawIO(dirname=dirname, include_filenames=filename, gap_tolerance_ms=0.0) rawio.parse_header() self.assertEqual(rawio._nb_segment, 2) @@ -133,7 +138,7 @@ def test_include_filenames(self): # test one single electrode channel fname = self.get_local_path("neuralynx/Cheetah_v5.5.1/original_data/STet3a.nse") dirname, filename = os.path.split(fname) - rawio = NeuralynxRawIO(dirname=dirname, include_filenames=filename) + rawio = NeuralynxRawIO(dirname=dirname, include_filenames=filename, gap_tolerance_ms=0.0) rawio.parse_header() self.assertEqual(rawio._nb_segment, 1) @@ -149,7 +154,7 @@ def test_include_filenames(self): def test_exclude_filenames(self): # exclude single ncs file from session dname = self.get_local_path("neuralynx/Cheetah_v5.6.3/original_data/") - rawio = NeuralynxRawIO(dirname=dname, exclude_filenames="CSC2.ncs") + rawio = NeuralynxRawIO(dirname=dname, exclude_filenames="CSC2.ncs", gap_tolerance_ms=0.0) rawio.parse_header() self.assertEqual(rawio._nb_segment, 2) @@ -163,7 +168,7 @@ def test_exclude_filenames(self): self.assertEqual(len(rawio.header["event_channels"]), 2) # exclude multiple files from session - rawio = NeuralynxRawIO(dirname=dname, exclude_filenames=["Events.nev", "CSC2.ncs"]) + rawio = NeuralynxRawIO(dirname=dname, exclude_filenames=["Events.nev", "CSC2.ncs"], gap_tolerance_ms=0.0) rawio.parse_header() self.assertEqual(rawio._nb_segment, 2) @@ -201,7 +206,7 @@ def test_directory_in_data_folder(self): f.write("test file content") # This should not raise an error despite the directory presence - rawio = NeuralynxRawIO(dirname=temp_data_dir) + rawio = NeuralynxRawIO(dirname=temp_data_dir, gap_tolerance_ms=0.0) rawio.parse_header() # Verify that the reader still works correctly @@ -224,7 +229,7 @@ def test_two_streams_different_header_encoding(self): dname = self.get_local_path("neuralynx/two_streams_different_header_encoding") # Test with Path object (as shown in user's notebook) - rawio = NeuralynxRawIO(dirname=Path(dname)) + rawio = NeuralynxRawIO(dirname=Path(dname), gap_tolerance_ms=0.0) rawio.parse_header() # Should have 2 streams due to different filter configurations @@ -255,6 +260,77 @@ def test_two_streams_different_header_encoding(self): filter_1 = rawio._dsp_filter_configurations[1] self.assertTrue(filter_1.get("DSPLowCutFilterEnabled", False)) + def test_gap_tolerance_ms_error_by_default(self): + """Test that gaps raise ValueError by default (no gap_tolerance_ms).""" + # Cheetah_v5.5.1 has 2 segments (a large gap between them) + rawio = NeuralynxRawIO(self.get_local_path("neuralynx/Cheetah_v5.5.1/original_data")) + with self.assertRaises(ValueError) as cm: + rawio.parse_header() + self.assertIn("timestamp gaps", str(cm.exception)) + self.assertIn("gap_tolerance_ms", str(cm.exception)) + + def test_gap_tolerance_ms_segmentation(self): + """Test that gap_tolerance_ms=0.0 segments on all gaps.""" + rawio = NeuralynxRawIO( + self.get_local_path("neuralynx/Cheetah_v5.5.1/original_data"), + gap_tolerance_ms=0.0, + ) + rawio.parse_header() + self.assertEqual(rawio._nb_segment, 2) + + def test_gap_tolerance_ms_large_tolerance(self): + """Test that a very large tolerance collapses everything to 1 segment.""" + rawio = NeuralynxRawIO( + self.get_local_path("neuralynx/Cheetah_v5.5.1/original_data"), + gap_tolerance_ms=1e9, + ) + rawio.parse_header() + self.assertEqual(rawio._nb_segment, 1) + + def test_no_gaps_no_error(self): + """Test that datasets without gaps load fine without gap_tolerance_ms.""" + rawio = NeuralynxRawIO(self.get_local_path("neuralynx/Cheetah_v4.0.2/original_data")) + rawio.parse_header() + self.assertEqual(rawio._nb_segment, 1) + + def test_strict_gap_mode_deprecation(self): + """Test that strict_gap_mode emits DeprecationWarning.""" + with self.assertWarns(DeprecationWarning): + rawio = NeuralynxRawIO( + self.get_local_path("neuralynx/BML/original_data"), + strict_gap_mode=True, + ) + + with self.assertWarns(DeprecationWarning): + rawio = NeuralynxRawIO( + self.get_local_path("neuralynx/BML/original_data"), + strict_gap_mode=False, + ) + + def test_strict_gap_mode_legacy_behavior(self): + """Test that strict_gap_mode still works for backward compatibility.""" + # strict_gap_mode=True should segment like gap_tolerance_ms=0.0 + with self.assertWarns(DeprecationWarning): + rawio = NeuralynxRawIO( + self.get_local_path("neuralynx/Cheetah_v5.5.1/original_data"), + strict_gap_mode=True, + ) + rawio.parse_header() + self.assertEqual(rawio._nb_segment, 2) + + def test_get_neuralynx_timestamps(self): + """Test that _get_neuralynx_timestamps returns record timestamps.""" + rawio = NeuralynxRawIO( + self.get_local_path("neuralynx/Cheetah_v4.0.2/original_data"), + gap_tolerance_ms=0.0, + ) + rawio.parse_header() + timestamps = rawio._get_neuralynx_timestamps(block_index=0, seg_index=0, stream_index=0) + self.assertIsInstance(timestamps, np.ndarray) + self.assertTrue(len(timestamps) > 0) + # Timestamps should be monotonically increasing + self.assertTrue(np.all(np.diff(timestamps) > 0)) + class TestNcsRecordingType(BaseTestRawIO, unittest.TestCase): """