Add noise-aware stitching, min_rows, concat, and calcium imaging config

.github/workflows/docs-test.yml

@@ -22,10 +22,9 @@ jobs:
           cache: "pip"
 
       - name: Install dependencies
-        run: pip install -e .[docs] pytest-md
+        run: |
+          pip install pdm
+          pdm install --with docs
 
       - name: Build docs
-        working-directory: docs
-        env:
-          SPHINXOPTS: "-W --keep-going"
-        run: make html
+        run: pdm run docs-prod

.gitignore

@@ -165,4 +165,8 @@ user_data/*
 # The default output directory for the process commands. Not necessary but doesn't hurt to have.
 mio_process/*
 !user_data/.gitkeep
+tests/data/stitch/*_timestamps.csv
+tests/data/stitch/*_noise.csv
+tests/data/stitch/*_scores.csv
+tests/data/stitch/*_stitched*
 

docs/api/models/dataset.md

@@ -0,0 +1,7 @@
+# dataset
+
+```{eval-rst}
+.. automodule:: mio.models.dataset
+    :members:
+    :undoc-members:
+```

docs/api/models/index.md

@@ -19,6 +19,7 @@ keep what is common common, and what is unique unique.
 buffer
 config
 data
+dataset
 mixins
 models
 sdcard

docs/conf.py

@@ -81,6 +81,8 @@
 # Mock imports for packages we don't have yet - this one is
 # for opal kelley stuff we need to figure out the licensing for
 autodoc_mock_imports = ["routine"]
+autodoc_pydantic_model_show_json = False
+autodoc_pydantic_model_show_json_error_strategy = "coerce"
 
 # todo
 todo_include_todos = True

docs/meta/changelog.md

@@ -1,8 +1,8 @@
 # Changelog
 
-## Upcoming
+## 0.10
 
-### *.*
+### 0.10.0
 
 #### CLI
 
@@ -12,11 +12,32 @@
   - `mio config open` to open the config in default text editor
 - [`#154`](https://github.com/miniscope/mio/pull/154) - add cli command for removing frames from video:
   - `mio process remove_frames` to remove frames by explicitly specified index from videos and metadata
+- [`#155`](https://github.com/miniscope/mio/pull/155) - `mio process concat` - concatenate videos and metadata
 
 #### CI/CD
 
 - [`#157`](https://github.com/miniscope/mio/pull/157) - Add continuous deployment to PyPI 
 
+#### New features
+
+- [`#133`](https://github.com/miniscope/mio/pull/133) - {class}`~mio.models.dataset.Dataset` 
+  organization - group recordings with their metadata, and group multiple recordings collected at the same time.
+- [`#133`](https://github.com/miniscope/mio/pull/133), [`#155`](https://github.com/miniscope/mio/pull/155)
+  Noise-aware stitching: Given two recordings of the same data stream,
+  create a stitched version that picks the best frames from each of them
+- [`#133`](https://github.com/miniscope/mio/pull/133), [`#155`](https://github.com/miniscope/mio/pull/155)
+  Alignment Maps - within a dataset, create an alignment map to align frames between recordings,
+  either by `frame_num` or by timestamps.
+- preserve noise scoring metadata in `_scores.csv` and use to pick frames during stitching
+
+#### Perf
+
+- [`#155`](https://github.com/miniscope/mio/pull/155) - Vectorized black area detection
+
+#### Removed
+
+- [`#155`](https://github.com/miniscope/mio/pull/155) - Inter-frame mean squared error noise detection, unused.
+
 ## 0.9
 
 ### 0.9.0 - 2026-01-27 - Batch device update, NTP sync, driver import fix

mio/cli/process.py

@@ -9,10 +9,12 @@
 from mio.logging import init_logger
 from mio.models.dataset import Recording
 from mio.models.process import DenoiseConfig
+from mio.process.stitch import concat_recordings
 from mio.process.stitch import stitch as run_stitch
 from mio.process.video import denoise as run_denoise
 from mio.process.video import remove_frames as run_remove_frames
 from mio.process.video import trim as run_trim
+from mio.types import ConfigSource
 
 logger = init_logger("mio.cli.process")
 
@@ -198,6 +200,42 @@ def remove_frames(input: str, output: str | None, frames: str, force: bool = Fal
     click.echo(f"Video written to {output} with frames {frames} removed from {input}")
 
 
+@process.command()
+@click.option(
+    "-i",
+    "--inputs",
+    required=True,
+    multiple=True,
+    type=click.Path(exists=True, dir_okay=False),
+    help="Paths to video files. Each requires a .csv with the same stem name.",
+)
+@click.option(
+    "-o",
+    "--output",
+    type=click.Path(dir_okay=False),
+    required=True,
+    help="Path to the output concatenated video file or directory. "
+    "If not specified, saves next to video with '_combined' suffix.",
+)
+def concat(
+    inputs: list[Path],
+    output: Path,
+) -> None:
+    """
+    Concatenate sequential recording segments from one DAQ into a single video.
+
+    Use this to combine multiple segment files (e.g. long-2.avi, long-3.avi, ...)
+    from the same DAQ before stitching across DAQs.
+    """
+    if len(inputs) < 2:
+        raise click.ClickException("Need at least 2 .avi files to concat")
+    recordings = [Recording.from_video(Path(p)) for p in inputs]
+    output = Path(output)
+
+    click.echo(f"Concatenating {len(recordings)} segments...")
+    concat_recordings(recordings=recordings, output_video_path=output, progress=True)
+
+
 @process.command()
 @click.option(
     "-i",
@@ -214,6 +252,13 @@ def remove_frames(input: str, output: str | None, frames: str, force: bool = Fal
     default=None,
     help="Directory for output videos and metadata. If none provided, same as the inputs.",
 )
+@click.option(
+    "-c",
+    "--config",
+    default=None,
+    help="A config id or path for a DenoiseConfig used to score frames if no noise score exists."
+    "If not provided, default config is used.",
+)
 @click.option(
     "--debug-video",
     default=False,
@@ -228,7 +273,11 @@ def remove_frames(input: str, output: str | None, frames: str, force: bool = Fal
     help="Overwrite any existing files",
 )
 def stitch(
-    inputs: tuple, output: Path | None = None, debug_video: bool = False, force: bool = False
+    inputs: tuple,
+    output: Path | None = None,
+    config: ConfigSource | None = None,
+    debug_video: bool = False,
+    force: bool = False,
 ) -> None:
     """
     Stitch multiple video recordings into one by selecting the best frame
@@ -239,9 +288,20 @@ def stitch(
     if len(inputs) < 2:
         raise click.ClickException("At least 2 input videos are required for stitching.")
 
+    if config is not None:
+        denoise_config: DenoiseConfig = DenoiseConfig.from_any(config)
+        patch_config = denoise_config.noise_patch
+    else:
+        patch_config = None
+
     recordings = [Recording.from_video(Path(p)) for p in inputs]
     stitched = run_stitch(
-        recordings, debug_video=debug_video, output_dir=output, progress=True, force=force
+        recordings,
+        debug_video=debug_video,
+        noise_config=patch_config,
+        output_dir=output,
+        progress=True,
+        force=force,
     )
     click.echo(f"Stitched videos to {stitched.video.path}")
 
@@ -309,13 +369,21 @@ def workflow(
         output_dir = Path(output).expanduser()
         output_dir.mkdir(parents=True, exist_ok=True)
 
+    denoise_config_parsed = DenoiseConfig.from_any(denoise_config)
+
     if len(inputs) == 1:
         click.echo("Only one input video provided, skipping stitching")
         stitched_video = inputs[0]
     else:
         click.echo("Stitching videos...")
         recordings = [Recording.from_video(p) for p in inputs]
-        stitched = run_stitch(recordings, output_dir=output_dir, progress=True, force=force)
+        stitched = run_stitch(
+            recordings,
+            output_dir=output_dir,
+            noise_config=denoise_config_parsed,
+            progress=True,
+            force=force,
+        )
         stitched_video = stitched.video.path
 
     if trim_start == 0 and trim_end == 0:
@@ -331,7 +399,6 @@ def workflow(
     if trimmed.metadata is None:
         raise FileNotFoundError(f"No metadata csv found for video {trimmed_video}")
 
-    denoise_config_parsed = DenoiseConfig.from_any(denoise_config)
     final_video = run_denoise(
         trimmed_video,
         denoise_config_parsed,

mio/data/config/process/denoise_example_mean_error.yml → mio/data/config/process/denoise_calcium_imaging.yml

@@ -1,45 +1,36 @@
-id: denoise_example_mean_error
-mio_model: mio.models.process.DenoiseConfig
-mio_version: 0.6.1
-noise_patch:
-  enable: true
-  method: [mean_error]
-  mean_error_config:
-    threshold: 40
-    device_config_id: wireless-200px
-    buffer_split: 8
-    comparison_unit: 1000
-    diff_multiply: 1
-  gradient_config:
-    threshold: 20
-  black_area_config:
-    consecutive_threshold: 5
-    value_threshold: 16
-  output_result: true
-  output_noise_patch: true
-  output_diff: true
-  output_noisy_frames: true
-frequency_masking:
-  id: frequency_masking_example_mean_error
-  mio_model: mio.models.process.FrequencyMaskingConfig
-  mio_version: 0.6.1
-  enable: true
-  spatial_LPF_cutoff_radius: 15
-  vertical_BEF_cutoff: 2
-  horizontal_BEF_cutoff: 0
-  output_mask: true
-  output_result: true
-  output_freq_domain: true
-minimum_projection:
-  enable: true
-  normalize: true
-  output_result: true
-  output_min_proj: true
-interactive_display:
-  show_videos: true
-  start_frame: 40
-  end_frame: 140
-  display_freq_mask: true
-end_frame: -1 #-1 means all frames
-output_result: true
-output_dir: user_data/output
+id: denoise_calcium_imaging
+mio_model: mio.models.process.DenoiseConfig
+mio_version: 0.6.1
+noise_patch:
+  enable: true
+  method: [gradient, black_area]
+  gradient_config:
+    threshold: 20
+  black_area_config:
+    consecutive_threshold: 30
+    value_threshold: 0
+    min_rows: 10
+  output_result: true
+  output_noise_patch: true
+  output_noisy_frames: true
+frequency_masking:
+  id: frequency_masking_calcium_imaging
+  mio_model: mio.models.process.FrequencyMaskingConfig
+  mio_version: 0.6.1
+  enable: true
+  cast_float32: true
+  spatial_LPF_cutoff_radius: 15
+  vertical_BEF_cutoff: 2
+  horizontal_BEF_cutoff: 0
+  output_result: true
+minimum_projection:
+  enable: true
+  normalize: true
+  output_result: true
+interactive_display:
+  show_videos: false
+  start_frame: 0
+  end_frame: 100
+  display_freq_mask: false
+end_frame: -1
+output_result: true

mio/data/config/process/denoise_example.yml

@@ -4,20 +4,13 @@ mio_version: 0.6.1
 noise_patch:
   enable: true
   method: [gradient, black_area]
-  mean_error_config:
-    threshold: 40
-    device_config_id: wireless-200px
-    buffer_split: 8
-    comparison_unit: 1000
-    diff_multiply: 1
   gradient_config:
     threshold: 20
   black_area_config:
     consecutive_threshold: 5
     value_threshold: 0
   output_result: true
   output_noise_patch: true
-  output_diff: true
   output_noisy_frames: true
 frequency_masking:
   id: frequency_masking_example

mio/data/config/process/denoise_patchonly.yml

@@ -6,20 +6,13 @@ noise_patch:
   method:
   - gradient
   - black_area
-  mean_error_config:
-    threshold: 40
-    device_config_id: wireless-200px
-    buffer_split: 8
-    comparison_unit: 1000
-    diff_multiply: 1
   gradient_config:
     threshold: 20
   black_area_config:
     consecutive_threshold: 200
     value_threshold: 0
   output_result: true
   output_noise_patch: true
-  output_diff: false
   output_noisy_frames: true
 frequency_masking:
   id: frequency_masking_example