From 736f32f84ca7ba963bee32f347c017058b1e2e27 Mon Sep 17 00:00:00 2001 From: MarcelMB Date: Fri, 21 Feb 2025 15:09:05 -0800 Subject: [PATCH 1/3] close butter filter figure with Esc --- mio/process/video.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/mio/process/video.py b/mio/process/video.py index 4e7ad248..a30ed508 100644 --- a/mio/process/video.py +++ b/mio/process/video.py @@ -572,7 +572,6 @@ def batch_export_videos(self) -> None: logger.info(f"Processing {len(self.frames)} frames with Butterworth filter") filtered_data = self.apply_filter() if len(filtered_data) > 0: - np.save(self.output_dir / f"{self.name}_filtered_intensity.npy", filtered_data) filtered_frames = self.apply_filter_to_frames(filtered_data) @@ -583,6 +582,9 @@ def batch_export_videos(self) -> None: else: logger.warning("No frames to save after Butterworth filtering") + if self.config.plot: + self.plot_filtered_data(filtered_data) + def plot_filtered_data(self, filtered_data: np.ndarray) -> None: """Plot the original and filtered intensity data.""" if not self.config.plot or plt is None: @@ -613,13 +615,21 @@ def plot_filtered_data(self, filtered_data: np.ndarray) -> None: plt.ylabel("Mean Intensity") plt.grid(True, alpha=0.3) plt.legend() - plt.savefig( self.output_dir / f"{self.name}_intensity_plot.png", dpi=300, bbox_inches="tight" ) if plt.get_backend() != "agg": - plt.show() + plt.savefig("temp_plot.png") # Save temporarily + plot_img = cv2.imread("temp_plot.png") + cv2.imshow("Butterworth Filter Plot", plot_img) + while True: + if cv2.waitKey(1) == 27: # Wait for ESC key + break + cv2.destroyAllWindows() + cv2.waitKey(1) # Extra waitKey to properly close the window + Path("temp_plot.png").unlink() # Remove temporary file + plt.close() From 31fdbf008835c155b1c3988b747e6b9702b01335 Mon Sep 17 00:00:00 2001 From: MarcelMB Date: Fri, 21 Feb 2025 16:09:26 -0800 Subject: [PATCH 2/3] saving denoise_example.yml user settings to a json file --- mio/process/video.py | 92 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 92 insertions(+) diff --git a/mio/process/video.py b/mio/process/video.py index a30ed508..b2089793 100644 --- a/mio/process/video.py +++ b/mio/process/video.py @@ -2,6 +2,8 @@ This module contains functions for pre-processing video data. """ +import json +from datetime import datetime from pathlib import Path from typing import Optional @@ -777,3 +779,93 @@ def denoise_run( end_frame=config.interactive_display.end_frame, ) video_plotter.show() + + # Keep just the JSON logging + log_processing_metadata(output_dir, video_path, config, pathstem) + + +def log_processing_metadata( + output_dir: Path, video_path: str, config: DenoiseConfig, pathstem: str +) -> None: + """Log processing metadata to a JSON file.""" + + metadata = { + "timestamp": datetime.now().isoformat(), + "input_video": str(video_path), + "output_directory": str(output_dir), + "processing_config": { + "noise_patch": { + "enabled": config.noise_patch.enable if config.noise_patch else False, + "method": config.noise_patch.method if config.noise_patch else None, + "gradient_config": ( + { + "threshold": ( + config.noise_patch.gradient_config.threshold + if config.noise_patch + else None + ), + } + if config.noise_patch and config.noise_patch.gradient_config + else None + ), + "black_area_config": ( + { + "consecutive_threshold": ( + config.noise_patch.black_area_config.consecutive_threshold + if config.noise_patch + else None + ), + "value_threshold": ( + config.noise_patch.black_area_config.value_threshold + if config.noise_patch + else None + ), + } + if config.noise_patch and config.noise_patch.black_area_config + else None + ), + }, + "frequency_masking": { + "enabled": config.frequency_masking.enable if config.frequency_masking else False, + "cast_float32": ( + config.frequency_masking.cast_float32 if config.frequency_masking else False + ), + "cutoff_radius": ( + config.frequency_masking.spatial_LPF_cutoff_radius + if config.frequency_masking + else None + ), + "vertical_BEF": ( + config.frequency_masking.vertical_BEF_cutoff + if config.frequency_masking + else None + ), + "horizontal_BEF": ( + config.frequency_masking.horizontal_BEF_cutoff + if config.frequency_masking + else None + ), + }, + "butterworth": { + "enabled": config.butter_filter.enable, + "order": config.butter_filter.order, + "cutoff_frequency": config.butter_filter.cutoff_frequency, + "sampling_rate": config.butter_filter.sampling_rate, + }, + }, + "output_files": { + "noise_patch": f"output_{pathstem}_patch.avi", + "freq_mask": f"output_{pathstem}_freq_mask.avi", + "butter_filter": f"output_{pathstem}_butter_filter.avi", + "butter_plot": f"{pathstem}_butter_filter_intensity_plot.png", + }, + } + + # Save as JSON (overwrite any existing file) + log_file = output_dir / "processing_log.json" + logs = {"processing_runs": [metadata]} # Just create new log with single run + + with open(log_file, "w") as f: # "w" mode overwrites existing file + json.dump(logs, f, indent=2) + + logger.info(f"Saved processing metadata to {log_file}") From 39305451530027a440c5f47efa4b06da056e36ff Mon Sep 17 00:00:00 2001 From: MarcelMB Date: Fri, 21 Feb 2025 16:51:21 -0800 Subject: [PATCH 3/3] loading and existing json file to update config, CLI --- mio/cli/process.py | 25 ++++++++++++++++++-- mio/utils/__init__.py | 1 + mio/utils/config.py | 54 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 78 insertions(+), 2 deletions(-) create mode 100644 mio/utils/__init__.py create mode 100644 mio/utils/config.py diff --git a/mio/cli/process.py b/mio/cli/process.py index bdd5938a..dfe67f1b 100644 --- a/mio/cli/process.py +++ b/mio/cli/process.py @@ -2,10 +2,14 @@ Command line interface for offline video pre-processing. """ +from pathlib import Path +from typing import Optional + import click from mio.models.process import DenoiseConfig from mio.process.video import denoise_run +from mio.utils.config import update_config_from_json @click.group() @@ -31,12 +35,29 @@ def process() -> None: type=str, help="Path to the YAML processing configuration file.", ) +@click.option( + "-u", + "--update-from-json", + type=click.Path(exists=True, path_type=Path), + help="Update config from a processing_log.json file", +) +@click.option( + "--save-yaml", + is_flag=True, + help="Save updated config back to denoise_example.yml", +) def denoise( input: str, denoise_config: str, + update_from_json: Optional[Path] = None, + save_yaml: bool = False, ) -> None: """ Denoise a video file. """ - denoise_config_parsed = DenoiseConfig.from_any(denoise_config) - denoise_run(input, denoise_config_parsed) + config = DenoiseConfig.from_any(denoise_config) + + if update_from_json: + config = update_config_from_json(config, update_from_json) + + denoise_run(input, config) diff --git a/mio/utils/__init__.py b/mio/utils/__init__.py new file mode 100644 index 00000000..c09c3593 --- /dev/null +++ b/mio/utils/__init__.py @@ -0,0 +1 @@ +"""Utility functions for miniscope-io.""" diff --git a/mio/utils/config.py b/mio/utils/config.py new file mode 100644 index 00000000..bd9fb6c5 --- /dev/null +++ b/mio/utils/config.py @@ -0,0 +1,54 @@ +"""Configuration utilities for miniscope-io.""" + +import json +from pathlib import Path + +from mio.models.process import DenoiseConfig + + +def update_config_from_json(config: DenoiseConfig, json_path: Path) -> DenoiseConfig: + """Update DenoiseConfig from a processing_log.json file.""" + with open(json_path) as f: + log_data = json.load(f) + + # Get most recent processing run + proc_config = log_data["processing_runs"][-1]["processing_config"] + + # Update noise patch config + config.noise_patch.enable = proc_config["noise_patch"]["enabled"] + if config.noise_patch.enable: + config.noise_patch.method = proc_config["noise_patch"]["method"] + if proc_config["noise_patch"]["gradient_config"]: + config.noise_patch.gradient_config.threshold = proc_config["noise_patch"][ + "gradient_config" + ]["threshold"] + if proc_config["noise_patch"]["black_area_config"]: + config.noise_patch.black_area_config.consecutive_threshold = proc_config["noise_patch"][ + "black_area_config" + ]["consecutive_threshold"] + config.noise_patch.black_area_config.value_threshold = proc_config["noise_patch"][ + "black_area_config" + ]["value_threshold"] + + # Update frequency masking config + config.frequency_masking.enable = proc_config["frequency_masking"]["enabled"] + if config.frequency_masking.enable: + config.frequency_masking.cast_float32 = proc_config["frequency_masking"]["cast_float32"] + config.frequency_masking.spatial_LPF_cutoff_radius = proc_config["frequency_masking"][ + "cutoff_radius" + ] + config.frequency_masking.vertical_BEF_cutoff = proc_config["frequency_masking"][ + "vertical_BEF" + ] + config.frequency_masking.horizontal_BEF_cutoff = proc_config["frequency_masking"][ + "horizontal_BEF" + ] + + # Update butterworth config + config.butter_filter.enable = proc_config["butterworth"]["enabled"] + if config.butter_filter.enable: + config.butter_filter.order = proc_config["butterworth"]["order"] + config.butter_filter.cutoff_frequency = proc_config["butterworth"]["cutoff_frequency"] + config.butter_filter.sampling_rate = proc_config["butterworth"]["sampling_rate"] + + return config