diff --git a/COCA_scripts.zip b/COCA_scripts.zip deleted file mode 100644 index a369776..0000000 Binary files a/COCA_scripts.zip and /dev/null differ diff --git a/Collaboration/Links.md b/Collaboration/Links.md deleted file mode 100644 index 5087dc0..0000000 --- a/Collaboration/Links.md +++ /dev/null @@ -1,7 +0,0 @@ -Link to shared ideas/collab document: https://bama365-my.sharepoint.com/:w:/g/personal/kebutler2_crimson_ua_edu/ET9znazsBptAukJhoX8cFCsBjJSPzr7HlVZ0llgPaEjYDg?e=C0hvnl - -Link to meeting notes document: https://bama365-my.sharepoint.com/:w:/g/personal/kebutler2_crimson_ua_edu/EbpOB8hlf8dKj5zfoQos0k0B7VaXBB6-SOXR9i7phhyz6g?e=uhV8uW - -Link to project roadmap document: https://bama365-my.sharepoint.com/:w:/g/personal/kebutler2_crimson_ua_edu/EUHGUEECqL1GsJUzY5cEHwsBppvCseBaL6_Y0ghsy2SOUQ?e=E5Hj1T - -Link to approaches document: https://bama365-my.sharepoint.com/:w:/g/personal/kebutler2_crimson_ua_edu/EUYBPzP4CBtEnNXG0R8ptL8BsEAu79vcnBxSOoTT9pYO8w?e=vytxT8 diff --git a/Stanford COCA download instructions.md b/Stanford COCA download instructions.md deleted file mode 100644 index c454a1b..0000000 --- a/Stanford COCA download instructions.md +++ /dev/null @@ -1,27 +0,0 @@ -Hello interested contributors! Here’s how to download the dataset from COCA. Unfortunately, we cannot directly deliver data because of Stanford’s Terms of Use. Each person must individually upload and parse the dataset. However, we will provide the tools with which to do so! - -1. Go to Stanford AIMI datasets, find COCA, or follow this link: - -https://stanfordaimi.azurewebsites.net/ - -2. Make an account with AIMI, login to view the dataset, accept the license - -3. Download the dataset using one of the available options - -4. Download the following scripts to the same folder. They are in the COCA_scripts zip file - - a. Unnester: Fixes the format of the gated files to remove the intermediate scan label folder - - b. COCA_processor: class that creates 3D files of both the DICOM image and segmentations, as well as an output table - - c. COCA_resampler: class that resamples the voxels to what you would like (recommended .7 x .7 x 3 mm) - - d. COCA_pipeline: runs COCA processor and resampler - -5. Run Unnester to fix the Gated dataset - -6. Make sure COCA_processor and COCA_resampler are set up correctly to run in COCA_pipeline, taking care to tailor them to your file organization - -7. Run COCA_pipeline - -8. If you would like to view the files, I recommend downloading 3D Slicer, and uploading the nii image and segmentation zip files together using the DATA upload option. Make sure to indicate the segmentation file is a segmentation rather than a volume. diff --git a/coca_project/src/00e44a7f29de_img.nii b/coca_project/src/00e44a7f29de_img.nii new file mode 100644 index 0000000..1961755 Binary files /dev/null and b/coca_project/src/00e44a7f29de_img.nii differ diff --git a/coca_project/src/COCA_metadata_compiler.py b/coca_project/src/COCA_metadata_compiler.py new file mode 100644 index 0000000..ac04332 --- /dev/null +++ b/coca_project/src/COCA_metadata_compiler.py @@ -0,0 +1,94 @@ +import json +import pandas as pd +import SimpleITK as sitk +from pathlib import Path +from tqdm import tqdm +import numpy as np + +class COCAMetadataCompiler: + def __init__(self, project_root: str): + self.project_root = Path(project_root) + self.resampled_dir = self.project_root / "data_resampled" + self.canonical_dir = self.project_root / "data_canonical" / "images" + self.output_path = self.project_root / "data_canonical" / "tables" / "metadata_summary.parquet" + + # Ensure table directory exists + self.output_path.parent.mkdir(parents=True, exist_ok=True) + + def run(self): + all_meta = [] + + # We iterate through the resampled folder because these are the files + # your model will actually use for training. + scan_folders = [f for f in self.resampled_dir.iterdir() if f.is_dir()] + print(f"Compiling metadata for {len(scan_folders)} resampled scans...") + + for scan_folder in tqdm(scan_folders): + scan_id = scan_folder.name + + # 1. Define Paths + img_path = scan_folder / f"{scan_id}_img.nii.gz" + seg_path = scan_folder / f"{scan_id}_seg.nii.gz" + json_path = self.canonical_dir / scan_id / f"{scan_id}_meta.json" + + if not img_path.exists() or not seg_path.exists(): + continue + + try: + # 2. Load the actual resampled mask to get accurate counts + # This fixes the 'Voxel sum mismatch' error + res_seg = sitk.ReadImage(str(seg_path)) + res_seg_array = sitk.GetArrayFromImage(res_seg) + + # Calculate counts based on the CURRENT file state + current_voxels = int(np.sum(res_seg_array > 0)) + current_slices = np.where(np.any(res_seg_array > 0, axis=(1, 2)))[0].tolist() + + # Get geometry info + spacing = res_seg.GetSpacing() + shape = res_seg.GetSize() + + # 3. Pull patient info from the original JSON if it exists + patient_id = "Unknown" + if json_path.exists(): + with open(json_path, 'r') as f: + meta_data = json.load(f) + patient_id = meta_data.get("patient_id", "Unknown") + + # 4. Build the Row + row = { + "scan_id": scan_id, + "patient_id": patient_id, + "image_path": str(img_path.resolve()), + "mask_path": str(seg_path.resolve()), + "has_pos": 1 if current_voxels > 0 else 0, + "n_pos_voxels": current_voxels, + "n_pos_slices": len(current_slices), + "spacing_x": spacing[0], + "spacing_y": spacing[1], + "spacing_z": spacing[2], + "shape_x": shape[0], + "shape_y": shape[1], + "shape_z": shape[2] + } + all_meta.append(row) + + except Exception as e: + print(f" [ERROR] Processing {scan_id}: {e}") + + if not all_meta: + print("No data found to compile.") + return + + # Create DataFrame and Export + df = pd.DataFrame(all_meta) + df.to_parquet(self.output_path, index=False, compression='snappy') + + print("-" * 50) + print(f"SUCCESS: Metadata compiled at {self.output_path}") + print(f"Positive cases: {df['has_pos'].sum()} / {len(df)}") + print("-" * 50) + +if __name__ == "__main__": + compiler = COCAMetadataCompiler(r"C:\coca_project") + compiler.run() \ No newline at end of file diff --git a/coca_project/src/COCA_processor.py b/coca_project/src/COCA_processor.py new file mode 100644 index 0000000..faa7f5d --- /dev/null +++ b/coca_project/src/COCA_processor.py @@ -0,0 +1,396 @@ +import json +import hashlib +import plistlib +import subprocess +from pathlib import Path + +import numpy as np +import pandas as pd +import SimpleITK as sitk +import cv2 +from tqdm import tqdm + + +# --------------------------------------------------------------------------- +# CONFIG +# --------------------------------------------------------------------------- +TARGET_SPACING = (.5, .5, 3) # mm — isotropic resampling target +USE_TOTAL_SEGMENTATOR = True # Set False to skip heart masking +TS_DEVICE = "gpu" # "gpu" or "cpu" +# --------------------------------------------------------------------------- + + +class COCAProcessor: + def __init__(self, project_root: str): + self.project_root = Path(project_root) + self.dicom_root = ( + self.project_root + / "data_raw" / "dicom" + / "Gated_release_final" / "Gated_release_final" / "patient" + ) + self.xml_root = self.project_root / "data_raw" / "xml" / "calcium_xml" + + self.out_images_base = self.project_root / "data_canonical" / "images" + self.out_tables = self.project_root / "data_canonical" / "tables" + + self.out_images_base.mkdir(parents=True, exist_ok=True) + self.out_tables.mkdir(parents=True, exist_ok=True) + + # ------------------------------------------------------------------ + # Utilities + # ------------------------------------------------------------------ + + @staticmethod + def generate_stable_id(*parts: str, n: int = 12) -> str: + h = hashlib.sha1("||".join(parts).encode("utf-8")).hexdigest() + return h[:n] + + @staticmethod + def resample_image( + image: sitk.Image, + new_spacing: tuple = TARGET_SPACING, + interpolator=sitk.sitkLinear, + ) -> sitk.Image: + """Resample to isotropic voxel spacing.""" + orig_spacing = image.GetSpacing() + orig_size = image.GetSize() + + new_size = [ + int(round(orig_size[i] * orig_spacing[i] / new_spacing[i])) + for i in range(3) + ] + + resampler = sitk.ResampleImageFilter() + resampler.SetOutputSpacing(new_spacing) + resampler.SetSize(new_size) + resampler.SetOutputDirection(image.GetDirection()) + resampler.SetOutputOrigin(image.GetOrigin()) + resampler.SetTransform(sitk.Transform()) + resampler.SetDefaultPixelValue(-1024) + resampler.SetInterpolator(interpolator) + return resampler.Execute(image) + + # ------------------------------------------------------------------ + # Label parsing + # ------------------------------------------------------------------ + + def parse_plist_filled(self, xml_path: Path, image_shape: tuple): + """ + Parse COCA XML annotation and return a binary 3D mask. + image_shape is (Z, Y, X) — SimpleITK array convention. + """ + mask = np.zeros(image_shape, dtype=np.uint8) + segmented_slices = set() + total_z, total_y, total_x = image_shape + + if not xml_path.exists(): + return mask, [] + + try: + with open(xml_path, "rb") as f: + data = plistlib.load(f) + + for img_entry in data.get("Images", []): + z = int(img_entry.get("ImageIndex", -1)) + if z < 0 or z >= total_z: + continue + + for roi in img_entry.get("ROIs", []): + points_str = roi.get("Point_px", []) + if not points_str: + continue + + poly_points = [] + for p_str in points_str: + cleaned = p_str.replace("(", "").replace(")", "") + parts = cleaned.split(",") + if len(parts) == 2: + poly_points.append([float(parts[0]), float(parts[1])]) + + if poly_points: + pts = np.array(poly_points, dtype=np.int32) + temp_slice = np.zeros((total_y, total_x), dtype=np.uint8) + if len(pts) > 2: + cv2.fillPoly(temp_slice, [pts], 1) + else: + for p in pts: + if 0 <= p[0] < total_x and 0 <= p[1] < total_y: + temp_slice[int(p[1]), int(p[0])] = 1 + + if np.any(temp_slice): + mask[z] = np.logical_or(mask[z], temp_slice).astype(np.uint8) + segmented_slices.add(z) + + except Exception as e: + print(f" [PARSING ERROR] {xml_path.name}: {e}") + + return mask, sorted(segmented_slices) + + # ------------------------------------------------------------------ + # TotalSegmentator heart masking + # ------------------------------------------------------------------ + + def run_total_segmentator( + self, input_nii: Path, output_dir: Path + ) -> sitk.Image | None: + """ + Run TotalSegmentator to generate a cardiac ROI mask. + Returns a binary SimpleITK image, or None on failure. + + The cardiac mask is the union of: heart, aorta, pulmonary_artery. + Keeping the aorta is important — aortic calcification is a common + false-positive source that you still want to capture in training labels. + """ + output_dir.mkdir(parents=True, exist_ok=True) + + cmd = [ + "TotalSegmentator", + "-i", str(input_nii), + "-o", str(output_dir), + "--roi_subset", "heart", "aorta", + # pulmonary_artery removed — not a valid class name in TS v2.x + # heart + aorta is sufficient for cardiac calcium ROI masking + "--device", TS_DEVICE, + "--ml", # multi-label output (one file per structure) + ] + + try: + # Capture stderr only (not stdout) so we can log errors without + # deadlocking on the large stdout progress bar output from TS. + result = subprocess.run( + cmd, + check=True, + timeout=300, + stdout=subprocess.DEVNULL, # discard progress bars + stderr=subprocess.PIPE, # capture errors only + text=True, + ) + except subprocess.TimeoutExpired: + print(" [TS ERROR] TotalSegmentator timed out (>5 min) — skipping") + return None + except subprocess.CalledProcessError as e: + print(f" [TS ERROR] TotalSegmentator returned exit code {e.returncode}") + if e.stderr: + print(f" [TS STDERR] {e.stderr[:600]}") + return None + except FileNotFoundError: + print(" [TS ERROR] TotalSegmentator not found — is it installed in this environment?") + return None + + # TS v2 with --ml writes a single multi-label file named after the + # output argument (ts_tmp.nii), sitting inside the scan folder — + # NOT separate per-structure files inside the ts_tmp subdirectory. + # Each structure is a different integer label; we union heart + aorta. + ts_output = output_dir.parent / f"{output_dir.name}.nii" + if not ts_output.exists(): + # Fallback: also check for .nii.gz variant + ts_output_gz = output_dir.parent / f"{output_dir.name}.nii.gz" + if ts_output_gz.exists(): + ts_output = ts_output_gz + else: + print(f" [TS WARNING] Expected output not found at {ts_output}") + return None + + seg = sitk.ReadImage(str(ts_output)) + arr = sitk.GetArrayFromImage(seg) + + # Print unique label values on first scan so we can verify heart/aorta IDs + unique_labels = sorted(np.unique(arr).tolist()) + if len(unique_labels) <= 10: + print(f" [TS INFO] Label values in output: {unique_labels}") + + # TS v2 total task label IDs: heart=51, aorta=52 + # If these look wrong check the printed label values above and update. + HEART_LABEL = 51 + AORTA_LABEL = 52 + combined = ((arr == HEART_LABEL) | (arr == AORTA_LABEL)).astype(np.uint8) + + if combined.max() == 0: + print(f" [TS WARNING] Heart/aorta labels ({HEART_LABEL},{AORTA_LABEL}) " + f"not found in output. Found labels: {unique_labels}. " + f"Update HEART_LABEL/AORTA_LABEL constants in run_total_segmentator().") + return None + + cardiac_mask = sitk.GetImageFromArray(combined) + cardiac_mask.CopyInformation(seg) + return cardiac_mask + + # ------------------------------------------------------------------ + # Discovery + # ------------------------------------------------------------------ + + def discover_series(self) -> list[Path]: + """ + Find all DICOM series under dicom_root. + Uses the *grandparent* folder name as the patient ID to be robust + against COCA's nested folder structure (patient/study/series). + """ + print(f"Scanning {self.dicom_root} for DICOM series...") + all_series, found_dirs = [], set() + for p in self.dicom_root.rglob("*.dcm"): + if p.parent not in found_dirs: + dcm_files = list(p.parent.glob("*.dcm")) + if len(dcm_files) >= 5: + all_series.append(p.parent) + found_dirs.add(p.parent) + print(f" → Found {len(all_series)} valid series.") + return all_series + + @staticmethod + def _patient_id_from_path(series_dir: Path, dicom_root: Path) -> str: + """ + Walk up from series_dir until we hit a direct child of dicom_root. + That child's name is the patient-level folder — much safer than + using s_dir.name which could be a series UID. + """ + parts = series_dir.relative_to(dicom_root).parts + return parts[0] if parts else series_dir.name + + # ------------------------------------------------------------------ + # Main loop + # ------------------------------------------------------------------ + + def process_all(self): + series_dirs = self.discover_series() + rows = [] + + for s_dir in tqdm(series_dirs, desc="Processing Scans"): + patient_id = self._patient_id_from_path(s_dir, self.dicom_root) + xml_path = self.xml_root / f"{patient_id}.xml" + + try: + # ── 1. Load DICOM ────────────────────────────────────── + reader = sitk.ImageSeriesReader() + dicom_names = reader.GetGDCMSeriesFileNames(str(s_dir)) + reader.SetFileNames(dicom_names) + image = reader.Execute() + + # ── 2. Resample to isotropic spacing ─────────────────── + image_iso = self.resample_image(image, TARGET_SPACING, sitk.sitkLinear) + + # ── 3. Parse calcium annotation ──────────────────────── + img_array = sitk.GetArrayFromImage(image) # original for mask drawing + mask_array, seg_slices = self.parse_plist_filled(xml_path, img_array.shape) + + # Resample binary mask with nearest-neighbour to preserve labels + mask_sitk = sitk.GetImageFromArray(mask_array) + mask_sitk.CopyInformation(image) + mask_sitk_iso = self.resample_image( + sitk.Cast(mask_sitk, sitk.sitkFloat32), + TARGET_SPACING, + sitk.sitkNearestNeighbor, + ) + mask_sitk_iso = sitk.Cast(mask_sitk_iso, sitk.sitkUInt8) + + voxel_count = int( + np.sum(sitk.GetArrayFromImage(mask_sitk_iso)) + ) + + if xml_path.exists() and voxel_count == 0: + print( + f"\n [WARNING] Patient {patient_id}: XML exists but " + f"0 voxels after resampling. Check slice alignment." + ) + + # ── 4. Set up output folder ──────────────────────────── + scan_id = self.generate_stable_id(str(s_dir.resolve()), patient_id) + scan_folder = self.out_images_base / scan_id + scan_folder.mkdir(parents=True, exist_ok=True) + + image_path = scan_folder / f"{scan_id}_img.nii.gz" + mask_path = scan_folder / f"{scan_id}_seg.nii.gz" + + sitk.WriteImage(image_iso, str(image_path), useCompression=True) + sitk.WriteImage(mask_sitk_iso, str(mask_path), useCompression=True) + + # ── 5. TotalSegmentator cardiac mask (optional) ──────── + cardiac_mask_path = None + if USE_TOTAL_SEGMENTATOR: + ts_tmp_dir = scan_folder / "ts_tmp" + cardiac_mask = self.run_total_segmentator(image_path, ts_tmp_dir) + + if cardiac_mask is not None: + ct_ref = sitk.ReadImage(str(image_path)) + resampler = sitk.ResampleImageFilter() + resampler.SetReferenceImage(ct_ref) + resampler.SetInterpolator(sitk.sitkNearestNeighbor) + resampler.SetDefaultPixelValue(0) + cardiac_mask = resampler.Execute(cardiac_mask) + # Fill internal holes slice by slice + fill = sitk.BinaryFillholeImageFilter() + fill.SetFullyConnected(True) + cardiac_mask = fill.Execute(cardiac_mask) + + # Then dilate outward to capture epicardial surface + dilate = sitk.BinaryDilateImageFilter() + dilate.SetKernelRadius(7) # ~7-10mm depending on your voxel spacing + cardiac_mask = dilate.Execute(cardiac_mask) + + if cardiac_mask is not None: + cardiac_mask_path = scan_folder / f"{scan_id}_cardiac_mask.nii.gz" + sitk.WriteImage( + cardiac_mask, str(cardiac_mask_path), useCompression=True + ) + + # Sanity check: do any calcium labels fall inside the mask? + ca_arr = sitk.GetArrayFromImage(mask_sitk_iso) + cm_arr = sitk.GetArrayFromImage(cardiac_mask) + overlap = int(np.sum(ca_arr & cm_arr)) + if voxel_count > 0 and overlap == 0: + print( + f"\n [WARNING] Patient {patient_id}: calcium labels " + f"do NOT overlap with cardiac mask — check registration." + ) + + # Clean up large intermediate TS files to save disk space + for f in ts_tmp_dir.glob("*.nii.gz"): + if f.name not in ( + "heart.nii.gz", "aorta.nii.gz", "pulmonary_artery.nii.gz" + ): + f.unlink(missing_ok=True) + + # ── 6. Metadata ──────────────────────────────────────── + meta = { + "scan_id": scan_id, + "patient_id": patient_id, + "original_spacing": list(image.GetSpacing()), + "resampled_spacing": list(TARGET_SPACING), + "original_size": list(image.GetSize()), + "resampled_size": list(image_iso.GetSize()), + "calcium_voxels": voxel_count, + "slices_with_calcium": seg_slices, + "cardiac_mask_path": str(cardiac_mask_path) if cardiac_mask_path else None, + "original_path": str(s_dir), + } + (scan_folder / f"{scan_id}_meta.json").write_text( + json.dumps(meta, indent=2) + ) + + rows.append({ + "patient_id": patient_id, + "scan_id": scan_id, + "image_path": str(image_path), + "mask_path": str(mask_path), + "cardiac_mask_path": str(cardiac_mask_path) if cardiac_mask_path else None, + "voxels": voxel_count, + "has_calcium": voxel_count > 0, + "num_slices": len(seg_slices), + "folder_path": str(scan_folder), + }) + + except Exception as e: + print(f" [ERROR] Patient {patient_id}: {e}") + + if rows: + df = pd.DataFrame(rows) + out_csv = self.out_tables / "scan_index.csv" + df.to_csv(out_csv, index=False) + print(f"\n✓ Processing complete.") + print(f" {len(df)} scans written → {out_csv}") + print(f" Positive (calcium > 0): {df['has_calcium'].sum()}") + print(f" Negative: {(~df['has_calcium']).sum()}") + + +if __name__ == "__main__": + processor = COCAProcessor(r"C:\coca_project") + processor.process_all() \ No newline at end of file diff --git a/coca_project/src/COCA_scripts.zip b/coca_project/src/COCA_scripts.zip new file mode 100644 index 0000000..1ad21e0 Binary files /dev/null and b/coca_project/src/COCA_scripts.zip differ diff --git a/coca_project/src/COCA_scripts/COCA_pipeline.py b/coca_project/src/COCA_scripts/COCA_pipeline.py new file mode 100644 index 0000000..f11c408 --- /dev/null +++ b/coca_project/src/COCA_scripts/COCA_pipeline.py @@ -0,0 +1,60 @@ +import sys +import os +from pathlib import Path + +# 1. Get the absolute path of the directory where THIS script is located +# This bypasses all 'current working directory' issues +SCRIPT_DIR = Path(__file__).resolve().parent + +# 2. Force this directory into the system path at the very beginning +if str(SCRIPT_DIR) not in sys.path: + sys.path.insert(0, str(SCRIPT_DIR)) + +# 3. Diagnostic check (optional but helpful) +print(f"Pipeline running from: {SCRIPT_DIR}") + +try: + # Now we import using the exact class names + from COCA_processor import COCAProcessor + from COCA_resampler import COCAResampler + print("Successfully imported COCA modules.") +except ImportError as e: + print(f"\n[STILL FAILING]: {e}") + print(f"I am looking for files in: {SCRIPT_DIR}") + print(f"Files found there: {os.listdir(SCRIPT_DIR)}") + sys.exit(1) + +def main(): + print("="*50) + print(" COCA DATA PREPROCESSING PIPELINE") + print("="*50) + + # 1. Configuration + default_root = r"C:\coca_project" + project_root = input(f"Project Root [{default_root}]: ").strip() or default_root + + # 2. Execution Logic + print("\n1) Full Pipeline\n2) Process Only\n3) Resample Only") + choice = input("Selection: ").strip() + + if choice in ['1', '2']: + print("\n--- Running Processor ---") + proc = COCAProcessor(project_root) + proc.process_all() + + if choice in ['1', '3']: + print("\n--- Running Resampler ---") + space = input("Voxel Spacing (mm) [1.0] or [x,y,z]:").strip() or "1.0" + if "," in space: + target = [float(x.strip()) for x in space.split(",")] + if len(target) ! =3: + raise ValueError("Please enter spacing as x,y,z") + else: + target = [float(space)] * 3 + resamp = COCAResampler(project_root, target_spacing=target) + resamp.run() + + print("\nPipeline Finished.") + +if __name__ == "__main__": + main() diff --git a/coca_project/src/COCA_scripts/COCA_processor.py b/coca_project/src/COCA_scripts/COCA_processor.py new file mode 100644 index 0000000..3cb8290 --- /dev/null +++ b/coca_project/src/COCA_scripts/COCA_processor.py @@ -0,0 +1,167 @@ +import os +import json +import hashlib +import plistlib +from pathlib import Path +from collections import defaultdict +import numpy as np +import pandas as pd +import SimpleITK as sitk +import cv2 +from tqdm import tqdm + +class COCAProcessor: + def __init__(self, project_root: str): + self.project_root = Path(project_root) + self.dicom_root = self.project_root / "data_raw" / "dicom" / "Gated_release_final" / "Gated_release_final" / "patient" + self.xml_root = self.project_root / "data_raw" / "xml" / "calcium_xml" + + self.out_images_base = self.project_root / "data_canonical" / "images" + self.out_tables = self.project_root / "data_canonical" / "tables" + + # Ensure output directories exist + self.out_images_base.mkdir(parents=True, exist_ok=True) + self.out_tables.mkdir(parents=True, exist_ok=True) + + @staticmethod + def generate_stable_id(*parts: str, n: int = 12) -> str: + """Generates a unique, reproducible ID for each scan.""" + h = hashlib.sha1("||".join(parts).encode("utf-8")).hexdigest() + return h[:n] + + def parse_plist_filled(self, xml_path: Path, image_shape: tuple): + """Parses XML and returns a binary 3D mask and list of segmented slices.""" + mask = np.zeros(image_shape, dtype=np.uint8) + segmented_slices = set() + total_z, total_y, total_x = image_shape + + if not xml_path.exists(): + return mask, [] + + try: + with open(xml_path, 'rb') as f: + data = plistlib.load(f) + + images = data.get('Images', []) + for img_entry in images: + z = int(img_entry.get('ImageIndex', -1)) + + if z < 0 or z >= total_z: + continue + + rois = img_entry.get('ROIs', []) + for roi in rois: + points_str = roi.get('Point_px', []) + if not points_str: + continue + + poly_points = [] + for p_str in points_str: + cleaned = p_str.replace("(", "").replace(")", "") + parts = cleaned.split(",") + if len(parts) == 2: + poly_points.append([float(parts[0]), float(parts[1])]) + + if poly_points: + pts = np.array(poly_points, dtype=np.int32) + temp_slice = np.zeros((total_y, total_x), dtype=np.uint8) + + if len(pts) > 2: + cv2.fillPoly(temp_slice, [pts], 1) + else: + for p in pts: + if 0 <= p[0] < total_x and 0 <= p[1] < total_y: + temp_slice[int(p[1]), int(p[0])] = 1 + + if np.any(temp_slice): + mask[z, :, :] = np.logical_or(mask[z, :, :], temp_slice).astype(np.uint8) + segmented_slices.add(z) + + except Exception as e: + print(f" [PARSING ERROR] {xml_path.name}: {e}") + + return mask, sorted(list(segmented_slices)) + + def discover_series(self): + """Scans the DICOM root for folders containing at least 5 DICOM files.""" + print(f"Scanning {self.dicom_root} for DICOM series...") + all_series = [] + found_dirs = set() + for p in self.dicom_root.rglob("*.dcm"): + if p.parent not in found_dirs: + if len(list(p.parent.glob("*.dcm"))) >= 5: + all_series.append(p.parent) + found_dirs.add(p.parent) + return all_series + + def process_all(self): + """Main execution loop to process all discovered DICOM series.""" + series_dirs = self.discover_series() + print(f"Found {len(series_dirs)} valid series. Starting processing...") + + rows = [] + for s_dir in tqdm(series_dirs, desc="Processing Scans"): + patient_id = s_dir.name + xml_path = self.xml_root / f"{patient_id}.xml" + + try: + # Load DICOM Volume + reader = sitk.ImageSeriesReader() + dicom_names = reader.GetGDCMSeriesFileNames(str(s_dir)) + reader.SetFileNames(dicom_names) + image = reader.Execute() + + img_array = sitk.GetArrayFromImage(image) + + # Generate Mask + mask_array, seg_slices = self.parse_plist_filled(xml_path, img_array.shape) + voxel_count = int(np.sum(mask_array)) + + if xml_path.exists() and voxel_count == 0: + print(f"\n [WARNING] Patient {patient_id}: XML exists but 0 voxels drawn. Check slice alignment.") + + # Setup output folder + scan_id = self.generate_stable_id(str(s_dir.resolve()), patient_id) + scan_folder = self.out_images_base / scan_id + scan_folder.mkdir(parents=True, exist_ok=True) + + # Save Image + sitk.WriteImage(image, str(scan_folder / f"{scan_id}_img.nii.gz"), useCompression=True) + + # Save Mask (inheriting geometry from original image) + mask_image = sitk.GetImageFromArray(mask_array) + mask_image.CopyInformation(image) + sitk.WriteImage(mask_image, str(scan_folder / f"{scan_id}_seg.nii.gz"), useCompression=True) + + # Metadata + meta = { + "scan_id": scan_id, + "patient_id": patient_id, + "calcium_voxels": voxel_count, + "slices_with_calcium": seg_slices, + "original_path": str(s_dir) + } + (scan_folder / f"{scan_id}_meta.json").write_text(json.dumps(meta, indent=2)) + + rows.append({ + "patient_id": patient_id, + "scan_id": scan_id, + "voxels": voxel_count, + "num_slices": len(seg_slices), + "folder_path": str(scan_folder) # Useful for the resampling script later + }) + + except Exception as e: + print(f" [ERROR] Patient {patient_id}: {e}") + + if rows: + df = pd.DataFrame(rows) + df.to_csv(self.out_tables / "scan_index.csv", index=False) + print(f"\nProcessing complete. Check {self.out_tables}/scan_index.csv for results.") + +if __name__ == "__main__": + # This part only runs if you run COCA_processor.py DIRECTLY. + # It will NOT run when imported by your pipeline script. + print("Running Processor in standalone mode...") + processor = COCAProcessor(r"C:\coca_project") + processor.process_all() \ No newline at end of file diff --git a/coca_project/src/COCA_scripts/COCA_resampler.py b/coca_project/src/COCA_scripts/COCA_resampler.py new file mode 100644 index 0000000..bc1673a --- /dev/null +++ b/coca_project/src/COCA_scripts/COCA_resampler.py @@ -0,0 +1,92 @@ +import pandas as pd +import SimpleITK as sitk +from pathlib import Path +from tqdm import tqdm + +class COCAResampler: + def __init__(self, project_root: str, target_spacing: list = [0.7, 0.7, 3.0]): + """ + Initializes the resampler. + target_spacing: [x, y, z] in mm. [1.0, 1.0, 1.0] creates isotropic voxels. + """ + self.project_root = Path(project_root) + self.input_csv = self.project_root / "data_canonical" / "tables" / "scan_index.csv" + self.output_dir = self.project_root / "data_resampled" + self.target_spacing = target_spacing + + # Create output directory + self.output_dir.mkdir(parents=True, exist_ok=True) + + def resample_volume(self, volume: sitk.Image, is_mask: bool = False) -> sitk.Image: + """ + Resamples a single SimpleITK image to the target spacing. + Uses Linear interpolation for images and Nearest Neighbor for masks. + """ + original_spacing = volume.GetSpacing() + original_size = volume.GetSize() + + # Calculate new size to maintain physical extent + # NewSize = OldSize * (OldSpacing / NewSpacing) + new_size = [ + int(round(original_size[i] * (original_spacing[i] / self.target_spacing[i]))) + for i in range(3) + ] + + resample = sitk.ResampleImageFilter() + resample.SetOutputSpacing(self.target_spacing) + resample.SetSize(new_size) + resample.SetOutputDirection(volume.GetDirection()) + resample.SetOutputOrigin(volume.GetOrigin()) + resample.SetTransform(sitk.Transform()) + resample.SetDefaultPixelValue(volume.GetPixelIDValue()) + + if is_mask: + # Nearest Neighbor prevents creating new label values (keeps it 0 and 1) + resample.SetInterpolator(sitk.sitkNearestNeighbor) + else: + # Linear provides smoother anatomical transitions + resample.SetInterpolator(sitk.sitkLinear) + + return resample.Execute(volume) + + def run(self): + """Processes all scans listed in the scan_index.csv.""" + if not self.input_csv.exists(): + print(f"[ERROR] Could not find {self.input_csv}. Run the Processor first.") + return + + df = pd.read_csv(self.input_csv) + print(f"Starting resampling of {len(df)} scans to {self.target_spacing} mm...") + + for _, row in tqdm(df.iterrows(), total=len(df), desc="Resampling"): + scan_id = row['scan_id'] + # We use the folder_path saved in the CSV by the previous class + input_folder = Path(row['folder_path']) + + resampled_folder = self.output_dir / scan_id + resampled_folder.mkdir(parents=True, exist_ok=True) + + try: + # 1. Load Original NIfTI files + img_path = input_folder / f"{scan_id}_img.nii.gz" + seg_path = input_folder / f"{scan_id}_seg.nii.gz" + + img = sitk.ReadImage(str(img_path)) + seg = sitk.ReadImage(str(seg_path)) + + # 2. Perform Resampling + res_img = self.resample_volume(img, is_mask=False) + res_seg = self.resample_volume(seg, is_mask=True) + + # 3. Save Resampled Results + sitk.WriteImage(res_img, str(resampled_folder / f"{scan_id}_img.nii.gz"), useCompression=True) + sitk.WriteImage(res_seg, str(resampled_folder / f"{scan_id}_seg.nii.gz"), useCompression=True) + + except Exception as e: + print(f" [ERROR] Failed to resample {scan_id}: {e}") + + print(f"\nResampling complete. Files saved to: {self.output_dir}") + +if __name__ == "__main__": + resampler = COCAResampler(r"C:\coca_project", target_spacing=[.7, .7, 3.0]) + resampler.run() \ No newline at end of file diff --git a/coca_project/src/COCA_scripts/unnester.py b/coca_project/src/COCA_scripts/unnester.py new file mode 100644 index 0000000..3bdde93 --- /dev/null +++ b/coca_project/src/COCA_scripts/unnester.py @@ -0,0 +1,45 @@ +import os +import shutil +from pathlib import Path +from tqdm import tqdm + +def flatten_dicom_folders(root_dir): + root_path = Path(root_dir) + # Get all patient folders (the numeric ones) + patient_folders = [p for p in root_path.iterdir() if p.is_dir() and p.name.isdigit()] + + print(f"Checking {len(patient_folders)} patient folders for intermediate nesting...") + + for patient_dir in tqdm(patient_folders, desc="Flattening"): + # Find all .dcm files anywhere inside this patient directory + all_dcms = list(patient_dir.rglob("*.dcm")) + + for dcm_path in all_dcms: + # If the file is not already directly in the patient folder + if dcm_path.parent != patient_dir: + # Move to the patient_dir + target_path = patient_dir / dcm_path.name + + # Handle potential filename collisions (unlikely in DICOM) + if target_path.exists(): + target_path = patient_dir / f"{dcm_path.parent.name}_{dcm_path.name}" + + shutil.move(str(dcm_path), str(target_path)) + + # Cleanup: Remove now-empty subdirectories + for subfolder in list(patient_dir.iterdir()): + if subfolder.is_dir(): + try: + # Only removes if empty + shutil.rmtree(subfolder) + except Exception: + # If it contains non-dcm files, it stays + pass + +if __name__ == "__main__": + # Update this to your exact patient root + DICOM_PATIENT_ROOT = r"C:\coca_project\data_raw\dicom\Gated_release_final\Gated_release_final\patient" + + # It's always a good idea to have a backup or try on one folder first! + flatten_dicom_folders(DICOM_PATIENT_ROOT) + print("\nFlattening complete. All slices should now be directly inside patient ID folders.") \ No newline at end of file diff --git a/coca_project/src/Loader_check.py b/coca_project/src/Loader_check.py new file mode 100644 index 0000000..9c928b1 --- /dev/null +++ b/coca_project/src/Loader_check.py @@ -0,0 +1,89 @@ +import pandas as pd +import SimpleITK as sitk +import numpy as np +from pathlib import Path +import random + +def sanity_check_load(parquet_path, num_samples=20): + # 1. Load your metadata contract + if not Path(parquet_path).exists(): + print(f"❌ Error: Parquet file not found at {parquet_path}") + return + + df = pd.read_parquet(parquet_path) + + # Check if required columns exist to avoid KeyErrors + required_cols = ['image_path', 'mask_path', 'n_pos_voxels', 'scan_id'] + missing_cols = [c for c in required_cols if c not in df.columns] + if missing_cols: + print(f"❌ Error: Parquet is missing columns: {missing_cols}") + print(f"Available columns: {list(df.columns)}") + return + + # Select random cases + samples = df.sample(min(num_samples, len(df))).to_dict('records') + + print(f"--- Starting Sanity Check on {len(samples)} samples ---") + print(f"Target Orientation: RAS") + + results = {"pass": 0, "fail": 0} + + for case in samples: + case_id = case['scan_id'] + img_p = case['image_path'] + msk_p = case['mask_path'] + + # Verify files actually exist on disk where the parquet says they are + if not Path(img_p).exists() or not Path(msk_p).exists(): + print(f"❌ [FILE MISSING] {case_id}: Check if paths in parquet are absolute and correct.") + results["fail"] += 1 + continue + + try: + # 2. Load Image and Mask + img = sitk.ReadImage(img_p) + mask = sitk.ReadImage(msk_p) + + # --- Check A: Geometry/Affine Match --- + # Using absolute tolerance (atol) for floating point comparisons + spacing_match = np.allclose(img.GetSpacing(), mask.GetSpacing(), atol=1e-5) + origin_match = np.allclose(img.GetOrigin(), mask.GetOrigin(), atol=1e-5) + direction_match = np.allclose(img.GetDirection(), mask.GetDirection(), atol=1e-5) + size_match = img.GetSize() == mask.GetSize() + + # --- Check B: Orientation Check (RAS) --- + # Standard RAS direction matrix is diagonal [1, 0, 0, 0, 1, 0, 0, 0, 1] + # SimpleITK returns directions as a flattened tuple + is_ras = np.allclose(img.GetDirection(), (1.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 1.0), atol=1e-3) + + if not (spacing_match and origin_match and direction_match and size_match): + print(f"❌ [GEOMETRY ERROR] {case_id}: Image/Mask spatial mismatch.") + results["fail"] += 1 + continue + + if not is_ras: + print(f"⚠️ [ORIENTATION WARNING] {case_id}: Volume is not in standard RAS orientation.") + + # --- Check C: Voxel Count Consistency --- + mask_array = sitk.GetArrayFromImage(mask) + current_voxels = int(np.sum(mask_array > 0)) + expected_voxels = int(case['n_pos_voxels']) + + if current_voxels != expected_voxels: + print(f"❌ [DATA ERROR] {case_id}: Voxel sum mismatch! Table: {expected_voxels}, File: {current_voxels}") + results["fail"] += 1 + else: + print(f"✅ {case_id}: Passed (Voxels: {current_voxels}, Size: {img.GetSize()})") + results["pass"] += 1 + + except Exception as e: + print(f"❌ [LOAD ERROR] {case_id}: {str(e)}") + results["fail"] += 1 + + print(f"\n--- Sanity Check Complete ---") + print(f"Passed: {results['pass']} | Failed: {results['fail']}") + +if __name__ == "__main__": + # Ensure this matches the EXACT filename of the parquet you just created + PARQUET_FILE = r"C:\coca_project\data_canonical\tables\metadata_summary.parquet" + sanity_check_load(PARQUET_FILE) \ No newline at end of file diff --git a/coca_project/src/__init__.py b/coca_project/src/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/coca_project/src/__pycache__/COCA_processor.cpython-310.pyc b/coca_project/src/__pycache__/COCA_processor.cpython-310.pyc new file mode 100644 index 0000000..376a8a4 Binary files /dev/null and b/coca_project/src/__pycache__/COCA_processor.cpython-310.pyc differ diff --git a/coca_project/src/__pycache__/COCA_resampler.cpython-310.pyc b/coca_project/src/__pycache__/COCA_resampler.cpython-310.pyc new file mode 100644 index 0000000..4abe601 Binary files /dev/null and b/coca_project/src/__pycache__/COCA_resampler.cpython-310.pyc differ diff --git a/coca_project/src/assign_xml_to_scan.py.py b/coca_project/src/assign_xml_to_scan.py.py new file mode 100644 index 0000000..0b95d9e --- /dev/null +++ b/coca_project/src/assign_xml_to_scan.py.py @@ -0,0 +1,231 @@ +import re +import numpy as np +import pandas as pd +import xml.etree.ElementTree as ET +import nibabel as nib + +# ---------------------------- +# Parsing helpers +# ---------------------------- + +def parse_triplet(s): + nums = re.findall(r"[-+]?\d*\.\d+|[-+]?\d+", s) + if len(nums) < 3: + return None + return tuple(float(x) for x in nums[:3]) + +def extract_roi_z_from_xml(xml_path): + tree = ET.parse(xml_path) + root = tree.getroot() + + z_vals = [] + + for d in root.findall(".//dict"): + children = list(d) + kv = {} + for i in range(0, len(children) - 1, 2): + if children[i].tag == "key": + kv[children[i].text] = children[i + 1] + + if "Center" in kv and kv["Center"].text: + t = parse_triplet(kv["Center"].text) + if t: + z_vals.append(t[2]) + + if "Point_mm" in kv and kv["Point_mm"].tag == "array": + for s in kv["Point_mm"].findall("string"): + if s.text: + t = parse_triplet(s.text) + if t: + z_vals.append(t[2]) + + z_vals = np.array(z_vals, dtype=float) + # downweight polygon density by using unique-ish z values + z_vals = np.unique(np.round(z_vals, 3)) + return z_vals + +def slice_z_positions_from_nifti(nifti_path, slice_axis=2): + img = nib.load(nifti_path) + aff = img.affine + shape = img.shape + + if slice_axis == 2: + ks = np.arange(shape[2]) + pts = np.stack([np.zeros_like(ks), np.zeros_like(ks), ks, np.ones_like(ks)], axis=0) + elif slice_axis == 0: + ks = np.arange(shape[0]) + pts = np.stack([ks, np.zeros_like(ks), np.zeros_like(ks), np.ones_like(ks)], axis=0) + elif slice_axis == 1: + ks = np.arange(shape[1]) + pts = np.stack([np.zeros_like(ks), ks, np.zeros_like(ks), np.ones_like(ks)], axis=0) + else: + raise ValueError("slice_axis must be 0, 1, or 2") + + world = aff @ pts + return world[2, :] + +def score_scan(z_roi, z_slices, spacing_z_mm=None): + z_roi = np.asarray(z_roi) + z_slices = np.asarray(z_slices) + + zs = np.sort(z_slices) + idx = np.searchsorted(zs, z_roi) + idx0 = np.clip(idx - 1, 0, len(zs) - 1) + idx1 = np.clip(idx, 0, len(zs) - 1) + d = np.minimum(np.abs(z_roi - zs[idx0]), np.abs(z_roi - zs[idx1])) + + tol = 2.0 + if spacing_z_mm is not None and float(spacing_z_mm) > 0: + tol = 0.6 * float(spacing_z_mm) + + return (d <= tol).mean(), d.mean(), tol + +# ---------------------------- +# Main: patient-folder-only matching +# ---------------------------- + +def assign_xml_to_scan_per_patient( + scans_df: pd.DataFrame, + xml_index_df: pd.DataFrame, + patient_col: str = "patient_folder", + xml_path_col: str = "xml_path", + scan_id_col: str = "scan_id", + nifti_col: str = "nifti_path", + spacing_col: str = "spacing_z_mm", +): + """ + xml_index_df: one row per patient_folder containing xml_path + scans_df: scan table with patient_folder and nifti paths + """ + results = [] + + for _, xr in xml_index_df.iterrows(): + pf = xr[patient_col] + xml_path = xr[xml_path_col] + + # Filter to ONLY scans for this patient_folder + cand = scans_df[scans_df[patient_col] == pf].copy() + + # If no scans or only 1 scan, skip heavy work + if len(cand) == 0: + results.append({ + patient_col: pf, + "xml_path": xml_path, + "status": "no_scans_found" + }) + continue + + if len(cand) == 1: + results.append({ + patient_col: pf, + "xml_path": xml_path, + "best_scan_id": cand.iloc[0][scan_id_col], + "status": "single_scan_only" + }) + continue + + # Only consider candidates with valid nifti paths + cand = cand[cand[nifti_col].astype(str).str.len() > 0] + if len(cand) == 0: + results.append({ + patient_col: pf, + "xml_path": xml_path, + "status": "no_valid_nifti_paths" + }) + continue + + z_roi = extract_roi_z_from_xml(xml_path) + if len(z_roi) == 0: + results.append({ + patient_col: pf, + "xml_path": xml_path, + "status": "no_roi_z_found" + }) + continue + + scored = [] + for _, sr in cand.iterrows(): + nifti_path = sr[nifti_col] + z_slices = slice_z_positions_from_nifti(nifti_path, slice_axis=2) + in_tol, mae, tol = score_scan(z_roi, z_slices, spacing_z_mm=sr.get(spacing_col, None)) + scored.append({ + patient_col: pf, + "xml_path": xml_path, + "scan_id": sr[scan_id_col], + "in_tol_frac": in_tol, + "mae_mm": mae, + "tol_mm": tol, + }) + + scored_df = pd.DataFrame(scored).sort_values(["in_tol_frac", "mae_mm"], ascending=[False, True]).reset_index(drop=True) + best = scored_df.iloc[0] + second = scored_df.iloc[1] if len(scored_df) > 1 else None + + out = { + patient_col: pf, + "xml_path": xml_path, + "best_scan_id": best["scan_id"], + "best_in_tol_frac": float(best["in_tol_frac"]), + "best_mae_mm": float(best["mae_mm"]), + "tol_mm": float(best["tol_mm"]), + "status": "matched" + } + if second is not None: + out.update({ + "second_scan_id": second["scan_id"], + "second_in_tol_frac": float(second["in_tol_frac"]), + "second_mae_mm": float(second["mae_mm"]), + }) + + results.append(out) + + return pd.DataFrame(results) + +# Example usage: +# xml_index_df = rois_df[['patient_folder','xml_path']].drop_duplicates() +# assignments = assign_xml_to_scan_per_patient(scans_df, xml_index_df) +# assignments.to_csv("roi_xml_scan_assignments.csv", index=False) + +if __name__ == "__main__": + import pandas as pd + + # ---- EDIT THESE PATHS ---- + SCANS_CSV = "data_canonical/tables/Gated_scan_index.csv" + ROIS_CSV = "data_canonical/tables/calcium_rois_sorted_by_patient_folder.csv" + OUTPUT_CSV = "data_canonical/tables/roi_xml_scan_assignments.csv" + + # ---- LOAD TABLES ---- + scans_df = pd.read_csv(SCANS_CSV) + + rois_df = pd.read_csv(ROIS_CSV) + rois_df["patient_folder"] = ( + rois_df["xml_path"] + .str.extract(r'\\calcium_xml\\(\d+)\.xml', expand=False) + .astype(int) + ) + + # one XML per patient_folder + xml_index_df = rois_df[["patient_folder", "xml_path"]].drop_duplicates() + + # ---- RUN MATCHING (patient-folder–restricted) ---- + assignments = assign_xml_to_scan_per_patient( + scans_df=scans_df, + xml_index_df=xml_index_df, + patient_col="patient_folder", + scan_id_col="scan_id", + nifti_col="nifti_path", + spacing_col="spacing_z_mm", + ) + + assignments.to_csv(OUTPUT_CSV, index=False) + print(f"Wrote {OUTPUT_CSV}") + + + + + + +for /l %i in (1,1,20) do robocopy ^ + C:\coca_project\data_raw\dicom\Gated_release_final\Gated_release_final\patient\%i ^ + C:\Users\Jagdf\PrediCT\coca_project\sample_scans\%i ^ + /E diff --git a/coca_project/src/audit_images/epoch0_batch0.png b/coca_project/src/audit_images/epoch0_batch0.png new file mode 100644 index 0000000..bba83fa Binary files /dev/null and b/coca_project/src/audit_images/epoch0_batch0.png differ diff --git a/coca_project/src/audit_images/epoch0_batch10.png b/coca_project/src/audit_images/epoch0_batch10.png new file mode 100644 index 0000000..f29c020 Binary files /dev/null and b/coca_project/src/audit_images/epoch0_batch10.png differ diff --git a/coca_project/src/audit_images/epoch0_batch100.png b/coca_project/src/audit_images/epoch0_batch100.png new file mode 100644 index 0000000..5e0492a Binary files /dev/null and b/coca_project/src/audit_images/epoch0_batch100.png differ diff --git a/coca_project/src/audit_images/epoch0_batch110.png b/coca_project/src/audit_images/epoch0_batch110.png new file mode 100644 index 0000000..938eea1 Binary files /dev/null and b/coca_project/src/audit_images/epoch0_batch110.png differ diff --git a/coca_project/src/audit_images/epoch0_batch120.png b/coca_project/src/audit_images/epoch0_batch120.png new file mode 100644 index 0000000..aff853e Binary files /dev/null and b/coca_project/src/audit_images/epoch0_batch120.png differ diff --git a/coca_project/src/audit_images/epoch0_batch130.png b/coca_project/src/audit_images/epoch0_batch130.png new file mode 100644 index 0000000..85e2990 Binary files /dev/null and b/coca_project/src/audit_images/epoch0_batch130.png differ diff --git a/coca_project/src/audit_images/epoch0_batch140.png b/coca_project/src/audit_images/epoch0_batch140.png new file mode 100644 index 0000000..cd88520 Binary files /dev/null and b/coca_project/src/audit_images/epoch0_batch140.png differ diff --git a/coca_project/src/audit_images/epoch0_batch150.png b/coca_project/src/audit_images/epoch0_batch150.png new file mode 100644 index 0000000..261495a Binary files /dev/null and b/coca_project/src/audit_images/epoch0_batch150.png differ diff --git a/coca_project/src/audit_images/epoch0_batch160.png b/coca_project/src/audit_images/epoch0_batch160.png new file mode 100644 index 0000000..da44c29 Binary files /dev/null and b/coca_project/src/audit_images/epoch0_batch160.png differ diff --git a/coca_project/src/audit_images/epoch0_batch170.png b/coca_project/src/audit_images/epoch0_batch170.png new file mode 100644 index 0000000..16e406f Binary files /dev/null and b/coca_project/src/audit_images/epoch0_batch170.png differ diff --git a/coca_project/src/audit_images/epoch0_batch180.png b/coca_project/src/audit_images/epoch0_batch180.png new file mode 100644 index 0000000..ac40385 Binary files /dev/null and b/coca_project/src/audit_images/epoch0_batch180.png differ diff --git a/coca_project/src/audit_images/epoch0_batch190.png b/coca_project/src/audit_images/epoch0_batch190.png new file mode 100644 index 0000000..d135977 Binary files /dev/null and b/coca_project/src/audit_images/epoch0_batch190.png differ diff --git a/coca_project/src/audit_images/epoch0_batch20.png b/coca_project/src/audit_images/epoch0_batch20.png new file mode 100644 index 0000000..24f8f5b Binary files /dev/null and b/coca_project/src/audit_images/epoch0_batch20.png differ diff --git a/coca_project/src/audit_images/epoch0_batch200.png b/coca_project/src/audit_images/epoch0_batch200.png new file mode 100644 index 0000000..2e5accf Binary files /dev/null and b/coca_project/src/audit_images/epoch0_batch200.png differ diff --git a/coca_project/src/audit_images/epoch0_batch210.png b/coca_project/src/audit_images/epoch0_batch210.png new file mode 100644 index 0000000..5a9ed2c Binary files /dev/null and b/coca_project/src/audit_images/epoch0_batch210.png differ diff --git a/coca_project/src/audit_images/epoch0_batch220.png b/coca_project/src/audit_images/epoch0_batch220.png new file mode 100644 index 0000000..c732954 Binary files /dev/null and b/coca_project/src/audit_images/epoch0_batch220.png differ diff --git a/coca_project/src/audit_images/epoch0_batch230.png b/coca_project/src/audit_images/epoch0_batch230.png new file mode 100644 index 0000000..9d7d244 Binary files /dev/null and b/coca_project/src/audit_images/epoch0_batch230.png differ diff --git a/coca_project/src/audit_images/epoch0_batch240.png b/coca_project/src/audit_images/epoch0_batch240.png new file mode 100644 index 0000000..783a4ea Binary files /dev/null and b/coca_project/src/audit_images/epoch0_batch240.png differ diff --git a/coca_project/src/audit_images/epoch0_batch250.png b/coca_project/src/audit_images/epoch0_batch250.png new file mode 100644 index 0000000..20db087 Binary files /dev/null and b/coca_project/src/audit_images/epoch0_batch250.png differ diff --git a/coca_project/src/audit_images/epoch0_batch260.png b/coca_project/src/audit_images/epoch0_batch260.png new file mode 100644 index 0000000..fbafb52 Binary files /dev/null and b/coca_project/src/audit_images/epoch0_batch260.png differ diff --git a/coca_project/src/audit_images/epoch0_batch270.png b/coca_project/src/audit_images/epoch0_batch270.png new file mode 100644 index 0000000..00549b3 Binary files /dev/null and b/coca_project/src/audit_images/epoch0_batch270.png differ diff --git a/coca_project/src/audit_images/epoch0_batch280.png b/coca_project/src/audit_images/epoch0_batch280.png new file mode 100644 index 0000000..970ba37 Binary files /dev/null and b/coca_project/src/audit_images/epoch0_batch280.png differ diff --git a/coca_project/src/audit_images/epoch0_batch290.png b/coca_project/src/audit_images/epoch0_batch290.png new file mode 100644 index 0000000..4888af1 Binary files /dev/null and b/coca_project/src/audit_images/epoch0_batch290.png differ diff --git a/coca_project/src/audit_images/epoch0_batch30.png b/coca_project/src/audit_images/epoch0_batch30.png new file mode 100644 index 0000000..06ae500 Binary files /dev/null and b/coca_project/src/audit_images/epoch0_batch30.png differ diff --git a/coca_project/src/audit_images/epoch0_batch300.png b/coca_project/src/audit_images/epoch0_batch300.png new file mode 100644 index 0000000..bea3eb8 Binary files /dev/null and b/coca_project/src/audit_images/epoch0_batch300.png differ diff --git a/coca_project/src/audit_images/epoch0_batch310.png b/coca_project/src/audit_images/epoch0_batch310.png new file mode 100644 index 0000000..0c1960f Binary files /dev/null and b/coca_project/src/audit_images/epoch0_batch310.png differ diff --git a/coca_project/src/audit_images/epoch0_batch320.png b/coca_project/src/audit_images/epoch0_batch320.png new file mode 100644 index 0000000..9309864 Binary files /dev/null and b/coca_project/src/audit_images/epoch0_batch320.png differ diff --git a/coca_project/src/audit_images/epoch0_batch330.png b/coca_project/src/audit_images/epoch0_batch330.png new file mode 100644 index 0000000..aa19b8e Binary files /dev/null and b/coca_project/src/audit_images/epoch0_batch330.png differ diff --git a/coca_project/src/audit_images/epoch0_batch340.png b/coca_project/src/audit_images/epoch0_batch340.png new file mode 100644 index 0000000..701b906 Binary files /dev/null and b/coca_project/src/audit_images/epoch0_batch340.png differ diff --git a/coca_project/src/audit_images/epoch0_batch350.png b/coca_project/src/audit_images/epoch0_batch350.png new file mode 100644 index 0000000..b96a4e3 Binary files /dev/null and b/coca_project/src/audit_images/epoch0_batch350.png differ diff --git a/coca_project/src/audit_images/epoch0_batch360.png b/coca_project/src/audit_images/epoch0_batch360.png new file mode 100644 index 0000000..17c3e90 Binary files /dev/null and b/coca_project/src/audit_images/epoch0_batch360.png differ diff --git a/coca_project/src/audit_images/epoch0_batch370.png b/coca_project/src/audit_images/epoch0_batch370.png new file mode 100644 index 0000000..156983e Binary files /dev/null and b/coca_project/src/audit_images/epoch0_batch370.png differ diff --git a/coca_project/src/audit_images/epoch0_batch380.png b/coca_project/src/audit_images/epoch0_batch380.png new file mode 100644 index 0000000..c2a623d Binary files /dev/null and b/coca_project/src/audit_images/epoch0_batch380.png differ diff --git a/coca_project/src/audit_images/epoch0_batch390.png b/coca_project/src/audit_images/epoch0_batch390.png new file mode 100644 index 0000000..6fb6f96 Binary files /dev/null and b/coca_project/src/audit_images/epoch0_batch390.png differ diff --git a/coca_project/src/audit_images/epoch0_batch40.png b/coca_project/src/audit_images/epoch0_batch40.png new file mode 100644 index 0000000..3630f23 Binary files /dev/null and b/coca_project/src/audit_images/epoch0_batch40.png differ diff --git a/coca_project/src/audit_images/epoch0_batch400.png b/coca_project/src/audit_images/epoch0_batch400.png new file mode 100644 index 0000000..953ac04 Binary files /dev/null and b/coca_project/src/audit_images/epoch0_batch400.png differ diff --git a/coca_project/src/audit_images/epoch0_batch410.png b/coca_project/src/audit_images/epoch0_batch410.png new file mode 100644 index 0000000..4d924f7 Binary files /dev/null and b/coca_project/src/audit_images/epoch0_batch410.png differ diff --git a/coca_project/src/audit_images/epoch0_batch420.png b/coca_project/src/audit_images/epoch0_batch420.png new file mode 100644 index 0000000..ba88bd3 Binary files /dev/null and b/coca_project/src/audit_images/epoch0_batch420.png differ diff --git a/coca_project/src/audit_images/epoch0_batch430.png b/coca_project/src/audit_images/epoch0_batch430.png new file mode 100644 index 0000000..1a6d527 Binary files /dev/null and b/coca_project/src/audit_images/epoch0_batch430.png differ diff --git a/coca_project/src/audit_images/epoch0_batch440.png b/coca_project/src/audit_images/epoch0_batch440.png new file mode 100644 index 0000000..9417c06 Binary files /dev/null and b/coca_project/src/audit_images/epoch0_batch440.png differ diff --git a/coca_project/src/audit_images/epoch0_batch450.png b/coca_project/src/audit_images/epoch0_batch450.png new file mode 100644 index 0000000..79aac96 Binary files /dev/null and b/coca_project/src/audit_images/epoch0_batch450.png differ diff --git a/coca_project/src/audit_images/epoch0_batch460.png b/coca_project/src/audit_images/epoch0_batch460.png new file mode 100644 index 0000000..f03edcf Binary files /dev/null and b/coca_project/src/audit_images/epoch0_batch460.png differ diff --git a/coca_project/src/audit_images/epoch0_batch470.png b/coca_project/src/audit_images/epoch0_batch470.png new file mode 100644 index 0000000..113d3be Binary files /dev/null and b/coca_project/src/audit_images/epoch0_batch470.png differ diff --git a/coca_project/src/audit_images/epoch0_batch480.png b/coca_project/src/audit_images/epoch0_batch480.png new file mode 100644 index 0000000..aa99c69 Binary files /dev/null and b/coca_project/src/audit_images/epoch0_batch480.png differ diff --git a/coca_project/src/audit_images/epoch0_batch490.png b/coca_project/src/audit_images/epoch0_batch490.png new file mode 100644 index 0000000..20fa010 Binary files /dev/null and b/coca_project/src/audit_images/epoch0_batch490.png differ diff --git a/coca_project/src/audit_images/epoch0_batch50.png b/coca_project/src/audit_images/epoch0_batch50.png new file mode 100644 index 0000000..a638eb5 Binary files /dev/null and b/coca_project/src/audit_images/epoch0_batch50.png differ diff --git a/coca_project/src/audit_images/epoch0_batch500.png b/coca_project/src/audit_images/epoch0_batch500.png new file mode 100644 index 0000000..01abec0 Binary files /dev/null and b/coca_project/src/audit_images/epoch0_batch500.png differ diff --git a/coca_project/src/audit_images/epoch0_batch510.png b/coca_project/src/audit_images/epoch0_batch510.png new file mode 100644 index 0000000..69b601d Binary files /dev/null and b/coca_project/src/audit_images/epoch0_batch510.png differ diff --git a/coca_project/src/audit_images/epoch0_batch520.png b/coca_project/src/audit_images/epoch0_batch520.png new file mode 100644 index 0000000..cc89449 Binary files /dev/null and b/coca_project/src/audit_images/epoch0_batch520.png differ diff --git a/coca_project/src/audit_images/epoch0_batch530.png b/coca_project/src/audit_images/epoch0_batch530.png new file mode 100644 index 0000000..e134741 Binary files /dev/null and b/coca_project/src/audit_images/epoch0_batch530.png differ diff --git a/coca_project/src/audit_images/epoch0_batch540.png b/coca_project/src/audit_images/epoch0_batch540.png new file mode 100644 index 0000000..0a9f4f8 Binary files /dev/null and b/coca_project/src/audit_images/epoch0_batch540.png differ diff --git a/coca_project/src/audit_images/epoch0_batch60.png b/coca_project/src/audit_images/epoch0_batch60.png new file mode 100644 index 0000000..bfe8644 Binary files /dev/null and b/coca_project/src/audit_images/epoch0_batch60.png differ diff --git a/coca_project/src/audit_images/epoch0_batch70.png b/coca_project/src/audit_images/epoch0_batch70.png new file mode 100644 index 0000000..2c0cc79 Binary files /dev/null and b/coca_project/src/audit_images/epoch0_batch70.png differ diff --git a/coca_project/src/audit_images/epoch0_batch80.png b/coca_project/src/audit_images/epoch0_batch80.png new file mode 100644 index 0000000..9052cce Binary files /dev/null and b/coca_project/src/audit_images/epoch0_batch80.png differ diff --git a/coca_project/src/audit_images/epoch0_batch90.png b/coca_project/src/audit_images/epoch0_batch90.png new file mode 100644 index 0000000..ef5faac Binary files /dev/null and b/coca_project/src/audit_images/epoch0_batch90.png differ diff --git a/coca_project/src/audit_images/epoch1_batch0.png b/coca_project/src/audit_images/epoch1_batch0.png new file mode 100644 index 0000000..37b1bef Binary files /dev/null and b/coca_project/src/audit_images/epoch1_batch0.png differ diff --git a/coca_project/src/audit_images/epoch1_batch10.png b/coca_project/src/audit_images/epoch1_batch10.png new file mode 100644 index 0000000..f1eee37 Binary files /dev/null and b/coca_project/src/audit_images/epoch1_batch10.png differ diff --git a/coca_project/src/audit_images/epoch1_batch100.png b/coca_project/src/audit_images/epoch1_batch100.png new file mode 100644 index 0000000..196c25c Binary files /dev/null and b/coca_project/src/audit_images/epoch1_batch100.png differ diff --git a/coca_project/src/audit_images/epoch1_batch110.png b/coca_project/src/audit_images/epoch1_batch110.png new file mode 100644 index 0000000..19abb1f Binary files /dev/null and b/coca_project/src/audit_images/epoch1_batch110.png differ diff --git a/coca_project/src/audit_images/epoch1_batch120.png b/coca_project/src/audit_images/epoch1_batch120.png new file mode 100644 index 0000000..569c2cf Binary files /dev/null and b/coca_project/src/audit_images/epoch1_batch120.png differ diff --git a/coca_project/src/audit_images/epoch1_batch130.png b/coca_project/src/audit_images/epoch1_batch130.png new file mode 100644 index 0000000..c33f423 Binary files /dev/null and b/coca_project/src/audit_images/epoch1_batch130.png differ diff --git a/coca_project/src/audit_images/epoch1_batch140.png b/coca_project/src/audit_images/epoch1_batch140.png new file mode 100644 index 0000000..8ecf30f Binary files /dev/null and b/coca_project/src/audit_images/epoch1_batch140.png differ diff --git a/coca_project/src/audit_images/epoch1_batch150.png b/coca_project/src/audit_images/epoch1_batch150.png new file mode 100644 index 0000000..e990fd5 Binary files /dev/null and b/coca_project/src/audit_images/epoch1_batch150.png differ diff --git a/coca_project/src/audit_images/epoch1_batch160.png b/coca_project/src/audit_images/epoch1_batch160.png new file mode 100644 index 0000000..a1f5ec9 Binary files /dev/null and b/coca_project/src/audit_images/epoch1_batch160.png differ diff --git a/coca_project/src/audit_images/epoch1_batch170.png b/coca_project/src/audit_images/epoch1_batch170.png new file mode 100644 index 0000000..38051a9 Binary files /dev/null and b/coca_project/src/audit_images/epoch1_batch170.png differ diff --git a/coca_project/src/audit_images/epoch1_batch180.png b/coca_project/src/audit_images/epoch1_batch180.png new file mode 100644 index 0000000..fa1828e Binary files /dev/null and b/coca_project/src/audit_images/epoch1_batch180.png differ diff --git a/coca_project/src/audit_images/epoch1_batch190.png b/coca_project/src/audit_images/epoch1_batch190.png new file mode 100644 index 0000000..1eb0074 Binary files /dev/null and b/coca_project/src/audit_images/epoch1_batch190.png differ diff --git a/coca_project/src/audit_images/epoch1_batch20.png b/coca_project/src/audit_images/epoch1_batch20.png new file mode 100644 index 0000000..dfcd026 Binary files /dev/null and b/coca_project/src/audit_images/epoch1_batch20.png differ diff --git a/coca_project/src/audit_images/epoch1_batch200.png b/coca_project/src/audit_images/epoch1_batch200.png new file mode 100644 index 0000000..267c671 Binary files /dev/null and b/coca_project/src/audit_images/epoch1_batch200.png differ diff --git a/coca_project/src/audit_images/epoch1_batch210.png b/coca_project/src/audit_images/epoch1_batch210.png new file mode 100644 index 0000000..fd4027b Binary files /dev/null and b/coca_project/src/audit_images/epoch1_batch210.png differ diff --git a/coca_project/src/audit_images/epoch1_batch220.png b/coca_project/src/audit_images/epoch1_batch220.png new file mode 100644 index 0000000..53b6ece Binary files /dev/null and b/coca_project/src/audit_images/epoch1_batch220.png differ diff --git a/coca_project/src/audit_images/epoch1_batch230.png b/coca_project/src/audit_images/epoch1_batch230.png new file mode 100644 index 0000000..b7bf170 Binary files /dev/null and b/coca_project/src/audit_images/epoch1_batch230.png differ diff --git a/coca_project/src/audit_images/epoch1_batch240.png b/coca_project/src/audit_images/epoch1_batch240.png new file mode 100644 index 0000000..e6b9f1e Binary files /dev/null and b/coca_project/src/audit_images/epoch1_batch240.png differ diff --git a/coca_project/src/audit_images/epoch1_batch250.png b/coca_project/src/audit_images/epoch1_batch250.png new file mode 100644 index 0000000..af2dc8b Binary files /dev/null and b/coca_project/src/audit_images/epoch1_batch250.png differ diff --git a/coca_project/src/audit_images/epoch1_batch260.png b/coca_project/src/audit_images/epoch1_batch260.png new file mode 100644 index 0000000..d866251 Binary files /dev/null and b/coca_project/src/audit_images/epoch1_batch260.png differ diff --git a/coca_project/src/audit_images/epoch1_batch270.png b/coca_project/src/audit_images/epoch1_batch270.png new file mode 100644 index 0000000..3eb1330 Binary files /dev/null and b/coca_project/src/audit_images/epoch1_batch270.png differ diff --git a/coca_project/src/audit_images/epoch1_batch280.png b/coca_project/src/audit_images/epoch1_batch280.png new file mode 100644 index 0000000..4d77394 Binary files /dev/null and b/coca_project/src/audit_images/epoch1_batch280.png differ diff --git a/coca_project/src/audit_images/epoch1_batch290.png b/coca_project/src/audit_images/epoch1_batch290.png new file mode 100644 index 0000000..1ebd095 Binary files /dev/null and b/coca_project/src/audit_images/epoch1_batch290.png differ diff --git a/coca_project/src/audit_images/epoch1_batch30.png b/coca_project/src/audit_images/epoch1_batch30.png new file mode 100644 index 0000000..6b81adc Binary files /dev/null and b/coca_project/src/audit_images/epoch1_batch30.png differ diff --git a/coca_project/src/audit_images/epoch1_batch300.png b/coca_project/src/audit_images/epoch1_batch300.png new file mode 100644 index 0000000..04c6767 Binary files /dev/null and b/coca_project/src/audit_images/epoch1_batch300.png differ diff --git a/coca_project/src/audit_images/epoch1_batch310.png b/coca_project/src/audit_images/epoch1_batch310.png new file mode 100644 index 0000000..c45d3c3 Binary files /dev/null and b/coca_project/src/audit_images/epoch1_batch310.png differ diff --git a/coca_project/src/audit_images/epoch1_batch320.png b/coca_project/src/audit_images/epoch1_batch320.png new file mode 100644 index 0000000..a8ce0be Binary files /dev/null and b/coca_project/src/audit_images/epoch1_batch320.png differ diff --git a/coca_project/src/audit_images/epoch1_batch330.png b/coca_project/src/audit_images/epoch1_batch330.png new file mode 100644 index 0000000..70f0bea Binary files /dev/null and b/coca_project/src/audit_images/epoch1_batch330.png differ diff --git a/coca_project/src/audit_images/epoch1_batch340.png b/coca_project/src/audit_images/epoch1_batch340.png new file mode 100644 index 0000000..5e4886c Binary files /dev/null and b/coca_project/src/audit_images/epoch1_batch340.png differ diff --git a/coca_project/src/audit_images/epoch1_batch350.png b/coca_project/src/audit_images/epoch1_batch350.png new file mode 100644 index 0000000..3ac163f Binary files /dev/null and b/coca_project/src/audit_images/epoch1_batch350.png differ diff --git a/coca_project/src/audit_images/epoch1_batch360.png b/coca_project/src/audit_images/epoch1_batch360.png new file mode 100644 index 0000000..de848f4 Binary files /dev/null and b/coca_project/src/audit_images/epoch1_batch360.png differ diff --git a/coca_project/src/audit_images/epoch1_batch370.png b/coca_project/src/audit_images/epoch1_batch370.png new file mode 100644 index 0000000..b586243 Binary files /dev/null and b/coca_project/src/audit_images/epoch1_batch370.png differ diff --git a/coca_project/src/audit_images/epoch1_batch380.png b/coca_project/src/audit_images/epoch1_batch380.png new file mode 100644 index 0000000..5093e7a Binary files /dev/null and b/coca_project/src/audit_images/epoch1_batch380.png differ diff --git a/coca_project/src/audit_images/epoch1_batch390.png b/coca_project/src/audit_images/epoch1_batch390.png new file mode 100644 index 0000000..eb300a6 Binary files /dev/null and b/coca_project/src/audit_images/epoch1_batch390.png differ diff --git a/coca_project/src/audit_images/epoch1_batch40.png b/coca_project/src/audit_images/epoch1_batch40.png new file mode 100644 index 0000000..a5cecef Binary files /dev/null and b/coca_project/src/audit_images/epoch1_batch40.png differ diff --git a/coca_project/src/audit_images/epoch1_batch400.png b/coca_project/src/audit_images/epoch1_batch400.png new file mode 100644 index 0000000..657626b Binary files /dev/null and b/coca_project/src/audit_images/epoch1_batch400.png differ diff --git a/coca_project/src/audit_images/epoch1_batch410.png b/coca_project/src/audit_images/epoch1_batch410.png new file mode 100644 index 0000000..e1d0809 Binary files /dev/null and b/coca_project/src/audit_images/epoch1_batch410.png differ diff --git a/coca_project/src/audit_images/epoch1_batch420.png b/coca_project/src/audit_images/epoch1_batch420.png new file mode 100644 index 0000000..d3eb963 Binary files /dev/null and b/coca_project/src/audit_images/epoch1_batch420.png differ diff --git a/coca_project/src/audit_images/epoch1_batch430.png b/coca_project/src/audit_images/epoch1_batch430.png new file mode 100644 index 0000000..ddde41b Binary files /dev/null and b/coca_project/src/audit_images/epoch1_batch430.png differ diff --git a/coca_project/src/audit_images/epoch1_batch440.png b/coca_project/src/audit_images/epoch1_batch440.png new file mode 100644 index 0000000..543f453 Binary files /dev/null and b/coca_project/src/audit_images/epoch1_batch440.png differ diff --git a/coca_project/src/audit_images/epoch1_batch450.png b/coca_project/src/audit_images/epoch1_batch450.png new file mode 100644 index 0000000..6a09162 Binary files /dev/null and b/coca_project/src/audit_images/epoch1_batch450.png differ diff --git a/coca_project/src/audit_images/epoch1_batch460.png b/coca_project/src/audit_images/epoch1_batch460.png new file mode 100644 index 0000000..16b138b Binary files /dev/null and b/coca_project/src/audit_images/epoch1_batch460.png differ diff --git a/coca_project/src/audit_images/epoch1_batch470.png b/coca_project/src/audit_images/epoch1_batch470.png new file mode 100644 index 0000000..401891b Binary files /dev/null and b/coca_project/src/audit_images/epoch1_batch470.png differ diff --git a/coca_project/src/audit_images/epoch1_batch480.png b/coca_project/src/audit_images/epoch1_batch480.png new file mode 100644 index 0000000..bae128f Binary files /dev/null and b/coca_project/src/audit_images/epoch1_batch480.png differ diff --git a/coca_project/src/audit_images/epoch1_batch490.png b/coca_project/src/audit_images/epoch1_batch490.png new file mode 100644 index 0000000..689ecff Binary files /dev/null and b/coca_project/src/audit_images/epoch1_batch490.png differ diff --git a/coca_project/src/audit_images/epoch1_batch50.png b/coca_project/src/audit_images/epoch1_batch50.png new file mode 100644 index 0000000..2cba3f7 Binary files /dev/null and b/coca_project/src/audit_images/epoch1_batch50.png differ diff --git a/coca_project/src/audit_images/epoch1_batch500.png b/coca_project/src/audit_images/epoch1_batch500.png new file mode 100644 index 0000000..316c289 Binary files /dev/null and b/coca_project/src/audit_images/epoch1_batch500.png differ diff --git a/coca_project/src/audit_images/epoch1_batch510.png b/coca_project/src/audit_images/epoch1_batch510.png new file mode 100644 index 0000000..7bbe09e Binary files /dev/null and b/coca_project/src/audit_images/epoch1_batch510.png differ diff --git a/coca_project/src/audit_images/epoch1_batch520.png b/coca_project/src/audit_images/epoch1_batch520.png new file mode 100644 index 0000000..de4d7c9 Binary files /dev/null and b/coca_project/src/audit_images/epoch1_batch520.png differ diff --git a/coca_project/src/audit_images/epoch1_batch530.png b/coca_project/src/audit_images/epoch1_batch530.png new file mode 100644 index 0000000..63efb16 Binary files /dev/null and b/coca_project/src/audit_images/epoch1_batch530.png differ diff --git a/coca_project/src/audit_images/epoch1_batch540.png b/coca_project/src/audit_images/epoch1_batch540.png new file mode 100644 index 0000000..bcaf728 Binary files /dev/null and b/coca_project/src/audit_images/epoch1_batch540.png differ diff --git a/coca_project/src/audit_images/epoch1_batch60.png b/coca_project/src/audit_images/epoch1_batch60.png new file mode 100644 index 0000000..a61d9f7 Binary files /dev/null and b/coca_project/src/audit_images/epoch1_batch60.png differ diff --git a/coca_project/src/audit_images/epoch1_batch70.png b/coca_project/src/audit_images/epoch1_batch70.png new file mode 100644 index 0000000..6ae260d Binary files /dev/null and b/coca_project/src/audit_images/epoch1_batch70.png differ diff --git a/coca_project/src/audit_images/epoch1_batch80.png b/coca_project/src/audit_images/epoch1_batch80.png new file mode 100644 index 0000000..c7ca33d Binary files /dev/null and b/coca_project/src/audit_images/epoch1_batch80.png differ diff --git a/coca_project/src/audit_images/epoch1_batch90.png b/coca_project/src/audit_images/epoch1_batch90.png new file mode 100644 index 0000000..bea326f Binary files /dev/null and b/coca_project/src/audit_images/epoch1_batch90.png differ diff --git a/coca_project/src/audit_images/epoch2_batch0.png b/coca_project/src/audit_images/epoch2_batch0.png new file mode 100644 index 0000000..d99e3a6 Binary files /dev/null and b/coca_project/src/audit_images/epoch2_batch0.png differ diff --git a/coca_project/src/audit_images/epoch2_batch10.png b/coca_project/src/audit_images/epoch2_batch10.png new file mode 100644 index 0000000..098f9f7 Binary files /dev/null and b/coca_project/src/audit_images/epoch2_batch10.png differ diff --git a/coca_project/src/audit_images/epoch2_batch100.png b/coca_project/src/audit_images/epoch2_batch100.png new file mode 100644 index 0000000..b1f2c2d Binary files /dev/null and b/coca_project/src/audit_images/epoch2_batch100.png differ diff --git a/coca_project/src/audit_images/epoch2_batch110.png b/coca_project/src/audit_images/epoch2_batch110.png new file mode 100644 index 0000000..5315d10 Binary files /dev/null and b/coca_project/src/audit_images/epoch2_batch110.png differ diff --git a/coca_project/src/audit_images/epoch2_batch120.png b/coca_project/src/audit_images/epoch2_batch120.png new file mode 100644 index 0000000..bcae687 Binary files /dev/null and b/coca_project/src/audit_images/epoch2_batch120.png differ diff --git a/coca_project/src/audit_images/epoch2_batch130.png b/coca_project/src/audit_images/epoch2_batch130.png new file mode 100644 index 0000000..f01fc6d Binary files /dev/null and b/coca_project/src/audit_images/epoch2_batch130.png differ diff --git a/coca_project/src/audit_images/epoch2_batch140.png b/coca_project/src/audit_images/epoch2_batch140.png new file mode 100644 index 0000000..61dd4ff Binary files /dev/null and b/coca_project/src/audit_images/epoch2_batch140.png differ diff --git a/coca_project/src/audit_images/epoch2_batch150.png b/coca_project/src/audit_images/epoch2_batch150.png new file mode 100644 index 0000000..3063378 Binary files /dev/null and b/coca_project/src/audit_images/epoch2_batch150.png differ diff --git a/coca_project/src/audit_images/epoch2_batch160.png b/coca_project/src/audit_images/epoch2_batch160.png new file mode 100644 index 0000000..88dfe84 Binary files /dev/null and b/coca_project/src/audit_images/epoch2_batch160.png differ diff --git a/coca_project/src/audit_images/epoch2_batch170.png b/coca_project/src/audit_images/epoch2_batch170.png new file mode 100644 index 0000000..3ca4647 Binary files /dev/null and b/coca_project/src/audit_images/epoch2_batch170.png differ diff --git a/coca_project/src/audit_images/epoch2_batch180.png b/coca_project/src/audit_images/epoch2_batch180.png new file mode 100644 index 0000000..d887b5d Binary files /dev/null and b/coca_project/src/audit_images/epoch2_batch180.png differ diff --git a/coca_project/src/audit_images/epoch2_batch190.png b/coca_project/src/audit_images/epoch2_batch190.png new file mode 100644 index 0000000..219f2d2 Binary files /dev/null and b/coca_project/src/audit_images/epoch2_batch190.png differ diff --git a/coca_project/src/audit_images/epoch2_batch20.png b/coca_project/src/audit_images/epoch2_batch20.png new file mode 100644 index 0000000..cb0ea25 Binary files /dev/null and b/coca_project/src/audit_images/epoch2_batch20.png differ diff --git a/coca_project/src/audit_images/epoch2_batch200.png b/coca_project/src/audit_images/epoch2_batch200.png new file mode 100644 index 0000000..7913936 Binary files /dev/null and b/coca_project/src/audit_images/epoch2_batch200.png differ diff --git a/coca_project/src/audit_images/epoch2_batch210.png b/coca_project/src/audit_images/epoch2_batch210.png new file mode 100644 index 0000000..ecf61f1 Binary files /dev/null and b/coca_project/src/audit_images/epoch2_batch210.png differ diff --git a/coca_project/src/audit_images/epoch2_batch220.png b/coca_project/src/audit_images/epoch2_batch220.png new file mode 100644 index 0000000..f0deab6 Binary files /dev/null and b/coca_project/src/audit_images/epoch2_batch220.png differ diff --git a/coca_project/src/audit_images/epoch2_batch230.png b/coca_project/src/audit_images/epoch2_batch230.png new file mode 100644 index 0000000..e139973 Binary files /dev/null and b/coca_project/src/audit_images/epoch2_batch230.png differ diff --git a/coca_project/src/audit_images/epoch2_batch240.png b/coca_project/src/audit_images/epoch2_batch240.png new file mode 100644 index 0000000..d721599 Binary files /dev/null and b/coca_project/src/audit_images/epoch2_batch240.png differ diff --git a/coca_project/src/audit_images/epoch2_batch250.png b/coca_project/src/audit_images/epoch2_batch250.png new file mode 100644 index 0000000..c2dc3e9 Binary files /dev/null and b/coca_project/src/audit_images/epoch2_batch250.png differ diff --git a/coca_project/src/audit_images/epoch2_batch260.png b/coca_project/src/audit_images/epoch2_batch260.png new file mode 100644 index 0000000..d6afe21 Binary files /dev/null and b/coca_project/src/audit_images/epoch2_batch260.png differ diff --git a/coca_project/src/audit_images/epoch2_batch270.png b/coca_project/src/audit_images/epoch2_batch270.png new file mode 100644 index 0000000..1b2be71 Binary files /dev/null and b/coca_project/src/audit_images/epoch2_batch270.png differ diff --git a/coca_project/src/audit_images/epoch2_batch280.png b/coca_project/src/audit_images/epoch2_batch280.png new file mode 100644 index 0000000..f65fcc6 Binary files /dev/null and b/coca_project/src/audit_images/epoch2_batch280.png differ diff --git a/coca_project/src/audit_images/epoch2_batch290.png b/coca_project/src/audit_images/epoch2_batch290.png new file mode 100644 index 0000000..62ab34f Binary files /dev/null and b/coca_project/src/audit_images/epoch2_batch290.png differ diff --git a/coca_project/src/audit_images/epoch2_batch30.png b/coca_project/src/audit_images/epoch2_batch30.png new file mode 100644 index 0000000..df750a8 Binary files /dev/null and b/coca_project/src/audit_images/epoch2_batch30.png differ diff --git a/coca_project/src/audit_images/epoch2_batch300.png b/coca_project/src/audit_images/epoch2_batch300.png new file mode 100644 index 0000000..a5d124b Binary files /dev/null and b/coca_project/src/audit_images/epoch2_batch300.png differ diff --git a/coca_project/src/audit_images/epoch2_batch310.png b/coca_project/src/audit_images/epoch2_batch310.png new file mode 100644 index 0000000..af59073 Binary files /dev/null and b/coca_project/src/audit_images/epoch2_batch310.png differ diff --git a/coca_project/src/audit_images/epoch2_batch320.png b/coca_project/src/audit_images/epoch2_batch320.png new file mode 100644 index 0000000..4bbd87b Binary files /dev/null and b/coca_project/src/audit_images/epoch2_batch320.png differ diff --git a/coca_project/src/audit_images/epoch2_batch330.png b/coca_project/src/audit_images/epoch2_batch330.png new file mode 100644 index 0000000..5ac08c5 Binary files /dev/null and b/coca_project/src/audit_images/epoch2_batch330.png differ diff --git a/coca_project/src/audit_images/epoch2_batch340.png b/coca_project/src/audit_images/epoch2_batch340.png new file mode 100644 index 0000000..0be37f0 Binary files /dev/null and b/coca_project/src/audit_images/epoch2_batch340.png differ diff --git a/coca_project/src/audit_images/epoch2_batch350.png b/coca_project/src/audit_images/epoch2_batch350.png new file mode 100644 index 0000000..09cb0e8 Binary files /dev/null and b/coca_project/src/audit_images/epoch2_batch350.png differ diff --git a/coca_project/src/audit_images/epoch2_batch360.png b/coca_project/src/audit_images/epoch2_batch360.png new file mode 100644 index 0000000..fe5bcc7 Binary files /dev/null and b/coca_project/src/audit_images/epoch2_batch360.png differ diff --git a/coca_project/src/audit_images/epoch2_batch370.png b/coca_project/src/audit_images/epoch2_batch370.png new file mode 100644 index 0000000..d02eb11 Binary files /dev/null and b/coca_project/src/audit_images/epoch2_batch370.png differ diff --git a/coca_project/src/audit_images/epoch2_batch380.png b/coca_project/src/audit_images/epoch2_batch380.png new file mode 100644 index 0000000..0c529a3 Binary files /dev/null and b/coca_project/src/audit_images/epoch2_batch380.png differ diff --git a/coca_project/src/audit_images/epoch2_batch390.png b/coca_project/src/audit_images/epoch2_batch390.png new file mode 100644 index 0000000..30ab759 Binary files /dev/null and b/coca_project/src/audit_images/epoch2_batch390.png differ diff --git a/coca_project/src/audit_images/epoch2_batch40.png b/coca_project/src/audit_images/epoch2_batch40.png new file mode 100644 index 0000000..49277e7 Binary files /dev/null and b/coca_project/src/audit_images/epoch2_batch40.png differ diff --git a/coca_project/src/audit_images/epoch2_batch400.png b/coca_project/src/audit_images/epoch2_batch400.png new file mode 100644 index 0000000..b74bbff Binary files /dev/null and b/coca_project/src/audit_images/epoch2_batch400.png differ diff --git a/coca_project/src/audit_images/epoch2_batch410.png b/coca_project/src/audit_images/epoch2_batch410.png new file mode 100644 index 0000000..a3dea15 Binary files /dev/null and b/coca_project/src/audit_images/epoch2_batch410.png differ diff --git a/coca_project/src/audit_images/epoch2_batch420.png b/coca_project/src/audit_images/epoch2_batch420.png new file mode 100644 index 0000000..8c4c6e6 Binary files /dev/null and b/coca_project/src/audit_images/epoch2_batch420.png differ diff --git a/coca_project/src/audit_images/epoch2_batch430.png b/coca_project/src/audit_images/epoch2_batch430.png new file mode 100644 index 0000000..7cb4bdf Binary files /dev/null and b/coca_project/src/audit_images/epoch2_batch430.png differ diff --git a/coca_project/src/audit_images/epoch2_batch440.png b/coca_project/src/audit_images/epoch2_batch440.png new file mode 100644 index 0000000..179ae0e Binary files /dev/null and b/coca_project/src/audit_images/epoch2_batch440.png differ diff --git a/coca_project/src/audit_images/epoch2_batch450.png b/coca_project/src/audit_images/epoch2_batch450.png new file mode 100644 index 0000000..1e39a6b Binary files /dev/null and b/coca_project/src/audit_images/epoch2_batch450.png differ diff --git a/coca_project/src/audit_images/epoch2_batch460.png b/coca_project/src/audit_images/epoch2_batch460.png new file mode 100644 index 0000000..7110c8d Binary files /dev/null and b/coca_project/src/audit_images/epoch2_batch460.png differ diff --git a/coca_project/src/audit_images/epoch2_batch470.png b/coca_project/src/audit_images/epoch2_batch470.png new file mode 100644 index 0000000..7002f67 Binary files /dev/null and b/coca_project/src/audit_images/epoch2_batch470.png differ diff --git a/coca_project/src/audit_images/epoch2_batch480.png b/coca_project/src/audit_images/epoch2_batch480.png new file mode 100644 index 0000000..149f59f Binary files /dev/null and b/coca_project/src/audit_images/epoch2_batch480.png differ diff --git a/coca_project/src/audit_images/epoch2_batch490.png b/coca_project/src/audit_images/epoch2_batch490.png new file mode 100644 index 0000000..72162a5 Binary files /dev/null and b/coca_project/src/audit_images/epoch2_batch490.png differ diff --git a/coca_project/src/audit_images/epoch2_batch50.png b/coca_project/src/audit_images/epoch2_batch50.png new file mode 100644 index 0000000..7b4e78e Binary files /dev/null and b/coca_project/src/audit_images/epoch2_batch50.png differ diff --git a/coca_project/src/audit_images/epoch2_batch500.png b/coca_project/src/audit_images/epoch2_batch500.png new file mode 100644 index 0000000..c40ad05 Binary files /dev/null and b/coca_project/src/audit_images/epoch2_batch500.png differ diff --git a/coca_project/src/audit_images/epoch2_batch510.png b/coca_project/src/audit_images/epoch2_batch510.png new file mode 100644 index 0000000..cd35fb2 Binary files /dev/null and b/coca_project/src/audit_images/epoch2_batch510.png differ diff --git a/coca_project/src/audit_images/epoch2_batch520.png b/coca_project/src/audit_images/epoch2_batch520.png new file mode 100644 index 0000000..916dcf3 Binary files /dev/null and b/coca_project/src/audit_images/epoch2_batch520.png differ diff --git a/coca_project/src/audit_images/epoch2_batch530.png b/coca_project/src/audit_images/epoch2_batch530.png new file mode 100644 index 0000000..5e46c99 Binary files /dev/null and b/coca_project/src/audit_images/epoch2_batch530.png differ diff --git a/coca_project/src/audit_images/epoch2_batch540.png b/coca_project/src/audit_images/epoch2_batch540.png new file mode 100644 index 0000000..90700bc Binary files /dev/null and b/coca_project/src/audit_images/epoch2_batch540.png differ diff --git a/coca_project/src/audit_images/epoch2_batch60.png b/coca_project/src/audit_images/epoch2_batch60.png new file mode 100644 index 0000000..013407d Binary files /dev/null and b/coca_project/src/audit_images/epoch2_batch60.png differ diff --git a/coca_project/src/audit_images/epoch2_batch70.png b/coca_project/src/audit_images/epoch2_batch70.png new file mode 100644 index 0000000..db3e221 Binary files /dev/null and b/coca_project/src/audit_images/epoch2_batch70.png differ diff --git a/coca_project/src/audit_images/epoch2_batch80.png b/coca_project/src/audit_images/epoch2_batch80.png new file mode 100644 index 0000000..fc668e1 Binary files /dev/null and b/coca_project/src/audit_images/epoch2_batch80.png differ diff --git a/coca_project/src/audit_images/epoch2_batch90.png b/coca_project/src/audit_images/epoch2_batch90.png new file mode 100644 index 0000000..0858d3c Binary files /dev/null and b/coca_project/src/audit_images/epoch2_batch90.png differ diff --git a/coca_project/src/audit_images/epoch3_batch0.png b/coca_project/src/audit_images/epoch3_batch0.png new file mode 100644 index 0000000..48c6a38 Binary files /dev/null and b/coca_project/src/audit_images/epoch3_batch0.png differ diff --git a/coca_project/src/audit_images/epoch3_batch10.png b/coca_project/src/audit_images/epoch3_batch10.png new file mode 100644 index 0000000..0d4b86b Binary files /dev/null and b/coca_project/src/audit_images/epoch3_batch10.png differ diff --git a/coca_project/src/audit_images/epoch3_batch100.png b/coca_project/src/audit_images/epoch3_batch100.png new file mode 100644 index 0000000..b4b257f Binary files /dev/null and b/coca_project/src/audit_images/epoch3_batch100.png differ diff --git a/coca_project/src/audit_images/epoch3_batch110.png b/coca_project/src/audit_images/epoch3_batch110.png new file mode 100644 index 0000000..b988c24 Binary files /dev/null and b/coca_project/src/audit_images/epoch3_batch110.png differ diff --git a/coca_project/src/audit_images/epoch3_batch120.png b/coca_project/src/audit_images/epoch3_batch120.png new file mode 100644 index 0000000..b74c5b8 Binary files /dev/null and b/coca_project/src/audit_images/epoch3_batch120.png differ diff --git a/coca_project/src/audit_images/epoch3_batch130.png b/coca_project/src/audit_images/epoch3_batch130.png new file mode 100644 index 0000000..cd57377 Binary files /dev/null and b/coca_project/src/audit_images/epoch3_batch130.png differ diff --git a/coca_project/src/audit_images/epoch3_batch140.png b/coca_project/src/audit_images/epoch3_batch140.png new file mode 100644 index 0000000..5fb2843 Binary files /dev/null and b/coca_project/src/audit_images/epoch3_batch140.png differ diff --git a/coca_project/src/audit_images/epoch3_batch150.png b/coca_project/src/audit_images/epoch3_batch150.png new file mode 100644 index 0000000..8b0a8ad Binary files /dev/null and b/coca_project/src/audit_images/epoch3_batch150.png differ diff --git a/coca_project/src/audit_images/epoch3_batch160.png b/coca_project/src/audit_images/epoch3_batch160.png new file mode 100644 index 0000000..1e157ae Binary files /dev/null and b/coca_project/src/audit_images/epoch3_batch160.png differ diff --git a/coca_project/src/audit_images/epoch3_batch170.png b/coca_project/src/audit_images/epoch3_batch170.png new file mode 100644 index 0000000..ba80f46 Binary files /dev/null and b/coca_project/src/audit_images/epoch3_batch170.png differ diff --git a/coca_project/src/audit_images/epoch3_batch180.png b/coca_project/src/audit_images/epoch3_batch180.png new file mode 100644 index 0000000..245c558 Binary files /dev/null and b/coca_project/src/audit_images/epoch3_batch180.png differ diff --git a/coca_project/src/audit_images/epoch3_batch190.png b/coca_project/src/audit_images/epoch3_batch190.png new file mode 100644 index 0000000..4059bf7 Binary files /dev/null and b/coca_project/src/audit_images/epoch3_batch190.png differ diff --git a/coca_project/src/audit_images/epoch3_batch20.png b/coca_project/src/audit_images/epoch3_batch20.png new file mode 100644 index 0000000..3b946c4 Binary files /dev/null and b/coca_project/src/audit_images/epoch3_batch20.png differ diff --git a/coca_project/src/audit_images/epoch3_batch200.png b/coca_project/src/audit_images/epoch3_batch200.png new file mode 100644 index 0000000..a262d3d Binary files /dev/null and b/coca_project/src/audit_images/epoch3_batch200.png differ diff --git a/coca_project/src/audit_images/epoch3_batch210.png b/coca_project/src/audit_images/epoch3_batch210.png new file mode 100644 index 0000000..3e79df0 Binary files /dev/null and b/coca_project/src/audit_images/epoch3_batch210.png differ diff --git a/coca_project/src/audit_images/epoch3_batch220.png b/coca_project/src/audit_images/epoch3_batch220.png new file mode 100644 index 0000000..4a2b301 Binary files /dev/null and b/coca_project/src/audit_images/epoch3_batch220.png differ diff --git a/coca_project/src/audit_images/epoch3_batch230.png b/coca_project/src/audit_images/epoch3_batch230.png new file mode 100644 index 0000000..cb1e1ab Binary files /dev/null and b/coca_project/src/audit_images/epoch3_batch230.png differ diff --git a/coca_project/src/audit_images/epoch3_batch240.png b/coca_project/src/audit_images/epoch3_batch240.png new file mode 100644 index 0000000..e02b24e Binary files /dev/null and b/coca_project/src/audit_images/epoch3_batch240.png differ diff --git a/coca_project/src/audit_images/epoch3_batch250.png b/coca_project/src/audit_images/epoch3_batch250.png new file mode 100644 index 0000000..da2fd30 Binary files /dev/null and b/coca_project/src/audit_images/epoch3_batch250.png differ diff --git a/coca_project/src/audit_images/epoch3_batch260.png b/coca_project/src/audit_images/epoch3_batch260.png new file mode 100644 index 0000000..43c8e30 Binary files /dev/null and b/coca_project/src/audit_images/epoch3_batch260.png differ diff --git a/coca_project/src/audit_images/epoch3_batch270.png b/coca_project/src/audit_images/epoch3_batch270.png new file mode 100644 index 0000000..234dfc1 Binary files /dev/null and b/coca_project/src/audit_images/epoch3_batch270.png differ diff --git a/coca_project/src/audit_images/epoch3_batch280.png b/coca_project/src/audit_images/epoch3_batch280.png new file mode 100644 index 0000000..dd40c96 Binary files /dev/null and b/coca_project/src/audit_images/epoch3_batch280.png differ diff --git a/coca_project/src/audit_images/epoch3_batch290.png b/coca_project/src/audit_images/epoch3_batch290.png new file mode 100644 index 0000000..5ad60c2 Binary files /dev/null and b/coca_project/src/audit_images/epoch3_batch290.png differ diff --git a/coca_project/src/audit_images/epoch3_batch30.png b/coca_project/src/audit_images/epoch3_batch30.png new file mode 100644 index 0000000..a13535f Binary files /dev/null and b/coca_project/src/audit_images/epoch3_batch30.png differ diff --git a/coca_project/src/audit_images/epoch3_batch300.png b/coca_project/src/audit_images/epoch3_batch300.png new file mode 100644 index 0000000..cbd1d53 Binary files /dev/null and b/coca_project/src/audit_images/epoch3_batch300.png differ diff --git a/coca_project/src/audit_images/epoch3_batch310.png b/coca_project/src/audit_images/epoch3_batch310.png new file mode 100644 index 0000000..457ef6a Binary files /dev/null and b/coca_project/src/audit_images/epoch3_batch310.png differ diff --git a/coca_project/src/audit_images/epoch3_batch320.png b/coca_project/src/audit_images/epoch3_batch320.png new file mode 100644 index 0000000..e7607bc Binary files /dev/null and b/coca_project/src/audit_images/epoch3_batch320.png differ diff --git a/coca_project/src/audit_images/epoch3_batch330.png b/coca_project/src/audit_images/epoch3_batch330.png new file mode 100644 index 0000000..ddd4524 Binary files /dev/null and b/coca_project/src/audit_images/epoch3_batch330.png differ diff --git a/coca_project/src/audit_images/epoch3_batch340.png b/coca_project/src/audit_images/epoch3_batch340.png new file mode 100644 index 0000000..edf0d4c Binary files /dev/null and b/coca_project/src/audit_images/epoch3_batch340.png differ diff --git a/coca_project/src/audit_images/epoch3_batch350.png b/coca_project/src/audit_images/epoch3_batch350.png new file mode 100644 index 0000000..9de5e51 Binary files /dev/null and b/coca_project/src/audit_images/epoch3_batch350.png differ diff --git a/coca_project/src/audit_images/epoch3_batch360.png b/coca_project/src/audit_images/epoch3_batch360.png new file mode 100644 index 0000000..c7b63c9 Binary files /dev/null and b/coca_project/src/audit_images/epoch3_batch360.png differ diff --git a/coca_project/src/audit_images/epoch3_batch370.png b/coca_project/src/audit_images/epoch3_batch370.png new file mode 100644 index 0000000..fca8975 Binary files /dev/null and b/coca_project/src/audit_images/epoch3_batch370.png differ diff --git a/coca_project/src/audit_images/epoch3_batch380.png b/coca_project/src/audit_images/epoch3_batch380.png new file mode 100644 index 0000000..cbffde9 Binary files /dev/null and b/coca_project/src/audit_images/epoch3_batch380.png differ diff --git a/coca_project/src/audit_images/epoch3_batch390.png b/coca_project/src/audit_images/epoch3_batch390.png new file mode 100644 index 0000000..6795c0c Binary files /dev/null and b/coca_project/src/audit_images/epoch3_batch390.png differ diff --git a/coca_project/src/audit_images/epoch3_batch40.png b/coca_project/src/audit_images/epoch3_batch40.png new file mode 100644 index 0000000..14b3f72 Binary files /dev/null and b/coca_project/src/audit_images/epoch3_batch40.png differ diff --git a/coca_project/src/audit_images/epoch3_batch400.png b/coca_project/src/audit_images/epoch3_batch400.png new file mode 100644 index 0000000..5e1196c Binary files /dev/null and b/coca_project/src/audit_images/epoch3_batch400.png differ diff --git a/coca_project/src/audit_images/epoch3_batch410.png b/coca_project/src/audit_images/epoch3_batch410.png new file mode 100644 index 0000000..e069ff1 Binary files /dev/null and b/coca_project/src/audit_images/epoch3_batch410.png differ diff --git a/coca_project/src/audit_images/epoch3_batch420.png b/coca_project/src/audit_images/epoch3_batch420.png new file mode 100644 index 0000000..5814f25 Binary files /dev/null and b/coca_project/src/audit_images/epoch3_batch420.png differ diff --git a/coca_project/src/audit_images/epoch3_batch430.png b/coca_project/src/audit_images/epoch3_batch430.png new file mode 100644 index 0000000..c4e7a86 Binary files /dev/null and b/coca_project/src/audit_images/epoch3_batch430.png differ diff --git a/coca_project/src/audit_images/epoch3_batch440.png b/coca_project/src/audit_images/epoch3_batch440.png new file mode 100644 index 0000000..e0856fb Binary files /dev/null and b/coca_project/src/audit_images/epoch3_batch440.png differ diff --git a/coca_project/src/audit_images/epoch3_batch450.png b/coca_project/src/audit_images/epoch3_batch450.png new file mode 100644 index 0000000..579baf0 Binary files /dev/null and b/coca_project/src/audit_images/epoch3_batch450.png differ diff --git a/coca_project/src/audit_images/epoch3_batch460.png b/coca_project/src/audit_images/epoch3_batch460.png new file mode 100644 index 0000000..5c0d115 Binary files /dev/null and b/coca_project/src/audit_images/epoch3_batch460.png differ diff --git a/coca_project/src/audit_images/epoch3_batch470.png b/coca_project/src/audit_images/epoch3_batch470.png new file mode 100644 index 0000000..849dbcd Binary files /dev/null and b/coca_project/src/audit_images/epoch3_batch470.png differ diff --git a/coca_project/src/audit_images/epoch3_batch480.png b/coca_project/src/audit_images/epoch3_batch480.png new file mode 100644 index 0000000..d1f8d16 Binary files /dev/null and b/coca_project/src/audit_images/epoch3_batch480.png differ diff --git a/coca_project/src/audit_images/epoch3_batch490.png b/coca_project/src/audit_images/epoch3_batch490.png new file mode 100644 index 0000000..5e2ebd5 Binary files /dev/null and b/coca_project/src/audit_images/epoch3_batch490.png differ diff --git a/coca_project/src/audit_images/epoch3_batch50.png b/coca_project/src/audit_images/epoch3_batch50.png new file mode 100644 index 0000000..bc8124c Binary files /dev/null and b/coca_project/src/audit_images/epoch3_batch50.png differ diff --git a/coca_project/src/audit_images/epoch3_batch500.png b/coca_project/src/audit_images/epoch3_batch500.png new file mode 100644 index 0000000..e9fcdea Binary files /dev/null and b/coca_project/src/audit_images/epoch3_batch500.png differ diff --git a/coca_project/src/audit_images/epoch3_batch510.png b/coca_project/src/audit_images/epoch3_batch510.png new file mode 100644 index 0000000..4bdf4aa Binary files /dev/null and b/coca_project/src/audit_images/epoch3_batch510.png differ diff --git a/coca_project/src/audit_images/epoch3_batch520.png b/coca_project/src/audit_images/epoch3_batch520.png new file mode 100644 index 0000000..f137d3c Binary files /dev/null and b/coca_project/src/audit_images/epoch3_batch520.png differ diff --git a/coca_project/src/audit_images/epoch3_batch530.png b/coca_project/src/audit_images/epoch3_batch530.png new file mode 100644 index 0000000..818c4d5 Binary files /dev/null and b/coca_project/src/audit_images/epoch3_batch530.png differ diff --git a/coca_project/src/audit_images/epoch3_batch540.png b/coca_project/src/audit_images/epoch3_batch540.png new file mode 100644 index 0000000..f532414 Binary files /dev/null and b/coca_project/src/audit_images/epoch3_batch540.png differ diff --git a/coca_project/src/audit_images/epoch3_batch60.png b/coca_project/src/audit_images/epoch3_batch60.png new file mode 100644 index 0000000..7342946 Binary files /dev/null and b/coca_project/src/audit_images/epoch3_batch60.png differ diff --git a/coca_project/src/audit_images/epoch3_batch70.png b/coca_project/src/audit_images/epoch3_batch70.png new file mode 100644 index 0000000..471640f Binary files /dev/null and b/coca_project/src/audit_images/epoch3_batch70.png differ diff --git a/coca_project/src/audit_images/epoch3_batch80.png b/coca_project/src/audit_images/epoch3_batch80.png new file mode 100644 index 0000000..06d2d75 Binary files /dev/null and b/coca_project/src/audit_images/epoch3_batch80.png differ diff --git a/coca_project/src/audit_images/epoch3_batch90.png b/coca_project/src/audit_images/epoch3_batch90.png new file mode 100644 index 0000000..be53dbe Binary files /dev/null and b/coca_project/src/audit_images/epoch3_batch90.png differ diff --git a/coca_project/src/audit_images/epoch4_batch0.png b/coca_project/src/audit_images/epoch4_batch0.png new file mode 100644 index 0000000..4cd2465 Binary files /dev/null and b/coca_project/src/audit_images/epoch4_batch0.png differ diff --git a/coca_project/src/audit_images/epoch4_batch10.png b/coca_project/src/audit_images/epoch4_batch10.png new file mode 100644 index 0000000..89bff73 Binary files /dev/null and b/coca_project/src/audit_images/epoch4_batch10.png differ diff --git a/coca_project/src/audit_images/epoch4_batch20.png b/coca_project/src/audit_images/epoch4_batch20.png new file mode 100644 index 0000000..c9f7940 Binary files /dev/null and b/coca_project/src/audit_images/epoch4_batch20.png differ diff --git a/coca_project/src/audit_images/epoch4_batch30.png b/coca_project/src/audit_images/epoch4_batch30.png new file mode 100644 index 0000000..6d4bfaf Binary files /dev/null and b/coca_project/src/audit_images/epoch4_batch30.png differ diff --git a/coca_project/src/audit_images/epoch4_batch40.png b/coca_project/src/audit_images/epoch4_batch40.png new file mode 100644 index 0000000..30e1509 Binary files /dev/null and b/coca_project/src/audit_images/epoch4_batch40.png differ diff --git a/coca_project/src/audit_images/epoch4_batch50.png b/coca_project/src/audit_images/epoch4_batch50.png new file mode 100644 index 0000000..df74463 Binary files /dev/null and b/coca_project/src/audit_images/epoch4_batch50.png differ diff --git a/coca_project/src/audit_images/epoch4_batch60.png b/coca_project/src/audit_images/epoch4_batch60.png new file mode 100644 index 0000000..15f2ac0 Binary files /dev/null and b/coca_project/src/audit_images/epoch4_batch60.png differ diff --git a/coca_project/src/audit_images/epoch4_batch70.png b/coca_project/src/audit_images/epoch4_batch70.png new file mode 100644 index 0000000..cdc7d33 Binary files /dev/null and b/coca_project/src/audit_images/epoch4_batch70.png differ diff --git a/coca_project/src/audit_images/epoch4_batch80.png b/coca_project/src/audit_images/epoch4_batch80.png new file mode 100644 index 0000000..1355440 Binary files /dev/null and b/coca_project/src/audit_images/epoch4_batch80.png differ diff --git a/coca_project/src/best_coca_model.pth b/coca_project/src/best_coca_model.pth new file mode 100644 index 0000000..d5d4a8e Binary files /dev/null and b/coca_project/src/best_coca_model.pth differ diff --git a/coca_project/src/big_data_loader.py b/coca_project/src/big_data_loader.py new file mode 100644 index 0000000..ef8cb94 --- /dev/null +++ b/coca_project/src/big_data_loader.py @@ -0,0 +1,225 @@ +import pandas as pd +import torch +from monai.transforms import ( + Compose, LoadImaged, EnsureChannelFirstd, + RandRotate90d, RandCropByPosNegLabeld, + RandFlipd, SpatialPadd, ToTensord, RandGaussianNoised, + ScaleIntensityRanged, CropForegroundd, CastToTyped, AsDiscreted, + ConcatItemsd, DeleteItemsd, +) +from monai.data import DataLoader, CacheDataset + + +# --------------------------------------------------------------------------- +# CONFIG +# --------------------------------------------------------------------------- +USE_CARDIAC_MASK = True # Set False if cardiac masks weren't generated +# --------------------------------------------------------------------------- + + +def _base_load_transforms(use_cardiac_mask: bool) -> list: + """ + Transforms shared by train and val: + - Load image + label (+ optional cardiac mask) + - Ensure channel first + - Foreground crop on image + - Intensity scaling + """ + keys = ["image", "label", "cardiac_mask"] if use_cardiac_mask else ["image", "label"] + img_keys = ["image", "cardiac_mask"] if use_cardiac_mask else ["image"] + + transforms = [ + LoadImaged(keys=keys), + EnsureChannelFirstd(keys=keys), + + # Crop to body, ignoring air/lungs — applied to all keys together + # so spatial alignment is preserved + CropForegroundd( + keys=keys, + source_key="image", + select_fn=lambda x: x > -200, + margin=10, + ), + + # Intensity normalisation: calcium is 200–700 HU, bone 700–1000 HU. + # Clipping at 1000 compresses bone without zeroing it out entirely, + # which helps the model learn to distinguish calcium from bone edges. + ScaleIntensityRanged( + keys=["image"], + a_min=-100, a_max=1000, + b_min=0.0, b_max=1.0, + clip=True, + ), + ] + + if use_cardiac_mask: + # Multiply image by cardiac mask to zero out non-cardiac regions, + # then drop the mask key — the model only sees the masked image. + # This is done AFTER intensity scaling so values stay in [0, 1]. + transforms += [ + # cardiac_mask is binary {0,1} — simple elementwise multiply + # We use a Lambda here since MONAI has no built-in MaskIntensityd + # that handles the channel dim correctly post-EnsureChannelFirst. + AsDiscreted(keys=["cardiac_mask"], threshold=0.5), + # Now apply: image = image * cardiac_mask + # ConcatItemsd stacks along channel, we then use a custom fn. + # Simpler: just use a direct torch op via ToTensord + manual mask. + # We'll apply the mask after ToTensord in the dataset __getitem__ + # by keeping cardiac_mask in the dict — see MaskedCacheDataset below. + ] + + return transforms + + +def get_transforms(patch_size: tuple, use_cardiac_mask: bool = USE_CARDIAC_MASK): + keys = ["image", "label", "cardiac_mask"] if use_cardiac_mask else ["image", "label"] + + shared = _base_load_transforms(use_cardiac_mask) + + train_transforms = Compose(shared + [ + # Sample patches — bias toward calcium-containing regions + RandCropByPosNegLabeld( + keys=keys, + label_key="label", + spatial_size=(96, 96, 32), + pos=2, # increased from 1 — calcium is very sparse + neg=1, + num_samples=4, + ), + + # Pad if any patch is smaller than patch_size (edge cases) + SpatialPadd(keys=keys, spatial_size=patch_size, mode="constant"), + + # Cast before augmentation to avoid dtype surprises + CastToTyped( + keys=["image"] + (["cardiac_mask"] if use_cardiac_mask else []), + dtype=torch.float32, + ), + CastToTyped(keys=["label"], dtype=torch.uint8), + AsDiscreted(keys=["label"], threshold=0.5), + + # Augmentation — axis-aligned only, since calcium deposits are small + # and aggressive rotation would destroy the HU-based signal + RandFlipd(keys=keys, spatial_axis=[0], prob=0.5), + RandFlipd(keys=keys, spatial_axis=[1], prob=0.5), + RandRotate90d(keys=keys, prob=0.5, max_k=3), + + # Gaussian noise on image only, AFTER all spatial ops + RandGaussianNoised(keys=["image"], prob=0.2, mean=0.0, std=0.05), + + ToTensord(keys=keys), + ]) + + val_transforms = Compose(shared + [ + # Val needs the same spatial padding as train for consistent tensor shapes + SpatialPadd(keys=keys, spatial_size=patch_size, mode="constant"), + CastToTyped( + keys=["image"] + (["cardiac_mask"] if use_cardiac_mask else []), + dtype=torch.float32, + ), + CastToTyped(keys=["label"], dtype=torch.uint8), + AsDiscreted(keys=["label"], threshold=0.5), + ToTensord(keys=keys), + ]) + + return train_transforms, val_transforms + + +class MaskedCacheDataset(CacheDataset): + """ + Thin wrapper around CacheDataset that applies the cardiac mask to the + image after all cached transforms have run. Doing it here rather than + inside the transform pipeline avoids caching the masked image — we cache + the raw normalised image and apply the mask on the fly, which means we + can ablate USE_CARDIAC_MASK without reprocessing the cache. + """ + def __init__(self, use_cardiac_mask: bool, **kwargs): + super().__init__(**kwargs) + self.use_cardiac_mask = use_cardiac_mask + + def __getitem__(self, index): + items = super().__getitem__(index) + if not self.use_cardiac_mask: + return items + + # items is a list of dicts when num_samples > 1 + if isinstance(items, list): + for item in items: + self._apply_mask(item) + else: + self._apply_mask(items) + return items + + @staticmethod + def _apply_mask(item: dict): + if "cardiac_mask" in item and "image" in item: + item["image"] = item["image"] * item["cardiac_mask"] + del item["cardiac_mask"] + + +def build_file_dicts(df: pd.DataFrame, use_cardiac_mask: bool) -> list[dict]: + """Build MONAI-style file dicts from the scan_index DataFrame.""" + rows = [] + for _, r in df.iterrows(): + d = {"image": r["image_path"], "label": r["mask_path"]} + if use_cardiac_mask and pd.notna(r.get("cardiac_mask_path")): + d["cardiac_mask"] = r["cardiac_mask_path"] + elif use_cardiac_mask: + # Cardiac mask missing for this scan — fall back gracefully + d["cardiac_mask"] = r["image_path"] # will be overwritten by ones + rows.append(d) + return rows + + +def get_coca_loaders( + train_parquet: str, + val_parquet: str, + patch_size: tuple = (128, 128, 32), + batch_size: int = 2, + use_cardiac_mask: bool = USE_CARDIAC_MASK, +): + train_df = pd.read_parquet(train_parquet) + val_df = pd.read_parquet(val_parquet) + + # Use has_calcium column from processor — no need to open each NIfTI + pos_df = train_df[train_df["has_calcium"]] + neg_df = train_df[~train_df["has_calcium"]] + + # All positives + 10% negatives for context + neg_sample = neg_df.sample(frac=0.1, random_state=42) + curated_df = pd.concat([pos_df, neg_sample]).reset_index(drop=True) + + print(f" Train: {len(pos_df)} positive, {len(neg_sample)} negative sampled " + f"({len(curated_df)} total)") + print(f" Val: {len(val_df)} scans") + + train_files = build_file_dicts(curated_df, use_cardiac_mask) + val_files = build_file_dicts(val_df, use_cardiac_mask) + + train_trans, val_trans = get_transforms(patch_size, use_cardiac_mask) + + train_ds = MaskedCacheDataset( + use_cardiac_mask=use_cardiac_mask, + data=train_files, + transform=train_trans, + cache_rate=1.0, + ) + val_ds = MaskedCacheDataset( + use_cardiac_mask=use_cardiac_mask, + data=val_files, + transform=val_trans, + cache_rate=1.0, + ) + + train_loader = DataLoader( + train_ds, batch_size=batch_size, shuffle=True, + num_workers=0, # keep 0 for CacheDataset on Windows + pin_memory=True, + ) + val_loader = DataLoader( + val_ds, batch_size=1, + num_workers=0, # CacheDataset is not fork-safe, keep 0 + pin_memory=True, + ) + + return train_loader, val_loader diff --git a/coca_project/src/calcium_roi_extract.py b/coca_project/src/calcium_roi_extract.py new file mode 100644 index 0000000..c538288 --- /dev/null +++ b/coca_project/src/calcium_roi_extract.py @@ -0,0 +1,244 @@ +# calcium_roi_extract.py +# +# Extract calcium ROI annotations from plist-XML files (saved with .xml extension) +# into a single Parquet file with a stable, normalized schema. +# +# Expected schema per file: +# top-level dict with key "Images" -> list[dict] +# each image dict: "ImageIndex", "ROIs" -> list[dict] +# each ROI dict: "Name", "Area", "Mean", "Max", "Point_px", "Point_mm", etc. + +from __future__ import annotations + +import json +import plistlib +import re +from pathlib import Path +from typing import Any, Dict, List, Optional, Tuple + +import pandas as pd + + +# ----------------------------- +# Configuration (edit as needed) +# ----------------------------- +XML_DIR = Path(r"C:\coca_project\data_raw\xml\calcium_xml") +OUT_PARQUET = Path(r"C:\coca_project\derived\calcium_rois_raw.parquet") + +# If True, removes placeholder ROIs where area==0 or n_points==0 +DROP_EMPTY_ROIS = False + + +# ----------------------------- +# Parsing helpers +# ----------------------------- +_NUM_RE = re.compile(r"[-+]?\d*\.?\d+(?:[eE][-+]?\d+)?") + + +def parse_point_tuple(s: str) -> Tuple[float, ...]: + """ + Parses strings like: + "(269.003922, 377.000000)" -> (269.003922, 377.0) + "(38.165314, -122.242188, -151.250000)" -> (38.165314, -122.242188, -151.25) + """ + nums = [float(x) for x in _NUM_RE.findall(s)] + if not nums: + raise ValueError(f"Could not parse point from: {s!r}") + return tuple(nums) + + +def load_plist_xml(path: Path) -> Dict[str, Any]: + """ + Loads an Apple plist stored as XML (even if the extension is .xml). + """ + with open(path, "rb") as f: + obj = plistlib.load(f) + if not isinstance(obj, dict): + raise ValueError(f"Top-level plist is not a dict in {path}") + return obj + + +def safe_points(arr: Any) -> Optional[List[Tuple[float, ...]]]: + """ + Converts a list of "(x,y)" or "(x,y,z)" strings into a list of float tuples. + Returns None if empty or not parseable. + """ + if not isinstance(arr, list) or len(arr) == 0: + return None + + pts: List[Tuple[float, ...]] = [] + for item in arr: + if not isinstance(item, str): + continue + pts.append(parse_point_tuple(item)) + + return pts if pts else None + + +def extract_rows_from_file(xml_path: Path) -> List[Dict[str, Any]]: + """ + Extract one row per ROI, with normalized column names. + """ + data = load_plist_xml(xml_path) + images = data.get("Images", []) + + if not isinstance(images, list): + return [{ + "xml_path": str(xml_path), + "image_index": None, + "vessel": None, + "area": None, + "mean_hu": None, + "max_hu": None, + "point_px": None, + "point_mm": None, + "roi_idx": None, + "n_points": None, + "roi_type": None, + "total_hu": None, + "center_mm": None, + "raw_roi": None, + "note": "Top-level 'Images' is not a list." + }] + + rows: List[Dict[str, Any]] = [] + + for img in images: + if not isinstance(img, dict): + continue + + image_index = img.get("ImageIndex", None) + rois = img.get("ROIs", []) + if not isinstance(rois, list): + rois = [] + + for roi in rois: + if not isinstance(roi, dict): + continue + + # Normalized fields + vessel = roi.get("Name", None) + area = roi.get("Area", None) + mean_hu = roi.get("Mean", None) + max_hu = roi.get("Max", None) + + point_px = safe_points(roi.get("Point_px", None)) + point_mm = safe_points(roi.get("Point_mm", None)) + + roi_idx = roi.get("IndexInImage", None) + n_points = roi.get("NumberOfPoints", None) + roi_type = roi.get("Type", None) + total_hu = roi.get("Total", None) + center_mm = roi.get("Center", None) + + rows.append({ + "xml_path": str(xml_path), + "image_index": image_index, + "vessel": vessel, + "area": area, + "mean_hu": mean_hu, + "max_hu": max_hu, + "point_px": point_px, + "point_mm": point_mm, + "roi_idx": roi_idx, + "n_points": n_points, + "roi_type": roi_type, + "total_hu": total_hu, + "center_mm": center_mm, + # Keep raw ROI dict for debugging/traceability + "raw_roi": json.dumps(roi, default=str), + }) + + # Optional: keep a QC marker row if file contains no ROIs at all + if not rows: + rows.append({ + "xml_path": str(xml_path), + "image_index": None, + "vessel": None, + "area": None, + "mean_hu": None, + "max_hu": None, + "point_px": None, + "point_mm": None, + "roi_idx": None, + "n_points": None, + "roi_type": None, + "total_hu": None, + "center_mm": None, + "raw_roi": None, + "note": "No ROIs found in this file." + }) + + return rows + + +def coerce_types(df: pd.DataFrame) -> pd.DataFrame: + """ + Enforce consistent dtypes. + """ + int_cols = ["image_index", "roi_idx", "n_points", "roi_type"] + float_cols = ["area", "mean_hu", "max_hu", "total_hu"] + + for c in int_cols: + if c in df.columns: + df[c] = pd.to_numeric(df[c], errors="coerce").astype("Int64") + + for c in float_cols: + if c in df.columns: + df[c] = pd.to_numeric(df[c], errors="coerce") + + return df + + +def main() -> None: + if not XML_DIR.exists(): + raise FileNotFoundError(f"XML_DIR does not exist: {XML_DIR}") + + xml_files = sorted([p for p in XML_DIR.rglob("*.xml") if p.is_file()]) + if not xml_files: + raise FileNotFoundError(f"No .xml files found under: {XML_DIR}") + + all_rows: List[Dict[str, Any]] = [] + for xp in xml_files: + try: + all_rows.extend(extract_rows_from_file(xp)) + except Exception as e: + # Keep a row so failures are visible in the output parquet + all_rows.append({ + "xml_path": str(xp), + "image_index": None, + "vessel": None, + "area": None, + "mean_hu": None, + "max_hu": None, + "point_px": None, + "point_mm": None, + "roi_idx": None, + "n_points": None, + "roi_type": None, + "total_hu": None, + "center_mm": None, + "raw_roi": None, + "note": f"Failed to parse file: {e}" + }) + + df = pd.DataFrame(all_rows) + df = coerce_types(df) + + if DROP_EMPTY_ROIS: + # Drop placeholders: either area==0 or n_points==0 (or missing) + df = df[(df["area"].fillna(0) > 0) & (df["n_points"].fillna(0) > 0)].copy() + df.reset_index(drop=True, inplace=True) + + OUT_PARQUET.parent.mkdir(parents=True, exist_ok=True) + + # Requires pyarrow or fastparquet installed; pyarrow recommended + df.to_parquet(OUT_PARQUET, index=False) + + print(f"Wrote {len(df):,} rows to: {OUT_PARQUET}") + print("Columns:", df.columns.tolist()) + print(df[["vessel", "image_index", "area", "mean_hu", "max_hu"]].head(10)) + + +if __name__ == "__main__": + main() diff --git a/coca_project/src/data/COCA_pipeline.py b/coca_project/src/data/COCA_pipeline.py new file mode 100644 index 0000000..46d5aad --- /dev/null +++ b/coca_project/src/data/COCA_pipeline.py @@ -0,0 +1,55 @@ +import sys +import os +from pathlib import Path + +# 1. Get the absolute path of the directory where THIS script is located +# This bypasses all 'current working directory' issues +SCRIPT_DIR = Path(__file__).resolve().parent + +# 2. Force this directory into the system path at the very beginning +if str(SCRIPT_DIR) not in sys.path: + sys.path.insert(0, str(SCRIPT_DIR)) + +# 3. Diagnostic check (optional but helpful) +print(f"Pipeline running from: {SCRIPT_DIR}") + +try: + # Now we import using the exact class names + from COCA_processor import COCAProcessor + from COCA_resampler import COCAResampler + print("Successfully imported COCA modules.") +except ImportError as e: + print(f"\n[STILL FAILING]: {e}") + print(f"I am looking for files in: {SCRIPT_DIR}") + print(f"Files found there: {os.listdir(SCRIPT_DIR)}") + sys.exit(1) + +def main(): + print("="*50) + print(" COCA DATA PREPROCESSING PIPELINE") + print("="*50) + + # 1. Configuration + default_root = r"C:\coca_project" + project_root = input(f"Project Root [{default_root}]: ").strip() or default_root + + # 2. Execution Logic + print("\n1) Full Pipeline\n2) Process Only\n3) Resample Only") + choice = input("Selection: ").strip() + + if choice in ['1', '2']: + print("\n--- Running Processor ---") + proc = COCAProcessor(project_root) + proc.process_all() + + if choice in ['1', '3']: + print("\n--- Running Resampler ---") + space = input("Voxel Spacing (mm) [1.0]: ").strip() or "1.0" + target = [float(space)] * 3 + resamp = COCAResampler(project_root, target_spacing=target) + resamp.run() + + print("\nPipeline Finished.") + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/coca_project/src/data/COCA_processor.py b/coca_project/src/data/COCA_processor.py new file mode 100644 index 0000000..72751a8 --- /dev/null +++ b/coca_project/src/data/COCA_processor.py @@ -0,0 +1,167 @@ +import os +import json +import hashlib +import plistlib +from pathlib import Path +from collections import defaultdict +import numpy as np +import pandas as pd +import SimpleITK as sitk +import cv2 +from tqdm import tqdm + +class COCAProcessor: + def __init__(self, project_root: str): + self.project_root = Path(project_root) + self.dicom_root = self.project_root / "data_raw" / "dicom" / "Gated_release_final" / "Gated_release_final" / "patient" + self.xml_root = self.project_root / "data_raw" / "xml" / "calcium_xml" + + self.out_images_base = self.project_root / "data_canonical" / "images" + self.out_tables = self.project_root / "data_canonical" / "tables" + + # Ensure output directories exist + self.out_images_base.mkdir(parents=True, exist_ok=True) + self.out_tables.mkdir(parents=True, exist_ok=True) + + @staticmethod + def generate_stable_id(*parts: str, n: int = 12) -> str: + """Generates a unique, reproducible ID for each scan.""" + h = hashlib.sha1("||".join(parts).encode("utf-8")).hexdigest() + return h[:n] + + def parse_plist_filled(self, xml_path: Path, image_shape: tuple): + """Parses XML and returns a binary 3D mask and list of segmented slices.""" + mask = np.zeros(image_shape, dtype=np.uint8) + segmented_slices = set() + total_z, total_y, total_x = image_shape + + if not xml_path.exists(): + return mask, [] + + try: + with open(xml_path, 'rb') as f: + data = plistlib.load(f) + + images = data.get('Images', []) + for img_entry in images: + z = int(img_entry.get('ImageIndex', -1)) + + if z < 0 or z >= total_z: + continue + + rois = img_entry.get('ROIs', []) + for roi in rois: + points_str = roi.get('Point_px', []) + if not points_str: + continue + + poly_points = [] + for p_str in points_str: + cleaned = p_str.replace("(", "").replace(")", "") + parts = cleaned.split(",") + if len(parts) == 2: + poly_points.append([float(parts[0]), float(parts[1])]) + + if poly_points: + pts = np.array(poly_points, dtype=np.int32) + temp_slice = np.zeros((total_y, total_x), dtype=np.uint8) + + if len(pts) > 2: + cv2.fillPoly(temp_slice, [pts], 1) + else: + for p in pts: + if 0 <= p[0] < total_x and 0 <= p[1] < total_y: + temp_slice[int(p[1]), int(p[0])] = 1 + + if np.any(temp_slice): + mask[z, :, :] = np.logical_or(mask[z, :, :], temp_slice).astype(np.uint8) + segmented_slices.add(z) + + except Exception as e: + print(f" [PARSING ERROR] {xml_path.name}: {e}") + + return mask, sorted(list(segmented_slices)) + + def discover_series(self): + """Scans the DICOM root for folders containing at least 5 DICOM files.""" + print(f"Scanning {self.dicom_root} for DICOM series...") + all_series = [] + found_dirs = set() + for p in self.dicom_root.rglob("*.dcm"): + if p.parent not in found_dirs: + if len(list(p.parent.glob("*.dcm"))) >= 5: + all_series.append(p.parent) + found_dirs.add(p.parent) + return all_series + + def process_all(self): + """Main execution loop to process all discovered DICOM series.""" + series_dirs = self.discover_series() + print(f"Found {len(series_dirs)} valid series. Starting processing...") + + rows = [] + for s_dir in tqdm(series_dirs, desc="Processing Scans"): + patient_id = s_dir.name + xml_path = self.xml_root / f"{patient_id}.xml" + + try: + # Load DICOM Volume + reader = sitk.ImageSeriesReader() + dicom_names = reader.GetGDCMSeriesFileNames(str(s_dir)) + reader.SetFileNames(dicom_names) + image = reader.Execute() + + img_array = sitk.GetArrayFromImage(image) + + # Generate Mask + mask_array, seg_slices = self.parse_plist_filled(xml_path, img_array.shape) + voxel_count = int(np.sum(mask_array)) + + if xml_path.exists() and voxel_count == 0: + print(f"\n [WARNING] Patient {patient_id}: XML exists but 0 voxels drawn. Check slice alignment.") + + # Setup output folder + scan_id = self.generate_stable_id(str(s_dir.resolve()), patient_id) + scan_folder = self.out_images_base / scan_id + scan_folder.mkdir(parents=True, exist_ok=True) + + # Save Image + sitk.WriteImage(image, str(scan_folder / f"{scan_id}_img.nii.gz"), useCompression=True) + + # Save Mask (inheriting geometry from original image) + mask_image = sitk.GetImageFromArray(mask_array) + mask_image.CopyInformation(image) + sitk.WriteImage(mask_image, str(scan_folder / f"{scan_id}_seg.nii.gz"), useCompression=True) + + # Metadata + meta = { + "scan_id": scan_id, + "patient_id": patient_id, + "calcium_voxels": voxel_count, + "slices_with_calcium": seg_slices, + "original_path": str(s_dir) + } + (scan_folder / f"{scan_id}_meta.json").write_text(json.dumps(meta, indent=2)) + + rows.append({ + "patient_id": patient_id, + "scan_id": scan_id, + "voxels": voxel_count, + "num_slices": len(seg_slices), + "folder_path": str(scan_folder) # Useful for the resampling script later + }) + + except Exception as e: + print(f" [ERROR] Patient {patient_id}: {e}") + + if rows: + df = pd.DataFrame(rows) + df.to_csv(self.out_tables / "scan_index.csv", index=False) + print(f"\nProcessing complete. Check {self.out_tables}/scan_index.csv for results.") + +#if __name__ == "__main__": + # This part only runs if you run COCA_processor.py DIRECTLY. + # It will NOT run when imported by your pipeline script. + #print("Running Processor in standalone mode...") + #processor = COCAProcessor(r"C:\coca_project") + #processor.process_all() \ No newline at end of file diff --git a/coca_project/src/data/COCA_resampler.py b/coca_project/src/data/COCA_resampler.py new file mode 100644 index 0000000..3d75cca --- /dev/null +++ b/coca_project/src/data/COCA_resampler.py @@ -0,0 +1,90 @@ +import pandas as pd +import SimpleITK as sitk +from pathlib import Path +from tqdm import tqdm + +class COCAResampler: + def __init__(self, project_root: str, target_spacing: list = [0.7, 0.7, 3.0]): + self.project_root = Path(project_root) + self.input_csv = self.project_root / "data_canonical" / "tables" / "scan_index.csv" + self.output_dir = self.project_root / "data_resampled" + self.target_spacing = target_spacing + self.output_dir.mkdir(parents=True, exist_ok=True) + + def process_volume(self, volume, is_mask=False): + # 1. Reorient to RAS (Right, Anterior, Superior) + # This standardizes the axis order across all scanners + volume = sitk.DICOMOrient(volume, 'RAS') + + # 2. Resample logic + original_spacing = volume.GetSpacing() + original_size = volume.GetSize() + new_size = [ + int(round(original_size[i] * (original_spacing[i] / self.target_spacing[i]))) + for i in range(3) + ] + + resample = sitk.ResampleImageFilter() + resample.SetOutputSpacing(self.target_spacing) + resample.SetSize(new_size) + resample.SetOutputDirection(volume.GetDirection()) + resample.SetOutputOrigin(volume.GetOrigin()) + resample.SetTransform(sitk.Transform()) + resample.SetDefaultPixelValue(volume.GetPixelIDValue()) + resample.SetInterpolator(sitk.sitkNearestNeighbor if is_mask else sitk.sitkLinear) + + return resample.Execute(volume) + + def run(self): + if not self.input_csv.exists(): + print("Index CSV not found.") + return + + df = pd.read_csv(self.input_csv) + updated_rows = [] + + print(f"Resampling to {self.target_spacing} and orienting to RAS...") + + for _, row in tqdm(df.iterrows(), total=len(df)): + scan_id = row['scan_id'] + input_f = Path(row['folder_path']) + out_f = self.output_dir / scan_id + out_f.mkdir(parents=True, exist_ok=True) + + try: + # Load + img = sitk.ReadImage(str(input_f / f"{scan_id}_img.nii.gz")) + seg = sitk.ReadImage(str(input_f / f"{scan_id}_seg.nii.gz")) + + # Process (Orient + Resample) + res_img = self.process_volume(img, is_mask=False) + res_seg = self.process_volume(seg, is_mask=True) + + # Save + img_out = out_f / f"{scan_id}_img.nii.gz" + seg_out = out_f / f"{scan_id}_seg.nii.gz" + sitk.WriteImage(res_img, str(img_out), useCompression=True) + sitk.WriteImage(res_seg, str(seg_out), useCompression=True) + + # Update the row for the final "Frozen" contract + row_dict = row.to_dict() + row_dict.update({ + "image_path": str(img_out), + "mask_path": str(seg_out), + "spacing": str(self.target_spacing), + "shape": str(res_img.GetSize()), + "has_pos": 1 if row['voxels'] > 0 else 0 + }) + updated_rows.append(row_dict) + + except Exception as e: + print(f"Error on {scan_id}: {e}") + + # Save the final Frozen Dataset Contract + final_df = pd.DataFrame(updated_rows) + final_df.to_parquet(self.project_root / "data_canonical" / "tables" / "frozen_dataset_v1.parquet") + print("Dataset contract frozen at frozen_dataset_v1.parquet") + +#if __name__ == "__main__": + #resampler = COCAResampler(r"C:\coca_project") + #resampler.run() \ No newline at end of file diff --git a/coca_project/src/data/__pycache__/big_data_loader.cpython-310.pyc b/coca_project/src/data/__pycache__/big_data_loader.cpython-310.pyc new file mode 100644 index 0000000..cbe177b Binary files /dev/null and b/coca_project/src/data/__pycache__/big_data_loader.cpython-310.pyc differ diff --git a/coca_project/src/data/big_data_loader.py b/coca_project/src/data/big_data_loader.py new file mode 100644 index 0000000..5f03409 --- /dev/null +++ b/coca_project/src/data/big_data_loader.py @@ -0,0 +1,112 @@ +import os +import pandas as pd +import torch +from monai.transforms import ( + Compose, LoadImaged, EnsureChannelFirstd, + RandRotate90d, RandCropByPosNegLabeld, + RandFlipd, SpatialPadd, ToTensord, RandGaussianNoised, CenterSpatialCropd, + ScaleIntensityRanged, CropForegroundd +) +from monai.data import Dataset, DataLoader, CacheDataset +# Rule: Standardize Heart CT intensities (-100 to 400 HU) +from monai.transforms import CastToTyped, AsDiscreted +import nibabel as nib + +def get_transforms(patch_size): + # Ensure patch_size is explicitly (128, 128, 32) + print("🚀 LOADER CHECK: THE CODE IS ACTUALLY RUNNING!") + train_transforms = Compose([ + LoadImaged(keys=["image", "label"]), + EnsureChannelFirstd(keys=["image", "label"]), + # 1. CROP: Find the patient's body (HU > -200 ignores the lungs/air) + # This centers the "camera" on the soft tissue of the chest. + CropForegroundd( + keys=["image", "label"], + source_key="image", + select_fn=lambda x: x > -200, + margin=10 + ), + + # 2. INTENSITY: The "Anti-Bone" Filter + # We clip the max at 600. Why? Because calcium is 200-500. + # Anything 1000+ (Bone) gets squashed down to 1.0, + # making it look less "special" to the model. + ScaleIntensityRanged( + keys=["image"], + a_min=-100, a_max=1000, + b_min=0.0, b_max=1.0, + clip=True + ), + + # 3. SAMPLE: Force the model to look at labels + RandCropByPosNegLabeld( + keys=["image", "label"], + label_key="label", + spatial_size=(96, 96, 32), + pos=1, # Focus heavily on the calcium labels + neg=1, + num_samples=4 + ), + ToTensord(keys=["image", "label"]), + # 1. CAST FIRST: Prevents rounding issues on ARM/Windows + CastToTyped(keys=["image", "label"], dtype=(torch.float32, torch.uint8)), + + # 2. EXPLICIT PADDING: Added mode to ensure it's not trying to 'reflect' + SpatialPadd( + keys=["image", "label"], + spatial_size=patch_size, + mode="constant" + ), + + AsDiscreted(keys=["label"], threshold=0.5), + + # Adding back the augmentations to help the model learn + RandFlipd(keys=["image", "label"], spatial_axis=[0], prob=0.5), + RandRotate90d(keys=["image", "label"], prob=0.5, max_k=3), + ToTensord(keys=["image", "label"]), + RandGaussianNoised(keys=["image"], prob=0.2, mean=0.0, std=0.1), + ]) + + val_transforms = Compose([ + LoadImaged(keys=["image", "label"]), + EnsureChannelFirstd(keys=["image", "label"]), + # Padding is still needed here for consistency + SpatialPadd(keys=["image", "label"], spatial_size=patch_size, mode="constant"), + AsDiscreted(keys=["label"], threshold=0.5), + ToTensord(keys=["image", "label"]), + ]) + return train_transforms, val_transforms +# 1. Define your helper function +def has_calcium(file_dict): + lbl_img = nib.load(file_dict['label']) + return lbl_img.get_fdata().max() > 0 + +def get_coca_loaders(train_parquet, val_parquet, patch_size=(128, 128, 32), batch_size=2): + train_df = pd.read_parquet(train_parquet) + val_df = pd.read_parquet(val_parquet) + + train_files = [{"image": r["image_path"], "label": r["mask_path"]} for _, r in train_df.iterrows()] + val_files = [{"image": r["image_path"], "label": r["mask_path"]} for _, r in val_df.iterrows()] + + train_trans, val_trans = get_transforms(patch_size) + + # 2. Split the list after you've defined your 'train_files' list + pos_files = [f for f in train_files if has_calcium(f)] + neg_files = [f for f in train_files if f not in pos_files] + + # 3. Create the 'Curated' list (e.g., all positive + 10% negative for context) + curated_train_files = pos_files + neg_files[:int(len(pos_files) * 0.1)] + + # 4. Pass the curated list into your MONAI Dataset + train_ds = CacheDataset( + data=curated_train_files, + transform=train_trans, + cache_rate=1.0 # Since your curated set is smaller, try to cache it all! + ) + val_ds = CacheDataset(data=val_files, transform=val_trans, cache_rate=1.0) + train_loader = DataLoader(train_ds, batch_size=batch_size, shuffle=True, num_workers=0) + + + val_loader = DataLoader(val_ds, batch_size=1, num_workers=4) + + return train_loader, val_loader \ No newline at end of file diff --git a/coca_project/src/data/resampler.py b/coca_project/src/data/resampler.py new file mode 100644 index 0000000..628b1f9 --- /dev/null +++ b/coca_project/src/data/resampler.py @@ -0,0 +1,78 @@ +import pandas as pd +from pathlib import Path +import SimpleITK as sitk +from tqdm import tqdm + +def resample_volume(volume, new_spacing=[1.0, 1.0, 1.0], is_mask=False): + """Resamples a volume to a target voxel spacing.""" + original_spacing = volume.GetSpacing() + original_size = volume.GetSize() + + # Calculate new size to keep the physical extent the same + new_size = [ + int(round(original_size[i] * (original_spacing[i] / new_spacing[i]))) + for i in range(3) + ] + + resample = sitk.ResampleImageFilter() + resample.SetOutputSpacing(new_spacing) + resample.SetSize(new_size) + resample.SetOutputDirection(volume.GetDirection()) + resample.SetOutputOrigin(volume.GetOrigin()) + resample.SetTransform(sitk.Transform()) + resample.SetDefaultPixelValue(volume.GetPixelIDValue()) + + # Use Nearest Neighbor for masks to keep values 0 and 1 + # Use Linear for images to preserve anatomical detail + if is_mask: + resample.SetInterpolator(sitk.sitkNearestNeighbor) + else: + resample.SetInterpolator(sitk.sitkLinear) + + return resample.Execute(volume) + +def main(): + PROJECT_ROOT = Path(r"C:\coca_project") + INPUT_CSV = PROJECT_ROOT / "data_canonical" / "tables" / "scan_index.csv" + OUTPUT_DIR = PROJECT_ROOT / "data_resampled" + + # Target spacing in mm: [x, y, z] + # Common standard is 1.0mm isotropic or native in-plane (0.5) with 1.5-3.0 slice thickness + TARGET_SPACING = [1.0, 1.0, 1.0] + + if not INPUT_CSV.exists(): + print(f"Index file not found: {INPUT_CSV}") + return + + df = pd.read_csv(INPUT_CSV) + OUTPUT_DIR.mkdir(parents=True, exist_ok=True) + + print(f"Resampling {len(df)} scans to {TARGET_SPACING} mm...") + + for _, row in tqdm(df.iterrows(), total=len(df)): + scan_id = row['scan_id'] + input_folder = Path(row['folder_path']) + + # Output folder for this specific scan + resampled_scan_folder = OUTPUT_DIR / scan_id + resampled_scan_folder.mkdir(parents=True, exist_ok=True) + + # 1. Load Original Files + img_path = input_folder / f"{scan_id}_img.nii.gz" + seg_path = input_folder / f"{scan_id}_seg.nii.gz" + + img = sitk.ReadImage(str(img_path)) + seg = sitk.ReadImage(str(seg_path)) + + # 2. Resample + res_img = resample_volume(img, new_spacing=TARGET_SPACING, is_mask=False) + res_seg = resample_volume(seg, new_spacing=TARGET_SPACING, is_mask=True) + + # 3. Save Resampled Files + sitk.WriteImage(res_img, str(resampled_scan_folder / f"{scan_id}_img.nii.gz"), useCompression=True) + sitk.WriteImage(res_seg, str(resampled_scan_folder / f"{scan_id}_seg.nii.gz"), useCompression=True) + + print("\nResampling complete. Data is now ready for ML training.") + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/coca_project/src/data/unnester.py b/coca_project/src/data/unnester.py new file mode 100644 index 0000000..3bdde93 --- /dev/null +++ b/coca_project/src/data/unnester.py @@ -0,0 +1,45 @@ +import os +import shutil +from pathlib import Path +from tqdm import tqdm + +def flatten_dicom_folders(root_dir): + root_path = Path(root_dir) + # Get all patient folders (the numeric ones) + patient_folders = [p for p in root_path.iterdir() if p.is_dir() and p.name.isdigit()] + + print(f"Checking {len(patient_folders)} patient folders for intermediate nesting...") + + for patient_dir in tqdm(patient_folders, desc="Flattening"): + # Find all .dcm files anywhere inside this patient directory + all_dcms = list(patient_dir.rglob("*.dcm")) + + for dcm_path in all_dcms: + # If the file is not already directly in the patient folder + if dcm_path.parent != patient_dir: + # Move to the patient_dir + target_path = patient_dir / dcm_path.name + + # Handle potential filename collisions (unlikely in DICOM) + if target_path.exists(): + target_path = patient_dir / f"{dcm_path.parent.name}_{dcm_path.name}" + + shutil.move(str(dcm_path), str(target_path)) + + # Cleanup: Remove now-empty subdirectories + for subfolder in list(patient_dir.iterdir()): + if subfolder.is_dir(): + try: + # Only removes if empty + shutil.rmtree(subfolder) + except Exception: + # If it contains non-dcm files, it stays + pass + +if __name__ == "__main__": + # Update this to your exact patient root + DICOM_PATIENT_ROOT = r"C:\coca_project\data_raw\dicom\Gated_release_final\Gated_release_final\patient" + + # It's always a good idea to have a backup or try on one folder first! + flatten_dicom_folders(DICOM_PATIENT_ROOT) + print("\nFlattening complete. All slices should now be directly inside patient ID folders.") \ No newline at end of file diff --git a/coca_project/src/dataset_splitter.py b/coca_project/src/dataset_splitter.py new file mode 100644 index 0000000..c6822ff --- /dev/null +++ b/coca_project/src/dataset_splitter.py @@ -0,0 +1,41 @@ +import pandas as pd +from sklearn.model_selection import train_test_split +from pathlib import Path + +def create_nnunet_split(parquet_path, output_dir): + # Load the frozen contract + df = pd.read_parquet(parquet_path) + + # 1. First split: Separate out the TEST set (15%) + # Stratify by 'has_pos' to keep calcium cases balanced + train_val_df, test_df = train_test_split( + df, + test_size=0.15, + random_state=42, + stratify=df['has_pos'] + ) + + # 2. Second split: Separate TRAIN and VAL from the remaining 85% + # 0.176 of 85% is roughly 15% of the total + train_df, val_df = train_test_split( + train_val_df, + test_size=0.176, + random_state=42, + stratify=train_val_df['has_pos'] + ) + + # 3. Save the splits + output_dir = Path(output_dir) + train_df.to_parquet(output_dir / "train_split.parquet", index=False) + val_df.to_parquet(output_dir / "val_split.parquet", index=False) + test_df.to_parquet(output_dir / "test_split.parquet", index=False) + + print(f"--- Data Split Complete ---") + print(f"Train: {len(train_df)} cases ({train_df['has_pos'].sum()} pos)") + print(f"Val: {len(val_df)} cases ({val_df['has_pos'].sum()} pos)") + print(f"Test: {len(test_df)} cases ({test_df['has_pos'].sum()} pos)") + +if __name__ == "__main__": + PARQUET = r"C:\coca_project\data_canonical\tables\metadata_summary.parquet" + OUT = r"C:\coca_project\data_canonical\tables" + create_nnunet_split(PARQUET, OUT) \ No newline at end of file diff --git a/coca_project/src/fix_calcium_roi.py b/coca_project/src/fix_calcium_roi.py new file mode 100644 index 0000000..9109c2c --- /dev/null +++ b/coca_project/src/fix_calcium_roi.py @@ -0,0 +1,221 @@ +import re +import numpy as np +import pandas as pd +import xml.etree.ElementTree as ET +import nibabel as nib + +# ---------------------------- +# Parsing helpers +# ---------------------------- + +def parse_triplet(s): + nums = re.findall(r"[-+]?\d*\.\d+|[-+]?\d+", s) + if len(nums) < 3: + return None + return tuple(float(x) for x in nums[:3]) + +def extract_roi_z_from_xml(xml_path): + tree = ET.parse(xml_path) + root = tree.getroot() + + z_vals = [] + + for d in root.findall(".//dict"): + children = list(d) + kv = {} + for i in range(0, len(children) - 1, 2): + if children[i].tag == "key": + kv[children[i].text] = children[i + 1] + + if "Center" in kv and kv["Center"].text: + t = parse_triplet(kv["Center"].text) + if t: + z_vals.append(t[2]) + + if "Point_mm" in kv and kv["Point_mm"].tag == "array": + for s in kv["Point_mm"].findall("string"): + if s.text: + t = parse_triplet(s.text) + if t: + z_vals.append(t[2]) + + z_vals = np.array(z_vals, dtype=float) + # downweight polygon density by using unique-ish z values + z_vals = np.unique(np.round(z_vals, 3)) + return z_vals + +def slice_z_positions_from_nifti(nifti_path, slice_axis=2): + img = nib.load(nifti_path) + aff = img.affine + shape = img.shape + + if slice_axis == 2: + ks = np.arange(shape[2]) + pts = np.stack([np.zeros_like(ks), np.zeros_like(ks), ks, np.ones_like(ks)], axis=0) + elif slice_axis == 0: + ks = np.arange(shape[0]) + pts = np.stack([ks, np.zeros_like(ks), np.zeros_like(ks), np.ones_like(ks)], axis=0) + elif slice_axis == 1: + ks = np.arange(shape[1]) + pts = np.stack([np.zeros_like(ks), ks, np.zeros_like(ks), np.ones_like(ks)], axis=0) + else: + raise ValueError("slice_axis must be 0, 1, or 2") + + world = aff @ pts + return world[2, :] + +def score_scan(z_roi, z_slices, spacing_z_mm=None): + z_roi = np.asarray(z_roi) + z_slices = np.asarray(z_slices) + + zs = np.sort(z_slices) + idx = np.searchsorted(zs, z_roi) + idx0 = np.clip(idx - 1, 0, len(zs) - 1) + idx1 = np.clip(idx, 0, len(zs) - 1) + d = np.minimum(np.abs(z_roi - zs[idx0]), np.abs(z_roi - zs[idx1])) + + tol = 2.0 + if spacing_z_mm is not None and float(spacing_z_mm) > 0: + tol = 0.6 * float(spacing_z_mm) + + return (d <= tol).mean(), d.mean(), tol + +# ---------------------------- +# Main: patient-folder-only matching +# ---------------------------- + +def assign_xml_to_scan_per_patient( + scans_df: pd.DataFrame, + xml_index_df: pd.DataFrame, + patient_col: str = "patient_folder", + xml_path_col: str = "xml_path", + scan_id_col: str = "scan_id", + nifti_col: str = "nifti_path", + spacing_col: str = "spacing_z_mm", +): + """ + xml_index_df: one row per patient_folder containing xml_path + scans_df: scan table with patient_folder and nifti paths + """ + results = [] + + for _, xr in xml_index_df.iterrows(): + pf = xr[patient_col] + xml_path = xr[xml_path_col] + + # Filter to ONLY scans for this patient_folder + cand = scans_df[scans_df[patient_col] == pf].copy() + + # If no scans or only 1 scan, skip heavy work + if len(cand) == 0: + results.append({ + patient_col: pf, + "xml_path": xml_path, + "status": "no_scans_found" + }) + continue + + if len(cand) == 1: + results.append({ + patient_col: pf, + "xml_path": xml_path, + "best_scan_id": cand.iloc[0][scan_id_col], + "status": "single_scan_only" + }) + continue + + # Only consider candidates with valid nifti paths + cand = cand[cand[nifti_col].astype(str).str.len() > 0] + if len(cand) == 0: + results.append({ + patient_col: pf, + "xml_path": xml_path, + "status": "no_valid_nifti_paths" + }) + continue + + z_roi = extract_roi_z_from_xml(xml_path) + if len(z_roi) == 0: + results.append({ + patient_col: pf, + "xml_path": xml_path, + "status": "no_roi_z_found" + }) + continue + + scored = [] + for _, sr in cand.iterrows(): + nifti_path = sr[nifti_col] + z_slices = slice_z_positions_from_nifti(nifti_path, slice_axis=2) + in_tol, mae, tol = score_scan(z_roi, z_slices, spacing_z_mm=sr.get(spacing_col, None)) + scored.append({ + patient_col: pf, + "xml_path": xml_path, + "scan_id": sr[scan_id_col], + "in_tol_frac": in_tol, + "mae_mm": mae, + "tol_mm": tol, + }) + + scored_df = pd.DataFrame(scored).sort_values(["in_tol_frac", "mae_mm"], ascending=[False, True]).reset_index(drop=True) + best = scored_df.iloc[0] + second = scored_df.iloc[1] if len(scored_df) > 1 else None + + out = { + patient_col: pf, + "xml_path": xml_path, + "best_scan_id": best["scan_id"], + "best_in_tol_frac": float(best["in_tol_frac"]), + "best_mae_mm": float(best["mae_mm"]), + "tol_mm": float(best["tol_mm"]), + "status": "matched" + } + if second is not None: + out.update({ + "second_scan_id": second["scan_id"], + "second_in_tol_frac": float(second["in_tol_frac"]), + "second_mae_mm": float(second["mae_mm"]), + }) + + results.append(out) + + return pd.DataFrame(results) + +# Example usage: +# xml_index_df = rois_df[['patient_folder','xml_path']].drop_duplicates() +# assignments = assign_xml_to_scan_per_patient(scans_df, xml_index_df) +# assignments.to_csv("roi_xml_scan_assignments.csv", index=False) + +if __name__ == "__main__": + import pandas as pd + + # ---- EDIT THESE PATHS ---- + SCANS_CSV = "data_canonical/tables/Gated_scan_index.csv" + ROIS_CSV = "data_canonical/tables/calcium_rois_sorted_by_patient_folder.csv" + OUTPUT_CSV = "data_canonical/tables/roi_xml_scan_assignments.csv" + + # ---- LOAD TABLES ---- + scans_df = pd.read_csv(SCANS_CSV) + + rois_df = pd.read_csv(ROIS_CSV) + rois_df["patient_folder"] = ( + rois_df["xml_path"] + .str.extract(r'\\calcium_xml\\(\d+)\.xml', expand=False) + .astype(int) + ) + + # one XML per patient_folder + xml_index_df = rois_df[["patient_folder", "xml_path"]].drop_duplicates() + + # ---- RUN MATCHING (patient-folder–restricted) ---- + assignments = assign_xml_to_scan_per_patient( + scans_df=scans_df, + xml_index_df=xml_index_df, + patient_col="patient_folder", + scan_id_col="scan_id", + nifti_col="nifti_path", + spacing_col="spacing_z_mm", + ) + + assignments.to_csv(OUTPUT_CSV, index=False) + print(f"Wrote {OUTPUT_CSV}") diff --git a/coca_project/src/image_cleaner.py b/coca_project/src/image_cleaner.py new file mode 100644 index 0000000..3388bc7 --- /dev/null +++ b/coca_project/src/image_cleaner.py @@ -0,0 +1,70 @@ +import os +import shutil +from pathlib import Path +from datetime import datetime, timedelta + +def cleanup_pre_yesterday_5pm(target_dir): + target_path = Path(target_dir) + + # 1. Calculate Yesterday at 5:00 PM + now = datetime.now() + yesterday_5pm = (now - timedelta(days=1)).replace(hour=21, minute=30, second=0, microsecond=0) + cutoff_ts = yesterday_5pm.timestamp() + + print(f"Target Directory: {target_path}") + print(f"Cutoff Time: {yesterday_5pm.strftime('%Y-%m-%d %I:%M %p')}") + print("-" * 40) + + files_deleted = 0 + folders_deleted = 0 + + # 2. Walk through directory (topdown=False is critical for deleting folders) + for root, dirs, files in os.walk(target_path, topdown=False): + current_dir = Path(root) + + # Handle Files + for name in files: + file_path = current_dir / name + if file_path.stat().st_mtime < cutoff_ts: + try: + file_path.unlink() + files_deleted += 1 + except Exception as e: + print(f" [ERROR] Could not delete file {name}: {e}") + + # Handle Folders + for name in dirs: + dir_path = current_dir / name + + # Check if folder is empty + is_empty = not any(dir_path.iterdir()) + + # Delete if it's empty OR if the folder itself is older than the cutoff + if is_empty or dir_path.stat().st_mtime < cutoff_ts: + try: + # We use rmtree if it's old, or rmdir if we only want to remove it if empty + if is_empty: + dir_path.rmdir() + folders_deleted += 1 + elif dir_path.stat().st_mtime < cutoff_ts: + shutil.rmtree(dir_path) + folders_deleted += 1 + except Exception: + # Folder might be in use or already deleted by a previous step + pass + + print(f"\nCleanup complete!") + print(f"Files removed: {files_deleted}") + print(f"Folders removed: {folders_deleted}") + +if __name__ == "__main__": + IMAGES_DIR = r"C:\coca_project\data_canonical\images" + + if Path(IMAGES_DIR).exists(): + confirm = input(f"Delete files/folders in {IMAGES_DIR} modified BEFORE yesterday 5PM? (y/n): ") + if confirm.lower() == 'y': + cleanup_pre_yesterday_5pm(IMAGES_DIR) + else: + print("Operation cancelled.") + else: + print(f"Directory not found: {IMAGES_DIR}") \ No newline at end of file diff --git a/coca_project/src/import nibabel as nib.py b/coca_project/src/import nibabel as nib.py new file mode 100644 index 0000000..7656098 --- /dev/null +++ b/coca_project/src/import nibabel as nib.py @@ -0,0 +1,11 @@ +import nibabel as nib +import numpy as np + +# Pick one of your resampled image paths from the parquet +img_path = rC:\coca_project\data_resampled\000a85335c17\000a85335c17_seg.nii.gz" +img = nib.load(img_path).get_fdata() + +print(f"Raw Min: {img.min()}") +print(f"Raw Max: {img.max()}") +print(f"Raw Mean: {img.mean()}") +print(f"Data Type: {img.dtype}") \ No newline at end of file diff --git a/coca_project/src/ingest_coca_dicom_to_nifti.py b/coca_project/src/ingest_coca_dicom_to_nifti.py new file mode 100644 index 0000000..cd445fa --- /dev/null +++ b/coca_project/src/ingest_coca_dicom_to_nifti.py @@ -0,0 +1,279 @@ +from __future__ import annotations +from pathlib import Path +from typing import Any, Dict, List, Optional +from collections import defaultdict +import json +import hashlib +from pathlib import Path +from typing import Dict, Any, List, Optional +import pandas as pd +import SimpleITK as sitk + + + + +def stable_id(*parts: str, n: int = 12) -> str: + h = hashlib.sha1("||".join(parts).encode("utf-8")).hexdigest() + return h[:n] + + +def safe_get_tag(reader: sitk.ImageSeriesReader, tag: str, file_index: int = 0) -> Optional[str]: + try: + if reader.HasMetaDataKey(file_index, tag): + return reader.GetMetaData(file_index, tag) + except Exception: + pass + return None + + +def extract_geometry(image: sitk.Image) -> Dict[str, Any]: + return { + "size_xyz": list(image.GetSize()), + "spacing_xyz": list(image.GetSpacing()), + "origin_xyz": list(image.GetOrigin()), + "direction_3x3_flat": list(image.GetDirection()), + } + + +def ingest_series(series_dir: Path, series_uid: str, out_images: Path, cohort: str, patient_folder: str) -> Optional[Dict[str, Any]]: + file_names = sitk.ImageSeriesReader.GetGDCMSeriesFileNames(str(series_dir), series_uid) + if not file_names: + return None + + reader = sitk.ImageSeriesReader() + reader.SetFileNames(file_names) + reader.MetaDataDictionaryArrayUpdateOn() + reader.LoadPrivateTagsOn() + + image = reader.Execute() + + modality = safe_get_tag(reader, "0008|0060") # Modality + if modality and modality.strip().upper() != "CT": + return None + + series_description = safe_get_tag(reader, "0008|103e") # SeriesDescription + protocol_name = safe_get_tag(reader, "0018|1030") # ProtocolName + slice_thickness = safe_get_tag(reader, "0018|0050") # SliceThickness + kernel = safe_get_tag(reader, "0018|1210") # ConvolutionKernel (often helpful) + + scan_id = stable_id(str(series_dir.resolve()), series_uid) + + nifti_path = out_images / f"{scan_id}.nii.gz" + sitk.WriteImage(image, str(nifti_path), useCompression=True) + + meta = extract_geometry(image) + meta.update({ + "scan_id": scan_id, + "cohort": cohort, # gated / nongated (or whatever you call it) + "patient_folder": patient_folder, + "series_uid": series_uid, + "source_series_dir": str(series_dir.resolve()), + "num_files_in_series": len(file_names), + "modality": modality, + "series_description": series_description, + "protocol_name": protocol_name, + "slice_thickness_tag_mm": slice_thickness, + "convolution_kernel": kernel, + "num_slices_z": int(image.GetSize()[2]), + "spacing_z_mm": float(image.GetSpacing()[2]), + }) + + json_path = out_images / f"{scan_id}.json" + json_path.write_text(json.dumps(meta, indent=2)) + + return { + "scan_id": scan_id, + "cohort": cohort, + "patient_folder": patient_folder, + "series_uid": series_uid, + "source_series_dir": str(series_dir.resolve()), + "nifti_path": str(nifti_path.resolve()), + "json_path": str(json_path.resolve()), + "modality": modality, + "series_description": series_description, + "protocol_name": protocol_name, + "slice_thickness_tag_mm": slice_thickness, + "convolution_kernel": kernel, + "num_slices_z": meta["num_slices_z"], + "spacing_x_mm": meta["spacing_xyz"][0], + "spacing_y_mm": meta["spacing_xyz"][1], + "spacing_z_mm": meta["spacing_xyz"][2], + "size_x": meta["size_xyz"][0], + "size_y": meta["size_xyz"][1], + "size_z": meta["size_xyz"][2], + } + + + +def _infer_patient_folder(dicom_root: Path, series_dir: Path) -> str: + """ + Heuristics to infer patient folder across different layouts. + + Priority: + 1) If there's an ancestor literally named 'patient', use the next part after it. + e.g., .../patient// + 2) Otherwise, use numeric directory names near the tail of the path. + e.g., .../deidentified_nongated/deidentified_nongated/9/9 -> patient_folder='9' + """ + try: + rel_parts = series_dir.relative_to(dicom_root).parts + except Exception: + rel_parts = series_dir.parts + + # Rule 1: COCA-like /patient//... + lower_parts = [p.lower() for p in rel_parts] + if "patient" in lower_parts: + i = lower_parts.index("patient") + if i + 1 < len(rel_parts): + return rel_parts[i + 1] + + # Rule 2: fallback: last numeric-ish dir in the relative path + numeric_parts = [p for p in rel_parts if p.isdigit()] + if numeric_parts: + # Usually the first numeric in the trailing chain corresponds to patient id, + # but layouts vary; this is a reasonable default. + return numeric_parts[-2] if len(numeric_parts) >= 2 else numeric_parts[-1] + + return "unknown" + + +def _infer_series_name(series_dir: Path) -> str: + return series_dir.name + + +def find_series_candidate_dirs( + dicom_root: Path, + *, + min_dcm_files: int = 5, +) -> List[Dict[str, Any]]: + """ + Layout-agnostic DICOM series discovery. + + Strategy: + - Walk all *.dcm files under dicom_root (case-insensitive by checking suffix). + - Group by parent directory (series_dir). + - Keep directories with >= min_dcm_files. + - Infer cohort/patient metadata from the path using heuristics. + + Returns list of dicts: + { + "cohort": str, + "patient_folder": str, + "series_dir": Path, + "series_name": str, + "dcm_count": int, + "example_dcm": Path + } + """ + dicom_root = Path(dicom_root) + + # Group DICOM files by their parent directory + counts: Dict[Path, int] = defaultdict(int) + example_file: Dict[Path, Path] = {} + + # Use rglob("*") and suffix check to catch .dcm, .DCM, etc. + SKIP_DIRS = {"gated_release_final"} + + for p in dicom_root.rglob("*"): + # Skip gated data entirely (already processed) + if any(part.lower() in SKIP_DIRS for part in p.parts): + continue + + if p.is_file() and p.suffix.lower() == ".dcm": + parent = p.parent + counts[parent] += 1 + if parent not in example_file: + example_file[parent] = p + + candidates: List[Dict[str, Any]] = [] + + for series_dir, n in counts.items(): + if n < min_dcm_files: + continue + + # cohort = first path component under dicom_root (if possible) + try: + cohort = series_dir.relative_to(dicom_root).parts[0] + except Exception: + cohort = "unknown" + + patient_folder = _infer_patient_folder(dicom_root, series_dir) + + candidates.append({ + "cohort": cohort, + "patient_folder": patient_folder, + "series_dir": series_dir, + "series_name": _infer_series_name(series_dir), + "dcm_count": n, + "example_dcm": example_file.get(series_dir), + }) + + # Optional: sort for stable output + candidates.sort(key=lambda d: (str(d["cohort"]), str(d["patient_folder"]), str(d["series_dir"]))) + return candidates + + +def main(): + project_root = Path(r"C:\coca_project") # <- change if needed + dicom_root = project_root / "data_raw" / "dicom" + + out_images = project_root / "data_canonical" / "images" + out_tables = project_root / "data_canonical" / "tables" + out_images.mkdir(parents=True, exist_ok=True) + out_tables.mkdir(parents=True, exist_ok=True) + + if not dicom_root.exists(): + raise FileNotFoundError(f"Missing: {dicom_root}. Copy DICOM folders there first.") + + candidates = find_series_candidate_dirs(dicom_root) + print(f"Found {len(candidates)} series candidate directories under {dicom_root}") + + rows: List[Dict[str, Any]] = [] + for item in candidates: + cohort = item["cohort"] + patient_folder = item["patient_folder"] + series_dir: Path = item["series_dir"] + + # Skip obvious non-image label folders if any slipped in + if series_dir.name.lower() in {"calcium_xml", "xml", "labels"}: + continue + + try: + series_uids = sitk.ImageSeriesReader.GetGDCMSeriesIDs(str(series_dir)) + except Exception: + series_uids = None + + if not series_uids: + continue + + for uid in series_uids: + try: + row = ingest_series(series_dir, uid, out_images, cohort=cohort, patient_folder=patient_folder) + if row: + rows.append(row) + print(f"[OK] {row['scan_id']} cohort={cohort} patient={patient_folder} z={row['num_slices_z']} dir={series_dir.name}") + except Exception as e: + print(f"[SKIP] cohort={cohort} patient={patient_folder} dir={series_dir.name} uid={uid}: {e}") + + if not rows: + print("No CT series ingested. If you expected some, tell me ONE example folder path that has .dcm files.") + return + + df = pd.DataFrame(rows).drop_duplicates(subset=["scan_id"]).reset_index(drop=True) + + csv_path = out_tables / "scan_index.csv" + df.to_csv(csv_path, index=False) + + parquet_path = out_tables / "scan_index.parquet" + try: + df.to_parquet(parquet_path, index=False) + print(f"Wrote: {parquet_path}") + except Exception as e: + print(f"[WARN] Parquet failed ({e}). CSV written: {csv_path}") + + print(f"Wrote: {csv_path}") + print(f"Total ingested CT series: {len(df)}") + + +if __name__ == "__main__": + main() diff --git a/coca_project/src/inspect_nii_files.py b/coca_project/src/inspect_nii_files.py new file mode 100644 index 0000000..1f4ab60 --- /dev/null +++ b/coca_project/src/inspect_nii_files.py @@ -0,0 +1,40 @@ +import os +import nibabel as nib +import numpy as np +from pathlib import Path + +def inspect_nifti_masks(target_dir): + target_path = Path(target_dir) + # Search for all files ending in _seg.nii.gz + seg_files = list(target_path.rglob("*_seg.nii.gz")) + + if not seg_files: + print(f"No segmentation files found in {target_dir}") + return + + print(f"{'Folder/File Name':<40} | {'Total Pixels':<12} | {'Calcium Pixels':<15}") + print("-" * 75) + + for seg_file in seg_files: + try: + # Load using nibabel (lightweight for quick inspection) + img = nib.load(seg_file) + data = img.get_fdata() + + total_voxels = data.size + calcium_voxels = np.sum(data == 1) + + # Identify the parent folder (ScanID) for clarity + display_name = f"{seg_file.parent.name}/{seg_file.name}" + + status = "✅ DATA FOUND" if calcium_voxels > 0 else "❌ EMPTY (ALL ZEROS)" + + print(f"{display_name[:40]:<40} | {total_voxels:<12} | {int(calcium_voxels):<15} {status}") + + except Exception as e: + print(f"Error reading {seg_file.name}: {e}") + +# --- RUN IT --- +PROJECT_ROOT = r"C:\coca_project" # Change if your path is different +images_dir = os.path.join(PROJECT_ROOT, "data_canonical", "images") +inspect_nifti_masks(images_dir) \ No newline at end of file diff --git a/coca_project/src/json_nii_renamer.py b/coca_project/src/json_nii_renamer.py new file mode 100644 index 0000000..c7b9d71 --- /dev/null +++ b/coca_project/src/json_nii_renamer.py @@ -0,0 +1,104 @@ +import json +import shutil +from pathlib import Path + +# ---- ADJUST THESE PATHS TO MATCH YOUR SETUP ---- + +# Folder that contains *.json and *.nii / *.nii.gz pairs (the folder in your screenshot) +SRC_ROOT = Path(r"C:\coca_project\data_canonical\images") + +# Your GitHub repo +REPO_ROOT = Path(r"C:\Users\Jagdf\PrediCT") + +# Destination sample-data folders +SAMPLE_NII_DEST = REPO_ROOT / "coca_project" / "sample_data" / "NIfTI" +SAMPLE_JSON_DEST = REPO_ROOT / "coca_project" / "sample_data" / "json" + +SAMPLE_NII_DEST.mkdir(parents=True, exist_ok=True) +SAMPLE_JSON_DEST.mkdir(parents=True, exist_ok=True) + +# Patients to include +PATIENT_RANGE = range(0, 21) # 0–20 inclusive + +# Name of the field in JSON that holds the patient ID +PATIENT_KEY = "patient_folder" # CHANGE THIS if your JSON uses a different key + + +def find_matching_nii(stem: str) -> Path | None: + """ + Given the JSON filename stem, find the matching NIfTI. + Tries .nii.gz first, then .nii. + """ + nii_gz = SRC_ROOT / f"{stem}.nii.gz" + nii = SRC_ROOT / f"{stem}.nii" + if nii_gz.is_file(): + return nii_gz + if nii.is_file(): + return nii + return None + + +def main(): + # track how many scans per patient so we can add _scan1, _scan2 if needed + scan_counts: dict[str, int] = {} + + for json_path in sorted(SRC_ROOT.glob("*.json")): + stem = json_path.stem + + with json_path.open("r", encoding="utf-8") as f: + data = json.load(f) + + # Extract patient ID from JSON + patient_raw = data.get(PATIENT_KEY) + if patient_raw is None: + print(f"[WARN] {json_path.name}: no '{PATIENT_KEY}' field, skipping") + continue + + # Normalize to string; if it's numeric, this will still work + patient_id_str = str(patient_raw).strip() + + # Only keep patients 0–20 + try: + pid_int = int(patient_id_str) + except ValueError: + print(f"[WARN] {json_path.name}: '{PATIENT_KEY}'='{patient_id_str}' is not an int, skipping") + continue + + if pid_int not in PATIENT_RANGE: + continue + + # Determine scan index for this patient + scan_counts.setdefault(patient_id_str, 0) + scan_counts[patient_id_str] += 1 + scan_idx = scan_counts[patient_id_str] + + # ---- JSON destination name ---- + if scan_idx == 1: + json_name = f"patient_{pid_int}.json" + else: + json_name = f"patient_{pid_int}_scan{scan_idx}.json" + json_dest = SAMPLE_JSON_DEST / json_name + + print(f"Copying JSON: {json_path} -> {json_dest}") + shutil.copy2(json_path, json_dest) + + # ---- Matching NIfTI ---- + nii_src = find_matching_nii(stem) + if nii_src is None: + print(f"[WARN] No NIfTI found matching {stem}, skipping NIfTI") + continue + + # preserve .nii or .nii.gz + ext = "".join(nii_src.suffixes) + if scan_idx == 1: + nii_name = f"patient_{pid_int}{ext}" + else: + nii_name = f"patient_{pid_int}_scan{scan_idx}{ext}" + + nii_dest = SAMPLE_NII_DEST / nii_name + print(f"Copying NIfTI: {nii_src} -> {nii_dest}") + shutil.copy2(nii_src, nii_dest) + + +if __name__ == "__main__": + main() diff --git a/coca_project/src/make_cac_masks_from_flat_images.py b/coca_project/src/make_cac_masks_from_flat_images.py new file mode 100644 index 0000000..6e72083 --- /dev/null +++ b/coca_project/src/make_cac_masks_from_flat_images.py @@ -0,0 +1,134 @@ +import os +import json +import xml.etree.ElementTree as ET +from pathlib import Path +import numpy as np +import pandas as pd +import SimpleITK as sitk +from tqdm import tqdm + +def parse_coca_xml(xml_path, image_shape): + """ + Parses COCA XML to find which slices have calcium and creates a binary mask. + image_shape: (slices, height, width) from SimpleITK (GetArrayFromImage) + """ + mask = np.zeros(image_shape, dtype=np.uint8) + segmented_slices = [] + + try: + tree = ET.parse(xml_path) + root = tree.getroot() + + # In COCA XML, annotations are usually under ImageAnnotation tags + for ann in root.findall(".//ImageAnnotation"): + # ImageIndex corresponds to the Z-slice (0-indexed) + slice_idx = int(ann.find("ImageIndex").text) + segmented_slices.append(slice_idx) + + # Find all coordinates for this slice + for roi in ann.findall(".//Coordinate"): + x = int(float(roi.find("X").text)) + y = int(float(roi.find("Y").text)) + + # Basic boundary check + if 0 <= slice_idx < image_shape[0] and 0 <= y < image_shape[1] and 0 <= x < image_shape[2]: + mask[slice_idx, y, x] = 1 + + except Exception as e: + print(f"Error parsing XML {xml_path}: {e}") + + return mask, sorted(list(set(segmented_slices))) + +def process_patient(patient_id, dicom_dir, xml_path, output_dir): + # 1. Load DICOM Series + reader = sitk.ImageSeriesReader() + dicom_names = reader.GetGDCMSeriesFileNames(str(dicom_dir)) + reader.SetFileNames(dicom_names) + image = reader.Execute() + + # 2. Setup Patient Directory + pat_out_dir = output_dir / "images" / patient_id + pat_out_dir.mkdir(parents=True, exist_ok=True) + + img_nifti_path = pat_out_dir / f"{patient_id}_original.nii.gz" + mask_nifti_path = pat_out_dir / f"{patient_id}_seg.nii.gz" + json_path = pat_out_dir / f"{patient_id}_meta.json" + + # 3. Create Segmentation Mask + # SimpleITK image to Numpy (Note: SITK is [z, y, x]) + img_array = sitk.GetArrayFromImage(image) + mask_array, seg_slices = parse_coca_xml(xml_path, img_array.shape) + + # Convert Numpy mask back to SITK Image and copy geometry (Affine Alignment) + mask_image = sitk.GetImageFromArray(mask_array) + mask_image.CopyInformation(image) + + # 4. Save NIfTI Files + sitk.WriteImage(image, str(img_nifti_path)) + sitk.WriteImage(mask_image, str(mask_nifti_path)) + + # 5. Extract Metadata + metadata = { + "patient_id": patient_id, + "size": image.GetSize(), + "spacing": image.GetSpacing(), + "origin": image.GetOrigin(), + "direction": image.GetDirection(), + "segmented_slices": seg_slices, + "num_segmented_slices": len(seg_slices), + "dicom_source": str(dicom_dir), + "xml_source": str(xml_path) + } + + with open(json_path, 'w') as f: + json.dump(metadata, f, indent=4) + + return { + "patient_id": patient_id, + "img_path": str(img_nifti_path), + "mask_path": str(mask_nifti_path), + "json_path": str(json_path), + "total_slices": image.GetSize()[2], + "seg_slices_count": len(seg_slices), + "seg_slice_indices": str(seg_slices) + } + +def main(): + # --- CONFIGURE PATHS HERE --- + PROJECT_ROOT = Path(r"C:\coca_project") + DICOM_ROOT = PROJECT_ROOT / "data_raw" / "dicom" + XML_ROOT = PROJECT_ROOT / "data_raw" / "xml" + OUTPUT_ROOT = PROJECT_ROOT / "data_canonical" + TABLE_DIR = OUTPUT_ROOT / "tables" + # ---------------------------- + + TABLE_DIR.mkdir(parents=True, exist_ok=True) + results = [] + + # Assumption: XML files are named by patient ID (e.g., 0a9b.xml) + # and match folder names in DICOM_ROOT + all_xmls = list(XML_ROOT.glob("*.xml")) + + print(f"Starting ingestion for {len(all_xmls)} patients...") + + for xml_file in tqdm(all_xmls): + patient_id = xml_file.stem + # Adjust this logic if your DICOM folders are nested differently + dicom_folder = DICOM_ROOT / patient_id + + if dicom_folder.exists(): + try: + row = process_patient(patient_id, dicom_folder, xml_file, OUTPUT_ROOT) + results.append(row) + except Exception as e: + print(f"Failed patient {patient_id}: {e}") + else: + print(f"DICOM folder missing for {patient_id}") + + # 6. Create Master Table + df = pd.DataFrame(results) + df.to_csv(TABLE_DIR / "summary_table.csv", index=False) + print(f"Ingestion complete. Table saved to {TABLE_DIR / 'summary_table.csv'}") + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/coca_project/src/make_cac_masks_from_flatimages.py b/coca_project/src/make_cac_masks_from_flatimages.py new file mode 100644 index 0000000..542743f --- /dev/null +++ b/coca_project/src/make_cac_masks_from_flatimages.py @@ -0,0 +1,391 @@ +#!/usr/bin/env python3 +""" +make_cac_masks_from_flat_images.py + +Build per-case folders with: + out_root/ + case___/ + ct.nii.gz + seg.nii.gz + +Inputs: +- ROI table: Parquet/CSV with columns: + xml_path, image_index, point_px + Optional: series_uid column (recommended if multiple series per patient) +- Flat images directory containing paired files: + .nii.gz and .json + JSON contains patient folder and SeriesInstanceUID (key names configurable) + +Rasterizes per-slice polygons (Point_px) into a 3D segmentation NIfTI aligned to CT. +""" + +import argparse +import ast +import json +import os +import re +import shutil +from pathlib import Path +from typing import Dict, List, Tuple, Optional, Any + +import numpy as np +import pandas as pd +import nibabel as nib + +try: + from skimage.draw import polygon as sk_polygon +except ImportError as e: + raise SystemExit("Missing scikit-image. Install: pip install scikit-image") from e + + +_NUM_PAIR_RE = re.compile(r"\(?\s*([+-]?\d+(?:\.\d+)?)\s*,\s*([+-]?\d+(?:\.\d+)?)\s*\)?") + +def parse_points_field(val) -> List[Tuple[float, float]]: + """Parse polygon points into list of (x,y) floats from many encodings.""" + if val is None or (isinstance(val, float) and np.isnan(val)): + return [] + if isinstance(val, (list, tuple)): + pts: List[Tuple[float, float]] = [] + for item in val: + pts.extend(parse_points_field(item)) + if len(pts) == 0 and len(val) > 0 and isinstance(val[0], (list, tuple)) and len(val[0]) == 2: + return [(float(p[0]), float(p[1])) for p in val] + return pts + + s = str(val).strip() + if s == "" or s.lower() in ("nan", "none", "null"): + return [] + + # Try literal eval first + try: + obj = ast.literal_eval(s) + if isinstance(obj, (list, tuple)): + pts: List[Tuple[float, float]] = [] + for item in obj: + if isinstance(item, (list, tuple)) and len(item) == 2: + pts.append((float(item[0]), float(item[1]))) + else: + pts.extend(parse_points_field(item)) + return pts + if isinstance(obj, str): + s = obj.strip() + except Exception: + pass + + matches = _NUM_PAIR_RE.findall(s) + return [(float(x), float(y)) for x, y in matches] + + +def safe_case_name(s: str) -> str: + s = str(s).strip().replace("\\", "/").strip("/") + s = s.replace("/", "_") + s = re.sub(r"[^A-Za-z0-9_\-\.]+", "_", s) + return s if s else "unknown" + + +def copy_or_link(src: Path, dst: Path, link: bool) -> None: + dst.parent.mkdir(parents=True, exist_ok=True) + if dst.exists(): + return + if link: + os.link(src, dst) + else: + shutil.copy2(src, dst) + + +def read_table(path: Path) -> pd.DataFrame: + suf = path.suffix.lower() + if suf == ".parquet": + return pd.read_parquet(path) + if suf in (".csv", ".tsv"): + return pd.read_csv(path, sep="\t" if suf == ".tsv" else ",") + raise SystemExit("ROI table must be .parquet, .csv, or .tsv") + + +def get_nested(d: Dict[str, Any], keys: List[str]) -> Optional[Any]: + """Return first matching value for any dotted key path in keys.""" + for k in keys: + cur: Any = d + ok = True + for part in k.split("."): + if isinstance(cur, dict) and part in cur: + cur = cur[part] + else: + ok = False + break + if ok: + return cur + return None + + +def build_image_index(images_dir: Path, + json_patient_keys: List[str], + json_series_keys: List[str]) -> Tuple[ + Dict[Tuple[str, str], Path], + Dict[str, List[Tuple[str, Path]]] + ]: + """ + Index JSON+NIfTI pairs. + Returns: + by_patient_series[(patient_folder, series_uid)] = nii_path + by_patient[patient_folder] = [(series_uid, nii_path), ...] + """ + by_patient_series: Dict[Tuple[str, str], Path] = {} + by_patient: Dict[str, List[Tuple[str, Path]]] = {} + + json_paths = sorted(images_dir.glob("*.json")) + for jp in json_paths: + stem = jp.stem + nii_candidates = [] + # support .nii.gz primarily, but accept .nii + p1 = images_dir / f"{stem}.nii.gz" + p2 = images_dir / f"{stem}.nii" + if p1.exists(): + nii_candidates.append(p1) + if p2.exists(): + nii_candidates.append(p2) + if len(nii_candidates) == 0: + continue + nii_path = nii_candidates[0] + + try: + meta = json.loads(jp.read_text(encoding="utf-8")) + except Exception: + continue + + patient_folder = get_nested(meta, json_patient_keys) + series_uid = get_nested(meta, json_series_keys) + + if patient_folder is None or series_uid is None: + # can't index reliably + continue + + patient_folder = str(patient_folder) + series_uid = str(series_uid) + + by_patient_series[(patient_folder, series_uid)] = nii_path + by_patient.setdefault(patient_folder, []).append((series_uid, nii_path)) + + # make deterministic order in by_patient + for pf in by_patient: + by_patient[pf] = sorted(by_patient[pf], key=lambda t: (t[0], str(t[1]))) + + return by_patient_series, by_patient + + +def detect_z_axis(ct_shape: Tuple[int, int, int], max_image_index: int) -> int: + """Choose axis whose length best matches max_image_index+1.""" + target = max_image_index + 1 + diffs = [abs(ct_shape[a] - target) for a in range(3)] + return int(np.argmin(diffs)) + + +def fill_polygon(mask_rc: np.ndarray, pts_xy: List[Tuple[float, float]]) -> None: + """mask_rc shape (rows, cols), pts_xy are (x,y).""" + if len(pts_xy) < 3: + return + xs = np.array([p[0] for p in pts_xy], dtype=np.float32) + ys = np.array([p[1] for p in pts_xy], dtype=np.float32) + rr, cc = sk_polygon(ys, xs, shape=mask_rc.shape) + mask_rc[rr, cc] = 1 + + +def make_slice_mask(ct_shape: Tuple[int, int, int], z_axis: int, pts_xy: List[Tuple[float, float]]) -> np.ndarray: + """ + Create slice mask in the plane orthogonal to z_axis. + Assumes Point_px is defined on the axial slice plane in pixel coords. + """ + a0, a1, a2 = ct_shape + if z_axis == 2: + X, Y = a0, a1 + mask_yx = np.zeros((Y, X), dtype=np.uint8) + fill_polygon(mask_yx, pts_xy) + return mask_yx.T # (X,Y) + if z_axis == 0: + Y, X = a1, a2 + mask_yx = np.zeros((Y, X), dtype=np.uint8) + fill_polygon(mask_yx, pts_xy) + return mask_yx # (Y,X) + if z_axis == 1: + R, C = a0, a2 + mask_rc = np.zeros((R, C), dtype=np.uint8) + fill_polygon(mask_rc, pts_xy) + return mask_rc # (axis0, axis2) + raise ValueError("z_axis must be 0,1,2") + + +def assign_slice(seg: np.ndarray, z_axis: int, z: int, slice_mask: np.ndarray) -> None: + """Write slice_mask into seg at slice index z along z_axis.""" + if z_axis == 2: + seg[..., z] = np.maximum(seg[..., z], slice_mask.astype(seg.dtype)) + elif z_axis == 0: + seg[z, :, :] = np.maximum(seg[z, :, :], slice_mask.astype(seg.dtype)) + elif z_axis == 1: + seg[:, z, :] = np.maximum(seg[:, z, :], slice_mask.astype(seg.dtype)) + else: + raise ValueError("z_axis must be 0,1,2") + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--roi-table", required=True, help="Parquet/CSV/TSV with ROIs (xml_path, image_index, point_px).") + ap.add_argument("--images-dir", required=True, help="Flat folder containing .nii(.gz) and .json pairs.") + ap.add_argument("--out-root", required=True, help="Output folder for case_*/ct.nii.gz and seg.nii.gz") + + ap.add_argument("--xml-path-col", default="xml_path") + ap.add_argument("--image-index-col", default="image_index") + ap.add_argument("--points-col", default="point_px") + + ap.add_argument("--series-uid-col", default=None, + help="Optional series UID column in ROI table. Strongly recommended if multiple series per patient.") + + ap.add_argument("--xml-to-patient-regex", default=None, + help="Optional regex with one capture group to extract patient_folder from xml_path. " + "If omitted, uses parent folder name of xml_path.") + + ap.add_argument("--json-patient-keys", default="patient_folder,patientFolder,patient.folder,PatientFolder", + help="Comma-separated JSON key paths for patient folder (supports dotted paths).") + ap.add_argument("--json-series-keys", default="series_uid,SeriesInstanceUID,seriesInstanceUID,series.uid,SeriesUID", + help="Comma-separated JSON key paths for series UID (supports dotted paths).") + + ap.add_argument("--flip-z", action="store_true", help="If ImageIndex is reversed relative to NIfTI z order.") + ap.add_argument("--z-offset", type=int, default=0, help="Constant offset added to ImageIndex.") + ap.add_argument("--z-axis", type=int, default=-1, help="Force z-axis 0/1/2. Default auto.") + ap.add_argument("--limit-cases", type=int, default=20, help="Process first N cases (default 20).") + ap.add_argument("--link-ct", action="store_true", help="Hardlink CT into output (same filesystem only).") + + args = ap.parse_args() + + roi_table = Path(args.roi_table) + images_dir = Path(args.images_dir) + out_root = Path(args.out_root) + out_root.mkdir(parents=True, exist_ok=True) + + df = read_table(roi_table) + + for col in [args.xml_path_col, args.image_index_col, args.points_col]: + if col not in df.columns: + raise SystemExit(f"Missing required column '{col}'. Available: {list(df.columns)}") + + if args.series_uid_col is not None and args.series_uid_col not in df.columns: + raise SystemExit(f"series-uid-col '{args.series_uid_col}' not found. Available: {list(df.columns)}") + + # Derive patient folder key from xml_path + if args.xml_to_patient_regex: + rx = re.compile(args.xml_to_patient_regex) + def extract_patient(p: str) -> str: + m = rx.search(str(p).replace("\\", "/")) + return m.group(1) if m else str(Path(str(p)).parent) + df["_patient_folder"] = df[args.xml_path_col].apply(extract_patient).astype(str) + else: + def parent_name(p: str) -> str: + p = str(p).replace("\\", "/") + return Path(p).parent.name if Path(p).parent.name else p + df["_patient_folder"] = df[args.xml_path_col].apply(parent_name).astype(str) + + # Index images + json_patient_keys = [k.strip() for k in args.json_patient_keys.split(",") if k.strip()] + json_series_keys = [k.strip() for k in args.json_series_keys.split(",") if k.strip()] + by_ps, by_p = build_image_index(images_dir, json_patient_keys, json_series_keys) + + # Group cases by patient (and series if available) + if args.series_uid_col: + df["_series_uid"] = df[args.series_uid_col].astype(str) + group_cols = ["_patient_folder", "_series_uid"] + else: + group_cols = ["_patient_folder"] + + grouped = df.groupby(group_cols, dropna=False) + + # Select up to N groups + group_keys = list(grouped.groups.keys())[: max(0, int(args.limit_cases))] + + skipped_ambiguous = 0 + skipped_missing = 0 + processed = 0 + + for gk in group_keys: + if args.series_uid_col: + patient_folder, series_uid = gk + series_uid = str(series_uid) + else: + patient_folder = gk + series_uid = None + + patient_folder = str(patient_folder) + + # Resolve CT nii + ct_path: Optional[Path] = None + if series_uid is not None: + ct_path = by_ps.get((patient_folder, series_uid)) + if ct_path is None: + skipped_missing += 1 + print(f"[SKIP missing] No CT for patient='{patient_folder}' series='{series_uid}'") + continue + else: + cands = by_p.get(patient_folder, []) + if len(cands) == 0: + skipped_missing += 1 + print(f"[SKIP missing] No CT candidates for patient='{patient_folder}'") + continue + if len(cands) > 1: + skipped_ambiguous += 1 + print(f"[SKIP ambiguous] patient='{patient_folder}' has {len(cands)} series; " + f"provide --series-uid-col to disambiguate.") + continue + series_uid, ct_path = cands[0] + + assert ct_path is not None + case_df = grouped.get_group(gk).copy() + + # Load CT + ct_img = nib.load(str(ct_path)) + ct_data = ct_img.get_fdata(dtype=np.float32) + if ct_data.ndim != 3: + print(f"[SKIP] CT not 3D: {ct_path} shape={ct_data.shape}") + continue + ct_shape = ct_data.shape + + max_idx = int(case_df[args.image_index_col].max()) + z_axis = args.z_axis if args.z_axis in (0, 1, 2) else detect_z_axis(ct_shape, max_idx) + Z = ct_shape[z_axis] + + seg = np.zeros(ct_shape, dtype=np.uint8) + + for _, row in case_df.iterrows(): + pts = parse_points_field(row[args.points_col]) + if len(pts) < 3: + continue + + idx = int(row[args.image_index_col]) + int(args.z_offset) + if args.flip_z: + idx = (Z - 1) - idx + if idx < 0 or idx >= Z: + continue + + slice_mask = make_slice_mask(ct_shape, z_axis, pts) + assign_slice(seg, z_axis, idx, slice_mask) + + # Write output case folder + pf_safe = safe_case_name(patient_folder) + su_short = safe_case_name(str(series_uid))[:12] if series_uid else "noseries" + case_dir = out_root / f"case_{pf_safe}__{su_short}" + case_dir.mkdir(parents=True, exist_ok=True) + + ct_out = case_dir / "ct.nii.gz" + if not ct_out.exists(): + copy_or_link(ct_path, ct_out, link=args.link_ct) + + seg_img = nib.Nifti1Image(seg.astype(np.uint8), affine=ct_img.affine, header=ct_img.header) + seg_img.set_data_dtype(np.uint8) + seg_out = case_dir / "seg.nii.gz" + nib.save(seg_img, str(seg_out)) + + processed += 1 + print(f"[OK] {case_dir.name} z_axis={z_axis} flip_z={args.flip_z} seg_voxels={int(seg.sum())}") + + print(f"Done. processed={processed} skipped_missing={skipped_missing} skipped_ambiguous={skipped_ambiguous}") + print(f"Output: {out_root}") + +if __name__ == "__main__": + main() diff --git a/coca_project/src/nib.py b/coca_project/src/nib.py new file mode 100644 index 0000000..3c66d73 --- /dev/null +++ b/coca_project/src/nib.py @@ -0,0 +1,11 @@ +import nibabel as nib +import numpy as np + +# Pick one of your resampled image paths from the parquet +img_path = r"C:\coca_project\data_resampled\000a85335c17\000a85335c17_seg.nii.gz" +img = nib.load(img_path).get_fdata() + +print(f"Raw Min: {img.min()}") +print(f"Raw Max: {img.max()}") +print(f"Raw Mean: {img.mean()}") +print(f"Data Type: {img.dtype}") \ No newline at end of file diff --git a/coca_project/src/nii_view.py b/coca_project/src/nii_view.py new file mode 100644 index 0000000..bb6a2a9 --- /dev/null +++ b/coca_project/src/nii_view.py @@ -0,0 +1,31 @@ +import nibabel as nib +import matplotlib.pyplot as plt +import numpy as np + +# 1. Load the file +img = nib.load(r'C:\coca_project\data_canonical\images\000a85335c17\000a85335c17_img.nii.gz') + +# 2. Get the actual image data as a NumPy array +data = img.get_fdata() + +# 3. Check the shape (e.g., 256, 256, 150) +print(f"Image shape: {data.shape}") + +# 4. Visualize a single slice (the middle slice on the Z-axis) +middle_slice = data.shape[2] // 2 +plt.imshow(data[:, :, middle_slice], cmap='gray') +plt.title(f"Slice {middle_slice}") +plt.show() + +# 5. Access metadata (Header and Affine) +print(img.header) +print(img.affine) # Translation/rotation matrix for world coordinates + +# Get unique values in the data +unique_values = np.unique(data) + +if len(unique_values) < 20: # Arbitrary threshold, segmentations rarely have many labels + print(f"Likely a SEGMENTATION. Labels found: {unique_values}") +else: + print(f"Likely an IMAGE. Total unique intensity values: {len(unique_values)}") + print(f"Value range: {np.min(data)} to {np.max(data)}") \ No newline at end of file diff --git a/coca_project/src/parquet_view.py b/coca_project/src/parquet_view.py new file mode 100644 index 0000000..e6dd36e --- /dev/null +++ b/coca_project/src/parquet_view.py @@ -0,0 +1,3 @@ +import pandas as pd +df = pd.read_parquet(r"C:\coca_project\data_canonical\tables\metadata_summary.parquet") +df.to_csv(r"C:\coca_project\data_canonical\tables\metadata_summary.csv", index=False) diff --git a/coca_project/src/parquet_view.txt b/coca_project/src/parquet_view.txt new file mode 100644 index 0000000..bb52549 --- /dev/null +++ b/coca_project/src/parquet_view.txt @@ -0,0 +1,9 @@ +import pandas as pd + +df = pd.read_parquet( + r"C:\coca_project\derived\calcium_rois_raw.parquet" +) + +print(df.head()) +print(df.columns) +print(df.dtypes) \ No newline at end of file diff --git a/coca_project/src/plist_check.py b/coca_project/src/plist_check.py new file mode 100644 index 0000000..9d45df2 --- /dev/null +++ b/coca_project/src/plist_check.py @@ -0,0 +1,22 @@ +import plistlib +from pathlib import Path + +p = Path(r"C:\coca_project\data_raw\xml\calcium_xml\30.xml") + +with open(p, "rb") as f: + data = plistlib.load(f) + +print(type(data)) +print(data.keys() if isinstance(data, dict) else type(data)) + +for k in data.keys(): + print(k, type(data[k])) + +images = data["Images"] +print(len(images)) +print(images[0].keys()) + +for img in images[:20]: + print(img["ImageIndex"], img["ROIs"][0]["Center"]) + + diff --git a/coca_project/src/requirements.txt b/coca_project/src/requirements.txt new file mode 100644 index 0000000..c41c223 --- /dev/null +++ b/coca_project/src/requirements.txt @@ -0,0 +1,5 @@ +numpy +pandas +SimpleITK +opencv-python +tqdm diff --git a/coca_project/src/split_check.py b/coca_project/src/split_check.py new file mode 100644 index 0000000..936faec --- /dev/null +++ b/coca_project/src/split_check.py @@ -0,0 +1,23 @@ +import pandas as pd + +def audit_splits(train_path, val_path): + for name, path in [("TRAIN", train_path), ("VAL", val_path)]: + df = pd.read_parquet(path) + total = len(df) + positives = df['has_pos'].sum() + negatives = total - positives + + print(f"--- {name} SPLIT AUDIT ---") + print(f"Total Scans: {total}") + print(f"Positive (with calcium): {positives} ({positives/total:.1%})") + print(f"Negative (healthy): {negatives} ({negatives/total:.1%})") + + if positives == 0: + print(f"❌ WARNING: {name} split has ZERO calcium. Check your splitting logic!") + print("\n") + +if __name__ == "__main__": + audit_splits( + r"C:\coca_project\data_canonical\tables\train_split.parquet", + r"C:\coca_project\data_canonical\tables\val_split.parquet" + ) \ No newline at end of file diff --git a/coca_project/src/test_out/heart.nii.gz b/coca_project/src/test_out/heart.nii.gz new file mode 100644 index 0000000..c315842 Binary files /dev/null and b/coca_project/src/test_out/heart.nii.gz differ diff --git a/coca_project/src/train.py b/coca_project/src/train.py new file mode 100644 index 0000000..a3124b2 --- /dev/null +++ b/coca_project/src/train.py @@ -0,0 +1,374 @@ +import os +os.environ["KMP_DUPLICATE_LIB_OK"] = "TRUE" + +import torch +import numpy as np +import matplotlib +import matplotlib.pyplot as plt +import pandas as pd +import SimpleITK as sitk +from tqdm import tqdm + +from monai.networks.nets import UNet +from monai.losses import DiceFocalLoss +from monai.inferers import sliding_window_inference +from monai.metrics import DiceMetric +from monai.transforms import AsDiscrete + +from data.big_data_loader import get_coca_loaders + + +# --------------------------------------------------------------------------- +# CONFIGURATION +# --------------------------------------------------------------------------- +DEBUG = True # True = ARM laptop / fast iteration. False = cluster. +MODEL_SIZE = "heavy" # "light" or "heavy" + +TRAIN_P = r"C:\coca_project\data_canonical\tables\train_split.parquet" +VAL_P = r"C:\coca_project\data_canonical\tables\val_split.parquet" +MODEL_PATH = "best_coca_model.pth" + +# Hardware-scaled parameters +if DEBUG: + PATCH_SIZE = (96, 96, 16) # 3 slices is too thin for sliding window + BATCH_SIZE = 1 + NUM_EPOCHS = 5 + AUDIT_EVERY = 2 # save audit image every N batches +else: + PATCH_SIZE = (128, 128, 64) + BATCH_SIZE = 4 + NUM_EPOCHS = 100 + AUDIT_EVERY = 10 + +# Model architecture presets +MODEL_CONFIGS = { + "light": { + "channels": (16, 32, 64, 128), + "strides": (2, 2, 2), + "num_res_units": 1, + }, + "heavy": { + "channels": (32, 64, 128, 256, 512), + "strides": (2, 2, 2, 2), + "num_res_units": 2, + }, +} +# --------------------------------------------------------------------------- + + +# --------------------------------------------------------------------------- +# VISUALISATION TOOLS +# --------------------------------------------------------------------------- + +def save_audit_image(inputs, labels, conf_map, batch_idx, epoch, item_idx=0): + """ + Save a 3-panel audit image for one item in the batch. + Finds the z-slice where the model is most confident and shows: + CT | Ground truth overlay | Prediction confidence overlay + """ + conf = conf_map[item_idx] # (1, H, W, D) + z = int(np.unravel_index(conf.cpu().numpy().argmax(), conf.shape)[3]) + + img_s = inputs [item_idx, 0, :, :, z].cpu().numpy() + gt_s = labels [item_idx, 0, :, :, z].cpu().numpy() + pred_s = conf_map [item_idx, 0, :, :, z].detach().cpu().numpy() + + gt_present = gt_s.max() > 0 + + fig, axes = plt.subplots(1, 3, figsize=(18, 6)) + for ax in axes: + ax.axis("off") + + axes[0].imshow(img_s, cmap="gray"); axes[0].set_title(f"CT z={z}") + axes[1].imshow(img_s, cmap="gray") + axes[1].imshow(np.ma.masked_where(gt_s == 0, gt_s), cmap="Reds", alpha=0.6) + axes[1].set_title(f"GT {'PRESENT' if gt_present else 'absent'}") + axes[2].imshow(img_s, cmap="gray") + axes[2].imshow(pred_s, cmap="hot", alpha=0.6) + axes[2].set_title(f"Pred max={pred_s.max():.2f}") + + match = "✅" if gt_present else "❌" + plt.suptitle(f"Epoch {epoch} | Batch {batch_idx} | Item {item_idx} | GT match: {match}", fontsize=14) + + os.makedirs("audit_images", exist_ok=True) + plt.savefig(f"audit_images/epoch{epoch:03d}_batch{batch_idx:04d}_item{item_idx}.png", dpi=80) + plt.close(fig) + + +class SliceScroller: + """Interactive matplotlib scroller. Scroll wheel moves through z-slices.""" + + def __init__(self, image, mask): + self.image = image.cpu().numpy() if torch.is_tensor(image) else image + self.mask = mask.cpu().numpy() if torch.is_tensor(mask) else mask + self.slices = self.image.shape[0] + self.ind = self.slices // 2 + + v_min, v_max = np.percentile(self.image, [1, 99]) + if v_min == v_max: + v_min, v_max = self.image.min(), self.image.max() + self.v_min, self.v_max = v_min, v_max + + print(f" Dynamic range: {v_min:.4f} → {v_max:.4f} | slices: {self.slices}") + + self.fig, self.ax = plt.subplots(figsize=(8, 8)) + self.im = self.ax.imshow(self.image[self.ind], cmap="gray", + vmin=v_min, vmax=v_max) + masked = np.ma.masked_where(self.mask[self.ind] == 0, self.mask[self.ind]) + self.mask_im = self.ax.imshow(masked, cmap="spring", alpha=0.8) + self._update_title() + + self.fig.canvas.mpl_connect("scroll_event", self._on_scroll) + self.fig.canvas.mpl_connect("key_press_event", self._on_key) + + def _on_scroll(self, event): + self.ind = (self.ind + (1 if event.button == "up" else -1)) % self.slices + self._redraw() + + def _on_key(self, event): + if event.key in ("up", "right"): self.ind = (self.ind + 1) % self.slices + elif event.key in ("down", "left"): self.ind = (self.ind - 1) % self.slices + self._redraw() + + def _redraw(self): + self.im.set_data(self.image[self.ind]) + self.im.set_clim(self.v_min, self.v_max) + masked = np.ma.masked_where(self.mask[self.ind] == 0, self.mask[self.ind]) + self.mask_im.set_data(masked) + self._update_title() + self.fig.canvas.draw_idle() + + def _update_title(self): + has_label = self.mask[self.ind].max() > 0 + self.ax.set_title(f"Slice {self.ind}/{self.slices-1} " + f"{'🔴 calcium' if has_label else '⬜ no label'}") + + +def debug_loader_sanity(val_loader): + """ + Quick checks on one batch pulled through the full transform pipeline. + This reflects what the model actually sees — NOT the raw HU values on disk. + Raw CT files will show ~-1024 to 3000 HU range — that is completely normal. + Expected range after ScaleIntensityRanged: 0.0 to 1.0. + """ + print("\n── Loader sanity check ──") + print(" (values below are POST-transform — what the model sees, not raw HU)") + batch = next(iter(val_loader)) + img_data = batch["image"][0, 0].numpy() + + i_min, i_max = img_data.min(), img_data.max() + n_unique = len(np.unique(img_data)) + print(f" Image range : {i_min:.4f} → {i_max:.4f}") + print(f" Unique vals : {n_unique}") + + if n_unique < 10: + print(" ❌ Image looks binarised — check ScaleIntensityRanged in dataloader") + elif i_max > 1.05: + print(" ❌ Values exceed 1.0 after transforms — ScaleIntensityRanged clip=True may not be active") + elif i_min < -0.05: + print(" ⚠️ Values below 0.0 — check a_min in ScaleIntensityRanged") + else: + print(" ✅ Intensity looks correct (0→1)") + + lbl_sum = batch["label"][0, 0].sum().item() + print(f" Label voxels in first batch: {int(lbl_sum)}") + if lbl_sum == 0: + print(" ⚠️ No label voxels — sampler may have returned an all-negative patch") + + +def debug_visualise_positive(val_parquet: str): + """Load the 7th positive case directly (bypassing the loader) and launch scroller.""" + print("\n── Direct file visualisation ──") + df = pd.read_parquet(val_parquet) + + # Graceful fallback if parquet was generated before the new processor added + # has_calcium — derive it from the voxels column, or scan the mask files. + if "has_calcium" not in df.columns: + print(" ⚠️ 'has_calcium' column missing — parquet needs regenerating with " + "the new COCA_processor. Deriving from 'voxels' column as fallback...") + if "voxels" in df.columns: + df["has_calcium"] = df["voxels"] > 0 + else: + print(" ❌ No 'voxels' column either — cannot determine positives. " + "Please re-run COCA_processor.py to regenerate the parquet.") + return + + pos_cases = df[df["has_calcium"]] + + if pos_cases.empty: + print(" ❌ No positive cases in val parquet") + return + + row = pos_cases.iloc[min(6, len(pos_cases) - 1)] + print(f" Scan: {row['scan_id']}") + + img_arr = sitk.GetArrayFromImage(sitk.ReadImage(row["image_path"])) + mask_arr = sitk.GetArrayFromImage(sitk.ReadImage(row["mask_path"])) + + print(f" Mask max: {mask_arr.max()} | voxel sum: {int(mask_arr.sum())}") + if mask_arr.sum() == 0: + print(" ❗ Parquet says has_calcium but NIfTI is empty — resampling may have failed") + return + + matplotlib.use("TkAgg") # force interactive backend for scroll events + scroller = SliceScroller(img_arr, mask_arr) + scroller.ind = int(np.argmax(np.sum(mask_arr, axis=(1, 2)))) + scroller._redraw() + plt.show() + + +# --------------------------------------------------------------------------- +# VALIDATION +# --------------------------------------------------------------------------- + +def run_validation(model, val_loader, device, patch_size): + model.eval() + dice_metric = DiceMetric(include_background=False, reduction="mean") + post = AsDiscrete(threshold=0.5) + + with torch.no_grad(): + for val_data in tqdm(val_loader, desc=" Validation", leave=False): + images = val_data["image"].to(device) + labels = val_data["label"].to(device) + outputs = sliding_window_inference(images, patch_size, 4, model, overlap=0.5) + dice_metric( + y_pred=[post(i) for i in outputs], + y =[post(i) for i in labels], + ) + + score = dice_metric.aggregate().item() + dice_metric.reset() + return score + + +# --------------------------------------------------------------------------- +# TRAINING +# --------------------------------------------------------------------------- + +def train(): + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + print(f"Device : {device}") + print(f"Mode : {'DEBUG' if DEBUG else 'CLUSTER'}") + print(f"Model : {MODEL_SIZE}") + + # ── Loaders ────────────────────────────────────────────────────────────── + train_loader, val_loader = get_coca_loaders(TRAIN_P, VAL_P, PATCH_SIZE, BATCH_SIZE) + + # ── Debug checks (run before model init to fail fast) ──────────────────── + if DEBUG: + debug_loader_sanity(val_loader) + debug_visualise_positive(VAL_P) + + # ── Model ───────────────────────────────────────────────────────────────── + cfg = MODEL_CONFIGS[MODEL_SIZE] + model = UNet( + spatial_dims = 3, + in_channels = 1, + out_channels = 1, + channels = cfg["channels"], + strides = cfg["strides"], + num_res_units = cfg["num_res_units"], + ).to(device) + + n_params = sum(p.numel() for p in model.parameters() if p.requires_grad) + print(f"Parameters: {n_params:,}") + + # ── Loss & optimiser ────────────────────────────────────────────────────── + loss_func = DiceFocalLoss( + sigmoid = True, + gamma = 2.0, + lambda_dice = 1.0, + lambda_focal = 0.5, + alpha = 0.75, + ) + optimizer = torch.optim.Adam(model.parameters(), lr=5e-5) + + # ── Resume from checkpoint ──────────────────────────────────────────────── + best_metric = 0.0 # MUST be outside the epoch loop + start_epoch = 0 + + if os.path.exists(MODEL_PATH): + ckpt = torch.load(MODEL_PATH, map_location=device) + # Support both raw state_dict saves and richer checkpoint dicts + if isinstance(ckpt, dict) and "model_state" in ckpt: + model.load_state_dict(ckpt["model_state"]) + best_metric = ckpt.get("best_metric", 0.0) + start_epoch = ckpt.get("epoch", 0) + 1 + else: + model.load_state_dict(ckpt) + print(f"Loaded weights from {MODEL_PATH} (best Dice so far: {best_metric:.4f})") + else: + print("No checkpoint found — training from scratch") + + # ── Epoch loop ──────────────────────────────────────────────────────────── + for epoch in range(start_epoch, start_epoch + NUM_EPOCHS): + model.train() + epoch_loss = 0.0 + success_count = 0 + hallucination_count = 0 + + step_bar = tqdm(train_loader, desc=f"Epoch {epoch:03d}") + for i, batch in enumerate(step_bar): + inputs = batch["image"].to(device) + labels = batch["label"].to(device) + + # Warn if sampler failed to include any label voxels + if DEBUG and (labels > 0).sum().item() == 0: + print(f" ⚠️ Batch {i}: zero label pixels (sampler fallback to random crop)") + + optimizer.zero_grad() + outputs = model(inputs) + loss = loss_func(outputs, labels) + loss.backward() + optimizer.step() + + conf_map = torch.sigmoid(outputs.detach()) + epoch_loss += loss.item() + step_bar.set_postfix(loss=f"{loss.item():.4f}") + + # Hallucination tracking — check every item in the batch + for b in range(inputs.shape[0]): + max_val = conf_map[b].max().item() + if max_val > 0.5: + z = int(np.unravel_index(conf_map[b].cpu().numpy().argmax(), + conf_map[b].shape)[3]) + hit = labels[b, 0, :, :, z].max().item() > 0 + if hit: success_count += 1 + else: hallucination_count += 1 + + # Audit images + if i % AUDIT_EVERY == 0: + for b in range(inputs.shape[0]): + save_audit_image(inputs, labels, conf_map, i, epoch, item_idx=b) + + # ── Epoch summary ───────────────────────────────────────────────────── + avg_loss = epoch_loss / len(train_loader) + total_dets = success_count + hallucination_count + halluc_rate = hallucination_count / total_dets if total_dets > 0 else 0.0 + + print(f"\nEpoch {epoch:03d} loss={avg_loss:.4f} " + f"detections={total_dets} " + f"✅={success_count} ❌={hallucination_count} " + f"halluc_rate={halluc_rate:.1%}") + + # ── Validation & checkpoint ─────────────────────────────────────────── + val_dice = run_validation(model, val_loader, device, PATCH_SIZE) + print(f" val_dice={val_dice:.4f} best={best_metric:.4f}") + + if val_dice > best_metric: + best_metric = val_dice + torch.save( + { + "epoch": epoch, + "model_state": model.state_dict(), + "best_metric": best_metric, + "patch_size": PATCH_SIZE, + "model_size": MODEL_SIZE, + }, + MODEL_PATH, + ) + print(" 🌟 New best — checkpoint saved") + + +if __name__ == "__main__": + train() diff --git a/coca_project/src/training_lite.py b/coca_project/src/training_lite.py new file mode 100644 index 0000000..a06ea98 --- /dev/null +++ b/coca_project/src/training_lite.py @@ -0,0 +1,306 @@ +import os +os.environ["KMP_DUPLICATE_LIB_OK"] = "TRUE" +import torch +import numpy as np +import matplotlib.pyplot as plt +import pandas as pd +import SimpleITK as sitk +from tqdm import tqdm +from monai.networks.nets import UNet +from monai.losses import DiceFocalLoss +from monai.visualize import matshow3d +from data.big_data_loader import get_coca_loaders +from monai.inferers import sliding_window_inference +from monai.metrics import DiceMetric +from monai.transforms import AsDiscrete + + +def save_audit_image(inputs, labels, conf_map, batch_idx, epoch): + # 1. Find the exact slice (z) where the model is most confident + max_idx = torch.argmax(conf_map[0]).item() + coords = np.unravel_index(max_idx, conf_map[0].shape) + z = coords[3] + + img_slice = inputs[0, 0, :, :, z].cpu().numpy() + gt_slice = labels[0, 0, :, :, z].cpu().numpy() + pred_slice = conf_map[0, 0, :, :, z].detach().cpu().numpy() + + # 2. Check if THIS SPECIFIC SLICE has a label + gt_present_on_slice = "YES" if gt_slice.max() > 0 else "NO" + + fig, axes = plt.subplots(1, 3, figsize=(18, 6)) + + # Panel 1: Pure CT (The "Judge" panel) + axes[0].imshow(img_slice, cmap='gray') + axes[0].set_title(f"CT Slice {z} (Raw)") + + # Panel 2: Ground Truth on this slice + axes[1].imshow(img_slice, cmap='gray') + axes[1].imshow(gt_slice, cmap='Reds', alpha=0.5) + axes[1].set_title(f"Label on this slice: {gt_present_on_slice}") + + # Panel 3: Model Prediction + axes[2].imshow(img_slice, cmap='gray') + axes[2].imshow(pred_slice, cmap='hot', alpha=0.5) + axes[2].set_title(f"Model Confidence: {pred_slice.max():.2f}") + + plt.suptitle(f"Epoch {epoch} | Batch {batch_idx} | Match: {'✅' if gt_present_on_slice == 'YES' else '❌'}", fontsize=16) + + os.makedirs("audit_images", exist_ok=True) + plt.savefig(f"audit_images/epoch{epoch}_batch{batch_idx}.png") + plt.close() + +def check_ground_truth(batch): + # This grabs the first 3D volume in the batch + img = batch["image"][0, 0].cpu().numpy() + mask = batch["label"][0, 0].cpu().numpy() + slice_idx = img.shape[2] // 2 # Grab the middle slice + + plt.imshow(img[:, :, slice_idx], cmap='gray') + plt.imshow(mask[:, :, slice_idx], cmap='Reds', alpha=0.5) + plt.title(f"Visual Check: Slice {slice_idx}") + plt.show() + +class SliceScroller: + def __init__(self, image, mask): + self.image = image.cpu().numpy() if torch.is_tensor(image) else image + self.mask = mask.cpu().numpy() if torch.is_tensor(mask) else mask + self.slices = self.image.shape[0] + self.ind = self.slices // 2 + self.v_min = np.percentile(self.image, 1) + self.v_max = np.percentile(self.image, 99) + + # AUTO-CONTRAST LOGIC + # We calculate the 99th percentile to avoid white-out from outliers + # If the image is still "flat", fall back to raw min/max + if self.v_min == self.v_max: + self.v_min, self.v_max = self.image.min(), self.image.max() + + print(f"👁️ Visualizing with dynamic range: {self.v_min:.4f} to {self.v_max:.4f}") + + self.fig, self.ax = plt.subplots(1, 1, figsize=(8, 8)) + self.im = self.ax.imshow(self.image[self.ind, :, :], + cmap="gray", + vmin=self.v_min, + vmax=self.v_max) + + masked_data = np.ma.masked_where(self.mask[self.ind, :, :] == 0, self.mask[self.ind, :, :]) + self.mask_im = self.ax.imshow(masked_data, cmap="spring", alpha=0.8) + + self.ax.set_title(f"Slice {self.ind} | Scroll to move") + self.fig.canvas.mpl_connect('scroll_event', self.on_scroll) + + def on_scroll(self, event): + self.ind = (self.ind + 1) % self.slices if event.button == 'up' else (self.ind - 1) % self.slices + self.update() + + def on_key(self, event): + if event.key == 'up': self.ind = (self.ind + 1) % self.slices + elif event.key == 'down': self.ind = (self.ind - 1) % self.slices + self.update() + + def update(self): + self.im.set_data(self.image[self.ind, :, :]) + # Force the display to keep the same contrast levels + self.im.set_clim(vmin=self.v_min, vmax=self.v_max) + + masked_data = np.ma.masked_where(self.mask[self.ind, :, :] == 0, self.mask[self.ind, :, :]) + self.mask_im.set_data(masked_data) + self.ax.set_title(f"Slice {self.ind}") + self.fig.canvas.draw() +# 1. ENVIRONMENT TOGGLE +# Set DEBUG = True for your ARM laptop (Weekend) +# Set DEBUG = False for the GPU Cluster (Monday) +DEBUG = True +def run_validation(model, val_loader, device, patch_size): + model.eval() + dice_metric = DiceMetric(include_background=False, reduction="mean") + post_label = AsDiscrete(threshold=0.5) + post_pred = AsDiscrete(threshold=0.5) + + print("🩺 Running Validation...") + with torch.no_grad(): + for val_data in val_loader: + val_images = val_data["image"].to(device) + val_labels = val_data["label"].to(device) + + # Use sliding window to handle full volumes + val_outputs = sliding_window_inference( + val_images, patch_size, 4, model, overlap=0.5 + ) + + # Post-process to binary (0 or 1) + val_outputs = [post_pred(i) for i in val_outputs] + val_labels = [post_label(i) for i in val_labels] + + dice_metric(y_pred=val_outputs, y=val_labels) + + metric = dice_metric.aggregate().item() + dice_metric.reset() + return metric + +def train(): + # --- CONFIGURATION --- + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + + # Paths (Update if paths differ on cluster) + TRAIN_P = r"C:\coca_project\data_canonical\tables\train_split.parquet" + VAL_P = r"C:\coca_project\data_canonical\tables\val_split.parquet" + + # Scale parameters based on hardware + if DEBUG: + print("🛠️ RUNNING IN DEBUG MODE (ARM LAPTOP)") + PATCH_SIZE = (128, 128, 3) + BATCH_SIZE = 1 # Safer for 8GB shared memory + NUM_EPOCHS = 10 + UNET_CHANNELS = (32, 64, 128) # Slim model + else: + print("🚀 RUNNING IN CLUSTER MODE (GPU CLUSTER)") + PATCH_SIZE = (128, 128, 64) # Increased depth + BATCH_SIZE = 4 + NUM_EPOCHS = 100 + UNET_CHANNELS = (32, 64, 128, 256, 512) # Full model + + # 2. LOADERS + train_loader, val_loader = get_coca_loaders(TRAIN_P, VAL_P, PATCH_SIZE, BATCH_SIZE) + + if DEBUG: + print("🔍 Testing loader output for binarization issues...") + # Grab one sample from the loader + test_batch = next(iter(val_loader)) + img_data = test_batch["image"][0][0].numpy() + + print(f"📊 Image Value Range: {img_data.min()} to {img_data.max()}") + print(f"📊 Number of Unique Values: {len(np.unique(img_data))}") + + if len(np.unique(img_data)) < 10: + print("❌ ERROR: Image is still binarized! Check ScaleIntensityRanged.") + else: + print("✅ Success: Image has grayscale gradients.") +# 3. VISUALIZATION (Only triggers in Debug) + if DEBUG: + + print("🔍 Bypassing loader to find the first POSITIVE scan in the Parquet...") + df_val = pd.read_parquet(VAL_P) + pos_cases = df_val[df_val['has_pos'] == 1] + + if pos_cases.empty: + print("❌ No positive cases found in the Validation Parquet!") + else: + # Grab the very first positive file path + sample_row = pos_cases.iloc[6] + print(f"✅ Found Positive Case: {sample_row['scan_id']}") + + # Load directly via SimpleITK to verify the file isn't corrupted/empty + test_img = sitk.GetArrayFromImage(sitk.ReadImage(sample_row['image_path'])) + test_mask = sitk.GetArrayFromImage(sitk.ReadImage(sample_row['mask_path'])) + + print(f"📊 Mask Max Value: {test_mask.max()}") + print(f"📊 Mask Voxel Sum: {test_mask.sum()}") + + if test_mask.sum() == 0: + print("❗ ALERT: Parquet says 'has_pos', but the NIfTI file is empty. Resampling failed.") + else: + # If the files are good, run the scroller + # Note: SimpleITK reads as [Z, H, W], which SliceScroller expects + tracker = SliceScroller(torch.from_numpy(test_img), torch.from_numpy(test_mask)) + tracker.ind = np.argmax(np.sum(test_mask, axis=(1, 2))) + tracker.update() + plt.show() + + + # 4. MODEL, LOSS, OPTIMIZER + model = UNet( + spatial_dims=3, + in_channels=1, + out_channels=1, + channels=UNET_CHANNELS, + strides=(2, 2,1), + num_res_units=2, + ).to(device) + + # Focal Loss handles the extreme sparsity of calcium voxels + loss_func = DiceFocalLoss( + sigmoid=True, + gamma=2.0, # Focal "bone-dimmer" power + lambda_dice=1.0, # High weight to force intersection + lambda_focal=.5, + alpha=.75 + # Optional: squared_pred=True can help "sharpen" the result by punishing + # mid-range probabilities (0.5) more than low (0.1) or high (0.9). +) + optimizer = torch.optim.Adam(model.parameters(), lr=5e-5) + + # 1. Load the existing weights + model_path = "best_coca_model.pth" + if os.path.exists(model_path): + model.load_state_dict(torch.load(model_path, map_location=device)) + print(f"🚀 Loaded existing weights from {model_path}. Continuing training...") + else: + print("🆕 No existing model found. Starting from scratch.") + +# 2. Update your loop range so you don't overwrite previous epoch numbers + for epoch in range(NUM_EPOCHS): + model.train() + epoch_loss = 0 + hallucination_count = 0 + success_count = 0 + step_bar = tqdm(train_loader, desc=f"Epoch {epoch}") + for i, batch in enumerate(step_bar): + inputs, labels = batch["image"].to(device), batch["label"].to(device) + # DEBUG: Check if the patches actually have labels in them + pixels_with_label = (labels > 0).sum().item() + if pixels_with_label == 0: + print(f"⚠️ Sampler Fail: Batch {i} contains ZERO label pixels! (Defaulted to random crop)") + else: + print(f"✅ Sampler Success: Batch {i} has {pixels_with_label} label pixels.") + optimizer.zero_grad() + outputs = model(inputs) + loss = loss_func(outputs, labels) + loss.backward() + optimizer.step() + + # --- TRACKING LOGIC (Every Batch) --- + conf_map = torch.sigmoid(outputs) + for b in range(inputs.shape[0]): # Loop through images in batch + max_val = conf_map[b].max().item() + if max_val > 0.5: # If model thinks it found calcium + max_idx = torch.argmax(conf_map[b]).item() + coords = np.unravel_index(max_idx, conf_map[b].shape) + z_slice = coords[3] # Depth slice + + # Check if ground truth has calcium at that exact spot + if labels[b, 0, :, :, z_slice].max() > 0: + success_count += 1 + else: + hallucination_count += 1 + + # --- VISUAL CHECK (Every 10 Batches) --- + if i % 10 == 0: + save_audit_image(inputs, labels, conf_map, i, epoch) + + epoch_loss += loss.item() + step_bar.set_postfix({"loss": f"{loss.item():.4f}"}) + conf_map = torch.sigmoid(outputs) + + # --- EPOCH SUMMARY --- + print(f"\n--- Epoch {epoch} Report ---") + print(f"Avg Loss: {epoch_loss/len(train_loader):.4f}") + print(f"Total Confident Detections: {success_count + hallucination_count}") + print(f"✅ Successes: {success_count} | ❌ Hallucinations: {hallucination_count}") + avg_loss = epoch_loss / len(train_loader) + print(f"Epoch {epoch} Complete. Avg Loss: {avg_loss:.4f}") + + +# --- VALIDATION STEP --- + val_dice = run_validation(model, val_loader, device, PATCH_SIZE) + print(f"Epoch {epoch} Metrics: | Dice: {val_dice:.4f}") + best_metric=0 + # Save if this is the best version yet + if val_dice > best_metric: + best_metric = val_dice + torch.save(model.state_dict(), "best_coca_model.pth") + print("🌟 New Best Model Saved!") + +if __name__ == "__main__": + train() \ No newline at end of file diff --git a/coca_project/src/xml_check.py b/coca_project/src/xml_check.py new file mode 100644 index 0000000..e7d4420 --- /dev/null +++ b/coca_project/src/xml_check.py @@ -0,0 +1,74 @@ +import plistlib +from pathlib import Path +import SimpleITK as sitk + +def debug_plist_alignment(xml_path, dicom_dir): + xml_path = Path(xml_path) + dicom_dir = Path(dicom_dir) + + # 1. Get DICOM slice count + reader = sitk.ImageSeriesReader() + dicom_names = reader.GetGDCMSeriesFileNames(str(dicom_dir)) + total_slices = len(dicom_names) + + if total_slices == 0: + print(f"Error: No DICOMs found in {dicom_dir}") + return + + # 2. Extract ImageIndices from Plist + if not xml_path.exists(): + print(f"Error: XML not found at {xml_path}") + return + + try: + with open(xml_path, 'rb') as f: + data = plistlib.load(f) + + images = data.get('Images', []) + xml_indices = [img.get('ImageIndex') for img in images if img.get('ImageIndex') is not None] + + # Filter for only slices that actually have ROIs with points + active_indices = [] + for img in images: + idx = img.get('ImageIndex') + rois = img.get('ROIs', []) + has_points = any(len(roi.get('Point_px', [])) > 0 for roi in rois) + if has_points: + active_indices.append(idx) + + except Exception as e: + print(f"Error parsing Plist: {e}") + return + + if not active_indices: + print(f"Result: XML found, but no calcium points were detected in {xml_path.name}") + return + + min_xml = min(active_indices) + max_xml = max(active_indices) + + print(f"--- Fast Plist Debug: {xml_path.name} ---") + print(f"Total DICOM Slices: {total_slices}") + print(f"XML Calcium Range: Slice {min_xml} to Slice {max_xml}") + + # 3. Check Alignment Logic + print(f"\n[Scenario A] reverse_z = False (Standard)") + print(f" XML Slice {min_xml} -> DICOM Index {min_xml}") + print(f" XML Slice {max_xml} -> DICOM Index {max_xml}") + if max_xml >= total_slices: + print(" !!! WARNING: Indices exceed slice count. This will return 0 pixels.") + + print(f"\n[Scenario B] reverse_z = True (Flipped)") + print(f" XML Slice {min_xml} -> DICOM Index {(total_slices - 1) - min_xml}") + print(f" XML Slice {max_xml} -> DICOM Index {(total_slices - 1) - max_xml}") + if ((total_slices - 1) - max_xml) < 0: + print(" !!! WARNING: Flipped indices are negative. This will return 0 pixels.") + +# --- RUN TEST --- +PROJECT_ROOT = Path(r"C:\coca_project") +# Update these two paths to one specific patient you want to test +TEST_PATIENT = "10" +DICOM_FOLDER = PROJECT_ROOT / "data_raw" / "dicom" / "Gated_release_final" / "Gated_release_final" / "patient" / TEST_PATIENT +XML_FILE = PROJECT_ROOT / "data_raw" / "xml" / "calcium_xml"/f"{TEST_PATIENT}.xml" + +debug_plist_alignment(XML_FILE, DICOM_FOLDER) \ No newline at end of file diff --git a/coca_project/src/xml_format.py b/coca_project/src/xml_format.py new file mode 100644 index 0000000..bec308e --- /dev/null +++ b/coca_project/src/xml_format.py @@ -0,0 +1,18 @@ +from pathlib import Path +import xml.etree.ElementTree as ET + +xml_path = Path(r"C:\coca_project\data_raw\xml\calcium_xml\9.xml") + +tree = ET.parse(xml_path) +root = tree.getroot() + +def walk(el, depth=0): + tag = el.tag.split("}")[-1] + indent = " " * depth + print(f"{indent}{tag}") + for k, v in el.attrib.items(): + print(f"{indent} @{k} = {v}") + for child in el: + walk(child, depth + 1) + +walk(root) \ No newline at end of file diff --git a/predict2/Readme.md b/predict2/Readme.md new file mode 100644 index 0000000..9684ddf --- /dev/null +++ b/predict2/Readme.md @@ -0,0 +1,102 @@ +# PREDICT2: Radiomics Feature Extraction and Calcium Phenotype Discovery + +## Overview +This project develops a feature extraction and phenotyping framework for coronary +artery calcium (CAC) using the Stanford COCA dataset. We extract radiomic features +from calcium segmentation masks, perform statistical analysis, and discover calcium +phenotypes via unsupervised clustering. + +## Dataset +- **Source**: Stanford COCA (Coronary Calcium and Chest CT) +- **Scans used**: 24 gated coronary CT scans with calcium segmentation masks +- **Format**: DICOM → converted to NIfTI (.nii.gz) via COCA pipeline + +## Pipeline + +### 1. Preprocessing +- DICOM series loaded using SimpleITK +- Calcium segmentation masks parsed from XML annotation files +- Agatston scores calculated per patient + +### 2. Feature Extraction (PyRadiomics) +Extracted **107 radiomic features** per patient across 6 categories: + +| Category | Features | +|----------|---------| +| Shape | 14 | +| First Order | 18 | +| GLCM (texture) | 24 | +| GLSZM | 16 | +| GLRLM | 16 | +| NGTDM | 5 | + +### 3. Statistical Analysis + +#### Spearman Correlation +- **81 out of 107 features** significantly correlated with Agatston score (p<0.05) +- Top feature: `glcm_JointEnergy` (r = -0.9885, p < 0.001) +- Strong texture features dominate correlation with calcium burden + +#### Kruskal-Wallis Test +- **76 out of 107 features** significantly different across Agatston categories (p<0.05) +- Shape and intensity features most discriminative between Minimal/Mild groups + +### 4. Calcium Phenotype Discovery (Clustering) +- Applied K-Means clustering (K=2 to 5) on PCA-reduced features +- **Best K=2** (Silhouette Score = 0.439) +- Cluster 0 (n=21): Minimal calcium burden +- Cluster 1 (n=3): Mild calcium burden +- Clusters perfectly align with Agatston categories + +### 5. Feature Importance (Random Forest + SHAP) +Top discriminating features: +1. `gldm_LargeDependenceEmphasis` +2. `gldm_LargeDependenceHighGrayLevelEmphasis` +3. `ngtdm_Complexity` +4. `firstorder_Energy` +5. `glszm_SizeZoneNonUniformity` + +## Key Results +- 107 radiomic features extracted from 24 patients +- 81/107 features show significant Spearman correlation with Agatston score +- 76/107 features pass Kruskal-Wallis significance test +- K=2 clustering perfectly separates calcium burden categories +- GLCM texture features most strongly associated with calcium severity + +## Visualizations +| File | Description | +|------|-------------| +| `pca_clustering.png` | PCA plot colored by Agatston category and cluster | +| `spearman_plot.png` | Top 15 features by Spearman correlation | +| `agatston_distribution.png` | Score distribution and category counts | +| `correlation_heatmap.png` | Feature correlation heatmap | +| `feature_importance.png` | Random Forest feature importance | + +## Output Files +| File | Description | +|------|-------------| +| `features.csv` | 107 radiomic features for 24 patients | +| `agatston_correct.csv` | Agatston scores per patient | +| `final_merged.csv` | Features merged with Agatston scores | +| `spearman_results.csv` | Full Spearman correlation results | +| `kruskal_results.csv` | Full Kruskal-Wallis test results | +| `top_features.csv` | Top 20 features by RF importance | + +## Requirements +``` +pip install pyradiomics SimpleITK pydicom scikit-learn +pip install matplotlib seaborn pandas numpy scipy +``` + +## How to Run +```bash +# 1. Preprocess DICOM data +python COCA_scripts/COCA_scripts/COCA_pipeline.py + +# 2. Run full analysis +python predict2/complete_analysis.py +``` + +## Author +Prince Bhadania +GSoC 2026 Applicant — ML4SCI / PREDICT Project \ No newline at end of file diff --git a/predict2/agatston_correct.csv b/predict2/agatston_correct.csv new file mode 100644 index 0000000..6eb6949 --- /dev/null +++ b/predict2/agatston_correct.csv @@ -0,0 +1,452 @@ +patient_id,agatston_score,calcium_volume,lesion_count,agatston_category +0,0.06,0.03,1,Minimal +1,3.7,1.1,16,Minimal +10,1.13,0.48,18,Minimal +100,2.15,0.87,28,Minimal +101,0.01,0.01,1,Minimal +102,0.51,0.2,6,Minimal +103,0.04,0.02,1,Minimal +104,0.63,0.19,4,Minimal +105,0.1,0.03,1,Minimal +106,0.04,0.02,2,Minimal +107,3.19,1.01,22,Minimal +108,0.71,0.3,10,Minimal +109,0.47,0.13,2,Minimal +11,1.49,0.47,12,Minimal +110,0.37,0.18,3,Minimal +111,0.02,0.02,1,Minimal +112,9.49,2.69,19,Minimal +113,0.54,0.2,2,Minimal +114,0.66,0.21,6,Minimal +115,1.61,0.48,8,Minimal +116,7.51,2.34,40,Minimal +117,6.81,2.15,27,Minimal +118,4.4,1.4,30,Minimal +119,13.67,4.03,52,Mild +12,0.33,0.12,2,Minimal +120,0.59,0.21,6,Minimal +121,0.11,0.04,1,Minimal +122,0.24,0.08,2,Minimal +123,9.25,2.57,27,Minimal +124,0.37,0.14,6,Minimal +125,9.21,2.57,30,Minimal +126,13.1,3.79,54,Mild +127,0.21,0.08,3,Minimal +128,1.87,0.67,16,Minimal +129,4.74,1.33,8,Minimal +13,16.7,4.62,44,Mild +130,7.68,2.16,37,Minimal +131,1.16,0.46,12,Minimal +132,28.24,7.79,57,Mild +133,0.06,0.04,2,Minimal +134,1.5,0.48,9,Minimal +135,0.02,0.02,3,Minimal +136,0.2,0.06,2,Minimal +137,0.14,0.09,3,Minimal +138,0.12,0.08,5,Minimal +139,8.04,2.3,23,Minimal +14,8.07,2.31,30,Minimal +140,0.01,0.01,1,Minimal +141,5.34,1.84,40,Minimal +142,2.37,0.81,9,Minimal +143,2.33,0.69,8,Minimal +144,2.24,0.71,13,Minimal +145,7.39,2.28,38,Minimal +146,1.39,0.48,9,Minimal +147,16.0,4.33,42,Mild +148,0.27,0.07,1,Minimal +149,0.02,0.02,2,Minimal +15,0.61,0.2,4,Minimal +150,1.93,0.69,18,Minimal +151,0.09,0.06,3,Minimal +152,0.04,0.02,1,Minimal +153,0.01,0.01,1,Minimal +154,1.99,0.78,16,Minimal +155,0.55,0.21,9,Minimal +156,0.08,0.04,1,Minimal +157,16.24,4.97,41,Mild +158,0.31,0.14,7,Minimal +159,2.18,0.72,13,Minimal +16,0.26,0.09,2,Minimal +160,4.65,1.31,20,Minimal +161,2.62,0.75,13,Minimal +162,0.61,0.17,4,Minimal +163,0.02,0.02,1,Minimal +164,0.16,0.08,2,Minimal +165,4.75,1.34,15,Minimal +166,0.54,0.17,3,Minimal +167,0.94,0.33,10,Minimal +168,1.15,0.41,11,Minimal +169,0.03,0.02,1,Minimal +17,0.02,0.02,2,Minimal +170,0.18,0.07,3,Minimal +171,1.0,0.25,2,Minimal +172,7.53,2.13,20,Minimal +173,39.38,10.59,38,Mild +174,0.09,0.04,2,Minimal +175,0.84,0.24,5,Minimal +176,1.3,0.54,11,Minimal +177,1.32,0.46,11,Minimal +178,6.56,1.94,30,Minimal +179,0.01,0.01,1,Minimal +18,0.63,0.21,3,Minimal +180,0.05,0.03,2,Minimal +181,4.06,1.13,12,Minimal +182,2.19,0.58,5,Minimal +183,1.06,0.52,16,Minimal +184,2.85,0.92,21,Minimal +185,1.47,0.49,10,Minimal +186,1.19,0.46,16,Minimal +187,0.16,0.06,2,Minimal +188,0.01,0.01,1,Minimal +189,0.04,0.02,2,Minimal +19,0.53,0.13,2,Minimal +190,17.93,4.81,32,Mild +191,0.06,0.03,1,Minimal +192,0.27,0.08,2,Minimal +193,0.7,0.17,2,Minimal +194,45.71,12.06,69,Mild +195,0.12,0.06,1,Minimal +196,24.23,6.51,49,Mild +197,0.12,0.06,3,Minimal +198,0.09,0.03,1,Minimal +199,1.55,0.51,13,Minimal +2,1.14,0.41,13,Minimal +20,1.46,0.41,6,Minimal +200,0.12,0.05,2,Minimal +201,0.8,0.29,8,Minimal +202,0.01,0.01,1,Minimal +203,3.79,1.03,10,Minimal +204,0.29,0.07,1,Minimal +205,0.71,0.21,4,Minimal +206,1.02,0.25,2,Minimal +207,0.15,0.05,1,Minimal +208,0.01,0.01,1,Minimal +209,11.52,3.25,42,Mild +21,0.19,0.05,1,Minimal +210,3.08,0.96,22,Minimal +211,27.13,7.55,58,Mild +212,0.63,0.18,2,Minimal +213,1.0,0.33,9,Minimal +214,0.03,0.02,1,Minimal +215,0.89,0.31,11,Minimal +216,3.69,1.24,25,Minimal +217,0.1,0.05,2,Minimal +218,28.24,7.85,41,Mild +219,7.06,2.03,36,Minimal +22,30.1,8.26,24,Mild +220,0.7,0.21,6,Minimal +221,0.04,0.03,2,Minimal +222,2.56,0.9,24,Minimal +223,14.89,4.09,46,Mild +224,0.1,0.08,4,Minimal +225,14.6,4.48,46,Mild +226,0.94,0.25,2,Minimal +227,0.44,0.15,5,Minimal +228,0.27,0.07,1,Minimal +229,0.6,0.25,7,Minimal +23,0.06,0.04,3,Minimal +230,1.14,0.38,12,Minimal +231,13.28,3.68,32,Mild +232,0.15,0.09,3,Minimal +233,0.01,0.01,1,Minimal +234,0.58,0.21,6,Minimal +235,2.69,0.83,6,Minimal +236,0.05,0.02,1,Minimal +237,0.25,0.11,3,Minimal +238,29.11,7.69,58,Mild +239,5.68,1.71,18,Minimal +24,0.12,0.06,1,Minimal +240,0.25,0.14,5,Minimal +241,0.17,0.1,5,Minimal +242,6.96,1.96,22,Minimal +243,5.13,1.42,17,Minimal +244,0.3,0.12,3,Minimal +245,6.09,1.68,24,Minimal +246,6.75,1.87,28,Minimal +247,0.32,0.11,2,Minimal +248,0.64,0.18,3,Minimal +249,10.09,3.15,38,Mild +25,3.86,1.38,27,Minimal +250,3.04,0.99,17,Minimal +251,0.1,0.04,2,Minimal +252,0.49,0.27,10,Minimal +253,1.69,0.48,8,Minimal +254,12.97,3.94,44,Mild +255,1.47,0.52,13,Minimal +256,1.03,0.33,7,Minimal +257,2.79,0.83,17,Minimal +258,3.79,0.99,13,Minimal +259,11.08,3.0,32,Mild +26,2.09,0.74,17,Minimal +260,0.83,0.3,8,Minimal +261,0.04,0.02,1,Minimal +262,0.49,0.21,8,Minimal +263,7.17,1.98,16,Minimal +264,0.05,0.03,2,Minimal +265,0.12,0.07,3,Minimal +266,0.78,0.31,9,Minimal +267,0.07,0.03,1,Minimal +268,12.99,3.32,14,Mild +269,6.96,1.94,26,Minimal +27,2.44,0.73,8,Minimal +270,3.89,1.03,14,Minimal +271,0.42,0.16,5,Minimal +272,2.4,0.8,22,Minimal +273,0.04,0.02,1,Minimal +274,4.08,1.28,23,Minimal +275,8.79,2.51,34,Minimal +276,0.96,0.27,3,Minimal +277,11.97,4.08,31,Mild +278,0.22,0.11,3,Minimal +279,0.36,0.12,2,Minimal +28,11.88,3.36,27,Mild +280,0.45,0.15,6,Minimal +281,6.45,2.34,50,Minimal +282,1.24,0.36,7,Minimal +283,4.19,1.53,35,Minimal +284,0.11,0.06,3,Minimal +285,0.2,0.09,3,Minimal +286,1.17,0.39,8,Minimal +287,0.31,0.1,2,Minimal +288,2.27,0.65,11,Minimal +289,5.55,1.91,33,Minimal +29,0.11,0.05,1,Minimal +290,0.18,0.08,4,Minimal +291,0.56,0.16,2,Minimal +292,5.18,1.55,27,Minimal +293,0.66,0.21,4,Minimal +294,1.39,0.38,5,Minimal +295,0.35,0.1,2,Minimal +296,1.61,0.5,9,Minimal +297,0.01,0.01,1,Minimal +298,27.02,7.09,38,Mild +299,8.82,2.34,25,Minimal +3,4.71,1.29,10,Minimal +30,8.9,2.46,23,Minimal +300,2.74,0.93,16,Minimal +301,12.15,3.27,28,Mild +302,0.01,0.01,1,Minimal +303,14.63,4.17,44,Mild +304,2.79,0.81,15,Minimal +305,5.92,1.85,22,Minimal +306,18.73,5.82,70,Mild +307,0.01,0.01,1,Minimal +308,0.26,0.09,4,Minimal +309,0.87,0.27,8,Minimal +31,0.08,0.03,1,Minimal +310,14.22,3.65,22,Mild +311,0.31,0.08,1,Minimal +312,13.89,4.06,51,Mild +313,0.29,0.1,3,Minimal +314,0.02,0.02,2,Minimal +315,0.91,0.28,4,Minimal +316,0.15,0.1,4,Minimal +317,1.26,0.41,9,Minimal +318,6.9,1.87,28,Minimal +319,0.97,0.32,10,Minimal +32,0.03,0.03,3,Minimal +320,1.34,0.34,4,Minimal +321,57.7,15.35,67,Mild +322,2.98,0.85,15,Minimal +323,3.14,0.98,20,Minimal +324,18.79,5.46,70,Mild +325,0.37,0.18,6,Minimal +326,28.68,7.6,42,Mild +327,0.09,0.04,1,Minimal +328,0.8,0.3,10,Minimal +329,0.29,0.08,2,Minimal +33,0.45,0.17,8,Minimal +330,1.61,0.41,4,Minimal +331,13.56,3.86,23,Mild +332,37.16,9.99,60,Mild +333,0.34,0.15,6,Minimal +334,0.08,0.04,1,Minimal +335,1.56,0.5,16,Minimal +336,10.12,3.3,46,Mild +337,3.6,1.17,18,Minimal +338,6.32,1.75,15,Minimal +339,1.85,0.52,6,Minimal +34,1.02,0.34,6,Minimal +340,8.92,2.88,32,Minimal +341,2.72,0.87,15,Minimal +342,53.81,13.66,62,Mild +343,0.93,0.25,3,Minimal +344,0.67,0.17,2,Minimal +345,0.09,0.04,1,Minimal +346,0.43,0.14,4,Minimal +347,1.46,0.54,16,Minimal +348,0.54,0.22,4,Minimal +349,31.88,8.5,47,Mild +35,0.47,0.15,4,Minimal +350,4.25,1.14,9,Minimal +351,0.79,0.24,5,Minimal +352,3.17,0.86,10,Minimal +353,1.66,0.63,11,Minimal +354,4.95,1.42,13,Minimal +355,0.01,0.01,1,Minimal +356,3.3,0.9,13,Minimal +357,0.01,0.01,1,Minimal +358,0.56,0.23,5,Minimal +359,0.51,0.15,3,Minimal +36,3.71,1.03,12,Minimal +360,12.38,3.98,50,Mild +361,15.55,4.1,30,Mild +362,2.83,0.84,15,Minimal +363,8.23,2.37,26,Minimal +364,2.53,0.69,8,Minimal +365,0.34,0.14,4,Minimal +366,0.54,0.15,4,Minimal +367,0.51,0.23,7,Minimal +368,0.08,0.05,2,Minimal +369,0.34,0.08,1,Minimal +37,11.14,3.14,37,Mild +370,2.3,0.64,11,Minimal +371,25.0,7.15,30,Mild +372,1.33,0.47,12,Minimal +373,0.16,0.04,1,Minimal +374,2.13,0.88,26,Minimal +375,1.88,0.56,10,Minimal +376,0.59,0.15,1,Minimal +377,0.59,0.33,5,Minimal +378,0.04,0.02,1,Minimal +379,0.49,0.19,4,Minimal +38,0.84,0.26,8,Minimal +380,1.4,0.66,10,Minimal +381,0.3,0.15,2,Minimal +382,8.91,2.46,24,Minimal +383,0.02,0.02,2,Minimal +384,0.01,0.01,1,Minimal +385,0.07,0.05,5,Minimal +386,32.34,8.71,70,Mild +387,1.83,0.75,22,Minimal +388,1.73,0.75,22,Minimal +389,3.38,1.37,26,Minimal +39,6.82,2.04,33,Minimal +390,1.26,0.45,14,Minimal +391,0.67,0.28,6,Minimal +392,10.8,3.33,44,Mild +393,11.61,3.03,21,Mild +394,12.53,3.25,24,Mild +395,8.69,2.47,29,Minimal +396,0.04,0.02,1,Minimal +397,4.27,1.19,28,Minimal +398,3.11,1.07,22,Minimal +399,7.62,2.12,26,Minimal +4,0.27,0.15,10,Minimal +40,18.65,4.91,29,Mild +400,2.15,0.67,12,Minimal +401,17.64,4.75,42,Mild +402,4.36,1.1,5,Minimal +403,0.16,0.08,2,Minimal +404,0.4,0.19,8,Minimal +405,11.53,3.25,39,Mild +406,35.01,9.14,49,Mild +407,2.28,0.8,19,Minimal +408,4.33,1.44,21,Minimal +409,0.48,0.19,6,Minimal +41,0.71,0.28,11,Minimal +410,6.75,2.2,40,Minimal +411,0.0,0.0,1,None +412,8.29,2.16,21,Minimal +413,10.26,3.06,37,Mild +414,0.51,0.21,4,Minimal +415,4.16,1.29,22,Minimal +416,1.18,0.33,2,Minimal +417,0.85,0.28,4,Minimal +418,1.37,0.42,8,Minimal +419,5.87,1.69,20,Minimal +42,1.16,0.39,13,Minimal +420,0.94,0.31,7,Minimal +421,0.12,0.07,2,Minimal +422,0.8,0.28,6,Minimal +423,0.15,0.07,2,Minimal +424,1.2,0.49,15,Minimal +425,0.21,0.11,4,Minimal +426,0.48,0.14,2,Minimal +427,0.37,0.13,2,Minimal +428,5.34,1.54,19,Minimal +429,0.01,0.01,1,Minimal +43,0.25,0.07,2,Minimal +430,7.29,1.99,27,Minimal +431,6.28,1.83,21,Minimal +432,0.38,0.11,2,Minimal +433,40.42,10.32,56,Mild +434,7.16,1.99,24,Minimal +435,0.43,0.13,3,Minimal +436,0.62,0.16,2,Minimal +437,8.49,2.56,39,Minimal +438,0.1,0.08,6,Minimal +439,0.77,0.26,7,Minimal +44,0.36,0.12,1,Minimal +440,3.15,0.88,11,Minimal +441,2.72,0.96,17,Minimal +442,0.06,0.03,1,Minimal +443,3.41,0.97,14,Minimal +444,0.62,0.23,4,Minimal +445,1.3,0.38,6,Minimal +446,4.38,1.26,15,Minimal +447,0.15,0.04,2,Minimal +448,7.06,1.91,19,Minimal +449,2.69,0.79,14,Minimal +45,1.82,0.55,9,Minimal +450,0.16,0.06,2,Minimal +46,4.31,1.37,16,Minimal +47,7.64,2.07,25,Minimal +48,3.41,1.03,17,Minimal +49,0.73,0.19,2,Minimal +5,22.99,6.03,41,Mild +50,4.22,1.16,10,Minimal +51,0.54,0.16,4,Minimal +52,0.5,0.15,3,Minimal +53,0.06,0.03,2,Minimal +54,3.65,1.02,12,Minimal +55,0.05,0.03,2,Minimal +56,2.58,0.72,7,Minimal +57,6.38,1.9,24,Minimal +58,0.39,0.2,11,Minimal +59,2.26,0.7,11,Minimal +6,3.41,1.0,18,Minimal +60,1.65,0.41,3,Minimal +61,0.02,0.02,1,Minimal +62,0.48,0.19,5,Minimal +63,0.17,0.06,3,Minimal +64,0.09,0.03,1,Minimal +65,0.04,0.02,1,Minimal +66,5.54,1.44,13,Minimal +67,2.1,0.57,9,Minimal +68,0.19,0.05,1,Minimal +69,3.61,1.27,23,Minimal +7,0.52,0.19,4,Minimal +70,0.05,0.02,1,Minimal +71,21.03,5.69,54,Mild +72,0.11,0.05,2,Minimal +73,0.23,0.1,3,Minimal +74,0.57,0.17,4,Minimal +75,0.06,0.03,1,Minimal +76,0.52,0.17,7,Minimal +77,16.53,4.21,26,Mild +78,0.33,0.16,6,Minimal +79,0.15,0.07,3,Minimal +8,0.7,0.2,2,Minimal +80,0.81,0.31,12,Minimal +81,28.34,7.65,51,Mild +82,3.08,0.88,14,Minimal +83,1.72,0.49,3,Minimal +84,0.06,0.03,1,Minimal +85,5.03,1.38,13,Minimal +86,0.11,0.04,1,Minimal +87,0.04,0.02,1,Minimal +88,3.17,0.94,18,Minimal +89,0.85,0.25,5,Minimal +9,0.72,0.23,4,Minimal +90,11.86,3.17,27,Mild +91,2.46,0.84,16,Minimal +92,1.16,0.61,18,Minimal +93,0.14,0.05,2,Minimal +94,3.25,1.18,28,Minimal +95,0.73,0.21,3,Minimal +96,0.18,0.07,3,Minimal +97,0.13,0.07,2,Minimal +98,20.93,5.5,58,Mild +99,4.87,1.34,19,Minimal diff --git a/predict2/agatston_distribution.png b/predict2/agatston_distribution.png new file mode 100644 index 0000000..5aeacd1 Binary files /dev/null and b/predict2/agatston_distribution.png differ diff --git a/predict2/complete_analysis.py b/predict2/complete_analysis.py new file mode 100644 index 0000000..507c5cc --- /dev/null +++ b/predict2/complete_analysis.py @@ -0,0 +1,401 @@ +import os +import plistlib +import pandas as pd +import numpy as np +from scipy import stats +from sklearn.preprocessing import StandardScaler +from sklearn.decomposition import PCA +from sklearn.cluster import KMeans, DBSCAN +from sklearn.ensemble import RandomForestRegressor +from sklearn.metrics import silhouette_score +import matplotlib.pyplot as plt +import matplotlib +matplotlib.use('Agg') + +# ============================================================ +# STEP 1 — CORRECT AGATSTON SCORE PARSING +# ============================================================ +print("="*60) +print("STEP 1: Parsing Agatston Scores") +print("="*60) + +XML_PATH = r"D:\COCA_data\calcium_xml\calcium_xml" + +def parse_agatston(xml_file): + """ + Real Agatston score calculation: + Score = Area(mm2) x Density_factor + Density factors: HU 130-199=1, 200-299=2, 300-399=3, 400+=4 + """ + try: + with open(xml_file, 'rb') as f: + plist = plistlib.load(f) + + total_agatston = 0 + total_volume = 0 + lesion_count = 0 + + for img in plist.get("Images", []): + for roi in img.get("ROIs", []): + area = roi.get("Area", 0) + max_hu = roi.get("Max", 0) + mean_hu = roi.get("Mean", max_hu) + + if area <= 0: + continue + + # Density factor based on peak HU + if max_hu >= 400: + density_factor = 4 + elif max_hu >= 300: + density_factor = 3 + elif max_hu >= 200: + density_factor = 2 + elif max_hu >= 130: + density_factor = 1 + else: + continue + + total_agatston += area * density_factor + total_volume += area + lesion_count += 1 + + return total_agatston, total_volume, lesion_count + + except Exception as e: + print(f" Error: {e}") + return 0, 0, 0 + +data = [] +for file in sorted(os.listdir(XML_PATH)): + if not file.endswith(".xml") or file == "000.DS_Store": + continue + patient_id = file.replace(".xml", "") + xml_file = os.path.join(XML_PATH, file) + agatston, volume, lesions = parse_agatston(xml_file) + data.append({ + "patient_id": patient_id, + "agatston_score": round(agatston, 2), + "calcium_volume": round(volume, 2), + "lesion_count": lesions + }) + +agatston_df = pd.DataFrame(data) + +# Agatston categories +def categorize(score): + if score == 0: return "None" + elif score <= 10: return "Minimal" + elif score <= 100: return "Mild" + elif score <= 400: return "Moderate" + else: return "Severe" + +agatston_df['agatston_category'] = agatston_df['agatston_score'].apply(categorize) +agatston_df.to_csv(r"D:\COCA_data\agatston_correct.csv", index=False) + +print(f"Parsed {len(agatston_df)} patients") +print(f"Score range: {agatston_df['agatston_score'].min():.1f} - {agatston_df['agatston_score'].max():.1f}") +print(f"\nCategory distribution:") +print(agatston_df['agatston_category'].value_counts()) +print(agatston_df[['patient_id','agatston_score','agatston_category']].head(10)) + +# ============================================================ +# STEP 2 — MERGE WITH FEATURES +# ============================================================ +print("\n" + "="*60) +print("STEP 2: Merging Features with Agatston Scores") +print("="*60) + +features_df = pd.read_csv(r"D:\COCA_data\features.csv") + +# Get patient_id from scan_index +scan_index = pd.read_csv(r"D:\COCA_data\processed\tables\scan_index.csv") +print("Scan index columns:", scan_index.columns.tolist()) + +# Merge features with scan_index to get patient_id +merged = features_df.merge(scan_index[['scan_id','patient_id']], on='scan_id', how='left') + +# Merge with agatston scores +merged['patient_id'] = merged['patient_id'].astype(str) +agatston_df['patient_id'] = agatston_df['patient_id'].astype(str) +final_df = merged.merge(agatston_df, on='patient_id', how='left') + +print(f"Final dataset: {final_df.shape}") +print(f"Patients with Agatston scores: {final_df['agatston_score'].notna().sum()}") +print(final_df[['patient_id','agatston_score','agatston_category']].head(10)) + +final_df.to_csv(r"D:\COCA_data\final_merged.csv", index=False) + +# ============================================================ +# STEP 3 — FEATURE EXTRACTION STATS +# ============================================================ +print("\n" + "="*60) +print("STEP 3: Dataset Statistics") +print("="*60) + +feature_cols = [c for c in features_df.columns if c != 'scan_id'] +print(f"Total features extracted: {len(feature_cols)}") +print(f"Total patients: {len(final_df)}") +print(f"\nFeature categories:") +for prefix in ['shape', 'firstorder', 'glcm', 'glszm', 'glrlm', 'ngtdm']: + count = len([c for c in feature_cols if prefix in c]) + print(f" {prefix}: {count} features") + +# ============================================================ +# STEP 4 — SPEARMAN CORRELATION +# ============================================================ +print("\n" + "="*60) +print("STEP 4: Spearman Correlation with Agatston Score") +print("="*60) + +valid_df = final_df.dropna(subset=['agatston_score']) +feature_cols_clean = [c for c in feature_cols if c in valid_df.columns] + +correlations = [] +for feat in feature_cols_clean: + try: + vals = valid_df[feat].fillna(0) + scores = valid_df['agatston_score'] + corr, pval = stats.spearmanr(vals, scores) + correlations.append({ + 'feature': feat, + 'spearman_r': round(corr, 4), + 'p_value': round(pval, 4), + 'abs_corr': abs(corr), + 'significant': pval < 0.05 + }) + except: + pass + +corr_df = pd.DataFrame(correlations).sort_values('abs_corr', ascending=False) +corr_df.to_csv(r"D:\COCA_data\spearman_results.csv", index=False) + +print(f"Significant features (p<0.05): {corr_df['significant'].sum()}") +print("\nTop 15 features by Spearman correlation:") +print(corr_df.head(15)[['feature','spearman_r','p_value','significant']].to_string()) + +# ============================================================ +# STEP 5 — KRUSKAL-WALLIS TEST +# ============================================================ +print("\n" + "="*60) +print("STEP 5: Kruskal-Wallis Test Across Agatston Categories") +print("="*60) + +kruskal_results = [] +categories = valid_df['agatston_category'].unique() +print(f"Categories present: {categories}") + +for feat in feature_cols_clean: + try: + groups = [valid_df[valid_df['agatston_category']==cat][feat].dropna().values + for cat in categories] + groups = [g for g in groups if len(g) > 0] + if len(groups) >= 2: + stat, pval = stats.kruskal(*groups) + kruskal_results.append({ + 'feature': feat, + 'kruskal_stat': round(stat, 4), + 'p_value': round(pval, 4), + 'significant': pval < 0.05 + }) + except: + pass + +kruskal_df = pd.DataFrame(kruskal_results).sort_values('p_value') +kruskal_df.to_csv(r"D:\COCA_data\kruskal_results.csv", index=False) + +print(f"Significant features (p<0.05): {kruskal_df['significant'].sum()}") +print("\nTop 10 significant features:") +print(kruskal_df.head(10)[['feature','kruskal_stat','p_value']].to_string()) + +# ============================================================ +# STEP 6 — CLUSTERING +# ============================================================ +print("\n" + "="*60) +print("STEP 6: Clustering") +print("="*60) + +X = valid_df[feature_cols_clean].fillna(0) +scaler = StandardScaler() +X_scaled = scaler.fit_transform(X) + +# PCA for dimensionality reduction +pca = PCA(n_components=min(10, len(feature_cols_clean))) +X_pca = pca.fit_transform(X_scaled) +print(f"PCA variance explained (10 components): {pca.explained_variance_ratio_.sum():.3f}") + +# Find best K +print("\nFinding best number of clusters:") +best_k, best_score = 2, -1 +for k in range(2, min(6, len(valid_df))): + km = KMeans(n_clusters=k, random_state=42, n_init=10) + labels = km.fit_predict(X_pca) + if len(set(labels)) > 1: + score = silhouette_score(X_pca, labels) + print(f" K={k}: Silhouette={score:.3f}") + if score > best_score: + best_score = score + best_k = k + +print(f"\nBest K={best_k} (Silhouette={best_score:.3f})") +kmeans = KMeans(n_clusters=best_k, random_state=42, n_init=10) +valid_df = valid_df.copy() +valid_df['cluster'] = kmeans.fit_predict(X_pca) + +print("\nCluster distribution:") +print(valid_df['cluster'].value_counts()) +print("\nCluster vs Agatston category:") +print(pd.crosstab(valid_df['cluster'], valid_df['agatston_category'])) + +# ============================================================ +# STEP 7 — VISUALIZATIONS +# ============================================================ +print("\n" + "="*60) +print("STEP 7: Creating Visualizations") +print("="*60) + +# Color maps +cat_colors = {'None':'green','Minimal':'blue','Mild':'orange', + 'Moderate':'red','Severe':'darkred','Unknown':'gray'} + +# --- Plot 1: PCA colored by Agatston category --- +pca2 = PCA(n_components=2) +X_pca2 = pca2.fit_transform(X_scaled) + +fig, axes = plt.subplots(1, 2, figsize=(16, 6)) + +cats = valid_df['agatston_category'].fillna('Unknown') +for cat, color in cat_colors.items(): + mask = cats == cat + if mask.any(): + axes[0].scatter(X_pca2[mask, 0], X_pca2[mask, 1], + c=color, label=cat, alpha=0.8, s=120, edgecolors='black', linewidth=0.5) +axes[0].set_title('PCA — Colored by Agatston Category', fontsize=13, fontweight='bold') +axes[0].set_xlabel(f'PC1 ({pca2.explained_variance_ratio_[0]*100:.1f}% variance)') +axes[0].set_ylabel(f'PC2 ({pca2.explained_variance_ratio_[1]*100:.1f}% variance)') +axes[0].legend(title='Agatston Category') +axes[0].grid(True, alpha=0.3) + +# Colored by cluster +scatter = axes[1].scatter(X_pca2[:, 0], X_pca2[:, 1], + c=valid_df['cluster'], cmap='Set1', + alpha=0.8, s=120, edgecolors='black', linewidth=0.5) +axes[1].set_title(f'PCA — K-Means Clusters (K={best_k})', fontsize=13, fontweight='bold') +axes[1].set_xlabel(f'PC1 ({pca2.explained_variance_ratio_[0]*100:.1f}% variance)') +axes[1].set_ylabel(f'PC2 ({pca2.explained_variance_ratio_[1]*100:.1f}% variance)') +plt.colorbar(scatter, ax=axes[1], label='Cluster') +axes[1].grid(True, alpha=0.3) + +plt.tight_layout() +plt.savefig(r"D:\COCA_data\pca_clustering.png", dpi=150, bbox_inches='tight') +plt.close() +print("Saved: pca_clustering.png") + +# --- Plot 2: Top 15 Spearman correlations --- +fig, ax = plt.subplots(figsize=(12, 8)) +top15 = corr_df.head(15) +colors_bar = ['green' if r > 0 else 'red' for r in top15['spearman_r']] +bars = ax.barh(range(len(top15)), top15['spearman_r'], color=colors_bar, alpha=0.7) +ax.set_yticks(range(len(top15))) +ax.set_yticklabels([f.replace('original_','') for f in top15['feature']], fontsize=9) +ax.set_xlabel('Spearman Correlation Coefficient') +ax.set_title('Top 15 Features Correlated with Agatston Score', fontsize=13, fontweight='bold') +ax.axvline(x=0, color='black', linewidth=0.8) +ax.grid(True, alpha=0.3, axis='x') +plt.tight_layout() +plt.savefig(r"D:\COCA_data\spearman_plot.png", dpi=150, bbox_inches='tight') +plt.close() +print("Saved: spearman_plot.png") + +# --- Plot 3: Agatston score distribution --- +fig, axes = plt.subplots(1, 2, figsize=(14, 5)) + +axes[0].hist(valid_df['agatston_score'], bins=10, color='steelblue', + edgecolor='black', alpha=0.7) +axes[0].set_title('Agatston Score Distribution', fontsize=13, fontweight='bold') +axes[0].set_xlabel('Agatston Score') +axes[0].set_ylabel('Count') +axes[0].grid(True, alpha=0.3) + +cat_counts = valid_df['agatston_category'].value_counts() +axes[1].bar(cat_counts.index, cat_counts.values, + color=[cat_colors.get(c,'gray') for c in cat_counts.index], + edgecolor='black', alpha=0.8) +axes[1].set_title('Patients per Agatston Category', fontsize=13, fontweight='bold') +axes[1].set_xlabel('Category') +axes[1].set_ylabel('Count') +axes[1].grid(True, alpha=0.3, axis='y') + +plt.tight_layout() +plt.savefig(r"D:\COCA_data\agatston_distribution.png", dpi=150, bbox_inches='tight') +plt.close() +print("Saved: agatston_distribution.png") + +# --- Plot 4: Feature correlation heatmap (top 12) --- +top12_feats = corr_df.head(12)['feature'].tolist() +if len(top12_feats) >= 3: + import seaborn as sns + fig, ax = plt.subplots(figsize=(12, 10)) + corr_matrix = valid_df[top12_feats].corr() + feat_labels = [f.replace('original_','') for f in top12_feats] + corr_matrix.columns = feat_labels + corr_matrix.index = feat_labels + sns.heatmap(corr_matrix, annot=True, fmt='.2f', cmap='coolwarm', + center=0, linewidths=0.5, ax=ax) + ax.set_title('Feature Correlation Heatmap (Top 12 Features)', fontsize=13, fontweight='bold') + plt.tight_layout() + plt.savefig(r"D:\COCA_data\correlation_heatmap.png", dpi=150, bbox_inches='tight') + plt.close() + print("Saved: correlation_heatmap.png") + +# ============================================================ +# STEP 8 — RANDOM FOREST + FEATURE IMPORTANCE +# ============================================================ +print("\n" + "="*60) +print("STEP 8: Random Forest Feature Importance") +print("="*60) + +rf = RandomForestRegressor(n_estimators=100, random_state=42, n_jobs=-1) +rf.fit(X, valid_df['agatston_score']) + +importances = pd.Series(rf.feature_importances_, index=feature_cols_clean) +top20 = importances.sort_values(ascending=False).head(20) + +fig, ax = plt.subplots(figsize=(12, 8)) +top20.plot(kind='barh', ax=ax, color='steelblue', alpha=0.8, edgecolor='black') +ax.set_title('Top 20 Features by Random Forest Importance', fontsize=13, fontweight='bold') +ax.set_xlabel('Feature Importance') +ax.set_yticklabels([f.replace('original_','') for f in top20.index], fontsize=9) +ax.grid(True, alpha=0.3, axis='x') +plt.tight_layout() +plt.savefig(r"D:\COCA_data\feature_importance.png", dpi=150, bbox_inches='tight') +plt.close() +print("Saved: feature_importance.png") + +top20.to_csv(r"D:\COCA_data\top_features.csv", header=['importance']) +print("\nTop 10 most important features:") +print(top20.head(10).to_string()) + +# ============================================================ +# STEP 9 — FINAL SUMMARY +# ============================================================ +print("\n" + "="*60) +print("STEP 9: FINAL SUMMARY") +print("="*60) +print(f"✅ Patients processed: {len(valid_df)}") +print(f"✅ Features extracted: {len(feature_cols_clean)}") +print(f"✅ Agatston score range: {valid_df['agatston_score'].min():.1f} - {valid_df['agatston_score'].max():.1f}") +print(f"✅ Significant Spearman features: {corr_df['significant'].sum()}") +print(f"✅ Significant Kruskal-Wallis features: {kruskal_df['significant'].sum()}") +print(f"✅ Best clustering K: {best_k} (Silhouette: {best_score:.3f})") +print(f"\nFiles saved:") +print(f" D:/COCA_data/agatston_correct.csv") +print(f" D:/COCA_data/spearman_results.csv") +print(f" D:/COCA_data/kruskal_results.csv") +print(f" D:/COCA_data/top_features.csv") +print(f" D:/COCA_data/pca_clustering.png") +print(f" D:/COCA_data/spearman_plot.png") +print(f" D:/COCA_data/agatston_distribution.png") +print(f" D:/COCA_data/correlation_heatmap.png") +print(f" D:/COCA_data/feature_importance.png") +print("\n🎉 PREDICT2 Analysis Complete!") \ No newline at end of file diff --git a/predict2/correlation_heatmap.png b/predict2/correlation_heatmap.png new file mode 100644 index 0000000..b6d838c Binary files /dev/null and b/predict2/correlation_heatmap.png differ diff --git a/predict2/feature_importance.png b/predict2/feature_importance.png new file mode 100644 index 0000000..d92372d Binary files /dev/null and b/predict2/feature_importance.png differ diff --git a/predict2/features.csv b/predict2/features.csv new file mode 100644 index 0000000..d9a2d2d --- /dev/null +++ b/predict2/features.csv @@ -0,0 +1,25 @@ +original_shape_Elongation,original_shape_Flatness,original_shape_LeastAxisLength,original_shape_MajorAxisLength,original_shape_Maximum2DDiameterColumn,original_shape_Maximum2DDiameterRow,original_shape_Maximum2DDiameterSlice,original_shape_Maximum3DDiameter,original_shape_MeshVolume,original_shape_MinorAxisLength,original_shape_Sphericity,original_shape_SurfaceArea,original_shape_SurfaceVolumeRatio,original_shape_VoxelVolume,original_firstorder_10Percentile,original_firstorder_90Percentile,original_firstorder_Energy,original_firstorder_Entropy,original_firstorder_InterquartileRange,original_firstorder_Kurtosis,original_firstorder_Maximum,original_firstorder_MeanAbsoluteDeviation,original_firstorder_Mean,original_firstorder_Median,original_firstorder_Minimum,original_firstorder_Range,original_firstorder_RobustMeanAbsoluteDeviation,original_firstorder_RootMeanSquared,original_firstorder_Skewness,original_firstorder_TotalEnergy,original_firstorder_Uniformity,original_firstorder_Variance,original_glcm_Autocorrelation,original_glcm_ClusterProminence,original_glcm_ClusterShade,original_glcm_ClusterTendency,original_glcm_Contrast,original_glcm_Correlation,original_glcm_DifferenceAverage,original_glcm_DifferenceEntropy,original_glcm_DifferenceVariance,original_glcm_Id,original_glcm_Idm,original_glcm_Idmn,original_glcm_Idn,original_glcm_Imc1,original_glcm_Imc2,original_glcm_InverseVariance,original_glcm_JointAverage,original_glcm_JointEnergy,original_glcm_JointEntropy,original_glcm_MCC,original_glcm_MaximumProbability,original_glcm_SumAverage,original_glcm_SumEntropy,original_glcm_SumSquares,original_gldm_DependenceEntropy,original_gldm_DependenceNonUniformity,original_gldm_DependenceNonUniformityNormalized,original_gldm_DependenceVariance,original_gldm_GrayLevelNonUniformity,original_gldm_GrayLevelVariance,original_gldm_HighGrayLevelEmphasis,original_gldm_LargeDependenceEmphasis,original_gldm_LargeDependenceHighGrayLevelEmphasis,original_gldm_LargeDependenceLowGrayLevelEmphasis,original_gldm_LowGrayLevelEmphasis,original_gldm_SmallDependenceEmphasis,original_gldm_SmallDependenceHighGrayLevelEmphasis,original_gldm_SmallDependenceLowGrayLevelEmphasis,original_glrlm_GrayLevelNonUniformity,original_glrlm_GrayLevelNonUniformityNormalized,original_glrlm_GrayLevelVariance,original_glrlm_HighGrayLevelRunEmphasis,original_glrlm_LongRunEmphasis,original_glrlm_LongRunHighGrayLevelEmphasis,original_glrlm_LongRunLowGrayLevelEmphasis,original_glrlm_LowGrayLevelRunEmphasis,original_glrlm_RunEntropy,original_glrlm_RunLengthNonUniformity,original_glrlm_RunLengthNonUniformityNormalized,original_glrlm_RunPercentage,original_glrlm_RunVariance,original_glrlm_ShortRunEmphasis,original_glrlm_ShortRunHighGrayLevelEmphasis,original_glrlm_ShortRunLowGrayLevelEmphasis,original_glszm_GrayLevelNonUniformity,original_glszm_GrayLevelNonUniformityNormalized,original_glszm_GrayLevelVariance,original_glszm_HighGrayLevelZoneEmphasis,original_glszm_LargeAreaEmphasis,original_glszm_LargeAreaHighGrayLevelEmphasis,original_glszm_LargeAreaLowGrayLevelEmphasis,original_glszm_LowGrayLevelZoneEmphasis,original_glszm_SizeZoneNonUniformity,original_glszm_SizeZoneNonUniformityNormalized,original_glszm_SmallAreaEmphasis,original_glszm_SmallAreaHighGrayLevelEmphasis,original_glszm_SmallAreaLowGrayLevelEmphasis,original_glszm_ZoneEntropy,original_glszm_ZonePercentage,original_glszm_ZoneVariance,original_ngtdm_Busyness,original_ngtdm_Coarseness,original_ngtdm_Complexity,original_ngtdm_Contrast,original_ngtdm_Strength,scan_id +0.9993625929219437,0.046860553771808006,12.37860976574501,264.1584183154097,229.0,229.0,323.1485726411305,323.1485726411305,52368.333333333336,263.9900418698473,0.06415716283734095,105507.85870727764,2.014726304839648,52615.0,2.0,2.0,11567286.0,0.044883290394348206,0.0,1122.9115614899351,877.0,1.2960630704725395,2.6456333745129714,2.0,-44.0,921.0,0.0,14.827261582001467,30.341709808443028,11567286.0,0.9935869555823512,212.84831006875973,119.66064285057824,44565.17466108682,923.6868317496888,94.22037163258454,57.523404260213866,0.39858661499685755,4.715474468153125,2.745191803162805,23.409901662799165,0.4789342288137195,0.41784593961047123,0.966731330000424,0.9042603414328695,-0.5630025406437277,0.7501733837735425,0.10893526741565479,9.555595223572142,0.3130219734446293,4.7887675010436315,0.8486742799574394,0.32403197250292826,19.11119044714429,3.4149106259035658,37.9359439731996,0.17700458851118128,50481.28459564763,0.9594466330067021,0.3229502832877294,52277.57766796541,0.3215665658050146,9.46961892996294,79.96504799011689,721.2328803573125,8.885385293077938,0.11091141314099398,0.014340696980042542,0.4442770122588615,0.001443587409270362,36270.48672928581,0.8283611848442824,8.602454622105002,24.69113861987565,6662.148270093807,59976.1099572755,740.232524134027,0.10413235201347064,1.6658465853022029,36452.15062147034,0.7796910958821593,0.6952214562971951,2825.506381595789,0.7954992680077096,22.577797703818455,0.08140272086363821,6.339285714285714,0.05660076530612245,47.72313456632653,171.6875,24554097.08035714,220987299.875,2728232.913817201,0.032573836221548226,54.0,0.48214285714285715,0.7176661706381673,142.60880208336258,0.016770152408281662,5.317836954365818,0.0021286705312173336,24333406.8519611,0.24616801040496997,0.026041132749717585,4.884164200188554,1.2485436319994676e-05,17.454353422243575,5b8eb0dfaafb +0.9989270127717552,0.07725001231372526,17.668238327643902,228.71502280012876,199.0,199.0,280.72228269234347,280.72228269234347,39828.875,228.46961450175652,0.07035678007011455,80163.5353537984,2.0126989615900124,40066.0,2.0,2.0,26984272.0,0.137992390503385,0.0,211.89587691903006,711.0,4.694394233541013,4.374082763440324,2.0,-18.0,729.0,0.0,25.95179246933797,13.331934837284162,26984272.0,0.9771292370334418,654.3629323501611,96.72694379872064,11779.746046015682,275.12730834828153,50.34204008730576,29.197327092070868,0.4537888720206611,3.442730678629307,2.5650569494905655,11.559502823398379,0.4997703603975673,0.43513790417611187,0.9720674022051301,0.909085254184057,-0.3798394471698749,0.7492937154798709,0.14250829154648745,8.530080045723428,0.307496605770526,5.301381666929735,0.6725944533124836,0.3177228808553251,17.06016009144685,3.429074271840662,19.884841794844156,0.2983677612241394,37608.60095841861,0.9386662246897272,0.7394675488882668,39149.66001098188,0.9752935242583278,5.344756152348625,79.22457944391753,322.76316577646884,19.793355505263538,0.24739005333100592,0.018072274111912874,0.8048152130774344,0.0032865909272608174,27158.209949586093,0.7350423937607047,9.624765823322688,26.73134334884942,2875.367145432649,11527.563148038114,718.7904147787232,0.20624221629197803,1.7677595102596189,27791.87971534655,0.8102206658716457,0.6977237558029251,1910.760547450922,0.8716167457962775,25.377656216289918,0.17604550885059303,17.18978102189781,0.06273642708721829,29.660250945708345,129.29562043795622,5723505.2919708025,22894523.098540146,1430875.2865734927,0.028268532351720374,120.00729927007299,0.4379828440513613,0.684685223169634,96.69430982017512,0.020402023104069956,5.5720185419486326,0.006838716118404632,5702123.167776654,0.24440471538190844,0.03612568398824158,13.803974343943468,8.195815144848356e-05,10.386027749033996,66dbc7756f54 +0.9999157212195828,0.04831661259371927,10.700458628469677,221.46541435855218,192.0,192.0,270.8228203087768,270.8228203087768,36743.0,221.44674952352545,0.07202974624831507,74203.0884240639,2.0195163275743377,36957.0,2.0,2.0,3219313.0,0.0340444167022229,0.0,1278.106608576696,582.0,0.7630398860896919,2.3822550531699003,2.0,-6.0,588.0,0.0,9.333257472662222,31.732928395729388,3219313.0,0.9950280851424353,81.43455591265169,59.40707601131984,6182.432442028242,269.9536935126543,33.0957757671301,28.457947709388193,0.3058301226631647,3.3463737101026605,2.3070939588472363,11.839017646144756,0.5051785274906953,0.43861143169921846,0.9631464519046035,0.8978011168894974,-0.6154967926578251,0.7382236896506656,0.1332953698999468,6.896050333930286,0.3232356907489774,3.865461048515882,0.9303784779794237,0.34010738979679006,13.792100667860574,2.627108878946954,15.388430869129573,0.18338162152333004,35278.584273615284,0.9545846327790481,0.31284905129534396,36773.252942608975,0.120176707140533,4.1778553454014125,79.87469220986551,320.1177314175934,19.966236055209794,0.24947295186170798,0.014057308677385163,0.16175890432558093,0.0032382152286375854,25529.22187027537,0.8658250855475479,3.176786552244525,9.671898720033868,5437.2753036465865,21755.42633743126,1359.299372083067,0.2335117861318827,1.5108405288110505,25607.44415944402,0.7844053910396356,0.6953944396918664,1798.9989640786748,0.7677734644287624,8.589596121655443,0.17612773942302093,5.548387096774194,0.08949011446409989,19.749479708636834,77.95161290322581,21918624.629032258,87674710.59677419,5479655.342441966,0.05348726956142245,32.38709677419355,0.522372528616025,0.7495270808561539,59.0252016129507,0.04502276667888866,4.536868842952556,0.0016776253483778444,21563312.49349636,0.07602019361682885,0.1825635977076956,1.29507755861101,5.541055692273879e-06,14.823293939470554,69ebf706870a +0.990236834207065,0.19461899802215785,46.068804246658544,236.71278094553492,205.0,205.0,289.2075379377239,289.2075379377239,42619.0,234.40171481985695,0.06881706247255312,85741.31981983998,2.011809751984795,42916.0,2.0,2.0,42804618.0,0.22363603293813605,0.0,167.0094056472295,895.0,7.310681720160203,5.728026843135427,2.0,-38.0,933.0,0.0,31.581713869675188,11.25503977676445,42804618.0,0.9600346611072309,964.5943594303543,96.19347434988597,29258.14392899947,779.8434387815064,54.5475679998829,30.040907100083757,0.44965347384415427,3.405842942061433,2.5990535365114034,12.923925535323331,0.5054604688263639,0.4410945555387714,0.9815302447115232,0.926251497848665,-0.31812160772865006,0.7200643575649829,0.14317514481373725,8.71449795845187,0.30480591789968436,5.436141019343809,0.7532483151287382,0.31631624330796027,17.428995916903727,3.3849048754661935,21.147118774991675,0.40225300006417053,39591.022369279526,0.9225235895535353,1.1700440988311493,41200.84751607792,1.4297179216747014,11.293526889738093,78.56953117718334,717.7217121819368,8.719682962252977,0.10930892615423153,0.023007335460946392,1.53296066087758,0.0017175346648685085,28588.03619659204,0.7029880744574354,9.250651501838329,36.01526066874179,1968.0924226853442,17743.0564022592,218.65240712491575,0.09023484680470896,1.7790302569766776,29833.1011041448,0.8513444095358332,0.7000885450647777,1530.6349016873583,0.9115567035723517,34.45850909573031,0.08124846784703843,33.445283018867926,0.06310430758276966,32.015108579565684,130.28679245283018,3332270.3056603773,29990922.266037736,370251.7916625332,0.03026188449195789,267.0452830188679,0.5038590245639017,0.7379264925036605,102.26474197835029,0.025614873538806135,5.618568150914065,0.012349706403206264,3325713.584207903,0.8662554156083611,0.006339707025630678,35.18842550930922,0.00019974876880337757,6.773706347156496,6b1dee6a31e8 +0.9950565618660016,0.21321925978452724,48.76270635186346,228.69747508335564,201.0,201.0,283.5507009337131,283.5507009337131,41801.208333333336,227.56692326387943,0.06972617390127515,83537.3793825111,1.998444129087443,42168.0,2.0,2.0,113152826.0,0.4226484789251693,0.0,76.78146053745364,928.0,16.543990949051274,10.610083475621325,2.0,-118.0,1046.0,0.0,51.801364646550816,7.791371520709379,113152826.0,0.9194728368059234,2570.807507885273,197.2121561473841,42264.972146976856,1066.4688151367184,85.07973097490802,35.42698257399276,0.5373195197960164,3.724624339569652,2.73265275244155,15.088994395814298,0.4932035041263862,0.4294705961154446,0.9826956147791405,0.9282325106144886,-0.29179292817837327,0.7488715027045452,0.14126656668514867,12.854574342842565,0.29458892091919464,5.941331718839267,0.6555541282261588,0.3089224415707909,25.70914868568514,3.7069550372781586,30.12667838722519,0.6415197226967717,37228.26935116676,0.8828559417370224,2.1448202276312713,38772.33058243218,3.8585900135398994,43.89914152912161,76.96518687156137,2805.860344336938,2.1337687518589266,0.027017854034731992,0.03280381887682207,5.583192595534071,0.0005439655070953346,26925.13704880726,0.6616551490289967,13.904289556055044,91.08001103970489,1069.1961697853112,38553.51434638951,29.69379362517119,0.02251890448972432,1.9067949309111467,29476.22310531032,0.8931094529920472,0.7057739737022604,927.18613745559,0.943019502033523,87.26872087532774,0.021132080345993974,60.542307692307695,0.058213757396449704,41.38801682692308,241.96442307692308,1569467.0365384615,56501612.67596154,43596.21549418869,0.01015580827534269,482.5980769230769,0.4640366124260355,0.7074141975042921,181.67375906042074,0.0071506651389832265,5.925260783478826,0.02466325175488522,1567823.0459467457,0.5143714373360457,0.004118175365561156,107.51434416812913,0.0006608259092110763,5.270615799233856,70a5b54337ec +0.9998582268614304,0.04433657954459684,9.258001439679713,208.81181035554096,181.0,181.0,255.266527378738,255.266527378738,32620.5,208.78220644981647,0.0748332458158519,65975.5902204224,2.022519281446403,32833.0,2.0,2.0,1100657.0,0.02838564785299938,0.0,847.4148572371255,263.0,0.4544576230974559,2.2277282002862973,2.0,2.0,261.0,0.0,5.789895383963707,27.24506453416987,1100657.0,0.995680509682262,28.56011562289341,21.555855159698943,126.85470033230348,6.962026003155013,5.8237378624423375,7.878076039187341,0.12832770867053228,1.9327092422869931,1.6543016467436025,2.418977263611483,0.5592516390124834,0.4916484335491019,0.9459874024994451,0.8699724180977009,-0.6579037760490845,0.7071637228087615,0.18172043834697058,4.180693409312345,0.3487819482126729,2.970225493495517,0.9090780217670758,0.37650557445320154,8.361386818624691,1.8184798256426746,3.42545347540742,0.18465397128831573,31283.779459689948,0.9528151390275011,0.3109966498522533,32691.178174397708,0.03928202288327202,1.0559193494350196,79.8408004142174,80.03067036213565,79.83358669136132,0.9979868914255038,0.013975348687503126,0.047730133000034224,0.012797523622682225,22701.29038370507,0.882571089544673,1.0153700708131956,2.755708637124329,5081.357587188288,5083.336141496774,5081.285425602373,0.9372607156617991,1.4223361372659606,22752.347641386594,0.7885726342160511,0.6954635228627859,1504.2143264639474,0.7578905974988975,2.4578878167826943,0.6975068885547011,7.423076923076923,0.14275147928994084,4.736686390532545,26.03846153846154,20640062.480769232,20640125.519230768,20640060.184897754,0.11293826628918618,27.192307692307693,0.5229289940828402,0.7501335470264647,19.9337606837786,0.08237791093276042,3.985048189755741,0.0015837724240855236,20241391.66383136,0.18463114289425708,0.27057617379345617,0.18062861876259462,2.776839222555751e-06,6.024493853719077,8234fbd8de27 +0.9999699433766066,0.010361449761750026,2.7867565275194663,268.9543057774561,233.0,233.0,328.80541358073776,328.80541358073776,54089.0,268.94622191917733,0.06343859150412466,109027.62723991457,2.015707948749553,54328.0,2.0,2.0,920684.0,0.01070080132542194,0.0,2611.92749264768,264.0,0.17194245006595205,2.086032984832867,2.0,2.0,262.0,0.0,4.116645209498442,48.22234422830545,920684.0,0.9985648657724137,12.595234167075745,26.29668090235278,129.49015647752708,8.663801340809117,6.571363754745481,9.40615502277632,0.159518243809279,2.0636261920666024,1.7662809167733005,3.1765208980824156,0.5610921382559136,0.49645954512706414,0.9382268778432878,0.8649836793186988,-0.6615446746784268,0.7102317473826683,0.17668848385607178,4.604582990398822,0.3474013424820231,3.0082301551108106,0.9406691383165928,0.3945379834754347,9.209165980797644,1.9923091253370833,3.9943796943804504,0.13679231833176272,52427.03721837726,0.9650095202911438,0.1854024977968862,54250.0320276837,0.017605318974624935,1.0239103224856427,80.17736710351936,80.33295906346635,80.17197767848667,0.9993282049951501,0.012949415280027488,0.024643651115037878,0.012631831761989086,37656.36916825702,0.9405224729286578,0.7154017799519005,2.058291753043101,9869.533088041822,9870.945867869827,9869.49262573483,0.9703997467095967,1.3697259378567845,37641.19181905416,0.8129827374357296,0.6944683574414101,1702.0895929748626,0.7224727043281663,1.7118480302471903,0.6947997407982935,2.9130434782608696,0.1266540642722117,7.323251417769377,35.0,128143289.47826087,128143454.47826087,128143283.83069055,0.11661962166455862,10.652173913043478,0.46313799621928164,0.7037372517592501,24.461788781549913,0.06218267430816872,3.653996738665704,0.00042335443969960244,122563834.68809076,0.00030668479082826463,163.01223913235947,0.055760906302400715,4.3569095461099485e-07,10.408727881446481,8322c647539e +0.999833039682875,0.015444271211463044,4.333679164035027,280.60107885301085,243.0,243.0,342.94751785076386,342.94751785076386,58817.583333333336,280.5542296078999,0.06170487878374131,118532.12544355597,2.01524984071185,59064.0,2.0,2.0,444228.0,0.003924182389155024,0.0,5124.9231066335215,188.0,0.05687299689045905,2.0284437220642015,2.0,2.0,186.0,0.0,2.7424677978245864,69.33497257764073,444228.0,0.9994921578007502,3.406545688523165,18.394548843262616,4.245555076503317,0.5257230474650177,0.7119063839613826,2.7000463926763087,-0.03587312624966757,1.0212399641674217,0.7895611963264776,0.8375211369425835,0.6850897937844361,0.6444264361682474,0.9637741936945684,0.9028866041489251,-0.7239521410222185,0.6041427688907951,0.2562021305277092,3.792521697578238,0.5096000570549085,1.501090246554147,0.8344138207643387,0.5439161886508929,7.585043395156475,0.7889489205017725,0.8529881941594227,0.12534383769585744,57130.09535419206,0.9672574724737922,0.15887493070141534,59034.00480834349,0.004553315057256285,1.006619937694704,80.24173777597183,80.27476974129758,80.24061280160315,0.999757683990624,0.012727261848624551,0.015765279930705006,0.012605136790307397,39473.17502100088,0.9723365285491439,0.24309168202758388,1.3689112257149392,12494.883765471155,12495.310894941713,12494.86827994448,0.9864218299805532,1.397978323298238,39413.57573838198,0.81693896811505,0.6687957131247461,1365.163605637803,0.6813463745126209,1.035703039017284,0.6682450436577628,2.3333333333333335,0.25925925925925924,3.333333333333333,22.11111111111111,387420494.2222222,387420637.6666667,387420489.2023534,0.16050154320987653,4.333333333333333,0.48148148148148145,0.6780555555874219,15.5833333333652,0.0422302469454466,2.2810361125534215,0.000152377082486794,344351900.4444444,9.269900956519828e-05,1078.8734276558605,0.002819815466372306,8.580929932917387e-08,12.336125263273882,9edcb930e6ab +0.9824913544136815,0.5042474170531516,98.64034908358403,195.61894765875732,198.31288409984865,179.61347388211163,242.53865671269807,242.53865671269807,31407.375,192.19392483423155,0.07765163821181058,61994.70465658467,1.9738900387754363,31690.0,2.0,2.0,264517584.0,0.6677753528218292,0.0,47.81200720571428,1350.0,35.38701228501375,20.862985168822973,2.0,-333.0,1683.0,0.0,91.36211695923511,6.134328748369493,264517584.0,0.8726455393200998,7911.772265118429,656.8035690673798,162468.14685516342,2791.261270458418,172.1332433917403,67.48290388429312,0.5470124961270595,5.16312895567581,3.090489551939622,28.744004373700214,0.45852197922929755,0.3967448635227037,0.9868388306305917,0.936539462756261,-0.29184692209054663,0.8122953558932254,0.10656474505794906,24.408684251220887,0.2824093236991598,6.596190696971028,0.6525828731565908,0.2995686322618593,48.8173685024417,4.137862076692862,59.90403681900837,0.9359571010162765,26418.59804354686,0.8336572434063383,3.365018321521771,27654.13714105396,12.116265802119596,259.47898390659515,74.94761754496686,16995.56443673083,0.33237041179718796,0.004308451542858408,0.04937483435545484,32.459576724615374,0.00015644756582871671,19218.60814603376,0.622042740936994,31.341119043368316,390.9067554293078,579.4051439692975,130549.88657060069,2.574388447248067,0.0037996778694657767,2.164647744540887,22326.973964772355,0.9153870237164443,0.7130519212564023,518.8967253786365,0.956640272022472,376.93439345441453,0.003630358956920187,54.62979351032448,0.04028745834094726,82.88913851689422,788.4653392330383,645441.4985250738,145226135.69100294,2868.619283665048,0.002655404965720147,745.7330383480826,0.5499506182507984,0.7701002314145036,623.8314837130974,0.002218529051426625,6.2307569095634845,0.04278952350899337,644895.3319737037,0.08665218518103404,0.006900607148447924,473.91739639892836,0.0022918073067590716,8.046515891280338,a72e38ef4265 +0.9999639083818775,0.014887116614408434,3.1964693144957113,214.7137956454255,186.0,186.0,262.33756879257686,262.33756879257686,34424.75,214.70604627710745,0.07358776272414458,69543.90208351391,2.0201715940860545,34617.0,2.0,2.0,373460.0,0.008867790124240481,0.0,2948.0329699335794,182.0,0.11397882519476137,2.057024005546408,2.0,2.0,180.0,0.0,3.284561003929086,51.79655290480338,373460.0,0.9987871605590137,6.5569932291374595,10.818575313876,24.5541780846222,2.822060073894849,1.7757257190285125,3.089098200446632,-0.04702476748917664,1.2696395830014378,0.8217451819922063,0.6923015724743963,0.6009759098151165,0.5402439487514571,0.9572626026785351,0.8773542486208588,-0.7811724919566841,0.6819336829957597,0.26670985395593705,3.028695642933211,0.44059716583567793,1.810055478729303,0.9429564998070836,0.46414713026487414,6.057391285866422,1.0084824003404753,1.216205979868786,0.15926837931281815,33127.38275991565,0.9569686211952407,0.21763126706704713,34575.01513707138,0.008719948321635675,1.012768293035214,79.98997602334113,80.04832885576451,79.98696342801306,0.9994468071144839,0.012874303177237189,0.016750048235060416,0.012687875771845577,24017.257774208723,0.9582083812602536,0.31846891949986766,1.4904848107489097,6541.64815734587,6542.215473485406,6541.621219386483,0.9799288398574149,1.278053190044749,23991.20010533836,0.8222660521818781,0.6949409027578715,904.6962141845022,0.7130378500527184,1.1843148286050764,0.6946833897214534,2.1666666666666665,0.18055555555555555,3.5763888888888893,20.25,99740271.91666667,99740348.25,99740268.38532124,0.1572388747165533,3.5,0.2916666666666667,0.44444444451406967,7.8310185185881425,0.03527486792571834,3.0849625007211534,0.00034665048964381665,91418489.35416666,0.0002815498291810353,296.1632790625324,0.013080562336226276,3.6557125230530757e-07,7.544671137554932,ae676e3b5447 +0.999981911804411,0.012420730560651898,2.5379716721141623,204.3335260934086,177.0,177.0,249.6096953245206,249.6096953245206,31164.083333333332,204.32983006862324,0.07604404549970262,62977.94463595891,2.020850219220061,31344.0,2.0,2.0,732294.0,0.00749485976239524,0.0,3023.3352595077117,317.0,0.1775566416281085,2.088820826952527,2.0,2.0,315.0,0.0,4.833542355234601,53.39074916948603,732294.0,0.999043135522468,18.999959252736232,52.71826054184746,266.7690322217189,-10.272049681791318,7.127979767165673,13.302001544446798,-0.04036020644790199,2.5424734477569015,1.220117204882325,3.635356135394718,0.5246340460546844,0.4568523164036228,0.9360104411682499,0.8585556278936566,-0.9181802518949461,0.7114748621766153,0.15009886115831755,6.406205827255591,0.4139517743257788,2.042712182086753,0.9999999531094832,0.4339149828117669,12.812411654511184,1.1499108608674409,5.107495327903117,0.16374342620540885,29938.131380806535,0.9551471216439043,0.22787421948306802,31314.00803981623,0.02710741851654684,1.0337544665645737,79.94888973966309,80.01837672281776,79.94798448088746,0.999535883404498,0.013093158987641717,0.040750410419778296,0.012704316653034004,21756.31267342562,0.9641050577688808,0.9974676065669071,2.3130400675537923,5989.525098337448,5990.839316437638,5989.5070292262835,0.9819454324477863,1.2639876658572187,21724.92831249252,0.8251082456201018,0.6950514391172888,772.6546326529686,0.7119693108151484,2.0247148702098383,0.6939183791661332,1.4285714285714286,0.10204081632653061,12.061224489795917,65.14285714285714,70107590.14285715,70107700.0,70107588.68048596,0.10200271946861665,10.428571428571429,0.7448979591836735,0.8650793651521395,59.928571428644204,0.029790295477115196,3.3787834934861722,0.00044665645737621234,65095108.8367347,0.00010416212417501098,479.9321244473511,0.0545300610147952,6.327359113922631e-07,31.45245541354373,b39c18fd990e +0.9796621769628845,0.3147385120013255,75.20966502050277,238.95920630197944,209.0,209.0,294.8643756034289,294.8643756034289,46173.208333333336,234.0992962511202,0.06870331876843126,90594.07108529742,1.9620484336128705,46498.0,2.0,2.0,166305119.0,0.5749792973960035,0.0,32.35719093911723,714.0,24.33157605726459,14.935932728289389,2.0,-44.0,758.0,0.0,59.804749014149046,5.171408430434639,166305119.0,0.883597191602844,3353.525918181376,116.10787244759973,10166.461255542943,279.1587705689161,43.983040071545325,18.715594482194586,0.5241026051699491,2.7869516275743917,2.4914618532337522,7.64883254757722,0.5178409356292972,0.45151057632827507,0.9822998218259728,0.9253098975310305,-0.24300713073228025,0.6590002377733867,0.1674888936332582,9.624499654824877,0.28313541585046037,5.766288202817564,0.5934566330864828,0.30142562304162224,19.248999309649754,3.528100714612909,15.674658638434973,0.8456941534851843,39543.48135403673,0.8504340262814902,2.5518755475288883,41085.50221514904,4.992783486533639,17.196309518688977,75.83767043743816,756.1167792163104,8.371260531716903,0.10535569664287683,0.033315651840848906,3.1762323381633757,0.001821210264976341,28585.36977367777,0.6357736137937363,10.955223963141899,51.70181482936835,828.5777619362452,7513.12986898298,92.0242685402825,0.08137879095453232,1.9813928529279359,32505.274946512458,0.8929182377307664,0.7092033735115156,746.1010667884829,0.945932789850512,48.44973235803278,0.0774667797636111,71.39093959731544,0.059891727850096846,27.298899120535115,144.90855704697987,1600712.312080537,14408563.954697987,177855.29282049809,0.022178657923479627,449.21812080536915,0.37686083960181976,0.63663444350025,90.50492700154484,0.01697740022268852,6.215686930664766,0.025635511204783003,1599190.6576477408,0.8486355617416927,0.0064879244123859005,60.33225592192009,0.0016985209614257986,1.9695045785321135,c4f0b2426103 +0.9999286309889949,0.015334458018747735,4.1583325552823345,271.1757109510101,235.0,235.0,331.6338342208165,331.6338342208165,55058.625,271.1563574087109,0.06306220801304793,110985.23160652645,2.0157646800392572,55306.0,2.0,2.0,2210070.0,0.020545515608809635,0.0,1567.329172212471,382.0,0.3972764872914399,2.198929591726033,2.0,2.0,380.0,0.0,6.32145266143538,36.61407723085877,2210070.0,0.9971093415645064,35.12547240140003,33.81070275415835,919.4560048946806,34.759750309298674,15.277864481451441,16.784068220174184,0.21687714777269207,2.667082077930242,2.0805951392868494,6.306718817503356,0.5290252614740006,0.46467341703929876,0.9477990149357646,0.8778368693459627,-0.5947820897714409,0.7186019460373282,0.17104627567847144,5.145857610290053,0.32788006596781244,3.6255970748777098,0.8924183464455311,0.34294886682850023,10.291715220580105,2.4707147811148746,8.015483175406407,0.14723640781278804,53306.44284526091,0.9638455654948995,0.22214004423336306,55146.1292445666,0.05008506968228414,1.0648573391675407,80.12646005858315,80.40865367229596,80.11874780730514,0.9986299009464114,0.013343194853504162,0.05533210403354088,0.012646502248516992,38270.23174721987,0.895037009659246,1.8010344383332024,3.726680034633075,8873.58946347513,8876.658660818985,8873.5211906035,0.9440338948432293,1.4836839347150779,38315.491581057366,0.7916124742029031,0.6946568601542745,2407.1662467832252,0.7491266119696854,3.40020292132567,0.695932262534718,5.03921568627451,0.09880815071126489,12.336793540945791,51.21568627450981,59800016.078431375,59800163.74509804,59800012.4578967,0.09037091745304386,24.372549019607842,0.47789311803152634,0.7183769063245119,42.850784313731914,0.05444946974801539,4.443581528970368,0.0009221422630456009,58624024.67666282,0.2100958217694009,0.18299258627588919,0.2781041486287365,1.9222299016827763e-06,9.31178219613705,c7c24b912e3f +0.9987180667376037,0.11476755684272932,25.868629714004843,225.40019519151812,196.0,196.0,276.4796556710819,276.4796556710819,38601.416666666664,225.1112471839515,0.0710345544591087,77758.87332559821,2.0144046524785977,38839.0,2.0,2.0,29815138.0,0.1299110118192883,0.0,310.2565504561779,911.0,4.524959290798204,4.287031077010222,2.0,-6.0,917.0,0.0,27.706673834447532,16.016154341540304,29815138.0,0.9789551812418025,749.2811395132081,115.6914055148982,45458.36439323794,1190.74041437451,98.60703012037742,37.99325539296469,0.5736300134477192,3.8502318605200894,2.6773633709852414,16.05045742078254,0.4965922534369328,0.4336038692272429,0.9768403057167186,0.9184255874671147,-0.44011097309884195,0.7746008055965745,0.12855431611558432,8.91660720982449,0.30817030986460464,5.32741462105216,0.764389916556381,0.32121494432563796,17.83321441964898,3.569239558723302,34.15007137833554,0.29408626427887585,36487.44177244522,0.9394536875935329,0.7126207969257836,38021.64028425037,1.1335631273261926,5.4901001570586265,79.26947140760576,322.8114524060867,19.804513580558766,0.24762188533037519,0.018189638416671504,0.9567265323546812,0.0033360030203833844,26379.418244442455,0.7429955581877891,12.83079551366936,29.92251703789453,2906.4182242402094,11655.106807521253,726.5572411351711,0.2087775785813471,1.7886181386735462,26936.852823336718,0.8085965837257818,0.6975680669905547,1891.8202112104107,0.868228744247821,28.528600715893635,0.17731670296389757,14.423076923076923,0.05547337278106509,47.324792899408294,156.6076923076923,5676116.088461539,22704928.39230769,1419028.056751405,0.035680703289630314,128.62307692307692,0.4947041420118343,0.7312103780284605,120.77064036669842,0.027351228112091508,5.671268487626294,0.006694302119004094,5653801.474245562,0.5041108492551972,0.01539537695108977,19.642023207030384,7.650290878770294e-05,15.638908919368257,d11d0c292b1b +0.9995850132556447,0.12311575554808088,23.692135690087092,192.43788566796803,167.0,167.0,235.46761985462035,235.46761985462035,27857.583333333332,192.3580264963041,0.07883820817270387,56369.3471409852,2.023483030329331,28057.0,2.0,2.0,6304613.0,0.0741506283416783,0.0,397.278996390419,497.0,1.9596891977685194,2.985030473678583,2.0,-17.0,514.0,0.0,14.990240495692088,18.34831536603996,6304613.0,0.9882051701762747,215.79690318989714,66.5004467226661,2948.598492568611,87.07978904522471,28.355980127233497,24.704711885612564,0.3093885050313243,3.2277611521162637,2.3860012392544325,9.38358694932178,0.5093715294669894,0.4448819913109451,0.9544107214230471,0.8851377294065051,-0.5402158744898002,0.7476402995914998,0.14071153556460023,7.289347343648289,0.3174342356948883,4.2444313551249495,0.8148172878545517,0.3309743208733543,14.578694687296577,2.8904768257422373,13.265173003211514,0.24539909308527008,26433.596571265636,0.9421390943887671,0.5245034424744095,27726.072459635743,0.31331475172559126,4.462522721602452,79.47061339416189,319.72894464839436,19.86233105967884,0.2487069690415779,0.016475329983010777,0.3573385924685066,0.0033195228552670604,19247.808152455113,0.8003498315947944,4.777422347537897,14.005201628795426,3142.4462661057264,12581.03896754554,785.5801864869168,0.22198886195727102,1.6020074914708753,19448.586446462366,0.7824269818893127,0.6967245250739567,1530.0022049393253,0.815712885857426,12.956852823803494,0.17675673364393715,10.15,0.08458333333333333,16.856597222222224,80.19166666666666,6481639.075,25926789.658333335,1620409.0821336007,0.04123660895812283,76.93333333333334,0.6411111111111111,0.8251700680379249,65.85035194637695,0.03556812394477638,4.576504829670242,0.00427700752040489,6426972.738263888,0.14220081810874607,0.09749912795052622,2.562523261570866,3.0847130168477354e-05,8.401676144056733,e01cefab0d59 +0.9999959743936842,0.014527156536899443,3.135861397280637,215.86202291655965,187.0,187.0,263.751777245197,263.751777245197,34799.333333333336,215.86115394103686,0.0733346978123877,70289.19461225385,2.019843137193831,34990.0,2.0,2.0,564161.0,0.008988864558083485,0.0,2766.098451053441,237.0,0.15315181139003406,2.076621891969134,2.0,2.0,235.0,0.0,4.015406881800093,50.53960581045358,564161.0,0.9988000719758172,11.811133944202084,25.76271769040925,111.37884534691852,-2.2106606143914806,5.294989509874144,10.110931198736601,0.1452384202282944,2.14951552899856,1.4562760315107335,2.635049833288353,0.5682920117562662,0.5041062445081413,0.9219653495902105,0.8514834349372742,-0.6364203665973874,0.6455257557437896,0.14215667474887098,4.477669426851201,0.4190219519424506,2.3377334551471125,0.863636078348161,0.44391495186131213,8.955338853702402,1.5510054093737937,3.8514801771526863,0.15900819493819135,33492.38113746785,0.9571986606878493,0.21872030885238025,34948.01451843384,0.017225163176904097,1.0230065733066591,79.99491283223779,80.11480422977993,79.99238045746758,0.9994341037474551,0.012931759162229774,0.021544338946930162,0.012685355607003238,24275.741369004852,0.9568912063207288,0.5903072147681044,1.8433482961377292,6581.605976202434,6582.712984403259,6581.579791109703,0.9787339512790478,1.2882950658618673,24249.682607251663,0.820842963503345,0.6949326181106689,940.4590105392891,0.714465160656172,1.491898480622518,0.6944288729376284,1.7692307692307692,0.13609467455621302,6.437869822485207,33.0,94063923.6923077,94064078.23076923,94063920.25594966,0.13247210351377017,4.230769230769231,0.3254437869822485,0.551816239379145,17.598290598353504,0.031478847700607535,3.546593564294935,0.00037153472420691625,86819544.40236688,0.00022462550043565095,278.21306449119237,0.033711360107770844,6.152304747276634e-07,11.06588651161052,e643d0f987b7 +0.9999557451136373,0.013680883440250076,3.9477399957775408,288.55885023938043,250.0,250.0,352.8469923352047,352.8469923352047,62297.25,288.54608010025413,0.06055160819214346,125507.88940951606,2.0146617934100792,62555.0,4.0,4.0,2815963.0,0.012756818906849118,0.0,2989.9144432430403,504.0,0.2598767746680103,4.128926544640716,4.0,-25.0,529.0,0.0,6.709381052018952,50.13402359044866,2815963.0,0.998306283372878,27.967759690152235,55.39517777135323,3198.422593981036,32.68254328298329,29.478107463369344,31.161259306506942,0.2166927096183969,3.576072576412432,2.1580901191487776,12.18655160375811,0.515322932332743,0.4534569291389372,0.9492401447027455,0.8809784658113045,-0.6530788902593441,0.7177459726587367,0.10649277939654993,6.757551811887897,0.3316099327266974,3.4182405609575266,0.9423952387077147,0.3464046340753982,13.515103623775794,2.5369415151242944,15.159841692469076,0.13277991901248865,60486.403916553434,0.9669315628895122,0.1811130240237923,62449.049556390375,0.04189118601607725,4.06148189593158,80.2171688913756,321.3334825353689,20.05303748746319,0.2498590816285778,0.012932446607479246,0.07643904228476471,0.0031880290803787893,43338.50062630011,0.9273563340730381,1.777793931331271,6.847394332291633,10911.581208602978,43649.88947422378,2727.8866892911474,0.24369498425475064,1.4392666029675723,43336.31500297592,0.8041900038523992,0.6943735666459669,2256.945664101266,0.7309839895784177,5.5920189207118085,0.17701794261226322,2.6,0.08666666666666667,21.965555555555554,71.43333333333334,130208338.23333333,520833706.6,32552083.633302014,0.1186841018578101,10.466666666666667,0.3488888888888889,0.6004074074159408,45.97584259262673,0.07625909955043153,4.573557262275177,0.000479577971385181,125860418.20555556,0.1902612523390828,0.10949710225996849,0.23449372921710265,1.5709970937712513e-06,9.692428703747582,ecaf9c8bec0b +0.9989322914759764,0.09723341101591709,22.04033596886509,226.67451176074746,197.0,197.0,277.89386463180506,277.89386463180506,38946.875,226.4324894523616,0.07068566423003106,78608.20129177917,2.0183442520556314,39205.0,2.0,2.0,14259310.0,0.11649020507480695,0.0,249.49282625496886,545.0,3.1321148223961037,3.5769927305190663,2.0,-31.0,576.0,0.0,19.07122220488807,14.591078030842255,14259310.0,0.9807161779859467,350.9166393940296,77.04587942311069,4017.4056367777916,75.15791213122165,31.52487485183665,28.378173033047172,0.2859229674671733,3.464144610596954,2.5015391728177403,10.533099596666972,0.49239553677428216,0.4260299486764077,0.9595742215315742,0.8902270120658107,-0.4043719716530071,0.7419004450203066,0.13726415540682746,8.059591375337154,0.31003651168298857,5.017166412529618,0.7017241720393574,0.3186503979525503,16.119182750674312,3.190665250129645,14.975761971220964,0.27828783347680464,36897.50860859584,0.9411429309678827,0.7027108078636163,38448.977757939036,0.511219572019002,9.867593419206734,79.31837775793903,716.9489350848106,8.809893788551818,0.11031469190396423,0.018423238249089546,0.6934026555569726,0.0016071642680142667,26673.56697780305,0.7507816949490488,5.870203047777531,24.7356148082348,3089.561666133403,27824.379889537573,343.26770696449796,0.09672556171586112,1.7140829784087832,27196.54395578961,0.8034566204828753,0.6974404756065259,1958.1679702998106,0.8621505769155786,22.876632919945816,0.08202628977358195,19.437037037037037,0.07198902606310013,19.481755829903978,97.83703703703704,5578293.611111111,50204879.29259259,619810.1613248707,0.03511213944564466,144.06666666666666,0.5335802469135802,0.7587592592617184,75.83655555557769,0.028351646951984666,5.1798609221356235,0.006886876673893636,5557209.495541838,0.6331613434448976,0.011424669500564059,8.000626706472483,5.666736650973535e-05,5.505151233024589,f19209eb4180 +0.9999391565084456,0.03578734477110683,7.346915872224026,205.29368465904216,178.0,178.0,251.02390324429265,251.02390324429265,31577.583333333332,205.28119387447342,0.07576022245732236,63771.81974509412,2.0195281909928973,31762.0,2.0,2.0,6739938.0,0.03473165340098726,0.0,1215.537389847223,786.0,1.154264395626719,2.5785529878471127,2.0,2.0,784.0,0.0,14.567131143131792,32.135541686272816,6739938.0,0.9951576837173299,205.55237423006494,108.90641459212709,25963.662630804887,421.956629701368,82.52296746565501,78.10444001313468,0.2715303655176358,5.637867662263646,2.614657731878446,28.70773612099371,0.4650481179968973,0.40750877369060634,0.9417624856828257,0.8739722626543377,-0.6929193695098536,0.7482711350604035,0.08566614783006997,8.925103651664791,0.3192237092295557,4.135760503054701,0.9336573903730506,0.3336671816414865,17.850207303329583,3.109769479177945,40.15685186969741,0.19234107682377977,30225.129966626788,0.951612932643624,0.32775455870009457,31608.198350229835,0.3120177306057167,1.3569674453749765,79.80372772495434,80.86650714690511,79.7957595335175,0.9976605820955488,0.01405602704062079,0.22042247471743937,0.012746273825669648,21949.103448999038,0.8710238400037846,8.122541149687546,12.0358131186029,4750.594928852005,4761.95384479597,4750.519007614984,0.9284156026451871,1.5243589585831037,22009.15820894191,0.7863277175828609,0.6955796234494049,1520.056789787467,0.7658766278457064,11.720914040107717,0.695376440407544,3.1481481481481484,0.05829903978052126,50.092249657064464,149.72222222222223,18590296.333333332,18590645.666666668,18590293.7345456,0.07257090545303217,26.074074074074073,0.4828532235939643,0.716820987672768,107.86844135804313,0.0451607738041023,4.896285132226415,0.0017001448271519426,18244334.521261998,0.26777061833469024,0.08472458286295069,3.172981599466828,1.5538012784694093e-05,25.90180143775459,f1dfa95835a7 +0.9999550261522356,0.015964595275674582,4.071007416208657,255.00223124426415,221.0,221.0,311.8348922106056,311.8348922106056,48686.5,254.9907628127366,0.06568539642255836,98164.4721820236,2.0162565019466094,48916.0,2.0,2.0,2857293.0,0.020761731712722747,0.0,1716.7699448880073,473.0,0.4739017983143,2.2371207784773897,2.0,-5.0,478.0,0.0,7.642789890136151,38.34775090691229,2857293.0,0.9971401824911033,53.407527927272056,56.39653040015197,3599.5318020782643,96.39047887109768,28.155685565812632,30.962207467859635,0.18981277926230206,3.5999919916831877,2.3001273699296236,11.487867174333902,0.49248049886335743,0.4257948863793599,0.941586363821677,0.8706672976055021,-0.6233630657820685,0.7293882003926787,0.1221059332060064,6.831838050007643,0.32450637620588363,3.7918655368512404,0.9058285848875621,0.3447351251430581,13.663676100015286,2.5611916222529185,14.779473258418067,0.15457072702161714,47040.37026739717,0.9616561098085937,0.23611476498106723,48776.10916673481,0.07827900852206143,4.114093548123313,80.07163300351623,320.83259056341484,20.016444834047935,0.2497003702870445,0.013329398015645868,0.09564341024522763,0.0032039445810386775,33856.28578973208,0.9007037192482716,2.5852808435292824,8.364796092345147,7845.710204179939,31388.205650108383,1961.4135694782367,0.23848356382376357,1.4885359283644974,33891.11553298421,0.7912852464746108,0.694804908886191,2132.871875501466,0.7486932497998938,7.109559767474602,0.17627315891675993,4.090909090909091,0.09297520661157024,15.34452479338843,67.25,54214624.31818182,216858843.6590909,13553655.197862316,0.06961830246798463,17.727272727272727,0.40289256198347106,0.6544760101105376,35.83048611114922,0.051583298988674614,4.390805482076973,0.0008995011857061084,52978686.78925621,0.5238369162316149,0.03409918452308164,0.4872251037670404,3.5788159835654174e-06,9.18497990360771,f74488b17fd4 +0.9987943387973272,0.08011752569087956,20.60896707709365,257.23419313534464,223.0,223.0,314.66331212901196,314.66331212901196,49679.291666666664,256.9240558486805,0.0651890923051392,100251.94038798186,2.0179824837407643,49954.0,2.0,2.0,4895851.0,0.054244875428966915,0.0,439.9154743872075,336.0,1.1502187978428406,2.577711494575009,2.0,2.0,334.0,0.0,9.899857908661254,19.54194528045253,4895851.0,0.9912935779525484,91.36259006241859,30.24976967978263,446.16254829468744,18.338033279450137,10.12609376711772,12.372397996178874,0.18642641826449136,2.291171239056678,2.0963867489157226,4.557596315736893,0.5436230551571375,0.48238858584467226,0.9495765096935954,0.8793865001721279,-0.3988531567398012,0.6745042650379107,0.21385119678839398,4.901308078751668,0.3195229270189114,4.167985552390869,0.6974713891757834,0.33515110282062505,9.802616157503333,2.599974514581947,5.624622940824149,0.19205869033901432,47768.78672378588,0.9562554895260816,0.38568126789229673,49519.0793930416,0.1293457426883099,1.1723986067181806,79.8529447091324,80.58077431236738,79.83451714979687,0.9958433999364866,0.014854805927626256,0.10459503916442632,0.012825049850722359,34356.096420110174,0.8081231939177477,2.5351668221597334,5.929762601375085,5649.480392479843,5655.145138004485,5649.342007280831,0.8820420299609221,1.5812787564737967,34613.64215999647,0.780396021300622,0.6955984120775729,2735.1877689369608,0.8128493355955275,5.559011260717881,0.6999923706175168,16.106382978723403,0.11422966651576882,8.239424576228561,39.63120567375886,17538822.31205674,17538957.985815603,17538818.927245043,0.1062326390608289,63.95035460992908,0.45354861425481613,0.6909003152116937,28.071266745471743,0.08626896844782654,4.635256117586233,0.002822596789045922,17413305.380514055,1.2790092291762711,0.029954561358476316,0.7931754726320799,1.3824087707672358e-05,3.362981721882907,f96bf216d7b9 +0.9996779120397962,0.07116958860099869,13.444808723213766,188.9122726083189,164.0,164.0,231.22499864850255,231.22499864850255,26867.541666666668,188.851426239777,0.07977938357157202,54376.6024841573,2.023877106390417,27064.0,2.0,2.0,5050533.0,0.07591478121605358,0.0,428.9245533884614,520.0,1.8111217378864557,2.9095846881466154,2.0,-41.0,561.0,0.0,13.660689311173124,18.754747750283233,5050533.0,0.987627852971802,178.14874939890242,67.26473694892735,1832.989654584468,73.60516802482657,19.74548344879937,25.739664011021645,0.1391922148460817,3.18005296502831,2.3767253356355402,10.518262463137141,0.5227037871564885,0.4624362491289518,0.9601868624315509,0.8956655655499078,-0.5059309853349264,0.7364974250274774,0.15977177186654054,7.678478398371012,0.31981004264657553,4.113733777693459,0.7941812926205453,0.33980440599654094,15.356956796742022,2.7454493739274732,11.371286864955255,0.24882242348111078,25464.794191545967,0.9409102199063688,0.5362973821306984,26729.16021282885,0.2588896637995144,9.464454626071534,79.43511676027195,716.5925953295891,8.82384000296539,0.11057961754842793,0.01634744391894111,0.4432119338522678,0.0015169007917563537,18556.982264858983,0.7970428233624083,3.8980986102119806,18.959459377040837,2982.428193367232,26852.364374845038,331.36872878844264,0.09978624852862311,1.5620988096555273,18759.36253000419,0.7846752108879309,0.6968268946542667,1475.6644060661615,0.819579088301694,17.20891087804587,0.07993571031214948,11.123893805309734,0.09844153810008614,13.95817996710784,87.79646017699115,6401727.451327434,57615775.18584071,711302.744648816,0.029597859569553416,64.929203539823,0.574594721591354,0.7845807998838044,71.15339734724955,0.024814988166223483,4.522513675591049,0.004175288205734555,6344365.081838828,0.0029753619485891327,3.2951017806125265,2.6358742408310905,3.1321496764337396e-05,6.890935253922127,f9afc5eead29 +0.9999231811616796,0.0196518462376234,3.8345709531734857,195.12522675005525,169.0,169.0,238.29603437741048,238.29603437741048,28397.0,195.1102374568093,0.07838924171220847,57421.68980196265,2.0221040885291632,28570.0,2.0,2.0,187410.0,0.004837316442880626,0.0,5143.2003865846955,148.0,0.05094649412204617,2.0254812740637034,2.0,2.0,146.0,0.0,2.561186831119353,69.00059154727282,187410.0,0.9993700885604309,2.4571035923164715,8.333599979875732,2.501499658908478,0.25685049005450133,0.6872071908358821,2.309538902724323,0.34067797060280963,0.8333484265338464,0.35152797585509815,0.6865230281138167,0.7603099137172037,0.720860800738679,0.9499005919766617,0.9036826510847339,-0.4880696788435229,0.42208299591094267,0.12286562230104227,2.4405239856111485,0.6725888630784264,1.012397646025856,0.5476190220434477,0.6904610972756773,4.881047971222297,0.5600016004443907,0.7491865233900512,0.16684160997877717,27239.853482674134,0.9534425440207958,0.2276696907999058,28552.00343017151,0.0029743830558696294,1.004725236261813,79.91375568778439,79.9235211760588,79.91327248862443,0.9997184109205461,0.01297214552085629,0.016437318779519223,0.012740958961528322,19842.05367455293,0.9766100475641628,0.10943860935816985,1.1799144595185458,5603.362146556135,5603.542254878285,5603.351415966452,0.9892771648218711,1.2101598800266467,19804.106674516424,0.8323928222121041,0.6951266794108937,590.3266660394175,0.7055488555030525,0.8854148493635561,0.6948279589512455,1.6666666666666667,0.18518518518518517,2.2222222222222223,13.333333333333334,90636748.0,90636768.33333333,90636746.89277777,0.2127777777777778,5.666666666666667,0.6296296296296297,0.8055555556917661,11.138888889025099,0.09833333346954386,2.503258334775644,0.0003150157507875394,80559650.46913579,8.356734524284718e-05,1197.067039106145,0.003935886323670904,9.045881653988449e-08,8.438608631198823,f9e71e8fdba7 +0.9994493631475545,0.06783971933560791,12.364202630878331,182.25609940559661,158.0,158.0,222.73975846265077,222.73975846265077,24873.666666666668,182.1557424806809,0.08190330549372282,50312.50044911622,2.0227215039646835,25042.0,2.0,2.0,2259932.0,0.040760185961855135,0.0,620.9834497385297,354.0,0.9197362997690434,2.4613050075872533,2.0,2.0,352.0,0.0,9.499771959314145,23.446380353823887,2259932.0,0.9937812995850543,84.18764493859726,37.681809096657545,661.05518489684,11.613378476626904,12.850774490743751,16.108970671881572,0.17786690143753686,2.7189100398627963,2.02937128548362,4.322629733896914,0.5188669458060192,0.4539386459249134,0.9420401010394374,0.8678665066421125,-0.5686789948318788,0.7335079720945508,0.16834839595742684,5.44848533133828,0.325959471838982,3.710307721151245,0.8543408451061616,0.3441775421601237,10.89697066267656,2.3969014043774726,7.239936290656333,0.21494748002938854,23665.55195271943,0.9450344202827023,0.37654641328675964,24886.27130420893,0.1194267602155876,1.15390144557144,79.62934270425684,80.31610893698587,79.61649242677254,0.9970161221755238,0.014060478396294226,0.07709724329792615,0.012766915501172198,17287.04995328893,0.8621606566731922,2.487850132579555,5.024585518052002,3653.153920717543,3657.6498651782113,3653.0655870908386,0.9225584395093444,1.4567275104749844,17353.949374197222,0.784107087226797,0.6960613861021177,1225.6995839907993,0.7695707503746267,4.676804718673671,0.694848816257526,5.133333333333334,0.11407407407407408,8.75358024691358,46.644444444444446,13848921.688888889,13849128.977777777,13848917.841466324,0.06780063889584925,15.8,0.3511111111111111,0.5826790123813372,26.124129629665287,0.031015977150413827,4.50597469684966,0.001796981071799377,13539241.805432098,0.0024789925621010373,15.494711511133561,0.5532662603322366,1.040476128227384e-05,8.507556771318315,fd95f8f7e74b diff --git a/predict2/final_merged.csv b/predict2/final_merged.csv new file mode 100644 index 0000000..9b99331 --- /dev/null +++ b/predict2/final_merged.csv @@ -0,0 +1,25 @@ +original_shape_Elongation,original_shape_Flatness,original_shape_LeastAxisLength,original_shape_MajorAxisLength,original_shape_Maximum2DDiameterColumn,original_shape_Maximum2DDiameterRow,original_shape_Maximum2DDiameterSlice,original_shape_Maximum3DDiameter,original_shape_MeshVolume,original_shape_MinorAxisLength,original_shape_Sphericity,original_shape_SurfaceArea,original_shape_SurfaceVolumeRatio,original_shape_VoxelVolume,original_firstorder_10Percentile,original_firstorder_90Percentile,original_firstorder_Energy,original_firstorder_Entropy,original_firstorder_InterquartileRange,original_firstorder_Kurtosis,original_firstorder_Maximum,original_firstorder_MeanAbsoluteDeviation,original_firstorder_Mean,original_firstorder_Median,original_firstorder_Minimum,original_firstorder_Range,original_firstorder_RobustMeanAbsoluteDeviation,original_firstorder_RootMeanSquared,original_firstorder_Skewness,original_firstorder_TotalEnergy,original_firstorder_Uniformity,original_firstorder_Variance,original_glcm_Autocorrelation,original_glcm_ClusterProminence,original_glcm_ClusterShade,original_glcm_ClusterTendency,original_glcm_Contrast,original_glcm_Correlation,original_glcm_DifferenceAverage,original_glcm_DifferenceEntropy,original_glcm_DifferenceVariance,original_glcm_Id,original_glcm_Idm,original_glcm_Idmn,original_glcm_Idn,original_glcm_Imc1,original_glcm_Imc2,original_glcm_InverseVariance,original_glcm_JointAverage,original_glcm_JointEnergy,original_glcm_JointEntropy,original_glcm_MCC,original_glcm_MaximumProbability,original_glcm_SumAverage,original_glcm_SumEntropy,original_glcm_SumSquares,original_gldm_DependenceEntropy,original_gldm_DependenceNonUniformity,original_gldm_DependenceNonUniformityNormalized,original_gldm_DependenceVariance,original_gldm_GrayLevelNonUniformity,original_gldm_GrayLevelVariance,original_gldm_HighGrayLevelEmphasis,original_gldm_LargeDependenceEmphasis,original_gldm_LargeDependenceHighGrayLevelEmphasis,original_gldm_LargeDependenceLowGrayLevelEmphasis,original_gldm_LowGrayLevelEmphasis,original_gldm_SmallDependenceEmphasis,original_gldm_SmallDependenceHighGrayLevelEmphasis,original_gldm_SmallDependenceLowGrayLevelEmphasis,original_glrlm_GrayLevelNonUniformity,original_glrlm_GrayLevelNonUniformityNormalized,original_glrlm_GrayLevelVariance,original_glrlm_HighGrayLevelRunEmphasis,original_glrlm_LongRunEmphasis,original_glrlm_LongRunHighGrayLevelEmphasis,original_glrlm_LongRunLowGrayLevelEmphasis,original_glrlm_LowGrayLevelRunEmphasis,original_glrlm_RunEntropy,original_glrlm_RunLengthNonUniformity,original_glrlm_RunLengthNonUniformityNormalized,original_glrlm_RunPercentage,original_glrlm_RunVariance,original_glrlm_ShortRunEmphasis,original_glrlm_ShortRunHighGrayLevelEmphasis,original_glrlm_ShortRunLowGrayLevelEmphasis,original_glszm_GrayLevelNonUniformity,original_glszm_GrayLevelNonUniformityNormalized,original_glszm_GrayLevelVariance,original_glszm_HighGrayLevelZoneEmphasis,original_glszm_LargeAreaEmphasis,original_glszm_LargeAreaHighGrayLevelEmphasis,original_glszm_LargeAreaLowGrayLevelEmphasis,original_glszm_LowGrayLevelZoneEmphasis,original_glszm_SizeZoneNonUniformity,original_glszm_SizeZoneNonUniformityNormalized,original_glszm_SmallAreaEmphasis,original_glszm_SmallAreaHighGrayLevelEmphasis,original_glszm_SmallAreaLowGrayLevelEmphasis,original_glszm_ZoneEntropy,original_glszm_ZonePercentage,original_glszm_ZoneVariance,original_ngtdm_Busyness,original_ngtdm_Coarseness,original_ngtdm_Complexity,original_ngtdm_Contrast,original_ngtdm_Strength,scan_id,patient_id,agatston_score,calcium_volume,lesion_count,agatston_category +0.9993625929219436,0.046860553771808,12.37860976574501,264.1584183154097,229.0,229.0,323.1485726411305,323.1485726411305,52368.333333333336,263.9900418698473,0.0641571628373409,105507.85870727764,2.014726304839648,52615.0,2.0,2.0,11567286.0,0.0448832903943482,0.0,1122.9115614899351,877.0,1.2960630704725395,2.6456333745129714,2.0,-44.0,921.0,0.0,14.827261582001467,30.341709808443028,11567286.0,0.9935869555823512,212.84831006875973,119.66064285057824,44565.17466108682,923.6868317496888,94.22037163258454,57.523404260213866,0.3985866149968575,4.715474468153125,2.745191803162805,23.409901662799165,0.4789342288137195,0.4178459396104712,0.966731330000424,0.9042603414328696,-0.5630025406437277,0.7501733837735425,0.1089352674156547,9.555595223572142,0.3130219734446293,4.788767501043632,0.8486742799574394,0.3240319725029282,19.11119044714429,3.4149106259035658,37.9359439731996,0.1770045885111812,50481.28459564763,0.959446633006702,0.3229502832877294,52277.57766796541,0.3215665658050146,9.46961892996294,79.96504799011689,721.2328803573125,8.885385293077938,0.1109114131409939,0.0143406969800425,0.4442770122588615,0.0014435874092703,36270.48672928581,0.8283611848442824,8.602454622105002,24.69113861987565,6662.148270093807,59976.1099572755,740.232524134027,0.1041323520134706,1.6658465853022029,36452.15062147034,0.7796910958821593,0.6952214562971951,2825.506381595789,0.7954992680077096,22.57779770381845,0.0814027208636382,6.339285714285714,0.0566007653061224,47.72313456632653,171.6875,24554097.08035714,220987299.875,2728232.913817201,0.0325738362215482,54.0,0.4821428571428571,0.7176661706381673,142.60880208336258,0.0167701524082816,5.317836954365818,0.0021286705312173,24333406.8519611,0.2461680104049699,0.0260411327497175,4.884164200188554,1.2485436319994676e-05,17.454353422243575,5b8eb0dfaafb,20,1.46,0.41,6,Minimal +0.9989270127717552,0.0772500123137252,17.668238327643902,228.7150228001288,199.0,199.0,280.72228269234347,280.72228269234347,39828.875,228.46961450175647,0.0703567800701145,80163.5353537984,2.0126989615900124,40066.0,2.0,2.0,26984272.0,0.137992390503385,0.0,211.8958769190301,711.0,4.694394233541013,4.374082763440324,2.0,-18.0,729.0,0.0,25.95179246933797,13.331934837284162,26984272.0,0.9771292370334418,654.3629323501611,96.72694379872064,11779.746046015682,275.1273083482816,50.34204008730576,29.197327092070868,0.4537888720206611,3.442730678629307,2.565056949490565,11.55950282339838,0.4997703603975673,0.4351379041761118,0.97206740220513,0.909085254184057,-0.3798394471698749,0.7492937154798709,0.1425082915464874,8.530080045723428,0.307496605770526,5.301381666929735,0.6725944533124836,0.3177228808553251,17.06016009144685,3.429074271840662,19.88484179484416,0.2983677612241394,37608.60095841861,0.9386662246897272,0.7394675488882668,39149.66001098188,0.9752935242583278,5.344756152348625,79.22457944391753,322.76316577646884,19.793355505263538,0.2473900533310059,0.0180722741119128,0.8048152130774344,0.0032865909272608,27158.209949586093,0.7350423937607047,9.624765823322688,26.73134334884942,2875.367145432649,11527.563148038114,718.7904147787232,0.206242216291978,1.7677595102596189,27791.87971534655,0.8102206658716457,0.6977237558029251,1910.760547450922,0.8716167457962775,25.37765621628992,0.176045508850593,17.18978102189781,0.0627364270872182,29.660250945708345,129.29562043795622,5723505.2919708025,22894523.098540142,1430875.2865734927,0.0282685323517203,120.007299270073,0.4379828440513613,0.684685223169634,96.69430982017512,0.0204020231040699,5.5720185419486326,0.0068387161184046,5702123.167776654,0.2444047153819084,0.0361256839882415,13.803974343943468,8.195815144848356e-05,10.386027749033996,66dbc7756f54,3,4.71,1.29,10,Minimal +0.9999157212195828,0.0483166125937192,10.700458628469676,221.46541435855215,192.0,192.0,270.8228203087768,270.8228203087768,36743.0,221.44674952352543,0.072029746248315,74203.0884240639,2.0195163275743377,36957.0,2.0,2.0,3219313.0,0.0340444167022229,0.0,1278.106608576696,582.0,0.7630398860896919,2.3822550531699003,2.0,-6.0,588.0,0.0,9.333257472662222,31.732928395729388,3219313.0,0.9950280851424352,81.43455591265169,59.40707601131984,6182.432442028242,269.9536935126543,33.0957757671301,28.457947709388197,0.3058301226631647,3.3463737101026605,2.3070939588472363,11.839017646144756,0.5051785274906953,0.4386114316992184,0.9631464519046036,0.8978011168894974,-0.6154967926578251,0.7382236896506656,0.1332953698999468,6.896050333930286,0.3232356907489774,3.865461048515882,0.9303784779794236,0.34010738979679,13.792100667860574,2.627108878946954,15.388430869129571,0.18338162152333,35278.584273615284,0.954584632779048,0.3128490512953439,36773.252942608975,0.120176707140533,4.1778553454014125,79.87469220986551,320.1177314175934,19.966236055209794,0.2494729518617079,0.0140573086773851,0.1617589043255809,0.0032382152286375,25529.22187027537,0.8658250855475479,3.176786552244525,9.671898720033868,5437.2753036465865,21755.42633743126,1359.299372083067,0.2335117861318827,1.5108405288110505,25607.44415944402,0.7844053910396356,0.6953944396918664,1798.9989640786748,0.7677734644287624,8.589596121655443,0.1761277394230209,5.548387096774194,0.0894901144640998,19.749479708636837,77.95161290322581,21918624.629032254,87674710.59677419,5479655.342441966,0.0534872695614224,32.38709677419355,0.522372528616025,0.7495270808561539,59.0252016129507,0.0450227666788886,4.536868842952556,0.0016776253483778,21563312.49349636,0.0760201936168288,0.1825635977076956,1.29507755861101,5.541055692273879e-06,14.823293939470554,69ebf706870a,9,0.72,0.23,4,Minimal +0.990236834207065,0.1946189980221578,46.068804246658544,236.7127809455349,205.0,205.0,289.2075379377239,289.2075379377239,42619.0,234.40171481985692,0.0688170624725531,85741.31981983998,2.011809751984795,42916.0,2.0,2.0,42804618.0,0.223636032938136,0.0,167.0094056472295,895.0,7.310681720160203,5.728026843135427,2.0,-38.0,933.0,0.0,31.581713869675188,11.25503977676445,42804618.0,0.9600346611072308,964.5943594303544,96.19347434988596,29258.14392899947,779.8434387815064,54.5475679998829,30.04090710008376,0.4496534738441542,3.405842942061433,2.599053536511404,12.923925535323333,0.5054604688263639,0.4410945555387714,0.9815302447115232,0.926251497848665,-0.31812160772865,0.7200643575649829,0.1431751448137372,8.71449795845187,0.3048059178996843,5.436141019343809,0.7532483151287382,0.3163162433079602,17.428995916903727,3.384904875466193,21.147118774991675,0.4022530000641705,39591.022369279526,0.9225235895535352,1.1700440988311491,41200.84751607792,1.4297179216747014,11.293526889738091,78.56953117718334,717.7217121819368,8.719682962252977,0.1093089261542315,0.0230073354609463,1.53296066087758,0.0017175346648685,28588.03619659204,0.7029880744574354,9.250651501838329,36.01526066874179,1968.092422685344,17743.0564022592,218.65240712491573,0.0902348468047089,1.7790302569766776,29833.1011041448,0.8513444095358332,0.7000885450647777,1530.6349016873585,0.9115567035723516,34.45850909573031,0.0812484678470384,33.445283018867926,0.0631043075827696,32.015108579565684,130.28679245283018,3332270.3056603773,29990922.266037736,370251.7916625332,0.0302618844919578,267.0452830188679,0.5038590245639017,0.7379264925036605,102.26474197835029,0.0256148735388061,5.618568150914065,0.0123497064032062,3325713.584207903,0.8662554156083611,0.0063397070256306,35.18842550930922,0.0001997487688033,6.773706347156496,6b1dee6a31e8,14,8.07,2.31,30,Minimal +0.9950565618660016,0.2132192597845272,48.76270635186346,228.69747508335564,201.0,201.0,283.5507009337131,283.5507009337131,41801.208333333336,227.56692326387943,0.0697261739012751,83537.3793825111,1.998444129087443,42168.0,2.0,2.0,113152826.0,0.4226484789251693,0.0,76.78146053745364,928.0,16.543990949051274,10.610083475621323,2.0,-118.0,1046.0,0.0,51.801364646550816,7.791371520709379,113152826.0,0.9194728368059234,2570.807507885273,197.2121561473841,42264.972146976856,1066.4688151367184,85.07973097490802,35.42698257399276,0.5373195197960164,3.724624339569652,2.73265275244155,15.088994395814298,0.4932035041263862,0.4294705961154446,0.9826956147791404,0.9282325106144886,-0.2917929281783732,0.7488715027045452,0.1412665666851486,12.854574342842565,0.2945889209191946,5.941331718839267,0.6555541282261588,0.3089224415707909,25.70914868568514,3.706955037278159,30.12667838722519,0.6415197226967717,37228.26935116676,0.8828559417370224,2.144820227631272,38772.33058243218,3.8585900135398994,43.89914152912161,76.96518687156137,2805.860344336938,2.1337687518589266,0.0270178540347319,0.032803818876822,5.583192595534071,0.0005439655070953,26925.13704880726,0.6616551490289967,13.904289556055044,91.08001103970489,1069.1961697853112,38553.51434638951,29.69379362517119,0.0225189044897243,1.9067949309111467,29476.22310531032,0.8931094529920472,0.7057739737022604,927.18613745559,0.943019502033523,87.26872087532774,0.0211320803459939,60.542307692307695,0.0582137573964497,41.38801682692308,241.96442307692308,1569467.0365384617,56501612.67596154,43596.21549418869,0.0101558082753426,482.5980769230769,0.4640366124260355,0.7074141975042921,181.67375906042076,0.0071506651389832,5.925260783478826,0.0246632517548852,1567823.0459467457,0.5143714373360457,0.0041181753655611,107.51434416812911,0.000660825909211,5.270615799233856,70a5b54337ec,13,16.7,4.62,44,Mild +0.9998582268614304,0.0443365795445968,9.258001439679711,208.81181035554096,181.0,181.0,255.266527378738,255.266527378738,32620.5,208.78220644981647,0.0748332458158519,65975.5902204224,2.022519281446403,32833.0,2.0,2.0,1100657.0,0.0283856478529993,0.0,847.4148572371255,263.0,0.4544576230974559,2.2277282002862973,2.0,2.0,261.0,0.0,5.789895383963707,27.24506453416987,1100657.0,0.995680509682262,28.56011562289341,21.555855159698943,126.85470033230348,6.962026003155013,5.8237378624423375,7.878076039187341,0.1283277086705322,1.9327092422869927,1.6543016467436025,2.418977263611483,0.5592516390124834,0.4916484335491019,0.9459874024994452,0.8699724180977009,-0.6579037760490845,0.7071637228087615,0.1817204383469705,4.180693409312345,0.3487819482126729,2.970225493495517,0.9090780217670758,0.3765055744532015,8.361386818624691,1.8184798256426744,3.42545347540742,0.1846539712883157,31283.779459689948,0.9528151390275011,0.3109966498522533,32691.178174397708,0.039282022883272,1.0559193494350196,79.8408004142174,80.03067036213565,79.83358669136132,0.9979868914255038,0.0139753486875031,0.0477301330000342,0.0127975236226822,22701.29038370507,0.882571089544673,1.0153700708131956,2.755708637124329,5081.357587188288,5083.336141496774,5081.285425602373,0.9372607156617992,1.4223361372659606,22752.347641386597,0.7885726342160511,0.6954635228627859,1504.2143264639474,0.7578905974988975,2.4578878167826943,0.6975068885547011,7.423076923076923,0.1427514792899408,4.736686390532545,26.03846153846154,20640062.480769232,20640125.519230768,20640060.184897754,0.1129382662891861,27.192307692307693,0.5229289940828402,0.7501335470264647,19.9337606837786,0.0823779109327604,3.985048189755741,0.0015837724240855,20241391.66383136,0.184631142894257,0.2705761737934561,0.1806286187625946,2.776839222555751e-06,6.024493853719077,8234fbd8de27,4,0.27,0.15,10,Minimal +0.9999699433766066,0.01036144976175,2.7867565275194663,268.9543057774561,233.0,233.0,328.80541358073776,328.80541358073776,54089.0,268.94622191917733,0.0634385915041246,109027.62723991455,2.015707948749553,54328.0,2.0,2.0,920684.0,0.0107008013254219,0.0,2611.92749264768,264.0,0.171942450065952,2.086032984832867,2.0,2.0,262.0,0.0,4.116645209498442,48.22234422830545,920684.0,0.9985648657724135,12.595234167075745,26.29668090235278,129.49015647752708,8.663801340809117,6.571363754745481,9.40615502277632,0.159518243809279,2.0636261920666024,1.7662809167733005,3.176520898082416,0.5610921382559136,0.4964595451270641,0.9382268778432878,0.8649836793186988,-0.6615446746784268,0.7102317473826683,0.1766884838560717,4.604582990398822,0.3474013424820231,3.0082301551108106,0.9406691383165928,0.3945379834754347,9.209165980797644,1.9923091253370837,3.9943796943804495,0.1367923183317627,52427.03721837726,0.9650095202911438,0.1854024977968862,54250.0320276837,0.0176053189746249,1.0239103224856427,80.17736710351936,80.33295906346635,80.17197767848667,0.99932820499515,0.0129494152800274,0.0246436511150378,0.012631831761989,37656.36916825702,0.9405224729286578,0.7154017799519005,2.058291753043101,9869.533088041822,9870.945867869828,9869.49262573483,0.9703997467095968,1.3697259378567843,37641.19181905416,0.8129827374357296,0.6944683574414101,1702.0895929748626,0.7224727043281663,1.7118480302471903,0.6947997407982935,2.9130434782608696,0.1266540642722117,7.323251417769377,35.0,128143289.47826087,128143454.47826087,128143283.83069056,0.1166196216645586,10.652173913043478,0.4631379962192816,0.7037372517592501,24.461788781549917,0.0621826743081687,3.653996738665704,0.0004233544396996,122563834.68809076,0.0003066847908282,163.01223913235947,0.0557609063024007,4.356909546109949e-07,10.40872788144648,8322c647539e,16,0.26,0.09,2,Minimal +0.999833039682875,0.015444271211463,4.333679164035027,280.60107885301085,243.0,243.0,342.94751785076386,342.94751785076386,58817.583333333336,280.5542296078999,0.0617048787837413,118532.12544355595,2.01524984071185,59064.0,2.0,2.0,444228.0,0.003924182389155,0.0,5124.9231066335215,188.0,0.056872996890459,2.028443722064201,2.0,2.0,186.0,0.0,2.7424677978245864,69.33497257764073,444228.0,0.9994921578007502,3.406545688523165,18.394548843262616,4.245555076503317,0.5257230474650177,0.7119063839613826,2.7000463926763087,-0.0358731262496675,1.0212399641674217,0.7895611963264776,0.8375211369425835,0.6850897937844361,0.6444264361682474,0.9637741936945684,0.9028866041489252,-0.7239521410222185,0.6041427688907951,0.2562021305277092,3.792521697578238,0.5096000570549085,1.501090246554147,0.8344138207643387,0.5439161886508929,7.585043395156475,0.7889489205017725,0.8529881941594227,0.1253438376958574,57130.09535419206,0.9672574724737922,0.1588749307014153,59034.00480834349,0.0045533150572562,1.006619937694704,80.24173777597183,80.27476974129758,80.24061280160315,0.999757683990624,0.0127272618486245,0.015765279930705,0.0126051367903073,39473.17502100088,0.972336528549144,0.2430916820275838,1.3689112257149392,12494.883765471155,12495.310894941713,12494.86827994448,0.9864218299805532,1.397978323298238,39413.57573838198,0.81693896811505,0.6687957131247461,1365.163605637803,0.6813463745126209,1.035703039017284,0.6682450436577628,2.333333333333333,0.2592592592592592,3.333333333333333,22.11111111111111,387420494.2222222,387420637.6666667,387420489.2023534,0.1605015432098765,4.333333333333333,0.4814814814814814,0.6780555555874219,15.5833333333652,0.0422302469454466,2.281036112553421,0.0001523770824867,344351900.4444444,9.269900956519828e-05,1078.8734276558605,0.0028198154663723,8.580929932917387e-08,12.336125263273882,9edcb930e6ab,0,0.06,0.03,1,Minimal +0.9824913544136816,0.5042474170531516,98.64034908358404,195.6189476587573,198.31288409984865,179.61347388211163,242.53865671269807,242.53865671269807,31407.375,192.19392483423155,0.0776516382118105,61994.70465658467,1.9738900387754363,31690.0,2.0,2.0,264517584.0,0.6677753528218292,0.0,47.81200720571428,1350.0,35.38701228501375,20.862985168822973,2.0,-333.0,1683.0,0.0,91.36211695923512,6.134328748369493,264517584.0,0.8726455393200998,7911.772265118429,656.8035690673798,162468.14685516342,2791.261270458418,172.1332433917403,67.48290388429312,0.5470124961270595,5.16312895567581,3.090489551939622,28.74400437370021,0.4585219792292975,0.3967448635227037,0.9868388306305916,0.936539462756261,-0.2918469220905466,0.8122953558932254,0.106564745057949,24.408684251220887,0.2824093236991598,6.596190696971028,0.6525828731565908,0.2995686322618593,48.8173685024417,4.137862076692862,59.90403681900837,0.9359571010162764,26418.59804354686,0.8336572434063383,3.365018321521771,27654.13714105396,12.116265802119596,259.47898390659515,74.94761754496686,16995.56443673083,0.3323704117971879,0.0043084515428584,0.0493748343554548,32.459576724615374,0.0001564475658287,19218.60814603376,0.622042740936994,31.34111904336832,390.9067554293078,579.4051439692975,130549.88657060068,2.574388447248067,0.0037996778694657,2.164647744540887,22326.973964772355,0.9153870237164444,0.7130519212564023,518.8967253786365,0.956640272022472,376.9343934544145,0.0036303589569201,54.62979351032448,0.0402874583409472,82.88913851689422,788.4653392330383,645441.4985250738,145226135.69100294,2868.619283665048,0.0026554049657201,745.7330383480826,0.5499506182507984,0.7701002314145036,623.8314837130974,0.0022185290514266,6.2307569095634845,0.0427895235089933,644895.3319737037,0.086652185181034,0.0069006071484479,473.9173963989284,0.002291807306759,8.046515891280338,a72e38ef4265,5,22.99,6.03,41,Mild +0.9999639083818777,0.0148871166144084,3.1964693144957117,214.7137956454255,186.0,186.0,262.33756879257686,262.33756879257686,34424.75,214.70604627710745,0.0735877627241445,69543.90208351391,2.0201715940860545,34617.0,2.0,2.0,373460.0,0.0088677901242404,0.0,2948.03296993358,182.0,0.1139788251947613,2.057024005546408,2.0,2.0,180.0,0.0,3.284561003929086,51.79655290480338,373460.0,0.9987871605590136,6.55699322913746,10.818575313876,24.5541780846222,2.822060073894849,1.7757257190285125,3.089098200446632,-0.0470247674891766,1.2696395830014378,0.8217451819922063,0.6923015724743963,0.6009759098151165,0.5402439487514571,0.9572626026785352,0.8773542486208588,-0.7811724919566841,0.6819336829957597,0.266709853955937,3.028695642933211,0.4405971658356779,1.810055478729303,0.9429564998070836,0.4641471302648741,6.057391285866422,1.008482400340475,1.216205979868786,0.1592683793128181,33127.38275991565,0.9569686211952408,0.2176312670670471,34575.01513707138,0.0087199483216356,1.012768293035214,79.98997602334113,80.04832885576451,79.98696342801306,0.999446807114484,0.0128743031772371,0.0167500482350604,0.0126878757718455,24017.257774208723,0.9582083812602536,0.3184689194998676,1.4904848107489097,6541.64815734587,6542.215473485406,6541.621219386483,0.9799288398574147,1.278053190044749,23991.20010533836,0.8222660521818781,0.6949409027578715,904.6962141845022,0.7130378500527184,1.1843148286050764,0.6946833897214534,2.1666666666666665,0.1805555555555555,3.5763888888888893,20.25,99740271.91666669,99740348.25,99740268.38532124,0.1572388747165533,3.5,0.2916666666666667,0.4444444445140696,7.8310185185881425,0.0352748679257183,3.0849625007211534,0.0003466504896438,91418489.35416666,0.000281549829181,296.1632790625324,0.0130805623362262,3.655712523053076e-07,7.544671137554932,ae676e3b5447,23,0.06,0.04,3,Minimal +0.999981911804411,0.0124207305606518,2.5379716721141623,204.3335260934086,177.0,177.0,249.6096953245206,249.6096953245206,31164.08333333333,204.32983006862324,0.0760440454997026,62977.94463595891,2.020850219220061,31344.0,2.0,2.0,732294.0,0.0074948597623952,0.0,3023.335259507712,317.0,0.1775566416281085,2.088820826952527,2.0,2.0,315.0,0.0,4.833542355234601,53.39074916948603,732294.0,0.999043135522468,18.999959252736232,52.71826054184746,266.7690322217189,-10.272049681791318,7.127979767165673,13.302001544446798,-0.0403602064479019,2.542473447756901,1.220117204882325,3.635356135394718,0.5246340460546844,0.4568523164036228,0.93601044116825,0.8585556278936566,-0.918180251894946,0.7114748621766153,0.1500988611583175,6.406205827255591,0.4139517743257788,2.042712182086753,0.9999999531094832,0.4339149828117669,12.812411654511184,1.1499108608674409,5.107495327903117,0.1637434262054088,29938.13138080653,0.9551471216439044,0.227874219483068,31314.00803981623,0.0271074185165468,1.0337544665645737,79.94888973966309,80.01837672281776,79.94798448088746,0.999535883404498,0.0130931589876417,0.0407504104197782,0.012704316653034,21756.31267342562,0.9641050577688808,0.9974676065669072,2.3130400675537923,5989.525098337448,5990.839316437638,5989.507029226284,0.9819454324477864,1.263987665857219,21724.92831249252,0.8251082456201018,0.6950514391172888,772.6546326529686,0.7119693108151484,2.0247148702098383,0.6939183791661332,1.4285714285714286,0.1020408163265306,12.061224489795917,65.14285714285714,70107590.14285715,70107700.0,70107588.68048596,0.1020027194686166,10.428571428571429,0.7448979591836735,0.8650793651521395,59.92857142864421,0.0297902954771151,3.378783493486172,0.0004466564573762,65095108.8367347,0.000104162124175,479.9321244473511,0.0545300610147952,6.327359113922631e-07,31.45245541354373,b39c18fd990e,21,0.19,0.05,1,Minimal +0.9796621769628844,0.3147385120013255,75.20966502050277,238.95920630197944,209.0,209.0,294.8643756034289,294.8643756034289,46173.208333333336,234.0992962511202,0.0687033187684312,90594.07108529742,1.9620484336128703,46498.0,2.0,2.0,166305119.0,0.5749792973960035,0.0,32.35719093911723,714.0,24.33157605726459,14.935932728289387,2.0,-44.0,758.0,0.0,59.804749014149046,5.171408430434639,166305119.0,0.883597191602844,3353.525918181376,116.10787244759972,10166.461255542945,279.1587705689161,43.983040071545325,18.715594482194582,0.5241026051699491,2.786951627574392,2.491461853233752,7.64883254757722,0.5178409356292972,0.451510576328275,0.9822998218259728,0.9253098975310304,-0.2430071307322802,0.6590002377733867,0.1674888936332582,9.624499654824875,0.2831354158504603,5.766288202817564,0.5934566330864828,0.3014256230416222,19.248999309649754,3.528100714612909,15.674658638434972,0.8456941534851843,39543.48135403673,0.8504340262814902,2.5518755475288883,41085.50221514904,4.992783486533639,17.196309518688977,75.83767043743816,756.1167792163104,8.371260531716903,0.1053556966428768,0.0333156518408489,3.176232338163376,0.0018212102649763,28585.36977367777,0.6357736137937363,10.9552239631419,51.70181482936835,828.5777619362452,7513.12986898298,92.0242685402825,0.0813787909545323,1.981392852927936,32505.274946512454,0.8929182377307664,0.7092033735115156,746.1010667884829,0.945932789850512,48.44973235803278,0.0774667797636111,71.39093959731544,0.0598917278500968,27.298899120535115,144.9085570469799,1600712.312080537,14408563.954697989,177855.29282049809,0.0221786579234796,449.2181208053692,0.3768608396018197,0.63663444350025,90.50492700154484,0.0169774002226885,6.215686930664766,0.025635511204783,1599190.6576477408,0.8486355617416927,0.0064879244123859,60.33225592192009,0.0016985209614257,1.9695045785321132,c4f0b2426103,22,30.1,8.26,24,Mild +0.9999286309889948,0.0153344580187477,4.158332555282335,271.1757109510101,235.0,235.0,331.6338342208165,331.6338342208165,55058.625,271.1563574087109,0.0630622080130479,110985.23160652645,2.015764680039257,55306.0,2.0,2.0,2210070.0,0.0205455156088096,0.0,1567.329172212471,382.0,0.3972764872914399,2.198929591726033,2.0,2.0,380.0,0.0,6.32145266143538,36.61407723085877,2210070.0,0.9971093415645064,35.12547240140003,33.81070275415835,919.4560048946806,34.759750309298674,15.27786448145144,16.784068220174184,0.216877147772692,2.667082077930242,2.0805951392868494,6.306718817503356,0.5290252614740006,0.4646734170392987,0.9477990149357646,0.8778368693459627,-0.5947820897714409,0.7186019460373282,0.1710462756784714,5.145857610290053,0.3278800659678124,3.6255970748777098,0.8924183464455311,0.3429488668285002,10.291715220580103,2.470714781114874,8.015483175406407,0.147236407812788,53306.44284526091,0.9638455654948996,0.222140044233363,55146.1292445666,0.0500850696822841,1.0648573391675409,80.12646005858315,80.40865367229596,80.11874780730514,0.9986299009464114,0.0133431948535041,0.0553321040335408,0.0126465022485169,38270.23174721987,0.895037009659246,1.8010344383332024,3.726680034633075,8873.58946347513,8876.658660818985,8873.5211906035,0.9440338948432292,1.483683934715078,38315.491581057366,0.7916124742029031,0.6946568601542745,2407.166246783225,0.7491266119696854,3.40020292132567,0.695932262534718,5.03921568627451,0.0988081507112648,12.336793540945791,51.21568627450981,59800016.078431375,59800163.74509804,59800012.4578967,0.0903709174530438,24.37254901960784,0.4778931180315263,0.7183769063245119,42.85078431373192,0.0544494697480153,4.443581528970368,0.0009221422630456,58624024.67666282,0.2100958217694009,0.1829925862758891,0.2781041486287365,1.9222299016827763e-06,9.31178219613705,c7c24b912e3f,15,0.61,0.2,4,Minimal +0.9987180667376035,0.1147675568427293,25.868629714004843,225.4001951915181,196.0,196.0,276.4796556710819,276.4796556710819,38601.41666666666,225.1112471839515,0.0710345544591087,77758.87332559821,2.0144046524785977,38839.0,2.0,2.0,29815138.0,0.1299110118192883,0.0,310.2565504561779,911.0,4.524959290798204,4.287031077010222,2.0,-6.0,917.0,0.0,27.70667383444753,16.016154341540304,29815138.0,0.9789551812418024,749.2811395132081,115.6914055148982,45458.36439323794,1190.74041437451,98.60703012037742,37.99325539296469,0.5736300134477192,3.85023186052009,2.6773633709852414,16.05045742078254,0.4965922534369328,0.4336038692272429,0.9768403057167186,0.9184255874671148,-0.4401109730988419,0.7746008055965745,0.1285543161155843,8.91660720982449,0.3081703098646046,5.32741462105216,0.764389916556381,0.3212149443256379,17.83321441964898,3.569239558723302,34.15007137833554,0.2940862642788758,36487.44177244522,0.9394536875935328,0.7126207969257836,38021.64028425037,1.1335631273261926,5.4901001570586265,79.26947140760576,322.8114524060867,19.80451358055877,0.2476218853303751,0.0181896384166715,0.9567265323546812,0.0033360030203833,26379.41824444245,0.7429955581877891,12.83079551366936,29.92251703789453,2906.4182242402094,11655.106807521252,726.5572411351711,0.2087775785813471,1.7886181386735462,26936.85282333672,0.8085965837257818,0.6975680669905547,1891.820211210411,0.868228744247821,28.528600715893635,0.1773167029638975,14.423076923076923,0.055473372781065,47.32479289940829,156.6076923076923,5676116.088461539,22704928.39230769,1419028.056751405,0.0356807032896303,128.62307692307692,0.4947041420118343,0.7312103780284605,120.77064036669842,0.0273512281120915,5.671268487626294,0.006694302119004,5653801.474245562,0.5041108492551972,0.0153953769510897,19.642023207030384,7.650290878770294e-05,15.638908919368255,d11d0c292b1b,1,3.7,1.1,16,Minimal +0.9995850132556447,0.1231157555480808,23.69213569008709,192.43788566796803,167.0,167.0,235.46761985462035,235.46761985462035,27857.58333333333,192.3580264963041,0.0788382081727038,56369.3471409852,2.023483030329331,28057.0,2.0,2.0,6304613.0,0.0741506283416783,0.0,397.278996390419,497.0,1.9596891977685196,2.985030473678583,2.0,-17.0,514.0,0.0,14.990240495692088,18.34831536603996,6304613.0,0.9882051701762748,215.79690318989717,66.5004467226661,2948.598492568611,87.07978904522471,28.355980127233497,24.704711885612564,0.3093885050313243,3.227761152116264,2.3860012392544325,9.38358694932178,0.5093715294669894,0.4448819913109451,0.9544107214230472,0.8851377294065051,-0.5402158744898002,0.7476402995914998,0.1407115355646002,7.289347343648289,0.3174342356948883,4.2444313551249495,0.8148172878545517,0.3309743208733543,14.578694687296576,2.8904768257422373,13.265173003211514,0.24539909308527,26433.59657126564,0.9421390943887672,0.5245034424744095,27726.072459635743,0.3133147517255912,4.462522721602452,79.47061339416189,319.7289446483944,19.86233105967884,0.2487069690415779,0.0164753299830107,0.3573385924685066,0.003319522855267,19247.808152455116,0.8003498315947944,4.777422347537897,14.005201628795426,3142.4462661057264,12581.03896754554,785.5801864869168,0.221988861957271,1.6020074914708753,19448.586446462366,0.7824269818893127,0.6967245250739567,1530.002204939325,0.815712885857426,12.956852823803494,0.1767567336439371,10.15,0.0845833333333333,16.856597222222224,80.19166666666666,6481639.075,25926789.65833333,1620409.0821336007,0.0412366089581228,76.93333333333334,0.6411111111111111,0.8251700680379249,65.85035194637695,0.0355681239447763,4.576504829670242,0.0042770075204048,6426972.738263888,0.142200818108746,0.0974991279505262,2.562523261570866,3.0847130168477354e-05,8.401676144056733,e01cefab0d59,11,1.49,0.47,12,Minimal +0.9999959743936842,0.0145271565368994,3.135861397280637,215.86202291655965,187.0,187.0,263.751777245197,263.751777245197,34799.333333333336,215.8611539410369,0.0733346978123877,70289.19461225385,2.019843137193831,34990.0,2.0,2.0,564161.0,0.0089888645580834,0.0,2766.098451053441,237.0,0.153151811390034,2.076621891969134,2.0,2.0,235.0,0.0,4.015406881800093,50.53960581045358,564161.0,0.9988000719758172,11.811133944202084,25.76271769040925,111.37884534691852,-2.2106606143914806,5.294989509874144,10.1109311987366,0.1452384202282944,2.14951552899856,1.4562760315107337,2.635049833288353,0.5682920117562662,0.5041062445081413,0.9219653495902104,0.8514834349372742,-0.6364203665973874,0.6455257557437896,0.1421566747488709,4.477669426851201,0.4190219519424506,2.3377334551471125,0.863636078348161,0.4439149518613121,8.955338853702402,1.5510054093737935,3.8514801771526863,0.1590081949381913,33492.38113746785,0.9571986606878492,0.2187203088523802,34948.01451843384,0.017225163176904,1.0230065733066591,79.99491283223779,80.11480422977993,79.99238045746758,0.9994341037474552,0.0129317591622297,0.0215443389469301,0.0126853556070032,24275.74136900485,0.9568912063207288,0.5903072147681044,1.8433482961377288,6581.605976202434,6582.712984403259,6581.579791109703,0.9787339512790478,1.2882950658618673,24249.682607251663,0.820842963503345,0.6949326181106689,940.4590105392892,0.714465160656172,1.491898480622518,0.6944288729376284,1.7692307692307692,0.136094674556213,6.437869822485207,33.0,94063923.6923077,94064078.23076925,94063920.25594966,0.1324721035137701,4.230769230769231,0.3254437869822485,0.551816239379145,17.598290598353504,0.0314788477006075,3.546593564294935,0.0003715347242069,86819544.40236688,0.0002246255004356,278.2130644911924,0.0337113601077708,6.152304747276634e-07,11.06588651161052,e643d0f987b7,24,0.12,0.06,1,Minimal +0.9999557451136372,0.01368088344025,3.9477399957775408,288.55885023938043,250.0,250.0,352.8469923352047,352.8469923352047,62297.25,288.54608010025413,0.0605516081921434,125507.88940951606,2.0146617934100792,62555.0,4.0,4.0,2815963.0,0.0127568189068491,0.0,2989.9144432430403,504.0,0.2598767746680103,4.128926544640716,4.0,-25.0,529.0,0.0,6.709381052018952,50.13402359044866,2815963.0,0.998306283372878,27.967759690152235,55.39517777135323,3198.422593981036,32.68254328298329,29.478107463369344,31.161259306506945,0.2166927096183969,3.576072576412432,2.1580901191487776,12.18655160375811,0.515322932332743,0.4534569291389372,0.9492401447027456,0.8809784658113045,-0.6530788902593441,0.7177459726587367,0.1064927793965499,6.757551811887897,0.3316099327266974,3.4182405609575266,0.9423952387077148,0.3464046340753982,13.515103623775794,2.5369415151242944,15.159841692469076,0.1327799190124886,60486.40391655344,0.9669315628895122,0.1811130240237923,62449.049556390375,0.0418911860160772,4.06148189593158,80.2171688913756,321.3334825353689,20.05303748746319,0.2498590816285778,0.0129324466074792,0.0764390422847647,0.0031880290803787,43338.50062630011,0.927356334073038,1.777793931331271,6.847394332291633,10911.581208602978,43649.88947422378,2727.8866892911474,0.2436949842547506,1.4392666029675725,43336.31500297592,0.8041900038523992,0.6943735666459669,2256.945664101266,0.7309839895784177,5.5920189207118085,0.1770179426122632,2.6,0.0866666666666666,21.965555555555557,71.43333333333334,130208338.23333332,520833706.6,32552083.63330201,0.1186841018578101,10.466666666666669,0.3488888888888889,0.6004074074159408,45.97584259262673,0.0762590995504315,4.573557262275177,0.0004795779713851,125860418.20555556,0.1902612523390828,0.1094971022599684,0.2344937292171026,1.5709970937712513e-06,9.692428703747582,ecaf9c8bec0b,19,0.53,0.13,2,Minimal +0.9989322914759764,0.097233411015917,22.04033596886509,226.67451176074744,197.0,197.0,277.89386463180506,277.89386463180506,38946.875,226.4324894523616,0.070685664230031,78608.20129177917,2.0183442520556314,39205.0,2.0,2.0,14259310.0,0.1164902050748069,0.0,249.4928262549689,545.0,3.132114822396104,3.5769927305190663,2.0,-31.0,576.0,0.0,19.07122220488807,14.591078030842256,14259310.0,0.9807161779859468,350.9166393940296,77.04587942311069,4017.405636777792,75.15791213122165,31.52487485183665,28.37817303304717,0.2859229674671733,3.464144610596954,2.5015391728177403,10.533099596666972,0.4923955367742821,0.4260299486764077,0.9595742215315742,0.8902270120658107,-0.4043719716530071,0.7419004450203066,0.1372641554068274,8.059591375337154,0.3100365116829885,5.017166412529618,0.7017241720393574,0.3186503979525503,16.119182750674312,3.190665250129645,14.975761971220964,0.2782878334768046,36897.50860859584,0.9411429309678828,0.7027108078636163,38448.97775793904,0.511219572019002,9.867593419206734,79.31837775793903,716.9489350848106,8.809893788551818,0.1103146919039642,0.0184232382490895,0.6934026555569726,0.0016071642680142,26673.56697780305,0.7507816949490488,5.870203047777531,24.7356148082348,3089.561666133403,27824.379889537573,343.267706964498,0.0967255617158611,1.7140829784087832,27196.54395578961,0.8034566204828753,0.6974404756065259,1958.1679702998103,0.8621505769155786,22.876632919945816,0.0820262897735819,19.43703703703704,0.0719890260631001,19.481755829903975,97.83703703703704,5578293.611111111,50204879.29259259,619810.1613248707,0.0351121394456446,144.06666666666666,0.5335802469135802,0.7587592592617184,75.83655555557769,0.0283516469519846,5.179860922135624,0.0068868766738936,5557209.495541838,0.6331613434448976,0.011424669500564,8.000626706472483,5.666736650973535e-05,5.505151233024589,f19209eb4180,6,3.41,1.0,18,Minimal +0.9999391565084456,0.0357873447711068,7.346915872224026,205.29368465904216,178.0,178.0,251.02390324429263,251.02390324429263,31577.58333333333,205.28119387447344,0.0757602224573223,63771.81974509412,2.0195281909928973,31762.0,2.0,2.0,6739938.0,0.0347316534009872,0.0,1215.537389847223,786.0,1.154264395626719,2.5785529878471127,2.0,2.0,784.0,0.0,14.567131143131792,32.135541686272816,6739938.0,0.99515768371733,205.5523742300649,108.90641459212708,25963.662630804887,421.956629701368,82.52296746565501,78.10444001313468,0.2715303655176358,5.637867662263646,2.614657731878446,28.70773612099371,0.4650481179968973,0.4075087736906063,0.9417624856828256,0.8739722626543377,-0.6929193695098536,0.7482711350604035,0.0856661478300699,8.925103651664791,0.3192237092295557,4.135760503054701,0.9336573903730506,0.3336671816414865,17.850207303329583,3.109769479177945,40.15685186969741,0.1923410768237797,30225.129966626788,0.951612932643624,0.3277545587000945,31608.198350229835,0.3120177306057167,1.3569674453749765,79.80372772495434,80.86650714690511,79.7957595335175,0.9976605820955488,0.0140560270406207,0.2204224747174393,0.0127462738256696,21949.10344899904,0.8710238400037846,8.122541149687546,12.0358131186029,4750.594928852005,4761.95384479597,4750.519007614984,0.9284156026451872,1.5243589585831037,22009.15820894191,0.7863277175828609,0.6955796234494049,1520.056789787467,0.7658766278457064,11.720914040107717,0.695376440407544,3.1481481481481484,0.0582990397805212,50.092249657064464,149.72222222222223,18590296.33333333,18590645.666666668,18590293.7345456,0.0725709054530321,26.074074074074076,0.4828532235939643,0.716820987672768,107.86844135804311,0.0451607738041023,4.896285132226415,0.0017001448271519,18244334.521262,0.2677706183346902,0.0847245828629506,3.172981599466828,1.5538012784694093e-05,25.90180143775459,f1dfa95835a7,8,0.7,0.2,2,Minimal +0.9999550261522356,0.0159645952756745,4.071007416208657,255.00223124426412,221.0,221.0,311.8348922106056,311.8348922106056,48686.5,254.9907628127366,0.0656853964225583,98164.4721820236,2.016256501946609,48916.0,2.0,2.0,2857293.0,0.0207617317127227,0.0,1716.769944888007,473.0,0.4739017983143,2.2371207784773897,2.0,-5.0,478.0,0.0,7.642789890136151,38.34775090691229,2857293.0,0.9971401824911033,53.407527927272056,56.39653040015197,3599.5318020782643,96.39047887109768,28.155685565812632,30.962207467859635,0.189812779262302,3.599991991683188,2.300127369929624,11.487867174333902,0.4924804988633574,0.4257948863793599,0.941586363821677,0.8706672976055021,-0.6233630657820685,0.7293882003926787,0.1221059332060064,6.831838050007643,0.3245063762058836,3.79186553685124,0.905828584887562,0.3447351251430581,13.663676100015286,2.5611916222529185,14.779473258418069,0.1545707270216171,47040.37026739717,0.9616561098085936,0.2361147649810672,48776.10916673481,0.0782790085220614,4.114093548123313,80.07163300351623,320.83259056341484,20.01644483404793,0.2497003702870445,0.0133293980156458,0.0956434102452276,0.0032039445810386,33856.28578973208,0.9007037192482716,2.5852808435292824,8.364796092345147,7845.710204179939,31388.205650108383,1961.4135694782367,0.2384835638237635,1.4885359283644974,33891.11553298421,0.7912852464746108,0.694804908886191,2132.871875501466,0.7486932497998938,7.109559767474602,0.1762731589167599,4.090909090909091,0.0929752066115702,15.34452479338843,67.25,54214624.31818182,216858843.6590909,13553655.197862316,0.0696183024679846,17.727272727272727,0.402892561983471,0.6544760101105376,35.83048611114922,0.0515832989886746,4.390805482076973,0.0008995011857061,52978686.78925621,0.5238369162316149,0.0340991845230816,0.4872251037670404,3.5788159835654174e-06,9.18497990360771,f74488b17fd4,18,0.63,0.21,3,Minimal +0.9987943387973272,0.0801175256908795,20.60896707709365,257.23419313534464,223.0,223.0,314.663312129012,314.663312129012,49679.29166666666,256.9240558486805,0.0651890923051392,100251.94038798186,2.0179824837407643,49954.0,2.0,2.0,4895851.0,0.0542448754289669,0.0,439.9154743872075,336.0,1.1502187978428406,2.577711494575009,2.0,2.0,334.0,0.0,9.899857908661254,19.54194528045253,4895851.0,0.9912935779525484,91.3625900624186,30.24976967978263,446.1625482946874,18.338033279450137,10.12609376711772,12.372397996178874,0.1864264182644913,2.291171239056678,2.096386748915722,4.557596315736893,0.5436230551571375,0.4823885858446722,0.9495765096935954,0.8793865001721279,-0.3988531567398012,0.6745042650379107,0.2138511967883939,4.901308078751668,0.3195229270189114,4.167985552390869,0.6974713891757834,0.335151102820625,9.802616157503332,2.599974514581947,5.624622940824149,0.1920586903390143,47768.78672378588,0.9562554895260816,0.3856812678922967,49519.0793930416,0.1293457426883099,1.1723986067181806,79.8529447091324,80.58077431236738,79.83451714979687,0.9958433999364866,0.0148548059276262,0.1045950391644263,0.0128250498507223,34356.096420110174,0.8081231939177477,2.535166822159733,5.929762601375085,5649.480392479843,5655.145138004485,5649.342007280831,0.8820420299609221,1.581278756473797,34613.64215999647,0.780396021300622,0.6955984120775729,2735.1877689369608,0.8128493355955275,5.559011260717881,0.6999923706175168,16.106382978723403,0.1142296665157688,8.239424576228561,39.63120567375886,17538822.31205674,17538957.985815603,17538818.927245043,0.1062326390608289,63.95035460992908,0.4535486142548161,0.6909003152116937,28.071266745471743,0.0862689684478265,4.635256117586233,0.0028225967890459,17413305.380514055,1.2790092291762711,0.0299545613584763,0.7931754726320799,1.3824087707672358e-05,3.362981721882907,f96bf216d7b9,10,1.13,0.48,18,Minimal +0.9996779120397962,0.0711695886009986,13.444808723213766,188.9122726083189,164.0,164.0,231.22499864850252,231.22499864850252,26867.541666666668,188.851426239777,0.079779383571572,54376.6024841573,2.023877106390417,27064.0,2.0,2.0,5050533.0,0.0759147812160535,0.0,428.9245533884614,520.0,1.811121737886456,2.9095846881466154,2.0,-41.0,561.0,0.0,13.660689311173124,18.754747750283236,5050533.0,0.987627852971802,178.14874939890242,67.26473694892735,1832.989654584468,73.60516802482657,19.74548344879937,25.739664011021645,0.1391922148460817,3.18005296502831,2.37672533563554,10.51826246313714,0.5227037871564885,0.4624362491289518,0.9601868624315508,0.8956655655499078,-0.5059309853349264,0.7364974250274774,0.1597717718665405,7.678478398371012,0.3198100426465755,4.113733777693459,0.7941812926205453,0.3398044059965409,15.356956796742022,2.745449373927473,11.371286864955255,0.2488224234811107,25464.794191545967,0.9409102199063688,0.5362973821306984,26729.16021282885,0.2588896637995144,9.464454626071534,79.43511676027195,716.5925953295891,8.82384000296539,0.1105796175484279,0.0163474439189411,0.4432119338522678,0.0015169007917563,18556.982264858983,0.7970428233624083,3.898098610211981,18.95945937704084,2982.428193367232,26852.36437484504,331.36872878844264,0.0997862485286231,1.5620988096555273,18759.36253000419,0.7846752108879309,0.6968268946542667,1475.6644060661615,0.819579088301694,17.20891087804587,0.0799357103121494,11.123893805309734,0.0984415381000861,13.95817996710784,87.79646017699115,6401727.451327434,57615775.18584071,711302.744648816,0.0295978595695534,64.929203539823,0.574594721591354,0.7845807998838044,71.15339734724955,0.0248149881662234,4.522513675591049,0.0041752882057345,6344365.081838828,0.0029753619485891,3.2951017806125265,2.6358742408310905,3.13214967643374e-05,6.890935253922127,f9afc5eead29,2,1.14,0.41,13,Minimal +0.9999231811616796,0.0196518462376234,3.8345709531734857,195.12522675005525,169.0,169.0,238.29603437741048,238.29603437741048,28397.0,195.1102374568093,0.0783892417122084,57421.68980196265,2.0221040885291632,28570.0,2.0,2.0,187410.0,0.0048373164428806,0.0,5143.200386584696,148.0,0.0509464941220461,2.025481274063704,2.0,2.0,146.0,0.0,2.561186831119353,69.00059154727282,187410.0,0.9993700885604307,2.4571035923164715,8.333599979875732,2.501499658908478,0.2568504900545013,0.6872071908358821,2.309538902724323,0.3406779706028096,0.8333484265338464,0.3515279758550981,0.6865230281138167,0.7603099137172037,0.720860800738679,0.9499005919766615,0.903682651084734,-0.4880696788435229,0.4220829959109426,0.1228656223010422,2.4405239856111485,0.6725888630784264,1.012397646025856,0.5476190220434477,0.6904610972756773,4.881047971222297,0.5600016004443907,0.7491865233900512,0.1668416099787771,27239.85348267413,0.9534425440207958,0.2276696907999058,28552.00343017151,0.0029743830558696,1.004725236261813,79.91375568778439,79.9235211760588,79.91327248862443,0.999718410920546,0.0129721455208562,0.0164373187795192,0.0127409589615283,19842.05367455293,0.9766100475641628,0.1094386093581698,1.1799144595185458,5603.362146556135,5603.542254878285,5603.351415966452,0.9892771648218712,1.2101598800266469,19804.106674516424,0.8323928222121041,0.6951266794108937,590.3266660394175,0.7055488555030525,0.8854148493635561,0.6948279589512455,1.6666666666666667,0.1851851851851851,2.2222222222222223,13.333333333333334,90636748.0,90636768.33333331,90636746.89277776,0.2127777777777778,5.666666666666667,0.6296296296296297,0.8055555556917661,11.1388888890251,0.0983333334695438,2.503258334775644,0.0003150157507875,80559650.46913579,8.356734524284718e-05,1197.067039106145,0.0039358863236709,9.045881653988449e-08,8.438608631198823,f9e71e8fdba7,17,0.02,0.02,2,Minimal +0.9994493631475544,0.0678397193356079,12.364202630878331,182.2560994055966,158.0,158.0,222.7397584626508,222.7397584626508,24873.666666666668,182.1557424806809,0.0819033054937228,50312.50044911622,2.0227215039646835,25042.0,2.0,2.0,2259932.0,0.0407601859618551,0.0,620.9834497385297,354.0,0.9197362997690434,2.4613050075872533,2.0,2.0,352.0,0.0,9.499771959314144,23.446380353823887,2259932.0,0.9937812995850543,84.18764493859726,37.681809096657545,661.05518489684,11.613378476626904,12.850774490743753,16.108970671881572,0.1778669014375368,2.7189100398627963,2.02937128548362,4.322629733896914,0.5188669458060192,0.4539386459249134,0.9420401010394374,0.8678665066421125,-0.5686789948318788,0.7335079720945508,0.1683483959574268,5.44848533133828,0.325959471838982,3.710307721151245,0.8543408451061616,0.3441775421601237,10.89697066267656,2.396901404377473,7.239936290656333,0.2149474800293885,23665.55195271943,0.9450344202827023,0.3765464132867596,24886.27130420893,0.1194267602155876,1.15390144557144,79.62934270425684,80.31610893698587,79.61649242677254,0.9970161221755238,0.0140604783962942,0.0770972432979261,0.0127669155011721,17287.04995328893,0.8621606566731922,2.487850132579555,5.024585518052002,3653.153920717543,3657.6498651782113,3653.065587090839,0.9225584395093444,1.4567275104749844,17353.949374197222,0.784107087226797,0.6960613861021177,1225.6995839907993,0.7695707503746267,4.676804718673671,0.694848816257526,5.133333333333334,0.114074074074074,8.75358024691358,46.644444444444446,13848921.688888889,13849128.977777775,13848917.841466324,0.0678006388958492,15.8,0.3511111111111111,0.5826790123813372,26.124129629665287,0.0310159771504138,4.50597469684966,0.0017969810717993,13539241.805432098,0.002478992562101,15.49471151113356,0.5532662603322366,1.040476128227384e-05,8.507556771318315,fd95f8f7e74b,7,0.52,0.19,4,Minimal diff --git a/predict2/kruskal_results.csv b/predict2/kruskal_results.csv new file mode 100644 index 0000000..6b2297c --- /dev/null +++ b/predict2/kruskal_results.csv @@ -0,0 +1,106 @@ +feature,kruskal_stat,p_value,significant +original_firstorder_Minimum,8.1041,0.0044,True +original_shape_Flatness,7.56,0.006,True +original_shape_SurfaceVolumeRatio,7.56,0.006,True +original_shape_LeastAxisLength,7.56,0.006,True +original_firstorder_Entropy,7.56,0.006,True +original_firstorder_TotalEnergy,7.56,0.006,True +original_firstorder_Uniformity,7.56,0.006,True +original_firstorder_Mean,7.56,0.006,True +original_firstorder_RootMeanSquared,7.56,0.006,True +original_firstorder_Skewness,7.56,0.006,True +original_firstorder_MeanAbsoluteDeviation,7.56,0.006,True +original_firstorder_Variance,7.56,0.006,True +original_firstorder_Energy,7.56,0.006,True +original_firstorder_Kurtosis,7.56,0.006,True +original_glcm_JointAverage,7.56,0.006,True +original_glcm_JointEnergy,7.56,0.006,True +original_gldm_LargeDependenceHighGrayLevelEmphasis,7.56,0.006,True +original_glcm_JointEntropy,7.56,0.006,True +original_glcm_SumAverage,7.56,0.006,True +original_glcm_MaximumProbability,7.56,0.006,True +original_gldm_HighGrayLevelEmphasis,7.56,0.006,True +original_gldm_GrayLevelVariance,7.56,0.006,True +original_gldm_DependenceVariance,7.56,0.006,True +original_gldm_DependenceNonUniformityNormalized,7.56,0.006,True +original_glcm_Idmn,7.56,0.006,True +original_gldm_LargeDependenceEmphasis,7.56,0.006,True +original_gldm_LargeDependenceLowGrayLevelEmphasis,7.56,0.006,True +original_gldm_DependenceEntropy,7.56,0.006,True +original_glcm_Imc1,7.56,0.006,True +original_gldm_SmallDependenceHighGrayLevelEmphasis,7.56,0.006,True +original_gldm_SmallDependenceEmphasis,7.56,0.006,True +original_gldm_LowGrayLevelEmphasis,7.56,0.006,True +original_glrlm_ShortRunEmphasis,7.56,0.006,True +original_glrlm_ShortRunHighGrayLevelEmphasis,7.56,0.006,True +original_glrlm_ShortRunLowGrayLevelEmphasis,7.56,0.006,True +original_glszm_GrayLevelNonUniformity,7.56,0.006,True +original_glszm_SizeZoneNonUniformity,7.56,0.006,True +original_glrlm_RunEntropy,7.56,0.006,True +original_glrlm_RunLengthNonUniformityNormalized,7.56,0.006,True +original_glrlm_RunPercentage,7.56,0.006,True +original_glszm_ZoneEntropy,7.56,0.006,True +original_glszm_ZonePercentage,7.56,0.006,True +original_glszm_ZoneVariance,7.56,0.006,True +original_ngtdm_Contrast,7.56,0.006,True +original_ngtdm_Complexity,7.56,0.006,True +original_glszm_LargeAreaEmphasis,7.56,0.006,True +original_glszm_LowGrayLevelZoneEmphasis,7.56,0.006,True +original_glszm_LargeAreaLowGrayLevelEmphasis,7.56,0.006,True +original_glrlm_LongRunEmphasis,7.56,0.006,True +original_glrlm_LongRunLowGrayLevelEmphasis,7.56,0.006,True +original_glrlm_LowGrayLevelRunEmphasis,7.56,0.006,True +original_glrlm_HighGrayLevelRunEmphasis,7.56,0.006,True +original_glrlm_GrayLevelNonUniformityNormalized,7.56,0.006,True +original_glcm_Idn,7.0876,0.0078,True +original_glrlm_GrayLevelVariance,7.0876,0.0078,True +original_glszm_SmallAreaLowGrayLevelEmphasis,7.0876,0.0078,True +original_glcm_SumEntropy,7.0876,0.0078,True +original_shape_Elongation,7.0876,0.0078,True +original_glcm_Autocorrelation,7.0876,0.0078,True +original_ngtdm_Coarseness,6.6305,0.01,True +original_glszm_HighGrayLevelZoneEmphasis,6.1886,0.0129,True +original_glcm_MCC,6.1886,0.0129,True +original_glcm_Correlation,6.1886,0.0129,True +original_firstorder_Range,5.7619,0.0164,True +original_gldm_SmallDependenceLowGrayLevelEmphasis,5.7619,0.0164,True +original_glrlm_RunVariance,5.7619,0.0164,True +original_firstorder_Maximum,5.7619,0.0164,True +original_glcm_ClusterShade,5.3505,0.0207,True +original_glszm_GrayLevelNonUniformityNormalized,5.3505,0.0207,True +original_glszm_SmallAreaHighGrayLevelEmphasis,5.3505,0.0207,True +original_ngtdm_Strength,4.5733,0.0325,True +original_glcm_ClusterTendency,4.5733,0.0325,True +original_glcm_DifferenceEntropy,4.5733,0.0325,True +original_glcm_ClusterProminence,4.5733,0.0325,True +original_glcm_SumSquares,4.2076,0.0402,True +original_glszm_GrayLevelVariance,4.2076,0.0402,True +original_glcm_Idm,2.6076,0.1064,False +original_glcm_Id,2.3333,0.1266,False +original_glcm_DifferenceVariance,2.3333,0.1266,False +original_glrlm_LongRunHighGrayLevelEmphasis,2.0743,0.1498,False +original_glcm_Contrast,2.0743,0.1498,False +original_glcm_DifferenceAverage,2.0743,0.1498,False +original_ngtdm_Busyness,1.3886,0.2386,False +original_glcm_Imc2,0.84,0.3594,False +original_shape_Maximum2DDiameterColumn,0.4286,0.5127,False +original_glcm_InverseVariance,0.3219,0.5705,False +original_glrlm_GrayLevelNonUniformity,0.1543,0.6945,False +original_glszm_LargeAreaHighGrayLevelEmphasis,0.1543,0.6945,False +original_shape_MinorAxisLength,0.1543,0.6945,False +original_gldm_DependenceNonUniformity,0.1543,0.6945,False +original_gldm_GrayLevelNonUniformity,0.1543,0.6945,False +original_firstorder_Median,0.1429,0.7055,False +original_firstorder_10Percentile,0.1429,0.7055,False +original_firstorder_90Percentile,0.1429,0.7055,False +original_glszm_SizeZoneNonUniformityNormalized,0.0476,0.8273,False +original_shape_Maximum2DDiameterRow,0.0171,0.8958,False +original_glszm_SmallAreaEmphasis,0.0171,0.8958,False +original_glrlm_RunLengthNonUniformity,0.0171,0.8958,False +original_shape_MajorAxisLength,0.0171,0.8958,False +original_shape_Maximum3DDiameter,0.0019,0.9652,False +original_shape_Sphericity,0.0019,0.9652,False +original_shape_SurfaceArea,0.0019,0.9652,False +original_shape_VoxelVolume,0.0019,0.9652,False +original_shape_MeshVolume,0.0019,0.9652,False +original_shape_Maximum2DDiameterSlice,0.0019,0.9652,False diff --git a/predict2/pca_clustering.png b/predict2/pca_clustering.png new file mode 100644 index 0000000..129800f Binary files /dev/null and b/predict2/pca_clustering.png differ diff --git a/predict2/spearman_plot.png b/predict2/spearman_plot.png new file mode 100644 index 0000000..692252c Binary files /dev/null and b/predict2/spearman_plot.png differ diff --git a/predict2/spearman_results.csv b/predict2/spearman_results.csv new file mode 100644 index 0000000..edec33e --- /dev/null +++ b/predict2/spearman_results.csv @@ -0,0 +1,108 @@ +feature,spearman_r,p_value,abs_corr,significant +original_glcm_JointEnergy,-0.9885,0.0,0.9884757789663802,True +original_glcm_JointEntropy,0.9867,0.0,0.9867362703364349,True +original_glcm_MaximumProbability,-0.9867,0.0,0.9867362703364349,True +original_gldm_GrayLevelVariance,0.9859,0.0,0.9858665160214624,True +original_firstorder_Energy,0.985,0.0,0.9849967617064898,True +original_firstorder_TotalEnergy,0.985,0.0,0.9849967617064898,True +original_gldm_SmallDependenceHighGrayLevelEmphasis,0.9846,0.0,0.9845618845490035,True +original_glrlm_RunEntropy,0.9833,0.0,0.9832572530765445,True +original_firstorder_RootMeanSquared,0.9815,0.0,0.9815177444465992,True +original_firstorder_MeanAbsoluteDeviation,0.9806,0.0,0.9806479901316266,True +original_firstorder_Variance,0.9806,0.0,0.9806479901316266,True +original_glrlm_HighGrayLevelRunEmphasis,0.9798,0.0,0.9797782358166541,True +original_glrlm_ShortRunHighGrayLevelEmphasis,0.9798,0.0,0.9797782358166541,True +original_ngtdm_Complexity,0.9767,0.0,0.9767340957142499,True +original_ngtdm_Contrast,0.9706,0.0,0.9706458155094415,True +original_glrlm_GrayLevelNonUniformityNormalized,-0.9698,0.0,0.969776061194469,True +original_glcm_SumEntropy,0.9693,0.0,0.9693411840369826,True +original_glszm_ZonePercentage,0.9689,0.0,0.9689063068794963,True +original_firstorder_Entropy,0.9685,0.0,0.9684714297220101,True +original_glrlm_ShortRunEmphasis,0.9685,0.0,0.9684714297220101,True +original_glszm_SizeZoneNonUniformity,0.9672,0.0,0.9671667982495511,True +original_firstorder_Uniformity,-0.9672,0.0,0.9671667982495511,True +original_glrlm_GrayLevelVariance,0.965,0.0,0.9649924124621196,True +original_gldm_HighGrayLevelEmphasis,0.9606,0.0,0.9606436408872565,True +original_glszm_ZoneEntropy,0.9602,0.0,0.9602087637297703,True +original_glrlm_LowGrayLevelRunEmphasis,-0.9541,0.0,0.9541204835249618,True +original_gldm_LowGrayLevelEmphasis,-0.9528,0.0,0.952815852052503,True +original_gldm_SmallDependenceEmphasis,0.9476,0.0,0.9475973261626671,True +original_glszm_LargeAreaLowGrayLevelEmphasis,-0.9472,0.0,0.9471624490051809,True +original_firstorder_Mean,0.9441,0.0,0.9441183089027766,True +original_firstorder_Skewness,-0.9432,0.0,0.9432485545878041,True +original_glszm_LowGrayLevelZoneEmphasis,-0.9346,0.0,0.934551011438078,True +original_firstorder_Kurtosis,-0.9337,0.0,0.9336812571231053,True +original_glszm_GrayLevelNonUniformity,0.9319,0.0,0.9319417484931601,True +original_ngtdm_Coarseness,-0.9228,0.0,0.9228093281859476,True +original_gldm_LargeDependenceHighGrayLevelEmphasis,0.915,0.0,0.9149815393511941,True +original_gldm_DependenceVariance,0.9119,0.0,0.9119373992487899,True +original_shape_LeastAxisLength,0.9111,0.0,0.9110676449338172,True +original_glcm_JointAverage,0.9041,0.0,0.9041096104140363,True +original_glcm_SumAverage,0.9041,0.0,0.9041096104140363,True +original_glcm_DifferenceEntropy,0.9032,0.0,0.9032398560990637,True +original_glcm_Autocorrelation,0.9024,0.0,0.902370101784091,True +original_firstorder_Maximum,0.9015,0.0,0.9015003474691183,True +original_firstorder_Range,0.9006,0.0,0.9006305931541458,True +original_gldm_LargeDependenceLowGrayLevelEmphasis,-0.9002,0.0,0.9001957159966595,True +original_glszm_LargeAreaEmphasis,-0.898,0.0,0.8980213302092279,True +original_glszm_ZoneVariance,-0.898,0.0,0.8980213302092279,True +original_glrlm_LongRunLowGrayLevelEmphasis,-0.8924,0.0,0.892367927161906,True +original_glszm_HighGrayLevelZoneEmphasis,0.8885,0.0,0.8884540327445292,True +original_glcm_ClusterShade,0.8841,0.0,0.884105261169666,True +original_glcm_ClusterProminence,0.8754,0.0,0.8754077180199399,True +original_glszm_GrayLevelNonUniformityNormalized,-0.8732,0.0,0.8732333322325083,True +original_glszm_SmallAreaHighGrayLevelEmphasis,0.8724,0.0,0.8723635779175358,True +original_shape_Flatness,0.8724,0.0,0.8723635779175358,True +original_glcm_ClusterTendency,0.8711,0.0,0.8710589464450768,True +original_glrlm_RunPercentage,0.8471,0.0,0.8471407027833299,True +original_gldm_DependenceEntropy,0.8454,0.0,0.8454011941533847,True +original_shape_Elongation,-0.8376,0.0,0.8375734053186311,True +original_firstorder_Minimum,-0.8373,0.0,0.837257087294408,True +original_glcm_SumSquares,0.8345,0.0,0.834529265216227,True +original_glszm_GrayLevelVariance,0.8258,0.0,0.8258317220665006,True +original_glcm_Imc1,0.8232,0.0,0.8232224591215829,True +original_gldm_LargeDependenceEmphasis,-0.8167,0.0,0.8166993017592881,True +original_glcm_Correlation,0.7915,0.0,0.7914764266250821,True +original_glrlm_LongRunEmphasis,-0.7797,0.0,0.7797347433729519,True +original_gldm_DependenceNonUniformityNormalized,-0.7615,0.0,0.761469902758527,True +original_glcm_DifferenceVariance,0.7597,0.0,0.7597303941285817,True +original_glcm_Id,-0.7449,0.0,0.7449445707740472,True +original_glcm_Idm,-0.7432,0.0,0.743205062144102,True +original_ngtdm_Busyness,0.7406,0.0,0.7405957991991841,True +original_glcm_Idmn,0.7254,0.0001,0.7253750986871633,True +original_glrlm_ShortRunLowGrayLevelEmphasis,-0.7154,0.0001,0.7153729240649782,True +original_glcm_DifferenceAverage,0.7136,0.0001,0.7136334154350329,True +original_glcm_Idn,0.7132,0.0001,0.7131985382775466,True +original_glcm_Contrast,0.7128,0.0001,0.7127636611200604,True +original_glszm_SmallAreaLowGrayLevelEmphasis,-0.6828,0.0002,0.6827571372535051,True +original_gldm_SmallDependenceLowGrayLevelEmphasis,-0.6793,0.0003,0.6792781199936145,True +original_glcm_Imc2,0.6715,0.0003,0.6714503311588609,True +original_glcm_MCC,-0.5971,0.0021,0.5970863372287021,True +original_shape_SurfaceVolumeRatio,-0.5893,0.0024,0.5892585483939485,True +original_glrlm_LongRunHighGrayLevelEmphasis,0.5084,0.0112,0.5083713971014951,True +original_glszm_LargeAreaHighGrayLevelEmphasis,-0.377,0.0693,0.37703849554062985,False +original_ngtdm_Strength,-0.3683,0.0766,0.36834095239090364,False +original_glcm_InverseVariance,-0.3162,0.1323,0.3161556934925466,False +original_glszm_SmallAreaEmphasis,0.1931,0.366,0.1930854579239212,False +original_glrlm_RunVariance,0.1479,0.4905,0.14785823354534505,False +original_glszm_SizeZoneNonUniformityNormalized,0.1439,0.5022,0.14394433912796825,False +original_shape_Maximum2DDiameterColumn,0.1361,0.526,0.13611655029321468,False +original_firstorder_10Percentile,-0.1055,0.6239,0.1054521064544387,False +original_firstorder_Median,-0.1055,0.6239,0.1054521064544387,False +original_firstorder_90Percentile,-0.1055,0.6239,0.1054521064544387,False +original_glrlm_RunLengthNonUniformityNormalized,0.0774,0.7192,0.07740813403256298,False +original_shape_Maximum2DDiameterRow,0.0713,0.7405,0.07131985382775467,False +original_glrlm_RunLengthNonUniformity,0.0713,0.7405,0.07131985382775467,False +original_shape_VoxelVolume,0.0618,0.7744,0.06175255636305587,False +original_shape_MeshVolume,0.0618,0.7744,0.06175255636305587,False +original_shape_Maximum2DDiameterSlice,0.0461,0.8306,0.04609697869354874,False +original_shape_Maximum3DDiameter,0.0461,0.8306,0.04609697869354874,False +original_shape_SurfaceArea,0.0461,0.8306,0.04609697869354874,False +original_shape_Sphericity,-0.0461,0.8306,0.04609697869354874,False +original_shape_MajorAxisLength,0.0444,0.8369,0.044357470063603514,False +original_shape_MinorAxisLength,0.0174,0.9357,0.017395086299452356,False +original_glrlm_GrayLevelNonUniformity,0.0174,0.9357,0.017395086299452356,False +original_gldm_DependenceNonUniformity,0.0174,0.9357,0.017395086299452356,False +original_gldm_GrayLevelNonUniformity,0.0174,0.9357,0.017395086299452356,False +original_firstorder_InterquartileRange,,,,False +original_firstorder_RobustMeanAbsoluteDeviation,,,,False diff --git a/predict2/top_features.csv b/predict2/top_features.csv new file mode 100644 index 0000000..704acec --- /dev/null +++ b/predict2/top_features.csv @@ -0,0 +1,21 @@ +,importance +original_gldm_LargeDependenceEmphasis,0.043659269155012115 +original_gldm_LargeDependenceHighGrayLevelEmphasis,0.036121262910002255 +original_ngtdm_Complexity,0.03095161727262849 +original_firstorder_Energy,0.029217150532204687 +original_glszm_SizeZoneNonUniformity,0.0288867179123807 +original_shape_Flatness,0.028198739652665377 +original_firstorder_Kurtosis,0.028136588286768665 +original_glcm_Imc1,0.028040739377574384 +original_firstorder_Skewness,0.02748349406951955 +original_glrlm_RunLengthNonUniformityNormalized,0.02478153597748112 +original_firstorder_Uniformity,0.021581353546722085 +original_glrlm_RunPercentage,0.02143804561532609 +original_glrlm_GrayLevelNonUniformityNormalized,0.021212302592516857 +original_gldm_SmallDependenceEmphasis,0.020842162826334224 +original_firstorder_Variance,0.02042651871125634 +original_glcm_MCC,0.02009748155219999 +original_firstorder_RootMeanSquared,0.019625509134551265 +original_shape_LeastAxisLength,0.019461978130348348 +original_glszm_LargeAreaLowGrayLevelEmphasis,0.01945527366181059 +original_gldm_HighGrayLevelEmphasis,0.01940454478106091 diff --git a/resources/Contacts.md b/resources/Contacts.md deleted file mode 100644 index 6ea529a..0000000 --- a/resources/Contacts.md +++ /dev/null @@ -1,36 +0,0 @@ -Dr. Ghosh - ---> imaging of neural plaques, but could definitely help with imaging of cardiac plaques perhaps ---> Not yet contacted - -Dr. Sharlene Newman UA --> Not contacted - -Suzanne Lapi PhD (UAB) -->Not contacted - - -Dr. Darryl Outlaw --> contacted, responded, follow up later? - - -Dr. Stephen Walker--> Contacted, responded, followed up, no response - - -Lacey Sellars Human Research UA --> Currently in communication - -Contacts she gave me: -Chris Crawford -MRI person (pending) - - -Dr. Harrison Kim (UAB now at OSU)--> contacted, need to follow up - -Dr. Matthew Tong @ OSU (Dr. Hahn will contact) - - -Patents: Jeff Capili (jccapili@ua.edu) or Alankriti Bajpai (AB)(abajpai@ua.edu) - -Phone: Jeff (205) 348-0621 or AB (205) 348-5833 - - -Potential Contact at UAB: Director of Service Learning and Undergraduate Research: Gareth Jones (ghjones@uab.edu) - -Phone: (205) 996-7786 diff --git a/resources/IRB/HRP-503A-TemplateTips.docx b/resources/IRB/HRP-503A-TemplateTips.docx deleted file mode 100644 index c6de68b..0000000 Binary files a/resources/IRB/HRP-503A-TemplateTips.docx and /dev/null differ diff --git a/resources/README.md b/resources/README.md deleted file mode 100644 index a71cf27..0000000 --- a/resources/README.md +++ /dev/null @@ -1,5 +0,0 @@ -This is a folder for all the miscellaneous resources we have for this project! - -IRB= helpful documents for IRB access - -Contacts= document with the people that could be helpful for this project, with their info and if we have reached out yet