diff --git a/templates/4_Generic_Cell_Clustering.ipynb b/templates/4_Generic_Cell_Clustering.ipynb new file mode 100644 index 0000000..72a0a2e --- /dev/null +++ b/templates/4_Generic_Cell_Clustering.ipynb @@ -0,0 +1,684 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "tags": [] + }, + "source": [ + "# Generic cell clustering notebook" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# install requirements\n", + "!pip install ark-analysis" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [ + "import" + ] + }, + "outputs": [], + "source": [ + "# import required packages\n", + "import json\n", + "import os\n", + "import subprocess\n", + "from datetime import datetime as dt\n", + "import sys\n", + "\n", + "import feather\n", + "import matplotlib.pyplot as plt\n", + "import numpy as np\n", + "import pandas as pd\n", + "import scipy.stats as stats\n", + "import seaborn as sns\n", + "import xarray as xr\n", + "from matplotlib import rc_file_defaults\n", + "from alpineer import io_utils, load_utils, misc_utils\n", + "\n", + "from ark.analysis import visualize\n", + "from ark.phenotyping import cell_cluster_utils, cell_som_clustering, cell_meta_clustering\n", + "from ark.utils import data_utils, example_dataset, plot_utils\n", + "from ark.utils.metacluster_remap_gui import (MetaClusterData, MetaClusterGui,\n", + " colormap_helper,\n", + " metaclusterdata_from_files)\n", + "from IPython.display import display, HTML\n", + "display(HTML(\"\"))" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "tags": [] + }, + "source": [ + "## 0: Set root directory and download the example dataset\n", + "\n", + "Here we are using the example data located in `/data/example_dataset/input_data`. To modify this notebook to run using your own data, simply change `base_dir` to point to your own sub-directory within the data folder, rather than `'example_dataset'`. \n", + "\n", + "* `base_dir`: the path to all of your imaging data. Should contain a directory for your images, segmentations, and cell table (generated from `1_Nimbus_Predict.ipynb`) as visualized below. This directory will also store all of the directories/files created during pixel clustering.\n", + "\n", + "```\n", + "|-- base_dir\n", + "| |-- image_data\n", + "| | |-- fov_1\n", + "| | |-- fov_2\n", + "| |-- segmentation\n", + "| | |-- deepcell_output\n", + "| |-- nimbus_output\n", + "| | |-- fov_1\n", + "| | |-- fov_2\n", + "| | |-- nimbus_cell_table.csv\n", + "```" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [ + "base_dir" + ] + }, + "outputs": [], + "source": [ + "# define the home directory containing pixel_output_dir\n", + "base_dir = \"../data/example_dataset\"" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "tags": [] + }, + "source": [ + "## 1: Set input paths" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Define the following paths:\n", + "\n", + "- `tiff_dir`: the path to the directory with extracted image data\n", + "- `cell_table_path`: the path to the cell table generated by `1_Nimbus_Predict.ipynb`" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [ + "input_set" + ] + }, + "outputs": [], + "source": [ + "tiff_dir = os.path.join(base_dir, 'image_data')\n", + "cell_table_path = os.path.join(base_dir, 'nimbus_output', 'nimbus_cell_table.csv')\n", + "\n", + "# load cell table\n", + "cell_table = pd.read_csv(cell_table_path)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Define the list of FOVs and expression columns to use for cell SOM clustering" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [ + "param_set" + ] + }, + "outputs": [], + "source": [ + "# explicitly set the SOM expression columns here\n", + "\n", + "cell_som_cluster_cols = [\n", + " 'CD14', 'HLADR', 'Ki67', 'Collagen1', 'CD45', 'GLUT1',\n", + " 'CK17', 'CD68', 'CD163', 'Fibronectin', 'Vim', 'CD8', 'CD4', 'ECAD',\n", + " 'SMA', 'CD31', 'IDO', 'CD20', 'PD1', 'CD3'\n", + "]\n", + "\n", + "# verify that the columns provided exist in cell_table\n", + "misc_utils.verify_in_list(\n", + " provided_cell_cols=cell_som_cluster_cols,\n", + " cell_som_input_cols=cell_table.columns.values\n", + ")\n", + "\n", + "# extract the set of FOVs used in the dataset\n", + "fovs = list(cell_table['fov'].unique())" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Set a prefix to be applied to all data directories/files created by cell clustering. If the prefix is not set, a default of the datetime at the start of the run is used." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [ + "cluster_prefix" + ] + }, + "outputs": [], + "source": [ + "# explicitly set cell_cluster_prefix to override datetime default\n", + "cell_cluster_prefix = \"example\"\n", + "\n", + "if cell_cluster_prefix is None:\n", + " cell_cluster_prefix = dt.now().strftime('%Y-%m-%dT%H:%M:%S')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The following folders/files will be created with names prefixed by `cell_cluster_prefix`:\n", + "\n", + "* `cell_output_dir`: the name of the folder to store the cell clustering directories/files\n", + "* `cell_som_weights_name`: file name to place the cell SOM weights\n", + "* `cell_som_expr_col_avg_name`: file name to store the average expression values of each `cell_som_cluster_cols` per cell SOM cluster\n", + "* `cell_meta_expr_col_avg_name`: same as above except for cell meta clusters\n", + "* `cell_meta_cluster_remap_name`: for the meta cluster remapping process, the file to store the new SOM to meta mappings" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [ + "cell_cluster_files" + ] + }, + "outputs": [], + "source": [ + "# define the base output cell folder\n", + "cell_output_dir = '%s_cell_output_dir_generic' % cell_cluster_prefix\n", + "if not os.path.exists(os.path.join(base_dir, \"pixie\", cell_output_dir)):\n", + " os.makedirs(os.path.normpath(os.path.join(base_dir, \"pixie\", cell_output_dir)))\n", + " \n", + "# define the paths to cell clustering files, explicitly set the variables to use custom names\n", + "cell_som_weights_name = os.path.join(\"pixie\", cell_output_dir, 'cell_som_weights.feather')\n", + "cell_som_expr_col_avg_name = os.path.join(\"pixie\", cell_output_dir, 'cell_som_expr_col_avg.csv')\n", + "cell_meta_expr_col_avg_name = os.path.join(\"pixie\", cell_output_dir, 'cell_meta_expr_col_avg.csv')\n", + "cell_meta_cluster_remap_name = os.path.join(\"pixie\", cell_output_dir, 'cell_meta_cluster_mapping.csv')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 2: Cell clustering" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 2.1: train cell SOM" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Train the cell SOM on the expression values provided per cell (the data stored in `cell_som_input_name`).\n", + "\n", + "For a full set of parameters you can customize for `train_cell_som`, please consult: cell training docs." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [ + "train_cell_som" + ] + }, + "outputs": [], + "source": [ + "# create the cell-level SOM weights\n", + "cell_pysom = cell_som_clustering.train_cell_som(\n", + " fovs,\n", + " base_dir,\n", + " cell_table_path,\n", + " cell_som_cluster_cols,\n", + " cell_table,\n", + " som_weights_name=cell_som_weights_name,\n", + " num_passes=10,\n", + " overwrite=True,\n", + " xdim=20,\n", + " ydim=20,\n", + " normalize=False,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 2.2: assign cell SOM clusters" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Use the weights learned from `train_cell_som` to assign cell clusters to the cell dataset.\n", + "\n", + "`generate_som_avg_files` will then compute the average values of all columns in `cell_som_cluster_cols` per cell SOM cluster as well as the number of cells in each cell SOM cluster (the data placed in `cell_som_expr_col_avg_name`). This is needed for cell consensus clustering." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [ + "cluster_cell_data" + ] + }, + "outputs": [], + "source": [ + "# use cell SOM weights to assign cell clusters\n", + "cell_table = cell_som_clustering.cluster_cells(\n", + " base_dir,\n", + " cell_pysom,\n", + " cell_som_cluster_cols=cell_som_cluster_cols\n", + ")\n", + "\n", + "# generate the SOM cluster summary files\n", + "cell_som_clustering.generate_som_avg_files(\n", + " base_dir,\n", + " cell_table,\n", + " cell_som_cluster_cols=cell_som_cluster_cols,\n", + " cell_som_expr_col_avg_name=cell_som_expr_col_avg_name\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 2.3: run cell consensus clustering" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "With the SOM cluster labels assigned to the cell data, assign consensus cluster labels. The consensus clusters are trained on the average values of all columns in `cell_som_cluster_cols` per cell SOM cluster (the data stored in `cell_som_expr_col_avg_name`). These values are z-scored and capped at the value specified in the `cap` argument prior to training: this helps improve the meta clustering process.\n", + "\n", + "After consensus clustering, the following are also computed by `generate_meta_avg_files`:\n", + "\n", + "* The average values of all columns in `cell_som_cluster_cols` per cell meta cluster, and the number of cells per meta cluster (the data placed in `cell_meta_expr_col_avg_name`)\n", + "* The meta cluster mapping for each cell SOM cluster in `cell_som_expr_col_avg_name` (data is resaved, same data except with an associated meta cluster column)\n", + "\n", + "For a full set of parameters you can customize for `cell_consensus_cluster`, please consult: cell consensus clustering docs. Do note that weighted cell channel computations are unsupported as it is outside the scope of generic cell clustering." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "* `max_k`: the number of consensus clusters desired\n", + "* `cap`: used to clip z-scored values prior to consensus clustering (in the range `[-cap, cap]`)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [ + "cell_consensus_cluster" + ] + }, + "outputs": [], + "source": [ + "max_k = 20\n", + "cap = 3\n", + "\n", + "# run hierarchical clustering based on cell SOM cluster assignments\n", + "cell_cc, cell_table = cell_meta_clustering.cell_consensus_cluster(\n", + " base_dir,\n", + " cell_som_cluster_cols=cell_som_cluster_cols,\n", + " cell_som_input_data=cell_table,\n", + " cell_som_expr_col_avg_name=cell_som_expr_col_avg_name,\n", + " max_k=max_k,\n", + " cap=cap,\n", + ")\n", + "\n", + "# generate the meta cluster summary files\n", + "cell_meta_clustering.generate_meta_avg_files(\n", + " base_dir,\n", + " cell_cc,\n", + " cell_som_cluster_cols=cell_som_cluster_cols,\n", + " cell_som_input_data=cell_table,\n", + " cell_som_expr_col_avg_name=cell_som_expr_col_avg_name,\n", + " cell_meta_expr_col_avg_name=cell_meta_expr_col_avg_name\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 3: visualize results" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 3.1: use the interactive reclustering results to relabel cell meta clusters" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The visualization shows the z-scored average `cell_som_cluster_cols` expression per cell SOM and meta cluster. The heatmaps are faceted by cell SOM clusters on the left and cell meta clusters on the right.\n", + "\n", + "## Usage\n", + "\n", + "### Quickstart\n", + "- **Select**: Left Click\n", + "- **Remap**: **New metacluster button** or Right Click\n", + "- **Edit Metacluster Name**: Textbox at bottom right of the heatmaps.\n", + "\n", + "### Selection and Remapping details\n", + "- To select a SOM cluster, click on its respective position in the **selected** bar. Click on it again to deselect.\n", + "- To select a meta cluster, click on its corresponding color in the **metacluster** bar. Click on it again to deselect.\n", + "- To remap the selected clusters, click the **New metacluster** button (alternatively, right click anywhere). Note that remapping an entire metacluster deletes it.\n", + "- To clear the selected SOM/meta clusters, use the **Clear Selection** button.\n", + "- **After remapping a meta cluster, make sure to deselect the newly created one to prevent unwanted combinations.**\n", + "\n", + "### Other features and notes\n", + "- You will likely need to zoom out to see the entire visualization. To toggle Zoom, use Ctrl -/Ctrl + on Windows or ⌘ +/⌘ - on Mac.\n", + "- The bars at the top show the number of cells in each SOM cluster.\n", + "- The text box at the bottom right allows you to rename a particular meta cluster. This can be useful as remapping may cause inconsistent numbering.\n", + "- Adjust the z-score limit using the slider on the bottom left to adjust your dynamic range.\n", + "- When meta clusters are combined or a meta cluster is renamed, the change is immediately saved to `cell_meta_cluster_remap_name`.\n", + "- You won't be able to advance until you've clicked `New metacluster` or renamed a meta cluster at least once. If you do not want to make changes, just click `New metacluster` to trigger a save before continuing." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "scrolled": false, + "tags": [ + "cell_interactive" + ] + }, + "outputs": [], + "source": [ + "%matplotlib widget\n", + "rc_file_defaults()\n", + "plt.ion()\n", + "\n", + "cell_mcd = metaclusterdata_from_files(\n", + " os.path.join(base_dir, cell_som_expr_col_avg_name),\n", + " cluster_type='cell'\n", + ")\n", + "cell_mcd.output_mapping_filename = os.path.join(base_dir, cell_meta_cluster_remap_name)\n", + "cell_mcg = MetaClusterGui(cell_mcd, width=17)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Relabel the cell meta clusters using the mapping." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [ + "cell_apply_remap" + ] + }, + "outputs": [], + "source": [ + "# rename the meta cluster values in the cell dataset\n", + "cell_table = cell_meta_clustering.apply_cell_meta_cluster_remapping(\n", + " base_dir,\n", + " cell_table,\n", + " cell_meta_cluster_remap_name\n", + ")\n", + "\n", + "# recompute the mean column expression per meta cluster and apply these new names to the SOM cluster average data\n", + "cell_meta_clustering.generate_remap_avg_count_files(\n", + " base_dir,\n", + " cell_table,\n", + " cell_meta_cluster_remap_name,\n", + " cell_som_cluster_cols,\n", + " cell_som_expr_col_avg_name,\n", + " cell_meta_expr_col_avg_name,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Generate the color scheme returned by the interactive reclustering process. This will be for visualizing the weighted channel average heatmaps and the cell cluster overlay." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [ + "cell_cmap_gen" + ] + }, + "outputs": [], + "source": [ + "raw_cmap, renamed_cmap = colormap_helper.generate_meta_cluster_colormap_dict(\n", + " cell_mcd.output_mapping_filename,\n", + " cell_mcg.im_cl.cmap,\n", + " cluster_type='cell'\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 3.2: cell cluster overlay (cell meta clusters only)\n", + "\n", + "This will take your FOVs and generate cell cluster images in batches. Run this cell if you wish to create a significant amount of cell cluster mask images for downstream analysis.\n", + "\n", + "Generating overlays will require you to define the following:\n", + "\n", + "- `segmentation_dir`: the path to the directory with Mesmer segmented images\n", + "- `seg_suffix`: the suffix used for the files in `segmentation_dir`\n", + "- `subset_cell_fovs`: the list of FOVs you want to visualize" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [ + "cell_overlay_fovs" + ] + }, + "outputs": [], + "source": [ + "segmentation_dir = os.path.join(\"segmentation\", \"deepcell_output\")\n", + "seg_suffix = '_whole_cell.tiff'\n", + "subset_cell_fovs = ['fov0', 'fov1']" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [ + "cell_mask_gen_save" + ] + }, + "outputs": [], + "source": [ + "# generate and save the cell cluster masks for each fov in subset_cell_fovs\n", + "data_utils.generate_and_save_cell_cluster_masks(\n", + " fovs=subset_cell_fovs,\n", + " save_dir=os.path.join(base_dir, \"pixie\", cell_output_dir),\n", + " seg_dir=os.path.join(base_dir, segmentation_dir),\n", + " cell_data=cell_table,\n", + " seg_suffix=seg_suffix,\n", + " sub_dir='cell_masks',\n", + " name_suffix='_cell_mask',\n", + " cluster_id_to_name_path=os.path.join(base_dir, cell_meta_cluster_remap_name)\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Load a subset of the cell cluster masks that you would like to preview." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [ + "cell_overlay_gen" + ] + }, + "outputs": [], + "source": [ + "for cell_fov in subset_cell_fovs:\n", + " cell_cluster_mask = load_utils.load_imgs_from_dir(\n", + " data_dir = os.path.join(base_dir, \"pixie\", cell_output_dir, \"cell_masks\"),\n", + " files=[cell_fov + \"_cell_mask.tiff\"],\n", + " trim_suffix=\"_cell_mask\",\n", + " match_substring=\"_cell_mask\",\n", + " xr_dim_name=\"cell_mask\",\n", + " xr_channel_names=None,\n", + " )\n", + "\n", + " plot_utils.plot_pixel_cell_cluster(\n", + " cell_cluster_mask,\n", + " [cell_fov],\n", + " os.path.join(base_dir, cell_meta_cluster_remap_name),\n", + " metacluster_colors=raw_cmap,\n", + " cluster_type='cell',\n", + " erode=True, \n", + " figsize=(5,5),\n", + " dpi=150\n", + " )" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 3.3: save the full results of Pixie cell clustering" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "`cell_table` with the SOM, meta, and renamed meta cluster labels, is resaved to `cell_table_path` as a `.csv` file." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [ + "pixie_cell_save" + ] + }, + "outputs": [], + "source": [ + "cell_table.to_csv(\n", + " cell_table_path,\n", + " index=False\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 3.4: Save Images a Mantis Viewer Project\n", + "\n", + "Mantis Viewer is a visualization tool for multi-dimensional imaging in pathology. Learn more about Mantis Viewer in the [README](../README.md#mantis-viewer)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [ + "cell_mantis_project" + ] + }, + "outputs": [], + "source": [ + "plot_utils.create_mantis_dir(\n", + " fovs=subset_cell_fovs,\n", + " mantis_project_path=os.path.join(base_dir, \"pixie\", cell_output_dir, \"mantis\"),\n", + " img_data_path=tiff_dir,\n", + " mask_output_dir=os.path.join(base_dir, \"pixie\", cell_output_dir, \"cell_masks\"),\n", + " mapping = os.path.join(base_dir, cell_meta_cluster_remap_name),\n", + " seg_dir=os.path.join(base_dir, segmentation_dir),\n", + " cluster_type='cell',\n", + " mask_suffix=\"_cell_mask\",\n", + " seg_suffix_name=seg_suffix\n", + ")" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "nimbus", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.11" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +}