From 10d27358235ac0f1421c336ee3e264c634d5d20b Mon Sep 17 00:00:00 2001 From: Karen Fang Date: Mon, 21 Jul 2025 13:52:59 -0400 Subject: [PATCH 01/20] Rename output files and output merged adata metadata --- README.md | 11 +++++------ docker/spatial_py/scripts/visium_merge_and_plot_qc | 14 ++++++++------ .../cohort_analysis/cohort_analysis.wdl | 14 +++++++++----- .../spatial_statistics/spatial_statistics.wdl | 6 +++--- workflows/pmdbs_spatial_visium/main.wdl | 2 ++ .../pmdbs_spatial_visium/preprocess/preprocess.wdl | 12 ++++++------ 6 files changed, 33 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index d1b0307..1ce3c17 100644 --- a/README.md +++ b/README.md @@ -342,21 +342,21 @@ asap-dev-{cohort,team-xxyy}-{source}-{dataset} └── pmdbs_spatial_visium ├── cohort_analysis │ ├── ${cohort_id}.sample_list.tsv - │ ├── ${cohort_id}.merged_adata_object.h5ad + │ ├── ${cohort_id}.merged.h5ad + │ ├── ${cohort_id}.merged_adata_metadata.csv │ ├── ${cohort_id}.qc_violin.png │ ├── ${cohort_id}.qc_dist.png │ ├── ${cohort_id}.hvg_dispersion.png │ ├── ${cohort_id}.clustered.h5ad │ ├── ${cohort_id}.umap_cluster.png │ ├── ${cohort_id}.spatial_scatter.png - │ ├── ${cohort_id}.final_adata_object.h5ad + │ ├── ${cohort_id}.final.h5ad │ ├── ${cohort_id}.moran_top_10_variable_genes.csv │ ├── ${cohort_id}.moran_top_4_variable_genes_spatial_scatter.png │ └── MANIFEST.tsv └── preprocess ├── ${sampleA_id}.raw_feature_bc_matrix.h5 ├── ${sampleA_id}.filtered_feature_bc_matrix.h5 - ├── ${sampleA_id}.initial_adata_object.h5ad ├── ${sampleA_id}.molecule_info.h5 ├── ${sampleA_id}.metrics_summary.csv ├── ${sampleA_id}.spaceranger_spatial_outputs.tar.gz @@ -367,13 +367,12 @@ asap-dev-{cohort,team-xxyy}-{source}-{dataset} ├── ${sampleA_id}.scalefactors_json.json ├── ${sampleA_id}.tissue_positions.csv ├── ${sampleA_id}.spatial_enrichment.csv - ├── ${sampleA_id}.initial_adata_object.h5ad + ├── ${sampleA_id}.initial.h5ad ├── ${sampleA_id}.qc.h5ad ├── MANIFEST.tsv ├── ... ├── ${sampleN_id}.raw_feature_bc_matrix.h5 ├── ${sampleN_id}.filtered_feature_bc_matrix.h5 - ├── ${sampleN_id}.initial_adata_object.h5ad ├── ${sampleN_id}.molecule_info.h5 ├── ${sampleN_id}.metrics_summary.csv ├── ${sampleN_id}.spaceranger_spatial_outputs.tar.gz @@ -384,7 +383,7 @@ asap-dev-{cohort,team-xxyy}-{source}-{dataset} ├── ${sampleN_id}.scalefactors_json.json ├── ${sampleN_id}.tissue_positions.csv ├── ${sampleN_id}.spatial_enrichment.csv - ├── ${sampleN_id}.initial_adata_object.h5ad + ├── ${sampleN_id}.initial.h5ad ├── ${sampleN_id}.qc.h5ad └── MANIFEST.tsv ``` diff --git a/docker/spatial_py/scripts/visium_merge_and_plot_qc b/docker/spatial_py/scripts/visium_merge_and_plot_qc index 8dadad5..65d8c51 100755 --- a/docker/spatial_py/scripts/visium_merge_and_plot_qc +++ b/docker/spatial_py/scripts/visium_merge_and_plot_qc @@ -41,12 +41,12 @@ def main(args): multi_panel=True, ) fig = plt.gcf() - fig.suptitle(f"QC violin plot - {args.qc_plots_prefix}", va="center", ha="center", fontsize=16) - plt.savefig(f"{args.qc_plots_prefix}.qc_violin.png", dpi=300, bbox_inches="tight") + fig.suptitle(f"QC violin plot - {args.output_prefix}", va="center", ha="center", fontsize=16) + plt.savefig(f"{args.output_prefix}.qc_violin.png", dpi=300, bbox_inches="tight") # Total counts and n genes by counts distribution plots fig, axs = plt.subplots(1, 4, figsize=(15,4)) - fig.suptitle(f"Covariates for filtering - {args.qc_plots_prefix}") + fig.suptitle(f"Covariates for filtering - {args.output_prefix}") sns.distplot( merged_adata.obs["total_counts"], kde=False, @@ -71,9 +71,11 @@ def main(args): ax=axs[3], ) - plt.savefig(f"{args.qc_plots_prefix}.qc_dist.png", dpi=300, bbox_inches="tight") + plt.savefig(f"{args.output_prefix}.qc_dist.png", dpi=300, bbox_inches="tight") # Save outputs + metadata = merged_adata.obs + metadata.to_csv(f"{args.output_prefix}.merged_adata_metadata.csv") merged_adata.write_h5ad(filename=args.merged_adata_output, compression="gzip") @@ -90,10 +92,10 @@ if __name__ == "__main__": ) parser.add_argument( "-p", - "--qc-plots-prefix", + "--output-prefix", type=str, required=True, - help="Output file name prefix for the QC violin and distribution plots" + help="Output file name prefix for the QC violin, distribution plots, and metadata" ) parser.add_argument( "-o", diff --git a/workflows/pmdbs_spatial_visium/cohort_analysis/cohort_analysis.wdl b/workflows/pmdbs_spatial_visium/cohort_analysis/cohort_analysis.wdl index 02c7314..eb5fc56 100644 --- a/workflows/pmdbs_spatial_visium/cohort_analysis/cohort_analysis.wdl +++ b/workflows/pmdbs_spatial_visium/cohort_analysis/cohort_analysis.wdl @@ -135,7 +135,8 @@ workflow cohort_analysis { write_cohort_sample_list.cohort_sample_list ], [ - merge_and_plot_qc_metrics.merged_adata_object + merge_and_plot_qc_metrics.merged_adata_object, + merge_and_plot_qc_metrics.merged_adata_metadata_csv ], merge_and_plot_qc_metrics.qc_plots_png, [ @@ -169,6 +170,7 @@ workflow cohort_analysis { # Merged adata objects and QC plots File merged_adata_object = merge_and_plot_qc_metrics.merged_adata_object #!FileCoercion + File merged_adata_metadata_csv = merge_and_plot_qc_metrics.merged_adata_metadata_csv #!FileCoercion Array[File] qc_plots_png = merge_and_plot_qc_metrics.qc_plots_png #!FileCoercion # Processed outputs @@ -242,20 +244,22 @@ task merge_and_plot_qc_metrics { visium_merge_and_plot_qc \ --adata-paths-input ~{sep=' ' preprocessed_adata_objects} \ - --qc-plots-prefix ~{cohort_id} \ - --merged-adata-output ~{cohort_id}.merged_adata_object.h5ad + --output-prefix ~{cohort_id} \ + --merged-adata-output ~{cohort_id}.merged.h5ad upload_outputs \ -b ~{billing_project} \ -d ~{raw_data_path} \ -i ~{write_tsv(workflow_info)} \ - -o "~{cohort_id}.merged_adata_object.h5ad" \ + -o "~{cohort_id}.merged.h5ad" \ + -o "~{cohort_id}.merged_adata_metadata.csv" \ -o "~{cohort_id}.qc_violin.png" \ -o "~{cohort_id}.qc_dist.png" >>> output { - String merged_adata_object = "~{raw_data_path}/~{cohort_id}.merged_adata_object.h5ad" + String merged_adata_object = "~{raw_data_path}/~{cohort_id}.merged.h5ad" + String merged_adata_metadata_csv = "~{raw_data_path}/~{cohort_id}.merged_adata_metadata.csv" Array[String] qc_plots_png = [ "~{raw_data_path}/~{cohort_id}.qc_violin.png", "~{raw_data_path}/~{cohort_id}.qc_dist.png" diff --git a/workflows/pmdbs_spatial_visium/cohort_analysis/spatial_statistics/spatial_statistics.wdl b/workflows/pmdbs_spatial_visium/cohort_analysis/spatial_statistics/spatial_statistics.wdl index f192e33..c2ba335 100644 --- a/workflows/pmdbs_spatial_visium/cohort_analysis/spatial_statistics/spatial_statistics.wdl +++ b/workflows/pmdbs_spatial_visium/cohort_analysis/spatial_statistics/spatial_statistics.wdl @@ -68,19 +68,19 @@ task spatially_variable_gene_analysis { visium_spatially_variable_genes \ --cohort-id ~{cohort_id} \ --adata-input ~{clustered_adata_object} \ - --adata-output ~{cohort_id}.final_adata_object.h5ad + --adata-output ~{cohort_id}.final.h5ad upload_outputs \ -b ~{billing_project} \ -d ~{raw_data_path} \ -i ~{write_tsv(workflow_info)} \ - -o "~{cohort_id}.final_adata_object.h5ad" \ + -o "~{cohort_id}.final.h5ad" \ -o "~{cohort_id}.moran_top_10_variable_genes.csv" \ -o "~{cohort_id}.moran_top_4_variable_genes_spatial_scatter.png" >>> output { - String final_adata_object = "~{raw_data_path}/~{cohort_id}.final_adata_object.h5ad" + String final_adata_object = "~{raw_data_path}/~{cohort_id}.final.h5ad" String moran_top_10_variable_genes_csv = "~{raw_data_path}/~{cohort_id}.moran_top_10_variable_genes.csv" String moran_top_4_variable_genes_spatial_scatter_plot_png = "~{raw_data_path}/~{cohort_id}.moran_top_4_variable_genes_spatial_scatter.png" } diff --git a/workflows/pmdbs_spatial_visium/main.wdl b/workflows/pmdbs_spatial_visium/main.wdl index c9ffa94..2af290a 100644 --- a/workflows/pmdbs_spatial_visium/main.wdl +++ b/workflows/pmdbs_spatial_visium/main.wdl @@ -163,6 +163,7 @@ workflow pmdbs_spatial_visium_analysis { # Merged, processed (filtered, normalized, dimensionality reduced), integrated, and clustered adata objects, and plots Array[File?] project_merged_adata_object = project_cohort_analysis.merged_adata_object + Array[File?] project_merged_adata_metadata_csv = project_cohort_analysis.merged_adata_metadata_csv Array[Array[File]?] project_qc_plots_png = project_cohort_analysis.qc_plots_png Array[File?] project_processed_adata_object = project_cohort_analysis.processed_adata_object Array[File?] project_hvg_plot_png = project_cohort_analysis.hvg_plot_png @@ -187,6 +188,7 @@ workflow pmdbs_spatial_visium_analysis { # Merged, processed (filtered, normalized, dimensionality reduced), integrated, and clustered adata objects, and plots File? cohort_merged_adata_object = cross_team_cohort_analysis.merged_adata_object + File? cohort_merged_adata_metadata_csv = cross_team_cohort_analysis.merged_adata_metadata_csv Array[File]? cohort_qc_plots_png = cross_team_cohort_analysis.qc_plots_png File? cohort_processed_adata_object = cross_team_cohort_analysis.processed_adata_object File? cohort_hvg_plot_png = cross_team_cohort_analysis.hvg_plot_png diff --git a/workflows/pmdbs_spatial_visium/preprocess/preprocess.wdl b/workflows/pmdbs_spatial_visium/preprocess/preprocess.wdl index a8d7341..cd96703 100644 --- a/workflows/pmdbs_spatial_visium/preprocess/preprocess.wdl +++ b/workflows/pmdbs_spatial_visium/preprocess/preprocess.wdl @@ -39,7 +39,7 @@ workflow preprocess { scatter (sample_object in samples) { String spaceranger_count_output = "~{spaceranger_raw_data_path}/~{sample_object.sample_id}.raw_feature_bc_matrix.h5" - String counts_to_adata_output = "~{adata_raw_data_path}/~{sample_object.sample_id}.initial_adata_object.h5ad" + String counts_to_adata_output = "~{adata_raw_data_path}/~{sample_object.sample_id}.initial.h5ad" String qc_output = "~{qc_raw_data_path}/~{sample_object.sample_id}.qc.h5ad" } @@ -108,7 +108,7 @@ workflow preprocess { File tissue_positions_csv_output = select_first([spaceranger_count.tissue_positions_csv, spaceranger_tissue_positions_csv]) #!FileCoercion File spatial_enrichment_csv_output = select_first([spaceranger_count.spatial_enrichment_csv, spaceranger_spatial_enrichment_csv]) #!FileCoercion - String counts_to_adata_object = "~{adata_raw_data_path}/~{sample.sample_id}.initial_adata_object.h5ad" + String counts_to_adata_object = "~{adata_raw_data_path}/~{sample.sample_id}.initial.h5ad" if (counts_to_adata_complete == "false") { call counts_to_adata { @@ -250,7 +250,7 @@ task check_output_files_exist { parameter_meta { spaceranger_count_output_files: {help: "Spaceranger count output file to detect (`.raw_feature_bc_matrix.h5`)."} - counts_to_adata_output_files: {help: "Converted AnnData object output file to detect (`.initial_adata_object.h5ad`)."} + counts_to_adata_output_files: {help: "Converted AnnData object output file to detect (`.initial.h5ad`)."} qc_output_files: {help: "QC'ed output file to detect (`.qc.h5ad`)."} billing_project: {help: "Billing project to charge GCP costs."} zones: {help: "Space-delimited set of GCP zones where compute will take place. ['us-central1-c us-central1-f']"} @@ -449,17 +449,17 @@ task counts_to_adata { --slide ~{visium_slide_serial_number} \ --area ~{visium_capture_area} \ --spaceranger-spatial-dir spatial_outputs \ - --adata-output ~{sample_id}.initial_adata_object.h5ad + --adata-output ~{sample_id}.initial.h5ad upload_outputs \ -b ~{billing_project} \ -d ~{raw_data_path} \ -i ~{write_tsv(workflow_info)} \ - -o "~{sample_id}.initial_adata_object.h5ad" + -o "~{sample_id}.initial.h5ad" >>> output { - String initial_adata_object = "~{raw_data_path}/~{sample_id}.initial_adata_object.h5ad" + String initial_adata_object = "~{raw_data_path}/~{sample_id}.initial.h5ad" } runtime { From 65f9055f159e4de540b28102be7bc816330cb469 Mon Sep 17 00:00:00 2001 From: Karen Fang Date: Mon, 21 Jul 2025 16:36:44 -0400 Subject: [PATCH 02/20] Use Pearsons residuals for HVG and output gene features --- README.md | 6 ++++- .../spatial_py/scripts/geomx_merge_and_prep | 20 ++++++++++++-- docker/spatial_py/scripts/visium_process | 26 +++++++++++++++---- .../cohort_analysis/cohort_analysis.wdl | 24 ++++++++++++----- workflows/pmdbs_spatial_geomx/main.wdl | 8 ++++-- .../cohort_analysis/cohort_analysis.wdl | 14 +++++++++- workflows/pmdbs_spatial_visium/main.wdl | 4 +++ 7 files changed, 85 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index 1ce3c17..c0e633e 100644 --- a/README.md +++ b/README.md @@ -308,8 +308,10 @@ asap-dev-{cohort,team-xxyy}-{source}-{dataset} ├── cohort_analysis │ ├── ${cohort_id}.sample_list.tsv │ ├── ${cohort_id}.merged.h5ad - │ ├── ${cohort_id}.hvg_dispersion.png │ ├── ${cohort_id}.merged_adata_metadata.csv + │ ├── ${cohort_id}.all_genes.csv + │ ├── ${cohort_id}.hvg_genes.csv + │ ├── ${cohort_id}.hvg_dispersion.png │ ├── ${cohort_id}.clustered.h5ad # Final │ ├── ${cohort_id}.umap_cluster.png │ └── MANIFEST.tsv @@ -344,6 +346,8 @@ asap-dev-{cohort,team-xxyy}-{source}-{dataset} │ ├── ${cohort_id}.sample_list.tsv │ ├── ${cohort_id}.merged.h5ad │ ├── ${cohort_id}.merged_adata_metadata.csv + │ ├── ${cohort_id}.all_genes.csv + │ ├── ${cohort_id}.hvg_genes.csv │ ├── ${cohort_id}.qc_violin.png │ ├── ${cohort_id}.qc_dist.png │ ├── ${cohort_id}.hvg_dispersion.png diff --git a/docker/spatial_py/scripts/geomx_merge_and_prep b/docker/spatial_py/scripts/geomx_merge_and_prep index 952db7c..e1c5033 100755 --- a/docker/spatial_py/scripts/geomx_merge_and_prep +++ b/docker/spatial_py/scripts/geomx_merge_and_prep @@ -28,10 +28,17 @@ def main(args): sc.pp.highly_variable_genes( merged_adata, - flavor="seurat", + layer="counts", n_top_genes=args.n_top_genes, + flavor="pearson_residuals", inplace=True, + batch_key=args.batch_key, ) + full_features = merged_adata.var.copy() + hvg_full = merged_adata.iloc[: args.n_top_genes].index.to_list() + hvg_merged_adata = merged_adata[:, merged_adata.var.index.isin(hvg_full)] + hvg_features = hvg_merged_adata.var.copy() + sc.pl.highly_variable_genes( merged_adata, ) @@ -52,12 +59,14 @@ def main(args): # Save outputs metadata = merged_adata.obs metadata.to_csv(f"{args.output_prefix}.merged_adata_metadata.csv") + full_features.to_csv(f"{args.output_prefix}.all_genes.csv", index=True) + hvg_features.to_csv(f"{args.output_prefix}.hvg_genes.csv", index=True) merged_adata.write_h5ad(filename=args.adata_output, compression="gzip") if __name__ == "__main__": parser = argparse.ArgumentParser( - description="Merge adata objects and process to prepare for downstream analysis" + description="Merge adata objects and process to prepare for downstream analysis by identifying highly variable genes (HVGs) and PCA" ) parser.add_argument( "-i", @@ -80,6 +89,13 @@ if __name__ == "__main__": required=True, help="Number of principal components to compute using `scanpy.pp.pca` [30]" ) + parser.add_argument( + "-b", + "--batch-key", + type=str, + required=True, + help="Key in AnnData object for batch information ['batch_id']" + ) parser.add_argument( "-p", "--output-prefix", diff --git a/docker/spatial_py/scripts/visium_process b/docker/spatial_py/scripts/visium_process index dedf5f0..0b9da99 100755 --- a/docker/spatial_py/scripts/visium_process +++ b/docker/spatial_py/scripts/visium_process @@ -43,16 +43,23 @@ def main(args): ################## sc.pp.highly_variable_genes( adata, - flavor="seurat", + layer="counts", n_top_genes=args.n_top_genes, + flavor="pearson_residuals", inplace=True, + batch_key=args.batch_key, ) + full_features = adata.var.copy() + hvg_full = adata.iloc[: args.n_top_genes].index.to_list() + hvg_merged_adata = adata[:, adata.var.index.isin(hvg_full)] + hvg_features = hvg_merged_adata.var.copy() + sc.pl.highly_variable_genes( adata, ) fig = plt.gcf() - fig.suptitle(f"Highly variable genes dispersion plot - {args.plots_prefix}", va="center", ha="center", fontsize=16) - plt.savefig(f"{args.plots_prefix}.hvg_dispersion.png", dpi=300, bbox_inches="tight") + fig.suptitle(f"Highly variable genes dispersion plot - {args.output_prefix}", va="center", ha="center", fontsize=16) + plt.savefig(f"{args.output_prefix}.hvg_dispersion.png", dpi=300, bbox_inches="tight") ############################## @@ -65,6 +72,8 @@ def main(args): ) # Save outputs + full_features.to_csv(f"{args.output_prefix}.all_genes.csv", index=True) + hvg_features.to_csv(f"{args.output_prefix}.hvg_genes.csv", index=True) adata.write_h5ad(filename=args.adata_output, compression="gzip") @@ -128,12 +137,19 @@ if __name__ == "__main__": required=True, help="Number of principal components to compute using `scanpy.pp.pca` [30]" ) + parser.add_argument( + "-b", + "--batch-key", + type=str, + required=True, + help="Key in AnnData object for batch information ['batch_id']" + ) parser.add_argument( "-p", - "--plots-prefix", + "--output-prefix", type=str, required=True, - help="Output file name prefix for the HVGs plot" + help="Output file name prefix for the HVGs plot and CSVs" ) parser.add_argument( "-o", diff --git a/workflows/pmdbs_spatial_geomx/cohort_analysis/cohort_analysis.wdl b/workflows/pmdbs_spatial_geomx/cohort_analysis/cohort_analysis.wdl index 230b259..60d48c8 100644 --- a/workflows/pmdbs_spatial_geomx/cohort_analysis/cohort_analysis.wdl +++ b/workflows/pmdbs_spatial_geomx/cohort_analysis/cohort_analysis.wdl @@ -57,6 +57,7 @@ workflow cohort_analysis { processed_adata_objects = processed_adata_objects, n_top_genes = n_top_genes, n_comps = n_comps, + batch_key = batch_key, raw_data_path = raw_data_path, workflow_info = workflow_info, billing_project = billing_project, @@ -102,8 +103,10 @@ workflow cohort_analysis { ], [ merge_and_prep.merged_adata_object, - merge_and_prep.hvg_plot_png, - merge_and_prep.merged_adata_metadata_csv + merge_and_prep.merged_adata_metadata_csv, + merge_and_prep.all_genes_csv, + merge_and_prep.hvg_genes_csv, + merge_and_prep.hvg_plot_png ], [ integrate_data.clustered_adata_object, @@ -125,8 +128,10 @@ workflow cohort_analysis { # Merged and prepped AnnData object File merged_adata_object = merge_and_prep.merged_adata_object #!FileCoercion - File hvg_plot_png = merge_and_prep.hvg_plot_png #!FileCoercion File merged_adata_metadata_csv = merge_and_prep.merged_adata_metadata_csv #!FileCoercion + File all_genes_csv = merge_and_prep.all_genes_csv #!FileCoercion + File hvg_genes_csv = merge_and_prep.hvg_genes_csv #!FileCoercion + File hvg_plot_png = merge_and_prep.hvg_plot_png #!FileCoercion # Integrate data outputs File integrated_adata_object = integrate_data.integrated_adata_object @@ -171,6 +176,7 @@ task merge_and_prep { Int n_top_genes Int n_comps + String batch_key String raw_data_path Array[Array[String]] workflow_info @@ -189,6 +195,7 @@ task merge_and_prep { --adata-paths-input ~{sep=' ' processed_adata_objects} \ --n-top-genes ~{n_top_genes} \ --n-comps ~{n_comps} \ + --batch-key ~{batch_key} \ --output-prefix ~{cohort_id} \ --adata-output ~{cohort_id}.merged.h5ad @@ -197,15 +204,19 @@ task merge_and_prep { -d ~{raw_data_path} \ -i ~{write_tsv(workflow_info)} \ -o "~{cohort_id}.merged.h5ad" \ - -o "~{cohort_id}.hvg_dispersion.png" \ - -o "~{cohort_id}.merged_adata_metadata.csv" + -o "~{cohort_id}.merged_adata_metadata.csv" \ + -o "~{cohort_id}.all_genes.csv" \ + -o "~{cohort_id}.hvg_genes.csv" \ + -o "~{cohort_id}.hvg_dispersion.png" >>> output { String merged_adata_object = "~{raw_data_path}/~{cohort_id}.merged.h5ad" - String hvg_plot_png = "~{raw_data_path}/~{cohort_id}.hvg_dispersion.png" String merged_adata_metadata_csv = "~{raw_data_path}/~{cohort_id}.merged_adata_metadata.csv" + String all_genes_csv = "~{raw_data_path}/~{cohort_id}.all_genes.csv" + String hvg_genes_csv = "~{raw_data_path}/~{cohort_id}.hvg_genes.csv" + String hvg_plot_png = "~{raw_data_path}/~{cohort_id}.hvg_dispersion.png" } runtime { @@ -228,6 +239,7 @@ task merge_and_prep { processed_adata_objects: {help: "An array of processed AnnData object to merge."} n_top_genes: {help: "Number of highly-variable genes to keep. [3000]"} n_comps: {help: "Number of principal components to compute. [30]"} + batch_key: {help: "Key in AnnData object for batch information. ['batch_id']"} raw_data_path: {help: "Raw data bucket path for merged adata and HVG plot outputs; location of raw bucket to upload task outputs to (`/workflow_execution/cohort_analysis//`)."} workflow_info: {help: "UTC timestamp, workflow name, workflow version, and GitHub release; stored in the file-level manifest and final manifest with all saved files."} billing_project: {help: "Billing project to charge GCP costs."} diff --git a/workflows/pmdbs_spatial_geomx/main.wdl b/workflows/pmdbs_spatial_geomx/main.wdl index 12e7931..5698661 100644 --- a/workflows/pmdbs_spatial_geomx/main.wdl +++ b/workflows/pmdbs_spatial_geomx/main.wdl @@ -198,8 +198,10 @@ workflow pmdbs_spatial_geomx_analysis { ## Merged, integrated and clustered adata objects, and plots Array[File?] project_merged_adata_object = project_cohort_analysis.merged_adata_object - Array[File?] project_hvg_plot_png = project_cohort_analysis.hvg_plot_png Array[File?] project_merged_adata_metadata_csv = project_cohort_analysis.merged_adata_metadata_csv + Array[File?] project_all_genes_csv = project_cohort_analysis.all_genes_csv + Array[File?] project_hvg_genes_csv = project_cohort_analysis.hvg_genes_csv + Array[File?] project_hvg_plot_png = project_cohort_analysis.hvg_plot_png Array[File?] project_integrated_adata_object = project_cohort_analysis.integrated_adata_object Array[File?] project_clustered_adata_object = project_cohort_analysis.clustered_adata_object Array[File?] project_umap_cluster_plots_png = project_cohort_analysis.umap_cluster_plots_png @@ -214,8 +216,10 @@ workflow pmdbs_spatial_geomx_analysis { ## Merged, integrated and clustered adata objects, and plots File? cohort_merged_adata_object = cross_team_cohort_analysis.merged_adata_object - File? cohort_hvg_plot_png = cross_team_cohort_analysis.hvg_plot_png File? cohort_merged_adata_metadata_csv = cross_team_cohort_analysis.merged_adata_metadata_csv + File? cohort_all_genes_csv = cross_team_cohort_analysis.all_genes_csv + File? cohort_hvg_genes_csv = cross_team_cohort_analysis.hvg_genes_csv + File? cohort_hvg_plot_png = cross_team_cohort_analysis.hvg_plot_png File? cohort_integrated_adata_object = cross_team_cohort_analysis.integrated_adata_object File? cohort_clustered_adata_object = cross_team_cohort_analysis.clustered_adata_object File? cohort_umap_cluster_plots_png = cross_team_cohort_analysis.umap_cluster_plots_png diff --git a/workflows/pmdbs_spatial_visium/cohort_analysis/cohort_analysis.wdl b/workflows/pmdbs_spatial_visium/cohort_analysis/cohort_analysis.wdl index eb5fc56..aa19cd4 100644 --- a/workflows/pmdbs_spatial_visium/cohort_analysis/cohort_analysis.wdl +++ b/workflows/pmdbs_spatial_visium/cohort_analysis/cohort_analysis.wdl @@ -78,6 +78,7 @@ workflow cohort_analysis { normalize_target_sum = normalize_target_sum, n_top_genes = n_top_genes, n_comps = n_comps, + batch_key = batch_key, raw_data_path = raw_data_path, workflow_info = workflow_info, billing_project = billing_project, @@ -140,6 +141,8 @@ workflow cohort_analysis { ], merge_and_plot_qc_metrics.qc_plots_png, [ + filter_and_normalize.all_genes_csv, + filter_and_normalize.hvg_genes_csv, filter_and_normalize.hvg_plot_png ], [ @@ -175,6 +178,8 @@ workflow cohort_analysis { # Processed outputs File processed_adata_object = filter_and_normalize.processed_adata_object + File all_genes_csv = filter_and_normalize.all_genes_csv #!FileCoercion + File hvg_genes_csv = filter_and_normalize.hvg_genes_csv #!FileCoercion File hvg_plot_png = filter_and_normalize.hvg_plot_png #!FileCoercion # Integrate data outputs @@ -304,6 +309,7 @@ task filter_and_normalize { Float normalize_target_sum Int n_top_genes Int n_comps + String batch_key String raw_data_path Array[Array[String]] workflow_info @@ -327,18 +333,23 @@ task filter_and_normalize { --target-sum ~{normalize_target_sum} \ --n-top-genes ~{n_top_genes} \ --n-comps ~{n_comps} \ - --plots-prefix ~{cohort_id} \ + --batch-key ~{batch_key} \ + --output-prefix ~{cohort_id} \ --adata-output ~{cohort_id}.processed.h5ad upload_outputs \ -b ~{billing_project} \ -d ~{raw_data_path} \ -i ~{write_tsv(workflow_info)} \ + -o "~{cohort_id}.all_genes.csv" \ + -o "~{cohort_id}.hvg_genes.csv" \ -o "~{cohort_id}.hvg_dispersion.png" >>> output { File processed_adata_object = "~{cohort_id}.processed.h5ad" + String all_genes_csv = "~{raw_data_path}/~{cohort_id}.all_genes.csv" + String hvg_genes_csv = "~{raw_data_path}/~{cohort_id}.hvg_genes.csv" String hvg_plot_png = "~{raw_data_path}/~{cohort_id}.hvg_dispersion.png" } @@ -367,6 +378,7 @@ task filter_and_normalize { normalize_target_sum: {help: "The total count to which each cell's gene expression values will be normalized. [10000]"} n_top_genes: {help: "Number of highly-variable genes to keep. [3000]"} n_comps: {help: "Number of principal components to compute. [30]"} + batch_key: {help: "Key in AnnData object for batch information. ['batch_id']"} raw_data_path: {help: "Raw data bucket path for processed adata and HVG plot outputs; location of raw bucket to upload task outputs to (`/workflow_execution/cohort_analysis//`)."} workflow_info: {help: "UTC timestamp, workflow name, workflow version, and GitHub release; stored in the file-level manifest and final manifest with all saved files."} billing_project: {help: "Billing project to charge GCP costs."} diff --git a/workflows/pmdbs_spatial_visium/main.wdl b/workflows/pmdbs_spatial_visium/main.wdl index 2af290a..45987e4 100644 --- a/workflows/pmdbs_spatial_visium/main.wdl +++ b/workflows/pmdbs_spatial_visium/main.wdl @@ -164,6 +164,8 @@ workflow pmdbs_spatial_visium_analysis { # Merged, processed (filtered, normalized, dimensionality reduced), integrated, and clustered adata objects, and plots Array[File?] project_merged_adata_object = project_cohort_analysis.merged_adata_object Array[File?] project_merged_adata_metadata_csv = project_cohort_analysis.merged_adata_metadata_csv + Array[File?] project_all_genes_csv = project_cohort_analysis.all_genes_csv + Array[File?] project_hvg_genes_csv = project_cohort_analysis.hvg_genes_csv Array[Array[File]?] project_qc_plots_png = project_cohort_analysis.qc_plots_png Array[File?] project_processed_adata_object = project_cohort_analysis.processed_adata_object Array[File?] project_hvg_plot_png = project_cohort_analysis.hvg_plot_png @@ -189,6 +191,8 @@ workflow pmdbs_spatial_visium_analysis { # Merged, processed (filtered, normalized, dimensionality reduced), integrated, and clustered adata objects, and plots File? cohort_merged_adata_object = cross_team_cohort_analysis.merged_adata_object File? cohort_merged_adata_metadata_csv = cross_team_cohort_analysis.merged_adata_metadata_csv + File? cohort_all_genes_csv = cross_team_cohort_analysis.all_genes_csv + File? cohort_hvg_genes_csv = cross_team_cohort_analysis.hvg_genes_csv Array[File]? cohort_qc_plots_png = cross_team_cohort_analysis.qc_plots_png File? cohort_processed_adata_object = cross_team_cohort_analysis.processed_adata_object File? cohort_hvg_plot_png = cross_team_cohort_analysis.hvg_plot_png From f7be1f9c25463bc4327db03688eababeccb7e8f9 Mon Sep 17 00:00:00 2001 From: Karen Fang Date: Mon, 21 Jul 2025 17:05:49 -0400 Subject: [PATCH 03/20] Remove cross-team cohort analysis and update README --- README.md | 81 +++++++++++-------------- workflows/pmdbs_spatial_geomx/main.wdl | 52 ---------------- workflows/pmdbs_spatial_visium/main.wdl | 66 -------------------- 3 files changed, 37 insertions(+), 162 deletions(-) diff --git a/README.md b/README.md index c0e633e..f4b4442 100644 --- a/README.md +++ b/README.md @@ -19,6 +19,8 @@ Worfklows are defined in [the `workflows` directory](workflows). There is [the ` These workflows are set up to analyze spatial transcriptomics data: Nanostring GeoMx in WDL using command line, R, and Python scripts and 10x Visium in WDL using command line and Python scripts. +_Note: Unlike our other workflows (e.g., [pmdbs-sc-rnaseq-wf](https://github.com/ASAP-CRN/pmdbs-sc-rnaseq-wf/tree/main)), the spatial workflows do not perform cross-team cohort analysis, as integrating spatial coordinates and mapping dimensions across datasets adds significant complexity._ + ## Nanostring GeoMx workflow overview For the Nanostring GeoMx workflow, we start with raw output files from the instrument and convert them into counts. Then, we clean the data by removing unreliable segments and genes, adjust for technical noise, and combine data from different slides. Finally, we cluster based on segment (which may contain many cells) and visualize their transcriptional profiles in a UMAP space. @@ -45,7 +47,7 @@ Run once per slide. Intermediate files from previous runs are not reused and are ### Cohort analysis -Run once per team (all slide from a single team) if `project.run_project_cohort_analysis` is set to `true`, and once for the whole cohort (all slides from all teams) if `run_cross_team_cohort_analysis` is set to `true`. Additional slides requires this entire analysis to be rerun. Intermediate files from previous runs are not reused and are stored in timestamped directories. +Run once per team (all slide from a single team) if `project.run_project_cohort_analysis` is set to `true`. Additional slides requires this entire analysis to be rerun. Intermediate files from previous runs are not reused and are stored in timestamped directories. ## 10x Visium workflow overview @@ -69,7 +71,7 @@ Run once per sample; only rerun when the preprocessing workflow version is updat ### Cohort analysis -Run once per team (all samples from a single team) if `project.run_project_cohort_analysis` is set to `true`, and once for the whole cohort (all samples from all teams) if `run_cross_team_cohort_analysis` is set to `true`. This can be rerun using different sample subsets; including additional samples requires this entire analysis to be rerun. Intermediate files from previous runs are not reused and are stored in timestamped directories. +Run once per team (all samples from a single team) if `project.run_project_cohort_analysis` is set to `true`. This can be rerun using different sample subsets; including additional samples requires this entire analysis to be rerun. Intermediate files from previous runs are not reused and are stored in timestamped directories. # Inputs @@ -80,7 +82,6 @@ An input template file can be found at [workflows/pmdbs_spatial_geomx/inputs.jso | Type | Name | Description | | :- | :- | :- | -| String | cohort_id | Name of the cohort; used to name output files during cross-team cohort analysis. | | Array[[Project](#nanostring-geomx-project)] | projects | The project ID, set of slides and their associated samples, reads and metadata, output bucket locations, and whether or not to run project-level cohort analysis. | | File | geomxngs_config_pkc | The GeoMx DSP configuration file to associate assay targets with GeoMx HybCode barcodes and Seq Code primers; see https://nanostring.com/products/geomx-digital-spatial-profiler/geomx-dsp-configuration-files/. | | Int? | min_segment_reads | Minimum number of segment reads. [1000] | @@ -97,9 +98,6 @@ An input template file can be found at [workflows/pmdbs_spatial_geomx/inputs.jso | Int? | n_comps | Number of principal components to compute. [30] | | String? | batch_key | Key in AnnData object for batch information. ['batch_id'] | | Float? | leiden_resolution | Value controlling the coarseness of the Leiden clustering. [0.4] | -| Boolean? | run_cross_team_cohort_analysis | Whether to run downstream harmonization steps on all samples across projects. If set to false, only preprocessing steps (GeoMxNGSPipeline and generating the initial adata object(s)) will run for samples. [false] | -| String | cohort_raw_data_bucket | Bucket to upload cross-team cohort analysis intermediate files to. | -| Array[String] | cohort_staging_data_buckets | Buckets to upload cross-team cohort analysis outputs to. | | String | container_registry | Container registry where workflow Docker images are hosted. | | String? | zones | Space-delimited set of GCP zones where compute will take place. ['us-central1-c us-central1-f'] | @@ -109,7 +107,6 @@ An input template file can be found at [workflows/pmdbs_spatial_visium/inputs.js | Type | Name | Description | | :- | :- | :- | -| String | cohort_id | Name of the cohort; used to name output files during cross-team cohort analysis. | | Array[[Project](#10x-visium-project)] | projects | The project ID, set of samples and their associated reads and metadata, output bucket locations, and whether or not to run project-level cohort analysis. | | File | spaceranger_reference_data | Space Ranger transcriptome reference data; see https://www.10xgenomics.com/support/software/space-ranger/downloads and [10x Visium notes](#10x-visium-notes). | | File | visium_probe_set_csv | Visium probe-based assays target genes in Space Ranger transcriptome; see https://www.10xgenomics.com/support/software/space-ranger/downloads and [10x Visium notes](#10x-visium-notes). | @@ -122,9 +119,6 @@ An input template file can be found at [workflows/pmdbs_spatial_visium/inputs.js | Int? | n_comps | Number of principal components to compute. [30] | | String? | batch_key | Key in AnnData object for batch information. ['batch_id'] | | Float? | leiden_resolution | Value controlling the coarseness of the Leiden clustering. [0.4] | -| Boolean? | run_cross_team_cohort_analysis | Whether to run downstream harmonization steps on all samples across projects. If set to false, only preprocessing steps ( and generating the initial adata object(s)) will run for samples. [false] | -| String | cohort_raw_data_bucket | Bucket to upload cross-team cohort analysis intermediate files to. | -| Array[String] | cohort_staging_data_buckets | Buckets to upload cross-team cohort analysis outputs to. | | String | container_registry | Container registry where workflow Docker images are hosted. | | String? | zones | Space-delimited set of GCP zones where compute will take place. ['us-central1-c us-central1-f'] | @@ -195,7 +189,7 @@ An input template file can be found at [workflows/pmdbs_spatial_visium/inputs.js ## Generating the inputs JSON -The inputs JSON may be generated manually, however when running a large number of samples, this can become unwieldly. The `generate_inputs` utility script may be used to automatically generate the inputs JSON (`inputs.{staging_env}.{source}-{cohort_dataset}.{date}.json` and a sample list TSV (`{team_id}.{source}-{cohort_dataset}.sample_list.{date}.tsv`; same as the one generated in [the write_cohort_sample_list task](https://github.com/ASAP-CRN/wf-common/wdl/tasks/write_cohort_sample_list.wdl)). The script requires the libraries outlined in [the requirements.txt file](https://github.com/ASAP-CRN/wf-common/util/requirements.txt) and the following inputs: +The inputs JSON may be generated manually, however when running a large number of samples, this can become unwieldly. The [`generate_inputs` utility script](https://github.com/ASAP-CRN/wf-common/blob/main/util/generate_inputs) may be used to automatically generate the inputs JSON (`inputs.{staging_env}.{source}-{cohort_dataset}.{date}.json` and a sample list TSV (`{team_id}.{source}-{cohort_dataset}.sample_list.{date}.tsv`; same as the one generated in [the write_cohort_sample_list task](https://github.com/ASAP-CRN/wf-common/wdl/tasks/write_cohort_sample_list.wdl)). The script requires the libraries outlined in [the requirements.txt file](https://github.com/ASAP-CRN/wf-common/util/requirements.txt) and the following inputs: - `project-tsv`: One or more project TSVs with one row per sample and columns team_id, ASAP_dataset_id, ASAP_sample_id, batch, fastq_R1s, fastq_R2s, fastq_I1s, fastq_I2s, embargoed, source, dataset, dataset_DOI_url, and SPATIAL columns if applicable: geomx_config, geomx_dsp_config, geomx_annotation_file, visium_cytassist, visium_probe_set, visium_slide_ref, and visium_capture_area. All samples from all projects may be included in the same project TSV, or multiple project TSVs may be provided. - `team_id`: A unique identifier for the team from which the sample(s) arose. @@ -226,14 +220,14 @@ Example usage: ```bash ./wf-common/util/generate_inputs \ --project-tsv metadata.tsv \ - --inputs-template workflows/inputs.json \ + --inputs-template workflows/pmdbs_spatial_geomx/inputs.json \ --run-project-cohort-analysis \ --workflow-name pmdbs_spatial_geomx_analysis \ --cohort-dataset spatial-geomx ./wf-common/util/generate_inputs \ --project-tsv metadata.tsv \ - --inputs-template workflows/inputs.json \ + --inputs-template workflows/pmdbs_spatial_visium/inputs.json \ --run-project-cohort-analysis \ --workflow-name pmdbs_spatial_visium_analysis \ --cohort-dataset spatial-visium @@ -246,7 +240,7 @@ Example usage: - `cohort_id`: either the `team_id` for project-level downstream analysis, or the `cohort_id` for the full cohort - `workflow_run_timestamp`: format: `%Y-%m-%dT%H-%M-%SZ` - The list of samples used to generate the cohort analysis will be output alongside other cohort analysis outputs in the staging data bucket (`${cohort_id}.sample_list.tsv`) -- The MANIFEST.tsv file in the staging data bucket describes the file name, md5 hash, timestamp, workflow version, workflow name, and workflow release for the run used to generate each file in that directory +- The `MANIFEST.tsv` file in the staging data bucket describes the file name, md5 hash, timestamp, workflow version, workflow name, and workflow release for the run used to generate each file in that directory ### Raw data (intermediate files and final outputs for all runs of the workflow) @@ -298,22 +292,22 @@ asap-raw-{cohort,team-xxyy}-{source}-{dataset} ### Staging data (intermediate workflow objects and final workflow outputs for the latest run of the workflow) -Following QC by researchers, the objects in the dev or uat bucket are synced into the curated data buckets, maintaining the same file structure. Curated data buckets are named `asap-curated-{cohort,team-xxyy}-{source}-{dataset}`. +Following QC by researchers, the objects in the dev or uat bucket are synced into the curated data buckets, maintaining the same file structure. Curated data buckets are named `asap-curated-{team-xxyy}-{source}-{dataset}`. Data may be synced using [the `promote_staging_data` script](#promoting-staging-data). ```bash -asap-dev-{cohort,team-xxyy}-{source}-{dataset} +asap-dev-{team-xxyy}-{source}-{dataset} └── pmdbs_spatial_geomx ├── cohort_analysis - │ ├── ${cohort_id}.sample_list.tsv - │ ├── ${cohort_id}.merged.h5ad - │ ├── ${cohort_id}.merged_adata_metadata.csv - │ ├── ${cohort_id}.all_genes.csv - │ ├── ${cohort_id}.hvg_genes.csv - │ ├── ${cohort_id}.hvg_dispersion.png - │ ├── ${cohort_id}.clustered.h5ad # Final - │ ├── ${cohort_id}.umap_cluster.png + │ ├── ${team_id}.sample_list.tsv + │ ├── ${team_id}.merged.h5ad + │ ├── ${team_id}.merged_adata_metadata.csv + │ ├── ${team_id}.all_genes.csv + │ ├── ${team_id}.hvg_genes.csv + │ ├── ${team_id}.hvg_dispersion.png + │ ├── ${team_id}.clustered.h5ad # Final + │ ├── ${team_id}.umap_cluster.png │ └── MANIFEST.tsv ├── process_to_adata │ ├── ${slideN_id}.segment_gene_detection_plot.png @@ -340,23 +334,23 @@ asap-dev-{cohort,team-xxyy}-{source}-{dataset} ├── ${slideN_id}.gene_count.csv └── MANIFEST.tsv -asap-dev-{cohort,team-xxyy}-{source}-{dataset} +asap-dev-{team-xxyy}-{source}-{dataset} └── pmdbs_spatial_visium ├── cohort_analysis - │ ├── ${cohort_id}.sample_list.tsv - │ ├── ${cohort_id}.merged.h5ad - │ ├── ${cohort_id}.merged_adata_metadata.csv - │ ├── ${cohort_id}.all_genes.csv - │ ├── ${cohort_id}.hvg_genes.csv - │ ├── ${cohort_id}.qc_violin.png - │ ├── ${cohort_id}.qc_dist.png - │ ├── ${cohort_id}.hvg_dispersion.png - │ ├── ${cohort_id}.clustered.h5ad - │ ├── ${cohort_id}.umap_cluster.png - │ ├── ${cohort_id}.spatial_scatter.png - │ ├── ${cohort_id}.final.h5ad - │ ├── ${cohort_id}.moran_top_10_variable_genes.csv - │ ├── ${cohort_id}.moran_top_4_variable_genes_spatial_scatter.png + │ ├── ${team_id}.sample_list.tsv + │ ├── ${team_id}.merged.h5ad + │ ├── ${team_id}.merged_adata_metadata.csv + │ ├── ${team_id}.all_genes.csv + │ ├── ${team_id}.hvg_genes.csv + │ ├── ${team_id}.qc_violin.png + │ ├── ${team_id}.qc_dist.png + │ ├── ${team_id}.hvg_dispersion.png + │ ├── ${team_id}.clustered.h5ad + │ ├── ${team_id}.umap_cluster.png + │ ├── ${team_id}.spatial_scatter.png + │ ├── ${team_id}.final.h5ad + │ ├── ${team_id}.moran_top_10_variable_genes.csv + │ ├── ${team_id}.moran_top_4_variable_genes_spatial_scatter.png │ └── MANIFEST.tsv └── preprocess ├── ${sampleA_id}.raw_feature_bc_matrix.h5 @@ -414,7 +408,6 @@ The script defaults to a dry run, printing out the files that would be copied or -d Space-delimited dataset name(s) in team bucket name, must follow the same order as {team} -w Workflow name used as a directory in bucket -p Promote data. If this option is not selected, data that would be copied or deleted is printed out, but files are not actually changed (dry run) --e Staging bucket type; options are 'uat' or 'dev' ['uat'] ``` ### Usage @@ -424,11 +417,11 @@ The script defaults to a dry run, printing out the files that would be copied or ./wf-common/util/promote_staging_data -t cohort -l -s pmdbs -d spatial-geomx -w pmdbs_spatial_geomx ./wf-common/util/promote_staging_data -t cohort -l -s pmdbs -d spatial-visium -w pmdbs_spatial_visium -# Print out the files that would be copied or deleted from the staging bucket to the curated bucket for teams team-hardy and team-biederer -./wf-common/util/promote_staging_data -t team-hardy team-biederer -s pmdbs -d spatial-geomx -w pmdbs_spatial_geomx +# Print out the files that would be copied or deleted from the staging bucket to the curated bucket for teams team-edwards and team-vila +./wf-common/util/promote_staging_data -t team-edwards team-vila -s pmdbs -d spatial-geomx-th spatial-geomx-thlc -w pmdbs_spatial_geomx -# Promote data for team-hardy and cohort -./wf-common/util/promote_staging_data -t team-hardy cohort -s pmdbs -d spatial-geomx -w pmdbs_spatial_geomx -p -e dev +# Promote data for team-edwards and team-vila +./wf-common/util/promote_staging_data -t team-edwards team-vila -s pmdbs -d spatial-geomx-th spatial-geomx-thlc -w pmdbs_spatial_geomx -p ``` # Docker images diff --git a/workflows/pmdbs_spatial_geomx/main.wdl b/workflows/pmdbs_spatial_geomx/main.wdl index 5698661..cc73c57 100644 --- a/workflows/pmdbs_spatial_geomx/main.wdl +++ b/workflows/pmdbs_spatial_geomx/main.wdl @@ -10,7 +10,6 @@ import "cohort_analysis/cohort_analysis.wdl" as CohortAnalysis workflow pmdbs_spatial_geomx_analysis { input { - String cohort_id Array[Project] projects File geomxngs_config_pkc @@ -36,11 +35,6 @@ workflow pmdbs_spatial_geomx_analysis { String batch_key = "batch_id" Float leiden_resolution = 0.4 - # Cohort analysis - Boolean run_cross_team_cohort_analysis = false - String cohort_raw_data_bucket - Array[String] cohort_staging_data_buckets - String container_registry String zones = "us-central1-c us-central1-f" } @@ -143,32 +137,6 @@ workflow pmdbs_spatial_geomx_analysis { } } - if (run_cross_team_cohort_analysis) { - String cohort_raw_data_path_prefix = "~{cohort_raw_data_bucket}/~{workflow_execution_path}/~{workflow_name}" - - call CohortAnalysis.cohort_analysis as cross_team_cohort_analysis { - input: - cohort_id = cohort_id, - project_sample_ids = flatten(preprocess.project_sample_ids), - processed_adata_objects = flatten(process_to_adata.processed_adata_objects), - preprocessing_output_file_paths = flatten(preprocessing_output_file_paths), - processing_output_file_paths = flatten(processing_output_file_paths), - n_top_genes = n_top_genes, - n_comps = n_comps, - batch_key = batch_key, - leiden_resolution = leiden_resolution, - workflow_name = workflow_name, - workflow_version = workflow_version, - workflow_release = workflow_release, - run_timestamp = get_workflow_metadata.timestamp, - raw_data_path_prefix = cohort_raw_data_path_prefix, - staging_data_buckets = cohort_staging_data_buckets, - billing_project = get_workflow_metadata.billing_project, - container_registry = container_registry, - zones = zones - } - } - output { # Sample-level outputs ## Sample list @@ -209,22 +177,6 @@ workflow pmdbs_spatial_geomx_analysis { Array[Array[File]?] preprocess_manifests = project_cohort_analysis.preprocess_manifest_tsvs Array[Array[File]?] process_to_adata_manifests = project_cohort_analysis.process_to_adata_manifest_tsvs Array[Array[File]?] project_manifests = project_cohort_analysis.cohort_analysis_manifest_tsvs - - # Cross-team cohort analysis outputs - ## List of samples included in the cohort - File? cohort_cohort_sample_list = cross_team_cohort_analysis.cohort_sample_list - - ## Merged, integrated and clustered adata objects, and plots - File? cohort_merged_adata_object = cross_team_cohort_analysis.merged_adata_object - File? cohort_merged_adata_metadata_csv = cross_team_cohort_analysis.merged_adata_metadata_csv - File? cohort_all_genes_csv = cross_team_cohort_analysis.all_genes_csv - File? cohort_hvg_genes_csv = cross_team_cohort_analysis.hvg_genes_csv - File? cohort_hvg_plot_png = cross_team_cohort_analysis.hvg_plot_png - File? cohort_integrated_adata_object = cross_team_cohort_analysis.integrated_adata_object - File? cohort_clustered_adata_object = cross_team_cohort_analysis.clustered_adata_object - File? cohort_umap_cluster_plots_png = cross_team_cohort_analysis.umap_cluster_plots_png - - Array[File]? cohort_manifests = cross_team_cohort_analysis.cohort_analysis_manifest_tsvs } meta { @@ -232,7 +184,6 @@ workflow pmdbs_spatial_geomx_analysis { } parameter_meta { - cohort_id: {help: "Name of the cohort; used to name output files during cross-team downstream analysis."} projects: {help: "The project ID, set of slides and their associated samples, reads and metadata, output bucket locations, and whether or not to run project-level downstream analysis."} geomxngs_config_pkc: {help: "The GeoMx DSP configuration file to associate assay targets with GeoMx HybCode barcodes and Seq Code primers."} min_segment_reads: {help: "Minimum number of segment reads. [1000]"} @@ -250,9 +201,6 @@ workflow pmdbs_spatial_geomx_analysis { n_comps: {help: "Number of principal components to compute. [30]"} batch_key: {help: "Key in AnnData object for batch information. ['batch_id']"} leiden_resolution: {help: "Value controlling the coarseness of the Leiden clustering. [0.4]"} - run_cross_team_cohort_analysis: {help: "Whether to run downstream harmonization steps on all samples across projects. If set to false, only preprocessing steps (GeoMxNGSPipeline and generating the initial adata object(s)) will run for samples. [false]"} - cohort_raw_data_bucket: {help: "Bucket to upload cross-team downstream intermediate files to."} - cohort_staging_data_buckets: {help: "Set of buckets to stage cross-team downstream analysis outputs in."} container_registry: {help: "Container registry where workflow Docker images are hosted."} zones: {help: "Space-delimited set of GCP zones where compute will take place."} } diff --git a/workflows/pmdbs_spatial_visium/main.wdl b/workflows/pmdbs_spatial_visium/main.wdl index 45987e4..6dc5d98 100644 --- a/workflows/pmdbs_spatial_visium/main.wdl +++ b/workflows/pmdbs_spatial_visium/main.wdl @@ -9,7 +9,6 @@ import "cohort_analysis/cohort_analysis.wdl" as CohortAnalysis workflow pmdbs_spatial_visium_analysis { input { - String cohort_id Array[Project] projects File spaceranger_reference_data @@ -26,11 +25,6 @@ workflow pmdbs_spatial_visium_analysis { String batch_key = "batch_id" Float leiden_resolution = 0.4 - # Cohort analysis - Boolean run_cross_team_cohort_analysis = false - String cohort_raw_data_bucket - Array[String] cohort_staging_data_buckets - String container_registry String zones = "us-central1-c us-central1-f" } @@ -109,36 +103,6 @@ workflow pmdbs_spatial_visium_analysis { } } - if (run_cross_team_cohort_analysis) { - String cohort_raw_data_path_prefix = "~{cohort_raw_data_bucket}/~{workflow_execution_path}/~{workflow_name}" - - call CohortAnalysis.cohort_analysis as cross_team_cohort_analysis { - input: - cohort_id = cohort_id, - project_sample_ids = flatten(preprocess.project_sample_ids), - preprocessed_adata_objects = flatten(preprocess.qc_adata_object), - preprocessing_output_file_paths = flatten(preprocessing_output_file_paths), - filter_cells_min_counts = filter_cells_min_counts, - filter_cells_min_genes = filter_cells_min_genes, - filter_genes_min_cells = filter_genes_min_cells, - filter_mt_max_percent = filter_mt_max_percent, - normalize_target_sum = normalize_target_sum, - n_top_genes = n_top_genes, - n_comps = n_comps, - batch_key = batch_key, - leiden_resolution = leiden_resolution, - workflow_name = workflow_name, - workflow_version = workflow_version, - workflow_release = workflow_release, - run_timestamp = get_workflow_metadata.timestamp, - raw_data_path_prefix = cohort_raw_data_path_prefix, - staging_data_buckets = cohort_staging_data_buckets, - billing_project = get_workflow_metadata.billing_project, - container_registry = container_registry, - zones = zones - } - } - output { # Sample-level outputs ## Sample list @@ -183,32 +147,6 @@ workflow pmdbs_spatial_visium_analysis { Array[Array[File]?] preprocess_manifests = project_cohort_analysis.preprocess_manifest_tsvs Array[Array[File]?] project_manifests = project_cohort_analysis.cohort_analysis_manifest_tsvs - - # Cross-team cohort analysis outputs - ## List of samples included in the cohort - File? cohort_cohort_sample_list = cross_team_cohort_analysis.cohort_sample_list - - # Merged, processed (filtered, normalized, dimensionality reduced), integrated, and clustered adata objects, and plots - File? cohort_merged_adata_object = cross_team_cohort_analysis.merged_adata_object - File? cohort_merged_adata_metadata_csv = cross_team_cohort_analysis.merged_adata_metadata_csv - File? cohort_all_genes_csv = cross_team_cohort_analysis.all_genes_csv - File? cohort_hvg_genes_csv = cross_team_cohort_analysis.hvg_genes_csv - Array[File]? cohort_qc_plots_png = cross_team_cohort_analysis.qc_plots_png - File? cohort_processed_adata_object = cross_team_cohort_analysis.processed_adata_object - File? cohort_hvg_plot_png = cross_team_cohort_analysis.hvg_plot_png - File? cohort_integrated_adata_object = cross_team_cohort_analysis.integrated_adata_object - File? cohort_clustered_adata_object = cross_team_cohort_analysis.clustered_adata_object - File? cohort_umap_cluster_plots_png = cross_team_cohort_analysis.umap_cluster_plots_png - - # Spatial plots - File? cohort_spatial_scatter_plot_png = cross_team_cohort_analysis.spatial_scatter_plot_png - - # Spatial statistics outputs - File? cohort_final_adata_object = cross_team_cohort_analysis.final_adata_object - File? cohort_moran_top_10_variable_genes_csv = cross_team_cohort_analysis.moran_top_10_variable_genes_csv - File? cohort_moran_top_4_variable_genes_spatial_scatter_plot_png = cross_team_cohort_analysis.moran_top_4_variable_genes_spatial_scatter_plot_png - - Array[File]? cohort_manifests = cross_team_cohort_analysis.cohort_analysis_manifest_tsvs } meta { @@ -216,7 +154,6 @@ workflow pmdbs_spatial_visium_analysis { } parameter_meta { - cohort_id: {help: "Name of the cohort; used to name output files during cross-team downstream analysis."} projects: {help: "The project ID, set of samples and their associated reads and metadata, output bucket locations, and whether or not to run project-level downstream analysis."} spaceranger_reference_data: {help: "Space Ranger transcriptome reference data; see https://www.10xgenomics.com/support/software/space-ranger/downloads."} visium_probe_set_csv: {help: "Visium probe-based assays target genes in Space Ranger transcriptome; see https://www.10xgenomics.com/support/software/space-ranger/downloads."} @@ -229,9 +166,6 @@ workflow pmdbs_spatial_visium_analysis { n_comps: {help: "Number of principal components to compute. [30]"} batch_key: {help: "Key in AnnData object for batch information. ['batch_id']"} leiden_resolution: {help: "Value controlling the coarseness of the Leiden clustering. [0.4]"} - run_cross_team_cohort_analysis: {help: "Whether to run downstream harmonization steps on all samples across projects. If set to false, only preprocessing steps (GeoMxNGSPipeline and generating the initial adata object(s)) will run for samples. [false]"} - cohort_raw_data_bucket: {help: "Bucket to upload cross-team downstream intermediate files to."} - cohort_staging_data_buckets: {help: "Set of buckets to stage cross-team downstream analysis outputs in."} container_registry: {help: "Container registry where workflow Docker images are hosted."} zones: {help: "Space-delimited set of GCP zones where compute will take place. ['us-central1-c us-central1-f']"} } From 44bcfc76a6717953cc6c83a380371512f826b048 Mon Sep 17 00:00:00 2001 From: Karen Fang Date: Mon, 21 Jul 2025 17:11:15 -0400 Subject: [PATCH 04/20] Update inputs JSON --- workflows/pmdbs_spatial_geomx/inputs.json | 4 ---- workflows/pmdbs_spatial_visium/inputs.json | 4 ---- 2 files changed, 8 deletions(-) diff --git a/workflows/pmdbs_spatial_geomx/inputs.json b/workflows/pmdbs_spatial_geomx/inputs.json index 68418e8..89a7f40 100644 --- a/workflows/pmdbs_spatial_geomx/inputs.json +++ b/workflows/pmdbs_spatial_geomx/inputs.json @@ -1,5 +1,4 @@ { - "pmdbs_spatial_geomx_analysis.cohort_id": "String", "pmdbs_spatial_geomx_analysis.projects": "Array[WomCompositeType {\n slides -> Array[WomCompositeType {\n slide_id -> String\ngeomx_lab_annotation_xlsx -> File\nsamples -> Array[WomCompositeType {\n fastq_I1s -> Array[File]\nsample_id -> String\nfastq_R1s -> Array[File]+\nbatch -> String?\nfastq_I2s -> Array[File]\nfastq_R2s -> Array[File]+ \n}] \n}]\nproject_sample_metadata_csv -> File\nteam_id -> String\ndataset_id -> String\ngeomx_config_ini -> File\nstaging_data_buckets -> Array[String]\nrun_project_cohort_analysis -> Boolean\ndataset_doi_url -> String\nraw_data_bucket -> String \n}]", "pmdbs_spatial_geomx_analysis.geomxngs_config_pkc": "File", "pmdbs_spatial_geomx_analysis.min_segment_reads": "Int (optional, default = 1000)", @@ -17,9 +16,6 @@ "pmdbs_spatial_geomx_analysis.n_comps": "Int (optional, default = 30)", "pmdbs_spatial_geomx_analysis.batch_key": "String (optional, default = \"batch_id\")", "pmdbs_spatial_geomx_analysis.leiden_resolution": "Float (optional, default = 0.4)", - "pmdbs_spatial_geomx_analysis.run_cross_team_cohort_analysis": "Boolean (optional, default = false)", - "pmdbs_spatial_geomx_analysis.cohort_raw_data_bucket": "String", - "pmdbs_spatial_geomx_analysis.cohort_staging_data_buckets": "Array[String]", "pmdbs_spatial_geomx_analysis.container_registry": "String", "pmdbs_spatial_geomx_analysis.zones": "String (optional, default = \"us-central1-c us-central1-f\")" } diff --git a/workflows/pmdbs_spatial_visium/inputs.json b/workflows/pmdbs_spatial_visium/inputs.json index b1a17aa..d165e2c 100644 --- a/workflows/pmdbs_spatial_visium/inputs.json +++ b/workflows/pmdbs_spatial_visium/inputs.json @@ -1,5 +1,4 @@ { - "pmdbs_spatial_visium_analysis.cohort_id": "String", "pmdbs_spatial_visium_analysis.projects": "Array[WomCompositeType {\n team_id -> String\ndataset_id -> String\nsamples -> Array[WomCompositeType {\n fastq_I1s -> Array[File]\nsample_id -> String\nfastq_R1s -> Array[File]+\nbatch -> String?\nvisium_capture_area -> String\nvisium_brightfield_image -> File\nfastq_I2s -> Array[File]\nvisium_slide_serial_number -> String\nfastq_R2s -> Array[File]+ \n}]\nstaging_data_buckets -> Array[String]\nrun_project_cohort_analysis -> Boolean\ndataset_doi_url -> String\nraw_data_bucket -> String \n}]", "pmdbs_spatial_visium_analysis.spaceranger_reference_data": "File", "pmdbs_spatial_visium_analysis.visium_probe_set_csv": "File? (optional)", @@ -12,9 +11,6 @@ "pmdbs_spatial_visium_analysis.n_comps": "Int (optional, default = 30)", "pmdbs_spatial_visium_analysis.batch_key": "String (optional, default = \"batch_id\")", "pmdbs_spatial_visium_analysis.leiden_resolution": "Float (optional, default = 0.4)", - "pmdbs_spatial_visium_analysis.run_cross_team_cohort_analysis": "Boolean (optional, default = false)", - "pmdbs_spatial_visium_analysis.cohort_raw_data_bucket": "String", - "pmdbs_spatial_visium_analysis.cohort_staging_data_buckets": "Array[String]", "pmdbs_spatial_visium_analysis.container_registry": "String", "pmdbs_spatial_visium_analysis.zones": "String (optional, default = \"us-central1-c us-central1-f\")" } From f40d88d319a1c68f6349e0bc79a6e4f84c1c2135 Mon Sep 17 00:00:00 2001 From: Karen Fang Date: Mon, 21 Jul 2025 17:29:33 -0400 Subject: [PATCH 05/20] Update wdl-ci --- wdl-ci.config.json | 2 ++ 1 file changed, 2 insertions(+) diff --git a/wdl-ci.config.json b/wdl-ci.config.json index 15d8e78..b1fba18 100644 --- a/wdl-ci.config.json +++ b/wdl-ci.config.json @@ -18,6 +18,7 @@ ], "n_top_genes": 3000, "n_comps": 30, + "batch_key": "batch_id", "raw_data_path": "${geomx_cohort_analysis_raw_data_path}", "workflow_info": "${geomx_workflow_info}", "billing_project": "${billing_project}", @@ -328,6 +329,7 @@ "normalize_target_sum": 10000, "n_top_genes": 3000, "n_comps": 30, + "batch_key": "batch_id", "raw_data_path": "${visium_cohort_analysis_raw_data_path}", "workflow_info": "${visium_workflow_info}", "billing_project": "${billing_project}", From 6186f36c797072f6a61bd414374dbcb8a41944c0 Mon Sep 17 00:00:00 2001 From: Karen Fang Date: Mon, 21 Jul 2025 18:22:37 -0400 Subject: [PATCH 06/20] Update wdl-ci --- wdl-ci.config.json | 58 +++++++++++++++++++++++++++++++++++++--------- 1 file changed, 47 insertions(+), 11 deletions(-) diff --git a/wdl-ci.config.json b/wdl-ci.config.json index b1fba18..7e016b3 100644 --- a/wdl-ci.config.json +++ b/wdl-ci.config.json @@ -33,22 +33,40 @@ "check_hdf5" ] }, - "hvg_plot_png": { - "value": "${geomx_input_file_path}/${cohort_analysis_workflow_name}/${geomx_team_id}.hvg_dispersion.png", + "merged_adata_metadata_csv": { + "value": "${geomx_input_file_path}/${cohort_analysis_workflow_name}/${geomx_team_id}.merged_adata_metadata.csv", "test_tasks": [ "compare_file_basename", - "png_validator", - "image_validator" + "check_empty_lines", + "check_comma_separated", + "count_columns" ] }, - "merged_adata_metadata_csv": { - "value": "${geomx_input_file_path}/${cohort_analysis_workflow_name}/${geomx_team_id}.merged_adata_metadata.csv", + "all_genes_csv": { + "value": "${geomx_input_file_path}/${cohort_analysis_workflow_name}/${geomx_team_id}.all_genes.csv", + "test_tasks": [ + "compare_file_basename", + "check_empty_lines", + "check_comma_separated", + "count_columns" + ] + }, + "hvg_genes_csv": { + "value": "${geomx_input_file_path}/${cohort_analysis_workflow_name}/${geomx_team_id}.hvg_genes.csv", "test_tasks": [ "compare_file_basename", "check_empty_lines", "check_comma_separated", "count_columns" ] + }, + "hvg_plot_png": { + "value": "${geomx_input_file_path}/${cohort_analysis_workflow_name}/${geomx_team_id}.hvg_dispersion.png", + "test_tasks": [ + "compare_file_basename", + "png_validator", + "image_validator" + ] } } } @@ -293,7 +311,7 @@ }, "output_tests": { "merged_adata_object": { - "value": "${visium_input_file_path}/${cohort_analysis_workflow_name}/${visium_team_id}.merged_adata_object.h5ad", + "value": "${visium_input_file_path}/${cohort_analysis_workflow_name}/${visium_team_id}.merged.h5ad", "test_tasks": [ "compare_file_basename", "check_hdf5" @@ -321,7 +339,7 @@ { "inputs": { "cohort_id": "${visium_team_id}", - "merged_adata_object": "${visium_input_file_path}/${cohort_analysis_workflow_name}/${visium_team_id}.merged_adata_object.h5ad", + "merged_adata_object": "${visium_input_file_path}/${cohort_analysis_workflow_name}/${visium_team_id}.merged.h5ad", "filter_cells_min_counts": 5000, "filter_cells_min_genes": 3000, "filter_genes_min_cells": 10, @@ -344,6 +362,24 @@ "check_hdf5" ] }, + "all_genes_csv": { + "value": "${visium_input_file_path}/${cohort_analysis_workflow_name}/${visium_team_id}.all_genes.csv", + "test_tasks": [ + "compare_file_basename", + "check_empty_lines", + "check_comma_separated", + "count_columns" + ] + }, + "hvg_genes_csv": { + "value": "${visium_input_file_path}/${cohort_analysis_workflow_name}/${visium_team_id}.hvg_genes.csv", + "test_tasks": [ + "compare_file_basename", + "check_empty_lines", + "check_comma_separated", + "count_columns" + ] + }, "hvg_plot_png": { "value": "${visium_input_file_path}/${cohort_analysis_workflow_name}/${visium_team_id}.hvg_dispersion.png", "test_tasks": [ @@ -406,7 +442,7 @@ }, "output_tests": { "final_adata_object": { - "value": "${visium_input_file_path}/${cohort_analysis_workflow_name}/${visium_team_id}.final_adata_object.h5ad", + "value": "${visium_input_file_path}/${cohort_analysis_workflow_name}/${visium_team_id}.final.h5ad", "test_tasks": [ "compare_file_basename", "check_hdf5" @@ -566,7 +602,7 @@ }, "output_tests": { "initial_adata_object": { - "value": "${visium_input_file_path}/${preprocess_workflow_name}/${visium_sample_id}.initial_adata_object.h5ad", + "value": "${visium_input_file_path}/${preprocess_workflow_name}/${visium_sample_id}.initial.h5ad", "test_tasks": [ "compare_file_basename", "check_hdf5" @@ -583,7 +619,7 @@ { "inputs": { "sample_id": "${visium_sample_id}", - "initial_adata_object": "${visium_input_file_path}/${preprocess_workflow_name}/${visium_sample_id}.initial_adata_object.h5ad", + "initial_adata_object": "${visium_input_file_path}/${preprocess_workflow_name}/${visium_sample_id}.initial.h5ad", "raw_data_path": "${visium_qc_raw_data_path}", "workflow_info": "${visium_workflow_info}", "billing_project": "${billing_project}", From 05ec36286c0e3bc4b58d80e7a763ec319a1f31b6 Mon Sep 17 00:00:00 2001 From: github-actions <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 21 Jul 2025 22:41:47 +0000 Subject: [PATCH 07/20] update wdl-ci config file after successful tests --- wdl-ci.config.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/wdl-ci.config.json b/wdl-ci.config.json index 7e016b3..6126ac4 100644 --- a/wdl-ci.config.json +++ b/wdl-ci.config.json @@ -294,7 +294,7 @@ "tasks": { "merge_and_plot_qc_metrics": { "key": "merge_and_plot_qc_metrics", - "digest": "wmyxopfjayx7b473xjcvghxm4zyu32fw", + "digest": "w5637hnyv47nzalkr72puv7cixhgx5pu", "tests": [ { "inputs": { @@ -428,7 +428,7 @@ "tasks": { "spatially_variable_gene_analysis": { "key": "spatially_variable_gene_analysis", - "digest": "evtq6h2q4bix4eglhh5ptj6hvgarmoqu", + "digest": "pbh5hevnohvtahgqfegk32kwzqefn7xo", "tests": [ { "inputs": { @@ -583,7 +583,7 @@ }, "counts_to_adata": { "key": "counts_to_adata", - "digest": "j53mpjhjsfrcldiwhxeqm5jyhmkganmh", + "digest": "itpk7kn2vwno4o5bthjsetdmk6bvq2cb", "tests": [ { "inputs": { From 5b5b4fb73e80c1c5f6e984ef86d7bee87f9ccd5c Mon Sep 17 00:00:00 2001 From: Karen Fang Date: Tue, 22 Jul 2025 09:28:56 -0400 Subject: [PATCH 08/20] Bump package versions --- docker/spatial_py/requirements.txt | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docker/spatial_py/requirements.txt b/docker/spatial_py/requirements.txt index b0daffd..4c74280 100644 --- a/docker/spatial_py/requirements.txt +++ b/docker/spatial_py/requirements.txt @@ -1,13 +1,13 @@ argparse==1.4.0 -anndata==0.10.9 -numpy==2.0.1 -pandas==2.2.2 -scipy==1.15.2 +anndata==0.11.4 +numpy==2.2.0 +pandas==2.3.1 +scipy==1.15.3 squidpy==1.6.2 matplotlib==3.10.0 seaborn==0.13.2 harmonypy==0.0.10 -scanpy==1.10.4 +scanpy==1.11.3 dask==2024.11.2 distributed==2024.11.2 numcodecs==0.11.0 From 25ed6d8811aeb054c9cd58e68b6aa0d049a28912 Mon Sep 17 00:00:00 2001 From: Karen Fang Date: Tue, 22 Jul 2025 10:34:06 -0400 Subject: [PATCH 09/20] Use scanpy experimental hvg --- docker/spatial_py/scripts/geomx_merge_and_prep | 3 ++- docker/spatial_py/scripts/visium_process | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/docker/spatial_py/scripts/geomx_merge_and_prep b/docker/spatial_py/scripts/geomx_merge_and_prep index e1c5033..d0c615b 100755 --- a/docker/spatial_py/scripts/geomx_merge_and_prep +++ b/docker/spatial_py/scripts/geomx_merge_and_prep @@ -26,7 +26,8 @@ def main(args): ################## sc.pp.log1p(merged_adata) - sc.pp.highly_variable_genes( + merged_adata.layers["counts"] = merged_adata.X.copy() + sc.experimental.pp.highly_variable_genes( merged_adata, layer="counts", n_top_genes=args.n_top_genes, diff --git a/docker/spatial_py/scripts/visium_process b/docker/spatial_py/scripts/visium_process index 0b9da99..858d50f 100755 --- a/docker/spatial_py/scripts/visium_process +++ b/docker/spatial_py/scripts/visium_process @@ -41,7 +41,7 @@ def main(args): ################## ## ANNOTATE HVG ## ################## - sc.pp.highly_variable_genes( + sc.experimental.pp.highly_variable_genes( adata, layer="counts", n_top_genes=args.n_top_genes, From 339878eb94e611cb15532bc0886762e1649b1bbc Mon Sep 17 00:00:00 2001 From: Karen Fang Date: Tue, 22 Jul 2025 12:07:35 -0400 Subject: [PATCH 10/20] Rename output files and export final metadata and adata --- README.md | 17 +-- .../scripts/geomx_export_final_artifacts | 47 ++++++++ .../spatial_py/scripts/geomx_merge_and_prep | 19 ++-- docker/spatial_py/scripts/visium_process | 17 ++- .../scripts/visium_spatially_variable_genes | 2 + wdl-ci.config.json | 62 +++++++++-- workflows/integrate_data/integrate_data.wdl | 5 +- .../cohort_analysis/cohort_analysis.wdl | 103 ++++++++++++++++-- workflows/pmdbs_spatial_geomx/main.wdl | 6 +- .../cohort_analysis/cohort_analysis.wdl | 17 +-- .../spatial_statistics/spatial_statistics.wdl | 3 + workflows/pmdbs_spatial_visium/main.wdl | 3 +- .../preprocess/preprocess.wdl | 12 +- 13 files changed, 252 insertions(+), 61 deletions(-) create mode 100755 docker/spatial_py/scripts/geomx_export_final_artifacts diff --git a/README.md b/README.md index f4b4442..f72782a 100644 --- a/README.md +++ b/README.md @@ -301,13 +301,14 @@ asap-dev-{team-xxyy}-{source}-{dataset} └── pmdbs_spatial_geomx ├── cohort_analysis │ ├── ${team_id}.sample_list.tsv - │ ├── ${team_id}.merged.h5ad - │ ├── ${team_id}.merged_adata_metadata.csv + │ ├── ${team_id}.merged_metadata.csv + │ ├── ${team_id}.merged_processed.h5ad │ ├── ${team_id}.all_genes.csv │ ├── ${team_id}.hvg_genes.csv │ ├── ${team_id}.hvg_dispersion.png - │ ├── ${team_id}.clustered.h5ad # Final │ ├── ${team_id}.umap_cluster.png + │ ├── ${team_id}.final.h5ad + │ ├── ${team_id}.final_metadata.csv │ └── MANIFEST.tsv ├── process_to_adata │ ├── ${slideN_id}.segment_gene_detection_plot.png @@ -338,17 +339,17 @@ asap-dev-{team-xxyy}-{source}-{dataset} └── pmdbs_spatial_visium ├── cohort_analysis │ ├── ${team_id}.sample_list.tsv - │ ├── ${team_id}.merged.h5ad - │ ├── ${team_id}.merged_adata_metadata.csv + │ ├── ${team_id}.merged_cleaned_unfiltered.h5ad + │ ├── ${team_id}.merged_metadata.csv │ ├── ${team_id}.all_genes.csv │ ├── ${team_id}.hvg_genes.csv │ ├── ${team_id}.qc_violin.png │ ├── ${team_id}.qc_dist.png │ ├── ${team_id}.hvg_dispersion.png - │ ├── ${team_id}.clustered.h5ad │ ├── ${team_id}.umap_cluster.png │ ├── ${team_id}.spatial_scatter.png │ ├── ${team_id}.final.h5ad + │ ├── ${team_id}.final_metadata.csv │ ├── ${team_id}.moran_top_10_variable_genes.csv │ ├── ${team_id}.moran_top_4_variable_genes_spatial_scatter.png │ └── MANIFEST.tsv @@ -365,7 +366,7 @@ asap-dev-{team-xxyy}-{source}-{dataset} ├── ${sampleA_id}.scalefactors_json.json ├── ${sampleA_id}.tissue_positions.csv ├── ${sampleA_id}.spatial_enrichment.csv - ├── ${sampleA_id}.initial.h5ad + ├── ${sampleA_id}.cleaned_unfiltered.h5ad ├── ${sampleA_id}.qc.h5ad ├── MANIFEST.tsv ├── ... @@ -381,7 +382,7 @@ asap-dev-{team-xxyy}-{source}-{dataset} ├── ${sampleN_id}.scalefactors_json.json ├── ${sampleN_id}.tissue_positions.csv ├── ${sampleN_id}.spatial_enrichment.csv - ├── ${sampleN_id}.initial.h5ad + ├── ${sampleN_id}.cleaned_unfiltered.h5ad ├── ${sampleN_id}.qc.h5ad └── MANIFEST.tsv ``` diff --git a/docker/spatial_py/scripts/geomx_export_final_artifacts b/docker/spatial_py/scripts/geomx_export_final_artifacts new file mode 100755 index 0000000..b1b5f6d --- /dev/null +++ b/docker/spatial_py/scripts/geomx_export_final_artifacts @@ -0,0 +1,47 @@ +#!/usr/bin/env python3 + +import argparse +import scanpy as sc + + +def main(args): + ############################ + ## EXPORT FINAL ARTIFACTS ## + ############################ + adata = sc.read_h5ad(args.adata_input) + + # Save outputs + metadata = adata.obs + metadata.to_csv(f"{args.cohort_id}.final_metadata.csv") + adata.write_h5ad(filename=args.adata_output, compression="gzip") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Export final AnnData object and its metadata" + ) + parser.add_argument( + "-c", + "--cohort-id", + type=str, + required=True, + help="Cohort ID" + ) + parser.add_argument( + "-i", + "--adata-input", + type=str, + required=True, + help="Adata object to finalize" + ) + parser.add_argument( + "-o", + "--adata-output", + type=str, + required=True, + help="Output file name for the final AnnData object" + ) + + args = parser.parse_args() + + main(args) diff --git a/docker/spatial_py/scripts/geomx_merge_and_prep b/docker/spatial_py/scripts/geomx_merge_and_prep index d0c615b..779683a 100755 --- a/docker/spatial_py/scripts/geomx_merge_and_prep +++ b/docker/spatial_py/scripts/geomx_merge_and_prep @@ -36,12 +36,17 @@ def main(args): batch_key=args.batch_key, ) full_features = merged_adata.var.copy() - hvg_full = merged_adata.iloc[: args.n_top_genes].index.to_list() - hvg_merged_adata = merged_adata[:, merged_adata.var.index.isin(hvg_full)] - hvg_features = hvg_merged_adata.var.copy() + hvg_full = ( + merged_adata.var[merged_adata.var["highly_variable"]] + .sort_values("highly_variable_rank") + .head(3000) + .index.tolist() + ) + hvg_adata = merged_adata[:, merged_adata.var.index.isin(hvg_full)] + hvg_features = hvg_adata.var.copy() sc.pl.highly_variable_genes( - merged_adata, + hvg_adata, ) fig = plt.gcf() fig.suptitle(f"Highly variable genes dispersion plot - {args.output_prefix}", va="center", ha="center", fontsize=16) @@ -52,17 +57,17 @@ def main(args): ## DIMENSIONALITY REDUCTION ## ############################## sc.pp.pca( - merged_adata, + hvg_adata, n_comps=args.n_comps, svd_solver="arpack", ) # Save outputs metadata = merged_adata.obs - metadata.to_csv(f"{args.output_prefix}.merged_adata_metadata.csv") + metadata.to_csv(f"{args.output_prefix}.merged_metadata.csv") full_features.to_csv(f"{args.output_prefix}.all_genes.csv", index=True) hvg_features.to_csv(f"{args.output_prefix}.hvg_genes.csv", index=True) - merged_adata.write_h5ad(filename=args.adata_output, compression="gzip") + hvg_adata.write_h5ad(filename=args.adata_output, compression="gzip") if __name__ == "__main__": diff --git a/docker/spatial_py/scripts/visium_process b/docker/spatial_py/scripts/visium_process index 858d50f..a49c385 100755 --- a/docker/spatial_py/scripts/visium_process +++ b/docker/spatial_py/scripts/visium_process @@ -50,12 +50,17 @@ def main(args): batch_key=args.batch_key, ) full_features = adata.var.copy() - hvg_full = adata.iloc[: args.n_top_genes].index.to_list() - hvg_merged_adata = adata[:, adata.var.index.isin(hvg_full)] - hvg_features = hvg_merged_adata.var.copy() + hvg_full = ( + adata.var[adata.var["highly_variable"]] + .sort_values("highly_variable_rank") + .head(3000) + .index.tolist() + ) + hvg_adata = adata[:, adata.var.index.isin(hvg_full)] + hvg_features = hvg_adata.var.copy() sc.pl.highly_variable_genes( - adata, + hvg_adata, ) fig = plt.gcf() fig.suptitle(f"Highly variable genes dispersion plot - {args.output_prefix}", va="center", ha="center", fontsize=16) @@ -66,7 +71,7 @@ def main(args): ## DIMENSIONALITY REDUCTION ## ############################## sc.pp.pca( - adata, + hvg_adata, n_comps=args.n_comps, svd_solver="arpack", ) @@ -74,7 +79,7 @@ def main(args): # Save outputs full_features.to_csv(f"{args.output_prefix}.all_genes.csv", index=True) hvg_features.to_csv(f"{args.output_prefix}.hvg_genes.csv", index=True) - adata.write_h5ad(filename=args.adata_output, compression="gzip") + hvg_adata.write_h5ad(filename=args.adata_output, compression="gzip") if __name__ == "__main__": diff --git a/docker/spatial_py/scripts/visium_spatially_variable_genes b/docker/spatial_py/scripts/visium_spatially_variable_genes index ffe0d78..98dafc0 100755 --- a/docker/spatial_py/scripts/visium_spatially_variable_genes +++ b/docker/spatial_py/scripts/visium_spatially_variable_genes @@ -43,6 +43,8 @@ def main(args): # Save table and adata object top_10_variable_genes.to_csv(f"{args.cohort_id}.moran_top_10_variable_genes.csv") + metadata = adata.obs + metadata.to_csv(f"{args.cohort_id}.final_metadata.csv") adata.write_h5ad(filename=args.adata_output, compression="gzip") diff --git a/wdl-ci.config.json b/wdl-ci.config.json index 6126ac4..5fd4c49 100644 --- a/wdl-ci.config.json +++ b/wdl-ci.config.json @@ -26,15 +26,15 @@ "zones": "${zones}" }, "output_tests": { - "merged_adata_object": { - "value": "${geomx_input_file_path}/${cohort_analysis_workflow_name}/${geomx_team_id}.merged.h5ad", + "merged_and_processed_adata_object": { + "value": "${geomx_input_file_path}/${cohort_analysis_workflow_name}/${geomx_team_id}.merged_and_processed.h5ad", "test_tasks": [ "compare_file_basename", "check_hdf5" ] }, - "merged_adata_metadata_csv": { - "value": "${geomx_input_file_path}/${cohort_analysis_workflow_name}/${geomx_team_id}.merged_adata_metadata.csv", + "merged_metadata_csv": { + "value": "${geomx_input_file_path}/${cohort_analysis_workflow_name}/${geomx_team_id}.merged_metadata.csv", "test_tasks": [ "compare_file_basename", "check_empty_lines", @@ -71,6 +71,41 @@ } } ] + }, + "export_final_artifacts": { + "key": "export_final_artifacts", + "digest": "", + "tests": [ + { + "inputs": { + "cohort_id": "${geomx_team_id}", + "clustered_adata_object": "${geomx_input_file_path}/${cohort_analysis_workflow_name}/${geomx_team_id}.clustered.h5ad", + "raw_data_path": "${geomx_cohort_analysis_raw_data_path}", + "workflow_info": "${geomx_workflow_info}", + "billing_project": "${billing_project}", + "container_registry": "${container_registry}", + "zones": "${zones}" + }, + "output_tests": { + "final_adata_object": { + "value": "${geomx_input_file_path}/${cohort_analysis_workflow_name}/${geomx_team_id}.final.h5ad", + "test_tasks": [ + "compare_file_basename", + "check_hdf5" + ] + }, + "final_metadata_csv": { + "value": "${geomx_input_file_path}/${cohort_analysis_workflow_name}/${geomx_team_id}.final_metadata.csv", + "test_tasks": [ + "compare_file_basename", + "check_empty_lines", + "check_comma_separated", + "count_columns" + ] + } + } + } + ] } } }, @@ -232,7 +267,7 @@ { "inputs": { "cohort_id": "${geomx_team_id}", - "merged_and_processed_adata_object": "${geomx_input_file_path}/${cohort_analysis_workflow_name}/${geomx_team_id}.merged.h5ad", + "merged_and_processed_adata_object": "${geomx_input_file_path}/${cohort_analysis_workflow_name}/${geomx_team_id}.merged_and_processed.h5ad", "batch_key": "batch_id", "container_registry": "${container_registry}", "zones": "${zones}" @@ -311,7 +346,7 @@ }, "output_tests": { "merged_adata_object": { - "value": "${visium_input_file_path}/${cohort_analysis_workflow_name}/${visium_team_id}.merged.h5ad", + "value": "${visium_input_file_path}/${cohort_analysis_workflow_name}/${visium_team_id}.merged_cleaned_unfiltered.h5ad", "test_tasks": [ "compare_file_basename", "check_hdf5" @@ -339,7 +374,7 @@ { "inputs": { "cohort_id": "${visium_team_id}", - "merged_adata_object": "${visium_input_file_path}/${cohort_analysis_workflow_name}/${visium_team_id}.merged.h5ad", + "merged_adata_object": "${visium_input_file_path}/${cohort_analysis_workflow_name}/${visium_team_id}.merged_cleaned_unfiltered.h5ad", "filter_cells_min_counts": 5000, "filter_cells_min_genes": 3000, "filter_genes_min_cells": 10, @@ -448,6 +483,15 @@ "check_hdf5" ] }, + "final_metadata_csv": { + "value": "${visium_input_file_path}/${cohort_analysis_workflow_name}/${visium_team_id}.final_metadata.csv", + "test_tasks": [ + "compare_file_basename", + "check_empty_lines", + "check_comma_separated", + "count_columns" + ] + }, "moran_top_10_variable_genes_csv": { "value": "${visium_input_file_path}/${cohort_analysis_workflow_name}/${visium_team_id}.moran_top_10_variable_genes.csv", "test_tasks": [ @@ -602,7 +646,7 @@ }, "output_tests": { "initial_adata_object": { - "value": "${visium_input_file_path}/${preprocess_workflow_name}/${visium_sample_id}.initial.h5ad", + "value": "${visium_input_file_path}/${preprocess_workflow_name}/${visium_sample_id}.cleaned_unfiltered.h5ad", "test_tasks": [ "compare_file_basename", "check_hdf5" @@ -619,7 +663,7 @@ { "inputs": { "sample_id": "${visium_sample_id}", - "initial_adata_object": "${visium_input_file_path}/${preprocess_workflow_name}/${visium_sample_id}.initial.h5ad", + "initial_adata_object": "${visium_input_file_path}/${preprocess_workflow_name}/${visium_sample_id}.cleaned_unfiltered.h5ad", "raw_data_path": "${visium_qc_raw_data_path}", "workflow_info": "${visium_workflow_info}", "billing_project": "${billing_project}", diff --git a/workflows/integrate_data/integrate_data.wdl b/workflows/integrate_data/integrate_data.wdl index c5dc59a..1241820 100644 --- a/workflows/integrate_data/integrate_data.wdl +++ b/workflows/integrate_data/integrate_data.wdl @@ -42,7 +42,7 @@ workflow integrate_data { output { File integrated_adata_object = integrate_sample_data.integrated_adata_object - File clustered_adata_object = cluster.clustered_adata_object #!FileCoercion + File clustered_adata_object = cluster.clustered_adata_object File umap_cluster_plots_png = cluster.umap_cluster_plots_png #!FileCoercion } @@ -147,12 +147,11 @@ task cluster { -b ~{billing_project} \ -d ~{raw_data_path} \ -i ~{write_tsv(workflow_info)} \ - -o "~{cohort_id}.clustered.h5ad" \ -o "~{cohort_id}.umap_cluster.png" >>> output { - String clustered_adata_object = "~{raw_data_path}/~{cohort_id}.clustered.h5ad" + File clustered_adata_object = "~{cohort_id}.clustered.h5ad" String umap_cluster_plots_png = "~{raw_data_path}/~{cohort_id}.umap_cluster.png" } diff --git a/workflows/pmdbs_spatial_geomx/cohort_analysis/cohort_analysis.wdl b/workflows/pmdbs_spatial_geomx/cohort_analysis/cohort_analysis.wdl index 60d48c8..34a3bde 100644 --- a/workflows/pmdbs_spatial_geomx/cohort_analysis/cohort_analysis.wdl +++ b/workflows/pmdbs_spatial_geomx/cohort_analysis/cohort_analysis.wdl @@ -68,7 +68,7 @@ workflow cohort_analysis { call IntegrateData.integrate_data { input: cohort_id = cohort_id, - merged_and_processed_adata_object = merge_and_prep.merged_adata_object, #!FileCoercion + merged_and_processed_adata_object = merge_and_prep.merged_and_processed_adata_object, #!FileCoercion n_comps = n_comps, batch_key = batch_key, leiden_resolution = leiden_resolution, @@ -79,6 +79,17 @@ workflow cohort_analysis { zones = zones } + call export_final_artifacts { + input: + cohort_id = cohort_id, + clustered_adata_object = integrate_data.clustered_adata_object, + raw_data_path = raw_data_path, + workflow_info = workflow_info, + billing_project = billing_project, + container_registry = container_registry, + zones = zones + } + call UploadFinalOutputs.upload_final_outputs as upload_preprocess_files { input: output_file_paths = preprocessing_output_file_paths, @@ -102,15 +113,18 @@ workflow cohort_analysis { write_cohort_sample_list.cohort_sample_list ], [ - merge_and_prep.merged_adata_object, - merge_and_prep.merged_adata_metadata_csv, + merge_and_prep.merged_metadata_csv, + merge_and_prep.merged_and_processed_adata_object, merge_and_prep.all_genes_csv, merge_and_prep.hvg_genes_csv, merge_and_prep.hvg_plot_png ], [ - integrate_data.clustered_adata_object, integrate_data.umap_cluster_plots_png + ], + [ + export_final_artifacts.final_adata_object, + export_final_artifacts.final_metadata_csv ] ]) #!StringCoercion @@ -127,8 +141,8 @@ workflow cohort_analysis { File cohort_sample_list = write_cohort_sample_list.cohort_sample_list #!FileCoercion # Merged and prepped AnnData object - File merged_adata_object = merge_and_prep.merged_adata_object #!FileCoercion - File merged_adata_metadata_csv = merge_and_prep.merged_adata_metadata_csv #!FileCoercion + File merged_metadata_csv = merge_and_prep.merged_metadata_csv #!FileCoercion + File merged_and_processed_adata_object = merge_and_prep.merged_and_processed_adata_object #!FileCoercion File all_genes_csv = merge_and_prep.all_genes_csv #!FileCoercion File hvg_genes_csv = merge_and_prep.hvg_genes_csv #!FileCoercion File hvg_plot_png = merge_and_prep.hvg_plot_png #!FileCoercion @@ -138,6 +152,10 @@ workflow cohort_analysis { File clustered_adata_object = integrate_data.clustered_adata_object File umap_cluster_plots_png = integrate_data.umap_cluster_plots_png + # Export final outputs + File final_adata_object = export_final_artifacts.final_adata_object #!FileCoercion + File final_metadata_csv = export_final_artifacts.final_metadata_csv #!FileCoercion + Array[File] preprocess_manifest_tsvs = upload_preprocess_files.manifests #!FileCoercion Array[File] process_to_adata_manifest_tsvs = upload_process_to_adata_files.manifests #!FileCoercion Array[File] cohort_analysis_manifest_tsvs = upload_cohort_analysis_files.manifests #!FileCoercion @@ -197,14 +215,14 @@ task merge_and_prep { --n-comps ~{n_comps} \ --batch-key ~{batch_key} \ --output-prefix ~{cohort_id} \ - --adata-output ~{cohort_id}.merged.h5ad + --adata-output ~{cohort_id}.merged_processed.h5ad upload_outputs \ -b ~{billing_project} \ -d ~{raw_data_path} \ -i ~{write_tsv(workflow_info)} \ - -o "~{cohort_id}.merged.h5ad" \ - -o "~{cohort_id}.merged_adata_metadata.csv" \ + -o "~{cohort_id}.merged_metadata.csv" \ + -o "~{cohort_id}.merged_processed.h5ad" \ -o "~{cohort_id}.all_genes.csv" \ -o "~{cohort_id}.hvg_genes.csv" \ -o "~{cohort_id}.hvg_dispersion.png" @@ -212,8 +230,8 @@ task merge_and_prep { >>> output { - String merged_adata_object = "~{raw_data_path}/~{cohort_id}.merged.h5ad" - String merged_adata_metadata_csv = "~{raw_data_path}/~{cohort_id}.merged_adata_metadata.csv" + String merged_metadata_csv = "~{raw_data_path}/~{cohort_id}.merged_metadata.csv" + String merged_and_processed_adata_object = "~{raw_data_path}/~{cohort_id}.merged_processed.h5ad" String all_genes_csv = "~{raw_data_path}/~{cohort_id}.all_genes.csv" String hvg_genes_csv = "~{raw_data_path}/~{cohort_id}.hvg_genes.csv" String hvg_plot_png = "~{raw_data_path}/~{cohort_id}.hvg_dispersion.png" @@ -247,3 +265,66 @@ task merge_and_prep { zones: {help: "Space-delimited set of GCP zones where compute will take place. ['us-central1-c us-central1-f']"} } } + +task export_final_artifacts { + input { + String cohort_id + File clustered_adata_object + + String raw_data_path + Array[Array[String]] workflow_info + String billing_project + String container_registry + String zones + } + + Int mem_gb = ceil(size(clustered_adata_object, "GB") * 2 + 20) + Int disk_size = ceil(size(clustered_adata_object, "GB") * 2 + 50) + + command <<< + set -euo pipefail + + geomx_export_final_artifacts \ + --cohort-id ~{cohort_id} \ + --adata-input ~{clustered_adata_object} \ + --adata-output ~{cohort_id}.final.h5ad + + upload_outputs \ + -b ~{billing_project} \ + -d ~{raw_data_path} \ + -i ~{write_tsv(workflow_info)} \ + -o "~{cohort_id}.final.h5ad" \ + -o "~{cohort_id}.final_metadata.csv" + + >>> + + output { + String final_adata_object = "~{raw_data_path}/~{cohort_id}.final.h5ad" + String final_metadata_csv = "~{raw_data_path}/~{cohort_id}.final_metadata.csv" + } + + runtime { + docker: "~{container_registry}/spatial_py:1.0.0" + cpu: 2 + memory: "~{mem_gb} GB" + disks: "local-disk ~{disk_size} HDD" + preemptible: 3 + maxRetries: 2 + bootDiskSizeGb: 15 + zones: zones + } + + meta { + description: "Export clustered AnnData object as final and grab the metadata." + } + + parameter_meta { + cohort_id: {help: "Name of the cohort; used to name output files."} + clustered_adata_object: {help: "Clustered AnnData object."} + raw_data_path: {help: "Raw data bucket path for merged adata and HVG plot outputs; location of raw bucket to upload task outputs to (`/workflow_execution/cohort_analysis//`)."} + workflow_info: {help: "UTC timestamp, workflow name, workflow version, and GitHub release; stored in the file-level manifest and final manifest with all saved files."} + billing_project: {help: "Billing project to charge GCP costs."} + container_registry: {help: "Container registry where workflow Docker images are hosted."} + zones: {help: "Space-delimited set of GCP zones where compute will take place. ['us-central1-c us-central1-f']"} + } +} diff --git a/workflows/pmdbs_spatial_geomx/main.wdl b/workflows/pmdbs_spatial_geomx/main.wdl index cc73c57..8fa6423 100644 --- a/workflows/pmdbs_spatial_geomx/main.wdl +++ b/workflows/pmdbs_spatial_geomx/main.wdl @@ -165,14 +165,16 @@ workflow pmdbs_spatial_geomx_analysis { Array[File?] project_cohort_sample_list = project_cohort_analysis.cohort_sample_list ## Merged, integrated and clustered adata objects, and plots - Array[File?] project_merged_adata_object = project_cohort_analysis.merged_adata_object - Array[File?] project_merged_adata_metadata_csv = project_cohort_analysis.merged_adata_metadata_csv + Array[File?] project_merged_metadata_csv = project_cohort_analysis.merged_metadata_csv + Array[File?] project_merged_and_processed_adata_object = project_cohort_analysis.merged_and_processed_adata_object Array[File?] project_all_genes_csv = project_cohort_analysis.all_genes_csv Array[File?] project_hvg_genes_csv = project_cohort_analysis.hvg_genes_csv Array[File?] project_hvg_plot_png = project_cohort_analysis.hvg_plot_png Array[File?] project_integrated_adata_object = project_cohort_analysis.integrated_adata_object Array[File?] project_clustered_adata_object = project_cohort_analysis.clustered_adata_object Array[File?] project_umap_cluster_plots_png = project_cohort_analysis.umap_cluster_plots_png + Array[File?] project_final_adata_object = project_cohort_analysis.final_adata_object + Array[File?] project_final_metadata_csv = project_cohort_analysis.final_metadata_csv Array[Array[File]?] preprocess_manifests = project_cohort_analysis.preprocess_manifest_tsvs Array[Array[File]?] process_to_adata_manifests = project_cohort_analysis.process_to_adata_manifest_tsvs diff --git a/workflows/pmdbs_spatial_visium/cohort_analysis/cohort_analysis.wdl b/workflows/pmdbs_spatial_visium/cohort_analysis/cohort_analysis.wdl index aa19cd4..0cadcfd 100644 --- a/workflows/pmdbs_spatial_visium/cohort_analysis/cohort_analysis.wdl +++ b/workflows/pmdbs_spatial_visium/cohort_analysis/cohort_analysis.wdl @@ -137,7 +137,7 @@ workflow cohort_analysis { ], [ merge_and_plot_qc_metrics.merged_adata_object, - merge_and_plot_qc_metrics.merged_adata_metadata_csv + merge_and_plot_qc_metrics.merged_metadata_csv ], merge_and_plot_qc_metrics.qc_plots_png, [ @@ -146,7 +146,6 @@ workflow cohort_analysis { filter_and_normalize.hvg_plot_png ], [ - integrate_data.clustered_adata_object, integrate_data.umap_cluster_plots_png ], [ @@ -154,6 +153,7 @@ workflow cohort_analysis { ], [ spatial_statistics.final_adata_object, + spatial_statistics.final_metadata_csv, spatial_statistics.moran_top_10_variable_genes_csv, spatial_statistics.moran_top_4_variable_genes_spatial_scatter_plot_png ] @@ -173,7 +173,7 @@ workflow cohort_analysis { # Merged adata objects and QC plots File merged_adata_object = merge_and_plot_qc_metrics.merged_adata_object #!FileCoercion - File merged_adata_metadata_csv = merge_and_plot_qc_metrics.merged_adata_metadata_csv #!FileCoercion + File merged_metadata_csv = merge_and_plot_qc_metrics.merged_metadata_csv #!FileCoercion Array[File] qc_plots_png = merge_and_plot_qc_metrics.qc_plots_png #!FileCoercion # Processed outputs @@ -192,6 +192,7 @@ workflow cohort_analysis { # Spatial statistics outputs File final_adata_object = spatial_statistics.final_adata_object + File final_metadata_csv = spatial_statistics.final_metadata_csv File moran_top_10_variable_genes_csv = spatial_statistics.moran_top_10_variable_genes_csv File moran_top_4_variable_genes_spatial_scatter_plot_png = spatial_statistics.moran_top_4_variable_genes_spatial_scatter_plot_png @@ -250,21 +251,21 @@ task merge_and_plot_qc_metrics { visium_merge_and_plot_qc \ --adata-paths-input ~{sep=' ' preprocessed_adata_objects} \ --output-prefix ~{cohort_id} \ - --merged-adata-output ~{cohort_id}.merged.h5ad + --merged-adata-output ~{cohort_id}.merged_cleaned_unfiltered.h5ad upload_outputs \ -b ~{billing_project} \ -d ~{raw_data_path} \ -i ~{write_tsv(workflow_info)} \ - -o "~{cohort_id}.merged.h5ad" \ - -o "~{cohort_id}.merged_adata_metadata.csv" \ + -o "~{cohort_id}.merged_cleaned_unfiltered.h5ad" \ + -o "~{cohort_id}.merged_metadata.csv" \ -o "~{cohort_id}.qc_violin.png" \ -o "~{cohort_id}.qc_dist.png" >>> output { - String merged_adata_object = "~{raw_data_path}/~{cohort_id}.merged.h5ad" - String merged_adata_metadata_csv = "~{raw_data_path}/~{cohort_id}.merged_adata_metadata.csv" + String merged_adata_object = "~{raw_data_path}/~{cohort_id}.merged_cleaned_unfiltered.h5ad" + String merged_metadata_csv = "~{raw_data_path}/~{cohort_id}.merged_metadata.csv" Array[String] qc_plots_png = [ "~{raw_data_path}/~{cohort_id}.qc_violin.png", "~{raw_data_path}/~{cohort_id}.qc_dist.png" diff --git a/workflows/pmdbs_spatial_visium/cohort_analysis/spatial_statistics/spatial_statistics.wdl b/workflows/pmdbs_spatial_visium/cohort_analysis/spatial_statistics/spatial_statistics.wdl index c2ba335..d5c8060 100644 --- a/workflows/pmdbs_spatial_visium/cohort_analysis/spatial_statistics/spatial_statistics.wdl +++ b/workflows/pmdbs_spatial_visium/cohort_analysis/spatial_statistics/spatial_statistics.wdl @@ -28,6 +28,7 @@ workflow spatial_statistics { output { # Moran’s I global spatial auto-correlation statistics File final_adata_object = spatially_variable_gene_analysis.final_adata_object #!FileCoercion + File final_metadata_csv = spatially_variable_gene_analysis.final_metadata_csv #!FileCoercion File moran_top_10_variable_genes_csv = spatially_variable_gene_analysis.moran_top_10_variable_genes_csv #!FileCoercion File moran_top_4_variable_genes_spatial_scatter_plot_png = spatially_variable_gene_analysis.moran_top_4_variable_genes_spatial_scatter_plot_png #!FileCoercion } @@ -75,12 +76,14 @@ task spatially_variable_gene_analysis { -d ~{raw_data_path} \ -i ~{write_tsv(workflow_info)} \ -o "~{cohort_id}.final.h5ad" \ + -o "~{cohort_id}.final_metadata.csv" \ -o "~{cohort_id}.moran_top_10_variable_genes.csv" \ -o "~{cohort_id}.moran_top_4_variable_genes_spatial_scatter.png" >>> output { String final_adata_object = "~{raw_data_path}/~{cohort_id}.final.h5ad" + String final_metadata_csv = "~{raw_data_path}/~{cohort_id}.final_metadata.csv" String moran_top_10_variable_genes_csv = "~{raw_data_path}/~{cohort_id}.moran_top_10_variable_genes.csv" String moran_top_4_variable_genes_spatial_scatter_plot_png = "~{raw_data_path}/~{cohort_id}.moran_top_4_variable_genes_spatial_scatter.png" } diff --git a/workflows/pmdbs_spatial_visium/main.wdl b/workflows/pmdbs_spatial_visium/main.wdl index 6dc5d98..8f8a6b0 100644 --- a/workflows/pmdbs_spatial_visium/main.wdl +++ b/workflows/pmdbs_spatial_visium/main.wdl @@ -127,7 +127,7 @@ workflow pmdbs_spatial_visium_analysis { # Merged, processed (filtered, normalized, dimensionality reduced), integrated, and clustered adata objects, and plots Array[File?] project_merged_adata_object = project_cohort_analysis.merged_adata_object - Array[File?] project_merged_adata_metadata_csv = project_cohort_analysis.merged_adata_metadata_csv + Array[File?] project_merged_metadata_csv = project_cohort_analysis.merged_metadata_csv Array[File?] project_all_genes_csv = project_cohort_analysis.all_genes_csv Array[File?] project_hvg_genes_csv = project_cohort_analysis.hvg_genes_csv Array[Array[File]?] project_qc_plots_png = project_cohort_analysis.qc_plots_png @@ -142,6 +142,7 @@ workflow pmdbs_spatial_visium_analysis { # Spatial statistics outputs Array[File?] project_final_adata_object = project_cohort_analysis.final_adata_object + Array[File?] project_final_metadata_csv = project_cohort_analysis.final_metadata_csv Array[File?] project_moran_top_10_variable_genes_csv = project_cohort_analysis.moran_top_10_variable_genes_csv Array[File?] project_moran_top_4_variable_genes_spatial_scatter_plot_png = project_cohort_analysis.moran_top_4_variable_genes_spatial_scatter_plot_png diff --git a/workflows/pmdbs_spatial_visium/preprocess/preprocess.wdl b/workflows/pmdbs_spatial_visium/preprocess/preprocess.wdl index cd96703..f282c8b 100644 --- a/workflows/pmdbs_spatial_visium/preprocess/preprocess.wdl +++ b/workflows/pmdbs_spatial_visium/preprocess/preprocess.wdl @@ -39,7 +39,7 @@ workflow preprocess { scatter (sample_object in samples) { String spaceranger_count_output = "~{spaceranger_raw_data_path}/~{sample_object.sample_id}.raw_feature_bc_matrix.h5" - String counts_to_adata_output = "~{adata_raw_data_path}/~{sample_object.sample_id}.initial.h5ad" + String counts_to_adata_output = "~{adata_raw_data_path}/~{sample_object.sample_id}.cleaned_unfiltered.h5ad" String qc_output = "~{qc_raw_data_path}/~{sample_object.sample_id}.qc.h5ad" } @@ -108,7 +108,7 @@ workflow preprocess { File tissue_positions_csv_output = select_first([spaceranger_count.tissue_positions_csv, spaceranger_tissue_positions_csv]) #!FileCoercion File spatial_enrichment_csv_output = select_first([spaceranger_count.spatial_enrichment_csv, spaceranger_spatial_enrichment_csv]) #!FileCoercion - String counts_to_adata_object = "~{adata_raw_data_path}/~{sample.sample_id}.initial.h5ad" + String counts_to_adata_object = "~{adata_raw_data_path}/~{sample.sample_id}.cleaned_unfiltered.h5ad" if (counts_to_adata_complete == "false") { call counts_to_adata { @@ -250,7 +250,7 @@ task check_output_files_exist { parameter_meta { spaceranger_count_output_files: {help: "Spaceranger count output file to detect (`.raw_feature_bc_matrix.h5`)."} - counts_to_adata_output_files: {help: "Converted AnnData object output file to detect (`.initial.h5ad`)."} + counts_to_adata_output_files: {help: "Converted AnnData object output file to detect (`.cleaned_unfiltered.h5ad`)."} qc_output_files: {help: "QC'ed output file to detect (`.qc.h5ad`)."} billing_project: {help: "Billing project to charge GCP costs."} zones: {help: "Space-delimited set of GCP zones where compute will take place. ['us-central1-c us-central1-f']"} @@ -449,17 +449,17 @@ task counts_to_adata { --slide ~{visium_slide_serial_number} \ --area ~{visium_capture_area} \ --spaceranger-spatial-dir spatial_outputs \ - --adata-output ~{sample_id}.initial.h5ad + --adata-output ~{sample_id}.cleaned_unfiltered.h5ad upload_outputs \ -b ~{billing_project} \ -d ~{raw_data_path} \ -i ~{write_tsv(workflow_info)} \ - -o "~{sample_id}.initial.h5ad" + -o "~{sample_id}.cleaned_unfiltered.h5ad" >>> output { - String initial_adata_object = "~{raw_data_path}/~{sample_id}.initial.h5ad" + String initial_adata_object = "~{raw_data_path}/~{sample_id}.cleaned_unfiltered.h5ad" } runtime { From d68e27ff8f65ba2bee5aa27930b2706acedd1342 Mon Sep 17 00:00:00 2001 From: github-actions <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 22 Jul 2025 16:27:52 +0000 Subject: [PATCH 11/20] update wdl-ci config file after successful tests --- wdl-ci.config.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wdl-ci.config.json b/wdl-ci.config.json index 5fd4c49..913b4c7 100644 --- a/wdl-ci.config.json +++ b/wdl-ci.config.json @@ -286,7 +286,7 @@ }, "cluster": { "key": "cluster", - "digest": "chquhvgfycakclmctuyqjyeanrbtlilo", + "digest": "qouf6flpm3rprnfz4fztlh3ybrumqcfb", "tests": [ { "inputs": { From 58f8182d35e9df3bc741fa20527f8112bbb0df3e Mon Sep 17 00:00:00 2001 From: Karen Fang Date: Tue, 22 Jul 2025 14:17:30 -0400 Subject: [PATCH 12/20] Use histplot instead of deprecated distplot --- docker/spatial_py/scripts/visium_merge_and_plot_qc | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docker/spatial_py/scripts/visium_merge_and_plot_qc b/docker/spatial_py/scripts/visium_merge_and_plot_qc index 65d8c51..512c4c4 100755 --- a/docker/spatial_py/scripts/visium_merge_and_plot_qc +++ b/docker/spatial_py/scripts/visium_merge_and_plot_qc @@ -47,24 +47,24 @@ def main(args): # Total counts and n genes by counts distribution plots fig, axs = plt.subplots(1, 4, figsize=(15,4)) fig.suptitle(f"Covariates for filtering - {args.output_prefix}") - sns.distplot( + sns.histplot( merged_adata.obs["total_counts"], kde=False, ax=axs[0], ) - sns.distplot( + sns.histplot( merged_adata.obs["total_counts"][merged_adata.obs["total_counts"]<10000], kde=False, bins=40, ax=axs[1], ) - sns.distplot( + sns.histplot( merged_adata.obs["n_genes_by_counts"], kde=False, bins=60, ax=axs[2], ) - sns.distplot( + sns.histplot( merged_adata.obs["n_genes_by_counts"][merged_adata.obs["n_genes_by_counts"]<4000], kde=False, bins=60, @@ -75,7 +75,7 @@ def main(args): # Save outputs metadata = merged_adata.obs - metadata.to_csv(f"{args.output_prefix}.merged_adata_metadata.csv") + metadata.to_csv(f"{args.output_prefix}.merged_metadata.csv") merged_adata.write_h5ad(filename=args.merged_adata_output, compression="gzip") From e01e92158c22445cd726a95859d0720b86c8adf5 Mon Sep 17 00:00:00 2001 From: github-actions <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 22 Jul 2025 18:36:32 +0000 Subject: [PATCH 13/20] update wdl-ci config file after successful tests --- wdl-ci.config.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/wdl-ci.config.json b/wdl-ci.config.json index 913b4c7..658c1ff 100644 --- a/wdl-ci.config.json +++ b/wdl-ci.config.json @@ -74,7 +74,7 @@ }, "export_final_artifacts": { "key": "export_final_artifacts", - "digest": "", + "digest": "i6i56p6qrv5iew6tthkzl7p6mlmkyvsl", "tests": [ { "inputs": { @@ -329,7 +329,7 @@ "tasks": { "merge_and_plot_qc_metrics": { "key": "merge_and_plot_qc_metrics", - "digest": "w5637hnyv47nzalkr72puv7cixhgx5pu", + "digest": "kppsqzekmaf2vvut6ijgchkdof56io7u", "tests": [ { "inputs": { @@ -627,7 +627,7 @@ }, "counts_to_adata": { "key": "counts_to_adata", - "digest": "itpk7kn2vwno4o5bthjsetdmk6bvq2cb", + "digest": "uty3rh5gqsqcymnegbiacpejn2mhklcm", "tests": [ { "inputs": { From b438aa0fbef2144e22da2918643c54a26246e31e Mon Sep 17 00:00:00 2001 From: Karen Fang Date: Tue, 22 Jul 2025 14:56:01 -0400 Subject: [PATCH 14/20] Revert back to seurat instead of pearsons residuals --- docker/spatial_py/scripts/geomx_merge_and_prep | 4 ++-- docker/spatial_py/scripts/visium_process | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docker/spatial_py/scripts/geomx_merge_and_prep b/docker/spatial_py/scripts/geomx_merge_and_prep index 779683a..ead3b41 100755 --- a/docker/spatial_py/scripts/geomx_merge_and_prep +++ b/docker/spatial_py/scripts/geomx_merge_and_prep @@ -27,11 +27,11 @@ def main(args): sc.pp.log1p(merged_adata) merged_adata.layers["counts"] = merged_adata.X.copy() - sc.experimental.pp.highly_variable_genes( + sc.pp.highly_variable_genes( merged_adata, layer="counts", n_top_genes=args.n_top_genes, - flavor="pearson_residuals", + flavor="seurat", inplace=True, batch_key=args.batch_key, ) diff --git a/docker/spatial_py/scripts/visium_process b/docker/spatial_py/scripts/visium_process index a49c385..150e1d0 100755 --- a/docker/spatial_py/scripts/visium_process +++ b/docker/spatial_py/scripts/visium_process @@ -41,11 +41,11 @@ def main(args): ################## ## ANNOTATE HVG ## ################## - sc.experimental.pp.highly_variable_genes( + sc.pp.highly_variable_genes( adata, layer="counts", n_top_genes=args.n_top_genes, - flavor="pearson_residuals", + flavor="seurat", inplace=True, batch_key=args.batch_key, ) From f444913712190dac0c6aa6b3a29fec0df20a0a7c Mon Sep 17 00:00:00 2001 From: github-actions <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 22 Jul 2025 19:15:56 +0000 Subject: [PATCH 15/20] update wdl-ci config file after successful tests --- wdl-ci.config.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wdl-ci.config.json b/wdl-ci.config.json index 658c1ff..a0a9616 100644 --- a/wdl-ci.config.json +++ b/wdl-ci.config.json @@ -463,7 +463,7 @@ "tasks": { "spatially_variable_gene_analysis": { "key": "spatially_variable_gene_analysis", - "digest": "pbh5hevnohvtahgqfegk32kwzqefn7xo", + "digest": "swz5i2qx4fxynsb77wgp53zvqraibph3", "tests": [ { "inputs": { From 4b61d411b68132eadfe091eb6a54ea4624abdde3 Mon Sep 17 00:00:00 2001 From: Karen Fang Date: Tue, 22 Jul 2025 15:25:03 -0400 Subject: [PATCH 16/20] Sort by flavor Seurat columns --- docker/spatial_py/scripts/geomx_merge_and_prep | 10 ++++++---- docker/spatial_py/scripts/visium_process | 8 +++++--- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/docker/spatial_py/scripts/geomx_merge_and_prep b/docker/spatial_py/scripts/geomx_merge_and_prep index ead3b41..385d61e 100755 --- a/docker/spatial_py/scripts/geomx_merge_and_prep +++ b/docker/spatial_py/scripts/geomx_merge_and_prep @@ -24,12 +24,11 @@ def main(args): ################## ## ANNOTATE HVG ## ################## + merged_adata.layers["counts"] = merged_adata.X.copy() sc.pp.log1p(merged_adata) - merged_adata.layers["counts"] = merged_adata.X.copy() sc.pp.highly_variable_genes( merged_adata, - layer="counts", n_top_genes=args.n_top_genes, flavor="seurat", inplace=True, @@ -38,8 +37,11 @@ def main(args): full_features = merged_adata.var.copy() hvg_full = ( merged_adata.var[merged_adata.var["highly_variable"]] - .sort_values("highly_variable_rank") - .head(3000) + .sort_values( + by=["highly_variable_nbatches", "dispersions_norm"], + ascending=[False, False] + ) + .head(args.n_top_genes) .index.tolist() ) hvg_adata = merged_adata[:, merged_adata.var.index.isin(hvg_full)] diff --git a/docker/spatial_py/scripts/visium_process b/docker/spatial_py/scripts/visium_process index 150e1d0..e06f2bb 100755 --- a/docker/spatial_py/scripts/visium_process +++ b/docker/spatial_py/scripts/visium_process @@ -43,7 +43,6 @@ def main(args): ################## sc.pp.highly_variable_genes( adata, - layer="counts", n_top_genes=args.n_top_genes, flavor="seurat", inplace=True, @@ -52,8 +51,11 @@ def main(args): full_features = adata.var.copy() hvg_full = ( adata.var[adata.var["highly_variable"]] - .sort_values("highly_variable_rank") - .head(3000) + .sort_values( + by=["highly_variable_nbatches", "dispersions_norm"], + ascending=[False, False] + ) + .head(args.n_top_genes) .index.tolist() ) hvg_adata = adata[:, adata.var.index.isin(hvg_full)] From a597ca3f4cad9362b0b7259c63142ac2f2b36bbe Mon Sep 17 00:00:00 2001 From: github-actions <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 22 Jul 2025 19:49:53 +0000 Subject: [PATCH 17/20] update wdl-ci config file after successful tests --- wdl-ci.config.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wdl-ci.config.json b/wdl-ci.config.json index a0a9616..d077df0 100644 --- a/wdl-ci.config.json +++ b/wdl-ci.config.json @@ -369,7 +369,7 @@ }, "filter_and_normalize": { "key": "filter_and_normalize", - "digest": "hrix2jd7skzunlvqtbddvosxosslqlac", + "digest": "lujek6d7l3dinbjtu23rvf2bplcbdkl5", "tests": [ { "inputs": { From 5c777629a6df6c14ce8987b87ffb78a6bd4b08cd Mon Sep 17 00:00:00 2001 From: Karen Fang Date: Tue, 22 Jul 2025 15:58:27 -0400 Subject: [PATCH 18/20] Fix naming in wdl-ci --- wdl-ci.config.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/wdl-ci.config.json b/wdl-ci.config.json index d077df0..916f321 100644 --- a/wdl-ci.config.json +++ b/wdl-ci.config.json @@ -27,7 +27,7 @@ }, "output_tests": { "merged_and_processed_adata_object": { - "value": "${geomx_input_file_path}/${cohort_analysis_workflow_name}/${geomx_team_id}.merged_and_processed.h5ad", + "value": "${geomx_input_file_path}/${cohort_analysis_workflow_name}/${geomx_team_id}.merged_processed.h5ad", "test_tasks": [ "compare_file_basename", "check_hdf5" @@ -267,7 +267,7 @@ { "inputs": { "cohort_id": "${geomx_team_id}", - "merged_and_processed_adata_object": "${geomx_input_file_path}/${cohort_analysis_workflow_name}/${geomx_team_id}.merged_and_processed.h5ad", + "merged_and_processed_adata_object": "${geomx_input_file_path}/${cohort_analysis_workflow_name}/${geomx_team_id}.merged_processed.h5ad", "batch_key": "batch_id", "container_registry": "${container_registry}", "zones": "${zones}" From 046b1f699cabb9be26ed97ef8bb09c362f0b60e5 Mon Sep 17 00:00:00 2001 From: github-actions <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 22 Jul 2025 20:10:08 +0000 Subject: [PATCH 19/20] update wdl-ci config file after successful tests --- wdl-ci.config.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wdl-ci.config.json b/wdl-ci.config.json index 916f321..93f805e 100644 --- a/wdl-ci.config.json +++ b/wdl-ci.config.json @@ -7,7 +7,7 @@ "tasks": { "merge_and_prep": { "key": "merge_and_prep", - "digest": "z6qqmruro2r42bxwxxsbig34t5qfjhir", + "digest": "cgyktioycaor6b3vkycc5qmefyw65lmo", "tests": [ { "inputs": { From 51a6c0f409529e46075ad654c3e5ac40bb76d2f7 Mon Sep 17 00:00:00 2001 From: Karen Fang Date: Tue, 22 Jul 2025 16:27:40 -0400 Subject: [PATCH 20/20] Clean up README and update wf-common --- README.md | 2 +- wf-common | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index f72782a..a17d8f7 100644 --- a/README.md +++ b/README.md @@ -395,7 +395,7 @@ This script compiles bucket and file information for both the initial (staging) If data integrity tests pass, this script will upload a combined MANIFEST.tsv and the data promotion Markdown report under a metadata/{timestamp} directory in the staging bucket. Previous manifest files and reports will be kept. Next, it will rsync all files in the staging bucket to the curated bucket's upstream, downstream, cohort_analysis, and metadata directories. **Exercise caution when using this script**; files that are not present in the source (staging) bucket will be deleted at the destination (curated) bucket. -If data integrity tests fail, staging data cannot be promoted. The combined MANFIEST.tsv, Markdown report, and promote_staging_data_script.log will be locally available. +If data integrity tests fail, staging data cannot be promoted. The combined `MANIFEST.tsv`, Markdown report, and `promote_staging_data_script.log` will be locally available. The script defaults to a dry run, printing out the files that would be copied or deleted for each selected team. diff --git a/wf-common b/wf-common index 68cccc9..4bf08b0 160000 --- a/wf-common +++ b/wf-common @@ -1 +1 @@ -Subproject commit 68cccc9b07eedb063ab92c1475c98fd5e4fa7318 +Subproject commit 4bf08b09cbf8f7cc0c6ee61ad31cf6e2e230696b