From 460f86c431c719605169a805410fc559a52d979d Mon Sep 17 00:00:00 2001 From: andrejdvornik Date: Tue, 18 Nov 2025 12:58:38 +0000 Subject: [PATCH 01/37] Some more optimisation in covariance constructor, more flexible file reading --- scripts/add_gt.sh | 2 +- scripts/add_wt.sh | 2 +- scripts/add_xipm.sh | 2 +- scripts/covariance_constructor_3x2pt.sh | 166 +++++++++++-------- scripts/run_measure_statistics_cats2stats.py | 2 +- 5 files changed, 97 insertions(+), 77 deletions(-) diff --git a/scripts/add_gt.sh b/scripts/add_gt.sh index 9bb6ef3b..f7fbfe37 100644 --- a/scripts/add_gt.sh +++ b/scripts/add_gt.sh @@ -25,7 +25,7 @@ then outlist=${file} elif [ -d @BV:TREECORRGT@ ] then - filelist=`ls @BV:TREECORRGT@/*.asc*` + filelist=`ls @BV:TREECORRGT@/*.txt*` for file in ${filelist} do cp ${file} @RUNROOT@/@STORAGEPATH@/@DATABLOCK@/gt/${file##*/} diff --git a/scripts/add_wt.sh b/scripts/add_wt.sh index c65c066d..19777fff 100644 --- a/scripts/add_wt.sh +++ b/scripts/add_wt.sh @@ -25,7 +25,7 @@ then outlist=${file} elif [ -d @BV:TREECORRWT@ ] then - filelist=`ls @BV:TREECORRWT@/*.asc*` + filelist=`ls @BV:TREECORRWT@/*.txt*` for file in ${filelist} do cp ${file} @RUNROOT@/@STORAGEPATH@/@DATABLOCK@/wt/${file##*/} diff --git a/scripts/add_xipm.sh b/scripts/add_xipm.sh index 4ad1a8fd..23364004 100644 --- a/scripts/add_xipm.sh +++ b/scripts/add_xipm.sh @@ -25,7 +25,7 @@ then outlist=${file} elif [ -d @BV:TREECORRXIPM@ ] then - filelist=`ls @BV:TREECORRXIPM@/*.asc*` + filelist=`ls @BV:TREECORRXIPM@/*.txt*` for file in ${filelist} do cp ${file} @RUNROOT@/@STORAGEPATH@/@DATABLOCK@/xipm/${file##*/} diff --git a/scripts/covariance_constructor_3x2pt.sh b/scripts/covariance_constructor_3x2pt.sh index 177452a7..fe0a18a6 100644 --- a/scripts/covariance_constructor_3x2pt.sh +++ b/scripts/covariance_constructor_3x2pt.sh @@ -174,46 +174,58 @@ then z_mins="" z_maxs="" file1="@RUNROOT@/@STORAGEPATH@/@DATABLOCK@/smf_lens_cats_metadata/stats_LB1.txt" - slice=`grep '^slice_in' ${file1} | awk '{printf $2}'` - if [ "${slice}" == "obs" ] + if [ -f ${file1} ] then - for i in `seq ${NOBS}` - do - file="@RUNROOT@/@STORAGEPATH@/@DATABLOCK@/smf_lens_cats_metadata/stats_LB${i}.txt" - x_lo=`grep '^x_lims_lo' ${file} | awk '{printf $2}'` - x_hi=`grep '^x_lims_hi' ${file} | awk '{printf $2}'` - y_lo=`grep '^y_lims_lo' ${file} | awk '{printf $2}'` - y_hi=`grep '^y_lims_hi' ${file} | awk '{printf $2}'` - obs_mins="${obs_mins} $(echo "$x_lo + l(@BV:H0_IN@)/l(10)" | bc -l)" - obs_maxs="${obs_maxs} $(echo "$x_hi + l(@BV:H0_IN@)/l(10)" | bc -l)" - z_mins="${z_mins} ${y_lo}" - z_maxs="${z_maxs} ${y_hi}" - done - elif [ "${slice}" == "z" ] - then - for i in `seq ${NOBS}` - do - file="@RUNROOT@/@STORAGEPATH@/@DATABLOCK@/smf_lens_cats_metadata/stats_LB${i}.txt" - x_lo=`grep '^x_lims_lo' ${file} | awk '{printf $2}'` - x_hi=`grep '^x_lims_hi' ${file} | awk '{printf $2}'` - y_lo=`grep '^y_lims_lo' ${file} | awk '{printf $2}'` - y_hi=`grep '^y_lims_hi' ${file} | awk '{printf $2}'` - obs_mins="${obs_mins} $(echo "$y_lo + l(@BV:H0_IN@)/l(10)" | bc -l)" - obs_maxs="${obs_maxs} $(echo "$y_hi + l(@BV:H0_IN@)/l(10)" | bc -l)" - z_mins="${z_mins} ${x_lo}" - z_maxs="${z_maxs} ${x_hi}" - done + slice=`grep '^slice_in' ${file1} | awk '{printf $2}'` + if [ "${slice}" == "obs" ] + then + for i in `seq ${NOBS}` + do + file="@RUNROOT@/@STORAGEPATH@/@DATABLOCK@/smf_lens_cats_metadata/stats_LB${i}.txt" + x_lo=`grep '^x_lims_lo' ${file} | awk '{printf $2}'` + x_hi=`grep '^x_lims_hi' ${file} | awk '{printf $2}'` + y_lo=`grep '^y_lims_lo' ${file} | awk '{printf $2}'` + y_hi=`grep '^y_lims_hi' ${file} | awk '{printf $2}'` + obs_mins="${obs_mins} $(echo "$x_lo + l(@BV:H0_IN@)/l(10)" | bc -l)" + obs_maxs="${obs_maxs} $(echo "$x_hi + l(@BV:H0_IN@)/l(10)" | bc -l)" + z_mins="${z_mins} ${y_lo}" + z_maxs="${z_maxs} ${y_hi}" + done + elif [ "${slice}" == "z" ] + then + for i in `seq ${NOBS}` + do + file="@RUNROOT@/@STORAGEPATH@/@DATABLOCK@/smf_lens_cats_metadata/stats_LB${i}.txt" + x_lo=`grep '^x_lims_lo' ${file} | awk '{printf $2}'` + x_hi=`grep '^x_lims_hi' ${file} | awk '{printf $2}'` + y_lo=`grep '^y_lims_lo' ${file} | awk '{printf $2}'` + y_hi=`grep '^y_lims_hi' ${file} | awk '{printf $2}'` + obs_mins="${obs_mins} $(echo "$y_lo + l(@BV:H0_IN@)/l(10)" | bc -l)" + obs_maxs="${obs_maxs} $(echo "$y_hi + l(@BV:H0_IN@)/l(10)" | bc -l)" + z_mins="${z_mins} ${x_lo}" + z_maxs="${z_maxs} ${x_hi}" + done + else + _message "Got wrong or no information about slicing of the lens sample.\n" + #exit 1 + fi + cstellar_mf=True + csmf_Mmin=`echo ${obs_mins} | sed 's/ /,/g'` + csmf_Mmax=`echo ${obs_maxs} | sed 's/ /,/g'` + csmf_N_log10M_bin=@BV:NSMFBINS@ + csmf_directory="@RUNROOT@/@STORAGEPATH@/@DATABLOCK@/smf/" + V_max_file="@DB:vmax@" # This assumes one file, currently we have NSMFLENSBINS + f_tomo_file="@DB:f_tomo@" # This assumes one file, currently we have NSMFLENSBINS else - _message "Got wrong or no information about slicing of the lens sample.\n" - #exit 1 + _message "No SMF lens catalog metadata found, setting default CSMF parameters from saved variables.\n" + cstellar_mf=True + csmf_Min=$(echo @BV:SMFLENSLIMSX@ | awk '{for(i=1; i Date: Tue, 18 Nov 2025 13:02:18 +0000 Subject: [PATCH 02/37] Include last element --- scripts/covariance_constructor_3x2pt.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/covariance_constructor_3x2pt.sh b/scripts/covariance_constructor_3x2pt.sh index fe0a18a6..fa79e3f7 100644 --- a/scripts/covariance_constructor_3x2pt.sh +++ b/scripts/covariance_constructor_3x2pt.sh @@ -220,7 +220,7 @@ then _message "No SMF lens catalog metadata found, setting default CSMF parameters from saved variables.\n" cstellar_mf=True csmf_Min=$(echo @BV:SMFLENSLIMSX@ | awk '{for(i=1; i Date: Tue, 18 Nov 2025 14:59:08 +0000 Subject: [PATCH 03/37] More changes to covariance constructor --- scripts/covariance_constructor_3x2pt.sh | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/scripts/covariance_constructor_3x2pt.sh b/scripts/covariance_constructor_3x2pt.sh index fa79e3f7..7d8e5da2 100644 --- a/scripts/covariance_constructor_3x2pt.sh +++ b/scripts/covariance_constructor_3x2pt.sh @@ -218,9 +218,15 @@ then f_tomo_file="@DB:f_tomo@" # This assumes one file, currently we have NSMFLENSBINS else _message "No SMF lens catalog metadata found, setting default CSMF parameters from saved variables.\n" + if [ "${NOBS}" = "1" ] + then + csmf_Mmin=$(echo @BV:SMFLENSLIMSX@ | awk '{print $1}') + csmf_Mmax=$(echo @BV:SMFLENSLIMSX@ | awk '{print $2}') + else + csmf_Mmin=$(echo @BV:SMFLENSLIMSX@ | awk '{for(i=1; i Date: Wed, 19 Nov 2025 14:17:48 +0000 Subject: [PATCH 04/37] Some fixes --- scripts/covariance_constructor_3x2pt.sh | 1 + scripts/make_data_vector_allstats.py | 2 +- scripts/run_covariance.sh | 14 +++++- scripts/save_and_check_sacc.py | 4 +- scripts/save_and_check_sacc.sh | 59 +++++++++++++++++++------ 5 files changed, 61 insertions(+), 19 deletions(-) diff --git a/scripts/covariance_constructor_3x2pt.sh b/scripts/covariance_constructor_3x2pt.sh index 7d8e5da2..05ac690a 100644 --- a/scripts/covariance_constructor_3x2pt.sh +++ b/scripts/covariance_constructor_3x2pt.sh @@ -622,6 +622,7 @@ est_clust = ${est_clust} cstellar_mf = ${cstellar_mf} cross_terms = True unbiased_clustering = True +# combinations_clustering = $(seq 0 $((NLENS-1)) | sed 's/.*/&-&/' | paste -sd, -) [csmf settings] csmf_log10Mmin = ${csmf_Mmin} diff --git a/scripts/make_data_vector_allstats.py b/scripts/make_data_vector_allstats.py index 160925aa..8ad15a2f 100644 --- a/scripts/make_data_vector_allstats.py +++ b/scripts/make_data_vector_allstats.py @@ -32,7 +32,7 @@ def make_2pt_vector(input_files, m_corr,col=1, xipm=False, correlations='EE'): for rp in range(len(input_files)): with open(input_files[rp]) as f: header=f.readline().strip('#').split() - data = pd.read_csv(input_files[rp], delim_whitespace=True, comment = '#', names = header) + data = pd.read_csv(input_files[rp], sep='\s+', comment = '#', names = header) if correlations == 'EE': data_xip = data['xip'] data_xim = data['xim'] diff --git a/scripts/run_covariance.sh b/scripts/run_covariance.sh index 91135596..ce81f9f4 100644 --- a/scripts/run_covariance.sh +++ b/scripts/run_covariance.sh @@ -41,9 +41,19 @@ _message " >@RED@ Done! (`date +'%a %H:%M'`)@DEF@\n" if [ "${SECONDSTATISTIC^^}" == "XIPM" ] || [ "${SECONDSTATISTIC^^}" == "2PCF" ] || [ "${SECONDSTATISTIC^^}" == "COSEBIS" ] || [ "${SECONDSTATISTIC^^}" == "BANDPOWERS" ] then - _write_datablock "covariance_@BV:STATISTIC@_@BV:SECONDSTATISTIC@" "covariance_matrix_${non_linear_model}.mat" + if [ -f @RUNROOT@/@STORAGEPATH@/@DATABLOCK@/covariance_@BV:STATISTIC@_@BV:SECONDSTATISTIC@/covariance_matrix_${non_linear_model}_reduced.mat ] + then + _write_datablock "covariance_@BV:STATISTIC@_@BV:SECONDSTATISTIC@" "covariance_matrix_${non_linear_model}_reduced.mat" + else + _write_datablock "covariance_@BV:STATISTIC@_@BV:SECONDSTATISTIC@" "covariance_matrix_${non_linear_model}.mat" + fi else - _write_datablock "covariance_@BV:STATISTIC@" "covariance_matrix_${non_linear_model}.mat" + if [ -f @RUNROOT@/@STORAGEPATH@/@DATABLOCK@/covariance_@BV:STATISTIC@/covariance_matrix_${non_linear_model}_reduced.mat ] + then + _write_datablock "covariance_@BV:STATISTIC@" "covariance_matrix_${non_linear_model}_reduced.mat" + else + _write_datablock "covariance_@BV:STATISTIC@" "covariance_matrix_${non_linear_model}.mat" + fi fi #Clean up the temporary input folder diff --git a/scripts/save_and_check_sacc.py b/scripts/save_and_check_sacc.py index f89bedd3..ec322063 100644 --- a/scripts/save_and_check_sacc.py +++ b/scripts/save_and_check_sacc.py @@ -378,8 +378,8 @@ def process_2pt_data(mode, nangle, min_val, max_val, n1, n2, datavec_files, stat add_1pt_points(sacc_data_no_mbias, smfvec) - covariance = np.loadtxt(args.covarianceFile[0]) - #covariance = 0.1 * np.eye(len(sacc_data.data)) # Placeholder for covariance matrix, replace with actual loading if needed + #covariance = np.loadtxt(args.covarianceFile[0]) + covariance = 0.1 * np.eye(len(sacc_data.data)) # Placeholder for covariance matrix, replace with actual loading if needed sacc_data.add_covariance(covariance) sacc_data_no_mbias.add_covariance(covariance) diff --git a/scripts/save_and_check_sacc.sh b/scripts/save_and_check_sacc.sh index cd669350..c7cc6a21 100644 --- a/scripts/save_and_check_sacc.sh +++ b/scripts/save_and_check_sacc.sh @@ -47,13 +47,24 @@ then else input_datavector_nn= fi - input_covariance="@RUNROOT@/@STORAGEPATH@/@DATABLOCK@/covariance_cosebis/covariance_matrix_${non_linear_model}.mat" - #input_covariance="@RUNROOT@/@STORAGEPATH@/@DATABLOCK@/covariance_cosebis/covariance_list_${non_linear_model}.dat" - if [ -n "$ITERATION" ] && [ "$ITERATION" -eq "$ITERATION" ] + if [ -f @RUNROOT@/@STORAGEPATH@/@DATABLOCK@/covariance_cosebis/covariance_matrix_${non_linear_model}_reduced.mat ] then - filename_extension=${CHAINSUFFIX}_iteration_${ITERATION} - input_covariance_iterative=@RUNROOT@/@STORAGEPATH@/@DATABLOCK@/covariance_cosebis/covariance_matrix_${non_linear_model}${filename_extension}.mat - #input_covariance_iterative=@RUNROOT@/@STORAGEPATH@/@DATABLOCK@/covariance_cosebis/covariance_list_${non_linear_model}${filename_extension}.dat + input_covariance="@RUNROOT@/@STORAGEPATH@/@DATABLOCK@/covariance_cosebis/covariance_matrix_${non_linear_model}_reduced.mat" + if [ -n "$ITERATION" ] && [ "$ITERATION" -eq "$ITERATION" ] + then + filename_extension=${CHAINSUFFIX}_iteration_${ITERATION} + input_covariance_iterative=@RUNROOT@/@STORAGEPATH@/@DATABLOCK@/covariance_cosebis/covariance_matrix_${non_linear_model}_reduced${filename_extension}.mat + fi + #input_covariance="@RUNROOT@/@STORAGEPATH@/@DATABLOCK@/covariance_cosebis/covariance_list_${non_linear_model}_reduced.dat" + else + input_covariance="@RUNROOT@/@STORAGEPATH@/@DATABLOCK@/covariance_cosebis/covariance_matrix_${non_linear_model}.mat" + #input_covariance="@RUNROOT@/@STORAGEPATH@/@DATABLOCK@/covariance_cosebis/covariance_list_${non_linear_model}.dat" + if [ -n "$ITERATION" ] && [ "$ITERATION" -eq "$ITERATION" ] + then + filename_extension=${CHAINSUFFIX}_iteration_${ITERATION} + input_covariance_iterative=@RUNROOT@/@STORAGEPATH@/@DATABLOCK@/covariance_cosebis/covariance_matrix_${non_linear_model}${filename_extension}.mat + #input_covariance_iterative=@RUNROOT@/@STORAGEPATH@/@DATABLOCK@/covariance_cosebis/covariance_list_${non_linear_model}${filename_extension}.dat + fi fi #}}} elif [ "${STATISTIC^^}" == "COSEBIS_DIMLESS" ] #{{{ @@ -88,11 +99,21 @@ then else input_datavector_nn= fi - input_covariance="@RUNROOT@/@STORAGEPATH@/@DATABLOCK@/covariance_bandpowers/covariance_matrix_${non_linear_model}.mat" - if [ -n "$ITERATION" ] && [ "$ITERATION" -eq "$ITERATION" ] + if [ -f @RUNROOT@/@STORAGEPATH@/@DATABLOCK@/covariance_bandpowers/covariance_matrix_${non_linear_model}_reduced.mat ] then - filename_extension=${CHAINSUFFIX}_iteration_${ITERATION} - input_covariance_iterative=@RUNROOT@/@STORAGEPATH@/@DATABLOCK@/covariance_bandpowers/covariance_matrix_${non_linear_model}${filename_extension}.mat + input_covariance="@RUNROOT@/@STORAGEPATH@/@DATABLOCK@/covariance_bandpowers/covariance_matrix_${non_linear_model}_reduced.mat" + if [ -n "$ITERATION" ] && [ "$ITERATION" -eq "$ITERATION" ] + then + filename_extension=${CHAINSUFFIX}_iteration_${ITERATION} + input_covariance_iterative=@RUNROOT@/@STORAGEPATH@/@DATABLOCK@/covariance_bandpowers/covariance_matrix_${non_linear_model}_reduced${filename_extension}.mat + fi + else + input_covariance="@RUNROOT@/@STORAGEPATH@/@DATABLOCK@/covariance_bandpowers/covariance_matrix_${non_linear_model}.mat" + if [ -n "$ITERATION" ] && [ "$ITERATION" -eq "$ITERATION" ] + then + filename_extension=${CHAINSUFFIX}_iteration_${ITERATION} + i nput_covariance_iterative=@RUNROOT@/@STORAGEPATH@/@DATABLOCK@/covariance_bandpowers/covariance_matrix_${non_linear_model}${filename_extension}.mat + fi fi #}}} elif [ "${STATISTIC^^}" == "2PCF" ] #{{{ @@ -115,11 +136,21 @@ then else input_datavector_nn= fi - input_covariance="@RUNROOT@/@STORAGEPATH@/@DATABLOCK@/covariance_2pcf/covariance_matrix_${non_linear_model}.mat" - if [ -n "$ITERATION" ] && [ "$ITERATION" -eq "$ITERATION" ] + if [ -f @RUNROOT@/@STORAGEPATH@/@DATABLOCK@/covariance_2pcf/covariance_matrix_${non_linear_model}_reduced.mat ] then - filename_extension=${CHAINSUFFIX}_iteration_${ITERATION} - input_covariance_iterative=@RUNROOT@/@STORAGEPATH@/@DATABLOCK@/covariance_2pcf/covariance_matrix_${non_linear_model}${filename_extension}.mat + input_covariance="@RUNROOT@/@STORAGEPATH@/@DATABLOCK@/covariance_2pcf/covariance_matrix_${non_linear_model}_reduced.mat" + if [ -n "$ITERATION" ] && [ "$ITERATION" -eq "$ITERATION" ] + then + filename_extension=${CHAINSUFFIX}_iteration_${ITERATION} + input_covariance_iterative=@RUNROOT@/@STORAGEPATH@/@DATABLOCK@/covariance_2pcf/covariance_matrix_${non_linear_model}_reduced${filename_extension}.mat + fi + else + input_covariance="@RUNROOT@/@STORAGEPATH@/@DATABLOCK@/covariance_2pcf/covariance_matrix_${non_linear_model}.mat" + if [ -n "$ITERATION" ] && [ "$ITERATION" -eq "$ITERATION" ] + then + filename_extension=${CHAINSUFFIX}_iteration_${ITERATION} + input_covariance_iterative=@RUNROOT@/@STORAGEPATH@/@DATABLOCK@/covariance_2pcf/covariance_matrix_${non_linear_model}${filename_extension}.mat + fi fi elif [ "${STATISTIC^^}" == "XIEB" ] #{{{ then From 60b9d070d951604a672fc0fe8f8bd7593d8160bd Mon Sep 17 00:00:00 2001 From: andrejdvornik Date: Thu, 20 Nov 2025 07:19:15 +0000 Subject: [PATCH 05/37] Create separate make_cosmosis_nz for 3x2pt --- man/make_cosmosis_nz_3x2pt.man.sh | 144 ++++++++++++++++ scripts/make_cosmosis_nz_3x2pt.sh | 262 ++++++++++++++++++++++++++++++ 2 files changed, 406 insertions(+) create mode 100644 man/make_cosmosis_nz_3x2pt.man.sh create mode 100644 scripts/make_cosmosis_nz_3x2pt.sh diff --git a/man/make_cosmosis_nz_3x2pt.man.sh b/man/make_cosmosis_nz_3x2pt.man.sh new file mode 100644 index 00000000..2aaa599b --- /dev/null +++ b/man/make_cosmosis_nz_3x2pt.man.sh @@ -0,0 +1,144 @@ +# +# make_cosmosis_nz.sh Documentation & Housekeeping functions +# + +#Starting Prompt {{{ +function _prompt { + #Check if we do want verbose output + if [ "$1" != "0" ] + then + _message "@BLU@========================================@DEF@\n" + _message "@BLU@== @RED@ Running make_cosmosis_nz_3x2pt.sh Mode @BLU@ ==@DEF@\n" + _message "@BLU@========================================@DEF@\n" + fi +} +#}}} + +#Mode description {{{ +function _description { + echo "#" + echo '# Construct an Nz file in the format expected by ' + echo '# cosmosis' + echo "#" + echo "# Function takes input data:" + echo "# `_inp_data`" + echo "#" +} +#}}} + +# Abort Message {{{ +_abort() +{ + #Message to print when script aborts + #$0 is the script that was running when this error occurred + _message "@BLU@ An error occured while running:\n@DEF@$0.\n" >&2 + _message "@BLU@ Check the logging file for this step in:\n" >&2 + _message "@DEF@@RUNROOT@/@LOGPATH@/\n" >&2 + exit 1 +} +trap '_abort' 0 +set -e +#}}} + +# Input variables {{{ +function _inp_var { + #Variable inputs (leave blank if none) + echo ALLPATCH BLU BV:NLENSBINS BV:NSMFLENSBINS BV:PATCHLIST BV:TOMOLIMS DATABLOCK DEF PYTHON3BIN RED RUNROOT SCRIPTPATH STORAGEPATH +} +#}}} + +# Input data {{{ +function _inp_data { + #Data inputs (leave blank if none) + outlist='' + #input is dynamic, depending on the value of BV:COSMOSIS_PATCHLIST + patchvar="@BV:COSMOSIS_PATCHLIST@" + patchvar=`_parse_blockvars ${patchvar}` + MODES_IN=`_parse_blockvars @BV:MODES@` + modes='' + if [[ .*\ $MODES_IN\ .* =~ " EE " ]] || [[ .*\ $MODES_IN\ .* =~ " NE " ]] + then + modes="${modes} source" + fi + if [[ .*\ $MODES_IN\ .* =~ " NE " ]] || [[ .*\ $MODES_IN\ .* =~ " NN " ]] + then + modes="${modes} lens" + fi + if [[ .*\ $MODES_IN\ .* =~ " OBS " ]] + then + modes="${modes} obs" + fi + #Define the patches to loop over {{{ + if [ "${patchvar}" == "ALL" ] || [ "${patchvar}" == "@BV:COSMOSIS_PATCHLIST@" ] + then + patchlist=`echo @BV:PATCHLIST@ @ALLPATCH@ @ALLPATCH@comb` + else + patchlist="${patchvar}" + fi + #}}} + for mode in ${modes} + do + for patch in ${patchlist} + do + #>&2 echo ${patch} + outlist="${outlist} cosmosis_neff_${mode} nz_${mode}_${patch}" + done + done + echo ${outlist} +} +#}}} + +# Output data {{{ +function _outputs { + #Data outputs (leave blank if none) + outlist='' + patchvar="@BV:COSMOSIS_PATCHLIST@" + patchvar=`_parse_blockvars ${patchvar}` + MODES_IN=`_parse_blockvars @BV:MODES@` + modes='' + if [[ .*\ $MODES_IN\ .* =~ " EE " ]] || [[ .*\ $MODES_IN\ .* =~ " NE " ]] + then + modes="${modes} source" + fi + if [[ .*\ $MODES_IN\ .* =~ " NE " ]] || [[ .*\ $MODES_IN\ .* =~ " NN " ]] + then + modes="${modes} lens" + fi + if [[ .*\ $MODES_IN\ .* =~ " OBS " ]] + then + modes="${modes} obs" + fi + #Define the patches to loop over {{{ + if [ "${patchvar}" == "ALL" ] || [ "${patchvar}" == "@BV:COSMOSIS_PATCHLIST@" ] + then + patchlist=`echo @BV:PATCHLIST@ @ALLPATCH@ @ALLPATCH@comb` + else + patchlist="${patchvar}" + fi + for mode in ${modes} + do + for patch in ${patchlist} + do + outlist="${outlist} cosmosis_nz_${mode}_${patch}" + done + done + echo ${outlist} +} +#}}} + +# Execution command {{{ +function _runcommand { + #Command for running the script + echo bash @RUNROOT@/@SCRIPTPATH@/make_cosmosis_nz_3x2pt.sh +} +#}}} + +# Unset Function command {{{ +function _unset_functions { + #Remove these functions from the environment + unset -f _prompt _description _inp_data _inp_var _abort _outputs _runcommand _unset_functions +} +#}}} + +#Additional Functions + diff --git a/scripts/make_cosmosis_nz_3x2pt.sh b/scripts/make_cosmosis_nz_3x2pt.sh new file mode 100644 index 00000000..bd44a49f --- /dev/null +++ b/scripts/make_cosmosis_nz_3x2pt.sh @@ -0,0 +1,262 @@ +#========================================= +# +# File Name : make_cosmosis_nz_3x2pt.sh +# Created By : awright +# Creation Date : 30-03-2023 +# Last Modified : Tue 21 Nov 2023 12:55:58 AM CET +# +#========================================= + +NTOMO=`echo @BV:TOMOLIMS@ | awk '{print NF-1}'` +NLENS="@BV:NLENSBINS@" +NOBS="@BV:NSMFLENSBINS@" +outputlist='' +found_source="FALSE" +found_lens="FALSE" +found_obs="FALSE" +for patch in @BV:PATCHLIST@ @ALLPATCH@ +do + _message " ->@BLU@ Patch @RED@${patch}@DEF@" + #Get all the files in this stat and patch {{{ + inputs=`_read_datablock "nz_source_${patch}"` + inputs=`_blockentry_to_filelist ${inputs}` + #}}} + filelist='' + #Get the file list {{{ + for ZBIN1 in `seq ${NTOMO}` + do + #Define the Z_B limits from the TOMOLIMS {{{ + ZB_lo=`echo @BV:TOMOLIMS@ | awk -v n=$ZBIN1 '{print $n}'` + ZB_hi=`echo @BV:TOMOLIMS@ | awk -v n=$ZBIN1 '{print $(n+1)}'` + #}}} + #Define the string to append to the file names {{{ + ZB_lo_str=`echo $ZB_lo | sed 's/\./p/g'` + ZB_hi_str=`echo $ZB_hi | sed 's/\./p/g'` + appendstr="_ZB${ZB_lo_str}t${ZB_hi_str}" + #}}} + #file=`echo ${inputs} | sed 's/ /\n/g' | grep "_${patch}_" | grep ${appendstr} || echo ` + file=`echo ${inputs} | sed 's/ /\n/g' | grep ${appendstr} || echo ` + #Check if the output file exists {{{ + if [ "${file}" == "" ] + then + #If not, loop + continue + fi + #}}} + filelist="${filelist} @RUNROOT@/@STORAGEPATH@/@DATABLOCK@/nz_source_${patch}/${file}" + done + #}}} + #If filelist is empty, skip {{{ + if [ "${filelist}" == "" ] + then + _message "@RED@ - skipping! (No matching source Nz files)@DEF@\n" + continue + fi + #}}} + found_source='TRUE' + #Construct the output directory {{{ + if [ ! -d @RUNROOT@/@STORAGEPATH@/@DATABLOCK@/cosmosis_nz_source_${patch} ] + then + mkdir @RUNROOT@/@STORAGEPATH@/@DATABLOCK@/cosmosis_nz_source_${patch}/ + fi + #}}} + _message "@RED@ - OK! (`date +'%a %H:%M'`)@DEF@\n" + #Construct the output base {{{ + file=${filelist##* } + output_base=${file##*/} + output_base=${output_base%%_ZB*} + output_base="@RUNROOT@/@STORAGEPATH@/@DATABLOCK@/cosmosis_nz_source_${patch}/${output_base}" + #}}} + #Remove existing files {{{ + if [ -f ${output_base}_comb_Nz.fits ] + then + _message " > @BLU@Removing previous COSMOSIS Nz file@DEF@ ${output_base##*/}_comb_Nz.fits@DEF@" + rm ${output_base}_comb_Nz.fits + _message " - @RED@Done! (`date +'%a %H:%M'`)@DEF@\n" + fi + #}}} + #Construct the redshift file {{{ + _message " > @BLU@Constructing COSMOSIS Nz file @RED@${output_base}_comb_Nz.fits@DEF@" + #Construct the Nz combined fits file and put into covariance/input/ + @PYTHON3BIN@ @RUNROOT@/@SCRIPTPATH@/MakeNofZForCosmosis_function.py \ + --inputs ${filelist} \ + --neff @DB:cosmosis_neff_source@ \ + --output_base ${output_base} \ + --suffix "source" 2>&1 + _message " - @RED@Done! (`date +'%a %H:%M'`)@DEF@\n" + #}}} + + #Update the datablock + _write_datablock cosmosis_nz_source_${patch} "${output_base##*/}_comb_Nz.fits" + + + + + #Get all the files in this stat and patch {{{ + inputs_lens=`_read_datablock "nz_lens_${patch}"` + inputs_lens=`_blockentry_to_filelist ${inputs_lens}` + #}}} + filelist_lens='' + #Get the file list {{{ + for LBIN in `seq ${NLENS}` + do + #Define the string to append to the file names {{{ + appendstr="_LB${LBIN}" + #}}} + #file=`echo ${inputs} | sed 's/ /\n/g' | grep "_${patch}_" | grep ${appendstr} || echo ` + file=`echo ${inputs_lens} | sed 's/ /\n/g' | grep ${appendstr} || echo ` + appendstr="_LB${LBIN}" + #Check if the output file exists {{{ + if [ "${file}" == "" ] + then + #If not, loop + continue + fi + #}}} + filelist_lens="${filelist_lens} @RUNROOT@/@STORAGEPATH@/@DATABLOCK@/nz_lens_${patch}/${file}" + done + #}}} + #If filelist is empty, skip {{{ + if [ "${filelist_lens}" == "" ] + then + _message "@RED@ - skipping! (No matching lens Nz files)@DEF@\n" + continue + fi + #}}} + found_lens='TRUE' + #Construct the output directory {{{ + if [ ! -d @RUNROOT@/@STORAGEPATH@/@DATABLOCK@/cosmosis_nz_lens_${patch} ] + then + mkdir @RUNROOT@/@STORAGEPATH@/@DATABLOCK@/cosmosis_nz_lens_${patch}/ + fi + #}}} + _message "@RED@ - OK! (`date +'%a %H:%M'`)@DEF@\n" + #Construct the output base {{{ + file=${filelist_lens##* } + output_base=${file##*/} + output_base=${output_base%%_ZB*} + output_base="@RUNROOT@/@STORAGEPATH@/@DATABLOCK@/cosmosis_nz_lens_${patch}/${output_base}" + #}}} + #Remove existing files {{{ + if [ -f ${output_base}_comb_Nz.fits ] + then + _message " > @BLU@Removing previous COSMOSIS Nz file@DEF@ ${output_base##*/}_comb_Nz.fits@DEF@" + rm ${output_base}_comb_Nz.fits + _message " - @RED@Done! (`date +'%a %H:%M'`)@DEF@\n" + fi + #}}} + #Construct the redshift file {{{ + _message " > @BLU@Constructing COSMOSIS Nz file @RED@${output_base}_comb_Nz.fits@DEF@" + #Construct the Nz combined fits file and put into covariance/input/ + @PYTHON3BIN@ @RUNROOT@/@SCRIPTPATH@/MakeNofZForCosmosis_function.py \ + --inputs ${filelist_lens} \ + --neff @DB:cosmosis_neff_lens@ \ + --output_base ${output_base} \ + --suffix "lens" 2>&1 + _message " - @RED@Done! (`date +'%a %H:%M'`)@DEF@\n" + #}}} + + #Update the datablock + _write_datablock cosmosis_nz_lens_${patch} "${output_base##*/}_comb_Nz.fits" + + + + #Get all the files in this stat and patch {{{ + inputs_obs=`_read_datablock "nz_obs_${patch}"` + inputs_obs=`_blockentry_to_filelist ${inputs_obs}` + #}}} + filelist_obs='' + #Get the file list {{{ + for LBIN in `seq ${NOBS}` + do + #Define the string to append to the file names {{{ + appendstr="_LB${LBIN}" + #}}} + #file=`echo ${inputs} | sed 's/ /\n/g' | grep "_${patch}_" | grep ${appendstr} || echo ` + file=`echo ${inputs_obs} | sed 's/ /\n/g' | grep ${appendstr} || echo ` + appendstr="_LB${LBIN}" + #Check if the output file exists {{{ + if [ "${file}" == "" ] + then + #If not, loop + continue + fi + #}}} + filelist_obs="${filelist_obs} @RUNROOT@/@STORAGEPATH@/@DATABLOCK@/nz_obs_${patch}/${file}" + done + #}}} + #If filelist is empty, skip {{{ + if [ "${filelist_obs}" == "" ] + then + _message "@RED@ - skipping! (No matching obs Nz files)@DEF@\n" + continue + fi + #}}} + found_obs='TRUE' + #Construct the output directory {{{ + if [ ! -d @RUNROOT@/@STORAGEPATH@/@DATABLOCK@/cosmosis_nz_obs_${patch} ] + then + mkdir @RUNROOT@/@STORAGEPATH@/@DATABLOCK@/cosmosis_nz_obs_${patch}/ + fi + #}}} + _message "@RED@ - OK! (`date +'%a %H:%M'`)@DEF@\n" + #Construct the output base {{{ + file=${filelist_obs##* } + output_base=${file##*/} + output_base=${output_base%%_ZB*} + output_base="@RUNROOT@/@STORAGEPATH@/@DATABLOCK@/cosmosis_nz_obs_${patch}/${output_base}" + #}}} + #Remove existing files {{{ + if [ -f ${output_base}_comb_Nz.fits ] + then + _message " > @BLU@Removing previous COSMOSIS Nz file@DEF@ ${output_base##*/}_comb_Nz.fits@DEF@" + rm ${output_base}_comb_Nz.fits + _message " - @RED@Done! (`date +'%a %H:%M'`)@DEF@\n" + fi + #}}} + #Construct the redshift file {{{ + _message " > @BLU@Constructing COSMOSIS Nz file @RED@${output_base}_comb_Nz.fits@DEF@" + #Construct the Nz combined fits file and put into covariance/input/ + @PYTHON3BIN@ @RUNROOT@/@SCRIPTPATH@/MakeNofZForCosmosis_function.py \ + --inputs ${filelist_obs} \ + --neff @DB:cosmosis_neff_obs@ \ + --output_base ${output_base} \ + --suffix "obs" 2>&1 + _message " - @RED@Done! (`date +'%a %H:%M'`)@DEF@\n" + #}}} + + #Update the datablock + _write_datablock cosmosis_nz_obs_${patch} "${output_base##*/}_comb_Nz.fits" + + +done + +#Error if no stat files found {{{ +if [ "${found_source}" == "FALSE" ] && [ "${NTOMO}" != 0 ] +then + #If not found, error + _message " - @RED@ERROR!@DEF@\n" + _message "@RED@There are no nz_source files in any patch?!@DEF@\n" + _message "@BLU@You probably didn't run rename an 'nz_source' block for a particular patch?!@DEF@\n" + exit 1 +fi + +if [ "${found_lens}" == "FALSE" ] && [ "${NLENS}" != 0 ] +then + #If not found, error + _message " - @RED@ERROR!@DEF@\n" + _message "@RED@There are no nz_lens files in any patch?!@DEF@\n" + _message "@BLU@You probably didn't run rename an 'nz_lens' block for a particular patch?!@DEF@\n" + exit 1 +fi + +if [ "${found_obs}" == "FALSE" ] && [ "${NOBS}" != 0 ] +then + #If not found, error + _message " - @RED@ERROR!@DEF@\n" + _message "@RED@There are no nz_obs files in any patch?!@DEF@\n" + _message "@BLU@You probably didn't run rename an 'nz_obs' block for a particular patch?!@DEF@\n" + exit 1 +fi +#}}} + From 4da0590840d363197cc1a014f0f6846875fbdc7b Mon Sep 17 00:00:00 2001 From: andrejdvornik Date: Thu, 20 Nov 2025 07:32:03 +0000 Subject: [PATCH 06/37] Fixed capitalisation --- man/calculate_psistats.man.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/man/calculate_psistats.man.sh b/man/calculate_psistats.man.sh index 0016f32f..4c4a61c0 100644 --- a/man/calculate_psistats.man.sh +++ b/man/calculate_psistats.man.sh @@ -57,6 +57,7 @@ function _inp_data { function _outputs { #Data outputs (leave blank if none) output=`_parse_blockvars psi_stats_@BV:MODES@` + output=${output,,} echo $output } #}}} From 7539da09a9068b3b5b05784bcd8a11007ec6b0df Mon Sep 17 00:00:00 2001 From: andrejdvornik Date: Thu, 20 Nov 2025 07:39:34 +0000 Subject: [PATCH 07/37] Revert comment --- scripts/cosmosis_constructor_3x2pt.sh | 48 +++++++++++++-------------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/scripts/cosmosis_constructor_3x2pt.sh b/scripts/cosmosis_constructor_3x2pt.sh index c6d2424a..63a54d84 100644 --- a/scripts/cosmosis_constructor_3x2pt.sh +++ b/scripts/cosmosis_constructor_3x2pt.sh @@ -591,7 +591,7 @@ input_section_name = shear_cl output_section_name = shear_xi [xip_conv] -file = %(CSL_PATH)s/utility/convert_theta/convert_theta.py +file = %(HMPATH)s/cosmosis_modules/convert_theta/convert_theta.py output_units = arcmin section_name = shear_xi_plus @@ -699,12 +699,12 @@ EOF #}}} elif [ "${SAMPLER^^}" == "MULTINEST" ] #{{{ then -fast_slow="T" +fast_slow="F" cat > @RUNROOT@/@STORAGEPATH@/@DATABLOCK@/cosmosis_inputs/@SURVEY@_CosmoPipe_constructed_sampler.ini <<- EOF [multinest] -max_iterations=100000 +max_iterations=1000 ; 100000 multinest_outfile_root= %(OUTPUT_FOLDER)s/%(RUN_NAME)s_ -resume=T +resume=F tolerance = 0.01 constant_efficiency = F live_points = 1000 @@ -738,17 +738,18 @@ fast_slow="F" n_batch=`echo "@BV:NTHREADS@" | awk '{printf "%d", 4*$1}'` cat > @RUNROOT@/@STORAGEPATH@/@DATABLOCK@/cosmosis_inputs/@SURVEY@_CosmoPipe_constructed_sampler.ini <<- EOF [nautilus] -live_points = 4000 -enlarge_per_dim = 1.1 +n_live = 1000 ; 4000 +enlarge_per_dim = 2.0 ; 1.1 split_threshold = 100 n_networks = 8 n_batch = $n_batch filepath = %(OUTPUT_FOLDER)s/run_nautilus.hdf5 -resume = False ; True -f_live = 0.01 +resume = False +f_live = 0.5 ; 0.01 discard_exploration = True verbose = True -n_eff = 10000 +n_eff = 2000 ; 1000 ; 10000 +; n_like_max = 10000 EOF @@ -1062,7 +1063,6 @@ EOF fi cat >> @RUNROOT@/@STORAGEPATH@/@DATABLOCK@/cosmosis_inputs/@SURVEY@_CosmoPipe_constructed_pipe.ini <<- EOF extra_output = ${extraparams} ${shifts} ${listparam} ${tpdparams} -quiet = T timing = F ; T debug = F fast_slow = ${fast_slow} @@ -1070,7 +1070,7 @@ first_fast_module = hod ; halo_model_ingredients_halomod [runtime] sampler = %(SAMPLER_NAME)s -verbosity = quiet +verbosity = quiet ; normal ; quiet pool_stdout = F ; T [output] @@ -1215,15 +1215,15 @@ do sets="" if [[ .*\ $MODES\ .* =~ " EE " ]] || [[ .*\ $MODES\ .* =~ " NE " ]] then - sets="${sets} %(redshift_name)s" + sets="${sets} %(redshift_name)s" fi if [[ .*\ $MODES\ .* =~ " NE " ]] || [[ .*\ $MODES\ .* =~ " NN " ]] then - sets="${sets} %(redshift_name_lens)s" + sets="${sets} %(redshift_name_lens)s" fi if [[ .*\ $MODES\ .* =~ " OBS " ]] then - sets="${sets} %(redshift_name_obs)s" + sets="${sets} %(redshift_name_obs)s" fi cat >> @RUNROOT@/@STORAGEPATH@/@DATABLOCK@/cosmosis_inputs/@SURVEY@_CosmoPipe_constructed_other.ini <<- EOF [$module] @@ -1432,8 +1432,8 @@ do if [ "$add_intrinsic" == "True" ] then - cat >> @RUNROOT@/@STORAGEPATH@/@DATABLOCK@/cosmosis_inputs/@SURVEY@_CosmoPipe_constructed_other.ini <<- EOF - [$module] + cat >> @RUNROOT@/@STORAGEPATH@/@DATABLOCK@/cosmosis_inputs/@SURVEY@_CosmoPipe_constructed_other.ini <<- EOF + [$module] file= %(HMPATH)s/cosmosis_modules/onepower_interface.py p_mm = ${ee} p_gg = ${nn} @@ -1445,15 +1445,15 @@ do EOF else - cat >> @RUNROOT@/@STORAGEPATH@/@DATABLOCK@/cosmosis_inputs/@SURVEY@_CosmoPipe_constructed_other.ini <<- EOF - [$module] + cat >> @RUNROOT@/@STORAGEPATH@/@DATABLOCK@/cosmosis_inputs/@SURVEY@_CosmoPipe_constructed_other.ini <<- EOF + [$module] file= %(HMPATH)s/cosmosis_modules/onepower_interface.py - p_mm = ${ee} - p_gg = ${nn} - p_gm = ${ne} - p_gI = False - p_mI = False - p_II = False + p_mm = ${ee} + p_gg = ${nn} + p_gm = ${ne} + p_gI = False + p_mI = False + p_II = False split_ia = True EOF From f368c8399337ca27bc16e772bef7cf5d3d8d5c80 Mon Sep 17 00:00:00 2001 From: andrejdvornik Date: Thu, 20 Nov 2025 07:51:32 +0000 Subject: [PATCH 08/37] Revert comment --- scripts/save_and_check_sacc.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/save_and_check_sacc.py b/scripts/save_and_check_sacc.py index ec322063..f89bedd3 100644 --- a/scripts/save_and_check_sacc.py +++ b/scripts/save_and_check_sacc.py @@ -378,8 +378,8 @@ def process_2pt_data(mode, nangle, min_val, max_val, n1, n2, datavec_files, stat add_1pt_points(sacc_data_no_mbias, smfvec) - #covariance = np.loadtxt(args.covarianceFile[0]) - covariance = 0.1 * np.eye(len(sacc_data.data)) # Placeholder for covariance matrix, replace with actual loading if needed + covariance = np.loadtxt(args.covarianceFile[0]) + #covariance = 0.1 * np.eye(len(sacc_data.data)) # Placeholder for covariance matrix, replace with actual loading if needed sacc_data.add_covariance(covariance) sacc_data_no_mbias.add_covariance(covariance) From 9771b8880e8b22934c14a78546a49a3a72ba5201 Mon Sep 17 00:00:00 2001 From: andrejdvornik Date: Thu, 20 Nov 2025 07:54:59 +0000 Subject: [PATCH 09/37] Revert changes --- scripts/cosmosis_constructor_3x2pt.sh | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/scripts/cosmosis_constructor_3x2pt.sh b/scripts/cosmosis_constructor_3x2pt.sh index 63a54d84..bc3f6d84 100644 --- a/scripts/cosmosis_constructor_3x2pt.sh +++ b/scripts/cosmosis_constructor_3x2pt.sh @@ -699,10 +699,10 @@ EOF #}}} elif [ "${SAMPLER^^}" == "MULTINEST" ] #{{{ then -fast_slow="F" +fast_slow="T" cat > @RUNROOT@/@STORAGEPATH@/@DATABLOCK@/cosmosis_inputs/@SURVEY@_CosmoPipe_constructed_sampler.ini <<- EOF [multinest] -max_iterations=1000 ; 100000 +max_iterations=100000 multinest_outfile_root= %(OUTPUT_FOLDER)s/%(RUN_NAME)s_ resume=F tolerance = 0.01 @@ -738,18 +738,17 @@ fast_slow="F" n_batch=`echo "@BV:NTHREADS@" | awk '{printf "%d", 4*$1}'` cat > @RUNROOT@/@STORAGEPATH@/@DATABLOCK@/cosmosis_inputs/@SURVEY@_CosmoPipe_constructed_sampler.ini <<- EOF [nautilus] -n_live = 1000 ; 4000 -enlarge_per_dim = 2.0 ; 1.1 +n_live = 4000 +enlarge_per_dim = 1.1 split_threshold = 100 n_networks = 8 n_batch = $n_batch filepath = %(OUTPUT_FOLDER)s/run_nautilus.hdf5 -resume = False -f_live = 0.5 ; 0.01 +resume = False ; True +f_live = 0.01 discard_exploration = True verbose = True -n_eff = 2000 ; 1000 ; 10000 -; n_like_max = 10000 +n_eff = 10000 EOF From afdabe89eff453f76fc2fd6527eff367198a90af Mon Sep 17 00:00:00 2001 From: andrejdvornik Date: Thu, 20 Nov 2025 11:23:16 +0000 Subject: [PATCH 10/37] More fixes to covariance constructor --- scripts/covariance_constructor_3x2pt.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/covariance_constructor_3x2pt.sh b/scripts/covariance_constructor_3x2pt.sh index 05ac690a..39928144 100644 --- a/scripts/covariance_constructor_3x2pt.sh +++ b/scripts/covariance_constructor_3x2pt.sh @@ -622,7 +622,7 @@ est_clust = ${est_clust} cstellar_mf = ${cstellar_mf} cross_terms = True unbiased_clustering = True -# combinations_clustering = $(seq 0 $((NLENS-1)) | sed 's/.*/&-&/' | paste -sd, -) +; combinations_clustering = $(seq 0 $((NLENS-1)) | sed 's/.*/&-&/' | paste -sd, -) [csmf settings] csmf_log10Mmin = ${csmf_Mmin} @@ -695,10 +695,10 @@ theta_type = log theta_type_lensing = log theta_type_clustering = log theta_list = 1, 2, 3 +clustering = ${clustering} +ggl = ${ggl} xi_pp = ${cosmic_shear} xi_mm = ${cosmic_shear} -ggl = ${ggl} -clustering = ${clustering} theta_accuracy = 1e-5 integration_intervals = 50 From 1b9d5d335d70507f546dded057397c1975d0444e Mon Sep 17 00:00:00 2001 From: andrejdvornik Date: Thu, 20 Nov 2025 11:49:21 +0000 Subject: [PATCH 11/37] Temp fix to psi_stats --- man/calculate_psistats.man.sh | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/man/calculate_psistats.man.sh b/man/calculate_psistats.man.sh index 4c4a61c0..aed784c3 100644 --- a/man/calculate_psistats.man.sh +++ b/man/calculate_psistats.man.sh @@ -56,9 +56,11 @@ function _inp_data { # Output data {{{ function _outputs { #Data outputs (leave blank if none) - output=`_parse_blockvars psi_stats_@BV:MODES@` - output=${output,,} - echo $output + # Need to fix this to output gg and gm instad of nn and ne + echo psi_stats_gm psi_stats_gg + #output=`_parse_blockvars psi_stats_@BV:MODES@` + #output=${output,,} + #echo $output } #}}} From 5891829e5ce98ccc6565f4d20e2afc51fb8bffe7 Mon Sep 17 00:00:00 2001 From: andrejdvornik Date: Fri, 21 Nov 2025 09:50:36 +0000 Subject: [PATCH 12/37] Fix blocklist --- scripts/prepare_cosmosis.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/prepare_cosmosis.sh b/scripts/prepare_cosmosis.sh index 1648e602..9b83c791 100644 --- a/scripts/prepare_cosmosis.sh +++ b/scripts/prepare_cosmosis.sh @@ -37,7 +37,6 @@ fi #}}} blocklist="neff_source neff_lens neff_obs sigmae" -blocklist="neff_source sigmae" #N_effective & sigmae {{{ for stat in $blocklist From 477d2fa9f2066c459ccaff721e3b73100f1f0006 Mon Sep 17 00:00:00 2001 From: andrejdvornik Date: Fri, 21 Nov 2025 12:41:16 +0000 Subject: [PATCH 13/37] Flexible stats selection given MODES --- scripts/prepare_cosmosis.sh | 32 ++++++++++++++++++++++++++++---- 1 file changed, 28 insertions(+), 4 deletions(-) diff --git a/scripts/prepare_cosmosis.sh b/scripts/prepare_cosmosis.sh index 9b83c791..8a3ab87a 100644 --- a/scripts/prepare_cosmosis.sh +++ b/scripts/prepare_cosmosis.sh @@ -10,17 +10,43 @@ #For each of the files in the nz directory MODES="@BV:MODES@" +required_stats="" if [[ .*\ $MODES\ .* =~ " EE " ]] then headfiles_xi="@DB:xipm_comb@" + # add neff_source if not already present + if [[ " $required_stats " != *" neff_source "* ]]; then + required_stats="$required_stats neff_source" + fi + if [[ " $required_stats " != *" sigmae "* ]]; then + required_stats="$required_stats sigmae" + fi fi if [[ .*\ $MODES\ .* =~ " NE " ]] then headfiles_gt="@DB:gt_comb@" + if [[ " $required_stats " != *" neff_source "* ]]; then + required_stats="$required_stats neff_source" + fi + if [[ " $required_stats " != *" neff_lens "* ]]; then + required_stats="$required_stats neff_lens" + fi + if [[ " $required_stats " != *" sigmae "* ]]; then + required_stats="$required_stats sigmae" + fi fi if [[ .*\ $MODES\ .* =~ " NN " ]] then headfiles_wt="@DB:wt_comb@" + if [[ " $required_stats " != *" neff_lens "* ]]; then + required_stats="$required_stats neff_lens" + fi +fi +if [[ .*\ $MODES\ .* =~ " OBS " ]] +then + if [[ " $required_stats " != *" neff_obs "* ]]; then + required_stats="$required_stats neff_obs" + fi fi #Number of tomographic bins @@ -36,10 +62,8 @@ else fi #}}} -blocklist="neff_source neff_lens neff_obs sigmae" - #N_effective & sigmae {{{ -for stat in $blocklist +for stat in $required_stats do found="FALSE" foundlist="" @@ -237,7 +261,7 @@ then _message " - @RED@ Done! (`date +'%a %H:%M'`)@DEF@\n" #}}} #Save the file to the output list {{{ - outlist_gt="${outlist_gt} GT_@SURVEY@_${patch}_nBins_${NLENS}_Bin${ZBIN1}_Bin${ZBIN2}.ascii" + outlist_gt="${outlist_gt} GT_@SURVEY@_${patch}_nBins_${NLENS}_Bin${LBIN1}_Bin${ZBIN2}.ascii" #}}} done #}}} From 4a9bb4ec18be182a29a242f1fd06f0ae47757ed1 Mon Sep 17 00:00:00 2001 From: andrejdvornik Date: Fri, 21 Nov 2025 12:42:53 +0000 Subject: [PATCH 14/37] Clearer error message --- scripts/prepare_cosmosis.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/prepare_cosmosis.sh b/scripts/prepare_cosmosis.sh index 8a3ab87a..4601bdb9 100644 --- a/scripts/prepare_cosmosis.sh +++ b/scripts/prepare_cosmosis.sh @@ -330,8 +330,8 @@ then then #If not, error _message " - @RED@ERROR!@DEF@\n" - _message "@RED@There were no catalogues added to the cosmosis xipm folder?!@DEF@" - _message "@BLU@You probably didn't load the xipm files into the datahead?!@DEF@" + _message "@RED@There were no catalogues added to the cosmosis npair folder?!@DEF@" + _message "@BLU@You probably didn't load the all correlation function files into the datahead?!@DEF@" exit 1 fi #}}} From 00c695b5c1c987a528257d625f92ee71d514c10a Mon Sep 17 00:00:00 2001 From: andrejdvornik Date: Fri, 21 Nov 2025 12:51:35 +0000 Subject: [PATCH 15/37] Better handling of nzs in make_cosmosis_nz_3x2pt --- scripts/make_cosmosis_nz_3x2pt.sh | 25 ++++++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/scripts/make_cosmosis_nz_3x2pt.sh b/scripts/make_cosmosis_nz_3x2pt.sh index bd44a49f..9ce68e75 100644 --- a/scripts/make_cosmosis_nz_3x2pt.sh +++ b/scripts/make_cosmosis_nz_3x2pt.sh @@ -14,6 +14,25 @@ outputlist='' found_source="FALSE" found_lens="FALSE" found_obs="FALSE" + +need_source="FALSE" +need_lens="FALSE" +need_obs="FALSE" + +if [[ .*\ $MODES\ .* =~ " EE " ]]; then + need_source="TRUE" +fi +if [[ .*\ $MODES\ .* =~ " NE " ]]; then + need_source="TRUE" + need_lens="TRUE" +fi +if [[ .*\ $MODES\ .* =~ " NN " ]]; then + need_lens="TRUE" +fi +if [[ .*\ $MODES\ .* =~ " OBS " ]]; then + need_obs="TRUE" +fi + for patch in @BV:PATCHLIST@ @ALLPATCH@ do _message " ->@BLU@ Patch @RED@${patch}@DEF@" @@ -232,7 +251,7 @@ do done #Error if no stat files found {{{ -if [ "${found_source}" == "FALSE" ] && [ "${NTOMO}" != 0 ] +if [ "${found_source}" == "FALSE" ] && [ "${need_source}" == "TRUE" ] then #If not found, error _message " - @RED@ERROR!@DEF@\n" @@ -241,7 +260,7 @@ then exit 1 fi -if [ "${found_lens}" == "FALSE" ] && [ "${NLENS}" != 0 ] +if [ "${found_lens}" == "FALSE" ] && [ "${need_lens}" == "TRUE" ] then #If not found, error _message " - @RED@ERROR!@DEF@\n" @@ -250,7 +269,7 @@ then exit 1 fi -if [ "${found_obs}" == "FALSE" ] && [ "${NOBS}" != 0 ] +if [ "${found_obs}" == "FALSE" ] && [ "${need_obs}" == "TRUE" ] then #If not found, error _message " - @RED@ERROR!@DEF@\n" From c8d37249e0aafc44422dacbad2e6ae7f6c466e5e Mon Sep 17 00:00:00 2001 From: andrejdvornik Date: Fri, 21 Nov 2025 12:57:43 +0000 Subject: [PATCH 16/37] Better logic in make_cosmosis_nz --- scripts/make_cosmosis_nz_3x2pt.sh | 380 +++++++++++++++--------------- 1 file changed, 192 insertions(+), 188 deletions(-) diff --git a/scripts/make_cosmosis_nz_3x2pt.sh b/scripts/make_cosmosis_nz_3x2pt.sh index 9ce68e75..ce36d202 100644 --- a/scripts/make_cosmosis_nz_3x2pt.sh +++ b/scripts/make_cosmosis_nz_3x2pt.sh @@ -35,218 +35,222 @@ fi for patch in @BV:PATCHLIST@ @ALLPATCH@ do - _message " ->@BLU@ Patch @RED@${patch}@DEF@" - #Get all the files in this stat and patch {{{ - inputs=`_read_datablock "nz_source_${patch}"` - inputs=`_blockentry_to_filelist ${inputs}` - #}}} - filelist='' - #Get the file list {{{ - for ZBIN1 in `seq ${NTOMO}` - do - #Define the Z_B limits from the TOMOLIMS {{{ - ZB_lo=`echo @BV:TOMOLIMS@ | awk -v n=$ZBIN1 '{print $n}'` - ZB_hi=`echo @BV:TOMOLIMS@ | awk -v n=$ZBIN1 '{print $(n+1)}'` + if [ ${need_source} == "TRUE" ] + then + _message " ->@BLU@ Patch @RED@${patch}@DEF@" + #Get all the files in this stat and patch {{{ + inputs=`_read_datablock "nz_source_${patch}"` + inputs=`_blockentry_to_filelist ${inputs}` #}}} - #Define the string to append to the file names {{{ - ZB_lo_str=`echo $ZB_lo | sed 's/\./p/g'` - ZB_hi_str=`echo $ZB_hi | sed 's/\./p/g'` - appendstr="_ZB${ZB_lo_str}t${ZB_hi_str}" + filelist='' + #Get the file list {{{ + for ZBIN1 in `seq ${NTOMO}` + do + #Define the Z_B limits from the TOMOLIMS {{{ + ZB_lo=`echo @BV:TOMOLIMS@ | awk -v n=$ZBIN1 '{print $n}'` + ZB_hi=`echo @BV:TOMOLIMS@ | awk -v n=$ZBIN1 '{print $(n+1)}'` + #}}} + #Define the string to append to the file names {{{ + ZB_lo_str=`echo $ZB_lo | sed 's/\./p/g'` + ZB_hi_str=`echo $ZB_hi | sed 's/\./p/g'` + appendstr="_ZB${ZB_lo_str}t${ZB_hi_str}" + #}}} + #file=`echo ${inputs} | sed 's/ /\n/g' | grep "_${patch}_" | grep ${appendstr} || echo ` + file=`echo ${inputs} | sed 's/ /\n/g' | grep ${appendstr} || echo ` + #Check if the output file exists {{{ + if [ "${file}" == "" ] + then + #If not, loop + continue + fi + #}}} + filelist="${filelist} @RUNROOT@/@STORAGEPATH@/@DATABLOCK@/nz_source_${patch}/${file}" + done #}}} - #file=`echo ${inputs} | sed 's/ /\n/g' | grep "_${patch}_" | grep ${appendstr} || echo ` - file=`echo ${inputs} | sed 's/ /\n/g' | grep ${appendstr} || echo ` - #Check if the output file exists {{{ - if [ "${file}" == "" ] + #If filelist is empty, skip {{{ + if [ "${filelist}" == "" ] then - #If not, loop + _message "@RED@ - skipping! (No matching source Nz files)@DEF@\n" continue fi #}}} - filelist="${filelist} @RUNROOT@/@STORAGEPATH@/@DATABLOCK@/nz_source_${patch}/${file}" - done - #}}} - #If filelist is empty, skip {{{ - if [ "${filelist}" == "" ] - then - _message "@RED@ - skipping! (No matching source Nz files)@DEF@\n" - continue - fi - #}}} - found_source='TRUE' - #Construct the output directory {{{ - if [ ! -d @RUNROOT@/@STORAGEPATH@/@DATABLOCK@/cosmosis_nz_source_${patch} ] - then - mkdir @RUNROOT@/@STORAGEPATH@/@DATABLOCK@/cosmosis_nz_source_${patch}/ - fi - #}}} - _message "@RED@ - OK! (`date +'%a %H:%M'`)@DEF@\n" - #Construct the output base {{{ - file=${filelist##* } - output_base=${file##*/} - output_base=${output_base%%_ZB*} - output_base="@RUNROOT@/@STORAGEPATH@/@DATABLOCK@/cosmosis_nz_source_${patch}/${output_base}" - #}}} - #Remove existing files {{{ - if [ -f ${output_base}_comb_Nz.fits ] - then - _message " > @BLU@Removing previous COSMOSIS Nz file@DEF@ ${output_base##*/}_comb_Nz.fits@DEF@" - rm ${output_base}_comb_Nz.fits + found_source='TRUE' + #Construct the output directory {{{ + if [ ! -d @RUNROOT@/@STORAGEPATH@/@DATABLOCK@/cosmosis_nz_source_${patch} ] + then + mkdir @RUNROOT@/@STORAGEPATH@/@DATABLOCK@/cosmosis_nz_source_${patch}/ + fi + #}}} + _message "@RED@ - OK! (`date +'%a %H:%M'`)@DEF@\n" + #Construct the output base {{{ + file=${filelist##* } + output_base=${file##*/} + output_base=${output_base%%_ZB*} + output_base="@RUNROOT@/@STORAGEPATH@/@DATABLOCK@/cosmosis_nz_source_${patch}/${output_base}" + #}}} + #Remove existing files {{{ + if [ -f ${output_base}_comb_Nz.fits ] + then + _message " > @BLU@Removing previous COSMOSIS Nz file@DEF@ ${output_base##*/}_comb_Nz.fits@DEF@" + rm ${output_base}_comb_Nz.fits + _message " - @RED@Done! (`date +'%a %H:%M'`)@DEF@\n" + fi + #}}} + #Construct the redshift file {{{ + _message " > @BLU@Constructing COSMOSIS Nz file @RED@${output_base}_comb_Nz.fits@DEF@" + #Construct the Nz combined fits file and put into covariance/input/ + @PYTHON3BIN@ @RUNROOT@/@SCRIPTPATH@/MakeNofZForCosmosis_function.py \ + --inputs ${filelist} \ + --neff @DB:cosmosis_neff_source@ \ + --output_base ${output_base} \ + --suffix "source" 2>&1 _message " - @RED@Done! (`date +'%a %H:%M'`)@DEF@\n" - fi - #}}} - #Construct the redshift file {{{ - _message " > @BLU@Constructing COSMOSIS Nz file @RED@${output_base}_comb_Nz.fits@DEF@" - #Construct the Nz combined fits file and put into covariance/input/ - @PYTHON3BIN@ @RUNROOT@/@SCRIPTPATH@/MakeNofZForCosmosis_function.py \ - --inputs ${filelist} \ - --neff @DB:cosmosis_neff_source@ \ - --output_base ${output_base} \ - --suffix "source" 2>&1 - _message " - @RED@Done! (`date +'%a %H:%M'`)@DEF@\n" - #}}} - - #Update the datablock - _write_datablock cosmosis_nz_source_${patch} "${output_base##*/}_comb_Nz.fits" - - + #}}} + + #Update the datablock + _write_datablock cosmosis_nz_source_${patch} "${output_base##*/}_comb_Nz.fits" + fi - #Get all the files in this stat and patch {{{ - inputs_lens=`_read_datablock "nz_lens_${patch}"` - inputs_lens=`_blockentry_to_filelist ${inputs_lens}` - #}}} - filelist_lens='' - #Get the file list {{{ - for LBIN in `seq ${NLENS}` - do - #Define the string to append to the file names {{{ - appendstr="_LB${LBIN}" + if [ ${need_lens} == "TRUE" ] + then + #Get all the files in this stat and patch {{{ + inputs_lens=`_read_datablock "nz_lens_${patch}"` + inputs_lens=`_blockentry_to_filelist ${inputs_lens}` + #}}} + filelist_lens='' + #Get the file list {{{ + for LBIN in `seq ${NLENS}` + do + #Define the string to append to the file names {{{ + appendstr="_LB${LBIN}" + #}}} + #file=`echo ${inputs} | sed 's/ /\n/g' | grep "_${patch}_" | grep ${appendstr} || echo ` + file=`echo ${inputs_lens} | sed 's/ /\n/g' | grep ${appendstr} || echo ` + appendstr="_LB${LBIN}" + #Check if the output file exists {{{ + if [ "${file}" == "" ] + then + #If not, loop + continue + fi + #}}} + filelist_lens="${filelist_lens} @RUNROOT@/@STORAGEPATH@/@DATABLOCK@/nz_lens_${patch}/${file}" + done #}}} - #file=`echo ${inputs} | sed 's/ /\n/g' | grep "_${patch}_" | grep ${appendstr} || echo ` - file=`echo ${inputs_lens} | sed 's/ /\n/g' | grep ${appendstr} || echo ` - appendstr="_LB${LBIN}" - #Check if the output file exists {{{ - if [ "${file}" == "" ] + #If filelist is empty, skip {{{ + if [ "${filelist_lens}" == "" ] then - #If not, loop + _message "@RED@ - skipping! (No matching lens Nz files)@DEF@\n" continue fi #}}} - filelist_lens="${filelist_lens} @RUNROOT@/@STORAGEPATH@/@DATABLOCK@/nz_lens_${patch}/${file}" - done - #}}} - #If filelist is empty, skip {{{ - if [ "${filelist_lens}" == "" ] - then - _message "@RED@ - skipping! (No matching lens Nz files)@DEF@\n" - continue - fi - #}}} - found_lens='TRUE' - #Construct the output directory {{{ - if [ ! -d @RUNROOT@/@STORAGEPATH@/@DATABLOCK@/cosmosis_nz_lens_${patch} ] - then - mkdir @RUNROOT@/@STORAGEPATH@/@DATABLOCK@/cosmosis_nz_lens_${patch}/ - fi - #}}} - _message "@RED@ - OK! (`date +'%a %H:%M'`)@DEF@\n" - #Construct the output base {{{ - file=${filelist_lens##* } - output_base=${file##*/} - output_base=${output_base%%_ZB*} - output_base="@RUNROOT@/@STORAGEPATH@/@DATABLOCK@/cosmosis_nz_lens_${patch}/${output_base}" - #}}} - #Remove existing files {{{ - if [ -f ${output_base}_comb_Nz.fits ] - then - _message " > @BLU@Removing previous COSMOSIS Nz file@DEF@ ${output_base##*/}_comb_Nz.fits@DEF@" - rm ${output_base}_comb_Nz.fits + found_lens='TRUE' + #Construct the output directory {{{ + if [ ! -d @RUNROOT@/@STORAGEPATH@/@DATABLOCK@/cosmosis_nz_lens_${patch} ] + then + mkdir @RUNROOT@/@STORAGEPATH@/@DATABLOCK@/cosmosis_nz_lens_${patch}/ + fi + #}}} + _message "@RED@ - OK! (`date +'%a %H:%M'`)@DEF@\n" + #Construct the output base {{{ + file=${filelist_lens##* } + output_base=${file##*/} + output_base=${output_base%%_ZB*} + output_base="@RUNROOT@/@STORAGEPATH@/@DATABLOCK@/cosmosis_nz_lens_${patch}/${output_base}" + #}}} + #Remove existing files {{{ + if [ -f ${output_base}_comb_Nz.fits ] + then + _message " > @BLU@Removing previous COSMOSIS Nz file@DEF@ ${output_base##*/}_comb_Nz.fits@DEF@" + rm ${output_base}_comb_Nz.fits + _message " - @RED@Done! (`date +'%a %H:%M'`)@DEF@\n" + fi + #}}} + #Construct the redshift file {{{ + _message " > @BLU@Constructing COSMOSIS Nz file @RED@${output_base}_comb_Nz.fits@DEF@" + #Construct the Nz combined fits file and put into covariance/input/ + @PYTHON3BIN@ @RUNROOT@/@SCRIPTPATH@/MakeNofZForCosmosis_function.py \ + --inputs ${filelist_lens} \ + --neff @DB:cosmosis_neff_lens@ \ + --output_base ${output_base} \ + --suffix "lens" 2>&1 _message " - @RED@Done! (`date +'%a %H:%M'`)@DEF@\n" + #}}} + + #Update the datablock + _write_datablock cosmosis_nz_lens_${patch} "${output_base##*/}_comb_Nz.fits" fi - #}}} - #Construct the redshift file {{{ - _message " > @BLU@Constructing COSMOSIS Nz file @RED@${output_base}_comb_Nz.fits@DEF@" - #Construct the Nz combined fits file and put into covariance/input/ - @PYTHON3BIN@ @RUNROOT@/@SCRIPTPATH@/MakeNofZForCosmosis_function.py \ - --inputs ${filelist_lens} \ - --neff @DB:cosmosis_neff_lens@ \ - --output_base ${output_base} \ - --suffix "lens" 2>&1 - _message " - @RED@Done! (`date +'%a %H:%M'`)@DEF@\n" - #}}} - - #Update the datablock - _write_datablock cosmosis_nz_lens_${patch} "${output_base##*/}_comb_Nz.fits" - - - #Get all the files in this stat and patch {{{ - inputs_obs=`_read_datablock "nz_obs_${patch}"` - inputs_obs=`_blockentry_to_filelist ${inputs_obs}` - #}}} - filelist_obs='' - #Get the file list {{{ - for LBIN in `seq ${NOBS}` - do - #Define the string to append to the file names {{{ - appendstr="_LB${LBIN}" + if [ ${need_obs} == "TRUE" ] + then + #Get all the files in this stat and patch {{{ + inputs_obs=`_read_datablock "nz_obs_${patch}"` + inputs_obs=`_blockentry_to_filelist ${inputs_obs}` + #}}} + filelist_obs='' + #Get the file list {{{ + for LBIN in `seq ${NOBS}` + do + #Define the string to append to the file names {{{ + appendstr="_LB${LBIN}" + #}}} + #file=`echo ${inputs} | sed 's/ /\n/g' | grep "_${patch}_" | grep ${appendstr} || echo ` + file=`echo ${inputs_obs} | sed 's/ /\n/g' | grep ${appendstr} || echo ` + appendstr="_LB${LBIN}" + #Check if the output file exists {{{ + if [ "${file}" == "" ] + then + #If not, loop + continue + fi + #}}} + filelist_obs="${filelist_obs} @RUNROOT@/@STORAGEPATH@/@DATABLOCK@/nz_obs_${patch}/${file}" + done #}}} - #file=`echo ${inputs} | sed 's/ /\n/g' | grep "_${patch}_" | grep ${appendstr} || echo ` - file=`echo ${inputs_obs} | sed 's/ /\n/g' | grep ${appendstr} || echo ` - appendstr="_LB${LBIN}" - #Check if the output file exists {{{ - if [ "${file}" == "" ] + #If filelist is empty, skip {{{ + if [ "${filelist_obs}" == "" ] then - #If not, loop + _message "@RED@ - skipping! (No matching obs Nz files)@DEF@\n" continue fi #}}} - filelist_obs="${filelist_obs} @RUNROOT@/@STORAGEPATH@/@DATABLOCK@/nz_obs_${patch}/${file}" - done - #}}} - #If filelist is empty, skip {{{ - if [ "${filelist_obs}" == "" ] - then - _message "@RED@ - skipping! (No matching obs Nz files)@DEF@\n" - continue - fi - #}}} - found_obs='TRUE' - #Construct the output directory {{{ - if [ ! -d @RUNROOT@/@STORAGEPATH@/@DATABLOCK@/cosmosis_nz_obs_${patch} ] - then - mkdir @RUNROOT@/@STORAGEPATH@/@DATABLOCK@/cosmosis_nz_obs_${patch}/ - fi - #}}} - _message "@RED@ - OK! (`date +'%a %H:%M'`)@DEF@\n" - #Construct the output base {{{ - file=${filelist_obs##* } - output_base=${file##*/} - output_base=${output_base%%_ZB*} - output_base="@RUNROOT@/@STORAGEPATH@/@DATABLOCK@/cosmosis_nz_obs_${patch}/${output_base}" - #}}} - #Remove existing files {{{ - if [ -f ${output_base}_comb_Nz.fits ] - then - _message " > @BLU@Removing previous COSMOSIS Nz file@DEF@ ${output_base##*/}_comb_Nz.fits@DEF@" - rm ${output_base}_comb_Nz.fits + found_obs='TRUE' + #Construct the output directory {{{ + if [ ! -d @RUNROOT@/@STORAGEPATH@/@DATABLOCK@/cosmosis_nz_obs_${patch} ] + then + mkdir @RUNROOT@/@STORAGEPATH@/@DATABLOCK@/cosmosis_nz_obs_${patch}/ + fi + #}}} + _message "@RED@ - OK! (`date +'%a %H:%M'`)@DEF@\n" + #Construct the output base {{{ + file=${filelist_obs##* } + output_base=${file##*/} + output_base=${output_base%%_ZB*} + output_base="@RUNROOT@/@STORAGEPATH@/@DATABLOCK@/cosmosis_nz_obs_${patch}/${output_base}" + #}}} + #Remove existing files {{{ + if [ -f ${output_base}_comb_Nz.fits ] + then + _message " > @BLU@Removing previous COSMOSIS Nz file@DEF@ ${output_base##*/}_comb_Nz.fits@DEF@" + rm ${output_base}_comb_Nz.fits + _message " - @RED@Done! (`date +'%a %H:%M'`)@DEF@\n" + fi + #}}} + #Construct the redshift file {{{ + _message " > @BLU@Constructing COSMOSIS Nz file @RED@${output_base}_comb_Nz.fits@DEF@" + #Construct the Nz combined fits file and put into covariance/input/ + @PYTHON3BIN@ @RUNROOT@/@SCRIPTPATH@/MakeNofZForCosmosis_function.py \ + --inputs ${filelist_obs} \ + --neff @DB:cosmosis_neff_obs@ \ + --output_base ${output_base} \ + --suffix "obs" 2>&1 _message " - @RED@Done! (`date +'%a %H:%M'`)@DEF@\n" + #}}} + + #Update the datablock + _write_datablock cosmosis_nz_obs_${patch} "${output_base##*/}_comb_Nz.fits" fi - #}}} - #Construct the redshift file {{{ - _message " > @BLU@Constructing COSMOSIS Nz file @RED@${output_base}_comb_Nz.fits@DEF@" - #Construct the Nz combined fits file and put into covariance/input/ - @PYTHON3BIN@ @RUNROOT@/@SCRIPTPATH@/MakeNofZForCosmosis_function.py \ - --inputs ${filelist_obs} \ - --neff @DB:cosmosis_neff_obs@ \ - --output_base ${output_base} \ - --suffix "obs" 2>&1 - _message " - @RED@Done! (`date +'%a %H:%M'`)@DEF@\n" - #}}} - - #Update the datablock - _write_datablock cosmosis_nz_obs_${patch} "${output_base##*/}_comb_Nz.fits" - done From 556090b55d685af7dbf6612beaf549250603ce13 Mon Sep 17 00:00:00 2001 From: andrejdvornik Date: Fri, 21 Nov 2025 13:54:28 +0000 Subject: [PATCH 17/37] More fixes --- man/make_cosmosis_nz_3x2pt.man.sh | 2 +- scripts/make_cosmosis_nz_3x2pt.sh | 51 ++++++++++++++++++------------- scripts/prepare_cosmosis.sh | 21 ++++++++----- 3 files changed, 45 insertions(+), 29 deletions(-) diff --git a/man/make_cosmosis_nz_3x2pt.man.sh b/man/make_cosmosis_nz_3x2pt.man.sh index 2aaa599b..ec664b9c 100644 --- a/man/make_cosmosis_nz_3x2pt.man.sh +++ b/man/make_cosmosis_nz_3x2pt.man.sh @@ -43,7 +43,7 @@ set -e # Input variables {{{ function _inp_var { #Variable inputs (leave blank if none) - echo ALLPATCH BLU BV:NLENSBINS BV:NSMFLENSBINS BV:PATCHLIST BV:TOMOLIMS DATABLOCK DEF PYTHON3BIN RED RUNROOT SCRIPTPATH STORAGEPATH + echo ALLPATCH BLU BV:MODES BV:NLENSBINS BV:NSMFLENSBINS BV:PATCHLIST BV:TOMOLIMS DATABLOCK DEF PYTHON3BIN RED RUNROOT SCRIPTPATH STORAGEPATH } #}}} diff --git a/scripts/make_cosmosis_nz_3x2pt.sh b/scripts/make_cosmosis_nz_3x2pt.sh index ce36d202..9e5144ae 100644 --- a/scripts/make_cosmosis_nz_3x2pt.sh +++ b/scripts/make_cosmosis_nz_3x2pt.sh @@ -7,6 +7,7 @@ # #========================================= +MODES="@BV:MODES@" NTOMO=`echo @BV:TOMOLIMS@ | awk '{print NF-1}'` NLENS="@BV:NLENSBINS@" NOBS="@BV:NSMFLENSBINS@" @@ -19,24 +20,28 @@ need_source="FALSE" need_lens="FALSE" need_obs="FALSE" -if [[ .*\ $MODES\ .* =~ " EE " ]]; then - need_source="TRUE" -fi -if [[ .*\ $MODES\ .* =~ " NE " ]]; then +if [[ .*\ $MODES\ .* =~ " EE " ]] || [[ .*\ $MODES\ .* =~ " NE " ]] +then need_source="TRUE" - need_lens="TRUE" fi -if [[ .*\ $MODES\ .* =~ " NN " ]]; then +if [[ .*\ $MODES\ .* =~ " NE " ]] || [[ .*\ $MODES\ .* =~ " NN " ]] +then need_lens="TRUE" fi -if [[ .*\ $MODES\ .* =~ " OBS " ]]; then +if [[ .*\ $MODES\ .* =~ " OBS " ]] +then need_obs="TRUE" fi -for patch in @BV:PATCHLIST@ @ALLPATCH@ -do - if [ ${need_source} == "TRUE" ] - then + +_message "${MODES}\n" +_message "${need_source} ${need_lens} ${need_obs}\n" +_message "\n" + +if [ ${need_source} == "TRUE" ] +then + for patch in @BV:PATCHLIST@ @ALLPATCH@ + do _message " ->@BLU@ Patch @RED@${patch}@DEF@" #Get all the files in this stat and patch {{{ inputs=`_read_datablock "nz_source_${patch}"` @@ -109,11 +114,13 @@ do #Update the datablock _write_datablock cosmosis_nz_source_${patch} "${output_base##*/}_comb_Nz.fits" - fi - + done +fi - if [ ${need_lens} == "TRUE" ] - then +if [ ${need_lens} == "TRUE" ] +then + for patch in @BV:PATCHLIST@ @ALLPATCH@ + do #Get all the files in this stat and patch {{{ inputs_lens=`_read_datablock "nz_lens_${patch}"` inputs_lens=`_blockentry_to_filelist ${inputs_lens}` @@ -180,10 +187,13 @@ do #Update the datablock _write_datablock cosmosis_nz_lens_${patch} "${output_base##*/}_comb_Nz.fits" - fi + done +fi - if [ ${need_obs} == "TRUE" ] - then +if [ ${need_obs} == "TRUE" ] +then + for patch in @BV:PATCHLIST@ @ALLPATCH@ + do #Get all the files in this stat and patch {{{ inputs_obs=`_read_datablock "nz_obs_${patch}"` inputs_obs=`_blockentry_to_filelist ${inputs_obs}` @@ -250,9 +260,8 @@ do #Update the datablock _write_datablock cosmosis_nz_obs_${patch} "${output_base##*/}_comb_Nz.fits" - fi - -done + done +fi #Error if no stat files found {{{ if [ "${found_source}" == "FALSE" ] && [ "${need_source}" == "TRUE" ] diff --git a/scripts/prepare_cosmosis.sh b/scripts/prepare_cosmosis.sh index 4601bdb9..51bb4585 100644 --- a/scripts/prepare_cosmosis.sh +++ b/scripts/prepare_cosmosis.sh @@ -15,36 +15,43 @@ if [[ .*\ $MODES\ .* =~ " EE " ]] then headfiles_xi="@DB:xipm_comb@" # add neff_source if not already present - if [[ " $required_stats " != *" neff_source "* ]]; then + if [[ " $required_stats " != *" neff_source "* ]] + then required_stats="$required_stats neff_source" fi - if [[ " $required_stats " != *" sigmae "* ]]; then + if [[ " $required_stats " != *" sigmae "* ]] + then required_stats="$required_stats sigmae" fi fi if [[ .*\ $MODES\ .* =~ " NE " ]] then headfiles_gt="@DB:gt_comb@" - if [[ " $required_stats " != *" neff_source "* ]]; then + if [[ " $required_stats " != *" neff_source "* ]] + then required_stats="$required_stats neff_source" fi - if [[ " $required_stats " != *" neff_lens "* ]]; then + if [[ " $required_stats " != *" neff_lens "* ]] + then required_stats="$required_stats neff_lens" fi - if [[ " $required_stats " != *" sigmae "* ]]; then + if [[ " $required_stats " != *" sigmae "* ]] + then required_stats="$required_stats sigmae" fi fi if [[ .*\ $MODES\ .* =~ " NN " ]] then headfiles_wt="@DB:wt_comb@" - if [[ " $required_stats " != *" neff_lens "* ]]; then + if [[ " $required_stats " != *" neff_lens "* ]] + then required_stats="$required_stats neff_lens" fi fi if [[ .*\ $MODES\ .* =~ " OBS " ]] then - if [[ " $required_stats " != *" neff_obs "* ]]; then + if [[ " $required_stats " != *" neff_obs "* ]] + then required_stats="$required_stats neff_obs" fi fi From 302c626e7d0d4407e0745b07492e9c37d6ea55b2 Mon Sep 17 00:00:00 2001 From: andrejdvornik Date: Fri, 21 Nov 2025 13:57:24 +0000 Subject: [PATCH 18/37] Remove message --- scripts/make_cosmosis_nz_3x2pt.sh | 5 ----- 1 file changed, 5 deletions(-) diff --git a/scripts/make_cosmosis_nz_3x2pt.sh b/scripts/make_cosmosis_nz_3x2pt.sh index 9e5144ae..0f9c7b96 100644 --- a/scripts/make_cosmosis_nz_3x2pt.sh +++ b/scripts/make_cosmosis_nz_3x2pt.sh @@ -33,11 +33,6 @@ then need_obs="TRUE" fi - -_message "${MODES}\n" -_message "${need_source} ${need_lens} ${need_obs}\n" -_message "\n" - if [ ${need_source} == "TRUE" ] then for patch in @BV:PATCHLIST@ @ALLPATCH@ From b7c232a0bece1580a43f95718bdb39dd83456bac Mon Sep 17 00:00:00 2001 From: andrejdvornik Date: Fri, 21 Nov 2025 13:58:43 +0000 Subject: [PATCH 19/37] Fix patch message --- scripts/make_cosmosis_nz_3x2pt.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/make_cosmosis_nz_3x2pt.sh b/scripts/make_cosmosis_nz_3x2pt.sh index 0f9c7b96..9e4bc867 100644 --- a/scripts/make_cosmosis_nz_3x2pt.sh +++ b/scripts/make_cosmosis_nz_3x2pt.sh @@ -116,6 +116,7 @@ if [ ${need_lens} == "TRUE" ] then for patch in @BV:PATCHLIST@ @ALLPATCH@ do + _message " ->@BLU@ Patch @RED@${patch}@DEF@" #Get all the files in this stat and patch {{{ inputs_lens=`_read_datablock "nz_lens_${patch}"` inputs_lens=`_blockentry_to_filelist ${inputs_lens}` @@ -189,6 +190,7 @@ if [ ${need_obs} == "TRUE" ] then for patch in @BV:PATCHLIST@ @ALLPATCH@ do + _message " ->@BLU@ Patch @RED@${patch}@DEF@" #Get all the files in this stat and patch {{{ inputs_obs=`_read_datablock "nz_obs_${patch}"` inputs_obs=`_blockentry_to_filelist ${inputs_obs}` From b7ccf10fabd85aa5bfe1a4e059d0d600e3d70195 Mon Sep 17 00:00:00 2001 From: andrejdvornik Date: Mon, 24 Nov 2025 12:39:19 +0000 Subject: [PATCH 20/37] Fix names of lens and obs nzs --- scripts/make_cosmosis_nz_3x2pt.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/make_cosmosis_nz_3x2pt.sh b/scripts/make_cosmosis_nz_3x2pt.sh index 9e4bc867..6fc489fb 100644 --- a/scripts/make_cosmosis_nz_3x2pt.sh +++ b/scripts/make_cosmosis_nz_3x2pt.sh @@ -159,7 +159,7 @@ then #Construct the output base {{{ file=${filelist_lens##* } output_base=${file##*/} - output_base=${output_base%%_ZB*} + output_base=${output_base%%_LB*} output_base="@RUNROOT@/@STORAGEPATH@/@DATABLOCK@/cosmosis_nz_lens_${patch}/${output_base}" #}}} #Remove existing files {{{ @@ -233,7 +233,7 @@ then #Construct the output base {{{ file=${filelist_obs##* } output_base=${file##*/} - output_base=${output_base%%_ZB*} + output_base=${output_base%%_LB*} output_base="@RUNROOT@/@STORAGEPATH@/@DATABLOCK@/cosmosis_nz_obs_${patch}/${output_base}" #}}} #Remove existing files {{{ From ced8fb25889be2c5e1623278e312a1fcf1e10467 Mon Sep 17 00:00:00 2001 From: andrejdvornik Date: Mon, 24 Nov 2025 13:45:32 +0000 Subject: [PATCH 21/37] More optimisation of prepare_cosmosis.sh --- scripts/prepare_cosmosis.sh | 67 +++++++++++++++++++++++++++++-------- 1 file changed, 53 insertions(+), 14 deletions(-) diff --git a/scripts/prepare_cosmosis.sh b/scripts/prepare_cosmosis.sh index 51bb4585..926f0a37 100644 --- a/scripts/prepare_cosmosis.sh +++ b/scripts/prepare_cosmosis.sh @@ -153,11 +153,13 @@ done #}}} #npair {{{ -if [ "${headfiles_xi}" != "" ] || [ "${headfiles_gt}" != "" ] || [ "${headfiles_wt}" != "" ] +outall='' +if [[ .*\ $MODES\ .* =~ " EE " ]] && [ "${headfiles_xi}" != "" ] then - _message "Copying 2pcf catalogues from datahead into cosmosis_npair {\n" + _message "\n" + _message "Copying xipm catalogues from datahead into cosmosis_npair {\n" #Loop over patches {{{ - outall='' + outlist_check='' for patch in ${patchlist} do outlist='' @@ -219,10 +221,27 @@ then #Update the datablock {{{ #_write_datablock "cosmosis_npair_${patch}_@BV:BLIND@" "${outlist}" outall="${outall} ${outlist}" + outlist_check="${outlist_check} ${outlist}" #}}} - #}}} - - + done + if [ "${outlist_check}" == "" ] + then + #If not, error + _message " - @RED@ERROR!@DEF@\n" + _message "@RED@There were no catalogues added to the cosmosis npair folder?!@DEF@" + _message "@BLU@You probably didn't load the all correlation function files into the datahead?!@DEF@" + exit 1 + fi +fi + +if [[ .*\ $MODES\ .* =~ " NE " ]] && [ "${headfiles_gt}" != "" ] +then + _message "\n" + _message "Copying gamma_t catalogues from datahead into cosmosis_npair {\n" + #Loop over patches {{{ + outlist_check='' + for patch in ${patchlist} + do outlist_gt='' #Loop over tomographic bins in this patch {{{ for LBIN1 in `seq ${NLENS}` @@ -276,9 +295,27 @@ then #Update the datablock {{{ #_write_datablock "cosmosis_npair_${patch}_@BV:BLIND@" "${outlist_gt}" outall="${outall} ${outlist_gt}" + outlist_check="${outlist_check} ${outlist_gt}" #}}} - #}}} - + done + if [ "${outlist_check}" == "" ] + then + #If not, error + _message " - @RED@ERROR!@DEF@\n" + _message "@RED@There were no catalogues added to the cosmosis npair folder?!@DEF@" + _message "@BLU@You probably didn't load the all correlation function files into the datahead?!@DEF@" + exit 1 + fi +fi + +if [[ .*\ $MODES\ .* =~ " NN " ]] && [ "${headfiles_wt}" != "" ] +then + _message "\n" + _message "Copying w(theta) catalogues from datahead into cosmosis_npair {\n" + #Loop over patches {{{ + outlist_check='' + for patch in ${patchlist} + do outlist_wt='' #Loop over tomographic bins in this patch {{{ for LBIN1 in `seq ${NLENS}` @@ -326,22 +363,24 @@ then #Update the datablock {{{ #_write_datablock "cosmosis_npair_${patch}_@BV:BLIND@" "${outlist_wt}" outall="${outall} ${outlist_wt}" - _write_datablock "cosmosis_npair_${patch}_@BV:BLIND@" "${outall}" + outlist_check="${outlist_check} ${outlist_wt}" #}}} #}}} done - #}}} #Were there any files in any of the patches? {{{ - if [ "${outall}" == "" ] - then + if [ "${outlist_check}" == "" ] + then #If not, error _message " - @RED@ERROR!@DEF@\n" _message "@RED@There were no catalogues added to the cosmosis npair folder?!@DEF@" _message "@BLU@You probably didn't load the all correlation function files into the datahead?!@DEF@" exit 1 fi - #}}} - _message "}\n" fi #}}} +#Update the datablock {{{ +_write_datablock "cosmosis_npair_${patch}_@BV:BLIND@" "${outall}" +#}}} +_message "}\n" + From 866b72be646cf48992126f9723a0d06b725b6f52 Mon Sep 17 00:00:00 2001 From: andrejdvornik Date: Thu, 27 Nov 2025 08:23:13 +0000 Subject: [PATCH 22/37] Implemented bin averaging for 1 point statistics --- scripts/save_and_check_sacc.py | 52 +++++++++++++++++++++++++++++----- 1 file changed, 45 insertions(+), 7 deletions(-) diff --git a/scripts/save_and_check_sacc.py b/scripts/save_and_check_sacc.py index f89bedd3..a69a3730 100644 --- a/scripts/save_and_check_sacc.py +++ b/scripts/save_and_check_sacc.py @@ -146,16 +146,54 @@ def add_2pt_points(sacc_data, statistic, type, auto, n1, n2, min_val, max_val, n def add_1pt_points(sacc_data, files): + dt = "galaxy_stellarmassfunction" + grid_resolution = 10000 # Number of points in mass grid for window functions + for i, file in enumerate(files): - # Load SMF + + tracer_name = f"lens_{i}" + mass, value, extra = np.loadtxt(file).T - mass_min = mass - 0.5 * (mass[1] - mass[0]) - mass_max = mass + 0.5 * (mass[1] - mass[0]) - for m, mmin, mmax, v in zip(mass, mass_min, mass_max, value): - tracer_name = f"lens_{i}" - dt = "galaxy_stellarmassfunction" - sacc_data.add_data_point(dt, (tracer_name,), v, mass=m, mass_min=mmin, mass_max=mmax) + # Bin edges + log_mass = np.log10(mass) + + # Compute log bin edges + dlog = log_mass[1] - log_mass[0] + log_mass_min = log_mass - 0.5 * dlog + log_mass_max = log_mass + 0.5 * dlog + + # Mass grid for window functions + log_grid = np.linspace(log_mass_min.min(), log_mass_max.max(), grid_resolution) + + mass_grid = 10**log_grid + + # Window array + w = np.zeros((len(mass), grid_resolution)) + + # Build windows + for j in range(len(mass)): + in_bin = np.where( + (log_grid >= log_mass_min[j]) & + (log_grid < log_mass_max[j]) + )[0] + w[j, in_bin] = 1.0 + + window = sacc.Window(values=mass_grid, weight=w.T) + + # Add data points + for j, (lm, lmmin, lmmax, v) in enumerate( + zip(log_mass, log_mass_min, log_mass_max, value) + ): + sacc_data.add_data_point( + dt, (tracer_name,), v, + mass=10**lm, + mass_min=10**lmmin, + mass_max=10**lmmax, + window=window, + window_ind=j + ) + def process_2pt_data(mode, nangle, min_val, max_val, n1, n2, datavec_files, statistic, sacc_data, sacc_data_no_mbias, auto=None): From 02548be94041e88ef4df3c11df0e55bb1065ea38 Mon Sep 17 00:00:00 2001 From: andrejdvornik Date: Wed, 3 Dec 2025 08:35:15 +0000 Subject: [PATCH 23/37] Moved normalisation to sacc creation instead of likelihood code --- scripts/save_and_check_sacc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/save_and_check_sacc.py b/scripts/save_and_check_sacc.py index a69a3730..d051434b 100644 --- a/scripts/save_and_check_sacc.py +++ b/scripts/save_and_check_sacc.py @@ -178,7 +178,7 @@ def add_1pt_points(sacc_data, files): (log_grid < log_mass_max[j]) )[0] w[j, in_bin] = 1.0 - + w /= w.sum(axis=1)[:, None] # Normalize window = sacc.Window(values=mass_grid, weight=w.T) # Add data points From b02491822c5346955b0c0327a3fd3a7e1d305a81 Mon Sep 17 00:00:00 2001 From: andrejdvornik Date: Fri, 5 Dec 2025 13:58:43 +0000 Subject: [PATCH 24/37] Fix covariance constructor --- man/add_datavector.man.sh | 3 ++- scripts/covariance_constructor_3x2pt.sh | 8 ++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/man/add_datavector.man.sh b/man/add_datavector.man.sh index 01c7e64e..14324464 100644 --- a/man/add_datavector.man.sh +++ b/man/add_datavector.man.sh @@ -97,10 +97,11 @@ function _outputs { then output="obs_vec" else - _message "BV:DATAVEC_TYPE is not set to a known value: ${DATAVEC_TYPE^^} is not one of {COSEBIS,PSI_STATS_GM,PSI_STATS_GG,BANDPOWERS_EE,BANDPOWERS_NE,BANDPOWERS_NN,XIPM,GT,WT}" + _message "BV:DATAVEC_TYPE is not set to a known value: ${DATAVEC_TYPE^^} is not one of {COSEBIS,PSI_STATS_GM,PSI_STATS_GG,BANDPOWERS_EE,BANDPOWERS_NE,BANDPOWERS_NN,XIPM,GT,WT,OBS}" exit 1 fi #}}} + _message "$output" echo ${output} } #}}} diff --git a/scripts/covariance_constructor_3x2pt.sh b/scripts/covariance_constructor_3x2pt.sh index 39928144..a4d8a2c2 100644 --- a/scripts/covariance_constructor_3x2pt.sh +++ b/scripts/covariance_constructor_3x2pt.sh @@ -227,10 +227,10 @@ then csmf_Mmax=$(echo @BV:SMFLENSLIMSX@ | awk '{for(i=2; i<=NF; i++) printf "%s,", $i; print ""}' | sed 's/,$//') fi cstellar_mf=True - csmf_N_log10M_bin=10 - csmf_directory="@RUNROOT@/INSTALL/OneCovariance/input/conditional_smf/" - V_max_file="V_max.asc" - f_tomo_file="f_tomo.asc" + csmf_N_log10M_bin=@BV:NSMFBINS@ + csmf_directory="@RUNROOT@/@STORAGEPATH@/@DATABLOCK@/smf/" + V_max_file="@DB:vmax@" # This assumes one file, currently we have NSMFLENSBINS + f_tomo_file="@DB:f_tomo@" # This assumes one file, currently we have NSMFLENSBINS fi else cstellar_mf=False From fdfd7499d60dbf1a3b67915fc9987ce459d3baab Mon Sep 17 00:00:00 2001 From: andrejdvornik Date: Tue, 9 Dec 2025 14:12:11 +0000 Subject: [PATCH 25/37] Optimisation of cov filters handling --- scripts/covariance_constructor_3x2pt.sh | 263 +++++++---------- scripts/run_cov_weights.sh | 360 ++++++++++++------------ 2 files changed, 280 insertions(+), 343 deletions(-) diff --git a/scripts/covariance_constructor_3x2pt.sh b/scripts/covariance_constructor_3x2pt.sh index a4d8a2c2..2aacb991 100644 --- a/scripts/covariance_constructor_3x2pt.sh +++ b/scripts/covariance_constructor_3x2pt.sh @@ -36,6 +36,40 @@ NTOMO=`echo @BV:TOMOLIMS@ | awk '{print NF-1}'` NLENS="@BV:NLENSBINS@" NOBS="@BV:NSMFLENSBINS@" +use_arbitrary=True +arb_base=@RUNROOT@/@CONFIGPATH@/covariance_arb_summary/ +outfold=@RUNROOT@/@STORAGEPATH@/@DATABLOCK@/covariance_inputs/arb_summary_filters/ + +# Function to check and copy files +check_and_copy_files() { + local n=$1 + shift + local files=("$@") + + for i in $(seq -f "%02g" 1 "$n"); do + missing=False + expanded_files=() + for f in "${files[@]}"; do + file="${f//\?/$i}" + expanded_files+=("$file") + if [ ! -f "$arb_base$file" ]; then + missing=True + fi + done + + if [ "$missing" == "True" ]; then + run_arbitrary=True + _message "One or more arbitrary input files do not exist. Please run the run_cov_weights.sh script before!\n" + break + else + for f in "${expanded_files[@]}"; do + cp "$arb_base$f" "$outfold" + done + fi + done +} + + if [ "${STATISTIC^^}" == "2PCF" ] then @@ -44,10 +78,10 @@ then est_shear=xi_pm cosmic_shear=True n_arb_ee=@BV:NXIPM@ - arb_fourier_filter_mmE_file_@BV:STATISTIC@="fourier_weight_realspace_cf_mm_p_?.table" - arb_fourier_filter_mmB_file_@BV:STATISTIC@="fourier_weight_realspace_cf_mm_m_?.table" - arb_real_filter_mm_p_file_@BV:STATISTIC@="real_weight_realspace_cf_mm_p_?.table" - arb_real_filter_mm_m_file_@BV:STATISTIC@="real_weight_realspace_cf_mm_m_?.table" + arb_fourier_filter_mmE_file="fourier_weight_realspace_cf_mm_p_?.table" + arb_fourier_filter_mmB_file="fourier_weight_realspace_cf_mm_m_?.table" + arb_real_filter_mm_p_file="real_weight_realspace_cf_mm_p_?.table" + arb_real_filter_mm_m_file="real_weight_realspace_cf_mm_m_?.table" else est_shear=xi_pm cosmic_shear=False @@ -57,8 +91,8 @@ then est_ggl=gamma_t ggl=True n_arb_ne=@BV:NGT@ - arb_fourier_filter_gm_file_@BV:STATISTIC@="fourier_weight_realspace_cf_gm_?.table" - arb_real_filter_gm_file_@BV:STATISTIC@="real_weight_realspace_cf_gm_?.table" + arb_fourier_filter_gm_file="fourier_weight_realspace_cf_gm_?.table" + arb_real_filter_gm_file="real_weight_realspace_cf_gm_?.table" else est_ggl=gamma_t ggl=False @@ -68,8 +102,8 @@ then est_clust=w clustering=True n_arb_nn=@BV:NWT@ - arb_fourier_filter_gg_file_@BV:STATISTIC@="fourier_weight_realspace_cf_gg_?.table" - arb_real_filter_gg_file_@BV:STATISTIC@="real_weight_realspace_cf_gg_?.table" + arb_fourier_filter_gg_file="fourier_weight_realspace_cf_gg_?.table" + arb_real_filter_gg_file="real_weight_realspace_cf_gg_?.table" else est_clust=w clustering=False @@ -81,10 +115,10 @@ then est_shear=cosebi cosmic_shear=True n_arb_ee=@BV:NMAXCOSEBIS@ - arb_fourier_filter_mmE_file_@BV:STATISTIC@="Wn_@BV:THETAMINXI@_to_@BV:THETAMAXXI@_?.table" - arb_fourier_filter_mmB_file_@BV:STATISTIC@="Wn_@BV:THETAMINXI@_to_@BV:THETAMAXXI@_?.table" - arb_real_filter_mm_p_file_@BV:STATISTIC@="Tp_@BV:THETAMINXI@_to_@BV:THETAMAXXI@_?.table" - arb_real_filter_mm_m_file_@BV:STATISTIC@="Tm_@BV:THETAMINXI@_to_@BV:THETAMAXXI@_?.table" + arb_fourier_filter_mmE_file="Wn_@BV:THETAMINXI@_to_@BV:THETAMAXXI@_?.table" + arb_fourier_filter_mmB_file="Wn_@BV:THETAMINXI@_to_@BV:THETAMAXXI@_?.table" + arb_real_filter_mm_p_file="Tp_@BV:THETAMINXI@_to_@BV:THETAMAXXI@_?.table" + arb_real_filter_mm_m_file="Tm_@BV:THETAMINXI@_to_@BV:THETAMAXXI@_?.table" else est_shear=cosebi cosmic_shear=False @@ -94,8 +128,8 @@ then est_ggl=cosebi ggl=True n_arb_ne=@BV:NMAXCOSEBISNE@ - arb_fourier_filter_gm_file_@BV:STATISTIC@="Qgm_@BV:THETAMINGT@_to_@BV:THETAMAXGT@_?.table" - arb_real_filter_gm_file_@BV:STATISTIC@="Wn_psigm_@BV:THETAMINGT@_to_@BV:THETAMAXGT@_?.table" + arb_fourier_filter_gm_file="Qgm_@BV:THETAMINGT@_to_@BV:THETAMAXGT@_?.table" + arb_real_filter_gm_file="Wn_psigm_@BV:THETAMINGT@_to_@BV:THETAMAXGT@_?.table" else est_ggl=cosebi ggl=False @@ -105,8 +139,8 @@ then est_clust=cosebi clustering=True n_arb_nn=@BV:NMAXCOSEBISNN@ - arb_fourier_filter_gg_file_@BV:STATISTIC@="Ugg_@BV:THETAMINWT@_to_@BV:THETAMAXWT@_?.table" - arb_real_filter_gg_file_@BV:STATISTIC@="Wn_psigg_@BV:THETAMINWT@_to_@BV:THETAMAXWT@_?.table" + arb_fourier_filter_gg_file="Ugg_@BV:THETAMINWT@_to_@BV:THETAMAXWT@_?.table" + arb_real_filter_gg_file="Wn_psigg_@BV:THETAMINWT@_to_@BV:THETAMAXWT@_?.table" else est_clust=cosebi clustering=False @@ -122,10 +156,10 @@ then theta_up=`echo 'e(l(@BV:THETAMAXXI@)-@BV:APODISATIONWIDTH@/2)' | bc -l | awk '{printf "%.9f", $0}'` t_lo=`printf "%.2f" $theta_lo` t_up=`printf "%.2f" $theta_up` - arb_fourier_filter_mmE_file_@BV:STATISTIC@="fourier_weight_bandpowers_mmE_?.table" - arb_fourier_filter_mmB_file_@BV:STATISTIC@="fourier_weight_bandpowers_mmB_?.table" - arb_real_filter_mm_p_file_@BV:STATISTIC@="real_weight_bandpowers_mmE_?.table" - arb_real_filter_mm_m_file_@BV:STATISTIC@="real_weight_bandpowers_mmB_?.table" + arb_fourier_filter_mmE_file="fourier_weight_bandpowers_mmE_?.table" + arb_fourier_filter_mmB_file="fourier_weight_bandpowers_mmB_?.table" + arb_real_filter_mm_p_file="real_weight_bandpowers_mmE_?.table" + arb_real_filter_mm_m_file="real_weight_bandpowers_mmB_?.table" else est_shear=bandpowers cosmic_shear=False @@ -139,8 +173,8 @@ then theta_up_lensing=`echo 'e(l(@BV:THETAMAXGT@)-@BV:APODISATIONWIDTH@/2)' | bc -l | awk '{printf "%.9f", $0}'` t_lo=`printf "%.2f" $theta_lo_lensing` t_up=`printf "%.2f" $theta_up_lensing` - arb_fourier_filter_gm_file_@BV:STATISTIC@="fourier_weight_bandpowers_gm_?.table" - arb_real_filter_gm_file_@BV:STATISTIC@="real_weight_bandpowers_gm_?.table" + arb_fourier_filter_gm_file="fourier_weight_bandpowers_gm_?.table" + arb_real_filter_gm_file="real_weight_bandpowers_gm_?.table" else est_ggl=bandpowers ggl=False @@ -154,8 +188,8 @@ then theta_up_clustering=`echo 'e(l(@BV:THETAMAXWT@)-@BV:APODISATIONWIDTH@/2)' | bc -l | awk '{printf "%.9f", $0}'` t_lo=`printf "%.2f" $theta_lo_clustering` t_up=`printf "%.2f" $theta_up_clustering` - arb_fourier_filter_gg_file_@BV:STATISTIC@="fourier_weight_bandpowers_gg_?.table" - arb_real_filter_gg_file_@BV:STATISTIC@="real_weight_bandpowers_gg_?.table" + arb_fourier_filter_gg_file="fourier_weight_bandpowers_gg_?.table" + arb_real_filter_gg_file="real_weight_bandpowers_gg_?.table" else est_clust=bandpowers clustering=False @@ -307,65 +341,19 @@ else fi # Check if the arbitrary input files exist and copy to input directory -use_arbitrary=True #False if [[ .*\ $MODES\ .* =~ " EE " ]] then - for i in $(seq -f "%02g" 1 $n_arb_ee) - do - file=`echo ${arb_fourier_filter_mmE_file_@BV:STATISTIC@} | sed "s/?/${i}/g"` - file2=`echo ${arb_fourier_filter_mmB_file_@BV:STATISTIC@} | sed "s/?/${i}/g"` - file3=`echo ${arb_real_filter_mm_p_file_@BV:STATISTIC@} | sed "s/?/${i}/g"` - file4=`echo ${arb_real_filter_mm_m_file_@BV:STATISTIC@} | sed "s/?/${i}/g"` - arb_base=@RUNROOT@/@CONFIGPATH@/covariance_arb_summary/ - if [ ! -f $arb_base${file} ] || [ ! -f $arb_base${file2} ] || [ ! -f $arb_base${file3} ] || [ ! -f $arb_base${file4} ] - then - #use_arbitrary=False - _message "One or more arbitrary input files do not exist. Calculating filters on the fly!\n" - break - else - cp ${arb_base}/{$file,$file2,$file3,$file4} @RUNROOT@/@STORAGEPATH@/@DATABLOCK@/covariance_inputs/arb_summary_filters/ - fi - done + check_and_copy_files "$n_arb_ee" "$arb_fourier_filter_mmE_file" "$arb_fourier_filter_mmB_file" "$arb_real_filter_mm_p_file" "$arb_real_filter_mm_m_file" fi if [[ .*\ $MODES\ .* =~ " NE " ]] then - for i in $(seq -f "%02g" 1 $n_arb_ne) - do - file=`echo ${arb_fourier_filter_gm_file_@BV:STATISTIC@} | sed "s/?/${i}/g"` - file2=`echo ${arb_real_filter_gm_file_@BV:STATISTIC@} | sed "s/?/${i}/g"` - arb_base=@RUNROOT@/@CONFIGPATH@/covariance_arb_summary/ - if [ ! -f $arb_base${file} ] || [ ! -f $arb_base${file2} ] - then - #use_arbitrary=False - _message "One or more arbitrary input files do not exist. Calculating filters on the fly!\n" - break - else - cp ${arb_base}/{$file,$file2} @RUNROOT@/@STORAGEPATH@/@DATABLOCK@/covariance_inputs/arb_summary_filters/ - fi - done + check_and_copy_files "$n_arb_ne" "$arb_fourier_filter_gm_file" "$arb_real_filter_gm_file" fi if [[ .*\ $MODES\ .* =~ " NN " ]] then - for i in $(seq -f "%02g" 1 $n_arb_nn) - do - file=`echo ${arb_fourier_filter_gg_file_@BV:STATISTIC@} | sed "s/?/${i}/g"` - file2=`echo ${arb_real_filter_gg_file_@BV:STATISTIC@} | sed "s/?/${i}/g"` - arb_base=@RUNROOT@/@CONFIGPATH@/covariance_arb_summary/ - if [ ! -f $arb_base${file} ] || [ ! -f $arb_base${file2} ] - then - #use_arbitrary=False - _message "One or more arbitrary input files do not exist. Calculating filters on the fly!\n" - break - else - cp ${arb_base}/{$file,$file2} @RUNROOT@/@STORAGEPATH@/@DATABLOCK@/covariance_inputs/arb_summary_filters/ - fi - done + check_and_copy_files "$n_arb_nn" "$arb_fourier_filter_gg_file" "$arb_real_filter_gg_file" fi -if [ "${use_arbitrary}" == "True" ] -then - _message "Using arbitrary input files!\n" -fi mix_term=@BV:MIXTERM@ if [ "${mix_term^^}" == "TRUE" ] @@ -946,44 +934,44 @@ then if [[ .*\ $MODES\ .* =~ " EE " ]] then n_arb_ee2=@BV:NXIPM@ - arb_fourier_filter_mmE_file_@BV:STATISTIC@="fourier_weight_realspace_cf_mm_p_?.table" - arb_fourier_filter_mmB_file_@BV:STATISTIC@="fourier_weight_realspace_cf_mm_m_?.table" - arb_real_filter_mm_p_file_@BV:STATISTIC@="real_weight_realspace_cf_mm_p_?.table" - arb_real_filter_mm_m_file_@BV:STATISTIC@="real_weight_realspace_cf_mm_m_?.table" + arb_fourier_filter_mmE_file_second_stat="fourier_weight_realspace_cf_mm_p_?.table" + arb_fourier_filter_mmB_file_second_stat="fourier_weight_realspace_cf_mm_m_?.table" + arb_real_filter_mm_p_file_second_stat="real_weight_realspace_cf_mm_p_?.table" + arb_real_filter_mm_m_file_second_stat="real_weight_realspace_cf_mm_m_?.table" fi if [[ .*\ $MODES\ .* =~ " NE " ]] then n_arb_ne2=@BV:NGT@ - arb_fourier_filter_gm_file_@BV:STATISTIC@="fourier_weight_realspace_cf_gm_?.table" - arb_real_filter_gm_file_@BV:STATISTIC@="real_weight_realspace_cf_gm_?.table" + arb_fourier_filter_gm_file_second_stat="fourier_weight_realspace_cf_gm_?.table" + arb_real_filter_gm_file_second_stat="real_weight_realspace_cf_gm_?.table" fi if [[ .*\ $MODES\ .* =~ " NN " ]] then n_arb_nn2=@BV:NWT@ - arb_fourier_filter_gg_file_@BV:STATISTIC@="fourier_weight_realspace_cf_gg_?.table" - arb_real_filter_gg_file_@BV:STATISTIC@="real_weight_realspace_cf_gg_?.table" + arb_fourier_filter_gg_file_second_stat="fourier_weight_realspace_cf_gg_?.table" + arb_real_filter_gg_file_second_stat="real_weight_realspace_cf_gg_?.table" fi elif [ "${SECONDSTATISTIC^^}" == "COSEBIS" ] then if [[ .*\ $MODES\ .* =~ " EE " ]] then n_arb_ee2=@BV:NMAXCOSEBIS@ - arb_fourier_filter_mmE_file_@BV:STATISTIC@="Wn_@BV:THETAMINXI@_to_@BV:THETAMAXXI@_?.table" - arb_fourier_filter_mmB_file_@BV:STATISTIC@="Wn_@BV:THETAMINXI@_to_@BV:THETAMAXXI@_?.table" - arb_real_filter_mm_p_file_@BV:STATISTIC@="Tp_@BV:THETAMINXI@_to_@BV:THETAMAXXI@_?.table" - arb_real_filter_mm_m_file_@BV:STATISTIC@="Tm_@BV:THETAMINXI@_to_@BV:THETAMAXXI@_?.table" + arb_fourier_filter_mmE_file_second_stat="Wn_@BV:THETAMINXI@_to_@BV:THETAMAXXI@_?.table" + arb_fourier_filter_mmB_file_second_stat="Wn_@BV:THETAMINXI@_to_@BV:THETAMAXXI@_?.table" + arb_real_filter_mm_p_file_second_stat="Tp_@BV:THETAMINXI@_to_@BV:THETAMAXXI@_?.table" + arb_real_filter_mm_m_file_second_stat="Tm_@BV:THETAMINXI@_to_@BV:THETAMAXXI@_?.table" fi if [[ .*\ $MODES\ .* =~ " NE " ]] then n_arb_ne2=@BV:NMAXCOSEBISNE@ - arb_fourier_filter_gm_file_@BV:STATISTIC@="Qgm_@BV:THETAMINGT@_to_@BV:THETAMAXGT@_?.table" - arb_real_filter_gm_file_@BV:STATISTIC@="Wn_psigm_@BV:THETAMINGT@_to_@BV:THETAMAXGT@_?.table" + arb_fourier_filter_gm_file_second_stat="Qgm_@BV:THETAMINGT@_to_@BV:THETAMAXGT@_?.table" + arb_real_filter_gm_file_second_stat="Wn_psigm_@BV:THETAMINGT@_to_@BV:THETAMAXGT@_?.table" fi if [[ .*\ $MODES\ .* =~ " NN " ]] then - n_arb2_nn=@BV:NMAXCOSEBISNN@ - arb_fourier_filter_gg_file_@BV:STATISTIC@="Ugg_@BV:THETAMINWT@_to_@BV:THETAMAXWT@_?.table" - arb_real_filter_gg_file_@BV:STATISTIC@="Wn_psigg_@BV:THETAMINWT@_to_@BV:THETAMAXWT@_?.table" + n_arb_nn2=@BV:NMAXCOSEBISNN@ + arb_fourier_filter_gg_file_second_stat="Ugg_@BV:THETAMINWT@_to_@BV:THETAMAXWT@_?.table" + arb_real_filter_gg_file_second_stat="Wn_psigg_@BV:THETAMINWT@_to_@BV:THETAMAXWT@_?.table" fi elif [ "${SECONDSTATISTIC^^}" == "BANDPOWERS" ] then @@ -994,10 +982,10 @@ then theta_up_lensing=`echo 'e(l(@BV:THETAMAXXI@)-@BV:APODISATIONWIDTH@/2)' | bc -l | awk '{printf "%.9f", $0}'` t_lo=`printf "%.2f" $theta_lo` t_up=`printf "%.2f" $theta_up` - arb_fourier_filter_mmE_file_@BV:STATISTIC@="fourier_weight_bandpowers_mmE_?.table" - arb_fourier_filter_mmB_file_@BV:STATISTIC@="fourier_weight_bandpowers_mmB_?.table" - arb_real_filter_mm_p_file_@BV:STATISTIC@="real_weight_bandpowers_mmE_?.table" - arb_real_filter_mm_m_file_@BV:STATISTIC@="real_weight_bandpowers_mmB_?.table" + arb_fourier_filter_mmE_file_second_stat="fourier_weight_bandpowers_mmE_?.table" + arb_fourier_filter_mmB_file_second_stat="fourier_weight_bandpowers_mmB_?.table" + arb_real_filter_mm_p_file_second_stat="real_weight_bandpowers_mmE_?.table" + arb_real_filter_mm_m_file_second_stat="real_weight_bandpowers_mmB_?.table" fi if [[ .*\ $MODES\ .* =~ " NE " ]] then @@ -1006,8 +994,8 @@ then theta_up_clustering=`echo 'e(l(@BV:THETAMAXGT@)-@BV:APODISATIONWIDTH@/2)' | bc -l | awk '{printf "%.9f", $0}'` t_lo=`printf "%.2f" $theta_lo` t_up=`printf "%.2f" $theta_up` - arb_fourier_filter_gm_file_@BV:STATISTIC@="fourier_weight_bandpowers_gm_?.table" - arb_real_filter_gm_file_@BV:STATISTIC@="real_weight_bandpowers_gm_?.table" + arb_fourier_filter_gm_file_second_stat="fourier_weight_bandpowers_gm_?.table" + arb_real_filter_gm_file_second_stat="real_weight_bandpowers_gm_?.table" fi if [[ .*\ $MODES\ .* =~ " NN " ]] then @@ -1016,77 +1004,36 @@ then theta_up_clustering=`echo 'e(l(@BV:THETAMAXWT@)-@BV:APODISATIONWIDTH@/2)' | bc -l | awk '{printf "%.9f", $0}'` t_lo=`printf "%.2f" $theta_lo` t_up=`printf "%.2f" $theta_up` - arb_fourier_filter_gg_file_@BV:STATISTIC@="fourier_weight_bandpowers_gg_?.table" - arb_real_filter_gg_file_@BV:STATISTIC@="real_weight_bandpowers_gg_?.table" + arb_fourier_filter_gg_file_second_stat="fourier_weight_bandpowers_gg_?.table" + arb_real_filter_gg_file_second_stat="real_weight_bandpowers_gg_?.table" fi fi # Check if the arbitrary input files for second statistic exist and copy to input directory if [[ .*\ $MODES\ .* =~ " EE " ]] then - for i in $(seq -f "%02g" 1 $n_arb_ee2) - do - file=`echo ${arb_fourier_filter_mmE_file_@BV:SECONDSTATISTIC@} | sed "s/?/${i}/g"` - file2=`echo ${arb_fourier_filter_mmB_file_@BV:SECONDSTATISTIC@} | sed "s/?/${i}/g"` - file3=`echo ${arb_real_filter_mm_p_file_@BV:SECONDSTATISTIC@} | sed "s/?/${i}/g"` - file4=`echo ${arb_real_filter_mm_m_file_@BV:SECONDSTATISTIC@} | sed "s/?/${i}/g"` - arb_base=@RUNROOT@/@CONFIGPATH@/covariance_arb_summary/ - if [ ! -f $arb_base${file} ] || [ ! -f $arb_base${file2} ] || [ ! -f $arb_base${file3} ] || [ ! -f $arb_base${file4} ] - then - #use_arbitrary=False - _message "One or more arbitrary input files do not exist. Calculating filters on the fly!\n" - break - else - cp ${arb_base}/{$file,$file2,$file3,$file4} @RUNROOT@/@STORAGEPATH@/@DATABLOCK@/covariance_inputs/arb_summary_filters/ - fi - done + check_and_copy_files "$n_arb_ee2" "$arb_fourier_filter_mmE_file_second_stat" "$arb_fourier_filter_mmB_file_second_stat" "$arb_real_filter_mm_p_file_second_stat" "$arb_real_filter_mm_m_file_second_stat" fi if [[ .*\ $MODES\ .* =~ " NE " ]] then - for i in $(seq -f "%02g" 1 $n_arb_ne2) - do - file=`echo ${arb_fourier_filter_gm_file_@BV:SECONDSTATISTIC@} | sed "s/?/${i}/g"` - file2=`echo ${arb_real_filter_gm_file_@BV:SECONDSTATISTIC@} | sed "s/?/${i}/g"` - arb_base=@RUNROOT@/@CONFIGPATH@/covariance_arb_summary/ - if [ ! -f $arb_base${file} ] || [ ! -f $arb_base${file2} ] - then - #use_arbitrary=False - _message "One or more arbitrary input files do not exist. Calculating filters on the fly!\n" - break - else - cp ${arb_base}/{$file,$file2} @RUNROOT@/@STORAGEPATH@/@DATABLOCK@/covariance_inputs/arb_summary_filters/ - fi - done + check_and_copy_files "$n_arb_ne2" "$arb_fourier_filter_gm_file_second_stat" "$arb_real_filter_gm_file_second_stat" fi if [[ .*\ $MODES\ .* =~ " NN " ]] then - for i in $(seq -f "%02g" 1 $n_arb_nn2) - do - file=`echo ${arb_fourier_filter_gg_file_@BV:SECONDSTATISTIC@} | sed "s/?/${i}/g"` - file2=`echo ${arb_real_filter_gg_file_@BV:SECONDSTATISTIC@} | sed "s/?/${i}/g"` - arb_base=@RUNROOT@/@CONFIGPATH@/covariance_arb_summary/ - if [ ! -f $arb_base${file} ] || [ ! -f $arb_base${file2} ] - then - #use_arbitrary=False - _message "One or more arbitrary input files do not exist. Calculating filters on the fly!\n" - break - else - cp ${arb_base}/{$file,$file2} @RUNROOT@/@STORAGEPATH@/@DATABLOCK@/covariance_inputs/arb_summary_filters/ - fi - done + check_and_copy_files "$n_arb_nn2" "$arb_fourier_filter_gg_file_second_stat" "$arb_real_filter_gg_file_second_stat" fi cat >> @RUNROOT@/@STORAGEPATH@/@DATABLOCK@/covariance_inputs/@SURVEY@_CosmoPipe_constructed_other.ini <<- EOF ;arb_summary_directory = @RUNROOT@/@CONFIGPATH@/covariance_arb_summary/ arb_summary_directory = @RUNROOT@/@STORAGEPATH@/@DATABLOCK@/covariance_inputs/arb_summary_filters/ -arb_fourier_filter_mmE_file = ${arb_fourier_filter_mmE_file_@BV:STATISTIC@}, ${arb_fourier_filter_mmE_file_@BV:SECONDSTATISTIC@} -arb_fourier_filter_mmB_file = ${arb_fourier_filter_mmB_file_@BV:STATISTIC@}, ${arb_fourier_filter_mmB_file_@BV:SECONDSTATISTIC@} -arb_real_filter_mm_p_file = ${arb_real_filter_mm_p_file_@BV:STATISTIC@}, ${arb_real_filter_mm_p_file_@BV:SECONDSTATISTIC@} -arb_real_filter_mm_m_file = ${arb_real_filter_mm_m_file_@BV:STATISTIC@}, ${arb_real_filter_mm_m_file_@BV:SECONDSTATISTIC@} -arb_fourier_filter_gm_file = ${arb_fourier_filter_gm_file_@BV:STATISTIC@}, ${arb_fourier_filter_gm_file_@BV:SECONDSTATISTIC@} -arb_real_filter_gm_file = ${arb_real_filter_gm_file_@BV:STATISTIC@}, ${arb_real_filter_gm_file_@BV:SECONDSTATISTIC@} -arb_fourier_filter_gg_file = ${arb_fourier_filter_gg_file_@BV:STATISTIC@}, ${arb_fourier_filter_gg_file_@BV:SECONDSTATISTIC@} -arb_real_filter_gg_file = ${arb_real_filter_gg_file_@BV:STATISTIC@}, ${arb_real_filter_gg_file_@BV:SECONDSTATISTIC@} +arb_fourier_filter_mmE_file = ${arb_fourier_filter_mmE_file}, ${arb_fourier_filter_mmE_file_second_stat} +arb_fourier_filter_mmB_file = ${arb_fourier_filter_mmB_file}, ${arb_fourier_filter_mmB_file_second_stat} +arb_real_filter_mm_p_file = ${arb_real_filter_mm_p_file}, ${arb_real_filter_mm_p_file_second_stat} +arb_real_filter_mm_m_file = ${arb_real_filter_mm_m_file}, ${arb_real_filter_mm_m_file_second_stat} +arb_fourier_filter_gm_file = ${arb_fourier_filter_gm_file}, ${arb_fourier_filter_gm_file_second_stat} +arb_real_filter_gm_file = ${arb_real_filter_gm_file}, ${arb_real_filter_gm_file_second_stat} +arb_fourier_filter_gg_file = ${arb_fourier_filter_gg_file}, ${arb_fourier_filter_gg_file_second_stat} +arb_real_filter_gg_file = ${arb_real_filter_gg_file}, ${arb_real_filter_gg_file_second_stat} [arbitrary_summary] do_arbitrary_obs = True @@ -1101,14 +1048,14 @@ then cat >> @RUNROOT@/@STORAGEPATH@/@DATABLOCK@/covariance_inputs/@SURVEY@_CosmoPipe_constructed_other.ini <<- EOF ;arb_summary_directory = @RUNROOT@/@CONFIGPATH@/covariance_arb_summary/ arb_summary_directory = @RUNROOT@/@STORAGEPATH@/@DATABLOCK@/covariance_inputs/arb_summary_filters/ -arb_fourier_filter_mmE_file = ${arb_fourier_filter_mmE_file_@BV:STATISTIC@} -arb_fourier_filter_mmB_file = ${arb_fourier_filter_mmB_file_@BV:STATISTIC@} -arb_real_filter_mm_p_file = ${arb_real_filter_mm_p_file_@BV:STATISTIC@} -arb_real_filter_mm_m_file = ${arb_real_filter_mm_m_file_@BV:STATISTIC@} -arb_fourier_filter_gm_file = ${arb_fourier_filter_gm_file_@BV:STATISTIC@} -arb_real_filter_gm_file = ${arb_real_filter_gm_file_@BV:STATISTIC@} -arb_fourier_filter_gg_file = ${arb_fourier_filter_gg_file_@BV:STATISTIC@} -arb_real_filter_gg_file = ${arb_real_filter_gg_file_@BV:STATISTIC@} +arb_fourier_filter_mmE_file = ${arb_fourier_filter_mmE_file} +arb_fourier_filter_mmB_file = ${arb_fourier_filter_mmB_file} +arb_real_filter_mm_p_file = ${arb_real_filter_mm_p_file} +arb_real_filter_mm_m_file = ${arb_real_filter_mm_m_file} +arb_fourier_filter_gm_file = ${arb_fourier_filter_gm_file} +arb_real_filter_gm_file = ${arb_real_filter_gm_file} +arb_fourier_filter_gg_file = ${arb_fourier_filter_gg_file} +arb_real_filter_gg_file = ${arb_real_filter_gg_file} [arbitrary_summary] do_arbitrary_obs = True diff --git a/scripts/run_cov_weights.sh b/scripts/run_cov_weights.sh index ee33521d..62f0751c 100644 --- a/scripts/run_cov_weights.sh +++ b/scripts/run_cov_weights.sh @@ -1,213 +1,203 @@ -# +#!/bin/bash # # Script to construct weights needed for arbitrary covariance stats # -# STATISTIC="@BV:STATISTIC@" -#Output folder: +# Output folder: outfold=@RUNROOT@/@CONFIGPATH@/covariance_arb_summary/ -if [ ! -d ${outfold} ] -then - mkdir ${outfold} -fi - +mkdir -p "${outfold}" -if [ "${STATISTIC^^}" == "2PCF" ] -then +# Set variables depending on the statistic +case "${STATISTIC^^}" in + "2PCF") n_arb_ee=@BV:NXIPM@ - arb_fourier_filter_mmE_file_@BV:STATISTIC@="fourier_weight_realspace_cf_mm_p_?.table" - arb_fourier_filter_mmB_file_@BV:STATISTIC@="fourier_weight_realspace_cf_mm_m_?.table" - arb_real_filter_mm_p_file_@BV:STATISTIC@="real_weight_realspace_cf_mm_p_?.table" - arb_real_filter_mm_m_file_@BV:STATISTIC@="real_weight_realspace_cf_mm_m_?.table" n_arb_ne=@BV:NGT@ - arb_fourier_filter_gm_file_@BV:STATISTIC@="fourier_weight_realspace_cf_gm_?.table" - arb_real_filter_gm_file_@BV:STATISTIC@="real_weight_realspace_cf_gm_?.table" n_arb_nn=@BV:NWT@ - arb_fourier_filter_gg_file_@BV:STATISTIC@="fourier_weight_realspace_cf_gg_?.table" - arb_real_filter_gg_file_@BV:STATISTIC@="real_weight_realspace_cf_gg_?.table" + + arb_fourier_filter_mmE_file="fourier_weight_realspace_cf_mm_p_?.table" + arb_fourier_filter_mmB_file="fourier_weight_realspace_cf_mm_m_?.table" + arb_real_filter_mm_p_file="real_weight_realspace_cf_mm_p_?.table" + arb_real_filter_mm_m_file="real_weight_realspace_cf_mm_m_?.table" + + arb_fourier_filter_gm_file="fourier_weight_realspace_cf_gm_?.table" + arb_real_filter_gm_file="real_weight_realspace_cf_gm_?.table" + + arb_fourier_filter_gg_file="fourier_weight_realspace_cf_gg_?.table" + arb_real_filter_gg_file="real_weight_realspace_cf_gg_?.table" + arb_base=@RUNROOT@/INSTALL/OneCovariance/input/arbitrary_summary/rcf/ - -elif [ "${STATISTIC^^}" == "COSEBIS" ] -then + ;; + "COSEBIS") n_arb_ee=@BV:NMAXCOSEBIS@ - arb_fourier_filter_mmE_file_@BV:STATISTIC@="Wn_@BV:THETAMINXI@_to_@BV:THETAMAXXI@_?.table" - arb_fourier_filter_mmB_file_@BV:STATISTIC@="Wn_@BV:THETAMINXI@_to_@BV:THETAMAXXI@_?.table" - arb_real_filter_mm_p_file_@BV:STATISTIC@="Tp_@BV:THETAMINXI@_to_@BV:THETAMAXXI@_?.table" - arb_real_filter_mm_m_file_@BV:STATISTIC@="Tm_@BV:THETAMINXI@_to_@BV:THETAMAXXI@_?.table" n_arb_ne=@BV:NMAXCOSEBISNE@ - arb_fourier_filter_gm_file_@BV:STATISTIC@="Qgm_@BV:THETAMINGT@_to_@BV:THETAMAXGT@_?.table" - arb_real_filter_gm_file_@BV:STATISTIC@="Wn_psigm_@BV:THETAMINGT@_to_@BV:THETAMAXGT@_?.table" n_arb_nn=@BV:NMAXCOSEBISNN@ - arb_fourier_filter_gg_file_@BV:STATISTIC@="Ugg_@BV:THETAMINWT@_to_@BV:THETAMAXWT@_?.table" - arb_real_filter_gg_file_@BV:STATISTIC@="Wn_psigg_@BV:THETAMINWT@_to_@BV:THETAMAXWT@_?.table" - arb_base=@RUNROOT@/INSTALL/OneCovariance/input/arbitrary_summary/cosebis/ -elif [ "${STATISTIC^^}" == "BANDPOWERS" ] -then + arb_fourier_filter_mmE_file="Wn_@BV:THETAMINXI@_to_@BV:THETAMAXXI@_?.table" + arb_fourier_filter_mmB_file="Wn_@BV:THETAMINXI@_to_@BV:THETAMAXXI@_?.table" + arb_real_filter_mm_p_file="Tp_@BV:THETAMINXI@_to_@BV:THETAMAXXI@_?.table" + arb_real_filter_mm_m_file="Tm_@BV:THETAMINXI@_to_@BV:THETAMAXXI@_?.table" + + arb_fourier_filter_gm_file="Qgm_@BV:THETAMINGT@_to_@BV:THETAMAXGT@_?.table" + arb_real_filter_gm_file="Wn_psigm_@BV:THETAMINGT@_to_@BV:THETAMAXGT@_?.table" + + arb_fourier_filter_gg_file="Ugg_@BV:THETAMINWT@_to_@BV:THETAMAXWT@_?.table" + arb_real_filter_gg_file="Wn_psigg_@BV:THETAMINWT@_to_@BV:THETAMAXWT@_?.table" + + arb_base=@RUNROOT@/INSTALL/OneCovariance/input/arbitrary_summary/cosebis/ + ;; + "BANDPOWERS") n_arb_ee=@BV:NBANDPOWERS@ - theta_lo_lensing=`echo 'e(l(@BV:THETAMINXI@)+@BV:APODISATIONWIDTH@/2)' | bc -l | awk '{printf "%.9f", $0}'` - theta_up_lensing=`echo 'e(l(@BV:THETAMAXXI@)-@BV:APODISATIONWIDTH@/2)' | bc -l | awk '{printf "%.9f", $0}'` - t_lo_mm=`printf "%.2f" $theta_lo_lensing` - t_up_mm=`printf "%.2f" $theta_up_lensing` - arb_fourier_filter_mmE_file_@BV:STATISTIC@="fourier_weight_bandpowers_mmE_?.table" - arb_fourier_filter_mmB_file_@BV:STATISTIC@="fourier_weight_bandpowers_mmB_?.table" - arb_real_filter_mm_p_file_@BV:STATISTIC@="real_weight_bandpowers_mmE_?.table" - arb_real_filter_mm_m_file_@BV:STATISTIC@="real_weight_bandpowers_mmB_?.table" n_arb_ne=@BV:NBANDPOWERSNE@ - theta_lo_ggl=`echo 'e(l(@BV:THETAMINGT@)+@BV:APODISATIONWIDTH@/2)' | bc -l | awk '{printf "%.9f", $0}'` - theta_up_ggl=`echo 'e(l(@BV:THETAMAXGT@)-@BV:APODISATIONWIDTH@/2)' | bc -l | awk '{printf "%.9f", $0}'` - t_lo_gm=`printf "%.2f" $theta_lo_ggl` - t_up_gm=`printf "%.2f" $theta_up_ggl` - arb_fourier_filter_gm_file_@BV:STATISTIC@="fourier_weight_bandpowers_gm_?.table" - arb_real_filter_gm_file_@BV:STATISTIC@="real_weight_bandpowers_gm_?.table" n_arb_nn=@BV:NBANDPOWERSNN@ - theta_lo_clustering=`echo 'e(l(@BV:THETAMINWT@)+@BV:APODISATIONWIDTH@/2)' | bc -l | awk '{printf "%.9f", $0}'` - theta_up_clustering=`echo 'e(l(@BV:THETAMAXWT@)-@BV:APODISATIONWIDTH@/2)' | bc -l | awk '{printf "%.9f", $0}'` - t_lo_gg=`printf "%.2f" $theta_lo_clustering` - t_up_gg=`printf "%.2f" $theta_up_clustering` - arb_fourier_filter_gg_file_@BV:STATISTIC@="fourier_weight_bandpowers_gg_?.table" - arb_real_filter_gg_file_@BV:STATISTIC@="real_weight_bandpowers_gg_?.table" - arb_base=@RUNROOT@/INSTALL/OneCovariance/input/arbitrary_summary/bandpowers/ -else - #ERROR: Unknown statistic {{{ - _message "Unknown statistic: ${STATISTIC^^}\n" - exit 1 - #}}} -fi + theta_lo_lensing=$(echo "e(l(@BV:THETAMINXI@)+@BV:APODISATIONWIDTH@/2)" | bc -l) + theta_up_lensing=$(echo "e(l(@BV:THETAMAXXI@)-@BV:APODISATIONWIDTH@/2)" | bc -l) + t_lo_mm=$(printf "%.2f" "$theta_lo_lensing") + t_up_mm=$(printf "%.2f" "$theta_up_lensing") + + theta_lo_ggl=$(echo "e(l(@BV:THETAMINGT@)+@BV:APODISATIONWIDTH@/2)" | bc -l) + theta_up_ggl=$(echo "e(l(@BV:THETAMAXGT@)-@BV:APODISATIONWIDTH@/2)" | bc -l) + t_lo_gm=$(printf "%.2f" "$theta_lo_ggl") + t_up_gm=$(printf "%.2f" "$theta_up_ggl") + + theta_lo_clustering=$(echo "e(l(@BV:THETAMINWT@)+@BV:APODISATIONWIDTH@/2)" | bc -l) + theta_up_clustering=$(echo "e(l(@BV:THETAMAXWT@)-@BV:APODISATIONWIDTH@/2)" | bc -l) + t_lo_gg=$(printf "%.2f" "$theta_lo_clustering") + t_up_gg=$(printf "%.2f" "$theta_up_clustering") + + arb_fourier_filter_mmE_file="fourier_weight_bandpowers_mmE_?.table" + arb_fourier_filter_mmB_file="fourier_weight_bandpowers_mmB_?.table" + arb_real_filter_mm_p_file="real_weight_bandpowers_mmE_?.table" + arb_real_filter_mm_m_file="real_weight_bandpowers_mmB_?.table" + + arb_fourier_filter_gm_file="fourier_weight_bandpowers_gm_?.table" + arb_real_filter_gm_file="real_weight_bandpowers_gm_?.table" + + arb_fourier_filter_gg_file="fourier_weight_bandpowers_gg_?.table" + arb_real_filter_gg_file="real_weight_bandpowers_gg_?.table" + + arb_base=@RUNROOT@/INSTALL/OneCovariance/input/arbitrary_summary/bandpowers/ + ;; + *) + _message "Unknown statistic: ${STATISTIC^^}\n" + exit 1 + ;; +esac run_arbitrary=False -for i in $(seq -f "%02g" 1 $n_arb_ee) -do - file=`echo ${arb_fourier_filter_mmE_file_@BV:STATISTIC@} | sed "s/?/${i}/g"` - file2=`echo ${arb_fourier_filter_mmB_file_@BV:STATISTIC@} | sed "s/?/${i}/g"` - file3=`echo ${arb_real_filter_mm_p_file_@BV:STATISTIC@} | sed "s/?/${i}/g"` - file4=`echo ${arb_real_filter_mm_m_file_@BV:STATISTIC@} | sed "s/?/${i}/g"` - if [ ! -f $arb_base${file} ] || [ ! -f $arb_base${file2} ] || [ ! -f $arb_base${file3} ] || [ ! -f $arb_base${file4} ] - then - run_arbitrary=True - _message "One or more arbitrary input files do not exist. Calculating filters now!\n" - break - else - cp ${arb_base}/{$file,$file2,$file3,$file4} ${outfold} - fi -done -for i in $(seq -f "%02g" 1 $n_arb_ne) -do - file=`echo ${arb_fourier_filter_gm_file_@BV:STATISTIC@} | sed "s/?/${i}/g"` - file2=`echo ${arb_real_filter_gm_file_@BV:STATISTIC@} | sed "s/?/${i}/g"` - if [ ! -f $arb_base${file} ] || [ ! -f $arb_base${file2} ] - then - run_arbitrary=True - _message "One or more arbitrary input files do not exist. Calculating filters now!\n" - break - else - cp ${arb_base}/{$file,$file2} ${outfold} - fi -done -for i in $(seq -f "%02g" 1 $n_arb_nn) -do - file=`echo ${arb_fourier_filter_gg_file_@BV:STATISTIC@} | sed "s/?/${i}/g"` - file2=`echo ${arb_real_filter_gg_file_@BV:STATISTIC@} | sed "s/?/${i}/g"` - if [ ! -f $arb_base${file} ] || [ ! -f $arb_base${file2} ] - then - run_arbitrary=True - _message "One or more arbitrary input files do not exist. Calculating filters now!\n" - break - else - cp ${arb_base}/{$file,$file2} ${outfold} - fi -done - - -if [ "${run_arbitrary}" == "True" ] -then - if [ "${STATISTIC^^}" == "2PCF" ] - then - _message " -> @BLU@Generating arbitrary statistic weights for 2pcfs @DEF@" - cd @RUNROOT@/INSTALL/OneCovariance/input/arbitrary_summary/script_weights/ - @PYTHON3BIN@ @RUNROOT@/INSTALL/OneCovariance/input/arbitrary_summary/script_weights/get_weights_realspace.py \ - -n @BV:NTHREADS@ \ - -nf 100000 \ - -nt 100000 \ - --theta_lo_mm @BV:THETAMINXI@ \ - --theta_up_mm @BV:THETAMAXXI@ \ - --t_bins_mm @BV:NXIPM@ \ - --t_type_mm "log" \ - --theta_lo_gm @BV:THETAMINGT@ \ - --theta_up_gm @BV:THETAMAXGT@ \ - --t_bins_gm @BV:NGT@ \ - --t_type_gm "log" \ - --theta_lo_gg @BV:THETAMINWT@\ - --theta_up_gg @BV:THETAMAXWT@\ - --t_bins_gg @BV:NWT@ \ - --t_type_gg "log" 2>&1 - - cd @RUNROOT@ - cp -a @RUNROOT@/INSTALL/OneCovariance/input/arbitrary_summary/rcf/. ${outfold} - - elif [ "${STATISTIC^^}" == "COSEBIS" ] - then - _message " -> @BLU@Generating arbitrary statistic weights for cosebis / psi stats @DEF@" - cd @RUNROOT@/INSTALL/OneCovariance/input/arbitrary_summary/script_weights/ - @PYTHON3BIN@ @RUNROOT@/INSTALL/OneCovariance/input/arbitrary_summary/script_weights/get_weights_cosebis.py \ - -n @BV:NTHREADS@ \ - -nf 100000 \ - -nt 10000 \ - --Nmax_mm @BV:NMAXCOSEBIS@ \ - --tmin_mm @BV:THETAMINXI@ \ - --tmax_mm @BV:THETAMAXXI@ \ - --Nmax_gm @BV:NMAXCOSEBISNE@ \ - --tmin_gm @BV:THETAMINGT@ \ - --tmax_gm @BV:THETAMAXGT@ \ - --Nmax_gg @BV:NMAXCOSEBISNN@ \ - --tmin_gg @BV:THETAMINWT@ \ - --tmax_gg @BV:THETAMAXWT@ 2>&1 - - cd @RUNROOT@ - cp -a @RUNROOT@/INSTALL/OneCovariance/input/arbitrary_summary/cosebis/. ${outfold} - - - elif [ "${STATISTIC^^}" == "BANDPOWERS" ] - then - _message " -> @BLU@Generating arbitrary statistic weights for bandpowers @DEF@" - cd @RUNROOT@/INSTALL/OneCovariance/input/arbitrary_summary/script_weights/ - @PYTHON3BIN@ @RUNROOT@/INSTALL/OneCovariance/input/arbitrary_summary/script_weights/get_weights_bandpowers.py \ - -n @BV:NTHREADS@ \ - -nf 10000 \ - -nt 10000 \ - --delta_ln_theta_mm @BV:APODISATIONWIDTH@ \ - --theta_lo_mm ${t_lo_mm} \ - --theta_up_mm ${t_up_mm} \ - --L_min_mm @BV:LMINBANDPOWERS@ \ - --L_max_mm @BV:LMAXBANDPOWERS@ \ - --L_bins_mm @BV:NBANDPOWERS@ \ - --L_type_mm "log" \ - --delta_ln_theta_gm @BV:APODISATIONWIDTH@ \ - --theta_lo_gm ${t_lo_gm} \ - --theta_up_gm ${t_up_gm} \ - --L_min_gm @BV:LMINBANDPOWERSNE@ \ - --L_max_gm @BV:LMAXBANDPOWERSNE@ \ - --L_bins_gm @BV:NBANDPOWERSNE@ \ - --L_type_gm "log" \ - --delta_ln_theta_gg @BV:APODISATIONWIDTH@ \ - --theta_lo_gg ${t_lo_gg} \ - --theta_up_gg ${t_up_gg} \ - --L_min_gg @BV:LMINBANDPOWERSNN@ \ - --L_max_gg @BV:LMAXBANDPOWERSNN@ \ - --L_bins_gg @BV:NBANDPOWERSNN@ \ - --L_type_gg "log" 2>&1 - - cd @RUNROOT@ - cp -a @RUNROOT@/INSTALL/OneCovariance/input/arbitrary_summary/bandpowers/. ${outfold} - fi -fi -_message " - @RED@Done! (`date +'%a %H:%M'`)@DEF@\n" +# Function to check and copy files +check_and_copy_files() { + local n=$1 + shift + local files=("$@") + + for i in $(seq -f "%02g" 1 "$n"); do + missing=False + expanded_files=() + for f in "${files[@]}"; do + file="${f//\?/$i}" + expanded_files+=("$file") + if [ ! -f "$arb_base$file" ]; then + missing=True + fi + done + + if [ "$missing" == "True" ]; then + run_arbitrary=True + _message "One or more arbitrary input files do not exist. Calculating filters now!\n" + break + else + for f in "${expanded_files[@]}"; do + cp "$arb_base$f" "$outfold" + done + fi + done +} +# Check files +check_and_copy_files "$n_arb_ee" "$arb_fourier_filter_mmE_file" "$arb_fourier_filter_mmB_file" "$arb_real_filter_mm_p_file" "$arb_real_filter_mm_m_file" +check_and_copy_files "$n_arb_ne" "$arb_fourier_filter_gm_file" "$arb_real_filter_gm_file" +check_and_copy_files "$n_arb_nn" "$arb_fourier_filter_gg_file" "$arb_real_filter_gg_file" + +# Run scripts if needed +if [ "$run_arbitrary" == "True" ]; then + cd @RUNROOT@/INSTALL/OneCovariance/input/arbitrary_summary/script_weights/ + case "${STATISTIC^^}" in + "2PCF") + _message " -> @BLU@Generating arbitrary statistic weights for 2pcfs @DEF@" + @PYTHON3BIN@ get_weights_realspace.py \ + -n @BV:NTHREADS@ \ + -nf 100000 \ + -nt 100000 \ + --theta_lo_mm @BV:THETAMINXI@ \ + --theta_up_mm @BV:THETAMAXXI@ \ + --t_bins_mm @BV:NXIPM@ \ + --t_type_mm "log" \ + --theta_lo_gm @BV:THETAMINGT@ \ + --theta_up_gm @BV:THETAMAXGT@ \ + --t_bins_gm @BV:NGT@ \ + --t_type_gm "log" \ + --theta_lo_gg @BV:THETAMINWT@ \ + --theta_up_gg @BV:THETAMAXWT@ \ + --t_bins_gg @BV:NWT@ \ + --t_type_gg "log" 2>&1 + cp -a @RUNROOT@/INSTALL/OneCovariance/input/arbitrary_summary/rcf/. "$outfold" + ;; + "COSEBIS") + _message " -> @BLU@Generating arbitrary statistic weights for cosebis / psi stats @DEF@" + @PYTHON3BIN@ get_weights_cosebis.py \ + -n @BV:NTHREADS@ \ + -nf 100000 \ + -nt 10000 \ + --Nmax_mm @BV:NMAXCOSEBIS@ \ + --tmin_mm @BV:THETAMINXI@ \ + --tmax_mm @BV:THETAMAXXI@ \ + --Nmax_gm @BV:NMAXCOSEBISNE@ \ + --tmin_gm @BV:THETAMINGT@ \ + --tmax_gm @BV:THETAMAXGT@ \ + --Nmax_gg @BV:NMAXCOSEBISNN@ \ + --tmin_gg @BV:THETAMINWT@ \ + --tmax_gg @BV:THETAMAXWT@ 2>&1 + cp -a @RUNROOT@/INSTALL/OneCovariance/input/arbitrary_summary/cosebis/. "$outfold" + ;; + "BANDPOWERS") + _message " -> @BLU@Generating arbitrary statistic weights for bandpowers @DEF@" + @PYTHON3BIN@ get_weights_bandpowers.py \ + -n @BV:NTHREADS@ \ + -nf 10000 \ + -nt 10000 \ + --delta_ln_theta_mm @BV:APODISATIONWIDTH@ \ + --theta_lo_mm ${t_lo_mm} \ + --theta_up_mm ${t_up_mm} \ + --L_min_mm @BV:LMINBANDPOWERS@ \ + --L_max_mm @BV:LMAXBANDPOWERS@ \ + --L_bins_mm @BV:NBANDPOWERS@ \ + --L_type_mm "log" \ + --delta_ln_theta_gm @BV:APODISATIONWIDTH@ \ + --theta_lo_gm ${t_lo_gm} \ + --theta_up_gm ${t_up_gm} \ + --L_min_gm @BV:LMINBANDPOWERSNE@ \ + --L_max_gm @BV:LMAXBANDPOWERSNE@ \ + --L_bins_gm @BV:NBANDPOWERSNE@ \ + --L_type_gm "log" \ + --delta_ln_theta_gg @BV:APODISATIONWIDTH@ \ + --theta_lo_gg ${t_lo_gg} \ + --theta_up_gg ${t_up_gg} \ + --L_min_gg @BV:LMINBANDPOWERSNN@ \ + --L_max_gg @BV:LMAXBANDPOWERSNN@ \ + --L_bins_gg @BV:NBANDPOWERSNN@ \ + --L_type_gg "log" 2>&1 + cp -a @RUNROOT@/INSTALL/OneCovariance/input/arbitrary_summary/bandpowers/. "$outfold" + ;; + esac + cd @RUNROOT@ +fi + +_message " - @RED@Done! (`date +'%a %H:%M'`)@DEF@\n" From 5db397a9921a36a217c116e1603a7bc630ab90a2 Mon Sep 17 00:00:00 2001 From: andrejdvornik Date: Fri, 16 Jan 2026 10:07:42 +0000 Subject: [PATCH 26/37] Update to onepower settings --- scripts/cosmosis_constructor_3x2pt.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/cosmosis_constructor_3x2pt.sh b/scripts/cosmosis_constructor_3x2pt.sh index bc3f6d84..272417df 100644 --- a/scripts/cosmosis_constructor_3x2pt.sh +++ b/scripts/cosmosis_constructor_3x2pt.sh @@ -43,8 +43,8 @@ zmax_def = 1.2 nmass_def = 100 ; 200 logmassmin_def = 9.0 logmassmax_def = 18.0 -beta_nl = True -mead2020_corrections = fit_feedback +nonlinear_mode = bnl +mead2020_corrections = fit SAMPLER_NAME = @BV:SAMPLER@ RUN_NAME = %(SAMPLER_NAME)s_%(blind)s${CHAINSUFFIX} @@ -1459,14 +1459,14 @@ do fi cat >> @RUNROOT@/@STORAGEPATH@/@DATABLOCK@/cosmosis_inputs/@SURVEY@_CosmoPipe_constructed_other.ini <<- EOF - bnl = %(beta_nl)s + nonlinear_mode = %(nonlinear_mode)s update_bnl = 10 poisson_type = constant point_mass = True dewiggle = True response = False output_suffix = onepower - use_mead2020_corrections = %(mead2020_corrections)s + hmcode_ingredients = %(mead2020_corrections)s log_mass_min = %(logmassmin_def)s log_mass_max = %(logmassmax_def)s From 31e70ca44777eb26440f05b10c06cdf98f181127 Mon Sep 17 00:00:00 2001 From: andrejdvornik Date: Fri, 16 Jan 2026 10:08:24 +0000 Subject: [PATCH 27/37] Rename variables --- scripts/cosmosis_constructor_3x2pt.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/cosmosis_constructor_3x2pt.sh b/scripts/cosmosis_constructor_3x2pt.sh index 272417df..621a5333 100644 --- a/scripts/cosmosis_constructor_3x2pt.sh +++ b/scripts/cosmosis_constructor_3x2pt.sh @@ -44,7 +44,7 @@ nmass_def = 100 ; 200 logmassmin_def = 9.0 logmassmax_def = 18.0 nonlinear_mode = bnl -mead2020_corrections = fit +hmcode_ingredients = fit SAMPLER_NAME = @BV:SAMPLER@ RUN_NAME = %(SAMPLER_NAME)s_%(blind)s${CHAINSUFFIX} @@ -1466,7 +1466,7 @@ do dewiggle = True response = False output_suffix = onepower - hmcode_ingredients = %(mead2020_corrections)s + hmcode_ingredients = %(hmcode_ingredients)s log_mass_min = %(logmassmin_def)s log_mass_max = %(logmassmax_def)s From d19abf02dbf331eef9d7a17a17335069983b7570 Mon Sep 17 00:00:00 2001 From: andrejdvornik Date: Fri, 16 Jan 2026 13:48:49 +0000 Subject: [PATCH 28/37] Added check for zmin/zmax limits in calc_smf.py --- scripts/calc_smf.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/scripts/calc_smf.py b/scripts/calc_smf.py index 0b4c4fd3..56b63402 100644 --- a/scripts/calc_smf.py +++ b/scripts/calc_smf.py @@ -140,6 +140,10 @@ stellar_mass_in = data[stellar_mass_column] z_in = data[z_column] + # Check that data covers min_z, max_z + if not np.any((z_in >= min_z) & (z_in <= max_z)): + raise ValueError(f"No data in redshift range {min_z} < z < {max_z}") + # We set the possible minimum z to 0.001 z_min = np.maximum(0.001, min_z) # max_z is given as an input, this is the maximum redshift in the stellar mass-redshift bin. @@ -151,7 +155,6 @@ # or maximum redshift of the stellar mass-redshift bin z_max_i = np.minimum(z_max_bin, z_max) - if estimator == 'simple': # Comoving distance at z_min and z_max_i in units of Mpc h # z_max_i is the maximum redshift at which galaxy i is visible in the sample From 1d8dd80d176158f98b89dd27ee395a6a2dddb4f0 Mon Sep 17 00:00:00 2001 From: andrejdvornik Date: Fri, 13 Feb 2026 08:40:46 +0000 Subject: [PATCH 29/37] Added working environment without conflicts that supports OnePower and CosmoPower requirements for tensorflow --- environment_3x2pt.yml | 409 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 409 insertions(+) create mode 100644 environment_3x2pt.yml diff --git a/environment_3x2pt.yml b/environment_3x2pt.yml new file mode 100644 index 00000000..b4b89f86 --- /dev/null +++ b/environment_3x2pt.yml @@ -0,0 +1,409 @@ +name: cosmopipe +channels: + - conda-forge + - defaults +dependencies: + - _libgcc_mutex=0.1=conda_forge + - _openmp_mutex=4.5=2_gnu + - _r-mutex=1.0.1=anacondar_1 + - _sysroot_linux-64_curr_repodata_hack=3=h69a702a_13 + - astropy-base=6.0.0=h67a7307_0 + - asttokens=3.0.0=pyhd8ed1ab_1 + - atk-1.0=2.38.0=hd4edc92_1 + - attrs=23.1.0=pyh71513ae_1 + - binutils=2.40=hdd6e379_0 + - binutils_impl_linux-64=2.40=hf600244_0 + - binutils_linux-64=2.40=hbdbef99_2 + - blas=1.0=openblas + - bleach=6.2.0=py311h06a4308_0 + - bottleneck=1.4.2=py311hf4808d0_0 + - brotli=1.1.0=hd590300_1 + - brotli-bin=1.1.0=hd590300_1 + - brotli-python=1.1.0=py311hb755f60_1 + - brotlipy=0.7.0=py311h459d7ec_1006 + - bwidget=1.9.14=ha770c72_1 + - bzip2=1.0.8=hd590300_5 + - c-ares=1.23.0=hd590300_0 + - c-compiler=1.7.0=hd590300_0 + - ca-certificates=2025.9.9=h06a4308_0 + - cairo=1.16.0=ha61ee94_1014 + - camb=1.5.2=py311hbc92942_0 + - certifi=2025.8.3=py311h06a4308_0 + - cffi=1.16.0=py311hb3a22ac_0 + - cfitsio=4.2.0=hd9d235c_0 + - charset-normalizer=3.3.2=pyhd8ed1ab_0 + - colorama=0.4.6=pyhd8ed1ab_0 + - comm=0.2.2=pyhd8ed1ab_1 + - compilers=1.7.0=ha770c72_0 + - contourpy=1.2.0=py311h9547e67_0 + - cosmosis-build-standard-library=1.2=py311h8783aee_3 + - cryptography=41.0.7=py311hcb13ee4_1 + - curl=8.1.2=h409715c_0 + - cxx-compiler=1.7.0=h00ab1b0_0 + - cycler=0.12.1=pyhd8ed1ab_0 + - debugpy=1.8.5=py311hf86e51f_0 + - defusedxml=0.7.1=pyhd3eb1b0_0 + - dill=0.3.7=py311h06a4308_0 + - docutils=0.21.2=py311h06a4308_0 + - dulwich=0.21.6=py311h459d7ec_2 + - dynesty=2.1.2=pyhd8ed1ab_0 + - emcee=3.1.4=pyhd8ed1ab_0 + - exceptiongroup=1.2.0=pyhd8ed1ab_0 + - executing=2.2.0=pyhd8ed1ab_0 + - expat=2.5.0=hcb278e6_1 + - fast-pt=3.1.1=pyhd8ed1ab_0 + - fftw=3.3.10=nompi_hc118613_108 + - font-ttf-dejavu-sans-mono=2.37=hab24e00_0 + - font-ttf-inconsolata=3.000=h77eed37_0 + - font-ttf-source-code-pro=2.038=h77eed37_0 + - font-ttf-ubuntu=0.83=h77eed37_1 + - fontconfig=2.14.2=h14ed4e7_0 + - fonts-conda-ecosystem=1=0 + - fonts-conda-forge=1=0 + - fonttools=4.46.0=py311h459d7ec_0 + - fortran-compiler=1.7.0=heb67821_0 + - freetype=2.12.1=h267a509_2 + - fribidi=1.0.10=h36c2ea0_0 + - future=0.18.3=pyhd8ed1ab_0 + - gcc=12.3.0=h8d2909c_2 + - gcc_impl_linux-64=12.3.0=he2b93b0_3 + - gcc_linux-64=12.3.0=h76fc315_2 + - gdk-pixbuf=2.42.10=h05c8ddd_0 + - gettext=0.21.1=h27087fc_0 + - gfortran=12.3.0=h499e0f7_2 + - gfortran_impl_linux-64=12.3.0=hfcedea8_3 + - gfortran_linux-64=12.3.0=h7fe76b4_2 + - giflib=5.2.2=hd590300_0 + - git=2.42.0=pl5321h86e50cf_0 + - gmp=6.3.0=h59595ed_0 + - gmpy2=2.1.2=py311h6a5fa03_1 + - graphite2=1.3.13=h58526e2_1001 + - graphviz=8.0.3=h2e5815a_0 + - gtk2=2.24.33=h90689f9_2 + - gts=0.7.6=h977cf35_4 + - gxx=12.3.0=h8d2909c_2 + - gxx_impl_linux-64=12.3.0=he2b93b0_3 + - gxx_linux-64=12.3.0=h8a814eb_2 + - harfbuzz=6.0.0=h8e241bc_0 + - icu=70.1=h27087fc_0 + - idna=3.6=pyhd8ed1ab_0 + - imagesize=1.4.1=py311h06a4308_0 + - importlib-metadata=7.0.0=pyha770c72_0 + - importlib-resources=6.1.1=pyhd8ed1ab_0 + - importlib_resources=6.1.1=pyhd8ed1ab_0 + - iniconfig=2.0.0=pyhd8ed1ab_0 + - ipykernel=6.29.5=pyh3099207_0 + - ipython_pygments_lexers=1.1.1=pyhd8ed1ab_0 + - jedi=0.19.2=pyhd8ed1ab_1 + - joblib=1.3.2=pyhd8ed1ab_0 + - jpeg=9e=h0b41bf4_3 + - jsonschema=4.23.0=py311h06a4308_0 + - jupyter_client=8.6.3=pyhd8ed1ab_1 + - jupyter_core=5.8.1=pyh31011fe_0 + - jupyterlab_pygments=0.3.0=py311h06a4308_0 + - kernel-headers_linux-64=4.18.0=he073ed8_0 + - keyutils=1.6.1=h166bdaf_0 + - kiwisolver=1.4.5=py311h9547e67_1 + - krb5=1.20.1=h81ceb04_0 + - lcms2=2.15=hfd0df8a_0 + - ld_impl_linux-64=2.40=h41732ed_0 + - lerc=4.0.0=h27087fc_0 + - libblas=3.9.0=20_linux64_openblas + - libbrotlicommon=1.1.0=hd590300_1 + - libbrotlidec=1.1.0=hd590300_1 + - libbrotlienc=1.1.0=hd590300_1 + - libcblas=3.9.0=20_linux64_openblas + - libcurl=8.1.2=h409715c_0 + - libdeflate=1.17=h0b41bf4_0 + - libedit=3.1.20191231=he28a2e2_2 + - libev=4.33=h516909a_1 + - libexpat=2.5.0=hcb278e6_1 + - libffi=3.4.2=h7f98852_5 + - libgcc-devel_linux-64=12.3.0=h8bca6fd_103 + - libgcc-ng=13.2.0=h807b86a_3 + - libgd=2.3.3=h5aea950_4 + - libgfortran-ng=13.2.0=h69a702a_3 + - libgfortran5=13.2.0=ha4646dd_3 + - libglib=2.78.1=hebfc3b9_0 + - libgomp=13.2.0=h807b86a_3 + - libiconv=1.17=h166bdaf_0 + - liblapack=3.9.0=20_linux64_openblas + - libllvm14=14.0.6=hecde1de_4 + - libnghttp2=1.58.0=h47da74e_0 + - libnsl=2.0.1=hd590300_0 + - libopenblas=0.3.25=pthreads_h413a1c8_0 + - libopenlibm4=0.8.1=hd590300_1 + - libpng=1.6.39=h753d276_0 + - librsvg=2.54.4=h7abd40a_0 + - libsanitizer=12.3.0=h0f45ef3_3 + - libsodium=1.0.18=h36c2ea0_1 + - libsqlite=3.44.2=h2797004_0 + - libssh2=1.11.0=h0841786_0 + - libstdcxx-devel_linux-64=12.3.0=h8bca6fd_103 + - libstdcxx-ng=13.2.0=h7e041cc_3 + - libtiff=4.5.0=h6adf6a1_2 + - libtool=2.4.7=he02047a_1 + - libuuid=2.38.1=h0b41bf4_0 + - libwebp=1.3.2=h11a3e52_0 + - libwebp-base=1.3.2=hd590300_0 + - libxcb=1.13=h7f98852_1004 + - libxml2=2.10.3=hca2bb57_4 + - libzlib=1.2.13=hd590300_5 + - llvm-openmp=8.0.1=hc9558a2_0 + - llvmlite=0.43.0=py311h6a678d5_0 + - lsstdesc.coord=1.3.0=py311h9547e67_1 + - make=4.3=hd18ef5c_1 + - matplotlib-inline=0.1.7=pyhd8ed1ab_1 + - minuit2_standalone=6.22.06=h9c3ff4c_100 + - mistune=3.1.2=py311h06a4308_0 + - mock=5.1.0=pyhd8ed1ab_0 + - mpc=1.3.1=hfe3b2da_0 + - mpfr=4.2.1=h9458935_0 + - mpi=1.0=openmpi + - mpi4py=3.1.5=py311h4267d7f_0 + - mpmath=1.3.0=pyhd8ed1ab_0 + - munkres=1.1.4=pyh9f0ad1d_0 + - nbclient=0.10.2=py311h06a4308_0 + - nbconvert=7.16.6=py311h06a4308_0 + - nbconvert-core=7.16.6=py311h06a4308_0 + - nbconvert-pandoc=7.16.6=py311h06a4308_0 + - nbformat=5.10.4=py311h06a4308_0 + - ncurses=6.4=h59595ed_2 + - nest-asyncio=1.6.0=pyhd8ed1ab_1 + - numba=0.60.0=py311h4bc866e_0 + - numexpr=2.10.1=py311hd28fd6d_0 + - openjpeg=2.5.0=hfec8fc6_2 + - openlibm=0.8.1=hd590300_1 + - openmp=8.0.1=0 + - openmpi=4.1.6=hc5af2df_101 + - openssl=3.3.1=hb9d3cd8_3 + - packaging=23.2=pyhd8ed1ab_0 + - pandas=2.3.2=py311h277b779_0 + - pango=1.50.14=hd33c08f_0 + - parso=0.8.4=pyhd8ed1ab_1 + - patsy=0.5.4=pyhd8ed1ab_0 + - pcre2=10.40=hc3806b6_0 + - perl=5.32.1=4_hd590300_perl5 + - pexpect=4.9.0=pyhd8ed1ab_1 + - pickleshare=0.7.5=pyhd8ed1ab_1004 + - pip=23.3.1=pyhd8ed1ab_0 + - pixman=0.42.2=h59595ed_0 + - platformdirs=4.1.0=pyhd8ed1ab_0 + - pluggy=1.3.0=pyhd8ed1ab_0 + - pooch=1.8.0=pyhd8ed1ab_0 + - psutil=6.0.0=py311h331c9d8_0 + - pthread-stubs=0.4=h36c2ea0_1001 + - ptyprocess=0.7.0=pyhd8ed1ab_1 + - pure_eval=0.2.3=pyhd8ed1ab_1 + - pybind11=2.11.1=py311h9547e67_2 + - pybind11-global=2.11.1=py311h9547e67_2 + - pyccl=3.0.2=py311h671ba89_0 + - pycparser=2.21=pyhd8ed1ab_0 + - pygraphviz=1.10=py311h15e4e5d_4 + - pyopenssl=23.3.0=pyhd8ed1ab_0 + - pyparsing=3.1.1=pyhd8ed1ab_0 + - pysocks=1.7.1=pyha2e5f31_6 + - pytest=7.4.3=pyhd8ed1ab_0 + - python=3.11.6=hab00c5b_0_cpython + - python-dateutil=2.8.2=pyhd8ed1ab_0 + - python-fastjsonschema=2.20.0=py311h06a4308_0 + - python-tzdata=2023.3=pyhd8ed1ab_0 + - python_abi=3.11=4_cp311 + - pytz=2023.3.post1=pyhd8ed1ab_0 + - pyyaml=6.0.1=py311h459d7ec_1 + - r=4.2=r42hd8ed1ab_1007 + - r-base=4.2.3=ha7d60f8_0 + - r-boot=1.3_28.1=r42hc72bb7e_1 + - r-class=7.3_22=r42h57805ef_1 + - r-cluster=2.1.6=r42h61816a4_0 + - r-codetools=0.2_19=r42hc72bb7e_1 + - r-foreign=0.8_86=r42h57805ef_0 + - r-kernsmooth=2.23_22=r42h13b3f57_0 + - r-lattice=0.22_5=r42h57805ef_0 + - r-mass=7.3_60=r42h57805ef_1 + - r-matrix=1.6_4=r42h316c678_0 + - r-mgcv=1.9_0=r42h316c678_0 + - r-nlme=3.1_164=r42h61816a4_0 + - r-nnet=7.3_19=r42h57805ef_1 + - r-recommended=4.2=r42hd8ed1ab_1006 + - r-remotes=2.4.2.1=r42hc72bb7e_0 + - r-rpart=4.1.21=r42h57805ef_0 + - r-spatial=7.3_17=r42h57805ef_0 + - r-survival=3.5_7=r42h57805ef_0 + - readline=8.2=h8228510_1 + - requests=2.31.0=pyhd8ed1ab_0 + - rpds-py=0.22.3=py311h4aa5aa6_0 + - sacc=0.16=pyhd8ed1ab_2 + - scikit-learn=1.7.1=py311hc74f9fe_0 + - seaborn=0.13.0=hd8ed1ab_0 + - seaborn-base=0.13.0=pyhd8ed1ab_0 + - sed=4.8=he412f7d_0 + - setuptools=68.2.2=pyhd8ed1ab_0 + - six=1.16.0=pyh6c4a22f_0 + - snowballstemmer=2.2.0=pyhd3eb1b0_0 + - sphinxcontrib-applehelp=2.0.0=pyhd3eb1b0_1 + - sphinxcontrib-devhelp=2.0.0=pyhd3eb1b0_0 + - sphinxcontrib-htmlhelp=2.1.0=pyhd3eb1b0_0 + - sphinxcontrib-jsmath=1.0.1=pyhd3eb1b0_0 + - sphinxcontrib-qthelp=2.0.0=pyhd3eb1b0_1 + - sphinxcontrib-serializinghtml=2.0.0=pyhd3eb1b0_0 + - stack_data=0.6.3=pyhd8ed1ab_1 + - statsmodels=0.14.0=py311h1f0f07a_2 + - sympy=1.12=pypyh9d50eac_103 + - sysroot_linux-64=2.28=he073ed8_0 + - threadpoolctl=3.2.0=pyha21a80b_0 + - tinycss2=1.4.0=py311h06a4308_0 + - tk=8.6.13=noxft_h4845f30_101 + - tktable=2.10=h0c5db8f_5 + - tomli=2.0.1=pyhd8ed1ab_0 + - tqdm=4.66.1=pyhd8ed1ab_0 + - traitlets=5.14.3=pyhd8ed1ab_1 + - tree=2.1.3=h4bc722e_0 + - tzdata=2023c=h71feb2d_0 + - unicodedata2=15.1.0=py311h459d7ec_0 + - urllib3=2.1.0=pyhd8ed1ab_0 + - wcwidth=0.2.13=pyhd8ed1ab_1 + - wheel=0.42.0=pyhd8ed1ab_0 + - xorg-kbproto=1.0.7=h7f98852_1002 + - xorg-libice=1.0.10=h7f98852_0 + - xorg-libsm=1.2.3=hd9c2040_1000 + - xorg-libx11=1.8.4=h0b41bf4_0 + - xorg-libxau=1.0.11=hd590300_0 + - xorg-libxdmcp=1.1.3=h7f98852_0 + - xorg-libxext=1.3.4=h0b41bf4_2 + - xorg-libxrender=0.9.10=h7f98852_1003 + - xorg-libxt=1.3.0=hd590300_0 + - xorg-renderproto=0.11.1=h7f98852_1002 + - xorg-xextproto=7.3.0=h0b41bf4_1003 + - xorg-xproto=7.0.31=h7f98852_1007 + - xz=5.2.6=h166bdaf_0 + - yaml=0.2.5=h7f98852_2 + - zeromq=4.3.5=h59595ed_1 + - zeus-mcmc=2.5.4=pyhd8ed1ab_0 + - zipp=3.17.0=pyhd8ed1ab_0 + - zlib=1.2.13=hd590300_5 + - zstd=1.5.5=hfc55251_0 + - pip: + - absl-py==2.0.0 + - alabaster==1.0.0 + - annotated-types==0.6.0 + - argparse==1.4.0 + - astropandas==1.2.2 + - astropy==7.1.0 + - astropy-iers-data==0.2025.9.8.0.36.17 + - astunparse==1.6.3 + - babel==2.17.0 + - beautifulsoup4==4.13.3 + - cached-property==1.5.2 + - cachetools==5.3.2 + - cfgv==3.4.0 + - chainconsumer==0.34.0 + - click==8.1.7 + - cloudpickle==3.0.0 + - colossus==1.3.6 + - corner==2.2.2 + - cosmopower==0.1.3 + - cosmosis==3.23 + - coverage==7.6.1 + - cryptograpy==0.0.0 + - cython==0.29.33 + - d2to1==0.2.12.post1 + - dark-emulator==1.1.2 + - decorator==5.1.1 + - deprecation==2.1.0 + - distlib==0.3.9 + - dm-tree==0.1.8 + - fastjsonschema==2.21.1 + - filelock==3.17.0 + - fitsio==1.2.6 + - flatbuffers==23.5.26 + - furo==2024.8.6 + - galselect @ git+https://github.com/jlvdb/galselect.git + - gast==0.4.0 + - george==0.4.3 + - google-auth==2.24.0 + - google-auth-oauthlib==1.0.0 + - google-pasta==0.2.0 + - grpcio==1.59.3 + - gsl==0.0.3 + - h5py==3.14.0 + - halomod @ git+https://github.com/halomod/halomod.git@main + - hankel==1.2.2 + - healpy==1.18.1 + - hmf==3.5.2 + - identify==2.6.7 + - ipython==8.32.0 + - jax==0.4.21 + - jinja2==3.1.5 + - jsonschema-specifications==2024.10.1 + - jupyter-core==5.7.2 + - keras==2.13.1 + - legacy-blinding==0.0.0 + - levin==0.0.1 + - libclang==16.0.6 + - markdown==3.5.1 + - markdown-it-py==3.0.0 + - markupsafe==2.1.3 + - matplotlib==3.10.6 + - mdurl==0.1.2 + - ml-dtypes==0.3.1 + - nautilus-sampler==1.0.5 + - nbsphinx==0.9.6 + - nodeenv==1.9.1 + - numpy==2.2.0 + - numpydoc==1.8.0 + - oauthlib==3.2.2 + - onepower==0.7.2 + - opt-einsum==3.3.0 + - palettable==3.3.3 + - pandoc==2.4 + - pandocfilters==1.5.1 + - pillow==10.1.0 + - plumbum==1.9.0 + - ply==3.11 + - pre-commit==4.1.0 + - prompt-toolkit==3.0.50 + - protobuf==4.25.1 + - py-bobyqa==1.5.0 + - pyasn1==0.5.1 + - pyasn1-modules==0.3.0 + - pydantic==2.5.2 + - pydantic-core==2.14.5 + - pyerfa==2.0.1.5 + - pygments==2.17.2 + - pytest-cov==5.0.0 + - pytest-plt==1.1.1 + - pytest-runner==6.0.1 + - pyzmq==26.2.1 + - referencing==0.36.2 + - requests-oauthlib==1.3.1 + - rich==13.7.0 + - roman-numerals-py==3.0.0 + - rsa==4.9 + - scipy==1.13.1 + - setuptools-scm==8.1.0 + - soupsieve==2.6 + - sphinx==8.2.0 + - sphinx-basic-ng==1.0.0b2 + - sphinx-rtd-theme==3.0.2 + - sphinxcontrib-jquery==4.1 + - tabeval==1.5 + - tabulate==0.9.0 + - tensorboard==2.13.0 + - tensorboard-data-server==0.7.2 + - tensorboard-plugin-wit==1.8.1 + - tensorflow==2.13.0 + - tensorflow-estimator==2.13.0 + - tensorflow-io-gcs-filesystem==0.32.0 + - tensorflow-probability==0.19.0 + - termcolor==2.4.0 + - toml==0.10.2 + - tornado==6.4.2 + - treecorr==5.1.2 + - typing-extensions==4.5.0 + - virtualenv==20.29.2 + - webencodings==0.5.1 + - werkzeug==3.0.1 + - wget==3.2 + - wrapt==1.14.1 +prefix: /net/home/fohlen13/dvornik/anaconda3/envs/cosmopipe From 05caed45f485f0fa522a28bd9e52024cc27d0e5f Mon Sep 17 00:00:00 2001 From: andrejdvornik Date: Mon, 23 Feb 2026 13:19:47 +0000 Subject: [PATCH 30/37] Added an option to specify limits by hand --- scripts/cosmosis_constructor_3x2pt.sh | 62 ++++++++++++++++++++++++--- 1 file changed, 57 insertions(+), 5 deletions(-) diff --git a/scripts/cosmosis_constructor_3x2pt.sh b/scripts/cosmosis_constructor_3x2pt.sh index 621a5333..d583ca5d 100644 --- a/scripts/cosmosis_constructor_3x2pt.sh +++ b/scripts/cosmosis_constructor_3x2pt.sh @@ -43,8 +43,8 @@ zmax_def = 1.2 nmass_def = 100 ; 200 logmassmin_def = 9.0 logmassmax_def = 18.0 -nonlinear_mode = bnl -hmcode_ingredients = fit +nonlinear_mode_ = bnl +hmcode_ingredients_ = fit SAMPLER_NAME = @BV:SAMPLER@ RUN_NAME = %(SAMPLER_NAME)s_%(blind)s${CHAINSUFFIX} @@ -1275,6 +1275,8 @@ do z_maxs="" suffix=`seq -s ' ' ${NSMFLENSBINS}` file1="@RUNROOT@/@STORAGEPATH@/@DATABLOCK@/smf_lens_cats_metadata/stats_LB1.txt" + if [ -f ${file1} ] + then slice=`grep '^slice_in' ${file1} | awk '{printf $2}'` if [ "${slice}" == "obs" ] then @@ -1301,6 +1303,17 @@ do else _message "Got wrong or no information about slicing of the lens sample.\n" #exit 1 + fi + else + _message "No SMF lens catalog metadata found, setting default CSMF parameters from saved variables.\n" + if [ "${NSMFLENSBINS}" = "1" ] + then + z_mins=$(echo @BV:SMFLENSLIMSY@ | awk '{print $1}') + z_maxs=$(echo @BV:SMFLENSLIMSY@ | awk '{print $2}') + else + z_mins=$(echo @BV:SMFLENSLIMSY@ | awk '{for(i=1; i> @RUNROOT@/@STORAGEPATH@/@DATABLOCK@/cosmosis_inputs/@SURVEY@_CosmoPipe_constructed_other.ini <<- EOF - nonlinear_mode = %(nonlinear_mode)s + nonlinear_mode = %(nonlinear_mode_)s update_bnl = 10 poisson_type = constant point_mass = True dewiggle = True response = False output_suffix = onepower - hmcode_ingredients = %(hmcode_ingredients)s + hmcode_ingredients = %(hmcode_ingredients_)s log_mass_min = %(logmassmin_def)s log_mass_max = %(logmassmax_def)s From 40ccffc00ac8bdec9827ed7f11d30d293d8da1e4 Mon Sep 17 00:00:00 2001 From: andrejdvornik Date: Wed, 18 Mar 2026 13:46:34 +0000 Subject: [PATCH 31/37] Fix good_args --- scripts/run_measure_statistics_cats2stats.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/run_measure_statistics_cats2stats.py b/scripts/run_measure_statistics_cats2stats.py index b64bd9ec..6154365b 100644 --- a/scripts/run_measure_statistics_cats2stats.py +++ b/scripts/run_measure_statistics_cats2stats.py @@ -389,8 +389,8 @@ for i in range(len(ell)-1): filter[i]=h(theta_mid*arcmin2rad, ell[i], ell[i+1])*theta_mid*arcmin2rad*T(theta_mid, thetamin_apod, thetamax_apod, logwidth) N = np.log(ell[i+1]/ell[i]) - IntegralE=sum(filter[i]*gamma_t*delta_theta) - IntegralB=sum(filter[i]*gamma_x*delta_theta) + IntegralE=sum(filter[i]*gamma_t[good_args]*delta_theta) + IntegralB=sum(filter[i]*gamma_x[good_args]*delta_theta) CnE[i]=IntegralE*2*np.pi/N*arcmin2rad CnB[i]=IntegralB*2*np.pi/N*arcmin2rad @@ -410,7 +410,7 @@ for i in range(len(ell)-1): filter[i]=f(theta_mid*arcmin2rad, ell[i], ell[i+1])*T(theta_mid, thetamin_apod, thetamax_apod, logwidth) N = np.log(ell[i+1]/ell[i]) - Integral=sum(filter[i]*w_theta*delta_theta) + Integral=sum(filter[i]*w_theta[good_args]*delta_theta) Cnn[i]=Integral*2*np.pi/N*arcmin2rad CnnfileName=cfoldername+"/Cnn_"+outputfile+".asc" np.savetxt(CnnfileName,Cnn) From 8e41ed81560bf941b5b093732b6fa1ab13f16eb7 Mon Sep 17 00:00:00 2001 From: andrejdvornik Date: Thu, 19 Mar 2026 12:57:13 +0000 Subject: [PATCH 32/37] Add nz shifts and biases for lenses and smf --- man/add_values_priors.man.sh | 34 ++++++++- scripts/add_values_priors.sh | 102 +++++++++++++++++++------ scripts/cosmosis_constructor_3x2pt.sh | 105 ++++++++++++++++++++++++-- 3 files changed, 209 insertions(+), 32 deletions(-) diff --git a/man/add_values_priors.man.sh b/man/add_values_priors.man.sh index e9969850..65aa0b79 100644 --- a/man/add_values_priors.man.sh +++ b/man/add_values_priors.man.sh @@ -42,21 +42,49 @@ set -e # Input variables {{{ function _inp_var { #Variable inputs (leave blank if none) - echo BLU BV:NLENSBINS BV:PRIORSINI BV:TOMOLIMS BV:VALUESINI DATABLOCK DEF RED RUNROOT STORAGEPATH SURVEY + echo BLU BV:NLENSBINS BV:NSMFLENSBINS BV:MODES BV:PRIORSINI BV:TOMOLIMS BV:VALUESINI DATABLOCK DEF RED RUNROOT STORAGEPATH SURVEY } #}}} # Input data {{{ function _inp_data { #Data inputs (leave blank if none) - echo nzbias_uncorr + MODES=`_parse_blockvars @BV:MODES@` + input="" + if [[ .*\ $MODES\ .* =~ " EE " ]] || [[ .*\ $MODES\ .* =~ " NE " ]] + then + input="${input} nzbias_uncorr" + fi + if [[ .*\ $MODES\ .* =~ " NE " ]] || [[ .*\ $MODES\ .* =~ " NN " ]] + then + input="${input} nzbias_uncorr_lens" + fi + if [[ .*\ $MODES\ .* =~ " OBS " ]] + then + input="${input} nzbias_uncorr_obs" + fi + echo "${input}" } #}}} # Output data {{{ function _outputs { #Data outputs (leave blank if none) - echo nzbias_uncorr + MODES=`_parse_blockvars @BV:MODES@` + output="" + if [[ .*\ $MODES\ .* =~ " EE " ]] || [[ .*\ $MODES\ .* =~ " NE " ]] + then + output="${output} nzbias_uncorr" + fi + if [[ .*\ $MODES\ .* =~ " NE " ]] || [[ .*\ $MODES\ .* =~ " NN " ]] + then + output="${output} nzbias_uncorr_lens" + fi + if [[ .*\ $MODES\ .* =~ " OBS " ]] + then + output="${output} nzbias_uncorr_obs" + fi + echo "${output}" } #}}} diff --git a/scripts/add_values_priors.sh b/scripts/add_values_priors.sh index 7642d776..34a8e643 100644 --- a/scripts/add_values_priors.sh +++ b/scripts/add_values_priors.sh @@ -10,7 +10,8 @@ #Number of tomographic bins NTOMO=`echo @BV:TOMOLIMS@ | awk '{print NF-1}'` NLENS=`echo @BV:NLENSBINS@` - +NOBS=`echo @BV:NSMFLENSBINS@` +MODES="@BV:MODES@" @@ -38,28 +39,83 @@ cp @BV:VALUESINI@ @RUNROOT@/@STORAGEPATH@/@DATABLOCK@/cosmosis_inputs/@SURVEY@_v cp @BV:PRIORSINI@ @RUNROOT@/@STORAGEPATH@/@DATABLOCK@/cosmosis_inputs/@SURVEY@_priors.ini -#Update the values with the uncorrelated Dz priors {{{ -echo "[nofz_shifts]" >> @RUNROOT@/@STORAGEPATH@/@DATABLOCK@/cosmosis_inputs/@SURVEY@_values.ini -#Add the uncorrelated tomographic bin shifts -#Note: we multiply the bin shift by -1 because cosmosis defines the shifts with a relative minus sign! -tomoval_all=`cat @DB:nzbias_uncorr@` -for tomo in `seq ${NTOMO}` -do - tomoval=`echo ${tomoval_all} | awk -v n=${tomo} '{print -1*$n}'` - tomolo=`echo $tomoval | awk '{print $1-5.00}'` - tomohi=`echo $tomoval | awk '{print $1+5.00}'` - echo "uncorr_bias_${tomo} = ${tomolo} ${tomoval} ${tomohi} " >> @RUNROOT@/@STORAGEPATH@/@DATABLOCK@/cosmosis_inputs/@SURVEY@_values.ini -done -#}}} -#Update the priors with the uncorrelated Dz priors {{{ -echo "[nofz_shifts]" >> @RUNROOT@/@STORAGEPATH@/@DATABLOCK@/cosmosis_inputs/@SURVEY@_priors.ini -#Add the uncorrelated tomographic bin shifts -for tomo in `seq ${NTOMO}` -do - tomoval=`echo ${tomoval_all} | awk -v n=${tomo} '{print -1*$n}'` - echo "uncorr_bias_${tomo} = gaussian ${tomoval} 1.0 " >> @RUNROOT@/@STORAGEPATH@/@DATABLOCK@/cosmosis_inputs/@SURVEY@_priors.ini -done -#}}} +if [[ .*\ $MODES\ .* =~ " EE " ]] || [[ .*\ $MODES\ .* =~ " NE " ]] +then + #Update the values with the uncorrelated Dz priors {{{ + echo "[nofz_shifts]" >> @RUNROOT@/@STORAGEPATH@/@DATABLOCK@/cosmosis_inputs/@SURVEY@_values.ini + #Add the uncorrelated tomographic bin shifts + #Note: we multiply the bin shift by -1 because cosmosis defines the shifts with a relative minus sign! + tomoval_all=`cat @DB:nzbias_uncorr@` + for tomo in `seq ${NTOMO}` + do + tomoval=`echo ${tomoval_all} | awk -v n=${tomo} '{print -1*$n}'` + tomolo=`echo $tomoval | awk '{print $1-5.00}'` + tomohi=`echo $tomoval | awk '{print $1+5.00}'` + echo "uncorr_bias_${tomo} = ${tomolo} ${tomoval} ${tomohi} " >> @RUNROOT@/@STORAGEPATH@/@DATABLOCK@/cosmosis_inputs/@SURVEY@_values.ini + done + #}}} + #Update the priors with the uncorrelated Dz priors {{{ + echo "[nofz_shifts]" >> @RUNROOT@/@STORAGEPATH@/@DATABLOCK@/cosmosis_inputs/@SURVEY@_priors.ini + #Add the uncorrelated tomographic bin shifts + for tomo in `seq ${NTOMO}` + do + tomoval=`echo ${tomoval_all} | awk -v n=${tomo} '{print -1*$n}'` + echo "uncorr_bias_${tomo} = gaussian ${tomoval} 1.0 " >> @RUNROOT@/@STORAGEPATH@/@DATABLOCK@/cosmosis_inputs/@SURVEY@_priors.ini + done + #}}} +fi + +if [[ .*\ $MODES\ .* =~ " NE " ]] || [[ .*\ $MODES\ .* =~ " NN " ]] +then + #Update the values with the uncorrelated Dz priors {{{ + echo "[nofz_shifts_lens]" >> @RUNROOT@/@STORAGEPATH@/@DATABLOCK@/cosmosis_inputs/@SURVEY@_values.ini + #Add the uncorrelated tomographic bin shifts + #Note: we multiply the bin shift by -1 because cosmosis defines the shifts with a relative minus sign! + tomoval_all=`cat @DB:nzbias_uncorr_lens@` + for tomo in `seq ${NLENS}` + do + tomoval=`echo ${tomoval_all} | awk -v n=${tomo} '{print -1*$n}'` + tomolo=`echo $tomoval | awk '{print $1-5.00}'` + tomohi=`echo $tomoval | awk '{print $1+5.00}'` + echo "uncorr_bias_${tomo} = ${tomolo} ${tomoval} ${tomohi} " >> @RUNROOT@/@STORAGEPATH@/@DATABLOCK@/cosmosis_inputs/@SURVEY@_values.ini + done + #}}} + #Update the priors with the uncorrelated Dz priors {{{ + echo "[nofz_shifts_lens]" >> @RUNROOT@/@STORAGEPATH@/@DATABLOCK@/cosmosis_inputs/@SURVEY@_priors.ini + #Add the uncorrelated tomographic bin shifts + for tomo in `seq ${NLENS}` + do + tomoval=`echo ${tomoval_all} | awk -v n=${tomo} '{print -1*$n}'` + echo "uncorr_bias_${tomo} = gaussian ${tomoval} 1.0 " >> @RUNROOT@/@STORAGEPATH@/@DATABLOCK@/cosmosis_inputs/@SURVEY@_priors.ini + done + #}}} +fi + +if [[ .*\ $MODES\ .* =~ " OBS " ]] +then + #Update the values with the uncorrelated Dz priors {{{ + echo "[nofz_shifts_obs]" >> @RUNROOT@/@STORAGEPATH@/@DATABLOCK@/cosmosis_inputs/@SURVEY@_values.ini + #Add the uncorrelated tomographic bin shifts + #Note: we multiply the bin shift by -1 because cosmosis defines the shifts with a relative minus sign! + tomoval_all=`cat @DB:nzbias_uncorr_obs@` + for tomo in `seq ${NOBS}` + do + tomoval=`echo ${tomoval_all} | awk -v n=${tomo} '{print -1*$n}'` + tomolo=`echo $tomoval | awk '{print $1-5.00}'` + tomohi=`echo $tomoval | awk '{print $1+5.00}'` + echo "uncorr_bias_${tomo} = ${tomolo} ${tomoval} ${tomohi} " >> @RUNROOT@/@STORAGEPATH@/@DATABLOCK@/cosmosis_inputs/@SURVEY@_values.ini + done + #}}} + #Update the priors with the uncorrelated Dz priors {{{ + echo "[nofz_shifts_obs]" >> @RUNROOT@/@STORAGEPATH@/@DATABLOCK@/cosmosis_inputs/@SURVEY@_priors.ini + #Add the uncorrelated tomographic bin shifts + for tomo in `seq ${NOBS}` + do + tomoval=`echo ${tomoval_all} | awk -v n=${tomo} '{print -1*$n}'` + echo "uncorr_bias_${tomo} = gaussian ${tomoval} 1.0 " >> @RUNROOT@/@STORAGEPATH@/@DATABLOCK@/cosmosis_inputs/@SURVEY@_priors.ini + done + #}}} +fi _write_datablock "cosmosis_inputs" "@SURVEY@_values.ini @SURVEY@_priors.ini" #}}} diff --git a/scripts/cosmosis_constructor_3x2pt.sh b/scripts/cosmosis_constructor_3x2pt.sh index d583ca5d..ce9c7efb 100644 --- a/scripts/cosmosis_constructor_3x2pt.sh +++ b/scripts/cosmosis_constructor_3x2pt.sh @@ -875,21 +875,55 @@ fi #Prepare the pipeline section {{{ extraparams="cosmological_parameters/S_8 cosmological_parameters/sigma_8 cosmological_parameters/A_s cosmological_parameters/omega_m cosmological_parameters/omega_nu cosmological_parameters/omega_lambda" -#Add source nz shift values to outputs {{{ shifts_source="" +shifts_lens="" +shifts_obs="" +if [[ .*\ $MODES\ .* =~ " EE " ]] || [[ .*\ $MODES\ .* =~ " NE " ]] +then + #Add source nz shift values to outputs {{{ for i in `seq ${NTOMO}` do shifts_source="${shifts_source} nofz_shifts/bias_${i}" done #}}} - +fi +if [[ .*\ $MODES\ .* =~ " NE " ]] || [[ .*\ $MODES\ .* =~ " NN " ]] +then #Add lens nz shift values to outputs {{{ -shifts_lens="" for i in `seq ${NLENSBINS}` do shifts_lens="${shifts_lens} nofz_shifts_lens/bias_${i}" done #}}} +fi +if [[ .*\ $MODES\ .* =~ " OBS " ]] +then + #Add obs nz shift values to outputs {{{ + for i in `seq ${NSMFLENSBINS}` + do + shifts_obs="${shifts_obs} nofz_shifts_obs/bias_${i}" + done + #}}} +fi + + +photo_z_bias="" +corr_dz_priors="" +if [[ .*\ $MODES\ .* =~ " EE " ]] || [[ .*\ $MODES\ .* =~ " NE " ]] +then + photo_z_bias="${photo_z_bias} source_photoz_bias" + corr_dz_priors="${corr_dz_priors} correlated_dz_priors" +fi +if [[ .*\ $MODES\ .* =~ " NE " ]] || [[ .*\ $MODES\ .* =~ " NN " ]] +then + photo_z_bias="${photo_z_bias} lens_photoz_bias" + corr_dz_priors="${corr_dz_priors} correlated_dz_priors_lens" +fi +if [[ .*\ $MODES\ .* =~ " OBS " ]] +then + photo_z_bias="${photo_z_bias} obs_photoz_bias" + corr_dz_priors="${corr_dz_priors} correlated_dz_priors_obs" +fi #Add the values information #{{{ @@ -928,7 +962,7 @@ then exit 1 fi - COSMOSIS_PIPELINE="sample_S8 correlated_dz_priors load_nz_sacc consistency ${boltzmann_pipeline} extrapolate onepower ${iamodel_pipeline} source_photoz_bias ${twopt_modules}" + COSMOSIS_PIPELINE="sample_S8 ${corr_dz_priors} load_nz_sacc consistency ${boltzmann_pipeline} extrapolate onepower ${iamodel_pipeline} ${photo_z_bias} ${twopt_modules}" elif [ "@BV:COSMOSIS_PIPELINE@" == "lin_bias" ] then @@ -961,7 +995,7 @@ then exit 1 fi - COSMOSIS_PIPELINE="sample_S8 correlated_dz_priors load_nz_sacc ${boltzmann_pipeline} extrapolate_power source_photoz_bias ${iamodel_pipeline} ${twopt_modules}" + COSMOSIS_PIPELINE="sample_S8 ${corr_dz_priors} load_nz_sacc ${boltzmann_pipeline} extrapolate_power ${photo_z_bias} ${iamodel_pipeline} ${twopt_modules}" else COSMOSIS_PIPELINE="@BV:COSMOSIS_PIPELINE@" fi @@ -1061,7 +1095,7 @@ cat >> @RUNROOT@/@STORAGEPATH@/@DATABLOCK@/cosmosis_inputs/@SURVEY@_CosmoPipe_co EOF fi cat >> @RUNROOT@/@STORAGEPATH@/@DATABLOCK@/cosmosis_inputs/@SURVEY@_CosmoPipe_constructed_pipe.ini <<- EOF -extra_output = ${extraparams} ${shifts} ${listparam} ${tpdparams} +extra_output = ${extraparams} ${shifts_source} ${shifts_lens} ${listparam} ${tpdparams} timing = F ; T debug = F fast_slow = ${fast_slow} @@ -1199,6 +1233,40 @@ do output_parameters = ${shifts_source} covariance = @DB:nzcov@ + EOF + ;; #}}} + "correlated_dz_priors_lens") #{{{ + shifts_lens="" + unc_shifts="" + for i in `seq ${NLENSBINS}` + do + shifts_lens="${shifts_lens} nofz_shifts_lens/bias_${i}" + unc_shifts="${unc_shifts} nofz_shifts_lens/uncorr_bias_${i}" + done + cat >> @RUNROOT@/@STORAGEPATH@/@DATABLOCK@/cosmosis_inputs/@SURVEY@_CosmoPipe_constructed_other.ini <<- EOF + [$module] + file = %(KCAP_PATH)s/utils/correlated_priors.py + uncorrelated_parameters = ${unc_shifts} + output_parameters = ${shifts_lens} + covariance = @DB:nzcov_lens@ + + EOF + ;; #}}} + "correlated_dz_priors_obs") #{{{ + shifts_obs="" + unc_shifts="" + for i in `seq ${NSMFLENSBINS}` + do + shifts_obs="${shifts_obs} nofz_shifts_obs/bias_${i}" + unc_shifts="${unc_shifts} nofz_shifts_obs/uncorr_bias_${i}" + done + cat >> @RUNROOT@/@STORAGEPATH@/@DATABLOCK@/cosmosis_inputs/@SURVEY@_CosmoPipe_constructed_other.ini <<- EOF + [$module] + file = %(KCAP_PATH)s/utils/correlated_priors.py + uncorrelated_parameters = ${unc_shifts} + output_parameters = ${shifts_obs} + covariance = @DB:nzcov_obs@ + EOF ;; #}}} "extrapolate") #{{{ @@ -1245,7 +1313,32 @@ do EOF ;; #}}} + "lens_photoz_bias") #{{{ + cat >> @RUNROOT@/@STORAGEPATH@/@DATABLOCK@/cosmosis_inputs/@SURVEY@_CosmoPipe_constructed_other.ini <<- EOF + [$module] + file = %(CSL_PATH)s/number_density/photoz_bias/photoz_bias.py + mode = additive + sample = nz_%(redshift_name_lens)s + bias_section = nofz_shifts_lens + interpolation = cubic + output_deltaz = T + output_section_name = delta_z_out_lens + + EOF + ;; #}}} + "obs_photoz_bias") #{{{ + cat >> @RUNROOT@/@STORAGEPATH@/@DATABLOCK@/cosmosis_inputs/@SURVEY@_CosmoPipe_constructed_other.ini <<- EOF + [$module] + file = %(CSL_PATH)s/number_density/photoz_bias/photoz_bias.py + mode = additive + sample = nz_%(redshift_name_obs)s + bias_section = nofz_shifts_obs + interpolation = cubic + output_deltaz = T + output_section_name = delta_z_out_obs + EOF + ;; #}}} "add_intrinsic") #{{{ if [[ .*\ $MODES\ .* =~ " EE " ]] then From 761392a793b4963f5df471964c6e4b4e913fd1f6 Mon Sep 17 00:00:00 2001 From: andrejdvornik Date: Thu, 19 Mar 2026 13:30:29 +0000 Subject: [PATCH 33/37] Force 2d array --- kcap/utils/correlated_priors.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kcap/utils/correlated_priors.py b/kcap/utils/correlated_priors.py index 2c16165f..b33a40a3 100644 --- a/kcap/utils/correlated_priors.py +++ b/kcap/utils/correlated_priors.py @@ -10,7 +10,7 @@ def setup(options): output_parameters = [p.strip().split("/") for p in output_parameters] covariance_file = options[option_section, "covariance"] - cov = np.loadtxt(covariance_file) + cov = np.atleast_2d(np.loadtxt(covariance_file)) L = np.linalg.cholesky(cov) return input_parameters, output_parameters, L From e3d164eb875aa4cd5f21b781c596bb1ac2dc7841 Mon Sep 17 00:00:00 2001 From: andrejdvornik Date: Thu, 19 Mar 2026 14:58:40 +0000 Subject: [PATCH 34/37] Fix CSL branch --- COSMOPIPE_MASTER_INSTALL.sh | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/COSMOPIPE_MASTER_INSTALL.sh b/COSMOPIPE_MASTER_INSTALL.sh index 60bd3a0f..33dd9eae 100644 --- a/COSMOPIPE_MASTER_INSTALL.sh +++ b/COSMOPIPE_MASTER_INSTALL.sh @@ -161,10 +161,7 @@ then then rm -fr cosmosis-standard-library fi - git clone --single-branch -b two-point-one-point https://github.com/andrejdvornik/cosmosis-standard-library.git >> gitclone_output.log 2>&1 - # The two-point-one-point branch has one-point function enabled and some further non-PR fixes to CAMB - # We will try to merge that with the oficial CSL, until then the rest of CSL is in sync! - #git clone https://github.com/joezuntz/cosmosis-standard-library.git >> gitclone_output.log 2>&1 + git clone https://github.com/joezuntz/cosmosis-standard-library.git >> gitclone_output.log 2>&1 _message "${BLU} - Done! ${DEF}\n" _message " >${RED} Installing cosmosis-standard-library ${DEF}" #Replace the cpdef instances with cdef in classy.pyx From 6a0d647866561bcc1493b560131315a073130cae Mon Sep 17 00:00:00 2001 From: andrejdvornik Date: Thu, 19 Mar 2026 15:00:10 +0000 Subject: [PATCH 35/37] More fix to CSL repo --- COSMOPIPE_MASTER_INSTALL.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/COSMOPIPE_MASTER_INSTALL.sh b/COSMOPIPE_MASTER_INSTALL.sh index 33dd9eae..b36cb515 100644 --- a/COSMOPIPE_MASTER_INSTALL.sh +++ b/COSMOPIPE_MASTER_INSTALL.sh @@ -161,7 +161,7 @@ then then rm -fr cosmosis-standard-library fi - git clone https://github.com/joezuntz/cosmosis-standard-library.git >> gitclone_output.log 2>&1 + git clone https://github.com/cosmosis-developers/cosmosis-standard-library.git >> gitclone_output.log 2>&1 _message "${BLU} - Done! ${DEF}\n" _message " >${RED} Installing cosmosis-standard-library ${DEF}" #Replace the cpdef instances with cdef in classy.pyx From 7132bbb9dc4215a12b4bf7108fa79d04cc1f1893 Mon Sep 17 00:00:00 2001 From: andrejdvornik Date: Thu, 19 Mar 2026 16:22:51 +0000 Subject: [PATCH 36/37] Add back OnePower as we need interface to it, that is in the repo --- COSMOPIPE_MASTER_INSTALL.sh | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/COSMOPIPE_MASTER_INSTALL.sh b/COSMOPIPE_MASTER_INSTALL.sh index b36cb515..55f72bc3 100644 --- a/COSMOPIPE_MASTER_INSTALL.sh +++ b/COSMOPIPE_MASTER_INSTALL.sh @@ -231,6 +231,17 @@ EOF _message "${BLU} - Done! ${DEF}\n" #}}} + #Clone the OnePower repository {{{ + _message " >${RED} Cloning the OnePower Git repository${DEF}" + #Clone the repository + if [ -d ${RUNROOT}/INSTALL/OnePower ] + then + rm -fr OnePower + fi + git clone https://github.com/KiDS-WL/onepower.git >> gitclone_output.log 2>&1 + _message "${BLU} - Done! ${DEF}\n" + #}}} + #Clone the Datavec Blinding repository {{{ _message " >${RED} Cloning the Datavector Blinding Git repository${DEF}" #Clone the repository From 4d33b6b453d737fce2cd8d20793c73bb272dca2b Mon Sep 17 00:00:00 2001 From: andrejdvornik Date: Fri, 20 Mar 2026 10:24:06 +0000 Subject: [PATCH 37/37] add dill (spice) --- environment_3x2pt.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/environment_3x2pt.yml b/environment_3x2pt.yml index b4b89f86..0da86516 100644 --- a/environment_3x2pt.yml +++ b/environment_3x2pt.yml @@ -311,6 +311,7 @@ dependencies: - dark-emulator==1.1.2 - decorator==5.1.1 - deprecation==2.1.0 + - dill==0.4.1 - distlib==0.3.9 - dm-tree==0.1.8 - fastjsonschema==2.21.1