From 46e30c4e10dbd0ea96b39de92f0d10b1e2e2cf71 Mon Sep 17 00:00:00 2001 From: Homer Neal Date: Fri, 23 Jan 2026 09:42:57 -0800 Subject: [PATCH 1/8] use mc instead of singularity to get pp_apdb yaml --- scripts/LSSTCam/submit_ap_daytime.sh | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/scripts/LSSTCam/submit_ap_daytime.sh b/scripts/LSSTCam/submit_ap_daytime.sh index d3aa3998..d08c3fa5 100755 --- a/scripts/LSSTCam/submit_ap_daytime.sh +++ b/scripts/LSSTCam/submit_ap_daytime.sh @@ -23,11 +23,15 @@ case "$TMP_APDB_REL" in esac # Copy APDB config from S3 using Singularity AWS CLI -singularity exec /sdf/sw/s3/aws-cli_latest.sif \ - aws --endpoint-url https://sdfembs3.sdf.slac.stanford.edu s3 \ - --profile embargo-s3 \ - cp s3://rubin-summit-users/apdb_config/cassandra/pp_apdb_lsstcam.yaml \ - "$TMP_APDB" +#singularity exec /sdf/sw/s3/aws-cli_latest.sif \ +# aws --endpoint-url https://sdfembs3.sdf.slac.stanford.edu s3 \ +# --profile embargo-s3 \ +# cp s3://rubin-summit-users/apdb_config/cassandra/pp_apdb_lsstcam.yaml \ +# "$TMP_APDB" + +echo "TMP_APDB = "$TMP_APDB +mc cp embargo/rubin-summit-users/apdb_config/cassandra/pp_apdb_lsstcam.yaml "$TMP_APDB" + # NOTE: # No cleanup of TMP_APDB here since the job is launched with nohup From 12854d8f5ec253225c99e27b3dde1e45243b31f1 Mon Sep 17 00:00:00 2001 From: Homer Neal Date: Sat, 24 Jan 2026 14:03:34 -0800 Subject: [PATCH 2/8] add qg option associateApdb:doRunForcedMeasurement=False --- scripts/LSSTCam/submit_ap_daytime.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/LSSTCam/submit_ap_daytime.sh b/scripts/LSSTCam/submit_ap_daytime.sh index d08c3fa5..d68ba1c1 100755 --- a/scripts/LSSTCam/submit_ap_daytime.sh +++ b/scripts/LSSTCam/submit_ap_daytime.sh @@ -63,7 +63,7 @@ BAD_DETECTORS_SQL="($(printf '%s,' $BAD_DETECTORS | sed 's/,$//'))" BLOCKS_SQL="($(printf "'%s'," $BLOCKS | sed 's/,$//'))" nohup bps submit "${AP_PIPE_DIR}/bps/LSSTCam/bps_Daytime.yaml" \ - --extra-qgraph-options "--skip-existing-in LSSTCam/prompt/output-${DATE} -c parameters:release_id=1 -c parameters:apdb_config=${TMP_APDB} --dataset-query-constraint template_coadd" \ + --extra-qgraph-options "--skip-existing-in LSSTCam/prompt/output-${DATE} -c parameters:release_id=1 -c parameters:apdb_config=${TMP_APDB} --dataset-query-constraint template_coadd -c associateApdb:doRunForcedMeasurement=False" \ --extra-run-quantum-options "--no-raise-on-partial-outputs" \ --input "LSSTCam/defaults,LSSTCam/templates,LSSTCam/prompt/output-${DATE}" \ --output "$OUTPUT_COLLECTION" \ From 4878fb30006bc978989d0fab94fcf486a2344b62 Mon Sep 17 00:00:00 2001 From: Homer Neal Date: Tue, 27 Jan 2026 12:16:17 -0800 Subject: [PATCH 3/8] remove unneeded already commented lines for cleanup --- scripts/LSSTCam/submit_ap_daytime.sh | 7 ------- 1 file changed, 7 deletions(-) diff --git a/scripts/LSSTCam/submit_ap_daytime.sh b/scripts/LSSTCam/submit_ap_daytime.sh index d68ba1c1..2cae40c0 100755 --- a/scripts/LSSTCam/submit_ap_daytime.sh +++ b/scripts/LSSTCam/submit_ap_daytime.sh @@ -23,13 +23,6 @@ case "$TMP_APDB_REL" in esac # Copy APDB config from S3 using Singularity AWS CLI -#singularity exec /sdf/sw/s3/aws-cli_latest.sif \ -# aws --endpoint-url https://sdfembs3.sdf.slac.stanford.edu s3 \ -# --profile embargo-s3 \ -# cp s3://rubin-summit-users/apdb_config/cassandra/pp_apdb_lsstcam.yaml \ -# "$TMP_APDB" - -echo "TMP_APDB = "$TMP_APDB mc cp embargo/rubin-summit-users/apdb_config/cassandra/pp_apdb_lsstcam.yaml "$TMP_APDB" From 3dc7078f8f1e93f434ee76ae0e4db57850e02218 Mon Sep 17 00:00:00 2001 From: Homer Neal Date: Wed, 28 Jan 2026 14:34:30 -0800 Subject: [PATCH 4/8] update skymap spec to lsst_cells_v2 --- scripts/LSSTCam/submit_ap_daytime.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/LSSTCam/submit_ap_daytime.sh b/scripts/LSSTCam/submit_ap_daytime.sh index 2cae40c0..b5f0dea1 100755 --- a/scripts/LSSTCam/submit_ap_daytime.sh +++ b/scripts/LSSTCam/submit_ap_daytime.sh @@ -61,7 +61,7 @@ nohup bps submit "${AP_PIPE_DIR}/bps/LSSTCam/bps_Daytime.yaml" \ --input "LSSTCam/defaults,LSSTCam/templates,LSSTCam/prompt/output-${DATE}" \ --output "$OUTPUT_COLLECTION" \ -d "instrument='$INSTRUMENT' \ - AND skymap='lsst_cells_v1' \ + AND skymap='lsst_cells_v2' \ AND detector NOT IN $BAD_DETECTORS_SQL \ AND day_obs=$DAY_OBS \ AND exposure.science_program IN $BLOCKS_SQL" \ From e9a59c3c47571136c955940467254276beabeac8 Mon Sep 17 00:00:00 2001 From: Homer Neal Date: Fri, 20 Feb 2026 03:15:56 -0800 Subject: [PATCH 5/8] data-query-constraint off --- scripts/LSSTCam/submit_ap_daytime.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/LSSTCam/submit_ap_daytime.sh b/scripts/LSSTCam/submit_ap_daytime.sh index b5f0dea1..37d9edd9 100755 --- a/scripts/LSSTCam/submit_ap_daytime.sh +++ b/scripts/LSSTCam/submit_ap_daytime.sh @@ -56,7 +56,7 @@ BAD_DETECTORS_SQL="($(printf '%s,' $BAD_DETECTORS | sed 's/,$//'))" BLOCKS_SQL="($(printf "'%s'," $BLOCKS | sed 's/,$//'))" nohup bps submit "${AP_PIPE_DIR}/bps/LSSTCam/bps_Daytime.yaml" \ - --extra-qgraph-options "--skip-existing-in LSSTCam/prompt/output-${DATE} -c parameters:release_id=1 -c parameters:apdb_config=${TMP_APDB} --dataset-query-constraint template_coadd -c associateApdb:doRunForcedMeasurement=False" \ + --extra-qgraph-options "--skip-existing-in LSSTCam/prompt/output-${DATE} -c parameters:release_id=1 -c parameters:apdb_config=${TMP_APDB} -c associateApdb:doRunForcedMeasurement=False --dataset-query-constraint off" \ --extra-run-quantum-options "--no-raise-on-partial-outputs" \ --input "LSSTCam/defaults,LSSTCam/templates,LSSTCam/prompt/output-${DATE}" \ --output "$OUTPUT_COLLECTION" \ From 20733561ecf3bc3deffdeb9fcd1159ff859f5edc Mon Sep 17 00:00:00 2001 From: Homer Neal Date: Fri, 27 Feb 2026 03:23:00 -0800 Subject: [PATCH 6/8] set nodeset name expand skip-existing-in to skip also those already processed by daily batch --- bps/LSSTCam/bps_Daytime.yaml | 2 ++ scripts/LSSTCam/submit_ap_daytime.sh | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/bps/LSSTCam/bps_Daytime.yaml b/bps/LSSTCam/bps_Daytime.yaml index 168453dd..96c26256 100644 --- a/bps/LSSTCam/bps_Daytime.yaml +++ b/bps/LSSTCam/bps_Daytime.yaml @@ -3,6 +3,8 @@ pipelineYaml: $AP_PIPE_DIR/pipelines/LSSTCam/ApPipe.yaml project: ApPipe campaign: AP-daytime +nodeset: daily + payload: # Use the prompt processing configuration for the embargo repo, which turns on compression. # This can be changed to 'embargo' once compression is turned on by default diff --git a/scripts/LSSTCam/submit_ap_daytime.sh b/scripts/LSSTCam/submit_ap_daytime.sh index 37d9edd9..c152b799 100755 --- a/scripts/LSSTCam/submit_ap_daytime.sh +++ b/scripts/LSSTCam/submit_ap_daytime.sh @@ -56,7 +56,7 @@ BAD_DETECTORS_SQL="($(printf '%s,' $BAD_DETECTORS | sed 's/,$//'))" BLOCKS_SQL="($(printf "'%s'," $BLOCKS | sed 's/,$//'))" nohup bps submit "${AP_PIPE_DIR}/bps/LSSTCam/bps_Daytime.yaml" \ - --extra-qgraph-options "--skip-existing-in LSSTCam/prompt/output-${DATE} -c parameters:release_id=1 -c parameters:apdb_config=${TMP_APDB} -c associateApdb:doRunForcedMeasurement=False --dataset-query-constraint off" \ + --extra-qgraph-options "--skip-existing-in LSSTCam/prompt/output-${DATE},LSSTCam/prompt/output-${DATE}/daytime -c parameters:release_id=1 -c parameters:apdb_config=${TMP_APDB} -c associateApdb:doRunForcedMeasurement=False --dataset-query-constraint off" \ --extra-run-quantum-options "--no-raise-on-partial-outputs" \ --input "LSSTCam/defaults,LSSTCam/templates,LSSTCam/prompt/output-${DATE}" \ --output "$OUTPUT_COLLECTION" \ From cbfc35ee3105e1dfb98459d2664c7e0407f5a460 Mon Sep 17 00:00:00 2001 From: Homer Neal Date: Mon, 9 Mar 2026 08:10:17 -0700 Subject: [PATCH 7/8] Remove extended skip-existing-in. It doesn't work as foreseen. --- scripts/LSSTCam/submit_ap_daytime.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/LSSTCam/submit_ap_daytime.sh b/scripts/LSSTCam/submit_ap_daytime.sh index c152b799..37d9edd9 100755 --- a/scripts/LSSTCam/submit_ap_daytime.sh +++ b/scripts/LSSTCam/submit_ap_daytime.sh @@ -56,7 +56,7 @@ BAD_DETECTORS_SQL="($(printf '%s,' $BAD_DETECTORS | sed 's/,$//'))" BLOCKS_SQL="($(printf "'%s'," $BLOCKS | sed 's/,$//'))" nohup bps submit "${AP_PIPE_DIR}/bps/LSSTCam/bps_Daytime.yaml" \ - --extra-qgraph-options "--skip-existing-in LSSTCam/prompt/output-${DATE},LSSTCam/prompt/output-${DATE}/daytime -c parameters:release_id=1 -c parameters:apdb_config=${TMP_APDB} -c associateApdb:doRunForcedMeasurement=False --dataset-query-constraint off" \ + --extra-qgraph-options "--skip-existing-in LSSTCam/prompt/output-${DATE} -c parameters:release_id=1 -c parameters:apdb_config=${TMP_APDB} -c associateApdb:doRunForcedMeasurement=False --dataset-query-constraint off" \ --extra-run-quantum-options "--no-raise-on-partial-outputs" \ --input "LSSTCam/defaults,LSSTCam/templates,LSSTCam/prompt/output-${DATE}" \ --output "$OUTPUT_COLLECTION" \ From e6345717d06548097475b9d53a62695e5f4bcf5a Mon Sep 17 00:00:00 2001 From: Homer Neal Date: Mon, 9 Mar 2026 08:47:07 -0700 Subject: [PATCH 8/8] reject tracts in COSMOS --- scripts/LSSTCam/submit_ap_daytime.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/LSSTCam/submit_ap_daytime.sh b/scripts/LSSTCam/submit_ap_daytime.sh index 34c7b1e1..6ff717f1 100755 --- a/scripts/LSSTCam/submit_ap_daytime.sh +++ b/scripts/LSSTCam/submit_ap_daytime.sh @@ -64,7 +64,8 @@ nohup bps submit "${AP_PIPE_DIR}/bps/LSSTCam/bps_Daytime.yaml" \ AND skymap='lsst_cells_v2' \ AND detector NOT IN $BAD_DETECTORS_SQL \ AND day_obs=$DAY_OBS \ - AND exposure.science_program IN $BLOCKS_SQL" \ + AND exposure.science_program IN $BLOCKS_SQL \ + AND tract NOT IN (9327, 9328, 9569, 9570, 9571, 9572, 9811, 9812, 9813, 9814, 10053, 10054, 10055, 10056)" \ > "${LOG_FILE}" 2>&1 & echo "Submission started for date ${DATE}"