From c920c6fa6c167d70f9ba2897b83a381cb3940f9b Mon Sep 17 00:00:00 2001 From: Homer Neal Date: Fri, 23 Jan 2026 09:42:57 -0800 Subject: [PATCH 1/4] use mc instead of singularity to get pp_apdb yaml --- scripts/LSSTCam/submit_ap_daytime.sh | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/scripts/LSSTCam/submit_ap_daytime.sh b/scripts/LSSTCam/submit_ap_daytime.sh index d551236..e4051e0 100755 --- a/scripts/LSSTCam/submit_ap_daytime.sh +++ b/scripts/LSSTCam/submit_ap_daytime.sh @@ -23,11 +23,15 @@ case "$TMP_APDB_REL" in esac # Copy APDB config from S3 using Singularity AWS CLI -singularity exec /sdf/sw/s3/aws-cli_latest.sif \ - aws --endpoint-url https://sdfembs3.sdf.slac.stanford.edu s3 \ - --profile embargo-s3 \ - cp s3://rubin-summit-users/apdb_config/cassandra/pp_apdb_lsstcam.yaml \ - "$TMP_APDB" +#singularity exec /sdf/sw/s3/aws-cli_latest.sif \ +# aws --endpoint-url https://sdfembs3.sdf.slac.stanford.edu s3 \ +# --profile embargo-s3 \ +# cp s3://rubin-summit-users/apdb_config/cassandra/pp_apdb_lsstcam.yaml \ +# "$TMP_APDB" + +echo "TMP_APDB = "$TMP_APDB +mc cp embargo/rubin-summit-users/apdb_config/cassandra/pp_apdb_lsstcam.yaml "$TMP_APDB" + # NOTE: # No cleanup of TMP_APDB here since the job is launched with nohup From 31cbbf209cbdf9cd4e90109668a0d50956e74fc2 Mon Sep 17 00:00:00 2001 From: Homer Neal Date: Wed, 28 Jan 2026 14:34:30 -0800 Subject: [PATCH 2/4] update skymap spec to lsst_cells_v2 --- scripts/LSSTCam/submit_ap_daytime.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/LSSTCam/submit_ap_daytime.sh b/scripts/LSSTCam/submit_ap_daytime.sh index e4051e0..9a7707f 100755 --- a/scripts/LSSTCam/submit_ap_daytime.sh +++ b/scripts/LSSTCam/submit_ap_daytime.sh @@ -70,7 +70,7 @@ nohup bps submit "${AP_PIPE_DIR}/bps/LSSTCam/bps_Daytime.yaml" \ --input "LSSTCam/defaults,LSSTCam/templates,LSSTCam/prompt/output-${DATE}" \ --output "$OUTPUT_COLLECTION" \ -d "instrument='$INSTRUMENT' \ - AND skymap='lsst_cells_v1' \ + AND skymap='lsst_cells_v2' \ AND detector NOT IN $BAD_DETECTORS_SQL \ AND day_obs=$DAY_OBS \ AND exposure.science_program IN $BLOCKS_SQL" \ From 95581865030184c15ddb54ccae4f35efde6266f8 Mon Sep 17 00:00:00 2001 From: Homer Neal Date: Fri, 27 Feb 2026 03:23:00 -0800 Subject: [PATCH 3/4] set nodeset name expand skip-existing-in to skip also those already processed by daily batch --- bps/LSSTCam/bps_Daytime.yaml | 2 ++ scripts/LSSTCam/submit_ap_daytime.sh | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/bps/LSSTCam/bps_Daytime.yaml b/bps/LSSTCam/bps_Daytime.yaml index 645b137..bc2f89d 100644 --- a/bps/LSSTCam/bps_Daytime.yaml +++ b/bps/LSSTCam/bps_Daytime.yaml @@ -3,6 +3,8 @@ pipelineYaml: $AP_PIPE_DIR/pipelines/LSSTCam/ApPipe.yaml project: ApPipe campaign: AP-daytime +nodeset: daily + payload: butlerConfig: "embargo" # payloadName: Do not set here, must pass in from command line or script diff --git a/scripts/LSSTCam/submit_ap_daytime.sh b/scripts/LSSTCam/submit_ap_daytime.sh index 9a7707f..31e5be7 100755 --- a/scripts/LSSTCam/submit_ap_daytime.sh +++ b/scripts/LSSTCam/submit_ap_daytime.sh @@ -65,7 +65,7 @@ BAD_DETECTORS_SQL="($(printf '%s,' $BAD_DETECTORS | sed 's/,$//'))" BLOCKS_SQL="($(printf "'%s'," $BLOCKS | sed 's/,$//'))" nohup bps submit "${AP_PIPE_DIR}/bps/LSSTCam/bps_Daytime.yaml" \ - --extra-qgraph-options "--skip-existing-in LSSTCam/prompt/output-${DATE} -c parameters:release_id=1 -c parameters:apdb_config=${TMP_APDB} -c associateApdb:doRunForcedMeasurement=False --dataset-query-constraint off" \ + --extra-qgraph-options "--skip-existing-in LSSTCam/prompt/output-${DATE},LSSTCam/prompt/output-${DATE}/daytime -c parameters:release_id=1 -c parameters:apdb_config=${TMP_APDB} -c associateApdb:doRunForcedMeasurement=False --dataset-query-constraint off" \ --extra-run-quantum-options "--no-raise-on-partial-outputs" \ --input "LSSTCam/defaults,LSSTCam/templates,LSSTCam/prompt/output-${DATE}" \ --output "$OUTPUT_COLLECTION" \ From 69ec2006c327ef90884493f77a19a2e3437cfbd4 Mon Sep 17 00:00:00 2001 From: Homer Neal Date: Mon, 9 Mar 2026 08:10:17 -0700 Subject: [PATCH 4/4] Remove extended skip-existing-in. It doesn't work as foreseen. --- scripts/LSSTCam/submit_ap_daytime.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/LSSTCam/submit_ap_daytime.sh b/scripts/LSSTCam/submit_ap_daytime.sh index 31e5be7..9a7707f 100755 --- a/scripts/LSSTCam/submit_ap_daytime.sh +++ b/scripts/LSSTCam/submit_ap_daytime.sh @@ -65,7 +65,7 @@ BAD_DETECTORS_SQL="($(printf '%s,' $BAD_DETECTORS | sed 's/,$//'))" BLOCKS_SQL="($(printf "'%s'," $BLOCKS | sed 's/,$//'))" nohup bps submit "${AP_PIPE_DIR}/bps/LSSTCam/bps_Daytime.yaml" \ - --extra-qgraph-options "--skip-existing-in LSSTCam/prompt/output-${DATE},LSSTCam/prompt/output-${DATE}/daytime -c parameters:release_id=1 -c parameters:apdb_config=${TMP_APDB} -c associateApdb:doRunForcedMeasurement=False --dataset-query-constraint off" \ + --extra-qgraph-options "--skip-existing-in LSSTCam/prompt/output-${DATE} -c parameters:release_id=1 -c parameters:apdb_config=${TMP_APDB} -c associateApdb:doRunForcedMeasurement=False --dataset-query-constraint off" \ --extra-run-quantum-options "--no-raise-on-partial-outputs" \ --input "LSSTCam/defaults,LSSTCam/templates,LSSTCam/prompt/output-${DATE}" \ --output "$OUTPUT_COLLECTION" \