DM-54463: Adapt new collection naming in Prompt Processing

python/initializer/write_init_outputs.py

@@ -142,13 +142,13 @@ def main(args=None):
         return 1
 
 
-def _get_current_day_obs() -> str:
+def _get_current_day_obs() -> int:
     """Generate the current day_obs value.
 
     Returns
     -------
-    day_obs : `str`
-        The day_obs value in YYYY-MM-DD format.
+    day_obs : `int`
+        The day_obs value in YYYYMMDD format.
     """
     return run_utils.get_day_obs(astropy.time.Time.now())
 

python/shared/run_utils.py

@@ -38,27 +38,27 @@
 _log_trace3.setLevel(logging.CRITICAL)  # Turn off by default.
 
 
-def get_output_chain(instrument: lsst.obs.base.Instrument, date: str) -> str:
+def get_output_chain(instrument: lsst.obs.base.Instrument, date: int) -> str:
     """Generate a deterministic output chain name that avoids
     configuration conflicts.
 
     Parameters
     ----------
     instrument : `lsst.obs.base.Instrument`
         The instrument for which to generate a collection.
-    date : `str`
-        Date of the processing run (not observation!).
+    date : `int`
+        Date of the processing run (not observation!) in YYYYMMDD format.
 
     Returns
     -------
     chain : `str`
         The chain in which to place all output collections.
     """
     # Order optimized for S3 bucket -- filter out as many files as soon as possible.
-    return instrument.makeCollectionName("prompt", f"output-{date}")
+    return instrument.makeCollectionName("runs", f"prompt-{date:08d}")
 
 
-def get_preload_run(instrument: lsst.obs.base.Instrument, deployment_id: str, date: str) -> str:
+def get_preload_run(instrument: lsst.obs.base.Instrument, deployment_id: str, date: int) -> str:
     """Generate a deterministic preload collection name that avoids
     configuration conflicts.
 
@@ -68,8 +68,8 @@ def get_preload_run(instrument: lsst.obs.base.Instrument, deployment_id: str, da
         The instrument for which to generate a collection.
     deployment_id : `str`
         A unique version ID of the active stack and pipeline configuration(s).
-    date : `str`
-        Date of the processing run (not observation!).
+    date : `int`
+        Date of the processing run (not observation!) in YYYYMMDD format.
 
     Returns
     -------
@@ -82,7 +82,7 @@ def get_preload_run(instrument: lsst.obs.base.Instrument, deployment_id: str, da
 def get_output_run(instrument: lsst.obs.base.Instrument,
                    deployment_id: str,
                    pipeline_file: str,
-                   date: str,
+                   date: int,
                    ) -> str:
     """Generate a deterministic collection name that avoids version or
     provenance conflicts.
@@ -95,8 +95,8 @@ def get_output_run(instrument: lsst.obs.base.Instrument,
         A unique version ID of the active stack and pipeline configuration(s).
     pipeline_file : `str`
         The pipeline name that the run will be used for.
-    date : `str`
-        Date of the processing run (not observation!).
+    date : `int`
+        Date of the processing run (not observation!) in YYYYMMDD format.
 
     Returns
     -------
@@ -105,10 +105,11 @@ def get_output_run(instrument: lsst.obs.base.Instrument,
     """
     pipeline_name, _ = os.path.splitext(os.path.basename(pipeline_file))
     # Order optimized for S3 bucket -- filter out as many files as soon as possible.
-    return "/".join([get_output_chain(instrument, date), pipeline_name, deployment_id])
+    output_run = instrument.makeCollectionName("runs", "prompt", f"{date:08d}")
+    return "/".join([output_run, pipeline_name, deployment_id])
 
 
-def get_day_obs(time: astropy.time.Time) -> str:
+def get_day_obs(time: astropy.time.Time) -> int:
     """Convert a timestamp into a day-obs string.
 
     The day-obs is defined as the TAI date of an instant 12 hours before
@@ -121,11 +122,12 @@ def get_day_obs(time: astropy.time.Time) -> str:
 
     Returns
     -------
-    day_obs : `str`
-        The day-obs corresponding to ``time``, in YYYY-MM-DD format.
+    day_obs : `int`
+        The day_obs corresponding to ``time``, in YYYYMMDD format.
     """
     day_obs_delta = astropy.time.TimeDelta(-12.0 * astropy.units.hour, scale="tai")
-    return (time + day_obs_delta).tai.to_value("iso", "date")
+    iso_date = (time + day_obs_delta).tai.to_value("iso", "date")
+    return int(iso_date.replace("-", ""))
 
 
 def _get_pp_hash():

tests/data/make_central_repo.sh

@@ -10,7 +10,6 @@ REPO="${PROMPT_PROCESSING_DIR:?Can\'t find prompt_processing repo; is it set up?
 # For compatibility testing, use the lowest version we offer support for
 butler create "$REPO" --dimension-config "$DAF_BUTLER_DIR/python/lsst/daf/butler/configs/old_dimensions/daf_butler_universe7.yaml"
 butler register-instrument "$REPO" lsst.obs.lsst.LsstCam
-butler register-dataset-type "$REPO" gain_correction IsrCalib instrument detector --is-calibration
 
 # Import datasets
 butler write-curated-calibrations "$REPO" LSSTCam --collection LSSTCam/calib/DM-50520
@@ -23,12 +22,13 @@ butler transfer-datasets /repo/main "$REPO" --transfer copy --register-dataset-t
 # write-curated-calibrations produces manual_defects but not defects
 butler transfer-datasets /repo/main "$REPO" --transfer copy --register-dataset-types --transfer-dimensions --dataset-type defects --collections LSSTCam/calib/DM-49175 --where "instrument='LSSTCam' and detector in (90, 91)"
 butler transfer-datasets /repo/main "$REPO" --transfer copy --register-dataset-types --transfer-dimensions --dataset-type flat --collections LSSTCam/calib/DM-52163 --where "instrument='LSSTCam' and detector in (90, 91) and physical_filter='g_6'"
+butler transfer-datasets /repo/main "$REPO" --transfer copy --register-dataset-types --transfer-dimensions --dataset-type gain_correction --collections LSSTCam/calib/DM-53620 --where "instrument='LSSTCam' and detector in (90, 91) and physical_filter='g_6'"
 butler transfer-datasets /repo/main "$REPO" --transfer copy --register-dataset-types --transfer-dimensions --dataset-type linearizer --collections LSSTCam/calib/DM-49175 --where "instrument='LSSTCam' and detector in (90, 91)"
 butler transfer-datasets /repo/main "$REPO" --transfer copy --register-dataset-types --transfer-dimensions --dataset-type ptc --collections LSSTCam/calib/DM-50336 --where "instrument='LSSTCam' and detector in (90, 91)"
 butler transfer-datasets /repo/main "$REPO" --transfer copy --register-dataset-types --transfer-dimensions --dataset-type pretrainedModelPackage --collections pretrained_models/dummy
 butler transfer-datasets /repo/main "$REPO" --transfer copy --register-dataset-types --transfer-dimensions --dataset-type skyMap --where "skymap='lsst_cells_v1'" --collections skymaps
 # The tract constraint is not strictly necessary but just to filter out templates from another overlapping tract.
-butler transfer-datasets embargo "$REPO" --transfer copy --register-dataset-types --transfer-dimensions --dataset-type template_coadd --collections LSSTCam/templates/DM-51716/SV_225/run --where "instrument='LSSTCam' and detector=90 and visit=2025052100138 and skymap='lsst_cells_v1' and tract=3534"
+butler transfer-datasets /repo/main "$REPO" --transfer copy --register-dataset-types --transfer-dimensions --dataset-type template_coadd --collections LSSTCam/templates/DM-51716/SV_225/run --where "instrument='LSSTCam' and detector=90 and visit=2025052100138 and skymap='lsst_cells_v1' and tract=3534"
 butler transfer-datasets /repo/main "$REPO" --transfer copy --register-dataset-types --transfer-dimensions --dataset-type the_monster_20250219 --collections refcats --where "instrument='LSSTCam' and detector=90 and visit=2025052100138"
 
 # Certify non-curated calibs
@@ -38,6 +38,7 @@ butler certify-calibrations "$REPO" LSSTCam/calib/DM-49175/run7/ctiGen.20250320a
 butler certify-calibrations "$REPO" LSSTCam/calib/DM-49175/run7/darkGen.20250320a/20250326T000943Z LSSTCam/calib/DM-50520 dark --begin-date "2025-05-15T12:00:00" --end-date "2025-06-25T12:00:00"
 butler certify-calibrations "$REPO" LSSTCam/calib/DM-49175/run7/defectGen.20250401a/20250401T232630Z LSSTCam/calib/DM-50520 defects --begin-date "2025-05-15T12:00:00" --end-date "2025-06-25T12:00:00"
 butler certify-calibrations "$REPO" LSSTCam/calib/DM-52163/flats-2s-v30-nograd-ugrizy/flatTwoLedGen-g.20250812a/20250812T182450Z LSSTCam/calib/DM-50520 flat --begin-date "2025-05-15T12:00:00" --end-date "2025-06-25T12:00:00"
+butler certify-calibrations "$REPO" LSSTCam/calib/DM-53620/3s_v1_dp2_v2_gain_correction_20250920/gainCorrectionGen.20251215a/20251218T193528Z LSSTCam/calib/DM-50520 gain_correction --begin-date "2025-05-15T12:00:00" --end-date "2025-06-25T12:00:00"
 butler certify-calibrations "$REPO" LSSTCam/calib/DM-49175/run7/linearizerGen.20250320a/20250321T052032Z LSSTCam/calib/DM-50520 linearizer --begin-date "2025-05-15T12:00:00" --end-date "2025-06-25T12:00:00"
 butler certify-calibrations "$REPO" LSSTCam/calib/DM-50336/run7/ptcGen.20250422a/20250422T162135Z LSSTCam/calib/DM-50520 ptc --begin-date "2025-05-15T12:00:00" --end-date "2025-06-25T12:00:00"
 
@@ -67,9 +68,12 @@ export MAIN_PIPELINES_CONFIG="- survey: SURVEY
   - ${PROMPT_PROCESSING_DIR}/tests/data/ISR.yaml
 "
 apdb-cli create-sql "sqlite:///${TEMP_APDB}" "${CONFIG_APDB}"
-butler ingest-raws "$REPO" s3://embargo@rubin-summit/LSSTCam/20250521/MC_O_20250521_000138/MC_O_20250521_000138_R22_S00.fits -t copy
+temp_dir=$(mktemp -d)
+unzip /sdf/data/rubin/lsstdata/offline/instrument/LSSTCam/20250521/MC_O_20250521_000138.zip -d "$temp_dir"
+butler ingest-raws "$REPO" "$temp_dir/MC_O_20250521_000138_R22_S00.fits" -t copy
+rm -rf "$temp_dir"
 butler define-visits "$REPO" lsst.obs.lsst.LsstCam
-pipetask run -b "$REPO" -i LSSTCam/raw/all,LSSTCam/defaults,LSSTCam/templates -o u/add-dataset-type -d "instrument='LSSTCam' and exposure=2025052100138 and detector=90" -p $AP_PIPE_DIR/pipelines/LSSTCam/ApPipe.yaml -c parameters:apdb_config=${CONFIG_APDB}  -c associateApdb:doPackageAlerts=False --register-dataset-types --init-only
+pipetask run -b "$REPO" -i LSSTCam/raw/all,LSSTCam/defaults,LSSTCam/templates/DM-51716/SV_225/run -o u/add-dataset-type -d "instrument='LSSTCam' and exposure=2025052100138 and detector=90" -p $AP_PIPE_DIR/pipelines/LSSTCam/ApPipe.yaml -c parameters:apdb_config=${CONFIG_APDB}  -c associateApdb:doPackageAlerts=False --register-dataset-types --init-only
 # Clean up data that are no longer needed.
 butler remove-runs "$REPO" LSSTCam/raw/all --no-confirm --force
 rm -rf "$REPO"/raw
@@ -87,4 +91,4 @@ for x in `find "$REPO/LSSTCam/calib/curated/" -name "*.fits"`; do : > $x; done
 for x in `find "$REPO/LSSTCam/templates/" -name "*.fits"`; do : > $x; done
 for x in `find "$REPO/pretrained_models/" -name "*.zip"`; do : > $x; done
 for x in `find "$REPO/refcats/DM-49042/" -name "*.fits"`; do : > $x; done
-for x in `find "$REPO/LSSTCam/prompt/" -type f`; do : > $x; done
+for x in `find "$REPO/LSSTCam/runs/prompt/" -type f`; do : > $x; done

tests/mock_central_repo.py

@@ -57,9 +57,9 @@ class TestRepo:
     skymap_name = "lsst_cells_v1"
     # The day_obs used for the init-output runs in the test repo.
     # Does not need to be synchronized with simulated metadata.
-    sim_date = astropy.time.Time("2025-09-13T00:00:00Z")
+    sim_date = astropy.time.Time("2026-04-02T00:00:00Z")
     # The deployment ID used in the test repo.
-    sim_deployment = "pipelines-cf62e06-config-8acfde6"
+    sim_deployment = "pipelines-e8bc8a9-config-8acfde6"
 
     @classmethod
     def fake_file_data(cls, filename, dimensions, instrument, visit):

tests/test_middleware_interface.py

@@ -311,8 +311,8 @@ def _check_imports(self, butler, group, detector, expected_shards, have_filter=T
 
         # Check that preloaded datasets have been generated
         date = (astropy.time.Time.now() - 12 * u.hour).to_value("ymdhms")
-        preload_collection = f"{TestRepo.instname}/prompt/" \
-                             f"output-{date.year:04d}-{date.month:02d}-{date.day:02d}/" \
+        preload_collection = f"{TestRepo.instname}/runs/prompt/" \
+                             f"{date.year:04d}{date.month:02d}{date.day:02d}/" \
                              f"NoPipeline/{self.deploy_id}"
         self.assertTrue(
             butler.exists('promptPreload_metrics',
@@ -1346,11 +1346,11 @@ def setUp(self):
         with unittest.mock.patch("activator.middleware_interface.MiddlewareInterface._run_preprocessing"):
             self.second_interface.prep_butler()
         date = (astropy.time.Time.now() - 12 * u.hour).to_value("ymdhms")
-        self.preprocessing_run = f"{TestRepo.instname}/prompt/" \
-                                 f"output-{date.year:04d}-{date.month:02d}-{date.day:02d}" \
+        self.preprocessing_run = f"{TestRepo.instname}/runs/prompt/" \
+                                 f"{date.year:04d}{date.month:02d}{date.day:02d}" \
                                  f"/Preprocess/{self.deploy_id}"
-        self.output_run = f"{TestRepo.instname}/prompt/" \
-                          f"output-{date.year:04d}-{date.month:02d}-{date.day:02d}" \
+        self.output_run = f"{TestRepo.instname}/runs/prompt/" \
+                          f"{date.year:04d}{date.month:02d}{date.day:02d}" \
                           f"/ApPipe/{self.deploy_id}"
 
         with unittest.mock.patch.object(self.interface.rawIngestTask, "extractMetadata") as mock:

tests/test_run_utils.py

@@ -39,10 +39,10 @@ def test_get_output_run(self):
         instrument = Instrument.from_string(instclass)
         deploy_id = "test-9660137"
         filename = "ApPipe.yaml"
-        date = "2023-01-22"
+        date = 20230122
         out_chain = get_output_chain(instrument, date)
-        self.assertEqual(out_chain, f"{instname}/prompt/output-2023-01-22")
+        self.assertEqual(out_chain, f"{instname}/runs/prompt-20230122")
         preload_run = get_preload_run(instrument, deploy_id, date)
-        self.assertEqual(preload_run, f"{instname}/prompt/output-2023-01-22/NoPipeline/{deploy_id}")
+        self.assertEqual(preload_run, f"{instname}/runs/prompt/20230122/NoPipeline/{deploy_id}")
         out_run = get_output_run(instrument, deploy_id, filename, date)
-        self.assertEqual(out_run, f"{instname}/prompt/output-2023-01-22/ApPipe/{deploy_id}")
+        self.assertEqual(out_run, f"{instname}/runs/prompt/20230122/ApPipe/{deploy_id}")

tests/test_write_init_outputs.py

@@ -98,7 +98,7 @@ def test_make_init_outputs_not_registered(self):
     def test_make_init_outputs_empty_run(self):
         pipe_file = "${PROMPT_PROCESSING_DIR}/tests/data/SingleFrame.yaml"
         instrument = lsst.obs.base.Instrument.from_string("lsst.obs.lsst.LsstCam")
-        expected_run = get_output_run(instrument, self.deploy_id, pipe_file, "2024-09-24")
+        expected_run = get_output_run(instrument, self.deploy_id, pipe_file, 20240924)
 
         self._register_dataset_types(pipe_file)
 
@@ -118,7 +118,7 @@ def test_make_init_outputs_empty_run(self):
     def test_make_init_outputs_filled_run(self):
         pipe_file = "${PROMPT_PROCESSING_DIR}/tests/data/SingleFrame.yaml"
         instrument = lsst.obs.base.Instrument.from_string("lsst.obs.lsst.LsstCam")
-        expected_run = get_output_run(instrument, self.deploy_id, pipe_file, "2024-09-24")
+        expected_run = get_output_run(instrument, self.deploy_id, pipe_file, 20240924)
 
         self._register_dataset_types(pipe_file)
 
@@ -139,7 +139,7 @@ def test_make_init_outputs_filled_run(self):
 
     def test_make_output_chain_new(self):
         instrument = lsst.obs.base.Instrument.from_string("lsst.obs.lsst.LsstCam")
-        expected_chain = get_output_chain(instrument, "2024-09-24")
+        expected_chain = get_output_chain(instrument, 20240924)
 
         self.base_butler.collections.register("run1", CollectionType.RUN)
         self.base_butler.collections.register("run2", CollectionType.RUN)
@@ -153,7 +153,7 @@ def test_make_output_chain_new(self):
 
     def test_make_output_chain_existing(self):
         instrument = lsst.obs.base.Instrument.from_string("lsst.obs.lsst.LsstCam")
-        expected_chain = get_output_chain(instrument, "2024-09-24")
+        expected_chain = get_output_chain(instrument, 20240924)
 
         self.base_butler.collections.register("run1", CollectionType.RUN)
         self.base_butler.collections.register("run2", CollectionType.RUN)
@@ -196,12 +196,12 @@ def _make_init_outputs(butler, instrument, apdb, deploy_id, pipeline):
                 unittest.mock.patch("initializer.write_init_outputs._make_init_outputs",
                                     side_effect=_make_init_outputs) as mock_make, \
                 unittest.mock.patch("initializer.write_init_outputs._get_current_day_obs",
-                                    return_value=""), \
+                                    return_value=0), \
                 unittest.mock.patch("initializer.write_init_outputs.import_iers_cache"):
             main(["--deploy-id", self.deploy_id])
 
         # The preload collection is not associated with a pipeline
-        preload_run = get_preload_run(lsst.obs.lsst.LsstCam(), self.deploy_id, "")
+        preload_run = get_preload_run(lsst.obs.lsst.LsstCam(), self.deploy_id, 0)
         with Butler(self.repo.name) as butler:
             self.assertTrue(butler.collections.query(preload_run, CollectionType.RUN))