From b4e551f8ccc45e7c462f3fcaea508db7b79979bf Mon Sep 17 00:00:00 2001 From: aaronm-2112 Date: Mon, 11 May 2026 15:11:20 -0700 Subject: [PATCH 1/3] refactor: if dates empty for datacite use registered or created --- add_dataset_records.py | 12 ++++++++++-- envision/scrapers/datacite.py | 5 +++++ 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/add_dataset_records.py b/add_dataset_records.py index 7548baa..041d680 100644 --- a/add_dataset_records.py +++ b/add_dataset_records.py @@ -177,6 +177,7 @@ def _build_record_from_result(record, source): sizes = [f"{record.get('size_mb', 0)} MB"] metadata_created_raw = "" metadata_modified_raw = "" + metadata_registered_raw = "" if metadata_dir and source_id: meta_path = os.path.join(metadata_dir, f"{source_id}.json") @@ -200,6 +201,7 @@ def _build_record_from_result(record, source): ) metadata_created_raw = meta.get("created", "") metadata_modified_raw = meta.get("modified", "") + metadata_registered_raw = meta.get("registered", "") publication_date = _extract_iso_date( m.get("publication_date", "") or meta.get("publication_date", "") @@ -223,6 +225,10 @@ def _build_record_from_result(record, source): if not publication_date and _dates_fallback: publication_date = _dates_fallback publication_date_source = "dates" + if not publication_date: + publication_date = _extract_iso_date(metadata_registered_raw) + if publication_date: + publication_date_source = "registered" if not publication_date: publication_date = _extract_iso_date(metadata_created_raw) if publication_date: @@ -232,6 +238,7 @@ def _build_record_from_result(record, source): if publication_date: publication_date_source = "modified" + print( f" Extracted publication date: {publication_date} from metadata for source_id: {source_id} (source: {source})" ) @@ -275,9 +282,10 @@ def _build_record_from_result(record, source): ) # Timestamp - if publication_date_source in {"created", "modified"}: + if publication_date_source in {"created", "modified", "registered"}: created_raw = ( - metadata_created_raw + metadata_registered_raw + or metadata_created_raw or metadata_modified_raw or publication_date or datetime.now().strftime("%Y-%m-%d %H:%M:%S") diff --git a/envision/scrapers/datacite.py b/envision/scrapers/datacite.py index e9a62d0..6f1ed15 100644 --- a/envision/scrapers/datacite.py +++ b/envision/scrapers/datacite.py @@ -146,6 +146,9 @@ def _item_to_metadata(self, item: dict) -> DatasetMetadata | None: "dateValue": d.get("date", ""), "dateType": d.get("dateType", "Other"), }) + registered = attrs.get("registered") + created = attrs.get("created") + pub_year = str(attrs.get("publicationYear", "")) @@ -205,4 +208,6 @@ def _item_to_metadata(self, item: dict) -> DatasetMetadata | None: dates=dates, related_identifiers=related, external_links=[], + registered=registered + created=created ) From 7c22ed99ab7213cb6336c67687e5809b9a68b3e5 Mon Sep 17 00:00:00 2001 From: aaronm-2112 Date: Wed, 13 May 2026 08:22:43 -0700 Subject: [PATCH 2/3] fix: datacite dates use registered/created if dates empty --- add_dataset_records.py | 2 +- envision/metadata.py | 2 ++ envision/scrapers/datacite.py | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/add_dataset_records.py b/add_dataset_records.py index 041d680..b161444 100644 --- a/add_dataset_records.py +++ b/add_dataset_records.py @@ -518,4 +518,4 @@ def add_dataset_records_to_database(): if __name__ == "__main__": generate_dataset_records() - add_dataset_records_to_database() + # add_dataset_records_to_database() diff --git a/envision/metadata.py b/envision/metadata.py index a7b011e..8a1d608 100644 --- a/envision/metadata.py +++ b/envision/metadata.py @@ -62,6 +62,8 @@ class DatasetMetadata: # Provenance creators: list[dict] = field(default_factory=list) publication_year: str | None = None + registered: str | None = None + created: str | None = None dates: list[dict] = field(default_factory=list) # Links diff --git a/envision/scrapers/datacite.py b/envision/scrapers/datacite.py index 6f1ed15..7ca7ef4 100644 --- a/envision/scrapers/datacite.py +++ b/envision/scrapers/datacite.py @@ -208,6 +208,6 @@ def _item_to_metadata(self, item: dict) -> DatasetMetadata | None: dates=dates, related_identifiers=related, external_links=[], - registered=registered + registered=registered, created=created ) From d644392ce8fd55d65378170c8a536cfdb50f742a Mon Sep 17 00:00:00 2001 From: aaronm-2112 Date: Thu, 14 May 2026 12:11:20 -0700 Subject: [PATCH 3/3] chore: add records to db --- add_dataset_records.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/add_dataset_records.py b/add_dataset_records.py index b161444..041d680 100644 --- a/add_dataset_records.py +++ b/add_dataset_records.py @@ -518,4 +518,4 @@ def add_dataset_records_to_database(): if __name__ == "__main__": generate_dataset_records() - # add_dataset_records_to_database() + add_dataset_records_to_database()