diff --git a/add_dataset_records.py b/add_dataset_records.py index 7548baa..041d680 100644 --- a/add_dataset_records.py +++ b/add_dataset_records.py @@ -177,6 +177,7 @@ def _build_record_from_result(record, source): sizes = [f"{record.get('size_mb', 0)} MB"] metadata_created_raw = "" metadata_modified_raw = "" + metadata_registered_raw = "" if metadata_dir and source_id: meta_path = os.path.join(metadata_dir, f"{source_id}.json") @@ -200,6 +201,7 @@ def _build_record_from_result(record, source): ) metadata_created_raw = meta.get("created", "") metadata_modified_raw = meta.get("modified", "") + metadata_registered_raw = meta.get("registered", "") publication_date = _extract_iso_date( m.get("publication_date", "") or meta.get("publication_date", "") @@ -223,6 +225,10 @@ def _build_record_from_result(record, source): if not publication_date and _dates_fallback: publication_date = _dates_fallback publication_date_source = "dates" + if not publication_date: + publication_date = _extract_iso_date(metadata_registered_raw) + if publication_date: + publication_date_source = "registered" if not publication_date: publication_date = _extract_iso_date(metadata_created_raw) if publication_date: @@ -232,6 +238,7 @@ def _build_record_from_result(record, source): if publication_date: publication_date_source = "modified" + print( f" Extracted publication date: {publication_date} from metadata for source_id: {source_id} (source: {source})" ) @@ -275,9 +282,10 @@ def _build_record_from_result(record, source): ) # Timestamp - if publication_date_source in {"created", "modified"}: + if publication_date_source in {"created", "modified", "registered"}: created_raw = ( - metadata_created_raw + metadata_registered_raw + or metadata_created_raw or metadata_modified_raw or publication_date or datetime.now().strftime("%Y-%m-%d %H:%M:%S") diff --git a/envision/metadata.py b/envision/metadata.py index a7b011e..8a1d608 100644 --- a/envision/metadata.py +++ b/envision/metadata.py @@ -62,6 +62,8 @@ class DatasetMetadata: # Provenance creators: list[dict] = field(default_factory=list) publication_year: str | None = None + registered: str | None = None + created: str | None = None dates: list[dict] = field(default_factory=list) # Links diff --git a/envision/scrapers/datacite.py b/envision/scrapers/datacite.py index e9a62d0..7ca7ef4 100644 --- a/envision/scrapers/datacite.py +++ b/envision/scrapers/datacite.py @@ -146,6 +146,9 @@ def _item_to_metadata(self, item: dict) -> DatasetMetadata | None: "dateValue": d.get("date", ""), "dateType": d.get("dateType", "Other"), }) + registered = attrs.get("registered") + created = attrs.get("created") + pub_year = str(attrs.get("publicationYear", "")) @@ -205,4 +208,6 @@ def _item_to_metadata(self, item: dict) -> DatasetMetadata | None: dates=dates, related_identifiers=related, external_links=[], + registered=registered, + created=created )