Skip to content

fix: No date values added to database for Datacite records with empty date array #5

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
aaronm-2112 wants to merge 3 commits into
base: main
Choose a base branch
from
Open

fix: No date values added to database for Datacite records with empty date array #5

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
b4e551f
refactor: if dates empty for datacite use registered or created
aaronm-2112 May 11, 2026
7c22ed9
fix: datacite dates use registered/created if dates empty
aaronm-2112 May 13, 2026
d644392
chore: add records to db
aaronm-2112 May 14, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 10 additions & 2 deletions add_dataset_records.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -177,6 +177,7 @@ def _build_record_from_result(record, source):
sizes = [f"{record.get('size_mb', 0)} MB"]
metadata_created_raw = ""
metadata_modified_raw = ""
metadata_registered_raw = ""

if metadata_dir and source_id:
meta_path = os.path.join(metadata_dir, f"{source_id}.json")
Expand All @@ -200,6 +201,7 @@ def _build_record_from_result(record, source):
)
metadata_created_raw = meta.get("created", "")
metadata_modified_raw = meta.get("modified", "")
metadata_registered_raw = meta.get("registered", "")

publication_date = _extract_iso_date(
m.get("publication_date", "") or meta.get("publication_date", "")
Expand All @@ -223,6 +225,10 @@ def _build_record_from_result(record, source):
if not publication_date and _dates_fallback:
publication_date = _dates_fallback
publication_date_source = "dates"
if not publication_date:
publication_date = _extract_iso_date(metadata_registered_raw)
if publication_date:
publication_date_source = "registered"
if not publication_date:
publication_date = _extract_iso_date(metadata_created_raw)
if publication_date:
Expand All @@ -232,6 +238,7 @@ def _build_record_from_result(record, source):
if publication_date:
publication_date_source = "modified"


print(
f" Extracted publication date: {publication_date} from metadata for source_id: {source_id} (source: {source})"
)
Expand Down Expand Up @@ -275,9 +282,10 @@ def _build_record_from_result(record, source):
)

# Timestamp
if publication_date_source in {"created", "modified"}:
if publication_date_source in {"created", "modified", "registered"}:
created_raw = (
metadata_created_raw
metadata_registered_raw
or metadata_created_raw
or metadata_modified_raw
or publication_date
or datetime.now().strftime("%Y-%m-%d %H:%M:%S")
Expand Down
2 changes: 2 additions & 0 deletions envision/metadata.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,8 @@ class DatasetMetadata:
# Provenance
creators: list[dict] = field(default_factory=list)
publication_year: str | None = None
registered: str | None = None
created: str | None = None
dates: list[dict] = field(default_factory=list)

# Links
Expand Down
5 changes: 5 additions & 0 deletions envision/scrapers/datacite.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -146,6 +146,9 @@ def _item_to_metadata(self, item: dict) -> DatasetMetadata | None:
"dateValue": d.get("date", ""),
"dateType": d.get("dateType", "Other"),
})
registered = attrs.get("registered")
created = attrs.get("created")


pub_year = str(attrs.get("publicationYear", ""))

Expand Down Expand Up @@ -205,4 +208,6 @@ def _item_to_metadata(self, item: dict) -> DatasetMetadata | None:
dates=dates,
related_identifiers=related,
external_links=[],
registered=registered,
created=created
)