From 712b5142817f3d630039298fb7f4f52675433613 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Fri, 12 Jun 2026 10:22:34 -0400 Subject: [PATCH 1/5] initial non-working script --- python/add_remote_files/add_remote_files.py | 326 ++++++++++++++++++++ 1 file changed, 326 insertions(+) create mode 100644 python/add_remote_files/add_remote_files.py diff --git a/python/add_remote_files/add_remote_files.py b/python/add_remote_files/add_remote_files.py new file mode 100644 index 0000000..f1bb8fd --- /dev/null +++ b/python/add_remote_files/add_remote_files.py @@ -0,0 +1,326 @@ +#!/usr/bin/env python3 +""" +add_remote_files.py + +Register files in a local directory tree as remote-store references in a +Dataverse dataset. No file bytes are transferred; only metadata (storage +identifier, filename, MIME type, MD5 hash) is sent to the Dataverse API. + +Dataverse and the specific dataset must be configured to use a remote store. +The remote store id must match the --store-id parameter in this script, and +the configured base-url must correspond to the --local-offset used, i.e. +for a base-url https://example.com/shareddata, the URL +https://example.com/sharedddata/file.txt must correspond the local path +/file.txt. Further, a file in the --base-dir is expected to +correspond to a URL matching the base-url plus the relative path difference +between the --local-offset and the file path. (With the usage example below, +a file.txt in the base dir should be accessible at +https://example.com/sharedddata/project/files/file.txt) + +Usage +----- +python dataverse_remote_store_upload.py \\ + --server https://dataverse.example.org \\ + --api-key xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx \\ + --store-id trs \\ + --local-offset /mnt/data \\ + --pid doi:10.5072/FK27U7YBV \\ + --base-dir /mnt/data/project/files + +Storage-identifier construction +-------------------------------- +Given: + --local-offset /mnt/data + --store-id trs + file path /mnt/data/project/files/subdir/file.csv + +The local offset is stripped from the absolute path to produce: + /project/files/subdir/file.csv + +The storage identifier becomes: + trs:///project/files/subdir/file.csv + +API used +-------- +POST /api/datasets/:persistentId/addFiles?persistentId= +with a multipart/form-data field "jsonData" containing a JSON array of +file-metadata objects (the "add multiple files" variant of the Direct +DataFile Upload/Replace API). +""" + +import argparse +import hashlib +import json +import mimetypes +import os +import sys +import urllib.parse +import urllib.request + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +def compute_md5(path: str, chunk: int = 1 << 20) -> str: + """Return the hex-encoded MD5 digest of a file.""" + h = hashlib.md5() + with open(path, "rb") as fh: + while True: + block = fh.read(chunk) + if not block: + break + h.update(block) + return h.hexdigest() + + +def guess_mime(filename: str) -> str: + """Return a best-guess MIME type for *filename*, defaulting to + 'application/octet-stream'.""" + mime, _ = mimetypes.guess_type(filename) + return mime or "application/octet-stream" + + +def build_storage_identifier(store_id: str, local_offset: str, abs_path: str) -> str: + """ + Strip *local_offset* from the start of *abs_path* to get the canonical + remote path, then format it as ://. + + Example + ------- + store-id = "trs" + local_offset = "/mnt/data" + abs_path = "/mnt/data/project/files/foo.csv" + → "trs:///project/files/foo.csv" + """ + # Normalise both paths so trailing slashes etc. don't cause problems. + local_offset = os.path.normpath(local_offset) + abs_path = os.path.normpath(abs_path) + + if not abs_path.startswith(local_offset): + raise ValueError( + f"File path '{abs_path}' does not start with " + f"local-offset '{local_offset}'" + ) + + remaining = abs_path[len(local_offset):] # starts with '/' on POSIX + if not remaining.startswith("/"): + remaining = "/" + remaining + + # Use double-slash after the scheme so the path is clearly absolute: + # trs:///some/path (scheme + empty authority + absolute path) + return f"{store_id}://{remaining}" + + +def collect_files(base_dir: str): + """Yield absolute paths for every regular file under *base_dir*.""" + for dirpath, _dirnames, filenames in os.walk(base_dir): + for name in filenames: + yield os.path.join(dirpath, name) + + +def build_file_metadata( + abs_path: str, + store_id: str, + local_offset: str, + base_dir: str, + verbose: bool = True, +) -> dict: + """Return the JSON-serialisable metadata dict for one file.""" + filename = os.path.basename(abs_path) + storage_id = build_storage_identifier(store_id, local_offset, abs_path) + mime = guess_mime(filename) + + if verbose: + print(f" Hashing {abs_path} …", end=" ", flush=True) + md5 = compute_md5(abs_path) + if verbose: + print(md5) + + # Derive a directoryLabel relative to base_dir (optional but useful). + rel = os.path.relpath(os.path.dirname(abs_path), base_dir) + dir_label = rel if rel != "." else "" + + entry = { + "storageIdentifier": storage_id, + "fileName": filename, + "mimeType": mime, + "md5Hash": md5, + "description": "", + } + if dir_label: + entry["directoryLabel"] = dir_label + + return entry + + +# --------------------------------------------------------------------------- +# Dataverse API call +# --------------------------------------------------------------------------- + +def add_files_to_dataset( + server: str, + api_key: str, + pid: str, + file_entries: list[dict], + dry_run: bool = False, +) -> dict | None: + """ + POST the file-metadata array to the Dataverse "addFiles" endpoint. + + Returns the parsed JSON response, or None on dry-run. + """ + encoded_pid = urllib.parse.quote(pid, safe="") + url = f"{server.rstrip('/')}/api/datasets/:persistentId/addFiles?persistentId={encoded_pid}" + + json_data = json.dumps(file_entries) + + print(f"\n→ POST {url}") + print(f" Files to register: {len(file_entries)}") + if dry_run: + print(" [DRY RUN] jsonData payload:") + print(json.dumps(file_entries, indent=2)) + return None + + # Build a minimal multipart/form-data request by hand so we only need + # the standard library (no 'requests' dependency). + boundary = "----DataverseRemoteUploadBoundary" + body_parts = [ + f"--{boundary}\r\n" + f'Content-Disposition: form-data; name="jsonData"\r\n\r\n' + f"{json_data}\r\n", + f"--{boundary}--\r\n", + ] + body = "".join(body_parts).encode("utf-8") + + req = urllib.request.Request( + url, + data=body, + method="POST", + headers={ + "X-Dataverse-key": api_key, + "Content-Type": f"multipart/form-data; boundary={boundary}", + }, + ) + + try: + with urllib.request.urlopen(req) as resp: + raw = resp.read().decode("utf-8") + except urllib.error.HTTPError as exc: + raw = exc.read().decode("utf-8") + print(f"\n[ERROR] HTTP {exc.code}: {exc.reason}", file=sys.stderr) + print(raw, file=sys.stderr) + sys.exit(1) + + return json.loads(raw) + + +# --------------------------------------------------------------------------- +# CLI +# --------------------------------------------------------------------------- + +def parse_args(): + p = argparse.ArgumentParser( + description="Register remote-store file references in a Dataverse dataset." + ) + p.add_argument( + "--server", required=True, + help="Base URL of the Dataverse server, e.g. https://demo.dataverse.org", + ) + p.add_argument( + "--api-key", required=True, + help="Dataverse API token.", + ) + p.add_argument( + "--store-id", required=True, + help="Remote store store-id configured in Dataverse, e.g. 'trs'.", + ) + p.add_argument( + "--local-offset", required=True, + help=( + "Local path prefix to strip before building the storage identifier, " + "e.g. /mnt/data → trs:///project/files/foo.csv" + ), + ) + p.add_argument( + "--pid", required=True, + help="Dataset persistent identifier, e.g. doi:10.5072/FK27U7YBV", + ) + p.add_argument( + "--base-dir", required=True, + help="Local directory tree to scan for files.", + ) + p.add_argument( + "--dry-run", action="store_true", + help="Build and print the JSON payload but do NOT call the API.", + ) + p.add_argument( + "--batch-size", type=int, default=100, + help="Number of files to send per API call (default: 100).", + ) + return p.parse_args() + + +def main(): + args = parse_args() + + base_dir = os.path.abspath(args.base_dir) + local_offset = os.path.abspath(args.local_offset) + + if not os.path.isdir(base_dir): + print(f"[ERROR] --base-dir '{base_dir}' is not a directory.", file=sys.stderr) + sys.exit(1) + + print(f"Scanning {base_dir} …") + all_files = sorted(collect_files(base_dir)) + print(f"Found {len(all_files)} file(s).\n") + + if not all_files: + print("Nothing to do.") + return + + # Build metadata for every file. + entries = [] + for path in all_files: + print(f"Processing: {path}") + try: + meta = build_file_metadata(path, args.store_id, local_offset, base_dir) + entries.append(meta) + except ValueError as exc: + print(f" [SKIP] {exc}", file=sys.stderr) + + if not entries: + print("No valid entries produced.") + return + + # Send in batches. + batch_size = max(1, args.batch_size) + for i in range(0, len(entries), batch_size): + batch = entries[i : i + batch_size] + print(f"\nBatch {i // batch_size + 1}: registering files {i + 1}–{i + len(batch)}") + result = add_files_to_dataset( + server=args.server, + api_key=args.api_key, + pid=args.pid, + file_entries=batch, + dry_run=args.dry_run, + ) + if result is not None: + status = result.get("status", "?") + print(f" Status: {status}") + data = result.get("data", {}) + summary = data.get("Result", {}) + if summary: + for k, v in summary.items(): + print(f" {k}: {v}") + # Print per-file errors if any. + for f in data.get("Files", []): + if "errorMessage" in f: + fname = f.get("fileDetails", {}).get("fileName", "?") + print(f" [FILE ERROR] {fname}: {f['errorMessage']}", file=sys.stderr) + + print("\nDone.") + + +if __name__ == "__main__": + main() From b4603de42624b4688259d376a2ee81bb4013cb85 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Fri, 12 Jun 2026 10:41:53 -0400 Subject: [PATCH 2/5] fix separators, add docs --- python/add_remote_files/README.md | 60 +++++++++++++++++++++ python/add_remote_files/add_remote_files.py | 12 +++-- python/add_remote_files/requirements.txt | 1 + 3 files changed, 68 insertions(+), 5 deletions(-) create mode 100644 python/add_remote_files/README.md create mode 100644 python/add_remote_files/requirements.txt diff --git a/python/add_remote_files/README.md b/python/add_remote_files/README.md new file mode 100644 index 0000000..e766936 --- /dev/null +++ b/python/add_remote_files/README.md @@ -0,0 +1,60 @@ +# Add Remote Files to Dataverse + +This script registers files in a local directory tree as remote-store references in a Dataverse dataset. + +## Overview + +No file bytes are transferred; only metadata (storage identifier, filename, MIME type, MD5 hash) is sent to the Dataverse API. + +Dataverse and the specific dataset must be configured to use a remote store - see https://guides.dataverse.org/en/latest/installation/config.html#trusted-remote-storage + +## Prerequisites + +- Python 3.x +- Dataverse API Token +- A dataset configured for a remote store + +## Installation + +This script uses only the Python standard library, so no external dependencies are required. + +```bash +python3 -m venv venv +# On Windows +venv\Scripts\activate +# On macOS/Linux +source venv/bin/activate +``` + +## Usage + +```bash +python add_remote_files.py \ + --server https://dataverse.example.org \ + --api-key xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx \ + --store-id trs \ + --local-offset /mnt/data \ + --pid doi:10.5072/FK27U7YBV \ + --base-dir /mnt/data/project/files +``` + +### Parameters + +- `--server`: Base URL of the Dataverse server. +- `--api-key`: Your Dataverse API token. +- `--store-id`: Remote store ID configured in Dataverse (e.g., `trs`). +- `--local-offset`: Local path prefix to strip before building the storage identifier. +- `--pid`: Dataset persistent identifier (e.g., DOI). +- `--base-dir`: Local directory tree to scan for files. +- `--dry-run`: (Optional) Build and print the JSON payload without calling the API. +- `--batch-size`: (Optional) Number of files to send per API call (default: 100). + +## How it works + +1. The script scans the `--base-dir` for all files. +2. For each file, it: + - Calculates the MD5 hash. + - Guesses the MIME type. + - Constructs a `storageIdentifier` by stripping the `--local-offset` from the absolute file path and prefixing it with the `--store-id`. + - Determines the `directoryLabel` based on the relative path from `--local-offset`. +3. It sends the metadata in batches to the Dataverse `addFiles` API. diff --git a/python/add_remote_files/add_remote_files.py b/python/add_remote_files/add_remote_files.py index f1bb8fd..f4a4b5e 100644 --- a/python/add_remote_files/add_remote_files.py +++ b/python/add_remote_files/add_remote_files.py @@ -10,16 +10,16 @@ The remote store id must match the --store-id parameter in this script, and the configured base-url must correspond to the --local-offset used, i.e. for a base-url https://example.com/shareddata, the URL -https://example.com/sharedddata/file.txt must correspond the local path +https://example.com/shareddata/file.txt must correspond the local path /file.txt. Further, a file in the --base-dir is expected to correspond to a URL matching the base-url plus the relative path difference between the --local-offset and the file path. (With the usage example below, a file.txt in the base dir should be accessible at -https://example.com/sharedddata/project/files/file.txt) +https://example.com/shareddata/project/files/file.txt) Usage ----- -python dataverse_remote_store_upload.py \\ +python add_remote_files.py \\ --server https://dataverse.example.org \\ --api-key xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx \\ --store-id trs \\ @@ -103,7 +103,9 @@ def build_storage_identifier(store_id: str, local_offset: str, abs_path: str) -> f"local-offset '{local_offset}'" ) - remaining = abs_path[len(local_offset):] # starts with '/' on POSIX + remaining = abs_path[len(local_offset):] + # Ensure we use forward slashes for the storage identifier + remaining = remaining.replace(os.sep, "/") if not remaining.startswith("/"): remaining = "/" + remaining @@ -149,7 +151,7 @@ def build_file_metadata( "description": "", } if dir_label: - entry["directoryLabel"] = dir_label + entry["directoryLabel"] = dir_label.replace(os.sep, "/") return entry diff --git a/python/add_remote_files/requirements.txt b/python/add_remote_files/requirements.txt new file mode 100644 index 0000000..eae3d58 --- /dev/null +++ b/python/add_remote_files/requirements.txt @@ -0,0 +1 @@ +# No external dependencies required (uses standard library only) From fa34e7572867acb82f4a15bcca33ea2508c4f65b Mon Sep 17 00:00:00 2001 From: qqmyers Date: Fri, 12 Jun 2026 10:55:20 -0400 Subject: [PATCH 3/5] rename to web-root --- python/add_remote_files/README.md | 8 ++--- python/add_remote_files/add_remote_files.py | 36 ++++++++++----------- 2 files changed, 22 insertions(+), 22 deletions(-) diff --git a/python/add_remote_files/README.md b/python/add_remote_files/README.md index e766936..11bbddf 100644 --- a/python/add_remote_files/README.md +++ b/python/add_remote_files/README.md @@ -33,7 +33,7 @@ python add_remote_files.py \ --server https://dataverse.example.org \ --api-key xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx \ --store-id trs \ - --local-offset /mnt/data \ + --web-root /mnt/data \ --pid doi:10.5072/FK27U7YBV \ --base-dir /mnt/data/project/files ``` @@ -43,7 +43,7 @@ python add_remote_files.py \ - `--server`: Base URL of the Dataverse server. - `--api-key`: Your Dataverse API token. - `--store-id`: Remote store ID configured in Dataverse (e.g., `trs`). -- `--local-offset`: Local path prefix to strip before building the storage identifier. +- `--web-root`: Local path prefix corresponding to the website root directory. - `--pid`: Dataset persistent identifier (e.g., DOI). - `--base-dir`: Local directory tree to scan for files. - `--dry-run`: (Optional) Build and print the JSON payload without calling the API. @@ -55,6 +55,6 @@ python add_remote_files.py \ 2. For each file, it: - Calculates the MD5 hash. - Guesses the MIME type. - - Constructs a `storageIdentifier` by stripping the `--local-offset` from the absolute file path and prefixing it with the `--store-id`. - - Determines the `directoryLabel` based on the relative path from `--local-offset`. + - Constructs a `storageIdentifier` by stripping the `--web-root` from the absolute file path and prefixing it with the `--store-id`. + - Determines the `directoryLabel` based on the relative path from `--web-root`. 3. It sends the metadata in batches to the Dataverse `addFiles` API. diff --git a/python/add_remote_files/add_remote_files.py b/python/add_remote_files/add_remote_files.py index f4a4b5e..4a57f52 100644 --- a/python/add_remote_files/add_remote_files.py +++ b/python/add_remote_files/add_remote_files.py @@ -8,12 +8,12 @@ Dataverse and the specific dataset must be configured to use a remote store. The remote store id must match the --store-id parameter in this script, and -the configured base-url must correspond to the --local-offset used, i.e. +the configured base-url must correspond to the --web-root used, i.e. for a base-url https://example.com/shareddata, the URL https://example.com/shareddata/file.txt must correspond the local path -/file.txt. Further, a file in the --base-dir is expected to +/file.txt. Further, a file in the --base-dir is expected to correspond to a URL matching the base-url plus the relative path difference -between the --local-offset and the file path. (With the usage example below, +between the --web-root and the file path. (With the usage example below, a file.txt in the base dir should be accessible at https://example.com/shareddata/project/files/file.txt) @@ -23,18 +23,18 @@ --server https://dataverse.example.org \\ --api-key xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx \\ --store-id trs \\ - --local-offset /mnt/data \\ + --web-root /mnt/data \\ --pid doi:10.5072/FK27U7YBV \\ --base-dir /mnt/data/project/files Storage-identifier construction -------------------------------- Given: - --local-offset /mnt/data + --web-root /mnt/data --store-id trs file path /mnt/data/project/files/subdir/file.csv -The local offset is stripped from the absolute path to produce: +The web root is stripped from the absolute path to produce: /project/files/subdir/file.csv The storage identifier becomes: @@ -81,29 +81,29 @@ def guess_mime(filename: str) -> str: return mime or "application/octet-stream" -def build_storage_identifier(store_id: str, local_offset: str, abs_path: str) -> str: +def build_storage_identifier(store_id: str, web_root: str, abs_path: str) -> str: """ - Strip *local_offset* from the start of *abs_path* to get the canonical + Strip *web_root* from the start of *abs_path* to get the canonical remote path, then format it as ://. Example ------- store-id = "trs" - local_offset = "/mnt/data" + web_root = "/mnt/data" abs_path = "/mnt/data/project/files/foo.csv" → "trs:///project/files/foo.csv" """ # Normalise both paths so trailing slashes etc. don't cause problems. - local_offset = os.path.normpath(local_offset) + web_root = os.path.normpath(web_root) abs_path = os.path.normpath(abs_path) - if not abs_path.startswith(local_offset): + if not abs_path.startswith(web_root): raise ValueError( f"File path '{abs_path}' does not start with " - f"local-offset '{local_offset}'" + f"web-root '{web_root}'" ) - remaining = abs_path[len(local_offset):] + remaining = abs_path[len(web_root):] # Ensure we use forward slashes for the storage identifier remaining = remaining.replace(os.sep, "/") if not remaining.startswith("/"): @@ -124,13 +124,13 @@ def collect_files(base_dir: str): def build_file_metadata( abs_path: str, store_id: str, - local_offset: str, + web_root: str, base_dir: str, verbose: bool = True, ) -> dict: """Return the JSON-serialisable metadata dict for one file.""" filename = os.path.basename(abs_path) - storage_id = build_storage_identifier(store_id, local_offset, abs_path) + storage_id = build_storage_identifier(store_id, web_root, abs_path) mime = guess_mime(filename) if verbose: @@ -238,7 +238,7 @@ def parse_args(): help="Remote store store-id configured in Dataverse, e.g. 'trs'.", ) p.add_argument( - "--local-offset", required=True, + "--web-root", required=True, help=( "Local path prefix to strip before building the storage identifier, " "e.g. /mnt/data → trs:///project/files/foo.csv" @@ -267,7 +267,7 @@ def main(): args = parse_args() base_dir = os.path.abspath(args.base_dir) - local_offset = os.path.abspath(args.local_offset) + web_root = os.path.abspath(args.web_root) if not os.path.isdir(base_dir): print(f"[ERROR] --base-dir '{base_dir}' is not a directory.", file=sys.stderr) @@ -286,7 +286,7 @@ def main(): for path in all_files: print(f"Processing: {path}") try: - meta = build_file_metadata(path, args.store_id, local_offset, base_dir) + meta = build_file_metadata(path, args.store_id, web_root, base_dir) entries.append(meta) except ValueError as exc: print(f" [SKIP] {exc}", file=sys.stderr) From e0aa99c986f466cb9b276311a726f5638417e99c Mon Sep 17 00:00:00 2001 From: qqmyers Date: Fri, 12 Jun 2026 11:49:42 -0400 Subject: [PATCH 4/5] check for existing files, add limit param, update docs --- README.md | 1 + python/add_remote_files/README.md | 11 +-- python/add_remote_files/add_remote_files.py | 80 ++++++++++++++++++--- 3 files changed, 78 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index afad2ca..3a956ae 100644 --- a/README.md +++ b/README.md @@ -32,6 +32,7 @@ In the following sections, you can find a list of available recipes for each lan - [Create datasets from Excel files](python/create_datasets_from_excel) 📊 - [Download Croissant from draft dataset](python/download_draft_croissant) - [Create Croissant from the client side](python/create_croissant_client_side) +- [Add remote files to a dataset](python/add_remote_files) 📂 ### Shell 🐚 diff --git a/python/add_remote_files/README.md b/python/add_remote_files/README.md index 11bbddf..ac39b41 100644 --- a/python/add_remote_files/README.md +++ b/python/add_remote_files/README.md @@ -46,15 +46,18 @@ python add_remote_files.py \ - `--web-root`: Local path prefix corresponding to the website root directory. - `--pid`: Dataset persistent identifier (e.g., DOI). - `--base-dir`: Local directory tree to scan for files. +- `--limit`: (Optional) Only register the first N files that don't exist on the server. - `--dry-run`: (Optional) Build and print the JSON payload without calling the API. - `--batch-size`: (Optional) Number of files to send per API call (default: 100). ## How it works 1. The script scans the `--base-dir` for all files. -2. For each file, it: - - Calculates the MD5 hash. +2. It fetches the existing file list from the Dataverse dataset to avoid duplicates. +3. For each file, it: + - Constructs a `storageIdentifier` and checks if it already exists in the dataset. + - Calculates the MD5 hash (only for new files). - Guesses the MIME type. - - Constructs a `storageIdentifier` by stripping the `--web-root` from the absolute file path and prefixing it with the `--store-id`. - Determines the `directoryLabel` based on the relative path from `--web-root`. -3. It sends the metadata in batches to the Dataverse `addFiles` API. +4. If `--limit` is specified, it stops after finding the requested number of new files. +5. It sends the metadata in batches to the Dataverse `addFiles` API. diff --git a/python/add_remote_files/add_remote_files.py b/python/add_remote_files/add_remote_files.py index 4a57f52..76deeb7 100644 --- a/python/add_remote_files/add_remote_files.py +++ b/python/add_remote_files/add_remote_files.py @@ -42,10 +42,13 @@ API used -------- -POST /api/datasets/:persistentId/addFiles?persistentId= -with a multipart/form-data field "jsonData" containing a JSON array of -file-metadata objects (the "add multiple files" variant of the Direct -DataFile Upload/Replace API). +1. GET /api/datasets/:persistentId/versions/:latest/files?persistentId= + to list existing files and avoid duplicates. + +2. POST /api/datasets/:persistentId/addFiles?persistentId= + with a multipart/form-data field "jsonData" containing a JSON array of + file-metadata objects (the "add multiple files" variant of the Direct + DataFile Upload/Replace API). """ import argparse @@ -123,14 +126,12 @@ def collect_files(base_dir: str): def build_file_metadata( abs_path: str, - store_id: str, - web_root: str, + storage_id: str, base_dir: str, verbose: bool = True, ) -> dict: """Return the JSON-serialisable metadata dict for one file.""" filename = os.path.basename(abs_path) - storage_id = build_storage_identifier(store_id, web_root, abs_path) mime = guess_mime(filename) if verbose: @@ -157,9 +158,48 @@ def build_file_metadata( # --------------------------------------------------------------------------- -# Dataverse API call +# Dataverse API calls # --------------------------------------------------------------------------- +def get_existing_storage_identifiers(server: str, api_key: str, pid: str, store_id: str) -> set[str]: + """Fetch the list of files in the dataset and return their storage identifiers.""" + encoded_pid = urllib.parse.quote(pid, safe="") + url = f"{server.rstrip('/')}/api/datasets/:persistentId/versions/:latest/files?persistentId={encoded_pid}" + + req = urllib.request.Request( + url, + headers={ + "X-Dataverse-key": api_key, + }, + ) + + try: + with urllib.request.urlopen(req) as resp: + raw = resp.read().decode("utf-8") + data = json.loads(raw) + except urllib.error.HTTPError as exc: + raw = exc.read().decode("utf-8") + print(f"\n[ERROR] Failed to list dataset contents: HTTP {exc.code}: {exc.reason}", file=sys.stderr) + print(raw, file=sys.stderr) + sys.exit(1) + except Exception as exc: + print(f"\n[ERROR] Failed to list dataset contents: {exc}", file=sys.stderr) + sys.exit(1) + + ids = set() + if data.get("status") == "OK": + for file_info in data.get("data", []): + storage_id = file_info.get("dataFile", {}).get("storageIdentifier") + if storage_id: + parsed = urllib.parse.urlparse(storage_id) + if parsed.scheme == store_id: + # Dataverse may return identifiers like ://// + # We normalize them to :/// for local comparison. + normalized = f"{parsed.scheme}:///{parsed.path.lstrip('/')}" + ids.add(normalized) + return ids + + def add_files_to_dataset( server: str, api_key: str, @@ -260,6 +300,10 @@ def parse_args(): "--batch-size", type=int, default=100, help="Number of files to send per API call (default: 100).", ) + p.add_argument( + "--limit", type=int, + help="Only register the first N files that don't exist on the server.", + ) return p.parse_args() @@ -281,16 +325,32 @@ def main(): print("Nothing to do.") return + print(f"Checking existing files in dataset {args.pid} …") + existing_ids = get_existing_storage_identifiers(args.server, args.api_key, args.pid, args.store_id) + print(f"Found {len(existing_ids)} matching existing file(s) in dataset.\n") + # Build metadata for every file. entries = [] + skipped_count = 0 for path in all_files: - print(f"Processing: {path}") + if args.limit is not None and len(entries) >= args.limit: + break + try: - meta = build_file_metadata(path, args.store_id, web_root, base_dir) + storage_id = build_storage_identifier(args.store_id, web_root, path) + if storage_id in existing_ids: + skipped_count += 1 + continue + + print(f"Processing: {path}") + meta = build_file_metadata(path, storage_id, base_dir) entries.append(meta) except ValueError as exc: print(f" [SKIP] {exc}", file=sys.stderr) + if skipped_count > 0: + print(f"\nSkipped {skipped_count} file(s) that already exist in the dataset.") + if not entries: print("No valid entries produced.") return From 997bcbf742f4c3c103ab10f4679b0150a45700d3 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Tue, 21 Jul 2026 16:11:52 -0400 Subject: [PATCH 5/5] add --web-path parameter to support prefixes not in path --- python/add_remote_files/README.md | 4 +- python/add_remote_files/add_remote_files.py | 102 ++++++++++++++------ 2 files changed, 76 insertions(+), 30 deletions(-) diff --git a/python/add_remote_files/README.md b/python/add_remote_files/README.md index ac39b41..09d5911 100644 --- a/python/add_remote_files/README.md +++ b/python/add_remote_files/README.md @@ -34,6 +34,7 @@ python add_remote_files.py \ --api-key xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx \ --store-id trs \ --web-root /mnt/data \ + --web-path /optional-prefix \ --pid doi:10.5072/FK27U7YBV \ --base-dir /mnt/data/project/files ``` @@ -44,6 +45,7 @@ python add_remote_files.py \ - `--api-key`: Your Dataverse API token. - `--store-id`: Remote store ID configured in Dataverse (e.g., `trs`). - `--web-root`: Local path prefix corresponding to the website root directory. +- `--web-path`: (Optional) Prefix to add to the web path in the storage identifier, useful when the web-root does not match the store's base-url. - `--pid`: Dataset persistent identifier (e.g., DOI). - `--base-dir`: Local directory tree to scan for files. - `--limit`: (Optional) Only register the first N files that don't exist on the server. @@ -58,6 +60,6 @@ python add_remote_files.py \ - Constructs a `storageIdentifier` and checks if it already exists in the dataset. - Calculates the MD5 hash (only for new files). - Guesses the MIME type. - - Determines the `directoryLabel` based on the relative path from `--web-root`. + - Determines the `directoryLabel` based on the relative path from `--web-root`, including any `--web-path` prefix. 4. If `--limit` is specified, it stops after finding the requested number of new files. 5. It sends the metadata in batches to the Dataverse `addFiles` API. diff --git a/python/add_remote_files/add_remote_files.py b/python/add_remote_files/add_remote_files.py index 76deeb7..1371e34 100644 --- a/python/add_remote_files/add_remote_files.py +++ b/python/add_remote_files/add_remote_files.py @@ -7,38 +7,52 @@ identifier, filename, MIME type, MD5 hash) is sent to the Dataverse API. Dataverse and the specific dataset must be configured to use a remote store. -The remote store id must match the --store-id parameter in this script, and -the configured base-url must correspond to the --web-root used, i.e. -for a base-url https://example.com/shareddata, the URL -https://example.com/shareddata/file.txt must correspond the local path -/file.txt. Further, a file in the --base-dir is expected to -correspond to a URL matching the base-url plus the relative path difference -between the --web-root and the file path. (With the usage example below, -a file.txt in the base dir should be accessible at -https://example.com/shareddata/project/files/file.txt) +The remote store id must match the --store-id parameter in this script. + +The configured base-url of the remote store usually corresponds to the +--web-root directory. If it doesn't, the --web-path parameter can be used +to specify the additional path components. + +Example: + base-url: https://example.com/shareddata + web-root: /mnt/data + File /mnt/data/file.txt is accessible at https://example.com/shareddata/file.txt + (No --web-path needed) + +Example with --web-path: + base-url: https://web.tacc.utexas.edu + web-root: /corral/Texas-Robotics/web + web-path: /texasrobotics + File /corral/Texas-Robotics/web/file.txt is accessible at + https://web.tacc.utexas.edu/texasrobotics/file.txt Usage ----- python add_remote_files.py \\ --server https://dataverse.example.org \\ --api-key xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx \\ - --store-id trs \\ + --store-id trs \\ --web-root /mnt/data \\ + --web-path /optional-prefix \\ --pid doi:10.5072/FK27U7YBV \\ --base-dir /mnt/data/project/files -Storage-identifier construction --------------------------------- +Storage-identifier and directoryLabel construction +------------------------------------------------ Given: --web-root /mnt/data - --store-id trs - file path /mnt/data/project/files/subdir/file.csv + --web-path /prefix (optional) + --store-id trs + file path /mnt/data/project/files/subdir/file.csv -The web root is stripped from the absolute path to produce: - /project/files/subdir/file.csv +The web root is stripped from the absolute path and the web path is prepended: + /prefix/project/files/subdir/file.csv The storage identifier becomes: - trs:///project/files/subdir/file.csv + trs:///prefix/project/files/subdir/file.csv + +The directoryLabel becomes: + prefix/project/files/subdir API used -------- @@ -84,17 +98,19 @@ def guess_mime(filename: str) -> str: return mime or "application/octet-stream" -def build_storage_identifier(store_id: str, web_root: str, abs_path: str) -> str: +def build_storage_identifier(store_id: str, web_root: str, abs_path: str, web_path: str = "") -> str: """ Strip *web_root* from the start of *abs_path* to get the canonical - remote path, then format it as ://. + remote path. If *web_path* is provided, it is prepended to the result. + The final path is formatted as ://. Example ------- store-id = "trs" - web_root = "/mnt/data" + web_root = "/mnt/data" abs_path = "/mnt/data/project/files/foo.csv" - → "trs:///project/files/foo.csv" + web_path = "/remote/url/prefix" + → "trs:///remote/url/prefix/project/files/foo.csv" """ # Normalise both paths so trailing slashes etc. don't cause problems. web_root = os.path.normpath(web_root) @@ -112,9 +128,17 @@ def build_storage_identifier(store_id: str, web_root: str, abs_path: str) -> str if not remaining.startswith("/"): remaining = "/" + remaining + full_path = remaining + if web_path: + # Prepend web_path, ensuring we don't double up on slashes + wp = web_path.replace(os.sep, "/").rstrip("/") + if not wp.startswith("/"): + wp = "/" + wp + full_path = wp + remaining + # Use double-slash after the scheme so the path is clearly absolute: # trs:///some/path (scheme + empty authority + absolute path) - return f"{store_id}://{remaining}" + return f"{store_id}://{full_path}" def collect_files(base_dir: str): @@ -127,7 +151,8 @@ def collect_files(base_dir: str): def build_file_metadata( abs_path: str, storage_id: str, - base_dir: str, + web_root: str, + web_path: str = "", verbose: bool = True, ) -> dict: """Return the JSON-serialisable metadata dict for one file.""" @@ -140,9 +165,19 @@ def build_file_metadata( if verbose: print(md5) - # Derive a directoryLabel relative to base_dir (optional but useful). - rel = os.path.relpath(os.path.dirname(abs_path), base_dir) - dir_label = rel if rel != "." else "" + # Derive a directoryLabel relative to web_root, prepending web_path if present. + rel = os.path.relpath(os.path.dirname(abs_path), web_root) + + parts = [] + if web_path: + wp = web_path.strip("/").replace(os.sep, "/") + if wp: + parts.append(wp) + + if rel != ".": + parts.append(rel.replace(os.sep, "/")) + + dir_label = "/".join(parts) entry = { "storageIdentifier": storage_id, @@ -152,7 +187,7 @@ def build_file_metadata( "description": "", } if dir_label: - entry["directoryLabel"] = dir_label.replace(os.sep, "/") + entry["directoryLabel"] = dir_label return entry @@ -284,6 +319,14 @@ def parse_args(): "e.g. /mnt/data → trs:///project/files/foo.csv" ), ) + p.add_argument( + "--web-path", default="", + help=( + "Optional prefix to add to the web path in the storage identifier, " + "useful when the web-root does not match the store's base-url. " + "e.g. /texasrobotics → trs:///texasrobotics/project/files/foo.csv" + ), + ) p.add_argument( "--pid", required=True, help="Dataset persistent identifier, e.g. doi:10.5072/FK27U7YBV", @@ -312,6 +355,7 @@ def main(): base_dir = os.path.abspath(args.base_dir) web_root = os.path.abspath(args.web_root) + web_path = args.web_path if not os.path.isdir(base_dir): print(f"[ERROR] --base-dir '{base_dir}' is not a directory.", file=sys.stderr) @@ -337,13 +381,13 @@ def main(): break try: - storage_id = build_storage_identifier(args.store_id, web_root, path) + storage_id = build_storage_identifier(args.store_id, web_root, path, web_path) if storage_id in existing_ids: skipped_count += 1 continue print(f"Processing: {path}") - meta = build_file_metadata(path, storage_id, base_dir) + meta = build_file_metadata(path, storage_id, web_root, web_path) entries.append(meta) except ValueError as exc: print(f" [SKIP] {exc}", file=sys.stderr)