From 205dc161d4a89e4766f9fa14abb3a4f2cb58029f Mon Sep 17 00:00:00 2001 From: Mike Chagnon Date: Tue, 21 Apr 2026 20:25:40 -0700 Subject: [PATCH 1/3] metadata_commands: first draft --- ifcbdb/dashboard/accession.py | 55 ++++++++++++++++++++++++++++++++++- 1 file changed, 54 insertions(+), 1 deletion(-) diff --git a/ifcbdb/dashboard/accession.py b/ifcbdb/dashboard/accession.py index 444f218e..77b6bd0e 100644 --- a/ifcbdb/dashboard/accession.py +++ b/ifcbdb/dashboard/accession.py @@ -13,7 +13,8 @@ import pandas as pd import numpy as np -from .models import Bin, DataDirectory, Instrument, Timeline, Dataset, normalize_tag_name, Team, TeamDataset +from .models import Bin, DataDirectory, Instrument, Timeline, Dataset, Tag, TagEvent, \ + normalize_tag_name, Team, TeamDataset from .qaqc import check_bad, check_no_rois import ifcb @@ -312,6 +313,10 @@ def import_metadata(metadata_dataframe, progress_callback=do_nothing): CAST_COLUMNS = ['cast'] NISKIN_COLUMNS = ['niskin','bottle'] SAMPLE_TYPE_COLUMNS = ['sampletype','sample_type'] + ADD_DATASET_COLUMNS = ['add_dataset', 'adddataset'] + REMOVE_DATASET_COLUMNS = ['remove_dataset', 'removedataset', 'delete_dataset', 'deletedataset'] + ADD_TAG_COLUMNS = ['add_tag', 'addtag'] + REMOVE_TAG_COLUMNS = ['remove_tag', 'removetag', 'delete_tag', 'deletetag'] SKIP_POSITIVE_VALUES = ['skip','yes','y','true','t','1'] SKIP_NEGATIVE_VALUES = ['noskip','no','n','false','f','0'] @@ -361,6 +366,11 @@ def get_cell(named_tup, key): sample_type_col = get_column(df, SAMPLE_TYPE_COLUMNS) + add_dataset_col = get_column(df, ADD_DATASET_COLUMNS) + remove_dataset_col = get_column(df, REMOVE_DATASET_COLUMNS) + add_tag_col = get_column(df, ADD_TAG_COLUMNS) + remove_tag_col = get_column(df, REMOVE_TAG_COLUMNS) + tag_cols = [] for c in df.columns: if c.startswith('tag'): @@ -467,6 +477,45 @@ def get_cell(named_tup, key): if body is not None: b.add_comment(body, skip_duplicates=True) + # command columns + + # TODO: the dataset being added or removed must be accessible by the user (for non-admins) + + if add_dataset_col is not None: + dataset_name = get_cell(row, add_dataset_col) + if str(dataset_name or "").strip(): + try: + dataset = Dataset.objects.get(name__iexact=dataset_name) + b.datasets.add(dataset) + except Dataset.DoesNotExist: + raise ValueError(f"Dataset '{dataset_name}' not found for {add_dataset_col}") + + if remove_dataset_col is not None: + dataset_name = get_cell(row, remove_dataset_col) + if str(dataset_name or "").strip(): + try: + dataset = Dataset.objects.get(name__iexact=dataset_name) + b.datasets.remove(dataset) + except Dataset.DoesNotExist: + pass + + if add_tag_col is not None: + tag = get_cell(row, add_tag_col) + if str(tag or "").strip(): + if re.match(r"^[0-9\.]+$", str(tag)): + raise ValueError(f"tag '{tag}' consists of only digits") + normalized = normalize_tag_name(str(tag)) + b.add_tag(normalized) + + if remove_tag_col is not None: + tag = get_cell(row, remove_tag_col) + if str(tag or "").strip(): + normalized = normalize_tag_name(str(tag)) + try: + b.delete_tag(normalized) + except (Tag.DoesNotExist, TagEvent.DoesNotExist): + pass + # skip flag if skip_col is not None: @@ -597,6 +646,10 @@ def add(field, rename=None): r['comment_summary'].append(comment_summary_by_id.get(item['id'], '')) r['trigger_selection'].append(trigger_selection_by_id.get(item['id'], '')) r['skip'].append(1 if item['skip'] else 0) + r['add_dataset'].append('') + r['remove_dataset'].append('') + r['add_tag'].append('') + r['remove_tag'].append('') df = pd.DataFrame(r) From 87b772cc15e5817a7c373dcb66b249c99dbc750a Mon Sep 17 00:00:00 2001 From: Mike Chagnon Date: Wed, 22 Apr 2026 08:47:14 -0700 Subject: [PATCH 2/3] metadata_commands: add helper text --- ifcbdb/templates/secure/upload-metadata.html | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/ifcbdb/templates/secure/upload-metadata.html b/ifcbdb/templates/secure/upload-metadata.html index 691eceb5..822ed053 100644 --- a/ifcbdb/templates/secure/upload-metadata.html +++ b/ifcbdb/templates/secure/upload-metadata.html @@ -49,6 +49,22 @@
+
+ + Supported Columns: +
    +
  • Bin ID: id, pid, lid, bin, bin_id, sample, sample_id, filename
    • +
  • Location: latitude/lat/y, longitude/lon/lng/x
    • +
  • Timestamp: date, timestamp, datetime
    • +
  • Depth: depth, dep, z
    • +
  • Comments: comment, comments, note, notes
    • +
  • Sample Info: cruise, cast, niskin/bottle, sampletype/sample_type
    • +
  • Tags: tag*, add_tag, remove_tag (or addtag/removetag/deletetag)
    • +
  • Datasets: add_dataset, remove_dataset (or adddataset/removedataset/deletedataset)
    • +
  • Other: ml_analyzed, skip
    • +
+ +
From 038974dd5266a1ec51d45865b39b54d1d5d6267b Mon Sep 17 00:00:00 2001 From: Mike Chagnon Date: Wed, 22 Apr 2026 13:33:46 -0700 Subject: [PATCH 3/3] metadata_commands: restrict datasets for non super admins --- ifcbdb/common/constants.py | 4 +++- ifcbdb/dashboard/accession.py | 2 -- ifcbdb/secure/views.py | 23 +++++++++++++++++++- ifcbdb/templates/secure/upload-metadata.html | 6 ++--- 4 files changed, 28 insertions(+), 7 deletions(-) diff --git a/ifcbdb/common/constants.py b/ifcbdb/common/constants.py index 4ddbea93..91970b15 100644 --- a/ifcbdb/common/constants.py +++ b/ifcbdb/common/constants.py @@ -17,4 +17,6 @@ class BinManagementActions(Enum): UNASSIGN_DATASET = "unassign-dataset" # Metadata column names -BIN_ID_COLUMNS = ['id','pid','lid','bin','bin_id','sample','sample_id','filename'] \ No newline at end of file +BIN_ID_COLUMNS = ['id','pid','lid','bin','bin_id','sample','sample_id','filename'] +ADD_DATASET_COLUMNS = ['add_dataset', 'adddataset'] +REMOVE_DATASET_COLUMNS = ['remove_dataset', 'removedataset', 'delete_dataset', 'deletedataset'] \ No newline at end of file diff --git a/ifcbdb/dashboard/accession.py b/ifcbdb/dashboard/accession.py index 77b6bd0e..f680cbc1 100644 --- a/ifcbdb/dashboard/accession.py +++ b/ifcbdb/dashboard/accession.py @@ -479,8 +479,6 @@ def get_cell(named_tup, key): # command columns - # TODO: the dataset being added or removed must be accessible by the user (for non-admins) - if add_dataset_col is not None: dataset_name = get_cell(row, add_dataset_col) if str(dataset_name or "").strip(): diff --git a/ifcbdb/secure/views.py b/ifcbdb/secure/views.py index 11c6c668..66d4fe8a 100644 --- a/ifcbdb/secure/views.py +++ b/ifcbdb/secure/views.py @@ -20,7 +20,7 @@ from dashboard.accession import export_metadata from common import auth -from common.constants import Features, TeamRoles, BinManagementActions, BIN_ID_COLUMNS +from common.constants import Features, TeamRoles, BinManagementActions, BIN_ID_COLUMNS, ADD_DATASET_COLUMNS, REMOVE_DATASET_COLUMNS from django.core.cache import cache @@ -869,6 +869,27 @@ def get_column(df, possible_names): bins_ids = list(bins_ids) df = df[df[pid_col].isin(bins_ids)] + + # Blank out any dataset names that are not associated with the user uploading the data to + # ensure they cannot assign or unassign bins to datasets they are not privy to + add_dataset_col = get_column(df, ADD_DATASET_COLUMNS) + remove_dataset_col = get_column(df, REMOVE_DATASET_COLUMNS) + + if add_dataset_col is not None or remove_dataset_col is not None: + valid_dataset_names = auth \ + .get_associated_datasets(request.user) \ + .values_list("name", flat=True) + + if add_dataset_col is not None: + df[add_dataset_col] = df[add_dataset_col].apply( + lambda x: x if pd.isna(x) or str(x).strip() in valid_dataset_names else "" + ) + + if remove_dataset_col is not None: + df[remove_dataset_col] = df[remove_dataset_col].apply( + lambda x: x if pd.isna(x) or str(x).strip() in valid_dataset_names else "" + ) + json_df = df.to_json() added = cache.add(METADATA_UPLOAD_LOCK_KEY, True, timeout=None) # this is atomic diff --git a/ifcbdb/templates/secure/upload-metadata.html b/ifcbdb/templates/secure/upload-metadata.html index 822ed053..8d8d2597 100644 --- a/ifcbdb/templates/secure/upload-metadata.html +++ b/ifcbdb/templates/secure/upload-metadata.html @@ -58,9 +58,9 @@
  • Timestamp: date, timestamp, datetime
  • Depth: depth, dep, z
  • Comments: comment, comments, note, notes
    • -
  • Sample Info: cruise, cast, niskin/bottle, sampletype/sample_type
    • -
  • Tags: tag*, add_tag, remove_tag (or addtag/removetag/deletetag)
    • -
  • Datasets: add_dataset, remove_dataset (or adddataset/removedataset/deletedataset)
    • +
  • Sample Info: cruise, cast, niskin, bottle, sample_type
    • +
  • Tags: tag*, add_tag, remove_tag
    • +
  • Datasets: add_dataset, remove_dataset
  • Other: ml_analyzed, skip