Metadata commands

ifcbdb/common/constants.py

@@ -17,4 +17,6 @@ class BinManagementActions(Enum):
     UNASSIGN_DATASET = "unassign-dataset"
 
 # Metadata column names
-BIN_ID_COLUMNS = ['id','pid','lid','bin','bin_id','sample','sample_id','filename']
+BIN_ID_COLUMNS = ['id','pid','lid','bin','bin_id','sample','sample_id','filename']
+ADD_DATASET_COLUMNS = ['add_dataset', 'adddataset']
+REMOVE_DATASET_COLUMNS = ['remove_dataset', 'removedataset', 'delete_dataset', 'deletedataset']

ifcbdb/dashboard/accession.py

@@ -13,7 +13,8 @@
 import pandas as pd
 import numpy as np
 
-from .models import Bin, DataDirectory, Instrument, Timeline, Dataset, normalize_tag_name, Team, TeamDataset
+from .models import Bin, DataDirectory, Instrument, Timeline, Dataset, Tag, TagEvent, \
+    normalize_tag_name, Team, TeamDataset
 from .qaqc import check_bad, check_no_rois
 
 import ifcb
@@ -312,6 +313,10 @@ def import_metadata(metadata_dataframe, progress_callback=do_nothing):
     CAST_COLUMNS = ['cast']
     NISKIN_COLUMNS = ['niskin','bottle']
     SAMPLE_TYPE_COLUMNS = ['sampletype','sample_type']
+    ADD_DATASET_COLUMNS = ['add_dataset', 'adddataset']
+    REMOVE_DATASET_COLUMNS = ['remove_dataset', 'removedataset', 'delete_dataset', 'deletedataset']
+    ADD_TAG_COLUMNS = ['add_tag', 'addtag']
+    REMOVE_TAG_COLUMNS = ['remove_tag', 'removetag', 'delete_tag', 'deletetag']
 
     SKIP_POSITIVE_VALUES = ['skip','yes','y','true','t','1']
     SKIP_NEGATIVE_VALUES = ['noskip','no','n','false','f','0']
@@ -361,6 +366,11 @@ def get_cell(named_tup, key):
 
     sample_type_col = get_column(df, SAMPLE_TYPE_COLUMNS)
 
+    add_dataset_col = get_column(df, ADD_DATASET_COLUMNS)
+    remove_dataset_col = get_column(df, REMOVE_DATASET_COLUMNS)
+    add_tag_col = get_column(df, ADD_TAG_COLUMNS)
+    remove_tag_col = get_column(df, REMOVE_TAG_COLUMNS)
+
     tag_cols = []
     for c in df.columns:
         if c.startswith('tag'):
@@ -467,6 +477,43 @@ def get_cell(named_tup, key):
                 if body is not None:
                     b.add_comment(body, skip_duplicates=True)
 
+            # command columns
+
+            if add_dataset_col is not None:
+                dataset_name = get_cell(row, add_dataset_col)
+                if str(dataset_name or "").strip():
+                    try:
+                        dataset = Dataset.objects.get(name__iexact=dataset_name)
+                        b.datasets.add(dataset)
+                    except Dataset.DoesNotExist:
+                        raise ValueError(f"Dataset '{dataset_name}' not found for {add_dataset_col}")
+
+            if remove_dataset_col is not None:
+                dataset_name = get_cell(row, remove_dataset_col)
+                if str(dataset_name or "").strip():
+                    try:
+                        dataset = Dataset.objects.get(name__iexact=dataset_name)
+                        b.datasets.remove(dataset)
+                    except Dataset.DoesNotExist:
+                        pass
+
+            if add_tag_col is not None:
+                tag = get_cell(row, add_tag_col)
+                if str(tag or "").strip():
+                    if re.match(r"^[0-9\.]+$", str(tag)):
+                        raise ValueError(f"tag '{tag}' consists of only digits")
+                    normalized = normalize_tag_name(str(tag))
+                    b.add_tag(normalized)
+
+            if remove_tag_col is not None:
+                tag = get_cell(row, remove_tag_col)
+                if str(tag or "").strip():
+                    normalized = normalize_tag_name(str(tag))
+                    try:
+                        b.delete_tag(normalized)
+                    except (Tag.DoesNotExist, TagEvent.DoesNotExist):
+                        pass
+
             # skip flag
 
             if skip_col is not None:
@@ -597,6 +644,10 @@ def add(field, rename=None):
         r['comment_summary'].append(comment_summary_by_id.get(item['id'], ''))
         r['trigger_selection'].append(trigger_selection_by_id.get(item['id'], ''))
         r['skip'].append(1 if item['skip'] else 0)
+        r['add_dataset'].append('')
+        r['remove_dataset'].append('')
+        r['add_tag'].append('')
+        r['remove_tag'].append('')
 
     df = pd.DataFrame(r)
 

ifcbdb/secure/views.py

@@ -20,7 +20,7 @@
 
 from dashboard.accession import export_metadata
 from common import auth
-from common.constants import Features, TeamRoles, BinManagementActions, BIN_ID_COLUMNS
+from common.constants import Features, TeamRoles, BinManagementActions, BIN_ID_COLUMNS, ADD_DATASET_COLUMNS, REMOVE_DATASET_COLUMNS
 
 
 from django.core.cache import cache
@@ -869,6 +869,27 @@ def get_column(df, possible_names):
                 bins_ids = list(bins_ids)
 
                 df = df[df[pid_col].isin(bins_ids)]
+
+                # Blank out any dataset names that are not associated with the user uploading the data to
+                #   ensure they cannot assign or unassign bins to datasets they are not privy to
+                add_dataset_col = get_column(df, ADD_DATASET_COLUMNS)
+                remove_dataset_col = get_column(df, REMOVE_DATASET_COLUMNS)
+
+                if add_dataset_col is not None or remove_dataset_col is not None:
+                    valid_dataset_names = auth \
+                        .get_associated_datasets(request.user) \
+                        .values_list("name", flat=True)
+
+                    if add_dataset_col is not None:
+                        df[add_dataset_col] = df[add_dataset_col].apply(
+                            lambda x: x if pd.isna(x) or str(x).strip() in valid_dataset_names else ""
+                        )
+
+                    if remove_dataset_col is not None:
+                        df[remove_dataset_col] = df[remove_dataset_col].apply(
+                            lambda x: x if pd.isna(x) or str(x).strip() in valid_dataset_names else ""
+                        )
+
                 json_df = df.to_json()
 
             added = cache.add(METADATA_UPLOAD_LOCK_KEY, True, timeout=None) # this is atomic

ifcbdb/templates/secure/upload-metadata.html

@@ -49,6 +49,22 @@
                         <div class="form-group">
                             <input id="submit" type="submit" value="Upload" class="btn btn-sm btn-mdb-color" />
                         </div>
+                        <div class="form-group mt-4">
+                            <small class="text-muted">
+                                <strong>Supported Columns:</strong>
+                                <ul class="mb-0 pl-3">
+                                    <li><strong>Bin ID:</strong> id, pid, lid, bin, bin_id, sample, sample_id, filename</li>
+                                    <li><strong>Location:</strong> latitude/lat/y, longitude/lon/lng/x</li>
+                                    <li><strong>Timestamp:</strong> date, timestamp, datetime</li>
+                                    <li><strong>Depth:</strong> depth, dep, z</li>
+                                    <li><strong>Comments:</strong> comment, comments, note, notes</li>
+                                    <li><strong>Sample Info:</strong> cruise, cast, niskin, bottle, sample_type</li>
+                                    <li><strong>Tags:</strong> tag*, add_tag, remove_tag</li>
+                                    <li><strong>Datasets:</strong> add_dataset, remove_dataset</li>
+                                    <li><strong>Other:</strong> ml_analyzed, skip</li>
+                                </ul>
+                            </small>
+                        </div>
                     </form>
                 </div>
             </div>