Add support for SamplesheetV2 and MiSeq sequencer

core/models.py

@@ -1791,7 +1791,7 @@ def create_sequencer_in_lab(self, sequencer_value):
             platform_obj = SequencingPlatform.objects.get(
                 pk__exact=sequencer_value["platformID"]
             )
-        except models.SequencingPlatform.DoesNotExist:
+        except SequencingPlatform.DoesNotExist:
             platform_obj = None
         new_sequencer = self.create(
             platform_id=platform_obj,

core/scripts/migrate_sample_type.py

@@ -1,6 +1,5 @@
 import core.models
 
-
 """
     The script is applicable for the upgrade from 2.3.0 to 3.0.0.
     Because the new version changes the value that is stored now is the field

core/scripts/rename_app_name.py

@@ -1,6 +1,5 @@
 import core.models
 
-
 """
     The script is applicable for the upgrade from 2.3.0 to 3.0.0.
     Because the application in iSkylims have been renamed, this required that

core/utils/samples.py

@@ -1878,7 +1878,7 @@ def record_extract_protocol(samples, excel_data, heading, user, app_name):
             return_data.append(r_data)
         # collect data for dropdown selection
         protocol_filter_selection = []
-        (protocols_dict, protocol_list) = get_molecule_protocols(app_name)
+        protocols_dict, protocol_list = get_molecule_protocols(app_name)
         for key, value in protocols_dict.items():
             protocol_filter_selection.append([key, value])
         return {

django_utils/templatetags/user_text.py

@@ -1,6 +1,5 @@
 from django import template
 
-
 register = template.Library()
 
 

drylab/scripts/drylab_service_state_migration.py

@@ -1,6 +1,5 @@
 from drylab.models import Service, ServiceState
 
-
 """
     The script is applicable for the upgrade from 2.3.0 to 2.3.1.
     Service state that was defined as option choice in  models,is replaced in

drylab/templatetags/upload_tags.py

@@ -6,8 +6,7 @@
 
 @register.simple_tag
 def upload_js():
-    return mark_safe(
-        """
+    return mark_safe("""
             <!-- The template to display files available for upload -->
             <script id="template-upload" type="text/x-tmpl">
             {% for (var i=0, file; file=o.files[i]; i++) { %}
@@ -96,5 +95,4 @@ def upload_js():
                 </tr>
             {% } %}
             </script>
-            """
-    )
+            """)

wetlab/config.py

@@ -77,8 +77,8 @@
 ]
 
 # ########### VALUE TAG FOR XML FILES #########################
-COMPLETION_TAG = "CompletionStatus"
-COMPLETION_SUCCESS = ["CompletedAsPlanned", "SuccessfullyCompleted"]
+COMPLETION_TAG = ["CompletionStatus", "RunStatus"]
+COMPLETION_SUCCESS = ["CompletedAsPlanned", "SuccessfullyCompleted", "RunCompleted"]
 EXPERIMENT_NAME_TAG = "ExperimentName"
 APPLICATION_NAME_TAG = "ApplicationName"
 NUMBER_CYCLES_TAG = "NumCycles"
@@ -226,15 +226,10 @@
 HEADING_FOR_SAMPLE_SHEET_TWO_INDEX = [
     "Unique_Sample_ID",
     "Sample_Name",
-    "Sample_Plate",
-    "Sample_Well",
-    "Index_Plate_Well",
-    "I7_Index_ID",
     "index",
-    "I5_Index_ID",
     "index2",
     "Sample_Project",
-    "Description",
+    "custom_description",
 ]
 
 
@@ -248,6 +243,7 @@
     ("index", "i7Index"),
     ("Sample_Project", "projectInSampleSheet"),
     ("Description", "userInSampleSheet"),
+    ("custom_description", "userInSampleSheet"),
 ]
 
 MAP_USER_SAMPLE_SHEET_TO_DATABASE_NEXTSEQ_PAIRED_END = [
@@ -261,6 +257,7 @@
     ("index2", "i5Index"),
     ("Sample_Project", "projectInSampleSheet"),
     ("Description", "userInSampleSheet"),
+    ("custom_description", "userInSampleSheet"),
 ]
 
 MAP_USER_SAMPLE_SHEET_TO_DATABASE_MISEQ_SINGLE_READ_VERSION_5 = [
@@ -272,6 +269,7 @@
     ("index", "i7Index"),
     ("Sample_Project", "projectInSampleSheet"),
     ("Description", "userInSampleSheet"),
+    ("custom_description", "userInSampleSheet"),
 ]
 
 MAP_USER_SAMPLE_SHEET_TO_DATABASE_MISEQ_PAiRED_END_VERSION_5 = [
@@ -285,6 +283,7 @@
     ("index2", "i5Index"),
     ("Sample_Project", "projectInSampleSheet"),
     ("Description", "userInSampleSheet"),
+    ("custom_description", "userInSampleSheet"),
 ]
 
 MAP_USER_SAMPLE_SHEET_TO_DATABASE_MISEQ_SINGLE_READ_VERSION_4 = [
@@ -296,6 +295,7 @@
     ("index", "i7Index"),
     ("Sample_Project", "projectInSampleSheet"),
     ("Description", "userInSampleSheet"),
+    ("custom_description", "userInSampleSheet"),
 ]
 
 MAP_USER_SAMPLE_SHEET_TO_DATABASE_MISEQ_PAiRED_END_VERSION_4 = [
@@ -309,6 +309,7 @@
     ("index2", "i5Index"),
     ("Sample_Project", "projectInSampleSheet"),
     ("Description", "userInSampleSheet"),
+    ("custom_description", "userInSampleSheet"),
 ]
 
 
@@ -326,6 +327,7 @@
     ("GenomeFolder", "genomeFolder"),
     ("Sample_Project", "projectInSampleSheet"),
     ("Description", "userInSampleSheet"),
+    ("custom_description", "userInSampleSheet"),
 ]
 # ######## MAPPING OPTIONAL COLUMNS THAT COULD BE IN SAMPLE SHEET FROM USER TO DATABASE   #############
 MAP_USER_SAMPLE_SHEET_ADDITIONAL_FIELDS_FROM_TYPE_OF_SECUENCER = [
@@ -335,7 +337,23 @@
 ]
 
 # Sections to check in the IEM file created by user
-SECTIONS_IN_IEM_SAMPLE_SHEET = ["[Header]", "[Reads]", "[Settings]", "[Data]"]
+SECTIONS_IN_IEM_SAMPLE_SHEET = ["Header", "Reads", "Settings", "Data"]
+SECTIONS_IN_V2_SAMPLE_SHEET = [
+    "Header",
+    "Reads",
+    "BCLConvert_Settings",
+    "BCLConvert_Data",
+    "CustomCustomer_Data",
+]
+
+# Possible names of the Tabular data iskylims user field
+
+TABULAR_DATA_ISKYLIMS_USER_COLUMN = {"1": "Description", "2": "custom_description"}
+
+# Tabular data sections in samplesheets.
+TABULAR_DATA_SECTIONS_SAMPLE_SHEET = {"1": "Data", "2": "BCLConvert_Data"}
+
+SETTINGS_SECTIONS_SAMPLE_SHEET = ["Settings", "BCLConvert_Settings"]
 
 FIELDS_IN_SAMPLE_SHEET_HEADER_IEM_VERSION_5 = [
     "Date",
@@ -349,6 +367,11 @@
     "Description",
 ]
 
+ADAPTER_1_FIELD_NAMES = ["Adapter", "AdapterRead1"]
+
+ADAPTER_2_FIELD_NAMES = ["Adapter", "AdapterRead2"]
+
+
 # #### HEADINGS VALUES
 
 # # Heading for pending Library Preparation state
@@ -686,6 +709,9 @@
 ERROR_SAMPLE_SHEET_USER_ARE_NOT_DEFINED = (
     "Sample sheet has users which are not defined : "
 )
+ERROR_SAMPLE_SHEET_HAS_INVALID_LINES = (
+    "Sample sheet has an invalid (Non-empty, non-comma-delimited) line: "
+)
 ERROR_USER_SAMPLE_SHEET_NO_LONGER_EXISTS = [
     "The Sample Sheet that you are uploaded does not longer exists",
     "Upload again the sample sheet",

wetlab/utils/crontab_process.py

@@ -299,9 +299,10 @@ def check_sequencer_status_from_completion_file(l_run_completion, experiment_nam
         experiment_name,
     )
     # check if NextSEq run have been successful completed
-    status_run = wetlab.utils.common.find_xml_tag_text(
-        l_run_completion, wetlab.config.COMPLETION_TAG
-    )
+    for xml_tag in wetlab.config.COMPLETION_TAG:
+        status_run = wetlab.utils.common.find_xml_tag_text(l_run_completion, xml_tag)
+        if status_run != "NOT FOUND":
+            break
     if status_run not in wetlab.config.COMPLETION_SUCCESS:
         logger.info(
             "%s : Run in sequencer was not completed but %s",
@@ -445,7 +446,7 @@ def check_sequencer_run_is_completed(
             "%s : End function check_sequencer_run_is_completed with exception",
             experiment_name,
         )
-        return "cancelled", ""
+        return "cancelled", None
     elif way_to_check == "txt_file":
         # l_run_completion = os.path.join(RUN_TEMP_DIRECTORY, RUN_COMPLETION_TXT_FILE)
         s_run_completion = os.path.join(
@@ -1119,6 +1120,8 @@ def parsing_run_info_and_parameter_information(
                 wetlab.utils.common.logging_warnings(string_message, False)
     # get date for miSeq and NextSeq with the format yymmdd
     date = p_run.find("Date").text
+    # Remove timestamp
+    date = date.split("T")[0]
     try:
         run_date = datetime.datetime.strptime(date, "%y%m%d")
     except Exception:

wetlab/utils/crontab_update_run.py

@@ -330,7 +330,7 @@ def search_update_new_runs(request_reason):
                     == "TRUE"
                 ):
                     user_id_list = wetlab.utils.common.get_userid_list()
-                    file_read = wetlab.utils.samplesheet.read_user_iem_file(
+                    file_read = wetlab.utils.samplesheet.read_file_from_path(
                         l_sample_sheet_path
                     )
                     users = wetlab.utils.samplesheet.validate_userid_in_user_iem_file(
@@ -584,10 +584,7 @@ def manage_run_in_recorded_state(conn, run_process_objs):
                 run_process_obj, l_sample_sheet_path, experiment_name
             )
 
-        if (
-            wetlab.config.COPY_SAMPLE_SHEET_TO_REMOTE
-            and "NextSeq" in run_process_obj.get_run_platform()
-        ):
+        if "NextSeq" in run_process_obj.get_run_platform():
             sample_sheet_path = run_process_obj.get_sample_file()
 
             try:

wetlab/utils/library.py

@@ -859,19 +859,19 @@ def format_sample_sheet_to_display_in_form(sample_sheet_data):
         display_data["adapter2"] = True
 
     display_data["sample_data"] = sample_sheet_data["sample_data"]
-    display_data["heading"] = sample_sheet_data["heading"]
+    display_data["headers"] = sample_sheet_data["headers"]
     main_values = []
     for item in extract_values:
         main_values.append(sample_sheet_data[item])
     summary_values = []
     summary_values.append(len(sample_sheet_data["samples"]))
-    summary_values.append(sample_sheet_data["proyects"])
+    summary_values.append(sample_sheet_data["projects"])
     summary_values.append(sample_sheet_data["userid_names"])
     display_data["main_data"] = list(zip(main_data_heading, main_values))
     display_data["summary_data"] = list(
         zip(wetlab.config.HEADING_SUMMARY_DATA_SAMPLE_SHEET, summary_values)
     )
-    display_data["heading_excel"] = ",".join(sample_sheet_data["heading"])
+    display_data["headers_excel"] = ",".join(sample_sheet_data["headers"])
     # if len(sample_sheet_data['userid_names']) == 0:
     #    display_data['no_user_defined'] = True
 

wetlab/utils/run.py

@@ -34,19 +34,21 @@ def check_run_already_defined_by_crontab(exp_name, pool_ids):
     # experiment name already exists, check if samples are the same
     # to confirm the match
     run_obj = wetlab.models.RunProcess.objects.filter(run_name__iexact=exp_name).last()
-    sample_sheet = run_obj.get_sample_file()
-    f_name = os.path.join(settings.MEDIA_ROOT, sample_sheet)
+    sample_sheet_path = run_obj.get_sample_file()
+    f_name = os.path.join(settings.MEDIA_ROOT, sample_sheet_path)
     try:
-        with open(f_name, "r") as fh:
-            file_data = fh.readlines()
+        file_data = wetlab.utils.samplesheet.read_file_from_path(f_name)
+        samplesheet = wetlab.utils.samplesheet.file_read_to_dictionary(file_data)
     except FileNotFoundError:
         error_message = str(
             wetlab.config.ERROR_RUN_NAME_BY_CRONTAB_ALREADY_CREATED
             + " but  "
             + wetlab.config.ERROR_SAMPLE_SHEET_NOT_FOUND_WHEN_CREATED_BY_CRONTAB
         )
         return {"ERROR": error_message}
-    sample_in_s_sheet = wetlab.utils.samplesheet.get_samples_in_sample_sheet(file_data)
+    sample_in_s_sheet = wetlab.utils.samplesheet.get_samples_in_sample_sheet(
+        samplesheet
+    )
     sample_in_pools = wetlab.utils.pool.get_sample_name_in_pools(pool_ids)
     if len(sample_in_pools) != len(sample_in_s_sheet["samples"]):
         return {"ERROR": wetlab.config.ERROR_EXISTING_RUN_WITH_DIF_SAMPLES_AS_IN_CRON}