Addition of WW artic parsers, large-scale shake-up of parser structure.

src/submissions/backend/db/functions.py

@@ -21,6 +21,7 @@ import numpy as np
 import yaml
 from pathlib import Path
 
+
 logger = logging.getLogger(f"submissions.{__name__}")
 
 # The below _should_ allow automatic creation of foreign keys in the database
@@ -41,12 +42,19 @@ def store_submission(ctx:dict, base_submission:models.BasicSubmission) -> None|d
     Returns:
         None|dict : object that indicates issue raised for reporting in gui
     """    
-    from tools import format_rsl_number
+    from tools import RSLNamer
     logger.debug(f"Hello from store_submission")
     # Add all samples to sample table
-    base_submission.rsl_plate_num = format_rsl_number(base_submission.rsl_plate_num)
+    typer = RSLNamer(base_submission.rsl_plate_num)
+    base_submission.rsl_plate_num = typer.parsed_name
     for sample in base_submission.samples:
-        sample.rsl_plate = base_submission
+        logger.debug(f"Typer: {typer.submission_type}")
+        # Suuuuuper hacky way to be sure that the artic doesn't overwrite the ww plate in a ww sample
+        # need something more elegant
+        if "_artic" not in typer.submission_type:
+            sample.rsl_plate = base_submission
+        else:
+            sample.artic_rsl_plate = base_submission
         logger.debug(f"Attempting to add sample: {sample.to_string()}")
         try:
             ctx['database_session'].add(sample)
@@ -152,7 +160,7 @@ def construct_submission_info(ctx:dict, info_dict:dict) -> models.BasicSubmissio
                 # Because of unique constraint, there will be problems with 
                 # multiple submissions named 'None', so...
                 logger.debug(f"Submitter plate id: {info_dict[item]}")
-                if info_dict[item] == None or info_dict[item] == "None":
+                if info_dict[item] == None or info_dict[item] == "None" or info_dict[item] == "":
                     logger.debug(f"Got None as a submitter plate number, inserting random string to preserve database unique constraint.")
                     info_dict[item] = uuid.uuid4().hex.upper()
                 field_value = info_dict[item]
@@ -170,8 +178,6 @@ def construct_submission_info(ctx:dict, info_dict:dict) -> models.BasicSubmissio
         # ceil(instance.sample_count / 8) will get number of columns
         # the cost of a full run multiplied by (that number / 12) is x twelfths the cost of a full run
         logger.debug(f"Calculating costs for procedure...")
-        # cols_count = ceil(int(instance.sample_count) / 8)
-        # instance.run_cost = instance.extraction_kit.constant_cost + (instance.extraction_kit.mutable_cost * (cols_count / 12))
         instance.calculate_base_cost()
     except (TypeError, AttributeError) as e:
         logger.debug(f"Looks like that kit doesn't have cost breakdown yet due to: {e}, using full plate cost.")
@@ -471,7 +477,7 @@ def create_kit_from_yaml(ctx:dict, exp:dict) -> dict:
     Returns:
         dict: a dictionary containing results of db addition
     """    
-    from tools import check_is_power_user
+    from tools import check_is_power_user, massage_common_reagents
     # Don't want just anyone adding kits
     if not check_is_power_user(ctx=ctx):
         logger.debug(f"{getuser()} does not have permission to add kits.")
@@ -491,6 +497,7 @@ def create_kit_from_yaml(ctx:dict, exp:dict) -> dict:
             # A kit contains multiple reagent types.
             for r in exp[type]['kits'][kt]['reagenttypes']:
                 # check if reagent type already exists.
+                r = massage_common_reagents(r)
                 look_up = ctx['database_session'].query(models.ReagentType).filter(models.ReagentType.name==r).first()
                 if look_up == None:
                     rt = models.ReagentType(name=r.replace(" ", "_").lower(), eol_ext=timedelta(30*exp[type]['kits'][kt]['reagenttypes'][r]['eol_ext']), kits=[kit])
@@ -689,7 +696,10 @@ def delete_submission_by_id(ctx:dict, id:int) -> None:
         pass
     sub.reagents = []
     for sample in sub.samples:
-        ctx['database_session'].delete(sample)
+        if sample.rsl_plate == sub:
+            ctx['database_session'].delete(sample)
+        else:
+            logger.warning(f"Not deleting sample {sample.ww_sample_full_id} because it belongs to another plate.")
     ctx["database_session"].delete(sub)
     ctx["database_session"].commit()
 
@@ -706,6 +716,19 @@ def lookup_ww_sample_by_rsl_sample_number(ctx:dict, rsl_number:str) -> models.WW
     """    
     return ctx['database_session'].query(models.WWSample).filter(models.WWSample.rsl_number==rsl_number).first()
 
+def lookup_ww_sample_by_ww_sample_num(ctx:dict, sample_number:str) -> models.WWSample:
+    """
+    Retrieves wastewater sample from database by ww sample number
+
+    Args:
+        ctx (dict): settings passed down from gui
+        sample_number (str): sample number assigned by wastewater
+
+    Returns:
+        models.WWSample: instance of wastewater sample
+    """    
+    return ctx['database_session'].query(models.WWSample).filter(models.WWSample.ww_sample_full_id==sample_number).first()
+
 def lookup_ww_sample_by_sub_sample_rsl(ctx:dict, sample_rsl:str, plate_rsl:str) -> models.WWSample:
     """
     Retrieves a wastewater sample from the database by its rsl sample number and parent rsl plate number.
@@ -775,7 +798,6 @@ def lookup_discounts_by_org_and_kit(ctx:dict, kit_id:int, lab_id:int):
         models.Organization.id==lab_id
         )).all()
 
-
 def hitpick_plate(submission:models.BasicSubmission, plate_number:int=0) -> list:
     plate_dicto = []
     for sample in submission.samples:

src/submissions/backend/db/models/__init__.py

@@ -10,4 +10,4 @@ from .controls import Control, ControlType
 from .kits import KitType, ReagentType, Reagent, Discount
 from .organizations import Organization, Contact
 from .samples import WWSample, BCSample
-from .submissions import BasicSubmission, BacterialCulture, Wastewater
+from .submissions import BasicSubmission, BacterialCulture, Wastewater, WastewaterArtic

src/submissions/backend/db/models/samples.py

@@ -37,6 +37,8 @@ class WWSample(Base):
     sample_type = Column(String(8))
     pcr_results = Column(JSON)
     elution_well = Column(String(8)) #: location on 96 well plate
+    artic_rsl_plate = relationship("WastewaterArtic", back_populates="samples")
+    artic_well_number = Column(String(8))
     
 
     def to_string(self) -> str:
@@ -131,3 +133,41 @@ class BCSample(Base):
             "well": self.well_number,
             "name": f"{self.sample_id} - ({self.organism})",
         }
+
+
+# class ArticSample(Base):
+#     """
+#     base of artic sample
+#     """    
+#     __tablename__ = "_artic_samples"
+
+#     id = Column(INTEGER, primary_key=True) #: primary key
+#     well_number = Column(String(8)) #: location on parent plate
+#     rsl_plate = relationship("WastewaterArtic", back_populates="samples") #: relationship to parent plate
+#     rsl_plate_id = Column(INTEGER, ForeignKey("_submissions.id", ondelete="SET NULL", name="fk_WWA_submission_id"))
+#     ww_sample_full_id = Column(String(64), nullable=False)
+#     lims_sample_id = Column(String(64), nullable=False)
+#     ct_1 = Column(FLOAT(2)) #: first ct value in column
+#     ct_2 = Column(FLOAT(2)) #: second ct value in column
+
+#     def to_string(self) -> str:
+#         """
+#         string representing sample object
+
+#         Returns:
+#             str: string representing location and sample id
+#         """        
+#         return f"{self.well_number}: {self.ww_sample_full_id}"
+    
+#     def to_sub_dict(self) -> dict:
+#         """
+#         gui friendly dictionary
+
+#         Returns:
+#             dict: well location and name (sample id, organism) NOTE: keys must sync with WWSample to_sub_dict above
+#         """
+#         return {
+#             "well": self.well_number,
+#             "name": self.ww_sample_full_id,
+#         }
+    

src/submissions/backend/db/models/submissions.py

@@ -161,7 +161,16 @@ class BasicSubmission(Base):
         }
         return output
     
-    
+    def calculate_base_cost(self):
+        try:
+            cols_count_96 = ceil(int(self.sample_count) / 8)
+        except Exception as e:
+            logger.error(f"Column count error: {e}")
+        # cols_count_24 = ceil(int(self.sample_count) / 3)
+        try:
+            self.run_cost = self.extraction_kit.constant_cost + (self.extraction_kit.mutable_cost_column * cols_count_96) + (self.extraction_kit.mutable_cost_sample * int(self.sample_count))
+        except Exception as e:
+            logger.error(f"Calculation error: {e}")
 
 # Below are the custom submission types
 
@@ -185,16 +194,16 @@ class  BacterialCulture(BasicSubmission):
         return output
     
 
-    def calculate_base_cost(self):
-        try:
-            cols_count_96 = ceil(int(self.sample_count) / 8)
-        except Exception as e:
-            logger.error(f"Column count error: {e}")
-        # cols_count_24 = ceil(int(self.sample_count) / 3)
-        try:
-            self.run_cost = self.extraction_kit.constant_cost + (self.extraction_kit.mutable_cost_column * cols_count_96) + (self.extraction_kit.mutable_cost_sample * int(self.sample_count))
-        except Exception as e:
-            logger.error(f"Calculation error: {e}")
+    # def calculate_base_cost(self):
+    #     try:
+    #         cols_count_96 = ceil(int(self.sample_count) / 8)
+    #     except Exception as e:
+    #         logger.error(f"Column count error: {e}")
+    #     # cols_count_24 = ceil(int(self.sample_count) / 3)
+    #     try:
+    #         self.run_cost = self.extraction_kit.constant_cost + (self.extraction_kit.mutable_cost_column * cols_count_96) + (self.extraction_kit.mutable_cost_sample * int(self.sample_count))
+    #     except Exception as e:
+    #         logger.error(f"Calculation error: {e}")
     
 
 class Wastewater(BasicSubmission):
@@ -220,14 +229,22 @@ class Wastewater(BasicSubmission):
             pass
         return output
     
-    def calculate_base_cost(self):
-        try:
-            cols_count_96 = ceil(int(self.sample_count) / 8) + 1 #: Adding in one column to account for 24 samples + ext negatives
-        except Exception as e:
-            logger.error(f"Column count error: {e}")
-        # cols_count_24 = ceil(int(self.sample_count) / 3)
-        try:
-            self.run_cost = self.extraction_kit.constant_cost + (self.extraction_kit.mutable_cost_column * cols_count_96) + (self.extraction_kit.mutable_cost_sample * int(self.sample_count))
-        except Exception as e:
-            logger.error(f"Calculation error: {e}")
-    
+    # def calculate_base_cost(self):
+    #     try:
+    #         cols_count_96 = ceil(int(self.sample_count) / 8) + 1 #: Adding in one column to account for 24 samples + ext negatives
+    #     except Exception as e:
+    #         logger.error(f"Column count error: {e}")
+    #     # cols_count_24 = ceil(int(self.sample_count) / 3)
+    #     try:
+    #         self.run_cost = self.extraction_kit.constant_cost + (self.extraction_kit.mutable_cost_column * cols_count_96) + (self.extraction_kit.mutable_cost_sample * int(self.sample_count))
+    #     except Exception as e:
+    #         logger.error(f"Calculation error: {e}")
+
+
+class WastewaterArtic(BasicSubmission):
+    """
+    derivative submission type for artic wastewater
+    """    
+    samples = relationship("WWSample", back_populates="artic_rsl_plate", uselist=True)
+    # Can in use the pcr_info from the wastewater? Cause I can't define pcr_info here due to conflicts with that
+    __mapper_args__ = {"polymorphic_identity": "wastewater_artic", "polymorphic_load": "inline"}

src/submissions/backend/excel/parser.py

@@ -2,18 +2,19 @@
 contains parser object for pulling values from client generated submission sheets.
 '''
 from getpass import getuser
+import math
 from typing import Tuple
 import pandas as pd
 from pathlib import Path
 from backend.db.models import WWSample, BCSample
-# from backend.db import lookup_ww_sample_by_rsl_sample_number
+from backend.db import lookup_ww_sample_by_ww_sample_num
 import logging
 from collections import OrderedDict
 import re
 import numpy as np
 from datetime import date, datetime
 import uuid
-from tools import check_not_nan, RSLNamer
+from tools import check_not_nan, RSLNamer, massage_common_reagents
 
 logger = logging.getLogger(f"submissions.{__name__}")
 
@@ -21,20 +22,22 @@ class SheetParser(object):
     """
     object to pull and contain data from excel file
     """
-    def __init__(self, filepath:Path|None = None, **kwargs):
+    def __init__(self, ctx:dict, filepath:Path|None = None):
         """
         Args:
             filepath (Path | None, optional): file path to excel sheet. Defaults to None.
         """
+        self.ctx = ctx
         logger.debug(f"Parsing {filepath.__str__()}")
         # set attributes based on kwargs from gui ctx
-        for kwarg in kwargs:
-            setattr(self, f"_{kwarg}", kwargs[kwarg])
+        # for kwarg in kwargs:
+        #     setattr(self, f"_{kwarg}", kwargs[kwarg])
         # self.__dict__.update(kwargs)
         if filepath == None:
             logger.error(f"No filepath given.")
             self.xl = None
         else:
+            self.filepath = filepath
             try:
                 self.xl = pd.ExcelFile(filepath.__str__())
             except ValueError as e:
@@ -55,8 +58,8 @@ class SheetParser(object):
             str: submission type name
         """        
         try:
-            for type in self._submission_types:
-                if self.xl.sheet_names == self._submission_types[type]['excel_map']:
+            for type in self.ctx['submission_types']:
+                if self.xl.sheet_names == self.ctx['submission_types'][type]['excel_map']:
                     return type.title()
             return "Unknown"
         except Exception as e:
@@ -74,7 +77,7 @@ class SheetParser(object):
 
     def parse_generic(self, sheet_name:str) -> pd.DataFrame:
         """
-        Pulls information common to all submission types and passes on dataframe
+        Pulls information common to all wasterwater/bacterial culture types and passes on dataframe
 
         Args:
             sheet_name (str): name of excel worksheet to pull from
@@ -107,8 +110,6 @@ class SheetParser(object):
             """            
             for ii, row in df.iterrows():
                 # skip positive control
-                # if ii == 12:
-                #     continue
                 logger.debug(f"Running reagent parse for {row[1]} with type {type(row[1])} and value: {row[2]} with type {type(row[2])}")
                 if not isinstance(row[2], float) and check_not_nan(row[1]):
                     # must be prefixed with 'lot_' to be recognized by gui
@@ -156,7 +157,7 @@ class SheetParser(object):
         logger.debug(reagent_range)
         parse_reagents(reagent_range)
         # get individual sample info
-        sample_parser = SampleParser(submission_info.iloc[16:112])
+        sample_parser = SampleParser(self.ctx, submission_info.iloc[16:112])
         sample_parse = getattr(sample_parser, f"parse_{self.sub['submission_type'].lower()}_samples")
         logger.debug(f"Parser result: {self.sub}")
         self.sub['samples'] = sample_parse()
@@ -181,6 +182,7 @@ class SheetParser(object):
                     # regex below will remove 80% from 80% ethanol in the Wastewater kit.
                     output_key = re.sub(r"^\d{1,3}%\s?", "", row[0].lower().strip().replace(' ', '_'))
                     output_key = output_key.strip("_")
+                    # output_var is the lot number
                     try:
                         output_var = row[5].upper()
                     except AttributeError:
@@ -214,24 +216,97 @@ class SheetParser(object):
         parse_reagents(ext_reagent_range)
         parse_reagents(pcr_reagent_range)
         # parse samples
-        sample_parser = SampleParser(submission_info.iloc[16:])
+        sample_parser = SampleParser(self.ctx, submission_info.iloc[16:])
         sample_parse = getattr(sample_parser, f"parse_{self.sub['submission_type'].lower()}_samples")
         self.sub['samples'] = sample_parse()
         self.sub['csv'] = self.xl.parse("Copy to import file", dtype=object)
 
 
+    def parse_wastewater_artic(self) -> None:
+        """
+        pulls info specific to wastewater_arctic submission type
+        """
+        def parse_reagents(df:pd.DataFrame):
+            logger.debug(df)
+            for ii, row in df.iterrows():
+                if check_not_nan(row[0]):
+                    try:
+                        output_key = re.sub(r"\(.+?\)", "", row[0].lower().strip().replace(' ', '_'))
+                    except AttributeError:
+                        continue
+                    output_key = output_key.strip("_")
+                    output_key = massage_common_reagents(output_key)
+                    try:
+                        output_var = row[1].upper()
+                    except AttributeError:
+                        logger.debug(f"Couldn't upperize {row[1]}, must be a number")
+                        output_var = row[1]
+                    logger.debug(f"Output variable is {output_var}")
+                    logger.debug(f"Expiry date for imported reagent: {row[2]}")
+                    if check_not_nan(row[2]):
+                        try:
+                            expiry = row[2].date()
+                        except AttributeError as e:
+                            try:
+                                expiry = datetime.strptime(row[2], "%Y-%m-%d")
+                            except TypeError as e:
+                                expiry = datetime.fromordinal(datetime(1900, 1, 1).toordinal() + row[2] - 2)
+                            except ValueError as e:
+                                continue
+                    else:
+                        logger.debug(f"Date: {row[2]}")
+                        expiry = date.today()
+                    self.sub[f"lot_{output_key}"] = {'lot':output_var, 'exp':expiry}
+                else:
+                    continue
+        def massage_samples(df:pd.DataFrame) -> pd.DataFrame:
+            df.set_index(df.columns[0], inplace=True)
+            df.columns = df.iloc[0]
+            logger.debug(f"df to massage\n: {df}")
+            return_list = []
+            for _, ii in df.iloc[1:,1:].iterrows():
+                for c in df.columns.to_list():
+                    logger.debug(f"Checking {ii.name}{c}")
+                    if check_not_nan(df.loc[ii.name, int(c)]) and df.loc[ii.name, int(c)] != "EMPTY":
+                        
+                        return_list.append(dict(sample_name=re.sub(r"\s?\(.*\)", "", df.loc[ii.name, int(c)]), \
+                                                well=f"{ii.name}{c}",
+                                                artic_plate=self.sub['rsl_plate_num']))
+            logger.debug(f"massaged sample list for {self.sub['rsl_plate_num']}: {return_list}")
+            return return_list
+        submission_info = self.xl.parse("First Strand", dtype=object)
+        biomek_info = self.xl.parse("ArticV4 Biomek", dtype=object)
+        sub_reagent_range = submission_info.iloc[56:, 1:4].dropna(how='all')
+        biomek_reagent_range = biomek_info.iloc[60:, 0:3].dropna(how='all')
+        self.sub['submitter_plate_num'] = ""
+        self.sub['rsl_plate_num'] =  RSLNamer(self.filepath.__str__()).parsed_name
+        self.sub['submitted_date'] = submission_info.iloc[0][2]
+        self.sub['submitting_lab'] = "Enterics Wastewater Genomics"
+        self.sub['sample_count'] = submission_info.iloc[4][6]
+        self.sub['extraction_kit'] = "ArticV4.1"
+        self.sub['technician'] = f"MM: {biomek_info.iloc[2][1]}, Bio: {biomek_info.iloc[3][1]}"
+        parse_reagents(sub_reagent_range)
+        parse_reagents(biomek_reagent_range)
+        samples = massage_samples(biomek_info.iloc[22:31, 0:])
+        sample_parser = SampleParser(self.ctx, pd.DataFrame.from_records(samples))
+        sample_parse = getattr(sample_parser, f"parse_{self.sub['submission_type'].lower()}_samples")
+        self.sub['samples'] = sample_parse()
+        
+
+
 class SampleParser(object):
     """
     object to pull data for samples in excel sheet and construct individual sample objects
     """
 
-    def __init__(self, df:pd.DataFrame) -> None:
+    def __init__(self, ctx:dict, df:pd.DataFrame) -> None:
         """
         convert sample sub-dataframe to dictionary of records
 
         Args:
             df (pd.DataFrame): input sample dataframe
         """        
+        self.ctx = ctx
         self.samples = df.to_dict("records")
 
 
@@ -295,6 +370,29 @@ class SampleParser(object):
             new.well_number = sample['Unnamed: 1']
             new_list.append(new)
         return new_list
+    
+    def parse_wastewater_artic_samples(self) -> list[WWSample]:
+        """
+        The artic samples are the wastewater samples that are to be sequenced
+        So we will need to lookup existing ww samples and append Artic well # and plate relation
+
+        Returns:
+            list[WWSample]: list of wastewater samples to be updated
+        """        
+        new_list = []
+        for sample in self.samples:
+            with self.ctx['database_session'].no_autoflush:
+                instance = lookup_ww_sample_by_ww_sample_num(ctx=self.ctx, sample_number=sample['sample_name'])
+            logger.debug(f"Checking: {sample['sample_name']}")
+            if instance == None:
+                logger.error(f"Unable to find match for: {sample['sample_name']}")
+                continue
+            logger.debug(f"Got instance: {instance.ww_sample_full_id}")
+            instance.artic_well_number = sample['well']
+            new_list.append(instance)
+        return new_list
+            
+
 
 
 class PCRParser(object):