moved frontend function check_not_nan to tools

src/submissions/backend/db/__init__.py

@@ -15,6 +15,7 @@ import json
 # from dateutil.relativedelta import relativedelta
 from getpass import getuser
 import numpy as np
+from tools import check_not_nan
 
 logger = logging.getLogger(f"submissions.{__name__}")
 
@@ -101,8 +102,13 @@ def construct_submission_info(ctx:dict, info_dict:dict) -> models.BasicSubmissio
     # convert submission type into model name
     query = info_dict['submission_type'].replace(" ", "")
     # check database for existing object
+    if info_dict["rsl_plate_num"] == 'nan' or info_dict["rsl_plate_num"] == None or not check_not_nan(info_dict["rsl_plate_num"]):
+        code = 2
+        instance = None
+        msg = "A proper RSL plate number is required."
+        return instance, {'code': 2, 'message': "A proper RSL plate number is required."}
     instance = ctx['database_session'].query(models.BasicSubmission).filter(models.BasicSubmission.rsl_plate_num==info_dict['rsl_plate_num']).first()
-    msg = "This submission already exists.\nWould you like to overwrite?"
+    
     # get model based on submission type converted above
     logger.debug(f"Looking at models for submission type: {query}")
     model = getattr(models, query)
@@ -113,6 +119,10 @@ def construct_submission_info(ctx:dict, info_dict:dict) -> models.BasicSubmissio
         instance = model()
         logger.debug(f"Submission doesn't exist yet, creating new instance: {instance}")
         msg = None
+        code =0
+    else:
+        code = 1
+        msg = "This submission already exists.\nWould you like to overwrite?"
     for item in info_dict:
         logger.debug(f"Setting {item} to {info_dict[item]}")
         # set fields based on keys in dictionary
@@ -151,9 +161,14 @@ def construct_submission_info(ctx:dict, info_dict:dict) -> models.BasicSubmissio
     except (TypeError, AttributeError):
         logger.debug(f"Looks like that kit doesn't have cost breakdown yet, using full plate cost.")
         instance.run_cost = instance.extraction_kit.cost_per_run
-    logger.debug(f"Constructed instance: {instance.to_string()}")
+    # We need to make sure there's a proper rsl plate number
+    
+    try:
+        logger.debug(f"Constructed instance: {instance.to_string()}")
+    except AttributeError as e:
+        logger.debug(f"Something went wrong constructing instance {info_dict['rsl_plate_num']}: {e}")
     logger.debug(msg)
-    return instance, {'message':msg}
+    return instance, {'code':code, 'message':msg}
     
 
 def construct_reagent(ctx:dict, info_dict:dict) -> models.Reagent:
@@ -244,7 +259,7 @@ def lookup_kittype_by_use(ctx:dict, used_by:str) -> list[models.KitType]:
     Returns:
         list[models.KitType]: list of kittypes that have that sample type in their uses
     """    
-    return ctx['database_session'].query(models.KitType).filter(models.KitType.used_for.contains(used_by))
+    return ctx['database_session'].query(models.KitType).filter(models.KitType.used_for.contains(used_by)).all()
 
 def lookup_kittype_by_name(ctx:dict, name:str) -> models.KitType:
     """
@@ -278,7 +293,7 @@ def lookup_regent_by_type_name(ctx:dict, type_name:str) -> list[models.Reagent]:
 
 def lookup_regent_by_type_name_and_kit_name(ctx:dict, type_name:str, kit_name:str) -> list[models.Reagent]:
     """
-    Lookup reagents by their type name and kits they belong to (Broken)
+    Lookup reagents by their type name and kits they belong to (Broken... maybe cursed, I'm not sure.)
 
     Args:
         ctx (dict): settings pass by gui
@@ -292,10 +307,6 @@ def lookup_regent_by_type_name_and_kit_name(ctx:dict, type_name:str, kit_name:st
     # Hang on, this is going to be a long one.
     # by_type = ctx['database_session'].query(models.Reagent).join(models.Reagent.type, aliased=True).filter(models.ReagentType.name.endswith(type_name)).all()
     rt_types = ctx['database_session'].query(models.ReagentType).filter(models.ReagentType.name.endswith(type_name))
-    
-    
-    
-
     # add filter for kit name... which I can not get to work.
     # add_in = by_type.join(models.ReagentType.kits).filter(models.KitType.name==kit_name)
     try:
@@ -315,7 +326,7 @@ def lookup_regent_by_type_name_and_kit_name(ctx:dict, type_name:str, kit_name:st
     return output
 
 
-def lookup_all_submissions_by_type(ctx:dict, type:str|None=None) -> list[models.BasicSubmission]:
+def lookup_all_submissions_by_type(ctx:dict, sub_type:str|None=None) -> list[models.BasicSubmission]:
     """
     Get all submissions, filtering by type if given
 
@@ -326,10 +337,10 @@ def lookup_all_submissions_by_type(ctx:dict, type:str|None=None) -> list[models.
     Returns:
         _type_: list of retrieved submissions
     """
-    if type == None:
+    if sub_type == None:
         subs = ctx['database_session'].query(models.BasicSubmission).all()
     else:
-        subs = ctx['database_session'].query(models.BasicSubmission).filter(models.BasicSubmission.submission_type==type.lower().replace(" ", "_")).all()
+        subs = ctx['database_session'].query(models.BasicSubmission).filter(models.BasicSubmission.submission_type==sub_type.lower().replace(" ", "_")).all()
     return subs
 
 def lookup_all_orgs(ctx:dict) -> list[models.Organization]:
@@ -358,7 +369,7 @@ def lookup_org_by_name(ctx:dict, name:str|None) -> models.Organization:
     logger.debug(f"Querying organization: {name}")
     return ctx['database_session'].query(models.Organization).filter(models.Organization.name==name).first()
 
-def submissions_to_df(ctx:dict, type:str|None=None) -> pd.DataFrame:
+def submissions_to_df(ctx:dict, sub_type:str|None=None) -> pd.DataFrame:
     """
     Convert submissions looked up by type to dataframe
 
@@ -369,9 +380,9 @@ def submissions_to_df(ctx:dict, type:str|None=None) -> pd.DataFrame:
     Returns:
         pd.DataFrame: dataframe constructed from retrieved submissions
     """    
-    logger.debug(f"Type: {type}")
+    logger.debug(f"Type: {sub_type}")
     # pass to lookup function
-    subs = [item.to_dict() for item in lookup_all_submissions_by_type(ctx=ctx, type=type)]
+    subs = [item.to_dict() for item in lookup_all_submissions_by_type(ctx=ctx, sub_type=sub_type)]
     df = pd.DataFrame.from_records(subs)
     # logger.debug(f"Pre: {df['Technician']}")
     try:
@@ -435,14 +446,17 @@ def get_all_Control_Types_names(ctx:dict) -> list[models.ControlType]:
     return conTypes
 
 
-def create_kit_from_yaml(ctx:dict, exp:dict) -> None:
+def create_kit_from_yaml(ctx:dict, exp:dict) -> dict:
     """
     Create and store a new kit in the database based on a .yml file
 
     Args:
         ctx (dict): Context dictionary passed down from frontend
         exp (dict): Experiment dictionary created from yaml file
-    """
+
+    Returns:
+        dict: a dictionary containing results of db addition
+    """    
     try:
         power_users = ctx['power_users']
     except KeyError:
@@ -474,6 +488,44 @@ def create_kit_from_yaml(ctx:dict, exp:dict) -> None:
     ctx['database_session'].commit()
     return {'code':0, 'message':'Kit has been added'}
 
+def create_org_from_yaml(ctx:dict, org:dict) -> dict:
+    """
+    Create and store a new organization based on a .yml file
+
+    Args:
+        ctx (dict): Context dictionary passed down from frontend
+        org (dict): Dictionary containing organization info.
+
+    Returns:
+        dict: dictionary containing results of db addition
+    """    
+    try:
+        power_users = ctx['power_users']
+    except KeyError:
+        logger.debug("This user does not have permission to add kits.")
+        return {'code':1,'message':"This user does not have permission to add organizations."}
+    logger.debug(f"Adding organization for user: {getuser()}")
+    if getuser() not in power_users:
+        logger.debug(f"{getuser()} does not have permission to add kits.")
+        return {'code':1, 'message':"This user does not have permission to add organizations."}
+    for client in org:
+        cli_org = models.Organization(name=client.replace(" ", "_").lower(), cost_centre=org[client]['cost centre'])
+        for contact in org[client]['contacts']:
+            cont_name = list(contact.keys())[0]
+            look_up = ctx['database_session'].query(models.Contact).filter(models.Contact.name==cont_name).first()
+            if look_up == None:
+                cli_cont = models.Contact(name=cont_name, phone=contact[cont_name]['phone'], email=contact[cont_name]['email'], organization=[cli_org])
+            else:
+                cli_cont = look_up
+                cli_cont.organization.append(cli_org)
+                # cli_org.contacts.append(cli_cont)
+            # cli_org.contact_ids.append_foreign_key(cli_cont.id)
+            ctx['database_session'].add(cli_cont)
+            logger.debug(cli_cont.__dict__)
+        ctx['database_session'].add(cli_org)
+    ctx["database_session"].commit()
+    return {"code":0, "message":"Organization has been added."}
+
 
 def lookup_all_sample_types(ctx:dict) -> list[str]:
     """
@@ -511,7 +563,7 @@ def get_all_available_modes(ctx:dict) -> list[str]:
 
 
 
-def get_all_controls_by_type(ctx:dict, con_type:str, start_date:date|None=None, end_date:date|None=None) -> list:
+def get_all_controls_by_type(ctx:dict, con_type:str, start_date:date|None=None, end_date:date|None=None) -> list[models.Control]:
     """
     Returns a list of control objects that are instances of the input controltype.
 
@@ -571,4 +623,4 @@ def lookup_submission_by_rsl_num(ctx:dict, rsl_num:str):
 
 
 def lookup_submissions_using_reagent(ctx:dict, reagent:models.Reagent) -> list[models.BasicSubmission]:
-    return ctx['database_session'].query(models.BasicSubmission).join(reagents_submissions).filter(reagents_submissions.c.reagent_id==reagent.id)
+    return ctx['database_session'].query(models.BasicSubmission).join(reagents_submissions).filter(reagents_submissions.c.reagent_id==reagent.id).all()

src/submissions/backend/db/functions/__init__.py

@@ -1,21 +1,21 @@
-from ..models import *
-import logging
+# from ..models import *
+# import logging
 
-logger = logging.getLogger(f"submissions.{__name__}")
+# logger = logging.getLogger(f"submissions.{__name__}")
 
-def check_kit_integrity(sub:BasicSubmission):
-    ext_kit_rtypes = [reagenttype.name for reagenttype in sub.extraction_kit.reagent_types]
-    logger.debug(f"Kit reagents: {ext_kit_rtypes}")
-    reagenttypes = [reagent.type.name for reagent in sub.reagents]
-    logger.debug(f"Submission reagents: {reagenttypes}")
-    check = set(ext_kit_rtypes) == set(reagenttypes)
-    logger.debug(f"Checking if reagents match kit contents: {check}")
-    common = list(set(ext_kit_rtypes).intersection(reagenttypes))
-    logger.debug(f"common reagents types: {common}")
-    if check:
-        result = None
-    else:
-        result = {'message' : f"Couldn't verify reagents match listed kit components.\n\nIt looks like you are missing: {[x.upper for x in ext_kit_rtypes if x not in common]}\n\nAlternatively, you may have set the wrong extraction kit."}
-    return result
+# def check_kit_integrity(sub:BasicSubmission):
+#     ext_kit_rtypes = [reagenttype.name for reagenttype in sub.extraction_kit.reagent_types]
+#     logger.debug(f"Kit reagents: {ext_kit_rtypes}")
+#     reagenttypes = [reagent.type.name for reagent in sub.reagents]
+#     logger.debug(f"Submission reagents: {reagenttypes}")
+#     check = set(ext_kit_rtypes) == set(reagenttypes)
+#     logger.debug(f"Checking if reagents match kit contents: {check}")
+#     common = list(set(ext_kit_rtypes).intersection(reagenttypes))
+#     logger.debug(f"common reagents types: {common}")
+#     if check:
+#         result = None
+#     else:
+#         result = {'message' : f"Couldn't verify reagents match listed kit components.\n\nIt looks like you are missing: {[x.upper for x in ext_kit_rtypes if x not in common]}\n\nAlternatively, you may have set the wrong extraction kit."}
+#     return result
     
     

src/submissions/backend/db/models/organizations.py

@@ -36,7 +36,7 @@ class Contact(Base):
     """
     __tablename__ = "_contacts"
 
-    id = id = Column(INTEGER, primary_key=True) #: primary key  
+    id = Column(INTEGER, primary_key=True) #: primary key  
     name = Column(String(64)) #: contact name
     email = Column(String(64)) #: contact email
     phone = Column(String(32)) #: contact phone number

src/submissions/backend/db/models/samples.py

@@ -1,5 +1,5 @@
 from . import Base
-from sqlalchemy import Column, String, TIMESTAMP, text, JSON, INTEGER, ForeignKey, FLOAT, BOOLEAN
+from sqlalchemy import Column, String, TIMESTAMP, INTEGER, ForeignKey, FLOAT, BOOLEAN
 from sqlalchemy.orm import relationship
 
 

src/submissions/backend/db/models/submissions.py

@@ -17,7 +17,7 @@ class BasicSubmission(Base):
     __tablename__ = "_submissions"
 
     id = Column(INTEGER, primary_key=True) #: primary key   
-    rsl_plate_num = Column(String(32), unique=True) #: RSL name (e.g. RSL-22-0012)
+    rsl_plate_num = Column(String(32), unique=True, nullable=False) #: RSL name (e.g. RSL-22-0012)
     submitter_plate_num = Column(String(127), unique=True) #: The number given to the submission by the submitting lab
     submitted_date = Column(TIMESTAMP) #: Date submission received
     submitting_lab = relationship("Organization", back_populates="submissions") #: client org

src/submissions/backend/excel/parser.py

@@ -1,13 +1,13 @@
 import pandas as pd
 from pathlib import Path
-from backend.db.models.samples import WWSample, BCSample
+from backend.db.models import WWSample, BCSample
 import logging
 from collections import OrderedDict
 import re
 import numpy as np
 from datetime import date
 import uuid
-from frontend.functions import check_not_nan
+from tools import check_not_nan
 
 logger = logging.getLogger(f"submissions.{__name__}")
 
@@ -15,7 +15,7 @@ class SheetParser(object):
     """
     object to pull and contain data from excel file
     """
-    def __init__(self, filepath:Path|None = None, **kwargs) -> None:
+    def __init__(self, filepath:Path|None = None, **kwargs):
         """
         Args:
             filepath (Path | None, optional): file path to excel sheet. Defaults to None.
@@ -77,12 +77,12 @@ class SheetParser(object):
         """        
         submission_info = self.xl.parse(sheet_name=sheet_name, dtype=object)
         
-        self.sub['submitter_plate_num'] = submission_info.iloc[0][1] #if pd.isnull(submission_info.iloc[0][1]) else string_formatter(submission_info.iloc[0][1])
-        self.sub['rsl_plate_num'] =  submission_info.iloc[10][1] #if pd.isnull(submission_info.iloc[10][1]) else string_formatter(submission_info.iloc[10][1])
-        self.sub['submitted_date'] = submission_info.iloc[1][1] #if pd.isnull(submission_info.iloc[1][1]) else submission_info.iloc[1][1].date()#.strftime("%Y-%m-%d")
-        self.sub['submitting_lab'] = submission_info.iloc[0][3] #if pd.isnull(submission_info.iloc[0][3]) else string_formatter(submission_info.iloc[0][3]) 
-        self.sub['sample_count'] = submission_info.iloc[2][3] #if pd.isnull(submission_info.iloc[2][3]) else string_formatter(submission_info.iloc[2][3])
-        self.sub['extraction_kit'] = submission_info.iloc[3][3] #if #pd.isnull(submission_info.iloc[3][3]) else string_formatter(submission_info.iloc[3][3])
+        self.sub['submitter_plate_num'] = submission_info.iloc[0][1]
+        self.sub['rsl_plate_num'] =  submission_info.iloc[10][1]
+        self.sub['submitted_date'] = submission_info.iloc[1][1]
+        self.sub['submitting_lab'] = submission_info.iloc[0][3]
+        self.sub['sample_count'] = submission_info.iloc[2][3]
+        self.sub['extraction_kit'] = submission_info.iloc[3][3]
         
         return submission_info
 
@@ -93,6 +93,12 @@ class SheetParser(object):
         """
 
         def _parse_reagents(df:pd.DataFrame) -> None:
+            """
+            Pulls reagents from the bacterial sub-dataframe
+
+            Args:
+                df (pd.DataFrame): input sub dataframe
+            """            
             for ii, row in df.iterrows():
                 # skip positive control
                 if ii == 11:
@@ -119,16 +125,15 @@ class SheetParser(object):
                     # self.sub[f"lot_{reagent_type}"] = output_var
                     # update 2023-02-10 to above allowing generation of expiry date in adding reagent to db.
                     logger.debug(f"Expiry date for imported reagent: {row[3]}")
-                    try:
-                        check = not np.isnan(row[3])
-                    except TypeError:
-                        check = True
-                    if check:
+                    # try:
+                    #     check = not np.isnan(row[3])
+                    # except TypeError:
+                    #     check = True
+                    if check_not_nan(row[3]):
                         expiry = row[3].date()
                     else:
                         expiry = date.today()
                     self.sub[f"lot_{reagent_type}"] = {'lot':output_var, 'exp':expiry}
-
         submission_info = self._parse_generic("Sample List")
         # iloc is [row][column] and the first row is set as header row so -2
         tech = str(submission_info.iloc[11][1])
@@ -160,9 +165,6 @@ class SheetParser(object):
         logger.debug(f"Parser result: {self.sub}")
         self.sub['samples'] = sample_parse()
 
-        
-
-
 
     def _parse_wastewater(self) -> None:
         """
@@ -170,23 +172,32 @@ class SheetParser(object):
         """        
 
         def _parse_reagents(df:pd.DataFrame) -> None:
-            logger.debug(df)
+            """
+            Pulls reagents from the bacterial sub-dataframe
+
+            Args:
+                df (pd.DataFrame): input sub dataframe
+            """
+            # logger.debug(df)
             for ii, row in df.iterrows():
-                try:
-                    check = not np.isnan(row[5])
-                except TypeError:
-                    check = True
-                if not isinstance(row[5], float) and check:
+                # try:
+                #     check = not np.isnan(row[5])
+                # except TypeError:
+                #     check = True
+                if not isinstance(row[5], float) and check_not_nan(row[5]):
                     # must be prefixed with 'lot_' to be recognized by gui
+                    # regex below will remove 80% from 80% ethanol in the Wastewater kit.
                     output_key = re.sub(r"\d{1,3}%", "", row[0].lower().strip().replace(' ', '_'))
                     try:
                         output_var = row[5].upper()
                     except AttributeError:
-                        logger.debug(f"Couldn't upperize {row[2]}, must be a number")
+                        logger.debug(f"Couldn't upperize {row[5]}, must be a number")
                         output_var = row[5]
-                    self.sub[f"lot_{output_key}"] = output_var
-
-        # submission_info = self.xl.parse("WW Submissions (ENTER HERE)")
+                    if check_not_nan(row[7]):
+                        expiry = row[7].date()
+                    else:
+                        expiry = date.today()
+                    self.sub[f"lot_{output_key}"] = {'lot':output_var, 'exp':expiry}
         submission_info = self._parse_generic("WW Submissions (ENTER HERE)")
         enrichment_info = self.xl.parse("Enrichment Worksheet", dtype=object)
         enr_reagent_range = enrichment_info.iloc[0:4, 9:20]
@@ -214,17 +225,11 @@ class SheetParser(object):
         # self.sub['lot_pre_mix_2'] = qprc_info.iloc[2][14] #if pd.isnull(qprc_info.iloc[2][14]) else string_formatter(qprc_info.iloc[2][14])
         # self.sub['lot_positive_control'] = qprc_info.iloc[3][14] #if pd.isnull(qprc_info.iloc[3][14]) else string_formatter(qprc_info.iloc[3][14])
         # self.sub['lot_ddh2o'] = qprc_info.iloc[4][14] #if pd.isnull(qprc_info.iloc[4][14]) else string_formatter(qprc_info.iloc[4][14])
-        # gt individual sample info
+        # get individual sample info
         sample_parser = SampleParser(submission_info.iloc[16:40])
         sample_parse = getattr(sample_parser, f"parse_{self.sub['submission_type'].lower()}_samples")
         self.sub['samples'] = sample_parse()
 
-        
-
-
-
-    
-
 
 class SampleParser(object):
     """
@@ -241,7 +246,8 @@ class SampleParser(object):
 
         Returns:
             list[BCSample]: list of sample objects
-        """        
+        """       
+        # logger.debug(f"Samples: {self.samples}") 
         new_list = []
         for sample in self.samples:
             new = BCSample()
@@ -297,13 +303,3 @@ class SampleParser(object):
             new.well_number = sample['Unnamed: 1']
             new_list.append(new)
         return new_list
-    
-
-# def string_formatter(input):
-#     logger.debug(f"{input} : {type(input)}")
-#     match input:
-#         case int() | float() | np.float64:
-#             return "{:0.0f}".format(input)
-#         case _:
-#             return input
-        

src/submissions/backend/excel/reports.py

@@ -10,6 +10,7 @@ from pathlib import Path
 
 logger = logging.getLogger(f"submissions.{__name__}")
 
+# set path of templates depending on pyinstaller/raw python
 if getattr(sys, 'frozen', False):
     loader_path = Path(sys._MEIPASS).joinpath("files", "templates")
 else: