Bug fixes and speed-ups

src/submissions/backend/db/functions.py

@@ -369,7 +369,7 @@ def lookup_regent_by_type_name_and_kit_name(ctx:Settings, type_name:str, kit_nam
     output = rt_types.instances
     return output
 
-def lookup_all_submissions_by_type(ctx:Settings, sub_type:str|None=None, chronologic:bool=False) -> list[models.BasicSubmission]:
+def lookup_all_submissions_by_type(ctx:Settings, sub_type:str|None=None, chronologic:bool=False, limit:int=None) -> list[models.BasicSubmission]:
     """
     Get all submissions, filtering by type if given
 
@@ -386,6 +386,8 @@ def lookup_all_submissions_by_type(ctx:Settings, sub_type:str|None=None, chronol
     else:
         # subs = ctx['database_session'].query(models.BasicSubmission).filter(models.BasicSubmission.submission_type==sub_type.lower().replace(" ", "_")).all()
         subs = ctx.database_session.query(models.BasicSubmission).filter(models.BasicSubmission.submission_type_name==sub_type)
+    if limit != None:
+        subs.limit(limit)
     if chronologic:
         subs.order_by(models.BasicSubmission.submitted_date)
     return subs.all()
@@ -418,7 +420,7 @@ def lookup_org_by_name(ctx:Settings, name:str|None) -> models.Organization:
     # return ctx['database_session'].query(models.Organization).filter(models.Organization.name.startswith(name)).first()
     return ctx.database_session.query(models.Organization).filter(models.Organization.name.startswith(name)).first()
 
-def submissions_to_df(ctx:Settings, sub_type:str|None=None) -> pd.DataFrame:
+def submissions_to_df(ctx:Settings, sub_type:str|None=None, limit:int=None) -> pd.DataFrame:
     """
     Convert submissions looked up by type to dataframe
 
@@ -429,9 +431,10 @@ def submissions_to_df(ctx:Settings, sub_type:str|None=None) -> pd.DataFrame:
     Returns:
         pd.DataFrame: dataframe constructed from retrieved submissions
     """    
-    logger.debug(f"Type: {sub_type}")
+    logger.debug(f"Querying Type: {sub_type}")
     # use lookup function to create list of dicts
-    subs = [item.to_dict() for item in lookup_all_submissions_by_type(ctx=ctx, sub_type=sub_type, chronologic=True)]
+    subs = [item.to_dict() for item in lookup_all_submissions_by_type(ctx=ctx, sub_type=sub_type, chronologic=True, limit=100)]
+    logger.debug(f"Got {len(subs)} results.")
     # make df from dicts (records) in list
     df = pd.DataFrame.from_records(subs)
     # Exclude sub information

src/submissions/backend/db/models/controls.py

@@ -55,7 +55,10 @@ class Control(Base):
             dict: output dictionary containing: Name, Type, Targets, Top Kraken results
         """        
         # load json string into dict
-        kraken = json.loads(self.kraken)
+        try:
+            kraken = json.loads(self.kraken)
+        except TypeError:
+            kraken = {}
         # calculate kraken count total to use in percentage
         kraken_cnt_total = sum([kraken[item]['kraken_count'] for item in kraken])
         new_kraken = []
@@ -91,7 +94,10 @@ class Control(Base):
         """    
         output = []
         # load json string for mode (i.e. contains, matches, kraken2)
-        data = json.loads(getattr(self, mode))
+        try:
+            data = json.loads(getattr(self, mode))
+        except TypeError:
+            data = {}
         logger.debug(f"Length of data: {len(data)}")
         # dict keys are genera of bacteria, e.g. 'Streptococcus'
         for genus in data:

src/submissions/backend/db/models/submissions.py

@@ -74,7 +74,7 @@ class BasicSubmission(Base):
         """        
         return f"{self.rsl_plate_num} - {self.submitter_plate_num}"
 
-    def to_dict(self) -> dict:
+    def to_dict(self, full_data:bool=False) -> dict:
         """
         dictionary used in submissions summary
 
@@ -82,6 +82,7 @@ class BasicSubmission(Base):
             dict: dictionary used in submissions summary
         """        
         # get lab from nested organization object
+        logger.debug(f"Converting {self.rsl_plate_num} to dict...")
         try:
             sub_lab = self.submitting_lab.name
         except AttributeError:
@@ -104,16 +105,20 @@ class BasicSubmission(Base):
             ext_info = None
             logger.debug(f"Json error in {self.rsl_plate_num}: {e}")
         # Updated 2023-09 to use the extraction kit to pull reagents.
-        try:
+        if full_data:
-            reagents = [item.to_sub_dict(extraction_kit=self.extraction_kit) for item in self.reagents]
+            try:
-        except Exception as e:
+                reagents = [item.to_sub_dict(extraction_kit=self.extraction_kit) for item in self.reagents]
-            logger.error(f"We got an error retrieving reagents: {e}")
+            except Exception as e:
+                logger.error(f"We got an error retrieving reagents: {e}")
+                reagents = None
+            samples = [item.sample.to_sub_dict(submission_rsl=self.rsl_plate_num) for item in self.submission_sample_associations]
+        else:
             reagents = None
-        samples = []
+            samples = None
         # Updated 2023-09 to get sample association with plate number
-        for item in self.submission_sample_associations:
+        # for item in self.submission_sample_associations:
-            sample = item.sample.to_sub_dict(submission_rsl=self.rsl_plate_num)
+        #     sample = item.sample.to_sub_dict(submission_rsl=self.rsl_plate_num)
-            samples.append(sample)
+        #     samples.append(sample)
         try:
             comments = self.comment
         except:
@@ -240,15 +245,16 @@ class BacterialCulture(BasicSubmission):
     controls = relationship("Control", back_populates="submission", uselist=True) #: A control sample added to submission
     __mapper_args__ = {"polymorphic_identity": "Bacterial Culture", "polymorphic_load": "inline"}
 
-    def to_dict(self) -> dict:
+    def to_dict(self, full_data:bool=False) -> dict:
         """
         Extends parent class method to add controls to dict
 
         Returns:
             dict: dictionary used in submissions summary
         """        
-        output = super().to_dict()
+        output = super().to_dict(full_data=full_data)
-        output['controls'] = [item.to_sub_dict() for item in self.controls]
+        if full_data:
+            output['controls'] = [item.to_sub_dict() for item in self.controls]
         return output   
 
 class Wastewater(BasicSubmission):
@@ -260,14 +266,14 @@ class Wastewater(BasicSubmission):
     pcr_technician = Column(String(64))
     __mapper_args__ = {"polymorphic_identity": "Wastewater", "polymorphic_load": "inline"}
 
-    def to_dict(self) -> dict:
+    def to_dict(self, full_data:bool=False) -> dict:
         """
         Extends parent class method to add controls to dict
 
         Returns:
             dict: dictionary used in submissions summary
         """        
-        output = super().to_dict()
+        output = super().to_dict(full_data=full_data)
         try:
             output['pcr_info'] = json.loads(self.pcr_info)
         except TypeError as e:

src/submissions/backend/excel/parser.py

@@ -335,7 +335,7 @@ class SampleParser(object):
         return df
     
     def create_basic_dictionaries_from_plate_map(self):
-        invalids = [0, "0"]
+        invalids = [0, "0", "EMPTY"]
         new_df = self.plate_map.dropna(axis=1, how='all')
         columns = new_df.columns.tolist()
         for _, iii in new_df.iterrows():

src/submissions/frontend/__init__.py

@@ -16,7 +16,7 @@ from backend.db import (
 )
 # from .main_window_functions import *
 from .all_window_functions import extract_form_info
-from tools import check_if_app
+from tools import check_if_app, Settings
 from frontend.custom_widgets import SubmissionsSheet, AlertPop, AddReagentForm, KitAdder, ControlsDatePicker
 import logging
 from datetime import date
@@ -27,8 +27,8 @@ logger.info("Hello, I am a logger")
 
 class App(QMainWindow):
 
-    def __init__(self, ctx: dict = {}):
+    def __init__(self, ctx: Settings = {}):
-        
+        logger.debug(f"Initializing main window...")
         super().__init__()
         self.ctx = ctx
         # indicate version and connected database in title bar
@@ -59,6 +59,7 @@ class App(QMainWindow):
         """
         adds items to menu bar
         """        
+        logger.debug(f"Creating menu bar...")
         menuBar = self.menuBar()
         fileMenu = menuBar.addMenu("&File")
         # Creating menus using a title
@@ -79,6 +80,7 @@ class App(QMainWindow):
         """
         adds items to toolbar
         """        
+        logger.debug(f"Creating toolbar...")
         toolbar = QToolBar("My main toolbar")
         self.addToolBar(toolbar)
         toolbar.addAction(self.addReagentAction)
@@ -89,6 +91,7 @@ class App(QMainWindow):
         """
         creates actions
         """        
+        logger.debug(f"Creating actions...")
         self.importAction = QAction("&Import Submission", self)
         self.importPCRAction = QAction("&Import PCR Results", self)
         self.addReagentAction = QAction("Add Reagent", self)
@@ -106,6 +109,7 @@ class App(QMainWindow):
         """
         connect menu and tool bar item to functions
         """
+        logger.debug(f"Connecting actions...")
         self.importAction.triggered.connect(self.importSubmission)
         self.importPCRAction.triggered.connect(self.importPCRResults)
         self.addReagentAction.triggered.connect(self.add_reagent)
@@ -126,7 +130,7 @@ class App(QMainWindow):
         """
         Show the 'about' message
         """        
-        output = f"Version: {self.ctx['package'].__version__}\n\nAuthor: {self.ctx['package'].__author__['name']} - {self.ctx['package'].__author__['email']}\n\nCopyright: {self.ctx['package'].__copyright__}"
+        output = f"Version: {self.ctx.package.__version__}\n\nAuthor: {self.ctx.package.__author__['name']} - {self.ctx.package.__author__['email']}\n\nCopyright: {self.ctx.package.__copyright__}"
         about = AlertPop(message=output, status="information")
         about.exec()
 
@@ -289,6 +293,7 @@ class App(QMainWindow):
 class AddSubForm(QWidget):
     
     def __init__(self, parent):
+        logger.debug(f"Initializating subform...")
         super(QWidget, self).__init__(parent)
         self.layout = QVBoxLayout(self)
         

src/submissions/frontend/custom_widgets/sub_details.py

@@ -17,7 +17,7 @@ from PyQt6.QtCore import Qt, QAbstractTableModel, QSortFilterProxyModel
 from PyQt6.QtGui import QAction, QCursor, QPixmap, QPainter
 from backend.db import submissions_to_df, lookup_submission_by_id, delete_submission_by_id, lookup_submission_by_rsl_num, hitpick_plate
 from backend.excel import make_hitpicks
-from tools import check_if_app
+from tools import check_if_app, Settings
 from tools import jinja_template_loading
 from xhtml2pdf import pisa
 from pathlib import Path
@@ -78,7 +78,7 @@ class SubmissionsSheet(QTableView):
     """
     presents submission summary to user in tab1
     """    
-    def __init__(self, ctx:dict) -> None:
+    def __init__(self, ctx:Settings) -> None:
         """
         initialize
 
@@ -98,7 +98,7 @@ class SubmissionsSheet(QTableView):
         """
         sets data in model
         """        
-        self.data = submissions_to_df(ctx=self.ctx)
+        self.data = submissions_to_df(ctx=self.ctx, limit=100)
         try:
             self.data['id'] = self.data['id'].apply(str)
             self.data['id'] = self.data['id'].str.zfill(3)
@@ -269,7 +269,7 @@ class SubmissionDetails(QDialog):
         # get submision from db
         data = lookup_submission_by_id(ctx=ctx, id=id)
         logger.debug(f"Submission details data:\n{pprint.pformat(data.to_dict())}")
-        self.base_dict = data.to_dict()
+        self.base_dict = data.to_dict(full_data=True)
         # don't want id
         del self.base_dict['id']
         # retrieve jinja template

src/submissions/tools/__init__.py

@@ -87,25 +87,6 @@ def convert_nans_to_nones(input_str) -> str|None:
         return input_str
     return None
 
-def check_is_power_user(ctx:dict) -> bool:
-    """
-    Check to ensure current user is in power users list.
-
-    Args:
-        ctx (dict): settings passed down from gui.
-
-    Returns:
-        bool: True if user is in power users, else false.
-    """    
-    try:
-        check = getpass.getuser() in ctx.power_users
-    except KeyError as e:
-        check = False
-    except Exception as e:
-        logger.debug(f"Check encountered unknown error: {type(e).__name__} - {e}")
-        check = False
-    return check
-
 def create_reagent_list(in_dict:dict) -> list[str]:
     """
     Makes list of reagent types without "lot\_" prefix for each key in a dictionary
@@ -118,21 +99,6 @@ def create_reagent_list(in_dict:dict) -> list[str]:
     """    
     return [item.strip("lot_") for item in in_dict.keys()]
 
-def check_if_app(ctx:dict=None) -> bool:
-    """
-    Checks if the program is running from pyinstaller compiled
-
-    Args:
-        ctx (dict, optional): Settings passed down from gui. Defaults to None.
-
-    Returns:
-        bool: True if running from pyinstaller. Else False.
-    """    
-    if getattr(sys, 'frozen', False):
-        return True
-    else:
-        return False
-    
 def retrieve_rsl_number(in_str:str) -> Tuple[str, str]:
     """
     Uses regex to retrieve the plate number and submission type from an input string
@@ -356,8 +322,8 @@ class Settings(BaseSettings):
     directory_path: Path
     database_path: Path|None = None
     backup_path: Path
-    super_users: list
+    super_users: list|None = None
-    power_users: list
+    power_users: list|None = None
     rerun_regex: str
     submission_types: dict|None = None
     database_session: Session|None = None
@@ -431,6 +397,7 @@ def get_config(settings_path: Path|str|None=None) -> dict:
     Returns:
         Settings: Pydantic settings object
     """    
+    logger.debug(f"Creating settings...")
     if isinstance(settings_path, str):
         settings_path = Path(settings_path)
     # custom pyyaml constructor to join fields
@@ -450,8 +417,8 @@ def get_config(settings_path: Path|str|None=None) -> dict:
         LOGDIR.mkdir(parents=True)
     except FileExistsError:
         pass
+    
     # if user hasn't defined config path in cli args
-    copy_settings_trigger = False
     if settings_path == None:
         # Check user .config/submissions directory
         if CONFIGDIR.joinpath("config.yml").exists():
@@ -466,10 +433,12 @@ def get_config(settings_path: Path|str|None=None) -> dict:
                 settings_path = Path(sys._MEIPASS).joinpath("files", "config.yml")
             else:
                 settings_path = package_dir.joinpath('config.yml')
+            with open(settings_path, "r") as dset:
+                default_settings = yaml.load(dset, Loader=yaml.Loader)
             # Tell program we need to copy the config.yml to the user directory
             # copy_settings_trigger = True
             # copy settings to config directory
-            return Settings(**copy_settings(settings_path=CONFIGDIR.joinpath("config.yml"), settings=settings))
+            return Settings(**copy_settings(settings_path=CONFIGDIR.joinpath("config.yml"), settings=default_settings))
     else:
         # check if user defined path is directory
         if settings_path.is_dir():
@@ -478,9 +447,11 @@ def get_config(settings_path: Path|str|None=None) -> dict:
         elif settings_path.is_file():
             settings_path = settings_path
         else:
-            logger.error("No config.yml file found. Cannot continue.")
+            logger.error("No config.yml file found. Writing to directory.")
-            raise FileNotFoundError("No config.yml file found. Cannot continue.")
+            # raise FileNotFoundError("No config.yml file found. Cannot continue.")
-            return {}
+            with open(settings_path, "r") as dset:
+                default_settings = yaml.load(dset, Loader=yaml.Loader)
+            return Settings(**copy_settings(settings_path=settings_path, settings=default_settings))
     logger.debug(f"Using {settings_path} for config file.")
     with open(settings_path, "r") as stream:
         # try:
@@ -595,8 +566,9 @@ def copy_settings(settings_path:Path, settings:dict) -> dict:
         del settings['super_users']
     if not getpass.getuser() in settings['power_users']:
         del settings['power_users']
-    with open(settings_path, 'w') as f:
+    if not settings_path.exists():
-        yaml.dump(settings, f)
+        with open(settings_path, 'w') as f:
+            yaml.dump(settings, f)
     return settings
 
 def jinja_template_loading():
@@ -615,3 +587,38 @@ def jinja_template_loading():
     loader = FileSystemLoader(loader_path)
     env = Environment(loader=loader)
     return env
+
+def check_is_power_user(ctx:Settings) -> bool:
+    """
+    Check to ensure current user is in power users list.
+
+    Args:
+        ctx (dict): settings passed down from gui.
+
+    Returns:
+        bool: True if user is in power users, else false.
+    """    
+    try:
+        check = getpass.getuser() in ctx.power_users
+    except KeyError as e:
+        check = False
+    except Exception as e:
+        logger.debug(f"Check encountered unknown error: {type(e).__name__} - {e}")
+        check = False
+    return check
+
+def check_if_app(ctx:Settings=None) -> bool:
+    """
+    Checks if the program is running from pyinstaller compiled
+
+    Args:
+        ctx (dict, optional): Settings passed down from gui. Defaults to None.
+
+    Returns:
+        bool: True if running from pyinstaller. Else False.
+    """    
+    if getattr(sys, 'frozen', False):
+        return True
+    else:
+        return False
+