From 0c843d1561bac70dc760619854f5831e370da0db Mon Sep 17 00:00:00 2001 From: Landon Wark Date: Tue, 12 Sep 2023 12:33:33 -0500 Subject: [PATCH] Bug fixes and speed-ups --- src/submissions/backend/db/functions.py | 11 ++- src/submissions/backend/db/models/controls.py | 10 +- .../backend/db/models/submissions.py | 34 ++++--- src/submissions/backend/excel/parser.py | 2 +- src/submissions/frontend/__init__.py | 13 ++- .../frontend/custom_widgets/sub_details.py | 8 +- src/submissions/tools/__init__.py | 93 ++++++++++--------- 7 files changed, 99 insertions(+), 72 deletions(-) diff --git a/src/submissions/backend/db/functions.py b/src/submissions/backend/db/functions.py index c434c02..c07c0c5 100644 --- a/src/submissions/backend/db/functions.py +++ b/src/submissions/backend/db/functions.py @@ -369,7 +369,7 @@ def lookup_regent_by_type_name_and_kit_name(ctx:Settings, type_name:str, kit_nam output = rt_types.instances return output -def lookup_all_submissions_by_type(ctx:Settings, sub_type:str|None=None, chronologic:bool=False) -> list[models.BasicSubmission]: +def lookup_all_submissions_by_type(ctx:Settings, sub_type:str|None=None, chronologic:bool=False, limit:int=None) -> list[models.BasicSubmission]: """ Get all submissions, filtering by type if given @@ -386,6 +386,8 @@ def lookup_all_submissions_by_type(ctx:Settings, sub_type:str|None=None, chronol else: # subs = ctx['database_session'].query(models.BasicSubmission).filter(models.BasicSubmission.submission_type==sub_type.lower().replace(" ", "_")).all() subs = ctx.database_session.query(models.BasicSubmission).filter(models.BasicSubmission.submission_type_name==sub_type) + if limit != None: + subs.limit(limit) if chronologic: subs.order_by(models.BasicSubmission.submitted_date) return subs.all() @@ -418,7 +420,7 @@ def lookup_org_by_name(ctx:Settings, name:str|None) -> models.Organization: # return ctx['database_session'].query(models.Organization).filter(models.Organization.name.startswith(name)).first() return ctx.database_session.query(models.Organization).filter(models.Organization.name.startswith(name)).first() -def submissions_to_df(ctx:Settings, sub_type:str|None=None) -> pd.DataFrame: +def submissions_to_df(ctx:Settings, sub_type:str|None=None, limit:int=None) -> pd.DataFrame: """ Convert submissions looked up by type to dataframe @@ -429,9 +431,10 @@ def submissions_to_df(ctx:Settings, sub_type:str|None=None) -> pd.DataFrame: Returns: pd.DataFrame: dataframe constructed from retrieved submissions """ - logger.debug(f"Type: {sub_type}") + logger.debug(f"Querying Type: {sub_type}") # use lookup function to create list of dicts - subs = [item.to_dict() for item in lookup_all_submissions_by_type(ctx=ctx, sub_type=sub_type, chronologic=True)] + subs = [item.to_dict() for item in lookup_all_submissions_by_type(ctx=ctx, sub_type=sub_type, chronologic=True, limit=100)] + logger.debug(f"Got {len(subs)} results.") # make df from dicts (records) in list df = pd.DataFrame.from_records(subs) # Exclude sub information diff --git a/src/submissions/backend/db/models/controls.py b/src/submissions/backend/db/models/controls.py index b483523..f1745c0 100644 --- a/src/submissions/backend/db/models/controls.py +++ b/src/submissions/backend/db/models/controls.py @@ -55,7 +55,10 @@ class Control(Base): dict: output dictionary containing: Name, Type, Targets, Top Kraken results """ # load json string into dict - kraken = json.loads(self.kraken) + try: + kraken = json.loads(self.kraken) + except TypeError: + kraken = {} # calculate kraken count total to use in percentage kraken_cnt_total = sum([kraken[item]['kraken_count'] for item in kraken]) new_kraken = [] @@ -91,7 +94,10 @@ class Control(Base): """ output = [] # load json string for mode (i.e. contains, matches, kraken2) - data = json.loads(getattr(self, mode)) + try: + data = json.loads(getattr(self, mode)) + except TypeError: + data = {} logger.debug(f"Length of data: {len(data)}") # dict keys are genera of bacteria, e.g. 'Streptococcus' for genus in data: diff --git a/src/submissions/backend/db/models/submissions.py b/src/submissions/backend/db/models/submissions.py index a83845f..84ab791 100644 --- a/src/submissions/backend/db/models/submissions.py +++ b/src/submissions/backend/db/models/submissions.py @@ -74,7 +74,7 @@ class BasicSubmission(Base): """ return f"{self.rsl_plate_num} - {self.submitter_plate_num}" - def to_dict(self) -> dict: + def to_dict(self, full_data:bool=False) -> dict: """ dictionary used in submissions summary @@ -82,6 +82,7 @@ class BasicSubmission(Base): dict: dictionary used in submissions summary """ # get lab from nested organization object + logger.debug(f"Converting {self.rsl_plate_num} to dict...") try: sub_lab = self.submitting_lab.name except AttributeError: @@ -104,16 +105,20 @@ class BasicSubmission(Base): ext_info = None logger.debug(f"Json error in {self.rsl_plate_num}: {e}") # Updated 2023-09 to use the extraction kit to pull reagents. - try: - reagents = [item.to_sub_dict(extraction_kit=self.extraction_kit) for item in self.reagents] - except Exception as e: - logger.error(f"We got an error retrieving reagents: {e}") + if full_data: + try: + reagents = [item.to_sub_dict(extraction_kit=self.extraction_kit) for item in self.reagents] + except Exception as e: + logger.error(f"We got an error retrieving reagents: {e}") + reagents = None + samples = [item.sample.to_sub_dict(submission_rsl=self.rsl_plate_num) for item in self.submission_sample_associations] + else: reagents = None - samples = [] + samples = None # Updated 2023-09 to get sample association with plate number - for item in self.submission_sample_associations: - sample = item.sample.to_sub_dict(submission_rsl=self.rsl_plate_num) - samples.append(sample) + # for item in self.submission_sample_associations: + # sample = item.sample.to_sub_dict(submission_rsl=self.rsl_plate_num) + # samples.append(sample) try: comments = self.comment except: @@ -240,15 +245,16 @@ class BacterialCulture(BasicSubmission): controls = relationship("Control", back_populates="submission", uselist=True) #: A control sample added to submission __mapper_args__ = {"polymorphic_identity": "Bacterial Culture", "polymorphic_load": "inline"} - def to_dict(self) -> dict: + def to_dict(self, full_data:bool=False) -> dict: """ Extends parent class method to add controls to dict Returns: dict: dictionary used in submissions summary """ - output = super().to_dict() - output['controls'] = [item.to_sub_dict() for item in self.controls] + output = super().to_dict(full_data=full_data) + if full_data: + output['controls'] = [item.to_sub_dict() for item in self.controls] return output class Wastewater(BasicSubmission): @@ -260,14 +266,14 @@ class Wastewater(BasicSubmission): pcr_technician = Column(String(64)) __mapper_args__ = {"polymorphic_identity": "Wastewater", "polymorphic_load": "inline"} - def to_dict(self) -> dict: + def to_dict(self, full_data:bool=False) -> dict: """ Extends parent class method to add controls to dict Returns: dict: dictionary used in submissions summary """ - output = super().to_dict() + output = super().to_dict(full_data=full_data) try: output['pcr_info'] = json.loads(self.pcr_info) except TypeError as e: diff --git a/src/submissions/backend/excel/parser.py b/src/submissions/backend/excel/parser.py index 3b131f2..b704074 100644 --- a/src/submissions/backend/excel/parser.py +++ b/src/submissions/backend/excel/parser.py @@ -335,7 +335,7 @@ class SampleParser(object): return df def create_basic_dictionaries_from_plate_map(self): - invalids = [0, "0"] + invalids = [0, "0", "EMPTY"] new_df = self.plate_map.dropna(axis=1, how='all') columns = new_df.columns.tolist() for _, iii in new_df.iterrows(): diff --git a/src/submissions/frontend/__init__.py b/src/submissions/frontend/__init__.py index 0c4c25d..e2daf25 100644 --- a/src/submissions/frontend/__init__.py +++ b/src/submissions/frontend/__init__.py @@ -16,7 +16,7 @@ from backend.db import ( ) # from .main_window_functions import * from .all_window_functions import extract_form_info -from tools import check_if_app +from tools import check_if_app, Settings from frontend.custom_widgets import SubmissionsSheet, AlertPop, AddReagentForm, KitAdder, ControlsDatePicker import logging from datetime import date @@ -27,8 +27,8 @@ logger.info("Hello, I am a logger") class App(QMainWindow): - def __init__(self, ctx: dict = {}): - + def __init__(self, ctx: Settings = {}): + logger.debug(f"Initializing main window...") super().__init__() self.ctx = ctx # indicate version and connected database in title bar @@ -59,6 +59,7 @@ class App(QMainWindow): """ adds items to menu bar """ + logger.debug(f"Creating menu bar...") menuBar = self.menuBar() fileMenu = menuBar.addMenu("&File") # Creating menus using a title @@ -79,6 +80,7 @@ class App(QMainWindow): """ adds items to toolbar """ + logger.debug(f"Creating toolbar...") toolbar = QToolBar("My main toolbar") self.addToolBar(toolbar) toolbar.addAction(self.addReagentAction) @@ -89,6 +91,7 @@ class App(QMainWindow): """ creates actions """ + logger.debug(f"Creating actions...") self.importAction = QAction("&Import Submission", self) self.importPCRAction = QAction("&Import PCR Results", self) self.addReagentAction = QAction("Add Reagent", self) @@ -106,6 +109,7 @@ class App(QMainWindow): """ connect menu and tool bar item to functions """ + logger.debug(f"Connecting actions...") self.importAction.triggered.connect(self.importSubmission) self.importPCRAction.triggered.connect(self.importPCRResults) self.addReagentAction.triggered.connect(self.add_reagent) @@ -126,7 +130,7 @@ class App(QMainWindow): """ Show the 'about' message """ - output = f"Version: {self.ctx['package'].__version__}\n\nAuthor: {self.ctx['package'].__author__['name']} - {self.ctx['package'].__author__['email']}\n\nCopyright: {self.ctx['package'].__copyright__}" + output = f"Version: {self.ctx.package.__version__}\n\nAuthor: {self.ctx.package.__author__['name']} - {self.ctx.package.__author__['email']}\n\nCopyright: {self.ctx.package.__copyright__}" about = AlertPop(message=output, status="information") about.exec() @@ -289,6 +293,7 @@ class App(QMainWindow): class AddSubForm(QWidget): def __init__(self, parent): + logger.debug(f"Initializating subform...") super(QWidget, self).__init__(parent) self.layout = QVBoxLayout(self) diff --git a/src/submissions/frontend/custom_widgets/sub_details.py b/src/submissions/frontend/custom_widgets/sub_details.py index 1b10aca..961b30e 100644 --- a/src/submissions/frontend/custom_widgets/sub_details.py +++ b/src/submissions/frontend/custom_widgets/sub_details.py @@ -17,7 +17,7 @@ from PyQt6.QtCore import Qt, QAbstractTableModel, QSortFilterProxyModel from PyQt6.QtGui import QAction, QCursor, QPixmap, QPainter from backend.db import submissions_to_df, lookup_submission_by_id, delete_submission_by_id, lookup_submission_by_rsl_num, hitpick_plate from backend.excel import make_hitpicks -from tools import check_if_app +from tools import check_if_app, Settings from tools import jinja_template_loading from xhtml2pdf import pisa from pathlib import Path @@ -78,7 +78,7 @@ class SubmissionsSheet(QTableView): """ presents submission summary to user in tab1 """ - def __init__(self, ctx:dict) -> None: + def __init__(self, ctx:Settings) -> None: """ initialize @@ -98,7 +98,7 @@ class SubmissionsSheet(QTableView): """ sets data in model """ - self.data = submissions_to_df(ctx=self.ctx) + self.data = submissions_to_df(ctx=self.ctx, limit=100) try: self.data['id'] = self.data['id'].apply(str) self.data['id'] = self.data['id'].str.zfill(3) @@ -269,7 +269,7 @@ class SubmissionDetails(QDialog): # get submision from db data = lookup_submission_by_id(ctx=ctx, id=id) logger.debug(f"Submission details data:\n{pprint.pformat(data.to_dict())}") - self.base_dict = data.to_dict() + self.base_dict = data.to_dict(full_data=True) # don't want id del self.base_dict['id'] # retrieve jinja template diff --git a/src/submissions/tools/__init__.py b/src/submissions/tools/__init__.py index 02507cb..42d4e0a 100644 --- a/src/submissions/tools/__init__.py +++ b/src/submissions/tools/__init__.py @@ -87,25 +87,6 @@ def convert_nans_to_nones(input_str) -> str|None: return input_str return None -def check_is_power_user(ctx:dict) -> bool: - """ - Check to ensure current user is in power users list. - - Args: - ctx (dict): settings passed down from gui. - - Returns: - bool: True if user is in power users, else false. - """ - try: - check = getpass.getuser() in ctx.power_users - except KeyError as e: - check = False - except Exception as e: - logger.debug(f"Check encountered unknown error: {type(e).__name__} - {e}") - check = False - return check - def create_reagent_list(in_dict:dict) -> list[str]: """ Makes list of reagent types without "lot\_" prefix for each key in a dictionary @@ -118,21 +99,6 @@ def create_reagent_list(in_dict:dict) -> list[str]: """ return [item.strip("lot_") for item in in_dict.keys()] -def check_if_app(ctx:dict=None) -> bool: - """ - Checks if the program is running from pyinstaller compiled - - Args: - ctx (dict, optional): Settings passed down from gui. Defaults to None. - - Returns: - bool: True if running from pyinstaller. Else False. - """ - if getattr(sys, 'frozen', False): - return True - else: - return False - def retrieve_rsl_number(in_str:str) -> Tuple[str, str]: """ Uses regex to retrieve the plate number and submission type from an input string @@ -356,8 +322,8 @@ class Settings(BaseSettings): directory_path: Path database_path: Path|None = None backup_path: Path - super_users: list - power_users: list + super_users: list|None = None + power_users: list|None = None rerun_regex: str submission_types: dict|None = None database_session: Session|None = None @@ -431,6 +397,7 @@ def get_config(settings_path: Path|str|None=None) -> dict: Returns: Settings: Pydantic settings object """ + logger.debug(f"Creating settings...") if isinstance(settings_path, str): settings_path = Path(settings_path) # custom pyyaml constructor to join fields @@ -450,8 +417,8 @@ def get_config(settings_path: Path|str|None=None) -> dict: LOGDIR.mkdir(parents=True) except FileExistsError: pass + # if user hasn't defined config path in cli args - copy_settings_trigger = False if settings_path == None: # Check user .config/submissions directory if CONFIGDIR.joinpath("config.yml").exists(): @@ -466,10 +433,12 @@ def get_config(settings_path: Path|str|None=None) -> dict: settings_path = Path(sys._MEIPASS).joinpath("files", "config.yml") else: settings_path = package_dir.joinpath('config.yml') + with open(settings_path, "r") as dset: + default_settings = yaml.load(dset, Loader=yaml.Loader) # Tell program we need to copy the config.yml to the user directory # copy_settings_trigger = True # copy settings to config directory - return Settings(**copy_settings(settings_path=CONFIGDIR.joinpath("config.yml"), settings=settings)) + return Settings(**copy_settings(settings_path=CONFIGDIR.joinpath("config.yml"), settings=default_settings)) else: # check if user defined path is directory if settings_path.is_dir(): @@ -478,9 +447,11 @@ def get_config(settings_path: Path|str|None=None) -> dict: elif settings_path.is_file(): settings_path = settings_path else: - logger.error("No config.yml file found. Cannot continue.") - raise FileNotFoundError("No config.yml file found. Cannot continue.") - return {} + logger.error("No config.yml file found. Writing to directory.") + # raise FileNotFoundError("No config.yml file found. Cannot continue.") + with open(settings_path, "r") as dset: + default_settings = yaml.load(dset, Loader=yaml.Loader) + return Settings(**copy_settings(settings_path=settings_path, settings=default_settings)) logger.debug(f"Using {settings_path} for config file.") with open(settings_path, "r") as stream: # try: @@ -595,8 +566,9 @@ def copy_settings(settings_path:Path, settings:dict) -> dict: del settings['super_users'] if not getpass.getuser() in settings['power_users']: del settings['power_users'] - with open(settings_path, 'w') as f: - yaml.dump(settings, f) + if not settings_path.exists(): + with open(settings_path, 'w') as f: + yaml.dump(settings, f) return settings def jinja_template_loading(): @@ -615,3 +587,38 @@ def jinja_template_loading(): loader = FileSystemLoader(loader_path) env = Environment(loader=loader) return env + +def check_is_power_user(ctx:Settings) -> bool: + """ + Check to ensure current user is in power users list. + + Args: + ctx (dict): settings passed down from gui. + + Returns: + bool: True if user is in power users, else false. + """ + try: + check = getpass.getuser() in ctx.power_users + except KeyError as e: + check = False + except Exception as e: + logger.debug(f"Check encountered unknown error: {type(e).__name__} - {e}") + check = False + return check + +def check_if_app(ctx:Settings=None) -> bool: + """ + Checks if the program is running from pyinstaller compiled + + Args: + ctx (dict, optional): Settings passed down from gui. Defaults to None. + + Returns: + bool: True if running from pyinstaller. Else False. + """ + if getattr(sys, 'frozen', False): + return True + else: + return False + \ No newline at end of file