landon/Submissions-App
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Bug fixes and speed-ups

Browse Source
This commit is contained in:
Landon Wark
2023-09-12 12:33:33 -05:00
parent 5a978c9bff
commit 0c843d1561
7 changed files with 99 additions and 72 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
src/submissions/backend/db/functions.py
View File

@@ -369,7 +369,7 @@ def lookup_regent_by_type_name_and_kit_name(ctx:Settings, type_name:str, kit_nam
output = rt_types.instances
return output
def lookup_all_submissions_by_type(ctx:Settings, sub_type:str|None=None, chronologic:bool=False) -> list[models.BasicSubmission]:
def lookup_all_submissions_by_type(ctx:Settings, sub_type:str|None=None, chronologic:bool=False, limit:int=None) -> list[models.BasicSubmission]:
"""
Get all submissions, filtering by type if given
@@ -386,6 +386,8 @@ def lookup_all_submissions_by_type(ctx:Settings, sub_type:str|None=None, chronol
else:
# subs = ctx['database_session'].query(models.BasicSubmission).filter(models.BasicSubmission.submission_type==sub_type.lower().replace(" ", "_")).all()
subs = ctx.database_session.query(models.BasicSubmission).filter(models.BasicSubmission.submission_type_name==sub_type)
if limit != None:
subs.limit(limit)
if chronologic:
subs.order_by(models.BasicSubmission.submitted_date)
return subs.all()
@@ -418,7 +420,7 @@ def lookup_org_by_name(ctx:Settings, name:str|None) -> models.Organization:
# return ctx['database_session'].query(models.Organization).filter(models.Organization.name.startswith(name)).first()
return ctx.database_session.query(models.Organization).filter(models.Organization.name.startswith(name)).first()
def submissions_to_df(ctx:Settings, sub_type:str|None=None) -> pd.DataFrame:
def submissions_to_df(ctx:Settings, sub_type:str|None=None, limit:int=None) -> pd.DataFrame:
"""
Convert submissions looked up by type to dataframe
@@ -429,9 +431,10 @@ def submissions_to_df(ctx:Settings, sub_type:str|None=None) -> pd.DataFrame:
Returns:
pd.DataFrame: dataframe constructed from retrieved submissions
"""
logger.debug(f"Type: {sub_type}")
logger.debug(f"Querying Type: {sub_type}")
# use lookup function to create list of dicts
subs = [item.to_dict() for item in lookup_all_submissions_by_type(ctx=ctx, sub_type=sub_type, chronologic=True)]
subs = [item.to_dict() for item in lookup_all_submissions_by_type(ctx=ctx, sub_type=sub_type, chronologic=True, limit=100)]
logger.debug(f"Got {len(subs)} results.")
# make df from dicts (records) in list
df = pd.DataFrame.from_records(subs)
# Exclude sub information

10
src/submissions/backend/db/models/controls.py
View File

@@ -55,7 +55,10 @@ class Control(Base):
dict: output dictionary containing: Name, Type, Targets, Top Kraken results
"""
# load json string into dict
kraken = json.loads(self.kraken)
try:
kraken = json.loads(self.kraken)
except TypeError:
kraken = {}
# calculate kraken count total to use in percentage
kraken_cnt_total = sum([kraken[item]['kraken_count'] for item in kraken])
new_kraken = []
@@ -91,7 +94,10 @@ class Control(Base):
"""
output = []
# load json string for mode (i.e. contains, matches, kraken2)
data = json.loads(getattr(self, mode))
try:
data = json.loads(getattr(self, mode))
except TypeError:
data = {}
logger.debug(f"Length of data: {len(data)}")
# dict keys are genera of bacteria, e.g. 'Streptococcus'
for genus in data:

34
src/submissions/backend/db/models/submissions.py
View File

@@ -74,7 +74,7 @@ class BasicSubmission(Base):
"""
return f"{self.rsl_plate_num} - {self.submitter_plate_num}"
def to_dict(self) -> dict:
def to_dict(self, full_data:bool=False) -> dict:
"""
dictionary used in submissions summary
@@ -82,6 +82,7 @@ class BasicSubmission(Base):
dict: dictionary used in submissions summary
"""
# get lab from nested organization object
logger.debug(f"Converting {self.rsl_plate_num} to dict...")
try:
sub_lab = self.submitting_lab.name
except AttributeError:
@@ -104,16 +105,20 @@ class BasicSubmission(Base):
ext_info = None
logger.debug(f"Json error in {self.rsl_plate_num}: {e}")
# Updated 2023-09 to use the extraction kit to pull reagents.
try:
reagents = [item.to_sub_dict(extraction_kit=self.extraction_kit) for item in self.reagents]
except Exception as e:
logger.error(f"We got an error retrieving reagents: {e}")
if full_data:
try:
reagents = [item.to_sub_dict(extraction_kit=self.extraction_kit) for item in self.reagents]
except Exception as e:
logger.error(f"We got an error retrieving reagents: {e}")
reagents = None
samples = [item.sample.to_sub_dict(submission_rsl=self.rsl_plate_num) for item in self.submission_sample_associations]
else:
reagents = None
samples = []
samples = None
# Updated 2023-09 to get sample association with plate number
for item in self.submission_sample_associations:
sample = item.sample.to_sub_dict(submission_rsl=self.rsl_plate_num)
samples.append(sample)
# for item in self.submission_sample_associations:
# sample = item.sample.to_sub_dict(submission_rsl=self.rsl_plate_num)
# samples.append(sample)
try:
comments = self.comment
except:
@@ -240,15 +245,16 @@ class BacterialCulture(BasicSubmission):
controls = relationship("Control", back_populates="submission", uselist=True) #: A control sample added to submission
__mapper_args__ = {"polymorphic_identity": "Bacterial Culture", "polymorphic_load": "inline"}
def to_dict(self) -> dict:
def to_dict(self, full_data:bool=False) -> dict:
"""
Extends parent class method to add controls to dict
Returns:
dict: dictionary used in submissions summary
"""
output = super().to_dict()
output['controls'] = [item.to_sub_dict() for item in self.controls]
output = super().to_dict(full_data=full_data)
if full_data:
output['controls'] = [item.to_sub_dict() for item in self.controls]
return output
class Wastewater(BasicSubmission):
@@ -260,14 +266,14 @@ class Wastewater(BasicSubmission):
pcr_technician = Column(String(64))
__mapper_args__ = {"polymorphic_identity": "Wastewater", "polymorphic_load": "inline"}
def to_dict(self) -> dict:
def to_dict(self, full_data:bool=False) -> dict:
"""
Extends parent class method to add controls to dict
Returns:
dict: dictionary used in submissions summary
"""
output = super().to_dict()
output = super().to_dict(full_data=full_data)
try:
output['pcr_info'] = json.loads(self.pcr_info)
except TypeError as e:

2
src/submissions/backend/excel/parser.py
View File

@@ -335,7 +335,7 @@ class SampleParser(object):
return df
def create_basic_dictionaries_from_plate_map(self):
invalids = [0, "0"]
invalids = [0, "0", "EMPTY"]
new_df = self.plate_map.dropna(axis=1, how='all')
columns = new_df.columns.tolist()
for _, iii in new_df.iterrows():

13
src/submissions/frontend/__init__.py
View File

@@ -16,7 +16,7 @@ from backend.db import (
)
# from .main_window_functions import *
from .all_window_functions import extract_form_info
from tools import check_if_app
from tools import check_if_app, Settings
from frontend.custom_widgets import SubmissionsSheet, AlertPop, AddReagentForm, KitAdder, ControlsDatePicker
import logging
from datetime import date
@@ -27,8 +27,8 @@ logger.info("Hello, I am a logger")
class App(QMainWindow):
def __init__(self, ctx: dict = {}):
def __init__(self, ctx: Settings = {}):
logger.debug(f"Initializing main window...")
super().__init__()
self.ctx = ctx
# indicate version and connected database in title bar
@@ -59,6 +59,7 @@ class App(QMainWindow):
"""
adds items to menu bar
"""
logger.debug(f"Creating menu bar...")
menuBar = self.menuBar()
fileMenu = menuBar.addMenu("&File")
# Creating menus using a title
@@ -79,6 +80,7 @@ class App(QMainWindow):
"""
adds items to toolbar
"""
logger.debug(f"Creating toolbar...")
toolbar = QToolBar("My main toolbar")
self.addToolBar(toolbar)
toolbar.addAction(self.addReagentAction)
@@ -89,6 +91,7 @@ class App(QMainWindow):
"""
creates actions
"""
logger.debug(f"Creating actions...")
self.importAction = QAction("&Import Submission", self)
self.importPCRAction = QAction("&Import PCR Results", self)
self.addReagentAction = QAction("Add Reagent", self)
@@ -106,6 +109,7 @@ class App(QMainWindow):
"""
connect menu and tool bar item to functions
"""
logger.debug(f"Connecting actions...")
self.importAction.triggered.connect(self.importSubmission)
self.importPCRAction.triggered.connect(self.importPCRResults)
self.addReagentAction.triggered.connect(self.add_reagent)
@@ -126,7 +130,7 @@ class App(QMainWindow):
"""
Show the 'about' message
"""
output = f"Version: {self.ctx['package'].__version__}\n\nAuthor: {self.ctx['package'].__author__['name']} - {self.ctx['package'].__author__['email']}\n\nCopyright: {self.ctx['package'].__copyright__}"
output = f"Version: {self.ctx.package.__version__}\n\nAuthor: {self.ctx.package.__author__['name']} - {self.ctx.package.__author__['email']}\n\nCopyright: {self.ctx.package.__copyright__}"
about = AlertPop(message=output, status="information")
about.exec()
@@ -289,6 +293,7 @@ class App(QMainWindow):
class AddSubForm(QWidget):
def __init__(self, parent):
logger.debug(f"Initializating subform...")
super(QWidget, self).__init__(parent)
self.layout = QVBoxLayout(self)

8
src/submissions/frontend/custom_widgets/sub_details.py
View File

@@ -17,7 +17,7 @@ from PyQt6.QtCore import Qt, QAbstractTableModel, QSortFilterProxyModel
from PyQt6.QtGui import QAction, QCursor, QPixmap, QPainter
from backend.db import submissions_to_df, lookup_submission_by_id, delete_submission_by_id, lookup_submission_by_rsl_num, hitpick_plate
from backend.excel import make_hitpicks
from tools import check_if_app
from tools import check_if_app, Settings
from tools import jinja_template_loading
from xhtml2pdf import pisa
from pathlib import Path
@@ -78,7 +78,7 @@ class SubmissionsSheet(QTableView):
"""
presents submission summary to user in tab1
"""
def __init__(self, ctx:dict) -> None:
def __init__(self, ctx:Settings) -> None:
"""
initialize
@@ -98,7 +98,7 @@ class SubmissionsSheet(QTableView):
"""
sets data in model
"""
self.data = submissions_to_df(ctx=self.ctx)
self.data = submissions_to_df(ctx=self.ctx, limit=100)
try:
self.data['id'] = self.data['id'].apply(str)
self.data['id'] = self.data['id'].str.zfill(3)
@@ -269,7 +269,7 @@ class SubmissionDetails(QDialog):
# get submision from db
data = lookup_submission_by_id(ctx=ctx, id=id)
logger.debug(f"Submission details data:\n{pprint.pformat(data.to_dict())}")
self.base_dict = data.to_dict()
self.base_dict = data.to_dict(full_data=True)
# don't want id
del self.base_dict['id']
# retrieve jinja template

93
src/submissions/tools/__init__.py
View File

@@ -87,25 +87,6 @@ def convert_nans_to_nones(input_str) -> str|None:
return input_str
return None
def check_is_power_user(ctx:dict) -> bool:
"""
Check to ensure current user is in power users list.
Args:
ctx (dict): settings passed down from gui.
Returns:
bool: True if user is in power users, else false.
"""
try:
check = getpass.getuser() in ctx.power_users
except KeyError as e:
check = False
except Exception as e:
logger.debug(f"Check encountered unknown error: {type(e).__name__} - {e}")
check = False
return check
def create_reagent_list(in_dict:dict) -> list[str]:
"""
Makes list of reagent types without "lot\_" prefix for each key in a dictionary
@@ -118,21 +99,6 @@ def create_reagent_list(in_dict:dict) -> list[str]:
"""
return [item.strip("lot_") for item in in_dict.keys()]
def check_if_app(ctx:dict=None) -> bool:
"""
Checks if the program is running from pyinstaller compiled
Args:
ctx (dict, optional): Settings passed down from gui. Defaults to None.
Returns:
bool: True if running from pyinstaller. Else False.
"""
if getattr(sys, 'frozen', False):
return True
else:
return False
def retrieve_rsl_number(in_str:str) -> Tuple[str, str]:
"""
Uses regex to retrieve the plate number and submission type from an input string
@@ -356,8 +322,8 @@ class Settings(BaseSettings):
directory_path: Path
database_path: Path|None = None
backup_path: Path
super_users: list
power_users: list
super_users: list|None = None
power_users: list|None = None
rerun_regex: str
submission_types: dict|None = None
database_session: Session|None = None
@@ -431,6 +397,7 @@ def get_config(settings_path: Path|str|None=None) -> dict:
Returns:
Settings: Pydantic settings object
"""
logger.debug(f"Creating settings...")
if isinstance(settings_path, str):
settings_path = Path(settings_path)
# custom pyyaml constructor to join fields
@@ -450,8 +417,8 @@ def get_config(settings_path: Path|str|None=None) -> dict:
LOGDIR.mkdir(parents=True)
except FileExistsError:
pass
# if user hasn't defined config path in cli args
copy_settings_trigger = False
if settings_path == None:
# Check user .config/submissions directory
if CONFIGDIR.joinpath("config.yml").exists():
@@ -466,10 +433,12 @@ def get_config(settings_path: Path|str|None=None) -> dict:
settings_path = Path(sys._MEIPASS).joinpath("files", "config.yml")
else:
settings_path = package_dir.joinpath('config.yml')
with open(settings_path, "r") as dset:
default_settings = yaml.load(dset, Loader=yaml.Loader)
# Tell program we need to copy the config.yml to the user directory
# copy_settings_trigger = True
# copy settings to config directory
return Settings(**copy_settings(settings_path=CONFIGDIR.joinpath("config.yml"), settings=settings))
return Settings(**copy_settings(settings_path=CONFIGDIR.joinpath("config.yml"), settings=default_settings))
else:
# check if user defined path is directory
if settings_path.is_dir():
@@ -478,9 +447,11 @@ def get_config(settings_path: Path|str|None=None) -> dict:
elif settings_path.is_file():
settings_path = settings_path
else:
logger.error("No config.yml file found. Cannot continue.")
raise FileNotFoundError("No config.yml file found. Cannot continue.")
return {}
logger.error("No config.yml file found. Writing to directory.")
# raise FileNotFoundError("No config.yml file found. Cannot continue.")
with open(settings_path, "r") as dset:
default_settings = yaml.load(dset, Loader=yaml.Loader)
return Settings(**copy_settings(settings_path=settings_path, settings=default_settings))
logger.debug(f"Using {settings_path} for config file.")
with open(settings_path, "r") as stream:
# try:
@@ -595,8 +566,9 @@ def copy_settings(settings_path:Path, settings:dict) -> dict:
del settings['super_users']
if not getpass.getuser() in settings['power_users']:
del settings['power_users']
with open(settings_path, 'w') as f:
yaml.dump(settings, f)
if not settings_path.exists():
with open(settings_path, 'w') as f:
yaml.dump(settings, f)
return settings
def jinja_template_loading():
@@ -615,3 +587,38 @@ def jinja_template_loading():
loader = FileSystemLoader(loader_path)
env = Environment(loader=loader)
return env
def check_is_power_user(ctx:Settings) -> bool:
"""
Check to ensure current user is in power users list.
Args:
ctx (dict): settings passed down from gui.
Returns:
bool: True if user is in power users, else false.
"""
try:
check = getpass.getuser() in ctx.power_users
except KeyError as e:
check = False
except Exception as e:
logger.debug(f"Check encountered unknown error: {type(e).__name__} - {e}")
check = False
return check
def check_if_app(ctx:Settings=None) -> bool:
"""
Checks if the program is running from pyinstaller compiled
Args:
ctx (dict, optional): Settings passed down from gui. Defaults to None.
Returns:
bool: True if running from pyinstaller. Else False.
"""
if getattr(sys, 'frozen', False):
return True
else:
return False