landon/Submissions-App
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactored to increase ui robustness.

Browse Source
This commit is contained in:
Landon Wark
2023-04-04 09:31:37 -05:00
parent 3c9f095937
commit 4398517a3e
15 changed files with 1580 additions and 672 deletions
Download Patch File Download Diff File Expand all files Collapse all files

83
src/submissions/backend/db/functions.py
View File

@@ -5,7 +5,7 @@ Convenience functions for interacting with the database.
from . import models
from .models.kits import reagenttypes_kittypes
from .models.submissions import reagents_submissions
from .models.samples import WWSample
# from .models.samples import WWSample
import pandas as pd
import sqlalchemy.exc
import sqlite3
@@ -18,7 +18,6 @@ from sqlalchemy.engine import Engine
import json
from getpass import getuser
import numpy as np
import yaml
from pathlib import Path
@@ -42,8 +41,10 @@ def store_submission(ctx:dict, base_submission:models.BasicSubmission) -> None|d
Returns:
None|dict : object that indicates issue raised for reporting in gui
"""
from tools import format_rsl_number
logger.debug(f"Hello from store_submission")
# Add all samples to sample table
base_submission.rsl_plate_num = format_rsl_number(base_submission.rsl_plate_num)
for sample in base_submission.samples:
sample.rsl_plate = base_submission
logger.debug(f"Attempting to add sample: {sample.to_string()}")
@@ -60,14 +61,13 @@ def store_submission(ctx:dict, base_submission:models.BasicSubmission) -> None|d
except (sqlite3.IntegrityError, sqlalchemy.exc.IntegrityError) as e:
logger.debug(f"Hit an integrity error : {e}")
ctx['database_session'].rollback()
return {"message":"This plate number already exists, so we can't add it."}
return {"message":"This plate number already exists, so we can't add it.", "status":"Critical"}
except (sqlite3.OperationalError, sqlalchemy.exc.IntegrityError) as e:
logger.debug(f"Hit an operational error: {e}")
ctx['database_session'].rollback()
return {"message":"The database is locked for editing."}
return {"message":"The database is locked for editing.", "status":"Critical"}
return None
def store_reagent(ctx:dict, reagent:models.Reagent) -> None|dict:
"""
Inserts a reagent into the database.
@@ -87,7 +87,6 @@ def store_reagent(ctx:dict, reagent:models.Reagent) -> None|dict:
return {"message":"The database is locked for editing."}
return None
def construct_submission_info(ctx:dict, info_dict:dict) -> models.BasicSubmission:
"""
Construct submission object from dictionary
@@ -99,14 +98,17 @@ def construct_submission_info(ctx:dict, info_dict:dict) -> models.BasicSubmissio
Returns:
models.BasicSubmission: Constructed submission object
"""
from tools import check_not_nan
from tools import check_regex_match, RSLNamer
# convert submission type into model name
query = info_dict['submission_type'].replace(" ", "")
# Ensure an rsl plate number exists for the plate
if info_dict["rsl_plate_num"] == 'nan' or info_dict["rsl_plate_num"] == None or not check_not_nan(info_dict["rsl_plate_num"]):
# if info_dict["rsl_plate_num"] == 'nan' or info_dict["rsl_plate_num"] == None or not check_not_nan(info_dict["rsl_plate_num"]):
if not check_regex_match("^RSL", info_dict["rsl_plate_num"]):
instance = None
msg = "A proper RSL plate number is required."
return instance, {'code': 2, 'message': "A proper RSL plate number is required."}
else:
info_dict['rsl_plate_num'] = RSLNamer(info_dict["rsl_plate_num"]).parsed_name
# check database for existing object
instance = ctx['database_session'].query(models.BasicSubmission).filter(models.BasicSubmission.rsl_plate_num==info_dict['rsl_plate_num']).first()
# get model based on submission type converted above
@@ -171,7 +173,6 @@ def construct_submission_info(ctx:dict, info_dict:dict) -> models.BasicSubmissio
logger.debug(f"Constructed submissions message: {msg}")
return instance, {'code':code, 'message':msg}
def construct_reagent(ctx:dict, info_dict:dict) -> models.Reagent:
"""
Construct reagent object from dictionary
@@ -204,7 +205,6 @@ def construct_reagent(ctx:dict, info_dict:dict) -> models.Reagent:
# pass
return reagent
def lookup_reagent(ctx:dict, reagent_lot:str) -> models.Reagent:
"""
Query db for reagent based on lot number
@@ -219,7 +219,6 @@ def lookup_reagent(ctx:dict, reagent_lot:str) -> models.Reagent:
lookedup = ctx['database_session'].query(models.Reagent).filter(models.Reagent.lot==reagent_lot).first()
return lookedup
def get_all_reagenttype_names(ctx:dict) -> list[str]:
"""
Lookup all reagent types and get names
@@ -233,7 +232,6 @@ def get_all_reagenttype_names(ctx:dict) -> list[str]:
lookedup = [item.__str__() for item in ctx['database_session'].query(models.ReagentType).all()]
return lookedup
def lookup_reagenttype_by_name(ctx:dict, rt_name:str) -> models.ReagentType:
"""
Lookup a single reagent type by name
@@ -250,7 +248,6 @@ def lookup_reagenttype_by_name(ctx:dict, rt_name:str) -> models.ReagentType:
logger.debug(f"Found ReagentType: {lookedup}")
return lookedup
def lookup_kittype_by_use(ctx:dict, used_by:str) -> list[models.KitType]:
"""
Lookup kits by a sample type its used for
@@ -264,7 +261,6 @@ def lookup_kittype_by_use(ctx:dict, used_by:str) -> list[models.KitType]:
"""
return ctx['database_session'].query(models.KitType).filter(models.KitType.used_for.contains(used_by)).all()
def lookup_kittype_by_name(ctx:dict, name:str) -> models.KitType:
"""
Lookup a kit type by name
@@ -279,7 +275,6 @@ def lookup_kittype_by_name(ctx:dict, name:str) -> models.KitType:
logger.debug(f"Querying kittype: {name}")
return ctx['database_session'].query(models.KitType).filter(models.KitType.name==name).first()
def lookup_regent_by_type_name(ctx:dict, type_name:str) -> list[models.Reagent]:
"""
Lookup reagents by their type name
@@ -293,7 +288,6 @@ def lookup_regent_by_type_name(ctx:dict, type_name:str) -> list[models.Reagent]:
"""
return ctx['database_session'].query(models.Reagent).join(models.Reagent.type, aliased=True).filter(models.ReagentType.name==type_name).all()
def lookup_regent_by_type_name_and_kit_name(ctx:dict, type_name:str, kit_name:str) -> list[models.Reagent]:
"""
Lookup reagents by their type name and kits they belong to (Broken... maybe cursed, I'm not sure.)
@@ -325,7 +319,6 @@ def lookup_regent_by_type_name_and_kit_name(ctx:dict, type_name:str, kit_name:st
output = rt_types.instances
return output
def lookup_all_submissions_by_type(ctx:dict, sub_type:str|None=None) -> list[models.BasicSubmission]:
"""
Get all submissions, filtering by type if given
@@ -399,7 +392,6 @@ def submissions_to_df(ctx:dict, sub_type:str|None=None) -> pd.DataFrame:
except:
logger.warning(f"Couldn't drop 'pcr_info' column from submissionsheet df.")
return df
def lookup_submission_by_id(ctx:dict, id:int) -> models.BasicSubmission:
"""
@@ -414,7 +406,6 @@ def lookup_submission_by_id(ctx:dict, id:int) -> models.BasicSubmission:
"""
return ctx['database_session'].query(models.BasicSubmission).filter(models.BasicSubmission.id==id).first()
def lookup_submissions_by_date_range(ctx:dict, start_date:datetime.date, end_date:datetime.date) -> list[models.BasicSubmission]:
"""
Lookup submissions greater than start_date and less than end_date
@@ -432,7 +423,6 @@ def lookup_submissions_by_date_range(ctx:dict, start_date:datetime.date, end_dat
end_date = end_date.strftime("%Y-%m-%d")
return ctx['database_session'].query(models.BasicSubmission).filter(models.BasicSubmission.submitted_date.between(start_date, end_date)).all()
def get_all_Control_Types_names(ctx:dict) -> list[str]:
"""
Grabs all control type names from db.
@@ -448,7 +438,6 @@ def get_all_Control_Types_names(ctx:dict) -> list[str]:
logger.debug(f"Control Types: {conTypes}")
return conTypes
def create_kit_from_yaml(ctx:dict, exp:dict) -> dict:
"""
Create and store a new kit in the database based on a .yml file
@@ -491,7 +480,6 @@ def create_kit_from_yaml(ctx:dict, exp:dict) -> dict:
ctx['database_session'].commit()
return {'code':0, 'message':'Kit has been added', 'status': 'information'}
def create_org_from_yaml(ctx:dict, org:dict) -> dict:
"""
Create and store a new organization based on a .yml file
@@ -528,7 +516,6 @@ def create_org_from_yaml(ctx:dict, org:dict) -> dict:
ctx["database_session"].commit()
return {"code":0, "message":"Organization has been added."}
def lookup_all_sample_types(ctx:dict) -> list[str]:
"""
Lookup all sample types and get names
@@ -544,7 +531,6 @@ def lookup_all_sample_types(ctx:dict) -> list[str]:
uses = list(set([item for sublist in uses for item in sublist]))
return uses
def get_all_available_modes(ctx:dict) -> list[str]:
"""
Get types of analysis for controls
@@ -564,7 +550,6 @@ def get_all_available_modes(ctx:dict) -> list[str]:
cols = []
return cols
def get_all_controls_by_type(ctx:dict, con_type:str, start_date:date|None=None, end_date:date|None=None) -> list[models.Control]:
"""
Returns a list of control objects that are instances of the input controltype.
@@ -589,7 +574,6 @@ def get_all_controls_by_type(ctx:dict, con_type:str, start_date:date|None=None,
logger.debug(f"Returned controls between dates: {output}")
return output
def get_control_subtypes(ctx:dict, type:str, mode:str) -> list[str]:
"""
Get subtypes for a control analysis mode
@@ -617,7 +601,6 @@ def get_control_subtypes(ctx:dict, type:str, mode:str) -> list[str]:
subtypes = [item for item in jsoner[genera] if "_hashes" not in item and "_ratio" not in item]
return subtypes
def get_all_controls(ctx:dict) -> list[models.Control]:
"""
Retrieve a list of all controls from the database
@@ -630,7 +613,6 @@ def get_all_controls(ctx:dict) -> list[models.Control]:
"""
return ctx['database_session'].query(models.Control).all()
def lookup_submission_by_rsl_num(ctx:dict, rsl_num:str) -> models.BasicSubmission:
"""
Retrieve a submission from the database based on rsl plate number
@@ -644,7 +626,6 @@ def lookup_submission_by_rsl_num(ctx:dict, rsl_num:str) -> models.BasicSubmissio
"""
return ctx['database_session'].query(models.BasicSubmission).filter(models.BasicSubmission.rsl_plate_num.startswith(rsl_num)).first()
def lookup_submissions_using_reagent(ctx:dict, reagent:models.Reagent) -> list[models.BasicSubmission]:
"""
Retrieves each submission using a specified reagent.
@@ -658,7 +639,6 @@ def lookup_submissions_using_reagent(ctx:dict, reagent:models.Reagent) -> list[m
"""
return ctx['database_session'].query(models.BasicSubmission).join(reagents_submissions).filter(reagents_submissions.c.reagent_id==reagent.id).all()
def delete_submission_by_id(ctx:dict, id:int) -> None:
"""
Deletes a submission and its associated samples from the database.
@@ -683,13 +663,12 @@ def delete_submission_by_id(ctx:dict, id:int) -> None:
ctx["database_session"].delete(sub)
ctx["database_session"].commit()
def lookup_ww_sample_by_rsl_sample_number(ctx:dict, rsl_number:str) -> models.WWSample:
"""
Retrieves wastewater sampel from database by rsl sample number
Retrieves wastewater sample from database by rsl sample number
Args:
ctx (dict): settings passed dwon from gui
ctx (dict): settings passed down from gui
rsl_number (str): sample number assigned by robotics lab
Returns:
@@ -697,6 +676,39 @@ def lookup_ww_sample_by_rsl_sample_number(ctx:dict, rsl_number:str) -> models.WW
"""
return ctx['database_session'].query(models.WWSample).filter(models.WWSample.rsl_number==rsl_number).first()
def lookup_ww_sample_by_sub_sample_rsl(ctx:dict, sample_rsl:str, plate_rsl:str) -> models.WWSample:
"""
Retrieves a wastewater sample from the database by its rsl sample number and parent rsl plate number.
This will likely replace simply looking up by the sample rsl above cine I need to control for repeats.
Args:
ctx (dict): settings passed down from the gui
sample_rsl (str): rsl number of the relevant sample
plate_rsl (str): rsl number of the parent plate
Returns:
models.WWSample: Relevant wastewater object
"""
return ctx['database_session'].query(models.WWSample).join(models.BasicSubmission).filter(models.BasicSubmission.rsl_plate_num==plate_rsl).filter(models.WWSample.rsl_number==sample_rsl).first()
def lookup_ww_sample_by_sub_sample_well(ctx:dict, sample_rsl:str, well_num:str, plate_rsl:str) -> models.WWSample:
"""
Retrieves a wastewater sample from the database by its rsl sample number and parent rsl plate number.
This will likely replace simply looking up by the sample rsl above cine I need to control for repeats.
Args:
ctx (dict): settings passed down from the gui
sample_rsl (str): rsl number of the relevant sample
well_num (str): well number of the relevant sample
plate_rsl (str): rsl number of the parent plate
Returns:
models.WWSample: Relevant wastewater object
"""
return ctx['database_session'].query(models.WWSample).join(models.BasicSubmission) \
.filter(models.BasicSubmission.rsl_plate_num==plate_rsl) \
.filter(models.WWSample.rsl_number==sample_rsl) \
.filter(models.WWSample.well_number==well_num).first()
def update_ww_sample(ctx:dict, sample_obj:dict):
"""
@@ -706,7 +718,10 @@ def update_ww_sample(ctx:dict, sample_obj:dict):
ctx (dict): settings passed down from gui
sample_obj (dict): dictionary representing new values for database object
"""
ww_samp = lookup_ww_sample_by_rsl_sample_number(ctx=ctx, rsl_number=sample_obj['sample'])
# ww_samp = lookup_ww_sample_by_rsl_sample_number(ctx=ctx, rsl_number=sample_obj['sample'])
logger.debug(f"Looking up {sample_obj['sample']} in plate {sample_obj['plate_rsl']}")
ww_samp = lookup_ww_sample_by_sub_sample_rsl(ctx=ctx, sample_rsl=sample_obj['sample'], plate_rsl=sample_obj['plate_rsl'])
# ww_samp = lookup_ww_sample_by_sub_sample_well(ctx=ctx, sample_rsl=sample_obj['sample'], well_num=sample_obj['well_num'], plate_rsl=sample_obj['plate_rsl'])
if ww_samp != None:
for key, value in sample_obj.items():
logger.debug(f"Setting {key} to {value}")

4
src/submissions/backend/db/models/samples.py
View File

@@ -27,6 +27,8 @@ class WWSample(Base):
notes = Column(String(2000))
ct_n1 = Column(FLOAT(2)) #: AKA ct for N1
ct_n2 = Column(FLOAT(2)) #: AKA ct for N2
n1_status = Column(String(32))
n2_status = Column(String(32))
seq_submitted = Column(BOOLEAN())
ww_seq_run_id = Column(String(64))
sample_type = Column(String(8))
@@ -50,7 +52,7 @@ class WWSample(Base):
dict: well location and id NOTE: keys must sync with BCSample to_sub_dict below
"""
if self.ct_n1 != None and self.ct_n2 != None:
name = f"{self.ww_sample_full_id}\n\t- ct N1: {'{:.2f}'.format(self.ct_n1)}, ct N2: {'{:.2f}'.format(self.ct_n1)}"
name = f"{self.ww_sample_full_id}\n\t- ct N1: {'{:.2f}'.format(self.ct_n1)} ({self.n1_status})\n\t- ct N2: {'{:.2f}'.format(self.ct_n2)} ({self.n2_status})"
else:
name = self.ww_sample_full_id
return {

82
src/submissions/backend/excel/parser.py
View File

@@ -2,17 +2,18 @@
contains parser object for pulling values from client generated submission sheets.
'''
from getpass import getuser
from typing import Tuple
import pandas as pd
from pathlib import Path
from backend.db.models import WWSample, BCSample
from backend.db import lookup_ww_sample_by_rsl_sample_number
# from backend.db import lookup_ww_sample_by_rsl_sample_number
import logging
from collections import OrderedDict
import re
import numpy as np
from datetime import date
import uuid
from tools import check_not_nan, retrieve_rsl_number
from tools import check_not_nan, RSLNamer
logger = logging.getLogger(f"submissions.{__name__}")
@@ -84,7 +85,7 @@ class SheetParser(object):
# self.xl is a pd.ExcelFile so we need to parse it into a df
submission_info = self.xl.parse(sheet_name=sheet_name, dtype=object)
self.sub['submitter_plate_num'] = submission_info.iloc[0][1]
self.sub['rsl_plate_num'] = submission_info.iloc[10][1]
self.sub['rsl_plate_num'] = RSLNamer(submission_info.iloc[10][1]).parsed_name
self.sub['submitted_date'] = submission_info.iloc[1][1]
self.sub['submitting_lab'] = submission_info.iloc[0][3]
self.sub['sample_count'] = submission_info.iloc[2][3]
@@ -202,7 +203,7 @@ class SheetParser(object):
parse_reagents(ext_reagent_range)
parse_reagents(pcr_reagent_range)
# parse samples
sample_parser = SampleParser(submission_info.iloc[16:40])
sample_parser = SampleParser(submission_info.iloc[16:])
sample_parse = getattr(sample_parser, f"parse_{self.sub['submission_type'].lower()}_samples")
self.sub['samples'] = sample_parse()
self.sub['csv'] = self.xl.parse("Copy to import file", dtype=object)
@@ -260,24 +261,20 @@ class SampleParser(object):
new_list = []
for sample in self.samples:
new = WWSample()
if check_not_nan(sample["Unnamed: 9"]):
new.rsl_number = sample['Unnamed: 9']
else:
logger.error(f"No RSL sample number found for this sample.")
continue
new.ww_processing_num = sample['Unnamed: 2']
# need to ensure we have a sample id for database integrity
try:
not_a_nan = not np.isnan(sample['Unnamed: 3'])
except TypeError:
not_a_nan = True
# if we don't have a sample full id, make one up
if not_a_nan:
if check_not_nan(sample['Unnamed: 3']):
new.ww_sample_full_id = sample['Unnamed: 3']
else:
new.ww_sample_full_id = uuid.uuid4().hex.upper()
new.rsl_number = sample['Unnamed: 9']
# need to ensure we get a collection date
try:
not_a_nan = not np.isnan(sample['Unnamed: 5'])
except TypeError:
not_a_nan = True
if not_a_nan:
if check_not_nan(sample['Unnamed: 5']):
new.collection_date = sample['Unnamed: 5']
else:
new.collection_date = date.today()
@@ -317,7 +314,9 @@ class PCRParser(object):
return
# self.pcr = OrderedDict()
self.pcr = {}
self.plate_num, self.submission_type = retrieve_rsl_number(filepath.__str__())
namer = RSLNamer(filepath.__str__())
self.plate_num = namer.parsed_name
self.submission_type = namer.submission_type
logger.debug(f"Set plate number to {self.plate_num} and type to {self.submission_type}")
self.samples = []
parser = getattr(self, f"parse_{self.submission_type}")
@@ -362,14 +361,25 @@ class PCRParser(object):
Parse specific to wastewater samples.
"""
df = self.parse_general(sheet_name="Results")
column_names = ["Well", "Well Position", "Omit","Sample","Target","Task"," Reporter","Quencher","Amp Status","Amp Score","Curve Quality","Result Quality Issues","Cq","Cq Confidence","Cq Mean","Cq SD","Auto Threshold","Threshold", "Auto Baseline", "Baseline Start", "Baseline End"]
self.samples_df = df.iloc[23:][0:]
self.samples_df.columns = column_names
logger.debug(f"Samples columns: {self.samples_df.columns}")
well_call_df = self.xl.parse(sheet_name="Well Call").iloc[24:][0:].iloc[:,-1:]
try:
self.samples_df['Assessment'] = well_call_df.values
except ValueError:
logger.error("Well call number doesn't match sample number")
logger.debug(f"Well call dr: {well_call_df}")
# iloc is [row][column]
for ii, row in self.samples_df.iterrows():
try:
sample_obj = [sample for sample in self.samples if sample['sample'] == row[3]][0]
except IndexError:
sample_obj = dict(
sample = row[3],
sample = row['Sample'],
plate_rsl = self.plate_num,
well_num = row['Well Position']
)
logger.debug(f"Got sample obj: {sample_obj}")
# logger.debug(f"row: {row}")
@@ -377,22 +387,30 @@ class PCRParser(object):
# # logger.debug(f"Looking up: {rsl_num}")
# ww_samp = lookup_ww_sample_by_rsl_sample_number(ctx=self.ctx, rsl_number=rsl_num)
# logger.debug(f"Got: {ww_samp}")
match row[4]:
case "N1":
if isinstance(row[12], float):
sample_obj['ct_n1'] = row[12]
else:
sample_obj['ct_n1'] = 0.0
case "N2":
if isinstance(row[12], float):
sample_obj['ct_n2'] = row[12]
else:
sample_obj['ct_n2'] = 0.0
case _:
logger.warning(f"Unexpected input for row[4]: {row[4]}")
if isinstance(row['Cq'], float):
sample_obj[f"ct_{row['Target'].lower()}"] = row['Cq']
else:
sample_obj[f"ct_{row['Target'].lower()}"] = 0.0
try:
sample_obj[f"{row['Target'].lower()}_status"] = row['Assessment']
except KeyError:
logger.error(f"No assessment for {sample_obj['sample']}")
# match row["Target"]:
# case "N1":
# if isinstance(row['Cq'], float):
# sample_obj['ct_n1'] = row["Cq"]
# else:
# sample_obj['ct_n1'] = 0.0
# sample_obj['n1_status'] = row['Assessment']
# case "N2":
# if isinstance(row['Cq'], float):
# sample_obj['ct_n2'] = row['Assessment']
# else:
# sample_obj['ct_n2'] = 0.0
# case _:
# logger.warning(f"Unexpected input for row[4]: {row["Target"]}")
self.samples.append(sample_obj)