landon/Submissions-App
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

mid refactor for improved rebustness and readability

Browse Source
This commit is contained in:
Landon Wark
2023-03-15 15:38:02 -05:00
parent fc334155ff
commit c645d3a9cf
15 changed files with 337 additions and 468 deletions
Download Patch File Download Diff File Expand all files Collapse all files

176
src/submissions/backend/db/__init__.py
View File

@@ -21,19 +21,13 @@ from pathlib import Path
logger = logging.getLogger(f"submissions.{__name__}")
# The below should allow automatic creation of foreign keys in the database
# The below _should_ allow automatic creation of foreign keys in the database
@event.listens_for(Engine, "connect")
def set_sqlite_pragma(dbapi_connection, connection_record):
cursor = dbapi_connection.cursor()
cursor.execute("PRAGMA foreign_keys=ON")
cursor.close()
def get_kits_by_use( ctx:dict, kittype_str:str|None) -> list:
pass
# ctx dict should contain the database session
def store_submission(ctx:dict, base_submission:models.BasicSubmission) -> None|dict:
"""
Upserts submissions into database
@@ -73,21 +67,22 @@ def store_submission(ctx:dict, base_submission:models.BasicSubmission) -> None|d
def store_reagent(ctx:dict, reagent:models.Reagent) -> None|dict:
"""
_summary_
Inserts a reagent into the database.
Args:
ctx (dict): settings passed down from gui
reagent (models.Reagent): Reagent object to be added to db
Returns:
None|dict: obejct indicating issue to be reported in the gui
None|dict: object indicating issue to be reported in the gui
"""
logger.debug(reagent.__dict__)
logger.debug(f"Reagent dictionary: {reagent.__dict__}")
ctx['database_session'].add(reagent)
try:
ctx['database_session'].commit()
except (sqlite3.OperationalError, sqlalchemy.exc.OperationalError):
return {"message":"The database is locked for editing."}
return None
def construct_submission_info(ctx:dict, info_dict:dict) -> models.BasicSubmission:
@@ -103,12 +98,12 @@ def construct_submission_info(ctx:dict, info_dict:dict) -> models.BasicSubmissio
"""
# convert submission type into model name
query = info_dict['submission_type'].replace(" ", "")
# check database for existing object
# Ensure an rsl plate number exists for the plate
if info_dict["rsl_plate_num"] == 'nan' or info_dict["rsl_plate_num"] == None or not check_not_nan(info_dict["rsl_plate_num"]):
code = 2
instance = None
msg = "A proper RSL plate number is required."
return instance, {'code': 2, 'message': "A proper RSL plate number is required."}
# check database for existing object
instance = ctx['database_session'].query(models.BasicSubmission).filter(models.BasicSubmission.rsl_plate_num==info_dict['rsl_plate_num']).first()
# get model based on submission type converted above
logger.debug(f"Looking at models for submission type: {query}")
@@ -142,7 +137,8 @@ def construct_submission_info(ctx:dict, info_dict:dict) -> models.BasicSubmissio
field_value = lookup_org_by_name(ctx=ctx, name=q_str)
logger.debug(f"Got {field_value} for organization {q_str}")
case "submitter_plate_num":
# Because of unique constraint, the submitter plate number cannot be None, so...
# Because of unique constraint, there will be problems with
# multiple submissions named 'None', so...
logger.debug(f"Submitter plate id: {info_dict[item]}")
if info_dict[item] == None or info_dict[item] == "None":
logger.debug(f"Got None as a submitter plate number, inserting random string to preserve database unique constraint.")
@@ -156,7 +152,8 @@ def construct_submission_info(ctx:dict, info_dict:dict) -> models.BasicSubmissio
except AttributeError:
logger.debug(f"Could not set attribute: {item} to {info_dict[item]}")
continue
# calculate cost of the run: immutable cost + mutable times number of columns
# calculate cost of the run: immutable cost + mutable times number of columns
# This is now attached to submission upon creation to preserve at-run costs incase of cost increase in the future.
try:
instance.run_cost = instance.extraction_kit.immutable_cost + (instance.extraction_kit.mutable_cost * ((instance.sample_count / 8)/12))
except (TypeError, AttributeError):
@@ -167,7 +164,7 @@ def construct_submission_info(ctx:dict, info_dict:dict) -> models.BasicSubmissio
logger.debug(f"Constructed instance: {instance.to_string()}")
except AttributeError as e:
logger.debug(f"Something went wrong constructing instance {info_dict['rsl_plate_num']}: {e}")
logger.debug(msg)
logger.debug(f"Constructed submissions message: {msg}")
return instance, {'code':code, 'message':msg}
@@ -194,7 +191,7 @@ def construct_reagent(ctx:dict, info_dict:dict) -> models.Reagent:
case "type":
reagent.type = lookup_reagenttype_by_name(ctx=ctx, rt_name=info_dict[item].replace(" ", "_").lower())
# add end-of-life extension from reagent type to expiry date
# Edit: this will now be done only in the reporting phase to account for potential changes in end-of-life extensions
# NOTE: this will now be done only in the reporting phase to account for potential changes in end-of-life extensions
# try:
# reagent.expiry = reagent.expiry + reagent.type.eol_ext
# except TypeError as e:
@@ -204,7 +201,6 @@ def construct_reagent(ctx:dict, info_dict:dict) -> models.Reagent:
return reagent
def lookup_reagent(ctx:dict, reagent_lot:str) -> models.Reagent:
"""
Query db for reagent based on lot number
@@ -219,6 +215,7 @@ def lookup_reagent(ctx:dict, reagent_lot:str) -> models.Reagent:
lookedup = ctx['database_session'].query(models.Reagent).filter(models.Reagent.lot==reagent_lot).first()
return lookedup
def get_all_reagenttype_names(ctx:dict) -> list[str]:
"""
Lookup all reagent types and get names
@@ -232,6 +229,7 @@ def get_all_reagenttype_names(ctx:dict) -> list[str]:
lookedup = [item.__str__() for item in ctx['database_session'].query(models.ReagentType).all()]
return lookedup
def lookup_reagenttype_by_name(ctx:dict, rt_name:str) -> models.ReagentType:
"""
Lookup a single reagent type by name
@@ -251,7 +249,7 @@ def lookup_reagenttype_by_name(ctx:dict, rt_name:str) -> models.ReagentType:
def lookup_kittype_by_use(ctx:dict, used_by:str) -> list[models.KitType]:
"""
Lookup a kit by an sample type its used for
Lookup kits by a sample type its used for
Args:
ctx (dict): settings passed from gui
@@ -262,6 +260,7 @@ def lookup_kittype_by_use(ctx:dict, used_by:str) -> list[models.KitType]:
"""
return ctx['database_session'].query(models.KitType).filter(models.KitType.used_for.contains(used_by)).all()
def lookup_kittype_by_name(ctx:dict, name:str) -> models.KitType:
"""
Lookup a kit type by name
@@ -288,7 +287,6 @@ def lookup_regent_by_type_name(ctx:dict, type_name:str) -> list[models.Reagent]:
Returns:
list[models.Reagent]: list of retrieved reagents
"""
# return [item for item in ctx['database_session'].query(models.Reagent).join(models.Reagent.type, aliased=True).filter(models.ReagentType.name==type_name).all()]
return ctx['database_session'].query(models.Reagent).join(models.Reagent.type, aliased=True).filter(models.ReagentType.name==type_name).all()
@@ -308,8 +306,7 @@ def lookup_regent_by_type_name_and_kit_name(ctx:dict, type_name:str, kit_name:st
# Hang on, this is going to be a long one.
# by_type = ctx['database_session'].query(models.Reagent).join(models.Reagent.type, aliased=True).filter(models.ReagentType.name.endswith(type_name)).all()
rt_types = ctx['database_session'].query(models.ReagentType).filter(models.ReagentType.name.endswith(type_name))
# add filter for kit name... which I can not get to work.
# add_in = by_type.join(models.ReagentType.kits).filter(models.KitType.name==kit_name)
# add filter for kit name...
try:
check = not np.isnan(kit_name)
except TypeError:
@@ -317,12 +314,10 @@ def lookup_regent_by_type_name_and_kit_name(ctx:dict, type_name:str, kit_name:st
if check:
kit_type = lookup_kittype_by_name(ctx=ctx, name=kit_name)
logger.debug(f"reagenttypes: {[item.name for item in rt_types.all()]}, kit: {kit_type.name}")
# add in lookup for related kit_id
rt_types = rt_types.join(reagenttypes_kittypes).filter(reagenttypes_kittypes.c.kits_id==kit_type.id).first()
# for item in by_type:
# logger.debug([thing.name for thing in item.type.kits])
# output = [item for item in by_type if kit_name in [thing.name for thing in item.type.kits]]
# else:
else:
rt_types = rt_types.first()
output = rt_types.instances
return output
@@ -336,7 +331,7 @@ def lookup_all_submissions_by_type(ctx:dict, sub_type:str|None=None) -> list[mod
type (str | None, optional): submission type (should be string in D3 of excel sheet). Defaults to None.
Returns:
_type_: list of retrieved submissions
list[models.BasicSubmission]: list of retrieved submissions
"""
if sub_type == None:
subs = ctx['database_session'].query(models.BasicSubmission).all()
@@ -358,7 +353,7 @@ def lookup_all_orgs(ctx:dict) -> list[models.Organization]:
def lookup_org_by_name(ctx:dict, name:str|None) -> models.Organization:
"""
Lookup organization (lab) by name.
Lookup organization (lab) by (startswith) name.
Args:
ctx (dict): settings passed from gui
@@ -368,7 +363,6 @@ def lookup_org_by_name(ctx:dict, name:str|None) -> models.Organization:
models.Organization: retrieved organization
"""
logger.debug(f"Querying organization: {name}")
# return ctx['database_session'].query(models.Organization).filter(models.Organization.name==name).first()
return ctx['database_session'].query(models.Organization).filter(models.Organization.name.startswith(name)).first()
def submissions_to_df(ctx:dict, sub_type:str|None=None) -> pd.DataFrame:
@@ -383,10 +377,11 @@ def submissions_to_df(ctx:dict, sub_type:str|None=None) -> pd.DataFrame:
pd.DataFrame: dataframe constructed from retrieved submissions
"""
logger.debug(f"Type: {sub_type}")
# pass to lookup function
# use lookup function to create list of dicts
subs = [item.to_dict() for item in lookup_all_submissions_by_type(ctx=ctx, sub_type=sub_type)]
# make df from dicts (records) in list
df = pd.DataFrame.from_records(subs)
# logger.debug(f"Pre: {df['Technician']}")
# Exclude sub information
try:
df = df.drop("controls", axis=1)
except:
@@ -395,7 +390,6 @@ def submissions_to_df(ctx:dict, sub_type:str|None=None) -> pd.DataFrame:
df = df.drop("ext_info", axis=1)
except:
logger.warning(f"Couldn't drop 'controls' column from submissionsheet df.")
# logger.debug(f"Post: {df['Technician']}")
return df
@@ -413,13 +407,9 @@ def lookup_submission_by_id(ctx:dict, id:int) -> models.BasicSubmission:
return ctx['database_session'].query(models.BasicSubmission).filter(models.BasicSubmission.id==id).first()
def create_submission_details(ctx:dict, sub_id:int) -> dict:
pass
def lookup_submissions_by_date_range(ctx:dict, start_date:datetime.date, end_date:datetime.date) -> list[models.BasicSubmission]:
"""
Lookup submissions by range of submitted dates
Lookup submissions greater than start_date and less than end_date
Args:
ctx (dict): settings passed from gui
@@ -429,18 +419,21 @@ def lookup_submissions_by_date_range(ctx:dict, start_date:datetime.date, end_dat
Returns:
list[models.BasicSubmission]: list of retrieved submissions
"""
return ctx['database_session'].query(models.BasicSubmission).filter(and_(models.BasicSubmission.submitted_date > start_date, models.BasicSubmission.submitted_date < end_date)).all()
# return ctx['database_session'].query(models.BasicSubmission).filter(and_(models.BasicSubmission.submitted_date > start_date, models.BasicSubmission.submitted_date < end_date)).all()
start_date = start_date.strftime("%Y-%m-%d")
end_date = end_date.strftime("%Y-%m-%d")
return ctx['database_session'].query(models.BasicSubmission).filter(models.BasicSubmission.submitted_date.between(start_date, end_date)).all()
def get_all_Control_Types_names(ctx:dict) -> list[models.ControlType]:
def get_all_Control_Types_names(ctx:dict) -> list[str]:
"""
Grabs all control type names from db.
Args:
settings (dict): settings passed down from click. Defaults to {}.
settings (dict): settings passed down from gui.
Returns:
list: names list
list: list of controltype names
"""
conTypes = ctx['database_session'].query(models.ControlType).all()
conTypes = [conType.name for conType in conTypes]
@@ -451,6 +444,7 @@ def get_all_Control_Types_names(ctx:dict) -> list[models.ControlType]:
def create_kit_from_yaml(ctx:dict, exp:dict) -> dict:
"""
Create and store a new kit in the database based on a .yml file
TODO: split into create and store functions
Args:
ctx (dict): Context dictionary passed down from frontend
@@ -459,18 +453,20 @@ def create_kit_from_yaml(ctx:dict, exp:dict) -> dict:
Returns:
dict: a dictionary containing results of db addition
"""
# try:
# power_users = ctx['power_users']
# except KeyError:
# Don't want just anyone adding kits
if not check_is_power_user(ctx=ctx):
logger.debug(f"{getuser()} does not have permission to add kits.")
return {'code':1, 'message':"This user does not have permission to add kits.", "status":"warning"}
# iterate through keys in dict
for type in exp:
if type == "password":
continue
# A submission type may use multiple kits.
for kt in exp[type]['kits']:
kit = models.KitType(name=kt, used_for=[type.replace("_", " ").title()], cost_per_run=exp[type]["kits"][kt]["cost"])
kit = models.KitType(name=kt, used_for=[type.replace("_", " ").title()], constant_cost=exp[type]["kits"][kt]["constant_cost"], mutable_cost=exp[type]["kits"][kt]["mutable_cost"])
# A kit contains multiple reagent types.
for r in exp[type]['kits'][kt]['reagenttypes']:
# check if reagent type already exists.
look_up = ctx['database_session'].query(models.ReagentType).filter(models.ReagentType.name==r).first()
if look_up == None:
rt = models.ReagentType(name=r.replace(" ", "_").lower(), eol_ext=timedelta(30*exp[type]['kits'][kt]['reagenttypes'][r]['eol_ext']), kits=[kit])
@@ -478,15 +474,15 @@ def create_kit_from_yaml(ctx:dict, exp:dict) -> dict:
rt = look_up
rt.kits.append(kit)
# add this because I think it's necessary to get proper back population
# rt.kit_id.append(kit.id)
kit.reagent_types_id.append(rt.id)
ctx['database_session'].add(rt)
logger.debug(rt.__dict__)
logger.debug(kit.__dict__)
logger.debug(f"Kit construction reagent type: {rt.__dict__}")
logger.debug(f"Kit construction kit: {kit.__dict__}")
ctx['database_session'].add(kit)
ctx['database_session'].commit()
return {'code':0, 'message':'Kit has been added', 'status': 'information'}
def create_org_from_yaml(ctx:dict, org:dict) -> dict:
"""
Create and store a new organization based on a .yml file
@@ -498,30 +494,26 @@ def create_org_from_yaml(ctx:dict, org:dict) -> dict:
Returns:
dict: dictionary containing results of db addition
"""
# try:
# power_users = ctx['power_users']
# except KeyError:
# logger.debug("This user does not have permission to add kits.")
# return {'code':1,'message':"This user does not have permission to add organizations."}
# logger.debug(f"Adding organization for user: {getuser()}")
# if getuser() not in power_users:
# Don't want just anyone adding in clients
if not check_is_power_user(ctx=ctx):
logger.debug(f"{getuser()} does not have permission to add kits.")
return {'code':1, 'message':"This user does not have permission to add organizations."}
# the yml can contain multiple clients
for client in org:
cli_org = models.Organization(name=client.replace(" ", "_").lower(), cost_centre=org[client]['cost centre'])
# a client can contain multiple contacts
for contact in org[client]['contacts']:
cont_name = list(contact.keys())[0]
# check if contact already exists
look_up = ctx['database_session'].query(models.Contact).filter(models.Contact.name==cont_name).first()
if look_up == None:
cli_cont = models.Contact(name=cont_name, phone=contact[cont_name]['phone'], email=contact[cont_name]['email'], organization=[cli_org])
else:
cli_cont = look_up
cli_cont.organization.append(cli_org)
# cli_org.contacts.append(cli_cont)
# cli_org.contact_ids.append_foreign_key(cli_cont.id)
ctx['database_session'].add(cli_cont)
logger.debug(cli_cont.__dict__)
logger.debug(f"Client creation contact: {cli_cont.__dict__}")
logger.debug(f"Client creation client: {cli_org.__dict__}")
ctx['database_session'].add(cli_org)
ctx["database_session"].commit()
return {"code":0, "message":"Organization has been added."}
@@ -538,11 +530,11 @@ def lookup_all_sample_types(ctx:dict) -> list[str]:
list[str]: list of sample type names
"""
uses = [item.used_for for item in ctx['database_session'].query(models.KitType).all()]
# flattened list of lists
uses = list(set([item for sublist in uses for item in sublist]))
return uses
def get_all_available_modes(ctx:dict) -> list[str]:
"""
Get types of analysis for controls
@@ -553,6 +545,7 @@ def get_all_available_modes(ctx:dict) -> list[str]:
Returns:
list[str]: list of analysis types
"""
# Only one control is necessary since they all share the same control types.
rel = ctx['database_session'].query(models.Control).first()
try:
cols = [item.name for item in list(rel.__table__.columns) if isinstance(item.type, JSON)]
@@ -562,54 +555,49 @@ def get_all_available_modes(ctx:dict) -> list[str]:
return cols
def get_all_controls_by_type(ctx:dict, con_type:str, start_date:date|None=None, end_date:date|None=None) -> list[models.Control]:
"""
Returns a list of control objects that are instances of the input controltype.
Between dates if supplied.
Args:
con_type (str): Name of the control type.
ctx (dict): Settings passed down from gui.
ctx (dict): Settings passed down from gui
con_type (str): Name of control type.
start_date (date | None, optional): Start date of query. Defaults to None.
end_date (date | None, optional): End date of query. Defaults to None.
Returns:
list: Control instances.
"""
list[models.Control]: list of control samples.
"""
logger.debug(f"Using dates: {start_date} to {end_date}")
if start_date != None and end_date != None:
output = ctx['database_session'].query(models.Control).join(models.ControlType).filter_by(name=con_type).filter(models.Control.submitted_date.between(start_date.strftime("%Y-%m-%d"), end_date.strftime("%Y-%m-%d"))).all()
start_date = start_date.strftime("%Y-%m-%d")
end_date = end_date.strftime("%Y-%m-%d")
output = ctx['database_session'].query(models.Control).join(models.ControlType).filter_by(name=con_type).filter(models.Control.submitted_date.between(start_date, end_date)).all()
else:
output = ctx['database_session'].query(models.Control).join(models.ControlType).filter_by(name=con_type).all()
logger.debug(f"Returned controls between dates: {output}")
return output
# query = ctx['database_session'].query(models.ControlType).filter_by(name=con_type)
# try:
# output = query.first().instances
# except AttributeError:
# output = None
# # Hacky solution to my not being able to get the sql query to work.
# if start_date != None and end_date != None:
# output = [item for item in output if item.submitted_date.date() > start_date and item.submitted_date.date() < end_date]
# # logger.debug(f"Type {con_type}: {query.first()}")
# return output
def get_control_subtypes(ctx:dict, type:str, mode:str) -> list[str]:
"""
Get subtypes for a control analysis type
Get subtypes for a control analysis mode
Args:
ctx (dict): settings passed from gui
type (str): control type name
mode (str): analysis type name
mode (str): analysis mode name
Returns:
list[str]: list of subtype names
"""
# Only the first control of type is necessary since they all share subtypes
try:
outs = get_all_controls_by_type(ctx=ctx, con_type=type)[0]
except TypeError:
return []
# Get analysis mode data as dict
jsoner = json.loads(getattr(outs, mode))
logger.debug(f"JSON out: {jsoner}")
try:
@@ -620,11 +608,30 @@ def get_control_subtypes(ctx:dict, type:str, mode:str) -> list[str]:
return subtypes
def get_all_controls(ctx:dict):
def get_all_controls(ctx:dict) -> list[models.Control]:
"""
Retrieve a list of all controls from the database
Args:
ctx (dict): settings passed down from the gui.
Returns:
list[models.Control]: list of all control objects
"""
return ctx['database_session'].query(models.Control).all()
def lookup_submission_by_rsl_num(ctx:dict, rsl_num:str):
def lookup_submission_by_rsl_num(ctx:dict, rsl_num:str) -> models.BasicSubmission:
"""
Retrieve a submission from the database based on rsl plate number
Args:
ctx (dict): settings passed down from gui
rsl_num (str): rsl plate number
Returns:
models.BasicSubmission: Submissions object retrieved from database
"""
return ctx['database_session'].query(models.BasicSubmission).filter(models.BasicSubmission.rsl_plate_num.startswith(rsl_num)).first()
@@ -641,10 +648,15 @@ def delete_submission_by_id(ctx:dict, id:int) -> None:
id (int): id of submission to be deleted.
"""
# In order to properly do this Im' going to have to delete all of the secondary table stuff as well.
# Retrieve submission
sub = ctx['database_session'].query(models.BasicSubmission).filter(models.BasicSubmission.id==id).first()
# Convert to dict for storing backup as a yml
backup = sub.to_dict()
with open(Path(ctx['backup_path']).joinpath(f"{sub.rsl_plate_num}-backup({date.today().strftime('%Y%m%d')}).yml"), "w") as f:
yaml.dump(backup, f)
try:
with open(Path(ctx['backup_path']).joinpath(f"{sub.rsl_plate_num}-backup({date.today().strftime('%Y%m%d')}).yml"), "w") as f:
yaml.dump(backup, f)
except KeyError:
pass
sub.reagents = []
for sample in sub.samples:
ctx['database_session'].delete(sample)