landon/Submissions-App
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Large scale code cleanup

Browse Source
This commit is contained in:
Landon Wark
2023-07-26 11:08:30 -05:00
parent 63308674c6
commit 82dffe3af2
16 changed files with 686 additions and 1289 deletions
Download Patch File Download Diff File Expand all files Collapse all files

668
src/submissions/backend/db/__init__.py
View File

@@ -1,670 +1,4 @@
'''
All database related operations.
'''
from .functions import *
# from . import models
# from .models.kits import reagenttypes_kittypes
# from .models.submissions import reagents_submissions
# import pandas as pd
# import sqlalchemy.exc
# import sqlite3
# import logging
# from datetime import date, datetime, timedelta
# from sqlalchemy import and_
# import uuid
# # import base64
# from sqlalchemy import JSON, event
# from sqlalchemy.engine import Engine
# import json
# # from dateutil.relativedelta import relativedelta
# from getpass import getuser
# import numpy as np
# from tools import check_not_nan, check_is_power_user
# import yaml
# from pathlib import Path
# logger = logging.getLogger(f"submissions.{__name__}")
# # The below _should_ allow automatic creation of foreign keys in the database
# @event.listens_for(Engine, "connect")
# def set_sqlite_pragma(dbapi_connection, connection_record):
# cursor = dbapi_connection.cursor()
# cursor.execute("PRAGMA foreign_keys=ON")
# cursor.close()
# def store_submission(ctx:dict, base_submission:models.BasicSubmission) -> None|dict:
# """
# Upserts submissions into database
# Args:
# ctx (dict): settings passed down from gui
# base_submission (models.BasicSubmission): submission to be add to db
# Returns:
# None|dict : object that indicates issue raised for reporting in gui
# """
# logger.debug(f"Hello from store_submission")
# # Add all samples to sample table
# for sample in base_submission.samples:
# sample.rsl_plate = base_submission
# logger.debug(f"Attempting to add sample: {sample.to_string()}")
# try:
# ctx['database_session'].add(sample)
# except (sqlite3.IntegrityError, sqlalchemy.exc.IntegrityError) as e:
# logger.debug(f"Hit an integrity error : {e}")
# continue
# # Add submission to submission table
# ctx['database_session'].add(base_submission)
# logger.debug(f"Attempting to add submission: {base_submission.rsl_plate_num}")
# try:
# ctx['database_session'].commit()
# except (sqlite3.IntegrityError, sqlalchemy.exc.IntegrityError) as e:
# logger.debug(f"Hit an integrity error : {e}")
# ctx['database_session'].rollback()
# return {"message":"This plate number already exists, so we can't add it."}
# except (sqlite3.OperationalError, sqlalchemy.exc.IntegrityError) as e:
# logger.debug(f"Hit an operational error: {e}")
# ctx['database_session'].rollback()
# return {"message":"The database is locked for editing."}
# return None
# def store_reagent(ctx:dict, reagent:models.Reagent) -> None|dict:
# """
# Inserts a reagent into the database.
# Args:
# ctx (dict): settings passed down from gui
# reagent (models.Reagent): Reagent object to be added to db
# Returns:
# None|dict: object indicating issue to be reported in the gui
# """
# logger.debug(f"Reagent dictionary: {reagent.__dict__}")
# ctx['database_session'].add(reagent)
# try:
# ctx['database_session'].commit()
# except (sqlite3.OperationalError, sqlalchemy.exc.OperationalError):
# return {"message":"The database is locked for editing."}
# return None
# def construct_submission_info(ctx:dict, info_dict:dict) -> models.BasicSubmission:
# """
# Construct submission object from dictionary
# Args:
# ctx (dict): settings passed down from gui
# info_dict (dict): dictionary to be transformed
# Returns:
# models.BasicSubmission: Constructed submission object
# """
# # convert submission type into model name
# query = info_dict['submission_type'].replace(" ", "")
# # Ensure an rsl plate number exists for the plate
# if info_dict["rsl_plate_num"] == 'nan' or info_dict["rsl_plate_num"] == None or not check_not_nan(info_dict["rsl_plate_num"]):
# instance = None
# msg = "A proper RSL plate number is required."
# return instance, {'code': 2, 'message': "A proper RSL plate number is required."}
# # check database for existing object
# instance = ctx['database_session'].query(models.BasicSubmission).filter(models.BasicSubmission.rsl_plate_num==info_dict['rsl_plate_num']).first()
# # get model based on submission type converted above
# logger.debug(f"Looking at models for submission type: {query}")
# model = getattr(models, query)
# logger.debug(f"We've got the model: {type(model)}")
# info_dict['submission_type'] = info_dict['submission_type'].replace(" ", "_").lower()
# # if query return nothing, ie doesn't already exist in db
# if instance == None:
# instance = model()
# logger.debug(f"Submission doesn't exist yet, creating new instance: {instance}")
# msg = None
# code = 0
# else:
# code = 1
# msg = "This submission already exists.\nWould you like to overwrite?"
# for item in info_dict:
# logger.debug(f"Setting {item} to {info_dict[item]}")
# # set fields based on keys in dictionary
# match item:
# case "extraction_kit":
# q_str = info_dict[item]
# logger.debug(f"Looking up kit {q_str}")
# try:
# field_value = lookup_kittype_by_name(ctx=ctx, name=q_str)
# except (sqlite3.IntegrityError, sqlalchemy.exc.IntegrityError) as e:
# logger.error(f"Hit an integrity error: {e}")
# logger.debug(f"Got {field_value} for kit {q_str}")
# case "submitting_lab":
# q_str = info_dict[item].replace(" ", "_").lower()
# logger.debug(f"Looking up organization: {q_str}")
# field_value = lookup_org_by_name(ctx=ctx, name=q_str)
# logger.debug(f"Got {field_value} for organization {q_str}")
# case "submitter_plate_num":
# # Because of unique constraint, there will be problems with
# # multiple submissions named 'None', so...
# logger.debug(f"Submitter plate id: {info_dict[item]}")
# if info_dict[item] == None or info_dict[item] == "None":
# logger.debug(f"Got None as a submitter plate number, inserting random string to preserve database unique constraint.")
# info_dict[item] = uuid.uuid4().hex.upper()
# field_value = info_dict[item]
# case _:
# field_value = info_dict[item]
# # insert into field
# try:
# setattr(instance, item, field_value)
# except AttributeError:
# logger.debug(f"Could not set attribute: {item} to {info_dict[item]}")
# continue
# # calculate cost of the run: immutable cost + mutable times number of columns
# # This is now attached to submission upon creation to preserve at-run costs incase of cost increase in the future.
# try:
# instance.run_cost = instance.extraction_kit.immutable_cost + (instance.extraction_kit.mutable_cost * ((instance.sample_count / 8)/12))
# except (TypeError, AttributeError):
# logger.debug(f"Looks like that kit doesn't have cost breakdown yet, using full plate cost.")
# instance.run_cost = instance.extraction_kit.cost_per_run
# # We need to make sure there's a proper rsl plate number
# try:
# logger.debug(f"Constructed instance: {instance.to_string()}")
# except AttributeError as e:
# logger.debug(f"Something went wrong constructing instance {info_dict['rsl_plate_num']}: {e}")
# logger.debug(f"Constructed submissions message: {msg}")
# return instance, {'code':code, 'message':msg}
# def construct_reagent(ctx:dict, info_dict:dict) -> models.Reagent:
# """
# Construct reagent object from dictionary
# Args:
# ctx (dict): settings passed down from gui
# info_dict (dict): dictionary to be converted
# Returns:
# models.Reagent: Constructed reagent object
# """
# reagent = models.Reagent()
# for item in info_dict:
# logger.debug(f"Reagent info item: {item}")
# # set fields based on keys in dictionary
# match item:
# case "lot":
# reagent.lot = info_dict[item].upper()
# case "expiry":
# reagent.expiry = info_dict[item]
# case "type":
# reagent.type = lookup_reagenttype_by_name(ctx=ctx, rt_name=info_dict[item].replace(" ", "_").lower())
# # add end-of-life extension from reagent type to expiry date
# # NOTE: this will now be done only in the reporting phase to account for potential changes in end-of-life extensions
# # try:
# # reagent.expiry = reagent.expiry + reagent.type.eol_ext
# # except TypeError as e:
# # logger.debug(f"We got a type error: {e}.")
# # except AttributeError:
# # pass
# return reagent
# def lookup_reagent(ctx:dict, reagent_lot:str) -> models.Reagent:
# """
# Query db for reagent based on lot number
# Args:
# ctx (dict): settings passed down from gui
# reagent_lot (str): lot number to query
# Returns:
# models.Reagent: looked up reagent
# """
# lookedup = ctx['database_session'].query(models.Reagent).filter(models.Reagent.lot==reagent_lot).first()
# return lookedup
# def get_all_reagenttype_names(ctx:dict) -> list[str]:
# """
# Lookup all reagent types and get names
# Args:
# ctx (dict): settings passed from gui
# Returns:
# list[str]: reagent type names
# """
# lookedup = [item.__str__() for item in ctx['database_session'].query(models.ReagentType).all()]
# return lookedup
# def lookup_reagenttype_by_name(ctx:dict, rt_name:str) -> models.ReagentType:
# """
# Lookup a single reagent type by name
# Args:
# ctx (dict): settings passed from gui
# rt_name (str): reagent type name to look up
# Returns:
# models.ReagentType: looked up reagent type
# """
# logger.debug(f"Looking up ReagentType by name: {rt_name}")
# lookedup = ctx['database_session'].query(models.ReagentType).filter(models.ReagentType.name==rt_name).first()
# logger.debug(f"Found ReagentType: {lookedup}")
# return lookedup
# def lookup_kittype_by_use(ctx:dict, used_by:str) -> list[models.KitType]:
# """
# Lookup kits by a sample type its used for
# Args:
# ctx (dict): settings passed from gui
# used_by (str): sample type (should be string in D3 of excel sheet)
# Returns:
# list[models.KitType]: list of kittypes that have that sample type in their uses
# """
# return ctx['database_session'].query(models.KitType).filter(models.KitType.used_for.contains(used_by)).all()
# def lookup_kittype_by_name(ctx:dict, name:str) -> models.KitType:
# """
# Lookup a kit type by name
# Args:
# ctx (dict): settings passed from bui
# name (str): name of kit to query
# Returns:
# models.KitType: retrieved kittype
# """
# logger.debug(f"Querying kittype: {name}")
# return ctx['database_session'].query(models.KitType).filter(models.KitType.name==name).first()
# def lookup_regent_by_type_name(ctx:dict, type_name:str) -> list[models.Reagent]:
# """
# Lookup reagents by their type name
# Args:
# ctx (dict): settings passed from gui
# type_name (str): reagent type name
# Returns:
# list[models.Reagent]: list of retrieved reagents
# """
# return ctx['database_session'].query(models.Reagent).join(models.Reagent.type, aliased=True).filter(models.ReagentType.name==type_name).all()
# def lookup_regent_by_type_name_and_kit_name(ctx:dict, type_name:str, kit_name:str) -> list[models.Reagent]:
# """
# Lookup reagents by their type name and kits they belong to (Broken... maybe cursed, I'm not sure.)
# Args:
# ctx (dict): settings pass by gui
# type_name (str): reagent type name
# kit_name (str): kit name
# Returns:
# list[models.Reagent]: list of retrieved reagents
# """
# # What I want to do is get the reagent type by name
# # Hang on, this is going to be a long one.
# # by_type = ctx['database_session'].query(models.Reagent).join(models.Reagent.type, aliased=True).filter(models.ReagentType.name.endswith(type_name)).all()
# rt_types = ctx['database_session'].query(models.ReagentType).filter(models.ReagentType.name.endswith(type_name))
# # add filter for kit name...
# try:
# check = not np.isnan(kit_name)
# except TypeError:
# check = True
# if check:
# kit_type = lookup_kittype_by_name(ctx=ctx, name=kit_name)
# logger.debug(f"reagenttypes: {[item.name for item in rt_types.all()]}, kit: {kit_type.name}")
# # add in lookup for related kit_id
# rt_types = rt_types.join(reagenttypes_kittypes).filter(reagenttypes_kittypes.c.kits_id==kit_type.id).first()
# else:
# rt_types = rt_types.first()
# output = rt_types.instances
# return output
# def lookup_all_submissions_by_type(ctx:dict, sub_type:str|None=None) -> list[models.BasicSubmission]:
# """
# Get all submissions, filtering by type if given
# Args:
# ctx (dict): settings pass from gui
# type (str | None, optional): submission type (should be string in D3 of excel sheet). Defaults to None.
# Returns:
# list[models.BasicSubmission]: list of retrieved submissions
# """
# if sub_type == None:
# subs = ctx['database_session'].query(models.BasicSubmission).all()
# else:
# subs = ctx['database_session'].query(models.BasicSubmission).filter(models.BasicSubmission.submission_type==sub_type.lower().replace(" ", "_")).all()
# return subs
# def lookup_all_orgs(ctx:dict) -> list[models.Organization]:
# """
# Lookup all organizations (labs)
# Args:
# ctx (dict): settings passed from gui
# Returns:
# list[models.Organization]: list of retrieved organizations
# """
# return ctx['database_session'].query(models.Organization).all()
# def lookup_org_by_name(ctx:dict, name:str|None) -> models.Organization:
# """
# Lookup organization (lab) by (startswith) name.
# Args:
# ctx (dict): settings passed from gui
# name (str | None): name of organization
# Returns:
# models.Organization: retrieved organization
# """
# logger.debug(f"Querying organization: {name}")
# return ctx['database_session'].query(models.Organization).filter(models.Organization.name.startswith(name)).first()
# def submissions_to_df(ctx:dict, sub_type:str|None=None) -> pd.DataFrame:
# """
# Convert submissions looked up by type to dataframe
# Args:
# ctx (dict): settings passed by gui
# type (str | None, optional): submission type (should be string in D3 of excel sheet) Defaults to None.
# Returns:
# pd.DataFrame: dataframe constructed from retrieved submissions
# """
# logger.debug(f"Type: {sub_type}")
# # use lookup function to create list of dicts
# subs = [item.to_dict() for item in lookup_all_submissions_by_type(ctx=ctx, sub_type=sub_type)]
# # make df from dicts (records) in list
# df = pd.DataFrame.from_records(subs)
# # Exclude sub information
# try:
# df = df.drop("controls", axis=1)
# except:
# logger.warning(f"Couldn't drop 'controls' column from submissionsheet df.")
# try:
# df = df.drop("ext_info", axis=1)
# except:
# logger.warning(f"Couldn't drop 'controls' column from submissionsheet df.")
# return df
# def lookup_submission_by_id(ctx:dict, id:int) -> models.BasicSubmission:
# """
# Lookup submission by id number
# Args:
# ctx (dict): settings passed from gui
# id (int): submission id number
# Returns:
# models.BasicSubmission: retrieved submission
# """
# return ctx['database_session'].query(models.BasicSubmission).filter(models.BasicSubmission.id==id).first()
# def lookup_submissions_by_date_range(ctx:dict, start_date:datetime.date, end_date:datetime.date) -> list[models.BasicSubmission]:
# """
# Lookup submissions greater than start_date and less than end_date
# Args:
# ctx (dict): settings passed from gui
# start_date (datetime.date): date to start looking
# end_date (datetime.date): date to end looking
# Returns:
# list[models.BasicSubmission]: list of retrieved submissions
# """
# # return ctx['database_session'].query(models.BasicSubmission).filter(and_(models.BasicSubmission.submitted_date > start_date, models.BasicSubmission.submitted_date < end_date)).all()
# start_date = start_date.strftime("%Y-%m-%d")
# end_date = end_date.strftime("%Y-%m-%d")
# return ctx['database_session'].query(models.BasicSubmission).filter(models.BasicSubmission.submitted_date.between(start_date, end_date)).all()
# def get_all_Control_Types_names(ctx:dict) -> list[str]:
# """
# Grabs all control type names from db.
# Args:
# settings (dict): settings passed down from gui.
# Returns:
# list: list of controltype names
# """
# conTypes = ctx['database_session'].query(models.ControlType).all()
# conTypes = [conType.name for conType in conTypes]
# logger.debug(f"Control Types: {conTypes}")
# return conTypes
# def create_kit_from_yaml(ctx:dict, exp:dict) -> dict:
# """
# Create and store a new kit in the database based on a .yml file
# TODO: split into create and store functions
# Args:
# ctx (dict): Context dictionary passed down from frontend
# exp (dict): Experiment dictionary created from yaml file
# Returns:
# dict: a dictionary containing results of db addition
# """
# # Don't want just anyone adding kits
# if not check_is_power_user(ctx=ctx):
# logger.debug(f"{getuser()} does not have permission to add kits.")
# return {'code':1, 'message':"This user does not have permission to add kits.", "status":"warning"}
# # iterate through keys in dict
# for type in exp:
# if type == "password":
# continue
# # A submission type may use multiple kits.
# for kt in exp[type]['kits']:
# kit = models.KitType(name=kt, used_for=[type.replace("_", " ").title()], constant_cost=exp[type]["kits"][kt]["constant_cost"], mutable_cost=exp[type]["kits"][kt]["mutable_cost"])
# # A kit contains multiple reagent types.
# for r in exp[type]['kits'][kt]['reagenttypes']:
# # check if reagent type already exists.
# look_up = ctx['database_session'].query(models.ReagentType).filter(models.ReagentType.name==r).first()
# if look_up == None:
# rt = models.ReagentType(name=r.replace(" ", "_").lower(), eol_ext=timedelta(30*exp[type]['kits'][kt]['reagenttypes'][r]['eol_ext']), kits=[kit])
# else:
# rt = look_up
# rt.kits.append(kit)
# # add this because I think it's necessary to get proper back population
# kit.reagent_types_id.append(rt.id)
# ctx['database_session'].add(rt)
# logger.debug(f"Kit construction reagent type: {rt.__dict__}")
# logger.debug(f"Kit construction kit: {kit.__dict__}")
# ctx['database_session'].add(kit)
# ctx['database_session'].commit()
# return {'code':0, 'message':'Kit has been added', 'status': 'information'}
# def create_org_from_yaml(ctx:dict, org:dict) -> dict:
# """
# Create and store a new organization based on a .yml file
# Args:
# ctx (dict): Context dictionary passed down from frontend
# org (dict): Dictionary containing organization info.
# Returns:
# dict: dictionary containing results of db addition
# """
# # Don't want just anyone adding in clients
# if not check_is_power_user(ctx=ctx):
# logger.debug(f"{getuser()} does not have permission to add kits.")
# return {'code':1, 'message':"This user does not have permission to add organizations."}
# # the yml can contain multiple clients
# for client in org:
# cli_org = models.Organization(name=client.replace(" ", "_").lower(), cost_centre=org[client]['cost centre'])
# # a client can contain multiple contacts
# for contact in org[client]['contacts']:
# cont_name = list(contact.keys())[0]
# # check if contact already exists
# look_up = ctx['database_session'].query(models.Contact).filter(models.Contact.name==cont_name).first()
# if look_up == None:
# cli_cont = models.Contact(name=cont_name, phone=contact[cont_name]['phone'], email=contact[cont_name]['email'], organization=[cli_org])
# else:
# cli_cont = look_up
# cli_cont.organization.append(cli_org)
# ctx['database_session'].add(cli_cont)
# logger.debug(f"Client creation contact: {cli_cont.__dict__}")
# logger.debug(f"Client creation client: {cli_org.__dict__}")
# ctx['database_session'].add(cli_org)
# ctx["database_session"].commit()
# return {"code":0, "message":"Organization has been added."}
# def lookup_all_sample_types(ctx:dict) -> list[str]:
# """
# Lookup all sample types and get names
# Args:
# ctx (dict): settings pass from gui
# Returns:
# list[str]: list of sample type names
# """
# uses = [item.used_for for item in ctx['database_session'].query(models.KitType).all()]
# # flattened list of lists
# uses = list(set([item for sublist in uses for item in sublist]))
# return uses
# def get_all_available_modes(ctx:dict) -> list[str]:
# """
# Get types of analysis for controls
# Args:
# ctx (dict): settings passed from gui
# Returns:
# list[str]: list of analysis types
# """
# # Only one control is necessary since they all share the same control types.
# rel = ctx['database_session'].query(models.Control).first()
# try:
# cols = [item.name for item in list(rel.__table__.columns) if isinstance(item.type, JSON)]
# except AttributeError as e:
# logger.debug(f"Failed to get available modes from db: {e}")
# cols = []
# return cols
# def get_all_controls_by_type(ctx:dict, con_type:str, start_date:date|None=None, end_date:date|None=None) -> list[models.Control]:
# """
# Returns a list of control objects that are instances of the input controltype.
# Between dates if supplied.
# Args:
# ctx (dict): Settings passed down from gui
# con_type (str): Name of control type.
# start_date (date | None, optional): Start date of query. Defaults to None.
# end_date (date | None, optional): End date of query. Defaults to None.
# Returns:
# list[models.Control]: list of control samples.
# """
# logger.debug(f"Using dates: {start_date} to {end_date}")
# if start_date != None and end_date != None:
# start_date = start_date.strftime("%Y-%m-%d")
# end_date = end_date.strftime("%Y-%m-%d")
# output = ctx['database_session'].query(models.Control).join(models.ControlType).filter_by(name=con_type).filter(models.Control.submitted_date.between(start_date, end_date)).all()
# else:
# output = ctx['database_session'].query(models.Control).join(models.ControlType).filter_by(name=con_type).all()
# logger.debug(f"Returned controls between dates: {output}")
# return output
# def get_control_subtypes(ctx:dict, type:str, mode:str) -> list[str]:
# """
# Get subtypes for a control analysis mode
# Args:
# ctx (dict): settings passed from gui
# type (str): control type name
# mode (str): analysis mode name
# Returns:
# list[str]: list of subtype names
# """
# # Only the first control of type is necessary since they all share subtypes
# try:
# outs = get_all_controls_by_type(ctx=ctx, con_type=type)[0]
# except TypeError:
# return []
# # Get analysis mode data as dict
# jsoner = json.loads(getattr(outs, mode))
# logger.debug(f"JSON out: {jsoner}")
# try:
# genera = list(jsoner.keys())[0]
# except IndexError:
# return []
# subtypes = [item for item in jsoner[genera] if "_hashes" not in item and "_ratio" not in item]
# return subtypes
# def get_all_controls(ctx:dict) -> list[models.Control]:
# """
# Retrieve a list of all controls from the database
# Args:
# ctx (dict): settings passed down from the gui.
# Returns:
# list[models.Control]: list of all control objects
# """
# return ctx['database_session'].query(models.Control).all()
# def lookup_submission_by_rsl_num(ctx:dict, rsl_num:str) -> models.BasicSubmission:
# """
# Retrieve a submission from the database based on rsl plate number
# Args:
# ctx (dict): settings passed down from gui
# rsl_num (str): rsl plate number
# Returns:
# models.BasicSubmission: Submissions object retrieved from database
# """
# return ctx['database_session'].query(models.BasicSubmission).filter(models.BasicSubmission.rsl_plate_num.startswith(rsl_num)).first()
# def lookup_submissions_using_reagent(ctx:dict, reagent:models.Reagent) -> list[models.BasicSubmission]:
# return ctx['database_session'].query(models.BasicSubmission).join(reagents_submissions).filter(reagents_submissions.c.reagent_id==reagent.id).all()
# def delete_submission_by_id(ctx:dict, id:int) -> None:
# """
# Deletes a submission and its associated samples from the database.
# Args:
# ctx (dict): settings passed down from gui
# id (int): id of submission to be deleted.
# """
# # In order to properly do this Im' going to have to delete all of the secondary table stuff as well.
# # Retrieve submission
# sub = ctx['database_session'].query(models.BasicSubmission).filter(models.BasicSubmission.id==id).first()
# # Convert to dict for storing backup as a yml
# backup = sub.to_dict()
# try:
# with open(Path(ctx['backup_path']).joinpath(f"{sub.rsl_plate_num}-backup({date.today().strftime('%Y%m%d')}).yml"), "w") as f:
# yaml.dump(backup, f)
# except KeyError:
# pass
# sub.reagents = []
# for sample in sub.samples:
# ctx['database_session'].delete(sample)
# ctx["database_session"].delete(sub)
# ctx["database_session"].commit()
from .functions import *

360
src/submissions/backend/db/functions.py
View File

@@ -3,8 +3,8 @@ Convenience functions for interacting with the database.
'''
from . import models
from .models.kits import reagenttypes_kittypes
from .models.submissions import reagents_submissions
from .models.kits import reagenttypes_kittypes, KitType
from .models.submissions import reagents_submissions, BasicSubmission
import pandas as pd
import sqlalchemy.exc
import sqlite3
@@ -18,6 +18,7 @@ from getpass import getuser
import numpy as np
import yaml
from pathlib import Path
from tools import Settings
@@ -31,12 +32,12 @@ def set_sqlite_pragma(dbapi_connection, connection_record):
cursor.close()
def store_submission(ctx:dict, base_submission:models.BasicSubmission) -> None|dict:
def store_submission(ctx:Settings, base_submission:models.BasicSubmission) -> None|dict:
"""
Upserts submissions into database
Args:
ctx (dict): settings passed down from gui
ctx (Settings): settings object passed down from gui
base_submission (models.BasicSubmission): submission to be add to db
Returns:
@@ -57,50 +58,57 @@ def store_submission(ctx:dict, base_submission:models.BasicSubmission) -> None|d
sample.artic_rsl_plate = base_submission
logger.debug(f"Attempting to add sample: {sample.to_string()}")
try:
ctx['database_session'].add(sample)
# ctx['database_session'].add(sample)
ctx.database_session.add(sample)
except (sqlite3.IntegrityError, sqlalchemy.exc.IntegrityError) as e:
logger.debug(f"Hit an integrity error : {e}")
continue
# Add submission to submission table
ctx['database_session'].add(base_submission)
# ctx['database_session'].add(base_submission)
ctx.database_session.add(base_submission)
logger.debug(f"Attempting to add submission: {base_submission.rsl_plate_num}")
try:
ctx['database_session'].commit()
# ctx['database_session'].commit()
ctx.database_session.commit()
except (sqlite3.IntegrityError, sqlalchemy.exc.IntegrityError) as e:
logger.debug(f"Hit an integrity error : {e}")
ctx['database_session'].rollback()
# ctx['database_session'].rollback()
ctx.database_session.rollback()
return {"message":"This plate number already exists, so we can't add it.", "status":"Critical"}
except (sqlite3.OperationalError, sqlalchemy.exc.IntegrityError) as e:
logger.debug(f"Hit an operational error: {e}")
ctx['database_session'].rollback()
# ctx['database_session'].rollback()
ctx.database_session.rollback()
return {"message":"The database is locked for editing.", "status":"Critical"}
return None
def store_reagent(ctx:dict, reagent:models.Reagent) -> None|dict:
def store_reagent(ctx:Settings, reagent:models.Reagent) -> None|dict:
"""
Inserts a reagent into the database.
Args:
ctx (dict): settings passed down from gui
ctx (Settings): settings object passed down from gui
reagent (models.Reagent): Reagent object to be added to db
Returns:
None|dict: object indicating issue to be reported in the gui
"""
logger.debug(f"Reagent dictionary: {reagent.__dict__}")
ctx['database_session'].add(reagent)
# ctx['database_session'].add(reagent)
ctx.database_session.add(reagent)
try:
ctx['database_session'].commit()
# ctx['database_session'].commit()
ctx.database_session.commit()
except (sqlite3.OperationalError, sqlalchemy.exc.OperationalError):
return {"message":"The database is locked for editing."}
return None
def construct_submission_info(ctx:dict, info_dict:dict) -> models.BasicSubmission:
def construct_submission_info(ctx:Settings, info_dict:dict) -> models.BasicSubmission:
"""
Construct submission object from dictionary
Args:
ctx (dict): settings passed down from gui
ctx (Settings): settings object passed down from gui
info_dict (dict): dictionary to be transformed
Returns:
@@ -118,7 +126,8 @@ def construct_submission_info(ctx:dict, info_dict:dict) -> models.BasicSubmissio
# enforce conventions on the rsl plate number from the form
info_dict['rsl_plate_num'] = RSLNamer(ctx=ctx, instr=info_dict["rsl_plate_num"]).parsed_name
# check database for existing object
instance = ctx['database_session'].query(models.BasicSubmission).filter(models.BasicSubmission.rsl_plate_num==info_dict['rsl_plate_num']).first()
# instance = ctx['database_session'].query(models.BasicSubmission).filter(models.BasicSubmission.rsl_plate_num==info_dict['rsl_plate_num']).first()
instance = ctx.database_session.query(models.BasicSubmission).filter(models.BasicSubmission.rsl_plate_num==info_dict['rsl_plate_num']).first()
# get model based on submission type converted above
logger.debug(f"Looking at models for submission type: {query}")
model = getattr(models, query)
@@ -201,12 +210,12 @@ def construct_submission_info(ctx:dict, info_dict:dict) -> models.BasicSubmissio
logger.debug(f"Constructed submissions message: {msg}")
return instance, {'code':code, 'message':msg}
def construct_reagent(ctx:dict, info_dict:dict) -> models.Reagent:
def construct_reagent(ctx:Settings, info_dict:dict) -> models.Reagent:
"""
Construct reagent object from dictionary
Args:
ctx (dict): settings passed down from gui
ctx (Settings): settings object passed down from gui
info_dict (dict): dictionary to be converted
Returns:
@@ -233,84 +242,90 @@ def construct_reagent(ctx:dict, info_dict:dict) -> models.Reagent:
# pass
return reagent
def get_all_reagenttype_names(ctx:dict) -> list[str]:
def get_all_reagenttype_names(ctx:Settings) -> list[str]:
"""
Lookup all reagent types and get names
Args:
ctx (dict): settings passed from gui
ctx (Settings): settings object passed from gui
Returns:
list[str]: reagent type names
"""
lookedup = [item.__str__() for item in ctx['database_session'].query(models.ReagentType).all()]
# lookedup = [item.__str__() for item in ctx['database_session'].query(models.ReagentType).all()]
lookedup = [item.__str__() for item in ctx.database_session.query(models.ReagentType).all()]
return lookedup
def lookup_reagenttype_by_name(ctx:dict, rt_name:str) -> models.ReagentType:
def lookup_reagenttype_by_name(ctx:Settings, rt_name:str) -> models.ReagentType:
"""
Lookup a single reagent type by name
Args:
ctx (dict): settings passed from gui
ctx (Settings): settings object passed from gui
rt_name (str): reagent type name to look up
Returns:
models.ReagentType: looked up reagent type
"""
logger.debug(f"Looking up ReagentType by name: {rt_name}")
lookedup = ctx['database_session'].query(models.ReagentType).filter(models.ReagentType.name==rt_name).first()
# lookedup = ctx['database_session'].query(models.ReagentType).filter(models.ReagentType.name==rt_name).first()
lookedup = ctx.database_session.query(models.ReagentType).filter(models.ReagentType.name==rt_name).first()
logger.debug(f"Found ReagentType: {lookedup}")
return lookedup
def lookup_kittype_by_use(ctx:dict, used_by:str|None=None) -> list[models.KitType]:
def lookup_kittype_by_use(ctx:Settings, used_by:str|None=None) -> list[models.KitType]:
"""
Lookup kits by a sample type its used for
Args:
ctx (dict): settings passed from gui
ctx (Settings): settings object from gui
used_by (str): sample type (should be string in D3 of excel sheet)
Returns:
list[models.KitType]: list of kittypes that have that sample type in their uses
"""
if used_by != None:
return ctx['database_session'].query(models.KitType).filter(models.KitType.used_for.contains(used_by)).all()
# return ctx['database_session'].query(models.KitType).filter(models.KitType.used_for.contains(used_by)).all()
return ctx.database_session.query(models.KitType).filter(models.KitType.used_for.contains(used_by)).all()
else:
return ctx['database_session'].query(models.KitType).all()
# return ctx['database_session'].query(models.KitType).all()
return ctx.database_session.query(models.KitType).all()
def lookup_kittype_by_name(ctx:dict, name:str) -> models.KitType:
def lookup_kittype_by_name(ctx:Settings, name:str) -> models.KitType:
"""
Lookup a kit type by name
Args:
ctx (dict): settings passed from bui
ctx (Settings): settings object passed from bui
name (str): name of kit to query
Returns:
models.KitType: retrieved kittype
"""
logger.debug(f"Querying kittype: {name}")
return ctx['database_session'].query(models.KitType).filter(models.KitType.name==name).first()
# return ctx['database_session'].query(models.KitType).filter(models.KitType.name==name).first()
return ctx.database_session.query(models.KitType).filter(models.KitType.name==name).first()
def lookup_regent_by_type_name(ctx:dict, type_name:str) -> list[models.Reagent]:
def lookup_regent_by_type_name(ctx:Settings, type_name:str) -> list[models.Reagent]:
"""
Lookup reagents by their type name
Args:
ctx (dict): settings passed from gui
ctx (Settings): settings object passed from gui
type_name (str): reagent type name
Returns:
list[models.Reagent]: list of retrieved reagents
"""
return ctx['database_session'].query(models.Reagent).join(models.Reagent.type, aliased=True).filter(models.ReagentType.name==type_name).all()
# return ctx['database_session'].query(models.Reagent).join(models.Reagent.type, aliased=True).filter(models.ReagentType.name==type_name).all()
return ctx.database_session.query(models.Reagent).join(models.Reagent.type, aliased=True).filter(models.ReagentType.name==type_name).all()
def lookup_regent_by_type_name_and_kit_name(ctx:dict, type_name:str, kit_name:str) -> list[models.Reagent]:
def lookup_regent_by_type_name_and_kit_name(ctx:Settings, type_name:str, kit_name:str) -> list[models.Reagent]:
"""
Lookup reagents by their type name and kits they belong to (Broken... maybe cursed, I'm not sure.)
Args:
ctx (dict): settings pass by gui
ctx (Settings): settings object pass by gui
type_name (str): reagent type name
kit_name (str): kit name
@@ -320,7 +335,8 @@ def lookup_regent_by_type_name_and_kit_name(ctx:dict, type_name:str, kit_name:st
# What I want to do is get the reagent type by name
# Hang on, this is going to be a long one.
# by_type = ctx['database_session'].query(models.Reagent).join(models.Reagent.type, aliased=True).filter(models.ReagentType.name.endswith(type_name)).all()
rt_types = ctx['database_session'].query(models.ReagentType).filter(models.ReagentType.name.endswith(type_name))
# rt_types = ctx['database_session'].query(models.ReagentType).filter(models.ReagentType.name.endswith(type_name))
rt_types = ctx.database_session.query(models.ReagentType).filter(models.ReagentType.name.endswith(type_name))
# add filter for kit name...
try:
check = not np.isnan(kit_name)
@@ -336,55 +352,59 @@ def lookup_regent_by_type_name_and_kit_name(ctx:dict, type_name:str, kit_name:st
output = rt_types.instances
return output
def lookup_all_submissions_by_type(ctx:dict, sub_type:str|None=None) -> list[models.BasicSubmission]:
def lookup_all_submissions_by_type(ctx:Settings, sub_type:str|None=None) -> list[models.BasicSubmission]:
"""
Get all submissions, filtering by type if given
Args:
ctx (dict): settings pass from gui
ctx (Settings): settings object pass from gui
type (str | None, optional): submission type (should be string in D3 of excel sheet). Defaults to None.
Returns:
list[models.BasicSubmission]: list of retrieved submissions
"""
if sub_type == None:
subs = ctx['database_session'].query(models.BasicSubmission).all()
# subs = ctx['database_session'].query(models.BasicSubmission).all()
subs = ctx.database_session.query(models.BasicSubmission).all()
else:
subs = ctx['database_session'].query(models.BasicSubmission).filter(models.BasicSubmission.submission_type==sub_type.lower().replace(" ", "_")).all()
# subs = ctx['database_session'].query(models.BasicSubmission).filter(models.BasicSubmission.submission_type==sub_type.lower().replace(" ", "_")).all()
subs = ctx.database_session.query(models.BasicSubmission).filter(models.BasicSubmission.submission_type==sub_type.lower().replace(" ", "_")).all()
return subs
def lookup_all_orgs(ctx:dict) -> list[models.Organization]:
def lookup_all_orgs(ctx:Settings) -> list[models.Organization]:
"""
Lookup all organizations (labs)
Args:
ctx (dict): settings passed from gui
ctx (Settings): settings object passed from gui
Returns:
list[models.Organization]: list of retrieved organizations
"""
return ctx['database_session'].query(models.Organization).all()
# return ctx['database_session'].query(models.Organization).all()
return ctx.database_session.query(models.Organization).all()
def lookup_org_by_name(ctx:dict, name:str|None) -> models.Organization:
def lookup_org_by_name(ctx:Settings, name:str|None) -> models.Organization:
"""
Lookup organization (lab) by (startswith) name.
Args:
ctx (dict): settings passed from gui
ctx (Settings): settings passed from gui
name (str | None): name of organization
Returns:
models.Organization: retrieved organization
"""
logger.debug(f"Querying organization: {name}")
return ctx['database_session'].query(models.Organization).filter(models.Organization.name.startswith(name)).first()
# return ctx['database_session'].query(models.Organization).filter(models.Organization.name.startswith(name)).first()
return ctx.database_session.query(models.Organization).filter(models.Organization.name.startswith(name)).first()
def submissions_to_df(ctx:dict, sub_type:str|None=None) -> pd.DataFrame:
def submissions_to_df(ctx:Settings, sub_type:str|None=None) -> pd.DataFrame:
"""
Convert submissions looked up by type to dataframe
Args:
ctx (dict): settings passed by gui
ctx (Settings): settings object passed by gui
type (str | None, optional): submission type (should be string in D3 of excel sheet) Defaults to None.
Returns:
@@ -410,25 +430,26 @@ def submissions_to_df(ctx:dict, sub_type:str|None=None) -> pd.DataFrame:
logger.warning(f"Couldn't drop 'pcr_info' column from submissionsheet df.")
return df
def lookup_submission_by_id(ctx:dict, id:int) -> models.BasicSubmission:
def lookup_submission_by_id(ctx:Settings, id:int) -> models.BasicSubmission:
"""
Lookup submission by id number
Args:
ctx (dict): settings passed from gui
ctx (Settings): settings object passed from gui
id (int): submission id number
Returns:
models.BasicSubmission: retrieved submission
"""
return ctx['database_session'].query(models.BasicSubmission).filter(models.BasicSubmission.id==id).first()
# return ctx['database_session'].query(models.BasicSubmission).filter(models.BasicSubmission.id==id).first()
return ctx.database_session.query(models.BasicSubmission).filter(models.BasicSubmission.id==id).first()
def lookup_submissions_by_date_range(ctx:dict, start_date:datetime.date, end_date:datetime.date) -> list[models.BasicSubmission]:
def lookup_submissions_by_date_range(ctx:Settings, start_date:datetime.date, end_date:datetime.date) -> list[models.BasicSubmission]:
"""
Lookup submissions greater than start_date and less than end_date
Args:
ctx (dict): settings passed from gui
ctx (Settings): settings object passed from gui
start_date (datetime.date): date to start looking
end_date (datetime.date): date to end looking
@@ -438,30 +459,32 @@ def lookup_submissions_by_date_range(ctx:dict, start_date:datetime.date, end_dat
# return ctx['database_session'].query(models.BasicSubmission).filter(and_(models.BasicSubmission.submitted_date > start_date, models.BasicSubmission.submitted_date < end_date)).all()
start_date = start_date.strftime("%Y-%m-%d")
end_date = end_date.strftime("%Y-%m-%d")
return ctx['database_session'].query(models.BasicSubmission).filter(models.BasicSubmission.submitted_date.between(start_date, end_date)).all()
# return ctx['database_session'].query(models.BasicSubmission).filter(models.BasicSubmission.submitted_date.between(start_date, end_date)).all()
return ctx.database_session.query(models.BasicSubmission).filter(models.BasicSubmission.submitted_date.between(start_date, end_date)).all()
def get_all_Control_Types_names(ctx:dict) -> list[str]:
def get_all_Control_Types_names(ctx:Settings) -> list[str]:
"""
Grabs all control type names from db.
Args:
settings (dict): settings passed down from gui.
settings (Settings): settings object passed down from gui.
Returns:
list: list of controltype names
"""
conTypes = ctx['database_session'].query(models.ControlType).all()
# conTypes = ctx['database_session'].query(models.ControlType).all()
conTypes = ctx.database_session.query(models.ControlType).all()
conTypes = [conType.name for conType in conTypes]
logger.debug(f"Control Types: {conTypes}")
return conTypes
def create_kit_from_yaml(ctx:dict, exp:dict) -> dict:
def create_kit_from_yaml(ctx:Settings, exp:dict) -> dict:
"""
Create and store a new kit in the database based on a .yml file
TODO: split into create and store functions
Args:
ctx (dict): Context dictionary passed down from frontend
ctx (Settings): Context object passed down from frontend
exp (dict): Experiment dictionary created from yaml file
Returns:
@@ -474,8 +497,8 @@ def create_kit_from_yaml(ctx:dict, exp:dict) -> dict:
return {'code':1, 'message':"This user does not have permission to add kits.", "status":"warning"}
# iterate through keys in dict
for type in exp:
if type == "password":
continue
# if type == "password":
# continue
# A submission type may use multiple kits.
for kt in exp[type]['kits']:
kit = models.KitType(name=kt,
@@ -488,7 +511,8 @@ def create_kit_from_yaml(ctx:dict, exp:dict) -> dict:
for r in exp[type]['kits'][kt]['reagenttypes']:
# check if reagent type already exists.
r = massage_common_reagents(r)
look_up = ctx['database_session'].query(models.ReagentType).filter(models.ReagentType.name==r).first()
# look_up = ctx['database_session'].query(models.ReagentType).filter(models.ReagentType.name==r).first()
look_up = ctx.database_session.query(models.ReagentType).filter(models.ReagentType.name==r).first()
if look_up == None:
rt = models.ReagentType(name=r.replace(" ", "_").lower(), eol_ext=timedelta(30*exp[type]['kits'][kt]['reagenttypes'][r]['eol_ext']), kits=[kit], required=1)
else:
@@ -500,19 +524,22 @@ def create_kit_from_yaml(ctx:dict, exp:dict) -> dict:
except AttributeError as e:
logger.error(f"Error appending reagent id to kit.reagent_types_id: {e}, creating new.")
# kit.reagent_types_id = [rt.id]
ctx['database_session'].add(rt)
# ctx['database_session'].add(rt)
ctx.database_session.add(rt)
logger.debug(f"Kit construction reagent type: {rt.__dict__}")
logger.debug(f"Kit construction kit: {kit.__dict__}")
ctx['database_session'].add(kit)
ctx['database_session'].commit()
# ctx['database_session'].add(kit)
ctx.database_session.add(kit)
# ctx['database_session'].commit()
ctx.database_session.commit()
return {'code':0, 'message':'Kit has been added', 'status': 'information'}
def create_org_from_yaml(ctx:dict, org:dict) -> dict:
def create_org_from_yaml(ctx:Settings, org:dict) -> dict:
"""
Create and store a new organization based on a .yml file
Args:
ctx (dict): Context dictionary passed down from frontend
ctx (Settings): Context object passed down from frontend
org (dict): Dictionary containing organization info.
Returns:
@@ -530,46 +557,52 @@ def create_org_from_yaml(ctx:dict, org:dict) -> dict:
for contact in org[client]['contacts']:
cont_name = list(contact.keys())[0]
# check if contact already exists
look_up = ctx['database_session'].query(models.Contact).filter(models.Contact.name==cont_name).first()
# look_up = ctx['database_session'].query(models.Contact).filter(models.Contact.name==cont_name).first()
look_up = ctx.database_session.query(models.Contact).filter(models.Contact.name==cont_name).first()
if look_up == None:
cli_cont = models.Contact(name=cont_name, phone=contact[cont_name]['phone'], email=contact[cont_name]['email'], organization=[cli_org])
else:
cli_cont = look_up
cli_cont.organization.append(cli_org)
ctx['database_session'].add(cli_cont)
# ctx['database_session'].add(cli_cont)
ctx.database_session.add(cli_cont)
logger.debug(f"Client creation contact: {cli_cont.__dict__}")
logger.debug(f"Client creation client: {cli_org.__dict__}")
ctx['database_session'].add(cli_org)
ctx["database_session"].commit()
# ctx['database_session'].add(cli_org)
ctx.database_session.add(cli_org)
# ctx["database_session"].commit()
ctx.database_session.commit()
return {"code":0, "message":"Organization has been added."}
def lookup_all_sample_types(ctx:dict) -> list[str]:
def lookup_all_sample_types(ctx:Settings) -> list[str]:
"""
Lookup all sample types and get names
Args:
ctx (dict): settings pass from gui
ctx (Settings): settings object pass from gui
Returns:
list[str]: list of sample type names
"""
uses = [item.used_for for item in ctx['database_session'].query(models.KitType).all()]
# uses = [item.used_for for item in ctx['database_session'].query(models.KitType).all()]
uses = [item.used_for for item in ctx.database_session.query(models.KitType).all()]
# flattened list of lists
uses = list(set([item for sublist in uses for item in sublist]))
return uses
def get_all_available_modes(ctx:dict) -> list[str]:
def get_all_available_modes(ctx:Settings) -> list[str]:
"""
Get types of analysis for controls
Args:
ctx (dict): settings passed from gui
ctx (Settings): settings object passed from gui
Returns:
list[str]: list of analysis types
"""
# Only one control is necessary since they all share the same control types.
rel = ctx['database_session'].query(models.Control).first()
# rel = ctx['database_session'].query(models.Control).first()
rel = ctx.database_session.query(models.Control).first()
try:
cols = [item.name for item in list(rel.__table__.columns) if isinstance(item.type, JSON)]
except AttributeError as e:
@@ -577,13 +610,13 @@ def get_all_available_modes(ctx:dict) -> list[str]:
cols = []
return cols
def get_all_controls_by_type(ctx:dict, con_type:str, start_date:date|None=None, end_date:date|None=None) -> list[models.Control]:
def get_all_controls_by_type(ctx:Settings, con_type:str, start_date:date|None=None, end_date:date|None=None) -> list[models.Control]:
"""
Returns a list of control objects that are instances of the input controltype.
Between dates if supplied.
Args:
ctx (dict): Settings passed down from gui
ctx (Settings): Settings object passed down from gui
con_type (str): Name of control type.
start_date (date | None, optional): Start date of query. Defaults to None.
end_date (date | None, optional): End date of query. Defaults to None.
@@ -595,18 +628,19 @@ def get_all_controls_by_type(ctx:dict, con_type:str, start_date:date|None=None,
if start_date != None and end_date != None:
start_date = start_date.strftime("%Y-%m-%d")
end_date = end_date.strftime("%Y-%m-%d")
output = ctx['database_session'].query(models.Control).join(models.ControlType).filter_by(name=con_type).filter(models.Control.submitted_date.between(start_date, end_date)).all()
# output = ctx['database_session'].query(models.Control).join(models.ControlType).filter_by(name=con_type).filter(models.Control.submitted_date.between(start_date, end_date)).all()
output = ctx.database_session.query(models.Control).join(models.ControlType).filter_by(name=con_type).filter(models.Control.submitted_date.between(start_date, end_date)).all()
else:
output = ctx['database_session'].query(models.Control).join(models.ControlType).filter_by(name=con_type).all()
output = ctx.database_session.query(models.Control).join(models.ControlType).filter_by(name=con_type).all()
logger.debug(f"Returned controls between dates: {[item.submitted_date for item in output]}")
return output
def get_control_subtypes(ctx:dict, type:str, mode:str) -> list[str]:
def get_control_subtypes(ctx:Settings, type:str, mode:str) -> list[str]:
"""
Get subtypes for a control analysis mode
Args:
ctx (dict): settings passed from gui
ctx (Settings): settings object passed from gui
type (str): control type name
mode (str): analysis mode name
@@ -628,7 +662,7 @@ def get_control_subtypes(ctx:dict, type:str, mode:str) -> list[str]:
subtypes = [item for item in jsoner[genera] if "_hashes" not in item and "_ratio" not in item]
return subtypes
def get_all_controls(ctx:dict) -> list[models.Control]:
def get_all_controls(ctx:Settings) -> list[models.Control]:
"""
Retrieve a list of all controls from the database
@@ -638,109 +672,118 @@ def get_all_controls(ctx:dict) -> list[models.Control]:
Returns:
list[models.Control]: list of all control objects
"""
return ctx['database_session'].query(models.Control).all()
return ctx.database_session.query(models.Control).all()
def lookup_submission_by_rsl_num(ctx:dict, rsl_num:str) -> models.BasicSubmission:
def lookup_submission_by_rsl_num(ctx:Settings, rsl_num:str) -> models.BasicSubmission:
"""
Retrieve a submission from the database based on rsl plate number
Args:
ctx (dict): settings passed down from gui
ctx (Settings): settings object passed down from gui
rsl_num (str): rsl plate number
Returns:
models.BasicSubmission: Submissions object retrieved from database
"""
return ctx['database_session'].query(models.BasicSubmission).filter(models.BasicSubmission.rsl_plate_num.startswith(rsl_num)).first()
# return ctx['database_session'].query(models.BasicSubmission).filter(models.BasicSubmission.rsl_plate_num.startswith(rsl_num)).first()
return ctx.database_session.query(models.BasicSubmission).filter(models.BasicSubmission.rsl_plate_num.startswith(rsl_num)).first()
def lookup_submissions_using_reagent(ctx:dict, reagent:models.Reagent) -> list[models.BasicSubmission]:
def lookup_submissions_using_reagent(ctx:Settings, reagent:models.Reagent) -> list[models.BasicSubmission]:
"""
Retrieves each submission using a specified reagent.
Args:
ctx (dict): settings passed down from gui
ctx (Settings): settings passed down from gui
reagent (models.Reagent): reagent object in question
Returns:
list[models.BasicSubmission]: list of all submissions using specified reagent.
"""
return ctx['database_session'].query(models.BasicSubmission).join(reagents_submissions).filter(reagents_submissions.c.reagent_id==reagent.id).all()
# return ctx['database_session'].query(models.BasicSubmission).join(reagents_submissions).filter(reagents_submissions.c.reagent_id==reagent.id).all()
return ctx.database_session.query(models.BasicSubmission).join(reagents_submissions).filter(reagents_submissions.c.reagent_id==reagent.id).all()
def delete_submission_by_id(ctx:dict, id:int) -> None:
def delete_submission_by_id(ctx:Settings, id:int) -> None:
"""
Deletes a submission and its associated samples from the database.
Args:
ctx (dict): settings passed down from gui
ctx (Settings): settings object passed down from gui
id (int): id of submission to be deleted.
"""
# In order to properly do this Im' going to have to delete all of the secondary table stuff as well.
# Retrieve submission
sub = ctx['database_session'].query(models.BasicSubmission).filter(models.BasicSubmission.id==id).first()
# sub = ctx['database_session'].query(models.BasicSubmission).filter(models.BasicSubmission.id==id).first()
sub = ctx.database_session.query(models.BasicSubmission).filter(models.BasicSubmission.id==id).first()
# Convert to dict for storing backup as a yml
backup = sub.to_dict()
try:
with open(Path(ctx['backup_path']).joinpath(f"{sub.rsl_plate_num}-backup({date.today().strftime('%Y%m%d')}).yml"), "w") as f:
# with open(Path(ctx['backup_path']).joinpath(f"{sub.rsl_plate_num}-backup({date.today().strftime('%Y%m%d')}).yml"), "w") as f:
with open(Path(ctx.backup_path).joinpath(f"{sub.rsl_plate_num}-backup({date.today().strftime('%Y%m%d')}).yml"), "w") as f:
yaml.dump(backup, f)
except KeyError:
pass
sub.reagents = []
for sample in sub.samples:
if sample.rsl_plate == sub:
ctx['database_session'].delete(sample)
# ctx['database_session'].delete(sample)
ctx.database_session.delete(sample)
else:
logger.warning(f"Not deleting sample {sample.ww_sample_full_id} because it belongs to another plate.")
ctx["database_session"].delete(sub)
ctx["database_session"].commit()
# ctx["database_session"].delete(sub)
# ctx["database_session"].commit()
ctx.database_session.delete(sub)
ctx.database_session.commit()
def lookup_ww_sample_by_rsl_sample_number(ctx:dict, rsl_number:str) -> models.WWSample:
def lookup_ww_sample_by_rsl_sample_number(ctx:Settings, rsl_number:str) -> models.WWSample:
"""
Retrieves wastewater sample from database by rsl sample number
Args:
ctx (dict): settings passed down from gui
ctx (Settings): settings object passed down from gui
rsl_number (str): sample number assigned by robotics lab
Returns:
models.WWSample: instance of wastewater sample
"""
return ctx['database_session'].query(models.WWSample).filter(models.WWSample.rsl_number==rsl_number).first()
# return ctx['database_session'].query(models.WWSample).filter(models.WWSample.rsl_number==rsl_number).first()
return ctx.database_session.query(models.WWSample).filter(models.WWSample.rsl_number==rsl_number).first()
def lookup_ww_sample_by_ww_sample_num(ctx:dict, sample_number:str) -> models.WWSample:
def lookup_ww_sample_by_ww_sample_num(ctx:Settings, sample_number:str) -> models.WWSample:
"""
Retrieves wastewater sample from database by ww sample number
Args:
ctx (dict): settings passed down from gui
ctx (Settings): settings object passed down from gui
sample_number (str): sample number assigned by wastewater
Returns:
models.WWSample: instance of wastewater sample
"""
return ctx['database_session'].query(models.WWSample).filter(models.WWSample.ww_sample_full_id==sample_number).first()
return ctx.database_session.query(models.WWSample).filter(models.WWSample.ww_sample_full_id==sample_number).first()
def lookup_ww_sample_by_sub_sample_rsl(ctx:dict, sample_rsl:str, plate_rsl:str) -> models.WWSample:
def lookup_ww_sample_by_sub_sample_rsl(ctx:Settings, sample_rsl:str, plate_rsl:str) -> models.WWSample:
"""
Retrieves a wastewater sample from the database by its rsl sample number and parent rsl plate number.
This will likely replace simply looking up by the sample rsl above cine I need to control for repeats.
Args:
ctx (dict): settings passed down from the gui
ctx (Settings): settings passed down from the gui
sample_rsl (str): rsl number of the relevant sample
plate_rsl (str): rsl number of the parent plate
Returns:
models.WWSample: Relevant wastewater object
"""
return ctx['database_session'].query(models.WWSample).join(models.BasicSubmission).filter(models.BasicSubmission.rsl_plate_num==plate_rsl).filter(models.WWSample.rsl_number==sample_rsl).first()
# return ctx['database_session'].query(models.WWSample).join(models.BasicSubmission).filter(models.BasicSubmission.rsl_plate_num==plate_rsl).filter(models.WWSample.rsl_number==sample_rsl).first()
return ctx.database_session.query(models.WWSample).join(models.BasicSubmission).filter(models.BasicSubmission.rsl_plate_num==plate_rsl).filter(models.WWSample.rsl_number==sample_rsl).first()
def lookup_ww_sample_by_sub_sample_well(ctx:dict, sample_rsl:str, well_num:str, plate_rsl:str) -> models.WWSample:
def lookup_ww_sample_by_sub_sample_well(ctx:Settings, sample_rsl:str, well_num:str, plate_rsl:str) -> models.WWSample:
"""
Retrieves a wastewater sample from the database by its rsl sample number and parent rsl plate number.
This will likely replace simply looking up by the sample rsl above cine I need to control for repeats.
Args:
ctx (dict): settings passed down from the gui
ctx (Settings): settings object passed down from the gui
sample_rsl (str): rsl number of the relevant sample
well_num (str): well number of the relevant sample
plate_rsl (str): rsl number of the parent plate
@@ -748,17 +791,21 @@ def lookup_ww_sample_by_sub_sample_well(ctx:dict, sample_rsl:str, well_num:str,
Returns:
models.WWSample: Relevant wastewater object
"""
return ctx['database_session'].query(models.WWSample).join(models.BasicSubmission) \
# return ctx['database_session'].query(models.WWSample).join(models.BasicSubmission) \
# .filter(models.BasicSubmission.rsl_plate_num==plate_rsl) \
# .filter(models.WWSample.rsl_number==sample_rsl) \
# .filter(models.WWSample.well_number==well_num).first()
return ctx.database_session.query(models.WWSample).join(models.BasicSubmission) \
.filter(models.BasicSubmission.rsl_plate_num==plate_rsl) \
.filter(models.WWSample.rsl_number==sample_rsl) \
.filter(models.WWSample.well_number==well_num).first()
def update_ww_sample(ctx:dict, sample_obj:dict):
def update_ww_sample(ctx:Settings, sample_obj:dict):
"""
Retrieves wastewater sample by rsl number (sample_obj['sample']) and updates values from constructed dictionary
Args:
ctx (dict): settings passed down from gui
ctx (Settings): settings object passed down from gui
sample_obj (dict): dictionary representing new values for database object
"""
# ww_samp = lookup_ww_sample_by_rsl_sample_number(ctx=ctx, rsl_number=sample_obj['sample'])
@@ -779,11 +826,28 @@ def update_ww_sample(ctx:dict, sample_obj:dict):
else:
logger.error(f"Unable to find sample {sample_obj['sample']}")
return
ctx['database_session'].add(ww_samp)
ctx["database_session"].commit()
# ctx['database_session'].add(ww_samp)
# ctx["database_session"].commit()
ctx.database_session.add(ww_samp)
ctx.database_session.commit()
def lookup_discounts_by_org_and_kit(ctx:dict, kit_id:int, lab_id:int):
return ctx['database_session'].query(models.Discount).join(models.KitType).join(models.Organization).filter(and_(
def lookup_discounts_by_org_and_kit(ctx:Settings, kit_id:int, lab_id:int) -> list:
"""
Find discounts for kit for specified client
Args:
ctx (Settings): settings object passed down from gui
kit_id (int): Id number of desired kit
lab_id (int): Id number of desired client
Returns:
list: list of Discount objects
"""
# return ctx['database_session'].query(models.Discount).join(models.KitType).join(models.Organization).filter(and_(
# models.KitType.id==kit_id,
# models.Organization.id==lab_id
# )).all()
return ctx.database_session.query(models.Discount).join(models.KitType).join(models.Organization).filter(and_(
models.KitType.id==kit_id,
models.Organization.id==lab_id
)).all()
@@ -861,12 +925,12 @@ def platemap_plate(submission:models.BasicSubmission) -> list:
# image = make_plate_map(plate_dicto)
return plate_dicto
def lookup_reagent(ctx:dict, reagent_lot:str, type_name:str|None=None) -> models.Reagent:
def lookup_reagent(ctx:Settings, reagent_lot:str, type_name:str|None=None) -> models.Reagent:
"""
Query db for reagent based on lot number, with optional reagent type to enforce
Args:
ctx (dict): settings passed down from gui
ctx (Settings): settings passed down from gui
reagent_lot (str): lot number to query
type_name (str | None, optional): name of reagent type. Defaults to None.
@@ -874,21 +938,67 @@ def lookup_reagent(ctx:dict, reagent_lot:str, type_name:str|None=None) -> models
models.Reagent: looked up reagent
"""
if reagent_lot != None and type_name != None:
return ctx['database_session'].query(models.Reagent).join(models.Reagent.type, aliased=True).filter(models.ReagentType.name==type_name).filter(models.Reagent.lot==reagent_lot).first()
# return ctx['database_session'].query(models.Reagent).join(models.Reagent.type, aliased=True).filter(models.ReagentType.name==type_name).filter(models.Reagent.lot==reagent_lot).first()
return ctx.database_session.query(models.Reagent).join(models.Reagent.type, aliased=True).filter(models.ReagentType.name==type_name).filter(models.Reagent.lot==reagent_lot).first()
elif type_name == None:
return ctx['database_session'].query(models.Reagent).filter(models.Reagent.lot==reagent_lot).first()
# return ctx['database_session'].query(models.Reagent).filter(models.Reagent.lot==reagent_lot).first()
return ctx.database_session.query(models.Reagent).filter(models.Reagent.lot==reagent_lot).first()
def lookup_last_used_reagenttype_lot(ctx:dict, type_name:str) -> models.Reagent:
def lookup_last_used_reagenttype_lot(ctx:Settings, type_name:str) -> models.Reagent:
"""
Look up the last used reagent of the reagent type
Args:
ctx (dict): Settings passed down from gui
ctx (Settings): Settings object passed down from gui
type_name (str): Name of reagent type
Returns:
models.Reagent: Reagent object with last used lot.
"""
rt = ctx['database_session'].query(models.ReagentType).filter(models.ReagentType.name==type_name).first()
# rt = ctx['database_session'].query(models.ReagentType).filter(models.ReagentType.name==type_name).first()
rt = ctx.database_session.query(models.ReagentType).filter(models.ReagentType.name==type_name).first()
logger.debug(f"Reagent type looked up for {type_name}: {rt.__str__()}")
return lookup_reagent(ctx=ctx, reagent_lot=rt.last_used, type_name=type_name)
try:
return lookup_reagent(ctx=ctx, reagent_lot=rt.last_used, type_name=type_name)
except AttributeError:
return None
def check_kit_integrity(sub:BasicSubmission|KitType, reagenttypes:list|None=None) -> dict|None:
"""
Ensures all reagents expected in kit are listed in Submission
Args:
sub (BasicSubmission | KitType): Object containing complete list of reagent types.
reagenttypes (list | None, optional): List to check against complete list. Defaults to None.
Returns:
dict|None: Result object containing a message and any missing components.
"""
logger.debug(type(sub))
# What type is sub?
match sub:
case BasicSubmission():
# Get all required reagent types for this kit.
ext_kit_rtypes = [reagenttype.name for reagenttype in sub.extraction_kit.reagent_types if reagenttype.required == 1]
# Overwrite function parameter reagenttypes
try:
reagenttypes = [reagent.type.name for reagent in sub.reagents]
except AttributeError as e:
logger.error(f"Problem parsing reagents: {[f'{reagent.lot}, {reagent.type}' for reagent in sub.reagents]}")
case KitType():
ext_kit_rtypes = [reagenttype.name for reagenttype in sub.reagent_types if reagenttype.required == 1]
logger.debug(f"Kit reagents: {ext_kit_rtypes}")
logger.debug(f"Submission reagents: {reagenttypes}")
# check if lists are equal
check = set(ext_kit_rtypes) == set(reagenttypes)
logger.debug(f"Checking if reagents match kit contents: {check}")
# what reagent types are in both lists?
missing = list(set(ext_kit_rtypes).difference(reagenttypes))
logger.debug(f"Missing reagents types: {missing}")
# if lists are equal return no problem
if len(missing)==0:
result = None
else:
result = {'message' : f"The submission you are importing is missing some reagents expected by the kit.\n\nIt looks like you are missing: {[item.upper() for item in missing]}\n\nAlternatively, you may have set the wrong extraction kit.\n\nThe program will populate lists using existing reagents.\n\nPlease make sure you check the lots carefully!", 'missing': missing}
return result

34
src/submissions/backend/excel/parser.py
View File

@@ -15,7 +15,7 @@ import re
import numpy as np
from datetime import date, datetime
import uuid
from tools import check_not_nan, RSLNamer, massage_common_reagents, convert_nans_to_nones
from tools import check_not_nan, RSLNamer, massage_common_reagents, convert_nans_to_nones, Settings
logger = logging.getLogger(f"submissions.{__name__}")
@@ -23,10 +23,10 @@ class SheetParser(object):
"""
object to pull and contain data from excel file
"""
def __init__(self, ctx:dict, filepath:Path|None = None):
def __init__(self, ctx:Settings, filepath:Path|None = None):
"""
Args:
ctx (dict): Settings passed down from gui
ctx (Settings): Settings object passed down from gui
filepath (Path | None, optional): file path to excel sheet. Defaults to None.
"""
self.ctx = ctx
@@ -59,15 +59,17 @@ class SheetParser(object):
"""
# Check metadata for category, return first category
if self.xl.book.properties.category != None:
logger.debug("Using file properties to find type...")
categories = [item.strip().title() for item in self.xl.book.properties.category.split(";")]
return categories[0].replace(" ", "_")
else:
# This code is going to be depreciated once there is full adoption of the client sheets
# with updated metadata... but how will it work for Artic?
logger.debug("Using excel map to find type...")
try:
for type in self.ctx['submission_types']:
for type in self.ctx.submission_types:
# This gets the *first* submission type that matches the sheet names in the workbook
if self.xl.sheet_names == self.ctx['submission_types'][type]['excel_map']:
if self.xl.sheet_names == self.ctx.submission_types[type]['excel_map']:
return type.title()
return "Unknown"
except Exception as e:
@@ -299,6 +301,8 @@ class SheetParser(object):
return_list = []
for _, ii in df.iloc[1:,1:].iterrows():
for c in df.columns.to_list():
if not check_not_nan(c):
continue
logger.debug(f"Checking {ii.name}{c}")
if check_not_nan(df.loc[ii.name, int(c)]) and df.loc[ii.name, int(c)] != "EMPTY":
try:
@@ -310,21 +314,23 @@ class SheetParser(object):
continue
logger.debug(f"massaged sample list for {self.sub['rsl_plate_num']}: {pprint.pprint(return_list)}")
return return_list
submission_info = self.xl.parse("First Strand", dtype=object)
biomek_info = self.xl.parse("ArticV4 Biomek", dtype=object)
sub_reagent_range = submission_info.iloc[56:, 1:4].dropna(how='all')
biomek_reagent_range = biomek_info.iloc[60:, 0:3].dropna(how='all')
submission_info = self.xl.parse("cDNA", dtype=object)
biomek_info = self.xl.parse("ArticV4_1 Biomek", dtype=object)
# Reminder that the iloc uses row, column ordering
# sub_reagent_range = submission_info.iloc[56:, 1:4].dropna(how='all')
sub_reagent_range = submission_info.iloc[7:15, 5:9].dropna(how='all')
biomek_reagent_range = biomek_info.iloc[62:, 0:3].dropna(how='all')
self.sub['submitter_plate_num'] = ""
self.sub['rsl_plate_num'] = RSLNamer(ctx=self.ctx, instr=self.filepath.__str__()).parsed_name
self.sub['submitted_date'] = biomek_info.iloc[1][1]
self.sub['submitting_lab'] = "Enterics Wastewater Genomics"
self.sub['sample_count'] = submission_info.iloc[4][6]
self.sub['sample_count'] = submission_info.iloc[34][6]
self.sub['extraction_kit'] = "ArticV4.1"
self.sub['technician'] = f"MM: {biomek_info.iloc[2][1]}, Bio: {biomek_info.iloc[3][1]}"
self.sub['reagents'] = []
parse_reagents(sub_reagent_range)
parse_reagents(biomek_reagent_range)
samples = massage_samples(biomek_info.iloc[22:31, 0:])
samples = massage_samples(biomek_info.iloc[25:33, 0:])
sample_parser = SampleParser(self.ctx, pd.DataFrame.from_records(samples))
sample_parse = getattr(sample_parser, f"parse_{self.sub['submission_type']['value'].lower()}_samples")
self.sample_result, self.sub['samples'] = sample_parse()
@@ -347,12 +353,12 @@ class SampleParser(object):
object to pull data for samples in excel sheet and construct individual sample objects
"""
def __init__(self, ctx:dict, df:pd.DataFrame, elution_map:pd.DataFrame|None=None) -> None:
def __init__(self, ctx:Settings, df:pd.DataFrame, elution_map:pd.DataFrame|None=None) -> None:
"""
convert sample sub-dataframe to dictionary of records
Args:
ctx (dict): setting passed down from gui
ctx (Settings): settings object passed down from gui
df (pd.DataFrame): input sample dataframe
elution_map (pd.DataFrame | None, optional): optional map of elution plate. Defaults to None.
"""
@@ -460,7 +466,7 @@ class SampleParser(object):
new_list = []
missed_samples = []
for sample in self.samples:
with self.ctx['database_session'].no_autoflush:
with self.ctx.database_session.no_autoflush:
instance = lookup_ww_sample_by_ww_sample_num(ctx=self.ctx, sample_number=sample['sample_name'])
logger.debug(f"Checking: {sample['sample_name']}")
if instance == None:

2
src/submissions/backend/excel/reports.py
View File

@@ -6,7 +6,7 @@ import logging
from datetime import date, timedelta
import re
from typing import Tuple
from configure import jinja_template_loading
from tools import jinja_template_loading
logger = logging.getLogger(f"submissions.{__name__}")

6
src/submissions/backend/pydant/__init__.py
View File

@@ -6,7 +6,7 @@ from tools import RSLNamer
from pathlib import Path
import re
import logging
from tools import check_not_nan, convert_nans_to_nones
from tools import check_not_nan, convert_nans_to_nones, Settings
import numpy as np
@@ -45,7 +45,7 @@ class PydReagent(BaseModel):
class PydSubmission(BaseModel, extra=Extra.allow):
ctx: dict
ctx: Settings
filepath: Path
submission_type: str|dict|None
submitter_plate_num: str|None
@@ -62,6 +62,8 @@ class PydSubmission(BaseModel, extra=Extra.allow):
@field_validator("submitted_date", mode="before")
@classmethod
def strip_datetime_string(cls, value):
if not check_not_nan(value):
value = date.today()
if isinstance(value, datetime):
return value
if isinstance(value, date):