landon/Submissions-App
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

new parsers/DB objects, pre code cleanup

Browse Source
This commit is contained in:
Landon Wark
2023-09-06 08:41:08 -05:00
parent bc7a3b8f5f
commit e0b80f6c7a
12 changed files with 263 additions and 137 deletions
Download Patch File Download Diff File Expand all files Collapse all files

87
src/submissions/backend/db/functions.py
View File

@@ -2,6 +2,7 @@
Convenience functions for interacting with the database.
'''
import pprint
from . import models
# from .models.kits import KitType
# from .models.submissions import BasicSample, reagents_submissions, BasicSubmission, SubmissionSampleAssociation
@@ -33,7 +34,6 @@ def set_sqlite_pragma(dbapi_connection, connection_record):
cursor.execute("PRAGMA foreign_keys=ON")
cursor.close()
def store_submission(ctx:Settings, base_submission:models.BasicSubmission, samples:List[dict]=[]) -> None|dict:
"""
Upserts submissions into database
@@ -206,10 +206,12 @@ def construct_submission_info(ctx:Settings, info_dict:dict) -> models.BasicSubmi
try:
with ctx.database_session.no_autoflush:
try:
logger.debug(f"Here is the sample instance type: {sample_instance.sample_type}")
sample_query = sample_instance.sample_type.replace('Sample', '').strip()
logger.debug(f"Here is the sample instance type: {sample_query}")
try:
assoc = getattr(models, f"{sample_instance.sample_type.replace('_sample', '').replace('_', ' ').title().replace(' ', '')}Association")
assoc = getattr(models, f"{sample_query}Association")
except AttributeError as e:
logger.error(f"Couldn't get type specific association. Getting generic.")
assoc = models.SubmissionSampleAssociation
# assoc = models.SubmissionSampleAssociation(submission=instance, sample=sample_instance, row=sample['row'], column=sample['column'])
assoc = assoc(submission=instance, sample=sample_instance, row=sample['row'], column=sample['column'])
@@ -287,7 +289,9 @@ def construct_reagent(ctx:Settings, info_dict:dict) -> models.Reagent:
case "expiry":
reagent.expiry = info_dict[item]
case "type":
reagent.type = lookup_reagenttype_by_name(ctx=ctx, rt_name=info_dict[item])
reagent_type = lookup_reagenttype_by_name(ctx=ctx, rt_name=info_dict[item])
if reagent_type != None:
reagent.type.append(reagent_type)
case "name":
if item == None:
reagent.name = reagent.type.name
@@ -420,7 +424,7 @@ def lookup_regent_by_type_name_and_kit_name(ctx:Settings, type_name:str, kit_nam
output = rt_types.instances
return output
def lookup_all_submissions_by_type(ctx:Settings, sub_type:str|None=None) -> list[models.BasicSubmission]:
def lookup_all_submissions_by_type(ctx:Settings, sub_type:str|None=None, chronologic:bool=False) -> list[models.BasicSubmission]:
"""
Get all submissions, filtering by type if given
@@ -433,11 +437,13 @@ def lookup_all_submissions_by_type(ctx:Settings, sub_type:str|None=None) -> list
"""
if sub_type == None:
# subs = ctx['database_session'].query(models.BasicSubmission).all()
subs = ctx.database_session.query(models.BasicSubmission).all()
subs = ctx.database_session.query(models.BasicSubmission)
else:
# subs = ctx['database_session'].query(models.BasicSubmission).filter(models.BasicSubmission.submission_type==sub_type.lower().replace(" ", "_")).all()
subs = ctx.database_session.query(models.BasicSubmission).filter(models.BasicSubmission.submission_type==sub_type.lower().replace(" ", "_")).all()
return subs
subs = ctx.database_session.query(models.BasicSubmission).filter(models.BasicSubmission.submission_type==sub_type.lower().replace(" ", "_"))
if chronologic:
subs.order_by(models.BasicSubmission.submitted_date)
return subs.all()
def lookup_all_orgs(ctx:Settings) -> list[models.Organization]:
"""
@@ -480,7 +486,7 @@ def submissions_to_df(ctx:Settings, sub_type:str|None=None) -> pd.DataFrame:
"""
logger.debug(f"Type: {sub_type}")
# use lookup function to create list of dicts
subs = [item.to_dict() for item in lookup_all_submissions_by_type(ctx=ctx, sub_type=sub_type)]
subs = [item.to_dict() for item in lookup_all_submissions_by_type(ctx=ctx, sub_type=sub_type, chronologic=True)]
# make df from dicts (records) in list
df = pd.DataFrame.from_records(subs)
# Exclude sub information
@@ -569,7 +575,9 @@ def create_kit_from_yaml(ctx:Settings, exp:dict) -> dict:
# continue
# A submission type may use multiple kits.
for kt in exp[type]['kits']:
logger.debug(f"Looking up submission type: {type}")
submission_type = lookup_submissiontype_by_name(ctx=ctx, type_name=type)
logger.debug(f"Looked up submission type: {submission_type}")
kit = models.KitType(name=kt,
# constant_cost=exp[type]["kits"][kt]["constant_cost"],
# mutable_cost_column=exp[type]["kits"][kt]["mutable_cost_column"],
@@ -588,7 +596,7 @@ def create_kit_from_yaml(ctx:Settings, exp:dict) -> dict:
look_up = ctx.database_session.query(models.ReagentType).filter(models.ReagentType.name==r).first()
if look_up == None:
# rt = models.ReagentType(name=r.replace(" ", "_").lower(), eol_ext=timedelta(30*exp[type]['kits'][kt]['reagenttypes'][r]['eol_ext']), kits=[kit], required=1)
rt = models.ReagentType(name=r.replace(" ", "_").lower().strip(), eol_ext=timedelta(30*exp[type]['kits'][kt]['reagenttypes'][r]['eol_ext']), last_used="")
rt = models.ReagentType(name=r.strip(), eol_ext=timedelta(30*exp[type]['kits'][kt]['reagenttypes'][r]['eol_ext']), last_used="")
else:
rt = look_up
# rt.kits.append(kit)
@@ -893,9 +901,10 @@ def update_ww_sample(ctx:Settings, sample_obj:dict):
sample_obj (dict): dictionary representing new values for database object
"""
# ww_samp = lookup_ww_sample_by_rsl_sample_number(ctx=ctx, rsl_number=sample_obj['sample'])
logger.debug(f"dictionary to use for update: {pprint.pformat(sample_obj)}")
logger.debug(f"Looking up {sample_obj['sample']} in plate {sample_obj['plate_rsl']}")
# ww_samp = lookup_ww_sample_by_sub_sample_rsl(ctx=ctx, sample_rsl=sample_obj['sample'], plate_rsl=sample_obj['plate_rsl'])
assoc = lookup_ww_association_by_plate_sample(ctx=ctx, rsl_plate_num=sample_obj['plate_rsl'], rsl_sample_num=sample_obj['sample'])
assoc = lookup_subsamp_association_by_plate_sample(ctx=ctx, rsl_plate_num=sample_obj['plate_rsl'], rsl_sample_num=sample_obj['sample'])
# ww_samp = lookup_ww_sample_by_sub_sample_well(ctx=ctx, sample_rsl=sample_obj['sample'], well_num=sample_obj['well_num'], plate_rsl=sample_obj['plate_rsl'])
if assoc != None:
# del sample_obj['well_number']
@@ -903,11 +912,16 @@ def update_ww_sample(ctx:Settings, sample_obj:dict):
# set attribute 'key' to 'value'
try:
check = getattr(assoc, key)
except AttributeError:
except AttributeError as e:
logger.error(f"Item doesn't have field {key} due to {e}")
continue
if check == None:
logger.debug(f"Setting {key} to {value}")
setattr(assoc, key, value)
if check != value:
logger.debug(f"Setting association key: {key} to {value}")
try:
setattr(assoc, key, value)
except AttributeError as e:
logger.error(f"Can't set field {key} to {value} due to {e}")
continue
else:
logger.error(f"Unable to find sample {sample_obj['sample']}")
return
@@ -1059,16 +1073,22 @@ def check_kit_integrity(sub:models.BasicSubmission|models.KitType, reagenttypes:
"""
logger.debug(type(sub))
# What type is sub?
reagenttypes = []
match sub:
case models.BasicSubmission():
# Get all required reagent types for this kit.
# ext_kit_rtypes = [reagenttype.name for reagenttype in sub.extraction_kit.reagent_types if reagenttype.required == 1]
ext_kit_rtypes = [item.name for item in sub.extraction_kit.get_reagents(required=True)]
# Overwrite function parameter reagenttypes
try:
reagenttypes = [reagent.type.name for reagent in sub.reagents]
except AttributeError as e:
logger.error(f"Problem parsing reagents: {[f'{reagent.lot}, {reagent.type}' for reagent in sub.reagents]}")
for reagent in sub.reagents:
try:
# reagenttypes = [reagent.type.name for reagent in sub.reagents]
rt = list(set(reagent.type).intersection(sub.extraction_kit.reagent_types))[0].name
logger.debug(f"Got reagent type: {rt}")
reagenttypes.append(rt)
except AttributeError as e:
logger.error(f"Problem parsing reagents: {[f'{reagent.lot}, {reagent.type}' for reagent in sub.reagents]}")
reagenttypes.append(reagent.type[0].name)
case models.KitType():
# ext_kit_rtypes = [reagenttype.name for reagenttype in sub.reagent_types if reagenttype.required == 1]
ext_kit_rtypes = [item.name for item in sub.get_reagents(required=True)]
@@ -1133,7 +1153,7 @@ def get_reagents_in_extkit(ctx:Settings, kit_name:str) -> List[str]:
kit = lookup_kittype_by_name(ctx=ctx, name=kit_name)
return kit.get_reagents(required=False)
def lookup_ww_association_by_plate_sample(ctx:Settings, rsl_plate_num:str, rsl_sample_num:str) -> models.SubmissionSampleAssociation:
def lookup_subsamp_association_by_plate_sample(ctx:Settings, rsl_plate_num:str, rsl_sample_num:str) -> models.SubmissionSampleAssociation:
"""
_summary_
@@ -1147,9 +1167,9 @@ def lookup_ww_association_by_plate_sample(ctx:Settings, rsl_plate_num:str, rsl_s
"""
return ctx.database_session.query(models.SubmissionSampleAssociation)\
.join(models.BasicSubmission)\
.join(models.WastewaterSample)\
.join(models.BasicSample)\
.filter(models.BasicSubmission.rsl_plate_num==rsl_plate_num)\
.filter(models.WastewaterSample.rsl_number==rsl_sample_num)\
.filter(models.BasicSample.submitter_id==rsl_sample_num)\
.first()
def lookup_all_reagent_names_by_role(ctx:Settings, role_name:str) -> List[str]:
@@ -1181,4 +1201,25 @@ def lookup_submissiontype_by_name(ctx:Settings, type_name:str) -> models.Submiss
models.SubmissionType: _description_
"""
return ctx.database_session.query(models.SubmissionType).filter(models.SubmissionType.name==type_name).first()
return ctx.database_session.query(models.SubmissionType).filter(models.SubmissionType.name==type_name).first()
def add_reagenttype_to_kit(ctx:Settings, rt_name:str, kit_name:str, eol:int=0):
"""
Mostly commandline procedure to add missing reagenttypes to kits
Args:
ctx (Settings): _description_
rt_name (str): _description_
kit_name (str): _description_
eol (int, optional): _description_. Defaults to 0.
"""
kit = lookup_kittype_by_name(ctx=ctx, name=kit_name)
rt = lookup_reagenttype_by_name(ctx=ctx, rt_name=rt_name)
if rt == None:
rt = models.ReagentType(name=rt_name.strip(), eol_ext=timedelta(30*eol), last_used="")
ctx.database_session.add(rt)
assoc = models.KitTypeReagentTypeAssociation(kit_type=kit, reagent_type=rt, uses={})
kit.kit_reagenttype_associations.append(assoc)
ctx.database_session.add(kit)
ctx.database_session.commit()

34
src/submissions/backend/db/models/kits.py
View File

@@ -2,7 +2,7 @@
All kit and reagent related models
'''
from . import Base
from sqlalchemy import Column, String, TIMESTAMP, JSON, INTEGER, ForeignKey, Interval, Table, FLOAT, CheckConstraint
from sqlalchemy import Column, String, TIMESTAMP, JSON, INTEGER, ForeignKey, Interval, Table, FLOAT
from sqlalchemy.orm import relationship, validates
from sqlalchemy.ext.associationproxy import association_proxy
@@ -22,6 +22,8 @@ logger = logging.getLogger(f'submissions.{__name__}')
# Column("required", INTEGER)
# )
reagenttypes_reagents = Table("_reagenttypes_reagents", Base.metadata, Column("reagent_id", INTEGER, ForeignKey("_reagents.id")), Column("reagenttype_id", INTEGER, ForeignKey("_reagent_types.id")))
class KitType(Base):
"""
@@ -47,7 +49,7 @@ class KitType(Base):
# association proxy of "user_keyword_associations" collection
# to "keyword" attribute
reagent_types = association_proxy("kit_reagenttype_associations", "reagenttype")
reagent_types = association_proxy("kit_reagenttype_associations", "reagent_type")
kit_submissiontype_associations = relationship(
@@ -139,7 +141,7 @@ class ReagentType(Base):
name = Column(String(64)) #: name of reagent type
# kit_id = Column(INTEGER, ForeignKey("_kits.id", ondelete="SET NULL", use_alter=True, name="fk_RT_kits_id")) #: id of joined kit type
# kits = relationship("KitType", back_populates="reagent_types", uselist=True, foreign_keys=[kit_id]) #: kits this reagent is used in
instances = relationship("Reagent", back_populates="type") #: concrete instances of this reagent type
instances = relationship("Reagent", back_populates="type", secondary=reagenttypes_reagents) #: concrete instances of this reagent type
eol_ext = Column(Interval()) #: extension of life interval
# required = Column(INTEGER, server_default="1") #: sqlite boolean to determine if reagent type is essential for the kit
last_used = Column(String(32)) #: last used lot number of this type of reagent
@@ -169,7 +171,7 @@ class Reagent(Base):
__tablename__ = "_reagents"
id = Column(INTEGER, primary_key=True) #: primary key
type = relationship("ReagentType", back_populates="instances") #: joined parent reagent type
type = relationship("ReagentType", back_populates="instances", secondary=reagenttypes_reagents) #: joined parent reagent type
type_id = Column(INTEGER, ForeignKey("_reagent_types.id", ondelete='SET NULL', name="fk_reagent_type_id")) #: id of parent reagent type
name = Column(String(64)) #: reagent name
lot = Column(String(64)) #: lot number of reagent
@@ -192,19 +194,26 @@ class Reagent(Base):
"""
return str(self.lot)
def to_sub_dict(self) -> dict:
def to_sub_dict(self, extraction_kit:KitType=None) -> dict:
"""
dictionary containing values necessary for gui
Returns:
dict: gui friendly dictionary
"""
if extraction_kit != None:
try:
reagent_role = list(set(self.type).intersection(extraction_kit.reagent_types))[0]
except:
reagent_role = self.type[0]
else:
reagent_role = self.type[0]
try:
type = self.type.name.replace("_", " ").title()
rtype = reagent_role.name.replace("_", " ").title()
except AttributeError:
type = "Unknown"
rtype = "Unknown"
try:
place_holder = self.expiry + self.type.eol_ext
place_holder = self.expiry + reagent_role.eol_ext
# logger.debug(f"EOL_ext for {self.lot} -- {self.expiry} + {self.type.eol_ext} = {place_holder}")
except TypeError as e:
place_holder = date.today()
@@ -213,14 +222,14 @@ class Reagent(Base):
place_holder = date.today()
logger.debug(f"We got an attribute error setting {self.lot} expiry: {e}. Setting to today for testing")
return {
"type": type,
"type": rtype,
"lot": self.lot,
"expiry": place_holder.strftime("%Y-%m-%d")
}
def to_reagent_dict(self) -> dict:
return {
"type": self.type.name,
"type": type,
"lot": self.lot,
"expiry": self.expiry.strftime("%Y-%m-%d")
}
@@ -279,4 +288,7 @@ class SubmissionTypeKitTypeAssociation(Base):
self.submission_type = submission_type
self.mutable_cost_column = 0.00
self.mutable_cost_sample = 0.00
self.constant_cost = 0.00
self.constant_cost = 0.00
def __repr__(self) -> str:
return f"<SubmissionTypeKitTypeAssociation({self.submission_type.name})"

24
src/submissions/backend/db/models/submissions.py
View File

@@ -3,7 +3,7 @@ Models for the main submission types.
'''
import math
from . import Base
from sqlalchemy import Column, String, TIMESTAMP, INTEGER, ForeignKey, Table, JSON, FLOAT, BOOLEAN
from sqlalchemy import Column, String, TIMESTAMP, INTEGER, ForeignKey, Table, JSON, FLOAT
from sqlalchemy.orm import relationship, validates
import logging
import json
@@ -106,7 +106,7 @@ class BasicSubmission(Base):
ext_info = None
logger.debug(f"Json error in {self.rsl_plate_num}: {e}")
try:
reagents = [item.to_sub_dict() for item in self.reagents]
reagents = [item.to_sub_dict(extraction_kit=self.extraction_kit) for item in self.reagents]
except Exception as e:
logger.error(f"We got an error retrieving reagents: {e}")
reagents = None
@@ -252,6 +252,8 @@ class Wastewater(BasicSubmission):
"""
# samples = relationship("WWSample", back_populates="rsl_plate", uselist=True)
pcr_info = Column(JSON)
ext_technician = Column(String(64))
pcr_technician = Column(String(64))
# ww_sample_id = Column(String, ForeignKey("_ww_samples.id", ondelete="SET NULL", name="fk_WW_sample_id"))
__mapper_args__ = {"polymorphic_identity": "Wastewater", "polymorphic_load": "inline"}
@@ -267,6 +269,7 @@ class Wastewater(BasicSubmission):
output['pcr_info'] = json.loads(self.pcr_info)
except TypeError as e:
pass
output['Technician'] = f"Enr: {self.technician}, Ext: {self.ext_technician}, PCR: {self.pcr_technician}"
return output
class WastewaterArtic(BasicSubmission):
@@ -460,10 +463,10 @@ class WastewaterSample(BasicSample):
except AttributeError as e:
check = False
if check:
logger.debug(f"Using well info in name.")
# logger.debug(f"Using well info in name.")
sample['name'] = f"{self.submitter_id}\n\t- ct N1: {'{:.2f}'.format(self.assoc.ct_n1)} ({self.assoc.n1_status})\n\t- ct N2: {'{:.2f}'.format(self.assoc.ct_n2)} ({self.assoc.n2_status})"
else:
logger.error(f"Couldn't get the pcr info")
# else:
# logger.error(f"Couldn't get the pcr info")
return sample
def to_hitpick(self, submission_rsl:str) -> dict|None:
@@ -511,7 +514,7 @@ class BacterialCultureSample(BasicSample):
# rsl_plate_id = Column(INTEGER, ForeignKey("_submissions.id", ondelete="SET NULL", name="fk_BCS_sample_id")) #: id of parent plate
# rsl_plate = relationship("BacterialCulture", back_populates="samples") #: relationship to parent plate
__mapper_args__ = {"polymorphic_identity": "bacterial_culture_sample", "polymorphic_load": "inline"}
__mapper_args__ = {"polymorphic_identity": "Bacterial Culture Sample", "polymorphic_load": "inline"}
# def to_string(self) -> str:
# """
@@ -543,10 +546,10 @@ class SubmissionSampleAssociation(Base):
DOC: https://docs.sqlalchemy.org/en/14/orm/extensions/associationproxy.html
"""
__tablename__ = "_submission_sample"
sample_id = Column(INTEGER, ForeignKey("_samples.id"), primary_key=True)
sample_id = Column(INTEGER, ForeignKey("_samples.id"), nullable=False)
submission_id = Column(INTEGER, ForeignKey("_submissions.id"), primary_key=True)
row = Column(INTEGER)
column = Column(INTEGER)
row = Column(INTEGER, primary_key=True)
column = Column(INTEGER, primary_key=True)
submission = relationship(BasicSubmission, back_populates="submission_sample_associations")
@@ -569,6 +572,9 @@ class SubmissionSampleAssociation(Base):
self.row = row
self.column = column
def __repr__(self) -> str:
return f"<SubmissionSampleAssociation({self.submission.rsl_plate_num} & {self.sample.submitter_id})"
class WastewaterAssociation(SubmissionSampleAssociation):
ct_n1 = Column(FLOAT(2)) #: AKA ct for N1