Big update, semi-functional

This commit is contained in:
Landon Wark
2023-08-28 13:20:30 -05:00
parent b6de159631
commit bc7a3b8f5f
13 changed files with 824 additions and 466 deletions

View File

@@ -150,7 +150,7 @@ def construct_submission_info(ctx:Settings, info_dict:dict) -> models.BasicSubmi
logger.debug(f"Looking at models for submission type: {query}")
model = getattr(models, query)
logger.debug(f"We've got the model: {type(model)}")
info_dict['submission_type'] = info_dict['submission_type'].replace(" ", "_").lower()
# info_dict['submission_type'] = info_dict['submission_type'].replace(" ", "_").lower()
# if query return nothing, ie doesn't already exist in db
if instance == None:
instance = model()
@@ -224,6 +224,9 @@ def construct_submission_info(ctx:Settings, info_dict:dict) -> models.BasicSubmi
logger.critical(e)
continue
continue
case "submission_type":
# item = "submission_type"
field_value = lookup_submissiontype_by_name(ctx=ctx, type_name=value)
case _:
field_value = value
# insert into field
@@ -276,7 +279,7 @@ def construct_reagent(ctx:Settings, info_dict:dict) -> models.Reagent:
"""
reagent = models.Reagent()
for item in info_dict:
logger.debug(f"Reagent info item: {item}")
logger.debug(f"Reagent info item for {item}: {info_dict[item]}")
# set fields based on keys in dictionary
match item:
case "lot":
@@ -284,7 +287,12 @@ def construct_reagent(ctx:Settings, info_dict:dict) -> models.Reagent:
case "expiry":
reagent.expiry = info_dict[item]
case "type":
reagent.type = lookup_reagenttype_by_name(ctx=ctx, rt_name=info_dict[item].replace(" ", "_").lower())
reagent.type = lookup_reagenttype_by_name(ctx=ctx, rt_name=info_dict[item])
case "name":
if item == None:
reagent.name = reagent.type.name
else:
reagent.name = info_dict[item]
# add end-of-life extension from reagent type to expiry date
# NOTE: this will now be done only in the reporting phase to account for potential changes in end-of-life extensions
# try:
@@ -320,7 +328,7 @@ def lookup_reagenttype_by_name(ctx:Settings, rt_name:str) -> models.ReagentType:
Returns:
models.ReagentType: looked up reagent type
"""
logger.debug(f"Looking up ReagentType by name: {rt_name}")
logger.debug(f"Looking up ReagentType by name: {rt_name.title()}")
# lookedup = ctx['database_session'].query(models.ReagentType).filter(models.ReagentType.name==rt_name).first()
lookedup = ctx.database_session.query(models.ReagentType).filter(models.ReagentType.name==rt_name).first()
logger.debug(f"Found ReagentType: {lookedup}")
@@ -339,12 +347,13 @@ def lookup_kittype_by_use(ctx:Settings, used_by:str|None=None) -> list[models.Ki
"""
if used_by != None:
# return ctx['database_session'].query(models.KitType).filter(models.KitType.used_for.contains(used_by)).all()
return ctx.database_session.query(models.KitType).filter(models.KitType.used_for.contains(used_by)).all()
# return ctx.database_session.query(models.KitType).filter(models.KitType.used_for.contains(used_by)).all()
return ctx.database_session.query(models.KitType).filter(models.KitType.used_for.any(name=used_by)).all()
else:
# return ctx['database_session'].query(models.KitType).all()
return ctx.database_session.query(models.KitType).all()
def lookup_kittype_by_name(ctx:Settings, name:str) -> models.KitType:
def lookup_kittype_by_name(ctx:Settings, name:str|dict) -> models.KitType:
"""
Lookup a kit type by name
@@ -359,7 +368,8 @@ def lookup_kittype_by_name(ctx:Settings, name:str) -> models.KitType:
name = name['value']
logger.debug(f"Querying kittype: {name}")
# return ctx['database_session'].query(models.KitType).filter(models.KitType.name==name).first()
return ctx.database_session.query(models.KitType).filter(models.KitType.name==name).first()
with ctx.database_session.no_autoflush:
return ctx.database_session.query(models.KitType).filter(models.KitType.name==name).first()
def lookup_kittype_by_id(ctx:Settings, id:int) -> models.KitType:
return ctx.database_session.query(models.KitType).filter(models.KitType.id==id).first()
@@ -559,12 +569,17 @@ def create_kit_from_yaml(ctx:Settings, exp:dict) -> dict:
# continue
# A submission type may use multiple kits.
for kt in exp[type]['kits']:
submission_type = lookup_submissiontype_by_name(ctx=ctx, type_name=type)
kit = models.KitType(name=kt,
used_for=[type.replace("_", " ").title()],
constant_cost=exp[type]["kits"][kt]["constant_cost"],
mutable_cost_column=exp[type]["kits"][kt]["mutable_cost_column"],
mutable_cost_sample=exp[type]["kits"][kt]["mutable_cost_sample"]
# constant_cost=exp[type]["kits"][kt]["constant_cost"],
# mutable_cost_column=exp[type]["kits"][kt]["mutable_cost_column"],
# mutable_cost_sample=exp[type]["kits"][kt]["mutable_cost_sample"]
)
kt_st_assoc = models.SubmissionTypeKitTypeAssociation(kit_type=kit, submission_type=submission_type)
kt_st_assoc.constant_cost = exp[type]["kits"][kt]["constant_cost"]
kt_st_assoc.mutable_cost_column = exp[type]["kits"][kt]["mutable_cost_column"]
kt_st_assoc.mutable_cost_sample = exp[type]["kits"][kt]["mutable_cost_sample"]
kit.kit_submissiontype_associations.append(kt_st_assoc)
# A kit contains multiple reagent types.
for r in exp[type]['kits'][kt]['reagenttypes']:
# check if reagent type already exists.
@@ -573,7 +588,7 @@ def create_kit_from_yaml(ctx:Settings, exp:dict) -> dict:
look_up = ctx.database_session.query(models.ReagentType).filter(models.ReagentType.name==r).first()
if look_up == None:
# rt = models.ReagentType(name=r.replace(" ", "_").lower(), eol_ext=timedelta(30*exp[type]['kits'][kt]['reagenttypes'][r]['eol_ext']), kits=[kit], required=1)
rt = models.ReagentType(name=r.replace(" ", "_").lower(), eol_ext=timedelta(30*exp[type]['kits'][kt]['reagenttypes'][r]['eol_ext']), last_used="")
rt = models.ReagentType(name=r.replace(" ", "_").lower().strip(), eol_ext=timedelta(30*exp[type]['kits'][kt]['reagenttypes'][r]['eol_ext']), last_used="")
else:
rt = look_up
# rt.kits.append(kit)
@@ -583,7 +598,7 @@ def create_kit_from_yaml(ctx:Settings, exp:dict) -> dict:
# except AttributeError as e:
# logger.error(f"Error appending reagent id to kit.reagent_types_id: {e}, creating new.")
# kit.reagent_types_id = [rt.id]
assoc = models.KitTypeReagentTypeAssociation(kit_type=kit, reagent_type=rt, uses=kit.used_for)
assoc = models.KitTypeReagentTypeAssociation(kit_type=kit, reagent_type=rt, uses={})
# ctx['database_session'].add(rt)
ctx.database_session.add(rt)
kit.kit_reagenttype_associations.append(assoc)
@@ -646,10 +661,11 @@ def lookup_all_sample_types(ctx:Settings) -> list[str]:
list[str]: list of sample type names
"""
# uses = [item.used_for for item in ctx['database_session'].query(models.KitType).all()]
uses = [item.used_for for item in ctx.database_session.query(models.KitType).all()]
# uses = [item.used_for for item in ctx.database_session.query(models.KitType).all()]
# flattened list of lists
uses = list(set([item for sublist in uses for item in sublist]))
return uses
# uses = list(set([item for sublist in uses for item in sublist]))
return [item.name for item in ctx.database_session.query(models.SubmissionType).all()]
def get_all_available_modes(ctx:Settings) -> list[str]:
"""
@@ -1084,7 +1100,8 @@ def lookup_sample_by_submitter_id(ctx:Settings, submitter_id:str) -> models.Basi
Returns:
BasicSample: _description_
"""
return ctx.database_session.query(models.BasicSample).filter(models.BasicSample.submitter_id==submitter_id).first()
with ctx.database_session.no_autoflush:
return ctx.database_session.query(models.BasicSample).filter(models.BasicSample.submitter_id==submitter_id).first()
def get_all_submission_types(ctx:Settings) -> List[str]:
"""
@@ -1150,4 +1167,18 @@ def lookup_all_reagent_names_by_role(ctx:Settings, role_name:str) -> List[str]:
try:
return [reagent.name for reagent in role.instances]
except AttributeError:
return []
return []
def lookup_submissiontype_by_name(ctx:Settings, type_name:str) -> models.SubmissionType:
"""
_summary_
Args:
ctx (Settings): _description_
type_name (str): _description_
Returns:
models.SubmissionType: _description_
"""
return ctx.database_session.query(models.SubmissionType).filter(models.SubmissionType.name==type_name).first()

View File

@@ -7,7 +7,7 @@ Base = declarative_base()
metadata = Base.metadata
from .controls import Control, ControlType
from .kits import KitType, ReagentType, Reagent, Discount, KitTypeReagentTypeAssociation
from .kits import KitType, ReagentType, Reagent, Discount, KitTypeReagentTypeAssociation, SubmissionType, SubmissionTypeKitTypeAssociation
from .organizations import Organization, Contact
# from .samples import WWSample, BCSample, BasicSample
from .submissions import BasicSubmission, BacterialCulture, Wastewater, WastewaterArtic, WastewaterSample, BacterialCultureSample, BasicSample, SubmissionSampleAssociation, WastewaterAssociation

View File

@@ -32,11 +32,9 @@ class KitType(Base):
id = Column(INTEGER, primary_key=True) #: primary key
name = Column(String(64), unique=True) #: name of kit
submissions = relationship("BasicSubmission", back_populates="extraction_kit") #: submissions this kit was used for
used_for = Column(JSON) #: list of names of sample types this kit can process
cost_per_run = Column(FLOAT(2)) #: dollar amount for each full run of this kit NOTE: depreciated, use the constant and mutable costs instead
mutable_cost_column = Column(FLOAT(2)) #: dollar amount per 96 well plate that can change with number of columns (reagents, tips, etc)
mutable_cost_sample = Column(FLOAT(2)) #: dollar amount that can change with number of samples (reagents, tips, etc)
constant_cost = Column(FLOAT(2)) #: dollar amount per plate that will remain constant (plates, man hours, etc)
# used_for = Column(JSON) #: list of names of sample types this kit can process
# used_for = relationship("SubmissionType", back_populates="extraction_kits", uselist=True, secondary=submissiontype_kittypes)
# cost_per_run = Column(FLOAT(2)) #: dollar amount for each full run of this kit NOTE: depreciated, use the constant and mutable costs instead
# reagent_types = relationship("ReagentType", back_populates="kits", uselist=True, secondary=reagenttypes_kittypes) #: reagent types this kit contains
# reagent_types_id = Column(INTEGER, ForeignKey("_reagent_types.id", ondelete='SET NULL', use_alter=True, name="fk_KT_reagentstype_id")) #: joined reagent type id
# kit_reagenttype_association =
@@ -46,12 +44,23 @@ class KitType(Base):
back_populates="kit_type",
cascade="all, delete-orphan",
)
# association proxy of "user_keyword_associations" collection
# to "keyword" attribute
reagent_types = association_proxy("kit_reagenttype_associations", "reagenttype")
kit_submissiontype_associations = relationship(
"SubmissionTypeKitTypeAssociation",
back_populates="kit_type",
cascade="all, delete-orphan",
)
used_for = association_proxy("kit_submissiontype_associations", "submission_type")
def __repr__(self) -> str:
return f"KitType({self.name})"
return f"<KitType({self.name})>"
def __str__(self) -> str:
"""
@@ -64,9 +73,9 @@ class KitType(Base):
def get_reagents(self, required:bool=False) -> list:
if required:
return [item.reagenttype for item in self.kit_reagenttype_associations if item.required == 1]
return [item.reagent_type for item in self.kit_reagenttype_associations if item.required == 1]
else:
return [item.reagenttype for item in self.kit_reagenttype_associations]
return [item.reagent_type for item in self.kit_reagenttype_associations]
def construct_xl_map_for_use(self, use:str) -> dict:
@@ -75,12 +84,16 @@ class KitType(Base):
assocs = [item for item in self.kit_reagenttype_associations if use in item.uses]
for assoc in assocs:
try:
map[assoc.reagenttype.name] = assoc.uses[use]
map[assoc.reagent_type.name] = assoc.uses[use]
except TypeError:
continue
try:
st_assoc = [item for item in self.used_for if use == item.name][0]
map['info'] = st_assoc.info_map
except IndexError as e:
map['info'] = {}
return map
class KitTypeReagentTypeAssociation(Base):
"""
table containing reagenttype/kittype associations
@@ -96,11 +109,11 @@ class KitTypeReagentTypeAssociation(Base):
kit_type = relationship(KitType, back_populates="kit_reagenttype_associations")
# reference to the "ReagentType" object
reagenttype = relationship("ReagentType")
reagent_type = relationship("ReagentType")
def __init__(self, kit_type=None, reagent_type=None, uses=None, required=1):
self.kit = kit_type
self.reagenttype = reagent_type
self.kit_type = kit_type
self.reagent_type = reagent_type
self.uses = uses
self.required = required
@@ -116,8 +129,6 @@ class KitTypeReagentTypeAssociation(Base):
raise ValueError(f'{value} is not a reagenttype')
return value
class ReagentType(Base):
"""
Base of reagent type abstract
@@ -151,7 +162,6 @@ class ReagentType(Base):
def __repr__(self):
return f"ReagentType({self.name})"
class Reagent(Base):
"""
Concrete reagent instance
@@ -215,7 +225,6 @@ class Reagent(Base):
"expiry": self.expiry.strftime("%Y-%m-%d")
}
class Discount(Base):
"""
Relationship table for client labs for certain kits.
@@ -230,4 +239,44 @@ class Discount(Base):
name = Column(String(128))
amount = Column(FLOAT(2))
class SubmissionType(Base):
__tablename__ = "_submission_types"
id = Column(INTEGER, primary_key=True) #: primary key
name = Column(String(128), unique=True) #: name of submission type
info_map = Column(JSON)
instances = relationship("BasicSubmission", backref="submission_type")
submissiontype_kit_associations = relationship(
"SubmissionTypeKitTypeAssociation",
back_populates="submission_type",
cascade="all, delete-orphan",
)
kit_types = association_proxy("kit_submissiontype_associations", "kit_type")
def __repr__(self) -> str:
return f"<SubmissionType({self.name})>"
class SubmissionTypeKitTypeAssociation(Base):
__tablename__ = "_submissiontypes_kittypes"
submission_types_id = Column(INTEGER, ForeignKey("_submission_types.id"), primary_key=True)
kits_id = Column(INTEGER, ForeignKey("_kits.id"), primary_key=True)
mutable_cost_column = Column(FLOAT(2)) #: dollar amount per 96 well plate that can change with number of columns (reagents, tips, etc)
mutable_cost_sample = Column(FLOAT(2)) #: dollar amount that can change with number of samples (reagents, tips, etc)
constant_cost = Column(FLOAT(2)) #: dollar amount per plate that will remain constant (plates, man hours, etc)
# reagent_type_name = Column(INTEGER, ForeignKey("_reagent_types.name"))
kit_type = relationship(KitType, back_populates="kit_submissiontype_associations")
# reference to the "ReagentType" object
submission_type = relationship(SubmissionType, back_populates="submissiontype_kit_associations")
def __init__(self, kit_type=None, submission_type=None):
self.kit_type = kit_type
self.submission_type = submission_type
self.mutable_cost_column = 0.00
self.mutable_cost_sample = 0.00
self.constant_cost = 0.00

View File

@@ -31,6 +31,9 @@ class Organization(Base):
str: string representing organization name
"""
return self.name.replace("_", " ").title()
def __repr__(self) -> str:
return f"<Organization({self.name})>"
class Contact(Base):

View File

@@ -12,6 +12,9 @@ from math import ceil
from sqlalchemy.ext.associationproxy import association_proxy
import uuid
from . import Base
from pandas import Timestamp
from dateutil.parser import parse
import pprint
logger = logging.getLogger(f"submissions.{__name__}")
@@ -33,7 +36,8 @@ class BasicSubmission(Base):
sample_count = Column(INTEGER) #: Number of samples in the submission
extraction_kit = relationship("KitType", back_populates="submissions") #: The extraction kit used
extraction_kit_id = Column(INTEGER, ForeignKey("_kits.id", ondelete="SET NULL", name="fk_BS_extkit_id"))
submission_type = Column(String(32)) #: submission type (should be string in D3 of excel sheet)
# submission_type = Column(String(32)) #: submission type (should be string in D3 of excel sheet)
submission_type_name = Column(String, ForeignKey("_submission_types.name", ondelete="SET NULL", name="fk_BS_subtype_name"))
technician = Column(String(64)) #: initials of processing tech(s)
# Move this into custom types?
reagents = relationship("Reagent", back_populates="submissions", secondary=reagents_submissions) #: relationship to reagents
@@ -55,7 +59,7 @@ class BasicSubmission(Base):
# Allows for subclassing into ex. BacterialCulture, Wastewater, etc.
__mapper_args__ = {
"polymorphic_identity": "basic_submission",
"polymorphic_on": submission_type,
"polymorphic_on": submission_type_name,
"with_polymorphic": "*",
}
@@ -128,7 +132,7 @@ class BasicSubmission(Base):
output = {
"id": self.id,
"Plate Number": self.rsl_plate_num,
"Submission Type": self.submission_type.replace("_", " ").title(),
"Submission Type": self.submission_type_name,
"Submitter Plate Number": self.submitter_plate_num,
"Submitted Date": self.submitted_date.strftime("%Y-%m-%d"),
"Submitting Lab": sub_lab,
@@ -184,14 +188,18 @@ class BasicSubmission(Base):
except Exception as e:
logger.error(f"Column count error: {e}")
# cols_count_24 = ceil(int(self.sample_count) / 3)
if all(item == 0.0 for item in [self.extraction_kit.constant_cost, self.extraction_kit.mutable_cost_column, self.extraction_kit.mutable_cost_sample]):
logger.debug(f"Pre-association check. {pprint.pformat(self.__dict__)}")
assoc = [item for item in self.extraction_kit.kit_submissiontype_associations if item.submission_type == self.submission_type][0]
logger.debug(f"Came up with association: {assoc}")
# if all(item == 0.0 for item in [self.extraction_kit.constant_cost, self.extraction_kit.mutable_cost_column, self.extraction_kit.mutable_cost_sample]):
if all(item == 0.0 for item in [assoc.constant_cost, assoc.mutable_cost_column, assoc.mutable_cost_sample]):
try:
self.run_cost = self.extraction_kit.cost_per_run
except Exception as e:
logger.error(f"Calculation error: {e}")
else:
try:
self.run_cost = self.extraction_kit.constant_cost + (self.extraction_kit.mutable_cost_column * cols_count_96) + (self.extraction_kit.mutable_cost_sample * int(self.sample_count))
self.run_cost = assoc.constant_cost + (assoc.mutable_cost_column * cols_count_96) + (assoc.mutable_cost_sample * int(self.sample_count))
except Exception as e:
logger.error(f"Calculation error: {e}")
@@ -225,7 +233,7 @@ class BacterialCulture(BasicSubmission):
"""
controls = relationship("Control", back_populates="submission", uselist=True) #: A control sample added to submission
# samples = relationship("BCSample", back_populates="rsl_plate", uselist=True)
__mapper_args__ = {"polymorphic_identity": "bacterial_culture", "polymorphic_load": "inline"}
__mapper_args__ = {"polymorphic_identity": "Bacterial Culture", "polymorphic_load": "inline"}
def to_dict(self) -> dict:
"""
@@ -245,7 +253,7 @@ class Wastewater(BasicSubmission):
# samples = relationship("WWSample", back_populates="rsl_plate", uselist=True)
pcr_info = Column(JSON)
# ww_sample_id = Column(String, ForeignKey("_ww_samples.id", ondelete="SET NULL", name="fk_WW_sample_id"))
__mapper_args__ = {"polymorphic_identity": "wastewater", "polymorphic_load": "inline"}
__mapper_args__ = {"polymorphic_identity": "Wastewater", "polymorphic_load": "inline"}
def to_dict(self) -> dict:
"""
@@ -315,14 +323,14 @@ class BasicSample(Base):
@validates('submitter_id')
def create_id(self, key, value):
logger.debug(f"validating sample_id of: {value}")
# logger.debug(f"validating sample_id of: {value}")
if value == None:
return uuid.uuid4().hex.upper()
else:
return value
def __repr__(self) -> str:
return f"{self.sample_type}Sample({self.submitter_id})"
return f"<{self.sample_type.replace('_', ' ').title(). replace(' ', '')}({self.submitter_id})>"
def to_sub_dict(self, submission_rsl:str) -> dict:
row_map = {1:"A", 2:"B", 3:"C", 4:"D", 5:"E", 6:"F", 7:"G", 8:"H"}
@@ -363,30 +371,31 @@ class WastewaterSample(BasicSample):
# id = Column(INTEGER, primary_key=True) #: primary key
ww_processing_num = Column(String(64)) #: wastewater processing number
# ww_sample_full_id = Column(String(64), nullable=False, unique=True)
ww_sample_full_id = Column(String(64))
rsl_number = Column(String(64)) #: rsl plate identification number
# rsl_plate = relationship("Wastewater", back_populates="samples") #: relationship to parent plate
# rsl_plate_id = Column(INTEGER, ForeignKey("_submissions.id", ondelete="SET NULL", name="fk_WWS_submission_id"))
collection_date = Column(TIMESTAMP) #: Date submission received
collection_date = Column(TIMESTAMP) #: Date sample collected
received_date = Column(TIMESTAMP) #: Date sample received
# well_number = Column(String(8)) #: location on 96 well plate
# The following are fields from the sample tracking excel sheet Ruth put together.
# I have no idea when they will be implemented or how.
testing_type = Column(String(64))
site_status = Column(String(64))
# testing_type = Column(String(64))
# site_status = Column(String(64))
notes = Column(String(2000))
# ct_n1 = Column(FLOAT(2)) #: AKA ct for N1
# ct_n2 = Column(FLOAT(2)) #: AKA ct for N2
# n1_status = Column(String(32))
# n2_status = Column(String(32))
seq_submitted = Column(BOOLEAN())
ww_seq_run_id = Column(String(64))
# seq_submitted = Column(BOOLEAN())
# ww_seq_run_id = Column(String(64))
# sample_type = Column(String(16))
# pcr_results = Column(JSON)
well_24 = Column(String(8)) #: location on 24 well plate
sample_location = Column(String(8)) #: location on 24 well plate
# artic_rsl_plate = relationship("WastewaterArtic", back_populates="samples")
# artic_well_number = Column(String(8))
__mapper_args__ = {"polymorphic_identity": "wastewater_sample", "polymorphic_load": "inline"}
__mapper_args__ = {"polymorphic_identity": "Wastewater Sample", "polymorphic_load": "inline"}
# def to_string(self) -> str:
# """
@@ -397,6 +406,42 @@ class WastewaterSample(BasicSample):
# """
# return f"{self.well_number}: {self.ww_sample_full_id}"
# @validates("received-date")
# def convert_rdate_time(self, key, value):
# if isinstance(value, Timestamp):
# return value.date()
# return value
@validates("collected-date")
def convert_cdate_time(self, key, value):
logger.debug(f"Validating {key}: {value}")
if isinstance(value, Timestamp):
return value.date()
if isinstance(value, str):
return parse(value)
return value
# @collection_date.setter
# def collection_date(self, value):
# match value:
# case Timestamp():
# self.collection_date = value.date()
# case str():
# self.collection_date = parse(value)
# case _:
# self.collection_date = value
def __init__(self, **kwargs):
if 'collection_date' in kwargs.keys():
logger.debug(f"Got collection_date: {kwargs['collection_date']}. Attempting parse.")
if isinstance(kwargs['collection_date'], str):
logger.debug(f"collection_date is a string...")
kwargs['collection_date'] = parse(kwargs['collection_date'])
logger.debug(f"output is {kwargs['collection_date']}")
super().__init__(**kwargs)
def to_sub_dict(self, submission_rsl:str) -> dict:
"""
Gui friendly dictionary. Inherited from BasicSample
@@ -451,7 +496,6 @@ class WastewaterSample(BasicSample):
# return None
return sample
class BacterialCultureSample(BasicSample):
"""
base of bacterial culture sample
@@ -493,8 +537,6 @@ class BacterialCultureSample(BasicSample):
# }
return sample
class SubmissionSampleAssociation(Base):
"""
table containing submission/sample associations