landon/Submissions-App
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Pre-sample/control connect

Browse Source
This commit is contained in:
Landon Wark
2023-12-05 10:20:46 -06:00
parent 283e77fee5
commit cddb947ec8
29 changed files with 1357 additions and 1042 deletions
Download Patch File Download Diff File Expand all files Collapse all files

44
src/submissions/backend/db/models/__init__.py
View File

@@ -1,8 +1,50 @@
'''
Contains all models for sqlalchemy
'''
import sys
from sqlalchemy.orm import DeclarativeMeta, declarative_base
from sqlalchemy.ext.declarative import declared_attr
if 'pytest' in sys.modules:
from pathlib import Path
sys.path.append(Path(__file__).parents[4].absolute().joinpath("tests").__str__())
Base: DeclarativeMeta = declarative_base()
class BaseClass(Base):
"""
Abstract class to pass ctx values to all SQLAlchemy objects.
Args:
Base (DeclarativeMeta): Declarative base for metadata.
"""
__abstract__ = True
__table_args__ = {'extend_existing': True}
@declared_attr
def __database_session__(cls):
if not 'pytest' in sys.modules:
from tools import ctx
else:
from test_settings import ctx
return ctx.database_session
@declared_attr
def __directory_path__(cls):
if not 'pytest' in sys.modules:
from tools import ctx
else:
from test_settings import ctx
return ctx.directory_path
@declared_attr
def __backup_path__(cls):
if not 'pytest' in sys.modules:
from tools import ctx
else:
from test_settings import ctx
return ctx.backup_path
from tools import Base
from .controls import *
# import order must go: orgs, kit, subs due to circular import issues
from .organizations import *

62
src/submissions/backend/db/models/controls.py
View File

@@ -7,7 +7,7 @@ from sqlalchemy.orm import relationship, Query
import logging
from operator import itemgetter
import json
from . import Base
from . import BaseClass
from tools import setup_lookup, query_return
from datetime import date, datetime
from typing import List
@@ -15,12 +15,11 @@ from dateutil.parser import parse
logger = logging.getLogger(f"submissions.{__name__}")
class ControlType(Base):
class ControlType(BaseClass):
"""
Base class of a control archetype.
"""
__tablename__ = '_control_types'
__table_args__ = {'extend_existing': True}
id = Column(INTEGER, primary_key=True) #: primary key
name = Column(String(255), unique=True) #: controltype name (e.g. MCS)
@@ -29,7 +28,7 @@ class ControlType(Base):
@classmethod
@setup_lookup
def query(cls,
def query(cls,
name:str=None,
limit:int=0
) -> ControlType|List[ControlType]:
@@ -37,14 +36,13 @@ class ControlType(Base):
Lookup control archetypes in the database
Args:
ctx (Settings): Settings object passed down from gui.
name (str, optional): Control type name (limits results to 1). Defaults to None.
limit (int, optional): Maximum number of results to return. Defaults to 0.
Returns:
models.ControlType|List[models.ControlType]: ControlType(s) of interest.
"""
query = cls.metadata.session.query(cls)
query = cls.__database_session__.query(cls)
match name:
case str():
query = query.filter(cls.name==name)
@@ -52,14 +50,13 @@ class ControlType(Base):
case _:
pass
return query_return(query=query, limit=limit)
class Control(Base):
class Control(BaseClass):
"""
Base class of a control sample.
"""
__tablename__ = '_control_samples'
__table_args__ = {'extend_existing': True}
id = Column(INTEGER, primary_key=True) #: primary key
parent_id = Column(String, ForeignKey("_control_types.id", name="fk_control_parent_id")) #: primary key of control type
@@ -114,10 +111,9 @@ class Control(Base):
def convert_by_mode(self, mode:str) -> list[dict]:
"""
split control object into analysis types for controls graphs
split this instance into analysis types for controls graphs
Args:
control (models.Control): control to be parsed into list
mode (str): analysis type, 'contains', etc
Returns:
@@ -168,6 +164,21 @@ class Control(Base):
data = {}
return data
@classmethod
def get_modes(cls) -> List[str]:
"""
Get all control modes from database
Returns:
List[str]: List of control mode names.
"""
try:
cols = [item.name for item in list(cls.__table__.columns) if isinstance(item.type, JSON)]
except AttributeError as e:
logger.error(f"Failed to get available modes from db: {e}")
cols = []
return cols
@classmethod
@setup_lookup
def query(cls,
@@ -190,15 +201,14 @@ class Control(Base):
Returns:
models.Control|List[models.Control]: Control object of interest.
"""
query: Query = cls.metadata.session.query(cls)
query: Query = cls.__database_session__.query(cls)
# by control type
match control_type:
case ControlType():
logger.debug(f"Looking up control by control type: {control_type}")
# query = query.join(models.ControlType).filter(models.ControlType==control_type)
# logger.debug(f"Looking up control by control type: {control_type}")
query = query.filter(cls.controltype==control_type)
case str():
logger.debug(f"Looking up control by control type: {control_type}")
# logger.debug(f"Looking up control by control type: {control_type}")
query = query.join(ControlType).filter(ControlType.name==control_type)
case _:
pass
@@ -224,7 +234,7 @@ class Control(Base):
end_date = datetime.fromordinal(datetime(1900, 1, 1).toordinal() + end_date - 2).date().strftime("%Y-%m-%d")
case _:
end_date = parse(end_date).strftime("%Y-%m-%d")
logger.debug(f"Looking up BasicSubmissions from start date: {start_date} and end date: {end_date}")
# logger.debug(f"Looking up BasicSubmissions from start date: {start_date} and end date: {end_date}")
query = query.filter(cls.submitted_date.between(start_date, end_date))
match control_name:
case str():
@@ -233,23 +243,3 @@ class Control(Base):
case _:
pass
return query_return(query=query, limit=limit)
@classmethod
def get_modes(cls):
"""
Get all control modes from database
Args:
ctx (Settings): Settings object passed down from gui.
Returns:
List[str]: List of control mode names.
"""
rel = cls.metadata.session.query(cls).first()
try:
cols = [item.name for item in list(rel.__table__.columns) if isinstance(item.type, JSON)]
except AttributeError as e:
logger.debug(f"Failed to get available modes from db: {e}")
cols = []
return cols

394
src/submissions/backend/db/models/kits.py
View File

@@ -2,14 +2,15 @@
All kit and reagent related models
'''
from __future__ import annotations
from sqlalchemy import Column, String, TIMESTAMP, JSON, INTEGER, ForeignKey, Interval, Table, FLOAT, func, BLOB
from sqlalchemy import Column, String, TIMESTAMP, JSON, INTEGER, ForeignKey, Interval, Table, FLOAT, BLOB
from sqlalchemy.orm import relationship, validates, Query
from sqlalchemy.ext.associationproxy import association_proxy
from datetime import date
import logging
from tools import check_authorization, setup_lookup, query_return, Report, Result
from tools import check_authorization, setup_lookup, query_return, Report, Result, Settings
from typing import List
from . import Base, Organization
from pandas import ExcelFile
from . import Base, BaseClass, Organization
logger = logging.getLogger(f'submissions.{__name__}')
@@ -21,12 +22,12 @@ reagenttypes_reagents = Table(
extend_existing = True
)
class KitType(Base):
class KitType(BaseClass):
"""
Base of kits used in submission processing
"""
__tablename__ = "_kits"
__table_args__ = {'extend_existing': True}
# __table_args__ = {'extend_existing': True}
id = Column(INTEGER, primary_key=True) #: primary key
name = Column(String(64), unique=True) #: name of kit
@@ -54,16 +55,7 @@ class KitType(Base):
def __repr__(self) -> str:
return f"<KitType({self.name})>"
def __str__(self) -> str:
"""
a string representing this object
Returns:
str: a string representing this object's name
"""
return self.name
def get_reagents(self, required:bool=False, submission_type:str|None=None) -> list:
def get_reagents(self, required:bool=False, submission_type:str|SubmissionType|None=None) -> list:
"""
Return ReagentTypes linked to kit through KitTypeReagentTypeAssociation.
@@ -74,10 +66,13 @@ class KitType(Base):
Returns:
list: List of reagent types
"""
if submission_type != None:
relevant_associations = [item for item in self.kit_reagenttype_associations if submission_type in item.uses.keys()]
else:
relevant_associations = [item for item in self.kit_reagenttype_associations]
match submission_type:
case SubmissionType():
relevant_associations = [item for item in self.kit_reagenttype_associations if submission_type.name in item.uses.keys()]
case str():
relevant_associations = [item for item in self.kit_reagenttype_associations if submission_type in item.uses.keys()]
case _:
relevant_associations = [item for item in self.kit_reagenttype_associations]
if required:
return [item.reagent_type for item in relevant_associations if item.required == 1]
else:
@@ -109,14 +104,9 @@ class KitType(Base):
map['info'] = {}
return map
@check_authorization
def save(self):
self.metadata.session.add(self)
self.metadata.session.commit()
@classmethod
@setup_lookup
def query(cls,
def query(cls,
name:str=None,
used_for:str|SubmissionType|None=None,
id:int|None=None,
@@ -126,7 +116,6 @@ class KitType(Base):
Lookup a list of or single KitType.
Args:
ctx (Settings): Settings object passed down from gui
name (str, optional): Name of desired kit (returns single instance). Defaults to None.
used_for (str | models.Submissiontype | None, optional): Submission type the kit is used for. Defaults to None.
id (int | None, optional): Kit id in the database. Defaults to None.
@@ -135,10 +124,10 @@ class KitType(Base):
Returns:
models.KitType|List[models.KitType]: KitType(s) of interest.
"""
query: Query = cls.metadata.session.query(cls)
query: Query = cls.__database_session__.query(cls)
match used_for:
case str():
logger.debug(f"Looking up kit type by use: {used_for}")
# logger.debug(f"Looking up kit type by use: {used_for}")
query = query.filter(cls.used_for.any(name=used_for))
case SubmissionType():
query = query.filter(cls.used_for.contains(used_for))
@@ -146,30 +135,37 @@ class KitType(Base):
pass
match name:
case str():
logger.debug(f"Looking up kit type by name: {name}")
# logger.debug(f"Looking up kit type by name: {name}")
query = query.filter(cls.name==name)
limit = 1
case _:
pass
match id:
case int():
logger.debug(f"Looking up kit type by id: {id}")
# logger.debug(f"Looking up kit type by id: {id}")
query = query.filter(cls.id==id)
limit = 1
case str():
logger.debug(f"Looking up kit type by id: {id}")
# logger.debug(f"Looking up kit type by id: {id}")
query = query.filter(cls.id==int(id))
limit = 1
case _:
pass
return query_return(query=query, limit=limit)
@check_authorization
def save(self, ctx:Settings):
"""
Add this instance to database and commit
"""
self.__database_session__.add(self)
self.__database_session__.commit()
class ReagentType(Base):
class ReagentType(BaseClass):
"""
Base of reagent type abstract
"""
__tablename__ = "_reagent_types"
__table_args__ = {'extend_existing': True}
id = Column(INTEGER, primary_key=True) #: primary key
name = Column(String(64)) #: name of reagent type
@@ -187,21 +183,21 @@ class ReagentType(Base):
# creator function: https://stackoverflow.com/questions/11091491/keyerror-when-adding-objects-to-sqlalchemy-association-object/11116291#11116291
kit_types = association_proxy("reagenttype_kit_associations", "kit_type", creator=lambda kit: KitTypeReagentTypeAssociation(kit_type=kit))
def __str__(self) -> str:
"""
string representing this object
# def __str__(self) -> str:
# """
# string representing this object
Returns:
str: string representing this object's name
"""
return self.name
# Returns:
# str: string representing this object's name
# """
# return self.name
def __repr__(self):
return f"ReagentType({self.name})"
return f"<ReagentType({self.name})>"
@classmethod
@setup_lookup
def query(cls,
def query(cls,
name: str|None=None,
kit_type: KitType|str|None=None,
reagent: Reagent|str|None=None,
@@ -211,14 +207,18 @@ class ReagentType(Base):
Lookup reagent types in the database.
Args:
ctx (Settings): Settings object passed down from gui.
name (str | None, optional): Reagent type name. Defaults to None.
kit_type (KitType | str | None, optional): Kit the type of interest belongs to. Defaults to None.
reagent (Reagent | str | None, optional): Concrete instance of the type of interest. Defaults to None.
limit (int, optional): maxmimum number of results to return (0 = all). Defaults to 0.
Raises:
ValueError: Raised if only kit_type or reagent, not both, given.
Returns:
models.ReagentType|List[models.ReagentType]: ReagentType or list of ReagentTypes matching filter.
"""
query: Query = cls.metadata.session.query(cls)
ReagentType|List[ReagentType]: ReagentType or list of ReagentTypes matching filter.
"""
query: Query = cls.__database_session__.query(cls)
if (kit_type != None and reagent == None) or (reagent != None and kit_type == None):
raise ValueError("Cannot filter without both reagent and kit type.")
elif kit_type == None and reagent == None:
@@ -235,9 +235,8 @@ class ReagentType(Base):
case _:
pass
assert reagent.type != []
logger.debug(f"Looking up reagent type for {type(kit_type)} {kit_type} and {type(reagent)} {reagent}")
logger.debug(f"Kit reagent types: {kit_type.reagent_types}")
# logger.debug(f"Reagent reagent types: {reagent._sa_instance_state}")
# logger.debug(f"Looking up reagent type for {type(kit_type)} {kit_type} and {type(reagent)} {reagent}")
# logger.debug(f"Kit reagent types: {kit_type.reagent_types}")
result = list(set(kit_type.reagent_types).intersection(reagent.type))
logger.debug(f"Result: {result}")
try:
@@ -246,34 +245,33 @@ class ReagentType(Base):
return None
match name:
case str():
logger.debug(f"Looking up reagent type by name: {name}")
# logger.debug(f"Looking up reagent type by name: {name}")
query = query.filter(cls.name==name)
limit = 1
case _:
pass
return query_return(query=query, limit=limit)
class KitTypeReagentTypeAssociation(Base):
class KitTypeReagentTypeAssociation(BaseClass):
"""
table containing reagenttype/kittype associations
DOC: https://docs.sqlalchemy.org/en/14/orm/extensions/associationproxy.html
"""
__tablename__ = "_reagenttypes_kittypes"
__table_args__ = {'extend_existing': True}
reagent_types_id = Column(INTEGER, ForeignKey("_reagent_types.id"), primary_key=True)
kits_id = Column(INTEGER, ForeignKey("_kits.id"), primary_key=True)
uses = Column(JSON)
required = Column(INTEGER)
reagent_types_id = Column(INTEGER, ForeignKey("_reagent_types.id"), primary_key=True) #: id of associated reagent type
kits_id = Column(INTEGER, ForeignKey("_kits.id"), primary_key=True) #: id of associated reagent type
uses = Column(JSON) #: map to location on excel sheets of different submission types
required = Column(INTEGER) #: whether the reagent type is required for the kit (Boolean 1 or 0)
last_used = Column(String(32)) #: last used lot number of this type of reagent
kit_type = relationship(KitType, back_populates="kit_reagenttype_associations")
kit_type = relationship(KitType, back_populates="kit_reagenttype_associations") #: relationship to associated kit
# reference to the "ReagentType" object
reagent_type = relationship(ReagentType, back_populates="reagenttype_kit_associations")
reagent_type = relationship(ReagentType, back_populates="reagenttype_kit_associations") #: relationship to associated reagent type
def __init__(self, kit_type=None, reagent_type=None, uses=None, required=1):
logger.debug(f"Parameters: Kit={kit_type}, RT={reagent_type}, Uses={uses}, Required={required}")
# logger.debug(f"Parameters: Kit={kit_type}, RT={reagent_type}, Uses={uses}, Required={required}")
self.kit_type = kit_type
self.reagent_type = reagent_type
self.uses = uses
@@ -284,12 +282,38 @@ class KitTypeReagentTypeAssociation(Base):
@validates('required')
def validate_age(self, key, value):
"""
Ensures only 1 & 0 used in 'required'
Args:
key (str): name of attribute
value (_type_): value of attribute
Raises:
ValueError: Raised if bad value given
Returns:
_type_: value
"""
if not 0 <= value < 2:
raise ValueError(f'Invalid required value {value}. Must be 0 or 1.')
return value
@validates('reagenttype')
def validate_reagenttype(self, key, value):
"""
Ensures reagenttype is an actual ReagentType
Args:
key (str)): name of attribute
value (_type_): value of attribute
Raises:
ValueError: raised if reagenttype is not a ReagentType
Returns:
_type_: ReagentType
"""
if not isinstance(value, ReagentType):
raise ValueError(f'{value} is not a reagenttype')
return value
@@ -297,15 +321,14 @@ class KitTypeReagentTypeAssociation(Base):
@classmethod
@setup_lookup
def query(cls,
kit_type:KitType|str|None,
reagent_type:ReagentType|str|None,
kit_type:KitType|str|None=None,
reagent_type:ReagentType|str|None=None,
limit:int=0
) -> KitTypeReagentTypeAssociation|List[KitTypeReagentTypeAssociation]:
"""
Lookup junction of ReagentType and KitType
Args:
ctx (Settings): Settings object passed down from gui.
kit_type (models.KitType | str | None): KitType of interest.
reagent_type (models.ReagentType | str | None): ReagentType of interest.
limit (int, optional): Maximum number of results to return (0 = all). Defaults to 0.
@@ -313,7 +336,7 @@ class KitTypeReagentTypeAssociation(Base):
Returns:
models.KitTypeReagentTypeAssociation|List[models.KitTypeReagentTypeAssociation]: Junction of interest.
"""
query: Query = cls.metadata.session.query(cls)
query: Query = cls.__database_session__.query(cls)
match kit_type:
case KitType():
query = query.filter(cls.kit_type==kit_type)
@@ -333,17 +356,22 @@ class KitTypeReagentTypeAssociation(Base):
return query_return(query=query, limit=limit)
def save(self) -> Report:
"""
Adds this instance to the database and commits.
Returns:
Report: Result of save action
"""
report = Report()
self.metadata.session.add(self)
self.metadata.session.commit()
self.__database_session__.add(self)
self.__database_session__.commit()
return report
class Reagent(Base):
class Reagent(BaseClass):
"""
Concrete reagent instance
"""
__tablename__ = "_reagents"
__table_args__ = {'extend_existing': True}
id = Column(INTEGER, primary_key=True) #: primary key
type = relationship("ReagentType", back_populates="instances", secondary=reagenttypes_reagents) #: joined parent reagent type
@@ -358,16 +386,7 @@ class Reagent(Base):
return f"<Reagent({self.name}-{self.lot})>"
else:
return f"<Reagent({self.type.name}-{self.lot})>"
def __str__(self) -> str:
"""
string representing this object
Returns:
str: string representing this object's type and lot number
"""
return str(self.lot)
def to_sub_dict(self, extraction_kit:KitType=None) -> dict:
"""
dictionary containing values necessary for gui
@@ -376,7 +395,7 @@ class Reagent(Base):
extraction_kit (KitType, optional): KitType to use to get reagent type. Defaults to None.
Returns:
dict: _description_
dict: representation of the reagent's attributes
"""
if extraction_kit != None:
# Get the intersection of this reagent's ReagentType and all ReagentTypes in KitType
@@ -388,73 +407,59 @@ class Reagent(Base):
else:
reagent_role = self.type[0]
try:
rtype = reagent_role.name.replace("_", " ").title()
rtype = reagent_role.name.replace("_", " ")
except AttributeError:
rtype = "Unknown"
# Calculate expiry with EOL from ReagentType
try:
place_holder = self.expiry + reagent_role.eol_ext
except TypeError as e:
except (TypeError, AttributeError) as e:
place_holder = date.today()
logger.debug(f"We got a type error setting {self.lot} expiry: {e}. setting to today for testing")
except AttributeError as e:
place_holder = date.today()
logger.debug(f"We got an attribute error setting {self.lot} expiry: {e}. Setting to today for testing")
return {
"type": rtype,
"lot": self.lot,
"expiry": place_holder.strftime("%Y-%m-%d")
}
return dict(
name=self.name,
type=rtype,
lot=self.lot,
expiry=place_holder.strftime("%Y-%m-%d")
)
def to_reagent_dict(self, extraction_kit:KitType|str=None) -> dict:
def update_last_used(self, kit:KitType) -> Report:
"""
Returns basic reagent dictionary.
Updates last used reagent lot for ReagentType/KitType
Args:
extraction_kit (KitType, optional): KitType to use to get reagent type. Defaults to None.
kit (KitType): Kit this instance is used in.
Returns:
dict: Basic reagent dictionary of 'type', 'lot', 'expiry'
Report: Result of operation
"""
if extraction_kit != None:
# Get the intersection of this reagent's ReagentType and all ReagentTypes in KitType
try:
reagent_role = list(set(self.type).intersection(extraction_kit.reagent_types))[0]
# Most will be able to fall back to first ReagentType in itself because most will only have 1.
except:
reagent_role = self.type[0]
else:
reagent_role = self.type[0]
try:
rtype = reagent_role.name
except AttributeError:
rtype = "Unknown"
try:
expiry = self.expiry.strftime("%Y-%m-%d")
except:
expiry = date.today()
return {
"name":self.name,
"type": rtype,
"lot": self.lot,
"expiry": self.expiry.strftime("%Y-%m-%d")
}
def save(self):
self.metadata.session.add(self)
self.metadata.session.commit()
report = Report()
logger.debug(f"Attempting update of reagent type at intersection of ({self}), ({kit})")
rt = ReagentType.query(kit_type=kit, reagent=self, limit=1)
if rt != None:
logger.debug(f"got reagenttype {rt}")
assoc = KitTypeReagentTypeAssociation.query(kit_type=kit, reagent_type=rt)
if assoc != None:
if assoc.last_used != self.lot:
logger.debug(f"Updating {assoc} last used to {self.lot}")
assoc.last_used = self.lot
result = assoc.save()
report.add_result(result)
return report
report.add_result(Result(msg=f"Updating last used {rt} was not performed.", status="Information"))
return report
@classmethod
@setup_lookup
def query(cls, reagent_type:str|ReagentType|None=None,
lot_number:str|None=None,
limit:int=0
) -> Reagent|List[Reagent]:
def query(cls,
reagent_type:str|ReagentType|None=None,
lot_number:str|None=None,
limit:int=0
) -> Reagent|List[Reagent]:
"""
Lookup a list of reagents from the database.
Args:
ctx (Settings): Settings object passed down from gui
reagent_type (str | models.ReagentType | None, optional): Reagent type. Defaults to None.
lot_number (str | None, optional): Reagent lot number. Defaults to None.
limit (int, optional): limit of results returned. Defaults to 0.
@@ -462,13 +467,14 @@ class Reagent(Base):
Returns:
models.Reagent | List[models.Reagent]: reagent or list of reagents matching filter.
"""
query: Query = cls.metadata.session.query(cls)
# super().query(session)
query: Query = cls.__database_session__.query(cls)
match reagent_type:
case str():
logger.debug(f"Looking up reagents by reagent type: {reagent_type}")
query = query.join(cls.type, aliased=True).filter(ReagentType.name==reagent_type)
# logger.debug(f"Looking up reagents by reagent type: {reagent_type}")
query = query.join(cls.type).filter(ReagentType.name==reagent_type)
case ReagentType():
logger.debug(f"Looking up reagents by reagent type: {reagent_type}")
# logger.debug(f"Looking up reagents by reagent type: {reagent_type}")
query = query.filter(cls.type.contains(reagent_type))
case _:
pass
@@ -482,42 +488,33 @@ class Reagent(Base):
pass
return query_return(query=query, limit=limit)
def update_last_used(self, kit:KitType):
report = Report()
logger.debug(f"Attempting update of reagent type at intersection of ({self}), ({kit})")
rt = ReagentType.query(kit_type=kit, reagent=self, limit=1)
if rt != None:
logger.debug(f"got reagenttype {rt}")
assoc = KitTypeReagentTypeAssociation.query(kit_type=kit, reagent_type=rt)
if assoc != None:
if assoc.last_used != self.lot:
logger.debug(f"Updating {assoc} last used to {self.lot}")
assoc.last_used = self.lot
result = assoc.save()
return(report.add_result(result))
return report.add_result(Result(msg=f"Updating last used {rt} was not performed.", status="Information"))
class Discount(Base):
def save(self):
"""
Add this instance to the database and commit
"""
self.__database_session__.add(self)
self.__database_session__.commit()
class Discount(BaseClass):
"""
Relationship table for client labs for certain kits.
"""
__tablename__ = "_discounts"
__table_args__ = {'extend_existing': True}
id = Column(INTEGER, primary_key=True) #: primary key
kit = relationship("KitType") #: joined parent reagent type
kit_id = Column(INTEGER, ForeignKey("_kits.id", ondelete='SET NULL', name="fk_kit_type_id"))
kit_id = Column(INTEGER, ForeignKey("_kits.id", ondelete='SET NULL', name="fk_kit_type_id")) #: id of joined kit
client = relationship("Organization") #: joined client lab
client_id = Column(INTEGER, ForeignKey("_organizations.id", ondelete='SET NULL', name="fk_org_id"))
name = Column(String(128))
amount = Column(FLOAT(2))
client_id = Column(INTEGER, ForeignKey("_organizations.id", ondelete='SET NULL', name="fk_org_id")) #: id of joined client
name = Column(String(128)) #: Short description
amount = Column(FLOAT(2)) #: Dollar amount of discount
def __repr__(self) -> str:
return f"<Discount({self.name})>"
@classmethod
@setup_lookup
def query(cls,
def query(cls,
organization:Organization|str|int|None=None,
kit_type:KitType|str|int|None=None,
) -> Discount|List[Discount]:
@@ -525,7 +522,6 @@ class Discount(Base):
Lookup discount objects (union of kit and organization)
Args:
ctx (Settings): Settings object passed down from the gui.
organization (models.Organization | str | int): Organization receiving discount.
kit_type (models.KitType | str | int): Kit discount received on.
@@ -536,60 +532,68 @@ class Discount(Base):
Returns:
models.Discount|List[models.Discount]: Discount(s) of interest.
"""
query: Query = cls.metadata.session.query(cls)
query: Query = cls.__database_session__.query(cls)
match organization:
case Organization():
logger.debug(f"Looking up discount with organization: {organization}")
# logger.debug(f"Looking up discount with organization: {organization}")
query = query.filter(cls.client==Organization)
case str():
logger.debug(f"Looking up discount with organization: {organization}")
# logger.debug(f"Looking up discount with organization: {organization}")
query = query.join(Organization).filter(Organization.name==organization)
case int():
logger.debug(f"Looking up discount with organization id: {organization}")
# logger.debug(f"Looking up discount with organization id: {organization}")
query = query.join(Organization).filter(Organization.id==organization)
case _:
# raise ValueError(f"Invalid value for organization: {organization}")
pass
match kit_type:
case KitType():
logger.debug(f"Looking up discount with kit type: {kit_type}")
# logger.debug(f"Looking up discount with kit type: {kit_type}")
query = query.filter(cls.kit==kit_type)
case str():
logger.debug(f"Looking up discount with kit type: {kit_type}")
# logger.debug(f"Looking up discount with kit type: {kit_type}")
query = query.join(KitType).filter(KitType.name==kit_type)
case int():
logger.debug(f"Looking up discount with kit type id: {organization}")
# logger.debug(f"Looking up discount with kit type id: {organization}")
query = query.join(KitType).filter(KitType.id==kit_type)
case _:
# raise ValueError(f"Invalid value for kit type: {kit_type}")
pass
return query.all()
class SubmissionType(Base):
class SubmissionType(BaseClass):
"""
Abstract of types of submissions.
"""
__tablename__ = "_submission_types"
__table_args__ = {'extend_existing': True}
id = Column(INTEGER, primary_key=True) #: primary key
name = Column(String(128), unique=True) #: name of submission type
info_map = Column(JSON) #: Where basic information is found in the excel workbook corresponding to this type.
instances = relationship("BasicSubmission", backref="submission_type")
instances = relationship("BasicSubmission", backref="submission_type") #: Concrete instances of this type.
# regex = Column(String(512))
# template_file = Column(BLOB)
template_file = Column(BLOB) #: Blank form for this type stored as binary.
submissiontype_kit_associations = relationship(
"SubmissionTypeKitTypeAssociation",
back_populates="submission_type",
cascade="all, delete-orphan",
)
) #: Association of kittypes
kit_types = association_proxy("submissiontype_kit_associations", "kit_type")
kit_types = association_proxy("submissiontype_kit_associations", "kit_type") #: Proxy of kittype association
def __repr__(self) -> str:
return f"<SubmissionType({self.name})>"
def get_template_file_sheets(self) -> List[str]:
"""
Gets names of sheet in the stored blank form.
Returns:
List[str]: List of sheet names
"""
return ExcelFile(self.template_file).sheet_names
@classmethod
@setup_lookup
def query(cls,
@@ -603,15 +607,16 @@ class SubmissionType(Base):
Args:
ctx (Settings): Settings object passed down from gui
name (str | None, optional): Name of submission type. Defaults to None.
key (str | None, optional): A key present in the info-map to lookup. Defaults to None.
limit (int, optional): Maximum number of results to return (0 = all). Defaults to 0.
Returns:
models.SubmissionType|List[models.SubmissionType]: SubmissionType(s) of interest.
"""
query: Query = cls.metadata.session.query(cls)
query: Query = cls.__database_session__.query(cls)
match name:
case str():
logger.debug(f"Looking up submission type by name: {name}")
# logger.debug(f"Looking up submission type by name: {name}")
query = query.filter(cls.name==name)
limit = 1
case _:
@@ -624,27 +629,28 @@ class SubmissionType(Base):
return query_return(query=query, limit=limit)
def save(self):
self.metadata.session.add(self)
self.metadata.session.commit()
return None
"""
Adds this instances to the database and commits.
"""
self.__database_session__.add(self)
self.__database_session__.commit()
class SubmissionTypeKitTypeAssociation(Base):
class SubmissionTypeKitTypeAssociation(BaseClass):
"""
Abstract of relationship between kits and their submission type.
"""
__tablename__ = "_submissiontypes_kittypes"
__table_args__ = {'extend_existing': True}
submission_types_id = Column(INTEGER, ForeignKey("_submission_types.id"), primary_key=True)
kits_id = Column(INTEGER, ForeignKey("_kits.id"), primary_key=True)
submission_types_id = Column(INTEGER, ForeignKey("_submission_types.id"), primary_key=True) #: id of joined submission type
kits_id = Column(INTEGER, ForeignKey("_kits.id"), primary_key=True) #: id of joined kit
mutable_cost_column = Column(FLOAT(2)) #: dollar amount per 96 well plate that can change with number of columns (reagents, tips, etc)
mutable_cost_sample = Column(FLOAT(2)) #: dollar amount that can change with number of samples (reagents, tips, etc)
constant_cost = Column(FLOAT(2)) #: dollar amount per plate that will remain constant (plates, man hours, etc)
kit_type = relationship(KitType, back_populates="kit_submissiontype_associations")
kit_type = relationship(KitType, back_populates="kit_submissiontype_associations") #: joined kittype
# reference to the "SubmissionType" object
submission_type = relationship(SubmissionType, back_populates="submissiontype_kit_associations")
submission_type = relationship(SubmissionType, back_populates="submissiontype_kit_associations") #: joined submission type
def __init__(self, kit_type=None, submission_type=None):
self.kit_type = kit_type
@@ -661,32 +667,42 @@ class SubmissionTypeKitTypeAssociation(Base):
@classmethod
@setup_lookup
def query(cls,
submission_type:SubmissionType|str|int|None=None,
def query(cls,
submission_type:SubmissionType|str|int|None=None,
kit_type:KitType|str|int|None=None,
limit:int=0
):
query: Query = cls.metadata.session.query(cls)
) -> SubmissionTypeKitTypeAssociation|List[SubmissionTypeKitTypeAssociation]:
"""
Lookup SubmissionTypeKitTypeAssociations of interest.
Args:
submission_type (SubmissionType | str | int | None, optional): Identifier of submission type. Defaults to None.
kit_type (KitType | str | int | None, optional): Identifier of kit type. Defaults to None.
limit (int, optional): Maximum number of results to return (0 = all). Defaults to 0.
Returns:
SubmissionTypeKitTypeAssociation|List[SubmissionTypeKitTypeAssociation]: SubmissionTypeKitTypeAssociation(s) of interest
"""
query: Query = cls.__database_session__.query(cls)
match submission_type:
case SubmissionType():
logger.debug(f"Looking up {cls.__name__} by SubmissionType {submission_type}")
# logger.debug(f"Looking up {cls.__name__} by SubmissionType {submission_type}")
query = query.filter(cls.submission_type==submission_type)
case str():
logger.debug(f"Looking up {cls.__name__} by name {submission_type}")
# logger.debug(f"Looking up {cls.__name__} by name {submission_type}")
query = query.join(SubmissionType).filter(SubmissionType.name==submission_type)
case int():
logger.debug(f"Looking up {cls.__name__} by id {submission_type}")
# logger.debug(f"Looking up {cls.__name__} by id {submission_type}")
query = query.join(SubmissionType).filter(SubmissionType.id==submission_type)
match kit_type:
case KitType():
logger.debug(f"Looking up {cls.__name__} by KitType {kit_type}")
# logger.debug(f"Looking up {cls.__name__} by KitType {kit_type}")
query = query.filter(cls.kit_type==kit_type)
case str():
logger.debug(f"Looking up {cls.__name__} by name {kit_type}")
# logger.debug(f"Looking up {cls.__name__} by name {kit_type}")
query = query.join(KitType).filter(KitType.name==kit_type)
case int():
logger.debug(f"Looking up {cls.__name__} by id {kit_type}")
# logger.debug(f"Looking up {cls.__name__} by id {kit_type}")
query = query.join(KitType).filter(KitType.id==kit_type)
limit = query.count()
return query_return(query=query, limit=limit)

62
src/submissions/backend/db/models/organizations.py
View File

@@ -4,8 +4,8 @@ All client organization related models.
from __future__ import annotations
from sqlalchemy import Column, String, INTEGER, ForeignKey, Table
from sqlalchemy.orm import relationship, Query
from . import Base
from tools import check_authorization, setup_lookup, query_return
from . import Base, BaseClass
from tools import check_authorization, setup_lookup, query_return, Settings
from typing import List
import logging
@@ -21,12 +21,11 @@ orgs_contacts = Table(
extend_existing = True
)
class Organization(Base):
class Organization(BaseClass):
"""
Base of organization
"""
__tablename__ = "_organizations"
__table_args__ = {'extend_existing': True}
id = Column(INTEGER, primary_key=True) #: primary key
name = Column(String(64)) #: organization name
@@ -34,29 +33,15 @@ class Organization(Base):
cost_centre = Column(String()) #: cost centre used by org for payment
contacts = relationship("Contact", back_populates="organization", secondary=orgs_contacts) #: contacts involved with this org
def __str__(self) -> str:
"""
String representing organization
Returns:
str: string representing organization name
"""
return self.name.replace("_", " ").title()
def __repr__(self) -> str:
return f"<Organization({self.name})>"
@check_authorization
def save(self, ctx):
ctx.database_session.add(self)
ctx.database_session.commit()
def set_attribute(self, name:str, value):
setattr(self, name, value)
@classmethod
@setup_lookup
def query(cls,
def query(cls,
name:str|None=None,
limit:int=0,
) -> Organization|List[Organization]:
@@ -68,24 +53,34 @@ class Organization(Base):
limit (int, optional): Maximum number of results to return (0 = all). Defaults to 0.
Returns:
Organization|List[Organization]: _description_
Organization|List[Organization]:
"""
query: Query = cls.metadata.session.query(cls)
query: Query = cls.__database_session__.query(cls)
match name:
case str():
logger.debug(f"Looking up organization with name: {name}")
# logger.debug(f"Looking up organization with name: {name}")
query = query.filter(cls.name==name)
limit = 1
case _:
pass
return query_return(query=query, limit=limit)
@check_authorization
def save(self, ctx:Settings):
"""
Adds this instance to the database and commits
class Contact(Base):
Args:
ctx (Settings): Settings object passed down from GUI. Necessary to check authorization
"""
ctx.database_session.add(self)
ctx.database_session.commit()
class Contact(BaseClass):
"""
Base of Contact
"""
__tablename__ = "_contacts"
__table_args__ = {'extend_existing': True}
id = Column(INTEGER, primary_key=True) #: primary key
name = Column(String(64)) #: contact name
@@ -98,7 +93,7 @@ class Contact(Base):
@classmethod
@setup_lookup
def query(cls,
def query(cls,
name:str|None=None,
email:str|None=None,
phone:str|None=None,
@@ -109,32 +104,35 @@ class Contact(Base):
Args:
name (str | None, optional): Name of the contact. Defaults to None.
email (str | None, optional): Email of the contact. Defaults to None.
phone (str | None, optional): Phone number of the contact. Defaults to None.
limit (int, optional): Maximum number of results to return (0 = all). Defaults to 0.
Returns:
Contact|List[Contact]: _description_
"""
query: Query = cls.metadata.session.query(cls)
Contact|List[Contact]: Contact(s) of interest.
"""
# super().query(session)
query: Query = cls.__database_session__.query(cls)
match name:
case str():
logger.debug(f"Looking up contact with name: {name}")
# logger.debug(f"Looking up contact with name: {name}")
query = query.filter(cls.name==name)
limit = 1
case _:
pass
match email:
case str():
logger.debug(f"Looking up contact with email: {name}")
# logger.debug(f"Looking up contact with email: {name}")
query = query.filter(cls.email==email)
limit = 1
case _:
pass
match phone:
case str():
logger.debug(f"Looking up contact with phone: {name}")
# logger.debug(f"Looking up contact with phone: {name}")
query = query.filter(cls.phone==phone)
limit = 1
case _:
pass
return query_return(query=query, limit=limit)

730
src/submissions/backend/db/models/submissions.py
View File

File diff suppressed because it is too large Load Diff

100
src/submissions/backend/excel/parser.py
View File

@@ -16,7 +16,6 @@ from datetime import date
from dateutil.parser import parse, ParserError
from tools import check_not_nan, convert_nans_to_nones, Settings
logger = logging.getLogger(f"submissions.{__name__}")
row_keys = dict(A=1, B=2, C=3, D=4, E=5, F=6, G=7, H=8)
@@ -28,7 +27,7 @@ class SheetParser(object):
def __init__(self, ctx:Settings, filepath:Path|None = None):
"""
Args:
ctx (Settings): Settings object passed down from gui
ctx (Settings): Settings object passed down from gui. Necessary for Bacterial to get directory path.
filepath (Path | None, optional): file path to excel sheet. Defaults to None.
"""
self.ctx = ctx
@@ -56,6 +55,7 @@ class SheetParser(object):
self.import_reagent_validation_check()
self.parse_samples()
self.finalize_parse()
logger.debug(f"Parser.sub after info scrape: {pformat(self.sub)}")
def parse_info(self):
"""
@@ -70,15 +70,17 @@ class SheetParser(object):
pass
case _:
self.sub[k] = v
logger.debug(f"Parser.sub after info scrape: {pformat(self.sub)}")
def parse_reagents(self, extraction_kit:str|None=None):
"""
Pulls reagent info from the excel sheet
Args:
extraction_kit (str | None, optional): Relevant extraction kit for reagent map. Defaults to None.
"""
if extraction_kit == None:
extraction_kit = extraction_kit=self.sub['extraction_kit']
logger.debug(f"Parsing reagents for {extraction_kit}")
# logger.debug(f"Parsing reagents for {extraction_kit}")
self.sub['reagents'] = ReagentParser(xl=self.xl, submission_type=self.sub['submission_type'], extraction_kit=extraction_kit).parse_reagents()
def parse_samples(self):
@@ -92,13 +94,6 @@ class SheetParser(object):
def import_kit_validation_check(self):
"""
Enforce that the parser has an extraction kit
Args:
ctx (Settings): Settings obj passed down from gui
parser_sub (dict): The parser dictionary before going to pydantic
Returns:
List[PydReagent]: List of reagents
"""
from frontend.widgets.pop_ups import KitSelector
if not check_not_nan(self.sub['extraction_kit']['value']):
@@ -115,18 +110,18 @@ class SheetParser(object):
"""
Enforce that only allowed reagents get into the Pydantic Model
"""
# kit = lookup_kit_types(ctx=self.ctx, name=self.sub['extraction_kit']['value'])
kit = KitType.query(name=self.sub['extraction_kit']['value'])
allowed_reagents = [item.name for item in kit.get_reagents()]
logger.debug(f"List of reagents for comparison with allowed_reagents: {pformat(self.sub['reagents'])}")
# self.sub['reagents'] = [reagent for reagent in self.sub['reagents'] if reagent['value'].type in allowed_reagents]
# logger.debug(f"List of reagents for comparison with allowed_reagents: {pformat(self.sub['reagents'])}")
self.sub['reagents'] = [reagent for reagent in self.sub['reagents'] if reagent.type in allowed_reagents]
def finalize_parse(self):
"""
Run custom final validations of data for submission subclasses.
"""
finisher = BasicSubmission.find_polymorphic_subclass(polymorphic_identity=self.sub['submission_type']).finalize_parse
self.sub = finisher(input_dict=self.sub, xl=self.xl, info_map=self.info_map, plate_map=self.plate_map)
def to_pydantic(self) -> PydSubmission:
"""
Generates a pydantic model of scraped data for validation
@@ -134,21 +129,19 @@ class SheetParser(object):
Returns:
PydSubmission: output pydantic model
"""
logger.debug(f"Submission dictionary coming into 'to_pydantic':\n{pformat(self.sub)}")
# logger.debug(f"Submission dictionary coming into 'to_pydantic':\n{pformat(self.sub)}")
psm = PydSubmission(filepath=self.filepath, **self.sub)
# delattr(psm, "filepath")
return psm
class InfoParser(object):
def __init__(self, xl:pd.ExcelFile, submission_type:str):
logger.debug(f"\n\nHello from InfoParser!")
logger.info(f"\n\Hello from InfoParser!\n\n")
# self.ctx = ctx
self.map = self.fetch_submission_info_map(submission_type=submission_type)
self.xl = xl
logger.debug(f"Info map for InfoParser: {pformat(self.map)}")
def fetch_submission_info_map(self, submission_type:str|dict) -> dict:
"""
Gets location of basic info from the submission_type object in the database.
@@ -192,6 +185,11 @@ class InfoParser(object):
continue
for item in relevant:
value = df.iat[relevant[item]['row']-1, relevant[item]['column']-1]
match item:
case "submission_type":
value = value.title()
case _:
pass
logger.debug(f"Setting {item} on {sheet} to {value}")
if check_not_nan(value):
if value != "None":
@@ -206,10 +204,6 @@ class InfoParser(object):
continue
else:
dicto[item] = dict(value=convert_nans_to_nones(value), missing=True)
try:
check = dicto['submission_category'] not in ["", None]
except KeyError:
check = False
return self.custom_parser(input_dict=dicto, xl=self.xl)
class ReagentParser(object):
@@ -220,7 +214,17 @@ class ReagentParser(object):
self.map = self.fetch_kit_info_map(extraction_kit=extraction_kit, submission_type=submission_type)
self.xl = xl
def fetch_kit_info_map(self, extraction_kit:dict, submission_type:str):
def fetch_kit_info_map(self, extraction_kit:dict, submission_type:str) -> dict:
"""
Gets location of kit reagents from database
Args:
extraction_kit (dict): Relevant kit information.
submission_type (str): Name of submission type.
Returns:
dict: locations of reagent info for the kit.
"""
if isinstance(extraction_kit, dict):
extraction_kit = extraction_kit['value']
# kit = lookup_kit_types(ctx=self.ctx, name=extraction_kit)
@@ -231,7 +235,13 @@ class ReagentParser(object):
del reagent_map['info']
return reagent_map
def parse_reagents(self) -> list:
def parse_reagents(self) -> List[PydReagent]:
"""
Extracts reagent information from the excel form.
Returns:
List[PydReagent]: List of parsed reagents.
"""
listo = []
for sheet in self.xl.sheet_names:
df = self.xl.parse(sheet, header=None, dtype=object)
@@ -271,11 +281,10 @@ class SampleParser(object):
convert sample sub-dataframe to dictionary of records
Args:
ctx (Settings): settings object passed down from gui
df (pd.DataFrame): input sample dataframe
elution_map (pd.DataFrame | None, optional): optional map of elution plate. Defaults to None.
"""
logger.debug("\n\nHello from SampleParser!")
logger.debug("\n\nHello from SampleParser!\n\n")
self.samples = []
# self.ctx = ctx
self.xl = xl
@@ -454,40 +463,6 @@ class SampleParser(object):
new_samples.append(PydSample(**translated_dict))
return result, new_samples
# def generate_sample_object(self, input_dict) -> BasicSample:
# """
# Constructs sample object from dict.
# NOTE: Depreciated due to using Pydantic object up until db saving.
# Args:
# input_dict (dict): sample information
# Returns:
# models.BasicSample: Sample object
# """
# database_obj = BasicSample.find_polymorphic_subclass(polymorphic_identity=input_dict['sample_type'])
# # query = input_dict['sample_type'].replace(" ", "")
# # try:
# # # database_obj = getattr(models, query)
# # except AttributeError as e:
# # logger.error(f"Could not find the model {query}. Using generic.")
# # database_obj = models.BasicSample
# logger.debug(f"Searching database for {input_dict['submitter_id']}...")
# # instance = lookup_samples(ctx=self.ctx, submitter_id=str(input_dict['submitter_id']))
# instance = BasicSample.query(submitter_id=str(input_dict['submitter_id']))
# if instance == None:
# logger.debug(f"Couldn't find sample {input_dict['submitter_id']}. Creating new sample.")
# instance = database_obj()
# for k,v in input_dict.items():
# try:
# instance.set_attribute(k, v)
# except Exception as e:
# logger.error(f"Failed to set {k} due to {type(e).__name__}: {e}")
# else:
# logger.debug(f"Sample {instance.submitter_id} already exists, will run update.")
# return dict(sample=instance, row=input_dict['row'], column=input_dict['column'])
def grab_plates(self) -> List[str]:
"""
Parse plate names from
@@ -514,7 +489,6 @@ class PCRParser(object):
Initializes object.
Args:
ctx (dict): settings passed down from gui.
filepath (Path | None, optional): file to parse. Defaults to None.
"""
# self.ctx = ctx

25
src/submissions/backend/excel/reports.py
View File

@@ -5,7 +5,7 @@ from pandas import DataFrame
import logging
from datetime import date, timedelta
import re
from typing import Tuple
from typing import List, Tuple
from tools import jinja_template_loading, Settings
logger = logging.getLogger(f"submissions.{__name__}")
@@ -27,7 +27,7 @@ def make_report_xlsx(records:list[dict]) -> Tuple[DataFrame, DataFrame]:
df = df.sort_values("Submitting Lab")
# aggregate cost and sample count columns
df2 = df.groupby(["Submitting Lab", "Extraction Kit"]).agg({'Extraction Kit':'count', 'Cost': 'sum', 'Sample Count':'sum'})
df2 = df2.rename(columns={"Extraction Kit": 'Plate Count'})
df2 = df2.rename(columns={"Extraction Kit": 'Run Count'})
logger.debug(f"Output daftaframe for xlsx: {df2.columns}")
df = df.drop('id', axis=1)
df = df.sort_values(['Submitting Lab', "Submitted Date"])
@@ -57,16 +57,16 @@ def make_report_html(df:DataFrame, start_date:date, end_date:date) -> str:
logger.debug(f"Old lab: {old_lab}, Current lab: {lab}")
logger.debug(f"Name: {row[0][1]}")
data = [item for item in row[1]]
kit = dict(name=row[0][1], cost=data[1], plate_count=int(data[0]), sample_count=int(data[2]))
kit = dict(name=row[0][1], cost=data[1], run_count=int(data[0]), sample_count=int(data[2]))
# if this is the same lab as before add together
if lab == old_lab:
output[-1]['kits'].append(kit)
output[-1]['total_cost'] += kit['cost']
output[-1]['total_samples'] += kit['sample_count']
output[-1]['total_plates'] += kit['plate_count']
output[-1]['total_runs'] += kit['run_count']
# if not the same lab, make a new one
else:
adder = dict(lab=lab, kits=[kit], total_cost=kit['cost'], total_samples=kit['sample_count'], total_plates=kit['plate_count'])
adder = dict(lab=lab, kits=[kit], total_cost=kit['cost'], total_samples=kit['sample_count'], total_runs=kit['run_count'])
output.append(adder)
old_lab = lab
logger.debug(output)
@@ -83,10 +83,10 @@ def convert_data_list_to_df(input:list[dict], subtype:str|None=None) -> DataFram
Args:
ctx (dict): settings passed from gui
input (list[dict]): list of dictionaries containing records
subtype (str | None, optional): _description_. Defaults to None.
subtype (str | None, optional): name of submission type. Defaults to None.
Returns:
DataFrame: _description_
DataFrame: dataframe of controls
"""
df = DataFrame.from_records(input)
@@ -218,5 +218,14 @@ def drop_reruns_from_df(ctx:Settings, df: DataFrame) -> DataFrame:
df = df.drop(df[df.name == first_run].index)
return df
def make_hitpicks(input:list) -> DataFrame:
def make_hitpicks(input:List[dict]) -> DataFrame:
"""
Converts lsit of dictionaries constructed by hitpicking to dataframe
Args:
input (List[dict]): list of hitpicked dictionaries
Returns:
DataFrame: constructed dataframe.
"""
return DataFrame.from_records(input)

13
src/submissions/backend/validators/__init__.py
View File

@@ -12,7 +12,6 @@ class RSLNamer(object):
"""
def __init__(self, instr:str, sub_type:str|None=None, data:dict|None=None):
self.submission_type = sub_type
if self.submission_type == None:
self.submission_type = self.retrieve_submission_type(instr=instr)
logger.debug(f"got submission type: {self.submission_type}")
@@ -23,6 +22,15 @@ class RSLNamer(object):
@classmethod
def retrieve_submission_type(cls, instr:str|Path) -> str:
"""
Gets submission type from excel file properties or sheet names or regex pattern match or user input
Args:
instr (str | Path): filename
Returns:
str: parsed submission type
"""
match instr:
case Path():
logger.debug(f"Using path method for {instr}.")
@@ -32,7 +40,8 @@ class RSLNamer(object):
submission_type = [item.strip().title() for item in wb.properties.category.split(";")][0]
except AttributeError:
try:
sts = {item.name:item.info_map['all_sheets'] for item in SubmissionType.query(key="all_sheets")}
# sts = {item.name:item.info_map['all_sheets'] for item in SubmissionType.query(key="all_sheets")}
sts = {item.name:item.get_template_file_sheets() for item in SubmissionType.query()}
for k,v in sts.items():
# This gets the *first* submission type that matches the sheet names in the workbook
if wb.sheetnames == v:

226
src/submissions/backend/validators/pydant.py
View File

@@ -1,22 +1,24 @@
'''
Contains pydantic models and accompanying validators
'''
from operator import attrgetter
import uuid
from pydantic import BaseModel, field_validator, Field
from datetime import date, datetime, timedelta
from dateutil.parser import parse
from dateutil.parser._parser import ParserError
from typing import List, Any, Tuple, Literal
from typing import List, Any, Tuple
from . import RSLNamer
from pathlib import Path
import re
import logging
from tools import check_not_nan, convert_nans_to_nones, jinja_template_loading, Report, Result
from tools import check_not_nan, convert_nans_to_nones, jinja_template_loading, Report, Result, row_map
from backend.db.models import *
from sqlalchemy.exc import StatementError, IntegrityError
from PyQt6.QtWidgets import QComboBox, QWidget
from pprint import pformat
from openpyxl import load_workbook
# from pprint import pformat
from openpyxl import load_workbook, Workbook
from io import BytesIO
logger = logging.getLogger(f"submissions.{__name__}")
@@ -87,9 +89,14 @@ class PydReagent(BaseModel):
return values.data['type']
def toSQL(self) -> Tuple[Reagent, Report]:
"""
Converts this instance into a backend.db.models.kit.Reagent instance
Returns:
Tuple[Reagent, Report]: Reagent instance and result of function
"""
report = Report()
logger.debug(f"Reagent SQL constructor is looking up type: {self.type}, lot: {self.lot}")
# reagent = lookup_reagents(ctx=self.ctx, lot_number=self.lot)
reagent = Reagent.query(lot_number=self.lot)
logger.debug(f"Result: {reagent}")
if reagent == None:
@@ -105,7 +112,6 @@ class PydReagent(BaseModel):
case "expiry":
reagent.expiry = value
case "type":
# reagent_type = lookup_reagent_types(ctx=self.ctx, name=value)
reagent_type = ReagentType.query(name=value)
if reagent_type != None:
reagent.type.append(reagent_type)
@@ -116,6 +122,16 @@ class PydReagent(BaseModel):
return reagent, report
def toForm(self, parent:QWidget, extraction_kit:str) -> QComboBox:
"""
Converts this instance into a form widget
Args:
parent (QWidget): Parent widget of the constructed object
extraction_kit (str): Name of extraction kit used
Returns:
QComboBox: Form object.
"""
from frontend.widgets.submission_widget import ReagentFormWidget
return ReagentFormWidget(parent=parent, reagent=self, extraction_kit=extraction_kit)
@@ -138,16 +154,19 @@ class PydSample(BaseModel, extra='allow'):
def int_to_str(cls, value):
return str(value)
def toSQL(self, submission=None):
result = None
def toSQL(self, submission:BasicSubmission|str=None) -> Tuple[BasicSample, Result]:
"""
Converts this instance into a backend.db.models.submissions.Sample object
Args:
submission (BasicSubmission | str, optional): Submission joined to this sample. Defaults to None.
Returns:
Tuple[BasicSample, Result]: Sample object and result object.
"""
report = None
self.__dict__.update(self.model_extra)
logger.debug(f"Here is the incoming sample dict: \n{self.__dict__}")
# instance = lookup_samples(ctx=ctx, submitter_id=self.submitter_id)
# instance = BasicSample.query(submitter_id=self.submitter_id)
# if instance == None:
# logger.debug(f"Sample {self.submitter_id} doesn't exist yet. Looking up sample object with polymorphic identity: {self.sample_type}")
# instance = BasicSample.find_polymorphic_subclass(polymorphic_identity=self.sample_type)()
# instance = BasicSample.query_or_create(**{k:v for k,v in self.__dict__.items() if k not in ['row', 'column']})
instance = BasicSample.query_or_create(sample_type=self.sample_type, submitter_id=self.submitter_id)
for key, value in self.__dict__.items():
# logger.debug(f"Setting sample field {key} to {value}")
@@ -161,13 +180,6 @@ class PydSample(BaseModel, extra='allow'):
for row, column in zip(self.row, self.column):
# logger.debug(f"Looking up association with identity: ({submission.submission_type_name} Association)")
logger.debug(f"Looking up association with identity: ({assoc_type} Association)")
# association = lookup_submission_sample_association(ctx=ctx, submission=submission, row=row, column=column)
# association = SubmissionSampleAssociation.query(submission=submission, row=row, column=column)
# logger.debug(f"Returned association: {association}")
# if association == None or association == []:
# logger.debug(f"Looked up association at row {row}, column {column} didn't exist, creating new association.")
# association = SubmissionSampleAssociation.find_polymorphic_subclass(polymorphic_identity=f"{submission.submission_type_name} Association")
# association = association(submission=submission, sample=instance, row=row, column=column)
association = SubmissionSampleAssociation.query_or_create(association_type=f"{assoc_type} Association",
submission=submission,
sample=instance,
@@ -176,7 +188,7 @@ class PydSample(BaseModel, extra='allow'):
instance.sample_submission_associations.append(association)
except IntegrityError:
instance.metadata.session.rollback()
return instance, result
return instance, report
class PydSubmission(BaseModel, extra='allow'):
filepath: Path
@@ -185,7 +197,7 @@ class PydSubmission(BaseModel, extra='allow'):
submitter_plate_num: dict|None = Field(default=dict(value=None, missing=True), validate_default=True)
submitted_date: dict|None
rsl_plate_num: dict|None = Field(default=dict(value=None, missing=True), validate_default=True)
# submitted_date: dict|None
submitted_date: dict|None
submitting_lab: dict|None
sample_count: dict|None
extraction_kit: dict|None
@@ -197,7 +209,7 @@ class PydSubmission(BaseModel, extra='allow'):
@field_validator("submitter_plate_num")
@classmethod
def enforce_with_uuid(cls, value):
logger.debug(f"submitter plate id: {value}")
# logger.debug(f"submitter_plate_num coming into pydantic: {value}")
if value['value'] == None or value['value'] == "None":
return dict(value=uuid.uuid4().hex.upper(), missing=True)
else:
@@ -250,14 +262,6 @@ class PydSubmission(BaseModel, extra='allow'):
logger.debug(f"RSL-plate initial value: {value['value']} and other values: {values.data}")
sub_type = values.data['submission_type']['value']
if check_not_nan(value['value']):
# if lookup_submissions(ctx=values.data['ctx'], rsl_number=value['value']) == None:
# if BasicSubmission.query(rsl_number=value['value']) == None:
# return dict(value=value['value'], missing=False)
# else:
# logger.warning(f"Submission number {value} already exists in DB, attempting salvage with filepath")
# # output = RSLNamer(ctx=values.data['ctx'], instr=values.data['filepath'].__str__(), sub_type=sub_type).parsed_name
# output = RSLNamer(instr=values.data['filepath'].__str__(), sub_type=sub_type).parsed_name
# return dict(value=output, missing=True)
return value
else:
output = RSLNamer(instr=values.data['filepath'].__str__(), sub_type=sub_type, data=values.data).parsed_name
@@ -278,7 +282,6 @@ class PydSubmission(BaseModel, extra='allow'):
return value
else:
return dict(value=convert_nans_to_nones(value['value']), missing=True)
return value
@field_validator("sample_count", mode='before')
@classmethod
@@ -290,7 +293,6 @@ class PydSubmission(BaseModel, extra='allow'):
@field_validator("extraction_kit", mode='before')
@classmethod
def rescue_kit(cls, value):
if check_not_nan(value):
if isinstance(value, str):
return dict(value=value, missing=False)
@@ -305,6 +307,7 @@ class PydSubmission(BaseModel, extra='allow'):
@field_validator("submission_type", mode='before')
@classmethod
def make_submission_type(cls, value, values):
logger.debug(f"Submission type coming into pydantic: {value}")
if not isinstance(value, dict):
value = {"value": value}
if check_not_nan(value['value']):
@@ -313,6 +316,12 @@ class PydSubmission(BaseModel, extra='allow'):
else:
return dict(value=RSLNamer(instr=values.data['filepath'].__str__()).submission_type.title(), missing=True)
@field_validator("submission_category", mode="before")
def create_category(cls, value):
if not isinstance(value, dict):
return dict(value=value, missing=True)
return value
@field_validator("submission_category")
@classmethod
def rescue_category(cls, value, values):
@@ -321,6 +330,10 @@ class PydSubmission(BaseModel, extra='allow'):
return value
def handle_duplicate_samples(self):
"""
Collapses multiple samples with same submitter id into one with lists for rows, columns
TODO: Find out if this is really necessary
"""
submitter_ids = list(set([sample.submitter_id for sample in self.samples]))
output = []
for id in submitter_ids:
@@ -336,7 +349,16 @@ class PydSubmission(BaseModel, extra='allow'):
output.append(dummy)
self.samples = output
def improved_dict(self, dictionaries:bool=True):
def improved_dict(self, dictionaries:bool=True) -> dict:
"""
Adds model_extra to fields.
Args:
dictionaries (bool, optional): Are dictionaries expected as input? i.e. Should key['value'] be retrieved. Defaults to True.
Returns:
dict: This instance as a dictionary
"""
fields = list(self.model_fields.keys()) + list(self.model_extra.keys())
if dictionaries:
output = {k:getattr(self, k) for k in fields}
@@ -344,14 +366,25 @@ class PydSubmission(BaseModel, extra='allow'):
output = {k:(getattr(self, k) if not isinstance(getattr(self, k), dict) else getattr(self, k)['value']) for k in fields}
return output
def find_missing(self):
def find_missing(self) -> Tuple[dict, dict]:
"""
Retrieves info and reagents marked as missing.
Returns:
Tuple[dict, dict]: Dict for missing info, dict for missing reagents.
"""
info = {k:v for k,v in self.improved_dict().items() if isinstance(v, dict)}
missing_info = {k:v for k,v in info.items() if v['missing']}
missing_reagents = [reagent for reagent in self.reagents if reagent.missing]
return missing_info, missing_reagents
def toSQL(self) -> Tuple[BasicSubmission, Result]:
"""
Converts this instance into a backend.db.models.submissions.BasicSubmission instance
Returns:
Tuple[BasicSubmission, Result]: BasicSubmission instance, result object
"""
self.__dict__.update(self.model_extra)
instance, code, msg = BasicSubmission.query_or_create(submission_type=self.submission_type['value'], rsl_plate_num=self.rsl_plate_num['value'])
result = Result(msg=msg, code=code)
@@ -395,10 +428,42 @@ class PydSubmission(BaseModel, extra='allow'):
return instance, result
def toForm(self, parent:QWidget):
"""
Converts this instance into a frontend.widgets.submission_widget.SubmissionFormWidget
Args:
parent (QWidget): parent widget of the constructed object
Returns:
SubmissionFormWidget: Submission form widget
"""
from frontend.widgets.submission_widget import SubmissionFormWidget
return SubmissionFormWidget(parent=parent, **self.improved_dict())
def autofill_excel(self, missing_only:bool=True):
def autofill_excel(self, missing_only:bool=True, backup:bool=False) -> Workbook:
"""
Fills in relevant information/reagent cells in an excel workbook.
Args:
missing_only (bool, optional): Only fill missing items or all. Defaults to True.
backup (bool, optional): Do a full backup of the submission (adds samples). Defaults to False.
Returns:
Workbook: Filled in workbook
"""
# open a new workbook using openpyxl
if self.filepath.stem.startswith("tmp"):
template = SubmissionType.query(name=self.submission_type['value']).template_file
workbook = load_workbook(BytesIO(template))
missing_only = False
else:
try:
workbook = load_workbook(self.filepath)
except Exception as e:
logger.error(f"Couldn't open workbook due to {e}")
template = SubmissionType.query(name=self.submission_type).template_file
workbook = load_workbook(BytesIO(template))
missing_only = False
if missing_only:
info, reagents = self.find_missing()
else:
@@ -442,8 +507,6 @@ class PydSubmission(BaseModel, extra='allow'):
logger.error(f"Unable to fill in {k}, not found in relevant info.")
logger.debug(f"New reagents: {new_reagents}")
logger.debug(f"New info: {new_info}")
# open a new workbook using openpyxl
workbook = load_workbook(self.filepath)
# get list of sheet names
sheets = workbook.sheetnames
# logger.debug(workbook.sheetnames)
@@ -468,12 +531,48 @@ class PydSubmission(BaseModel, extra='allow'):
logger.debug(f"Attempting: {item['type']} in row {item['location']['row']}, column {item['location']['column']}")
worksheet.cell(row=item['location']['row'], column=item['location']['column'], value=item['value'])
# Hacky way to pop in 'signed by'
# custom_parser = get_polymorphic_subclass(BasicSubmission, info['submission_type'])
custom_parser = BasicSubmission.find_polymorphic_subclass(polymorphic_identity=self.submission_type['value'])
workbook = custom_parser.custom_autofill(workbook)
workbook = custom_parser.custom_autofill(workbook, info=self.improved_dict(), backup=backup)
return workbook
def autofill_samples(self, workbook:Workbook) -> Workbook:
"""
Fill in sample rows on the excel sheet
Args:
workbook (Workbook): Input excel workbook
Returns:
Workbook: Updated excel workbook
"""
sample_info = SubmissionType.query(name=self.submission_type['value']).info_map['samples']
worksheet = workbook[sample_info["lookup_table"]['sheet']]
samples = sorted(self.samples, key=attrgetter('column', 'row'))
logger.debug(f"Samples: {samples}")
# Fail safe against multiple instances of the same sample
for iii, sample in enumerate(samples, start=1):
row = sample_info['lookup_table']['start_row'] + iii
fields = [field for field in list(sample.model_fields.keys()) + list(sample.model_extra.keys()) if field in sample_info['sample_columns'].keys()]
for field in fields:
column = sample_info['sample_columns'][field]
value = getattr(sample, field)
match value:
case list():
value = value[0]
case _:
value = value
if field == "row":
value = row_map[value]
worksheet.cell(row=row, column=column, value=value)
return workbook
def construct_filename(self):
def construct_filename(self) -> str:
"""
Creates filename for this instance
Returns:
str: Output filename
"""
env = jinja_template_loading()
template = BasicSubmission.find_polymorphic_subclass(polymorphic_identity=self.submission_type).filename_template()
logger.debug(f"Using template string: {template}")
@@ -483,12 +582,19 @@ class PydSubmission(BaseModel, extra='allow'):
return render
class PydContact(BaseModel):
name: str
phone: str|None
email: str|None
def toSQL(self):
def toSQL(self) -> Contact:
"""
Converts this instance into a backend.db.models.organization.Contact instance
Returns:
Contact: Contact instance
"""
return Contact(name=self.name, phone=self.phone, email=self.email)
class PydOrganization(BaseModel):
@@ -497,7 +603,13 @@ class PydOrganization(BaseModel):
cost_centre: str
contacts: List[PydContact]|None
def toSQL(self):
def toSQL(self) -> Organization:
"""
Converts this instance into a backend.db.models.organization.Organization instance.
Returns:
Organization: Organization instance
"""
instance = Organization()
for field in self.model_fields:
match field:
@@ -522,7 +634,16 @@ class PydReagentType(BaseModel):
return timedelta(days=value)
return value
def toSQL(self, kit:KitType):
def toSQL(self, kit:KitType) -> ReagentType:
"""
Converts this instance into a backend.db.models.ReagentType instance
Args:
kit (KitType): KitType joined to the reagenttype
Returns:
ReagentType: ReagentType instance
"""
# instance: ReagentType = lookup_reagent_types(ctx=ctx, name=self.name)
instance: ReagentType = ReagentType.query(name=self.name)
if instance == None:
@@ -543,14 +664,21 @@ class PydKit(BaseModel):
name: str
reagent_types: List[PydReagentType] = []
def toSQL(self):
result = dict(message=None, status='Information')
def toSQL(self) -> Tuple[KitType, Report]:
"""
Converts this instance into a backend.db.models.kits.KitType instance
Returns:
Tuple[KitType, Report]: KitType instance and report of results.
"""
# result = dict(message=None, status='Information')
report = Report()
# instance = lookup_kit_types(ctx=ctx, name=self.name)
instance = KitType.query(name=self.name)
if instance == None:
instance = KitType(name=self.name)
# instance.reagent_types = [item.toSQL(ctx, instance) for item in self.reagent_types]
[item.toSQL(instance) for item in self.reagent_types]
return instance, result
return instance, report