landon/Submissions-App
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Improved form regeneration for artic.

Browse Source
This commit is contained in:
Landon Wark
2024-01-04 13:29:18 -06:00
parent c688aa160c
commit 19448cc8f3
18 changed files with 519 additions and 123 deletions
Download Patch File Download Diff File Expand all files Collapse all files

144
src/submissions/backend/db/models/kits.py
View File

@@ -6,7 +6,7 @@ from sqlalchemy import Column, String, TIMESTAMP, JSON, INTEGER, ForeignKey, Int
from sqlalchemy.orm import relationship, validates, Query
from sqlalchemy.ext.associationproxy import association_proxy
from datetime import date
import logging
import logging, re
from tools import check_authorization, setup_lookup, query_return, Report, Result, Settings
from typing import List
from pandas import ExcelFile
@@ -24,6 +24,14 @@ reagenttypes_reagents = Table(
extend_existing = True
)
equipmentroles_equipment = Table(
"_equipmentroles_equipment",
Base.metadata,
Column("equipment_id", INTEGER, ForeignKey("_equipment.id")),
Column("equipmentroles_id", INTEGER, ForeignKey("_equipment_roles.id")),
extend_existing=True
)
class KitType(BaseClass):
"""
Base of kits used in submission processing
@@ -589,13 +597,13 @@ class SubmissionType(BaseClass):
kit_types = association_proxy("submissiontype_kit_associations", "kit_type") #: Proxy of kittype association
submissiontype_equipment_associations = relationship(
"SubmissionTypeEquipmentAssociation",
submissiontype_equipmentrole_associations = relationship(
"SubmissionTypeEquipmentRoleAssociation",
back_populates="submission_type",
cascade="all, delete-orphan"
)
equipment = association_proxy("submissiontype_equipment_associations", "equipment")
equipment = association_proxy("submissiontype_equipmentrole_associations", "equipment_role")
def __repr__(self) -> str:
return f"<SubmissionType({self.name})>"
@@ -609,34 +617,35 @@ class SubmissionType(BaseClass):
"""
return ExcelFile(self.template_file).sheet_names
def set_template_file(self, filepath:Path|str):
def set_template_file(self, ctx:Settings, filepath:Path|str):
if isinstance(filepath, str):
filepath = Path(filepath)
with open (filepath, "rb") as f:
data = f.read()
self.template_file = data
self.save()
self.save(ctx=ctx)
def get_equipment(self) -> list:
from backend.validators.pydant import PydEquipmentPool
# if static:
# return [item.equipment.to_pydantic() for item in self.submissiontype_equipment_associations if item.static==1]
# else:
preliminary1 = [item.equipment.to_pydantic(static=item.static) for item in self.submissiontype_equipment_associations]# if item.static==0]
preliminary2 = [item.equipment.to_pydantic(static=item.static) for item in self.submissiontype_equipment_associations]# if item.static==0]
def construct_equipment_map(self):
output = []
pools = list(set([item.pool_name for item in preliminary1 if item.pool_name != None]))
for pool in pools:
c_ = []
for item in preliminary1:
if item.pool_name == pool:
c_.append(item)
preliminary2.remove(item)
if len(c_) > 0:
output.append(PydEquipmentPool(name=pool, equipment=c_))
for item in preliminary2:
output.append(item)
for item in self.submissiontype_equipmentrole_associations:
map = item.uses
map['role'] = item.equipment_role.name
output.append(map)
return output
# return [item.uses for item in self.submissiontype_equipmentrole_associations]
def get_equipment(self) -> List['PydEquipmentRole']:
return [item.to_pydantic(submission_type=self) for item in self.equipment]
def get_processes_for_role(self, equipment_role:str|EquipmentRole):
match equipment_role:
case str():
relevant = [item.get_all_processes() for item in self.submissiontype_equipmentrole_associations if item.equipment_role.name==equipment_role]
case EquipmentRole():
relevant = [item.get_all_processes() for item in self.submissiontype_equipmentrole_associations if item.equipment_role==equipment_role]
case _:
raise TypeError(f"Type {type(equipment_role)} is not allowed")
return list(set([item for items in relevant for item in items if item != None ]))
@classmethod
@setup_lookup
@@ -832,7 +841,7 @@ class Equipment(BaseClass):
name = Column(String(64))
nickname = Column(String(64))
asset_number = Column(String(16))
pool_name = Column(String(16))
roles = relationship("EquipmentRole", back_populates="instances", secondary=equipmentroles_equipment)
equipment_submission_associations = relationship(
"SubmissionEquipmentAssociation",
@@ -842,16 +851,11 @@ class Equipment(BaseClass):
submissions = association_proxy("equipment_submission_associations", "submission")
equipment_submissiontype_associations = relationship(
"SubmissionTypeEquipmentAssociation",
back_populates="equipment",
cascade="all, delete-orphan",
)
submission_types = association_proxy("equipment_submission_associations", "submission_type")
def __repr__(self):
return f"<Equipment({self.name})>"
def get_processes(self, submission_type:SubmissionType):
return [assoc.process for assoc in self.equipment_submission_associations if assoc.submission.submission_type_name==submission_type.name]
@classmethod
@setup_lookup
@@ -882,14 +886,66 @@ class Equipment(BaseClass):
pass
return query_return(query=query, limit=limit)
def to_pydantic(self, static):
def to_pydantic(self, submission_type:SubmissionType):
from backend.validators.pydant import PydEquipment
return PydEquipment(static=static, **self.__dict__)
return PydEquipment(processes=self.get_processes(submission_type=submission_type), role=None, **self.__dict__)
def save(self):
self.__database_session__.add(self)
self.__database_session__.commit()
@classmethod
def get_regex(cls) -> re.Pattern:
return re.compile(r"""
(?P<PHAC>50\d{5}$)|
(?P<HC>HC-\d{6}$)|
(?P<Beckman>[^\d][A-Z0-9]{6}$)|
(?P<Axygen>[A-Z]{3}-\d{2}-[A-Z]-[A-Z]$)|
(?P<Labcon>\d{4}-\d{3}-\d{3}-\d$)""",
re.VERBOSE)
class EquipmentRole(BaseClass):
__tablename__ = "_equipment_roles"
id = Column(INTEGER, primary_key=True)
name = Column(String(32))
instances = relationship("Equipment", back_populates="roles", secondary=equipmentroles_equipment)
equipmentrole_submissiontype_associations = relationship(
"SubmissionTypeEquipmentRoleAssociation",
back_populates="equipment_role",
cascade="all, delete-orphan",
)
submission_types = association_proxy("equipmentrole_submission_associations", "submission_type")
def __repr__(self):
return f"<EquipmentRole({self.name})>"
def to_pydantic(self, submission_type:SubmissionType):
from backend.validators.pydant import PydEquipmentRole
equipment = [item.to_pydantic(submission_type=submission_type) for item in self.instances]
return PydEquipmentRole(equipment=equipment, **self.__dict__)
@classmethod
@setup_lookup
def query(cls, name:str|None=None, id:int|None=None, limit:int=0) -> EquipmentRole|List[EquipmentRole]:
query = cls.__database_session__.query(cls)
match id:
case int():
query = query.filter(cls.id==id)
limit = 1
case _:
pass
match name:
case str():
query = query.filter(cls.name==name)
limit = 1
case _:
pass
return query_return(query=query, limit=limit)
class SubmissionEquipmentAssociation(BaseClass):
# Currently abstract until ready to implement
@@ -899,6 +955,7 @@ class SubmissionEquipmentAssociation(BaseClass):
equipment_id = Column(INTEGER, ForeignKey("_equipment.id"), primary_key=True) #: id of associated equipment
submission_id = Column(INTEGER, ForeignKey("_submissions.id"), primary_key=True) #: id of associated submission
role = Column(String(64), primary_key=True) #: name of the role the equipment fills
process = Column(String(64)) #: name of the process run on this equipment
start_time = Column(TIMESTAMP)
end_time = Column(TIMESTAMP)
@@ -913,27 +970,27 @@ class SubmissionEquipmentAssociation(BaseClass):
self.equipment = equipment
def to_sub_dict(self) -> dict:
output = dict(name=self.equipment.name, asset_number=self.equipment.asset_number, comment=self.comments)
output = dict(name=self.equipment.name, asset_number=self.equipment.asset_number, comment=self.comments, process=[self.process], role=self.role, nickname=self.equipment.nickname)
return output
def save(self):
self.__database_session__.add(self)
self.__database_session__.commit()
class SubmissionTypeEquipmentAssociation(BaseClass):
class SubmissionTypeEquipmentRoleAssociation(BaseClass):
# __abstract__ = True
__tablename__ = "_submissiontype_equipment"
__tablename__ = "_submissiontype_equipmentrole"
equipment_id = Column(INTEGER, ForeignKey("_equipment.id"), primary_key=True) #: id of associated equipment
equipmentrole_id = Column(INTEGER, ForeignKey("_equipment_roles.id"), primary_key=True) #: id of associated equipment
submissiontype_id = Column(INTEGER, ForeignKey("_submission_types.id"), primary_key=True) #: id of associated submission
uses = Column(JSON) #: locations of equipment on the submission type excel sheet.
static = Column(INTEGER, default=1) #: if 1 this piece of equipment will always be used, otherwise it will need to be selected from list?
submission_type = relationship(SubmissionType, back_populates="submissiontype_equipment_associations") #: associated submission
submission_type = relationship(SubmissionType, back_populates="submissiontype_equipmentrole_associations") #: associated submission
equipment = relationship(Equipment, back_populates="equipment_submissiontype_associations") #: associated equipment
equipment_role = relationship(EquipmentRole, back_populates="equipmentrole_submissiontype_associations") #: associated equipment
@validates('static')
def validate_age(self, key, value):
@@ -954,6 +1011,11 @@ class SubmissionTypeEquipmentAssociation(BaseClass):
raise ValueError(f'Invalid required value {value}. Must be 0 or 1.')
return value
def get_all_processes(self):
processes = [equipment.get_processes(self.submission_type) for equipment in self.equipment_role.instances]
processes = [item for items in processes for item in items if item != None ]
return processes
@check_authorization
def save(self, ctx:Settings):
self.__database_session__.add(self)

126
src/submissions/backend/db/models/submissions.py
View File

@@ -4,8 +4,9 @@ Models for the main submission types.
from __future__ import annotations
from getpass import getuser
import math, json, logging, uuid, tempfile, re, yaml
from operator import attrgetter
from pprint import pformat
from . import Reagent, SubmissionType, KitType, Organization, Equipment, SubmissionEquipmentAssociation
from . import Reagent, SubmissionType, KitType, Organization
from sqlalchemy import Column, String, TIMESTAMP, INTEGER, ForeignKey, JSON, FLOAT, case
from sqlalchemy.orm import relationship, validates, Query
from json.decoder import JSONDecodeError
@@ -13,7 +14,7 @@ from sqlalchemy.ext.associationproxy import association_proxy
import pandas as pd
from openpyxl import Workbook
from . import BaseClass
from tools import check_not_nan, row_map, query_return, setup_lookup
from tools import check_not_nan, row_map, query_return, setup_lookup, jinja_template_loading
from datetime import datetime, date
from typing import List
from dateutil.parser import parse
@@ -137,21 +138,23 @@ class BasicSubmission(BaseClass):
reagents = None
# samples = [item.sample.to_sub_dict(submission_rsl=self.rsl_plate_num) for item in self.submission_sample_associations]
samples = [item.to_sub_dict() for item in self.submission_sample_associations]
try:
equipment = [item.to_sub_dict() for item in self.submission_equipment_associations]
if len(equipment) == 0:
equipment = None
except Exception as e:
logger.error(f"Error setting equipment: {self.equipment}")
equipment = None
else:
reagents = None
samples = None
equipment = None
try:
comments = self.comment
except Exception as e:
logger.error(f"Error setting comment: {self.comment}")
comments = None
try:
equipment = [item.to_sub_dict() for item in self.submission_equipment_associations]
if len(equipment) == 0:
equipment = None
except Exception as e:
logger.error(f"Error setting equipment: {self.equipment}")
equipment = None
output = {
"id": self.id,
"Plate Number": self.rsl_plate_num,
@@ -508,7 +511,7 @@ class BasicSubmission(BaseClass):
field_value = len(self.samples)
else:
field_value = value
case "ctx" | "csv" | "filepath":
case "ctx" | "csv" | "filepath" | "equipment":
return
case "comment":
if value == "" or value == None or value == 'null':
@@ -552,8 +555,9 @@ class BasicSubmission(BaseClass):
Returns:
PydSubmission: converted object.
"""
from backend.validators import PydSubmission, PydSample, PydReagent
from backend.validators import PydSubmission, PydSample, PydReagent, PydEquipment
dicto = self.to_dict(full_data=True)
logger.debug(f"Backup dictionary: {pformat(dicto)}")
# dicto['filepath'] = Path(tempfile.TemporaryFile().name)
new_dict = {}
for key, value in dicto.items():
@@ -562,6 +566,8 @@ class BasicSubmission(BaseClass):
new_dict[key] = [PydReagent(**reagent) for reagent in value]
case "samples":
new_dict[key] = [PydSample(**sample) for sample in dicto['samples']]
case "equipment":
new_dict[key] = [PydEquipment(**equipment) for equipment in dicto['equipment']]
case "Plate Number":
new_dict['rsl_plate_num'] = dict(value=value, missing=True)
case "Submitter Plate Number":
@@ -576,19 +582,20 @@ class BasicSubmission(BaseClass):
# sys.exit()
return PydSubmission(**new_dict)
def backup(self, fname:Path):
def backup(self, fname:Path, full_backup:bool=True):
"""
Exports xlsx and yml info files for this instance.
Args:
fname (Path): Filename of xlsx file.
"""
backup = self.to_dict(full_data=True)
try:
with open(self.__backup_path__.joinpath(fname.with_suffix(".yml")), "w") as f:
yaml.dump(backup, f)
except KeyError as e:
logger.error(f"Problem saving yml backup file: {e}")
if full_backup:
backup = self.to_dict(full_data=True)
try:
with open(self.__backup_path__.joinpath(fname.with_suffix(".yml")), "w") as f:
yaml.dump(backup, f)
except KeyError as e:
logger.error(f"Problem saving yml backup file: {e}")
pyd = self.to_pydantic()
wb = pyd.autofill_excel()
wb = pyd.autofill_samples(wb)
@@ -766,6 +773,8 @@ class BasicSubmission(BaseClass):
msg = "This submission already exists.\nWould you like to overwrite?"
return instance, code, msg
def get_used_equipment(self) -> List[str]:
return [item.role for item in self.submission_equipment_associations]
# Below are the custom submission types
@@ -882,7 +891,8 @@ class BacterialCulture(BasicSubmission):
Returns:
str: string for regex construction
"""
return "(?P<Bacterial_Culture>RSL-?\\d{2}-?\\d{4})"
# return "(?P<Bacterial_Culture>RSL-?\\d{2}-?\\d{4})"
return "(?P<Bacterial_Culture>RSL(?:-|_)?BC(?:-|_)?20\d{2}-?\d{2}-?\d{2}(?:(_|-)?\d?([^_0123456789\s]|$)?R?\d?)?)"
@classmethod
def filename_template(cls):
@@ -1175,7 +1185,36 @@ class WastewaterArtic(BasicSubmission):
logger.error(f"Couldn't construct df due to {e}")
input_dict['csv'] = df
return input_dict
@classmethod
def custom_autofill(cls, input_excel: Workbook, info: dict | None = None, backup: bool = False) -> Workbook:
input_excel = super().custom_autofill(input_excel, info, backup)
worksheet = input_excel["First Strand List"]
samples = cls.query(rsl_number=info['rsl_plate_num']['value']).submission_sample_associations
samples = sorted(samples, key=attrgetter('column', 'row'))
source_plates = []
first_samples = []
for sample in samples:
sample = sample.sample
try:
assoc = [item.submission.rsl_plate_num for item in sample.sample_submission_associations if item.submission.submission_type_name=="Wastewater"][-1]
except IndexError:
logger.error(f"Association not found for {sample}")
continue
if assoc not in source_plates:
source_plates.append(assoc)
first_samples.append(sample.ww_processing_num)
# Pad list to length of 3
# source_plates = list(set(source_plates))
source_plates += ['None'] * (3 - len(source_plates))
first_samples += [''] * (3 - len(first_samples))
source_plates = zip(source_plates, first_samples, strict=False)
for iii, plate in enumerate(source_plates, start=8):
logger.debug(f"Plate: {plate}")
for jjj, value in enumerate(plate, start=3):
worksheet.cell(row=iii, column=jjj, value=value)
return input_excel
# Sample Classes
class BasicSample(BaseClass):
@@ -1286,11 +1325,15 @@ class BasicSample(BaseClass):
dict: dictionary of sample id, row and column in elution plate
"""
# Since there is no PCR, negliable result is necessary.
assoc = [item for item in self.sample_submission_associations if item.submission.rsl_plate_num==submission_rsl][0]
tooltip_text = f"""
Sample name: {self.submitter_id}<br>
Well: {row_map[assoc.row]}{assoc.column}
"""
# assoc = [item for item in self.sample_submission_associations if item.submission.rsl_plate_num==submission_rsl][0]
fields = self.to_sub_dict(submission_rsl=submission_rsl)
env = jinja_template_loading()
template = env.get_template("tooltip.html")
tooltip_text = template.render(fields=fields)
# tooltip_text = f"""
# Sample name: {self.submitter_id}<br>
# Well: {row_map[assoc.row]}{assoc.column}
# """
return dict(name=self.submitter_id[:10], positive=False, tooltip=tooltip_text)
@classmethod
@@ -1436,6 +1479,7 @@ class BasicSample(BaseClass):
used_class = cls.find_subclasses(attrs=kwargs, sample_type=sample_type)
instance = used_class(**kwargs)
instance.sample_type = sample_type
logger.debug(f"Creating instance: {instance}")
return instance
def save(self):
@@ -1523,6 +1567,25 @@ class WastewaterSample(BasicSample):
del output_dict['collection_date']
return output_dict
def to_sub_dict(self, submission_rsl: str | BasicSubmission) -> dict:
sample = super().to_sub_dict(submission_rsl)
if self.ww_processing_num != None:
sample['ww_processing_num'] = self.ww_processing_num
else:
sample['ww_processing_num'] = self.submitter_id
try:
assoc = [item for item in self.sample_submission_associations if item.submission.submission_type_name=="Wastewater"][-1]
except:
assoc = None
if assoc != None:
try:
sample['ct'] = f"{assoc.ct_n1:.2f}, {assoc.ct_n2:.2f}"
except TypeError:
sample['ct'] = "None, None"
sample['source_plate'] = assoc.submission.rsl_plate_num
sample['source_well'] = f"{row_map[assoc.row]}{assoc.column}"
return sample
class BacterialCultureSample(BasicSample):
"""
base of bacterial culture sample
@@ -1541,7 +1604,9 @@ class BacterialCultureSample(BasicSample):
dict: well location and name (sample id, organism) NOTE: keys must sync with WWSample to_sub_dict above
"""
sample = super().to_sub_dict(submission_rsl=submission_rsl)
sample['name'] = f"{self.submitter_id} - ({self.organism})"
sample['name'] = self.submitter_id
sample['organism'] = self.organism
sample['concentration'] = self.concentration
return sample
def to_hitpick(self, submission_rsl: str | None = None) -> dict | None:
@@ -1622,13 +1687,15 @@ class SubmissionSampleAssociation(BaseClass):
if isinstance(polymorphic_identity, dict):
polymorphic_identity = polymorphic_identity['value']
if polymorphic_identity == None:
return cls
output = cls
else:
try:
return [item for item in cls.__subclasses__() if item.__mapper_args__['polymorphic_identity']==polymorphic_identity][0]
output = [item for item in cls.__subclasses__() if item.__mapper_args__['polymorphic_identity']==polymorphic_identity][0]
except Exception as e:
logger.error(f"Could not get polymorph {polymorphic_identity} of {cls} due to {e}")
return cls
output = cls
logger.debug(f"Using SubmissionSampleAssociation subclass: {output}")
return output
@classmethod
@setup_lookup
@@ -1707,6 +1774,7 @@ class SubmissionSampleAssociation(BaseClass):
Returns:
SubmissionSampleAssociation: Queried or new association.
"""
logger.debug(f"Attempting create or query with {kwargs}")
match submission:
case BasicSubmission():
pass