Moments before disaster.

This commit is contained in:
Landon Wark
2024-01-08 08:51:23 -06:00
parent 19448cc8f3
commit 8c688df75f
10 changed files with 445 additions and 82 deletions

View File

@@ -32,6 +32,22 @@ equipmentroles_equipment = Table(
extend_existing=True
)
equipmentroles_processes = Table(
"_equipmentroles_processes",
Base.metadata,
Column("process_id", INTEGER, ForeignKey("_process.id")),
Column("equipmentroles_id", INTEGER, ForeignKey("_equipment_roles.id")),
extend_existing=True
)
submissiontypes_processes = Table(
"_submissiontypes_processes",
Base.metadata,
Column("process_id", INTEGER, ForeignKey("_process.id")),
Column("equipmentroles_id", INTEGER, ForeignKey("_submission_types.id")),
extend_existing=True
)
class KitType(BaseClass):
"""
Base of kits used in submission processing
@@ -588,6 +604,7 @@ class SubmissionType(BaseClass):
instances = relationship("BasicSubmission", backref="submission_type") #: Concrete instances of this type.
# regex = Column(String(512))
template_file = Column(BLOB) #: Blank form for this type stored as binary.
processes = relationship("Process", back_populates="submission_types", secondary=submissiontypes_processes)
submissiontype_kit_associations = relationship(
"SubmissionTypeKitTypeAssociation",
@@ -855,7 +872,10 @@ class Equipment(BaseClass):
return f"<Equipment({self.name})>"
def get_processes(self, submission_type:SubmissionType):
return [assoc.process for assoc in self.equipment_submission_associations if assoc.submission.submission_type_name==submission_type.name]
processes = [assoc.process for assoc in self.equipment_submission_associations if assoc.submission.submission_type_name==submission_type.name]
if len(processes) == 0:
processes = ['']
return processes
@classmethod
@setup_lookup
@@ -888,7 +908,8 @@ class Equipment(BaseClass):
def to_pydantic(self, submission_type:SubmissionType):
from backend.validators.pydant import PydEquipment
return PydEquipment(processes=self.get_processes(submission_type=submission_type), role=None, **self.__dict__)
# return PydEquipment(process=self.get_processes(submission_type=submission_type), role=None, **self.__dict__)
return PydEquipment(process=None, role=None, **self.__dict__)
def save(self):
self.__database_session__.add(self)
@@ -911,6 +932,7 @@ class EquipmentRole(BaseClass):
id = Column(INTEGER, primary_key=True)
name = Column(String(32))
instances = relationship("Equipment", back_populates="roles", secondary=equipmentroles_equipment)
processes = relationship("Process", back_populates="equipment_roles", secondary=equipmentroles_processes)
equipmentrole_submissiontype_associations = relationship(
"SubmissionTypeEquipmentRoleAssociation",
@@ -926,7 +948,9 @@ class EquipmentRole(BaseClass):
def to_pydantic(self, submission_type:SubmissionType):
from backend.validators.pydant import PydEquipmentRole
equipment = [item.to_pydantic(submission_type=submission_type) for item in self.instances]
return PydEquipmentRole(equipment=equipment, **self.__dict__)
pyd_dict = self.__dict__
pyd_dict['processes'] = self.get_processes(submission_type=submission_type)
return PydEquipmentRole(equipment=equipment, **pyd_dict)
@classmethod
@setup_lookup
@@ -945,6 +969,25 @@ class EquipmentRole(BaseClass):
case _:
pass
return query_return(query=query, limit=limit)
def get_processes(self, submission_type:str|SubmissionType|None) -> List[Process]:
if isinstance(submission_type, str):
submission_type = SubmissionType.query(name=submission_type)
if submission_type != None:
output = [process.name for process in self.processes if submission_type in process.submission_types]
else:
output = [process.name for process in self.processes]
if len(output) == 0:
return ['']
else:
return output
def save(self):
try:
self.__database_session__.add(self)
self.__database_session__.commit()
except:
self.__database_session__.rollback()
class SubmissionEquipmentAssociation(BaseClass):
@@ -956,7 +999,8 @@ class SubmissionEquipmentAssociation(BaseClass):
equipment_id = Column(INTEGER, ForeignKey("_equipment.id"), primary_key=True) #: id of associated equipment
submission_id = Column(INTEGER, ForeignKey("_submissions.id"), primary_key=True) #: id of associated submission
role = Column(String(64), primary_key=True) #: name of the role the equipment fills
process = Column(String(64)) #: name of the process run on this equipment
# process = Column(String(64)) #: name of the process run on this equipment
process_id = Column(INTEGER, ForeignKey("_process.id",ondelete="SET NULL", name="SEA_Process_id"))
start_time = Column(TIMESTAMP)
end_time = Column(TIMESTAMP)
comments = Column(String(1024))
@@ -970,7 +1014,7 @@ class SubmissionEquipmentAssociation(BaseClass):
self.equipment = equipment
def to_sub_dict(self) -> dict:
output = dict(name=self.equipment.name, asset_number=self.equipment.asset_number, comment=self.comments, process=[self.process], role=self.role, nickname=self.equipment.nickname)
output = dict(name=self.equipment.name, asset_number=self.equipment.asset_number, comment=self.comments, process=self.process.name, role=self.role, nickname=self.equipment.nickname)
return output
def save(self):
@@ -1021,3 +1065,149 @@ class SubmissionTypeEquipmentRoleAssociation(BaseClass):
self.__database_session__.add(self)
self.__database_session__.commit()
class Process(BaseClass):
__tablename__ = "_process"
id = Column(INTEGER, primary_key=True)
name = Column(String(64))
submission_types = relationship("SubmissionType", back_populates='processes', secondary=submissiontypes_processes)
equipment_roles = relationship("EquipmentRole", back_populates='processes', secondary=equipmentroles_processes)
submissions = relationship("SubmissionEquipmentAssociation", backref='process')
def __repr__(self):
return f"<Process({self.name})"
@classmethod
@setup_lookup
def query(cls, name:str|None, limit:int=0):
query = cls.__database_session__.query(cls)
match name:
case str():
query = query.filter(cls.name==name)
limit = 1
case _:
pass
return query_return(query=query, limit=limit)
def save(self):
try:
self.__database_session__.add(self)
self.__database_session__.commit()
except:
self.__database_session__.rollback()
# class KitTypeReagentTypeAssociation(BaseClass):
# """
# table containing reagenttype/kittype associations
# DOC: https://docs.sqlalchemy.org/en/14/orm/extensions/associationproxy.html
# """
# __tablename__ = "_reagenttypes_kittypes"
# reagent_types_id = Column(INTEGER, ForeignKey("_reagent_types.id"), primary_key=True) #: id of associated reagent type
# kits_id = Column(INTEGER, ForeignKey("_kits.id"), primary_key=True) #: id of associated reagent type
# uses = Column(JSON) #: map to location on excel sheets of different submission types
# required = Column(INTEGER) #: whether the reagent type is required for the kit (Boolean 1 or 0)
# last_used = Column(String(32)) #: last used lot number of this type of reagent
# kit_type = relationship(KitType, back_populates="kit_reagenttype_associations") #: relationship to associated kit
# # reference to the "ReagentType" object
# reagent_type = relationship(ReagentType, back_populates="reagenttype_kit_associations") #: relationship to associated reagent type
# def __init__(self, kit_type=None, reagent_type=None, uses=None, required=1):
# # logger.debug(f"Parameters: Kit={kit_type}, RT={reagent_type}, Uses={uses}, Required={required}")
# self.kit_type = kit_type
# self.reagent_type = reagent_type
# self.uses = uses
# self.required = required
# def __repr__(self) -> str:
# return f"<KitTypeReagentTypeAssociation({self.kit_type} & {self.reagent_type})>"
# @validates('required')
# def validate_age(self, key, value):
# """
# Ensures only 1 & 0 used in 'required'
# Args:
# key (str): name of attribute
# value (_type_): value of attribute
# Raises:
# ValueError: Raised if bad value given
# Returns:
# _type_: value
# """
# if not 0 <= value < 2:
# raise ValueError(f'Invalid required value {value}. Must be 0 or 1.')
# return value
# @validates('reagenttype')
# def validate_reagenttype(self, key, value):
# """
# Ensures reagenttype is an actual ReagentType
# Args:
# key (str)): name of attribute
# value (_type_): value of attribute
# Raises:
# ValueError: raised if reagenttype is not a ReagentType
# Returns:
# _type_: ReagentType
# """
# if not isinstance(value, ReagentType):
# raise ValueError(f'{value} is not a reagenttype')
# return value
# @classmethod
# @setup_lookup
# def query(cls,
# kit_type:KitType|str|None=None,
# reagent_type:ReagentType|str|None=None,
# limit:int=0
# ) -> KitTypeReagentTypeAssociation|List[KitTypeReagentTypeAssociation]:
# """
# Lookup junction of ReagentType and KitType
# Args:
# kit_type (models.KitType | str | None): KitType of interest.
# reagent_type (models.ReagentType | str | None): ReagentType of interest.
# limit (int, optional): Maximum number of results to return (0 = all). Defaults to 0.
# Returns:
# models.KitTypeReagentTypeAssociation|List[models.KitTypeReagentTypeAssociation]: Junction of interest.
# """
# query: Query = cls.__database_session__.query(cls)
# match kit_type:
# case KitType():
# query = query.filter(cls.kit_type==kit_type)
# case str():
# query = query.join(KitType).filter(KitType.name==kit_type)
# case _:
# pass
# match reagent_type:
# case ReagentType():
# query = query.filter(cls.reagent_type==reagent_type)
# case str():
# query = query.join(ReagentType).filter(ReagentType.name==reagent_type)
# case _:
# pass
# if kit_type != None and reagent_type != None:
# limit = 1
# return query_return(query=query, limit=limit)
# def save(self) -> Report:
# """
# Adds this instance to the database and commits.
# Returns:
# Report: Result of save action
# """
# report = Report()
# self.__database_session__.add(self)
# self.__database_session__.commit()
# return report

View File

@@ -15,7 +15,7 @@ import pandas as pd
from openpyxl import Workbook
from . import BaseClass
from tools import check_not_nan, row_map, query_return, setup_lookup, jinja_template_loading
from datetime import datetime, date
from datetime import datetime, date, time
from typing import List
from dateutil.parser import parse
from dateutil.parser._parser import ParserError
@@ -397,7 +397,7 @@ class BasicSubmission(BaseClass):
return cls.find_polymorphic_subclass(submission_type.name)
case _:
pass
if len(attrs) == 0 or attrs == None:
if attrs == None or len(attrs) == 0:
return cls
if any([not hasattr(cls, attr) for attr in attrs]):
# looks for first model that has all included kwargs
@@ -675,6 +675,7 @@ class BasicSubmission(BaseClass):
logger.warning(f"End date with no start date, using Jan 1, 2023")
start_date = date(2023, 1, 1)
if start_date != None:
logger.debug(f"Querying with start date: {start_date} and end date: {end_date}")
match start_date:
case date():
start_date = start_date.strftime("%Y-%m-%d")
@@ -683,14 +684,19 @@ class BasicSubmission(BaseClass):
case _:
start_date = parse(start_date).strftime("%Y-%m-%d")
match end_date:
case date():
case date() | datetime():
end_date = end_date.strftime("%Y-%m-%d")
case int():
end_date = datetime.fromordinal(datetime(1900, 1, 1).toordinal() + end_date - 2).date().strftime("%Y-%m-%d")
case _:
end_date = parse(end_date).strftime("%Y-%m-%d")
# logger.debug(f"Looking up BasicSubmissions from start date: {start_date} and end date: {end_date}")
query = query.filter(cls.submitted_date.between(start_date, end_date))
logger.debug(f"Start date {start_date} == End date {end_date}: {start_date==end_date}")
if start_date == end_date:
start_date = datetime.strptime(start_date, "%Y-%m-%d").strftime("%Y-%m-%d %H:%M:%S.%f")
query = query.filter(cls.submitted_date==start_date)
else:
query = query.filter(cls.submitted_date.between(start_date, end_date))
# by reagent (for some reason)
match reagent:
case str():
@@ -846,42 +852,55 @@ class BacterialCulture(BasicSubmission):
"""
Extends parent
"""
from backend.validators import RSLNamer
data['abbreviation'] = "BC"
outstr = super().enforce_name(instr=instr, data=data)
def construct(data:dict|None=None) -> str:
"""
Create default plate name.
# def construct(data:dict|None=None) -> str:
# """
# Create default plate name.
Returns:
str: new RSL number
"""
# logger.debug(f"Attempting to construct RSL number from scratch...")
directory = cls.__directory_path__.joinpath("Bacteria")
year = str(datetime.now().year)[-2:]
if directory.exists():
logger.debug(f"Year: {year}")
relevant_rsls = []
all_xlsx = [item.stem for item in directory.rglob("*.xlsx") if bool(re.search(r"RSL-\d{2}-\d{4}", item.stem)) and year in item.stem[4:6]]
# logger.debug(f"All rsls: {all_xlsx}")
for item in all_xlsx:
try:
relevant_rsls.append(re.match(r"RSL-\d{2}-\d{4}", item).group(0))
except Exception as e:
logger.error(f"Regex error: {e}")
continue
# logger.debug(f"Initial xlsx: {relevant_rsls}")
max_number = max([int(item[-4:]) for item in relevant_rsls])
# logger.debug(f"The largest sample number is: {max_number}")
return f"RSL-{year}-{str(max_number+1).zfill(4)}"
else:
# raise FileNotFoundError(f"Unable to locate the directory: {directory.__str__()}")
return f"RSL-{year}-0000"
# Returns:
# str: new RSL number
# """
# # logger.debug(f"Attempting to construct RSL number from scratch...")
# directory = cls.__directory_path__.joinpath("Bacteria")
# year = str(datetime.now().year)[-2:]
# if directory.exists():
# logger.debug(f"Year: {year}")
# relevant_rsls = []
# all_xlsx = [item.stem for item in directory.rglob("*.xlsx") if bool(re.search(r"RSL-\d{2}-\d{4}", item.stem)) and year in item.stem[4:6]]
# # logger.debug(f"All rsls: {all_xlsx}")
# for item in all_xlsx:
# try:
# relevant_rsls.append(re.match(r"RSL-\d{2}-\d{4}", item).group(0))
# except Exception as e:
# logger.error(f"Regex error: {e}")
# continue
# # logger.debug(f"Initial xlsx: {relevant_rsls}")
# max_number = max([int(item[-4:]) for item in relevant_rsls])
# # logger.debug(f"The largest sample number is: {max_number}")
# return f"RSL-{year}-{str(max_number+1).zfill(4)}"
# else:
# # raise FileNotFoundError(f"Unable to locate the directory: {directory.__str__()}")
# return f"RSL-{year}-0000"
# try:
# outstr = re.sub(r"RSL(\d{2})", r"RSL-\1", outstr, flags=re.IGNORECASE)
# except (AttributeError, TypeError) as e:
# outstr = construct()
# # year = datetime.now().year
# # self.parsed_name = f"RSL-{str(year)[-2:]}-0000"
# return re.sub(r"RSL-(\d{2})(\d{4})", r"RSL-\1-\2", outstr, flags=re.IGNORECASE)
# def construct():
# previous = cls.query(start_date=date.today(), end_date=date.today(), submission_type=cls.__name__)
# max = len(previous)
# return f"RSL-BC-{date.today().strftime('%Y%m%d')}-{max+1}"
try:
outstr = re.sub(r"RSL(\d{2})", r"RSL-\1", outstr, flags=re.IGNORECASE)
outstr = re.sub(r"(\d{4})-(\d{2})-(\d{2})", r"\1\2\3", outstr)
outstr = re.sub(r"BC(\d{6})", r"BC-\1", outstr, flags=re.IGNORECASE)
except (AttributeError, TypeError) as e:
outstr = construct()
# year = datetime.now().year
# self.parsed_name = f"RSL-{str(year)[-2:]}-0000"
return re.sub(r"RSL-(\d{2})(\d{4})", r"RSL-\1-\2", outstr, flags=re.IGNORECASE)
# outstr = construct()
outstr = RSLNamer.construct_new_plate_name(data=data)
return outstr
@classmethod
def get_regex(cls) -> str:
@@ -992,27 +1011,30 @@ class Wastewater(BasicSubmission):
"""
Extends parent
"""
from backend.validators import RSLNamer
data['abbreviation'] = "WW"
outstr = super().enforce_name(instr=instr, data=data)
def construct(data:dict|None=None):
if "submitted_date" in data.keys():
if data['submitted_date']['value'] != None:
today = data['submitted_date']['value']
else:
today = datetime.now()
else:
today = re.search(r"\d{4}(_|-)?\d{2}(_|-)?\d{2}", instr)
try:
today = parse(today.group())
except AttributeError:
today = datetime.now()
return f"RSL-WW-{today.year}{str(today.month).zfill(2)}{str(today.day).zfill(2)}"
if outstr == None:
outstr = construct(data)
# def construct(data:dict|None=None):
# if "submitted_date" in data.keys():
# if data['submitted_date']['value'] != None:
# today = data['submitted_date']['value']
# else:
# today = datetime.now()
# else:
# today = re.search(r"\d{4}(_|-)?\d{2}(_|-)?\d{2}", instr)
# try:
# today = parse(today.group())
# except AttributeError:
# today = datetime.now()
# return f"RSL-WW-{today.year}{str(today.month).zfill(2)}{str(today.day).zfill(2)}"
# if outstr == None:
# outstr = construct(data)
try:
outstr = re.sub(r"PCR(-|_)", "", outstr)
except AttributeError as e:
logger.error(f"Problem using regex: {e}")
outstr = construct(data)
# outstr = construct(data)
outstr = RSLNamer.construct_new_plate_name(instr=outstr)
outstr = outstr.replace("RSLWW", "RSL-WW")
outstr = re.sub(r"WW(\d{4})", r"WW-\1", outstr, flags=re.IGNORECASE)
outstr = re.sub(r"(\d{4})-(\d{2})-(\d{2})", r"\1\2\3", outstr)
@@ -1094,14 +1116,17 @@ class WastewaterArtic(BasicSubmission):
"""
Extends parent
"""
from backend.validators import RSLNamer
data['abbreviation'] = "AR"
outstr = super().enforce_name(instr=instr, data=data)
def construct(data:dict|None=None):
today = datetime.now()
return f"RSL-AR-{today.year}{str(today.month).zfill(2)}{str(today.day).zfill(2)}"
# def construct(data:dict|None=None):
# today = datetime.now()
# return f"RSL-AR-{today.year}{str(today.month).zfill(2)}{str(today.day).zfill(2)}"
try:
outstr = re.sub(r"(\d{4})-(\d{2})-(\d{2})", r"RSL-AR-\1\2\3", outstr, flags=re.IGNORECASE)
except AttributeError:
outstr = construct()
# outstr = construct()
outstr = RSLNamer.construct_new_plate_name(instr=outstr, data=data)
try:
plate_number = int(re.search(r"_|-\d?_", outstr).group().strip("_").strip("-"))
except (AttributeError, ValueError) as e:

View File

@@ -2,6 +2,7 @@ import logging, re
from pathlib import Path
from openpyxl import load_workbook
from backend.db.models import BasicSubmission, SubmissionType
from datetime import date
logger = logging.getLogger(f"submissions.{__name__}")
@@ -17,6 +18,10 @@ class RSLNamer(object):
if self.submission_type != None:
enforcer = BasicSubmission.find_polymorphic_subclass(polymorphic_identity=self.submission_type)
self.parsed_name = self.retrieve_rsl_number(instr=instr, regex=enforcer.get_regex())
if data == None:
data = dict(submission_type=self.submission_type)
if "submission_type" not in data.keys():
data['submission_type'] = self.submission_type
self.parsed_name = enforcer.enforce_name(instr=self.parsed_name, data=data)
@classmethod
@@ -104,5 +109,23 @@ class RSLNamer(object):
parsed_name = None
logger.debug(f"Got parsed submission name: {parsed_name}")
return parsed_name
@classmethod
def construct_new_plate_name(cls, data:dict) -> str:
if "submitted_date" in data.keys():
if data['submitted_date']['value'] != None:
today = data['submitted_date']['value']
else:
today = datetime.now()
else:
today = re.search(r"\d{4}(_|-)?\d{2}(_|-)?\d{2}", instr)
try:
today = parse(today.group())
except AttributeError:
today = datetime.now()
previous = BasicSubmission.query(start_date=today, end_date=today, submission_type=data['submission_type'])
plate_number = len(previous) + 1
return f"RSL-{data['abbreviation']}-{today.year}{str(today.month).zfill(2)}{str(today.day).zfill(2)}-{plate_number}"
from .pydant import *

View File

@@ -355,8 +355,9 @@ class PydSubmission(BaseModel, extra='allow'):
value = value['value'].title()
return dict(value=value, missing=False)
else:
return dict(value=RSLNamer(instr=values.data['filepath'].__str__()).submission_type.title(), missing=True)
# return dict(value=RSLNamer(instr=values.data['filepath'].__str__()).submission_type.title(), missing=True)
return dict(value=RSLNamer.retrieve_submission_type(instr=values.data['filepath']).title(), missing=True)
@field_validator("submission_category", mode="before")
def create_category(cls, value):
if not isinstance(value, dict):
@@ -444,6 +445,11 @@ class PydSubmission(BaseModel, extra='allow'):
instance.submission_sample_associations.append(assoc)
case "equipment":
logger.debug(f"Equipment: {pformat(self.equipment)}")
try:
if equip == None:
continue
except UnboundLocalError:
continue
for equip in self.equipment:
equip, association = equip.toSQL(submission=instance)
if association != None:
@@ -773,20 +779,20 @@ class PydEquipment(BaseModel, extra='ignore'):
asset_number: str
name: str
nickname: str|None
process: List[str]|None
process: str|None
role: str|None
@field_validator('process')
@classmethod
def remove_dupes(cls, value):
if isinstance(value, list):
return list(set(value))
else:
return value
# @field_validator('process')
# @classmethod
# def remove_dupes(cls, value):
# if isinstance(value, list):
# return list(set(value))
# else:
# return value
def toForm(self, parent):
from frontend.widgets.equipment_usage import EquipmentCheckBox
return EquipmentCheckBox(parent=parent, equipment=self)
# def toForm(self, parent):
# from frontend.widgets.equipment_usage import EquipmentCheckBox
# return EquipmentCheckBox(parent=parent, equipment=self)
def toSQL(self, submission:BasicSubmission|str=None):
if isinstance(submission, str):
@@ -796,7 +802,7 @@ class PydEquipment(BaseModel, extra='ignore'):
return
if submission != None:
assoc = SubmissionEquipmentAssociation(submission=submission, equipment=equipment)
assoc.process = self.process[0]
assoc.process = self.process
assoc.role = self.role
# equipment.equipment_submission_associations.append(assoc)
equipment.equipment_submission_associations.append(assoc)
@@ -808,6 +814,7 @@ class PydEquipmentRole(BaseModel):
name: str
equipment: List[PydEquipment]
processes: List[str]|None
def toForm(self, parent, submission_type, used):
from frontend.widgets.equipment_usage import RoleComboBox